1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2017 NXP 5 * 6 */ 7 8 #include <fcntl.h> 9 #include <unistd.h> 10 #include <sched.h> 11 #include <net/if.h> 12 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 #include <rte_cryptodev_pmd.h> 16 #include <rte_crypto.h> 17 #include <rte_cryptodev.h> 18 #include <rte_security_driver.h> 19 #include <rte_cycles.h> 20 #include <rte_dev.h> 21 #include <rte_kvargs.h> 22 #include <rte_malloc.h> 23 #include <rte_mbuf.h> 24 #include <rte_memcpy.h> 25 #include <rte_string_fns.h> 26 27 #include <fsl_usd.h> 28 #include <fsl_qman.h> 29 #include <of.h> 30 31 /* RTA header files */ 32 #include <hw/desc/common.h> 33 #include <hw/desc/algo.h> 34 #include <hw/desc/ipsec.h> 35 36 #include <rte_dpaa_bus.h> 37 #include <dpaa_sec.h> 38 #include <dpaa_sec_log.h> 39 40 enum rta_sec_era rta_sec_era; 41 42 static uint8_t cryptodev_driver_id; 43 44 static __thread struct rte_crypto_op **dpaa_sec_ops; 45 static __thread int dpaa_sec_op_nb; 46 47 static int 48 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess); 49 50 static inline void 51 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx) 52 { 53 if (!ctx->fd_status) { 54 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 55 } else { 56 PMD_RX_LOG(ERR, "SEC return err: 0x%x", ctx->fd_status); 57 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR; 58 } 59 60 /* report op status to sym->op and then free the ctx memeory */ 61 rte_mempool_put(ctx->ctx_pool, (void *)ctx); 62 } 63 64 static inline struct dpaa_sec_op_ctx * 65 dpaa_sec_alloc_ctx(dpaa_sec_session *ses) 66 { 67 struct dpaa_sec_op_ctx *ctx; 68 int retval; 69 70 retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx)); 71 if (!ctx || retval) { 72 PMD_TX_LOG(ERR, "Alloc sec descriptor failed!"); 73 return NULL; 74 } 75 /* 76 * Clear SG memory. There are 16 SG entries of 16 Bytes each. 77 * one call to dcbz_64() clear 64 bytes, hence calling it 4 times 78 * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for 79 * each packet, memset is costlier than dcbz_64(). 80 */ 81 dcbz_64(&ctx->job.sg[SG_CACHELINE_0]); 82 dcbz_64(&ctx->job.sg[SG_CACHELINE_1]); 83 dcbz_64(&ctx->job.sg[SG_CACHELINE_2]); 84 dcbz_64(&ctx->job.sg[SG_CACHELINE_3]); 85 86 ctx->ctx_pool = ses->ctx_pool; 87 ctx->vtop_offset = (uint64_t) ctx 88 - rte_mempool_virt2iova(ctx); 89 90 return ctx; 91 } 92 93 static inline rte_iova_t 94 dpaa_mem_vtop(void *vaddr) 95 { 96 const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); 97 uint64_t vaddr_64, paddr; 98 int i; 99 100 vaddr_64 = (uint64_t)vaddr; 101 for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { 102 if (vaddr_64 >= memseg[i].addr_64 && 103 vaddr_64 < memseg[i].addr_64 + memseg[i].len) { 104 paddr = memseg[i].iova + 105 (vaddr_64 - memseg[i].addr_64); 106 107 return (rte_iova_t)paddr; 108 } 109 } 110 return (rte_iova_t)(NULL); 111 } 112 113 /* virtual address conversin when mempool support is available for ctx */ 114 static inline phys_addr_t 115 dpaa_mem_vtop_ctx(struct dpaa_sec_op_ctx *ctx, void *vaddr) 116 { 117 return (uint64_t)vaddr - ctx->vtop_offset; 118 } 119 120 static inline void * 121 dpaa_mem_ptov(rte_iova_t paddr) 122 { 123 const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); 124 int i; 125 126 for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { 127 if (paddr >= memseg[i].iova && 128 (char *)paddr < (char *)memseg[i].iova + memseg[i].len) 129 return (void *)(memseg[i].addr_64 + 130 (paddr - memseg[i].iova)); 131 } 132 return NULL; 133 } 134 135 static void 136 ern_sec_fq_handler(struct qman_portal *qm __rte_unused, 137 struct qman_fq *fq, 138 const struct qm_mr_entry *msg) 139 { 140 RTE_LOG_DP(ERR, PMD, "sec fq %d error, RC = %x, seqnum = %x\n", 141 fq->fqid, msg->ern.rc, msg->ern.seqnum); 142 } 143 144 /* initialize the queue with dest chan as caam chan so that 145 * all the packets in this queue could be dispatched into caam 146 */ 147 static int 148 dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc, 149 uint32_t fqid_out) 150 { 151 struct qm_mcc_initfq fq_opts; 152 uint32_t flags; 153 int ret = -1; 154 155 /* Clear FQ options */ 156 memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq)); 157 158 flags = QMAN_INITFQ_FLAG_SCHED; 159 fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA | 160 QM_INITFQ_WE_CONTEXTB; 161 162 qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc); 163 fq_opts.fqd.context_b = fqid_out; 164 fq_opts.fqd.dest.channel = qm_channel_caam; 165 fq_opts.fqd.dest.wq = 0; 166 167 fq_in->cb.ern = ern_sec_fq_handler; 168 169 PMD_INIT_LOG(DEBUG, "in-%x out-%x", fq_in->fqid, fqid_out); 170 171 ret = qman_init_fq(fq_in, flags, &fq_opts); 172 if (unlikely(ret != 0)) 173 PMD_INIT_LOG(ERR, "qman_init_fq failed %d", ret); 174 175 return ret; 176 } 177 178 /* something is put into in_fq and caam put the crypto result into out_fq */ 179 static enum qman_cb_dqrr_result 180 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, 181 struct qman_fq *fq __always_unused, 182 const struct qm_dqrr_entry *dqrr) 183 { 184 const struct qm_fd *fd; 185 struct dpaa_sec_job *job; 186 struct dpaa_sec_op_ctx *ctx; 187 188 if (dpaa_sec_op_nb >= DPAA_SEC_BURST) 189 return qman_cb_dqrr_defer; 190 191 if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID)) 192 return qman_cb_dqrr_consume; 193 194 fd = &dqrr->fd; 195 /* sg is embedded in an op ctx, 196 * sg[0] is for output 197 * sg[1] for input 198 */ 199 job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); 200 201 ctx = container_of(job, struct dpaa_sec_op_ctx, job); 202 ctx->fd_status = fd->status; 203 if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 204 struct qm_sg_entry *sg_out; 205 uint32_t len; 206 207 sg_out = &job->sg[0]; 208 hw_sg_to_cpu(sg_out); 209 len = sg_out->length; 210 ctx->op->sym->m_src->pkt_len = len; 211 ctx->op->sym->m_src->data_len = len; 212 } 213 dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op; 214 dpaa_sec_op_ending(ctx); 215 216 return qman_cb_dqrr_consume; 217 } 218 219 /* caam result is put into this queue */ 220 static int 221 dpaa_sec_init_tx(struct qman_fq *fq) 222 { 223 int ret; 224 struct qm_mcc_initfq opts; 225 uint32_t flags; 226 227 flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED | 228 QMAN_FQ_FLAG_DYNAMIC_FQID; 229 230 ret = qman_create_fq(0, flags, fq); 231 if (unlikely(ret)) { 232 PMD_INIT_LOG(ERR, "qman_create_fq failed"); 233 return ret; 234 } 235 236 memset(&opts, 0, sizeof(opts)); 237 opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL | 238 QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB; 239 240 /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */ 241 242 fq->cb.dqrr = dqrr_out_fq_cb_rx; 243 fq->cb.ern = ern_sec_fq_handler; 244 245 ret = qman_init_fq(fq, 0, &opts); 246 if (unlikely(ret)) { 247 PMD_INIT_LOG(ERR, "unable to init caam source fq!"); 248 return ret; 249 } 250 251 return ret; 252 } 253 254 static inline int is_cipher_only(dpaa_sec_session *ses) 255 { 256 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && 257 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL)); 258 } 259 260 static inline int is_auth_only(dpaa_sec_session *ses) 261 { 262 return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) && 263 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); 264 } 265 266 static inline int is_aead(dpaa_sec_session *ses) 267 { 268 return ((ses->cipher_alg == 0) && 269 (ses->auth_alg == 0) && 270 (ses->aead_alg != 0)); 271 } 272 273 static inline int is_auth_cipher(dpaa_sec_session *ses) 274 { 275 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && 276 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) && 277 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC)); 278 } 279 280 static inline int is_proto_ipsec(dpaa_sec_session *ses) 281 { 282 return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC); 283 } 284 285 static inline int is_encode(dpaa_sec_session *ses) 286 { 287 return ses->dir == DIR_ENC; 288 } 289 290 static inline int is_decode(dpaa_sec_session *ses) 291 { 292 return ses->dir == DIR_DEC; 293 } 294 295 static inline void 296 caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a) 297 { 298 switch (ses->auth_alg) { 299 case RTE_CRYPTO_AUTH_NULL: 300 ses->digest_length = 0; 301 break; 302 case RTE_CRYPTO_AUTH_MD5_HMAC: 303 alginfo_a->algtype = 304 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 305 OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5; 306 alginfo_a->algmode = OP_ALG_AAI_HMAC; 307 break; 308 case RTE_CRYPTO_AUTH_SHA1_HMAC: 309 alginfo_a->algtype = 310 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 311 OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1; 312 alginfo_a->algmode = OP_ALG_AAI_HMAC; 313 break; 314 case RTE_CRYPTO_AUTH_SHA224_HMAC: 315 alginfo_a->algtype = 316 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 317 OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224; 318 alginfo_a->algmode = OP_ALG_AAI_HMAC; 319 break; 320 case RTE_CRYPTO_AUTH_SHA256_HMAC: 321 alginfo_a->algtype = 322 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 323 OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256; 324 alginfo_a->algmode = OP_ALG_AAI_HMAC; 325 break; 326 case RTE_CRYPTO_AUTH_SHA384_HMAC: 327 alginfo_a->algtype = 328 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 329 OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384; 330 alginfo_a->algmode = OP_ALG_AAI_HMAC; 331 break; 332 case RTE_CRYPTO_AUTH_SHA512_HMAC: 333 alginfo_a->algtype = 334 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 335 OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512; 336 alginfo_a->algmode = OP_ALG_AAI_HMAC; 337 break; 338 default: 339 PMD_INIT_LOG(ERR, "unsupported auth alg %u", ses->auth_alg); 340 } 341 } 342 343 static inline void 344 caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c) 345 { 346 switch (ses->cipher_alg) { 347 case RTE_CRYPTO_CIPHER_NULL: 348 break; 349 case RTE_CRYPTO_CIPHER_AES_CBC: 350 alginfo_c->algtype = 351 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 352 OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES; 353 alginfo_c->algmode = OP_ALG_AAI_CBC; 354 break; 355 case RTE_CRYPTO_CIPHER_3DES_CBC: 356 alginfo_c->algtype = 357 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 358 OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES; 359 alginfo_c->algmode = OP_ALG_AAI_CBC; 360 break; 361 case RTE_CRYPTO_CIPHER_AES_CTR: 362 alginfo_c->algtype = 363 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 364 OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES; 365 alginfo_c->algmode = OP_ALG_AAI_CTR; 366 break; 367 default: 368 PMD_INIT_LOG(ERR, "unsupported cipher alg %d", ses->cipher_alg); 369 } 370 } 371 372 static inline void 373 caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo) 374 { 375 switch (ses->aead_alg) { 376 case RTE_CRYPTO_AEAD_AES_GCM: 377 alginfo->algtype = OP_ALG_ALGSEL_AES; 378 alginfo->algmode = OP_ALG_AAI_GCM; 379 break; 380 default: 381 PMD_INIT_LOG(ERR, "unsupported AEAD alg %d", ses->aead_alg); 382 } 383 } 384 385 386 /* prepare command block of the session */ 387 static int 388 dpaa_sec_prep_cdb(dpaa_sec_session *ses) 389 { 390 struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0}; 391 uint32_t shared_desc_len = 0; 392 struct sec_cdb *cdb = &ses->cdb; 393 int err; 394 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 395 int swap = false; 396 #else 397 int swap = true; 398 #endif 399 400 memset(cdb, 0, sizeof(struct sec_cdb)); 401 402 if (is_cipher_only(ses)) { 403 caam_cipher_alg(ses, &alginfo_c); 404 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 405 PMD_TX_LOG(ERR, "not supported cipher alg\n"); 406 return -ENOTSUP; 407 } 408 409 alginfo_c.key = (uint64_t)ses->cipher_key.data; 410 alginfo_c.keylen = ses->cipher_key.length; 411 alginfo_c.key_enc_flags = 0; 412 alginfo_c.key_type = RTA_DATA_IMM; 413 414 shared_desc_len = cnstr_shdsc_blkcipher( 415 cdb->sh_desc, true, 416 swap, &alginfo_c, 417 NULL, 418 ses->iv.length, 419 ses->dir); 420 } else if (is_auth_only(ses)) { 421 caam_auth_alg(ses, &alginfo_a); 422 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 423 PMD_TX_LOG(ERR, "not supported auth alg\n"); 424 return -ENOTSUP; 425 } 426 427 alginfo_a.key = (uint64_t)ses->auth_key.data; 428 alginfo_a.keylen = ses->auth_key.length; 429 alginfo_a.key_enc_flags = 0; 430 alginfo_a.key_type = RTA_DATA_IMM; 431 432 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true, 433 swap, &alginfo_a, 434 !ses->dir, 435 ses->digest_length); 436 } else if (is_aead(ses)) { 437 caam_aead_alg(ses, &alginfo); 438 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 439 PMD_TX_LOG(ERR, "not supported aead alg\n"); 440 return -ENOTSUP; 441 } 442 alginfo.key = (uint64_t)ses->aead_key.data; 443 alginfo.keylen = ses->aead_key.length; 444 alginfo.key_enc_flags = 0; 445 alginfo.key_type = RTA_DATA_IMM; 446 447 if (ses->dir == DIR_ENC) 448 shared_desc_len = cnstr_shdsc_gcm_encap( 449 cdb->sh_desc, true, swap, 450 &alginfo, 451 ses->iv.length, 452 ses->digest_length); 453 else 454 shared_desc_len = cnstr_shdsc_gcm_decap( 455 cdb->sh_desc, true, swap, 456 &alginfo, 457 ses->iv.length, 458 ses->digest_length); 459 } else { 460 caam_cipher_alg(ses, &alginfo_c); 461 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 462 PMD_TX_LOG(ERR, "not supported cipher alg\n"); 463 return -ENOTSUP; 464 } 465 466 alginfo_c.key = (uint64_t)ses->cipher_key.data; 467 alginfo_c.keylen = ses->cipher_key.length; 468 alginfo_c.key_enc_flags = 0; 469 alginfo_c.key_type = RTA_DATA_IMM; 470 471 caam_auth_alg(ses, &alginfo_a); 472 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 473 PMD_TX_LOG(ERR, "not supported auth alg\n"); 474 return -ENOTSUP; 475 } 476 477 alginfo_a.key = (uint64_t)ses->auth_key.data; 478 alginfo_a.keylen = ses->auth_key.length; 479 alginfo_a.key_enc_flags = 0; 480 alginfo_a.key_type = RTA_DATA_IMM; 481 482 cdb->sh_desc[0] = alginfo_c.keylen; 483 cdb->sh_desc[1] = alginfo_a.keylen; 484 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, 485 MIN_JOB_DESC_SIZE, 486 (unsigned int *)cdb->sh_desc, 487 &cdb->sh_desc[2], 2); 488 489 if (err < 0) { 490 PMD_TX_LOG(ERR, "Crypto: Incorrect key lengths"); 491 return err; 492 } 493 if (cdb->sh_desc[2] & 1) 494 alginfo_c.key_type = RTA_DATA_IMM; 495 else { 496 alginfo_c.key = (uint64_t)dpaa_mem_vtop( 497 (void *)alginfo_c.key); 498 alginfo_c.key_type = RTA_DATA_PTR; 499 } 500 if (cdb->sh_desc[2] & (1<<1)) 501 alginfo_a.key_type = RTA_DATA_IMM; 502 else { 503 alginfo_a.key = (uint64_t)dpaa_mem_vtop( 504 (void *)alginfo_a.key); 505 alginfo_a.key_type = RTA_DATA_PTR; 506 } 507 cdb->sh_desc[0] = 0; 508 cdb->sh_desc[1] = 0; 509 cdb->sh_desc[2] = 0; 510 if (is_proto_ipsec(ses)) { 511 if (ses->dir == DIR_ENC) { 512 shared_desc_len = cnstr_shdsc_ipsec_new_encap( 513 cdb->sh_desc, 514 true, swap, &ses->encap_pdb, 515 (uint8_t *)&ses->ip4_hdr, 516 &alginfo_c, &alginfo_a); 517 } else if (ses->dir == DIR_DEC) { 518 shared_desc_len = cnstr_shdsc_ipsec_new_decap( 519 cdb->sh_desc, 520 true, swap, &ses->decap_pdb, 521 &alginfo_c, &alginfo_a); 522 } 523 } else { 524 /* Auth_only_len is set as 0 here and it will be 525 * overwritten in fd for each packet. 526 */ 527 shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc, 528 true, swap, &alginfo_c, &alginfo_a, 529 ses->iv.length, 0, 530 ses->digest_length, ses->dir); 531 } 532 } 533 cdb->sh_hdr.hi.field.idlen = shared_desc_len; 534 cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word); 535 cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word); 536 537 return 0; 538 } 539 540 /* qp is lockless, should be accessed by only one thread */ 541 static int 542 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops) 543 { 544 struct qman_fq *fq; 545 unsigned int pkts = 0; 546 int ret; 547 struct qm_dqrr_entry *dq; 548 549 fq = &qp->outq; 550 ret = qman_set_vdq(fq, (nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES) ? 551 DPAA_MAX_DEQUEUE_NUM_FRAMES : nb_ops); 552 if (ret) 553 return 0; 554 555 do { 556 const struct qm_fd *fd; 557 struct dpaa_sec_job *job; 558 struct dpaa_sec_op_ctx *ctx; 559 struct rte_crypto_op *op; 560 561 dq = qman_dequeue(fq); 562 if (!dq) 563 continue; 564 565 fd = &dq->fd; 566 /* sg is embedded in an op ctx, 567 * sg[0] is for output 568 * sg[1] for input 569 */ 570 job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); 571 572 ctx = container_of(job, struct dpaa_sec_op_ctx, job); 573 ctx->fd_status = fd->status; 574 op = ctx->op; 575 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 576 struct qm_sg_entry *sg_out; 577 uint32_t len; 578 579 sg_out = &job->sg[0]; 580 hw_sg_to_cpu(sg_out); 581 len = sg_out->length; 582 op->sym->m_src->pkt_len = len; 583 op->sym->m_src->data_len = len; 584 } 585 if (!ctx->fd_status) { 586 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 587 } else { 588 printf("\nSEC return err: 0x%x", ctx->fd_status); 589 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 590 } 591 ops[pkts++] = op; 592 593 /* report op status to sym->op and then free the ctx memeory */ 594 rte_mempool_put(ctx->ctx_pool, (void *)ctx); 595 596 qman_dqrr_consume(fq, dq); 597 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 598 599 return pkts; 600 } 601 602 /** 603 * packet looks like: 604 * |<----data_len------->| 605 * |ip_header|ah_header|icv|payload| 606 * ^ 607 * | 608 * mbuf->pkt.data 609 */ 610 static inline struct dpaa_sec_job * 611 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) 612 { 613 struct rte_crypto_sym_op *sym = op->sym; 614 struct rte_mbuf *mbuf = sym->m_src; 615 struct dpaa_sec_job *cf; 616 struct dpaa_sec_op_ctx *ctx; 617 struct qm_sg_entry *sg; 618 rte_iova_t start_addr; 619 uint8_t *old_digest; 620 621 ctx = dpaa_sec_alloc_ctx(ses); 622 if (!ctx) 623 return NULL; 624 625 cf = &ctx->job; 626 ctx->op = op; 627 old_digest = ctx->digest; 628 629 start_addr = rte_pktmbuf_iova(mbuf); 630 /* output */ 631 sg = &cf->sg[0]; 632 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); 633 sg->length = ses->digest_length; 634 cpu_to_hw_sg(sg); 635 636 /* input */ 637 sg = &cf->sg[1]; 638 if (is_decode(ses)) { 639 /* need to extend the input to a compound frame */ 640 sg->extension = 1; 641 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2])); 642 sg->length = sym->auth.data.length + ses->digest_length; 643 sg->final = 1; 644 cpu_to_hw_sg(sg); 645 646 sg = &cf->sg[2]; 647 /* hash result or digest, save digest first */ 648 rte_memcpy(old_digest, sym->auth.digest.data, 649 ses->digest_length); 650 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); 651 sg->length = sym->auth.data.length; 652 cpu_to_hw_sg(sg); 653 654 /* let's check digest by hw */ 655 start_addr = dpaa_mem_vtop_ctx(ctx, old_digest); 656 sg++; 657 qm_sg_entry_set64(sg, start_addr); 658 sg->length = ses->digest_length; 659 sg->final = 1; 660 cpu_to_hw_sg(sg); 661 } else { 662 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); 663 sg->length = sym->auth.data.length; 664 sg->final = 1; 665 cpu_to_hw_sg(sg); 666 } 667 668 return cf; 669 } 670 671 static inline struct dpaa_sec_job * 672 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses) 673 { 674 struct rte_crypto_sym_op *sym = op->sym; 675 struct dpaa_sec_job *cf; 676 struct dpaa_sec_op_ctx *ctx; 677 struct qm_sg_entry *sg; 678 rte_iova_t src_start_addr, dst_start_addr; 679 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 680 ses->iv.offset); 681 682 ctx = dpaa_sec_alloc_ctx(ses); 683 if (!ctx) 684 return NULL; 685 686 cf = &ctx->job; 687 ctx->op = op; 688 689 src_start_addr = rte_pktmbuf_iova(sym->m_src); 690 691 if (sym->m_dst) 692 dst_start_addr = rte_pktmbuf_iova(sym->m_dst); 693 else 694 dst_start_addr = src_start_addr; 695 696 /* output */ 697 sg = &cf->sg[0]; 698 qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); 699 sg->length = sym->cipher.data.length + ses->iv.length; 700 cpu_to_hw_sg(sg); 701 702 /* input */ 703 sg = &cf->sg[1]; 704 705 /* need to extend the input to a compound frame */ 706 sg->extension = 1; 707 sg->final = 1; 708 sg->length = sym->cipher.data.length + ses->iv.length; 709 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2])); 710 cpu_to_hw_sg(sg); 711 712 sg = &cf->sg[2]; 713 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 714 sg->length = ses->iv.length; 715 cpu_to_hw_sg(sg); 716 717 sg++; 718 qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset); 719 sg->length = sym->cipher.data.length; 720 sg->final = 1; 721 cpu_to_hw_sg(sg); 722 723 return cf; 724 } 725 726 static inline struct dpaa_sec_job * 727 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) 728 { 729 struct rte_crypto_sym_op *sym = op->sym; 730 struct dpaa_sec_job *cf; 731 struct dpaa_sec_op_ctx *ctx; 732 struct qm_sg_entry *sg; 733 uint32_t length = 0; 734 rte_iova_t src_start_addr, dst_start_addr; 735 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 736 ses->iv.offset); 737 738 src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off; 739 740 if (sym->m_dst) 741 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off; 742 else 743 dst_start_addr = src_start_addr; 744 745 ctx = dpaa_sec_alloc_ctx(ses); 746 if (!ctx) 747 return NULL; 748 749 cf = &ctx->job; 750 ctx->op = op; 751 752 /* input */ 753 rte_prefetch0(cf->sg); 754 sg = &cf->sg[2]; 755 qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop_ctx(ctx, sg)); 756 if (is_encode(ses)) { 757 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 758 sg->length = ses->iv.length; 759 length += sg->length; 760 cpu_to_hw_sg(sg); 761 762 sg++; 763 if (ses->auth_only_len) { 764 qm_sg_entry_set64(sg, 765 dpaa_mem_vtop(sym->aead.aad.data)); 766 sg->length = ses->auth_only_len; 767 length += sg->length; 768 cpu_to_hw_sg(sg); 769 sg++; 770 } 771 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset); 772 sg->length = sym->aead.data.length; 773 length += sg->length; 774 sg->final = 1; 775 cpu_to_hw_sg(sg); 776 } else { 777 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 778 sg->length = ses->iv.length; 779 length += sg->length; 780 cpu_to_hw_sg(sg); 781 782 sg++; 783 if (ses->auth_only_len) { 784 qm_sg_entry_set64(sg, 785 dpaa_mem_vtop(sym->aead.aad.data)); 786 sg->length = ses->auth_only_len; 787 length += sg->length; 788 cpu_to_hw_sg(sg); 789 sg++; 790 } 791 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset); 792 sg->length = sym->aead.data.length; 793 length += sg->length; 794 cpu_to_hw_sg(sg); 795 796 memcpy(ctx->digest, sym->aead.digest.data, 797 ses->digest_length); 798 sg++; 799 800 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest)); 801 sg->length = ses->digest_length; 802 length += sg->length; 803 sg->final = 1; 804 cpu_to_hw_sg(sg); 805 } 806 /* input compound frame */ 807 cf->sg[1].length = length; 808 cf->sg[1].extension = 1; 809 cf->sg[1].final = 1; 810 cpu_to_hw_sg(&cf->sg[1]); 811 812 /* output */ 813 sg++; 814 qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop_ctx(ctx, sg)); 815 qm_sg_entry_set64(sg, 816 dst_start_addr + sym->aead.data.offset - ses->auth_only_len); 817 sg->length = sym->aead.data.length + ses->auth_only_len; 818 length = sg->length; 819 if (is_encode(ses)) { 820 cpu_to_hw_sg(sg); 821 /* set auth output */ 822 sg++; 823 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr); 824 sg->length = ses->digest_length; 825 length += sg->length; 826 } 827 sg->final = 1; 828 cpu_to_hw_sg(sg); 829 830 /* output compound frame */ 831 cf->sg[0].length = length; 832 cf->sg[0].extension = 1; 833 cpu_to_hw_sg(&cf->sg[0]); 834 835 return cf; 836 } 837 838 static inline struct dpaa_sec_job * 839 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) 840 { 841 struct rte_crypto_sym_op *sym = op->sym; 842 struct dpaa_sec_job *cf; 843 struct dpaa_sec_op_ctx *ctx; 844 struct qm_sg_entry *sg; 845 rte_iova_t src_start_addr, dst_start_addr; 846 uint32_t length = 0; 847 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 848 ses->iv.offset); 849 850 src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off; 851 if (sym->m_dst) 852 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off; 853 else 854 dst_start_addr = src_start_addr; 855 856 ctx = dpaa_sec_alloc_ctx(ses); 857 if (!ctx) 858 return NULL; 859 860 cf = &ctx->job; 861 ctx->op = op; 862 863 /* input */ 864 rte_prefetch0(cf->sg); 865 sg = &cf->sg[2]; 866 qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop_ctx(ctx, sg)); 867 if (is_encode(ses)) { 868 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 869 sg->length = ses->iv.length; 870 length += sg->length; 871 cpu_to_hw_sg(sg); 872 873 sg++; 874 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset); 875 sg->length = sym->auth.data.length; 876 length += sg->length; 877 sg->final = 1; 878 cpu_to_hw_sg(sg); 879 } else { 880 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 881 sg->length = ses->iv.length; 882 length += sg->length; 883 cpu_to_hw_sg(sg); 884 885 sg++; 886 887 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset); 888 sg->length = sym->auth.data.length; 889 length += sg->length; 890 cpu_to_hw_sg(sg); 891 892 memcpy(ctx->digest, sym->auth.digest.data, 893 ses->digest_length); 894 sg++; 895 896 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest)); 897 sg->length = ses->digest_length; 898 length += sg->length; 899 sg->final = 1; 900 cpu_to_hw_sg(sg); 901 } 902 /* input compound frame */ 903 cf->sg[1].length = length; 904 cf->sg[1].extension = 1; 905 cf->sg[1].final = 1; 906 cpu_to_hw_sg(&cf->sg[1]); 907 908 /* output */ 909 sg++; 910 qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop_ctx(ctx, sg)); 911 qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); 912 sg->length = sym->cipher.data.length; 913 length = sg->length; 914 if (is_encode(ses)) { 915 cpu_to_hw_sg(sg); 916 /* set auth output */ 917 sg++; 918 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); 919 sg->length = ses->digest_length; 920 length += sg->length; 921 } 922 sg->final = 1; 923 cpu_to_hw_sg(sg); 924 925 /* output compound frame */ 926 cf->sg[0].length = length; 927 cf->sg[0].extension = 1; 928 cpu_to_hw_sg(&cf->sg[0]); 929 930 return cf; 931 } 932 933 static inline struct dpaa_sec_job * 934 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses) 935 { 936 struct rte_crypto_sym_op *sym = op->sym; 937 struct dpaa_sec_job *cf; 938 struct dpaa_sec_op_ctx *ctx; 939 struct qm_sg_entry *sg; 940 phys_addr_t src_start_addr, dst_start_addr; 941 942 ctx = dpaa_sec_alloc_ctx(ses); 943 if (!ctx) 944 return NULL; 945 cf = &ctx->job; 946 ctx->op = op; 947 948 src_start_addr = rte_pktmbuf_mtophys(sym->m_src); 949 950 if (sym->m_dst) 951 dst_start_addr = rte_pktmbuf_mtophys(sym->m_dst); 952 else 953 dst_start_addr = src_start_addr; 954 955 /* input */ 956 sg = &cf->sg[1]; 957 qm_sg_entry_set64(sg, src_start_addr); 958 sg->length = sym->m_src->pkt_len; 959 sg->final = 1; 960 cpu_to_hw_sg(sg); 961 962 sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK; 963 /* output */ 964 sg = &cf->sg[0]; 965 qm_sg_entry_set64(sg, dst_start_addr); 966 sg->length = sym->m_src->buf_len - sym->m_src->data_off; 967 cpu_to_hw_sg(sg); 968 969 return cf; 970 } 971 972 static uint16_t 973 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, 974 uint16_t nb_ops) 975 { 976 /* Function to transmit the frames to given device and queuepair */ 977 uint32_t loop; 978 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; 979 uint16_t num_tx = 0; 980 struct qm_fd fds[DPAA_SEC_BURST], *fd; 981 uint32_t frames_to_send; 982 struct rte_crypto_op *op; 983 struct dpaa_sec_job *cf; 984 dpaa_sec_session *ses; 985 struct dpaa_sec_op_ctx *ctx; 986 uint32_t auth_only_len; 987 struct qman_fq *inq[DPAA_SEC_BURST]; 988 989 while (nb_ops) { 990 frames_to_send = (nb_ops > DPAA_SEC_BURST) ? 991 DPAA_SEC_BURST : nb_ops; 992 for (loop = 0; loop < frames_to_send; loop++) { 993 op = *(ops++); 994 switch (op->sess_type) { 995 case RTE_CRYPTO_OP_WITH_SESSION: 996 ses = (dpaa_sec_session *) 997 get_session_private_data( 998 op->sym->session, 999 cryptodev_driver_id); 1000 break; 1001 case RTE_CRYPTO_OP_SECURITY_SESSION: 1002 ses = (dpaa_sec_session *) 1003 get_sec_session_private_data( 1004 op->sym->sec_session); 1005 break; 1006 default: 1007 PMD_TX_LOG(ERR, 1008 "sessionless crypto op not supported"); 1009 frames_to_send = loop; 1010 nb_ops = loop; 1011 goto send_pkts; 1012 } 1013 if (unlikely(!ses->qp || ses->qp != qp)) { 1014 PMD_INIT_LOG(DEBUG, "sess->qp - %p qp %p", 1015 ses->qp, qp); 1016 if (dpaa_sec_attach_sess_q(qp, ses)) { 1017 frames_to_send = loop; 1018 nb_ops = loop; 1019 goto send_pkts; 1020 } 1021 } 1022 1023 /* 1024 * Segmented buffer is not supported. 1025 */ 1026 if (!rte_pktmbuf_is_contiguous(op->sym->m_src)) { 1027 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1028 frames_to_send = loop; 1029 nb_ops = loop; 1030 goto send_pkts; 1031 } 1032 auth_only_len = op->sym->auth.data.length - 1033 op->sym->cipher.data.length; 1034 1035 if (is_auth_only(ses)) { 1036 cf = build_auth_only(op, ses); 1037 } else if (is_cipher_only(ses)) { 1038 cf = build_cipher_only(op, ses); 1039 } else if (is_aead(ses)) { 1040 cf = build_cipher_auth_gcm(op, ses); 1041 auth_only_len = ses->auth_only_len; 1042 } else if (is_auth_cipher(ses)) { 1043 cf = build_cipher_auth(op, ses); 1044 } else if (is_proto_ipsec(ses)) { 1045 cf = build_proto(op, ses); 1046 } else { 1047 PMD_TX_LOG(ERR, "not supported sec op"); 1048 frames_to_send = loop; 1049 nb_ops = loop; 1050 goto send_pkts; 1051 } 1052 if (unlikely(!cf)) { 1053 frames_to_send = loop; 1054 nb_ops = loop; 1055 goto send_pkts; 1056 } 1057 1058 fd = &fds[loop]; 1059 inq[loop] = ses->inq; 1060 fd->opaque_addr = 0; 1061 fd->cmd = 0; 1062 ctx = container_of(cf, struct dpaa_sec_op_ctx, job); 1063 qm_fd_addr_set64(fd, dpaa_mem_vtop_ctx(ctx, cf->sg)); 1064 fd->_format1 = qm_fd_compound; 1065 fd->length29 = 2 * sizeof(struct qm_sg_entry); 1066 /* Auth_only_len is set as 0 in descriptor and it is 1067 * overwritten here in the fd.cmd which will update 1068 * the DPOVRD reg. 1069 */ 1070 if (auth_only_len) 1071 fd->cmd = 0x80000000 | auth_only_len; 1072 1073 } 1074 send_pkts: 1075 loop = 0; 1076 while (loop < frames_to_send) { 1077 loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop], 1078 frames_to_send - loop); 1079 } 1080 nb_ops -= frames_to_send; 1081 num_tx += frames_to_send; 1082 } 1083 1084 dpaa_qp->tx_pkts += num_tx; 1085 dpaa_qp->tx_errs += nb_ops - num_tx; 1086 1087 return num_tx; 1088 } 1089 1090 static uint16_t 1091 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, 1092 uint16_t nb_ops) 1093 { 1094 uint16_t num_rx; 1095 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; 1096 1097 num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops); 1098 1099 dpaa_qp->rx_pkts += num_rx; 1100 dpaa_qp->rx_errs += nb_ops - num_rx; 1101 1102 PMD_RX_LOG(DEBUG, "SEC Received %d Packets\n", num_rx); 1103 1104 return num_rx; 1105 } 1106 1107 /** Release queue pair */ 1108 static int 1109 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev, 1110 uint16_t qp_id) 1111 { 1112 struct dpaa_sec_dev_private *internals; 1113 struct dpaa_sec_qp *qp = NULL; 1114 1115 PMD_INIT_FUNC_TRACE(); 1116 1117 PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d", dev, qp_id); 1118 1119 internals = dev->data->dev_private; 1120 if (qp_id >= internals->max_nb_queue_pairs) { 1121 PMD_INIT_LOG(ERR, "Max supported qpid %d", 1122 internals->max_nb_queue_pairs); 1123 return -EINVAL; 1124 } 1125 1126 qp = &internals->qps[qp_id]; 1127 qp->internals = NULL; 1128 dev->data->queue_pairs[qp_id] = NULL; 1129 1130 return 0; 1131 } 1132 1133 /** Setup a queue pair */ 1134 static int 1135 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, 1136 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, 1137 __rte_unused int socket_id, 1138 __rte_unused struct rte_mempool *session_pool) 1139 { 1140 struct dpaa_sec_dev_private *internals; 1141 struct dpaa_sec_qp *qp = NULL; 1142 1143 PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d, conf =%p", 1144 dev, qp_id, qp_conf); 1145 1146 internals = dev->data->dev_private; 1147 if (qp_id >= internals->max_nb_queue_pairs) { 1148 PMD_INIT_LOG(ERR, "Max supported qpid %d", 1149 internals->max_nb_queue_pairs); 1150 return -EINVAL; 1151 } 1152 1153 qp = &internals->qps[qp_id]; 1154 qp->internals = internals; 1155 dev->data->queue_pairs[qp_id] = qp; 1156 1157 return 0; 1158 } 1159 1160 /** Start queue pair */ 1161 static int 1162 dpaa_sec_queue_pair_start(__rte_unused struct rte_cryptodev *dev, 1163 __rte_unused uint16_t queue_pair_id) 1164 { 1165 PMD_INIT_FUNC_TRACE(); 1166 1167 return 0; 1168 } 1169 1170 /** Stop queue pair */ 1171 static int 1172 dpaa_sec_queue_pair_stop(__rte_unused struct rte_cryptodev *dev, 1173 __rte_unused uint16_t queue_pair_id) 1174 { 1175 PMD_INIT_FUNC_TRACE(); 1176 1177 return 0; 1178 } 1179 1180 /** Return the number of allocated queue pairs */ 1181 static uint32_t 1182 dpaa_sec_queue_pair_count(struct rte_cryptodev *dev) 1183 { 1184 PMD_INIT_FUNC_TRACE(); 1185 1186 return dev->data->nb_queue_pairs; 1187 } 1188 1189 /** Returns the size of session structure */ 1190 static unsigned int 1191 dpaa_sec_session_get_size(struct rte_cryptodev *dev __rte_unused) 1192 { 1193 PMD_INIT_FUNC_TRACE(); 1194 1195 return sizeof(dpaa_sec_session); 1196 } 1197 1198 static int 1199 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused, 1200 struct rte_crypto_sym_xform *xform, 1201 dpaa_sec_session *session) 1202 { 1203 session->cipher_alg = xform->cipher.algo; 1204 session->iv.length = xform->cipher.iv.length; 1205 session->iv.offset = xform->cipher.iv.offset; 1206 session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, 1207 RTE_CACHE_LINE_SIZE); 1208 if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) { 1209 PMD_INIT_LOG(ERR, "No Memory for cipher key\n"); 1210 return -ENOMEM; 1211 } 1212 session->cipher_key.length = xform->cipher.key.length; 1213 1214 memcpy(session->cipher_key.data, xform->cipher.key.data, 1215 xform->cipher.key.length); 1216 session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 1217 DIR_ENC : DIR_DEC; 1218 1219 return 0; 1220 } 1221 1222 static int 1223 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused, 1224 struct rte_crypto_sym_xform *xform, 1225 dpaa_sec_session *session) 1226 { 1227 session->auth_alg = xform->auth.algo; 1228 session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, 1229 RTE_CACHE_LINE_SIZE); 1230 if (session->auth_key.data == NULL && xform->auth.key.length > 0) { 1231 PMD_INIT_LOG(ERR, "No Memory for auth key\n"); 1232 return -ENOMEM; 1233 } 1234 session->auth_key.length = xform->auth.key.length; 1235 session->digest_length = xform->auth.digest_length; 1236 1237 memcpy(session->auth_key.data, xform->auth.key.data, 1238 xform->auth.key.length); 1239 session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? 1240 DIR_ENC : DIR_DEC; 1241 1242 return 0; 1243 } 1244 1245 static int 1246 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused, 1247 struct rte_crypto_sym_xform *xform, 1248 dpaa_sec_session *session) 1249 { 1250 session->aead_alg = xform->aead.algo; 1251 session->iv.length = xform->aead.iv.length; 1252 session->iv.offset = xform->aead.iv.offset; 1253 session->auth_only_len = xform->aead.aad_length; 1254 session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length, 1255 RTE_CACHE_LINE_SIZE); 1256 if (session->aead_key.data == NULL && xform->aead.key.length > 0) { 1257 PMD_INIT_LOG(ERR, "No Memory for aead key\n"); 1258 return -ENOMEM; 1259 } 1260 session->aead_key.length = xform->aead.key.length; 1261 session->digest_length = xform->aead.digest_length; 1262 1263 memcpy(session->aead_key.data, xform->aead.key.data, 1264 xform->aead.key.length); 1265 session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 1266 DIR_ENC : DIR_DEC; 1267 1268 return 0; 1269 } 1270 1271 static struct qman_fq * 1272 dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi) 1273 { 1274 unsigned int i; 1275 1276 for (i = 0; i < qi->max_nb_sessions; i++) { 1277 if (qi->inq_attach[i] == 0) { 1278 qi->inq_attach[i] = 1; 1279 return &qi->inq[i]; 1280 } 1281 } 1282 PMD_DRV_LOG(ERR, "All ses session in use %x", qi->max_nb_sessions); 1283 1284 return NULL; 1285 } 1286 1287 static int 1288 dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq) 1289 { 1290 unsigned int i; 1291 1292 for (i = 0; i < qi->max_nb_sessions; i++) { 1293 if (&qi->inq[i] == fq) { 1294 qman_retire_fq(fq, NULL); 1295 qman_oos_fq(fq); 1296 qi->inq_attach[i] = 0; 1297 return 0; 1298 } 1299 } 1300 return -1; 1301 } 1302 1303 static int 1304 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess) 1305 { 1306 int ret; 1307 1308 sess->qp = qp; 1309 ret = dpaa_sec_prep_cdb(sess); 1310 if (ret) { 1311 PMD_DRV_LOG(ERR, "Unable to prepare sec cdb"); 1312 return -1; 1313 } 1314 1315 ret = dpaa_sec_init_rx(sess->inq, dpaa_mem_vtop(&sess->cdb), 1316 qman_fq_fqid(&qp->outq)); 1317 if (ret) 1318 PMD_DRV_LOG(ERR, "Unable to init sec queue"); 1319 1320 return ret; 1321 } 1322 1323 static int 1324 dpaa_sec_qp_attach_sess(struct rte_cryptodev *dev __rte_unused, 1325 uint16_t qp_id __rte_unused, 1326 void *ses __rte_unused) 1327 { 1328 PMD_INIT_FUNC_TRACE(); 1329 return 0; 1330 } 1331 1332 static int 1333 dpaa_sec_qp_detach_sess(struct rte_cryptodev *dev, 1334 uint16_t qp_id __rte_unused, 1335 void *ses) 1336 { 1337 dpaa_sec_session *sess = ses; 1338 struct dpaa_sec_dev_private *qi = dev->data->dev_private; 1339 1340 PMD_INIT_FUNC_TRACE(); 1341 1342 if (sess->inq) 1343 dpaa_sec_detach_rxq(qi, sess->inq); 1344 sess->inq = NULL; 1345 1346 sess->qp = NULL; 1347 1348 return 0; 1349 } 1350 1351 static int 1352 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, 1353 struct rte_crypto_sym_xform *xform, void *sess) 1354 { 1355 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 1356 dpaa_sec_session *session = sess; 1357 1358 PMD_INIT_FUNC_TRACE(); 1359 1360 if (unlikely(sess == NULL)) { 1361 RTE_LOG(ERR, PMD, "invalid session struct\n"); 1362 return -EINVAL; 1363 } 1364 1365 /* Default IV length = 0 */ 1366 session->iv.length = 0; 1367 1368 /* Cipher Only */ 1369 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { 1370 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 1371 dpaa_sec_cipher_init(dev, xform, session); 1372 1373 /* Authentication Only */ 1374 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1375 xform->next == NULL) { 1376 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; 1377 dpaa_sec_auth_init(dev, xform, session); 1378 1379 /* Cipher then Authenticate */ 1380 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 1381 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 1382 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1383 dpaa_sec_cipher_init(dev, xform, session); 1384 dpaa_sec_auth_init(dev, xform->next, session); 1385 } else { 1386 PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher"); 1387 return -EINVAL; 1388 } 1389 1390 /* Authenticate then Cipher */ 1391 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1392 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 1393 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 1394 dpaa_sec_auth_init(dev, xform, session); 1395 dpaa_sec_cipher_init(dev, xform->next, session); 1396 } else { 1397 PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher"); 1398 return -EINVAL; 1399 } 1400 1401 /* AEAD operation for AES-GCM kind of Algorithms */ 1402 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && 1403 xform->next == NULL) { 1404 dpaa_sec_aead_init(dev, xform, session); 1405 1406 } else { 1407 PMD_DRV_LOG(ERR, "Invalid crypto type"); 1408 return -EINVAL; 1409 } 1410 session->ctx_pool = internals->ctx_pool; 1411 session->inq = dpaa_sec_attach_rxq(internals); 1412 if (session->inq == NULL) { 1413 PMD_DRV_LOG(ERR, "unable to attach sec queue"); 1414 goto err1; 1415 } 1416 1417 return 0; 1418 1419 err1: 1420 rte_free(session->cipher_key.data); 1421 rte_free(session->auth_key.data); 1422 memset(session, 0, sizeof(dpaa_sec_session)); 1423 1424 return -EINVAL; 1425 } 1426 1427 static int 1428 dpaa_sec_session_configure(struct rte_cryptodev *dev, 1429 struct rte_crypto_sym_xform *xform, 1430 struct rte_cryptodev_sym_session *sess, 1431 struct rte_mempool *mempool) 1432 { 1433 void *sess_private_data; 1434 int ret; 1435 1436 PMD_INIT_FUNC_TRACE(); 1437 1438 if (rte_mempool_get(mempool, &sess_private_data)) { 1439 CDEV_LOG_ERR( 1440 "Couldn't get object from session mempool"); 1441 return -ENOMEM; 1442 } 1443 1444 ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data); 1445 if (ret != 0) { 1446 PMD_DRV_LOG(ERR, "DPAA PMD: failed to configure " 1447 "session parameters"); 1448 1449 /* Return session to mempool */ 1450 rte_mempool_put(mempool, sess_private_data); 1451 return ret; 1452 } 1453 1454 set_session_private_data(sess, dev->driver_id, 1455 sess_private_data); 1456 1457 1458 return 0; 1459 } 1460 1461 /** Clear the memory of session so it doesn't leave key material behind */ 1462 static void 1463 dpaa_sec_session_clear(struct rte_cryptodev *dev, 1464 struct rte_cryptodev_sym_session *sess) 1465 { 1466 struct dpaa_sec_dev_private *qi = dev->data->dev_private; 1467 uint8_t index = dev->driver_id; 1468 void *sess_priv = get_session_private_data(sess, index); 1469 1470 PMD_INIT_FUNC_TRACE(); 1471 1472 dpaa_sec_session *s = (dpaa_sec_session *)sess_priv; 1473 1474 if (sess_priv) { 1475 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 1476 1477 if (s->inq) 1478 dpaa_sec_detach_rxq(qi, s->inq); 1479 rte_free(s->cipher_key.data); 1480 rte_free(s->auth_key.data); 1481 memset(s, 0, sizeof(dpaa_sec_session)); 1482 set_session_private_data(sess, index, NULL); 1483 rte_mempool_put(sess_mp, sess_priv); 1484 } 1485 } 1486 1487 static int 1488 dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, 1489 struct rte_security_session_conf *conf, 1490 void *sess) 1491 { 1492 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 1493 struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec; 1494 struct rte_crypto_auth_xform *auth_xform; 1495 struct rte_crypto_cipher_xform *cipher_xform; 1496 dpaa_sec_session *session = (dpaa_sec_session *)sess; 1497 1498 PMD_INIT_FUNC_TRACE(); 1499 1500 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { 1501 cipher_xform = &conf->crypto_xform->cipher; 1502 auth_xform = &conf->crypto_xform->next->auth; 1503 } else { 1504 auth_xform = &conf->crypto_xform->auth; 1505 cipher_xform = &conf->crypto_xform->next->cipher; 1506 } 1507 session->proto_alg = conf->protocol; 1508 session->cipher_key.data = rte_zmalloc(NULL, 1509 cipher_xform->key.length, 1510 RTE_CACHE_LINE_SIZE); 1511 if (session->cipher_key.data == NULL && 1512 cipher_xform->key.length > 0) { 1513 RTE_LOG(ERR, PMD, "No Memory for cipher key\n"); 1514 return -ENOMEM; 1515 } 1516 1517 session->cipher_key.length = cipher_xform->key.length; 1518 session->auth_key.data = rte_zmalloc(NULL, 1519 auth_xform->key.length, 1520 RTE_CACHE_LINE_SIZE); 1521 if (session->auth_key.data == NULL && 1522 auth_xform->key.length > 0) { 1523 RTE_LOG(ERR, PMD, "No Memory for auth key\n"); 1524 rte_free(session->cipher_key.data); 1525 return -ENOMEM; 1526 } 1527 session->auth_key.length = auth_xform->key.length; 1528 memcpy(session->cipher_key.data, cipher_xform->key.data, 1529 cipher_xform->key.length); 1530 memcpy(session->auth_key.data, auth_xform->key.data, 1531 auth_xform->key.length); 1532 1533 switch (auth_xform->algo) { 1534 case RTE_CRYPTO_AUTH_SHA1_HMAC: 1535 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; 1536 break; 1537 case RTE_CRYPTO_AUTH_MD5_HMAC: 1538 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; 1539 break; 1540 case RTE_CRYPTO_AUTH_SHA256_HMAC: 1541 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; 1542 break; 1543 case RTE_CRYPTO_AUTH_SHA384_HMAC: 1544 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; 1545 break; 1546 case RTE_CRYPTO_AUTH_SHA512_HMAC: 1547 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; 1548 break; 1549 case RTE_CRYPTO_AUTH_AES_CMAC: 1550 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC; 1551 break; 1552 case RTE_CRYPTO_AUTH_NULL: 1553 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 1554 break; 1555 case RTE_CRYPTO_AUTH_SHA224_HMAC: 1556 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 1557 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 1558 case RTE_CRYPTO_AUTH_SHA1: 1559 case RTE_CRYPTO_AUTH_SHA256: 1560 case RTE_CRYPTO_AUTH_SHA512: 1561 case RTE_CRYPTO_AUTH_SHA224: 1562 case RTE_CRYPTO_AUTH_SHA384: 1563 case RTE_CRYPTO_AUTH_MD5: 1564 case RTE_CRYPTO_AUTH_AES_GMAC: 1565 case RTE_CRYPTO_AUTH_KASUMI_F9: 1566 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 1567 case RTE_CRYPTO_AUTH_ZUC_EIA3: 1568 RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u\n", 1569 auth_xform->algo); 1570 goto out; 1571 default: 1572 RTE_LOG(ERR, PMD, "Crypto: Undefined Auth specified %u\n", 1573 auth_xform->algo); 1574 goto out; 1575 } 1576 1577 switch (cipher_xform->algo) { 1578 case RTE_CRYPTO_CIPHER_AES_CBC: 1579 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; 1580 break; 1581 case RTE_CRYPTO_CIPHER_3DES_CBC: 1582 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; 1583 break; 1584 case RTE_CRYPTO_CIPHER_AES_CTR: 1585 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; 1586 break; 1587 case RTE_CRYPTO_CIPHER_NULL: 1588 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 1589 case RTE_CRYPTO_CIPHER_3DES_ECB: 1590 case RTE_CRYPTO_CIPHER_AES_ECB: 1591 case RTE_CRYPTO_CIPHER_KASUMI_F8: 1592 RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u\n", 1593 cipher_xform->algo); 1594 goto out; 1595 default: 1596 RTE_LOG(ERR, PMD, "Crypto: Undefined Cipher specified %u\n", 1597 cipher_xform->algo); 1598 goto out; 1599 } 1600 1601 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { 1602 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) + 1603 sizeof(session->ip4_hdr)); 1604 session->ip4_hdr.ip_v = IPVERSION; 1605 session->ip4_hdr.ip_hl = 5; 1606 session->ip4_hdr.ip_len = rte_cpu_to_be_16( 1607 sizeof(session->ip4_hdr)); 1608 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp; 1609 session->ip4_hdr.ip_id = 0; 1610 session->ip4_hdr.ip_off = 0; 1611 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl; 1612 session->ip4_hdr.ip_p = (ipsec_xform->proto == 1613 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP 1614 : IPPROTO_AH; 1615 session->ip4_hdr.ip_sum = 0; 1616 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip; 1617 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip; 1618 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *) 1619 (void *)&session->ip4_hdr, 1620 sizeof(struct ip)); 1621 1622 session->encap_pdb.options = 1623 (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) | 1624 PDBOPTS_ESP_OIHI_PDB_INL | 1625 PDBOPTS_ESP_IVSRC | 1626 PDBHMO_ESP_ENCAP_DTTL; 1627 session->encap_pdb.spi = ipsec_xform->spi; 1628 session->encap_pdb.ip_hdr_len = sizeof(struct ip); 1629 1630 session->dir = DIR_ENC; 1631 } else if (ipsec_xform->direction == 1632 RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { 1633 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb)); 1634 session->decap_pdb.options = sizeof(struct ip) << 16; 1635 session->dir = DIR_DEC; 1636 } else 1637 goto out; 1638 session->ctx_pool = internals->ctx_pool; 1639 session->inq = dpaa_sec_attach_rxq(internals); 1640 if (session->inq == NULL) { 1641 PMD_DRV_LOG(ERR, "unable to attach sec queue"); 1642 goto out; 1643 } 1644 1645 1646 return 0; 1647 out: 1648 rte_free(session->auth_key.data); 1649 rte_free(session->cipher_key.data); 1650 memset(session, 0, sizeof(dpaa_sec_session)); 1651 return -1; 1652 } 1653 1654 static int 1655 dpaa_sec_security_session_create(void *dev, 1656 struct rte_security_session_conf *conf, 1657 struct rte_security_session *sess, 1658 struct rte_mempool *mempool) 1659 { 1660 void *sess_private_data; 1661 struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev; 1662 int ret; 1663 1664 if (rte_mempool_get(mempool, &sess_private_data)) { 1665 CDEV_LOG_ERR( 1666 "Couldn't get object from session mempool"); 1667 return -ENOMEM; 1668 } 1669 1670 switch (conf->protocol) { 1671 case RTE_SECURITY_PROTOCOL_IPSEC: 1672 ret = dpaa_sec_set_ipsec_session(cdev, conf, 1673 sess_private_data); 1674 break; 1675 case RTE_SECURITY_PROTOCOL_MACSEC: 1676 return -ENOTSUP; 1677 default: 1678 return -EINVAL; 1679 } 1680 if (ret != 0) { 1681 PMD_DRV_LOG(ERR, 1682 "DPAA2 PMD: failed to configure session parameters"); 1683 1684 /* Return session to mempool */ 1685 rte_mempool_put(mempool, sess_private_data); 1686 return ret; 1687 } 1688 1689 set_sec_session_private_data(sess, sess_private_data); 1690 1691 return ret; 1692 } 1693 1694 /** Clear the memory of session so it doesn't leave key material behind */ 1695 static int 1696 dpaa_sec_security_session_destroy(void *dev __rte_unused, 1697 struct rte_security_session *sess) 1698 { 1699 PMD_INIT_FUNC_TRACE(); 1700 void *sess_priv = get_sec_session_private_data(sess); 1701 1702 dpaa_sec_session *s = (dpaa_sec_session *)sess_priv; 1703 1704 if (sess_priv) { 1705 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 1706 1707 rte_free(s->cipher_key.data); 1708 rte_free(s->auth_key.data); 1709 memset(sess, 0, sizeof(dpaa_sec_session)); 1710 set_sec_session_private_data(sess, NULL); 1711 rte_mempool_put(sess_mp, sess_priv); 1712 } 1713 return 0; 1714 } 1715 1716 1717 static int 1718 dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, 1719 struct rte_cryptodev_config *config __rte_unused) 1720 { 1721 PMD_INIT_FUNC_TRACE(); 1722 1723 return 0; 1724 } 1725 1726 static int 1727 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused) 1728 { 1729 PMD_INIT_FUNC_TRACE(); 1730 return 0; 1731 } 1732 1733 static void 1734 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused) 1735 { 1736 PMD_INIT_FUNC_TRACE(); 1737 } 1738 1739 static int 1740 dpaa_sec_dev_close(struct rte_cryptodev *dev __rte_unused) 1741 { 1742 PMD_INIT_FUNC_TRACE(); 1743 return 0; 1744 } 1745 1746 static void 1747 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev, 1748 struct rte_cryptodev_info *info) 1749 { 1750 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 1751 1752 PMD_INIT_FUNC_TRACE(); 1753 if (info != NULL) { 1754 info->max_nb_queue_pairs = internals->max_nb_queue_pairs; 1755 info->feature_flags = dev->feature_flags; 1756 info->capabilities = dpaa_sec_capabilities; 1757 info->sym.max_nb_sessions = internals->max_nb_sessions; 1758 info->sym.max_nb_sessions_per_qp = 1759 RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS / 1760 RTE_DPAA_MAX_NB_SEC_QPS; 1761 info->driver_id = cryptodev_driver_id; 1762 } 1763 } 1764 1765 static struct rte_cryptodev_ops crypto_ops = { 1766 .dev_configure = dpaa_sec_dev_configure, 1767 .dev_start = dpaa_sec_dev_start, 1768 .dev_stop = dpaa_sec_dev_stop, 1769 .dev_close = dpaa_sec_dev_close, 1770 .dev_infos_get = dpaa_sec_dev_infos_get, 1771 .queue_pair_setup = dpaa_sec_queue_pair_setup, 1772 .queue_pair_release = dpaa_sec_queue_pair_release, 1773 .queue_pair_start = dpaa_sec_queue_pair_start, 1774 .queue_pair_stop = dpaa_sec_queue_pair_stop, 1775 .queue_pair_count = dpaa_sec_queue_pair_count, 1776 .session_get_size = dpaa_sec_session_get_size, 1777 .session_configure = dpaa_sec_session_configure, 1778 .session_clear = dpaa_sec_session_clear, 1779 .qp_attach_session = dpaa_sec_qp_attach_sess, 1780 .qp_detach_session = dpaa_sec_qp_detach_sess, 1781 }; 1782 1783 static const struct rte_security_capability * 1784 dpaa_sec_capabilities_get(void *device __rte_unused) 1785 { 1786 return dpaa_sec_security_cap; 1787 } 1788 1789 struct rte_security_ops dpaa_sec_security_ops = { 1790 .session_create = dpaa_sec_security_session_create, 1791 .session_update = NULL, 1792 .session_stats_get = NULL, 1793 .session_destroy = dpaa_sec_security_session_destroy, 1794 .set_pkt_metadata = NULL, 1795 .capabilities_get = dpaa_sec_capabilities_get 1796 }; 1797 1798 static int 1799 dpaa_sec_uninit(struct rte_cryptodev *dev) 1800 { 1801 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 1802 1803 if (dev == NULL) 1804 return -ENODEV; 1805 1806 rte_free(dev->security_ctx); 1807 1808 rte_mempool_free(internals->ctx_pool); 1809 rte_free(internals); 1810 1811 PMD_INIT_LOG(INFO, "Closing DPAA_SEC device %s on numa socket %u\n", 1812 dev->data->name, rte_socket_id()); 1813 1814 return 0; 1815 } 1816 1817 static int 1818 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) 1819 { 1820 struct dpaa_sec_dev_private *internals; 1821 struct rte_security_ctx *security_instance; 1822 struct dpaa_sec_qp *qp; 1823 uint32_t i, flags; 1824 int ret; 1825 char str[20]; 1826 1827 PMD_INIT_FUNC_TRACE(); 1828 1829 cryptodev->driver_id = cryptodev_driver_id; 1830 cryptodev->dev_ops = &crypto_ops; 1831 1832 cryptodev->enqueue_burst = dpaa_sec_enqueue_burst; 1833 cryptodev->dequeue_burst = dpaa_sec_dequeue_burst; 1834 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 1835 RTE_CRYPTODEV_FF_HW_ACCELERATED | 1836 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 1837 RTE_CRYPTODEV_FF_SECURITY; 1838 1839 internals = cryptodev->data->dev_private; 1840 internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS; 1841 internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS; 1842 1843 /* 1844 * For secondary processes, we don't initialise any further as primary 1845 * has already done this work. Only check we don't need a different 1846 * RX function 1847 */ 1848 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1849 PMD_INIT_LOG(DEBUG, "Device already init by primary process"); 1850 return 0; 1851 } 1852 1853 /* Initialize security_ctx only for primary process*/ 1854 security_instance = rte_malloc("rte_security_instances_ops", 1855 sizeof(struct rte_security_ctx), 0); 1856 if (security_instance == NULL) 1857 return -ENOMEM; 1858 security_instance->device = (void *)cryptodev; 1859 security_instance->ops = &dpaa_sec_security_ops; 1860 security_instance->sess_cnt = 0; 1861 cryptodev->security_ctx = security_instance; 1862 1863 for (i = 0; i < internals->max_nb_queue_pairs; i++) { 1864 /* init qman fq for queue pair */ 1865 qp = &internals->qps[i]; 1866 ret = dpaa_sec_init_tx(&qp->outq); 1867 if (ret) { 1868 PMD_INIT_LOG(ERR, "config tx of queue pair %d", i); 1869 goto init_error; 1870 } 1871 } 1872 1873 flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID | 1874 QMAN_FQ_FLAG_TO_DCPORTAL; 1875 for (i = 0; i < internals->max_nb_sessions; i++) { 1876 /* create rx qman fq for sessions*/ 1877 ret = qman_create_fq(0, flags, &internals->inq[i]); 1878 if (unlikely(ret != 0)) { 1879 PMD_INIT_LOG(ERR, "sec qman_create_fq failed"); 1880 goto init_error; 1881 } 1882 } 1883 1884 sprintf(str, "ctx_pool_%d", cryptodev->data->dev_id); 1885 internals->ctx_pool = rte_mempool_create((const char *)str, 1886 CTX_POOL_NUM_BUFS, 1887 CTX_POOL_BUF_SIZE, 1888 CTX_POOL_CACHE_SIZE, 0, 1889 NULL, NULL, NULL, NULL, 1890 SOCKET_ID_ANY, 0); 1891 if (!internals->ctx_pool) { 1892 RTE_LOG(ERR, PMD, "%s create failed\n", str); 1893 goto init_error; 1894 } 1895 1896 PMD_INIT_LOG(DEBUG, "driver %s: created\n", cryptodev->data->name); 1897 return 0; 1898 1899 init_error: 1900 PMD_INIT_LOG(ERR, "driver %s: create failed\n", cryptodev->data->name); 1901 1902 dpaa_sec_uninit(cryptodev); 1903 return -EFAULT; 1904 } 1905 1906 static int 1907 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv, 1908 struct rte_dpaa_device *dpaa_dev) 1909 { 1910 struct rte_cryptodev *cryptodev; 1911 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; 1912 1913 int retval; 1914 1915 sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id); 1916 1917 cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id()); 1918 if (cryptodev == NULL) 1919 return -ENOMEM; 1920 1921 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 1922 cryptodev->data->dev_private = rte_zmalloc_socket( 1923 "cryptodev private structure", 1924 sizeof(struct dpaa_sec_dev_private), 1925 RTE_CACHE_LINE_SIZE, 1926 rte_socket_id()); 1927 1928 if (cryptodev->data->dev_private == NULL) 1929 rte_panic("Cannot allocate memzone for private " 1930 "device data"); 1931 } 1932 1933 dpaa_dev->crypto_dev = cryptodev; 1934 cryptodev->device = &dpaa_dev->device; 1935 cryptodev->device->driver = &dpaa_drv->driver; 1936 1937 /* init user callbacks */ 1938 TAILQ_INIT(&(cryptodev->link_intr_cbs)); 1939 1940 /* if sec device version is not configured */ 1941 if (!rta_get_sec_era()) { 1942 const struct device_node *caam_node; 1943 1944 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { 1945 const uint32_t *prop = of_get_property(caam_node, 1946 "fsl,sec-era", 1947 NULL); 1948 if (prop) { 1949 rta_set_sec_era( 1950 INTL_SEC_ERA(rte_cpu_to_be_32(*prop))); 1951 break; 1952 } 1953 } 1954 } 1955 1956 /* Invoke PMD device initialization function */ 1957 retval = dpaa_sec_dev_init(cryptodev); 1958 if (retval == 0) 1959 return 0; 1960 1961 /* In case of error, cleanup is done */ 1962 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 1963 rte_free(cryptodev->data->dev_private); 1964 1965 rte_cryptodev_pmd_release_device(cryptodev); 1966 1967 return -ENXIO; 1968 } 1969 1970 static int 1971 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev) 1972 { 1973 struct rte_cryptodev *cryptodev; 1974 int ret; 1975 1976 cryptodev = dpaa_dev->crypto_dev; 1977 if (cryptodev == NULL) 1978 return -ENODEV; 1979 1980 ret = dpaa_sec_uninit(cryptodev); 1981 if (ret) 1982 return ret; 1983 1984 return rte_cryptodev_pmd_destroy(cryptodev); 1985 } 1986 1987 static struct rte_dpaa_driver rte_dpaa_sec_driver = { 1988 .drv_type = FSL_DPAA_CRYPTO, 1989 .driver = { 1990 .name = "DPAA SEC PMD" 1991 }, 1992 .probe = cryptodev_dpaa_sec_probe, 1993 .remove = cryptodev_dpaa_sec_remove, 1994 }; 1995 1996 static struct cryptodev_driver dpaa_sec_crypto_drv; 1997 1998 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver); 1999 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver, 2000 cryptodev_driver_id); 2001