1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 5 * Copyright 2017 NXP. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of NXP nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <fcntl.h> 35 #include <unistd.h> 36 #include <sched.h> 37 #include <net/if.h> 38 39 #include <rte_byteorder.h> 40 #include <rte_common.h> 41 #include <rte_cryptodev_pmd.h> 42 #include <rte_crypto.h> 43 #include <rte_cryptodev.h> 44 #include <rte_cycles.h> 45 #include <rte_dev.h> 46 #include <rte_kvargs.h> 47 #include <rte_malloc.h> 48 #include <rte_mbuf.h> 49 #include <rte_memcpy.h> 50 #include <rte_string_fns.h> 51 52 #include <fsl_usd.h> 53 #include <fsl_qman.h> 54 #include <of.h> 55 56 /* RTA header files */ 57 #include <hw/desc/common.h> 58 #include <hw/desc/algo.h> 59 #include <hw/desc/ipsec.h> 60 61 #include <rte_dpaa_bus.h> 62 #include <dpaa_sec.h> 63 #include <dpaa_sec_log.h> 64 65 enum rta_sec_era rta_sec_era; 66 67 static uint8_t cryptodev_driver_id; 68 69 static __thread struct rte_crypto_op **dpaa_sec_ops; 70 static __thread int dpaa_sec_op_nb; 71 72 static inline void 73 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx) 74 { 75 if (!ctx->fd_status) { 76 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 77 } else { 78 PMD_RX_LOG(ERR, "SEC return err: 0x%x", ctx->fd_status); 79 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR; 80 } 81 82 /* report op status to sym->op and then free the ctx memeory */ 83 rte_mempool_put(ctx->ctx_pool, (void *)ctx); 84 } 85 86 static inline struct dpaa_sec_op_ctx * 87 dpaa_sec_alloc_ctx(dpaa_sec_session *ses) 88 { 89 struct dpaa_sec_op_ctx *ctx; 90 int retval; 91 92 retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx)); 93 if (!ctx || retval) { 94 PMD_TX_LOG(ERR, "Alloc sec descriptor failed!"); 95 return NULL; 96 } 97 /* 98 * Clear SG memory. There are 16 SG entries of 16 Bytes each. 99 * one call to dcbz_64() clear 64 bytes, hence calling it 4 times 100 * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for 101 * each packet, memset is costlier than dcbz_64(). 102 */ 103 dcbz_64(&ctx->job.sg[SG_CACHELINE_0]); 104 dcbz_64(&ctx->job.sg[SG_CACHELINE_1]); 105 dcbz_64(&ctx->job.sg[SG_CACHELINE_2]); 106 dcbz_64(&ctx->job.sg[SG_CACHELINE_3]); 107 108 ctx->ctx_pool = ses->ctx_pool; 109 110 return ctx; 111 } 112 113 static inline phys_addr_t 114 dpaa_mem_vtop(void *vaddr) 115 { 116 const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); 117 uint64_t vaddr_64, paddr; 118 int i; 119 120 vaddr_64 = (uint64_t)vaddr; 121 for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { 122 if (vaddr_64 >= memseg[i].addr_64 && 123 vaddr_64 < memseg[i].addr_64 + memseg[i].len) { 124 paddr = memseg[i].phys_addr + 125 (vaddr_64 - memseg[i].addr_64); 126 127 return (phys_addr_t)paddr; 128 } 129 } 130 return (phys_addr_t)(NULL); 131 } 132 133 static inline void * 134 dpaa_mem_ptov(phys_addr_t paddr) 135 { 136 const struct rte_memseg *memseg = rte_eal_get_physmem_layout(); 137 int i; 138 139 for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) { 140 if (paddr >= memseg[i].phys_addr && 141 (char *)paddr < (char *)memseg[i].phys_addr + memseg[i].len) 142 return (void *)(memseg[i].addr_64 + 143 (paddr - memseg[i].phys_addr)); 144 } 145 return NULL; 146 } 147 148 static void 149 ern_sec_fq_handler(struct qman_portal *qm __rte_unused, 150 struct qman_fq *fq, 151 const struct qm_mr_entry *msg) 152 { 153 RTE_LOG_DP(ERR, PMD, "sec fq %d error, RC = %x, seqnum = %x\n", 154 fq->fqid, msg->ern.rc, msg->ern.seqnum); 155 } 156 157 /* initialize the queue with dest chan as caam chan so that 158 * all the packets in this queue could be dispatched into caam 159 */ 160 static int 161 dpaa_sec_init_rx(struct qman_fq *fq_in, phys_addr_t hwdesc, 162 uint32_t fqid_out) 163 { 164 struct qm_mcc_initfq fq_opts; 165 uint32_t flags; 166 int ret = -1; 167 168 /* Clear FQ options */ 169 memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq)); 170 171 flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID | 172 QMAN_FQ_FLAG_TO_DCPORTAL; 173 174 ret = qman_create_fq(0, flags, fq_in); 175 if (unlikely(ret != 0)) { 176 PMD_INIT_LOG(ERR, "qman_create_fq failed"); 177 return ret; 178 } 179 180 flags = QMAN_INITFQ_FLAG_SCHED; 181 fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA | 182 QM_INITFQ_WE_CONTEXTB; 183 184 qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc); 185 fq_opts.fqd.context_b = fqid_out; 186 fq_opts.fqd.dest.channel = qm_channel_caam; 187 fq_opts.fqd.dest.wq = 0; 188 189 fq_in->cb.ern = ern_sec_fq_handler; 190 191 ret = qman_init_fq(fq_in, flags, &fq_opts); 192 if (unlikely(ret != 0)) 193 PMD_INIT_LOG(ERR, "qman_init_fq failed"); 194 195 return ret; 196 } 197 198 /* something is put into in_fq and caam put the crypto result into out_fq */ 199 static enum qman_cb_dqrr_result 200 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, 201 struct qman_fq *fq __always_unused, 202 const struct qm_dqrr_entry *dqrr) 203 { 204 const struct qm_fd *fd; 205 struct dpaa_sec_job *job; 206 struct dpaa_sec_op_ctx *ctx; 207 208 if (dpaa_sec_op_nb >= DPAA_SEC_BURST) 209 return qman_cb_dqrr_defer; 210 211 if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID)) 212 return qman_cb_dqrr_consume; 213 214 fd = &dqrr->fd; 215 /* sg is embedded in an op ctx, 216 * sg[0] is for output 217 * sg[1] for input 218 */ 219 job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); 220 ctx = container_of(job, struct dpaa_sec_op_ctx, job); 221 ctx->fd_status = fd->status; 222 dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op; 223 dpaa_sec_op_ending(ctx); 224 225 return qman_cb_dqrr_consume; 226 } 227 228 /* caam result is put into this queue */ 229 static int 230 dpaa_sec_init_tx(struct qman_fq *fq) 231 { 232 int ret; 233 struct qm_mcc_initfq opts; 234 uint32_t flags; 235 236 flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED | 237 QMAN_FQ_FLAG_DYNAMIC_FQID; 238 239 ret = qman_create_fq(0, flags, fq); 240 if (unlikely(ret)) { 241 PMD_INIT_LOG(ERR, "qman_create_fq failed"); 242 return ret; 243 } 244 245 memset(&opts, 0, sizeof(opts)); 246 opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL | 247 QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB; 248 249 /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */ 250 251 fq->cb.dqrr = dqrr_out_fq_cb_rx; 252 fq->cb.ern = ern_sec_fq_handler; 253 254 ret = qman_init_fq(fq, 0, &opts); 255 if (unlikely(ret)) { 256 PMD_INIT_LOG(ERR, "unable to init caam source fq!"); 257 return ret; 258 } 259 260 return ret; 261 } 262 263 static inline int is_cipher_only(dpaa_sec_session *ses) 264 { 265 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && 266 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL)); 267 } 268 269 static inline int is_auth_only(dpaa_sec_session *ses) 270 { 271 return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) && 272 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); 273 } 274 275 static inline int is_aead(dpaa_sec_session *ses) 276 { 277 return ((ses->cipher_alg == 0) && 278 (ses->auth_alg == 0) && 279 (ses->aead_alg != 0)); 280 } 281 282 static inline int is_auth_cipher(dpaa_sec_session *ses) 283 { 284 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && 285 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); 286 } 287 288 static inline int is_encode(dpaa_sec_session *ses) 289 { 290 return ses->dir == DIR_ENC; 291 } 292 293 static inline int is_decode(dpaa_sec_session *ses) 294 { 295 return ses->dir == DIR_DEC; 296 } 297 298 static inline void 299 caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a) 300 { 301 switch (ses->auth_alg) { 302 case RTE_CRYPTO_AUTH_NULL: 303 ses->digest_length = 0; 304 break; 305 case RTE_CRYPTO_AUTH_MD5_HMAC: 306 alginfo_a->algtype = OP_ALG_ALGSEL_MD5; 307 alginfo_a->algmode = OP_ALG_AAI_HMAC; 308 break; 309 case RTE_CRYPTO_AUTH_SHA1_HMAC: 310 alginfo_a->algtype = OP_ALG_ALGSEL_SHA1; 311 alginfo_a->algmode = OP_ALG_AAI_HMAC; 312 break; 313 case RTE_CRYPTO_AUTH_SHA224_HMAC: 314 alginfo_a->algtype = OP_ALG_ALGSEL_SHA224; 315 alginfo_a->algmode = OP_ALG_AAI_HMAC; 316 break; 317 case RTE_CRYPTO_AUTH_SHA256_HMAC: 318 alginfo_a->algtype = OP_ALG_ALGSEL_SHA256; 319 alginfo_a->algmode = OP_ALG_AAI_HMAC; 320 break; 321 case RTE_CRYPTO_AUTH_SHA384_HMAC: 322 alginfo_a->algtype = OP_ALG_ALGSEL_SHA384; 323 alginfo_a->algmode = OP_ALG_AAI_HMAC; 324 break; 325 case RTE_CRYPTO_AUTH_SHA512_HMAC: 326 alginfo_a->algtype = OP_ALG_ALGSEL_SHA512; 327 alginfo_a->algmode = OP_ALG_AAI_HMAC; 328 break; 329 default: 330 PMD_INIT_LOG(ERR, "unsupported auth alg %u", ses->auth_alg); 331 } 332 } 333 334 static inline void 335 caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c) 336 { 337 switch (ses->cipher_alg) { 338 case RTE_CRYPTO_CIPHER_NULL: 339 break; 340 case RTE_CRYPTO_CIPHER_AES_CBC: 341 alginfo_c->algtype = OP_ALG_ALGSEL_AES; 342 alginfo_c->algmode = OP_ALG_AAI_CBC; 343 break; 344 case RTE_CRYPTO_CIPHER_3DES_CBC: 345 alginfo_c->algtype = OP_ALG_ALGSEL_3DES; 346 alginfo_c->algmode = OP_ALG_AAI_CBC; 347 break; 348 case RTE_CRYPTO_CIPHER_AES_CTR: 349 alginfo_c->algtype = OP_ALG_ALGSEL_AES; 350 alginfo_c->algmode = OP_ALG_AAI_CTR; 351 break; 352 default: 353 PMD_INIT_LOG(ERR, "unsupported cipher alg %d", ses->cipher_alg); 354 } 355 } 356 357 static inline void 358 caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo) 359 { 360 switch (ses->aead_alg) { 361 case RTE_CRYPTO_AEAD_AES_GCM: 362 alginfo->algtype = OP_ALG_ALGSEL_AES; 363 alginfo->algmode = OP_ALG_AAI_GCM; 364 break; 365 default: 366 PMD_INIT_LOG(ERR, "unsupported AEAD alg %d", ses->aead_alg); 367 } 368 } 369 370 371 /* prepare command block of the session */ 372 static int 373 dpaa_sec_prep_cdb(dpaa_sec_session *ses) 374 { 375 struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0}; 376 uint32_t shared_desc_len = 0; 377 struct sec_cdb *cdb = &ses->qp->cdb; 378 int err; 379 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 380 int swap = false; 381 #else 382 int swap = true; 383 #endif 384 385 memset(cdb, 0, sizeof(struct sec_cdb)); 386 387 if (is_cipher_only(ses)) { 388 caam_cipher_alg(ses, &alginfo_c); 389 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 390 PMD_TX_LOG(ERR, "not supported cipher alg\n"); 391 return -ENOTSUP; 392 } 393 394 alginfo_c.key = (uint64_t)ses->cipher_key.data; 395 alginfo_c.keylen = ses->cipher_key.length; 396 alginfo_c.key_enc_flags = 0; 397 alginfo_c.key_type = RTA_DATA_IMM; 398 399 shared_desc_len = cnstr_shdsc_blkcipher( 400 cdb->sh_desc, true, 401 swap, &alginfo_c, 402 NULL, 403 ses->iv.length, 404 ses->dir); 405 } else if (is_auth_only(ses)) { 406 caam_auth_alg(ses, &alginfo_a); 407 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 408 PMD_TX_LOG(ERR, "not supported auth alg\n"); 409 return -ENOTSUP; 410 } 411 412 alginfo_a.key = (uint64_t)ses->auth_key.data; 413 alginfo_a.keylen = ses->auth_key.length; 414 alginfo_a.key_enc_flags = 0; 415 alginfo_a.key_type = RTA_DATA_IMM; 416 417 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true, 418 swap, &alginfo_a, 419 !ses->dir, 420 ses->digest_length); 421 } else if (is_aead(ses)) { 422 caam_aead_alg(ses, &alginfo); 423 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 424 PMD_TX_LOG(ERR, "not supported aead alg\n"); 425 return -ENOTSUP; 426 } 427 alginfo.key = (uint64_t)ses->aead_key.data; 428 alginfo.keylen = ses->aead_key.length; 429 alginfo.key_enc_flags = 0; 430 alginfo.key_type = RTA_DATA_IMM; 431 432 if (ses->dir == DIR_ENC) 433 shared_desc_len = cnstr_shdsc_gcm_encap( 434 cdb->sh_desc, true, swap, 435 &alginfo, 436 ses->iv.length, 437 ses->digest_length); 438 else 439 shared_desc_len = cnstr_shdsc_gcm_decap( 440 cdb->sh_desc, true, swap, 441 &alginfo, 442 ses->iv.length, 443 ses->digest_length); 444 } else { 445 caam_cipher_alg(ses, &alginfo_c); 446 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 447 PMD_TX_LOG(ERR, "not supported cipher alg\n"); 448 return -ENOTSUP; 449 } 450 451 alginfo_c.key = (uint64_t)ses->cipher_key.data; 452 alginfo_c.keylen = ses->cipher_key.length; 453 alginfo_c.key_enc_flags = 0; 454 alginfo_c.key_type = RTA_DATA_IMM; 455 456 caam_auth_alg(ses, &alginfo_a); 457 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 458 PMD_TX_LOG(ERR, "not supported auth alg\n"); 459 return -ENOTSUP; 460 } 461 462 alginfo_a.key = (uint64_t)ses->auth_key.data; 463 alginfo_a.keylen = ses->auth_key.length; 464 alginfo_a.key_enc_flags = 0; 465 alginfo_a.key_type = RTA_DATA_IMM; 466 467 cdb->sh_desc[0] = alginfo_c.keylen; 468 cdb->sh_desc[1] = alginfo_a.keylen; 469 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, 470 MIN_JOB_DESC_SIZE, 471 (unsigned int *)cdb->sh_desc, 472 &cdb->sh_desc[2], 2); 473 474 if (err < 0) { 475 PMD_TX_LOG(ERR, "Crypto: Incorrect key lengths"); 476 return err; 477 } 478 if (cdb->sh_desc[2] & 1) 479 alginfo_c.key_type = RTA_DATA_IMM; 480 else { 481 alginfo_c.key = (uint64_t)dpaa_mem_vtop( 482 (void *)alginfo_c.key); 483 alginfo_c.key_type = RTA_DATA_PTR; 484 } 485 if (cdb->sh_desc[2] & (1<<1)) 486 alginfo_a.key_type = RTA_DATA_IMM; 487 else { 488 alginfo_a.key = (uint64_t)dpaa_mem_vtop( 489 (void *)alginfo_a.key); 490 alginfo_a.key_type = RTA_DATA_PTR; 491 } 492 cdb->sh_desc[0] = 0; 493 cdb->sh_desc[1] = 0; 494 cdb->sh_desc[2] = 0; 495 496 /* Auth_only_len is set as 0 here and it will be overwritten 497 * in fd for each packet. 498 */ 499 shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc, 500 true, swap, &alginfo_c, &alginfo_a, 501 ses->iv.length, 0, 502 ses->digest_length, ses->dir); 503 } 504 cdb->sh_hdr.hi.field.idlen = shared_desc_len; 505 cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word); 506 cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word); 507 508 return 0; 509 } 510 511 static inline unsigned int 512 dpaa_volatile_deq(struct qman_fq *fq, unsigned int len, bool exact) 513 { 514 unsigned int pkts = 0; 515 int ret; 516 struct qm_mcr_queryfq_np np; 517 enum qman_fq_state state; 518 uint32_t flags; 519 uint32_t vdqcr; 520 521 qman_query_fq_np(fq, &np); 522 if (np.frm_cnt) { 523 vdqcr = QM_VDQCR_NUMFRAMES_SET(len); 524 if (exact) 525 vdqcr |= QM_VDQCR_EXACT; 526 ret = qman_volatile_dequeue(fq, 0, vdqcr); 527 if (ret) 528 return 0; 529 do { 530 pkts += qman_poll_dqrr(len); 531 qman_fq_state(fq, &state, &flags); 532 } while (flags & QMAN_FQ_STATE_VDQCR); 533 } 534 return pkts; 535 } 536 537 /* qp is lockless, should be accessed by only one thread */ 538 static int 539 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops) 540 { 541 struct qman_fq *fq; 542 543 fq = &qp->outq; 544 dpaa_sec_op_nb = 0; 545 dpaa_sec_ops = ops; 546 547 if (unlikely(nb_ops > DPAA_SEC_BURST)) 548 nb_ops = DPAA_SEC_BURST; 549 550 return dpaa_volatile_deq(fq, nb_ops, 1); 551 } 552 553 /** 554 * packet looks like: 555 * |<----data_len------->| 556 * |ip_header|ah_header|icv|payload| 557 * ^ 558 * | 559 * mbuf->pkt.data 560 */ 561 static inline struct dpaa_sec_job * 562 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) 563 { 564 struct rte_crypto_sym_op *sym = op->sym; 565 struct rte_mbuf *mbuf = sym->m_src; 566 struct dpaa_sec_job *cf; 567 struct dpaa_sec_op_ctx *ctx; 568 struct qm_sg_entry *sg; 569 phys_addr_t start_addr; 570 uint8_t *old_digest; 571 572 ctx = dpaa_sec_alloc_ctx(ses); 573 if (!ctx) 574 return NULL; 575 576 cf = &ctx->job; 577 ctx->op = op; 578 old_digest = ctx->digest; 579 580 start_addr = rte_pktmbuf_mtophys(mbuf); 581 /* output */ 582 sg = &cf->sg[0]; 583 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); 584 sg->length = ses->digest_length; 585 cpu_to_hw_sg(sg); 586 587 /* input */ 588 sg = &cf->sg[1]; 589 if (is_decode(ses)) { 590 /* need to extend the input to a compound frame */ 591 sg->extension = 1; 592 qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2])); 593 sg->length = sym->auth.data.length + ses->digest_length; 594 sg->final = 1; 595 cpu_to_hw_sg(sg); 596 597 sg = &cf->sg[2]; 598 /* hash result or digest, save digest first */ 599 rte_memcpy(old_digest, sym->auth.digest.data, 600 ses->digest_length); 601 memset(sym->auth.digest.data, 0, ses->digest_length); 602 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); 603 sg->length = sym->auth.data.length; 604 cpu_to_hw_sg(sg); 605 606 /* let's check digest by hw */ 607 start_addr = dpaa_mem_vtop(old_digest); 608 sg++; 609 qm_sg_entry_set64(sg, start_addr); 610 sg->length = ses->digest_length; 611 sg->final = 1; 612 cpu_to_hw_sg(sg); 613 } else { 614 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); 615 sg->length = sym->auth.data.length; 616 sg->final = 1; 617 cpu_to_hw_sg(sg); 618 } 619 620 return cf; 621 } 622 623 static inline struct dpaa_sec_job * 624 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses) 625 { 626 struct rte_crypto_sym_op *sym = op->sym; 627 struct rte_mbuf *mbuf = sym->m_src; 628 struct dpaa_sec_job *cf; 629 struct dpaa_sec_op_ctx *ctx; 630 struct qm_sg_entry *sg; 631 phys_addr_t start_addr; 632 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 633 ses->iv.offset); 634 635 ctx = dpaa_sec_alloc_ctx(ses); 636 if (!ctx) 637 return NULL; 638 639 cf = &ctx->job; 640 ctx->op = op; 641 start_addr = rte_pktmbuf_mtophys(mbuf); 642 643 /* output */ 644 sg = &cf->sg[0]; 645 qm_sg_entry_set64(sg, start_addr + sym->cipher.data.offset); 646 sg->length = sym->cipher.data.length + ses->iv.length; 647 cpu_to_hw_sg(sg); 648 649 /* input */ 650 sg = &cf->sg[1]; 651 652 /* need to extend the input to a compound frame */ 653 sg->extension = 1; 654 sg->final = 1; 655 sg->length = sym->cipher.data.length + ses->iv.length; 656 qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2])); 657 cpu_to_hw_sg(sg); 658 659 sg = &cf->sg[2]; 660 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 661 sg->length = ses->iv.length; 662 cpu_to_hw_sg(sg); 663 664 sg++; 665 qm_sg_entry_set64(sg, start_addr + sym->cipher.data.offset); 666 sg->length = sym->cipher.data.length; 667 sg->final = 1; 668 cpu_to_hw_sg(sg); 669 670 return cf; 671 } 672 673 static inline struct dpaa_sec_job * 674 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) 675 { 676 struct rte_crypto_sym_op *sym = op->sym; 677 struct rte_mbuf *mbuf = sym->m_src; 678 struct dpaa_sec_job *cf; 679 struct dpaa_sec_op_ctx *ctx; 680 struct qm_sg_entry *sg; 681 phys_addr_t start_addr; 682 uint32_t length = 0; 683 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 684 ses->iv.offset); 685 686 start_addr = mbuf->buf_physaddr + mbuf->data_off; 687 688 ctx = dpaa_sec_alloc_ctx(ses); 689 if (!ctx) 690 return NULL; 691 692 cf = &ctx->job; 693 ctx->op = op; 694 695 /* input */ 696 rte_prefetch0(cf->sg); 697 sg = &cf->sg[2]; 698 qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); 699 if (is_encode(ses)) { 700 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 701 sg->length = ses->iv.length; 702 length += sg->length; 703 cpu_to_hw_sg(sg); 704 705 sg++; 706 if (ses->auth_only_len) { 707 qm_sg_entry_set64(sg, 708 dpaa_mem_vtop(sym->aead.aad.data)); 709 sg->length = ses->auth_only_len; 710 length += sg->length; 711 cpu_to_hw_sg(sg); 712 sg++; 713 } 714 qm_sg_entry_set64(sg, start_addr + sym->aead.data.offset); 715 sg->length = sym->aead.data.length; 716 length += sg->length; 717 sg->final = 1; 718 cpu_to_hw_sg(sg); 719 } else { 720 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 721 sg->length = ses->iv.length; 722 length += sg->length; 723 cpu_to_hw_sg(sg); 724 725 sg++; 726 if (ses->auth_only_len) { 727 qm_sg_entry_set64(sg, 728 dpaa_mem_vtop(sym->aead.aad.data)); 729 sg->length = ses->auth_only_len; 730 length += sg->length; 731 cpu_to_hw_sg(sg); 732 sg++; 733 } 734 qm_sg_entry_set64(sg, start_addr + sym->aead.data.offset); 735 sg->length = sym->aead.data.length; 736 length += sg->length; 737 cpu_to_hw_sg(sg); 738 739 memcpy(ctx->digest, sym->aead.digest.data, 740 ses->digest_length); 741 memset(sym->aead.digest.data, 0, ses->digest_length); 742 sg++; 743 744 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); 745 sg->length = ses->digest_length; 746 length += sg->length; 747 sg->final = 1; 748 cpu_to_hw_sg(sg); 749 } 750 /* input compound frame */ 751 cf->sg[1].length = length; 752 cf->sg[1].extension = 1; 753 cf->sg[1].final = 1; 754 cpu_to_hw_sg(&cf->sg[1]); 755 756 /* output */ 757 sg++; 758 qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); 759 qm_sg_entry_set64(sg, 760 start_addr + sym->aead.data.offset - ses->auth_only_len); 761 sg->length = sym->aead.data.length + ses->auth_only_len; 762 length = sg->length; 763 if (is_encode(ses)) { 764 cpu_to_hw_sg(sg); 765 /* set auth output */ 766 sg++; 767 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr); 768 sg->length = ses->digest_length; 769 length += sg->length; 770 } 771 sg->final = 1; 772 cpu_to_hw_sg(sg); 773 774 /* output compound frame */ 775 cf->sg[0].length = length; 776 cf->sg[0].extension = 1; 777 cpu_to_hw_sg(&cf->sg[0]); 778 779 return cf; 780 } 781 782 static inline struct dpaa_sec_job * 783 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) 784 { 785 struct rte_crypto_sym_op *sym = op->sym; 786 struct rte_mbuf *mbuf = sym->m_src; 787 struct dpaa_sec_job *cf; 788 struct dpaa_sec_op_ctx *ctx; 789 struct qm_sg_entry *sg; 790 phys_addr_t start_addr; 791 uint32_t length = 0; 792 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 793 ses->iv.offset); 794 795 start_addr = mbuf->buf_physaddr + mbuf->data_off; 796 797 ctx = dpaa_sec_alloc_ctx(ses); 798 if (!ctx) 799 return NULL; 800 801 cf = &ctx->job; 802 ctx->op = op; 803 804 /* input */ 805 rte_prefetch0(cf->sg); 806 sg = &cf->sg[2]; 807 qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); 808 if (is_encode(ses)) { 809 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 810 sg->length = ses->iv.length; 811 length += sg->length; 812 cpu_to_hw_sg(sg); 813 814 sg++; 815 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); 816 sg->length = sym->auth.data.length; 817 length += sg->length; 818 sg->final = 1; 819 cpu_to_hw_sg(sg); 820 } else { 821 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 822 sg->length = ses->iv.length; 823 length += sg->length; 824 cpu_to_hw_sg(sg); 825 826 sg++; 827 828 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); 829 sg->length = sym->auth.data.length; 830 length += sg->length; 831 cpu_to_hw_sg(sg); 832 833 memcpy(ctx->digest, sym->auth.digest.data, 834 ses->digest_length); 835 memset(sym->auth.digest.data, 0, ses->digest_length); 836 sg++; 837 838 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); 839 sg->length = ses->digest_length; 840 length += sg->length; 841 sg->final = 1; 842 cpu_to_hw_sg(sg); 843 } 844 /* input compound frame */ 845 cf->sg[1].length = length; 846 cf->sg[1].extension = 1; 847 cf->sg[1].final = 1; 848 cpu_to_hw_sg(&cf->sg[1]); 849 850 /* output */ 851 sg++; 852 qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); 853 qm_sg_entry_set64(sg, start_addr + sym->cipher.data.offset); 854 sg->length = sym->cipher.data.length; 855 length = sg->length; 856 if (is_encode(ses)) { 857 cpu_to_hw_sg(sg); 858 /* set auth output */ 859 sg++; 860 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); 861 sg->length = ses->digest_length; 862 length += sg->length; 863 } 864 sg->final = 1; 865 cpu_to_hw_sg(sg); 866 867 /* output compound frame */ 868 cf->sg[0].length = length; 869 cf->sg[0].extension = 1; 870 cpu_to_hw_sg(&cf->sg[0]); 871 872 return cf; 873 } 874 875 static int 876 dpaa_sec_enqueue_op(struct rte_crypto_op *op, struct dpaa_sec_qp *qp) 877 { 878 struct dpaa_sec_job *cf; 879 dpaa_sec_session *ses; 880 struct qm_fd fd; 881 int ret; 882 uint32_t auth_only_len = op->sym->auth.data.length - 883 op->sym->cipher.data.length; 884 885 ses = (dpaa_sec_session *)get_session_private_data(op->sym->session, 886 cryptodev_driver_id); 887 888 if (unlikely(!qp->ses || qp->ses != ses)) { 889 qp->ses = ses; 890 ses->qp = qp; 891 ret = dpaa_sec_prep_cdb(ses); 892 if (ret) 893 return ret; 894 } 895 896 if (is_auth_only(ses)) { 897 cf = build_auth_only(op, ses); 898 } else if (is_cipher_only(ses)) { 899 cf = build_cipher_only(op, ses); 900 } else if (is_aead(ses)) { 901 cf = build_cipher_auth_gcm(op, ses); 902 auth_only_len = ses->auth_only_len; 903 } else if (is_auth_cipher(ses)) { 904 cf = build_cipher_auth(op, ses); 905 } else { 906 PMD_TX_LOG(ERR, "not supported sec op"); 907 return -ENOTSUP; 908 } 909 if (unlikely(!cf)) 910 return -ENOMEM; 911 912 memset(&fd, 0, sizeof(struct qm_fd)); 913 qm_fd_addr_set64(&fd, dpaa_mem_vtop(cf->sg)); 914 fd._format1 = qm_fd_compound; 915 fd.length29 = 2 * sizeof(struct qm_sg_entry); 916 /* Auth_only_len is set as 0 in descriptor and it is overwritten 917 * here in the fd.cmd which will update the DPOVRD reg. 918 */ 919 if (auth_only_len) 920 fd.cmd = 0x80000000 | auth_only_len; 921 do { 922 ret = qman_enqueue(&qp->inq, &fd, 0); 923 } while (ret != 0); 924 925 return 0; 926 } 927 928 static uint16_t 929 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, 930 uint16_t nb_ops) 931 { 932 /* Function to transmit the frames to given device and queuepair */ 933 uint32_t loop; 934 int32_t ret; 935 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; 936 uint16_t num_tx = 0; 937 938 if (unlikely(nb_ops == 0)) 939 return 0; 940 941 /*Prepare each packet which is to be sent*/ 942 for (loop = 0; loop < nb_ops; loop++) { 943 if (ops[loop]->sess_type != RTE_CRYPTO_OP_WITH_SESSION) { 944 PMD_TX_LOG(ERR, "sessionless crypto op not supported"); 945 return 0; 946 } 947 ret = dpaa_sec_enqueue_op(ops[loop], dpaa_qp); 948 if (!ret) 949 num_tx++; 950 } 951 dpaa_qp->tx_pkts += num_tx; 952 dpaa_qp->tx_errs += nb_ops - num_tx; 953 954 return num_tx; 955 } 956 957 static uint16_t 958 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, 959 uint16_t nb_ops) 960 { 961 uint16_t num_rx; 962 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; 963 964 num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops); 965 966 dpaa_qp->rx_pkts += num_rx; 967 dpaa_qp->rx_errs += nb_ops - num_rx; 968 969 PMD_RX_LOG(DEBUG, "SEC Received %d Packets\n", num_rx); 970 971 return num_rx; 972 } 973 974 /** Release queue pair */ 975 static int 976 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev, 977 uint16_t qp_id) 978 { 979 struct dpaa_sec_dev_private *internals; 980 struct dpaa_sec_qp *qp = NULL; 981 982 PMD_INIT_FUNC_TRACE(); 983 984 PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d", dev, qp_id); 985 986 internals = dev->data->dev_private; 987 if (qp_id >= internals->max_nb_queue_pairs) { 988 PMD_INIT_LOG(ERR, "Max supported qpid %d", 989 internals->max_nb_queue_pairs); 990 return -EINVAL; 991 } 992 993 qp = &internals->qps[qp_id]; 994 qp->internals = NULL; 995 dev->data->queue_pairs[qp_id] = NULL; 996 997 return 0; 998 } 999 1000 /** Setup a queue pair */ 1001 static int 1002 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, 1003 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, 1004 __rte_unused int socket_id, 1005 __rte_unused struct rte_mempool *session_pool) 1006 { 1007 struct dpaa_sec_dev_private *internals; 1008 struct dpaa_sec_qp *qp = NULL; 1009 1010 PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d, conf =%p", 1011 dev, qp_id, qp_conf); 1012 1013 internals = dev->data->dev_private; 1014 if (qp_id >= internals->max_nb_queue_pairs) { 1015 PMD_INIT_LOG(ERR, "Max supported qpid %d", 1016 internals->max_nb_queue_pairs); 1017 return -EINVAL; 1018 } 1019 1020 qp = &internals->qps[qp_id]; 1021 qp->internals = internals; 1022 dev->data->queue_pairs[qp_id] = qp; 1023 1024 return 0; 1025 } 1026 1027 /** Start queue pair */ 1028 static int 1029 dpaa_sec_queue_pair_start(__rte_unused struct rte_cryptodev *dev, 1030 __rte_unused uint16_t queue_pair_id) 1031 { 1032 PMD_INIT_FUNC_TRACE(); 1033 1034 return 0; 1035 } 1036 1037 /** Stop queue pair */ 1038 static int 1039 dpaa_sec_queue_pair_stop(__rte_unused struct rte_cryptodev *dev, 1040 __rte_unused uint16_t queue_pair_id) 1041 { 1042 PMD_INIT_FUNC_TRACE(); 1043 1044 return 0; 1045 } 1046 1047 /** Return the number of allocated queue pairs */ 1048 static uint32_t 1049 dpaa_sec_queue_pair_count(struct rte_cryptodev *dev) 1050 { 1051 PMD_INIT_FUNC_TRACE(); 1052 1053 return dev->data->nb_queue_pairs; 1054 } 1055 1056 /** Returns the size of session structure */ 1057 static unsigned int 1058 dpaa_sec_session_get_size(struct rte_cryptodev *dev __rte_unused) 1059 { 1060 PMD_INIT_FUNC_TRACE(); 1061 1062 return sizeof(dpaa_sec_session); 1063 } 1064 1065 static int 1066 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused, 1067 struct rte_crypto_sym_xform *xform, 1068 dpaa_sec_session *session) 1069 { 1070 session->cipher_alg = xform->cipher.algo; 1071 session->iv.length = xform->cipher.iv.length; 1072 session->iv.offset = xform->cipher.iv.offset; 1073 session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, 1074 RTE_CACHE_LINE_SIZE); 1075 if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) { 1076 PMD_INIT_LOG(ERR, "No Memory for cipher key\n"); 1077 return -ENOMEM; 1078 } 1079 session->cipher_key.length = xform->cipher.key.length; 1080 1081 memcpy(session->cipher_key.data, xform->cipher.key.data, 1082 xform->cipher.key.length); 1083 session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 1084 DIR_ENC : DIR_DEC; 1085 1086 return 0; 1087 } 1088 1089 static int 1090 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused, 1091 struct rte_crypto_sym_xform *xform, 1092 dpaa_sec_session *session) 1093 { 1094 session->auth_alg = xform->auth.algo; 1095 session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, 1096 RTE_CACHE_LINE_SIZE); 1097 if (session->auth_key.data == NULL && xform->auth.key.length > 0) { 1098 PMD_INIT_LOG(ERR, "No Memory for auth key\n"); 1099 return -ENOMEM; 1100 } 1101 session->auth_key.length = xform->auth.key.length; 1102 session->digest_length = xform->auth.digest_length; 1103 1104 memcpy(session->auth_key.data, xform->auth.key.data, 1105 xform->auth.key.length); 1106 session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? 1107 DIR_ENC : DIR_DEC; 1108 1109 return 0; 1110 } 1111 1112 static int 1113 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused, 1114 struct rte_crypto_sym_xform *xform, 1115 dpaa_sec_session *session) 1116 { 1117 session->aead_alg = xform->aead.algo; 1118 session->iv.length = xform->aead.iv.length; 1119 session->iv.offset = xform->aead.iv.offset; 1120 session->auth_only_len = xform->aead.aad_length; 1121 session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length, 1122 RTE_CACHE_LINE_SIZE); 1123 if (session->aead_key.data == NULL && xform->aead.key.length > 0) { 1124 PMD_INIT_LOG(ERR, "No Memory for aead key\n"); 1125 return -ENOMEM; 1126 } 1127 session->aead_key.length = xform->aead.key.length; 1128 session->digest_length = xform->aead.digest_length; 1129 1130 memcpy(session->aead_key.data, xform->aead.key.data, 1131 xform->aead.key.length); 1132 session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 1133 DIR_ENC : DIR_DEC; 1134 1135 return 0; 1136 } 1137 1138 static int 1139 dpaa_sec_qp_attach_sess(struct rte_cryptodev *dev, uint16_t qp_id, void *ses) 1140 { 1141 dpaa_sec_session *sess = ses; 1142 struct dpaa_sec_qp *qp; 1143 1144 PMD_INIT_FUNC_TRACE(); 1145 1146 qp = dev->data->queue_pairs[qp_id]; 1147 if (qp->ses != NULL) { 1148 PMD_INIT_LOG(ERR, "qp in-use by another session\n"); 1149 return -EBUSY; 1150 } 1151 1152 qp->ses = sess; 1153 sess->qp = qp; 1154 1155 return dpaa_sec_prep_cdb(sess); 1156 } 1157 1158 static int 1159 dpaa_sec_qp_detach_sess(struct rte_cryptodev *dev, uint16_t qp_id, void *ses) 1160 { 1161 dpaa_sec_session *sess = ses; 1162 struct dpaa_sec_qp *qp; 1163 1164 PMD_INIT_FUNC_TRACE(); 1165 1166 qp = dev->data->queue_pairs[qp_id]; 1167 if (qp->ses != NULL) { 1168 qp->ses = NULL; 1169 sess->qp = NULL; 1170 return 0; 1171 } 1172 1173 PMD_DRV_LOG(ERR, "No session attached to qp"); 1174 return -EINVAL; 1175 } 1176 1177 static int 1178 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, 1179 struct rte_crypto_sym_xform *xform, void *sess) 1180 { 1181 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 1182 dpaa_sec_session *session = sess; 1183 1184 PMD_INIT_FUNC_TRACE(); 1185 1186 if (unlikely(sess == NULL)) { 1187 RTE_LOG(ERR, PMD, "invalid session struct\n"); 1188 return -EINVAL; 1189 } 1190 1191 /* Default IV length = 0 */ 1192 session->iv.length = 0; 1193 1194 /* Cipher Only */ 1195 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { 1196 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 1197 dpaa_sec_cipher_init(dev, xform, session); 1198 1199 /* Authentication Only */ 1200 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1201 xform->next == NULL) { 1202 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; 1203 dpaa_sec_auth_init(dev, xform, session); 1204 1205 /* Cipher then Authenticate */ 1206 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 1207 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 1208 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1209 dpaa_sec_cipher_init(dev, xform, session); 1210 dpaa_sec_auth_init(dev, xform->next, session); 1211 } else { 1212 PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher"); 1213 return -EINVAL; 1214 } 1215 1216 /* Authenticate then Cipher */ 1217 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1218 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 1219 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 1220 dpaa_sec_auth_init(dev, xform, session); 1221 dpaa_sec_cipher_init(dev, xform->next, session); 1222 } else { 1223 PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher"); 1224 return -EINVAL; 1225 } 1226 1227 /* AEAD operation for AES-GCM kind of Algorithms */ 1228 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && 1229 xform->next == NULL) { 1230 dpaa_sec_aead_init(dev, xform, session); 1231 1232 } else { 1233 PMD_DRV_LOG(ERR, "Invalid crypto type"); 1234 return -EINVAL; 1235 } 1236 session->ctx_pool = internals->ctx_pool; 1237 1238 return 0; 1239 } 1240 1241 static int 1242 dpaa_sec_session_configure(struct rte_cryptodev *dev, 1243 struct rte_crypto_sym_xform *xform, 1244 struct rte_cryptodev_sym_session *sess, 1245 struct rte_mempool *mempool) 1246 { 1247 void *sess_private_data; 1248 int ret; 1249 1250 PMD_INIT_FUNC_TRACE(); 1251 1252 if (rte_mempool_get(mempool, &sess_private_data)) { 1253 CDEV_LOG_ERR( 1254 "Couldn't get object from session mempool"); 1255 return -ENOMEM; 1256 } 1257 1258 ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data); 1259 if (ret != 0) { 1260 PMD_DRV_LOG(ERR, "DPAA PMD: failed to configure " 1261 "session parameters"); 1262 1263 /* Return session to mempool */ 1264 rte_mempool_put(mempool, sess_private_data); 1265 return ret; 1266 } 1267 1268 set_session_private_data(sess, dev->driver_id, 1269 sess_private_data); 1270 1271 return 0; 1272 } 1273 1274 /** Clear the memory of session so it doesn't leave key material behind */ 1275 static void 1276 dpaa_sec_session_clear(struct rte_cryptodev *dev, 1277 struct rte_cryptodev_sym_session *sess) 1278 { 1279 PMD_INIT_FUNC_TRACE(); 1280 uint8_t index = dev->driver_id; 1281 void *sess_priv = get_session_private_data(sess, index); 1282 dpaa_sec_session *s = (dpaa_sec_session *)sess_priv; 1283 1284 if (sess_priv) { 1285 rte_free(s->cipher_key.data); 1286 rte_free(s->auth_key.data); 1287 memset(s, 0, sizeof(dpaa_sec_session)); 1288 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 1289 set_session_private_data(sess, index, NULL); 1290 rte_mempool_put(sess_mp, sess_priv); 1291 } 1292 } 1293 1294 static int 1295 dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, 1296 struct rte_cryptodev_config *config __rte_unused) 1297 { 1298 PMD_INIT_FUNC_TRACE(); 1299 1300 return 0; 1301 } 1302 1303 static int 1304 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused) 1305 { 1306 PMD_INIT_FUNC_TRACE(); 1307 return 0; 1308 } 1309 1310 static void 1311 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused) 1312 { 1313 PMD_INIT_FUNC_TRACE(); 1314 } 1315 1316 static int 1317 dpaa_sec_dev_close(struct rte_cryptodev *dev __rte_unused) 1318 { 1319 PMD_INIT_FUNC_TRACE(); 1320 return 0; 1321 } 1322 1323 static void 1324 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev, 1325 struct rte_cryptodev_info *info) 1326 { 1327 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 1328 1329 PMD_INIT_FUNC_TRACE(); 1330 if (info != NULL) { 1331 info->max_nb_queue_pairs = internals->max_nb_queue_pairs; 1332 info->feature_flags = dev->feature_flags; 1333 info->capabilities = dpaa_sec_capabilities; 1334 info->sym.max_nb_sessions = internals->max_nb_sessions; 1335 info->sym.max_nb_sessions_per_qp = 1336 RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS / RTE_MAX_NB_SEC_QPS; 1337 info->driver_id = cryptodev_driver_id; 1338 } 1339 } 1340 1341 static struct rte_cryptodev_ops crypto_ops = { 1342 .dev_configure = dpaa_sec_dev_configure, 1343 .dev_start = dpaa_sec_dev_start, 1344 .dev_stop = dpaa_sec_dev_stop, 1345 .dev_close = dpaa_sec_dev_close, 1346 .dev_infos_get = dpaa_sec_dev_infos_get, 1347 .queue_pair_setup = dpaa_sec_queue_pair_setup, 1348 .queue_pair_release = dpaa_sec_queue_pair_release, 1349 .queue_pair_start = dpaa_sec_queue_pair_start, 1350 .queue_pair_stop = dpaa_sec_queue_pair_stop, 1351 .queue_pair_count = dpaa_sec_queue_pair_count, 1352 .session_get_size = dpaa_sec_session_get_size, 1353 .session_configure = dpaa_sec_session_configure, 1354 .session_clear = dpaa_sec_session_clear, 1355 .qp_attach_session = dpaa_sec_qp_attach_sess, 1356 .qp_detach_session = dpaa_sec_qp_detach_sess, 1357 }; 1358 1359 static int 1360 dpaa_sec_uninit(struct rte_cryptodev *dev) 1361 { 1362 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 1363 1364 if (dev == NULL) 1365 return -ENODEV; 1366 1367 rte_mempool_free(internals->ctx_pool); 1368 rte_free(internals); 1369 1370 PMD_INIT_LOG(INFO, "Closing DPAA_SEC device %s on numa socket %u\n", 1371 dev->data->name, rte_socket_id()); 1372 1373 return 0; 1374 } 1375 1376 static int 1377 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) 1378 { 1379 struct dpaa_sec_dev_private *internals; 1380 struct dpaa_sec_qp *qp; 1381 uint32_t i; 1382 int ret; 1383 char str[20]; 1384 1385 PMD_INIT_FUNC_TRACE(); 1386 1387 cryptodev->driver_id = cryptodev_driver_id; 1388 cryptodev->dev_ops = &crypto_ops; 1389 1390 cryptodev->enqueue_burst = dpaa_sec_enqueue_burst; 1391 cryptodev->dequeue_burst = dpaa_sec_dequeue_burst; 1392 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 1393 RTE_CRYPTODEV_FF_HW_ACCELERATED | 1394 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING; 1395 1396 internals = cryptodev->data->dev_private; 1397 internals->max_nb_queue_pairs = RTE_MAX_NB_SEC_QPS; 1398 internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS; 1399 1400 for (i = 0; i < internals->max_nb_queue_pairs; i++) { 1401 /* init qman fq for queue pair */ 1402 qp = &internals->qps[i]; 1403 ret = dpaa_sec_init_tx(&qp->outq); 1404 if (ret) { 1405 PMD_INIT_LOG(ERR, "config tx of queue pair %d", i); 1406 goto init_error; 1407 } 1408 ret = dpaa_sec_init_rx(&qp->inq, dpaa_mem_vtop(&qp->cdb), 1409 qman_fq_fqid(&qp->outq)); 1410 if (ret) { 1411 PMD_INIT_LOG(ERR, "config rx of queue pair %d", i); 1412 goto init_error; 1413 } 1414 } 1415 1416 sprintf(str, "ctx_pool_%d", cryptodev->data->dev_id); 1417 internals->ctx_pool = rte_mempool_create((const char *)str, 1418 CTX_POOL_NUM_BUFS, 1419 CTX_POOL_BUF_SIZE, 1420 CTX_POOL_CACHE_SIZE, 0, 1421 NULL, NULL, NULL, NULL, 1422 SOCKET_ID_ANY, 0); 1423 if (!internals->ctx_pool) { 1424 RTE_LOG(ERR, PMD, "%s create failed\n", str); 1425 goto init_error; 1426 } 1427 1428 PMD_INIT_LOG(DEBUG, "driver %s: created\n", cryptodev->data->name); 1429 return 0; 1430 1431 init_error: 1432 PMD_INIT_LOG(ERR, "driver %s: create failed\n", cryptodev->data->name); 1433 1434 dpaa_sec_uninit(cryptodev); 1435 return -EFAULT; 1436 } 1437 1438 static int 1439 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv, 1440 struct rte_dpaa_device *dpaa_dev) 1441 { 1442 struct rte_cryptodev *cryptodev; 1443 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; 1444 1445 int retval; 1446 1447 sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id); 1448 1449 cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id()); 1450 if (cryptodev == NULL) 1451 return -ENOMEM; 1452 1453 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 1454 cryptodev->data->dev_private = rte_zmalloc_socket( 1455 "cryptodev private structure", 1456 sizeof(struct dpaa_sec_dev_private), 1457 RTE_CACHE_LINE_SIZE, 1458 rte_socket_id()); 1459 1460 if (cryptodev->data->dev_private == NULL) 1461 rte_panic("Cannot allocate memzone for private " 1462 "device data"); 1463 } 1464 1465 dpaa_dev->crypto_dev = cryptodev; 1466 cryptodev->device = &dpaa_dev->device; 1467 cryptodev->device->driver = &dpaa_drv->driver; 1468 1469 /* init user callbacks */ 1470 TAILQ_INIT(&(cryptodev->link_intr_cbs)); 1471 1472 /* if sec device version is not configured */ 1473 if (!rta_get_sec_era()) { 1474 const struct device_node *caam_node; 1475 1476 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { 1477 const uint32_t *prop = of_get_property(caam_node, 1478 "fsl,sec-era", 1479 NULL); 1480 if (prop) { 1481 rta_set_sec_era( 1482 INTL_SEC_ERA(rte_cpu_to_be_32(*prop))); 1483 break; 1484 } 1485 } 1486 } 1487 1488 /* Invoke PMD device initialization function */ 1489 retval = dpaa_sec_dev_init(cryptodev); 1490 if (retval == 0) 1491 return 0; 1492 1493 /* In case of error, cleanup is done */ 1494 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 1495 rte_free(cryptodev->data->dev_private); 1496 1497 rte_cryptodev_pmd_release_device(cryptodev); 1498 1499 return -ENXIO; 1500 } 1501 1502 static int 1503 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev) 1504 { 1505 struct rte_cryptodev *cryptodev; 1506 int ret; 1507 1508 cryptodev = dpaa_dev->crypto_dev; 1509 if (cryptodev == NULL) 1510 return -ENODEV; 1511 1512 ret = dpaa_sec_uninit(cryptodev); 1513 if (ret) 1514 return ret; 1515 1516 /* free crypto device */ 1517 rte_cryptodev_pmd_release_device(cryptodev); 1518 1519 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 1520 rte_free(cryptodev->data->dev_private); 1521 1522 PMD_INIT_LOG(INFO, "Closing dpaa crypto device %s", 1523 cryptodev->data->name); 1524 1525 cryptodev->device = NULL; 1526 cryptodev->data = NULL; 1527 1528 return 0; 1529 } 1530 1531 static struct rte_dpaa_driver rte_dpaa_sec_driver = { 1532 .drv_type = FSL_DPAA_CRYPTO, 1533 .driver = { 1534 .name = "DPAA SEC PMD" 1535 }, 1536 .probe = cryptodev_dpaa_sec_probe, 1537 .remove = cryptodev_dpaa_sec_remove, 1538 }; 1539 1540 static struct cryptodev_driver dpaa_sec_crypto_drv; 1541 1542 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver); 1543 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver, 1544 cryptodev_driver_id); 1545