1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2017-2018 NXP 5 * 6 */ 7 8 #include <fcntl.h> 9 #include <unistd.h> 10 #include <sched.h> 11 #include <net/if.h> 12 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 #include <rte_cryptodev_pmd.h> 16 #include <rte_crypto.h> 17 #include <rte_cryptodev.h> 18 #include <rte_security_driver.h> 19 #include <rte_cycles.h> 20 #include <rte_dev.h> 21 #include <rte_kvargs.h> 22 #include <rte_malloc.h> 23 #include <rte_mbuf.h> 24 #include <rte_memcpy.h> 25 #include <rte_string_fns.h> 26 #include <rte_spinlock.h> 27 28 #include <fsl_usd.h> 29 #include <fsl_qman.h> 30 #include <of.h> 31 32 /* RTA header files */ 33 #include <hw/desc/common.h> 34 #include <hw/desc/algo.h> 35 #include <hw/desc/ipsec.h> 36 37 #include <rte_dpaa_bus.h> 38 #include <dpaa_sec.h> 39 #include <dpaa_sec_log.h> 40 41 enum rta_sec_era rta_sec_era; 42 43 int dpaa_logtype_sec; 44 45 static uint8_t cryptodev_driver_id; 46 47 static __thread struct rte_crypto_op **dpaa_sec_ops; 48 static __thread int dpaa_sec_op_nb; 49 50 static int 51 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess); 52 53 static inline void 54 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx) 55 { 56 if (!ctx->fd_status) { 57 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 58 } else { 59 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status); 60 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR; 61 } 62 63 /* report op status to sym->op and then free the ctx memeory */ 64 rte_mempool_put(ctx->ctx_pool, (void *)ctx); 65 } 66 67 static inline struct dpaa_sec_op_ctx * 68 dpaa_sec_alloc_ctx(dpaa_sec_session *ses) 69 { 70 struct dpaa_sec_op_ctx *ctx; 71 int retval; 72 73 retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx)); 74 if (!ctx || retval) { 75 DPAA_SEC_DP_WARN("Alloc sec descriptor failed!"); 76 return NULL; 77 } 78 /* 79 * Clear SG memory. There are 16 SG entries of 16 Bytes each. 80 * one call to dcbz_64() clear 64 bytes, hence calling it 4 times 81 * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for 82 * each packet, memset is costlier than dcbz_64(). 83 */ 84 dcbz_64(&ctx->job.sg[SG_CACHELINE_0]); 85 dcbz_64(&ctx->job.sg[SG_CACHELINE_1]); 86 dcbz_64(&ctx->job.sg[SG_CACHELINE_2]); 87 dcbz_64(&ctx->job.sg[SG_CACHELINE_3]); 88 89 ctx->ctx_pool = ses->ctx_pool; 90 ctx->vtop_offset = (size_t) ctx 91 - rte_mempool_virt2iova(ctx); 92 93 return ctx; 94 } 95 96 static inline rte_iova_t 97 dpaa_mem_vtop(void *vaddr) 98 { 99 const struct rte_memseg *ms; 100 101 ms = rte_mem_virt2memseg(vaddr, NULL); 102 if (ms) 103 return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr); 104 return (size_t)NULL; 105 } 106 107 static inline void * 108 dpaa_mem_ptov(rte_iova_t paddr) 109 { 110 void *va; 111 112 va = (void *)dpaax_iova_table_get_va(paddr); 113 if (likely(va)) 114 return va; 115 116 return rte_mem_iova2virt(paddr); 117 } 118 119 static void 120 ern_sec_fq_handler(struct qman_portal *qm __rte_unused, 121 struct qman_fq *fq, 122 const struct qm_mr_entry *msg) 123 { 124 DPAA_SEC_DP_ERR("sec fq %d error, RC = %x, seqnum = %x\n", 125 fq->fqid, msg->ern.rc, msg->ern.seqnum); 126 } 127 128 /* initialize the queue with dest chan as caam chan so that 129 * all the packets in this queue could be dispatched into caam 130 */ 131 static int 132 dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc, 133 uint32_t fqid_out) 134 { 135 struct qm_mcc_initfq fq_opts; 136 uint32_t flags; 137 int ret = -1; 138 139 /* Clear FQ options */ 140 memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq)); 141 142 flags = QMAN_INITFQ_FLAG_SCHED; 143 fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA | 144 QM_INITFQ_WE_CONTEXTB; 145 146 qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc); 147 fq_opts.fqd.context_b = fqid_out; 148 fq_opts.fqd.dest.channel = qm_channel_caam; 149 fq_opts.fqd.dest.wq = 0; 150 151 fq_in->cb.ern = ern_sec_fq_handler; 152 153 DPAA_SEC_DEBUG("in-%x out-%x", fq_in->fqid, fqid_out); 154 155 ret = qman_init_fq(fq_in, flags, &fq_opts); 156 if (unlikely(ret != 0)) 157 DPAA_SEC_ERR("qman_init_fq failed %d", ret); 158 159 return ret; 160 } 161 162 /* something is put into in_fq and caam put the crypto result into out_fq */ 163 static enum qman_cb_dqrr_result 164 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, 165 struct qman_fq *fq __always_unused, 166 const struct qm_dqrr_entry *dqrr) 167 { 168 const struct qm_fd *fd; 169 struct dpaa_sec_job *job; 170 struct dpaa_sec_op_ctx *ctx; 171 172 if (dpaa_sec_op_nb >= DPAA_SEC_BURST) 173 return qman_cb_dqrr_defer; 174 175 if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID)) 176 return qman_cb_dqrr_consume; 177 178 fd = &dqrr->fd; 179 /* sg is embedded in an op ctx, 180 * sg[0] is for output 181 * sg[1] for input 182 */ 183 job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); 184 185 ctx = container_of(job, struct dpaa_sec_op_ctx, job); 186 ctx->fd_status = fd->status; 187 if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 188 struct qm_sg_entry *sg_out; 189 uint32_t len; 190 191 sg_out = &job->sg[0]; 192 hw_sg_to_cpu(sg_out); 193 len = sg_out->length; 194 ctx->op->sym->m_src->pkt_len = len; 195 ctx->op->sym->m_src->data_len = len; 196 } 197 dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op; 198 dpaa_sec_op_ending(ctx); 199 200 return qman_cb_dqrr_consume; 201 } 202 203 /* caam result is put into this queue */ 204 static int 205 dpaa_sec_init_tx(struct qman_fq *fq) 206 { 207 int ret; 208 struct qm_mcc_initfq opts; 209 uint32_t flags; 210 211 flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED | 212 QMAN_FQ_FLAG_DYNAMIC_FQID; 213 214 ret = qman_create_fq(0, flags, fq); 215 if (unlikely(ret)) { 216 DPAA_SEC_ERR("qman_create_fq failed"); 217 return ret; 218 } 219 220 memset(&opts, 0, sizeof(opts)); 221 opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL | 222 QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB; 223 224 /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */ 225 226 fq->cb.dqrr = dqrr_out_fq_cb_rx; 227 fq->cb.ern = ern_sec_fq_handler; 228 229 ret = qman_init_fq(fq, 0, &opts); 230 if (unlikely(ret)) { 231 DPAA_SEC_ERR("unable to init caam source fq!"); 232 return ret; 233 } 234 235 return ret; 236 } 237 238 static inline int is_cipher_only(dpaa_sec_session *ses) 239 { 240 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && 241 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL)); 242 } 243 244 static inline int is_auth_only(dpaa_sec_session *ses) 245 { 246 return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) && 247 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); 248 } 249 250 static inline int is_aead(dpaa_sec_session *ses) 251 { 252 return ((ses->cipher_alg == 0) && 253 (ses->auth_alg == 0) && 254 (ses->aead_alg != 0)); 255 } 256 257 static inline int is_auth_cipher(dpaa_sec_session *ses) 258 { 259 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && 260 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) && 261 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC)); 262 } 263 264 static inline int is_proto_ipsec(dpaa_sec_session *ses) 265 { 266 return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC); 267 } 268 269 static inline int is_encode(dpaa_sec_session *ses) 270 { 271 return ses->dir == DIR_ENC; 272 } 273 274 static inline int is_decode(dpaa_sec_session *ses) 275 { 276 return ses->dir == DIR_DEC; 277 } 278 279 static inline void 280 caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a) 281 { 282 switch (ses->auth_alg) { 283 case RTE_CRYPTO_AUTH_NULL: 284 alginfo_a->algtype = 285 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 286 OP_PCL_IPSEC_HMAC_NULL : 0; 287 ses->digest_length = 0; 288 break; 289 case RTE_CRYPTO_AUTH_MD5_HMAC: 290 alginfo_a->algtype = 291 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 292 OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5; 293 alginfo_a->algmode = OP_ALG_AAI_HMAC; 294 break; 295 case RTE_CRYPTO_AUTH_SHA1_HMAC: 296 alginfo_a->algtype = 297 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 298 OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1; 299 alginfo_a->algmode = OP_ALG_AAI_HMAC; 300 break; 301 case RTE_CRYPTO_AUTH_SHA224_HMAC: 302 alginfo_a->algtype = 303 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 304 OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224; 305 alginfo_a->algmode = OP_ALG_AAI_HMAC; 306 break; 307 case RTE_CRYPTO_AUTH_SHA256_HMAC: 308 alginfo_a->algtype = 309 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 310 OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256; 311 alginfo_a->algmode = OP_ALG_AAI_HMAC; 312 break; 313 case RTE_CRYPTO_AUTH_SHA384_HMAC: 314 alginfo_a->algtype = 315 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 316 OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384; 317 alginfo_a->algmode = OP_ALG_AAI_HMAC; 318 break; 319 case RTE_CRYPTO_AUTH_SHA512_HMAC: 320 alginfo_a->algtype = 321 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 322 OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512; 323 alginfo_a->algmode = OP_ALG_AAI_HMAC; 324 break; 325 default: 326 DPAA_SEC_ERR("unsupported auth alg %u", ses->auth_alg); 327 } 328 } 329 330 static inline void 331 caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c) 332 { 333 switch (ses->cipher_alg) { 334 case RTE_CRYPTO_CIPHER_NULL: 335 alginfo_c->algtype = 336 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 337 OP_PCL_IPSEC_NULL : 0; 338 break; 339 case RTE_CRYPTO_CIPHER_AES_CBC: 340 alginfo_c->algtype = 341 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 342 OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES; 343 alginfo_c->algmode = OP_ALG_AAI_CBC; 344 break; 345 case RTE_CRYPTO_CIPHER_3DES_CBC: 346 alginfo_c->algtype = 347 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 348 OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES; 349 alginfo_c->algmode = OP_ALG_AAI_CBC; 350 break; 351 case RTE_CRYPTO_CIPHER_AES_CTR: 352 alginfo_c->algtype = 353 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 354 OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES; 355 alginfo_c->algmode = OP_ALG_AAI_CTR; 356 break; 357 default: 358 DPAA_SEC_ERR("unsupported cipher alg %d", ses->cipher_alg); 359 } 360 } 361 362 static inline void 363 caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo) 364 { 365 switch (ses->aead_alg) { 366 case RTE_CRYPTO_AEAD_AES_GCM: 367 alginfo->algtype = OP_ALG_ALGSEL_AES; 368 alginfo->algmode = OP_ALG_AAI_GCM; 369 break; 370 default: 371 DPAA_SEC_ERR("unsupported AEAD alg %d", ses->aead_alg); 372 } 373 } 374 375 /* prepare ipsec proto command block of the session */ 376 static int 377 dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses) 378 { 379 struct alginfo cipherdata = {0}, authdata = {0}; 380 struct sec_cdb *cdb = &ses->cdb; 381 int32_t shared_desc_len = 0; 382 int err; 383 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 384 int swap = false; 385 #else 386 int swap = true; 387 #endif 388 389 caam_cipher_alg(ses, &cipherdata); 390 if (cipherdata.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 391 DPAA_SEC_ERR("not supported cipher alg"); 392 return -ENOTSUP; 393 } 394 395 cipherdata.key = (size_t)ses->cipher_key.data; 396 cipherdata.keylen = ses->cipher_key.length; 397 cipherdata.key_enc_flags = 0; 398 cipherdata.key_type = RTA_DATA_IMM; 399 400 caam_auth_alg(ses, &authdata); 401 if (authdata.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 402 DPAA_SEC_ERR("not supported auth alg"); 403 return -ENOTSUP; 404 } 405 406 authdata.key = (size_t)ses->auth_key.data; 407 authdata.keylen = ses->auth_key.length; 408 authdata.key_enc_flags = 0; 409 authdata.key_type = RTA_DATA_IMM; 410 411 cdb->sh_desc[0] = cipherdata.keylen; 412 cdb->sh_desc[1] = authdata.keylen; 413 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, 414 MIN_JOB_DESC_SIZE, 415 (unsigned int *)cdb->sh_desc, 416 &cdb->sh_desc[2], 2); 417 418 if (err < 0) { 419 DPAA_SEC_ERR("Crypto: Incorrect key lengths"); 420 return err; 421 } 422 if (cdb->sh_desc[2] & 1) 423 cipherdata.key_type = RTA_DATA_IMM; 424 else { 425 cipherdata.key = (size_t)dpaa_mem_vtop( 426 (void *)(size_t)cipherdata.key); 427 cipherdata.key_type = RTA_DATA_PTR; 428 } 429 if (cdb->sh_desc[2] & (1<<1)) 430 authdata.key_type = RTA_DATA_IMM; 431 else { 432 authdata.key = (size_t)dpaa_mem_vtop( 433 (void *)(size_t)authdata.key); 434 authdata.key_type = RTA_DATA_PTR; 435 } 436 437 cdb->sh_desc[0] = 0; 438 cdb->sh_desc[1] = 0; 439 cdb->sh_desc[2] = 0; 440 if (ses->dir == DIR_ENC) { 441 shared_desc_len = cnstr_shdsc_ipsec_new_encap( 442 cdb->sh_desc, 443 true, swap, SHR_SERIAL, 444 &ses->encap_pdb, 445 (uint8_t *)&ses->ip4_hdr, 446 &cipherdata, &authdata); 447 } else if (ses->dir == DIR_DEC) { 448 shared_desc_len = cnstr_shdsc_ipsec_new_decap( 449 cdb->sh_desc, 450 true, swap, SHR_SERIAL, 451 &ses->decap_pdb, 452 &cipherdata, &authdata); 453 } 454 return shared_desc_len; 455 } 456 457 /* prepare command block of the session */ 458 static int 459 dpaa_sec_prep_cdb(dpaa_sec_session *ses) 460 { 461 struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0}; 462 int32_t shared_desc_len = 0; 463 struct sec_cdb *cdb = &ses->cdb; 464 int err; 465 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 466 int swap = false; 467 #else 468 int swap = true; 469 #endif 470 471 memset(cdb, 0, sizeof(struct sec_cdb)); 472 473 if (is_proto_ipsec(ses)) { 474 shared_desc_len = dpaa_sec_prep_ipsec_cdb(ses); 475 } else if (is_cipher_only(ses)) { 476 caam_cipher_alg(ses, &alginfo_c); 477 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 478 DPAA_SEC_ERR("not supported cipher alg"); 479 return -ENOTSUP; 480 } 481 482 alginfo_c.key = (size_t)ses->cipher_key.data; 483 alginfo_c.keylen = ses->cipher_key.length; 484 alginfo_c.key_enc_flags = 0; 485 alginfo_c.key_type = RTA_DATA_IMM; 486 487 shared_desc_len = cnstr_shdsc_blkcipher( 488 cdb->sh_desc, true, 489 swap, &alginfo_c, 490 NULL, 491 ses->iv.length, 492 ses->dir); 493 } else if (is_auth_only(ses)) { 494 caam_auth_alg(ses, &alginfo_a); 495 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 496 DPAA_SEC_ERR("not supported auth alg"); 497 return -ENOTSUP; 498 } 499 500 alginfo_a.key = (size_t)ses->auth_key.data; 501 alginfo_a.keylen = ses->auth_key.length; 502 alginfo_a.key_enc_flags = 0; 503 alginfo_a.key_type = RTA_DATA_IMM; 504 505 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true, 506 swap, &alginfo_a, 507 !ses->dir, 508 ses->digest_length); 509 } else if (is_aead(ses)) { 510 caam_aead_alg(ses, &alginfo); 511 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 512 DPAA_SEC_ERR("not supported aead alg"); 513 return -ENOTSUP; 514 } 515 alginfo.key = (size_t)ses->aead_key.data; 516 alginfo.keylen = ses->aead_key.length; 517 alginfo.key_enc_flags = 0; 518 alginfo.key_type = RTA_DATA_IMM; 519 520 if (ses->dir == DIR_ENC) 521 shared_desc_len = cnstr_shdsc_gcm_encap( 522 cdb->sh_desc, true, swap, 523 &alginfo, 524 ses->iv.length, 525 ses->digest_length); 526 else 527 shared_desc_len = cnstr_shdsc_gcm_decap( 528 cdb->sh_desc, true, swap, 529 &alginfo, 530 ses->iv.length, 531 ses->digest_length); 532 } else { 533 caam_cipher_alg(ses, &alginfo_c); 534 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 535 DPAA_SEC_ERR("not supported cipher alg"); 536 return -ENOTSUP; 537 } 538 539 alginfo_c.key = (size_t)ses->cipher_key.data; 540 alginfo_c.keylen = ses->cipher_key.length; 541 alginfo_c.key_enc_flags = 0; 542 alginfo_c.key_type = RTA_DATA_IMM; 543 544 caam_auth_alg(ses, &alginfo_a); 545 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 546 DPAA_SEC_ERR("not supported auth alg"); 547 return -ENOTSUP; 548 } 549 550 alginfo_a.key = (size_t)ses->auth_key.data; 551 alginfo_a.keylen = ses->auth_key.length; 552 alginfo_a.key_enc_flags = 0; 553 alginfo_a.key_type = RTA_DATA_IMM; 554 555 cdb->sh_desc[0] = alginfo_c.keylen; 556 cdb->sh_desc[1] = alginfo_a.keylen; 557 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, 558 MIN_JOB_DESC_SIZE, 559 (unsigned int *)cdb->sh_desc, 560 &cdb->sh_desc[2], 2); 561 562 if (err < 0) { 563 DPAA_SEC_ERR("Crypto: Incorrect key lengths"); 564 return err; 565 } 566 if (cdb->sh_desc[2] & 1) 567 alginfo_c.key_type = RTA_DATA_IMM; 568 else { 569 alginfo_c.key = (size_t)dpaa_mem_vtop( 570 (void *)(size_t)alginfo_c.key); 571 alginfo_c.key_type = RTA_DATA_PTR; 572 } 573 if (cdb->sh_desc[2] & (1<<1)) 574 alginfo_a.key_type = RTA_DATA_IMM; 575 else { 576 alginfo_a.key = (size_t)dpaa_mem_vtop( 577 (void *)(size_t)alginfo_a.key); 578 alginfo_a.key_type = RTA_DATA_PTR; 579 } 580 cdb->sh_desc[0] = 0; 581 cdb->sh_desc[1] = 0; 582 cdb->sh_desc[2] = 0; 583 /* Auth_only_len is set as 0 here and it will be 584 * overwritten in fd for each packet. 585 */ 586 shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc, 587 true, swap, &alginfo_c, &alginfo_a, 588 ses->iv.length, 0, 589 ses->digest_length, ses->dir); 590 } 591 592 if (shared_desc_len < 0) { 593 DPAA_SEC_ERR("error in preparing command block"); 594 return shared_desc_len; 595 } 596 597 cdb->sh_hdr.hi.field.idlen = shared_desc_len; 598 cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word); 599 cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word); 600 601 return 0; 602 } 603 604 /* qp is lockless, should be accessed by only one thread */ 605 static int 606 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops) 607 { 608 struct qman_fq *fq; 609 unsigned int pkts = 0; 610 int num_rx_bufs, ret; 611 struct qm_dqrr_entry *dq; 612 uint32_t vdqcr_flags = 0; 613 614 fq = &qp->outq; 615 /* 616 * Until request for four buffers, we provide exact number of buffers. 617 * Otherwise we do not set the QM_VDQCR_EXACT flag. 618 * Not setting QM_VDQCR_EXACT flag can provide two more buffers than 619 * requested, so we request two less in this case. 620 */ 621 if (nb_ops < 4) { 622 vdqcr_flags = QM_VDQCR_EXACT; 623 num_rx_bufs = nb_ops; 624 } else { 625 num_rx_bufs = nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES ? 626 (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_ops - 2); 627 } 628 ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags); 629 if (ret) 630 return 0; 631 632 do { 633 const struct qm_fd *fd; 634 struct dpaa_sec_job *job; 635 struct dpaa_sec_op_ctx *ctx; 636 struct rte_crypto_op *op; 637 638 dq = qman_dequeue(fq); 639 if (!dq) 640 continue; 641 642 fd = &dq->fd; 643 /* sg is embedded in an op ctx, 644 * sg[0] is for output 645 * sg[1] for input 646 */ 647 job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); 648 649 ctx = container_of(job, struct dpaa_sec_op_ctx, job); 650 ctx->fd_status = fd->status; 651 op = ctx->op; 652 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 653 struct qm_sg_entry *sg_out; 654 uint32_t len; 655 656 sg_out = &job->sg[0]; 657 hw_sg_to_cpu(sg_out); 658 len = sg_out->length; 659 op->sym->m_src->pkt_len = len; 660 op->sym->m_src->data_len = len; 661 } 662 if (!ctx->fd_status) { 663 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 664 } else { 665 DPAA_SEC_DP_WARN("SEC return err:0x%x", ctx->fd_status); 666 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 667 } 668 ops[pkts++] = op; 669 670 /* report op status to sym->op and then free the ctx memeory */ 671 rte_mempool_put(ctx->ctx_pool, (void *)ctx); 672 673 qman_dqrr_consume(fq, dq); 674 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 675 676 return pkts; 677 } 678 679 static inline struct dpaa_sec_job * 680 build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) 681 { 682 struct rte_crypto_sym_op *sym = op->sym; 683 struct rte_mbuf *mbuf = sym->m_src; 684 struct dpaa_sec_job *cf; 685 struct dpaa_sec_op_ctx *ctx; 686 struct qm_sg_entry *sg, *out_sg, *in_sg; 687 phys_addr_t start_addr; 688 uint8_t *old_digest, extra_segs; 689 690 if (is_decode(ses)) 691 extra_segs = 3; 692 else 693 extra_segs = 2; 694 695 if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) { 696 DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d", 697 MAX_SG_ENTRIES); 698 return NULL; 699 } 700 ctx = dpaa_sec_alloc_ctx(ses); 701 if (!ctx) 702 return NULL; 703 704 cf = &ctx->job; 705 ctx->op = op; 706 old_digest = ctx->digest; 707 708 /* output */ 709 out_sg = &cf->sg[0]; 710 qm_sg_entry_set64(out_sg, sym->auth.digest.phys_addr); 711 out_sg->length = ses->digest_length; 712 cpu_to_hw_sg(out_sg); 713 714 /* input */ 715 in_sg = &cf->sg[1]; 716 /* need to extend the input to a compound frame */ 717 in_sg->extension = 1; 718 in_sg->final = 1; 719 in_sg->length = sym->auth.data.length; 720 qm_sg_entry_set64(in_sg, dpaa_mem_vtop(&cf->sg[2])); 721 722 /* 1st seg */ 723 sg = in_sg + 1; 724 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 725 sg->length = mbuf->data_len - sym->auth.data.offset; 726 sg->offset = sym->auth.data.offset; 727 728 /* Successive segs */ 729 mbuf = mbuf->next; 730 while (mbuf) { 731 cpu_to_hw_sg(sg); 732 sg++; 733 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 734 sg->length = mbuf->data_len; 735 mbuf = mbuf->next; 736 } 737 738 if (is_decode(ses)) { 739 /* Digest verification case */ 740 cpu_to_hw_sg(sg); 741 sg++; 742 rte_memcpy(old_digest, sym->auth.digest.data, 743 ses->digest_length); 744 start_addr = dpaa_mem_vtop(old_digest); 745 qm_sg_entry_set64(sg, start_addr); 746 sg->length = ses->digest_length; 747 in_sg->length += ses->digest_length; 748 } else { 749 /* Digest calculation case */ 750 sg->length -= ses->digest_length; 751 } 752 sg->final = 1; 753 cpu_to_hw_sg(sg); 754 cpu_to_hw_sg(in_sg); 755 756 return cf; 757 } 758 759 /** 760 * packet looks like: 761 * |<----data_len------->| 762 * |ip_header|ah_header|icv|payload| 763 * ^ 764 * | 765 * mbuf->pkt.data 766 */ 767 static inline struct dpaa_sec_job * 768 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) 769 { 770 struct rte_crypto_sym_op *sym = op->sym; 771 struct rte_mbuf *mbuf = sym->m_src; 772 struct dpaa_sec_job *cf; 773 struct dpaa_sec_op_ctx *ctx; 774 struct qm_sg_entry *sg; 775 rte_iova_t start_addr; 776 uint8_t *old_digest; 777 778 ctx = dpaa_sec_alloc_ctx(ses); 779 if (!ctx) 780 return NULL; 781 782 cf = &ctx->job; 783 ctx->op = op; 784 old_digest = ctx->digest; 785 786 start_addr = rte_pktmbuf_iova(mbuf); 787 /* output */ 788 sg = &cf->sg[0]; 789 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); 790 sg->length = ses->digest_length; 791 cpu_to_hw_sg(sg); 792 793 /* input */ 794 sg = &cf->sg[1]; 795 if (is_decode(ses)) { 796 /* need to extend the input to a compound frame */ 797 sg->extension = 1; 798 qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2])); 799 sg->length = sym->auth.data.length + ses->digest_length; 800 sg->final = 1; 801 cpu_to_hw_sg(sg); 802 803 sg = &cf->sg[2]; 804 /* hash result or digest, save digest first */ 805 rte_memcpy(old_digest, sym->auth.digest.data, 806 ses->digest_length); 807 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); 808 sg->length = sym->auth.data.length; 809 cpu_to_hw_sg(sg); 810 811 /* let's check digest by hw */ 812 start_addr = dpaa_mem_vtop(old_digest); 813 sg++; 814 qm_sg_entry_set64(sg, start_addr); 815 sg->length = ses->digest_length; 816 sg->final = 1; 817 cpu_to_hw_sg(sg); 818 } else { 819 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); 820 sg->length = sym->auth.data.length; 821 sg->final = 1; 822 cpu_to_hw_sg(sg); 823 } 824 825 return cf; 826 } 827 828 static inline struct dpaa_sec_job * 829 build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) 830 { 831 struct rte_crypto_sym_op *sym = op->sym; 832 struct dpaa_sec_job *cf; 833 struct dpaa_sec_op_ctx *ctx; 834 struct qm_sg_entry *sg, *out_sg, *in_sg; 835 struct rte_mbuf *mbuf; 836 uint8_t req_segs; 837 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 838 ses->iv.offset); 839 840 if (sym->m_dst) { 841 mbuf = sym->m_dst; 842 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3; 843 } else { 844 mbuf = sym->m_src; 845 req_segs = mbuf->nb_segs * 2 + 3; 846 } 847 848 if (req_segs > MAX_SG_ENTRIES) { 849 DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d", 850 MAX_SG_ENTRIES); 851 return NULL; 852 } 853 854 ctx = dpaa_sec_alloc_ctx(ses); 855 if (!ctx) 856 return NULL; 857 858 cf = &ctx->job; 859 ctx->op = op; 860 861 /* output */ 862 out_sg = &cf->sg[0]; 863 out_sg->extension = 1; 864 out_sg->length = sym->cipher.data.length; 865 qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2])); 866 cpu_to_hw_sg(out_sg); 867 868 /* 1st seg */ 869 sg = &cf->sg[2]; 870 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 871 sg->length = mbuf->data_len - sym->cipher.data.offset; 872 sg->offset = sym->cipher.data.offset; 873 874 /* Successive segs */ 875 mbuf = mbuf->next; 876 while (mbuf) { 877 cpu_to_hw_sg(sg); 878 sg++; 879 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 880 sg->length = mbuf->data_len; 881 mbuf = mbuf->next; 882 } 883 sg->final = 1; 884 cpu_to_hw_sg(sg); 885 886 /* input */ 887 mbuf = sym->m_src; 888 in_sg = &cf->sg[1]; 889 in_sg->extension = 1; 890 in_sg->final = 1; 891 in_sg->length = sym->cipher.data.length + ses->iv.length; 892 893 sg++; 894 qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); 895 cpu_to_hw_sg(in_sg); 896 897 /* IV */ 898 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 899 sg->length = ses->iv.length; 900 cpu_to_hw_sg(sg); 901 902 /* 1st seg */ 903 sg++; 904 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 905 sg->length = mbuf->data_len - sym->cipher.data.offset; 906 sg->offset = sym->cipher.data.offset; 907 908 /* Successive segs */ 909 mbuf = mbuf->next; 910 while (mbuf) { 911 cpu_to_hw_sg(sg); 912 sg++; 913 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 914 sg->length = mbuf->data_len; 915 mbuf = mbuf->next; 916 } 917 sg->final = 1; 918 cpu_to_hw_sg(sg); 919 920 return cf; 921 } 922 923 static inline struct dpaa_sec_job * 924 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses) 925 { 926 struct rte_crypto_sym_op *sym = op->sym; 927 struct dpaa_sec_job *cf; 928 struct dpaa_sec_op_ctx *ctx; 929 struct qm_sg_entry *sg; 930 rte_iova_t src_start_addr, dst_start_addr; 931 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 932 ses->iv.offset); 933 934 ctx = dpaa_sec_alloc_ctx(ses); 935 if (!ctx) 936 return NULL; 937 938 cf = &ctx->job; 939 ctx->op = op; 940 941 src_start_addr = rte_pktmbuf_iova(sym->m_src); 942 943 if (sym->m_dst) 944 dst_start_addr = rte_pktmbuf_iova(sym->m_dst); 945 else 946 dst_start_addr = src_start_addr; 947 948 /* output */ 949 sg = &cf->sg[0]; 950 qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); 951 sg->length = sym->cipher.data.length + ses->iv.length; 952 cpu_to_hw_sg(sg); 953 954 /* input */ 955 sg = &cf->sg[1]; 956 957 /* need to extend the input to a compound frame */ 958 sg->extension = 1; 959 sg->final = 1; 960 sg->length = sym->cipher.data.length + ses->iv.length; 961 qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2])); 962 cpu_to_hw_sg(sg); 963 964 sg = &cf->sg[2]; 965 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 966 sg->length = ses->iv.length; 967 cpu_to_hw_sg(sg); 968 969 sg++; 970 qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset); 971 sg->length = sym->cipher.data.length; 972 sg->final = 1; 973 cpu_to_hw_sg(sg); 974 975 return cf; 976 } 977 978 static inline struct dpaa_sec_job * 979 build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) 980 { 981 struct rte_crypto_sym_op *sym = op->sym; 982 struct dpaa_sec_job *cf; 983 struct dpaa_sec_op_ctx *ctx; 984 struct qm_sg_entry *sg, *out_sg, *in_sg; 985 struct rte_mbuf *mbuf; 986 uint8_t req_segs; 987 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 988 ses->iv.offset); 989 990 if (sym->m_dst) { 991 mbuf = sym->m_dst; 992 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4; 993 } else { 994 mbuf = sym->m_src; 995 req_segs = mbuf->nb_segs * 2 + 4; 996 } 997 998 if (ses->auth_only_len) 999 req_segs++; 1000 1001 if (req_segs > MAX_SG_ENTRIES) { 1002 DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d", 1003 MAX_SG_ENTRIES); 1004 return NULL; 1005 } 1006 1007 ctx = dpaa_sec_alloc_ctx(ses); 1008 if (!ctx) 1009 return NULL; 1010 1011 cf = &ctx->job; 1012 ctx->op = op; 1013 1014 rte_prefetch0(cf->sg); 1015 1016 /* output */ 1017 out_sg = &cf->sg[0]; 1018 out_sg->extension = 1; 1019 if (is_encode(ses)) 1020 out_sg->length = sym->aead.data.length + ses->auth_only_len 1021 + ses->digest_length; 1022 else 1023 out_sg->length = sym->aead.data.length + ses->auth_only_len; 1024 1025 /* output sg entries */ 1026 sg = &cf->sg[2]; 1027 qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg)); 1028 cpu_to_hw_sg(out_sg); 1029 1030 /* 1st seg */ 1031 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1032 sg->length = mbuf->data_len - sym->aead.data.offset + 1033 ses->auth_only_len; 1034 sg->offset = sym->aead.data.offset - ses->auth_only_len; 1035 1036 /* Successive segs */ 1037 mbuf = mbuf->next; 1038 while (mbuf) { 1039 cpu_to_hw_sg(sg); 1040 sg++; 1041 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1042 sg->length = mbuf->data_len; 1043 mbuf = mbuf->next; 1044 } 1045 sg->length -= ses->digest_length; 1046 1047 if (is_encode(ses)) { 1048 cpu_to_hw_sg(sg); 1049 /* set auth output */ 1050 sg++; 1051 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr); 1052 sg->length = ses->digest_length; 1053 } 1054 sg->final = 1; 1055 cpu_to_hw_sg(sg); 1056 1057 /* input */ 1058 mbuf = sym->m_src; 1059 in_sg = &cf->sg[1]; 1060 in_sg->extension = 1; 1061 in_sg->final = 1; 1062 if (is_encode(ses)) 1063 in_sg->length = ses->iv.length + sym->aead.data.length 1064 + ses->auth_only_len; 1065 else 1066 in_sg->length = ses->iv.length + sym->aead.data.length 1067 + ses->auth_only_len + ses->digest_length; 1068 1069 /* input sg entries */ 1070 sg++; 1071 qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); 1072 cpu_to_hw_sg(in_sg); 1073 1074 /* 1st seg IV */ 1075 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 1076 sg->length = ses->iv.length; 1077 cpu_to_hw_sg(sg); 1078 1079 /* 2nd seg auth only */ 1080 if (ses->auth_only_len) { 1081 sg++; 1082 qm_sg_entry_set64(sg, dpaa_mem_vtop(sym->aead.aad.data)); 1083 sg->length = ses->auth_only_len; 1084 cpu_to_hw_sg(sg); 1085 } 1086 1087 /* 3rd seg */ 1088 sg++; 1089 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1090 sg->length = mbuf->data_len - sym->aead.data.offset; 1091 sg->offset = sym->aead.data.offset; 1092 1093 /* Successive segs */ 1094 mbuf = mbuf->next; 1095 while (mbuf) { 1096 cpu_to_hw_sg(sg); 1097 sg++; 1098 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1099 sg->length = mbuf->data_len; 1100 mbuf = mbuf->next; 1101 } 1102 1103 if (is_decode(ses)) { 1104 cpu_to_hw_sg(sg); 1105 sg++; 1106 memcpy(ctx->digest, sym->aead.digest.data, 1107 ses->digest_length); 1108 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); 1109 sg->length = ses->digest_length; 1110 } 1111 sg->final = 1; 1112 cpu_to_hw_sg(sg); 1113 1114 return cf; 1115 } 1116 1117 static inline struct dpaa_sec_job * 1118 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) 1119 { 1120 struct rte_crypto_sym_op *sym = op->sym; 1121 struct dpaa_sec_job *cf; 1122 struct dpaa_sec_op_ctx *ctx; 1123 struct qm_sg_entry *sg; 1124 uint32_t length = 0; 1125 rte_iova_t src_start_addr, dst_start_addr; 1126 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1127 ses->iv.offset); 1128 1129 src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off; 1130 1131 if (sym->m_dst) 1132 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off; 1133 else 1134 dst_start_addr = src_start_addr; 1135 1136 ctx = dpaa_sec_alloc_ctx(ses); 1137 if (!ctx) 1138 return NULL; 1139 1140 cf = &ctx->job; 1141 ctx->op = op; 1142 1143 /* input */ 1144 rte_prefetch0(cf->sg); 1145 sg = &cf->sg[2]; 1146 qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); 1147 if (is_encode(ses)) { 1148 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 1149 sg->length = ses->iv.length; 1150 length += sg->length; 1151 cpu_to_hw_sg(sg); 1152 1153 sg++; 1154 if (ses->auth_only_len) { 1155 qm_sg_entry_set64(sg, 1156 dpaa_mem_vtop(sym->aead.aad.data)); 1157 sg->length = ses->auth_only_len; 1158 length += sg->length; 1159 cpu_to_hw_sg(sg); 1160 sg++; 1161 } 1162 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset); 1163 sg->length = sym->aead.data.length; 1164 length += sg->length; 1165 sg->final = 1; 1166 cpu_to_hw_sg(sg); 1167 } else { 1168 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 1169 sg->length = ses->iv.length; 1170 length += sg->length; 1171 cpu_to_hw_sg(sg); 1172 1173 sg++; 1174 if (ses->auth_only_len) { 1175 qm_sg_entry_set64(sg, 1176 dpaa_mem_vtop(sym->aead.aad.data)); 1177 sg->length = ses->auth_only_len; 1178 length += sg->length; 1179 cpu_to_hw_sg(sg); 1180 sg++; 1181 } 1182 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset); 1183 sg->length = sym->aead.data.length; 1184 length += sg->length; 1185 cpu_to_hw_sg(sg); 1186 1187 memcpy(ctx->digest, sym->aead.digest.data, 1188 ses->digest_length); 1189 sg++; 1190 1191 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); 1192 sg->length = ses->digest_length; 1193 length += sg->length; 1194 sg->final = 1; 1195 cpu_to_hw_sg(sg); 1196 } 1197 /* input compound frame */ 1198 cf->sg[1].length = length; 1199 cf->sg[1].extension = 1; 1200 cf->sg[1].final = 1; 1201 cpu_to_hw_sg(&cf->sg[1]); 1202 1203 /* output */ 1204 sg++; 1205 qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); 1206 qm_sg_entry_set64(sg, 1207 dst_start_addr + sym->aead.data.offset - ses->auth_only_len); 1208 sg->length = sym->aead.data.length + ses->auth_only_len; 1209 length = sg->length; 1210 if (is_encode(ses)) { 1211 cpu_to_hw_sg(sg); 1212 /* set auth output */ 1213 sg++; 1214 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr); 1215 sg->length = ses->digest_length; 1216 length += sg->length; 1217 } 1218 sg->final = 1; 1219 cpu_to_hw_sg(sg); 1220 1221 /* output compound frame */ 1222 cf->sg[0].length = length; 1223 cf->sg[0].extension = 1; 1224 cpu_to_hw_sg(&cf->sg[0]); 1225 1226 return cf; 1227 } 1228 1229 static inline struct dpaa_sec_job * 1230 build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) 1231 { 1232 struct rte_crypto_sym_op *sym = op->sym; 1233 struct dpaa_sec_job *cf; 1234 struct dpaa_sec_op_ctx *ctx; 1235 struct qm_sg_entry *sg, *out_sg, *in_sg; 1236 struct rte_mbuf *mbuf; 1237 uint8_t req_segs; 1238 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1239 ses->iv.offset); 1240 1241 if (sym->m_dst) { 1242 mbuf = sym->m_dst; 1243 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4; 1244 } else { 1245 mbuf = sym->m_src; 1246 req_segs = mbuf->nb_segs * 2 + 4; 1247 } 1248 1249 if (req_segs > MAX_SG_ENTRIES) { 1250 DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d", 1251 MAX_SG_ENTRIES); 1252 return NULL; 1253 } 1254 1255 ctx = dpaa_sec_alloc_ctx(ses); 1256 if (!ctx) 1257 return NULL; 1258 1259 cf = &ctx->job; 1260 ctx->op = op; 1261 1262 rte_prefetch0(cf->sg); 1263 1264 /* output */ 1265 out_sg = &cf->sg[0]; 1266 out_sg->extension = 1; 1267 if (is_encode(ses)) 1268 out_sg->length = sym->auth.data.length + ses->digest_length; 1269 else 1270 out_sg->length = sym->auth.data.length; 1271 1272 /* output sg entries */ 1273 sg = &cf->sg[2]; 1274 qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg)); 1275 cpu_to_hw_sg(out_sg); 1276 1277 /* 1st seg */ 1278 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1279 sg->length = mbuf->data_len - sym->auth.data.offset; 1280 sg->offset = sym->auth.data.offset; 1281 1282 /* Successive segs */ 1283 mbuf = mbuf->next; 1284 while (mbuf) { 1285 cpu_to_hw_sg(sg); 1286 sg++; 1287 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1288 sg->length = mbuf->data_len; 1289 mbuf = mbuf->next; 1290 } 1291 sg->length -= ses->digest_length; 1292 1293 if (is_encode(ses)) { 1294 cpu_to_hw_sg(sg); 1295 /* set auth output */ 1296 sg++; 1297 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); 1298 sg->length = ses->digest_length; 1299 } 1300 sg->final = 1; 1301 cpu_to_hw_sg(sg); 1302 1303 /* input */ 1304 mbuf = sym->m_src; 1305 in_sg = &cf->sg[1]; 1306 in_sg->extension = 1; 1307 in_sg->final = 1; 1308 if (is_encode(ses)) 1309 in_sg->length = ses->iv.length + sym->auth.data.length; 1310 else 1311 in_sg->length = ses->iv.length + sym->auth.data.length 1312 + ses->digest_length; 1313 1314 /* input sg entries */ 1315 sg++; 1316 qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); 1317 cpu_to_hw_sg(in_sg); 1318 1319 /* 1st seg IV */ 1320 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 1321 sg->length = ses->iv.length; 1322 cpu_to_hw_sg(sg); 1323 1324 /* 2nd seg */ 1325 sg++; 1326 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1327 sg->length = mbuf->data_len - sym->auth.data.offset; 1328 sg->offset = sym->auth.data.offset; 1329 1330 /* Successive segs */ 1331 mbuf = mbuf->next; 1332 while (mbuf) { 1333 cpu_to_hw_sg(sg); 1334 sg++; 1335 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1336 sg->length = mbuf->data_len; 1337 mbuf = mbuf->next; 1338 } 1339 1340 sg->length -= ses->digest_length; 1341 if (is_decode(ses)) { 1342 cpu_to_hw_sg(sg); 1343 sg++; 1344 memcpy(ctx->digest, sym->auth.digest.data, 1345 ses->digest_length); 1346 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); 1347 sg->length = ses->digest_length; 1348 } 1349 sg->final = 1; 1350 cpu_to_hw_sg(sg); 1351 1352 return cf; 1353 } 1354 1355 static inline struct dpaa_sec_job * 1356 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) 1357 { 1358 struct rte_crypto_sym_op *sym = op->sym; 1359 struct dpaa_sec_job *cf; 1360 struct dpaa_sec_op_ctx *ctx; 1361 struct qm_sg_entry *sg; 1362 rte_iova_t src_start_addr, dst_start_addr; 1363 uint32_t length = 0; 1364 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1365 ses->iv.offset); 1366 1367 src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off; 1368 if (sym->m_dst) 1369 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off; 1370 else 1371 dst_start_addr = src_start_addr; 1372 1373 ctx = dpaa_sec_alloc_ctx(ses); 1374 if (!ctx) 1375 return NULL; 1376 1377 cf = &ctx->job; 1378 ctx->op = op; 1379 1380 /* input */ 1381 rte_prefetch0(cf->sg); 1382 sg = &cf->sg[2]; 1383 qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); 1384 if (is_encode(ses)) { 1385 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 1386 sg->length = ses->iv.length; 1387 length += sg->length; 1388 cpu_to_hw_sg(sg); 1389 1390 sg++; 1391 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset); 1392 sg->length = sym->auth.data.length; 1393 length += sg->length; 1394 sg->final = 1; 1395 cpu_to_hw_sg(sg); 1396 } else { 1397 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 1398 sg->length = ses->iv.length; 1399 length += sg->length; 1400 cpu_to_hw_sg(sg); 1401 1402 sg++; 1403 1404 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset); 1405 sg->length = sym->auth.data.length; 1406 length += sg->length; 1407 cpu_to_hw_sg(sg); 1408 1409 memcpy(ctx->digest, sym->auth.digest.data, 1410 ses->digest_length); 1411 sg++; 1412 1413 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); 1414 sg->length = ses->digest_length; 1415 length += sg->length; 1416 sg->final = 1; 1417 cpu_to_hw_sg(sg); 1418 } 1419 /* input compound frame */ 1420 cf->sg[1].length = length; 1421 cf->sg[1].extension = 1; 1422 cf->sg[1].final = 1; 1423 cpu_to_hw_sg(&cf->sg[1]); 1424 1425 /* output */ 1426 sg++; 1427 qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); 1428 qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); 1429 sg->length = sym->cipher.data.length; 1430 length = sg->length; 1431 if (is_encode(ses)) { 1432 cpu_to_hw_sg(sg); 1433 /* set auth output */ 1434 sg++; 1435 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); 1436 sg->length = ses->digest_length; 1437 length += sg->length; 1438 } 1439 sg->final = 1; 1440 cpu_to_hw_sg(sg); 1441 1442 /* output compound frame */ 1443 cf->sg[0].length = length; 1444 cf->sg[0].extension = 1; 1445 cpu_to_hw_sg(&cf->sg[0]); 1446 1447 return cf; 1448 } 1449 1450 static inline struct dpaa_sec_job * 1451 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses) 1452 { 1453 struct rte_crypto_sym_op *sym = op->sym; 1454 struct dpaa_sec_job *cf; 1455 struct dpaa_sec_op_ctx *ctx; 1456 struct qm_sg_entry *sg; 1457 phys_addr_t src_start_addr, dst_start_addr; 1458 1459 ctx = dpaa_sec_alloc_ctx(ses); 1460 if (!ctx) 1461 return NULL; 1462 cf = &ctx->job; 1463 ctx->op = op; 1464 1465 src_start_addr = rte_pktmbuf_mtophys(sym->m_src); 1466 1467 if (sym->m_dst) 1468 dst_start_addr = rte_pktmbuf_mtophys(sym->m_dst); 1469 else 1470 dst_start_addr = src_start_addr; 1471 1472 /* input */ 1473 sg = &cf->sg[1]; 1474 qm_sg_entry_set64(sg, src_start_addr); 1475 sg->length = sym->m_src->pkt_len; 1476 sg->final = 1; 1477 cpu_to_hw_sg(sg); 1478 1479 sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK; 1480 /* output */ 1481 sg = &cf->sg[0]; 1482 qm_sg_entry_set64(sg, dst_start_addr); 1483 sg->length = sym->m_src->buf_len - sym->m_src->data_off; 1484 cpu_to_hw_sg(sg); 1485 1486 return cf; 1487 } 1488 1489 static uint16_t 1490 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, 1491 uint16_t nb_ops) 1492 { 1493 /* Function to transmit the frames to given device and queuepair */ 1494 uint32_t loop; 1495 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; 1496 uint16_t num_tx = 0; 1497 struct qm_fd fds[DPAA_SEC_BURST], *fd; 1498 uint32_t frames_to_send; 1499 struct rte_crypto_op *op; 1500 struct dpaa_sec_job *cf; 1501 dpaa_sec_session *ses; 1502 uint32_t auth_only_len; 1503 struct qman_fq *inq[DPAA_SEC_BURST]; 1504 1505 while (nb_ops) { 1506 frames_to_send = (nb_ops > DPAA_SEC_BURST) ? 1507 DPAA_SEC_BURST : nb_ops; 1508 for (loop = 0; loop < frames_to_send; loop++) { 1509 op = *(ops++); 1510 switch (op->sess_type) { 1511 case RTE_CRYPTO_OP_WITH_SESSION: 1512 ses = (dpaa_sec_session *) 1513 get_sym_session_private_data( 1514 op->sym->session, 1515 cryptodev_driver_id); 1516 break; 1517 case RTE_CRYPTO_OP_SECURITY_SESSION: 1518 ses = (dpaa_sec_session *) 1519 get_sec_session_private_data( 1520 op->sym->sec_session); 1521 break; 1522 default: 1523 DPAA_SEC_DP_ERR( 1524 "sessionless crypto op not supported"); 1525 frames_to_send = loop; 1526 nb_ops = loop; 1527 goto send_pkts; 1528 } 1529 if (unlikely(!ses->qp)) { 1530 if (dpaa_sec_attach_sess_q(qp, ses)) { 1531 frames_to_send = loop; 1532 nb_ops = loop; 1533 goto send_pkts; 1534 } 1535 } else if (unlikely(ses->qp != qp)) { 1536 DPAA_SEC_DP_ERR("Old:sess->qp = %p" 1537 " New qp = %p\n", ses->qp, qp); 1538 frames_to_send = loop; 1539 nb_ops = loop; 1540 goto send_pkts; 1541 } 1542 1543 auth_only_len = op->sym->auth.data.length - 1544 op->sym->cipher.data.length; 1545 if (rte_pktmbuf_is_contiguous(op->sym->m_src)) { 1546 if (is_proto_ipsec(ses)) { 1547 cf = build_proto(op, ses); 1548 } else if (is_auth_only(ses)) { 1549 cf = build_auth_only(op, ses); 1550 } else if (is_cipher_only(ses)) { 1551 cf = build_cipher_only(op, ses); 1552 } else if (is_aead(ses)) { 1553 cf = build_cipher_auth_gcm(op, ses); 1554 auth_only_len = ses->auth_only_len; 1555 } else if (is_auth_cipher(ses)) { 1556 cf = build_cipher_auth(op, ses); 1557 } else { 1558 DPAA_SEC_DP_ERR("not supported ops"); 1559 frames_to_send = loop; 1560 nb_ops = loop; 1561 goto send_pkts; 1562 } 1563 } else { 1564 if (is_auth_only(ses)) { 1565 cf = build_auth_only_sg(op, ses); 1566 } else if (is_cipher_only(ses)) { 1567 cf = build_cipher_only_sg(op, ses); 1568 } else if (is_aead(ses)) { 1569 cf = build_cipher_auth_gcm_sg(op, ses); 1570 auth_only_len = ses->auth_only_len; 1571 } else if (is_auth_cipher(ses)) { 1572 cf = build_cipher_auth_sg(op, ses); 1573 } else { 1574 DPAA_SEC_DP_ERR("not supported ops"); 1575 frames_to_send = loop; 1576 nb_ops = loop; 1577 goto send_pkts; 1578 } 1579 } 1580 if (unlikely(!cf)) { 1581 frames_to_send = loop; 1582 nb_ops = loop; 1583 goto send_pkts; 1584 } 1585 1586 fd = &fds[loop]; 1587 inq[loop] = ses->inq; 1588 fd->opaque_addr = 0; 1589 fd->cmd = 0; 1590 qm_fd_addr_set64(fd, dpaa_mem_vtop(cf->sg)); 1591 fd->_format1 = qm_fd_compound; 1592 fd->length29 = 2 * sizeof(struct qm_sg_entry); 1593 /* Auth_only_len is set as 0 in descriptor and it is 1594 * overwritten here in the fd.cmd which will update 1595 * the DPOVRD reg. 1596 */ 1597 if (auth_only_len) 1598 fd->cmd = 0x80000000 | auth_only_len; 1599 1600 } 1601 send_pkts: 1602 loop = 0; 1603 while (loop < frames_to_send) { 1604 loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop], 1605 frames_to_send - loop); 1606 } 1607 nb_ops -= frames_to_send; 1608 num_tx += frames_to_send; 1609 } 1610 1611 dpaa_qp->tx_pkts += num_tx; 1612 dpaa_qp->tx_errs += nb_ops - num_tx; 1613 1614 return num_tx; 1615 } 1616 1617 static uint16_t 1618 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, 1619 uint16_t nb_ops) 1620 { 1621 uint16_t num_rx; 1622 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; 1623 1624 num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops); 1625 1626 dpaa_qp->rx_pkts += num_rx; 1627 dpaa_qp->rx_errs += nb_ops - num_rx; 1628 1629 DPAA_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx); 1630 1631 return num_rx; 1632 } 1633 1634 /** Release queue pair */ 1635 static int 1636 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev, 1637 uint16_t qp_id) 1638 { 1639 struct dpaa_sec_dev_private *internals; 1640 struct dpaa_sec_qp *qp = NULL; 1641 1642 PMD_INIT_FUNC_TRACE(); 1643 1644 DPAA_SEC_DEBUG("dev =%p, queue =%d", dev, qp_id); 1645 1646 internals = dev->data->dev_private; 1647 if (qp_id >= internals->max_nb_queue_pairs) { 1648 DPAA_SEC_ERR("Max supported qpid %d", 1649 internals->max_nb_queue_pairs); 1650 return -EINVAL; 1651 } 1652 1653 qp = &internals->qps[qp_id]; 1654 qp->internals = NULL; 1655 dev->data->queue_pairs[qp_id] = NULL; 1656 1657 return 0; 1658 } 1659 1660 /** Setup a queue pair */ 1661 static int 1662 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, 1663 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, 1664 __rte_unused int socket_id, 1665 __rte_unused struct rte_mempool *session_pool) 1666 { 1667 struct dpaa_sec_dev_private *internals; 1668 struct dpaa_sec_qp *qp = NULL; 1669 1670 DPAA_SEC_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf); 1671 1672 internals = dev->data->dev_private; 1673 if (qp_id >= internals->max_nb_queue_pairs) { 1674 DPAA_SEC_ERR("Max supported qpid %d", 1675 internals->max_nb_queue_pairs); 1676 return -EINVAL; 1677 } 1678 1679 qp = &internals->qps[qp_id]; 1680 qp->internals = internals; 1681 dev->data->queue_pairs[qp_id] = qp; 1682 1683 return 0; 1684 } 1685 1686 /** Return the number of allocated queue pairs */ 1687 static uint32_t 1688 dpaa_sec_queue_pair_count(struct rte_cryptodev *dev) 1689 { 1690 PMD_INIT_FUNC_TRACE(); 1691 1692 return dev->data->nb_queue_pairs; 1693 } 1694 1695 /** Returns the size of session structure */ 1696 static unsigned int 1697 dpaa_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) 1698 { 1699 PMD_INIT_FUNC_TRACE(); 1700 1701 return sizeof(dpaa_sec_session); 1702 } 1703 1704 static int 1705 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused, 1706 struct rte_crypto_sym_xform *xform, 1707 dpaa_sec_session *session) 1708 { 1709 session->cipher_alg = xform->cipher.algo; 1710 session->iv.length = xform->cipher.iv.length; 1711 session->iv.offset = xform->cipher.iv.offset; 1712 session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, 1713 RTE_CACHE_LINE_SIZE); 1714 if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) { 1715 DPAA_SEC_ERR("No Memory for cipher key"); 1716 return -ENOMEM; 1717 } 1718 session->cipher_key.length = xform->cipher.key.length; 1719 1720 memcpy(session->cipher_key.data, xform->cipher.key.data, 1721 xform->cipher.key.length); 1722 session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 1723 DIR_ENC : DIR_DEC; 1724 1725 return 0; 1726 } 1727 1728 static int 1729 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused, 1730 struct rte_crypto_sym_xform *xform, 1731 dpaa_sec_session *session) 1732 { 1733 session->auth_alg = xform->auth.algo; 1734 session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, 1735 RTE_CACHE_LINE_SIZE); 1736 if (session->auth_key.data == NULL && xform->auth.key.length > 0) { 1737 DPAA_SEC_ERR("No Memory for auth key"); 1738 return -ENOMEM; 1739 } 1740 session->auth_key.length = xform->auth.key.length; 1741 session->digest_length = xform->auth.digest_length; 1742 1743 memcpy(session->auth_key.data, xform->auth.key.data, 1744 xform->auth.key.length); 1745 session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? 1746 DIR_ENC : DIR_DEC; 1747 1748 return 0; 1749 } 1750 1751 static int 1752 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused, 1753 struct rte_crypto_sym_xform *xform, 1754 dpaa_sec_session *session) 1755 { 1756 session->aead_alg = xform->aead.algo; 1757 session->iv.length = xform->aead.iv.length; 1758 session->iv.offset = xform->aead.iv.offset; 1759 session->auth_only_len = xform->aead.aad_length; 1760 session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length, 1761 RTE_CACHE_LINE_SIZE); 1762 if (session->aead_key.data == NULL && xform->aead.key.length > 0) { 1763 DPAA_SEC_ERR("No Memory for aead key\n"); 1764 return -ENOMEM; 1765 } 1766 session->aead_key.length = xform->aead.key.length; 1767 session->digest_length = xform->aead.digest_length; 1768 1769 memcpy(session->aead_key.data, xform->aead.key.data, 1770 xform->aead.key.length); 1771 session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 1772 DIR_ENC : DIR_DEC; 1773 1774 return 0; 1775 } 1776 1777 static struct qman_fq * 1778 dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi) 1779 { 1780 unsigned int i; 1781 1782 for (i = 0; i < qi->max_nb_sessions; i++) { 1783 if (qi->inq_attach[i] == 0) { 1784 qi->inq_attach[i] = 1; 1785 return &qi->inq[i]; 1786 } 1787 } 1788 DPAA_SEC_WARN("All ses session in use %x", qi->max_nb_sessions); 1789 1790 return NULL; 1791 } 1792 1793 static int 1794 dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq) 1795 { 1796 unsigned int i; 1797 1798 for (i = 0; i < qi->max_nb_sessions; i++) { 1799 if (&qi->inq[i] == fq) { 1800 qman_retire_fq(fq, NULL); 1801 qman_oos_fq(fq); 1802 qi->inq_attach[i] = 0; 1803 return 0; 1804 } 1805 } 1806 return -1; 1807 } 1808 1809 static int 1810 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess) 1811 { 1812 int ret; 1813 1814 sess->qp = qp; 1815 ret = dpaa_sec_prep_cdb(sess); 1816 if (ret) { 1817 DPAA_SEC_ERR("Unable to prepare sec cdb"); 1818 return -1; 1819 } 1820 if (unlikely(!RTE_PER_LCORE(dpaa_io))) { 1821 ret = rte_dpaa_portal_init((void *)0); 1822 if (ret) { 1823 DPAA_SEC_ERR("Failure in affining portal"); 1824 return ret; 1825 } 1826 } 1827 ret = dpaa_sec_init_rx(sess->inq, dpaa_mem_vtop(&sess->cdb), 1828 qman_fq_fqid(&qp->outq)); 1829 if (ret) 1830 DPAA_SEC_ERR("Unable to init sec queue"); 1831 1832 return ret; 1833 } 1834 1835 static int 1836 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, 1837 struct rte_crypto_sym_xform *xform, void *sess) 1838 { 1839 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 1840 dpaa_sec_session *session = sess; 1841 1842 PMD_INIT_FUNC_TRACE(); 1843 1844 if (unlikely(sess == NULL)) { 1845 DPAA_SEC_ERR("invalid session struct"); 1846 return -EINVAL; 1847 } 1848 memset(session, 0, sizeof(dpaa_sec_session)); 1849 1850 /* Default IV length = 0 */ 1851 session->iv.length = 0; 1852 1853 /* Cipher Only */ 1854 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { 1855 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 1856 dpaa_sec_cipher_init(dev, xform, session); 1857 1858 /* Authentication Only */ 1859 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1860 xform->next == NULL) { 1861 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; 1862 dpaa_sec_auth_init(dev, xform, session); 1863 1864 /* Cipher then Authenticate */ 1865 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 1866 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 1867 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1868 dpaa_sec_cipher_init(dev, xform, session); 1869 dpaa_sec_auth_init(dev, xform->next, session); 1870 } else { 1871 DPAA_SEC_ERR("Not supported: Auth then Cipher"); 1872 return -EINVAL; 1873 } 1874 1875 /* Authenticate then Cipher */ 1876 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1877 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 1878 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 1879 dpaa_sec_auth_init(dev, xform, session); 1880 dpaa_sec_cipher_init(dev, xform->next, session); 1881 } else { 1882 DPAA_SEC_ERR("Not supported: Auth then Cipher"); 1883 return -EINVAL; 1884 } 1885 1886 /* AEAD operation for AES-GCM kind of Algorithms */ 1887 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && 1888 xform->next == NULL) { 1889 dpaa_sec_aead_init(dev, xform, session); 1890 1891 } else { 1892 DPAA_SEC_ERR("Invalid crypto type"); 1893 return -EINVAL; 1894 } 1895 session->ctx_pool = internals->ctx_pool; 1896 rte_spinlock_lock(&internals->lock); 1897 session->inq = dpaa_sec_attach_rxq(internals); 1898 rte_spinlock_unlock(&internals->lock); 1899 if (session->inq == NULL) { 1900 DPAA_SEC_ERR("unable to attach sec queue"); 1901 goto err1; 1902 } 1903 1904 return 0; 1905 1906 err1: 1907 rte_free(session->cipher_key.data); 1908 rte_free(session->auth_key.data); 1909 memset(session, 0, sizeof(dpaa_sec_session)); 1910 1911 return -EINVAL; 1912 } 1913 1914 static int 1915 dpaa_sec_sym_session_configure(struct rte_cryptodev *dev, 1916 struct rte_crypto_sym_xform *xform, 1917 struct rte_cryptodev_sym_session *sess, 1918 struct rte_mempool *mempool) 1919 { 1920 void *sess_private_data; 1921 int ret; 1922 1923 PMD_INIT_FUNC_TRACE(); 1924 1925 if (rte_mempool_get(mempool, &sess_private_data)) { 1926 DPAA_SEC_ERR("Couldn't get object from session mempool"); 1927 return -ENOMEM; 1928 } 1929 1930 ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data); 1931 if (ret != 0) { 1932 DPAA_SEC_ERR("failed to configure session parameters"); 1933 1934 /* Return session to mempool */ 1935 rte_mempool_put(mempool, sess_private_data); 1936 return ret; 1937 } 1938 1939 set_sym_session_private_data(sess, dev->driver_id, 1940 sess_private_data); 1941 1942 1943 return 0; 1944 } 1945 1946 /** Clear the memory of session so it doesn't leave key material behind */ 1947 static void 1948 dpaa_sec_sym_session_clear(struct rte_cryptodev *dev, 1949 struct rte_cryptodev_sym_session *sess) 1950 { 1951 struct dpaa_sec_dev_private *qi = dev->data->dev_private; 1952 uint8_t index = dev->driver_id; 1953 void *sess_priv = get_sym_session_private_data(sess, index); 1954 1955 PMD_INIT_FUNC_TRACE(); 1956 1957 dpaa_sec_session *s = (dpaa_sec_session *)sess_priv; 1958 1959 if (sess_priv) { 1960 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 1961 1962 if (s->inq) 1963 dpaa_sec_detach_rxq(qi, s->inq); 1964 rte_free(s->cipher_key.data); 1965 rte_free(s->auth_key.data); 1966 memset(s, 0, sizeof(dpaa_sec_session)); 1967 set_sym_session_private_data(sess, index, NULL); 1968 rte_mempool_put(sess_mp, sess_priv); 1969 } 1970 } 1971 1972 static int 1973 dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, 1974 struct rte_security_session_conf *conf, 1975 void *sess) 1976 { 1977 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 1978 struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec; 1979 struct rte_crypto_auth_xform *auth_xform = NULL; 1980 struct rte_crypto_cipher_xform *cipher_xform = NULL; 1981 dpaa_sec_session *session = (dpaa_sec_session *)sess; 1982 1983 PMD_INIT_FUNC_TRACE(); 1984 1985 memset(session, 0, sizeof(dpaa_sec_session)); 1986 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { 1987 cipher_xform = &conf->crypto_xform->cipher; 1988 if (conf->crypto_xform->next) 1989 auth_xform = &conf->crypto_xform->next->auth; 1990 } else { 1991 auth_xform = &conf->crypto_xform->auth; 1992 if (conf->crypto_xform->next) 1993 cipher_xform = &conf->crypto_xform->next->cipher; 1994 } 1995 session->proto_alg = conf->protocol; 1996 1997 if (cipher_xform && cipher_xform->algo != RTE_CRYPTO_CIPHER_NULL) { 1998 session->cipher_key.data = rte_zmalloc(NULL, 1999 cipher_xform->key.length, 2000 RTE_CACHE_LINE_SIZE); 2001 if (session->cipher_key.data == NULL && 2002 cipher_xform->key.length > 0) { 2003 DPAA_SEC_ERR("No Memory for cipher key"); 2004 return -ENOMEM; 2005 } 2006 memcpy(session->cipher_key.data, cipher_xform->key.data, 2007 cipher_xform->key.length); 2008 session->cipher_key.length = cipher_xform->key.length; 2009 2010 switch (cipher_xform->algo) { 2011 case RTE_CRYPTO_CIPHER_AES_CBC: 2012 case RTE_CRYPTO_CIPHER_3DES_CBC: 2013 case RTE_CRYPTO_CIPHER_AES_CTR: 2014 break; 2015 default: 2016 DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u", 2017 cipher_xform->algo); 2018 goto out; 2019 } 2020 session->cipher_alg = cipher_xform->algo; 2021 } else { 2022 session->cipher_key.data = NULL; 2023 session->cipher_key.length = 0; 2024 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; 2025 } 2026 2027 if (auth_xform && auth_xform->algo != RTE_CRYPTO_AUTH_NULL) { 2028 session->auth_key.data = rte_zmalloc(NULL, 2029 auth_xform->key.length, 2030 RTE_CACHE_LINE_SIZE); 2031 if (session->auth_key.data == NULL && 2032 auth_xform->key.length > 0) { 2033 DPAA_SEC_ERR("No Memory for auth key"); 2034 rte_free(session->cipher_key.data); 2035 return -ENOMEM; 2036 } 2037 memcpy(session->auth_key.data, auth_xform->key.data, 2038 auth_xform->key.length); 2039 session->auth_key.length = auth_xform->key.length; 2040 2041 switch (auth_xform->algo) { 2042 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2043 case RTE_CRYPTO_AUTH_MD5_HMAC: 2044 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2045 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2046 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2047 case RTE_CRYPTO_AUTH_AES_CMAC: 2048 break; 2049 default: 2050 DPAA_SEC_ERR("Crypto: Unsupported auth alg %u", 2051 auth_xform->algo); 2052 goto out; 2053 } 2054 session->auth_alg = auth_xform->algo; 2055 } else { 2056 session->auth_key.data = NULL; 2057 session->auth_key.length = 0; 2058 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 2059 } 2060 2061 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { 2062 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) + 2063 sizeof(session->ip4_hdr)); 2064 session->ip4_hdr.ip_v = IPVERSION; 2065 session->ip4_hdr.ip_hl = 5; 2066 session->ip4_hdr.ip_len = rte_cpu_to_be_16( 2067 sizeof(session->ip4_hdr)); 2068 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp; 2069 session->ip4_hdr.ip_id = 0; 2070 session->ip4_hdr.ip_off = 0; 2071 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl; 2072 session->ip4_hdr.ip_p = (ipsec_xform->proto == 2073 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP 2074 : IPPROTO_AH; 2075 session->ip4_hdr.ip_sum = 0; 2076 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip; 2077 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip; 2078 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *) 2079 (void *)&session->ip4_hdr, 2080 sizeof(struct ip)); 2081 2082 session->encap_pdb.options = 2083 (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) | 2084 PDBOPTS_ESP_OIHI_PDB_INL | 2085 PDBOPTS_ESP_IVSRC | 2086 PDBHMO_ESP_ENCAP_DTTL | 2087 PDBHMO_ESP_SNR; 2088 session->encap_pdb.spi = ipsec_xform->spi; 2089 session->encap_pdb.ip_hdr_len = sizeof(struct ip); 2090 2091 session->dir = DIR_ENC; 2092 } else if (ipsec_xform->direction == 2093 RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { 2094 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb)); 2095 session->decap_pdb.options = sizeof(struct ip) << 16; 2096 session->dir = DIR_DEC; 2097 } else 2098 goto out; 2099 session->ctx_pool = internals->ctx_pool; 2100 rte_spinlock_lock(&internals->lock); 2101 session->inq = dpaa_sec_attach_rxq(internals); 2102 rte_spinlock_unlock(&internals->lock); 2103 if (session->inq == NULL) { 2104 DPAA_SEC_ERR("unable to attach sec queue"); 2105 goto out; 2106 } 2107 2108 2109 return 0; 2110 out: 2111 rte_free(session->auth_key.data); 2112 rte_free(session->cipher_key.data); 2113 memset(session, 0, sizeof(dpaa_sec_session)); 2114 return -1; 2115 } 2116 2117 static int 2118 dpaa_sec_security_session_create(void *dev, 2119 struct rte_security_session_conf *conf, 2120 struct rte_security_session *sess, 2121 struct rte_mempool *mempool) 2122 { 2123 void *sess_private_data; 2124 struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev; 2125 int ret; 2126 2127 if (rte_mempool_get(mempool, &sess_private_data)) { 2128 DPAA_SEC_ERR("Couldn't get object from session mempool"); 2129 return -ENOMEM; 2130 } 2131 2132 switch (conf->protocol) { 2133 case RTE_SECURITY_PROTOCOL_IPSEC: 2134 ret = dpaa_sec_set_ipsec_session(cdev, conf, 2135 sess_private_data); 2136 break; 2137 case RTE_SECURITY_PROTOCOL_MACSEC: 2138 return -ENOTSUP; 2139 default: 2140 return -EINVAL; 2141 } 2142 if (ret != 0) { 2143 DPAA_SEC_ERR("failed to configure session parameters"); 2144 /* Return session to mempool */ 2145 rte_mempool_put(mempool, sess_private_data); 2146 return ret; 2147 } 2148 2149 set_sec_session_private_data(sess, sess_private_data); 2150 2151 return ret; 2152 } 2153 2154 /** Clear the memory of session so it doesn't leave key material behind */ 2155 static int 2156 dpaa_sec_security_session_destroy(void *dev __rte_unused, 2157 struct rte_security_session *sess) 2158 { 2159 PMD_INIT_FUNC_TRACE(); 2160 void *sess_priv = get_sec_session_private_data(sess); 2161 2162 dpaa_sec_session *s = (dpaa_sec_session *)sess_priv; 2163 2164 if (sess_priv) { 2165 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 2166 2167 rte_free(s->cipher_key.data); 2168 rte_free(s->auth_key.data); 2169 memset(sess, 0, sizeof(dpaa_sec_session)); 2170 set_sec_session_private_data(sess, NULL); 2171 rte_mempool_put(sess_mp, sess_priv); 2172 } 2173 return 0; 2174 } 2175 2176 2177 static int 2178 dpaa_sec_dev_configure(struct rte_cryptodev *dev, 2179 struct rte_cryptodev_config *config __rte_unused) 2180 { 2181 2182 char str[20]; 2183 struct dpaa_sec_dev_private *internals; 2184 2185 PMD_INIT_FUNC_TRACE(); 2186 2187 internals = dev->data->dev_private; 2188 sprintf(str, "ctx_pool_%d", dev->data->dev_id); 2189 if (!internals->ctx_pool) { 2190 internals->ctx_pool = rte_mempool_create((const char *)str, 2191 CTX_POOL_NUM_BUFS, 2192 CTX_POOL_BUF_SIZE, 2193 CTX_POOL_CACHE_SIZE, 0, 2194 NULL, NULL, NULL, NULL, 2195 SOCKET_ID_ANY, 0); 2196 if (!internals->ctx_pool) { 2197 DPAA_SEC_ERR("%s create failed\n", str); 2198 return -ENOMEM; 2199 } 2200 } else 2201 DPAA_SEC_INFO("mempool already created for dev_id : %d", 2202 dev->data->dev_id); 2203 2204 return 0; 2205 } 2206 2207 static int 2208 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused) 2209 { 2210 PMD_INIT_FUNC_TRACE(); 2211 return 0; 2212 } 2213 2214 static void 2215 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused) 2216 { 2217 PMD_INIT_FUNC_TRACE(); 2218 } 2219 2220 static int 2221 dpaa_sec_dev_close(struct rte_cryptodev *dev) 2222 { 2223 struct dpaa_sec_dev_private *internals; 2224 2225 PMD_INIT_FUNC_TRACE(); 2226 2227 if (dev == NULL) 2228 return -ENOMEM; 2229 2230 internals = dev->data->dev_private; 2231 rte_mempool_free(internals->ctx_pool); 2232 internals->ctx_pool = NULL; 2233 2234 return 0; 2235 } 2236 2237 static void 2238 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev, 2239 struct rte_cryptodev_info *info) 2240 { 2241 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 2242 2243 PMD_INIT_FUNC_TRACE(); 2244 if (info != NULL) { 2245 info->max_nb_queue_pairs = internals->max_nb_queue_pairs; 2246 info->feature_flags = dev->feature_flags; 2247 info->capabilities = dpaa_sec_capabilities; 2248 info->sym.max_nb_sessions = internals->max_nb_sessions; 2249 info->driver_id = cryptodev_driver_id; 2250 } 2251 } 2252 2253 static struct rte_cryptodev_ops crypto_ops = { 2254 .dev_configure = dpaa_sec_dev_configure, 2255 .dev_start = dpaa_sec_dev_start, 2256 .dev_stop = dpaa_sec_dev_stop, 2257 .dev_close = dpaa_sec_dev_close, 2258 .dev_infos_get = dpaa_sec_dev_infos_get, 2259 .queue_pair_setup = dpaa_sec_queue_pair_setup, 2260 .queue_pair_release = dpaa_sec_queue_pair_release, 2261 .queue_pair_count = dpaa_sec_queue_pair_count, 2262 .sym_session_get_size = dpaa_sec_sym_session_get_size, 2263 .sym_session_configure = dpaa_sec_sym_session_configure, 2264 .sym_session_clear = dpaa_sec_sym_session_clear 2265 }; 2266 2267 static const struct rte_security_capability * 2268 dpaa_sec_capabilities_get(void *device __rte_unused) 2269 { 2270 return dpaa_sec_security_cap; 2271 } 2272 2273 static const struct rte_security_ops dpaa_sec_security_ops = { 2274 .session_create = dpaa_sec_security_session_create, 2275 .session_update = NULL, 2276 .session_stats_get = NULL, 2277 .session_destroy = dpaa_sec_security_session_destroy, 2278 .set_pkt_metadata = NULL, 2279 .capabilities_get = dpaa_sec_capabilities_get 2280 }; 2281 2282 static int 2283 dpaa_sec_uninit(struct rte_cryptodev *dev) 2284 { 2285 struct dpaa_sec_dev_private *internals; 2286 2287 if (dev == NULL) 2288 return -ENODEV; 2289 2290 internals = dev->data->dev_private; 2291 rte_free(dev->security_ctx); 2292 2293 /* In case close has been called, internals->ctx_pool would be NULL */ 2294 rte_mempool_free(internals->ctx_pool); 2295 rte_free(internals); 2296 2297 DPAA_SEC_INFO("Closing DPAA_SEC device %s on numa socket %u", 2298 dev->data->name, rte_socket_id()); 2299 2300 return 0; 2301 } 2302 2303 static int 2304 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) 2305 { 2306 struct dpaa_sec_dev_private *internals; 2307 struct rte_security_ctx *security_instance; 2308 struct dpaa_sec_qp *qp; 2309 uint32_t i, flags; 2310 int ret; 2311 2312 PMD_INIT_FUNC_TRACE(); 2313 2314 cryptodev->driver_id = cryptodev_driver_id; 2315 cryptodev->dev_ops = &crypto_ops; 2316 2317 cryptodev->enqueue_burst = dpaa_sec_enqueue_burst; 2318 cryptodev->dequeue_burst = dpaa_sec_dequeue_burst; 2319 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 2320 RTE_CRYPTODEV_FF_HW_ACCELERATED | 2321 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 2322 RTE_CRYPTODEV_FF_SECURITY | 2323 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 2324 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | 2325 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 2326 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | 2327 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; 2328 2329 internals = cryptodev->data->dev_private; 2330 internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS; 2331 internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS; 2332 2333 /* 2334 * For secondary processes, we don't initialise any further as primary 2335 * has already done this work. Only check we don't need a different 2336 * RX function 2337 */ 2338 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 2339 DPAA_SEC_WARN("Device already init by primary process"); 2340 return 0; 2341 } 2342 2343 /* Initialize security_ctx only for primary process*/ 2344 security_instance = rte_malloc("rte_security_instances_ops", 2345 sizeof(struct rte_security_ctx), 0); 2346 if (security_instance == NULL) 2347 return -ENOMEM; 2348 security_instance->device = (void *)cryptodev; 2349 security_instance->ops = &dpaa_sec_security_ops; 2350 security_instance->sess_cnt = 0; 2351 cryptodev->security_ctx = security_instance; 2352 2353 rte_spinlock_init(&internals->lock); 2354 for (i = 0; i < internals->max_nb_queue_pairs; i++) { 2355 /* init qman fq for queue pair */ 2356 qp = &internals->qps[i]; 2357 ret = dpaa_sec_init_tx(&qp->outq); 2358 if (ret) { 2359 DPAA_SEC_ERR("config tx of queue pair %d", i); 2360 goto init_error; 2361 } 2362 } 2363 2364 flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID | 2365 QMAN_FQ_FLAG_TO_DCPORTAL; 2366 for (i = 0; i < internals->max_nb_sessions; i++) { 2367 /* create rx qman fq for sessions*/ 2368 ret = qman_create_fq(0, flags, &internals->inq[i]); 2369 if (unlikely(ret != 0)) { 2370 DPAA_SEC_ERR("sec qman_create_fq failed"); 2371 goto init_error; 2372 } 2373 } 2374 2375 RTE_LOG(INFO, PMD, "%s cryptodev init\n", cryptodev->data->name); 2376 return 0; 2377 2378 init_error: 2379 DPAA_SEC_ERR("driver %s: create failed\n", cryptodev->data->name); 2380 2381 dpaa_sec_uninit(cryptodev); 2382 return -EFAULT; 2383 } 2384 2385 static int 2386 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused, 2387 struct rte_dpaa_device *dpaa_dev) 2388 { 2389 struct rte_cryptodev *cryptodev; 2390 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; 2391 2392 int retval; 2393 2394 sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id); 2395 2396 cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id()); 2397 if (cryptodev == NULL) 2398 return -ENOMEM; 2399 2400 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 2401 cryptodev->data->dev_private = rte_zmalloc_socket( 2402 "cryptodev private structure", 2403 sizeof(struct dpaa_sec_dev_private), 2404 RTE_CACHE_LINE_SIZE, 2405 rte_socket_id()); 2406 2407 if (cryptodev->data->dev_private == NULL) 2408 rte_panic("Cannot allocate memzone for private " 2409 "device data"); 2410 } 2411 2412 dpaa_dev->crypto_dev = cryptodev; 2413 cryptodev->device = &dpaa_dev->device; 2414 2415 /* init user callbacks */ 2416 TAILQ_INIT(&(cryptodev->link_intr_cbs)); 2417 2418 /* if sec device version is not configured */ 2419 if (!rta_get_sec_era()) { 2420 const struct device_node *caam_node; 2421 2422 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { 2423 const uint32_t *prop = of_get_property(caam_node, 2424 "fsl,sec-era", 2425 NULL); 2426 if (prop) { 2427 rta_set_sec_era( 2428 INTL_SEC_ERA(rte_cpu_to_be_32(*prop))); 2429 break; 2430 } 2431 } 2432 } 2433 2434 /* Invoke PMD device initialization function */ 2435 retval = dpaa_sec_dev_init(cryptodev); 2436 if (retval == 0) 2437 return 0; 2438 2439 /* In case of error, cleanup is done */ 2440 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 2441 rte_free(cryptodev->data->dev_private); 2442 2443 rte_cryptodev_pmd_release_device(cryptodev); 2444 2445 return -ENXIO; 2446 } 2447 2448 static int 2449 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev) 2450 { 2451 struct rte_cryptodev *cryptodev; 2452 int ret; 2453 2454 cryptodev = dpaa_dev->crypto_dev; 2455 if (cryptodev == NULL) 2456 return -ENODEV; 2457 2458 ret = dpaa_sec_uninit(cryptodev); 2459 if (ret) 2460 return ret; 2461 2462 return rte_cryptodev_pmd_destroy(cryptodev); 2463 } 2464 2465 static struct rte_dpaa_driver rte_dpaa_sec_driver = { 2466 .drv_type = FSL_DPAA_CRYPTO, 2467 .driver = { 2468 .name = "DPAA SEC PMD" 2469 }, 2470 .probe = cryptodev_dpaa_sec_probe, 2471 .remove = cryptodev_dpaa_sec_remove, 2472 }; 2473 2474 static struct cryptodev_driver dpaa_sec_crypto_drv; 2475 2476 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver); 2477 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver, 2478 cryptodev_driver_id); 2479 2480 RTE_INIT(dpaa_sec_init_log) 2481 { 2482 dpaa_logtype_sec = rte_log_register("pmd.crypto.dpaa"); 2483 if (dpaa_logtype_sec >= 0) 2484 rte_log_set_level(dpaa_logtype_sec, RTE_LOG_NOTICE); 2485 } 2486