1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021-2022 NXP 3 */ 4 5 #include <rte_byteorder.h> 6 #include <rte_common.h> 7 #include <cryptodev_pmd.h> 8 #include <rte_crypto.h> 9 #include <rte_cryptodev.h> 10 #include <rte_security_driver.h> 11 12 /* RTA header files */ 13 #include <desc/algo.h> 14 #include <desc/ipsec.h> 15 16 #include <bus_dpaa_driver.h> 17 #include <dpaa_sec.h> 18 #include <dpaa_sec_log.h> 19 20 struct dpaa_sec_raw_dp_ctx { 21 dpaa_sec_session *session; 22 uint32_t tail; 23 uint32_t head; 24 uint16_t cached_enqueue; 25 uint16_t cached_dequeue; 26 }; 27 28 static inline int 29 is_encode(dpaa_sec_session *ses) 30 { 31 return ses->dir == DIR_ENC; 32 } 33 34 static inline int is_decode(dpaa_sec_session *ses) 35 { 36 return ses->dir == DIR_DEC; 37 } 38 39 static __rte_always_inline int 40 dpaa_sec_raw_enqueue_done(void *qp_data, uint8_t *drv_ctx, uint32_t n) 41 { 42 RTE_SET_USED(qp_data); 43 RTE_SET_USED(drv_ctx); 44 RTE_SET_USED(n); 45 46 return 0; 47 } 48 49 static __rte_always_inline int 50 dpaa_sec_raw_dequeue_done(void *qp_data, uint8_t *drv_ctx, uint32_t n) 51 { 52 RTE_SET_USED(qp_data); 53 RTE_SET_USED(drv_ctx); 54 RTE_SET_USED(n); 55 56 return 0; 57 } 58 59 static inline struct dpaa_sec_op_ctx * 60 dpaa_sec_alloc_raw_ctx(dpaa_sec_session *ses, int sg_count) 61 { 62 struct dpaa_sec_op_ctx *ctx; 63 int i, retval; 64 65 retval = rte_mempool_get( 66 ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool, 67 (void **)(&ctx)); 68 if (!ctx || retval) { 69 DPAA_SEC_DP_WARN("Alloc sec descriptor failed!"); 70 return NULL; 71 } 72 /* 73 * Clear SG memory. There are 16 SG entries of 16 Bytes each. 74 * one call to dcbz_64() clear 64 bytes, hence calling it 4 times 75 * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for 76 * each packet, memset is costlier than dcbz_64(). 77 */ 78 for (i = 0; i < sg_count && i < MAX_JOB_SG_ENTRIES; i += 4) 79 dcbz_64(&ctx->job.sg[i]); 80 81 ctx->ctx_pool = ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool; 82 ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx); 83 84 return ctx; 85 } 86 87 static struct dpaa_sec_job * 88 build_dpaa_raw_dp_auth_fd(uint8_t *drv_ctx, 89 struct rte_crypto_sgl *sgl, 90 struct rte_crypto_sgl *dest_sgl, 91 struct rte_crypto_va_iova_ptr *iv, 92 struct rte_crypto_va_iova_ptr *digest, 93 struct rte_crypto_va_iova_ptr *auth_iv, 94 union rte_crypto_sym_ofs ofs, 95 void *userdata, 96 struct qm_fd *fd) 97 { 98 RTE_SET_USED(dest_sgl); 99 RTE_SET_USED(iv); 100 RTE_SET_USED(auth_iv); 101 RTE_SET_USED(fd); 102 103 dpaa_sec_session *ses = 104 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session; 105 struct dpaa_sec_job *cf; 106 struct dpaa_sec_op_ctx *ctx; 107 struct qm_sg_entry *sg, *out_sg, *in_sg; 108 phys_addr_t start_addr; 109 uint8_t *old_digest, extra_segs; 110 int data_len, data_offset, total_len = 0; 111 unsigned int i; 112 113 for (i = 0; i < sgl->num; i++) 114 total_len += sgl->vec[i].len; 115 116 data_len = total_len - ofs.ofs.auth.head - ofs.ofs.auth.tail; 117 data_offset = ofs.ofs.auth.head; 118 119 /* Support only length in bits for SNOW3G and ZUC */ 120 121 if (is_decode(ses)) 122 extra_segs = 3; 123 else 124 extra_segs = 2; 125 126 if (sgl->num > MAX_SG_ENTRIES) { 127 DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d", 128 MAX_SG_ENTRIES); 129 return NULL; 130 } 131 ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + extra_segs); 132 if (!ctx) 133 return NULL; 134 135 cf = &ctx->job; 136 ctx->userdata = (void *)userdata; 137 old_digest = ctx->digest; 138 139 /* output */ 140 out_sg = &cf->sg[0]; 141 qm_sg_entry_set64(out_sg, digest->iova); 142 out_sg->length = ses->digest_length; 143 cpu_to_hw_sg(out_sg); 144 145 /* input */ 146 in_sg = &cf->sg[1]; 147 /* need to extend the input to a compound frame */ 148 in_sg->extension = 1; 149 in_sg->final = 1; 150 in_sg->length = data_len; 151 qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2])); 152 153 /* 1st seg */ 154 sg = in_sg + 1; 155 156 if (ses->iv.length) { 157 uint8_t *iv_ptr; 158 159 iv_ptr = rte_crypto_op_ctod_offset(userdata, uint8_t *, 160 ses->iv.offset); 161 162 if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) { 163 iv_ptr = conv_to_snow_f9_iv(iv_ptr); 164 sg->length = 12; 165 } else if (ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) { 166 iv_ptr = conv_to_zuc_eia_iv(iv_ptr); 167 sg->length = 8; 168 } else { 169 sg->length = ses->iv.length; 170 } 171 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr)); 172 in_sg->length += sg->length; 173 cpu_to_hw_sg(sg); 174 sg++; 175 } 176 177 qm_sg_entry_set64(sg, sgl->vec[0].iova); 178 sg->offset = data_offset; 179 180 if (data_len <= (int)(sgl->vec[0].len - data_offset)) { 181 sg->length = data_len; 182 } else { 183 sg->length = sgl->vec[0].len - data_offset; 184 185 /* remaining i/p segs */ 186 for (i = 1; i < sgl->num; i++) { 187 cpu_to_hw_sg(sg); 188 sg++; 189 qm_sg_entry_set64(sg, sgl->vec[i].iova); 190 if (data_len > (int)sgl->vec[i].len) 191 sg->length = sgl->vec[0].len; 192 else 193 sg->length = data_len; 194 195 data_len = data_len - sg->length; 196 if (data_len < 1) 197 break; 198 } 199 } 200 201 if (is_decode(ses)) { 202 /* Digest verification case */ 203 cpu_to_hw_sg(sg); 204 sg++; 205 rte_memcpy(old_digest, digest->va, 206 ses->digest_length); 207 start_addr = rte_dpaa_mem_vtop(old_digest); 208 qm_sg_entry_set64(sg, start_addr); 209 sg->length = ses->digest_length; 210 in_sg->length += ses->digest_length; 211 } 212 sg->final = 1; 213 cpu_to_hw_sg(sg); 214 cpu_to_hw_sg(in_sg); 215 216 return cf; 217 } 218 219 static inline struct dpaa_sec_job * 220 build_raw_cipher_auth_gcm_sg(uint8_t *drv_ctx, 221 struct rte_crypto_sgl *sgl, 222 struct rte_crypto_sgl *dest_sgl, 223 struct rte_crypto_va_iova_ptr *iv, 224 struct rte_crypto_va_iova_ptr *digest, 225 struct rte_crypto_va_iova_ptr *auth_iv, 226 union rte_crypto_sym_ofs ofs, 227 void *userdata, 228 struct qm_fd *fd) 229 { 230 dpaa_sec_session *ses = 231 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session; 232 struct dpaa_sec_job *cf; 233 struct dpaa_sec_op_ctx *ctx; 234 struct qm_sg_entry *sg, *out_sg, *in_sg; 235 uint8_t extra_req_segs; 236 uint8_t *IV_ptr = iv->va; 237 int data_len = 0, aead_len = 0; 238 unsigned int i; 239 240 for (i = 0; i < sgl->num; i++) 241 data_len += sgl->vec[i].len; 242 243 extra_req_segs = 4; 244 aead_len = data_len - ofs.ofs.cipher.head - ofs.ofs.cipher.tail; 245 246 if (ses->auth_only_len) 247 extra_req_segs++; 248 249 if (sgl->num > MAX_SG_ENTRIES) { 250 DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d", 251 MAX_SG_ENTRIES); 252 return NULL; 253 } 254 255 ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + extra_req_segs); 256 if (!ctx) 257 return NULL; 258 259 cf = &ctx->job; 260 ctx->userdata = (void *)userdata; 261 262 rte_prefetch0(cf->sg); 263 264 /* output */ 265 out_sg = &cf->sg[0]; 266 out_sg->extension = 1; 267 if (is_encode(ses)) 268 out_sg->length = aead_len + ses->digest_length; 269 else 270 out_sg->length = aead_len; 271 272 /* output sg entries */ 273 sg = &cf->sg[2]; 274 qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg)); 275 cpu_to_hw_sg(out_sg); 276 277 if (dest_sgl) { 278 /* 1st seg */ 279 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova); 280 sg->length = dest_sgl->vec[0].len - ofs.ofs.cipher.head; 281 sg->offset = ofs.ofs.cipher.head; 282 283 /* Successive segs */ 284 for (i = 1; i < dest_sgl->num; i++) { 285 cpu_to_hw_sg(sg); 286 sg++; 287 qm_sg_entry_set64(sg, dest_sgl->vec[i].iova); 288 sg->length = dest_sgl->vec[i].len; 289 } 290 } else { 291 /* 1st seg */ 292 qm_sg_entry_set64(sg, sgl->vec[0].iova); 293 sg->length = sgl->vec[0].len - ofs.ofs.cipher.head; 294 sg->offset = ofs.ofs.cipher.head; 295 296 /* Successive segs */ 297 for (i = 1; i < sgl->num; i++) { 298 cpu_to_hw_sg(sg); 299 sg++; 300 qm_sg_entry_set64(sg, sgl->vec[i].iova); 301 sg->length = sgl->vec[i].len; 302 } 303 304 } 305 306 if (is_encode(ses)) { 307 cpu_to_hw_sg(sg); 308 /* set auth output */ 309 sg++; 310 qm_sg_entry_set64(sg, digest->iova); 311 sg->length = ses->digest_length; 312 } 313 sg->final = 1; 314 cpu_to_hw_sg(sg); 315 316 /* input */ 317 in_sg = &cf->sg[1]; 318 in_sg->extension = 1; 319 in_sg->final = 1; 320 if (is_encode(ses)) 321 in_sg->length = ses->iv.length + aead_len 322 + ses->auth_only_len; 323 else 324 in_sg->length = ses->iv.length + aead_len 325 + ses->auth_only_len + ses->digest_length; 326 327 /* input sg entries */ 328 sg++; 329 qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); 330 cpu_to_hw_sg(in_sg); 331 332 /* 1st seg IV */ 333 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); 334 sg->length = ses->iv.length; 335 cpu_to_hw_sg(sg); 336 337 /* 2 seg auth only */ 338 if (ses->auth_only_len) { 339 sg++; 340 qm_sg_entry_set64(sg, auth_iv->iova); 341 sg->length = ses->auth_only_len; 342 cpu_to_hw_sg(sg); 343 } 344 345 /* 3rd seg */ 346 sg++; 347 qm_sg_entry_set64(sg, sgl->vec[0].iova); 348 sg->length = sgl->vec[0].len - ofs.ofs.cipher.head; 349 sg->offset = ofs.ofs.cipher.head; 350 351 /* Successive segs */ 352 for (i = 1; i < sgl->num; i++) { 353 cpu_to_hw_sg(sg); 354 sg++; 355 qm_sg_entry_set64(sg, sgl->vec[i].iova); 356 sg->length = sgl->vec[i].len; 357 } 358 359 if (is_decode(ses)) { 360 cpu_to_hw_sg(sg); 361 sg++; 362 memcpy(ctx->digest, digest->va, 363 ses->digest_length); 364 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest)); 365 sg->length = ses->digest_length; 366 } 367 sg->final = 1; 368 cpu_to_hw_sg(sg); 369 370 if (ses->auth_only_len) 371 fd->cmd = 0x80000000 | ses->auth_only_len; 372 373 return cf; 374 } 375 376 static inline struct dpaa_sec_job * 377 build_dpaa_raw_dp_chain_fd(uint8_t *drv_ctx, 378 struct rte_crypto_sgl *sgl, 379 struct rte_crypto_sgl *dest_sgl, 380 struct rte_crypto_va_iova_ptr *iv, 381 struct rte_crypto_va_iova_ptr *digest, 382 struct rte_crypto_va_iova_ptr *auth_iv, 383 union rte_crypto_sym_ofs ofs, 384 void *userdata, 385 struct qm_fd *fd) 386 { 387 RTE_SET_USED(auth_iv); 388 389 dpaa_sec_session *ses = 390 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session; 391 struct dpaa_sec_job *cf; 392 struct dpaa_sec_op_ctx *ctx; 393 struct qm_sg_entry *sg, *out_sg, *in_sg; 394 uint8_t *IV_ptr = iv->va; 395 unsigned int i; 396 uint16_t auth_hdr_len = ofs.ofs.cipher.head - 397 ofs.ofs.auth.head; 398 uint16_t auth_tail_len; 399 uint32_t auth_only_len; 400 int data_len = 0, auth_len = 0, cipher_len = 0; 401 402 for (i = 0; i < sgl->num; i++) 403 data_len += sgl->vec[i].len; 404 405 cipher_len = data_len - ofs.ofs.cipher.head - ofs.ofs.cipher.tail; 406 auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail; 407 auth_tail_len = auth_len - cipher_len - auth_hdr_len; 408 auth_only_len = (auth_tail_len << 16) | auth_hdr_len; 409 410 if (sgl->num > MAX_SG_ENTRIES) { 411 DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d", 412 MAX_SG_ENTRIES); 413 return NULL; 414 } 415 416 ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + 4); 417 if (!ctx) 418 return NULL; 419 420 cf = &ctx->job; 421 ctx->userdata = (void *)userdata; 422 423 rte_prefetch0(cf->sg); 424 425 /* output */ 426 out_sg = &cf->sg[0]; 427 out_sg->extension = 1; 428 if (is_encode(ses)) 429 out_sg->length = cipher_len + ses->digest_length; 430 else 431 out_sg->length = cipher_len; 432 433 /* output sg entries */ 434 sg = &cf->sg[2]; 435 qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg)); 436 cpu_to_hw_sg(out_sg); 437 438 /* 1st seg */ 439 if (dest_sgl) { 440 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova); 441 sg->length = dest_sgl->vec[0].len - ofs.ofs.cipher.head; 442 sg->offset = ofs.ofs.cipher.head; 443 444 /* Successive segs */ 445 for (i = 1; i < dest_sgl->num; i++) { 446 cpu_to_hw_sg(sg); 447 sg++; 448 qm_sg_entry_set64(sg, dest_sgl->vec[i].iova); 449 sg->length = dest_sgl->vec[i].len; 450 } 451 sg->length -= ofs.ofs.cipher.tail; 452 } else { 453 qm_sg_entry_set64(sg, sgl->vec[0].iova); 454 sg->length = sgl->vec[0].len - ofs.ofs.cipher.head; 455 sg->offset = ofs.ofs.cipher.head; 456 457 /* Successive segs */ 458 for (i = 1; i < sgl->num; i++) { 459 cpu_to_hw_sg(sg); 460 sg++; 461 qm_sg_entry_set64(sg, sgl->vec[i].iova); 462 sg->length = sgl->vec[i].len; 463 } 464 sg->length -= ofs.ofs.cipher.tail; 465 } 466 467 if (is_encode(ses)) { 468 cpu_to_hw_sg(sg); 469 /* set auth output */ 470 sg++; 471 qm_sg_entry_set64(sg, digest->iova); 472 sg->length = ses->digest_length; 473 } 474 sg->final = 1; 475 cpu_to_hw_sg(sg); 476 477 /* input */ 478 in_sg = &cf->sg[1]; 479 in_sg->extension = 1; 480 in_sg->final = 1; 481 if (is_encode(ses)) 482 in_sg->length = ses->iv.length + auth_len; 483 else 484 in_sg->length = ses->iv.length + auth_len 485 + ses->digest_length; 486 487 /* input sg entries */ 488 sg++; 489 qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); 490 cpu_to_hw_sg(in_sg); 491 492 /* 1st seg IV */ 493 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); 494 sg->length = ses->iv.length; 495 cpu_to_hw_sg(sg); 496 497 /* 2 seg */ 498 sg++; 499 qm_sg_entry_set64(sg, sgl->vec[0].iova); 500 sg->length = sgl->vec[0].len - ofs.ofs.auth.head; 501 sg->offset = ofs.ofs.auth.head; 502 503 /* Successive segs */ 504 for (i = 1; i < sgl->num; i++) { 505 cpu_to_hw_sg(sg); 506 sg++; 507 qm_sg_entry_set64(sg, sgl->vec[i].iova); 508 sg->length = sgl->vec[i].len; 509 } 510 511 if (is_decode(ses)) { 512 cpu_to_hw_sg(sg); 513 sg++; 514 memcpy(ctx->digest, digest->va, 515 ses->digest_length); 516 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest)); 517 sg->length = ses->digest_length; 518 } 519 sg->final = 1; 520 cpu_to_hw_sg(sg); 521 522 if (auth_only_len) 523 fd->cmd = 0x80000000 | auth_only_len; 524 525 return cf; 526 } 527 528 static struct dpaa_sec_job * 529 build_dpaa_raw_dp_cipher_fd(uint8_t *drv_ctx, 530 struct rte_crypto_sgl *sgl, 531 struct rte_crypto_sgl *dest_sgl, 532 struct rte_crypto_va_iova_ptr *iv, 533 struct rte_crypto_va_iova_ptr *digest, 534 struct rte_crypto_va_iova_ptr *auth_iv, 535 union rte_crypto_sym_ofs ofs, 536 void *userdata, 537 struct qm_fd *fd) 538 { 539 RTE_SET_USED(digest); 540 RTE_SET_USED(auth_iv); 541 RTE_SET_USED(fd); 542 543 dpaa_sec_session *ses = 544 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session; 545 struct dpaa_sec_job *cf; 546 struct dpaa_sec_op_ctx *ctx; 547 struct qm_sg_entry *sg, *out_sg, *in_sg; 548 unsigned int i; 549 uint8_t *IV_ptr = iv->va; 550 int data_len, total_len = 0, data_offset; 551 552 for (i = 0; i < sgl->num; i++) 553 total_len += sgl->vec[i].len; 554 555 data_len = total_len - ofs.ofs.cipher.head - ofs.ofs.cipher.tail; 556 data_offset = ofs.ofs.cipher.head; 557 558 /* Support lengths in bits only for SNOW3G and ZUC */ 559 if (sgl->num > MAX_SG_ENTRIES) { 560 DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d", 561 MAX_SG_ENTRIES); 562 return NULL; 563 } 564 565 ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + 3); 566 if (!ctx) 567 return NULL; 568 569 cf = &ctx->job; 570 ctx->userdata = (void *)userdata; 571 572 /* output */ 573 out_sg = &cf->sg[0]; 574 out_sg->extension = 1; 575 out_sg->length = data_len; 576 qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2])); 577 cpu_to_hw_sg(out_sg); 578 579 if (dest_sgl) { 580 /* 1st seg */ 581 sg = &cf->sg[2]; 582 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova); 583 sg->length = dest_sgl->vec[0].len - data_offset; 584 sg->offset = data_offset; 585 586 /* Successive segs */ 587 for (i = 1; i < dest_sgl->num; i++) { 588 cpu_to_hw_sg(sg); 589 sg++; 590 qm_sg_entry_set64(sg, dest_sgl->vec[i].iova); 591 sg->length = dest_sgl->vec[i].len; 592 } 593 } else { 594 /* 1st seg */ 595 sg = &cf->sg[2]; 596 qm_sg_entry_set64(sg, sgl->vec[0].iova); 597 sg->length = sgl->vec[0].len - data_offset; 598 sg->offset = data_offset; 599 600 /* Successive segs */ 601 for (i = 1; i < sgl->num; i++) { 602 cpu_to_hw_sg(sg); 603 sg++; 604 qm_sg_entry_set64(sg, sgl->vec[i].iova); 605 sg->length = sgl->vec[i].len; 606 } 607 608 } 609 sg->final = 1; 610 cpu_to_hw_sg(sg); 611 612 /* input */ 613 in_sg = &cf->sg[1]; 614 in_sg->extension = 1; 615 in_sg->final = 1; 616 in_sg->length = data_len + ses->iv.length; 617 618 sg++; 619 qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); 620 cpu_to_hw_sg(in_sg); 621 622 /* IV */ 623 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); 624 sg->length = ses->iv.length; 625 cpu_to_hw_sg(sg); 626 627 /* 1st seg */ 628 sg++; 629 qm_sg_entry_set64(sg, sgl->vec[0].iova); 630 sg->length = sgl->vec[0].len - data_offset; 631 sg->offset = data_offset; 632 633 /* Successive segs */ 634 for (i = 1; i < sgl->num; i++) { 635 cpu_to_hw_sg(sg); 636 sg++; 637 qm_sg_entry_set64(sg, sgl->vec[i].iova); 638 sg->length = sgl->vec[i].len; 639 } 640 sg->final = 1; 641 cpu_to_hw_sg(sg); 642 643 return cf; 644 } 645 646 static inline struct dpaa_sec_job * 647 build_dpaa_raw_proto_sg(uint8_t *drv_ctx, 648 struct rte_crypto_sgl *sgl, 649 struct rte_crypto_sgl *dest_sgl, 650 struct rte_crypto_va_iova_ptr *iv, 651 struct rte_crypto_va_iova_ptr *digest, 652 struct rte_crypto_va_iova_ptr *auth_iv, 653 union rte_crypto_sym_ofs ofs, 654 void *userdata, 655 struct qm_fd *fd) 656 { 657 RTE_SET_USED(iv); 658 RTE_SET_USED(digest); 659 RTE_SET_USED(auth_iv); 660 RTE_SET_USED(ofs); 661 662 dpaa_sec_session *ses = 663 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session; 664 struct dpaa_sec_job *cf; 665 struct dpaa_sec_op_ctx *ctx; 666 struct qm_sg_entry *sg, *out_sg, *in_sg; 667 uint32_t in_len = 0, out_len = 0; 668 unsigned int i; 669 670 if (sgl->num > MAX_SG_ENTRIES) { 671 DPAA_SEC_DP_ERR("Proto: Max sec segs supported is %d", 672 MAX_SG_ENTRIES); 673 return NULL; 674 } 675 676 ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + 4); 677 if (!ctx) 678 return NULL; 679 cf = &ctx->job; 680 ctx->userdata = (void *)userdata; 681 /* output */ 682 out_sg = &cf->sg[0]; 683 out_sg->extension = 1; 684 qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2])); 685 686 if (dest_sgl) { 687 /* 1st seg */ 688 sg = &cf->sg[2]; 689 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova); 690 sg->offset = 0; 691 sg->length = dest_sgl->vec[0].len; 692 out_len += sg->length; 693 694 for (i = 1; i < dest_sgl->num; i++) { 695 /* Successive segs */ 696 cpu_to_hw_sg(sg); 697 sg++; 698 qm_sg_entry_set64(sg, dest_sgl->vec[i].iova); 699 sg->offset = 0; 700 sg->length = dest_sgl->vec[i].len; 701 out_len += sg->length; 702 } 703 sg->length = dest_sgl->vec[i - 1].tot_len; 704 } else { 705 /* 1st seg */ 706 sg = &cf->sg[2]; 707 qm_sg_entry_set64(sg, sgl->vec[0].iova); 708 sg->offset = 0; 709 sg->length = sgl->vec[0].len; 710 out_len += sg->length; 711 712 for (i = 1; i < sgl->num; i++) { 713 /* Successive segs */ 714 cpu_to_hw_sg(sg); 715 sg++; 716 qm_sg_entry_set64(sg, sgl->vec[i].iova); 717 sg->offset = 0; 718 sg->length = sgl->vec[i].len; 719 out_len += sg->length; 720 } 721 sg->length = sgl->vec[i - 1].tot_len; 722 723 } 724 out_len += sg->length; 725 sg->final = 1; 726 cpu_to_hw_sg(sg); 727 728 out_sg->length = out_len; 729 cpu_to_hw_sg(out_sg); 730 731 /* input */ 732 in_sg = &cf->sg[1]; 733 in_sg->extension = 1; 734 in_sg->final = 1; 735 in_len = sgl->vec[0].len; 736 737 sg++; 738 qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); 739 740 /* 1st seg */ 741 qm_sg_entry_set64(sg, sgl->vec[0].iova); 742 sg->length = sgl->vec[0].len; 743 sg->offset = 0; 744 745 /* Successive segs */ 746 for (i = 1; i < sgl->num; i++) { 747 cpu_to_hw_sg(sg); 748 sg++; 749 qm_sg_entry_set64(sg, sgl->vec[i].iova); 750 sg->length = sgl->vec[i].len; 751 sg->offset = 0; 752 in_len += sg->length; 753 } 754 sg->final = 1; 755 cpu_to_hw_sg(sg); 756 757 in_sg->length = in_len; 758 cpu_to_hw_sg(in_sg); 759 760 if ((ses->ctxt == DPAA_SEC_PDCP) && ses->pdcp.hfn_ovd) { 761 fd->cmd = 0x80000000 | 762 *((uint32_t *)((uint8_t *)userdata + 763 ses->pdcp.hfn_ovd_offset)); 764 DPAA_SEC_DP_DEBUG("Per packet HFN: %x, ovd:%u", 765 *((uint32_t *)((uint8_t *)userdata + 766 ses->pdcp.hfn_ovd_offset)), 767 ses->pdcp.hfn_ovd); 768 } 769 770 return cf; 771 } 772 773 static uint32_t 774 dpaa_sec_raw_enqueue_burst(void *qp_data, uint8_t *drv_ctx, 775 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs, 776 void *user_data[], int *status) 777 { 778 /* Function to transmit the frames to given device and queuepair */ 779 uint32_t loop; 780 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp_data; 781 uint16_t num_tx = 0; 782 struct qm_fd fds[DPAA_SEC_BURST], *fd; 783 uint32_t frames_to_send; 784 struct dpaa_sec_job *cf; 785 dpaa_sec_session *ses = 786 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session; 787 uint32_t flags[DPAA_SEC_BURST] = {0}; 788 struct qman_fq *inq[DPAA_SEC_BURST]; 789 790 if (unlikely(!DPAA_PER_LCORE_PORTAL)) { 791 if (rte_dpaa_portal_init((void *)0)) { 792 DPAA_SEC_ERR("Failure in affining portal"); 793 return 0; 794 } 795 } 796 797 while (vec->num) { 798 frames_to_send = (vec->num > DPAA_SEC_BURST) ? 799 DPAA_SEC_BURST : vec->num; 800 for (loop = 0; loop < frames_to_send; loop++) { 801 if (unlikely(!ses->qp[rte_lcore_id() % MAX_DPAA_CORES])) { 802 if (dpaa_sec_attach_sess_q(dpaa_qp, ses)) { 803 frames_to_send = loop; 804 goto send_pkts; 805 } 806 } else if (unlikely(ses->qp[rte_lcore_id() % 807 MAX_DPAA_CORES] != dpaa_qp)) { 808 DPAA_SEC_DP_ERR("Old:sess->qp = %p" 809 " New qp = %p", 810 ses->qp[rte_lcore_id() % 811 MAX_DPAA_CORES], dpaa_qp); 812 frames_to_send = loop; 813 goto send_pkts; 814 } 815 816 /*Clear the unused FD fields before sending*/ 817 fd = &fds[loop]; 818 memset(fd, 0, sizeof(struct qm_fd)); 819 cf = ses->build_raw_dp_fd(drv_ctx, 820 &vec->src_sgl[loop], 821 &vec->dest_sgl[loop], 822 &vec->iv[loop], 823 &vec->digest[loop], 824 &vec->auth_iv[loop], 825 ofs, 826 user_data[loop], 827 fd); 828 if (!cf) { 829 DPAA_SEC_ERR("error: Improper packet contents" 830 " for crypto operation"); 831 goto skip_tx; 832 } 833 inq[loop] = ses->inq[rte_lcore_id() % MAX_DPAA_CORES]; 834 qm_fd_addr_set64(fd, rte_dpaa_mem_vtop(cf->sg)); 835 fd->_format1 = qm_fd_compound; 836 fd->length29 = 2 * sizeof(struct qm_sg_entry); 837 838 status[loop] = 1; 839 } 840 send_pkts: 841 loop = 0; 842 while (loop < frames_to_send) { 843 loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop], 844 &flags[loop], frames_to_send - loop); 845 } 846 vec->num -= frames_to_send; 847 num_tx += frames_to_send; 848 } 849 850 skip_tx: 851 dpaa_qp->tx_pkts += num_tx; 852 dpaa_qp->tx_errs += vec->num - num_tx; 853 854 return num_tx; 855 } 856 857 static int 858 dpaa_sec_deq_raw(struct dpaa_sec_qp *qp, void **out_user_data, 859 uint8_t is_user_data_array, 860 rte_cryptodev_raw_post_dequeue_t post_dequeue, 861 int nb_ops) 862 { 863 struct qman_fq *fq; 864 unsigned int pkts = 0; 865 int num_rx_bufs, ret; 866 struct qm_dqrr_entry *dq; 867 uint32_t vdqcr_flags = 0; 868 uint8_t is_success = 0; 869 870 fq = &qp->outq; 871 /* 872 * Until request for four buffers, we provide exact number of buffers. 873 * Otherwise we do not set the QM_VDQCR_EXACT flag. 874 * Not setting QM_VDQCR_EXACT flag can provide two more buffers than 875 * requested, so we request two less in this case. 876 */ 877 if (nb_ops < 4) { 878 vdqcr_flags = QM_VDQCR_EXACT; 879 num_rx_bufs = nb_ops; 880 } else { 881 num_rx_bufs = nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES ? 882 (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_ops - 2); 883 } 884 ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags); 885 if (ret) 886 return 0; 887 888 do { 889 const struct qm_fd *fd; 890 struct dpaa_sec_job *job; 891 struct dpaa_sec_op_ctx *ctx; 892 893 dq = qman_dequeue(fq); 894 if (!dq) 895 continue; 896 897 fd = &dq->fd; 898 /* sg is embedded in an op ctx, 899 * sg[0] is for output 900 * sg[1] for input 901 */ 902 job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd)); 903 904 ctx = container_of(job, struct dpaa_sec_op_ctx, job); 905 ctx->fd_status = fd->status; 906 if (is_user_data_array) 907 out_user_data[pkts] = ctx->userdata; 908 else 909 out_user_data[0] = ctx->userdata; 910 911 if (!ctx->fd_status) { 912 is_success = true; 913 } else { 914 is_success = false; 915 DPAA_SEC_DP_WARN("SEC return err:0x%x", ctx->fd_status); 916 } 917 post_dequeue(ctx->op, pkts, is_success); 918 pkts++; 919 920 /* report op status to sym->op and then free the ctx memory */ 921 rte_mempool_put(ctx->ctx_pool, (void *)ctx); 922 923 qman_dqrr_consume(fq, dq); 924 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 925 926 return pkts; 927 } 928 929 930 static __rte_always_inline uint32_t 931 dpaa_sec_raw_dequeue_burst(void *qp_data, uint8_t *drv_ctx, 932 rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count, 933 uint32_t max_nb_to_dequeue, 934 rte_cryptodev_raw_post_dequeue_t post_dequeue, 935 void **out_user_data, uint8_t is_user_data_array, 936 uint32_t *n_success, int *dequeue_status) 937 { 938 RTE_SET_USED(drv_ctx); 939 RTE_SET_USED(get_dequeue_count); 940 uint16_t num_rx; 941 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp_data; 942 uint32_t nb_ops = max_nb_to_dequeue; 943 944 if (unlikely(!DPAA_PER_LCORE_PORTAL)) { 945 if (rte_dpaa_portal_init((void *)0)) { 946 DPAA_SEC_ERR("Failure in affining portal"); 947 return 0; 948 } 949 } 950 951 num_rx = dpaa_sec_deq_raw(dpaa_qp, out_user_data, 952 is_user_data_array, post_dequeue, nb_ops); 953 954 dpaa_qp->rx_pkts += num_rx; 955 *dequeue_status = 1; 956 *n_success = num_rx; 957 958 DPAA_SEC_DP_DEBUG("SEC Received %d Packets", num_rx); 959 960 return num_rx; 961 } 962 963 static __rte_always_inline int 964 dpaa_sec_raw_enqueue(void *qp_data, uint8_t *drv_ctx, 965 struct rte_crypto_vec *data_vec, 966 uint16_t n_data_vecs, union rte_crypto_sym_ofs ofs, 967 struct rte_crypto_va_iova_ptr *iv, 968 struct rte_crypto_va_iova_ptr *digest, 969 struct rte_crypto_va_iova_ptr *aad_or_auth_iv, 970 void *user_data) 971 { 972 RTE_SET_USED(qp_data); 973 RTE_SET_USED(drv_ctx); 974 RTE_SET_USED(data_vec); 975 RTE_SET_USED(n_data_vecs); 976 RTE_SET_USED(ofs); 977 RTE_SET_USED(iv); 978 RTE_SET_USED(digest); 979 RTE_SET_USED(aad_or_auth_iv); 980 RTE_SET_USED(user_data); 981 982 return 0; 983 } 984 985 static __rte_always_inline void * 986 dpaa_sec_raw_dequeue(void *qp_data, uint8_t *drv_ctx, int *dequeue_status, 987 enum rte_crypto_op_status *op_status) 988 { 989 RTE_SET_USED(qp_data); 990 RTE_SET_USED(drv_ctx); 991 RTE_SET_USED(dequeue_status); 992 RTE_SET_USED(op_status); 993 994 return NULL; 995 } 996 997 int 998 dpaa_sec_configure_raw_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id, 999 struct rte_crypto_raw_dp_ctx *raw_dp_ctx, 1000 enum rte_crypto_op_sess_type sess_type, 1001 union rte_cryptodev_session_ctx session_ctx, uint8_t is_update) 1002 { 1003 dpaa_sec_session *sess; 1004 struct dpaa_sec_raw_dp_ctx *dp_ctx; 1005 RTE_SET_USED(qp_id); 1006 1007 if (!is_update) { 1008 memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx)); 1009 raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id]; 1010 } 1011 1012 if (sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) 1013 sess = SECURITY_GET_SESS_PRIV(session_ctx.sec_sess); 1014 else if (sess_type == RTE_CRYPTO_OP_WITH_SESSION) 1015 sess = (dpaa_sec_session *) 1016 CRYPTODEV_GET_SYM_SESS_PRIV(session_ctx.crypto_sess); 1017 else 1018 return -ENOTSUP; 1019 raw_dp_ctx->dequeue_burst = dpaa_sec_raw_dequeue_burst; 1020 raw_dp_ctx->dequeue = dpaa_sec_raw_dequeue; 1021 raw_dp_ctx->dequeue_done = dpaa_sec_raw_dequeue_done; 1022 raw_dp_ctx->enqueue_burst = dpaa_sec_raw_enqueue_burst; 1023 raw_dp_ctx->enqueue = dpaa_sec_raw_enqueue; 1024 raw_dp_ctx->enqueue_done = dpaa_sec_raw_enqueue_done; 1025 1026 if (sess->ctxt == DPAA_SEC_CIPHER) 1027 sess->build_raw_dp_fd = build_dpaa_raw_dp_cipher_fd; 1028 else if (sess->ctxt == DPAA_SEC_AUTH) 1029 sess->build_raw_dp_fd = build_dpaa_raw_dp_auth_fd; 1030 else if (sess->ctxt == DPAA_SEC_CIPHER_HASH) 1031 sess->build_raw_dp_fd = build_dpaa_raw_dp_chain_fd; 1032 else if (sess->ctxt == DPAA_SEC_AEAD) 1033 sess->build_raw_dp_fd = build_raw_cipher_auth_gcm_sg; 1034 else if (sess->ctxt == DPAA_SEC_IPSEC || 1035 sess->ctxt == DPAA_SEC_PDCP) 1036 sess->build_raw_dp_fd = build_dpaa_raw_proto_sg; 1037 else 1038 return -ENOTSUP; 1039 dp_ctx = (struct dpaa_sec_raw_dp_ctx *)raw_dp_ctx->drv_ctx_data; 1040 dp_ctx->session = sess; 1041 1042 return 0; 1043 } 1044 1045 int 1046 dpaa_sec_get_dp_ctx_size(__rte_unused struct rte_cryptodev *dev) 1047 { 1048 return sizeof(struct dpaa_sec_raw_dp_ctx); 1049 } 1050