1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <rte_cryptodev.h> 6 #include <cryptodev_pmd.h> 7 #include <rte_event_crypto_adapter.h> 8 #include <rte_ip.h> 9 10 #include "cn10k_cryptodev.h" 11 #include "cn10k_cryptodev_ops.h" 12 #include "cn10k_ipsec.h" 13 #include "cn10k_ipsec_la_ops.h" 14 #include "cnxk_ae.h" 15 #include "cnxk_cryptodev.h" 16 #include "cnxk_cryptodev_ops.h" 17 #include "cnxk_eventdev.h" 18 #include "cnxk_se.h" 19 20 #include "roc_api.h" 21 22 #define PKTS_PER_LOOP 32 23 #define PKTS_PER_STEORL 16 24 25 /* Holds information required to send crypto operations in one burst */ 26 struct ops_burst { 27 struct rte_crypto_op *op[PKTS_PER_LOOP]; 28 uint64_t w2[PKTS_PER_LOOP]; 29 struct cn10k_sso_hws *ws; 30 struct cnxk_cpt_qp *qp; 31 uint16_t nb_ops; 32 }; 33 34 /* Holds information required to send vector of operations */ 35 struct vec_request { 36 struct cpt_inflight_req *req; 37 struct rte_event_vector *vec; 38 uint64_t w2; 39 }; 40 41 static inline struct cnxk_se_sess * 42 cn10k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op) 43 { 44 struct rte_crypto_sym_op *sym_op = op->sym; 45 struct rte_cryptodev_sym_session *sess; 46 struct cnxk_se_sess *priv; 47 int ret; 48 49 /* Create temporary session */ 50 if (rte_mempool_get(qp->sess_mp, (void **)&sess) < 0) 51 return NULL; 52 53 ret = sym_session_configure(qp->lf.roc_cpt, sym_op->xform, 54 sess); 55 if (ret) { 56 rte_mempool_put(qp->sess_mp, (void *)sess); 57 goto sess_put; 58 } 59 60 priv = (void *)sess->driver_priv_data; 61 sym_op->session = sess; 62 63 return priv; 64 65 sess_put: 66 rte_mempool_put(qp->sess_mp, sess); 67 return NULL; 68 } 69 70 static __rte_always_inline int __rte_hot 71 cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op, 72 struct cn10k_sec_session *sess, struct cpt_inst_s *inst) 73 { 74 struct rte_crypto_sym_op *sym_op = op->sym; 75 struct cn10k_ipsec_sa *sa; 76 int ret; 77 78 if (unlikely(sym_op->m_dst && sym_op->m_dst != sym_op->m_src)) { 79 plt_dp_err("Out of place is not supported"); 80 return -ENOTSUP; 81 } 82 83 if (unlikely(!rte_pktmbuf_is_contiguous(sym_op->m_src))) { 84 plt_dp_err("Scatter Gather mode is not supported"); 85 return -ENOTSUP; 86 } 87 88 sa = &sess->sa; 89 90 if (sa->is_outbound) 91 ret = process_outb_sa(&qp->lf, op, sa, inst); 92 else 93 ret = process_inb_sa(op, sa, inst); 94 95 return ret; 96 } 97 98 static inline int 99 cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[], 100 struct cpt_inst_s inst[], struct cpt_inflight_req *infl_req) 101 { 102 struct cn10k_sec_session *sec_sess; 103 struct rte_crypto_asym_op *asym_op; 104 struct rte_crypto_sym_op *sym_op; 105 struct cnxk_ae_sess *ae_sess; 106 struct cnxk_se_sess *sess; 107 struct rte_crypto_op *op; 108 uint64_t w7; 109 int ret; 110 111 const union cpt_res_s res = { 112 .cn10k.compcode = CPT_COMP_NOT_DONE, 113 }; 114 115 op = ops[0]; 116 117 inst[0].w0.u64 = 0; 118 inst[0].w2.u64 = 0; 119 inst[0].w3.u64 = 0; 120 121 sym_op = op->sym; 122 123 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 124 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 125 sec_sess = SECURITY_GET_SESS_PRIV(sym_op->session); 126 ret = cpt_sec_inst_fill(qp, op, sec_sess, &inst[0]); 127 if (unlikely(ret)) 128 return 0; 129 w7 = sec_sess->sa.inst.w7; 130 } else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 131 sess = CRYPTODEV_GET_SYM_SESS_PRIV(sym_op->session); 132 ret = cpt_sym_inst_fill(qp, op, sess, infl_req, 133 &inst[0]); 134 if (unlikely(ret)) 135 return 0; 136 w7 = sess->cpt_inst_w7; 137 } else { 138 sess = cn10k_cpt_sym_temp_sess_create(qp, op); 139 if (unlikely(sess == NULL)) { 140 plt_dp_err("Could not create temp session"); 141 return 0; 142 } 143 144 ret = cpt_sym_inst_fill(qp, op, sess, infl_req, 145 &inst[0]); 146 if (unlikely(ret)) { 147 sym_session_clear(op->sym->session); 148 rte_mempool_put(qp->sess_mp, op->sym->session); 149 return 0; 150 } 151 w7 = sess->cpt_inst_w7; 152 } 153 } else if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) { 154 155 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 156 asym_op = op->asym; 157 ae_sess = (struct cnxk_ae_sess *) 158 asym_op->session->sess_private_data; 159 ret = cnxk_ae_enqueue(qp, op, infl_req, &inst[0], 160 ae_sess); 161 if (unlikely(ret)) 162 return 0; 163 w7 = ae_sess->cpt_inst_w7; 164 } else { 165 plt_dp_err("Not supported Asym op without session"); 166 return 0; 167 } 168 } else { 169 plt_dp_err("Unsupported op type"); 170 return 0; 171 } 172 173 inst[0].res_addr = (uint64_t)&infl_req->res; 174 __atomic_store_n(&infl_req->res.u64[0], res.u64[0], __ATOMIC_RELAXED); 175 infl_req->cop = op; 176 177 inst[0].w7.u64 = w7; 178 179 return 1; 180 } 181 182 static uint16_t 183 cn10k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops) 184 { 185 uint64_t lmt_base, lmt_arg, io_addr; 186 struct cpt_inflight_req *infl_req; 187 uint16_t nb_allowed, count = 0; 188 struct cnxk_cpt_qp *qp = qptr; 189 struct pending_queue *pend_q; 190 struct cpt_inst_s *inst; 191 union cpt_fc_write_s fc; 192 uint64_t *fc_addr; 193 uint16_t lmt_id; 194 uint64_t head; 195 int ret, i; 196 197 pend_q = &qp->pend_q; 198 199 const uint64_t pq_mask = pend_q->pq_mask; 200 201 head = pend_q->head; 202 nb_allowed = pending_queue_free_cnt(head, pend_q->tail, pq_mask); 203 nb_ops = RTE_MIN(nb_ops, nb_allowed); 204 205 if (unlikely(nb_ops == 0)) 206 return 0; 207 208 lmt_base = qp->lmtline.lmt_base; 209 io_addr = qp->lmtline.io_addr; 210 fc_addr = qp->lmtline.fc_addr; 211 212 const uint32_t fc_thresh = qp->lmtline.fc_thresh; 213 214 ROC_LMT_BASE_ID_GET(lmt_base, lmt_id); 215 inst = (struct cpt_inst_s *)lmt_base; 216 217 again: 218 fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED); 219 if (unlikely(fc.s.qsize > fc_thresh)) { 220 i = 0; 221 goto pend_q_commit; 222 } 223 224 for (i = 0; i < RTE_MIN(PKTS_PER_LOOP, nb_ops); i++) { 225 infl_req = &pend_q->req_queue[head]; 226 infl_req->op_flags = 0; 227 228 ret = cn10k_cpt_fill_inst(qp, ops + i, &inst[2 * i], infl_req); 229 if (unlikely(ret != 1)) { 230 plt_dp_err("Could not process op: %p", ops + i); 231 if (i == 0) 232 goto pend_q_commit; 233 break; 234 } 235 236 pending_queue_advance(&head, pq_mask); 237 } 238 239 if (i > PKTS_PER_STEORL) { 240 lmt_arg = ROC_CN10K_CPT_LMT_ARG | (PKTS_PER_STEORL - 1) << 12 | 241 (uint64_t)lmt_id; 242 roc_lmt_submit_steorl(lmt_arg, io_addr); 243 lmt_arg = ROC_CN10K_CPT_LMT_ARG | 244 (i - PKTS_PER_STEORL - 1) << 12 | 245 (uint64_t)(lmt_id + PKTS_PER_STEORL); 246 roc_lmt_submit_steorl(lmt_arg, io_addr); 247 } else { 248 lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - 1) << 12 | 249 (uint64_t)lmt_id; 250 roc_lmt_submit_steorl(lmt_arg, io_addr); 251 } 252 253 rte_io_wmb(); 254 255 if (nb_ops - i > 0 && i == PKTS_PER_LOOP) { 256 nb_ops -= i; 257 ops += i; 258 count += i; 259 goto again; 260 } 261 262 pend_q_commit: 263 rte_atomic_thread_fence(__ATOMIC_RELEASE); 264 265 pend_q->head = head; 266 pend_q->time_out = rte_get_timer_cycles() + 267 DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz(); 268 269 return count + i; 270 } 271 272 static int 273 cn10k_cpt_crypto_adapter_ev_mdata_set(struct rte_cryptodev *dev __rte_unused, 274 void *sess, 275 enum rte_crypto_op_type op_type, 276 enum rte_crypto_op_sess_type sess_type, 277 void *mdata) 278 { 279 union rte_event_crypto_metadata *ec_mdata = mdata; 280 struct rte_event *rsp_info; 281 struct cnxk_cpt_qp *qp; 282 uint64_t w2, tag_type; 283 uint8_t cdev_id; 284 int16_t qp_id; 285 286 /* Get queue pair */ 287 cdev_id = ec_mdata->request_info.cdev_id; 288 qp_id = ec_mdata->request_info.queue_pair_id; 289 qp = rte_cryptodevs[cdev_id].data->queue_pairs[qp_id]; 290 291 /* Prepare w2 */ 292 tag_type = qp->ca.vector_sz ? RTE_EVENT_TYPE_CRYPTODEV_VECTOR : RTE_EVENT_TYPE_CRYPTODEV; 293 rsp_info = &ec_mdata->response_info; 294 w2 = CNXK_CPT_INST_W2((tag_type << 28) | (rsp_info->sub_event_type << 20) | 295 rsp_info->flow_id, 296 rsp_info->sched_type, rsp_info->queue_id, 0); 297 298 /* Set meta according to session type */ 299 if (op_type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 300 if (sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 301 struct cn10k_sec_session *priv; 302 struct cn10k_ipsec_sa *sa; 303 304 priv = SECURITY_GET_SESS_PRIV(sess); 305 sa = &priv->sa; 306 sa->qp = qp; 307 sa->inst.w2 = w2; 308 } else if (sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 309 struct cnxk_se_sess *priv; 310 311 priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess); 312 priv->qp = qp; 313 priv->cpt_inst_w2 = w2; 314 } else 315 return -EINVAL; 316 } else if (op_type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) { 317 if (sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 318 struct rte_cryptodev_asym_session *asym_sess = sess; 319 struct cnxk_ae_sess *priv; 320 321 priv = (struct cnxk_ae_sess *)asym_sess->sess_private_data; 322 priv->qp = qp; 323 priv->cpt_inst_w2 = w2; 324 } else 325 return -EINVAL; 326 } else 327 return -EINVAL; 328 329 return 0; 330 } 331 332 static inline int 333 cn10k_ca_meta_info_extract(struct rte_crypto_op *op, 334 struct cnxk_cpt_qp **qp, uint64_t *w2) 335 { 336 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 337 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 338 struct cn10k_sec_session *priv; 339 struct cn10k_ipsec_sa *sa; 340 341 priv = SECURITY_GET_SESS_PRIV(op->sym->session); 342 sa = &priv->sa; 343 *qp = sa->qp; 344 *w2 = sa->inst.w2; 345 } else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 346 struct cnxk_se_sess *priv; 347 348 priv = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session); 349 *qp = priv->qp; 350 *w2 = priv->cpt_inst_w2; 351 } else { 352 union rte_event_crypto_metadata *ec_mdata; 353 struct rte_event *rsp_info; 354 uint8_t cdev_id; 355 uint16_t qp_id; 356 357 if (unlikely(op->private_data_offset == 0)) 358 return -EINVAL; 359 ec_mdata = (union rte_event_crypto_metadata *) 360 ((uint8_t *)op + op->private_data_offset); 361 rsp_info = &ec_mdata->response_info; 362 cdev_id = ec_mdata->request_info.cdev_id; 363 qp_id = ec_mdata->request_info.queue_pair_id; 364 *qp = rte_cryptodevs[cdev_id].data->queue_pairs[qp_id]; 365 *w2 = CNXK_CPT_INST_W2( 366 (RTE_EVENT_TYPE_CRYPTODEV << 28) | rsp_info->flow_id, 367 rsp_info->sched_type, rsp_info->queue_id, 0); 368 } 369 } else if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) { 370 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 371 struct rte_cryptodev_asym_session *asym_sess; 372 struct cnxk_ae_sess *priv; 373 374 asym_sess = op->asym->session; 375 priv = (struct cnxk_ae_sess *)asym_sess->sess_private_data; 376 *qp = priv->qp; 377 *w2 = priv->cpt_inst_w2; 378 } else 379 return -EINVAL; 380 } else 381 return -EINVAL; 382 383 return 0; 384 } 385 386 static inline void 387 cn10k_cpt_vec_inst_fill(struct vec_request *vec_req, struct cpt_inst_s *inst, 388 struct cnxk_cpt_qp *qp) 389 { 390 const union cpt_res_s res = {.cn10k.compcode = CPT_COMP_NOT_DONE}; 391 struct cpt_inflight_req *infl_req = vec_req->req; 392 393 const union cpt_inst_w4 w4 = { 394 .s.opcode_major = ROC_SE_MAJOR_OP_MISC, 395 .s.opcode_minor = ROC_SE_MISC_MINOR_OP_PASSTHROUGH, 396 .s.param1 = 1, 397 .s.param2 = 1, 398 .s.dlen = 0, 399 }; 400 401 infl_req->vec = vec_req->vec; 402 infl_req->qp = qp; 403 404 inst->res_addr = (uint64_t)&infl_req->res; 405 __atomic_store_n(&infl_req->res.u64[0], res.u64[0], __ATOMIC_RELAXED); 406 407 inst->w0.u64 = 0; 408 inst->w2.u64 = vec_req->w2; 409 inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req); 410 inst->w4.u64 = w4.u64; 411 inst->w7.u64 = ROC_CPT_DFLT_ENG_GRP_SE << 61; 412 } 413 414 static void 415 cn10k_cpt_vec_pkt_submission_timeout_handle(void) 416 { 417 plt_dp_err("Vector packet submission timedout"); 418 abort(); 419 } 420 421 static inline void 422 cn10k_cpt_vec_submit(struct vec_request vec_tbl[], uint16_t vec_tbl_len, struct cnxk_cpt_qp *qp) 423 { 424 uint64_t lmt_base, lmt_arg, lmt_id, io_addr; 425 union cpt_fc_write_s fc; 426 struct cpt_inst_s *inst; 427 uint16_t burst_size; 428 uint64_t *fc_addr; 429 int i; 430 431 if (vec_tbl_len == 0) 432 return; 433 434 const uint32_t fc_thresh = qp->lmtline.fc_thresh; 435 /* 436 * Use 10 mins timeout for the poll. It is not possible to recover from partial submission 437 * of vector packet. Actual packets for processing are submitted to CPT prior to this 438 * routine. Hence, any failure for submission of vector packet would indicate an 439 * unrecoverable error for the application. 440 */ 441 const uint64_t timeout = rte_get_timer_cycles() + 10 * 60 * rte_get_timer_hz(); 442 443 lmt_base = qp->lmtline.lmt_base; 444 io_addr = qp->lmtline.io_addr; 445 fc_addr = qp->lmtline.fc_addr; 446 ROC_LMT_BASE_ID_GET(lmt_base, lmt_id); 447 inst = (struct cpt_inst_s *)lmt_base; 448 449 again: 450 burst_size = RTE_MIN(PKTS_PER_STEORL, vec_tbl_len); 451 for (i = 0; i < burst_size; i++) 452 cn10k_cpt_vec_inst_fill(&vec_tbl[i], &inst[i * 2], qp); 453 454 do { 455 fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED); 456 if (likely(fc.s.qsize < fc_thresh)) 457 break; 458 if (unlikely(rte_get_timer_cycles() > timeout)) 459 cn10k_cpt_vec_pkt_submission_timeout_handle(); 460 } while (true); 461 462 lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - 1) << 12 | lmt_id; 463 roc_lmt_submit_steorl(lmt_arg, io_addr); 464 465 rte_io_wmb(); 466 467 vec_tbl_len -= i; 468 469 if (vec_tbl_len > 0) { 470 vec_tbl += i; 471 goto again; 472 } 473 } 474 475 static inline int 476 ca_lmtst_vec_submit(struct ops_burst *burst, struct vec_request vec_tbl[], uint16_t *vec_tbl_len) 477 { 478 struct cpt_inflight_req *infl_reqs[PKTS_PER_LOOP]; 479 uint64_t lmt_base, lmt_arg, io_addr; 480 uint16_t lmt_id, len = *vec_tbl_len; 481 struct cpt_inst_s *inst, *inst_base; 482 struct cpt_inflight_req *infl_req; 483 struct rte_event_vector *vec; 484 union cpt_fc_write_s fc; 485 struct cnxk_cpt_qp *qp; 486 uint64_t *fc_addr; 487 int ret, i, vi; 488 489 qp = burst->qp; 490 491 lmt_base = qp->lmtline.lmt_base; 492 io_addr = qp->lmtline.io_addr; 493 fc_addr = qp->lmtline.fc_addr; 494 495 const uint32_t fc_thresh = qp->lmtline.fc_thresh; 496 497 ROC_LMT_BASE_ID_GET(lmt_base, lmt_id); 498 inst_base = (struct cpt_inst_s *)lmt_base; 499 500 #ifdef CNXK_CRYPTODEV_DEBUG 501 if (unlikely(!qp->ca.enabled)) { 502 rte_errno = EINVAL; 503 return 0; 504 } 505 #endif 506 507 /* Perform fc check before putting packets into vectors */ 508 fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED); 509 if (unlikely(fc.s.qsize > fc_thresh)) { 510 rte_errno = EAGAIN; 511 return 0; 512 } 513 514 if (unlikely(rte_mempool_get_bulk(qp->ca.req_mp, (void **)infl_reqs, burst->nb_ops))) { 515 rte_errno = ENOMEM; 516 return 0; 517 } 518 519 for (i = 0; i < burst->nb_ops; i++) { 520 inst = &inst_base[2 * i]; 521 infl_req = infl_reqs[i]; 522 infl_req->op_flags = 0; 523 524 ret = cn10k_cpt_fill_inst(qp, &burst->op[i], inst, infl_req); 525 if (unlikely(ret != 1)) { 526 plt_cpt_dbg("Could not process op: %p", burst->op[i]); 527 if (i != 0) 528 goto submit; 529 else 530 goto put; 531 } 532 533 infl_req->res.cn10k.compcode = CPT_COMP_NOT_DONE; 534 infl_req->qp = qp; 535 inst->w3.u64 = 0x1; 536 537 /* Lookup for existing vector by w2 */ 538 for (vi = len - 1; vi >= 0; vi--) { 539 if (vec_tbl[vi].w2 != burst->w2[i]) 540 continue; 541 vec = vec_tbl[vi].vec; 542 if (unlikely(vec->nb_elem == qp->ca.vector_sz)) 543 continue; 544 vec->ptrs[vec->nb_elem++] = infl_req; 545 goto next_op; /* continue outer loop */ 546 } 547 548 /* No available vectors found, allocate a new one */ 549 if (unlikely(rte_mempool_get(qp->ca.vector_mp, (void **)&vec_tbl[len].vec))) { 550 rte_errno = ENOMEM; 551 if (i != 0) 552 goto submit; 553 else 554 goto put; 555 } 556 /* Also preallocate in-flight request, that will be used to 557 * submit misc passthrough instruction 558 */ 559 if (unlikely(rte_mempool_get(qp->ca.req_mp, (void **)&vec_tbl[len].req))) { 560 rte_mempool_put(qp->ca.vector_mp, vec_tbl[len].vec); 561 rte_errno = ENOMEM; 562 if (i != 0) 563 goto submit; 564 else 565 goto put; 566 } 567 vec_tbl[len].w2 = burst->w2[i]; 568 vec_tbl[len].vec->ptrs[0] = infl_req; 569 vec_tbl[len].vec->nb_elem = 1; 570 len++; 571 572 next_op:; 573 } 574 575 /* Submit operations in burst */ 576 submit: 577 if (CNXK_TT_FROM_TAG(burst->ws->gw_rdata) == SSO_TT_ORDERED) 578 roc_sso_hws_head_wait(burst->ws->base); 579 580 if (i > PKTS_PER_STEORL) { 581 lmt_arg = ROC_CN10K_CPT_LMT_ARG | (PKTS_PER_STEORL - 1) << 12 | (uint64_t)lmt_id; 582 roc_lmt_submit_steorl(lmt_arg, io_addr); 583 lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - PKTS_PER_STEORL - 1) << 12 | 584 (uint64_t)(lmt_id + PKTS_PER_STEORL); 585 roc_lmt_submit_steorl(lmt_arg, io_addr); 586 } else { 587 lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - 1) << 12 | (uint64_t)lmt_id; 588 roc_lmt_submit_steorl(lmt_arg, io_addr); 589 } 590 591 rte_io_wmb(); 592 593 put: 594 if (i != burst->nb_ops) 595 rte_mempool_put_bulk(qp->ca.req_mp, (void *)&infl_reqs[i], burst->nb_ops - i); 596 597 *vec_tbl_len = len; 598 599 return i; 600 } 601 602 static inline uint16_t 603 ca_lmtst_burst_submit(struct ops_burst *burst) 604 { 605 struct cpt_inflight_req *infl_reqs[PKTS_PER_LOOP]; 606 uint64_t lmt_base, lmt_arg, io_addr; 607 struct cpt_inst_s *inst, *inst_base; 608 struct cpt_inflight_req *infl_req; 609 union cpt_fc_write_s fc; 610 struct cnxk_cpt_qp *qp; 611 uint64_t *fc_addr; 612 uint16_t lmt_id; 613 int ret, i, j; 614 615 qp = burst->qp; 616 617 lmt_base = qp->lmtline.lmt_base; 618 io_addr = qp->lmtline.io_addr; 619 fc_addr = qp->lmtline.fc_addr; 620 621 const uint32_t fc_thresh = qp->lmtline.fc_thresh; 622 623 ROC_LMT_BASE_ID_GET(lmt_base, lmt_id); 624 inst_base = (struct cpt_inst_s *)lmt_base; 625 626 #ifdef CNXK_CRYPTODEV_DEBUG 627 if (unlikely(!qp->ca.enabled)) { 628 rte_errno = EINVAL; 629 return 0; 630 } 631 #endif 632 633 if (unlikely(rte_mempool_get_bulk(qp->ca.req_mp, (void **)infl_reqs, burst->nb_ops))) { 634 rte_errno = ENOMEM; 635 return 0; 636 } 637 638 for (i = 0; i < burst->nb_ops; i++) { 639 inst = &inst_base[2 * i]; 640 infl_req = infl_reqs[i]; 641 infl_req->op_flags = 0; 642 643 ret = cn10k_cpt_fill_inst(qp, &burst->op[i], inst, infl_req); 644 if (unlikely(ret != 1)) { 645 plt_dp_dbg("Could not process op: %p", burst->op[i]); 646 if (i != 0) 647 goto submit; 648 else 649 goto put; 650 } 651 652 infl_req->res.cn10k.compcode = CPT_COMP_NOT_DONE; 653 infl_req->qp = qp; 654 inst->w0.u64 = 0; 655 inst->res_addr = (uint64_t)&infl_req->res; 656 inst->w2.u64 = burst->w2[i]; 657 inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req); 658 } 659 660 fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED); 661 if (unlikely(fc.s.qsize > fc_thresh)) { 662 rte_errno = EAGAIN; 663 for (j = 0; j < i; j++) { 664 infl_req = infl_reqs[j]; 665 if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF)) 666 rte_mempool_put(qp->meta_info.pool, infl_req->mdata); 667 } 668 i = 0; 669 goto put; 670 } 671 672 submit: 673 if (CNXK_TT_FROM_TAG(burst->ws->gw_rdata) == SSO_TT_ORDERED) 674 roc_sso_hws_head_wait(burst->ws->base); 675 676 if (i > PKTS_PER_STEORL) { 677 lmt_arg = ROC_CN10K_CPT_LMT_ARG | (PKTS_PER_STEORL - 1) << 12 | (uint64_t)lmt_id; 678 roc_lmt_submit_steorl(lmt_arg, io_addr); 679 lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - PKTS_PER_STEORL - 1) << 12 | 680 (uint64_t)(lmt_id + PKTS_PER_STEORL); 681 roc_lmt_submit_steorl(lmt_arg, io_addr); 682 } else { 683 lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - 1) << 12 | (uint64_t)lmt_id; 684 roc_lmt_submit_steorl(lmt_arg, io_addr); 685 } 686 687 rte_io_wmb(); 688 689 put: 690 if (unlikely(i != burst->nb_ops)) 691 rte_mempool_put_bulk(qp->ca.req_mp, (void *)&infl_reqs[i], burst->nb_ops - i); 692 693 return i; 694 } 695 696 uint16_t __rte_hot 697 cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events) 698 { 699 uint16_t submitted, count = 0, vec_tbl_len = 0; 700 struct vec_request vec_tbl[nb_events]; 701 struct rte_crypto_op *op; 702 struct ops_burst burst; 703 struct cnxk_cpt_qp *qp; 704 bool is_vector = false; 705 uint64_t w2; 706 int ret, i; 707 708 burst.ws = ws; 709 burst.qp = NULL; 710 burst.nb_ops = 0; 711 712 for (i = 0; i < nb_events; i++) { 713 op = ev[i].event_ptr; 714 ret = cn10k_ca_meta_info_extract(op, &qp, &w2); 715 if (unlikely(ret)) { 716 rte_errno = EINVAL; 717 goto vec_submit; 718 } 719 720 /* Queue pair change check */ 721 if (qp != burst.qp) { 722 if (burst.nb_ops) { 723 if (is_vector) { 724 submitted = 725 ca_lmtst_vec_submit(&burst, vec_tbl, &vec_tbl_len); 726 /* 727 * Vector submission is required on qp change, but not in 728 * other cases, since we could send several vectors per 729 * lmtst instruction only for same qp 730 */ 731 cn10k_cpt_vec_submit(vec_tbl, vec_tbl_len, burst.qp); 732 vec_tbl_len = 0; 733 } else { 734 submitted = ca_lmtst_burst_submit(&burst); 735 } 736 count += submitted; 737 if (unlikely(submitted != burst.nb_ops)) 738 goto vec_submit; 739 burst.nb_ops = 0; 740 } 741 is_vector = qp->ca.vector_sz; 742 burst.qp = qp; 743 } 744 burst.w2[burst.nb_ops] = w2; 745 burst.op[burst.nb_ops] = op; 746 747 /* Max nb_ops per burst check */ 748 if (++burst.nb_ops == PKTS_PER_LOOP) { 749 if (is_vector) 750 submitted = ca_lmtst_vec_submit(&burst, vec_tbl, &vec_tbl_len); 751 else 752 submitted = ca_lmtst_burst_submit(&burst); 753 count += submitted; 754 if (unlikely(submitted != burst.nb_ops)) 755 goto vec_submit; 756 burst.nb_ops = 0; 757 } 758 } 759 /* Submit the rest of crypto operations */ 760 if (burst.nb_ops) { 761 if (is_vector) 762 count += ca_lmtst_vec_submit(&burst, vec_tbl, &vec_tbl_len); 763 else 764 count += ca_lmtst_burst_submit(&burst); 765 } 766 767 vec_submit: 768 cn10k_cpt_vec_submit(vec_tbl, vec_tbl_len, burst.qp); 769 return count; 770 } 771 772 static inline void 773 cn10k_cpt_sec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *res) 774 { 775 struct rte_mbuf *mbuf = cop->sym->m_src; 776 const uint16_t m_len = res->rlen; 777 778 mbuf->data_len = m_len; 779 mbuf->pkt_len = m_len; 780 781 switch (res->uc_compcode) { 782 case ROC_IE_OT_UCC_SUCCESS: 783 break; 784 case ROC_IE_OT_UCC_SUCCESS_PKT_IP_BADCSUM: 785 mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 786 break; 787 case ROC_IE_OT_UCC_SUCCESS_PKT_L4_GOODCSUM: 788 mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD | 789 RTE_MBUF_F_RX_IP_CKSUM_GOOD; 790 break; 791 case ROC_IE_OT_UCC_SUCCESS_PKT_L4_BADCSUM: 792 mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD | 793 RTE_MBUF_F_RX_IP_CKSUM_GOOD; 794 break; 795 case ROC_IE_OT_UCC_SUCCESS_PKT_IP_GOODCSUM: 796 mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; 797 break; 798 case ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_FIRST: 799 cop->aux_flags = RTE_CRYPTO_OP_AUX_FLAGS_IPSEC_SOFT_EXPIRY; 800 break; 801 default: 802 plt_dp_err("Success with unknown microcode completion code"); 803 break; 804 } 805 } 806 807 static inline void 808 cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, 809 struct rte_crypto_op *cop, 810 struct cpt_inflight_req *infl_req, 811 struct cpt_cn10k_res_s *res) 812 { 813 const uint8_t uc_compcode = res->uc_compcode; 814 const uint8_t compcode = res->compcode; 815 816 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 817 818 if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC && 819 cop->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 820 if (likely(compcode == CPT_COMP_WARN)) { 821 /* Success with additional info */ 822 cn10k_cpt_sec_post_process(cop, res); 823 } else { 824 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 825 plt_dp_info("HW completion code 0x%x", res->compcode); 826 if (compcode == CPT_COMP_GOOD) { 827 plt_dp_info( 828 "Request failed with microcode error"); 829 plt_dp_info("MC completion code 0x%x", 830 uc_compcode); 831 } 832 } 833 834 return; 835 } 836 837 if (likely(compcode == CPT_COMP_GOOD || compcode == CPT_COMP_WARN)) { 838 if (unlikely(uc_compcode)) { 839 if (uc_compcode == ROC_SE_ERR_GC_ICV_MISCOMPARE) 840 cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 841 else 842 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 843 844 plt_dp_info("Request failed with microcode error"); 845 plt_dp_info("MC completion code 0x%x", 846 res->uc_compcode); 847 goto temp_sess_free; 848 } 849 850 if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 851 /* Verify authentication data if required */ 852 if (unlikely(infl_req->op_flags & 853 CPT_OP_FLAGS_AUTH_VERIFY)) { 854 uintptr_t *rsp = infl_req->mdata; 855 compl_auth_verify(cop, (uint8_t *)rsp[0], 856 rsp[1]); 857 } 858 } else if (cop->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) { 859 struct rte_crypto_asym_op *op = cop->asym; 860 uintptr_t *mdata = infl_req->mdata; 861 struct cnxk_ae_sess *sess; 862 863 sess = (struct cnxk_ae_sess *) 864 op->session->sess_private_data; 865 866 cnxk_ae_post_process(cop, sess, (uint8_t *)mdata[0]); 867 } 868 } else { 869 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 870 plt_dp_info("HW completion code 0x%x", res->compcode); 871 872 switch (compcode) { 873 case CPT_COMP_INSTERR: 874 plt_dp_err("Request failed with instruction error"); 875 break; 876 case CPT_COMP_FAULT: 877 plt_dp_err("Request failed with DMA fault"); 878 break; 879 case CPT_COMP_HWERR: 880 plt_dp_err("Request failed with hardware error"); 881 break; 882 default: 883 plt_dp_err( 884 "Request failed with unknown completion code"); 885 } 886 } 887 888 temp_sess_free: 889 if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) { 890 if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 891 sym_session_clear(cop->sym->session); 892 rte_mempool_put(qp->sess_mp, cop->sym->session); 893 cop->sym->session = NULL; 894 } 895 } 896 } 897 898 uintptr_t 899 cn10k_cpt_crypto_adapter_dequeue(uintptr_t get_work1) 900 { 901 struct cpt_inflight_req *infl_req; 902 struct rte_crypto_op *cop; 903 struct cnxk_cpt_qp *qp; 904 union cpt_res_s res; 905 906 infl_req = (struct cpt_inflight_req *)(get_work1); 907 cop = infl_req->cop; 908 qp = infl_req->qp; 909 910 res.u64[0] = __atomic_load_n(&infl_req->res.u64[0], __ATOMIC_RELAXED); 911 912 cn10k_cpt_dequeue_post_process(qp, infl_req->cop, infl_req, &res.cn10k); 913 914 if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF)) 915 rte_mempool_put(qp->meta_info.pool, infl_req->mdata); 916 917 rte_mempool_put(qp->ca.req_mp, infl_req); 918 return (uintptr_t)cop; 919 } 920 921 uintptr_t 922 cn10k_cpt_crypto_adapter_vector_dequeue(uintptr_t get_work1) 923 { 924 struct cpt_inflight_req *infl_req, *vec_infl_req; 925 struct rte_mempool *meta_mp, *req_mp; 926 struct rte_event_vector *vec; 927 struct rte_crypto_op *cop; 928 struct cnxk_cpt_qp *qp; 929 union cpt_res_s res; 930 int i; 931 932 vec_infl_req = (struct cpt_inflight_req *)(get_work1); 933 934 vec = vec_infl_req->vec; 935 qp = vec_infl_req->qp; 936 meta_mp = qp->meta_info.pool; 937 req_mp = qp->ca.req_mp; 938 939 #ifdef CNXK_CRYPTODEV_DEBUG 940 res.u64[0] = __atomic_load_n(&vec_infl_req->res.u64[0], __ATOMIC_RELAXED); 941 PLT_ASSERT(res.cn10k.compcode == CPT_COMP_WARN); 942 PLT_ASSERT(res.cn10k.uc_compcode == 0); 943 #endif 944 945 for (i = 0; i < vec->nb_elem; i++) { 946 infl_req = vec->ptrs[i]; 947 cop = infl_req->cop; 948 949 res.u64[0] = __atomic_load_n(&infl_req->res.u64[0], __ATOMIC_RELAXED); 950 cn10k_cpt_dequeue_post_process(qp, cop, infl_req, &res.cn10k); 951 952 vec->ptrs[i] = cop; 953 if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF)) 954 rte_mempool_put(meta_mp, infl_req->mdata); 955 956 rte_mempool_put(req_mp, infl_req); 957 } 958 959 rte_mempool_put(req_mp, vec_infl_req); 960 961 return (uintptr_t)vec; 962 } 963 964 static uint16_t 965 cn10k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops) 966 { 967 struct cpt_inflight_req *infl_req; 968 struct cnxk_cpt_qp *qp = qptr; 969 struct pending_queue *pend_q; 970 uint64_t infl_cnt, pq_tail; 971 struct rte_crypto_op *cop; 972 union cpt_res_s res; 973 int i; 974 975 pend_q = &qp->pend_q; 976 977 const uint64_t pq_mask = pend_q->pq_mask; 978 979 pq_tail = pend_q->tail; 980 infl_cnt = pending_queue_infl_cnt(pend_q->head, pq_tail, pq_mask); 981 nb_ops = RTE_MIN(nb_ops, infl_cnt); 982 983 /* Ensure infl_cnt isn't read before data lands */ 984 rte_atomic_thread_fence(__ATOMIC_ACQUIRE); 985 986 for (i = 0; i < nb_ops; i++) { 987 infl_req = &pend_q->req_queue[pq_tail]; 988 989 res.u64[0] = __atomic_load_n(&infl_req->res.u64[0], 990 __ATOMIC_RELAXED); 991 992 if (unlikely(res.cn10k.compcode == CPT_COMP_NOT_DONE)) { 993 if (unlikely(rte_get_timer_cycles() > 994 pend_q->time_out)) { 995 plt_err("Request timed out"); 996 cnxk_cpt_dump_on_err(qp); 997 pend_q->time_out = rte_get_timer_cycles() + 998 DEFAULT_COMMAND_TIMEOUT * 999 rte_get_timer_hz(); 1000 } 1001 break; 1002 } 1003 1004 pending_queue_advance(&pq_tail, pq_mask); 1005 1006 cop = infl_req->cop; 1007 1008 ops[i] = cop; 1009 1010 cn10k_cpt_dequeue_post_process(qp, cop, infl_req, &res.cn10k); 1011 1012 if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF)) 1013 rte_mempool_put(qp->meta_info.pool, infl_req->mdata); 1014 } 1015 1016 pend_q->tail = pq_tail; 1017 1018 return i; 1019 } 1020 1021 void 1022 cn10k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev) 1023 { 1024 dev->enqueue_burst = cn10k_cpt_enqueue_burst; 1025 dev->dequeue_burst = cn10k_cpt_dequeue_burst; 1026 1027 rte_mb(); 1028 } 1029 1030 static void 1031 cn10k_cpt_dev_info_get(struct rte_cryptodev *dev, 1032 struct rte_cryptodev_info *info) 1033 { 1034 if (info != NULL) { 1035 cnxk_cpt_dev_info_get(dev, info); 1036 info->driver_id = cn10k_cryptodev_driver_id; 1037 } 1038 } 1039 1040 struct rte_cryptodev_ops cn10k_cpt_ops = { 1041 /* Device control ops */ 1042 .dev_configure = cnxk_cpt_dev_config, 1043 .dev_start = cnxk_cpt_dev_start, 1044 .dev_stop = cnxk_cpt_dev_stop, 1045 .dev_close = cnxk_cpt_dev_close, 1046 .dev_infos_get = cn10k_cpt_dev_info_get, 1047 1048 .stats_get = NULL, 1049 .stats_reset = NULL, 1050 .queue_pair_setup = cnxk_cpt_queue_pair_setup, 1051 .queue_pair_release = cnxk_cpt_queue_pair_release, 1052 1053 /* Symmetric crypto ops */ 1054 .sym_session_get_size = cnxk_cpt_sym_session_get_size, 1055 .sym_session_configure = cnxk_cpt_sym_session_configure, 1056 .sym_session_clear = cnxk_cpt_sym_session_clear, 1057 1058 /* Asymmetric crypto ops */ 1059 .asym_session_get_size = cnxk_ae_session_size_get, 1060 .asym_session_configure = cnxk_ae_session_cfg, 1061 .asym_session_clear = cnxk_ae_session_clear, 1062 1063 /* Event crypto ops */ 1064 .session_ev_mdata_set = cn10k_cpt_crypto_adapter_ev_mdata_set, 1065 }; 1066