1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2022 NVIDIA Corporation & Affiliates 3 */ 4 5 #include "mlx5dr_internal.h" 6 7 struct mlx5dr_send_ring_dep_wqe * 8 mlx5dr_send_add_new_dep_wqe(struct mlx5dr_send_engine *queue) 9 { 10 struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq; 11 unsigned int idx = send_sq->head_dep_idx++ & (queue->num_entries - 1); 12 13 memset(&send_sq->dep_wqe[idx].wqe_data.tag, 0, MLX5DR_MATCH_TAG_SZ); 14 15 return &send_sq->dep_wqe[idx]; 16 } 17 18 void mlx5dr_send_abort_new_dep_wqe(struct mlx5dr_send_engine *queue) 19 { 20 queue->send_ring->send_sq.head_dep_idx--; 21 } 22 23 void mlx5dr_send_all_dep_wqe(struct mlx5dr_send_engine *queue) 24 { 25 struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq; 26 struct mlx5dr_send_ste_attr ste_attr = {0}; 27 struct mlx5dr_send_ring_dep_wqe *dep_wqe; 28 29 ste_attr.send_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE; 30 ste_attr.send_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS; 31 ste_attr.send_attr.len = MLX5DR_WQE_SZ_GTA_CTRL + MLX5DR_WQE_SZ_GTA_DATA; 32 ste_attr.gta_opcode = MLX5DR_WQE_GTA_OP_ACTIVATE; 33 34 /* Fence first from previous depend WQEs */ 35 ste_attr.send_attr.fence = 1; 36 37 while (send_sq->head_dep_idx != send_sq->tail_dep_idx) { 38 dep_wqe = &send_sq->dep_wqe[send_sq->tail_dep_idx++ & (queue->num_entries - 1)]; 39 40 /* Notify HW on the last WQE */ 41 ste_attr.send_attr.notify_hw = (send_sq->tail_dep_idx == send_sq->head_dep_idx); 42 ste_attr.send_attr.user_data = dep_wqe->user_data; 43 ste_attr.send_attr.rule = dep_wqe->rule; 44 45 ste_attr.rtc_0 = dep_wqe->rtc_0; 46 ste_attr.rtc_1 = dep_wqe->rtc_1; 47 ste_attr.retry_rtc_0 = dep_wqe->retry_rtc_0; 48 ste_attr.retry_rtc_1 = dep_wqe->retry_rtc_1; 49 ste_attr.used_id_rtc_0 = &dep_wqe->rule->rtc_0; 50 ste_attr.used_id_rtc_1 = &dep_wqe->rule->rtc_1; 51 ste_attr.wqe_ctrl = &dep_wqe->wqe_ctrl; 52 ste_attr.wqe_data = &dep_wqe->wqe_data; 53 54 mlx5dr_send_ste(queue, &ste_attr); 55 56 /* Fencing is done only on the first WQE */ 57 ste_attr.send_attr.fence = 0; 58 } 59 } 60 61 struct mlx5dr_send_engine_post_ctrl 62 mlx5dr_send_engine_post_start(struct mlx5dr_send_engine *queue) 63 { 64 struct mlx5dr_send_engine_post_ctrl ctrl; 65 66 ctrl.queue = queue; 67 /* Currently only one send ring is supported */ 68 ctrl.send_ring = &queue->send_ring[0]; 69 ctrl.num_wqebbs = 0; 70 71 return ctrl; 72 } 73 74 void mlx5dr_send_engine_post_req_wqe(struct mlx5dr_send_engine_post_ctrl *ctrl, 75 char **buf, size_t *len) 76 { 77 struct mlx5dr_send_ring_sq *send_sq = &ctrl->send_ring->send_sq; 78 unsigned int idx; 79 80 idx = (send_sq->cur_post + ctrl->num_wqebbs) & send_sq->buf_mask; 81 82 *buf = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT); 83 *len = MLX5_SEND_WQE_BB; 84 85 if (!ctrl->num_wqebbs) { 86 *buf += sizeof(struct mlx5dr_wqe_ctrl_seg); 87 *len -= sizeof(struct mlx5dr_wqe_ctrl_seg); 88 } 89 90 ctrl->num_wqebbs++; 91 } 92 93 static void mlx5dr_send_engine_post_ring(struct mlx5dr_send_ring_sq *sq, 94 struct mlx5dv_devx_uar *uar, 95 struct mlx5dr_wqe_ctrl_seg *wqe_ctrl) 96 { 97 rte_compiler_barrier(); 98 sq->db[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->cur_post); 99 100 rte_wmb(); 101 mlx5dr_uar_write64_relaxed(*((uint64_t *)wqe_ctrl), uar->reg_addr); 102 rte_wmb(); 103 } 104 105 static void 106 mlx5dr_send_wqe_set_tag(struct mlx5dr_wqe_gta_data_seg_ste *wqe_data, 107 struct mlx5dr_rule_match_tag *tag, 108 bool is_jumbo) 109 { 110 if (is_jumbo) { 111 /* Clear previous possibly dirty control */ 112 memset(wqe_data, 0, MLX5DR_STE_CTRL_SZ); 113 memcpy(wqe_data->action, tag->jumbo, MLX5DR_JUMBO_TAG_SZ); 114 } else { 115 /* Clear previous possibly dirty control and actions */ 116 memset(wqe_data, 0, MLX5DR_STE_CTRL_SZ + MLX5DR_ACTIONS_SZ); 117 memcpy(wqe_data->tag, tag->match, MLX5DR_MATCH_TAG_SZ); 118 } 119 } 120 121 void mlx5dr_send_engine_post_end(struct mlx5dr_send_engine_post_ctrl *ctrl, 122 struct mlx5dr_send_engine_post_attr *attr) 123 { 124 struct mlx5dr_wqe_ctrl_seg *wqe_ctrl; 125 struct mlx5dr_send_ring_sq *sq; 126 uint32_t flags = 0; 127 unsigned int idx; 128 129 sq = &ctrl->send_ring->send_sq; 130 idx = sq->cur_post & sq->buf_mask; 131 sq->last_idx = idx; 132 133 wqe_ctrl = (void *)(sq->buf + (idx << MLX5_SEND_WQE_SHIFT)); 134 135 wqe_ctrl->opmod_idx_opcode = 136 rte_cpu_to_be_32((attr->opmod << 24) | 137 ((sq->cur_post & 0xffff) << 8) | 138 attr->opcode); 139 wqe_ctrl->qpn_ds = 140 rte_cpu_to_be_32((attr->len + sizeof(struct mlx5dr_wqe_ctrl_seg)) / 16 | 141 sq->sqn << 8); 142 143 wqe_ctrl->imm = rte_cpu_to_be_32(attr->id); 144 145 flags |= attr->notify_hw ? MLX5_WQE_CTRL_CQ_UPDATE : 0; 146 flags |= attr->fence ? MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE : 0; 147 wqe_ctrl->flags = rte_cpu_to_be_32(flags); 148 149 sq->wr_priv[idx].id = attr->id; 150 sq->wr_priv[idx].retry_id = attr->retry_id; 151 152 sq->wr_priv[idx].rule = attr->rule; 153 sq->wr_priv[idx].user_data = attr->user_data; 154 sq->wr_priv[idx].num_wqebbs = ctrl->num_wqebbs; 155 156 if (attr->rule) { 157 sq->wr_priv[idx].rule->pending_wqes++; 158 sq->wr_priv[idx].used_id = attr->used_id; 159 } 160 161 sq->cur_post += ctrl->num_wqebbs; 162 163 if (attr->notify_hw) 164 mlx5dr_send_engine_post_ring(sq, ctrl->queue->uar, wqe_ctrl); 165 } 166 167 static void mlx5dr_send_wqe(struct mlx5dr_send_engine *queue, 168 struct mlx5dr_send_engine_post_attr *send_attr, 169 struct mlx5dr_wqe_gta_ctrl_seg *send_wqe_ctrl, 170 void *send_wqe_data, 171 void *send_wqe_tag, 172 bool is_jumbo, 173 uint8_t gta_opcode, 174 uint32_t direct_index) 175 { 176 struct mlx5dr_wqe_gta_data_seg_ste *wqe_data; 177 struct mlx5dr_wqe_gta_ctrl_seg *wqe_ctrl; 178 struct mlx5dr_send_engine_post_ctrl ctrl; 179 size_t wqe_len; 180 181 ctrl = mlx5dr_send_engine_post_start(queue); 182 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_ctrl, &wqe_len); 183 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_data, &wqe_len); 184 185 wqe_ctrl->op_dirix = htobe32(gta_opcode << 28 | direct_index); 186 memcpy(wqe_ctrl->stc_ix, send_wqe_ctrl->stc_ix, sizeof(send_wqe_ctrl->stc_ix)); 187 188 if (send_wqe_data) 189 memcpy(wqe_data, send_wqe_data, sizeof(*wqe_data)); 190 else 191 mlx5dr_send_wqe_set_tag(wqe_data, send_wqe_tag, is_jumbo); 192 193 mlx5dr_send_engine_post_end(&ctrl, send_attr); 194 } 195 196 void mlx5dr_send_ste(struct mlx5dr_send_engine *queue, 197 struct mlx5dr_send_ste_attr *ste_attr) 198 { 199 struct mlx5dr_send_engine_post_attr *send_attr = &ste_attr->send_attr; 200 uint8_t notify_hw = send_attr->notify_hw; 201 uint8_t fence = send_attr->fence; 202 203 if (ste_attr->rtc_1) { 204 send_attr->id = ste_attr->rtc_1; 205 send_attr->used_id = ste_attr->used_id_rtc_1; 206 send_attr->retry_id = ste_attr->retry_rtc_1; 207 send_attr->fence = fence; 208 send_attr->notify_hw = notify_hw && !ste_attr->rtc_0; 209 mlx5dr_send_wqe(queue, send_attr, 210 ste_attr->wqe_ctrl, 211 ste_attr->wqe_data, 212 ste_attr->wqe_tag, 213 ste_attr->wqe_tag_is_jumbo, 214 ste_attr->gta_opcode, 215 ste_attr->direct_index); 216 } 217 218 if (ste_attr->rtc_0) { 219 send_attr->id = ste_attr->rtc_0; 220 send_attr->used_id = ste_attr->used_id_rtc_0; 221 send_attr->retry_id = ste_attr->retry_rtc_0; 222 send_attr->fence = fence && !ste_attr->rtc_1; 223 send_attr->notify_hw = notify_hw; 224 mlx5dr_send_wqe(queue, send_attr, 225 ste_attr->wqe_ctrl, 226 ste_attr->wqe_data, 227 ste_attr->wqe_tag, 228 ste_attr->wqe_tag_is_jumbo, 229 ste_attr->gta_opcode, 230 ste_attr->direct_index); 231 } 232 233 /* Restore to ortginal requested values */ 234 send_attr->notify_hw = notify_hw; 235 send_attr->fence = fence; 236 } 237 238 static void mlx5dr_send_engine_retry_post_send(struct mlx5dr_send_engine *queue, 239 struct mlx5dr_send_ring_priv *priv, 240 uint16_t wqe_cnt) 241 { 242 struct mlx5dr_send_engine_post_attr send_attr = {0}; 243 struct mlx5dr_wqe_gta_data_seg_ste *wqe_data; 244 struct mlx5dr_wqe_gta_ctrl_seg *wqe_ctrl; 245 struct mlx5dr_send_engine_post_ctrl ctrl; 246 struct mlx5dr_send_ring_sq *send_sq; 247 unsigned int idx; 248 size_t wqe_len; 249 char *p; 250 251 send_attr.rule = priv->rule; 252 send_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS; 253 send_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE; 254 send_attr.len = MLX5_SEND_WQE_BB * 2 - sizeof(struct mlx5dr_wqe_ctrl_seg); 255 send_attr.notify_hw = 1; 256 send_attr.fence = 0; 257 send_attr.user_data = priv->user_data; 258 send_attr.id = priv->retry_id; 259 send_attr.used_id = priv->used_id; 260 261 ctrl = mlx5dr_send_engine_post_start(queue); 262 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_ctrl, &wqe_len); 263 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_data, &wqe_len); 264 265 send_sq = &ctrl.send_ring->send_sq; 266 idx = wqe_cnt & send_sq->buf_mask; 267 p = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT); 268 269 /* Copy old gta ctrl */ 270 memcpy(wqe_ctrl, p + sizeof(struct mlx5dr_wqe_ctrl_seg), 271 MLX5_SEND_WQE_BB - sizeof(struct mlx5dr_wqe_ctrl_seg)); 272 273 idx = (wqe_cnt + 1) & send_sq->buf_mask; 274 p = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT); 275 276 /* Copy old gta data */ 277 memcpy(wqe_data, p, MLX5_SEND_WQE_BB); 278 279 mlx5dr_send_engine_post_end(&ctrl, &send_attr); 280 } 281 282 void mlx5dr_send_engine_flush_queue(struct mlx5dr_send_engine *queue) 283 { 284 struct mlx5dr_send_ring_sq *sq = &queue->send_ring[0].send_sq; 285 struct mlx5dr_wqe_ctrl_seg *wqe_ctrl; 286 287 wqe_ctrl = (void *)(sq->buf + (sq->last_idx << MLX5_SEND_WQE_SHIFT)); 288 289 wqe_ctrl->flags |= rte_cpu_to_be_32(MLX5_WQE_CTRL_CQ_UPDATE); 290 291 mlx5dr_send_engine_post_ring(sq, queue->uar, wqe_ctrl); 292 } 293 294 static void mlx5dr_send_engine_update_rule(struct mlx5dr_send_engine *queue, 295 struct mlx5dr_send_ring_priv *priv, 296 uint16_t wqe_cnt, 297 enum rte_flow_op_status *status) 298 { 299 priv->rule->pending_wqes--; 300 301 if (*status == RTE_FLOW_OP_ERROR) { 302 if (priv->retry_id) { 303 mlx5dr_send_engine_retry_post_send(queue, priv, wqe_cnt); 304 return; 305 } 306 /* Some part of the rule failed */ 307 priv->rule->status = MLX5DR_RULE_STATUS_FAILING; 308 *priv->used_id = 0; 309 } else { 310 *priv->used_id = priv->id; 311 } 312 313 /* Update rule status for the last completion */ 314 if (!priv->rule->pending_wqes) { 315 if (unlikely(priv->rule->status == MLX5DR_RULE_STATUS_FAILING)) { 316 /* Rule completely failed and doesn't require cleanup */ 317 if (!priv->rule->rtc_0 && !priv->rule->rtc_1) 318 priv->rule->status = MLX5DR_RULE_STATUS_FAILED; 319 320 *status = RTE_FLOW_OP_ERROR; 321 } else { 322 /* Increase the status, this only works on good flow as the enum 323 * is arrange it away creating -> created -> deleting -> deleted 324 */ 325 priv->rule->status++; 326 *status = RTE_FLOW_OP_SUCCESS; 327 /* Rule was deleted now we can safely release action STEs */ 328 if (priv->rule->status == MLX5DR_RULE_STATUS_DELETED) 329 mlx5dr_rule_free_action_ste_idx(priv->rule); 330 } 331 } 332 } 333 334 static void mlx5dr_send_engine_update(struct mlx5dr_send_engine *queue, 335 struct mlx5_cqe64 *cqe, 336 struct mlx5dr_send_ring_priv *priv, 337 struct rte_flow_op_result res[], 338 int64_t *i, 339 uint32_t res_nb, 340 uint16_t wqe_cnt) 341 { 342 enum rte_flow_op_status status; 343 344 if (!cqe || (likely(rte_be_to_cpu_32(cqe->byte_cnt) >> 31 == 0) && 345 likely(mlx5dv_get_cqe_opcode(cqe) == MLX5_CQE_REQ))) { 346 status = RTE_FLOW_OP_SUCCESS; 347 } else { 348 status = RTE_FLOW_OP_ERROR; 349 } 350 351 if (priv->user_data) { 352 if (priv->rule) { 353 mlx5dr_send_engine_update_rule(queue, priv, wqe_cnt, &status); 354 /* Completion is provided on the last rule WQE */ 355 if (priv->rule->pending_wqes) 356 return; 357 } 358 359 if (*i < res_nb) { 360 res[*i].user_data = priv->user_data; 361 res[*i].status = status; 362 (*i)++; 363 mlx5dr_send_engine_dec_rule(queue); 364 } else { 365 mlx5dr_send_engine_gen_comp(queue, priv->user_data, status); 366 } 367 } 368 } 369 370 static void mlx5dr_send_engine_poll_cq(struct mlx5dr_send_engine *queue, 371 struct mlx5dr_send_ring *send_ring, 372 struct rte_flow_op_result res[], 373 int64_t *i, 374 uint32_t res_nb) 375 { 376 struct mlx5dr_send_ring_cq *cq = &send_ring->send_cq; 377 struct mlx5dr_send_ring_sq *sq = &send_ring->send_sq; 378 uint32_t cq_idx = cq->cons_index & cq->ncqe_mask; 379 struct mlx5dr_send_ring_priv *priv; 380 struct mlx5_cqe64 *cqe; 381 uint32_t offset_cqe64; 382 uint8_t cqe_opcode; 383 uint8_t cqe_owner; 384 uint16_t wqe_cnt; 385 uint8_t sw_own; 386 387 offset_cqe64 = RTE_CACHE_LINE_SIZE - sizeof(struct mlx5_cqe64); 388 cqe = (void *)(cq->buf + (cq_idx << cq->cqe_log_sz) + offset_cqe64); 389 390 sw_own = (cq->cons_index & cq->ncqe) ? 1 : 0; 391 cqe_opcode = mlx5dv_get_cqe_opcode(cqe); 392 cqe_owner = mlx5dv_get_cqe_owner(cqe); 393 394 if (cqe_opcode == MLX5_CQE_INVALID || 395 cqe_owner != sw_own) 396 return; 397 398 if (unlikely(mlx5dv_get_cqe_opcode(cqe) != MLX5_CQE_REQ)) 399 queue->err = true; 400 401 rte_io_rmb(); 402 403 wqe_cnt = be16toh(cqe->wqe_counter) & sq->buf_mask; 404 405 while (cq->poll_wqe != wqe_cnt) { 406 priv = &sq->wr_priv[cq->poll_wqe]; 407 mlx5dr_send_engine_update(queue, NULL, priv, res, i, res_nb, 0); 408 cq->poll_wqe = (cq->poll_wqe + priv->num_wqebbs) & sq->buf_mask; 409 } 410 411 priv = &sq->wr_priv[wqe_cnt]; 412 cq->poll_wqe = (wqe_cnt + priv->num_wqebbs) & sq->buf_mask; 413 mlx5dr_send_engine_update(queue, cqe, priv, res, i, res_nb, wqe_cnt); 414 cq->cons_index++; 415 } 416 417 static void mlx5dr_send_engine_poll_cqs(struct mlx5dr_send_engine *queue, 418 struct rte_flow_op_result res[], 419 int64_t *polled, 420 uint32_t res_nb) 421 { 422 int j; 423 424 for (j = 0; j < MLX5DR_NUM_SEND_RINGS; j++) { 425 mlx5dr_send_engine_poll_cq(queue, &queue->send_ring[j], 426 res, polled, res_nb); 427 428 *queue->send_ring[j].send_cq.db = 429 htobe32(queue->send_ring[j].send_cq.cons_index & 0xffffff); 430 } 431 } 432 433 static void mlx5dr_send_engine_poll_list(struct mlx5dr_send_engine *queue, 434 struct rte_flow_op_result res[], 435 int64_t *polled, 436 uint32_t res_nb) 437 { 438 struct mlx5dr_completed_poll *comp = &queue->completed; 439 440 while (comp->ci != comp->pi) { 441 if (*polled < res_nb) { 442 res[*polled].status = 443 comp->entries[comp->ci].status; 444 res[*polled].user_data = 445 comp->entries[comp->ci].user_data; 446 (*polled)++; 447 comp->ci = (comp->ci + 1) & comp->mask; 448 mlx5dr_send_engine_dec_rule(queue); 449 } else { 450 return; 451 } 452 } 453 } 454 455 static int mlx5dr_send_engine_poll(struct mlx5dr_send_engine *queue, 456 struct rte_flow_op_result res[], 457 uint32_t res_nb) 458 { 459 int64_t polled = 0; 460 461 mlx5dr_send_engine_poll_list(queue, res, &polled, res_nb); 462 463 if (polled >= res_nb) 464 return polled; 465 466 mlx5dr_send_engine_poll_cqs(queue, res, &polled, res_nb); 467 468 return polled; 469 } 470 471 int mlx5dr_send_queue_poll(struct mlx5dr_context *ctx, 472 uint16_t queue_id, 473 struct rte_flow_op_result res[], 474 uint32_t res_nb) 475 { 476 return mlx5dr_send_engine_poll(&ctx->send_queue[queue_id], 477 res, res_nb); 478 } 479 480 static int mlx5dr_send_ring_create_sq_obj(struct mlx5dr_context *ctx, 481 struct mlx5dr_send_engine *queue, 482 struct mlx5dr_send_ring_sq *sq, 483 struct mlx5dr_send_ring_cq *cq, 484 size_t log_wq_sz) 485 { 486 struct mlx5dr_cmd_sq_create_attr attr = {0}; 487 int err; 488 489 attr.cqn = cq->cqn; 490 attr.pdn = ctx->pd_num; 491 attr.page_id = queue->uar->page_id; 492 attr.dbr_id = sq->db_umem->umem_id; 493 attr.wq_id = sq->buf_umem->umem_id; 494 attr.log_wq_sz = log_wq_sz; 495 if (ctx->caps->sq_ts_format == MLX5_HCA_CAP_TIMESTAMP_FORMAT_FR) 496 attr.ts_format = MLX5_QPC_TIMESTAMP_FORMAT_FREE_RUNNING; 497 else 498 attr.ts_format = MLX5_QPC_TIMESTAMP_FORMAT_DEFAULT; 499 500 sq->obj = mlx5dr_cmd_sq_create(ctx->ibv_ctx, &attr); 501 if (!sq->obj) 502 return rte_errno; 503 504 sq->sqn = sq->obj->id; 505 506 err = mlx5dr_cmd_sq_modify_rdy(sq->obj); 507 if (err) 508 goto free_sq; 509 510 return 0; 511 512 free_sq: 513 mlx5dr_cmd_destroy_obj(sq->obj); 514 515 return err; 516 } 517 518 static inline unsigned long align(unsigned long val, unsigned long align) 519 { 520 return (val + align - 1) & ~(align - 1); 521 } 522 523 static int mlx5dr_send_ring_open_sq(struct mlx5dr_context *ctx, 524 struct mlx5dr_send_engine *queue, 525 struct mlx5dr_send_ring_sq *sq, 526 struct mlx5dr_send_ring_cq *cq) 527 { 528 size_t sq_log_buf_sz; 529 size_t buf_aligned; 530 size_t sq_buf_sz; 531 size_t page_size; 532 size_t buf_sz; 533 int err; 534 535 buf_sz = queue->num_entries * MAX_WQES_PER_RULE; 536 sq_log_buf_sz = log2above(buf_sz); 537 sq_buf_sz = 1 << (sq_log_buf_sz + log2above(MLX5_SEND_WQE_BB)); 538 sq->reg_addr = queue->uar->reg_addr; 539 540 page_size = sysconf(_SC_PAGESIZE); 541 buf_aligned = align(sq_buf_sz, page_size); 542 err = posix_memalign((void **)&sq->buf, page_size, buf_aligned); 543 if (err) { 544 rte_errno = ENOMEM; 545 return err; 546 } 547 memset(sq->buf, 0, buf_aligned); 548 549 err = posix_memalign((void **)&sq->db, 8, 8); 550 if (err) 551 goto free_buf; 552 553 sq->buf_umem = mlx5_glue->devx_umem_reg(ctx->ibv_ctx, sq->buf, sq_buf_sz, 0); 554 555 if (!sq->buf_umem) { 556 err = errno; 557 goto free_db; 558 } 559 560 sq->db_umem = mlx5_glue->devx_umem_reg(ctx->ibv_ctx, sq->db, 8, 0); 561 if (!sq->db_umem) { 562 err = errno; 563 goto free_buf_umem; 564 } 565 566 err = mlx5dr_send_ring_create_sq_obj(ctx, queue, sq, cq, sq_log_buf_sz); 567 568 if (err) 569 goto free_db_umem; 570 571 sq->wr_priv = simple_malloc(sizeof(*sq->wr_priv) * buf_sz); 572 if (!sq->wr_priv) { 573 err = ENOMEM; 574 goto destroy_sq_obj; 575 } 576 577 sq->dep_wqe = simple_calloc(queue->num_entries, sizeof(*sq->dep_wqe)); 578 if (!sq->dep_wqe) { 579 err = ENOMEM; 580 goto destroy_wr_priv; 581 } 582 583 sq->buf_mask = buf_sz - 1; 584 585 return 0; 586 587 destroy_wr_priv: 588 simple_free(sq->wr_priv); 589 destroy_sq_obj: 590 mlx5dr_cmd_destroy_obj(sq->obj); 591 free_db_umem: 592 mlx5_glue->devx_umem_dereg(sq->db_umem); 593 free_buf_umem: 594 mlx5_glue->devx_umem_dereg(sq->buf_umem); 595 free_db: 596 free(sq->db); 597 free_buf: 598 free(sq->buf); 599 rte_errno = err; 600 return err; 601 } 602 603 static void mlx5dr_send_ring_close_sq(struct mlx5dr_send_ring_sq *sq) 604 { 605 simple_free(sq->dep_wqe); 606 mlx5dr_cmd_destroy_obj(sq->obj); 607 mlx5_glue->devx_umem_dereg(sq->db_umem); 608 mlx5_glue->devx_umem_dereg(sq->buf_umem); 609 simple_free(sq->wr_priv); 610 free(sq->db); 611 free(sq->buf); 612 } 613 614 static int mlx5dr_send_ring_open_cq(struct mlx5dr_context *ctx, 615 struct mlx5dr_send_engine *queue, 616 struct mlx5dr_send_ring_cq *cq) 617 { 618 struct mlx5dv_cq mlx5_cq = {0}; 619 struct mlx5dv_obj obj; 620 struct ibv_cq *ibv_cq; 621 size_t cq_size; 622 int err; 623 624 cq_size = queue->num_entries; 625 ibv_cq = mlx5_glue->create_cq(ctx->ibv_ctx, cq_size, NULL, NULL, 0); 626 if (!ibv_cq) { 627 DR_LOG(ERR, "Failed to create CQ"); 628 rte_errno = errno; 629 return rte_errno; 630 } 631 632 obj.cq.in = ibv_cq; 633 obj.cq.out = &mlx5_cq; 634 err = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ); 635 if (err) { 636 err = errno; 637 goto close_cq; 638 } 639 640 cq->buf = mlx5_cq.buf; 641 cq->db = mlx5_cq.dbrec; 642 cq->ncqe = mlx5_cq.cqe_cnt; 643 cq->cqe_sz = mlx5_cq.cqe_size; 644 cq->cqe_log_sz = log2above(cq->cqe_sz); 645 cq->ncqe_mask = cq->ncqe - 1; 646 cq->buf_sz = cq->cqe_sz * cq->ncqe; 647 cq->cqn = mlx5_cq.cqn; 648 cq->ibv_cq = ibv_cq; 649 650 return 0; 651 652 close_cq: 653 mlx5_glue->destroy_cq(ibv_cq); 654 rte_errno = err; 655 return err; 656 } 657 658 static void mlx5dr_send_ring_close_cq(struct mlx5dr_send_ring_cq *cq) 659 { 660 mlx5_glue->destroy_cq(cq->ibv_cq); 661 } 662 663 static void mlx5dr_send_ring_close(struct mlx5dr_send_ring *ring) 664 { 665 mlx5dr_send_ring_close_sq(&ring->send_sq); 666 mlx5dr_send_ring_close_cq(&ring->send_cq); 667 } 668 669 static int mlx5dr_send_ring_open(struct mlx5dr_context *ctx, 670 struct mlx5dr_send_engine *queue, 671 struct mlx5dr_send_ring *ring) 672 { 673 int err; 674 675 err = mlx5dr_send_ring_open_cq(ctx, queue, &ring->send_cq); 676 if (err) 677 return err; 678 679 err = mlx5dr_send_ring_open_sq(ctx, queue, &ring->send_sq, &ring->send_cq); 680 if (err) 681 goto close_cq; 682 683 return err; 684 685 close_cq: 686 mlx5dr_send_ring_close_cq(&ring->send_cq); 687 688 return err; 689 } 690 691 static void __mlx5dr_send_rings_close(struct mlx5dr_send_engine *queue, 692 uint16_t i) 693 { 694 while (i--) 695 mlx5dr_send_ring_close(&queue->send_ring[i]); 696 } 697 698 static void mlx5dr_send_rings_close(struct mlx5dr_send_engine *queue) 699 { 700 __mlx5dr_send_rings_close(queue, queue->rings); 701 } 702 703 static int mlx5dr_send_rings_open(struct mlx5dr_context *ctx, 704 struct mlx5dr_send_engine *queue) 705 { 706 uint16_t i; 707 int err; 708 709 for (i = 0; i < queue->rings; i++) { 710 err = mlx5dr_send_ring_open(ctx, queue, &queue->send_ring[i]); 711 if (err) 712 goto free_rings; 713 } 714 715 return 0; 716 717 free_rings: 718 __mlx5dr_send_rings_close(queue, i); 719 720 return err; 721 } 722 723 void mlx5dr_send_queue_close(struct mlx5dr_send_engine *queue) 724 { 725 mlx5dr_send_rings_close(queue); 726 simple_free(queue->completed.entries); 727 mlx5_glue->devx_free_uar(queue->uar); 728 } 729 730 int mlx5dr_send_queue_open(struct mlx5dr_context *ctx, 731 struct mlx5dr_send_engine *queue, 732 uint16_t queue_size) 733 { 734 struct mlx5dv_devx_uar *uar; 735 int err; 736 737 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 738 uar = mlx5_glue->devx_alloc_uar(ctx->ibv_ctx, MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC); 739 if (!uar) { 740 rte_errno = errno; 741 return rte_errno; 742 } 743 #else 744 uar = NULL; 745 rte_errno = ENOTSUP; 746 return rte_errno; 747 #endif 748 749 queue->uar = uar; 750 queue->rings = MLX5DR_NUM_SEND_RINGS; 751 queue->num_entries = roundup_pow_of_two(queue_size); 752 queue->used_entries = 0; 753 queue->th_entries = queue->num_entries; 754 755 queue->completed.entries = simple_calloc(queue->num_entries, 756 sizeof(queue->completed.entries[0])); 757 if (!queue->completed.entries) { 758 rte_errno = ENOMEM; 759 goto free_uar; 760 } 761 queue->completed.pi = 0; 762 queue->completed.ci = 0; 763 queue->completed.mask = queue->num_entries - 1; 764 765 err = mlx5dr_send_rings_open(ctx, queue); 766 if (err) 767 goto free_completed_entries; 768 769 return 0; 770 771 free_completed_entries: 772 simple_free(queue->completed.entries); 773 free_uar: 774 mlx5_glue->devx_free_uar(uar); 775 return rte_errno; 776 } 777 778 static void __mlx5dr_send_queues_close(struct mlx5dr_context *ctx, uint16_t queues) 779 { 780 struct mlx5dr_send_engine *queue; 781 782 while (queues--) { 783 queue = &ctx->send_queue[queues]; 784 785 mlx5dr_send_queue_close(queue); 786 } 787 } 788 789 void mlx5dr_send_queues_close(struct mlx5dr_context *ctx) 790 { 791 __mlx5dr_send_queues_close(ctx, ctx->queues); 792 simple_free(ctx->send_queue); 793 } 794 795 int mlx5dr_send_queues_open(struct mlx5dr_context *ctx, 796 uint16_t queues, 797 uint16_t queue_size) 798 { 799 int err = 0; 800 uint32_t i; 801 802 /* Open one extra queue for control path */ 803 ctx->queues = queues + 1; 804 805 ctx->send_queue = simple_calloc(ctx->queues, sizeof(*ctx->send_queue)); 806 if (!ctx->send_queue) { 807 rte_errno = ENOMEM; 808 return rte_errno; 809 } 810 811 for (i = 0; i < ctx->queues; i++) { 812 err = mlx5dr_send_queue_open(ctx, &ctx->send_queue[i], queue_size); 813 if (err) 814 goto close_send_queues; 815 } 816 817 return 0; 818 819 close_send_queues: 820 __mlx5dr_send_queues_close(ctx, i); 821 822 simple_free(ctx->send_queue); 823 824 return err; 825 } 826 827 int mlx5dr_send_queue_action(struct mlx5dr_context *ctx, 828 uint16_t queue_id, 829 uint32_t actions) 830 { 831 struct mlx5dr_send_ring_sq *send_sq; 832 struct mlx5dr_send_engine *queue; 833 834 queue = &ctx->send_queue[queue_id]; 835 send_sq = &queue->send_ring->send_sq; 836 837 if (actions == MLX5DR_SEND_QUEUE_ACTION_DRAIN) { 838 if (send_sq->head_dep_idx != send_sq->tail_dep_idx) 839 /* Send dependent WQEs to drain the queue */ 840 mlx5dr_send_all_dep_wqe(queue); 841 else 842 /* Signal on the last posted WQE */ 843 mlx5dr_send_engine_flush_queue(queue); 844 } else { 845 rte_errno = -EINVAL; 846 return rte_errno; 847 } 848 849 return 0; 850 } 851