1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2022 NVIDIA Corporation & Affiliates 3 */ 4 5 #include "mlx5dr_internal.h" 6 7 struct mlx5dr_send_ring_dep_wqe * 8 mlx5dr_send_add_new_dep_wqe(struct mlx5dr_send_engine *queue) 9 { 10 struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq; 11 unsigned int idx = send_sq->head_dep_idx++ & (queue->num_entries - 1); 12 13 memset(&send_sq->dep_wqe[idx].wqe_data.tag, 0, MLX5DR_MATCH_TAG_SZ); 14 15 return &send_sq->dep_wqe[idx]; 16 } 17 18 void mlx5dr_send_abort_new_dep_wqe(struct mlx5dr_send_engine *queue) 19 { 20 queue->send_ring->send_sq.head_dep_idx--; 21 } 22 23 void mlx5dr_send_all_dep_wqe(struct mlx5dr_send_engine *queue) 24 { 25 struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq; 26 struct mlx5dr_send_ste_attr ste_attr = {0}; 27 struct mlx5dr_send_ring_dep_wqe *dep_wqe; 28 29 ste_attr.send_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE; 30 ste_attr.send_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS; 31 ste_attr.send_attr.len = MLX5DR_WQE_SZ_GTA_CTRL + MLX5DR_WQE_SZ_GTA_DATA; 32 ste_attr.gta_opcode = MLX5DR_WQE_GTA_OP_ACTIVATE; 33 34 /* Fence first from previous depend WQEs */ 35 ste_attr.send_attr.fence = 1; 36 37 while (send_sq->head_dep_idx != send_sq->tail_dep_idx) { 38 dep_wqe = &send_sq->dep_wqe[send_sq->tail_dep_idx++ & (queue->num_entries - 1)]; 39 40 /* Notify HW on the last WQE */ 41 ste_attr.send_attr.notify_hw = (send_sq->tail_dep_idx == send_sq->head_dep_idx); 42 ste_attr.send_attr.user_data = dep_wqe->user_data; 43 ste_attr.send_attr.rule = dep_wqe->rule; 44 45 ste_attr.rtc_0 = dep_wqe->rtc_0; 46 ste_attr.rtc_1 = dep_wqe->rtc_1; 47 ste_attr.retry_rtc_0 = dep_wqe->retry_rtc_0; 48 ste_attr.retry_rtc_1 = dep_wqe->retry_rtc_1; 49 ste_attr.used_id_rtc_0 = &dep_wqe->rule->rtc_0; 50 ste_attr.used_id_rtc_1 = &dep_wqe->rule->rtc_1; 51 ste_attr.wqe_ctrl = &dep_wqe->wqe_ctrl; 52 ste_attr.wqe_data = &dep_wqe->wqe_data; 53 54 mlx5dr_send_ste(queue, &ste_attr); 55 56 /* Fencing is done only on the first WQE */ 57 ste_attr.send_attr.fence = 0; 58 } 59 } 60 61 struct mlx5dr_send_engine_post_ctrl 62 mlx5dr_send_engine_post_start(struct mlx5dr_send_engine *queue) 63 { 64 struct mlx5dr_send_engine_post_ctrl ctrl; 65 66 ctrl.queue = queue; 67 /* Currently only one send ring is supported */ 68 ctrl.send_ring = &queue->send_ring[0]; 69 ctrl.num_wqebbs = 0; 70 71 return ctrl; 72 } 73 74 void mlx5dr_send_engine_post_req_wqe(struct mlx5dr_send_engine_post_ctrl *ctrl, 75 char **buf, size_t *len) 76 { 77 struct mlx5dr_send_ring_sq *send_sq = &ctrl->send_ring->send_sq; 78 unsigned int idx; 79 80 idx = (send_sq->cur_post + ctrl->num_wqebbs) & send_sq->buf_mask; 81 82 *buf = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT); 83 *len = MLX5_SEND_WQE_BB; 84 85 if (!ctrl->num_wqebbs) { 86 *buf += sizeof(struct mlx5dr_wqe_ctrl_seg); 87 *len -= sizeof(struct mlx5dr_wqe_ctrl_seg); 88 } 89 90 ctrl->num_wqebbs++; 91 } 92 93 static void mlx5dr_send_engine_post_ring(struct mlx5dr_send_ring_sq *sq, 94 struct mlx5dv_devx_uar *uar, 95 struct mlx5dr_wqe_ctrl_seg *wqe_ctrl) 96 { 97 rte_compiler_barrier(); 98 sq->db[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->cur_post); 99 100 rte_wmb(); 101 mlx5dr_uar_write64_relaxed(*((uint64_t *)wqe_ctrl), uar->reg_addr); 102 rte_wmb(); 103 } 104 105 static void 106 mlx5dr_send_wqe_set_tag(struct mlx5dr_wqe_gta_data_seg_ste *wqe_data, 107 struct mlx5dr_rule_match_tag *tag, 108 bool is_jumbo) 109 { 110 if (is_jumbo) { 111 /* Clear previous possibly dirty control */ 112 memset(wqe_data, 0, MLX5DR_STE_CTRL_SZ); 113 memcpy(wqe_data->action, tag->jumbo, MLX5DR_JUMBO_TAG_SZ); 114 } else { 115 /* Clear previous possibly dirty control and actions */ 116 memset(wqe_data, 0, MLX5DR_STE_CTRL_SZ + MLX5DR_ACTIONS_SZ); 117 memcpy(wqe_data->tag, tag->match, MLX5DR_MATCH_TAG_SZ); 118 } 119 } 120 121 void mlx5dr_send_engine_post_end(struct mlx5dr_send_engine_post_ctrl *ctrl, 122 struct mlx5dr_send_engine_post_attr *attr) 123 { 124 struct mlx5dr_wqe_ctrl_seg *wqe_ctrl; 125 struct mlx5dr_send_ring_sq *sq; 126 uint32_t flags = 0; 127 unsigned int idx; 128 129 sq = &ctrl->send_ring->send_sq; 130 idx = sq->cur_post & sq->buf_mask; 131 sq->last_idx = idx; 132 133 wqe_ctrl = (void *)(sq->buf + (idx << MLX5_SEND_WQE_SHIFT)); 134 135 wqe_ctrl->opmod_idx_opcode = 136 rte_cpu_to_be_32((attr->opmod << 24) | 137 ((sq->cur_post & 0xffff) << 8) | 138 attr->opcode); 139 wqe_ctrl->qpn_ds = 140 rte_cpu_to_be_32((attr->len + sizeof(struct mlx5dr_wqe_ctrl_seg)) / 16 | 141 sq->sqn << 8); 142 143 wqe_ctrl->imm = rte_cpu_to_be_32(attr->id); 144 145 flags |= attr->notify_hw ? MLX5_WQE_CTRL_CQ_UPDATE : 0; 146 flags |= attr->fence ? MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE : 0; 147 wqe_ctrl->flags = rte_cpu_to_be_32(flags); 148 149 sq->wr_priv[idx].id = attr->id; 150 sq->wr_priv[idx].retry_id = attr->retry_id; 151 152 sq->wr_priv[idx].rule = attr->rule; 153 sq->wr_priv[idx].user_data = attr->user_data; 154 sq->wr_priv[idx].num_wqebbs = ctrl->num_wqebbs; 155 156 if (attr->rule) { 157 sq->wr_priv[idx].rule->pending_wqes++; 158 sq->wr_priv[idx].used_id = attr->used_id; 159 } 160 161 sq->cur_post += ctrl->num_wqebbs; 162 163 if (attr->notify_hw) 164 mlx5dr_send_engine_post_ring(sq, ctrl->queue->uar, wqe_ctrl); 165 } 166 167 static void mlx5dr_send_wqe(struct mlx5dr_send_engine *queue, 168 struct mlx5dr_send_engine_post_attr *send_attr, 169 struct mlx5dr_wqe_gta_ctrl_seg *send_wqe_ctrl, 170 void *send_wqe_data, 171 void *send_wqe_tag, 172 bool is_jumbo, 173 uint8_t gta_opcode, 174 uint32_t direct_index) 175 { 176 struct mlx5dr_wqe_gta_data_seg_ste *wqe_data; 177 struct mlx5dr_wqe_gta_ctrl_seg *wqe_ctrl; 178 struct mlx5dr_send_engine_post_ctrl ctrl; 179 size_t wqe_len; 180 181 ctrl = mlx5dr_send_engine_post_start(queue); 182 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_ctrl, &wqe_len); 183 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_data, &wqe_len); 184 185 wqe_ctrl->op_dirix = htobe32(gta_opcode << 28 | direct_index); 186 memcpy(wqe_ctrl->stc_ix, send_wqe_ctrl->stc_ix, sizeof(send_wqe_ctrl->stc_ix)); 187 188 if (send_wqe_data) 189 memcpy(wqe_data, send_wqe_data, sizeof(*wqe_data)); 190 else 191 mlx5dr_send_wqe_set_tag(wqe_data, send_wqe_tag, is_jumbo); 192 193 mlx5dr_send_engine_post_end(&ctrl, send_attr); 194 } 195 196 void mlx5dr_send_ste(struct mlx5dr_send_engine *queue, 197 struct mlx5dr_send_ste_attr *ste_attr) 198 { 199 struct mlx5dr_send_engine_post_attr *send_attr = &ste_attr->send_attr; 200 uint8_t notify_hw = send_attr->notify_hw; 201 uint8_t fence = send_attr->fence; 202 203 if (ste_attr->rtc_1) { 204 send_attr->id = ste_attr->rtc_1; 205 send_attr->used_id = ste_attr->used_id_rtc_1; 206 send_attr->retry_id = ste_attr->retry_rtc_1; 207 send_attr->fence = fence; 208 send_attr->notify_hw = notify_hw && !ste_attr->rtc_0; 209 mlx5dr_send_wqe(queue, send_attr, 210 ste_attr->wqe_ctrl, 211 ste_attr->wqe_data, 212 ste_attr->wqe_tag, 213 ste_attr->wqe_tag_is_jumbo, 214 ste_attr->gta_opcode, 215 ste_attr->direct_index); 216 } 217 218 if (ste_attr->rtc_0) { 219 send_attr->id = ste_attr->rtc_0; 220 send_attr->used_id = ste_attr->used_id_rtc_0; 221 send_attr->retry_id = ste_attr->retry_rtc_0; 222 send_attr->fence = fence && !ste_attr->rtc_1; 223 send_attr->notify_hw = notify_hw; 224 mlx5dr_send_wqe(queue, send_attr, 225 ste_attr->wqe_ctrl, 226 ste_attr->wqe_data, 227 ste_attr->wqe_tag, 228 ste_attr->wqe_tag_is_jumbo, 229 ste_attr->gta_opcode, 230 ste_attr->direct_index); 231 } 232 233 /* Restore to ortginal requested values */ 234 send_attr->notify_hw = notify_hw; 235 send_attr->fence = fence; 236 } 237 238 static 239 int mlx5dr_send_wqe_fw(struct ibv_context *ibv_ctx, 240 uint32_t pd_num, 241 struct mlx5dr_send_engine_post_attr *send_attr, 242 struct mlx5dr_wqe_gta_ctrl_seg *send_wqe_ctrl, 243 void *send_wqe_match_data, 244 void *send_wqe_match_tag, 245 void *send_wqe_range_data, 246 void *send_wqe_range_tag, 247 bool is_jumbo, 248 uint8_t gta_opcode) 249 { 250 bool has_range = send_wqe_range_data || send_wqe_range_tag; 251 bool has_match = send_wqe_match_data || send_wqe_match_tag; 252 struct mlx5dr_wqe_gta_data_seg_ste gta_wqe_data0 = {0}; 253 struct mlx5dr_wqe_gta_data_seg_ste gta_wqe_data1 = {0}; 254 struct mlx5dr_wqe_gta_ctrl_seg gta_wqe_ctrl = {0}; 255 struct mlx5dr_cmd_generate_wqe_attr attr = {0}; 256 struct mlx5dr_wqe_ctrl_seg wqe_ctrl = {0}; 257 struct mlx5_cqe64 cqe; 258 uint32_t flags = 0; 259 int ret; 260 261 /* Set WQE control */ 262 wqe_ctrl.opmod_idx_opcode = 263 rte_cpu_to_be_32((send_attr->opmod << 24) | send_attr->opcode); 264 wqe_ctrl.qpn_ds = 265 rte_cpu_to_be_32((send_attr->len + sizeof(struct mlx5dr_wqe_ctrl_seg)) / 16); 266 flags |= send_attr->notify_hw ? MLX5_WQE_CTRL_CQ_UPDATE : 0; 267 wqe_ctrl.flags = rte_cpu_to_be_32(flags); 268 wqe_ctrl.imm = rte_cpu_to_be_32(send_attr->id); 269 270 /* Set GTA WQE CTRL */ 271 memcpy(gta_wqe_ctrl.stc_ix, send_wqe_ctrl->stc_ix, sizeof(send_wqe_ctrl->stc_ix)); 272 gta_wqe_ctrl.op_dirix = htobe32(gta_opcode << 28); 273 274 /* Set GTA match WQE DATA */ 275 if (has_match) { 276 if (send_wqe_match_data) 277 memcpy(>a_wqe_data0, send_wqe_match_data, sizeof(gta_wqe_data0)); 278 else 279 mlx5dr_send_wqe_set_tag(>a_wqe_data0, send_wqe_match_tag, is_jumbo); 280 281 gta_wqe_data0.rsvd1_definer = htobe32(send_attr->match_definer_id << 8); 282 attr.gta_data_0 = (uint8_t *)>a_wqe_data0; 283 } 284 285 /* Set GTA range WQE DATA */ 286 if (has_range) { 287 if (send_wqe_range_data) 288 memcpy(>a_wqe_data1, send_wqe_range_data, sizeof(gta_wqe_data1)); 289 else 290 mlx5dr_send_wqe_set_tag(>a_wqe_data1, send_wqe_range_tag, false); 291 292 gta_wqe_data1.rsvd1_definer = htobe32(send_attr->range_definer_id << 8); 293 attr.gta_data_1 = (uint8_t *)>a_wqe_data1; 294 } 295 296 attr.pdn = pd_num; 297 attr.wqe_ctrl = (uint8_t *)&wqe_ctrl; 298 attr.gta_ctrl = (uint8_t *)>a_wqe_ctrl; 299 300 send_wqe: 301 ret = mlx5dr_cmd_generate_wqe(ibv_ctx, &attr, &cqe); 302 if (ret) { 303 DR_LOG(ERR, "Failed to write WQE using command"); 304 return ret; 305 } 306 307 if ((mlx5dv_get_cqe_opcode(&cqe) == MLX5_CQE_REQ) && 308 (rte_be_to_cpu_32(cqe.byte_cnt) >> 31 == 0)) { 309 *send_attr->used_id = send_attr->id; 310 return 0; 311 } 312 313 /* Retry if rule failed */ 314 if (send_attr->retry_id) { 315 wqe_ctrl.imm = rte_cpu_to_be_32(send_attr->retry_id); 316 send_attr->id = send_attr->retry_id; 317 send_attr->retry_id = 0; 318 goto send_wqe; 319 } 320 321 return -1; 322 } 323 324 void mlx5dr_send_stes_fw(struct mlx5dr_send_engine *queue, 325 struct mlx5dr_send_ste_attr *ste_attr) 326 { 327 struct mlx5dr_send_engine_post_attr *send_attr = &ste_attr->send_attr; 328 struct mlx5dr_rule *rule = send_attr->rule; 329 struct ibv_context *ibv_ctx; 330 struct mlx5dr_context *ctx; 331 uint16_t queue_id; 332 uint32_t pdn; 333 int ret; 334 335 ctx = rule->matcher->tbl->ctx; 336 queue_id = queue - ctx->send_queue; 337 ibv_ctx = ctx->ibv_ctx; 338 pdn = ctx->pd_num; 339 340 /* Writing through FW can't HW fence, therefore we drain the queue */ 341 if (send_attr->fence) 342 mlx5dr_send_queue_action(ctx, 343 queue_id, 344 MLX5DR_SEND_QUEUE_ACTION_DRAIN_SYNC); 345 346 if (ste_attr->rtc_1) { 347 send_attr->id = ste_attr->rtc_1; 348 send_attr->used_id = ste_attr->used_id_rtc_1; 349 send_attr->retry_id = ste_attr->retry_rtc_1; 350 ret = mlx5dr_send_wqe_fw(ibv_ctx, pdn, send_attr, 351 ste_attr->wqe_ctrl, 352 ste_attr->wqe_data, 353 ste_attr->wqe_tag, 354 ste_attr->range_wqe_data, 355 ste_attr->range_wqe_tag, 356 ste_attr->wqe_tag_is_jumbo, 357 ste_attr->gta_opcode); 358 if (ret) 359 goto fail_rule; 360 } 361 362 if (ste_attr->rtc_0) { 363 send_attr->id = ste_attr->rtc_0; 364 send_attr->used_id = ste_attr->used_id_rtc_0; 365 send_attr->retry_id = ste_attr->retry_rtc_0; 366 ret = mlx5dr_send_wqe_fw(ibv_ctx, pdn, send_attr, 367 ste_attr->wqe_ctrl, 368 ste_attr->wqe_data, 369 ste_attr->wqe_tag, 370 ste_attr->range_wqe_data, 371 ste_attr->range_wqe_tag, 372 ste_attr->wqe_tag_is_jumbo, 373 ste_attr->gta_opcode); 374 if (ret) 375 goto fail_rule; 376 } 377 378 /* Increase the status, this only works on good flow as the enum 379 * is arrange it away creating -> created -> deleting -> deleted 380 */ 381 rule->status++; 382 mlx5dr_send_engine_gen_comp(queue, send_attr->user_data, RTE_FLOW_OP_SUCCESS); 383 return; 384 385 fail_rule: 386 rule->status = !rule->rtc_0 && !rule->rtc_1 ? 387 MLX5DR_RULE_STATUS_FAILED : MLX5DR_RULE_STATUS_FAILING; 388 mlx5dr_send_engine_gen_comp(queue, send_attr->user_data, RTE_FLOW_OP_ERROR); 389 } 390 391 static void mlx5dr_send_engine_retry_post_send(struct mlx5dr_send_engine *queue, 392 struct mlx5dr_send_ring_priv *priv, 393 uint16_t wqe_cnt) 394 { 395 struct mlx5dr_send_engine_post_attr send_attr = {0}; 396 struct mlx5dr_wqe_gta_data_seg_ste *wqe_data; 397 struct mlx5dr_wqe_gta_ctrl_seg *wqe_ctrl; 398 struct mlx5dr_send_engine_post_ctrl ctrl; 399 struct mlx5dr_send_ring_sq *send_sq; 400 unsigned int idx; 401 size_t wqe_len; 402 char *p; 403 404 send_attr.rule = priv->rule; 405 send_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS; 406 send_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE; 407 send_attr.len = MLX5_SEND_WQE_BB * 2 - sizeof(struct mlx5dr_wqe_ctrl_seg); 408 send_attr.notify_hw = 1; 409 send_attr.fence = 0; 410 send_attr.user_data = priv->user_data; 411 send_attr.id = priv->retry_id; 412 send_attr.used_id = priv->used_id; 413 414 ctrl = mlx5dr_send_engine_post_start(queue); 415 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_ctrl, &wqe_len); 416 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_data, &wqe_len); 417 418 send_sq = &ctrl.send_ring->send_sq; 419 idx = wqe_cnt & send_sq->buf_mask; 420 p = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT); 421 422 /* Copy old gta ctrl */ 423 memcpy(wqe_ctrl, p + sizeof(struct mlx5dr_wqe_ctrl_seg), 424 MLX5_SEND_WQE_BB - sizeof(struct mlx5dr_wqe_ctrl_seg)); 425 426 idx = (wqe_cnt + 1) & send_sq->buf_mask; 427 p = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT); 428 429 /* Copy old gta data */ 430 memcpy(wqe_data, p, MLX5_SEND_WQE_BB); 431 432 mlx5dr_send_engine_post_end(&ctrl, &send_attr); 433 } 434 435 void mlx5dr_send_engine_flush_queue(struct mlx5dr_send_engine *queue) 436 { 437 struct mlx5dr_send_ring_sq *sq = &queue->send_ring[0].send_sq; 438 struct mlx5dr_wqe_ctrl_seg *wqe_ctrl; 439 440 wqe_ctrl = (void *)(sq->buf + (sq->last_idx << MLX5_SEND_WQE_SHIFT)); 441 442 wqe_ctrl->flags |= rte_cpu_to_be_32(MLX5_WQE_CTRL_CQ_UPDATE); 443 444 mlx5dr_send_engine_post_ring(sq, queue->uar, wqe_ctrl); 445 } 446 447 static void mlx5dr_send_engine_update_rule(struct mlx5dr_send_engine *queue, 448 struct mlx5dr_send_ring_priv *priv, 449 uint16_t wqe_cnt, 450 enum rte_flow_op_status *status) 451 { 452 priv->rule->pending_wqes--; 453 454 if (*status == RTE_FLOW_OP_ERROR) { 455 if (priv->retry_id) { 456 mlx5dr_send_engine_retry_post_send(queue, priv, wqe_cnt); 457 return; 458 } 459 /* Some part of the rule failed */ 460 priv->rule->status = MLX5DR_RULE_STATUS_FAILING; 461 *priv->used_id = 0; 462 } else { 463 *priv->used_id = priv->id; 464 } 465 466 /* Update rule status for the last completion */ 467 if (!priv->rule->pending_wqes) { 468 if (unlikely(priv->rule->status == MLX5DR_RULE_STATUS_FAILING)) { 469 /* Rule completely failed and doesn't require cleanup */ 470 if (!priv->rule->rtc_0 && !priv->rule->rtc_1) 471 priv->rule->status = MLX5DR_RULE_STATUS_FAILED; 472 473 *status = RTE_FLOW_OP_ERROR; 474 } else { 475 /* Increase the status, this only works on good flow as the enum 476 * is arrange it away creating -> created -> deleting -> deleted 477 */ 478 priv->rule->status++; 479 *status = RTE_FLOW_OP_SUCCESS; 480 /* Rule was deleted now we can safely release action STEs */ 481 if (priv->rule->status == MLX5DR_RULE_STATUS_DELETED) 482 mlx5dr_rule_free_action_ste_idx(priv->rule); 483 } 484 } 485 } 486 487 static void mlx5dr_send_engine_update(struct mlx5dr_send_engine *queue, 488 struct mlx5_cqe64 *cqe, 489 struct mlx5dr_send_ring_priv *priv, 490 struct rte_flow_op_result res[], 491 int64_t *i, 492 uint32_t res_nb, 493 uint16_t wqe_cnt) 494 { 495 enum rte_flow_op_status status; 496 497 if (!cqe || (likely(rte_be_to_cpu_32(cqe->byte_cnt) >> 31 == 0) && 498 likely(mlx5dv_get_cqe_opcode(cqe) == MLX5_CQE_REQ))) { 499 status = RTE_FLOW_OP_SUCCESS; 500 } else { 501 status = RTE_FLOW_OP_ERROR; 502 } 503 504 if (priv->user_data) { 505 if (priv->rule) { 506 mlx5dr_send_engine_update_rule(queue, priv, wqe_cnt, &status); 507 /* Completion is provided on the last rule WQE */ 508 if (priv->rule->pending_wqes) 509 return; 510 } 511 512 if (*i < res_nb) { 513 res[*i].user_data = priv->user_data; 514 res[*i].status = status; 515 (*i)++; 516 mlx5dr_send_engine_dec_rule(queue); 517 } else { 518 mlx5dr_send_engine_gen_comp(queue, priv->user_data, status); 519 } 520 } 521 } 522 523 static void mlx5dr_send_engine_poll_cq(struct mlx5dr_send_engine *queue, 524 struct mlx5dr_send_ring *send_ring, 525 struct rte_flow_op_result res[], 526 int64_t *i, 527 uint32_t res_nb) 528 { 529 struct mlx5dr_send_ring_cq *cq = &send_ring->send_cq; 530 struct mlx5dr_send_ring_sq *sq = &send_ring->send_sq; 531 uint32_t cq_idx = cq->cons_index & cq->ncqe_mask; 532 struct mlx5dr_send_ring_priv *priv; 533 struct mlx5_cqe64 *cqe; 534 uint32_t offset_cqe64; 535 uint8_t cqe_opcode; 536 uint8_t cqe_owner; 537 uint16_t wqe_cnt; 538 uint8_t sw_own; 539 540 offset_cqe64 = RTE_CACHE_LINE_SIZE - sizeof(struct mlx5_cqe64); 541 cqe = (void *)(cq->buf + (cq_idx << cq->cqe_log_sz) + offset_cqe64); 542 543 sw_own = (cq->cons_index & cq->ncqe) ? 1 : 0; 544 cqe_opcode = mlx5dv_get_cqe_opcode(cqe); 545 cqe_owner = mlx5dv_get_cqe_owner(cqe); 546 547 if (cqe_opcode == MLX5_CQE_INVALID || 548 cqe_owner != sw_own) 549 return; 550 551 if (unlikely(mlx5dv_get_cqe_opcode(cqe) != MLX5_CQE_REQ)) 552 queue->err = true; 553 554 rte_io_rmb(); 555 556 wqe_cnt = be16toh(cqe->wqe_counter) & sq->buf_mask; 557 558 while (cq->poll_wqe != wqe_cnt) { 559 priv = &sq->wr_priv[cq->poll_wqe]; 560 mlx5dr_send_engine_update(queue, NULL, priv, res, i, res_nb, 0); 561 cq->poll_wqe = (cq->poll_wqe + priv->num_wqebbs) & sq->buf_mask; 562 } 563 564 priv = &sq->wr_priv[wqe_cnt]; 565 cq->poll_wqe = (wqe_cnt + priv->num_wqebbs) & sq->buf_mask; 566 mlx5dr_send_engine_update(queue, cqe, priv, res, i, res_nb, wqe_cnt); 567 cq->cons_index++; 568 } 569 570 static void mlx5dr_send_engine_poll_cqs(struct mlx5dr_send_engine *queue, 571 struct rte_flow_op_result res[], 572 int64_t *polled, 573 uint32_t res_nb) 574 { 575 int j; 576 577 for (j = 0; j < MLX5DR_NUM_SEND_RINGS; j++) { 578 mlx5dr_send_engine_poll_cq(queue, &queue->send_ring[j], 579 res, polled, res_nb); 580 581 *queue->send_ring[j].send_cq.db = 582 htobe32(queue->send_ring[j].send_cq.cons_index & 0xffffff); 583 } 584 } 585 586 static void mlx5dr_send_engine_poll_list(struct mlx5dr_send_engine *queue, 587 struct rte_flow_op_result res[], 588 int64_t *polled, 589 uint32_t res_nb) 590 { 591 struct mlx5dr_completed_poll *comp = &queue->completed; 592 593 while (comp->ci != comp->pi) { 594 if (*polled < res_nb) { 595 res[*polled].status = 596 comp->entries[comp->ci].status; 597 res[*polled].user_data = 598 comp->entries[comp->ci].user_data; 599 (*polled)++; 600 comp->ci = (comp->ci + 1) & comp->mask; 601 mlx5dr_send_engine_dec_rule(queue); 602 } else { 603 return; 604 } 605 } 606 } 607 608 static int mlx5dr_send_engine_poll(struct mlx5dr_send_engine *queue, 609 struct rte_flow_op_result res[], 610 uint32_t res_nb) 611 { 612 int64_t polled = 0; 613 614 mlx5dr_send_engine_poll_list(queue, res, &polled, res_nb); 615 616 if (polled >= res_nb) 617 return polled; 618 619 mlx5dr_send_engine_poll_cqs(queue, res, &polled, res_nb); 620 621 return polled; 622 } 623 624 int mlx5dr_send_queue_poll(struct mlx5dr_context *ctx, 625 uint16_t queue_id, 626 struct rte_flow_op_result res[], 627 uint32_t res_nb) 628 { 629 return mlx5dr_send_engine_poll(&ctx->send_queue[queue_id], 630 res, res_nb); 631 } 632 633 static int mlx5dr_send_ring_create_sq_obj(struct mlx5dr_context *ctx, 634 struct mlx5dr_send_engine *queue, 635 struct mlx5dr_send_ring_sq *sq, 636 struct mlx5dr_send_ring_cq *cq, 637 size_t log_wq_sz) 638 { 639 struct mlx5dr_cmd_sq_create_attr attr = {0}; 640 int err; 641 642 attr.cqn = cq->cqn; 643 attr.pdn = ctx->pd_num; 644 attr.page_id = queue->uar->page_id; 645 attr.dbr_id = sq->db_umem->umem_id; 646 attr.wq_id = sq->buf_umem->umem_id; 647 attr.log_wq_sz = log_wq_sz; 648 if (ctx->caps->sq_ts_format == MLX5_HCA_CAP_TIMESTAMP_FORMAT_FR) 649 attr.ts_format = MLX5_QPC_TIMESTAMP_FORMAT_FREE_RUNNING; 650 else 651 attr.ts_format = MLX5_QPC_TIMESTAMP_FORMAT_DEFAULT; 652 653 sq->obj = mlx5dr_cmd_sq_create(ctx->ibv_ctx, &attr); 654 if (!sq->obj) 655 return rte_errno; 656 657 sq->sqn = sq->obj->id; 658 659 err = mlx5dr_cmd_sq_modify_rdy(sq->obj); 660 if (err) 661 goto free_sq; 662 663 return 0; 664 665 free_sq: 666 mlx5dr_cmd_destroy_obj(sq->obj); 667 668 return err; 669 } 670 671 static inline unsigned long align(unsigned long val, unsigned long align) 672 { 673 return (val + align - 1) & ~(align - 1); 674 } 675 676 static int mlx5dr_send_ring_open_sq(struct mlx5dr_context *ctx, 677 struct mlx5dr_send_engine *queue, 678 struct mlx5dr_send_ring_sq *sq, 679 struct mlx5dr_send_ring_cq *cq) 680 { 681 size_t sq_log_buf_sz; 682 size_t buf_aligned; 683 size_t sq_buf_sz; 684 size_t page_size; 685 size_t buf_sz; 686 int err; 687 688 buf_sz = queue->num_entries * MAX_WQES_PER_RULE; 689 sq_log_buf_sz = log2above(buf_sz); 690 sq_buf_sz = 1 << (sq_log_buf_sz + log2above(MLX5_SEND_WQE_BB)); 691 sq->reg_addr = queue->uar->reg_addr; 692 693 page_size = sysconf(_SC_PAGESIZE); 694 buf_aligned = align(sq_buf_sz, page_size); 695 err = posix_memalign((void **)&sq->buf, page_size, buf_aligned); 696 if (err) { 697 rte_errno = ENOMEM; 698 return err; 699 } 700 memset(sq->buf, 0, buf_aligned); 701 702 err = posix_memalign((void **)&sq->db, 8, 8); 703 if (err) 704 goto free_buf; 705 706 sq->buf_umem = mlx5_glue->devx_umem_reg(ctx->ibv_ctx, sq->buf, sq_buf_sz, 0); 707 708 if (!sq->buf_umem) { 709 err = errno; 710 goto free_db; 711 } 712 713 sq->db_umem = mlx5_glue->devx_umem_reg(ctx->ibv_ctx, sq->db, 8, 0); 714 if (!sq->db_umem) { 715 err = errno; 716 goto free_buf_umem; 717 } 718 719 err = mlx5dr_send_ring_create_sq_obj(ctx, queue, sq, cq, sq_log_buf_sz); 720 721 if (err) 722 goto free_db_umem; 723 724 sq->wr_priv = simple_malloc(sizeof(*sq->wr_priv) * buf_sz); 725 if (!sq->wr_priv) { 726 err = ENOMEM; 727 goto destroy_sq_obj; 728 } 729 730 sq->dep_wqe = simple_calloc(queue->num_entries, sizeof(*sq->dep_wqe)); 731 if (!sq->dep_wqe) { 732 err = ENOMEM; 733 goto destroy_wr_priv; 734 } 735 736 sq->buf_mask = buf_sz - 1; 737 738 return 0; 739 740 destroy_wr_priv: 741 simple_free(sq->wr_priv); 742 destroy_sq_obj: 743 mlx5dr_cmd_destroy_obj(sq->obj); 744 free_db_umem: 745 mlx5_glue->devx_umem_dereg(sq->db_umem); 746 free_buf_umem: 747 mlx5_glue->devx_umem_dereg(sq->buf_umem); 748 free_db: 749 free(sq->db); 750 free_buf: 751 free(sq->buf); 752 rte_errno = err; 753 return err; 754 } 755 756 static void mlx5dr_send_ring_close_sq(struct mlx5dr_send_ring_sq *sq) 757 { 758 simple_free(sq->dep_wqe); 759 mlx5dr_cmd_destroy_obj(sq->obj); 760 mlx5_glue->devx_umem_dereg(sq->db_umem); 761 mlx5_glue->devx_umem_dereg(sq->buf_umem); 762 simple_free(sq->wr_priv); 763 free(sq->db); 764 free(sq->buf); 765 } 766 767 static int mlx5dr_send_ring_open_cq(struct mlx5dr_context *ctx, 768 struct mlx5dr_send_engine *queue, 769 struct mlx5dr_send_ring_cq *cq) 770 { 771 struct mlx5dv_cq mlx5_cq = {0}; 772 struct mlx5dv_obj obj; 773 struct ibv_cq *ibv_cq; 774 size_t cq_size; 775 int err; 776 777 cq_size = queue->num_entries; 778 ibv_cq = mlx5_glue->create_cq(ctx->ibv_ctx, cq_size, NULL, NULL, 0); 779 if (!ibv_cq) { 780 DR_LOG(ERR, "Failed to create CQ"); 781 rte_errno = errno; 782 return rte_errno; 783 } 784 785 obj.cq.in = ibv_cq; 786 obj.cq.out = &mlx5_cq; 787 err = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ); 788 if (err) { 789 err = errno; 790 goto close_cq; 791 } 792 793 cq->buf = mlx5_cq.buf; 794 cq->db = mlx5_cq.dbrec; 795 cq->ncqe = mlx5_cq.cqe_cnt; 796 cq->cqe_sz = mlx5_cq.cqe_size; 797 cq->cqe_log_sz = log2above(cq->cqe_sz); 798 cq->ncqe_mask = cq->ncqe - 1; 799 cq->buf_sz = cq->cqe_sz * cq->ncqe; 800 cq->cqn = mlx5_cq.cqn; 801 cq->ibv_cq = ibv_cq; 802 803 return 0; 804 805 close_cq: 806 mlx5_glue->destroy_cq(ibv_cq); 807 rte_errno = err; 808 return err; 809 } 810 811 static void mlx5dr_send_ring_close_cq(struct mlx5dr_send_ring_cq *cq) 812 { 813 mlx5_glue->destroy_cq(cq->ibv_cq); 814 } 815 816 static void mlx5dr_send_ring_close(struct mlx5dr_send_ring *ring) 817 { 818 mlx5dr_send_ring_close_sq(&ring->send_sq); 819 mlx5dr_send_ring_close_cq(&ring->send_cq); 820 } 821 822 static int mlx5dr_send_ring_open(struct mlx5dr_context *ctx, 823 struct mlx5dr_send_engine *queue, 824 struct mlx5dr_send_ring *ring) 825 { 826 int err; 827 828 err = mlx5dr_send_ring_open_cq(ctx, queue, &ring->send_cq); 829 if (err) 830 return err; 831 832 err = mlx5dr_send_ring_open_sq(ctx, queue, &ring->send_sq, &ring->send_cq); 833 if (err) 834 goto close_cq; 835 836 return err; 837 838 close_cq: 839 mlx5dr_send_ring_close_cq(&ring->send_cq); 840 841 return err; 842 } 843 844 static void __mlx5dr_send_rings_close(struct mlx5dr_send_engine *queue, 845 uint16_t i) 846 { 847 while (i--) 848 mlx5dr_send_ring_close(&queue->send_ring[i]); 849 } 850 851 static void mlx5dr_send_rings_close(struct mlx5dr_send_engine *queue) 852 { 853 __mlx5dr_send_rings_close(queue, queue->rings); 854 } 855 856 static int mlx5dr_send_rings_open(struct mlx5dr_context *ctx, 857 struct mlx5dr_send_engine *queue) 858 { 859 uint16_t i; 860 int err; 861 862 for (i = 0; i < queue->rings; i++) { 863 err = mlx5dr_send_ring_open(ctx, queue, &queue->send_ring[i]); 864 if (err) 865 goto free_rings; 866 } 867 868 return 0; 869 870 free_rings: 871 __mlx5dr_send_rings_close(queue, i); 872 873 return err; 874 } 875 876 void mlx5dr_send_queue_close(struct mlx5dr_send_engine *queue) 877 { 878 mlx5dr_send_rings_close(queue); 879 simple_free(queue->completed.entries); 880 mlx5_glue->devx_free_uar(queue->uar); 881 } 882 883 int mlx5dr_send_queue_open(struct mlx5dr_context *ctx, 884 struct mlx5dr_send_engine *queue, 885 uint16_t queue_size) 886 { 887 struct mlx5dv_devx_uar *uar; 888 int err; 889 890 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 891 uar = mlx5_glue->devx_alloc_uar(ctx->ibv_ctx, MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC); 892 if (!uar) { 893 rte_errno = errno; 894 return rte_errno; 895 } 896 #else 897 uar = NULL; 898 rte_errno = ENOTSUP; 899 return rte_errno; 900 #endif 901 902 queue->uar = uar; 903 queue->rings = MLX5DR_NUM_SEND_RINGS; 904 queue->num_entries = roundup_pow_of_two(queue_size); 905 queue->used_entries = 0; 906 queue->th_entries = queue->num_entries; 907 908 queue->completed.entries = simple_calloc(queue->num_entries, 909 sizeof(queue->completed.entries[0])); 910 if (!queue->completed.entries) { 911 rte_errno = ENOMEM; 912 goto free_uar; 913 } 914 queue->completed.pi = 0; 915 queue->completed.ci = 0; 916 queue->completed.mask = queue->num_entries - 1; 917 918 err = mlx5dr_send_rings_open(ctx, queue); 919 if (err) 920 goto free_completed_entries; 921 922 return 0; 923 924 free_completed_entries: 925 simple_free(queue->completed.entries); 926 free_uar: 927 mlx5_glue->devx_free_uar(uar); 928 return rte_errno; 929 } 930 931 static void __mlx5dr_send_queues_close(struct mlx5dr_context *ctx, uint16_t queues) 932 { 933 struct mlx5dr_send_engine *queue; 934 935 while (queues--) { 936 queue = &ctx->send_queue[queues]; 937 938 mlx5dr_send_queue_close(queue); 939 } 940 } 941 942 void mlx5dr_send_queues_close(struct mlx5dr_context *ctx) 943 { 944 __mlx5dr_send_queues_close(ctx, ctx->queues); 945 simple_free(ctx->send_queue); 946 } 947 948 int mlx5dr_send_queues_open(struct mlx5dr_context *ctx, 949 uint16_t queues, 950 uint16_t queue_size) 951 { 952 int err = 0; 953 uint32_t i; 954 955 /* Open one extra queue for control path */ 956 ctx->queues = queues + 1; 957 958 ctx->send_queue = simple_calloc(ctx->queues, sizeof(*ctx->send_queue)); 959 if (!ctx->send_queue) { 960 rte_errno = ENOMEM; 961 return rte_errno; 962 } 963 964 for (i = 0; i < ctx->queues; i++) { 965 err = mlx5dr_send_queue_open(ctx, &ctx->send_queue[i], queue_size); 966 if (err) 967 goto close_send_queues; 968 } 969 970 return 0; 971 972 close_send_queues: 973 __mlx5dr_send_queues_close(ctx, i); 974 975 simple_free(ctx->send_queue); 976 977 return err; 978 } 979 980 int mlx5dr_send_queue_action(struct mlx5dr_context *ctx, 981 uint16_t queue_id, 982 uint32_t actions) 983 { 984 struct mlx5dr_send_ring_sq *send_sq; 985 struct mlx5dr_send_engine *queue; 986 bool wait_comp = false; 987 int64_t polled = 0; 988 989 queue = &ctx->send_queue[queue_id]; 990 send_sq = &queue->send_ring->send_sq; 991 992 switch (actions) { 993 case MLX5DR_SEND_QUEUE_ACTION_DRAIN_SYNC: 994 wait_comp = true; 995 /* FALLTHROUGH */ 996 case MLX5DR_SEND_QUEUE_ACTION_DRAIN_ASYNC: 997 if (send_sq->head_dep_idx != send_sq->tail_dep_idx) 998 /* Send dependent WQEs to drain the queue */ 999 mlx5dr_send_all_dep_wqe(queue); 1000 else 1001 /* Signal on the last posted WQE */ 1002 mlx5dr_send_engine_flush_queue(queue); 1003 1004 /* Poll queue until empty */ 1005 while (wait_comp && !mlx5dr_send_engine_empty(queue)) 1006 mlx5dr_send_engine_poll_cqs(queue, NULL, &polled, 0); 1007 1008 break; 1009 default: 1010 rte_errno = -EINVAL; 1011 return rte_errno; 1012 } 1013 1014 return 0; 1015 } 1016