1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2022 NVIDIA Corporation & Affiliates 3 */ 4 5 #include "mlx5dr_internal.h" 6 7 struct mlx5dr_send_ring_dep_wqe * 8 mlx5dr_send_add_new_dep_wqe(struct mlx5dr_send_engine *queue) 9 { 10 struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq; 11 unsigned int idx = send_sq->head_dep_idx++ & (queue->num_entries - 1); 12 13 memset(&send_sq->dep_wqe[idx].wqe_data.tag, 0, MLX5DR_MATCH_TAG_SZ); 14 15 return &send_sq->dep_wqe[idx]; 16 } 17 18 void mlx5dr_send_abort_new_dep_wqe(struct mlx5dr_send_engine *queue) 19 { 20 queue->send_ring->send_sq.head_dep_idx--; 21 } 22 23 void mlx5dr_send_all_dep_wqe(struct mlx5dr_send_engine *queue) 24 { 25 struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq; 26 struct mlx5dr_send_ste_attr ste_attr = {0}; 27 struct mlx5dr_send_ring_dep_wqe *dep_wqe; 28 29 ste_attr.send_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE; 30 ste_attr.send_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS; 31 ste_attr.send_attr.len = MLX5DR_WQE_SZ_GTA_CTRL + MLX5DR_WQE_SZ_GTA_DATA; 32 ste_attr.gta_opcode = MLX5DR_WQE_GTA_OP_ACTIVATE; 33 34 /* Fence first from previous depend WQEs */ 35 ste_attr.send_attr.fence = 1; 36 37 while (send_sq->head_dep_idx != send_sq->tail_dep_idx) { 38 dep_wqe = &send_sq->dep_wqe[send_sq->tail_dep_idx++ & (queue->num_entries - 1)]; 39 40 /* Notify HW on the last WQE */ 41 ste_attr.send_attr.notify_hw = (send_sq->tail_dep_idx == send_sq->head_dep_idx); 42 ste_attr.send_attr.user_data = dep_wqe->user_data; 43 ste_attr.send_attr.rule = dep_wqe->rule; 44 45 ste_attr.rtc_0 = dep_wqe->rtc_0; 46 ste_attr.rtc_1 = dep_wqe->rtc_1; 47 ste_attr.retry_rtc_0 = dep_wqe->retry_rtc_0; 48 ste_attr.retry_rtc_1 = dep_wqe->retry_rtc_1; 49 ste_attr.used_id_rtc_0 = &dep_wqe->rule->rtc_0; 50 ste_attr.used_id_rtc_1 = &dep_wqe->rule->rtc_1; 51 ste_attr.wqe_ctrl = &dep_wqe->wqe_ctrl; 52 ste_attr.wqe_data = &dep_wqe->wqe_data; 53 ste_attr.direct_index = dep_wqe->direct_index; 54 55 mlx5dr_send_ste(queue, &ste_attr); 56 57 /* Fencing is done only on the first WQE */ 58 ste_attr.send_attr.fence = 0; 59 } 60 } 61 62 struct mlx5dr_send_engine_post_ctrl 63 mlx5dr_send_engine_post_start(struct mlx5dr_send_engine *queue) 64 { 65 struct mlx5dr_send_engine_post_ctrl ctrl; 66 67 ctrl.queue = queue; 68 /* Currently only one send ring is supported */ 69 ctrl.send_ring = &queue->send_ring[0]; 70 ctrl.num_wqebbs = 0; 71 72 return ctrl; 73 } 74 75 void mlx5dr_send_engine_post_req_wqe(struct mlx5dr_send_engine_post_ctrl *ctrl, 76 char **buf, size_t *len) 77 { 78 struct mlx5dr_send_ring_sq *send_sq = &ctrl->send_ring->send_sq; 79 unsigned int idx; 80 81 idx = (send_sq->cur_post + ctrl->num_wqebbs) & send_sq->buf_mask; 82 83 *buf = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT); 84 *len = MLX5_SEND_WQE_BB; 85 86 if (!ctrl->num_wqebbs) { 87 *buf += sizeof(struct mlx5dr_wqe_ctrl_seg); 88 *len -= sizeof(struct mlx5dr_wqe_ctrl_seg); 89 } 90 91 ctrl->num_wqebbs++; 92 } 93 94 static void mlx5dr_send_engine_post_ring(struct mlx5dr_send_ring_sq *sq, 95 struct mlx5dv_devx_uar *uar, 96 struct mlx5dr_wqe_ctrl_seg *wqe_ctrl) 97 { 98 rte_compiler_barrier(); 99 sq->db[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->cur_post); 100 101 rte_wmb(); 102 mlx5dr_uar_write64_relaxed(*((uint64_t *)wqe_ctrl), uar->reg_addr); 103 rte_wmb(); 104 } 105 106 static void 107 mlx5dr_send_wqe_set_tag(struct mlx5dr_wqe_gta_data_seg_ste *wqe_data, 108 struct mlx5dr_rule_match_tag *tag, 109 bool is_jumbo) 110 { 111 if (is_jumbo) { 112 /* Clear previous possibly dirty control */ 113 memset(wqe_data, 0, MLX5DR_STE_CTRL_SZ); 114 memcpy(wqe_data->jumbo, tag->jumbo, MLX5DR_JUMBO_TAG_SZ); 115 } else { 116 /* Clear previous possibly dirty control and actions */ 117 memset(wqe_data, 0, MLX5DR_STE_CTRL_SZ + MLX5DR_ACTIONS_SZ); 118 memcpy(wqe_data->tag, tag->match, MLX5DR_MATCH_TAG_SZ); 119 } 120 } 121 122 void mlx5dr_send_engine_post_end(struct mlx5dr_send_engine_post_ctrl *ctrl, 123 struct mlx5dr_send_engine_post_attr *attr) 124 { 125 struct mlx5dr_wqe_ctrl_seg *wqe_ctrl; 126 struct mlx5dr_send_ring_sq *sq; 127 uint32_t flags = 0; 128 unsigned int idx; 129 130 sq = &ctrl->send_ring->send_sq; 131 idx = sq->cur_post & sq->buf_mask; 132 sq->last_idx = idx; 133 134 wqe_ctrl = (void *)(sq->buf + (idx << MLX5_SEND_WQE_SHIFT)); 135 136 wqe_ctrl->opmod_idx_opcode = 137 rte_cpu_to_be_32((attr->opmod << 24) | 138 ((sq->cur_post & 0xffff) << 8) | 139 attr->opcode); 140 wqe_ctrl->qpn_ds = 141 rte_cpu_to_be_32((attr->len + sizeof(struct mlx5dr_wqe_ctrl_seg)) / 16 | 142 sq->sqn << 8); 143 144 wqe_ctrl->imm = rte_cpu_to_be_32(attr->id); 145 146 flags |= attr->notify_hw ? MLX5_WQE_CTRL_CQ_UPDATE : 0; 147 flags |= attr->fence ? MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE : 0; 148 wqe_ctrl->flags = rte_cpu_to_be_32(flags); 149 150 sq->wr_priv[idx].id = attr->id; 151 sq->wr_priv[idx].retry_id = attr->retry_id; 152 153 sq->wr_priv[idx].rule = attr->rule; 154 sq->wr_priv[idx].user_data = attr->user_data; 155 sq->wr_priv[idx].num_wqebbs = ctrl->num_wqebbs; 156 157 if (attr->rule) { 158 sq->wr_priv[idx].rule->pending_wqes++; 159 sq->wr_priv[idx].used_id = attr->used_id; 160 } 161 162 sq->cur_post += ctrl->num_wqebbs; 163 164 if (attr->notify_hw) 165 mlx5dr_send_engine_post_ring(sq, ctrl->queue->uar, wqe_ctrl); 166 } 167 168 static void mlx5dr_send_wqe(struct mlx5dr_send_engine *queue, 169 struct mlx5dr_send_engine_post_attr *send_attr, 170 struct mlx5dr_wqe_gta_ctrl_seg *send_wqe_ctrl, 171 void *send_wqe_data, 172 void *send_wqe_tag, 173 bool is_jumbo, 174 uint8_t gta_opcode, 175 uint32_t direct_index) 176 { 177 struct mlx5dr_wqe_gta_data_seg_ste *wqe_data; 178 struct mlx5dr_wqe_gta_ctrl_seg *wqe_ctrl; 179 struct mlx5dr_send_engine_post_ctrl ctrl; 180 size_t wqe_len; 181 182 ctrl = mlx5dr_send_engine_post_start(queue); 183 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_ctrl, &wqe_len); 184 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_data, &wqe_len); 185 186 wqe_ctrl->op_dirix = htobe32(gta_opcode << 28 | direct_index); 187 memcpy(wqe_ctrl->stc_ix, send_wqe_ctrl->stc_ix, sizeof(send_wqe_ctrl->stc_ix)); 188 189 if (send_wqe_data) 190 memcpy(wqe_data, send_wqe_data, sizeof(*wqe_data)); 191 else 192 mlx5dr_send_wqe_set_tag(wqe_data, send_wqe_tag, is_jumbo); 193 194 mlx5dr_send_engine_post_end(&ctrl, send_attr); 195 } 196 197 void mlx5dr_send_ste(struct mlx5dr_send_engine *queue, 198 struct mlx5dr_send_ste_attr *ste_attr) 199 { 200 struct mlx5dr_send_engine_post_attr *send_attr = &ste_attr->send_attr; 201 uint8_t notify_hw = send_attr->notify_hw; 202 uint8_t fence = send_attr->fence; 203 204 if (ste_attr->rtc_1) { 205 send_attr->id = ste_attr->rtc_1; 206 send_attr->used_id = ste_attr->used_id_rtc_1; 207 send_attr->retry_id = ste_attr->retry_rtc_1; 208 send_attr->fence = fence; 209 send_attr->notify_hw = notify_hw && !ste_attr->rtc_0; 210 mlx5dr_send_wqe(queue, send_attr, 211 ste_attr->wqe_ctrl, 212 ste_attr->wqe_data, 213 ste_attr->wqe_tag, 214 ste_attr->wqe_tag_is_jumbo, 215 ste_attr->gta_opcode, 216 ste_attr->direct_index); 217 } 218 219 if (ste_attr->rtc_0) { 220 send_attr->id = ste_attr->rtc_0; 221 send_attr->used_id = ste_attr->used_id_rtc_0; 222 send_attr->retry_id = ste_attr->retry_rtc_0; 223 send_attr->fence = fence && !ste_attr->rtc_1; 224 send_attr->notify_hw = notify_hw; 225 mlx5dr_send_wqe(queue, send_attr, 226 ste_attr->wqe_ctrl, 227 ste_attr->wqe_data, 228 ste_attr->wqe_tag, 229 ste_attr->wqe_tag_is_jumbo, 230 ste_attr->gta_opcode, 231 ste_attr->direct_index); 232 } 233 234 /* Restore to ortginal requested values */ 235 send_attr->notify_hw = notify_hw; 236 send_attr->fence = fence; 237 } 238 239 static 240 int mlx5dr_send_wqe_fw(struct ibv_context *ibv_ctx, 241 uint32_t pd_num, 242 struct mlx5dr_send_engine_post_attr *send_attr, 243 struct mlx5dr_wqe_gta_ctrl_seg *send_wqe_ctrl, 244 void *send_wqe_match_data, 245 void *send_wqe_match_tag, 246 void *send_wqe_range_data, 247 void *send_wqe_range_tag, 248 bool is_jumbo, 249 uint8_t gta_opcode) 250 { 251 bool has_range = send_wqe_range_data || send_wqe_range_tag; 252 bool has_match = send_wqe_match_data || send_wqe_match_tag; 253 struct mlx5dr_wqe_gta_data_seg_ste gta_wqe_data0 = {0}; 254 struct mlx5dr_wqe_gta_data_seg_ste gta_wqe_data1 = {0}; 255 struct mlx5dr_wqe_gta_ctrl_seg gta_wqe_ctrl = {0}; 256 struct mlx5dr_cmd_generate_wqe_attr attr = {0}; 257 struct mlx5dr_wqe_ctrl_seg wqe_ctrl = {0}; 258 struct mlx5_cqe64 cqe; 259 uint32_t flags = 0; 260 int ret; 261 262 /* Set WQE control */ 263 wqe_ctrl.opmod_idx_opcode = 264 rte_cpu_to_be_32((send_attr->opmod << 24) | send_attr->opcode); 265 wqe_ctrl.qpn_ds = 266 rte_cpu_to_be_32((send_attr->len + sizeof(struct mlx5dr_wqe_ctrl_seg)) / 16); 267 flags |= send_attr->notify_hw ? MLX5_WQE_CTRL_CQ_UPDATE : 0; 268 wqe_ctrl.flags = rte_cpu_to_be_32(flags); 269 wqe_ctrl.imm = rte_cpu_to_be_32(send_attr->id); 270 271 /* Set GTA WQE CTRL */ 272 memcpy(gta_wqe_ctrl.stc_ix, send_wqe_ctrl->stc_ix, sizeof(send_wqe_ctrl->stc_ix)); 273 gta_wqe_ctrl.op_dirix = htobe32(gta_opcode << 28); 274 275 /* Set GTA match WQE DATA */ 276 if (has_match) { 277 if (send_wqe_match_data) 278 memcpy(>a_wqe_data0, send_wqe_match_data, sizeof(gta_wqe_data0)); 279 else 280 mlx5dr_send_wqe_set_tag(>a_wqe_data0, send_wqe_match_tag, is_jumbo); 281 282 gta_wqe_data0.rsvd1_definer = htobe32(send_attr->match_definer_id << 8); 283 attr.gta_data_0 = (uint8_t *)>a_wqe_data0; 284 } 285 286 /* Set GTA range WQE DATA */ 287 if (has_range) { 288 if (send_wqe_range_data) 289 memcpy(>a_wqe_data1, send_wqe_range_data, sizeof(gta_wqe_data1)); 290 else 291 mlx5dr_send_wqe_set_tag(>a_wqe_data1, send_wqe_range_tag, false); 292 293 gta_wqe_data1.rsvd1_definer = htobe32(send_attr->range_definer_id << 8); 294 attr.gta_data_1 = (uint8_t *)>a_wqe_data1; 295 } 296 297 attr.pdn = pd_num; 298 attr.wqe_ctrl = (uint8_t *)&wqe_ctrl; 299 attr.gta_ctrl = (uint8_t *)>a_wqe_ctrl; 300 301 send_wqe: 302 ret = mlx5dr_cmd_generate_wqe(ibv_ctx, &attr, &cqe); 303 if (ret) { 304 DR_LOG(ERR, "Failed to write WQE using command"); 305 return ret; 306 } 307 308 if ((mlx5dv_get_cqe_opcode(&cqe) == MLX5_CQE_REQ) && 309 (rte_be_to_cpu_32(cqe.byte_cnt) >> 31 == 0)) { 310 *send_attr->used_id = send_attr->id; 311 return 0; 312 } 313 314 /* Retry if rule failed */ 315 if (send_attr->retry_id) { 316 wqe_ctrl.imm = rte_cpu_to_be_32(send_attr->retry_id); 317 send_attr->id = send_attr->retry_id; 318 send_attr->retry_id = 0; 319 goto send_wqe; 320 } 321 322 return -1; 323 } 324 325 void mlx5dr_send_stes_fw(struct mlx5dr_send_engine *queue, 326 struct mlx5dr_send_ste_attr *ste_attr) 327 { 328 struct mlx5dr_send_engine_post_attr *send_attr = &ste_attr->send_attr; 329 struct mlx5dr_rule *rule = send_attr->rule; 330 struct ibv_context *ibv_ctx; 331 struct mlx5dr_context *ctx; 332 uint16_t queue_id; 333 uint32_t pdn; 334 int ret; 335 336 ctx = rule->matcher->tbl->ctx; 337 queue_id = queue - ctx->send_queue; 338 ibv_ctx = ctx->ibv_ctx; 339 pdn = ctx->pd_num; 340 341 /* Writing through FW can't HW fence, therefore we drain the queue */ 342 if (send_attr->fence) 343 mlx5dr_send_queue_action(ctx, 344 queue_id, 345 MLX5DR_SEND_QUEUE_ACTION_DRAIN_SYNC); 346 347 if (ste_attr->rtc_1) { 348 send_attr->id = ste_attr->rtc_1; 349 send_attr->used_id = ste_attr->used_id_rtc_1; 350 send_attr->retry_id = ste_attr->retry_rtc_1; 351 ret = mlx5dr_send_wqe_fw(ibv_ctx, pdn, send_attr, 352 ste_attr->wqe_ctrl, 353 ste_attr->wqe_data, 354 ste_attr->wqe_tag, 355 ste_attr->range_wqe_data, 356 ste_attr->range_wqe_tag, 357 ste_attr->wqe_tag_is_jumbo, 358 ste_attr->gta_opcode); 359 if (ret) 360 goto fail_rule; 361 } 362 363 if (ste_attr->rtc_0) { 364 send_attr->id = ste_attr->rtc_0; 365 send_attr->used_id = ste_attr->used_id_rtc_0; 366 send_attr->retry_id = ste_attr->retry_rtc_0; 367 ret = mlx5dr_send_wqe_fw(ibv_ctx, pdn, send_attr, 368 ste_attr->wqe_ctrl, 369 ste_attr->wqe_data, 370 ste_attr->wqe_tag, 371 ste_attr->range_wqe_data, 372 ste_attr->range_wqe_tag, 373 ste_attr->wqe_tag_is_jumbo, 374 ste_attr->gta_opcode); 375 if (ret) 376 goto fail_rule; 377 } 378 379 /* Increase the status, this only works on good flow as the enum 380 * is arrange it away creating -> created -> deleting -> deleted 381 */ 382 rule->status++; 383 mlx5dr_send_engine_gen_comp(queue, send_attr->user_data, RTE_FLOW_OP_SUCCESS); 384 return; 385 386 fail_rule: 387 rule->status = !rule->rtc_0 && !rule->rtc_1 ? 388 MLX5DR_RULE_STATUS_FAILED : MLX5DR_RULE_STATUS_FAILING; 389 mlx5dr_send_engine_gen_comp(queue, send_attr->user_data, RTE_FLOW_OP_ERROR); 390 } 391 392 static void mlx5dr_send_engine_retry_post_send(struct mlx5dr_send_engine *queue, 393 struct mlx5dr_send_ring_priv *priv, 394 uint16_t wqe_cnt) 395 { 396 struct mlx5dr_send_engine_post_attr send_attr = {0}; 397 struct mlx5dr_wqe_gta_data_seg_ste *wqe_data; 398 struct mlx5dr_wqe_gta_ctrl_seg *wqe_ctrl; 399 struct mlx5dr_send_engine_post_ctrl ctrl; 400 struct mlx5dr_send_ring_sq *send_sq; 401 unsigned int idx; 402 size_t wqe_len; 403 char *p; 404 405 send_attr.rule = priv->rule; 406 send_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS; 407 send_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE; 408 send_attr.len = MLX5_SEND_WQE_BB * 2 - sizeof(struct mlx5dr_wqe_ctrl_seg); 409 send_attr.notify_hw = 1; 410 send_attr.fence = 0; 411 send_attr.user_data = priv->user_data; 412 send_attr.id = priv->retry_id; 413 send_attr.used_id = priv->used_id; 414 415 ctrl = mlx5dr_send_engine_post_start(queue); 416 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_ctrl, &wqe_len); 417 mlx5dr_send_engine_post_req_wqe(&ctrl, (void *)&wqe_data, &wqe_len); 418 419 send_sq = &ctrl.send_ring->send_sq; 420 idx = wqe_cnt & send_sq->buf_mask; 421 p = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT); 422 423 /* Copy old gta ctrl */ 424 memcpy(wqe_ctrl, p + sizeof(struct mlx5dr_wqe_ctrl_seg), 425 MLX5_SEND_WQE_BB - sizeof(struct mlx5dr_wqe_ctrl_seg)); 426 427 idx = (wqe_cnt + 1) & send_sq->buf_mask; 428 p = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT); 429 430 /* Copy old gta data */ 431 memcpy(wqe_data, p, MLX5_SEND_WQE_BB); 432 433 mlx5dr_send_engine_post_end(&ctrl, &send_attr); 434 } 435 436 void mlx5dr_send_engine_flush_queue(struct mlx5dr_send_engine *queue) 437 { 438 struct mlx5dr_send_ring_sq *sq = &queue->send_ring[0].send_sq; 439 struct mlx5dr_wqe_ctrl_seg *wqe_ctrl; 440 441 wqe_ctrl = (void *)(sq->buf + (sq->last_idx << MLX5_SEND_WQE_SHIFT)); 442 443 wqe_ctrl->flags |= rte_cpu_to_be_32(MLX5_WQE_CTRL_CQ_UPDATE); 444 445 mlx5dr_send_engine_post_ring(sq, queue->uar, wqe_ctrl); 446 } 447 448 static void 449 mlx5dr_send_engine_update_rule_resize(struct mlx5dr_send_engine *queue, 450 struct mlx5dr_send_ring_priv *priv, 451 enum rte_flow_op_status *status) 452 { 453 switch (priv->rule->resize_info->state) { 454 case MLX5DR_RULE_RESIZE_STATE_WRITING: 455 if (priv->rule->status == MLX5DR_RULE_STATUS_FAILING) { 456 /* Backup original RTCs */ 457 uint32_t orig_rtc_0 = priv->rule->resize_info->rtc_0; 458 uint32_t orig_rtc_1 = priv->rule->resize_info->rtc_1; 459 460 /* Delete partially failed move rule using resize_info */ 461 priv->rule->resize_info->rtc_0 = priv->rule->rtc_0; 462 priv->rule->resize_info->rtc_1 = priv->rule->rtc_1; 463 464 /* Move rule to original RTC for future delete */ 465 priv->rule->rtc_0 = orig_rtc_0; 466 priv->rule->rtc_1 = orig_rtc_1; 467 } 468 /* Clean leftovers */ 469 mlx5dr_rule_move_hws_remove(priv->rule, queue, priv->user_data); 470 break; 471 472 case MLX5DR_RULE_RESIZE_STATE_DELETING: 473 if (priv->rule->status == MLX5DR_RULE_STATUS_FAILING) { 474 *status = RTE_FLOW_OP_ERROR; 475 } else { 476 *status = RTE_FLOW_OP_SUCCESS; 477 priv->rule->matcher = priv->rule->matcher->resize_dst; 478 } 479 priv->rule->resize_info->state = MLX5DR_RULE_RESIZE_STATE_IDLE; 480 priv->rule->status = MLX5DR_RULE_STATUS_CREATED; 481 break; 482 483 default: 484 break; 485 } 486 } 487 488 static void mlx5dr_send_engine_update_rule(struct mlx5dr_send_engine *queue, 489 struct mlx5dr_send_ring_priv *priv, 490 uint16_t wqe_cnt, 491 enum rte_flow_op_status *status) 492 { 493 priv->rule->pending_wqes--; 494 495 if (*status == RTE_FLOW_OP_ERROR) { 496 if (priv->retry_id) { 497 mlx5dr_send_engine_retry_post_send(queue, priv, wqe_cnt); 498 return; 499 } 500 /* Some part of the rule failed */ 501 priv->rule->status = MLX5DR_RULE_STATUS_FAILING; 502 *priv->used_id = 0; 503 } else { 504 *priv->used_id = priv->id; 505 } 506 507 /* Update rule status for the last completion */ 508 if (!priv->rule->pending_wqes) { 509 if (unlikely(mlx5dr_rule_move_in_progress(priv->rule))) { 510 mlx5dr_send_engine_update_rule_resize(queue, priv, status); 511 return; 512 } 513 514 if (unlikely(priv->rule->status == MLX5DR_RULE_STATUS_FAILING)) { 515 /* Rule completely failed and doesn't require cleanup */ 516 if (!priv->rule->rtc_0 && !priv->rule->rtc_1) 517 priv->rule->status = MLX5DR_RULE_STATUS_FAILED; 518 519 *status = RTE_FLOW_OP_ERROR; 520 } else { 521 /* Increase the status, this only works on good flow as the enum 522 * is arrange it away creating -> created -> deleting -> deleted 523 */ 524 priv->rule->status++; 525 *status = RTE_FLOW_OP_SUCCESS; 526 /* Rule was deleted now we can safely release action STEs 527 * and clear resize info 528 */ 529 if (priv->rule->status == MLX5DR_RULE_STATUS_DELETED) { 530 mlx5dr_rule_free_action_ste_idx(priv->rule); 531 mlx5dr_rule_clear_resize_info(priv->rule); 532 } 533 } 534 } 535 } 536 537 static void mlx5dr_send_engine_update(struct mlx5dr_send_engine *queue, 538 struct mlx5_cqe64 *cqe, 539 struct mlx5dr_send_ring_priv *priv, 540 struct rte_flow_op_result res[], 541 int64_t *i, 542 uint32_t res_nb, 543 uint16_t wqe_cnt) 544 { 545 enum rte_flow_op_status status; 546 547 if (!cqe || (likely(rte_be_to_cpu_32(cqe->byte_cnt) >> 31 == 0) && 548 likely(mlx5dv_get_cqe_opcode(cqe) == MLX5_CQE_REQ))) { 549 status = RTE_FLOW_OP_SUCCESS; 550 } else { 551 status = RTE_FLOW_OP_ERROR; 552 } 553 554 if (priv->user_data) { 555 if (priv->rule) { 556 mlx5dr_send_engine_update_rule(queue, priv, wqe_cnt, &status); 557 /* Completion is provided on the last rule WQE */ 558 if (priv->rule->pending_wqes) 559 return; 560 } 561 562 if (*i < res_nb) { 563 res[*i].user_data = priv->user_data; 564 res[*i].status = status; 565 (*i)++; 566 mlx5dr_send_engine_dec_rule(queue); 567 } else { 568 mlx5dr_send_engine_gen_comp(queue, priv->user_data, status); 569 } 570 } 571 } 572 573 static void mlx5dr_send_engine_poll_cq(struct mlx5dr_send_engine *queue, 574 struct mlx5dr_send_ring *send_ring, 575 struct rte_flow_op_result res[], 576 int64_t *i, 577 uint32_t res_nb) 578 { 579 struct mlx5dr_send_ring_cq *cq = &send_ring->send_cq; 580 struct mlx5dr_send_ring_sq *sq = &send_ring->send_sq; 581 uint32_t cq_idx = cq->cons_index & cq->ncqe_mask; 582 struct mlx5dr_send_ring_priv *priv; 583 struct mlx5_cqe64 *cqe; 584 uint32_t offset_cqe64; 585 uint8_t cqe_opcode; 586 uint8_t cqe_owner; 587 uint16_t wqe_cnt; 588 uint8_t sw_own; 589 590 offset_cqe64 = RTE_CACHE_LINE_SIZE - sizeof(struct mlx5_cqe64); 591 cqe = (void *)(cq->buf + (cq_idx << cq->cqe_log_sz) + offset_cqe64); 592 593 sw_own = (cq->cons_index & cq->ncqe) ? 1 : 0; 594 cqe_opcode = mlx5dv_get_cqe_opcode(cqe); 595 cqe_owner = mlx5dv_get_cqe_owner(cqe); 596 597 if (cqe_opcode == MLX5_CQE_INVALID || 598 cqe_owner != sw_own) 599 return; 600 601 if (unlikely(cqe_opcode != MLX5_CQE_REQ)) 602 queue->err = true; 603 604 rte_io_rmb(); 605 606 wqe_cnt = be16toh(cqe->wqe_counter) & sq->buf_mask; 607 608 while (cq->poll_wqe != wqe_cnt) { 609 priv = &sq->wr_priv[cq->poll_wqe]; 610 mlx5dr_send_engine_update(queue, NULL, priv, res, i, res_nb, 0); 611 cq->poll_wqe = (cq->poll_wqe + priv->num_wqebbs) & sq->buf_mask; 612 } 613 614 priv = &sq->wr_priv[wqe_cnt]; 615 cq->poll_wqe = (wqe_cnt + priv->num_wqebbs) & sq->buf_mask; 616 mlx5dr_send_engine_update(queue, cqe, priv, res, i, res_nb, wqe_cnt); 617 cq->cons_index++; 618 *cq->db = htobe32(cq->cons_index & 0xffffff); 619 } 620 621 static void mlx5dr_send_engine_poll_cqs(struct mlx5dr_send_engine *queue, 622 struct rte_flow_op_result res[], 623 int64_t *polled, 624 uint32_t res_nb) 625 { 626 int j; 627 628 for (j = 0; j < MLX5DR_NUM_SEND_RINGS; j++) 629 mlx5dr_send_engine_poll_cq(queue, &queue->send_ring[j], 630 res, polled, res_nb); 631 } 632 633 static void mlx5dr_send_engine_poll_list(struct mlx5dr_send_engine *queue, 634 struct rte_flow_op_result res[], 635 int64_t *polled, 636 uint32_t res_nb) 637 { 638 struct mlx5dr_completed_poll *comp = &queue->completed; 639 640 while (comp->ci != comp->pi) { 641 if (*polled < res_nb) { 642 res[*polled].status = 643 comp->entries[comp->ci].status; 644 res[*polled].user_data = 645 comp->entries[comp->ci].user_data; 646 (*polled)++; 647 comp->ci = (comp->ci + 1) & comp->mask; 648 mlx5dr_send_engine_dec_rule(queue); 649 } else { 650 return; 651 } 652 } 653 } 654 655 static int mlx5dr_send_engine_poll(struct mlx5dr_send_engine *queue, 656 struct rte_flow_op_result res[], 657 uint32_t res_nb) 658 { 659 int64_t polled = 0; 660 661 mlx5dr_send_engine_poll_list(queue, res, &polled, res_nb); 662 663 if (polled >= res_nb) 664 return polled; 665 666 mlx5dr_send_engine_poll_cqs(queue, res, &polled, res_nb); 667 668 return polled; 669 } 670 671 int mlx5dr_send_queue_poll(struct mlx5dr_context *ctx, 672 uint16_t queue_id, 673 struct rte_flow_op_result res[], 674 uint32_t res_nb) 675 { 676 return mlx5dr_send_engine_poll(&ctx->send_queue[queue_id], 677 res, res_nb); 678 } 679 680 static int mlx5dr_send_ring_create_sq_obj(struct mlx5dr_context *ctx, 681 struct mlx5dr_send_engine *queue, 682 struct mlx5dr_send_ring_sq *sq, 683 struct mlx5dr_send_ring_cq *cq, 684 size_t log_wq_sz) 685 { 686 struct mlx5dr_cmd_sq_create_attr attr = {0}; 687 int err; 688 689 attr.cqn = cq->cqn; 690 attr.pdn = ctx->pd_num; 691 attr.page_id = queue->uar->page_id; 692 attr.dbr_id = sq->db_umem->umem_id; 693 attr.wq_id = sq->buf_umem->umem_id; 694 attr.log_wq_sz = log_wq_sz; 695 if (ctx->caps->sq_ts_format == MLX5_HCA_CAP_TIMESTAMP_FORMAT_FR) 696 attr.ts_format = MLX5_QPC_TIMESTAMP_FORMAT_FREE_RUNNING; 697 else 698 attr.ts_format = MLX5_QPC_TIMESTAMP_FORMAT_DEFAULT; 699 700 sq->obj = mlx5dr_cmd_sq_create(ctx->ibv_ctx, &attr); 701 if (!sq->obj) 702 return rte_errno; 703 704 sq->sqn = sq->obj->id; 705 706 err = mlx5dr_cmd_sq_modify_rdy(sq->obj); 707 if (err) 708 goto free_sq; 709 710 return 0; 711 712 free_sq: 713 mlx5dr_cmd_destroy_obj(sq->obj); 714 715 return err; 716 } 717 718 static int mlx5dr_send_ring_open_sq(struct mlx5dr_context *ctx, 719 struct mlx5dr_send_engine *queue, 720 struct mlx5dr_send_ring_sq *sq, 721 struct mlx5dr_send_ring_cq *cq) 722 { 723 size_t sq_log_buf_sz; 724 size_t buf_aligned; 725 size_t sq_buf_sz; 726 size_t page_size; 727 size_t buf_sz; 728 int err; 729 730 buf_sz = queue->num_entries * MAX_WQES_PER_RULE; 731 sq_log_buf_sz = log2above(buf_sz); 732 sq_buf_sz = 1 << (sq_log_buf_sz + log2above(MLX5_SEND_WQE_BB)); 733 734 page_size = sysconf(_SC_PAGESIZE); 735 buf_aligned = align(sq_buf_sz, page_size); 736 err = posix_memalign((void **)&sq->buf, page_size, buf_aligned); 737 if (err) { 738 rte_errno = ENOMEM; 739 return err; 740 } 741 memset(sq->buf, 0, buf_aligned); 742 743 err = posix_memalign((void **)&sq->db, 8, 8); 744 if (err) 745 goto free_buf; 746 747 sq->buf_umem = mlx5_glue->devx_umem_reg(ctx->ibv_ctx, sq->buf, sq_buf_sz, 0); 748 749 if (!sq->buf_umem) { 750 err = errno; 751 goto free_db; 752 } 753 754 sq->db_umem = mlx5_glue->devx_umem_reg(ctx->ibv_ctx, sq->db, 8, 0); 755 if (!sq->db_umem) { 756 err = errno; 757 goto free_buf_umem; 758 } 759 760 err = mlx5dr_send_ring_create_sq_obj(ctx, queue, sq, cq, sq_log_buf_sz); 761 762 if (err) 763 goto free_db_umem; 764 765 sq->wr_priv = simple_malloc(sizeof(*sq->wr_priv) * buf_sz); 766 if (!sq->wr_priv) { 767 err = ENOMEM; 768 goto destroy_sq_obj; 769 } 770 771 sq->dep_wqe = simple_calloc(queue->num_entries, sizeof(*sq->dep_wqe)); 772 if (!sq->dep_wqe) { 773 err = ENOMEM; 774 goto destroy_wr_priv; 775 } 776 777 sq->buf_mask = buf_sz - 1; 778 779 return 0; 780 781 destroy_wr_priv: 782 simple_free(sq->wr_priv); 783 destroy_sq_obj: 784 mlx5dr_cmd_destroy_obj(sq->obj); 785 free_db_umem: 786 mlx5_glue->devx_umem_dereg(sq->db_umem); 787 free_buf_umem: 788 mlx5_glue->devx_umem_dereg(sq->buf_umem); 789 free_db: 790 free(sq->db); 791 free_buf: 792 free(sq->buf); 793 rte_errno = err; 794 return err; 795 } 796 797 static void mlx5dr_send_ring_close_sq(struct mlx5dr_send_ring_sq *sq) 798 { 799 simple_free(sq->dep_wqe); 800 mlx5dr_cmd_destroy_obj(sq->obj); 801 mlx5_glue->devx_umem_dereg(sq->db_umem); 802 mlx5_glue->devx_umem_dereg(sq->buf_umem); 803 simple_free(sq->wr_priv); 804 free(sq->db); 805 free(sq->buf); 806 } 807 808 static int mlx5dr_send_ring_open_cq(struct mlx5dr_context *ctx, 809 struct mlx5dr_send_engine *queue, 810 struct mlx5dr_send_ring_cq *cq) 811 { 812 struct mlx5dv_cq mlx5_cq = {0}; 813 struct mlx5dv_obj obj; 814 struct ibv_cq *ibv_cq; 815 size_t cq_size; 816 int err; 817 818 cq_size = queue->num_entries; 819 ibv_cq = mlx5_glue->create_cq(ctx->ibv_ctx, cq_size, NULL, NULL, 0); 820 if (!ibv_cq) { 821 DR_LOG(ERR, "Failed to create CQ"); 822 rte_errno = errno; 823 return rte_errno; 824 } 825 826 obj.cq.in = ibv_cq; 827 obj.cq.out = &mlx5_cq; 828 err = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ); 829 if (err) { 830 err = errno; 831 goto close_cq; 832 } 833 834 cq->buf = mlx5_cq.buf; 835 cq->db = mlx5_cq.dbrec; 836 cq->ncqe = mlx5_cq.cqe_cnt; 837 cq->cqe_sz = mlx5_cq.cqe_size; 838 cq->cqe_log_sz = log2above(cq->cqe_sz); 839 cq->ncqe_mask = cq->ncqe - 1; 840 cq->buf_sz = cq->cqe_sz * cq->ncqe; 841 cq->cqn = mlx5_cq.cqn; 842 cq->ibv_cq = ibv_cq; 843 844 return 0; 845 846 close_cq: 847 mlx5_glue->destroy_cq(ibv_cq); 848 rte_errno = err; 849 return err; 850 } 851 852 static void mlx5dr_send_ring_close_cq(struct mlx5dr_send_ring_cq *cq) 853 { 854 mlx5_glue->destroy_cq(cq->ibv_cq); 855 } 856 857 static void mlx5dr_send_ring_close(struct mlx5dr_send_ring *ring) 858 { 859 mlx5dr_send_ring_close_sq(&ring->send_sq); 860 mlx5dr_send_ring_close_cq(&ring->send_cq); 861 } 862 863 static int mlx5dr_send_ring_open(struct mlx5dr_context *ctx, 864 struct mlx5dr_send_engine *queue, 865 struct mlx5dr_send_ring *ring) 866 { 867 int err; 868 869 err = mlx5dr_send_ring_open_cq(ctx, queue, &ring->send_cq); 870 if (err) 871 return err; 872 873 err = mlx5dr_send_ring_open_sq(ctx, queue, &ring->send_sq, &ring->send_cq); 874 if (err) 875 goto close_cq; 876 877 return err; 878 879 close_cq: 880 mlx5dr_send_ring_close_cq(&ring->send_cq); 881 882 return err; 883 } 884 885 static void __mlx5dr_send_rings_close(struct mlx5dr_send_engine *queue, 886 uint16_t i) 887 { 888 while (i--) 889 mlx5dr_send_ring_close(&queue->send_ring[i]); 890 } 891 892 static void mlx5dr_send_rings_close(struct mlx5dr_send_engine *queue) 893 { 894 __mlx5dr_send_rings_close(queue, queue->rings); 895 } 896 897 static int mlx5dr_send_rings_open(struct mlx5dr_context *ctx, 898 struct mlx5dr_send_engine *queue) 899 { 900 uint16_t i; 901 int err; 902 903 for (i = 0; i < queue->rings; i++) { 904 err = mlx5dr_send_ring_open(ctx, queue, &queue->send_ring[i]); 905 if (err) 906 goto free_rings; 907 } 908 909 return 0; 910 911 free_rings: 912 __mlx5dr_send_rings_close(queue, i); 913 914 return err; 915 } 916 917 void mlx5dr_send_queue_close(struct mlx5dr_send_engine *queue) 918 { 919 mlx5dr_send_rings_close(queue); 920 simple_free(queue->completed.entries); 921 mlx5_glue->devx_free_uar(queue->uar); 922 } 923 924 int mlx5dr_send_queue_open(struct mlx5dr_context *ctx, 925 struct mlx5dr_send_engine *queue, 926 uint16_t queue_size) 927 { 928 struct mlx5dv_devx_uar *uar; 929 int err; 930 931 #ifdef MLX5DV_UAR_ALLOC_TYPE_NC 932 uar = mlx5_glue->devx_alloc_uar(ctx->ibv_ctx, MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC); 933 if (!uar) { 934 rte_errno = errno; 935 return rte_errno; 936 } 937 #else 938 uar = NULL; 939 rte_errno = ENOTSUP; 940 return rte_errno; 941 #endif 942 943 queue->uar = uar; 944 queue->rings = MLX5DR_NUM_SEND_RINGS; 945 queue->num_entries = roundup_pow_of_two(queue_size); 946 queue->used_entries = 0; 947 queue->th_entries = queue->num_entries; 948 949 queue->completed.entries = simple_calloc(queue->num_entries, 950 sizeof(queue->completed.entries[0])); 951 if (!queue->completed.entries) { 952 rte_errno = ENOMEM; 953 goto free_uar; 954 } 955 queue->completed.pi = 0; 956 queue->completed.ci = 0; 957 queue->completed.mask = queue->num_entries - 1; 958 959 err = mlx5dr_send_rings_open(ctx, queue); 960 if (err) 961 goto free_completed_entries; 962 963 return 0; 964 965 free_completed_entries: 966 simple_free(queue->completed.entries); 967 free_uar: 968 mlx5_glue->devx_free_uar(uar); 969 return rte_errno; 970 } 971 972 static void __mlx5dr_send_queues_close(struct mlx5dr_context *ctx, uint16_t queues) 973 { 974 while (queues--) 975 mlx5dr_send_queue_close(&ctx->send_queue[queues]); 976 } 977 978 static int mlx5dr_bwc_send_queues_init(struct mlx5dr_context *ctx) 979 { 980 int bwc_queues = ctx->queues - 1; 981 int i; 982 983 if (!mlx5dr_context_bwc_supported(ctx)) 984 return 0; 985 986 ctx->queues += bwc_queues; 987 988 ctx->bwc_send_queue_locks = simple_calloc(bwc_queues, 989 sizeof(*ctx->bwc_send_queue_locks)); 990 if (!ctx->bwc_send_queue_locks) { 991 rte_errno = ENOMEM; 992 return rte_errno; 993 } 994 995 for (i = 0; i < bwc_queues; i++) 996 rte_spinlock_init(&ctx->bwc_send_queue_locks[i]); 997 998 return 0; 999 } 1000 1001 static void mlx5dr_send_queues_bwc_locks_destroy(struct mlx5dr_context *ctx) 1002 { 1003 if (!mlx5dr_context_bwc_supported(ctx)) 1004 return; 1005 1006 simple_free(ctx->bwc_send_queue_locks); 1007 } 1008 1009 void mlx5dr_send_queues_close(struct mlx5dr_context *ctx) 1010 { 1011 __mlx5dr_send_queues_close(ctx, ctx->queues); 1012 simple_free(ctx->send_queue); 1013 mlx5dr_send_queues_bwc_locks_destroy(ctx); 1014 } 1015 1016 int mlx5dr_send_queues_open(struct mlx5dr_context *ctx, 1017 uint16_t queues, 1018 uint16_t queue_size) 1019 { 1020 int err = 0; 1021 uint32_t i; 1022 1023 /* Open one extra queue for control path */ 1024 ctx->queues = queues + 1; 1025 1026 /* open a separate set of queues and locks for bwc API */ 1027 err = mlx5dr_bwc_send_queues_init(ctx); 1028 if (err) 1029 return err; 1030 1031 ctx->send_queue = simple_calloc(ctx->queues, sizeof(*ctx->send_queue)); 1032 if (!ctx->send_queue) { 1033 rte_errno = ENOMEM; 1034 err = rte_errno; 1035 goto free_bwc_locks; 1036 } 1037 1038 for (i = 0; i < ctx->queues; i++) { 1039 err = mlx5dr_send_queue_open(ctx, &ctx->send_queue[i], queue_size); 1040 if (err) 1041 goto close_send_queues; 1042 } 1043 1044 return 0; 1045 1046 close_send_queues: 1047 __mlx5dr_send_queues_close(ctx, i); 1048 1049 simple_free(ctx->send_queue); 1050 1051 free_bwc_locks: 1052 mlx5dr_send_queues_bwc_locks_destroy(ctx); 1053 1054 return err; 1055 } 1056 1057 int mlx5dr_send_queue_action(struct mlx5dr_context *ctx, 1058 uint16_t queue_id, 1059 uint32_t actions) 1060 { 1061 struct mlx5dr_send_ring_sq *send_sq; 1062 struct mlx5dr_send_engine *queue; 1063 bool wait_comp = false; 1064 int64_t polled = 0; 1065 1066 queue = &ctx->send_queue[queue_id]; 1067 send_sq = &queue->send_ring->send_sq; 1068 1069 switch (actions) { 1070 case MLX5DR_SEND_QUEUE_ACTION_DRAIN_SYNC: 1071 wait_comp = true; 1072 /* FALLTHROUGH */ 1073 case MLX5DR_SEND_QUEUE_ACTION_DRAIN_ASYNC: 1074 if (send_sq->head_dep_idx != send_sq->tail_dep_idx) 1075 /* Send dependent WQEs to drain the queue */ 1076 mlx5dr_send_all_dep_wqe(queue); 1077 else 1078 /* Signal on the last posted WQE */ 1079 mlx5dr_send_engine_flush_queue(queue); 1080 1081 /* Poll queue until empty */ 1082 while (wait_comp && !mlx5dr_send_engine_empty(queue)) 1083 mlx5dr_send_engine_poll_cqs(queue, NULL, &polled, 0); 1084 1085 break; 1086 default: 1087 rte_errno = EINVAL; 1088 return -rte_errno; 1089 } 1090 1091 return 0; 1092 } 1093