1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <stddef.h> 6 #include <errno.h> 7 #include <stdbool.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <sys/queue.h> 11 12 #include <rte_malloc.h> 13 #include <rte_common.h> 14 #include <rte_eal_paging.h> 15 16 #include <mlx5_glue.h> 17 #include <mlx5_devx_cmds.h> 18 #include <mlx5_common_devx.h> 19 #include <mlx5_malloc.h> 20 21 #include "mlx5.h" 22 #include "mlx5_common_os.h" 23 #include "mlx5_tx.h" 24 #include "mlx5_rx.h" 25 #include "mlx5_utils.h" 26 #include "mlx5_devx.h" 27 #include "mlx5_flow.h" 28 #include "mlx5_flow_os.h" 29 30 /** 31 * Validate given external queue's port is valid or not. 32 * 33 * @param[in] port_id 34 * The port identifier of the Ethernet device. 35 * 36 * @return 37 * 0 on success, non-0 otherwise 38 */ 39 int 40 mlx5_devx_extq_port_validate(uint16_t port_id) 41 { 42 struct rte_eth_dev *dev; 43 struct mlx5_priv *priv; 44 45 if (rte_eth_dev_is_valid_port(port_id) < 0) { 46 DRV_LOG(ERR, "There is no Ethernet device for port %u.", 47 port_id); 48 rte_errno = ENODEV; 49 return -rte_errno; 50 } 51 dev = &rte_eth_devices[port_id]; 52 priv = dev->data->dev_private; 53 if (!mlx5_imported_pd_and_ctx(priv->sh->cdev)) { 54 DRV_LOG(ERR, "Port %u " 55 "external queue isn't supported on local PD and CTX.", 56 port_id); 57 rte_errno = ENOTSUP; 58 return -rte_errno; 59 } 60 if (!mlx5_devx_obj_ops_en(priv->sh)) { 61 DRV_LOG(ERR, 62 "Port %u external queue isn't supported by Verbs API.", 63 port_id); 64 rte_errno = ENOTSUP; 65 return -rte_errno; 66 } 67 return 0; 68 } 69 70 /** 71 * Modify RQ vlan stripping offload 72 * 73 * @param rxq 74 * Rx queue. 75 * @param on 76 * Enable/disable VLAN stripping. 77 * 78 * @return 79 * 0 on success, non-0 otherwise 80 */ 81 static int 82 mlx5_rxq_obj_modify_rq_vlan_strip(struct mlx5_rxq_priv *rxq, int on) 83 { 84 struct mlx5_devx_modify_rq_attr rq_attr; 85 86 memset(&rq_attr, 0, sizeof(rq_attr)); 87 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 88 rq_attr.state = MLX5_RQC_STATE_RDY; 89 rq_attr.vsd = (on ? 0 : 1); 90 rq_attr.modify_bitmask = MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD; 91 return mlx5_devx_cmd_modify_rq(rxq->devx_rq.rq, &rq_attr); 92 } 93 94 /** 95 * Modify RQ using DevX API. 96 * 97 * @param rxq 98 * DevX rx queue. 99 * @param type 100 * Type of change queue state. 101 * 102 * @return 103 * 0 on success, a negative errno value otherwise and rte_errno is set. 104 */ 105 int 106 mlx5_devx_modify_rq(struct mlx5_rxq_priv *rxq, uint8_t type) 107 { 108 struct mlx5_devx_modify_rq_attr rq_attr; 109 110 memset(&rq_attr, 0, sizeof(rq_attr)); 111 switch (type) { 112 case MLX5_RXQ_MOD_ERR2RST: 113 rq_attr.rq_state = MLX5_RQC_STATE_ERR; 114 rq_attr.state = MLX5_RQC_STATE_RST; 115 break; 116 case MLX5_RXQ_MOD_RST2RDY: 117 rq_attr.rq_state = MLX5_RQC_STATE_RST; 118 rq_attr.state = MLX5_RQC_STATE_RDY; 119 if (rxq->lwm) { 120 rq_attr.modify_bitmask |= 121 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_WQ_LWM; 122 rq_attr.lwm = rxq->lwm; 123 } 124 break; 125 case MLX5_RXQ_MOD_RDY2ERR: 126 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 127 rq_attr.state = MLX5_RQC_STATE_ERR; 128 break; 129 case MLX5_RXQ_MOD_RDY2RST: 130 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 131 rq_attr.state = MLX5_RQC_STATE_RST; 132 break; 133 case MLX5_RXQ_MOD_RDY2RDY: 134 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 135 rq_attr.state = MLX5_RQC_STATE_RDY; 136 rq_attr.modify_bitmask |= MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_WQ_LWM; 137 rq_attr.lwm = rxq->lwm; 138 break; 139 default: 140 break; 141 } 142 if (rxq->ctrl->is_hairpin) 143 return mlx5_devx_cmd_modify_rq(rxq->ctrl->obj->rq, &rq_attr); 144 return mlx5_devx_cmd_modify_rq(rxq->devx_rq.rq, &rq_attr); 145 } 146 147 /** 148 * Modify SQ using DevX API. 149 * 150 * @param txq_obj 151 * DevX Tx queue object. 152 * @param type 153 * Type of change queue state. 154 * @param dev_port 155 * Unnecessary. 156 * 157 * @return 158 * 0 on success, a negative errno value otherwise and rte_errno is set. 159 */ 160 int 161 mlx5_txq_devx_modify(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type, 162 uint8_t dev_port) 163 { 164 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 165 int ret; 166 167 if (type != MLX5_TXQ_MOD_RST2RDY) { 168 /* Change queue state to reset. */ 169 if (type == MLX5_TXQ_MOD_ERR2RDY) 170 msq_attr.sq_state = MLX5_SQC_STATE_ERR; 171 else 172 msq_attr.sq_state = MLX5_SQC_STATE_RDY; 173 msq_attr.state = MLX5_SQC_STATE_RST; 174 ret = mlx5_devx_cmd_modify_sq(obj->sq_obj.sq, &msq_attr); 175 if (ret) { 176 DRV_LOG(ERR, "Cannot change the Tx SQ state to RESET" 177 " %s", strerror(errno)); 178 rte_errno = errno; 179 return ret; 180 } 181 } 182 if (type != MLX5_TXQ_MOD_RDY2RST) { 183 /* Change queue state to ready. */ 184 msq_attr.sq_state = MLX5_SQC_STATE_RST; 185 msq_attr.state = MLX5_SQC_STATE_RDY; 186 ret = mlx5_devx_cmd_modify_sq(obj->sq_obj.sq, &msq_attr); 187 if (ret) { 188 DRV_LOG(ERR, "Cannot change the Tx SQ state to READY" 189 " %s", strerror(errno)); 190 rte_errno = errno; 191 return ret; 192 } 193 } 194 /* 195 * The dev_port variable is relevant only in Verbs API, and there is a 196 * pointer that points to this function and a parallel function in verbs 197 * intermittently, so they should have the same parameters. 198 */ 199 (void)dev_port; 200 return 0; 201 } 202 203 /** 204 * Release an Rx DevX queue object. 205 * 206 * @param rxq 207 * DevX Rx queue. 208 */ 209 static void 210 mlx5_rxq_devx_obj_release(struct mlx5_rxq_priv *rxq) 211 { 212 struct mlx5_rxq_obj *rxq_obj = rxq->ctrl->obj; 213 214 if (rxq_obj == NULL) 215 return; 216 if (rxq_obj->rxq_ctrl->is_hairpin) { 217 if (rxq_obj->rq == NULL) 218 return; 219 mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RDY2RST); 220 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 221 } else { 222 if (rxq->devx_rq.rq == NULL) 223 return; 224 mlx5_devx_rq_destroy(&rxq->devx_rq); 225 if (rxq->devx_rq.rmp != NULL && rxq->devx_rq.rmp->ref_cnt > 0) 226 return; 227 mlx5_devx_cq_destroy(&rxq_obj->cq_obj); 228 memset(&rxq_obj->cq_obj, 0, sizeof(rxq_obj->cq_obj)); 229 if (rxq_obj->devx_channel) { 230 mlx5_os_devx_destroy_event_channel 231 (rxq_obj->devx_channel); 232 rxq_obj->devx_channel = NULL; 233 } 234 } 235 rxq->ctrl->started = false; 236 } 237 238 /** 239 * Get event for an Rx DevX queue object. 240 * 241 * @param rxq_obj 242 * DevX Rx queue object. 243 * 244 * @return 245 * 0 on success, a negative errno value otherwise and rte_errno is set. 246 */ 247 static int 248 mlx5_rx_devx_get_event(struct mlx5_rxq_obj *rxq_obj) 249 { 250 #ifdef HAVE_IBV_DEVX_EVENT 251 union { 252 struct mlx5dv_devx_async_event_hdr event_resp; 253 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 254 } out; 255 int ret = mlx5_glue->devx_get_event(rxq_obj->devx_channel, 256 &out.event_resp, 257 sizeof(out.buf)); 258 259 if (ret < 0) { 260 rte_errno = errno; 261 return -rte_errno; 262 } 263 if (out.event_resp.cookie != (uint64_t)(uintptr_t)rxq_obj->cq_obj.cq) { 264 rte_errno = EINVAL; 265 return -rte_errno; 266 } 267 return 0; 268 #else 269 (void)rxq_obj; 270 rte_errno = ENOTSUP; 271 return -rte_errno; 272 #endif /* HAVE_IBV_DEVX_EVENT */ 273 } 274 275 /** 276 * Get LWM event for shared context, return the correct port/rxq for this event. 277 * 278 * @param priv 279 * Mlx5_priv object. 280 * @param rxq_idx [out] 281 * Which rxq gets this event. 282 * @param port_id [out] 283 * Which port gets this event. 284 * 285 * @return 286 * 0 on success, a negative errno value otherwise and rte_errno is set. 287 */ 288 static int 289 mlx5_rx_devx_get_event_lwm(struct mlx5_priv *priv, int *rxq_idx, int *port_id) 290 { 291 #ifdef HAVE_IBV_DEVX_EVENT 292 union { 293 struct mlx5dv_devx_async_event_hdr event_resp; 294 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 295 } out; 296 int ret; 297 298 memset(&out, 0, sizeof(out)); 299 ret = mlx5_glue->devx_get_event(priv->sh->devx_channel_lwm, 300 &out.event_resp, 301 sizeof(out.buf)); 302 if (ret < 0) { 303 rte_errno = errno; 304 DRV_LOG(WARNING, "%s err\n", __func__); 305 return -rte_errno; 306 } 307 *port_id = (((uint32_t)out.event_resp.cookie) >> 308 LWM_COOKIE_PORTID_OFFSET) & LWM_COOKIE_PORTID_MASK; 309 *rxq_idx = (((uint32_t)out.event_resp.cookie) >> 310 LWM_COOKIE_RXQID_OFFSET) & LWM_COOKIE_RXQID_MASK; 311 return 0; 312 #else 313 (void)priv; 314 (void)rxq_idx; 315 (void)port_id; 316 rte_errno = ENOTSUP; 317 return -rte_errno; 318 #endif /* HAVE_IBV_DEVX_EVENT */ 319 } 320 321 /** 322 * Create a RQ object using DevX. 323 * 324 * @param rxq 325 * Pointer to Rx queue. 326 * 327 * @return 328 * 0 on success, a negative errno value otherwise and rte_errno is set. 329 */ 330 static int 331 mlx5_rxq_create_devx_rq_resources(struct mlx5_rxq_priv *rxq) 332 { 333 struct mlx5_priv *priv = rxq->priv; 334 struct mlx5_common_device *cdev = priv->sh->cdev; 335 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 336 struct mlx5_rxq_data *rxq_data = &rxq->ctrl->rxq; 337 struct mlx5_devx_create_rq_attr rq_attr = { 0 }; 338 uint16_t log_desc_n = rxq_data->elts_n - rxq_data->sges_n; 339 uint32_t wqe_size, log_wqe_size; 340 341 /* Fill RQ attributes. */ 342 rq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE; 343 rq_attr.flush_in_error_en = 1; 344 rq_attr.vsd = (rxq_data->vlan_strip) ? 0 : 1; 345 rq_attr.cqn = rxq_ctrl->obj->cq_obj.cq->id; 346 rq_attr.scatter_fcs = (rxq_data->crc_present) ? 1 : 0; 347 rq_attr.ts_format = 348 mlx5_ts_format_conv(cdev->config.hca_attr.rq_ts_format); 349 /* Fill WQ attributes for this RQ. */ 350 if (mlx5_rxq_mprq_enabled(rxq_data)) { 351 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ; 352 /* 353 * Number of strides in each WQE: 354 * 512*2^single_wqe_log_num_of_strides. 355 */ 356 rq_attr.wq_attr.single_wqe_log_num_of_strides = 357 rxq_data->log_strd_num - 358 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES; 359 /* Stride size = (2^single_stride_log_num_of_bytes)*64B. */ 360 rq_attr.wq_attr.single_stride_log_num_of_bytes = 361 rxq_data->log_strd_sz - 362 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES; 363 wqe_size = sizeof(struct mlx5_wqe_mprq); 364 } else { 365 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 366 wqe_size = sizeof(struct mlx5_wqe_data_seg); 367 } 368 log_wqe_size = log2above(wqe_size) + rxq_data->sges_n; 369 wqe_size = 1 << log_wqe_size; /* round up power of two.*/ 370 rq_attr.wq_attr.log_wq_stride = log_wqe_size; 371 rq_attr.wq_attr.log_wq_sz = log_desc_n; 372 rq_attr.wq_attr.end_padding_mode = priv->config.hw_padding ? 373 MLX5_WQ_END_PAD_MODE_ALIGN : 374 MLX5_WQ_END_PAD_MODE_NONE; 375 rq_attr.wq_attr.pd = cdev->pdn; 376 rq_attr.counter_set_id = priv->counter_set_id; 377 rq_attr.delay_drop_en = rxq_data->delay_drop; 378 rq_attr.user_index = rte_cpu_to_be_16(priv->dev_data->port_id); 379 if (rxq_data->shared) /* Create RMP based RQ. */ 380 rxq->devx_rq.rmp = &rxq_ctrl->obj->devx_rmp; 381 /* Create RQ using DevX API. */ 382 return mlx5_devx_rq_create(cdev->ctx, &rxq->devx_rq, wqe_size, 383 log_desc_n, &rq_attr, rxq_ctrl->socket); 384 } 385 386 /** 387 * Create a DevX CQ object for an Rx queue. 388 * 389 * @param rxq 390 * Pointer to Rx queue. 391 * 392 * @return 393 * 0 on success, a negative errno value otherwise and rte_errno is set. 394 */ 395 static int 396 mlx5_rxq_create_devx_cq_resources(struct mlx5_rxq_priv *rxq) 397 { 398 struct mlx5_devx_cq *cq_obj = 0; 399 struct mlx5_devx_cq_attr cq_attr = { 0 }; 400 struct mlx5_priv *priv = rxq->priv; 401 struct mlx5_dev_ctx_shared *sh = priv->sh; 402 uint16_t port_id = priv->dev_data->port_id; 403 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 404 struct mlx5_rxq_data *rxq_data = &rxq_ctrl->rxq; 405 unsigned int cqe_n = mlx5_rxq_cqe_num(rxq_data); 406 uint32_t log_cqe_n; 407 uint16_t event_nums[1] = { 0 }; 408 int ret = 0; 409 410 if (rxq_ctrl->started) 411 return 0; 412 if (priv->config.cqe_comp && !rxq_data->hw_timestamp && 413 !rxq_data->lro) { 414 cq_attr.cqe_comp_en = 1u; 415 cq_attr.cqe_comp_layout = priv->config.enh_cqe_comp; 416 rxq_data->cqe_comp_layout = cq_attr.cqe_comp_layout; 417 rxq_data->mcqe_format = priv->config.cqe_comp_fmt; 418 rxq_data->byte_mask = UINT32_MAX; 419 switch (priv->config.cqe_comp_fmt) { 420 case MLX5_CQE_RESP_FORMAT_HASH: 421 /* fallthrough */ 422 case MLX5_CQE_RESP_FORMAT_CSUM: 423 /* 424 * Select CSUM miniCQE format only for non-vectorized 425 * MPRQ Rx burst, use HASH miniCQE format for others. 426 */ 427 if (mlx5_rxq_check_vec_support(rxq_data) < 0 && 428 mlx5_rxq_mprq_enabled(rxq_data)) 429 cq_attr.mini_cqe_res_format = 430 MLX5_CQE_RESP_FORMAT_CSUM_STRIDX; 431 else 432 cq_attr.mini_cqe_res_format = 433 MLX5_CQE_RESP_FORMAT_HASH; 434 rxq_data->mcqe_format = cq_attr.mini_cqe_res_format; 435 break; 436 case MLX5_CQE_RESP_FORMAT_FTAG_STRIDX: 437 rxq_data->byte_mask = MLX5_LEN_WITH_MARK_MASK; 438 /* fallthrough */ 439 case MLX5_CQE_RESP_FORMAT_CSUM_STRIDX: 440 cq_attr.mini_cqe_res_format = priv->config.cqe_comp_fmt; 441 break; 442 case MLX5_CQE_RESP_FORMAT_L34H_STRIDX: 443 cq_attr.mini_cqe_res_format = 0; 444 cq_attr.mini_cqe_res_format_ext = 1; 445 break; 446 } 447 DRV_LOG(DEBUG, 448 "Port %u Rx CQE compression is enabled, format %d.", 449 port_id, priv->config.cqe_comp_fmt); 450 /* 451 * For vectorized Rx, it must not be doubled in order to 452 * make cq_ci and rq_ci aligned. 453 */ 454 if (mlx5_rxq_check_vec_support(rxq_data) < 0) 455 cqe_n *= 2; 456 } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) { 457 DRV_LOG(DEBUG, 458 "Port %u Rx CQE compression is disabled for HW timestamp.", 459 port_id); 460 } else if (priv->config.cqe_comp && rxq_data->lro) { 461 DRV_LOG(DEBUG, 462 "Port %u Rx CQE compression is disabled for LRO.", 463 port_id); 464 } 465 cq_attr.uar_page_id = mlx5_os_get_devx_uar_page_id(sh->rx_uar.obj); 466 log_cqe_n = log2above(cqe_n); 467 /* Create CQ using DevX API. */ 468 ret = mlx5_devx_cq_create(sh->cdev->ctx, &rxq_ctrl->obj->cq_obj, 469 log_cqe_n, &cq_attr, sh->numa_node); 470 if (ret) 471 return ret; 472 cq_obj = &rxq_ctrl->obj->cq_obj; 473 rxq_data->cqes = (volatile struct mlx5_cqe (*)[]) 474 (uintptr_t)cq_obj->cqes; 475 rxq_data->cq_db = cq_obj->db_rec; 476 rxq_data->uar_data = sh->rx_uar.cq_db; 477 rxq_data->cqe_n = log_cqe_n; 478 rxq_data->cqn = cq_obj->cq->id; 479 rxq_data->cq_ci = 0; 480 if (rxq_ctrl->obj->devx_channel) { 481 ret = mlx5_os_devx_subscribe_devx_event 482 (rxq_ctrl->obj->devx_channel, 483 cq_obj->cq->obj, 484 sizeof(event_nums), 485 event_nums, 486 (uint64_t)(uintptr_t)cq_obj->cq); 487 if (ret) { 488 DRV_LOG(ERR, "Fail to subscribe CQ to event channel."); 489 ret = errno; 490 mlx5_devx_cq_destroy(cq_obj); 491 memset(cq_obj, 0, sizeof(*cq_obj)); 492 rte_errno = ret; 493 return -ret; 494 } 495 } 496 return 0; 497 } 498 499 /** 500 * Create the Rx hairpin queue object. 501 * 502 * @param rxq 503 * Pointer to Rx queue. 504 * 505 * @return 506 * 0 on success, a negative errno value otherwise and rte_errno is set. 507 */ 508 static int 509 mlx5_rxq_obj_hairpin_new(struct mlx5_rxq_priv *rxq) 510 { 511 uint16_t idx = rxq->idx; 512 struct mlx5_priv *priv = rxq->priv; 513 struct mlx5_hca_attr *hca_attr __rte_unused = &priv->sh->cdev->config.hca_attr; 514 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 515 struct mlx5_devx_create_rq_attr unlocked_attr = { 0 }; 516 struct mlx5_devx_create_rq_attr locked_attr = { 0 }; 517 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 518 uint32_t max_wq_data; 519 520 MLX5_ASSERT(rxq != NULL && rxq->ctrl != NULL && tmpl != NULL); 521 tmpl->rxq_ctrl = rxq_ctrl; 522 unlocked_attr.hairpin = 1; 523 max_wq_data = 524 priv->sh->cdev->config.hca_attr.log_max_hairpin_wq_data_sz; 525 /* Jumbo frames > 9KB should be supported, and more packets. */ 526 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 527 if (priv->config.log_hp_size > max_wq_data) { 528 DRV_LOG(ERR, "Total data size %u power of 2 is " 529 "too large for hairpin.", 530 priv->config.log_hp_size); 531 rte_errno = ERANGE; 532 return -rte_errno; 533 } 534 unlocked_attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 535 } else { 536 unlocked_attr.wq_attr.log_hairpin_data_sz = 537 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 538 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 539 } 540 /* Set the packets number to the maximum value for performance. */ 541 unlocked_attr.wq_attr.log_hairpin_num_packets = 542 unlocked_attr.wq_attr.log_hairpin_data_sz - 543 MLX5_HAIRPIN_QUEUE_STRIDE; 544 unlocked_attr.counter_set_id = priv->counter_set_id; 545 rxq_ctrl->rxq.delay_drop = priv->config.hp_delay_drop; 546 unlocked_attr.delay_drop_en = priv->config.hp_delay_drop; 547 unlocked_attr.hairpin_data_buffer_type = 548 MLX5_RQC_HAIRPIN_DATA_BUFFER_TYPE_UNLOCKED_INTERNAL_BUFFER; 549 if (rxq->hairpin_conf.use_locked_device_memory) { 550 /* 551 * It is assumed that configuration is verified against capabilities 552 * during queue setup. 553 */ 554 MLX5_ASSERT(hca_attr->hairpin_data_buffer_locked); 555 rte_memcpy(&locked_attr, &unlocked_attr, sizeof(locked_attr)); 556 locked_attr.hairpin_data_buffer_type = 557 MLX5_RQC_HAIRPIN_DATA_BUFFER_TYPE_LOCKED_INTERNAL_BUFFER; 558 tmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->cdev->ctx, &locked_attr, 559 rxq_ctrl->socket); 560 if (!tmpl->rq && rxq->hairpin_conf.force_memory) { 561 DRV_LOG(ERR, "Port %u Rx hairpin queue %u can't create RQ object" 562 " with locked memory buffer", 563 priv->dev_data->port_id, idx); 564 return -rte_errno; 565 } else if (!tmpl->rq && !rxq->hairpin_conf.force_memory) { 566 DRV_LOG(WARNING, "Port %u Rx hairpin queue %u can't create RQ object" 567 " with locked memory buffer. Falling back to unlocked" 568 " device memory.", 569 priv->dev_data->port_id, idx); 570 rte_errno = 0; 571 goto create_rq_unlocked; 572 } 573 goto create_rq_set_state; 574 } 575 576 create_rq_unlocked: 577 tmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->cdev->ctx, &unlocked_attr, 578 rxq_ctrl->socket); 579 if (!tmpl->rq) { 580 DRV_LOG(ERR, 581 "Port %u Rx hairpin queue %u can't create rq object.", 582 priv->dev_data->port_id, idx); 583 rte_errno = errno; 584 return -rte_errno; 585 } 586 create_rq_set_state: 587 priv->dev_data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN; 588 return 0; 589 } 590 591 /** 592 * Create the Rx queue DevX object. 593 * 594 * @param rxq 595 * Pointer to Rx queue. 596 * 597 * @return 598 * 0 on success, a negative errno value otherwise and rte_errno is set. 599 */ 600 static int 601 mlx5_rxq_devx_obj_new(struct mlx5_rxq_priv *rxq) 602 { 603 struct mlx5_priv *priv = rxq->priv; 604 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 605 struct mlx5_rxq_data *rxq_data = &rxq_ctrl->rxq; 606 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 607 int ret = 0; 608 609 MLX5_ASSERT(rxq_data); 610 MLX5_ASSERT(tmpl); 611 if (rxq_ctrl->is_hairpin) 612 return mlx5_rxq_obj_hairpin_new(rxq); 613 tmpl->rxq_ctrl = rxq_ctrl; 614 if (rxq_ctrl->irq && !rxq_ctrl->started) { 615 int devx_ev_flag = 616 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA; 617 618 tmpl->devx_channel = mlx5_os_devx_create_event_channel 619 (priv->sh->cdev->ctx, 620 devx_ev_flag); 621 if (!tmpl->devx_channel) { 622 rte_errno = errno; 623 DRV_LOG(ERR, "Failed to create event channel %d.", 624 rte_errno); 625 goto error; 626 } 627 tmpl->fd = mlx5_os_get_devx_channel_fd(tmpl->devx_channel); 628 } 629 /* Create CQ using DevX API. */ 630 ret = mlx5_rxq_create_devx_cq_resources(rxq); 631 if (ret) { 632 DRV_LOG(ERR, "Failed to create CQ."); 633 goto error; 634 } 635 rxq_data->delay_drop = priv->config.std_delay_drop; 636 /* Create RQ using DevX API. */ 637 ret = mlx5_rxq_create_devx_rq_resources(rxq); 638 if (ret) { 639 DRV_LOG(ERR, "Port %u Rx queue %u RQ creation failure.", 640 priv->dev_data->port_id, rxq->idx); 641 rte_errno = ENOMEM; 642 goto error; 643 } 644 /* Change queue state to ready. */ 645 ret = mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RST2RDY); 646 if (ret) 647 goto error; 648 if (!rxq_data->shared) { 649 rxq_data->wqes = (void *)(uintptr_t)rxq->devx_rq.wq.umem_buf; 650 rxq_data->rq_db = (uint32_t *)(uintptr_t)rxq->devx_rq.wq.db_rec; 651 } else if (!rxq_ctrl->started) { 652 rxq_data->wqes = (void *)(uintptr_t)tmpl->devx_rmp.wq.umem_buf; 653 rxq_data->rq_db = 654 (uint32_t *)(uintptr_t)tmpl->devx_rmp.wq.db_rec; 655 } 656 if (!rxq_ctrl->started) { 657 mlx5_rxq_initialize(rxq_data); 658 rxq_ctrl->wqn = rxq->devx_rq.rq->id; 659 } 660 priv->dev_data->rx_queue_state[rxq->idx] = RTE_ETH_QUEUE_STATE_STARTED; 661 return 0; 662 error: 663 ret = rte_errno; /* Save rte_errno before cleanup. */ 664 mlx5_rxq_devx_obj_release(rxq); 665 rte_errno = ret; /* Restore rte_errno. */ 666 return -rte_errno; 667 } 668 669 /** 670 * Prepare RQT attribute structure for DevX RQT API. 671 * 672 * @param dev 673 * Pointer to Ethernet device. 674 * @param log_n 675 * Log of number of queues in the array. 676 * @param queues 677 * List of RX queue indices or NULL, in which case 678 * the attribute will be filled by drop queue ID. 679 * @param queues_n 680 * Size of @p queues array or 0 if it is NULL. 681 * @param ind_tbl 682 * DevX indirection table object. 683 * 684 * @return 685 * The RQT attr object initialized, NULL otherwise and rte_errno is set. 686 */ 687 static struct mlx5_devx_rqt_attr * 688 mlx5_devx_ind_table_create_rqt_attr(struct rte_eth_dev *dev, 689 const unsigned int log_n, 690 const uint16_t *queues, 691 const uint32_t queues_n) 692 { 693 struct mlx5_priv *priv = dev->data->dev_private; 694 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 695 const unsigned int rqt_n = 1 << log_n; 696 unsigned int i, j; 697 698 rqt_attr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rqt_attr) + 699 rqt_n * sizeof(uint32_t), 0, SOCKET_ID_ANY); 700 if (!rqt_attr) { 701 DRV_LOG(ERR, "Port %u cannot allocate RQT resources.", 702 dev->data->port_id); 703 rte_errno = ENOMEM; 704 return NULL; 705 } 706 rqt_attr->rqt_max_size = priv->sh->dev_cap.ind_table_max_size; 707 rqt_attr->rqt_actual_size = rqt_n; 708 if (queues == NULL) { 709 for (i = 0; i < rqt_n; i++) 710 rqt_attr->rq_list[i] = 711 priv->drop_queue.rxq->devx_rq.rq->id; 712 return rqt_attr; 713 } 714 for (i = 0; i != queues_n; ++i) { 715 if (mlx5_is_external_rxq(dev, queues[i])) { 716 struct mlx5_external_q *ext_rxq = 717 mlx5_ext_rxq_get(dev, queues[i]); 718 719 rqt_attr->rq_list[i] = ext_rxq->hw_id; 720 } else { 721 struct mlx5_rxq_priv *rxq = 722 mlx5_rxq_get(dev, queues[i]); 723 724 MLX5_ASSERT(rxq != NULL); 725 if (rxq->ctrl->is_hairpin) 726 rqt_attr->rq_list[i] = rxq->ctrl->obj->rq->id; 727 else 728 rqt_attr->rq_list[i] = rxq->devx_rq.rq->id; 729 } 730 } 731 MLX5_ASSERT(i > 0); 732 for (j = 0; i != rqt_n; ++j, ++i) 733 rqt_attr->rq_list[i] = rqt_attr->rq_list[j]; 734 return rqt_attr; 735 } 736 737 /** 738 * Create RQT using DevX API as a filed of indirection table. 739 * 740 * @param dev 741 * Pointer to Ethernet device. 742 * @param log_n 743 * Log of number of queues in the array. 744 * @param ind_tbl 745 * DevX indirection table object. 746 * 747 * @return 748 * 0 on success, a negative errno value otherwise and rte_errno is set. 749 */ 750 static int 751 mlx5_devx_ind_table_new(struct rte_eth_dev *dev, const unsigned int log_n, 752 struct mlx5_ind_table_obj *ind_tbl) 753 { 754 struct mlx5_priv *priv = dev->data->dev_private; 755 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 756 const uint16_t *queues = dev->data->dev_started ? ind_tbl->queues : 757 NULL; 758 759 MLX5_ASSERT(ind_tbl); 760 rqt_attr = mlx5_devx_ind_table_create_rqt_attr(dev, log_n, queues, 761 ind_tbl->queues_n); 762 if (!rqt_attr) 763 return -rte_errno; 764 ind_tbl->rqt = mlx5_devx_cmd_create_rqt(priv->sh->cdev->ctx, rqt_attr); 765 mlx5_free(rqt_attr); 766 if (!ind_tbl->rqt) { 767 DRV_LOG(ERR, "Port %u cannot create DevX RQT.", 768 dev->data->port_id); 769 rte_errno = errno; 770 return -rte_errno; 771 } 772 return 0; 773 } 774 775 /** 776 * Modify RQT using DevX API as a filed of indirection table. 777 * 778 * @param dev 779 * Pointer to Ethernet device. 780 * @param log_n 781 * Log of number of queues in the array. 782 * @param ind_tbl 783 * DevX indirection table object. 784 * 785 * @return 786 * 0 on success, a negative errno value otherwise and rte_errno is set. 787 */ 788 static int 789 mlx5_devx_ind_table_modify(struct rte_eth_dev *dev, const unsigned int log_n, 790 const uint16_t *queues, const uint32_t queues_n, 791 struct mlx5_ind_table_obj *ind_tbl) 792 { 793 int ret = 0; 794 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 795 796 MLX5_ASSERT(ind_tbl); 797 rqt_attr = mlx5_devx_ind_table_create_rqt_attr(dev, log_n, 798 queues, 799 queues_n); 800 if (!rqt_attr) 801 return -rte_errno; 802 ret = mlx5_devx_cmd_modify_rqt(ind_tbl->rqt, rqt_attr); 803 mlx5_free(rqt_attr); 804 if (ret) 805 DRV_LOG(ERR, "Port %u cannot modify DevX RQT.", 806 dev->data->port_id); 807 return ret; 808 } 809 810 /** 811 * Destroy the DevX RQT object. 812 * 813 * @param ind_table 814 * Indirection table to release. 815 */ 816 static void 817 mlx5_devx_ind_table_destroy(struct mlx5_ind_table_obj *ind_tbl) 818 { 819 claim_zero(mlx5_devx_cmd_destroy(ind_tbl->rqt)); 820 } 821 822 /** 823 * Set TIR attribute struct with relevant input values. 824 * 825 * @param[in] dev 826 * Pointer to Ethernet device. 827 * @param[in] rss_key 828 * RSS key for the Rx hash queue. 829 * @param[in] hash_fields 830 * Verbs protocol hash field to make the RSS on. 831 * @param[in] ind_tbl 832 * Indirection table for TIR. If table queues array is NULL, 833 * a TIR for drop queue is assumed. 834 * @param[in] tunnel 835 * Tunnel type. 836 * @param[out] tir_attr 837 * Parameters structure for TIR creation/modification. 838 * 839 * @return 840 * The Verbs/DevX object initialised index, 0 otherwise and rte_errno is set. 841 */ 842 static void 843 mlx5_devx_tir_attr_set(struct rte_eth_dev *dev, const uint8_t *rss_key, 844 uint64_t hash_fields, 845 const struct mlx5_ind_table_obj *ind_tbl, 846 int tunnel, bool symmetric_hash_function, 847 struct mlx5_devx_tir_attr *tir_attr) 848 { 849 struct mlx5_priv *priv = dev->data->dev_private; 850 bool is_hairpin; 851 bool lro = false; 852 uint32_t i; 853 854 /* NULL queues designate drop queue. */ 855 if (ind_tbl->queues == NULL) { 856 is_hairpin = priv->drop_queue.rxq->ctrl->is_hairpin; 857 } else if (mlx5_is_external_rxq(dev, ind_tbl->queues[0])) { 858 /* External RxQ supports neither Hairpin nor LRO. */ 859 is_hairpin = false; 860 } else { 861 is_hairpin = mlx5_rxq_is_hairpin(dev, ind_tbl->queues[0]); 862 lro = true; 863 /* Enable TIR LRO only if all the queues were configured for. */ 864 for (i = 0; i < ind_tbl->queues_n; ++i) { 865 struct mlx5_rxq_data *rxq_i = 866 mlx5_rxq_data_get(dev, ind_tbl->queues[i]); 867 868 if (rxq_i != NULL && !rxq_i->lro) { 869 lro = false; 870 break; 871 } 872 } 873 } 874 memset(tir_attr, 0, sizeof(*tir_attr)); 875 tir_attr->disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT; 876 tir_attr->rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ; 877 tir_attr->tunneled_offload_en = !!tunnel; 878 tir_attr->rx_hash_symmetric = symmetric_hash_function; 879 /* If needed, translate hash_fields bitmap to PRM format. */ 880 if (hash_fields) { 881 struct mlx5_rx_hash_field_select *rx_hash_field_select = 882 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 883 hash_fields & IBV_RX_HASH_INNER ? 884 &tir_attr->rx_hash_field_selector_inner : 885 #endif 886 &tir_attr->rx_hash_field_selector_outer; 887 /* 1 bit: 0: IPv4, 1: IPv6. */ 888 rx_hash_field_select->l3_prot_type = 889 !!(hash_fields & MLX5_IPV6_IBV_RX_HASH); 890 /* 1 bit: 0: TCP, 1: UDP. */ 891 rx_hash_field_select->l4_prot_type = 892 !!(hash_fields & MLX5_UDP_IBV_RX_HASH); 893 /* Bitmask which sets which fields to use in RX Hash. */ 894 rx_hash_field_select->selected_fields = 895 ((!!(hash_fields & MLX5_L3_SRC_IBV_RX_HASH)) << 896 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) | 897 (!!(hash_fields & MLX5_L3_DST_IBV_RX_HASH)) << 898 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP | 899 (!!(hash_fields & MLX5_L4_SRC_IBV_RX_HASH)) << 900 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT | 901 (!!(hash_fields & MLX5_L4_DST_IBV_RX_HASH)) << 902 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT | 903 (!!(hash_fields & IBV_RX_HASH_IPSEC_SPI)) << 904 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_IPSEC_SPI; 905 } 906 if (is_hairpin) 907 tir_attr->transport_domain = priv->sh->td->id; 908 else 909 tir_attr->transport_domain = priv->sh->tdn; 910 memcpy(tir_attr->rx_hash_toeplitz_key, rss_key, MLX5_RSS_HASH_KEY_LEN); 911 tir_attr->indirect_table = ind_tbl->rqt->id; 912 if (dev->data->dev_conf.lpbk_mode) 913 tir_attr->self_lb_block = MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST; 914 if (lro) { 915 MLX5_ASSERT(priv->sh->config.lro_allowed); 916 tir_attr->lro_timeout_period_usecs = priv->config.lro_timeout; 917 tir_attr->lro_max_msg_sz = 918 priv->max_lro_msg_size / MLX5_LRO_SEG_CHUNK_SIZE; 919 tir_attr->lro_enable_mask = 920 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | 921 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO; 922 } 923 } 924 925 /** 926 * Create an Rx Hash queue. 927 * 928 * @param dev 929 * Pointer to Ethernet device. 930 * @param hrxq 931 * Pointer to Rx Hash queue. 932 * @param tunnel 933 * Tunnel type. 934 * 935 * @return 936 * 0 on success, a negative errno value otherwise and rte_errno is set. 937 */ 938 static int 939 mlx5_devx_hrxq_new(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq, 940 int tunnel __rte_unused) 941 { 942 struct mlx5_priv *priv = dev->data->dev_private; 943 struct mlx5_devx_tir_attr tir_attr = {0}; 944 int err; 945 946 mlx5_devx_tir_attr_set(dev, hrxq->rss_key, hrxq->hash_fields, 947 hrxq->ind_table, tunnel, hrxq->symmetric_hash_function, 948 &tir_attr); 949 hrxq->tir = mlx5_devx_cmd_create_tir(priv->sh->cdev->ctx, &tir_attr); 950 if (!hrxq->tir) { 951 DRV_LOG(ERR, "Port %u cannot create DevX TIR.", 952 dev->data->port_id); 953 rte_errno = errno; 954 goto error; 955 } 956 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 957 #ifdef HAVE_MLX5_HWS_SUPPORT 958 if (hrxq->hws_flags) { 959 hrxq->action = mlx5dr_action_create_dest_tir 960 (priv->dr_ctx, 961 (struct mlx5dr_devx_obj *)hrxq->tir, hrxq->hws_flags, true); 962 if (!hrxq->action) 963 goto error; 964 return 0; 965 } 966 #endif 967 if (mlx5_flow_os_create_flow_action_dest_devx_tir(hrxq->tir, 968 &hrxq->action)) { 969 rte_errno = errno; 970 goto error; 971 } 972 #endif 973 return 0; 974 error: 975 err = rte_errno; /* Save rte_errno before cleanup. */ 976 if (hrxq->tir) 977 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 978 rte_errno = err; /* Restore rte_errno. */ 979 return -rte_errno; 980 } 981 982 /** 983 * Destroy a DevX TIR object. 984 * 985 * @param hrxq 986 * Hash Rx queue to release its tir. 987 */ 988 static void 989 mlx5_devx_tir_destroy(struct mlx5_hrxq *hrxq) 990 { 991 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 992 } 993 994 /** 995 * Modify an Rx Hash queue configuration. 996 * 997 * @param dev 998 * Pointer to Ethernet device. 999 * @param hrxq 1000 * Hash Rx queue to modify. 1001 * @param rss_key 1002 * RSS key for the Rx hash queue. 1003 * @param hash_fields 1004 * Verbs protocol hash field to make the RSS on. 1005 * @param[in] ind_tbl 1006 * Indirection table for TIR. 1007 * 1008 * @return 1009 * 0 on success, a negative errno value otherwise and rte_errno is set. 1010 */ 1011 static int 1012 mlx5_devx_hrxq_modify(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq, 1013 const uint8_t *rss_key, 1014 uint64_t hash_fields, 1015 bool symmetric_hash_function, 1016 const struct mlx5_ind_table_obj *ind_tbl) 1017 { 1018 struct mlx5_devx_modify_tir_attr modify_tir = {0}; 1019 1020 /* 1021 * untested for modification fields: 1022 * - rx_hash_fn set hard-coded in hrxq_new(), 1023 * - lro_xxx not set after rxq setup 1024 */ 1025 if (ind_tbl != hrxq->ind_table) 1026 modify_tir.modify_bitmask |= 1027 MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_INDIRECT_TABLE; 1028 if (hash_fields != hrxq->hash_fields || 1029 symmetric_hash_function != hrxq->symmetric_hash_function || 1030 memcmp(hrxq->rss_key, rss_key, MLX5_RSS_HASH_KEY_LEN)) 1031 modify_tir.modify_bitmask |= 1032 MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_HASH; 1033 mlx5_devx_tir_attr_set(dev, rss_key, hash_fields, ind_tbl, 1034 0, /* N/A - tunnel modification unsupported */ 1035 symmetric_hash_function, 1036 &modify_tir.tir); 1037 modify_tir.tirn = hrxq->tir->id; 1038 if (mlx5_devx_cmd_modify_tir(hrxq->tir, &modify_tir)) { 1039 DRV_LOG(ERR, "port %u cannot modify DevX TIR", 1040 dev->data->port_id); 1041 rte_errno = errno; 1042 return -rte_errno; 1043 } 1044 return 0; 1045 } 1046 1047 /** 1048 * Create a DevX drop Rx queue. 1049 * 1050 * @param dev 1051 * Pointer to Ethernet device. 1052 * 1053 * @return 1054 * 0 on success, a negative errno value otherwise and rte_errno is set. 1055 */ 1056 static int 1057 mlx5_rxq_devx_obj_drop_create(struct rte_eth_dev *dev) 1058 { 1059 struct mlx5_priv *priv = dev->data->dev_private; 1060 int socket_id = dev->device->numa_node; 1061 struct mlx5_rxq_priv *rxq; 1062 struct mlx5_rxq_ctrl *rxq_ctrl = NULL; 1063 struct mlx5_rxq_obj *rxq_obj = NULL; 1064 int ret; 1065 1066 /* 1067 * Initialize dummy control structures. 1068 * They are required to hold pointers for cleanup 1069 * and are only accessible via drop queue DevX objects. 1070 */ 1071 rxq = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq), 0, socket_id); 1072 if (rxq == NULL) { 1073 DRV_LOG(ERR, "Port %u could not allocate drop queue private", 1074 dev->data->port_id); 1075 rte_errno = ENOMEM; 1076 goto error; 1077 } 1078 rxq_ctrl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq_ctrl), 1079 0, socket_id); 1080 if (rxq_ctrl == NULL) { 1081 DRV_LOG(ERR, "Port %u could not allocate drop queue control", 1082 dev->data->port_id); 1083 rte_errno = ENOMEM; 1084 goto error; 1085 } 1086 rxq_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq_obj), 0, socket_id); 1087 if (rxq_obj == NULL) { 1088 DRV_LOG(ERR, "Port %u could not allocate drop queue object", 1089 dev->data->port_id); 1090 rte_errno = ENOMEM; 1091 goto error; 1092 } 1093 /* set the CPU socket ID where the rxq_ctrl was allocated */ 1094 rxq_ctrl->socket = socket_id; 1095 rxq_obj->rxq_ctrl = rxq_ctrl; 1096 rxq_ctrl->is_hairpin = false; 1097 rxq_ctrl->sh = priv->sh; 1098 rxq_ctrl->obj = rxq_obj; 1099 rxq->ctrl = rxq_ctrl; 1100 rxq->priv = priv; 1101 LIST_INSERT_HEAD(&rxq_ctrl->owners, rxq, owner_entry); 1102 /* Create CQ using DevX API. */ 1103 ret = mlx5_rxq_create_devx_cq_resources(rxq); 1104 if (ret != 0) { 1105 DRV_LOG(ERR, "Port %u drop queue CQ creation failed.", 1106 dev->data->port_id); 1107 goto error; 1108 } 1109 rxq_ctrl->rxq.delay_drop = 0; 1110 /* Create RQ using DevX API. */ 1111 ret = mlx5_rxq_create_devx_rq_resources(rxq); 1112 if (ret != 0) { 1113 DRV_LOG(ERR, "Port %u drop queue RQ creation failed.", 1114 dev->data->port_id); 1115 rte_errno = ENOMEM; 1116 goto error; 1117 } 1118 /* Change queue state to ready. */ 1119 ret = mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RST2RDY); 1120 if (ret != 0) 1121 goto error; 1122 /* Initialize drop queue. */ 1123 priv->drop_queue.rxq = rxq; 1124 return 0; 1125 error: 1126 ret = rte_errno; /* Save rte_errno before cleanup. */ 1127 if (rxq != NULL && rxq->devx_rq.rq != NULL) 1128 mlx5_devx_rq_destroy(&rxq->devx_rq); 1129 if (rxq_obj != NULL) { 1130 if (rxq_obj->cq_obj.cq != NULL) 1131 mlx5_devx_cq_destroy(&rxq_obj->cq_obj); 1132 if (rxq_obj->devx_channel) 1133 mlx5_os_devx_destroy_event_channel 1134 (rxq_obj->devx_channel); 1135 mlx5_free(rxq_obj); 1136 } 1137 if (rxq_ctrl != NULL) 1138 mlx5_free(rxq_ctrl); 1139 if (rxq != NULL) 1140 mlx5_free(rxq); 1141 rte_errno = ret; /* Restore rte_errno. */ 1142 return -rte_errno; 1143 } 1144 1145 /** 1146 * Release drop Rx queue resources. 1147 * 1148 * @param dev 1149 * Pointer to Ethernet device. 1150 */ 1151 static void 1152 mlx5_rxq_devx_obj_drop_release(struct rte_eth_dev *dev) 1153 { 1154 struct mlx5_priv *priv = dev->data->dev_private; 1155 struct mlx5_rxq_priv *rxq = priv->drop_queue.rxq; 1156 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 1157 1158 mlx5_rxq_devx_obj_release(rxq); 1159 mlx5_free(rxq_ctrl->obj); 1160 mlx5_free(rxq_ctrl); 1161 mlx5_free(rxq); 1162 priv->drop_queue.rxq = NULL; 1163 } 1164 1165 /** 1166 * Release a drop hash Rx queue. 1167 * 1168 * @param dev 1169 * Pointer to Ethernet device. 1170 */ 1171 static void 1172 mlx5_devx_drop_action_destroy(struct rte_eth_dev *dev) 1173 { 1174 struct mlx5_priv *priv = dev->data->dev_private; 1175 struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq; 1176 1177 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 1178 if (hrxq->action != NULL) 1179 mlx5_flow_os_destroy_flow_action(hrxq->action); 1180 #endif 1181 if (hrxq->tir != NULL) 1182 mlx5_devx_tir_destroy(hrxq); 1183 if (hrxq->ind_table->ind_table != NULL) 1184 mlx5_devx_ind_table_destroy(hrxq->ind_table); 1185 if (priv->drop_queue.rxq->devx_rq.rq != NULL) 1186 mlx5_rxq_devx_obj_drop_release(dev); 1187 } 1188 1189 /** 1190 * Create a DevX drop action for Rx Hash queue. 1191 * 1192 * @param dev 1193 * Pointer to Ethernet device. 1194 * 1195 * @return 1196 * 0 on success, a negative errno value otherwise and rte_errno is set. 1197 */ 1198 static int 1199 mlx5_devx_drop_action_create(struct rte_eth_dev *dev) 1200 { 1201 struct mlx5_priv *priv = dev->data->dev_private; 1202 struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq; 1203 int ret; 1204 1205 ret = mlx5_rxq_devx_obj_drop_create(dev); 1206 if (ret != 0) { 1207 DRV_LOG(ERR, "Cannot create drop RX queue"); 1208 return ret; 1209 } 1210 if (priv->sh->config.dv_flow_en == 2) 1211 return 0; 1212 /* hrxq->ind_table queues are NULL, drop RX queue ID will be used */ 1213 ret = mlx5_devx_ind_table_new(dev, 0, hrxq->ind_table); 1214 if (ret != 0) { 1215 DRV_LOG(ERR, "Cannot create drop hash RX queue indirection table"); 1216 goto error; 1217 } 1218 ret = mlx5_devx_hrxq_new(dev, hrxq, /* tunnel */ false); 1219 if (ret != 0) { 1220 DRV_LOG(ERR, "Cannot create drop hash RX queue"); 1221 goto error; 1222 } 1223 return 0; 1224 error: 1225 mlx5_devx_drop_action_destroy(dev); 1226 return ret; 1227 } 1228 1229 /** 1230 * Select TXQ TIS number. 1231 * 1232 * @param dev 1233 * Pointer to Ethernet device. 1234 * @param queue_idx 1235 * Queue index in DPDK Tx queue array. 1236 * 1237 * @return 1238 * > 0 on success, a negative errno value otherwise. 1239 */ 1240 static uint32_t 1241 mlx5_get_txq_tis_num(struct rte_eth_dev *dev, uint16_t queue_idx) 1242 { 1243 struct mlx5_priv *priv = dev->data->dev_private; 1244 struct mlx5_txq_data *txq_data = (*priv->txqs)[queue_idx]; 1245 int tis_idx = 0; 1246 1247 if (priv->sh->bond.n_port) { 1248 if (txq_data->tx_aggr_affinity) { 1249 tis_idx = txq_data->tx_aggr_affinity; 1250 } else if (priv->sh->lag.affinity_mode == MLX5_LAG_MODE_TIS) { 1251 tis_idx = (priv->lag_affinity_idx + queue_idx) % 1252 priv->sh->bond.n_port + 1; 1253 DRV_LOG(INFO, "port %d txq %d gets affinity %d and maps to PF %d.", 1254 dev->data->port_id, queue_idx, tis_idx, 1255 priv->sh->lag.tx_remap_affinity[tis_idx - 1]); 1256 } 1257 } 1258 MLX5_ASSERT(priv->sh->tis[tis_idx]); 1259 return priv->sh->tis[tis_idx]->id; 1260 } 1261 1262 /** 1263 * Create the Tx hairpin queue object. 1264 * 1265 * @param dev 1266 * Pointer to Ethernet device. 1267 * @param idx 1268 * Queue index in DPDK Tx queue array. 1269 * 1270 * @return 1271 * 0 on success, a negative errno value otherwise and rte_errno is set. 1272 */ 1273 static int 1274 mlx5_txq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) 1275 { 1276 struct mlx5_priv *priv = dev->data->dev_private; 1277 struct mlx5_hca_attr *hca_attr = &priv->sh->cdev->config.hca_attr; 1278 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1279 struct mlx5_txq_ctrl *txq_ctrl = 1280 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1281 struct mlx5_devx_create_sq_attr dev_mem_attr = { 0 }; 1282 struct mlx5_devx_create_sq_attr host_mem_attr = { 0 }; 1283 struct mlx5_txq_obj *tmpl = txq_ctrl->obj; 1284 void *umem_buf = NULL; 1285 void *umem_obj = NULL; 1286 uint32_t max_wq_data; 1287 1288 MLX5_ASSERT(txq_data); 1289 MLX5_ASSERT(tmpl); 1290 tmpl->txq_ctrl = txq_ctrl; 1291 dev_mem_attr.hairpin = 1; 1292 dev_mem_attr.tis_lst_sz = 1; 1293 dev_mem_attr.tis_num = mlx5_get_txq_tis_num(dev, idx); 1294 max_wq_data = 1295 priv->sh->cdev->config.hca_attr.log_max_hairpin_wq_data_sz; 1296 /* Jumbo frames > 9KB should be supported, and more packets. */ 1297 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 1298 if (priv->config.log_hp_size > max_wq_data) { 1299 DRV_LOG(ERR, "Total data size %u power of 2 is " 1300 "too large for hairpin.", 1301 priv->config.log_hp_size); 1302 rte_errno = ERANGE; 1303 return -rte_errno; 1304 } 1305 dev_mem_attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 1306 } else { 1307 dev_mem_attr.wq_attr.log_hairpin_data_sz = 1308 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 1309 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 1310 } 1311 /* Set the packets number to the maximum value for performance. */ 1312 dev_mem_attr.wq_attr.log_hairpin_num_packets = 1313 dev_mem_attr.wq_attr.log_hairpin_data_sz - 1314 MLX5_HAIRPIN_QUEUE_STRIDE; 1315 dev_mem_attr.hairpin_wq_buffer_type = MLX5_SQC_HAIRPIN_WQ_BUFFER_TYPE_INTERNAL_BUFFER; 1316 if (txq_ctrl->hairpin_conf.use_rte_memory) { 1317 uint32_t umem_size; 1318 uint32_t umem_dbrec; 1319 size_t alignment = MLX5_WQE_BUF_ALIGNMENT; 1320 1321 if (alignment == (size_t)-1) { 1322 DRV_LOG(ERR, "Failed to get WQE buf alignment."); 1323 rte_errno = ENOMEM; 1324 return -rte_errno; 1325 } 1326 /* 1327 * It is assumed that configuration is verified against capabilities 1328 * during queue setup. 1329 */ 1330 MLX5_ASSERT(hca_attr->hairpin_sq_wq_in_host_mem); 1331 MLX5_ASSERT(hca_attr->hairpin_sq_wqe_bb_size > 0); 1332 rte_memcpy(&host_mem_attr, &dev_mem_attr, sizeof(host_mem_attr)); 1333 umem_size = MLX5_WQE_SIZE * 1334 RTE_BIT32(host_mem_attr.wq_attr.log_hairpin_num_packets); 1335 umem_dbrec = RTE_ALIGN(umem_size, MLX5_DBR_SIZE); 1336 umem_size += MLX5_DBR_SIZE; 1337 umem_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, umem_size, 1338 alignment, priv->sh->numa_node); 1339 if (umem_buf == NULL && txq_ctrl->hairpin_conf.force_memory) { 1340 DRV_LOG(ERR, "Failed to allocate memory for hairpin TX queue"); 1341 rte_errno = ENOMEM; 1342 return -rte_errno; 1343 } else if (umem_buf == NULL && !txq_ctrl->hairpin_conf.force_memory) { 1344 DRV_LOG(WARNING, "Failed to allocate memory for hairpin TX queue." 1345 " Falling back to TX queue located on the device."); 1346 goto create_sq_on_device; 1347 } 1348 umem_obj = mlx5_os_umem_reg(priv->sh->cdev->ctx, 1349 (void *)(uintptr_t)umem_buf, 1350 umem_size, 1351 IBV_ACCESS_LOCAL_WRITE); 1352 if (umem_obj == NULL && txq_ctrl->hairpin_conf.force_memory) { 1353 DRV_LOG(ERR, "Failed to register UMEM for hairpin TX queue"); 1354 mlx5_free(umem_buf); 1355 return -rte_errno; 1356 } else if (umem_obj == NULL && !txq_ctrl->hairpin_conf.force_memory) { 1357 DRV_LOG(WARNING, "Failed to register UMEM for hairpin TX queue." 1358 " Falling back to TX queue located on the device."); 1359 rte_errno = 0; 1360 mlx5_free(umem_buf); 1361 goto create_sq_on_device; 1362 } 1363 host_mem_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 1364 host_mem_attr.wq_attr.wq_umem_valid = 1; 1365 host_mem_attr.wq_attr.wq_umem_id = mlx5_os_get_umem_id(umem_obj); 1366 host_mem_attr.wq_attr.wq_umem_offset = 0; 1367 host_mem_attr.wq_attr.dbr_umem_valid = 1; 1368 host_mem_attr.wq_attr.dbr_umem_id = host_mem_attr.wq_attr.wq_umem_id; 1369 host_mem_attr.wq_attr.dbr_addr = umem_dbrec; 1370 host_mem_attr.wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE); 1371 host_mem_attr.wq_attr.log_wq_sz = 1372 host_mem_attr.wq_attr.log_hairpin_num_packets * 1373 hca_attr->hairpin_sq_wqe_bb_size; 1374 host_mem_attr.wq_attr.log_wq_pg_sz = MLX5_LOG_PAGE_SIZE; 1375 host_mem_attr.hairpin_wq_buffer_type = MLX5_SQC_HAIRPIN_WQ_BUFFER_TYPE_HOST_MEMORY; 1376 tmpl->sq = mlx5_devx_cmd_create_sq(priv->sh->cdev->ctx, &host_mem_attr); 1377 if (!tmpl->sq && txq_ctrl->hairpin_conf.force_memory) { 1378 DRV_LOG(ERR, 1379 "Port %u tx hairpin queue %u can't create SQ object.", 1380 dev->data->port_id, idx); 1381 claim_zero(mlx5_os_umem_dereg(umem_obj)); 1382 mlx5_free(umem_buf); 1383 return -rte_errno; 1384 } else if (!tmpl->sq && !txq_ctrl->hairpin_conf.force_memory) { 1385 DRV_LOG(WARNING, 1386 "Port %u tx hairpin queue %u failed to allocate SQ object" 1387 " using host memory. Falling back to TX queue located" 1388 " on the device", 1389 dev->data->port_id, idx); 1390 rte_errno = 0; 1391 claim_zero(mlx5_os_umem_dereg(umem_obj)); 1392 mlx5_free(umem_buf); 1393 goto create_sq_on_device; 1394 } 1395 tmpl->umem_buf_wq_buffer = umem_buf; 1396 tmpl->umem_obj_wq_buffer = umem_obj; 1397 return 0; 1398 } 1399 1400 create_sq_on_device: 1401 tmpl->sq = mlx5_devx_cmd_create_sq(priv->sh->cdev->ctx, &dev_mem_attr); 1402 if (!tmpl->sq) { 1403 DRV_LOG(ERR, 1404 "Port %u tx hairpin queue %u can't create SQ object.", 1405 dev->data->port_id, idx); 1406 rte_errno = errno; 1407 return -rte_errno; 1408 } 1409 return 0; 1410 } 1411 1412 #if defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) || !defined(HAVE_INFINIBAND_VERBS_H) 1413 /** 1414 * Destroy the Tx queue DevX object. 1415 * 1416 * @param txq_obj 1417 * Txq object to destroy. 1418 */ 1419 static void 1420 mlx5_txq_release_devx_resources(struct mlx5_txq_obj *txq_obj) 1421 { 1422 mlx5_devx_sq_destroy(&txq_obj->sq_obj); 1423 memset(&txq_obj->sq_obj, 0, sizeof(txq_obj->sq_obj)); 1424 mlx5_devx_cq_destroy(&txq_obj->cq_obj); 1425 memset(&txq_obj->cq_obj, 0, sizeof(txq_obj->cq_obj)); 1426 } 1427 1428 /** 1429 * Create a SQ object and its resources using DevX. 1430 * 1431 * @param dev 1432 * Pointer to Ethernet device. 1433 * @param idx 1434 * Queue index in DPDK Tx queue array. 1435 * @param[in] log_desc_n 1436 * Log of number of descriptors in queue. 1437 * 1438 * @return 1439 * 0 on success, a negative errno value otherwise and rte_errno is set. 1440 */ 1441 static int 1442 mlx5_txq_create_devx_sq_resources(struct rte_eth_dev *dev, uint16_t idx, 1443 uint16_t log_desc_n) 1444 { 1445 struct mlx5_priv *priv = dev->data->dev_private; 1446 struct mlx5_common_device *cdev = priv->sh->cdev; 1447 struct mlx5_uar *uar = &priv->sh->tx_uar; 1448 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1449 struct mlx5_txq_ctrl *txq_ctrl = 1450 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1451 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1452 struct mlx5_devx_create_sq_attr sq_attr = { 1453 .flush_in_error_en = 1, 1454 .allow_multi_pkt_send_wqe = !!priv->config.mps, 1455 .min_wqe_inline_mode = cdev->config.hca_attr.vport_inline_mode, 1456 .allow_swp = !!priv->sh->dev_cap.swp, 1457 .cqn = txq_obj->cq_obj.cq->id, 1458 .tis_lst_sz = 1, 1459 .wq_attr = (struct mlx5_devx_wq_attr){ 1460 .pd = cdev->pdn, 1461 .uar_page = mlx5_os_get_devx_uar_page_id(uar->obj), 1462 }, 1463 .ts_format = 1464 mlx5_ts_format_conv(cdev->config.hca_attr.sq_ts_format), 1465 .tis_num = mlx5_get_txq_tis_num(dev, idx), 1466 }; 1467 1468 /* Create Send Queue object with DevX. */ 1469 return mlx5_devx_sq_create(cdev->ctx, &txq_obj->sq_obj, 1470 log_desc_n, &sq_attr, priv->sh->numa_node); 1471 } 1472 #endif 1473 1474 /** 1475 * Create the Tx queue DevX object. 1476 * 1477 * @param dev 1478 * Pointer to Ethernet device. 1479 * @param idx 1480 * Queue index in DPDK Tx queue array. 1481 * 1482 * @return 1483 * 0 on success, a negative errno value otherwise and rte_errno is set. 1484 */ 1485 int 1486 mlx5_txq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx) 1487 { 1488 struct mlx5_priv *priv = dev->data->dev_private; 1489 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1490 struct mlx5_txq_ctrl *txq_ctrl = 1491 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1492 1493 if (txq_ctrl->is_hairpin) 1494 return mlx5_txq_obj_hairpin_new(dev, idx); 1495 #if !defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) && defined(HAVE_INFINIBAND_VERBS_H) 1496 DRV_LOG(ERR, "Port %u Tx queue %u cannot create with DevX, no UAR.", 1497 dev->data->port_id, idx); 1498 rte_errno = ENOMEM; 1499 return -rte_errno; 1500 #else 1501 struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv)); 1502 struct mlx5_dev_ctx_shared *sh = priv->sh; 1503 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1504 struct mlx5_devx_cq_attr cq_attr = { 1505 .uar_page_id = mlx5_os_get_devx_uar_page_id(sh->tx_uar.obj), 1506 }; 1507 uint32_t cqe_n, log_desc_n; 1508 uint32_t wqe_n, wqe_size; 1509 int ret = 0; 1510 1511 MLX5_ASSERT(txq_data); 1512 MLX5_ASSERT(txq_obj); 1513 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1514 MLX5_ASSERT(ppriv); 1515 txq_obj->txq_ctrl = txq_ctrl; 1516 txq_obj->dev = dev; 1517 if (__rte_trace_point_fp_is_enabled() && 1518 txq_data->offloads & RTE_ETH_TX_OFFLOAD_SEND_ON_TIMESTAMP) 1519 cqe_n = UINT16_MAX / 2 - 1; 1520 else 1521 cqe_n = (1UL << txq_data->elts_n) / MLX5_TX_COMP_THRESH + 1522 1 + MLX5_TX_COMP_THRESH_INLINE_DIV; 1523 log_desc_n = log2above(cqe_n); 1524 cqe_n = 1UL << log_desc_n; 1525 if (cqe_n > UINT16_MAX) { 1526 DRV_LOG(ERR, "Port %u Tx queue %u requests to many CQEs %u.", 1527 dev->data->port_id, txq_data->idx, cqe_n); 1528 rte_errno = EINVAL; 1529 return 0; 1530 } 1531 /* Create completion queue object with DevX. */ 1532 ret = mlx5_devx_cq_create(sh->cdev->ctx, &txq_obj->cq_obj, log_desc_n, 1533 &cq_attr, priv->sh->numa_node); 1534 if (ret) { 1535 DRV_LOG(ERR, "Port %u Tx queue %u CQ creation failure.", 1536 dev->data->port_id, idx); 1537 goto error; 1538 } 1539 txq_data->cqe_n = log_desc_n; 1540 txq_data->cqe_s = cqe_n; 1541 txq_data->cqe_m = txq_data->cqe_s - 1; 1542 txq_data->cqes = txq_obj->cq_obj.cqes; 1543 txq_data->cq_ci = 0; 1544 txq_data->cq_pi = 0; 1545 txq_data->cq_db = txq_obj->cq_obj.db_rec; 1546 *txq_data->cq_db = 0; 1547 /* 1548 * Adjust the amount of WQEs depending on inline settings. 1549 * The number of descriptors should be enough to handle 1550 * the specified number of packets. If queue is being created 1551 * with Verbs the rdma-core does queue size adjustment 1552 * internally in the mlx5_calc_sq_size(), we do the same 1553 * for the queue being created with DevX at this point. 1554 */ 1555 wqe_size = txq_data->tso_en ? 1556 RTE_ALIGN(txq_ctrl->max_tso_header, MLX5_WSEG_SIZE) : 0; 1557 wqe_size += sizeof(struct mlx5_wqe_cseg) + 1558 sizeof(struct mlx5_wqe_eseg) + 1559 sizeof(struct mlx5_wqe_dseg); 1560 if (txq_data->inlen_send) 1561 wqe_size = RTE_MAX(wqe_size, sizeof(struct mlx5_wqe_cseg) + 1562 sizeof(struct mlx5_wqe_eseg) + 1563 RTE_ALIGN(txq_data->inlen_send + 1564 sizeof(uint32_t), 1565 MLX5_WSEG_SIZE)); 1566 wqe_size = RTE_ALIGN(wqe_size, MLX5_WQE_SIZE) / MLX5_WQE_SIZE; 1567 /* Create Send Queue object with DevX. */ 1568 wqe_n = RTE_MIN((1UL << txq_data->elts_n) * wqe_size, 1569 (uint32_t)priv->sh->dev_cap.max_qp_wr); 1570 log_desc_n = log2above(wqe_n); 1571 ret = mlx5_txq_create_devx_sq_resources(dev, idx, log_desc_n); 1572 if (ret) { 1573 DRV_LOG(ERR, "Port %u Tx queue %u SQ creation failure.", 1574 dev->data->port_id, idx); 1575 rte_errno = errno; 1576 goto error; 1577 } 1578 /* Create the Work Queue. */ 1579 txq_data->wqe_n = log_desc_n; 1580 txq_data->wqe_s = 1 << txq_data->wqe_n; 1581 txq_data->wqe_m = txq_data->wqe_s - 1; 1582 txq_data->wqes = (struct mlx5_wqe *)(uintptr_t)txq_obj->sq_obj.wqes; 1583 txq_data->wqes_end = txq_data->wqes + txq_data->wqe_s; 1584 txq_data->wqe_ci = 0; 1585 txq_data->wqe_pi = 0; 1586 txq_data->wqe_comp = 0; 1587 txq_data->wqe_thres = txq_data->wqe_s / MLX5_TX_COMP_THRESH_INLINE_DIV; 1588 txq_data->qp_db = &txq_obj->sq_obj.db_rec[MLX5_SND_DBR]; 1589 *txq_data->qp_db = 0; 1590 txq_data->qp_num_8s = txq_obj->sq_obj.sq->id << 8; 1591 txq_data->db_heu = sh->cdev->config.dbnc == MLX5_SQ_DB_HEURISTIC; 1592 txq_data->db_nc = sh->tx_uar.dbnc; 1593 txq_data->wait_on_time = !!(!sh->config.tx_pp && 1594 sh->cdev->config.hca_attr.wait_on_time); 1595 /* Change Send Queue state to Ready-to-Send. */ 1596 ret = mlx5_txq_devx_modify(txq_obj, MLX5_TXQ_MOD_RST2RDY, 0); 1597 if (ret) { 1598 rte_errno = errno; 1599 DRV_LOG(ERR, 1600 "Port %u Tx queue %u SQ state to SQC_STATE_RDY failed.", 1601 dev->data->port_id, idx); 1602 goto error; 1603 } 1604 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 1605 /* 1606 * If using DevX need to query and store TIS transport domain value. 1607 * This is done once per port. 1608 * Will use this value on Rx, when creating matching TIR. 1609 */ 1610 if (!priv->sh->tdn) 1611 priv->sh->tdn = priv->sh->td->id; 1612 #endif 1613 txq_ctrl->uar_mmap_offset = 1614 mlx5_os_get_devx_uar_mmap_offset(sh->tx_uar.obj); 1615 ppriv->uar_table[txq_data->idx] = sh->tx_uar.bf_db; 1616 dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED; 1617 return 0; 1618 error: 1619 ret = rte_errno; /* Save rte_errno before cleanup. */ 1620 mlx5_txq_release_devx_resources(txq_obj); 1621 rte_errno = ret; /* Restore rte_errno. */ 1622 return -rte_errno; 1623 #endif 1624 } 1625 1626 /** 1627 * Release an Tx DevX queue object. 1628 * 1629 * @param txq_obj 1630 * DevX Tx queue object. 1631 */ 1632 void 1633 mlx5_txq_devx_obj_release(struct mlx5_txq_obj *txq_obj) 1634 { 1635 MLX5_ASSERT(txq_obj); 1636 if (txq_obj->txq_ctrl->is_hairpin) { 1637 if (txq_obj->sq) { 1638 claim_zero(mlx5_devx_cmd_destroy(txq_obj->sq)); 1639 txq_obj->sq = NULL; 1640 } 1641 if (txq_obj->tis) 1642 claim_zero(mlx5_devx_cmd_destroy(txq_obj->tis)); 1643 if (txq_obj->umem_obj_wq_buffer) { 1644 claim_zero(mlx5_os_umem_dereg(txq_obj->umem_obj_wq_buffer)); 1645 txq_obj->umem_obj_wq_buffer = NULL; 1646 } 1647 if (txq_obj->umem_buf_wq_buffer) { 1648 mlx5_free(txq_obj->umem_buf_wq_buffer); 1649 txq_obj->umem_buf_wq_buffer = NULL; 1650 } 1651 #if defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) || !defined(HAVE_INFINIBAND_VERBS_H) 1652 } else { 1653 mlx5_txq_release_devx_resources(txq_obj); 1654 #endif 1655 } 1656 } 1657 1658 struct mlx5_obj_ops devx_obj_ops = { 1659 .rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_rq_vlan_strip, 1660 .rxq_obj_new = mlx5_rxq_devx_obj_new, 1661 .rxq_event_get = mlx5_rx_devx_get_event, 1662 .rxq_obj_modify = mlx5_devx_modify_rq, 1663 .rxq_obj_release = mlx5_rxq_devx_obj_release, 1664 .rxq_event_get_lwm = mlx5_rx_devx_get_event_lwm, 1665 .ind_table_new = mlx5_devx_ind_table_new, 1666 .ind_table_modify = mlx5_devx_ind_table_modify, 1667 .ind_table_destroy = mlx5_devx_ind_table_destroy, 1668 .hrxq_new = mlx5_devx_hrxq_new, 1669 .hrxq_destroy = mlx5_devx_tir_destroy, 1670 .hrxq_modify = mlx5_devx_hrxq_modify, 1671 .drop_action_create = mlx5_devx_drop_action_create, 1672 .drop_action_destroy = mlx5_devx_drop_action_destroy, 1673 .txq_obj_new = mlx5_txq_devx_obj_new, 1674 .txq_obj_modify = mlx5_txq_devx_modify, 1675 .txq_obj_release = mlx5_txq_devx_obj_release, 1676 .lb_dummy_queue_create = NULL, 1677 .lb_dummy_queue_release = NULL, 1678 }; 1679