1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <stddef.h> 6 #include <errno.h> 7 #include <stdbool.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <sys/queue.h> 11 12 #include <rte_malloc.h> 13 #include <rte_common.h> 14 #include <rte_eal_paging.h> 15 16 #include <mlx5_glue.h> 17 #include <mlx5_devx_cmds.h> 18 #include <mlx5_common_devx.h> 19 #include <mlx5_malloc.h> 20 21 #include "mlx5.h" 22 #include "mlx5_common_os.h" 23 #include "mlx5_tx.h" 24 #include "mlx5_rx.h" 25 #include "mlx5_utils.h" 26 #include "mlx5_devx.h" 27 #include "mlx5_flow.h" 28 #include "mlx5_flow_os.h" 29 30 /** 31 * Modify RQ vlan stripping offload 32 * 33 * @param rxq 34 * Rx queue. 35 * @param on 36 * Enable/disable VLAN stripping. 37 * 38 * @return 39 * 0 on success, non-0 otherwise 40 */ 41 static int 42 mlx5_rxq_obj_modify_rq_vlan_strip(struct mlx5_rxq_priv *rxq, int on) 43 { 44 struct mlx5_devx_modify_rq_attr rq_attr; 45 46 memset(&rq_attr, 0, sizeof(rq_attr)); 47 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 48 rq_attr.state = MLX5_RQC_STATE_RDY; 49 rq_attr.vsd = (on ? 0 : 1); 50 rq_attr.modify_bitmask = MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD; 51 return mlx5_devx_cmd_modify_rq(rxq->devx_rq.rq, &rq_attr); 52 } 53 54 /** 55 * Modify RQ using DevX API. 56 * 57 * @param rxq 58 * DevX rx queue. 59 * @param type 60 * Type of change queue state. 61 * 62 * @return 63 * 0 on success, a negative errno value otherwise and rte_errno is set. 64 */ 65 static int 66 mlx5_devx_modify_rq(struct mlx5_rxq_priv *rxq, uint8_t type) 67 { 68 struct mlx5_devx_modify_rq_attr rq_attr; 69 70 memset(&rq_attr, 0, sizeof(rq_attr)); 71 switch (type) { 72 case MLX5_RXQ_MOD_ERR2RST: 73 rq_attr.rq_state = MLX5_RQC_STATE_ERR; 74 rq_attr.state = MLX5_RQC_STATE_RST; 75 break; 76 case MLX5_RXQ_MOD_RST2RDY: 77 rq_attr.rq_state = MLX5_RQC_STATE_RST; 78 rq_attr.state = MLX5_RQC_STATE_RDY; 79 break; 80 case MLX5_RXQ_MOD_RDY2ERR: 81 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 82 rq_attr.state = MLX5_RQC_STATE_ERR; 83 break; 84 case MLX5_RXQ_MOD_RDY2RST: 85 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 86 rq_attr.state = MLX5_RQC_STATE_RST; 87 break; 88 default: 89 break; 90 } 91 return mlx5_devx_cmd_modify_rq(rxq->devx_rq.rq, &rq_attr); 92 } 93 94 /** 95 * Modify SQ using DevX API. 96 * 97 * @param txq_obj 98 * DevX Tx queue object. 99 * @param type 100 * Type of change queue state. 101 * @param dev_port 102 * Unnecessary. 103 * 104 * @return 105 * 0 on success, a negative errno value otherwise and rte_errno is set. 106 */ 107 int 108 mlx5_txq_devx_modify(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type, 109 uint8_t dev_port) 110 { 111 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 112 int ret; 113 114 if (type != MLX5_TXQ_MOD_RST2RDY) { 115 /* Change queue state to reset. */ 116 if (type == MLX5_TXQ_MOD_ERR2RDY) 117 msq_attr.sq_state = MLX5_SQC_STATE_ERR; 118 else 119 msq_attr.sq_state = MLX5_SQC_STATE_RDY; 120 msq_attr.state = MLX5_SQC_STATE_RST; 121 ret = mlx5_devx_cmd_modify_sq(obj->sq_obj.sq, &msq_attr); 122 if (ret) { 123 DRV_LOG(ERR, "Cannot change the Tx SQ state to RESET" 124 " %s", strerror(errno)); 125 rte_errno = errno; 126 return ret; 127 } 128 } 129 if (type != MLX5_TXQ_MOD_RDY2RST) { 130 /* Change queue state to ready. */ 131 msq_attr.sq_state = MLX5_SQC_STATE_RST; 132 msq_attr.state = MLX5_SQC_STATE_RDY; 133 ret = mlx5_devx_cmd_modify_sq(obj->sq_obj.sq, &msq_attr); 134 if (ret) { 135 DRV_LOG(ERR, "Cannot change the Tx SQ state to READY" 136 " %s", strerror(errno)); 137 rte_errno = errno; 138 return ret; 139 } 140 } 141 /* 142 * The dev_port variable is relevant only in Verbs API, and there is a 143 * pointer that points to this function and a parallel function in verbs 144 * intermittently, so they should have the same parameters. 145 */ 146 (void)dev_port; 147 return 0; 148 } 149 150 /** 151 * Release an Rx DevX queue object. 152 * 153 * @param rxq 154 * DevX Rx queue. 155 */ 156 static void 157 mlx5_rxq_devx_obj_release(struct mlx5_rxq_priv *rxq) 158 { 159 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 160 struct mlx5_rxq_obj *rxq_obj = rxq_ctrl->obj; 161 162 MLX5_ASSERT(rxq != NULL); 163 MLX5_ASSERT(rxq_ctrl != NULL); 164 if (rxq_obj->rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) { 165 MLX5_ASSERT(rxq_obj->rq); 166 mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RDY2RST); 167 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 168 } else { 169 mlx5_devx_rq_destroy(&rxq->devx_rq); 170 memset(&rxq->devx_rq, 0, sizeof(rxq->devx_rq)); 171 mlx5_devx_cq_destroy(&rxq_obj->cq_obj); 172 memset(&rxq_obj->cq_obj, 0, sizeof(rxq_obj->cq_obj)); 173 if (rxq_obj->devx_channel) { 174 mlx5_os_devx_destroy_event_channel 175 (rxq_obj->devx_channel); 176 rxq_obj->devx_channel = NULL; 177 } 178 } 179 } 180 181 /** 182 * Get event for an Rx DevX queue object. 183 * 184 * @param rxq_obj 185 * DevX Rx queue object. 186 * 187 * @return 188 * 0 on success, a negative errno value otherwise and rte_errno is set. 189 */ 190 static int 191 mlx5_rx_devx_get_event(struct mlx5_rxq_obj *rxq_obj) 192 { 193 #ifdef HAVE_IBV_DEVX_EVENT 194 union { 195 struct mlx5dv_devx_async_event_hdr event_resp; 196 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 197 } out; 198 int ret = mlx5_glue->devx_get_event(rxq_obj->devx_channel, 199 &out.event_resp, 200 sizeof(out.buf)); 201 202 if (ret < 0) { 203 rte_errno = errno; 204 return -rte_errno; 205 } 206 if (out.event_resp.cookie != (uint64_t)(uintptr_t)rxq_obj->cq_obj.cq) { 207 rte_errno = EINVAL; 208 return -rte_errno; 209 } 210 return 0; 211 #else 212 (void)rxq_obj; 213 rte_errno = ENOTSUP; 214 return -rte_errno; 215 #endif /* HAVE_IBV_DEVX_EVENT */ 216 } 217 218 /** 219 * Create a RQ object using DevX. 220 * 221 * @param rxq 222 * Pointer to Rx queue. 223 * 224 * @return 225 * 0 on success, a negative errno value otherwise and rte_errno is set. 226 */ 227 static int 228 mlx5_rxq_create_devx_rq_resources(struct mlx5_rxq_priv *rxq) 229 { 230 struct mlx5_priv *priv = rxq->priv; 231 struct mlx5_common_device *cdev = priv->sh->cdev; 232 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 233 struct mlx5_rxq_data *rxq_data = &rxq->ctrl->rxq; 234 struct mlx5_devx_create_rq_attr rq_attr = { 0 }; 235 uint16_t log_desc_n = rxq_data->elts_n - rxq_data->sges_n; 236 uint32_t wqe_size, log_wqe_size; 237 238 /* Fill RQ attributes. */ 239 rq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE; 240 rq_attr.flush_in_error_en = 1; 241 rq_attr.vsd = (rxq_data->vlan_strip) ? 0 : 1; 242 rq_attr.cqn = rxq_ctrl->obj->cq_obj.cq->id; 243 rq_attr.scatter_fcs = (rxq_data->crc_present) ? 1 : 0; 244 rq_attr.ts_format = 245 mlx5_ts_format_conv(cdev->config.hca_attr.rq_ts_format); 246 /* Fill WQ attributes for this RQ. */ 247 if (mlx5_rxq_mprq_enabled(rxq_data)) { 248 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ; 249 /* 250 * Number of strides in each WQE: 251 * 512*2^single_wqe_log_num_of_strides. 252 */ 253 rq_attr.wq_attr.single_wqe_log_num_of_strides = 254 rxq_data->strd_num_n - 255 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES; 256 /* Stride size = (2^single_stride_log_num_of_bytes)*64B. */ 257 rq_attr.wq_attr.single_stride_log_num_of_bytes = 258 rxq_data->strd_sz_n - 259 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES; 260 wqe_size = sizeof(struct mlx5_wqe_mprq); 261 } else { 262 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 263 wqe_size = sizeof(struct mlx5_wqe_data_seg); 264 } 265 log_wqe_size = log2above(wqe_size) + rxq_data->sges_n; 266 wqe_size = 1 << log_wqe_size; /* round up power of two.*/ 267 rq_attr.wq_attr.log_wq_stride = log_wqe_size; 268 rq_attr.wq_attr.log_wq_sz = log_desc_n; 269 rq_attr.wq_attr.end_padding_mode = priv->config.hw_padding ? 270 MLX5_WQ_END_PAD_MODE_ALIGN : 271 MLX5_WQ_END_PAD_MODE_NONE; 272 rq_attr.wq_attr.pd = cdev->pdn; 273 rq_attr.counter_set_id = priv->counter_set_id; 274 /* Create RQ using DevX API. */ 275 return mlx5_devx_rq_create(cdev->ctx, &rxq->devx_rq, wqe_size, 276 log_desc_n, &rq_attr, rxq_ctrl->socket); 277 } 278 279 /** 280 * Create a DevX CQ object for an Rx queue. 281 * 282 * @param rxq 283 * Pointer to Rx queue. 284 * 285 * @return 286 * 0 on success, a negative errno value otherwise and rte_errno is set. 287 */ 288 static int 289 mlx5_rxq_create_devx_cq_resources(struct mlx5_rxq_priv *rxq) 290 { 291 struct mlx5_devx_cq *cq_obj = 0; 292 struct mlx5_devx_cq_attr cq_attr = { 0 }; 293 struct mlx5_priv *priv = rxq->priv; 294 struct mlx5_dev_ctx_shared *sh = priv->sh; 295 uint16_t port_id = priv->dev_data->port_id; 296 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 297 struct mlx5_rxq_data *rxq_data = &rxq_ctrl->rxq; 298 unsigned int cqe_n = mlx5_rxq_cqe_num(rxq_data); 299 uint32_t log_cqe_n; 300 uint16_t event_nums[1] = { 0 }; 301 int ret = 0; 302 303 if (priv->config.cqe_comp && !rxq_data->hw_timestamp && 304 !rxq_data->lro) { 305 cq_attr.cqe_comp_en = 1u; 306 rxq_data->mcqe_format = priv->config.cqe_comp_fmt; 307 rxq_data->byte_mask = UINT32_MAX; 308 switch (priv->config.cqe_comp_fmt) { 309 case MLX5_CQE_RESP_FORMAT_HASH: 310 /* fallthrough */ 311 case MLX5_CQE_RESP_FORMAT_CSUM: 312 /* 313 * Select CSUM miniCQE format only for non-vectorized 314 * MPRQ Rx burst, use HASH miniCQE format for others. 315 */ 316 if (mlx5_rxq_check_vec_support(rxq_data) < 0 && 317 mlx5_rxq_mprq_enabled(rxq_data)) 318 cq_attr.mini_cqe_res_format = 319 MLX5_CQE_RESP_FORMAT_CSUM_STRIDX; 320 else 321 cq_attr.mini_cqe_res_format = 322 MLX5_CQE_RESP_FORMAT_HASH; 323 rxq_data->mcqe_format = cq_attr.mini_cqe_res_format; 324 break; 325 case MLX5_CQE_RESP_FORMAT_FTAG_STRIDX: 326 rxq_data->byte_mask = MLX5_LEN_WITH_MARK_MASK; 327 /* fallthrough */ 328 case MLX5_CQE_RESP_FORMAT_CSUM_STRIDX: 329 cq_attr.mini_cqe_res_format = priv->config.cqe_comp_fmt; 330 break; 331 case MLX5_CQE_RESP_FORMAT_L34H_STRIDX: 332 cq_attr.mini_cqe_res_format = 0; 333 cq_attr.mini_cqe_res_format_ext = 1; 334 break; 335 } 336 DRV_LOG(DEBUG, 337 "Port %u Rx CQE compression is enabled, format %d.", 338 port_id, priv->config.cqe_comp_fmt); 339 /* 340 * For vectorized Rx, it must not be doubled in order to 341 * make cq_ci and rq_ci aligned. 342 */ 343 if (mlx5_rxq_check_vec_support(rxq_data) < 0) 344 cqe_n *= 2; 345 } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) { 346 DRV_LOG(DEBUG, 347 "Port %u Rx CQE compression is disabled for HW timestamp.", 348 port_id); 349 } else if (priv->config.cqe_comp && rxq_data->lro) { 350 DRV_LOG(DEBUG, 351 "Port %u Rx CQE compression is disabled for LRO.", 352 port_id); 353 } 354 cq_attr.uar_page_id = mlx5_os_get_devx_uar_page_id(sh->devx_rx_uar); 355 log_cqe_n = log2above(cqe_n); 356 /* Create CQ using DevX API. */ 357 ret = mlx5_devx_cq_create(sh->cdev->ctx, &rxq_ctrl->obj->cq_obj, 358 log_cqe_n, &cq_attr, sh->numa_node); 359 if (ret) 360 return ret; 361 cq_obj = &rxq_ctrl->obj->cq_obj; 362 rxq_data->cqes = (volatile struct mlx5_cqe (*)[]) 363 (uintptr_t)cq_obj->cqes; 364 rxq_data->cq_db = cq_obj->db_rec; 365 rxq_data->cq_uar = mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar); 366 rxq_data->cqe_n = log_cqe_n; 367 rxq_data->cqn = cq_obj->cq->id; 368 if (rxq_ctrl->obj->devx_channel) { 369 ret = mlx5_os_devx_subscribe_devx_event 370 (rxq_ctrl->obj->devx_channel, 371 cq_obj->cq->obj, 372 sizeof(event_nums), 373 event_nums, 374 (uint64_t)(uintptr_t)cq_obj->cq); 375 if (ret) { 376 DRV_LOG(ERR, "Fail to subscribe CQ to event channel."); 377 ret = errno; 378 mlx5_devx_cq_destroy(cq_obj); 379 memset(cq_obj, 0, sizeof(*cq_obj)); 380 rte_errno = ret; 381 return -ret; 382 } 383 } 384 return 0; 385 } 386 387 /** 388 * Create the Rx hairpin queue object. 389 * 390 * @param rxq 391 * Pointer to Rx queue. 392 * 393 * @return 394 * 0 on success, a negative errno value otherwise and rte_errno is set. 395 */ 396 static int 397 mlx5_rxq_obj_hairpin_new(struct mlx5_rxq_priv *rxq) 398 { 399 uint16_t idx = rxq->idx; 400 struct mlx5_priv *priv = rxq->priv; 401 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 402 struct mlx5_devx_create_rq_attr attr = { 0 }; 403 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 404 uint32_t max_wq_data; 405 406 MLX5_ASSERT(rxq != NULL && rxq->ctrl != NULL && tmpl != NULL); 407 tmpl->rxq_ctrl = rxq_ctrl; 408 attr.hairpin = 1; 409 max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; 410 /* Jumbo frames > 9KB should be supported, and more packets. */ 411 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 412 if (priv->config.log_hp_size > max_wq_data) { 413 DRV_LOG(ERR, "Total data size %u power of 2 is " 414 "too large for hairpin.", 415 priv->config.log_hp_size); 416 rte_errno = ERANGE; 417 return -rte_errno; 418 } 419 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 420 } else { 421 attr.wq_attr.log_hairpin_data_sz = 422 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 423 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 424 } 425 /* Set the packets number to the maximum value for performance. */ 426 attr.wq_attr.log_hairpin_num_packets = 427 attr.wq_attr.log_hairpin_data_sz - 428 MLX5_HAIRPIN_QUEUE_STRIDE; 429 attr.counter_set_id = priv->counter_set_id; 430 tmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->cdev->ctx, &attr, 431 rxq_ctrl->socket); 432 if (!tmpl->rq) { 433 DRV_LOG(ERR, 434 "Port %u Rx hairpin queue %u can't create rq object.", 435 priv->dev_data->port_id, idx); 436 rte_errno = errno; 437 return -rte_errno; 438 } 439 priv->dev_data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN; 440 return 0; 441 } 442 443 /** 444 * Create the Rx queue DevX object. 445 * 446 * @param rxq 447 * Pointer to Rx queue. 448 * 449 * @return 450 * 0 on success, a negative errno value otherwise and rte_errno is set. 451 */ 452 static int 453 mlx5_rxq_devx_obj_new(struct mlx5_rxq_priv *rxq) 454 { 455 struct mlx5_priv *priv = rxq->priv; 456 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 457 struct mlx5_rxq_data *rxq_data = &rxq_ctrl->rxq; 458 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 459 int ret = 0; 460 461 MLX5_ASSERT(rxq_data); 462 MLX5_ASSERT(tmpl); 463 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) 464 return mlx5_rxq_obj_hairpin_new(rxq); 465 tmpl->rxq_ctrl = rxq_ctrl; 466 if (rxq_ctrl->irq) { 467 int devx_ev_flag = 468 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA; 469 470 tmpl->devx_channel = mlx5_os_devx_create_event_channel 471 (priv->sh->cdev->ctx, 472 devx_ev_flag); 473 if (!tmpl->devx_channel) { 474 rte_errno = errno; 475 DRV_LOG(ERR, "Failed to create event channel %d.", 476 rte_errno); 477 goto error; 478 } 479 tmpl->fd = mlx5_os_get_devx_channel_fd(tmpl->devx_channel); 480 } 481 /* Create CQ using DevX API. */ 482 ret = mlx5_rxq_create_devx_cq_resources(rxq); 483 if (ret) { 484 DRV_LOG(ERR, "Failed to create CQ."); 485 goto error; 486 } 487 /* Create RQ using DevX API. */ 488 ret = mlx5_rxq_create_devx_rq_resources(rxq); 489 if (ret) { 490 DRV_LOG(ERR, "Port %u Rx queue %u RQ creation failure.", 491 priv->dev_data->port_id, rxq->idx); 492 rte_errno = ENOMEM; 493 goto error; 494 } 495 /* Change queue state to ready. */ 496 ret = mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RST2RDY); 497 if (ret) 498 goto error; 499 rxq_data->wqes = (void *)(uintptr_t)rxq->devx_rq.wq.umem_buf; 500 rxq_data->rq_db = (uint32_t *)(uintptr_t)rxq->devx_rq.wq.db_rec; 501 mlx5_rxq_initialize(rxq_data); 502 priv->dev_data->rx_queue_state[rxq->idx] = RTE_ETH_QUEUE_STATE_STARTED; 503 rxq_ctrl->wqn = rxq->devx_rq.rq->id; 504 return 0; 505 error: 506 ret = rte_errno; /* Save rte_errno before cleanup. */ 507 mlx5_rxq_devx_obj_release(rxq); 508 rte_errno = ret; /* Restore rte_errno. */ 509 return -rte_errno; 510 } 511 512 /** 513 * Prepare RQT attribute structure for DevX RQT API. 514 * 515 * @param dev 516 * Pointer to Ethernet device. 517 * @param log_n 518 * Log of number of queues in the array. 519 * @param queues 520 * List of RX queue indices or NULL, in which case 521 * the attribute will be filled by drop queue ID. 522 * @param queues_n 523 * Size of @p queues array or 0 if it is NULL. 524 * @param ind_tbl 525 * DevX indirection table object. 526 * 527 * @return 528 * The RQT attr object initialized, NULL otherwise and rte_errno is set. 529 */ 530 static struct mlx5_devx_rqt_attr * 531 mlx5_devx_ind_table_create_rqt_attr(struct rte_eth_dev *dev, 532 const unsigned int log_n, 533 const uint16_t *queues, 534 const uint32_t queues_n) 535 { 536 struct mlx5_priv *priv = dev->data->dev_private; 537 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 538 const unsigned int rqt_n = 1 << log_n; 539 unsigned int i, j; 540 541 rqt_attr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rqt_attr) + 542 rqt_n * sizeof(uint32_t), 0, SOCKET_ID_ANY); 543 if (!rqt_attr) { 544 DRV_LOG(ERR, "Port %u cannot allocate RQT resources.", 545 dev->data->port_id); 546 rte_errno = ENOMEM; 547 return NULL; 548 } 549 rqt_attr->rqt_max_size = priv->config.ind_table_max_size; 550 rqt_attr->rqt_actual_size = rqt_n; 551 if (queues == NULL) { 552 for (i = 0; i < rqt_n; i++) 553 rqt_attr->rq_list[i] = 554 priv->drop_queue.rxq->devx_rq.rq->id; 555 return rqt_attr; 556 } 557 for (i = 0; i != queues_n; ++i) { 558 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, queues[i]); 559 560 MLX5_ASSERT(rxq != NULL); 561 rqt_attr->rq_list[i] = rxq->devx_rq.rq->id; 562 } 563 MLX5_ASSERT(i > 0); 564 for (j = 0; i != rqt_n; ++j, ++i) 565 rqt_attr->rq_list[i] = rqt_attr->rq_list[j]; 566 return rqt_attr; 567 } 568 569 /** 570 * Create RQT using DevX API as a filed of indirection table. 571 * 572 * @param dev 573 * Pointer to Ethernet device. 574 * @param log_n 575 * Log of number of queues in the array. 576 * @param ind_tbl 577 * DevX indirection table object. 578 * 579 * @return 580 * 0 on success, a negative errno value otherwise and rte_errno is set. 581 */ 582 static int 583 mlx5_devx_ind_table_new(struct rte_eth_dev *dev, const unsigned int log_n, 584 struct mlx5_ind_table_obj *ind_tbl) 585 { 586 struct mlx5_priv *priv = dev->data->dev_private; 587 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 588 const uint16_t *queues = dev->data->dev_started ? ind_tbl->queues : 589 NULL; 590 591 MLX5_ASSERT(ind_tbl); 592 rqt_attr = mlx5_devx_ind_table_create_rqt_attr(dev, log_n, queues, 593 ind_tbl->queues_n); 594 if (!rqt_attr) 595 return -rte_errno; 596 ind_tbl->rqt = mlx5_devx_cmd_create_rqt(priv->sh->cdev->ctx, rqt_attr); 597 mlx5_free(rqt_attr); 598 if (!ind_tbl->rqt) { 599 DRV_LOG(ERR, "Port %u cannot create DevX RQT.", 600 dev->data->port_id); 601 rte_errno = errno; 602 return -rte_errno; 603 } 604 return 0; 605 } 606 607 /** 608 * Modify RQT using DevX API as a filed of indirection table. 609 * 610 * @param dev 611 * Pointer to Ethernet device. 612 * @param log_n 613 * Log of number of queues in the array. 614 * @param ind_tbl 615 * DevX indirection table object. 616 * 617 * @return 618 * 0 on success, a negative errno value otherwise and rte_errno is set. 619 */ 620 static int 621 mlx5_devx_ind_table_modify(struct rte_eth_dev *dev, const unsigned int log_n, 622 const uint16_t *queues, const uint32_t queues_n, 623 struct mlx5_ind_table_obj *ind_tbl) 624 { 625 int ret = 0; 626 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 627 628 MLX5_ASSERT(ind_tbl); 629 rqt_attr = mlx5_devx_ind_table_create_rqt_attr(dev, log_n, 630 queues, 631 queues_n); 632 if (!rqt_attr) 633 return -rte_errno; 634 ret = mlx5_devx_cmd_modify_rqt(ind_tbl->rqt, rqt_attr); 635 mlx5_free(rqt_attr); 636 if (ret) 637 DRV_LOG(ERR, "Port %u cannot modify DevX RQT.", 638 dev->data->port_id); 639 return ret; 640 } 641 642 /** 643 * Destroy the DevX RQT object. 644 * 645 * @param ind_table 646 * Indirection table to release. 647 */ 648 static void 649 mlx5_devx_ind_table_destroy(struct mlx5_ind_table_obj *ind_tbl) 650 { 651 claim_zero(mlx5_devx_cmd_destroy(ind_tbl->rqt)); 652 } 653 654 /** 655 * Set TIR attribute struct with relevant input values. 656 * 657 * @param[in] dev 658 * Pointer to Ethernet device. 659 * @param[in] rss_key 660 * RSS key for the Rx hash queue. 661 * @param[in] hash_fields 662 * Verbs protocol hash field to make the RSS on. 663 * @param[in] ind_tbl 664 * Indirection table for TIR. If table queues array is NULL, 665 * a TIR for drop queue is assumed. 666 * @param[in] tunnel 667 * Tunnel type. 668 * @param[out] tir_attr 669 * Parameters structure for TIR creation/modification. 670 * 671 * @return 672 * The Verbs/DevX object initialised index, 0 otherwise and rte_errno is set. 673 */ 674 static void 675 mlx5_devx_tir_attr_set(struct rte_eth_dev *dev, const uint8_t *rss_key, 676 uint64_t hash_fields, 677 const struct mlx5_ind_table_obj *ind_tbl, 678 int tunnel, struct mlx5_devx_tir_attr *tir_attr) 679 { 680 struct mlx5_priv *priv = dev->data->dev_private; 681 enum mlx5_rxq_type rxq_obj_type; 682 bool lro = true; 683 uint32_t i; 684 685 /* NULL queues designate drop queue. */ 686 if (ind_tbl->queues != NULL) { 687 struct mlx5_rxq_data *rxq_data = 688 (*priv->rxqs)[ind_tbl->queues[0]]; 689 struct mlx5_rxq_ctrl *rxq_ctrl = 690 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 691 rxq_obj_type = rxq_ctrl->type; 692 693 /* Enable TIR LRO only if all the queues were configured for. */ 694 for (i = 0; i < ind_tbl->queues_n; ++i) { 695 if (!(*priv->rxqs)[ind_tbl->queues[i]]->lro) { 696 lro = false; 697 break; 698 } 699 } 700 } else { 701 rxq_obj_type = priv->drop_queue.rxq->ctrl->type; 702 } 703 memset(tir_attr, 0, sizeof(*tir_attr)); 704 tir_attr->disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT; 705 tir_attr->rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ; 706 tir_attr->tunneled_offload_en = !!tunnel; 707 /* If needed, translate hash_fields bitmap to PRM format. */ 708 if (hash_fields) { 709 struct mlx5_rx_hash_field_select *rx_hash_field_select = 710 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 711 hash_fields & IBV_RX_HASH_INNER ? 712 &tir_attr->rx_hash_field_selector_inner : 713 #endif 714 &tir_attr->rx_hash_field_selector_outer; 715 /* 1 bit: 0: IPv4, 1: IPv6. */ 716 rx_hash_field_select->l3_prot_type = 717 !!(hash_fields & MLX5_IPV6_IBV_RX_HASH); 718 /* 1 bit: 0: TCP, 1: UDP. */ 719 rx_hash_field_select->l4_prot_type = 720 !!(hash_fields & MLX5_UDP_IBV_RX_HASH); 721 /* Bitmask which sets which fields to use in RX Hash. */ 722 rx_hash_field_select->selected_fields = 723 ((!!(hash_fields & MLX5_L3_SRC_IBV_RX_HASH)) << 724 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) | 725 (!!(hash_fields & MLX5_L3_DST_IBV_RX_HASH)) << 726 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP | 727 (!!(hash_fields & MLX5_L4_SRC_IBV_RX_HASH)) << 728 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT | 729 (!!(hash_fields & MLX5_L4_DST_IBV_RX_HASH)) << 730 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT; 731 } 732 if (rxq_obj_type == MLX5_RXQ_TYPE_HAIRPIN) 733 tir_attr->transport_domain = priv->sh->td->id; 734 else 735 tir_attr->transport_domain = priv->sh->tdn; 736 memcpy(tir_attr->rx_hash_toeplitz_key, rss_key, MLX5_RSS_HASH_KEY_LEN); 737 tir_attr->indirect_table = ind_tbl->rqt->id; 738 if (dev->data->dev_conf.lpbk_mode) 739 tir_attr->self_lb_block = 740 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST; 741 if (lro) { 742 tir_attr->lro_timeout_period_usecs = priv->config.lro.timeout; 743 tir_attr->lro_max_msg_sz = priv->max_lro_msg_size; 744 tir_attr->lro_enable_mask = 745 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | 746 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO; 747 } 748 } 749 750 /** 751 * Create an Rx Hash queue. 752 * 753 * @param dev 754 * Pointer to Ethernet device. 755 * @param hrxq 756 * Pointer to Rx Hash queue. 757 * @param tunnel 758 * Tunnel type. 759 * 760 * @return 761 * 0 on success, a negative errno value otherwise and rte_errno is set. 762 */ 763 static int 764 mlx5_devx_hrxq_new(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq, 765 int tunnel __rte_unused) 766 { 767 struct mlx5_priv *priv = dev->data->dev_private; 768 struct mlx5_devx_tir_attr tir_attr = {0}; 769 int err; 770 771 mlx5_devx_tir_attr_set(dev, hrxq->rss_key, hrxq->hash_fields, 772 hrxq->ind_table, tunnel, &tir_attr); 773 hrxq->tir = mlx5_devx_cmd_create_tir(priv->sh->cdev->ctx, &tir_attr); 774 if (!hrxq->tir) { 775 DRV_LOG(ERR, "Port %u cannot create DevX TIR.", 776 dev->data->port_id); 777 rte_errno = errno; 778 goto error; 779 } 780 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 781 if (mlx5_flow_os_create_flow_action_dest_devx_tir(hrxq->tir, 782 &hrxq->action)) { 783 rte_errno = errno; 784 goto error; 785 } 786 #endif 787 return 0; 788 error: 789 err = rte_errno; /* Save rte_errno before cleanup. */ 790 if (hrxq->tir) 791 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 792 rte_errno = err; /* Restore rte_errno. */ 793 return -rte_errno; 794 } 795 796 /** 797 * Destroy a DevX TIR object. 798 * 799 * @param hrxq 800 * Hash Rx queue to release its tir. 801 */ 802 static void 803 mlx5_devx_tir_destroy(struct mlx5_hrxq *hrxq) 804 { 805 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 806 } 807 808 /** 809 * Modify an Rx Hash queue configuration. 810 * 811 * @param dev 812 * Pointer to Ethernet device. 813 * @param hrxq 814 * Hash Rx queue to modify. 815 * @param rss_key 816 * RSS key for the Rx hash queue. 817 * @param hash_fields 818 * Verbs protocol hash field to make the RSS on. 819 * @param[in] ind_tbl 820 * Indirection table for TIR. 821 * 822 * @return 823 * 0 on success, a negative errno value otherwise and rte_errno is set. 824 */ 825 static int 826 mlx5_devx_hrxq_modify(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq, 827 const uint8_t *rss_key, 828 uint64_t hash_fields, 829 const struct mlx5_ind_table_obj *ind_tbl) 830 { 831 struct mlx5_devx_modify_tir_attr modify_tir = {0}; 832 833 /* 834 * untested for modification fields: 835 * - rx_hash_symmetric not set in hrxq_new(), 836 * - rx_hash_fn set hard-coded in hrxq_new(), 837 * - lro_xxx not set after rxq setup 838 */ 839 if (ind_tbl != hrxq->ind_table) 840 modify_tir.modify_bitmask |= 841 MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_INDIRECT_TABLE; 842 if (hash_fields != hrxq->hash_fields || 843 memcmp(hrxq->rss_key, rss_key, MLX5_RSS_HASH_KEY_LEN)) 844 modify_tir.modify_bitmask |= 845 MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_HASH; 846 mlx5_devx_tir_attr_set(dev, rss_key, hash_fields, ind_tbl, 847 0, /* N/A - tunnel modification unsupported */ 848 &modify_tir.tir); 849 modify_tir.tirn = hrxq->tir->id; 850 if (mlx5_devx_cmd_modify_tir(hrxq->tir, &modify_tir)) { 851 DRV_LOG(ERR, "port %u cannot modify DevX TIR", 852 dev->data->port_id); 853 rte_errno = errno; 854 return -rte_errno; 855 } 856 return 0; 857 } 858 859 /** 860 * Create a DevX drop Rx queue. 861 * 862 * @param dev 863 * Pointer to Ethernet device. 864 * 865 * @return 866 * 0 on success, a negative errno value otherwise and rte_errno is set. 867 */ 868 static int 869 mlx5_rxq_devx_obj_drop_create(struct rte_eth_dev *dev) 870 { 871 struct mlx5_priv *priv = dev->data->dev_private; 872 int socket_id = dev->device->numa_node; 873 struct mlx5_rxq_priv *rxq; 874 struct mlx5_rxq_ctrl *rxq_ctrl = NULL; 875 struct mlx5_rxq_obj *rxq_obj = NULL; 876 int ret; 877 878 /* 879 * Initialize dummy control structures. 880 * They are required to hold pointers for cleanup 881 * and are only accessible via drop queue DevX objects. 882 */ 883 rxq = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq), 0, socket_id); 884 if (rxq == NULL) { 885 DRV_LOG(ERR, "Port %u could not allocate drop queue private", 886 dev->data->port_id); 887 rte_errno = ENOMEM; 888 goto error; 889 } 890 rxq_ctrl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq_ctrl), 891 0, socket_id); 892 if (rxq_ctrl == NULL) { 893 DRV_LOG(ERR, "Port %u could not allocate drop queue control", 894 dev->data->port_id); 895 rte_errno = ENOMEM; 896 goto error; 897 } 898 rxq_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq_obj), 0, socket_id); 899 if (rxq_obj == NULL) { 900 DRV_LOG(ERR, "Port %u could not allocate drop queue object", 901 dev->data->port_id); 902 rte_errno = ENOMEM; 903 goto error; 904 } 905 rxq_obj->rxq_ctrl = rxq_ctrl; 906 rxq_ctrl->type = MLX5_RXQ_TYPE_STANDARD; 907 rxq_ctrl->sh = priv->sh; 908 rxq_ctrl->obj = rxq_obj; 909 rxq->ctrl = rxq_ctrl; 910 rxq->priv = priv; 911 LIST_INSERT_HEAD(&rxq_ctrl->owners, rxq, owner_entry); 912 /* Create CQ using DevX API. */ 913 ret = mlx5_rxq_create_devx_cq_resources(rxq); 914 if (ret != 0) { 915 DRV_LOG(ERR, "Port %u drop queue CQ creation failed.", 916 dev->data->port_id); 917 goto error; 918 } 919 /* Create RQ using DevX API. */ 920 ret = mlx5_rxq_create_devx_rq_resources(rxq); 921 if (ret != 0) { 922 DRV_LOG(ERR, "Port %u drop queue RQ creation failed.", 923 dev->data->port_id); 924 rte_errno = ENOMEM; 925 goto error; 926 } 927 /* Change queue state to ready. */ 928 ret = mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RST2RDY); 929 if (ret != 0) 930 goto error; 931 /* Initialize drop queue. */ 932 priv->drop_queue.rxq = rxq; 933 return 0; 934 error: 935 ret = rte_errno; /* Save rte_errno before cleanup. */ 936 if (rxq != NULL && rxq->devx_rq.rq != NULL) 937 mlx5_devx_rq_destroy(&rxq->devx_rq); 938 if (rxq_obj != NULL) { 939 if (rxq_obj->cq_obj.cq != NULL) 940 mlx5_devx_cq_destroy(&rxq_obj->cq_obj); 941 if (rxq_obj->devx_channel) 942 mlx5_os_devx_destroy_event_channel 943 (rxq_obj->devx_channel); 944 mlx5_free(rxq_obj); 945 } 946 if (rxq_ctrl != NULL) 947 mlx5_free(rxq_ctrl); 948 if (rxq != NULL) 949 mlx5_free(rxq); 950 rte_errno = ret; /* Restore rte_errno. */ 951 return -rte_errno; 952 } 953 954 /** 955 * Release drop Rx queue resources. 956 * 957 * @param dev 958 * Pointer to Ethernet device. 959 */ 960 static void 961 mlx5_rxq_devx_obj_drop_release(struct rte_eth_dev *dev) 962 { 963 struct mlx5_priv *priv = dev->data->dev_private; 964 struct mlx5_rxq_priv *rxq = priv->drop_queue.rxq; 965 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 966 967 mlx5_rxq_devx_obj_release(rxq); 968 mlx5_free(rxq_ctrl->obj); 969 mlx5_free(rxq_ctrl); 970 mlx5_free(rxq); 971 priv->drop_queue.rxq = NULL; 972 } 973 974 /** 975 * Release a drop hash Rx queue. 976 * 977 * @param dev 978 * Pointer to Ethernet device. 979 */ 980 static void 981 mlx5_devx_drop_action_destroy(struct rte_eth_dev *dev) 982 { 983 struct mlx5_priv *priv = dev->data->dev_private; 984 struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq; 985 986 if (hrxq->tir != NULL) 987 mlx5_devx_tir_destroy(hrxq); 988 if (hrxq->ind_table->ind_table != NULL) 989 mlx5_devx_ind_table_destroy(hrxq->ind_table); 990 if (priv->drop_queue.rxq->devx_rq.rq != NULL) 991 mlx5_rxq_devx_obj_drop_release(dev); 992 } 993 994 /** 995 * Create a DevX drop action for Rx Hash queue. 996 * 997 * @param dev 998 * Pointer to Ethernet device. 999 * 1000 * @return 1001 * 0 on success, a negative errno value otherwise and rte_errno is set. 1002 */ 1003 static int 1004 mlx5_devx_drop_action_create(struct rte_eth_dev *dev) 1005 { 1006 struct mlx5_priv *priv = dev->data->dev_private; 1007 struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq; 1008 int ret; 1009 1010 ret = mlx5_rxq_devx_obj_drop_create(dev); 1011 if (ret != 0) { 1012 DRV_LOG(ERR, "Cannot create drop RX queue"); 1013 return ret; 1014 } 1015 /* hrxq->ind_table queues are NULL, drop RX queue ID will be used */ 1016 ret = mlx5_devx_ind_table_new(dev, 0, hrxq->ind_table); 1017 if (ret != 0) { 1018 DRV_LOG(ERR, "Cannot create drop hash RX queue indirection table"); 1019 goto error; 1020 } 1021 ret = mlx5_devx_hrxq_new(dev, hrxq, /* tunnel */ false); 1022 if (ret != 0) { 1023 DRV_LOG(ERR, "Cannot create drop hash RX queue"); 1024 goto error; 1025 } 1026 return 0; 1027 error: 1028 mlx5_devx_drop_action_destroy(dev); 1029 return ret; 1030 } 1031 1032 /** 1033 * Select TXQ TIS number. 1034 * 1035 * @param dev 1036 * Pointer to Ethernet device. 1037 * @param queue_idx 1038 * Queue index in DPDK Tx queue array. 1039 * 1040 * @return 1041 * > 0 on success, a negative errno value otherwise. 1042 */ 1043 static uint32_t 1044 mlx5_get_txq_tis_num(struct rte_eth_dev *dev, uint16_t queue_idx) 1045 { 1046 struct mlx5_priv *priv = dev->data->dev_private; 1047 int tis_idx; 1048 1049 if (priv->sh->bond.n_port && priv->sh->lag.affinity_mode == 1050 MLX5_LAG_MODE_TIS) { 1051 tis_idx = (priv->lag_affinity_idx + queue_idx) % 1052 priv->sh->bond.n_port; 1053 DRV_LOG(INFO, "port %d txq %d gets affinity %d and maps to PF %d.", 1054 dev->data->port_id, queue_idx, tis_idx + 1, 1055 priv->sh->lag.tx_remap_affinity[tis_idx]); 1056 } else { 1057 tis_idx = 0; 1058 } 1059 MLX5_ASSERT(priv->sh->tis[tis_idx]); 1060 return priv->sh->tis[tis_idx]->id; 1061 } 1062 1063 /** 1064 * Create the Tx hairpin queue object. 1065 * 1066 * @param dev 1067 * Pointer to Ethernet device. 1068 * @param idx 1069 * Queue index in DPDK Tx queue array. 1070 * 1071 * @return 1072 * 0 on success, a negative errno value otherwise and rte_errno is set. 1073 */ 1074 static int 1075 mlx5_txq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) 1076 { 1077 struct mlx5_priv *priv = dev->data->dev_private; 1078 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1079 struct mlx5_txq_ctrl *txq_ctrl = 1080 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1081 struct mlx5_devx_create_sq_attr attr = { 0 }; 1082 struct mlx5_txq_obj *tmpl = txq_ctrl->obj; 1083 uint32_t max_wq_data; 1084 1085 MLX5_ASSERT(txq_data); 1086 MLX5_ASSERT(tmpl); 1087 tmpl->txq_ctrl = txq_ctrl; 1088 attr.hairpin = 1; 1089 attr.tis_lst_sz = 1; 1090 max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; 1091 /* Jumbo frames > 9KB should be supported, and more packets. */ 1092 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 1093 if (priv->config.log_hp_size > max_wq_data) { 1094 DRV_LOG(ERR, "Total data size %u power of 2 is " 1095 "too large for hairpin.", 1096 priv->config.log_hp_size); 1097 rte_errno = ERANGE; 1098 return -rte_errno; 1099 } 1100 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 1101 } else { 1102 attr.wq_attr.log_hairpin_data_sz = 1103 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 1104 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 1105 } 1106 /* Set the packets number to the maximum value for performance. */ 1107 attr.wq_attr.log_hairpin_num_packets = 1108 attr.wq_attr.log_hairpin_data_sz - 1109 MLX5_HAIRPIN_QUEUE_STRIDE; 1110 1111 attr.tis_num = mlx5_get_txq_tis_num(dev, idx); 1112 tmpl->sq = mlx5_devx_cmd_create_sq(priv->sh->cdev->ctx, &attr); 1113 if (!tmpl->sq) { 1114 DRV_LOG(ERR, 1115 "Port %u tx hairpin queue %u can't create SQ object.", 1116 dev->data->port_id, idx); 1117 rte_errno = errno; 1118 return -rte_errno; 1119 } 1120 return 0; 1121 } 1122 1123 #if defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) || !defined(HAVE_INFINIBAND_VERBS_H) 1124 /** 1125 * Destroy the Tx queue DevX object. 1126 * 1127 * @param txq_obj 1128 * Txq object to destroy. 1129 */ 1130 static void 1131 mlx5_txq_release_devx_resources(struct mlx5_txq_obj *txq_obj) 1132 { 1133 mlx5_devx_sq_destroy(&txq_obj->sq_obj); 1134 memset(&txq_obj->sq_obj, 0, sizeof(txq_obj->sq_obj)); 1135 mlx5_devx_cq_destroy(&txq_obj->cq_obj); 1136 memset(&txq_obj->cq_obj, 0, sizeof(txq_obj->cq_obj)); 1137 } 1138 1139 /** 1140 * Create a SQ object and its resources using DevX. 1141 * 1142 * @param dev 1143 * Pointer to Ethernet device. 1144 * @param idx 1145 * Queue index in DPDK Tx queue array. 1146 * @param[in] log_desc_n 1147 * Log of number of descriptors in queue. 1148 * 1149 * @return 1150 * 0 on success, a negative errno value otherwise and rte_errno is set. 1151 */ 1152 static int 1153 mlx5_txq_create_devx_sq_resources(struct rte_eth_dev *dev, uint16_t idx, 1154 uint16_t log_desc_n) 1155 { 1156 struct mlx5_priv *priv = dev->data->dev_private; 1157 struct mlx5_common_device *cdev = priv->sh->cdev; 1158 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1159 struct mlx5_txq_ctrl *txq_ctrl = 1160 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1161 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1162 struct mlx5_devx_create_sq_attr sq_attr = { 1163 .flush_in_error_en = 1, 1164 .allow_multi_pkt_send_wqe = !!priv->config.mps, 1165 .min_wqe_inline_mode = priv->config.hca_attr.vport_inline_mode, 1166 .allow_swp = !!priv->config.swp, 1167 .cqn = txq_obj->cq_obj.cq->id, 1168 .tis_lst_sz = 1, 1169 .wq_attr = (struct mlx5_devx_wq_attr){ 1170 .pd = cdev->pdn, 1171 .uar_page = 1172 mlx5_os_get_devx_uar_page_id(priv->sh->tx_uar), 1173 }, 1174 .ts_format = 1175 mlx5_ts_format_conv(cdev->config.hca_attr.sq_ts_format), 1176 .tis_num = mlx5_get_txq_tis_num(dev, idx), 1177 }; 1178 1179 /* Create Send Queue object with DevX. */ 1180 return mlx5_devx_sq_create(cdev->ctx, &txq_obj->sq_obj, 1181 log_desc_n, &sq_attr, priv->sh->numa_node); 1182 } 1183 #endif 1184 1185 /** 1186 * Create the Tx queue DevX object. 1187 * 1188 * @param dev 1189 * Pointer to Ethernet device. 1190 * @param idx 1191 * Queue index in DPDK Tx queue array. 1192 * 1193 * @return 1194 * 0 on success, a negative errno value otherwise and rte_errno is set. 1195 */ 1196 int 1197 mlx5_txq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx) 1198 { 1199 struct mlx5_priv *priv = dev->data->dev_private; 1200 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1201 struct mlx5_txq_ctrl *txq_ctrl = 1202 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1203 1204 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) 1205 return mlx5_txq_obj_hairpin_new(dev, idx); 1206 #if !defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) && defined(HAVE_INFINIBAND_VERBS_H) 1207 DRV_LOG(ERR, "Port %u Tx queue %u cannot create with DevX, no UAR.", 1208 dev->data->port_id, idx); 1209 rte_errno = ENOMEM; 1210 return -rte_errno; 1211 #else 1212 struct mlx5_dev_ctx_shared *sh = priv->sh; 1213 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1214 struct mlx5_devx_cq_attr cq_attr = { 1215 .uar_page_id = mlx5_os_get_devx_uar_page_id(sh->tx_uar), 1216 }; 1217 void *reg_addr; 1218 uint32_t cqe_n, log_desc_n; 1219 uint32_t wqe_n, wqe_size; 1220 int ret = 0; 1221 1222 MLX5_ASSERT(txq_data); 1223 MLX5_ASSERT(txq_obj); 1224 txq_obj->txq_ctrl = txq_ctrl; 1225 txq_obj->dev = dev; 1226 cqe_n = (1UL << txq_data->elts_n) / MLX5_TX_COMP_THRESH + 1227 1 + MLX5_TX_COMP_THRESH_INLINE_DIV; 1228 log_desc_n = log2above(cqe_n); 1229 cqe_n = 1UL << log_desc_n; 1230 if (cqe_n > UINT16_MAX) { 1231 DRV_LOG(ERR, "Port %u Tx queue %u requests to many CQEs %u.", 1232 dev->data->port_id, txq_data->idx, cqe_n); 1233 rte_errno = EINVAL; 1234 return 0; 1235 } 1236 /* Create completion queue object with DevX. */ 1237 ret = mlx5_devx_cq_create(sh->cdev->ctx, &txq_obj->cq_obj, log_desc_n, 1238 &cq_attr, priv->sh->numa_node); 1239 if (ret) { 1240 DRV_LOG(ERR, "Port %u Tx queue %u CQ creation failure.", 1241 dev->data->port_id, idx); 1242 goto error; 1243 } 1244 txq_data->cqe_n = log_desc_n; 1245 txq_data->cqe_s = cqe_n; 1246 txq_data->cqe_m = txq_data->cqe_s - 1; 1247 txq_data->cqes = txq_obj->cq_obj.cqes; 1248 txq_data->cq_ci = 0; 1249 txq_data->cq_pi = 0; 1250 txq_data->cq_db = txq_obj->cq_obj.db_rec; 1251 *txq_data->cq_db = 0; 1252 /* 1253 * Adjust the amount of WQEs depending on inline settings. 1254 * The number of descriptors should be enough to handle 1255 * the specified number of packets. If queue is being created 1256 * with Verbs the rdma-core does queue size adjustment 1257 * internally in the mlx5_calc_sq_size(), we do the same 1258 * for the queue being created with DevX at this point. 1259 */ 1260 wqe_size = txq_data->tso_en ? 1261 RTE_ALIGN(txq_ctrl->max_tso_header, MLX5_WSEG_SIZE) : 0; 1262 wqe_size += sizeof(struct mlx5_wqe_cseg) + 1263 sizeof(struct mlx5_wqe_eseg) + 1264 sizeof(struct mlx5_wqe_dseg); 1265 if (txq_data->inlen_send) 1266 wqe_size = RTE_MAX(wqe_size, sizeof(struct mlx5_wqe_cseg) + 1267 sizeof(struct mlx5_wqe_eseg) + 1268 RTE_ALIGN(txq_data->inlen_send + 1269 sizeof(uint32_t), 1270 MLX5_WSEG_SIZE)); 1271 wqe_size = RTE_ALIGN(wqe_size, MLX5_WQE_SIZE) / MLX5_WQE_SIZE; 1272 /* Create Send Queue object with DevX. */ 1273 wqe_n = RTE_MIN((1UL << txq_data->elts_n) * wqe_size, 1274 (uint32_t)priv->sh->device_attr.max_qp_wr); 1275 log_desc_n = log2above(wqe_n); 1276 ret = mlx5_txq_create_devx_sq_resources(dev, idx, log_desc_n); 1277 if (ret) { 1278 DRV_LOG(ERR, "Port %u Tx queue %u SQ creation failure.", 1279 dev->data->port_id, idx); 1280 rte_errno = errno; 1281 goto error; 1282 } 1283 /* Create the Work Queue. */ 1284 txq_data->wqe_n = log_desc_n; 1285 txq_data->wqe_s = 1 << txq_data->wqe_n; 1286 txq_data->wqe_m = txq_data->wqe_s - 1; 1287 txq_data->wqes = (struct mlx5_wqe *)(uintptr_t)txq_obj->sq_obj.wqes; 1288 txq_data->wqes_end = txq_data->wqes + txq_data->wqe_s; 1289 txq_data->wqe_ci = 0; 1290 txq_data->wqe_pi = 0; 1291 txq_data->wqe_comp = 0; 1292 txq_data->wqe_thres = txq_data->wqe_s / MLX5_TX_COMP_THRESH_INLINE_DIV; 1293 txq_data->qp_db = &txq_obj->sq_obj.db_rec[MLX5_SND_DBR]; 1294 *txq_data->qp_db = 0; 1295 txq_data->qp_num_8s = txq_obj->sq_obj.sq->id << 8; 1296 /* Change Send Queue state to Ready-to-Send. */ 1297 ret = mlx5_txq_devx_modify(txq_obj, MLX5_TXQ_MOD_RST2RDY, 0); 1298 if (ret) { 1299 rte_errno = errno; 1300 DRV_LOG(ERR, 1301 "Port %u Tx queue %u SQ state to SQC_STATE_RDY failed.", 1302 dev->data->port_id, idx); 1303 goto error; 1304 } 1305 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 1306 /* 1307 * If using DevX need to query and store TIS transport domain value. 1308 * This is done once per port. 1309 * Will use this value on Rx, when creating matching TIR. 1310 */ 1311 if (!priv->sh->tdn) 1312 priv->sh->tdn = priv->sh->td->id; 1313 #endif 1314 MLX5_ASSERT(sh->tx_uar); 1315 reg_addr = mlx5_os_get_devx_uar_reg_addr(sh->tx_uar); 1316 MLX5_ASSERT(reg_addr); 1317 txq_ctrl->bf_reg = reg_addr; 1318 txq_ctrl->uar_mmap_offset = 1319 mlx5_os_get_devx_uar_mmap_offset(sh->tx_uar); 1320 txq_uar_init(txq_ctrl); 1321 dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED; 1322 return 0; 1323 error: 1324 ret = rte_errno; /* Save rte_errno before cleanup. */ 1325 mlx5_txq_release_devx_resources(txq_obj); 1326 rte_errno = ret; /* Restore rte_errno. */ 1327 return -rte_errno; 1328 #endif 1329 } 1330 1331 /** 1332 * Release an Tx DevX queue object. 1333 * 1334 * @param txq_obj 1335 * DevX Tx queue object. 1336 */ 1337 void 1338 mlx5_txq_devx_obj_release(struct mlx5_txq_obj *txq_obj) 1339 { 1340 MLX5_ASSERT(txq_obj); 1341 if (txq_obj->txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 1342 if (txq_obj->tis) 1343 claim_zero(mlx5_devx_cmd_destroy(txq_obj->tis)); 1344 #if defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) || !defined(HAVE_INFINIBAND_VERBS_H) 1345 } else { 1346 mlx5_txq_release_devx_resources(txq_obj); 1347 #endif 1348 } 1349 } 1350 1351 struct mlx5_obj_ops devx_obj_ops = { 1352 .rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_rq_vlan_strip, 1353 .rxq_obj_new = mlx5_rxq_devx_obj_new, 1354 .rxq_event_get = mlx5_rx_devx_get_event, 1355 .rxq_obj_modify = mlx5_devx_modify_rq, 1356 .rxq_obj_release = mlx5_rxq_devx_obj_release, 1357 .ind_table_new = mlx5_devx_ind_table_new, 1358 .ind_table_modify = mlx5_devx_ind_table_modify, 1359 .ind_table_destroy = mlx5_devx_ind_table_destroy, 1360 .hrxq_new = mlx5_devx_hrxq_new, 1361 .hrxq_destroy = mlx5_devx_tir_destroy, 1362 .hrxq_modify = mlx5_devx_hrxq_modify, 1363 .drop_action_create = mlx5_devx_drop_action_create, 1364 .drop_action_destroy = mlx5_devx_drop_action_destroy, 1365 .txq_obj_new = mlx5_txq_devx_obj_new, 1366 .txq_obj_modify = mlx5_txq_devx_modify, 1367 .txq_obj_release = mlx5_txq_devx_obj_release, 1368 .lb_dummy_queue_create = NULL, 1369 .lb_dummy_queue_release = NULL, 1370 }; 1371