1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <stddef.h> 6 #include <errno.h> 7 #include <stdbool.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <sys/queue.h> 11 12 #include <rte_malloc.h> 13 #include <rte_common.h> 14 #include <rte_eal_paging.h> 15 16 #include <mlx5_glue.h> 17 #include <mlx5_devx_cmds.h> 18 #include <mlx5_malloc.h> 19 20 #include "mlx5.h" 21 #include "mlx5_common_os.h" 22 #include "mlx5_rxtx.h" 23 #include "mlx5_utils.h" 24 #include "mlx5_devx.h" 25 #include "mlx5_flow.h" 26 27 28 /** 29 * Modify RQ vlan stripping offload 30 * 31 * @param rxq_obj 32 * Rx queue object. 33 * 34 * @return 35 * 0 on success, non-0 otherwise 36 */ 37 static int 38 mlx5_rxq_obj_modify_rq_vlan_strip(struct mlx5_rxq_obj *rxq_obj, int on) 39 { 40 struct mlx5_devx_modify_rq_attr rq_attr; 41 42 memset(&rq_attr, 0, sizeof(rq_attr)); 43 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 44 rq_attr.state = MLX5_RQC_STATE_RDY; 45 rq_attr.vsd = (on ? 0 : 1); 46 rq_attr.modify_bitmask = MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD; 47 return mlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr); 48 } 49 50 /** 51 * Modify RQ using DevX API. 52 * 53 * @param rxq_obj 54 * DevX Rx queue object. 55 * 56 * @return 57 * 0 on success, a negative errno value otherwise and rte_errno is set. 58 */ 59 static int 60 mlx5_devx_modify_rq(struct mlx5_rxq_obj *rxq_obj, bool is_start) 61 { 62 struct mlx5_devx_modify_rq_attr rq_attr; 63 64 memset(&rq_attr, 0, sizeof(rq_attr)); 65 if (is_start) { 66 rq_attr.rq_state = MLX5_RQC_STATE_RST; 67 rq_attr.state = MLX5_RQC_STATE_RDY; 68 } else { 69 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 70 rq_attr.state = MLX5_RQC_STATE_RST; 71 } 72 return mlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr); 73 } 74 75 /** 76 * Release the resources allocated for an RQ DevX object. 77 * 78 * @param rxq_ctrl 79 * DevX Rx queue object. 80 */ 81 static void 82 rxq_release_devx_rq_resources(struct mlx5_rxq_ctrl *rxq_ctrl) 83 { 84 struct mlx5_devx_dbr_page *dbr_page = rxq_ctrl->rq_dbrec_page; 85 86 if (rxq_ctrl->rxq.wqes) { 87 mlx5_free((void *)(uintptr_t)rxq_ctrl->rxq.wqes); 88 rxq_ctrl->rxq.wqes = NULL; 89 } 90 if (rxq_ctrl->wq_umem) { 91 mlx5_glue->devx_umem_dereg(rxq_ctrl->wq_umem); 92 rxq_ctrl->wq_umem = NULL; 93 } 94 if (dbr_page) { 95 claim_zero(mlx5_release_dbr(&rxq_ctrl->priv->dbrpgs, 96 mlx5_os_get_umem_id(dbr_page->umem), 97 rxq_ctrl->rq_dbr_offset)); 98 rxq_ctrl->rq_dbrec_page = NULL; 99 } 100 } 101 102 /** 103 * Release the resources allocated for the Rx CQ DevX object. 104 * 105 * @param rxq_ctrl 106 * DevX Rx queue object. 107 */ 108 static void 109 rxq_release_devx_cq_resources(struct mlx5_rxq_ctrl *rxq_ctrl) 110 { 111 struct mlx5_devx_dbr_page *dbr_page = rxq_ctrl->cq_dbrec_page; 112 113 if (rxq_ctrl->rxq.cqes) { 114 rte_free((void *)(uintptr_t)rxq_ctrl->rxq.cqes); 115 rxq_ctrl->rxq.cqes = NULL; 116 } 117 if (rxq_ctrl->cq_umem) { 118 mlx5_glue->devx_umem_dereg(rxq_ctrl->cq_umem); 119 rxq_ctrl->cq_umem = NULL; 120 } 121 if (dbr_page) { 122 claim_zero(mlx5_release_dbr(&rxq_ctrl->priv->dbrpgs, 123 mlx5_os_get_umem_id(dbr_page->umem), 124 rxq_ctrl->cq_dbr_offset)); 125 rxq_ctrl->cq_dbrec_page = NULL; 126 } 127 } 128 129 /** 130 * Release an Rx DevX queue object. 131 * 132 * @param rxq_obj 133 * DevX Rx queue object. 134 */ 135 static void 136 mlx5_rxq_devx_obj_release(struct mlx5_rxq_obj *rxq_obj) 137 { 138 MLX5_ASSERT(rxq_obj); 139 MLX5_ASSERT(rxq_obj->rq); 140 if (rxq_obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN) { 141 mlx5_devx_modify_rq(rxq_obj, false); 142 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 143 } else { 144 MLX5_ASSERT(rxq_obj->devx_cq); 145 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 146 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->devx_cq)); 147 if (rxq_obj->devx_channel) 148 mlx5_glue->devx_destroy_event_channel 149 (rxq_obj->devx_channel); 150 rxq_release_devx_rq_resources(rxq_obj->rxq_ctrl); 151 rxq_release_devx_cq_resources(rxq_obj->rxq_ctrl); 152 } 153 } 154 155 /** 156 * Get event for an Rx DevX queue object. 157 * 158 * @param rxq_obj 159 * DevX Rx queue object. 160 * 161 * @return 162 * 0 on success, a negative errno value otherwise and rte_errno is set. 163 */ 164 static int 165 mlx5_rx_devx_get_event(struct mlx5_rxq_obj *rxq_obj) 166 { 167 #ifdef HAVE_IBV_DEVX_EVENT 168 union { 169 struct mlx5dv_devx_async_event_hdr event_resp; 170 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 171 } out; 172 int ret = mlx5_glue->devx_get_event(rxq_obj->devx_channel, 173 &out.event_resp, 174 sizeof(out.buf)); 175 176 if (ret < 0) { 177 rte_errno = errno; 178 return -rte_errno; 179 } 180 if (out.event_resp.cookie != (uint64_t)(uintptr_t)rxq_obj->devx_cq) { 181 rte_errno = EINVAL; 182 return -rte_errno; 183 } 184 return 0; 185 #else 186 (void)rxq_obj; 187 rte_errno = ENOTSUP; 188 return -rte_errno; 189 #endif /* HAVE_IBV_DEVX_EVENT */ 190 } 191 192 /** 193 * Fill common fields of create RQ attributes structure. 194 * 195 * @param rxq_data 196 * Pointer to Rx queue data. 197 * @param cqn 198 * CQ number to use with this RQ. 199 * @param rq_attr 200 * RQ attributes structure to fill.. 201 */ 202 static void 203 mlx5_devx_create_rq_attr_fill(struct mlx5_rxq_data *rxq_data, uint32_t cqn, 204 struct mlx5_devx_create_rq_attr *rq_attr) 205 { 206 rq_attr->state = MLX5_RQC_STATE_RST; 207 rq_attr->vsd = (rxq_data->vlan_strip) ? 0 : 1; 208 rq_attr->cqn = cqn; 209 rq_attr->scatter_fcs = (rxq_data->crc_present) ? 1 : 0; 210 } 211 212 /** 213 * Fill common fields of DevX WQ attributes structure. 214 * 215 * @param priv 216 * Pointer to device private data. 217 * @param rxq_ctrl 218 * Pointer to Rx queue control structure. 219 * @param wq_attr 220 * WQ attributes structure to fill.. 221 */ 222 static void 223 mlx5_devx_wq_attr_fill(struct mlx5_priv *priv, struct mlx5_rxq_ctrl *rxq_ctrl, 224 struct mlx5_devx_wq_attr *wq_attr) 225 { 226 wq_attr->end_padding_mode = priv->config.cqe_pad ? 227 MLX5_WQ_END_PAD_MODE_ALIGN : 228 MLX5_WQ_END_PAD_MODE_NONE; 229 wq_attr->pd = priv->sh->pdn; 230 wq_attr->dbr_addr = rxq_ctrl->rq_dbr_offset; 231 wq_attr->dbr_umem_id = 232 mlx5_os_get_umem_id(rxq_ctrl->rq_dbrec_page->umem); 233 wq_attr->dbr_umem_valid = 1; 234 wq_attr->wq_umem_id = mlx5_os_get_umem_id(rxq_ctrl->wq_umem); 235 wq_attr->wq_umem_valid = 1; 236 } 237 238 /** 239 * Create a RQ object using DevX. 240 * 241 * @param dev 242 * Pointer to Ethernet device. 243 * @param idx 244 * Queue index in DPDK Rx queue array. 245 * 246 * @return 247 * The DevX RQ object initialized, NULL otherwise and rte_errno is set. 248 */ 249 static struct mlx5_devx_obj * 250 rxq_create_devx_rq_resources(struct rte_eth_dev *dev, uint16_t idx) 251 { 252 struct mlx5_priv *priv = dev->data->dev_private; 253 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 254 struct mlx5_rxq_ctrl *rxq_ctrl = 255 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 256 struct mlx5_devx_create_rq_attr rq_attr = { 0 }; 257 uint32_t wqe_n = 1 << (rxq_data->elts_n - rxq_data->sges_n); 258 uint32_t cqn = rxq_ctrl->obj->devx_cq->id; 259 struct mlx5_devx_dbr_page *dbr_page; 260 int64_t dbr_offset; 261 uint32_t wq_size = 0; 262 uint32_t wqe_size = 0; 263 uint32_t log_wqe_size = 0; 264 void *buf = NULL; 265 struct mlx5_devx_obj *rq; 266 267 /* Fill RQ attributes. */ 268 rq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE; 269 rq_attr.flush_in_error_en = 1; 270 mlx5_devx_create_rq_attr_fill(rxq_data, cqn, &rq_attr); 271 /* Fill WQ attributes for this RQ. */ 272 if (mlx5_rxq_mprq_enabled(rxq_data)) { 273 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ; 274 /* 275 * Number of strides in each WQE: 276 * 512*2^single_wqe_log_num_of_strides. 277 */ 278 rq_attr.wq_attr.single_wqe_log_num_of_strides = 279 rxq_data->strd_num_n - 280 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES; 281 /* Stride size = (2^single_stride_log_num_of_bytes)*64B. */ 282 rq_attr.wq_attr.single_stride_log_num_of_bytes = 283 rxq_data->strd_sz_n - 284 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES; 285 wqe_size = sizeof(struct mlx5_wqe_mprq); 286 } else { 287 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 288 wqe_size = sizeof(struct mlx5_wqe_data_seg); 289 } 290 log_wqe_size = log2above(wqe_size) + rxq_data->sges_n; 291 rq_attr.wq_attr.log_wq_stride = log_wqe_size; 292 rq_attr.wq_attr.log_wq_sz = rxq_data->elts_n - rxq_data->sges_n; 293 /* Calculate and allocate WQ memory space. */ 294 wqe_size = 1 << log_wqe_size; /* round up power of two.*/ 295 wq_size = wqe_n * wqe_size; 296 size_t alignment = MLX5_WQE_BUF_ALIGNMENT; 297 if (alignment == (size_t)-1) { 298 DRV_LOG(ERR, "Failed to get mem page size"); 299 rte_errno = ENOMEM; 300 return NULL; 301 } 302 buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, wq_size, 303 alignment, rxq_ctrl->socket); 304 if (!buf) 305 return NULL; 306 rxq_data->wqes = buf; 307 rxq_ctrl->wq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, 308 buf, wq_size, 0); 309 if (!rxq_ctrl->wq_umem) 310 goto error; 311 /* Allocate RQ door-bell. */ 312 dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs, &dbr_page); 313 if (dbr_offset < 0) { 314 DRV_LOG(ERR, "Failed to allocate RQ door-bell."); 315 goto error; 316 } 317 rxq_ctrl->rq_dbr_offset = dbr_offset; 318 rxq_ctrl->rq_dbrec_page = dbr_page; 319 rxq_data->rq_db = (uint32_t *)((uintptr_t)dbr_page->dbrs + 320 (uintptr_t)rxq_ctrl->rq_dbr_offset); 321 /* Create RQ using DevX API. */ 322 mlx5_devx_wq_attr_fill(priv, rxq_ctrl, &rq_attr.wq_attr); 323 rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &rq_attr, rxq_ctrl->socket); 324 if (!rq) 325 goto error; 326 return rq; 327 error: 328 rxq_release_devx_rq_resources(rxq_ctrl); 329 return NULL; 330 } 331 332 /** 333 * Create a DevX CQ object for an Rx queue. 334 * 335 * @param dev 336 * Pointer to Ethernet device. 337 * @param idx 338 * Queue index in DPDK Rx queue array. 339 * 340 * @return 341 * The DevX CQ object initialized, NULL otherwise and rte_errno is set. 342 */ 343 static struct mlx5_devx_obj * 344 rxq_create_devx_cq_resources(struct rte_eth_dev *dev, uint16_t idx) 345 { 346 struct mlx5_devx_obj *cq_obj = 0; 347 struct mlx5_devx_cq_attr cq_attr = { 0 }; 348 struct mlx5_priv *priv = dev->data->dev_private; 349 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 350 struct mlx5_rxq_ctrl *rxq_ctrl = 351 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 352 size_t page_size = rte_mem_page_size(); 353 uint32_t lcore = (uint32_t)rte_lcore_to_cpu_id(-1); 354 unsigned int cqe_n = mlx5_rxq_cqe_num(rxq_data); 355 struct mlx5_devx_dbr_page *dbr_page; 356 int64_t dbr_offset; 357 uint32_t eqn = 0; 358 void *buf = NULL; 359 uint16_t event_nums[1] = {0}; 360 uint32_t log_cqe_n; 361 uint32_t cq_size; 362 int ret = 0; 363 364 if (page_size == (size_t)-1) { 365 DRV_LOG(ERR, "Failed to get page_size."); 366 goto error; 367 } 368 if (priv->config.cqe_comp && !rxq_data->hw_timestamp && 369 !rxq_data->lro) { 370 cq_attr.cqe_comp_en = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE; 371 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT 372 cq_attr.mini_cqe_res_format = 373 mlx5_rxq_mprq_enabled(rxq_data) ? 374 MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX : 375 MLX5DV_CQE_RES_FORMAT_HASH; 376 #else 377 cq_attr.mini_cqe_res_format = MLX5DV_CQE_RES_FORMAT_HASH; 378 #endif 379 /* 380 * For vectorized Rx, it must not be doubled in order to 381 * make cq_ci and rq_ci aligned. 382 */ 383 if (mlx5_rxq_check_vec_support(rxq_data) < 0) 384 cqe_n *= 2; 385 } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) { 386 DRV_LOG(DEBUG, 387 "Port %u Rx CQE compression is disabled for HW" 388 " timestamp.", 389 dev->data->port_id); 390 } else if (priv->config.cqe_comp && rxq_data->lro) { 391 DRV_LOG(DEBUG, 392 "Port %u Rx CQE compression is disabled for LRO.", 393 dev->data->port_id); 394 } 395 #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD 396 if (priv->config.cqe_pad) 397 cq_attr.cqe_size = MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD; 398 #endif 399 log_cqe_n = log2above(cqe_n); 400 cq_size = sizeof(struct mlx5_cqe) * (1 << log_cqe_n); 401 /* Query the EQN for this core. */ 402 if (mlx5_glue->devx_query_eqn(priv->sh->ctx, lcore, &eqn)) { 403 DRV_LOG(ERR, "Failed to query EQN for CQ."); 404 goto error; 405 } 406 cq_attr.eqn = eqn; 407 buf = rte_calloc_socket(__func__, 1, cq_size, page_size, 408 rxq_ctrl->socket); 409 if (!buf) { 410 DRV_LOG(ERR, "Failed to allocate memory for CQ."); 411 goto error; 412 } 413 rxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)buf; 414 rxq_ctrl->cq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, buf, 415 cq_size, 416 IBV_ACCESS_LOCAL_WRITE); 417 if (!rxq_ctrl->cq_umem) { 418 DRV_LOG(ERR, "Failed to register umem for CQ."); 419 goto error; 420 } 421 /* Allocate CQ door-bell. */ 422 dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs, &dbr_page); 423 if (dbr_offset < 0) { 424 DRV_LOG(ERR, "Failed to allocate CQ door-bell."); 425 goto error; 426 } 427 rxq_ctrl->cq_dbr_offset = dbr_offset; 428 rxq_ctrl->cq_dbrec_page = dbr_page; 429 rxq_data->cq_db = (uint32_t *)((uintptr_t)dbr_page->dbrs + 430 (uintptr_t)rxq_ctrl->cq_dbr_offset); 431 rxq_data->cq_uar = 432 mlx5_os_get_devx_uar_base_addr(priv->sh->devx_rx_uar); 433 /* Create CQ using DevX API. */ 434 cq_attr.uar_page_id = 435 mlx5_os_get_devx_uar_page_id(priv->sh->devx_rx_uar); 436 cq_attr.q_umem_id = mlx5_os_get_umem_id(rxq_ctrl->cq_umem); 437 cq_attr.q_umem_valid = 1; 438 cq_attr.log_cq_size = log_cqe_n; 439 cq_attr.log_page_size = rte_log2_u32(page_size); 440 cq_attr.db_umem_offset = rxq_ctrl->cq_dbr_offset; 441 cq_attr.db_umem_id = mlx5_os_get_umem_id(dbr_page->umem); 442 cq_attr.db_umem_valid = 1; 443 cq_obj = mlx5_devx_cmd_create_cq(priv->sh->ctx, &cq_attr); 444 if (!cq_obj) 445 goto error; 446 rxq_data->cqe_n = log_cqe_n; 447 rxq_data->cqn = cq_obj->id; 448 if (rxq_ctrl->obj->devx_channel) { 449 ret = mlx5_glue->devx_subscribe_devx_event 450 (rxq_ctrl->obj->devx_channel, 451 cq_obj->obj, 452 sizeof(event_nums), 453 event_nums, 454 (uint64_t)(uintptr_t)cq_obj); 455 if (ret) { 456 DRV_LOG(ERR, "Fail to subscribe CQ to event channel."); 457 rte_errno = errno; 458 goto error; 459 } 460 } 461 /* Initialise CQ to 1's to mark HW ownership for all CQEs. */ 462 memset((void *)(uintptr_t)rxq_data->cqes, 0xFF, cq_size); 463 return cq_obj; 464 error: 465 if (cq_obj) 466 mlx5_devx_cmd_destroy(cq_obj); 467 rxq_release_devx_cq_resources(rxq_ctrl); 468 return NULL; 469 } 470 471 /** 472 * Create the Rx hairpin queue object. 473 * 474 * @param dev 475 * Pointer to Ethernet device. 476 * @param idx 477 * Queue index in DPDK Rx queue array. 478 * 479 * @return 480 * 0 on success, a negative errno value otherwise and rte_errno is set. 481 */ 482 static int 483 mlx5_rxq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) 484 { 485 struct mlx5_priv *priv = dev->data->dev_private; 486 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 487 struct mlx5_rxq_ctrl *rxq_ctrl = 488 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 489 struct mlx5_devx_create_rq_attr attr = { 0 }; 490 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 491 uint32_t max_wq_data; 492 493 MLX5_ASSERT(rxq_data); 494 MLX5_ASSERT(tmpl); 495 tmpl->type = MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN; 496 tmpl->rxq_ctrl = rxq_ctrl; 497 attr.hairpin = 1; 498 max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; 499 /* Jumbo frames > 9KB should be supported, and more packets. */ 500 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 501 if (priv->config.log_hp_size > max_wq_data) { 502 DRV_LOG(ERR, "Total data size %u power of 2 is " 503 "too large for hairpin.", 504 priv->config.log_hp_size); 505 rte_errno = ERANGE; 506 return -rte_errno; 507 } 508 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 509 } else { 510 attr.wq_attr.log_hairpin_data_sz = 511 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 512 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 513 } 514 /* Set the packets number to the maximum value for performance. */ 515 attr.wq_attr.log_hairpin_num_packets = 516 attr.wq_attr.log_hairpin_data_sz - 517 MLX5_HAIRPIN_QUEUE_STRIDE; 518 tmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &attr, 519 rxq_ctrl->socket); 520 if (!tmpl->rq) { 521 DRV_LOG(ERR, 522 "Port %u Rx hairpin queue %u can't create rq object.", 523 dev->data->port_id, idx); 524 rte_errno = errno; 525 return -rte_errno; 526 } 527 dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN; 528 return 0; 529 } 530 531 /** 532 * Create the Rx queue DevX object. 533 * 534 * @param dev 535 * Pointer to Ethernet device. 536 * @param idx 537 * Queue index in DPDK Rx queue array. 538 * 539 * @return 540 * 0 on success, a negative errno value otherwise and rte_errno is set. 541 */ 542 static int 543 mlx5_rxq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx) 544 { 545 struct mlx5_priv *priv = dev->data->dev_private; 546 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 547 struct mlx5_rxq_ctrl *rxq_ctrl = 548 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 549 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 550 int ret = 0; 551 552 MLX5_ASSERT(rxq_data); 553 MLX5_ASSERT(tmpl); 554 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) 555 return mlx5_rxq_obj_hairpin_new(dev, idx); 556 tmpl->type = MLX5_RXQ_OBJ_TYPE_DEVX_RQ; 557 tmpl->rxq_ctrl = rxq_ctrl; 558 if (rxq_ctrl->irq) { 559 int devx_ev_flag = 560 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA; 561 562 tmpl->devx_channel = mlx5_glue->devx_create_event_channel 563 (priv->sh->ctx, 564 devx_ev_flag); 565 if (!tmpl->devx_channel) { 566 rte_errno = errno; 567 DRV_LOG(ERR, "Failed to create event channel %d.", 568 rte_errno); 569 goto error; 570 } 571 tmpl->fd = mlx5_os_get_devx_channel_fd(tmpl->devx_channel); 572 } 573 /* Create CQ using DevX API. */ 574 tmpl->devx_cq = rxq_create_devx_cq_resources(dev, idx); 575 if (!tmpl->devx_cq) { 576 DRV_LOG(ERR, "Failed to create CQ."); 577 goto error; 578 } 579 /* Create RQ using DevX API. */ 580 tmpl->rq = rxq_create_devx_rq_resources(dev, idx); 581 if (!tmpl->rq) { 582 DRV_LOG(ERR, "Port %u Rx queue %u RQ creation failure.", 583 dev->data->port_id, idx); 584 rte_errno = ENOMEM; 585 goto error; 586 } 587 /* Change queue state to ready. */ 588 ret = mlx5_devx_modify_rq(tmpl, true); 589 if (ret) 590 goto error; 591 rxq_data->cq_arm_sn = 0; 592 mlx5_rxq_initialize(rxq_data); 593 rxq_data->cq_ci = 0; 594 dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED; 595 rxq_ctrl->wqn = tmpl->rq->id; 596 return 0; 597 error: 598 ret = rte_errno; /* Save rte_errno before cleanup. */ 599 if (tmpl->rq) 600 claim_zero(mlx5_devx_cmd_destroy(tmpl->rq)); 601 if (tmpl->devx_cq) 602 claim_zero(mlx5_devx_cmd_destroy(tmpl->devx_cq)); 603 if (tmpl->devx_channel) 604 mlx5_glue->devx_destroy_event_channel(tmpl->devx_channel); 605 rxq_release_devx_rq_resources(rxq_ctrl); 606 rxq_release_devx_cq_resources(rxq_ctrl); 607 rte_errno = ret; /* Restore rte_errno. */ 608 return -rte_errno; 609 } 610 611 /** 612 * Create RQT using DevX API as a filed of indirection table. 613 * 614 * @param dev 615 * Pointer to Ethernet device. 616 * @param log_n 617 * Log of number of queues in the array. 618 * @param ind_tbl 619 * DevX indirection table object. 620 * 621 * @return 622 * 0 on success, a negative errno value otherwise and rte_errno is set. 623 */ 624 static int 625 mlx5_devx_ind_table_new(struct rte_eth_dev *dev, const unsigned int log_n, 626 struct mlx5_ind_table_obj *ind_tbl) 627 { 628 struct mlx5_priv *priv = dev->data->dev_private; 629 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 630 const unsigned int rqt_n = 1 << log_n; 631 unsigned int i, j; 632 633 MLX5_ASSERT(ind_tbl); 634 rqt_attr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rqt_attr) + 635 rqt_n * sizeof(uint32_t), 0, SOCKET_ID_ANY); 636 if (!rqt_attr) { 637 DRV_LOG(ERR, "Port %u cannot allocate RQT resources.", 638 dev->data->port_id); 639 rte_errno = ENOMEM; 640 return -rte_errno; 641 } 642 rqt_attr->rqt_max_size = priv->config.ind_table_max_size; 643 rqt_attr->rqt_actual_size = rqt_n; 644 for (i = 0; i != ind_tbl->queues_n; ++i) { 645 struct mlx5_rxq_data *rxq = (*priv->rxqs)[ind_tbl->queues[i]]; 646 struct mlx5_rxq_ctrl *rxq_ctrl = 647 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 648 649 rqt_attr->rq_list[i] = rxq_ctrl->obj->rq->id; 650 } 651 MLX5_ASSERT(i > 0); 652 for (j = 0; i != rqt_n; ++j, ++i) 653 rqt_attr->rq_list[i] = rqt_attr->rq_list[j]; 654 ind_tbl->rqt = mlx5_devx_cmd_create_rqt(priv->sh->ctx, rqt_attr); 655 mlx5_free(rqt_attr); 656 if (!ind_tbl->rqt) { 657 DRV_LOG(ERR, "Port %u cannot create DevX RQT.", 658 dev->data->port_id); 659 rte_errno = errno; 660 return -rte_errno; 661 } 662 return 0; 663 } 664 665 /** 666 * Destroy the DevX RQT object. 667 * 668 * @param ind_table 669 * Indirection table to release. 670 */ 671 static void 672 mlx5_devx_ind_table_destroy(struct mlx5_ind_table_obj *ind_tbl) 673 { 674 claim_zero(mlx5_devx_cmd_destroy(ind_tbl->rqt)); 675 } 676 677 /** 678 * Create an Rx Hash queue. 679 * 680 * @param dev 681 * Pointer to Ethernet device. 682 * @param rss_key 683 * RSS key for the Rx hash queue. 684 * @param rss_key_len 685 * RSS key length. 686 * @param hash_fields 687 * Verbs protocol hash field to make the RSS on. 688 * @param queues 689 * Queues entering in hash queue. In case of empty hash_fields only the 690 * first queue index will be taken for the indirection table. 691 * @param queues_n 692 * Number of queues. 693 * @param tunnel 694 * Tunnel type. 695 * 696 * @return 697 * The DevX object initialized index, 0 otherwise and rte_errno is set. 698 */ 699 static uint32_t 700 mlx5_devx_hrxq_new(struct rte_eth_dev *dev, 701 const uint8_t *rss_key, uint32_t rss_key_len, 702 uint64_t hash_fields, 703 const uint16_t *queues, uint32_t queues_n, 704 int tunnel __rte_unused) 705 { 706 struct mlx5_priv *priv = dev->data->dev_private; 707 struct mlx5_hrxq *hrxq = NULL; 708 uint32_t hrxq_idx = 0; 709 struct mlx5_ind_table_obj *ind_tbl; 710 struct mlx5_devx_obj *tir = NULL; 711 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[queues[0]]; 712 struct mlx5_rxq_ctrl *rxq_ctrl = 713 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 714 struct mlx5_devx_tir_attr tir_attr; 715 int err; 716 uint32_t i; 717 bool lro = true; 718 719 queues_n = hash_fields ? queues_n : 1; 720 ind_tbl = mlx5_ind_table_obj_get(dev, queues, queues_n); 721 if (!ind_tbl) 722 ind_tbl = mlx5_ind_table_obj_new(dev, queues, queues_n); 723 if (!ind_tbl) { 724 rte_errno = ENOMEM; 725 return 0; 726 } 727 /* Enable TIR LRO only if all the queues were configured for. */ 728 for (i = 0; i < queues_n; ++i) { 729 if (!(*priv->rxqs)[queues[i]]->lro) { 730 lro = false; 731 break; 732 } 733 } 734 memset(&tir_attr, 0, sizeof(tir_attr)); 735 tir_attr.disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT; 736 tir_attr.rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ; 737 tir_attr.tunneled_offload_en = !!tunnel; 738 /* If needed, translate hash_fields bitmap to PRM format. */ 739 if (hash_fields) { 740 struct mlx5_rx_hash_field_select *rx_hash_field_select = NULL; 741 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 742 rx_hash_field_select = hash_fields & IBV_RX_HASH_INNER ? 743 &tir_attr.rx_hash_field_selector_inner : 744 &tir_attr.rx_hash_field_selector_outer; 745 #else 746 rx_hash_field_select = &tir_attr.rx_hash_field_selector_outer; 747 #endif 748 /* 1 bit: 0: IPv4, 1: IPv6. */ 749 rx_hash_field_select->l3_prot_type = 750 !!(hash_fields & MLX5_IPV6_IBV_RX_HASH); 751 /* 1 bit: 0: TCP, 1: UDP. */ 752 rx_hash_field_select->l4_prot_type = 753 !!(hash_fields & MLX5_UDP_IBV_RX_HASH); 754 /* Bitmask which sets which fields to use in RX Hash. */ 755 rx_hash_field_select->selected_fields = 756 ((!!(hash_fields & MLX5_L3_SRC_IBV_RX_HASH)) << 757 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) | 758 (!!(hash_fields & MLX5_L3_DST_IBV_RX_HASH)) << 759 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP | 760 (!!(hash_fields & MLX5_L4_SRC_IBV_RX_HASH)) << 761 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT | 762 (!!(hash_fields & MLX5_L4_DST_IBV_RX_HASH)) << 763 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT; 764 } 765 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) 766 tir_attr.transport_domain = priv->sh->td->id; 767 else 768 tir_attr.transport_domain = priv->sh->tdn; 769 memcpy(tir_attr.rx_hash_toeplitz_key, rss_key, MLX5_RSS_HASH_KEY_LEN); 770 tir_attr.indirect_table = ind_tbl->rqt->id; 771 if (dev->data->dev_conf.lpbk_mode) 772 tir_attr.self_lb_block = MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST; 773 if (lro) { 774 tir_attr.lro_timeout_period_usecs = priv->config.lro.timeout; 775 tir_attr.lro_max_msg_sz = priv->max_lro_msg_size; 776 tir_attr.lro_enable_mask = MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | 777 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO; 778 } 779 tir = mlx5_devx_cmd_create_tir(priv->sh->ctx, &tir_attr); 780 if (!tir) { 781 DRV_LOG(ERR, "Port %u cannot create DevX TIR.", 782 dev->data->port_id); 783 rte_errno = errno; 784 goto error; 785 } 786 hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx); 787 if (!hrxq) 788 goto error; 789 hrxq->ind_table = ind_tbl; 790 hrxq->tir = tir; 791 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 792 hrxq->action = mlx5_glue->dv_create_flow_action_dest_devx_tir 793 (hrxq->tir->obj); 794 if (!hrxq->action) { 795 rte_errno = errno; 796 goto error; 797 } 798 #endif 799 hrxq->rss_key_len = rss_key_len; 800 hrxq->hash_fields = hash_fields; 801 memcpy(hrxq->rss_key, rss_key, rss_key_len); 802 rte_atomic32_inc(&hrxq->refcnt); 803 ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_HRXQ], &priv->hrxqs, hrxq_idx, 804 hrxq, next); 805 return hrxq_idx; 806 error: 807 err = rte_errno; /* Save rte_errno before cleanup. */ 808 mlx5_ind_table_obj_release(dev, ind_tbl); 809 if (tir) 810 claim_zero(mlx5_devx_cmd_destroy(tir)); 811 if (hrxq) 812 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); 813 rte_errno = err; /* Restore rte_errno. */ 814 return 0; 815 } 816 817 /** 818 * Destroy a DevX TIR object. 819 * 820 * @param hrxq 821 * Hash Rx queue to release its tir. 822 */ 823 static void 824 mlx5_devx_tir_destroy(struct mlx5_hrxq *hrxq) 825 { 826 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 827 } 828 829 struct mlx5_obj_ops devx_obj_ops = { 830 .rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_rq_vlan_strip, 831 .rxq_obj_new = mlx5_rxq_devx_obj_new, 832 .rxq_event_get = mlx5_rx_devx_get_event, 833 .rxq_obj_modify = mlx5_devx_modify_rq, 834 .rxq_obj_release = mlx5_rxq_devx_obj_release, 835 .ind_table_new = mlx5_devx_ind_table_new, 836 .ind_table_destroy = mlx5_devx_ind_table_destroy, 837 .hrxq_new = mlx5_devx_hrxq_new, 838 .hrxq_destroy = mlx5_devx_tir_destroy, 839 }; 840