1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <stddef.h> 6 #include <errno.h> 7 #include <stdbool.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <sys/queue.h> 11 12 #include <rte_malloc.h> 13 #include <rte_common.h> 14 #include <rte_eal_paging.h> 15 16 #include <mlx5_glue.h> 17 #include <mlx5_devx_cmds.h> 18 #include <mlx5_malloc.h> 19 20 #include "mlx5.h" 21 #include "mlx5_common_os.h" 22 #include "mlx5_rxtx.h" 23 #include "mlx5_utils.h" 24 #include "mlx5_devx.h" 25 #include "mlx5_flow.h" 26 27 28 /** 29 * Modify RQ vlan stripping offload 30 * 31 * @param rxq_obj 32 * Rx queue object. 33 * 34 * @return 35 * 0 on success, non-0 otherwise 36 */ 37 static int 38 mlx5_rxq_obj_modify_rq_vlan_strip(struct mlx5_rxq_obj *rxq_obj, int on) 39 { 40 struct mlx5_devx_modify_rq_attr rq_attr; 41 42 memset(&rq_attr, 0, sizeof(rq_attr)); 43 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 44 rq_attr.state = MLX5_RQC_STATE_RDY; 45 rq_attr.vsd = (on ? 0 : 1); 46 rq_attr.modify_bitmask = MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD; 47 return mlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr); 48 } 49 50 /** 51 * Modify RQ using DevX API. 52 * 53 * @param rxq_obj 54 * DevX Rx queue object. 55 * @param type 56 * Type of change queue state. 57 * 58 * @return 59 * 0 on success, a negative errno value otherwise and rte_errno is set. 60 */ 61 static int 62 mlx5_devx_modify_rq(struct mlx5_rxq_obj *rxq_obj, uint8_t type) 63 { 64 struct mlx5_devx_modify_rq_attr rq_attr; 65 66 memset(&rq_attr, 0, sizeof(rq_attr)); 67 switch (type) { 68 case MLX5_RXQ_MOD_ERR2RST: 69 rq_attr.rq_state = MLX5_RQC_STATE_ERR; 70 rq_attr.state = MLX5_RQC_STATE_RST; 71 break; 72 case MLX5_RXQ_MOD_RST2RDY: 73 rq_attr.rq_state = MLX5_RQC_STATE_RST; 74 rq_attr.state = MLX5_RQC_STATE_RDY; 75 break; 76 case MLX5_RXQ_MOD_RDY2ERR: 77 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 78 rq_attr.state = MLX5_RQC_STATE_ERR; 79 break; 80 case MLX5_RXQ_MOD_RDY2RST: 81 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 82 rq_attr.state = MLX5_RQC_STATE_RST; 83 break; 84 default: 85 break; 86 } 87 return mlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr); 88 } 89 90 /** 91 * Modify SQ using DevX API. 92 * 93 * @param txq_obj 94 * DevX Tx queue object. 95 * @param type 96 * Type of change queue state. 97 * @param dev_port 98 * Unnecessary. 99 * 100 * @return 101 * 0 on success, a negative errno value otherwise and rte_errno is set. 102 */ 103 static int 104 mlx5_devx_modify_sq(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type, 105 uint8_t dev_port) 106 { 107 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 108 int ret; 109 110 if (type != MLX5_TXQ_MOD_RST2RDY) { 111 /* Change queue state to reset. */ 112 if (type == MLX5_TXQ_MOD_ERR2RDY) 113 msq_attr.sq_state = MLX5_SQC_STATE_ERR; 114 else 115 msq_attr.sq_state = MLX5_SQC_STATE_RDY; 116 msq_attr.state = MLX5_SQC_STATE_RST; 117 ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr); 118 if (ret) { 119 DRV_LOG(ERR, "Cannot change the Tx SQ state to RESET" 120 " %s", strerror(errno)); 121 rte_errno = errno; 122 return ret; 123 } 124 } 125 if (type != MLX5_TXQ_MOD_RDY2RST) { 126 /* Change queue state to ready. */ 127 msq_attr.sq_state = MLX5_SQC_STATE_RST; 128 msq_attr.state = MLX5_SQC_STATE_RDY; 129 ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr); 130 if (ret) { 131 DRV_LOG(ERR, "Cannot change the Tx SQ state to READY" 132 " %s", strerror(errno)); 133 rte_errno = errno; 134 return ret; 135 } 136 } 137 /* 138 * The dev_port variable is relevant only in Verbs API, and there is a 139 * pointer that points to this function and a parallel function in verbs 140 * intermittently, so they should have the same parameters. 141 */ 142 (void)dev_port; 143 return 0; 144 } 145 146 /** 147 * Release the resources allocated for an RQ DevX object. 148 * 149 * @param rxq_ctrl 150 * DevX Rx queue object. 151 */ 152 static void 153 mlx5_rxq_release_devx_rq_resources(struct mlx5_rxq_ctrl *rxq_ctrl) 154 { 155 struct mlx5_devx_dbr_page *dbr_page = rxq_ctrl->rq_dbrec_page; 156 157 if (rxq_ctrl->rxq.wqes) { 158 mlx5_free((void *)(uintptr_t)rxq_ctrl->rxq.wqes); 159 rxq_ctrl->rxq.wqes = NULL; 160 } 161 if (rxq_ctrl->wq_umem) { 162 mlx5_glue->devx_umem_dereg(rxq_ctrl->wq_umem); 163 rxq_ctrl->wq_umem = NULL; 164 } 165 if (dbr_page) { 166 claim_zero(mlx5_release_dbr(&rxq_ctrl->priv->dbrpgs, 167 mlx5_os_get_umem_id(dbr_page->umem), 168 rxq_ctrl->rq_dbr_offset)); 169 rxq_ctrl->rq_dbrec_page = NULL; 170 } 171 } 172 173 /** 174 * Release the resources allocated for the Rx CQ DevX object. 175 * 176 * @param rxq_ctrl 177 * DevX Rx queue object. 178 */ 179 static void 180 mlx5_rxq_release_devx_cq_resources(struct mlx5_rxq_ctrl *rxq_ctrl) 181 { 182 struct mlx5_devx_dbr_page *dbr_page = rxq_ctrl->cq_dbrec_page; 183 184 if (rxq_ctrl->rxq.cqes) { 185 rte_free((void *)(uintptr_t)rxq_ctrl->rxq.cqes); 186 rxq_ctrl->rxq.cqes = NULL; 187 } 188 if (rxq_ctrl->cq_umem) { 189 mlx5_glue->devx_umem_dereg(rxq_ctrl->cq_umem); 190 rxq_ctrl->cq_umem = NULL; 191 } 192 if (dbr_page) { 193 claim_zero(mlx5_release_dbr(&rxq_ctrl->priv->dbrpgs, 194 mlx5_os_get_umem_id(dbr_page->umem), 195 rxq_ctrl->cq_dbr_offset)); 196 rxq_ctrl->cq_dbrec_page = NULL; 197 } 198 } 199 200 /** 201 * Release an Rx DevX queue object. 202 * 203 * @param rxq_obj 204 * DevX Rx queue object. 205 */ 206 static void 207 mlx5_rxq_devx_obj_release(struct mlx5_rxq_obj *rxq_obj) 208 { 209 MLX5_ASSERT(rxq_obj); 210 MLX5_ASSERT(rxq_obj->rq); 211 if (rxq_obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN) { 212 mlx5_devx_modify_rq(rxq_obj, MLX5_RXQ_MOD_RDY2RST); 213 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 214 } else { 215 MLX5_ASSERT(rxq_obj->devx_cq); 216 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 217 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->devx_cq)); 218 if (rxq_obj->devx_channel) 219 mlx5_glue->devx_destroy_event_channel 220 (rxq_obj->devx_channel); 221 mlx5_rxq_release_devx_rq_resources(rxq_obj->rxq_ctrl); 222 mlx5_rxq_release_devx_cq_resources(rxq_obj->rxq_ctrl); 223 } 224 } 225 226 /** 227 * Get event for an Rx DevX queue object. 228 * 229 * @param rxq_obj 230 * DevX Rx queue object. 231 * 232 * @return 233 * 0 on success, a negative errno value otherwise and rte_errno is set. 234 */ 235 static int 236 mlx5_rx_devx_get_event(struct mlx5_rxq_obj *rxq_obj) 237 { 238 #ifdef HAVE_IBV_DEVX_EVENT 239 union { 240 struct mlx5dv_devx_async_event_hdr event_resp; 241 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 242 } out; 243 int ret = mlx5_glue->devx_get_event(rxq_obj->devx_channel, 244 &out.event_resp, 245 sizeof(out.buf)); 246 247 if (ret < 0) { 248 rte_errno = errno; 249 return -rte_errno; 250 } 251 if (out.event_resp.cookie != (uint64_t)(uintptr_t)rxq_obj->devx_cq) { 252 rte_errno = EINVAL; 253 return -rte_errno; 254 } 255 return 0; 256 #else 257 (void)rxq_obj; 258 rte_errno = ENOTSUP; 259 return -rte_errno; 260 #endif /* HAVE_IBV_DEVX_EVENT */ 261 } 262 263 /** 264 * Fill common fields of create RQ attributes structure. 265 * 266 * @param rxq_data 267 * Pointer to Rx queue data. 268 * @param cqn 269 * CQ number to use with this RQ. 270 * @param rq_attr 271 * RQ attributes structure to fill.. 272 */ 273 static void 274 mlx5_devx_create_rq_attr_fill(struct mlx5_rxq_data *rxq_data, uint32_t cqn, 275 struct mlx5_devx_create_rq_attr *rq_attr) 276 { 277 rq_attr->state = MLX5_RQC_STATE_RST; 278 rq_attr->vsd = (rxq_data->vlan_strip) ? 0 : 1; 279 rq_attr->cqn = cqn; 280 rq_attr->scatter_fcs = (rxq_data->crc_present) ? 1 : 0; 281 } 282 283 /** 284 * Fill common fields of DevX WQ attributes structure. 285 * 286 * @param priv 287 * Pointer to device private data. 288 * @param rxq_ctrl 289 * Pointer to Rx queue control structure. 290 * @param wq_attr 291 * WQ attributes structure to fill.. 292 */ 293 static void 294 mlx5_devx_wq_attr_fill(struct mlx5_priv *priv, struct mlx5_rxq_ctrl *rxq_ctrl, 295 struct mlx5_devx_wq_attr *wq_attr) 296 { 297 wq_attr->end_padding_mode = priv->config.cqe_pad ? 298 MLX5_WQ_END_PAD_MODE_ALIGN : 299 MLX5_WQ_END_PAD_MODE_NONE; 300 wq_attr->pd = priv->sh->pdn; 301 wq_attr->dbr_addr = rxq_ctrl->rq_dbr_offset; 302 wq_attr->dbr_umem_id = 303 mlx5_os_get_umem_id(rxq_ctrl->rq_dbrec_page->umem); 304 wq_attr->dbr_umem_valid = 1; 305 wq_attr->wq_umem_id = mlx5_os_get_umem_id(rxq_ctrl->wq_umem); 306 wq_attr->wq_umem_valid = 1; 307 } 308 309 /** 310 * Create a RQ object using DevX. 311 * 312 * @param dev 313 * Pointer to Ethernet device. 314 * @param idx 315 * Queue index in DPDK Rx queue array. 316 * 317 * @return 318 * The DevX RQ object initialized, NULL otherwise and rte_errno is set. 319 */ 320 static struct mlx5_devx_obj * 321 mlx5_rxq_create_devx_rq_resources(struct rte_eth_dev *dev, uint16_t idx) 322 { 323 struct mlx5_priv *priv = dev->data->dev_private; 324 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 325 struct mlx5_rxq_ctrl *rxq_ctrl = 326 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 327 struct mlx5_devx_create_rq_attr rq_attr = { 0 }; 328 uint32_t wqe_n = 1 << (rxq_data->elts_n - rxq_data->sges_n); 329 uint32_t cqn = rxq_ctrl->obj->devx_cq->id; 330 struct mlx5_devx_dbr_page *dbr_page; 331 int64_t dbr_offset; 332 uint32_t wq_size = 0; 333 uint32_t wqe_size = 0; 334 uint32_t log_wqe_size = 0; 335 void *buf = NULL; 336 struct mlx5_devx_obj *rq; 337 338 /* Fill RQ attributes. */ 339 rq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE; 340 rq_attr.flush_in_error_en = 1; 341 mlx5_devx_create_rq_attr_fill(rxq_data, cqn, &rq_attr); 342 /* Fill WQ attributes for this RQ. */ 343 if (mlx5_rxq_mprq_enabled(rxq_data)) { 344 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ; 345 /* 346 * Number of strides in each WQE: 347 * 512*2^single_wqe_log_num_of_strides. 348 */ 349 rq_attr.wq_attr.single_wqe_log_num_of_strides = 350 rxq_data->strd_num_n - 351 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES; 352 /* Stride size = (2^single_stride_log_num_of_bytes)*64B. */ 353 rq_attr.wq_attr.single_stride_log_num_of_bytes = 354 rxq_data->strd_sz_n - 355 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES; 356 wqe_size = sizeof(struct mlx5_wqe_mprq); 357 } else { 358 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 359 wqe_size = sizeof(struct mlx5_wqe_data_seg); 360 } 361 log_wqe_size = log2above(wqe_size) + rxq_data->sges_n; 362 rq_attr.wq_attr.log_wq_stride = log_wqe_size; 363 rq_attr.wq_attr.log_wq_sz = rxq_data->elts_n - rxq_data->sges_n; 364 /* Calculate and allocate WQ memory space. */ 365 wqe_size = 1 << log_wqe_size; /* round up power of two.*/ 366 wq_size = wqe_n * wqe_size; 367 size_t alignment = MLX5_WQE_BUF_ALIGNMENT; 368 if (alignment == (size_t)-1) { 369 DRV_LOG(ERR, "Failed to get mem page size"); 370 rte_errno = ENOMEM; 371 return NULL; 372 } 373 buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, wq_size, 374 alignment, rxq_ctrl->socket); 375 if (!buf) 376 return NULL; 377 rxq_data->wqes = buf; 378 rxq_ctrl->wq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, 379 buf, wq_size, 0); 380 if (!rxq_ctrl->wq_umem) 381 goto error; 382 /* Allocate RQ door-bell. */ 383 dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs, &dbr_page); 384 if (dbr_offset < 0) { 385 DRV_LOG(ERR, "Failed to allocate RQ door-bell."); 386 goto error; 387 } 388 rxq_ctrl->rq_dbr_offset = dbr_offset; 389 rxq_ctrl->rq_dbrec_page = dbr_page; 390 rxq_data->rq_db = (uint32_t *)((uintptr_t)dbr_page->dbrs + 391 (uintptr_t)rxq_ctrl->rq_dbr_offset); 392 /* Create RQ using DevX API. */ 393 mlx5_devx_wq_attr_fill(priv, rxq_ctrl, &rq_attr.wq_attr); 394 rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &rq_attr, rxq_ctrl->socket); 395 if (!rq) 396 goto error; 397 return rq; 398 error: 399 mlx5_rxq_release_devx_rq_resources(rxq_ctrl); 400 return NULL; 401 } 402 403 /** 404 * Create a DevX CQ object for an Rx queue. 405 * 406 * @param dev 407 * Pointer to Ethernet device. 408 * @param idx 409 * Queue index in DPDK Rx queue array. 410 * 411 * @return 412 * The DevX CQ object initialized, NULL otherwise and rte_errno is set. 413 */ 414 static struct mlx5_devx_obj * 415 mlx5_rxq_create_devx_cq_resources(struct rte_eth_dev *dev, uint16_t idx) 416 { 417 struct mlx5_devx_obj *cq_obj = 0; 418 struct mlx5_devx_cq_attr cq_attr = { 0 }; 419 struct mlx5_priv *priv = dev->data->dev_private; 420 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 421 struct mlx5_rxq_ctrl *rxq_ctrl = 422 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 423 size_t page_size = rte_mem_page_size(); 424 unsigned int cqe_n = mlx5_rxq_cqe_num(rxq_data); 425 struct mlx5_devx_dbr_page *dbr_page; 426 int64_t dbr_offset; 427 void *buf = NULL; 428 uint16_t event_nums[1] = {0}; 429 uint32_t log_cqe_n; 430 uint32_t cq_size; 431 int ret = 0; 432 433 if (page_size == (size_t)-1) { 434 DRV_LOG(ERR, "Failed to get page_size."); 435 goto error; 436 } 437 if (priv->config.cqe_comp && !rxq_data->hw_timestamp && 438 !rxq_data->lro) { 439 cq_attr.cqe_comp_en = 1u; 440 cq_attr.mini_cqe_res_format = 441 mlx5_rxq_mprq_enabled(rxq_data) ? 442 MLX5_CQE_RESP_FORMAT_CSUM_STRIDX : 443 MLX5_CQE_RESP_FORMAT_HASH; 444 /* 445 * For vectorized Rx, it must not be doubled in order to 446 * make cq_ci and rq_ci aligned. 447 */ 448 if (mlx5_rxq_check_vec_support(rxq_data) < 0) 449 cqe_n *= 2; 450 } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) { 451 DRV_LOG(DEBUG, 452 "Port %u Rx CQE compression is disabled for HW" 453 " timestamp.", 454 dev->data->port_id); 455 } else if (priv->config.cqe_comp && rxq_data->lro) { 456 DRV_LOG(DEBUG, 457 "Port %u Rx CQE compression is disabled for LRO.", 458 dev->data->port_id); 459 } 460 if (priv->config.cqe_pad) 461 cq_attr.cqe_size = MLX5_CQE_SIZE_128B; 462 log_cqe_n = log2above(cqe_n); 463 cq_size = sizeof(struct mlx5_cqe) * (1 << log_cqe_n); 464 buf = rte_calloc_socket(__func__, 1, cq_size, page_size, 465 rxq_ctrl->socket); 466 if (!buf) { 467 DRV_LOG(ERR, "Failed to allocate memory for CQ."); 468 goto error; 469 } 470 rxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)buf; 471 rxq_ctrl->cq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, buf, 472 cq_size, 473 IBV_ACCESS_LOCAL_WRITE); 474 if (!rxq_ctrl->cq_umem) { 475 DRV_LOG(ERR, "Failed to register umem for CQ."); 476 goto error; 477 } 478 /* Allocate CQ door-bell. */ 479 dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs, &dbr_page); 480 if (dbr_offset < 0) { 481 DRV_LOG(ERR, "Failed to allocate CQ door-bell."); 482 goto error; 483 } 484 rxq_ctrl->cq_dbr_offset = dbr_offset; 485 rxq_ctrl->cq_dbrec_page = dbr_page; 486 rxq_data->cq_db = (uint32_t *)((uintptr_t)dbr_page->dbrs + 487 (uintptr_t)rxq_ctrl->cq_dbr_offset); 488 rxq_data->cq_uar = 489 mlx5_os_get_devx_uar_base_addr(priv->sh->devx_rx_uar); 490 /* Create CQ using DevX API. */ 491 cq_attr.eqn = priv->sh->eqn; 492 cq_attr.uar_page_id = 493 mlx5_os_get_devx_uar_page_id(priv->sh->devx_rx_uar); 494 cq_attr.q_umem_id = mlx5_os_get_umem_id(rxq_ctrl->cq_umem); 495 cq_attr.q_umem_valid = 1; 496 cq_attr.log_cq_size = log_cqe_n; 497 cq_attr.log_page_size = rte_log2_u32(page_size); 498 cq_attr.db_umem_offset = rxq_ctrl->cq_dbr_offset; 499 cq_attr.db_umem_id = mlx5_os_get_umem_id(dbr_page->umem); 500 cq_attr.db_umem_valid = 1; 501 cq_obj = mlx5_devx_cmd_create_cq(priv->sh->ctx, &cq_attr); 502 if (!cq_obj) 503 goto error; 504 rxq_data->cqe_n = log_cqe_n; 505 rxq_data->cqn = cq_obj->id; 506 if (rxq_ctrl->obj->devx_channel) { 507 ret = mlx5_glue->devx_subscribe_devx_event 508 (rxq_ctrl->obj->devx_channel, 509 cq_obj->obj, 510 sizeof(event_nums), 511 event_nums, 512 (uint64_t)(uintptr_t)cq_obj); 513 if (ret) { 514 DRV_LOG(ERR, "Fail to subscribe CQ to event channel."); 515 rte_errno = errno; 516 goto error; 517 } 518 } 519 /* Initialise CQ to 1's to mark HW ownership for all CQEs. */ 520 memset((void *)(uintptr_t)rxq_data->cqes, 0xFF, cq_size); 521 return cq_obj; 522 error: 523 if (cq_obj) 524 mlx5_devx_cmd_destroy(cq_obj); 525 mlx5_rxq_release_devx_cq_resources(rxq_ctrl); 526 return NULL; 527 } 528 529 /** 530 * Create the Rx hairpin queue object. 531 * 532 * @param dev 533 * Pointer to Ethernet device. 534 * @param idx 535 * Queue index in DPDK Rx queue array. 536 * 537 * @return 538 * 0 on success, a negative errno value otherwise and rte_errno is set. 539 */ 540 static int 541 mlx5_rxq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) 542 { 543 struct mlx5_priv *priv = dev->data->dev_private; 544 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 545 struct mlx5_rxq_ctrl *rxq_ctrl = 546 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 547 struct mlx5_devx_create_rq_attr attr = { 0 }; 548 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 549 uint32_t max_wq_data; 550 551 MLX5_ASSERT(rxq_data); 552 MLX5_ASSERT(tmpl); 553 tmpl->type = MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN; 554 tmpl->rxq_ctrl = rxq_ctrl; 555 attr.hairpin = 1; 556 max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; 557 /* Jumbo frames > 9KB should be supported, and more packets. */ 558 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 559 if (priv->config.log_hp_size > max_wq_data) { 560 DRV_LOG(ERR, "Total data size %u power of 2 is " 561 "too large for hairpin.", 562 priv->config.log_hp_size); 563 rte_errno = ERANGE; 564 return -rte_errno; 565 } 566 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 567 } else { 568 attr.wq_attr.log_hairpin_data_sz = 569 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 570 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 571 } 572 /* Set the packets number to the maximum value for performance. */ 573 attr.wq_attr.log_hairpin_num_packets = 574 attr.wq_attr.log_hairpin_data_sz - 575 MLX5_HAIRPIN_QUEUE_STRIDE; 576 tmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &attr, 577 rxq_ctrl->socket); 578 if (!tmpl->rq) { 579 DRV_LOG(ERR, 580 "Port %u Rx hairpin queue %u can't create rq object.", 581 dev->data->port_id, idx); 582 rte_errno = errno; 583 return -rte_errno; 584 } 585 dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN; 586 return 0; 587 } 588 589 /** 590 * Create the Rx queue DevX object. 591 * 592 * @param dev 593 * Pointer to Ethernet device. 594 * @param idx 595 * Queue index in DPDK Rx queue array. 596 * 597 * @return 598 * 0 on success, a negative errno value otherwise and rte_errno is set. 599 */ 600 static int 601 mlx5_rxq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx) 602 { 603 struct mlx5_priv *priv = dev->data->dev_private; 604 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 605 struct mlx5_rxq_ctrl *rxq_ctrl = 606 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 607 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 608 int ret = 0; 609 610 MLX5_ASSERT(rxq_data); 611 MLX5_ASSERT(tmpl); 612 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) 613 return mlx5_rxq_obj_hairpin_new(dev, idx); 614 tmpl->type = MLX5_RXQ_OBJ_TYPE_DEVX_RQ; 615 tmpl->rxq_ctrl = rxq_ctrl; 616 if (rxq_ctrl->irq) { 617 int devx_ev_flag = 618 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA; 619 620 tmpl->devx_channel = mlx5_glue->devx_create_event_channel 621 (priv->sh->ctx, 622 devx_ev_flag); 623 if (!tmpl->devx_channel) { 624 rte_errno = errno; 625 DRV_LOG(ERR, "Failed to create event channel %d.", 626 rte_errno); 627 goto error; 628 } 629 tmpl->fd = mlx5_os_get_devx_channel_fd(tmpl->devx_channel); 630 } 631 /* Create CQ using DevX API. */ 632 tmpl->devx_cq = mlx5_rxq_create_devx_cq_resources(dev, idx); 633 if (!tmpl->devx_cq) { 634 DRV_LOG(ERR, "Failed to create CQ."); 635 goto error; 636 } 637 /* Create RQ using DevX API. */ 638 tmpl->rq = mlx5_rxq_create_devx_rq_resources(dev, idx); 639 if (!tmpl->rq) { 640 DRV_LOG(ERR, "Port %u Rx queue %u RQ creation failure.", 641 dev->data->port_id, idx); 642 rte_errno = ENOMEM; 643 goto error; 644 } 645 /* Change queue state to ready. */ 646 ret = mlx5_devx_modify_rq(tmpl, MLX5_RXQ_MOD_RST2RDY); 647 if (ret) 648 goto error; 649 rxq_data->cq_arm_sn = 0; 650 mlx5_rxq_initialize(rxq_data); 651 rxq_data->cq_ci = 0; 652 dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED; 653 rxq_ctrl->wqn = tmpl->rq->id; 654 return 0; 655 error: 656 ret = rte_errno; /* Save rte_errno before cleanup. */ 657 if (tmpl->rq) 658 claim_zero(mlx5_devx_cmd_destroy(tmpl->rq)); 659 if (tmpl->devx_cq) 660 claim_zero(mlx5_devx_cmd_destroy(tmpl->devx_cq)); 661 if (tmpl->devx_channel) 662 mlx5_glue->devx_destroy_event_channel(tmpl->devx_channel); 663 mlx5_rxq_release_devx_rq_resources(rxq_ctrl); 664 mlx5_rxq_release_devx_cq_resources(rxq_ctrl); 665 rte_errno = ret; /* Restore rte_errno. */ 666 return -rte_errno; 667 } 668 669 /** 670 * Create RQT using DevX API as a filed of indirection table. 671 * 672 * @param dev 673 * Pointer to Ethernet device. 674 * @param log_n 675 * Log of number of queues in the array. 676 * @param ind_tbl 677 * DevX indirection table object. 678 * 679 * @return 680 * 0 on success, a negative errno value otherwise and rte_errno is set. 681 */ 682 static int 683 mlx5_devx_ind_table_new(struct rte_eth_dev *dev, const unsigned int log_n, 684 struct mlx5_ind_table_obj *ind_tbl) 685 { 686 struct mlx5_priv *priv = dev->data->dev_private; 687 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 688 const unsigned int rqt_n = 1 << log_n; 689 unsigned int i, j; 690 691 MLX5_ASSERT(ind_tbl); 692 rqt_attr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rqt_attr) + 693 rqt_n * sizeof(uint32_t), 0, SOCKET_ID_ANY); 694 if (!rqt_attr) { 695 DRV_LOG(ERR, "Port %u cannot allocate RQT resources.", 696 dev->data->port_id); 697 rte_errno = ENOMEM; 698 return -rte_errno; 699 } 700 rqt_attr->rqt_max_size = priv->config.ind_table_max_size; 701 rqt_attr->rqt_actual_size = rqt_n; 702 for (i = 0; i != ind_tbl->queues_n; ++i) { 703 struct mlx5_rxq_data *rxq = (*priv->rxqs)[ind_tbl->queues[i]]; 704 struct mlx5_rxq_ctrl *rxq_ctrl = 705 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 706 707 rqt_attr->rq_list[i] = rxq_ctrl->obj->rq->id; 708 } 709 MLX5_ASSERT(i > 0); 710 for (j = 0; i != rqt_n; ++j, ++i) 711 rqt_attr->rq_list[i] = rqt_attr->rq_list[j]; 712 ind_tbl->rqt = mlx5_devx_cmd_create_rqt(priv->sh->ctx, rqt_attr); 713 mlx5_free(rqt_attr); 714 if (!ind_tbl->rqt) { 715 DRV_LOG(ERR, "Port %u cannot create DevX RQT.", 716 dev->data->port_id); 717 rte_errno = errno; 718 return -rte_errno; 719 } 720 return 0; 721 } 722 723 /** 724 * Destroy the DevX RQT object. 725 * 726 * @param ind_table 727 * Indirection table to release. 728 */ 729 static void 730 mlx5_devx_ind_table_destroy(struct mlx5_ind_table_obj *ind_tbl) 731 { 732 claim_zero(mlx5_devx_cmd_destroy(ind_tbl->rqt)); 733 } 734 735 /** 736 * Create an Rx Hash queue. 737 * 738 * @param dev 739 * Pointer to Ethernet device. 740 * @param hrxq 741 * Pointer to Rx Hash queue. 742 * @param tunnel 743 * Tunnel type. 744 * 745 * @return 746 * 0 on success, a negative errno value otherwise and rte_errno is set. 747 */ 748 static int 749 mlx5_devx_hrxq_new(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq, 750 int tunnel __rte_unused) 751 { 752 struct mlx5_priv *priv = dev->data->dev_private; 753 struct mlx5_ind_table_obj *ind_tbl = hrxq->ind_table; 754 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[ind_tbl->queues[0]]; 755 struct mlx5_rxq_ctrl *rxq_ctrl = 756 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 757 struct mlx5_devx_tir_attr tir_attr; 758 const uint8_t *rss_key = hrxq->rss_key; 759 uint64_t hash_fields = hrxq->hash_fields; 760 bool lro = true; 761 uint32_t i; 762 int err; 763 764 /* Enable TIR LRO only if all the queues were configured for. */ 765 for (i = 0; i < ind_tbl->queues_n; ++i) { 766 if (!(*priv->rxqs)[ind_tbl->queues[i]]->lro) { 767 lro = false; 768 break; 769 } 770 } 771 memset(&tir_attr, 0, sizeof(tir_attr)); 772 tir_attr.disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT; 773 tir_attr.rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ; 774 tir_attr.tunneled_offload_en = !!tunnel; 775 /* If needed, translate hash_fields bitmap to PRM format. */ 776 if (hash_fields) { 777 struct mlx5_rx_hash_field_select *rx_hash_field_select = NULL; 778 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 779 rx_hash_field_select = hash_fields & IBV_RX_HASH_INNER ? 780 &tir_attr.rx_hash_field_selector_inner : 781 &tir_attr.rx_hash_field_selector_outer; 782 #else 783 rx_hash_field_select = &tir_attr.rx_hash_field_selector_outer; 784 #endif 785 /* 1 bit: 0: IPv4, 1: IPv6. */ 786 rx_hash_field_select->l3_prot_type = 787 !!(hash_fields & MLX5_IPV6_IBV_RX_HASH); 788 /* 1 bit: 0: TCP, 1: UDP. */ 789 rx_hash_field_select->l4_prot_type = 790 !!(hash_fields & MLX5_UDP_IBV_RX_HASH); 791 /* Bitmask which sets which fields to use in RX Hash. */ 792 rx_hash_field_select->selected_fields = 793 ((!!(hash_fields & MLX5_L3_SRC_IBV_RX_HASH)) << 794 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) | 795 (!!(hash_fields & MLX5_L3_DST_IBV_RX_HASH)) << 796 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP | 797 (!!(hash_fields & MLX5_L4_SRC_IBV_RX_HASH)) << 798 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT | 799 (!!(hash_fields & MLX5_L4_DST_IBV_RX_HASH)) << 800 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT; 801 } 802 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) 803 tir_attr.transport_domain = priv->sh->td->id; 804 else 805 tir_attr.transport_domain = priv->sh->tdn; 806 memcpy(tir_attr.rx_hash_toeplitz_key, rss_key, MLX5_RSS_HASH_KEY_LEN); 807 tir_attr.indirect_table = ind_tbl->rqt->id; 808 if (dev->data->dev_conf.lpbk_mode) 809 tir_attr.self_lb_block = MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST; 810 if (lro) { 811 tir_attr.lro_timeout_period_usecs = priv->config.lro.timeout; 812 tir_attr.lro_max_msg_sz = priv->max_lro_msg_size; 813 tir_attr.lro_enable_mask = MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | 814 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO; 815 } 816 hrxq->tir = mlx5_devx_cmd_create_tir(priv->sh->ctx, &tir_attr); 817 if (!hrxq->tir) { 818 DRV_LOG(ERR, "Port %u cannot create DevX TIR.", 819 dev->data->port_id); 820 rte_errno = errno; 821 goto error; 822 } 823 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 824 hrxq->action = mlx5_glue->dv_create_flow_action_dest_devx_tir 825 (hrxq->tir->obj); 826 if (!hrxq->action) { 827 rte_errno = errno; 828 goto error; 829 } 830 #endif 831 return 0; 832 error: 833 err = rte_errno; /* Save rte_errno before cleanup. */ 834 if (hrxq->tir) 835 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 836 rte_errno = err; /* Restore rte_errno. */ 837 return -rte_errno; 838 } 839 840 /** 841 * Destroy a DevX TIR object. 842 * 843 * @param hrxq 844 * Hash Rx queue to release its tir. 845 */ 846 static void 847 mlx5_devx_tir_destroy(struct mlx5_hrxq *hrxq) 848 { 849 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 850 } 851 852 /** 853 * Create a DevX drop action for Rx Hash queue. 854 * 855 * @param dev 856 * Pointer to Ethernet device. 857 * 858 * @return 859 * 0 on success, a negative errno value otherwise and rte_errno is set. 860 */ 861 static int 862 mlx5_devx_drop_action_create(struct rte_eth_dev *dev) 863 { 864 (void)dev; 865 DRV_LOG(ERR, "DevX drop action is not supported yet."); 866 rte_errno = ENOTSUP; 867 return -rte_errno; 868 } 869 870 /** 871 * Release a drop hash Rx queue. 872 * 873 * @param dev 874 * Pointer to Ethernet device. 875 */ 876 static void 877 mlx5_devx_drop_action_destroy(struct rte_eth_dev *dev) 878 { 879 (void)dev; 880 DRV_LOG(ERR, "DevX drop action is not supported yet."); 881 rte_errno = ENOTSUP; 882 } 883 884 /** 885 * Create the Tx hairpin queue object. 886 * 887 * @param dev 888 * Pointer to Ethernet device. 889 * @param idx 890 * Queue index in DPDK Tx queue array. 891 * 892 * @return 893 * 0 on success, a negative errno value otherwise and rte_errno is set. 894 */ 895 static int 896 mlx5_txq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) 897 { 898 struct mlx5_priv *priv = dev->data->dev_private; 899 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 900 struct mlx5_txq_ctrl *txq_ctrl = 901 container_of(txq_data, struct mlx5_txq_ctrl, txq); 902 struct mlx5_devx_create_sq_attr attr = { 0 }; 903 struct mlx5_txq_obj *tmpl = txq_ctrl->obj; 904 uint32_t max_wq_data; 905 906 MLX5_ASSERT(txq_data); 907 MLX5_ASSERT(tmpl); 908 tmpl->txq_ctrl = txq_ctrl; 909 attr.hairpin = 1; 910 attr.tis_lst_sz = 1; 911 max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; 912 /* Jumbo frames > 9KB should be supported, and more packets. */ 913 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 914 if (priv->config.log_hp_size > max_wq_data) { 915 DRV_LOG(ERR, "Total data size %u power of 2 is " 916 "too large for hairpin.", 917 priv->config.log_hp_size); 918 rte_errno = ERANGE; 919 return -rte_errno; 920 } 921 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 922 } else { 923 attr.wq_attr.log_hairpin_data_sz = 924 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 925 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 926 } 927 /* Set the packets number to the maximum value for performance. */ 928 attr.wq_attr.log_hairpin_num_packets = 929 attr.wq_attr.log_hairpin_data_sz - 930 MLX5_HAIRPIN_QUEUE_STRIDE; 931 attr.tis_num = priv->sh->tis->id; 932 tmpl->sq = mlx5_devx_cmd_create_sq(priv->sh->ctx, &attr); 933 if (!tmpl->sq) { 934 DRV_LOG(ERR, 935 "Port %u tx hairpin queue %u can't create SQ object.", 936 dev->data->port_id, idx); 937 rte_errno = errno; 938 return -rte_errno; 939 } 940 return 0; 941 } 942 943 #ifdef HAVE_MLX5DV_DEVX_UAR_OFFSET 944 /** 945 * Release DevX SQ resources. 946 * 947 * @param txq_obj 948 * DevX Tx queue object. 949 */ 950 static void 951 mlx5_txq_release_devx_sq_resources(struct mlx5_txq_obj *txq_obj) 952 { 953 if (txq_obj->sq_devx) 954 claim_zero(mlx5_devx_cmd_destroy(txq_obj->sq_devx)); 955 if (txq_obj->sq_umem) 956 claim_zero(mlx5_glue->devx_umem_dereg(txq_obj->sq_umem)); 957 if (txq_obj->sq_buf) 958 mlx5_free(txq_obj->sq_buf); 959 if (txq_obj->sq_dbrec_page) 960 claim_zero(mlx5_release_dbr(&txq_obj->txq_ctrl->priv->dbrpgs, 961 mlx5_os_get_umem_id 962 (txq_obj->sq_dbrec_page->umem), 963 txq_obj->sq_dbrec_offset)); 964 } 965 966 /** 967 * Release DevX Tx CQ resources. 968 * 969 * @param txq_obj 970 * DevX Tx queue object. 971 */ 972 static void 973 mlx5_txq_release_devx_cq_resources(struct mlx5_txq_obj *txq_obj) 974 { 975 if (txq_obj->cq_devx) 976 claim_zero(mlx5_devx_cmd_destroy(txq_obj->cq_devx)); 977 if (txq_obj->cq_umem) 978 claim_zero(mlx5_glue->devx_umem_dereg(txq_obj->cq_umem)); 979 if (txq_obj->cq_buf) 980 mlx5_free(txq_obj->cq_buf); 981 if (txq_obj->cq_dbrec_page) 982 claim_zero(mlx5_release_dbr(&txq_obj->txq_ctrl->priv->dbrpgs, 983 mlx5_os_get_umem_id 984 (txq_obj->cq_dbrec_page->umem), 985 txq_obj->cq_dbrec_offset)); 986 } 987 988 /** 989 * Destroy the Tx queue DevX object. 990 * 991 * @param txq_obj 992 * Txq object to destroy. 993 */ 994 static void 995 mlx5_txq_release_devx_resources(struct mlx5_txq_obj *txq_obj) 996 { 997 mlx5_txq_release_devx_cq_resources(txq_obj); 998 mlx5_txq_release_devx_sq_resources(txq_obj); 999 } 1000 1001 /** 1002 * Create a DevX CQ object and its resources for an Tx queue. 1003 * 1004 * @param dev 1005 * Pointer to Ethernet device. 1006 * @param idx 1007 * Queue index in DPDK Tx queue array. 1008 * 1009 * @return 1010 * Number of CQEs in CQ, 0 otherwise and rte_errno is set. 1011 */ 1012 static uint32_t 1013 mlx5_txq_create_devx_cq_resources(struct rte_eth_dev *dev, uint16_t idx) 1014 { 1015 struct mlx5_priv *priv = dev->data->dev_private; 1016 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1017 struct mlx5_txq_ctrl *txq_ctrl = 1018 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1019 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1020 struct mlx5_devx_cq_attr cq_attr = { 0 }; 1021 struct mlx5_cqe *cqe; 1022 size_t page_size; 1023 size_t alignment; 1024 uint32_t cqe_n; 1025 uint32_t i; 1026 int ret; 1027 1028 MLX5_ASSERT(txq_data); 1029 MLX5_ASSERT(txq_obj); 1030 page_size = rte_mem_page_size(); 1031 if (page_size == (size_t)-1) { 1032 DRV_LOG(ERR, "Failed to get mem page size."); 1033 rte_errno = ENOMEM; 1034 return 0; 1035 } 1036 /* Allocate memory buffer for CQEs. */ 1037 alignment = MLX5_CQE_BUF_ALIGNMENT; 1038 if (alignment == (size_t)-1) { 1039 DRV_LOG(ERR, "Failed to get CQE buf alignment."); 1040 rte_errno = ENOMEM; 1041 return 0; 1042 } 1043 /* Create the Completion Queue. */ 1044 cqe_n = (1UL << txq_data->elts_n) / MLX5_TX_COMP_THRESH + 1045 1 + MLX5_TX_COMP_THRESH_INLINE_DIV; 1046 cqe_n = 1UL << log2above(cqe_n); 1047 if (cqe_n > UINT16_MAX) { 1048 DRV_LOG(ERR, 1049 "Port %u Tx queue %u requests to many CQEs %u.", 1050 dev->data->port_id, txq_data->idx, cqe_n); 1051 rte_errno = EINVAL; 1052 return 0; 1053 } 1054 txq_obj->cq_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, 1055 cqe_n * sizeof(struct mlx5_cqe), 1056 alignment, 1057 priv->sh->numa_node); 1058 if (!txq_obj->cq_buf) { 1059 DRV_LOG(ERR, 1060 "Port %u Tx queue %u cannot allocate memory (CQ).", 1061 dev->data->port_id, txq_data->idx); 1062 rte_errno = ENOMEM; 1063 return 0; 1064 } 1065 /* Register allocated buffer in user space with DevX. */ 1066 txq_obj->cq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, 1067 (void *)txq_obj->cq_buf, 1068 cqe_n * sizeof(struct mlx5_cqe), 1069 IBV_ACCESS_LOCAL_WRITE); 1070 if (!txq_obj->cq_umem) { 1071 rte_errno = errno; 1072 DRV_LOG(ERR, 1073 "Port %u Tx queue %u cannot register memory (CQ).", 1074 dev->data->port_id, txq_data->idx); 1075 goto error; 1076 } 1077 /* Allocate doorbell record for completion queue. */ 1078 txq_obj->cq_dbrec_offset = mlx5_get_dbr(priv->sh->ctx, 1079 &priv->dbrpgs, 1080 &txq_obj->cq_dbrec_page); 1081 if (txq_obj->cq_dbrec_offset < 0) { 1082 rte_errno = errno; 1083 DRV_LOG(ERR, "Failed to allocate CQ door-bell."); 1084 goto error; 1085 } 1086 cq_attr.cqe_size = (sizeof(struct mlx5_cqe) == 128) ? 1087 MLX5_CQE_SIZE_128B : MLX5_CQE_SIZE_64B; 1088 cq_attr.uar_page_id = mlx5_os_get_devx_uar_page_id(priv->sh->tx_uar); 1089 cq_attr.eqn = priv->sh->eqn; 1090 cq_attr.q_umem_valid = 1; 1091 cq_attr.q_umem_offset = (uintptr_t)txq_obj->cq_buf % page_size; 1092 cq_attr.q_umem_id = mlx5_os_get_umem_id(txq_obj->cq_umem); 1093 cq_attr.db_umem_valid = 1; 1094 cq_attr.db_umem_offset = txq_obj->cq_dbrec_offset; 1095 cq_attr.db_umem_id = mlx5_os_get_umem_id(txq_obj->cq_dbrec_page->umem); 1096 cq_attr.log_cq_size = rte_log2_u32(cqe_n); 1097 cq_attr.log_page_size = rte_log2_u32(page_size); 1098 /* Create completion queue object with DevX. */ 1099 txq_obj->cq_devx = mlx5_devx_cmd_create_cq(priv->sh->ctx, &cq_attr); 1100 if (!txq_obj->cq_devx) { 1101 rte_errno = errno; 1102 DRV_LOG(ERR, "Port %u Tx queue %u CQ creation failure.", 1103 dev->data->port_id, idx); 1104 goto error; 1105 } 1106 /* Initial fill CQ buffer with invalid CQE opcode. */ 1107 cqe = (struct mlx5_cqe *)txq_obj->cq_buf; 1108 for (i = 0; i < cqe_n; i++) { 1109 cqe->op_own = (MLX5_CQE_INVALID << 4) | MLX5_CQE_OWNER_MASK; 1110 ++cqe; 1111 } 1112 return cqe_n; 1113 error: 1114 ret = rte_errno; 1115 mlx5_txq_release_devx_cq_resources(txq_obj); 1116 rte_errno = ret; 1117 return 0; 1118 } 1119 1120 /** 1121 * Create a SQ object and its resources using DevX. 1122 * 1123 * @param dev 1124 * Pointer to Ethernet device. 1125 * @param idx 1126 * Queue index in DPDK Tx queue array. 1127 * 1128 * @return 1129 * Number of WQEs in SQ, 0 otherwise and rte_errno is set. 1130 */ 1131 static uint32_t 1132 mlx5_txq_create_devx_sq_resources(struct rte_eth_dev *dev, uint16_t idx) 1133 { 1134 struct mlx5_priv *priv = dev->data->dev_private; 1135 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1136 struct mlx5_txq_ctrl *txq_ctrl = 1137 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1138 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1139 struct mlx5_devx_create_sq_attr sq_attr = { 0 }; 1140 size_t page_size; 1141 uint32_t wqe_n; 1142 int ret; 1143 1144 MLX5_ASSERT(txq_data); 1145 MLX5_ASSERT(txq_obj); 1146 page_size = rte_mem_page_size(); 1147 if (page_size == (size_t)-1) { 1148 DRV_LOG(ERR, "Failed to get mem page size."); 1149 rte_errno = ENOMEM; 1150 return 0; 1151 } 1152 wqe_n = RTE_MIN(1UL << txq_data->elts_n, 1153 (uint32_t)priv->sh->device_attr.max_qp_wr); 1154 txq_obj->sq_buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, 1155 wqe_n * sizeof(struct mlx5_wqe), 1156 page_size, priv->sh->numa_node); 1157 if (!txq_obj->sq_buf) { 1158 DRV_LOG(ERR, 1159 "Port %u Tx queue %u cannot allocate memory (SQ).", 1160 dev->data->port_id, txq_data->idx); 1161 rte_errno = ENOMEM; 1162 goto error; 1163 } 1164 /* Register allocated buffer in user space with DevX. */ 1165 txq_obj->sq_umem = mlx5_glue->devx_umem_reg 1166 (priv->sh->ctx, 1167 (void *)txq_obj->sq_buf, 1168 wqe_n * sizeof(struct mlx5_wqe), 1169 IBV_ACCESS_LOCAL_WRITE); 1170 if (!txq_obj->sq_umem) { 1171 rte_errno = errno; 1172 DRV_LOG(ERR, 1173 "Port %u Tx queue %u cannot register memory (SQ).", 1174 dev->data->port_id, txq_data->idx); 1175 goto error; 1176 } 1177 /* Allocate doorbell record for send queue. */ 1178 txq_obj->sq_dbrec_offset = mlx5_get_dbr(priv->sh->ctx, 1179 &priv->dbrpgs, 1180 &txq_obj->sq_dbrec_page); 1181 if (txq_obj->sq_dbrec_offset < 0) { 1182 rte_errno = errno; 1183 DRV_LOG(ERR, "Failed to allocate SQ door-bell."); 1184 goto error; 1185 } 1186 sq_attr.tis_lst_sz = 1; 1187 sq_attr.tis_num = priv->sh->tis->id; 1188 sq_attr.state = MLX5_SQC_STATE_RST; 1189 sq_attr.cqn = txq_obj->cq_devx->id; 1190 sq_attr.flush_in_error_en = 1; 1191 sq_attr.allow_multi_pkt_send_wqe = !!priv->config.mps; 1192 sq_attr.allow_swp = !!priv->config.swp; 1193 sq_attr.min_wqe_inline_mode = priv->config.hca_attr.vport_inline_mode; 1194 sq_attr.wq_attr.uar_page = 1195 mlx5_os_get_devx_uar_page_id(priv->sh->tx_uar); 1196 sq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 1197 sq_attr.wq_attr.pd = priv->sh->pdn; 1198 sq_attr.wq_attr.log_wq_stride = rte_log2_u32(MLX5_WQE_SIZE); 1199 sq_attr.wq_attr.log_wq_sz = log2above(wqe_n); 1200 sq_attr.wq_attr.dbr_umem_valid = 1; 1201 sq_attr.wq_attr.dbr_addr = txq_obj->sq_dbrec_offset; 1202 sq_attr.wq_attr.dbr_umem_id = 1203 mlx5_os_get_umem_id(txq_obj->sq_dbrec_page->umem); 1204 sq_attr.wq_attr.wq_umem_valid = 1; 1205 sq_attr.wq_attr.wq_umem_id = mlx5_os_get_umem_id(txq_obj->sq_umem); 1206 sq_attr.wq_attr.wq_umem_offset = (uintptr_t)txq_obj->sq_buf % page_size; 1207 /* Create Send Queue object with DevX. */ 1208 txq_obj->sq_devx = mlx5_devx_cmd_create_sq(priv->sh->ctx, &sq_attr); 1209 if (!txq_obj->sq_devx) { 1210 rte_errno = errno; 1211 DRV_LOG(ERR, "Port %u Tx queue %u SQ creation failure.", 1212 dev->data->port_id, idx); 1213 goto error; 1214 } 1215 return wqe_n; 1216 error: 1217 ret = rte_errno; 1218 mlx5_txq_release_devx_sq_resources(txq_obj); 1219 rte_errno = ret; 1220 return 0; 1221 } 1222 #endif 1223 1224 /** 1225 * Create the Tx queue DevX object. 1226 * 1227 * @param dev 1228 * Pointer to Ethernet device. 1229 * @param idx 1230 * Queue index in DPDK Tx queue array. 1231 * 1232 * @return 1233 * 0 on success, a negative errno value otherwise and rte_errno is set. 1234 */ 1235 int 1236 mlx5_txq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx) 1237 { 1238 struct mlx5_priv *priv = dev->data->dev_private; 1239 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1240 struct mlx5_txq_ctrl *txq_ctrl = 1241 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1242 1243 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) 1244 return mlx5_txq_obj_hairpin_new(dev, idx); 1245 #ifndef HAVE_MLX5DV_DEVX_UAR_OFFSET 1246 DRV_LOG(ERR, "Port %u Tx queue %u cannot create with DevX, no UAR.", 1247 dev->data->port_id, idx); 1248 rte_errno = ENOMEM; 1249 return -rte_errno; 1250 #else 1251 struct mlx5_dev_ctx_shared *sh = priv->sh; 1252 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1253 void *reg_addr; 1254 uint32_t cqe_n; 1255 uint32_t wqe_n; 1256 int ret = 0; 1257 1258 MLX5_ASSERT(txq_data); 1259 MLX5_ASSERT(txq_obj); 1260 txq_obj->txq_ctrl = txq_ctrl; 1261 txq_obj->dev = dev; 1262 cqe_n = mlx5_txq_create_devx_cq_resources(dev, idx); 1263 if (!cqe_n) { 1264 rte_errno = errno; 1265 goto error; 1266 } 1267 txq_data->cqe_n = log2above(cqe_n); 1268 txq_data->cqe_s = 1 << txq_data->cqe_n; 1269 txq_data->cqe_m = txq_data->cqe_s - 1; 1270 txq_data->cqes = (volatile struct mlx5_cqe *)txq_obj->cq_buf; 1271 txq_data->cq_ci = 0; 1272 txq_data->cq_pi = 0; 1273 txq_data->cq_db = (volatile uint32_t *)(txq_obj->cq_dbrec_page->dbrs + 1274 txq_obj->cq_dbrec_offset); 1275 *txq_data->cq_db = 0; 1276 /* Create Send Queue object with DevX. */ 1277 wqe_n = mlx5_txq_create_devx_sq_resources(dev, idx); 1278 if (!wqe_n) { 1279 rte_errno = errno; 1280 goto error; 1281 } 1282 /* Create the Work Queue. */ 1283 txq_data->wqe_n = log2above(wqe_n); 1284 txq_data->wqe_s = 1 << txq_data->wqe_n; 1285 txq_data->wqe_m = txq_data->wqe_s - 1; 1286 txq_data->wqes = (struct mlx5_wqe *)txq_obj->sq_buf; 1287 txq_data->wqes_end = txq_data->wqes + txq_data->wqe_s; 1288 txq_data->wqe_ci = 0; 1289 txq_data->wqe_pi = 0; 1290 txq_data->wqe_comp = 0; 1291 txq_data->wqe_thres = txq_data->wqe_s / MLX5_TX_COMP_THRESH_INLINE_DIV; 1292 txq_data->qp_db = (volatile uint32_t *) 1293 (txq_obj->sq_dbrec_page->dbrs + 1294 txq_obj->sq_dbrec_offset + 1295 MLX5_SND_DBR * sizeof(uint32_t)); 1296 *txq_data->qp_db = 0; 1297 txq_data->qp_num_8s = txq_obj->sq_devx->id << 8; 1298 /* Change Send Queue state to Ready-to-Send. */ 1299 ret = mlx5_devx_modify_sq(txq_obj, MLX5_TXQ_MOD_RST2RDY, 0); 1300 if (ret) { 1301 rte_errno = errno; 1302 DRV_LOG(ERR, 1303 "Port %u Tx queue %u SQ state to SQC_STATE_RDY failed.", 1304 dev->data->port_id, idx); 1305 goto error; 1306 } 1307 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 1308 /* 1309 * If using DevX need to query and store TIS transport domain value. 1310 * This is done once per port. 1311 * Will use this value on Rx, when creating matching TIR. 1312 */ 1313 if (!priv->sh->tdn) 1314 priv->sh->tdn = priv->sh->td->id; 1315 #endif 1316 MLX5_ASSERT(sh->tx_uar); 1317 reg_addr = mlx5_os_get_devx_uar_reg_addr(sh->tx_uar); 1318 MLX5_ASSERT(reg_addr); 1319 txq_ctrl->bf_reg = reg_addr; 1320 txq_ctrl->uar_mmap_offset = 1321 mlx5_os_get_devx_uar_mmap_offset(sh->tx_uar); 1322 txq_uar_init(txq_ctrl); 1323 return 0; 1324 error: 1325 ret = rte_errno; /* Save rte_errno before cleanup. */ 1326 mlx5_txq_release_devx_resources(txq_obj); 1327 rte_errno = ret; /* Restore rte_errno. */ 1328 return -rte_errno; 1329 #endif 1330 } 1331 1332 /** 1333 * Release an Tx DevX queue object. 1334 * 1335 * @param txq_obj 1336 * DevX Tx queue object. 1337 */ 1338 void 1339 mlx5_txq_devx_obj_release(struct mlx5_txq_obj *txq_obj) 1340 { 1341 MLX5_ASSERT(txq_obj); 1342 if (txq_obj->txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 1343 if (txq_obj->tis) 1344 claim_zero(mlx5_devx_cmd_destroy(txq_obj->tis)); 1345 #ifdef HAVE_MLX5DV_DEVX_UAR_OFFSET 1346 } else { 1347 mlx5_txq_release_devx_resources(txq_obj); 1348 #endif 1349 } 1350 } 1351 1352 struct mlx5_obj_ops devx_obj_ops = { 1353 .rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_rq_vlan_strip, 1354 .rxq_obj_new = mlx5_rxq_devx_obj_new, 1355 .rxq_event_get = mlx5_rx_devx_get_event, 1356 .rxq_obj_modify = mlx5_devx_modify_rq, 1357 .rxq_obj_release = mlx5_rxq_devx_obj_release, 1358 .ind_table_new = mlx5_devx_ind_table_new, 1359 .ind_table_destroy = mlx5_devx_ind_table_destroy, 1360 .hrxq_new = mlx5_devx_hrxq_new, 1361 .hrxq_destroy = mlx5_devx_tir_destroy, 1362 .drop_action_create = mlx5_devx_drop_action_create, 1363 .drop_action_destroy = mlx5_devx_drop_action_destroy, 1364 .txq_obj_new = mlx5_txq_devx_obj_new, 1365 .txq_obj_modify = mlx5_devx_modify_sq, 1366 .txq_obj_release = mlx5_txq_devx_obj_release, 1367 }; 1368