1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <stddef.h> 6 #include <errno.h> 7 #include <stdbool.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <sys/queue.h> 11 12 #include <rte_malloc.h> 13 #include <rte_common.h> 14 #include <rte_eal_paging.h> 15 16 #include <mlx5_glue.h> 17 #include <mlx5_devx_cmds.h> 18 #include <mlx5_common_devx.h> 19 #include <mlx5_malloc.h> 20 21 #include "mlx5.h" 22 #include "mlx5_common_os.h" 23 #include "mlx5_tx.h" 24 #include "mlx5_rx.h" 25 #include "mlx5_utils.h" 26 #include "mlx5_devx.h" 27 #include "mlx5_flow.h" 28 #include "mlx5_flow_os.h" 29 30 /** 31 * Modify RQ vlan stripping offload 32 * 33 * @param rxq 34 * Rx queue. 35 * @param on 36 * Enable/disable VLAN stripping. 37 * 38 * @return 39 * 0 on success, non-0 otherwise 40 */ 41 static int 42 mlx5_rxq_obj_modify_rq_vlan_strip(struct mlx5_rxq_priv *rxq, int on) 43 { 44 struct mlx5_devx_modify_rq_attr rq_attr; 45 46 memset(&rq_attr, 0, sizeof(rq_attr)); 47 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 48 rq_attr.state = MLX5_RQC_STATE_RDY; 49 rq_attr.vsd = (on ? 0 : 1); 50 rq_attr.modify_bitmask = MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD; 51 return mlx5_devx_cmd_modify_rq(rxq->devx_rq.rq, &rq_attr); 52 } 53 54 /** 55 * Modify RQ using DevX API. 56 * 57 * @param rxq 58 * DevX rx queue. 59 * @param type 60 * Type of change queue state. 61 * 62 * @return 63 * 0 on success, a negative errno value otherwise and rte_errno is set. 64 */ 65 static int 66 mlx5_devx_modify_rq(struct mlx5_rxq_priv *rxq, uint8_t type) 67 { 68 struct mlx5_devx_modify_rq_attr rq_attr; 69 70 memset(&rq_attr, 0, sizeof(rq_attr)); 71 switch (type) { 72 case MLX5_RXQ_MOD_ERR2RST: 73 rq_attr.rq_state = MLX5_RQC_STATE_ERR; 74 rq_attr.state = MLX5_RQC_STATE_RST; 75 break; 76 case MLX5_RXQ_MOD_RST2RDY: 77 rq_attr.rq_state = MLX5_RQC_STATE_RST; 78 rq_attr.state = MLX5_RQC_STATE_RDY; 79 break; 80 case MLX5_RXQ_MOD_RDY2ERR: 81 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 82 rq_attr.state = MLX5_RQC_STATE_ERR; 83 break; 84 case MLX5_RXQ_MOD_RDY2RST: 85 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 86 rq_attr.state = MLX5_RQC_STATE_RST; 87 break; 88 default: 89 break; 90 } 91 if (rxq->ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) 92 return mlx5_devx_cmd_modify_rq(rxq->ctrl->obj->rq, &rq_attr); 93 return mlx5_devx_cmd_modify_rq(rxq->devx_rq.rq, &rq_attr); 94 } 95 96 /** 97 * Modify SQ using DevX API. 98 * 99 * @param txq_obj 100 * DevX Tx queue object. 101 * @param type 102 * Type of change queue state. 103 * @param dev_port 104 * Unnecessary. 105 * 106 * @return 107 * 0 on success, a negative errno value otherwise and rte_errno is set. 108 */ 109 int 110 mlx5_txq_devx_modify(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type, 111 uint8_t dev_port) 112 { 113 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 114 int ret; 115 116 if (type != MLX5_TXQ_MOD_RST2RDY) { 117 /* Change queue state to reset. */ 118 if (type == MLX5_TXQ_MOD_ERR2RDY) 119 msq_attr.sq_state = MLX5_SQC_STATE_ERR; 120 else 121 msq_attr.sq_state = MLX5_SQC_STATE_RDY; 122 msq_attr.state = MLX5_SQC_STATE_RST; 123 ret = mlx5_devx_cmd_modify_sq(obj->sq_obj.sq, &msq_attr); 124 if (ret) { 125 DRV_LOG(ERR, "Cannot change the Tx SQ state to RESET" 126 " %s", strerror(errno)); 127 rte_errno = errno; 128 return ret; 129 } 130 } 131 if (type != MLX5_TXQ_MOD_RDY2RST) { 132 /* Change queue state to ready. */ 133 msq_attr.sq_state = MLX5_SQC_STATE_RST; 134 msq_attr.state = MLX5_SQC_STATE_RDY; 135 ret = mlx5_devx_cmd_modify_sq(obj->sq_obj.sq, &msq_attr); 136 if (ret) { 137 DRV_LOG(ERR, "Cannot change the Tx SQ state to READY" 138 " %s", strerror(errno)); 139 rte_errno = errno; 140 return ret; 141 } 142 } 143 /* 144 * The dev_port variable is relevant only in Verbs API, and there is a 145 * pointer that points to this function and a parallel function in verbs 146 * intermittently, so they should have the same parameters. 147 */ 148 (void)dev_port; 149 return 0; 150 } 151 152 /** 153 * Release an Rx DevX queue object. 154 * 155 * @param rxq 156 * DevX Rx queue. 157 */ 158 static void 159 mlx5_rxq_devx_obj_release(struct mlx5_rxq_priv *rxq) 160 { 161 struct mlx5_rxq_obj *rxq_obj = rxq->ctrl->obj; 162 163 if (rxq_obj == NULL) 164 return; 165 if (rxq_obj->rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) { 166 if (rxq_obj->rq == NULL) 167 return; 168 mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RDY2RST); 169 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 170 } else { 171 if (rxq->devx_rq.rq == NULL) 172 return; 173 mlx5_devx_rq_destroy(&rxq->devx_rq); 174 if (rxq->devx_rq.rmp != NULL && rxq->devx_rq.rmp->ref_cnt > 0) 175 return; 176 mlx5_devx_cq_destroy(&rxq_obj->cq_obj); 177 memset(&rxq_obj->cq_obj, 0, sizeof(rxq_obj->cq_obj)); 178 if (rxq_obj->devx_channel) { 179 mlx5_os_devx_destroy_event_channel 180 (rxq_obj->devx_channel); 181 rxq_obj->devx_channel = NULL; 182 } 183 } 184 rxq->ctrl->started = false; 185 } 186 187 /** 188 * Get event for an Rx DevX queue object. 189 * 190 * @param rxq_obj 191 * DevX Rx queue object. 192 * 193 * @return 194 * 0 on success, a negative errno value otherwise and rte_errno is set. 195 */ 196 static int 197 mlx5_rx_devx_get_event(struct mlx5_rxq_obj *rxq_obj) 198 { 199 #ifdef HAVE_IBV_DEVX_EVENT 200 union { 201 struct mlx5dv_devx_async_event_hdr event_resp; 202 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 203 } out; 204 int ret = mlx5_glue->devx_get_event(rxq_obj->devx_channel, 205 &out.event_resp, 206 sizeof(out.buf)); 207 208 if (ret < 0) { 209 rte_errno = errno; 210 return -rte_errno; 211 } 212 if (out.event_resp.cookie != (uint64_t)(uintptr_t)rxq_obj->cq_obj.cq) { 213 rte_errno = EINVAL; 214 return -rte_errno; 215 } 216 return 0; 217 #else 218 (void)rxq_obj; 219 rte_errno = ENOTSUP; 220 return -rte_errno; 221 #endif /* HAVE_IBV_DEVX_EVENT */ 222 } 223 224 /** 225 * Create a RQ object using DevX. 226 * 227 * @param rxq 228 * Pointer to Rx queue. 229 * 230 * @return 231 * 0 on success, a negative errno value otherwise and rte_errno is set. 232 */ 233 static int 234 mlx5_rxq_create_devx_rq_resources(struct mlx5_rxq_priv *rxq) 235 { 236 struct mlx5_priv *priv = rxq->priv; 237 struct mlx5_common_device *cdev = priv->sh->cdev; 238 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 239 struct mlx5_rxq_data *rxq_data = &rxq->ctrl->rxq; 240 struct mlx5_devx_create_rq_attr rq_attr = { 0 }; 241 uint16_t log_desc_n = rxq_data->elts_n - rxq_data->sges_n; 242 uint32_t wqe_size, log_wqe_size; 243 244 /* Fill RQ attributes. */ 245 rq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE; 246 rq_attr.flush_in_error_en = 1; 247 rq_attr.vsd = (rxq_data->vlan_strip) ? 0 : 1; 248 rq_attr.cqn = rxq_ctrl->obj->cq_obj.cq->id; 249 rq_attr.scatter_fcs = (rxq_data->crc_present) ? 1 : 0; 250 rq_attr.ts_format = 251 mlx5_ts_format_conv(cdev->config.hca_attr.rq_ts_format); 252 /* Fill WQ attributes for this RQ. */ 253 if (mlx5_rxq_mprq_enabled(rxq_data)) { 254 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ; 255 /* 256 * Number of strides in each WQE: 257 * 512*2^single_wqe_log_num_of_strides. 258 */ 259 rq_attr.wq_attr.single_wqe_log_num_of_strides = 260 rxq_data->strd_num_n - 261 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES; 262 /* Stride size = (2^single_stride_log_num_of_bytes)*64B. */ 263 rq_attr.wq_attr.single_stride_log_num_of_bytes = 264 rxq_data->strd_sz_n - 265 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES; 266 wqe_size = sizeof(struct mlx5_wqe_mprq); 267 } else { 268 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 269 wqe_size = sizeof(struct mlx5_wqe_data_seg); 270 } 271 log_wqe_size = log2above(wqe_size) + rxq_data->sges_n; 272 wqe_size = 1 << log_wqe_size; /* round up power of two.*/ 273 rq_attr.wq_attr.log_wq_stride = log_wqe_size; 274 rq_attr.wq_attr.log_wq_sz = log_desc_n; 275 rq_attr.wq_attr.end_padding_mode = priv->config.hw_padding ? 276 MLX5_WQ_END_PAD_MODE_ALIGN : 277 MLX5_WQ_END_PAD_MODE_NONE; 278 rq_attr.wq_attr.pd = cdev->pdn; 279 rq_attr.counter_set_id = priv->counter_set_id; 280 rq_attr.delay_drop_en = rxq_data->delay_drop; 281 rq_attr.user_index = rte_cpu_to_be_16(priv->dev_data->port_id); 282 if (rxq_data->shared) /* Create RMP based RQ. */ 283 rxq->devx_rq.rmp = &rxq_ctrl->obj->devx_rmp; 284 /* Create RQ using DevX API. */ 285 return mlx5_devx_rq_create(cdev->ctx, &rxq->devx_rq, wqe_size, 286 log_desc_n, &rq_attr, rxq_ctrl->socket); 287 } 288 289 /** 290 * Create a DevX CQ object for an Rx queue. 291 * 292 * @param rxq 293 * Pointer to Rx queue. 294 * 295 * @return 296 * 0 on success, a negative errno value otherwise and rte_errno is set. 297 */ 298 static int 299 mlx5_rxq_create_devx_cq_resources(struct mlx5_rxq_priv *rxq) 300 { 301 struct mlx5_devx_cq *cq_obj = 0; 302 struct mlx5_devx_cq_attr cq_attr = { 0 }; 303 struct mlx5_priv *priv = rxq->priv; 304 struct mlx5_dev_ctx_shared *sh = priv->sh; 305 uint16_t port_id = priv->dev_data->port_id; 306 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 307 struct mlx5_rxq_data *rxq_data = &rxq_ctrl->rxq; 308 unsigned int cqe_n = mlx5_rxq_cqe_num(rxq_data); 309 uint32_t log_cqe_n; 310 uint16_t event_nums[1] = { 0 }; 311 int ret = 0; 312 313 if (rxq_ctrl->started) 314 return 0; 315 if (priv->config.cqe_comp && !rxq_data->hw_timestamp && 316 !rxq_data->lro) { 317 cq_attr.cqe_comp_en = 1u; 318 rxq_data->mcqe_format = priv->config.cqe_comp_fmt; 319 rxq_data->byte_mask = UINT32_MAX; 320 switch (priv->config.cqe_comp_fmt) { 321 case MLX5_CQE_RESP_FORMAT_HASH: 322 /* fallthrough */ 323 case MLX5_CQE_RESP_FORMAT_CSUM: 324 /* 325 * Select CSUM miniCQE format only for non-vectorized 326 * MPRQ Rx burst, use HASH miniCQE format for others. 327 */ 328 if (mlx5_rxq_check_vec_support(rxq_data) < 0 && 329 mlx5_rxq_mprq_enabled(rxq_data)) 330 cq_attr.mini_cqe_res_format = 331 MLX5_CQE_RESP_FORMAT_CSUM_STRIDX; 332 else 333 cq_attr.mini_cqe_res_format = 334 MLX5_CQE_RESP_FORMAT_HASH; 335 rxq_data->mcqe_format = cq_attr.mini_cqe_res_format; 336 break; 337 case MLX5_CQE_RESP_FORMAT_FTAG_STRIDX: 338 rxq_data->byte_mask = MLX5_LEN_WITH_MARK_MASK; 339 /* fallthrough */ 340 case MLX5_CQE_RESP_FORMAT_CSUM_STRIDX: 341 cq_attr.mini_cqe_res_format = priv->config.cqe_comp_fmt; 342 break; 343 case MLX5_CQE_RESP_FORMAT_L34H_STRIDX: 344 cq_attr.mini_cqe_res_format = 0; 345 cq_attr.mini_cqe_res_format_ext = 1; 346 break; 347 } 348 DRV_LOG(DEBUG, 349 "Port %u Rx CQE compression is enabled, format %d.", 350 port_id, priv->config.cqe_comp_fmt); 351 /* 352 * For vectorized Rx, it must not be doubled in order to 353 * make cq_ci and rq_ci aligned. 354 */ 355 if (mlx5_rxq_check_vec_support(rxq_data) < 0) 356 cqe_n *= 2; 357 } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) { 358 DRV_LOG(DEBUG, 359 "Port %u Rx CQE compression is disabled for HW timestamp.", 360 port_id); 361 } else if (priv->config.cqe_comp && rxq_data->lro) { 362 DRV_LOG(DEBUG, 363 "Port %u Rx CQE compression is disabled for LRO.", 364 port_id); 365 } 366 cq_attr.uar_page_id = mlx5_os_get_devx_uar_page_id(sh->rx_uar.obj); 367 log_cqe_n = log2above(cqe_n); 368 /* Create CQ using DevX API. */ 369 ret = mlx5_devx_cq_create(sh->cdev->ctx, &rxq_ctrl->obj->cq_obj, 370 log_cqe_n, &cq_attr, sh->numa_node); 371 if (ret) 372 return ret; 373 cq_obj = &rxq_ctrl->obj->cq_obj; 374 rxq_data->cqes = (volatile struct mlx5_cqe (*)[]) 375 (uintptr_t)cq_obj->cqes; 376 rxq_data->cq_db = cq_obj->db_rec; 377 rxq_data->uar_data = sh->rx_uar.cq_db; 378 rxq_data->cqe_n = log_cqe_n; 379 rxq_data->cqn = cq_obj->cq->id; 380 rxq_data->cq_ci = 0; 381 if (rxq_ctrl->obj->devx_channel) { 382 ret = mlx5_os_devx_subscribe_devx_event 383 (rxq_ctrl->obj->devx_channel, 384 cq_obj->cq->obj, 385 sizeof(event_nums), 386 event_nums, 387 (uint64_t)(uintptr_t)cq_obj->cq); 388 if (ret) { 389 DRV_LOG(ERR, "Fail to subscribe CQ to event channel."); 390 ret = errno; 391 mlx5_devx_cq_destroy(cq_obj); 392 memset(cq_obj, 0, sizeof(*cq_obj)); 393 rte_errno = ret; 394 return -ret; 395 } 396 } 397 return 0; 398 } 399 400 /** 401 * Create the Rx hairpin queue object. 402 * 403 * @param rxq 404 * Pointer to Rx queue. 405 * 406 * @return 407 * 0 on success, a negative errno value otherwise and rte_errno is set. 408 */ 409 static int 410 mlx5_rxq_obj_hairpin_new(struct mlx5_rxq_priv *rxq) 411 { 412 uint16_t idx = rxq->idx; 413 struct mlx5_priv *priv = rxq->priv; 414 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 415 struct mlx5_devx_create_rq_attr attr = { 0 }; 416 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 417 uint32_t max_wq_data; 418 419 MLX5_ASSERT(rxq != NULL && rxq->ctrl != NULL && tmpl != NULL); 420 tmpl->rxq_ctrl = rxq_ctrl; 421 attr.hairpin = 1; 422 max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; 423 /* Jumbo frames > 9KB should be supported, and more packets. */ 424 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 425 if (priv->config.log_hp_size > max_wq_data) { 426 DRV_LOG(ERR, "Total data size %u power of 2 is " 427 "too large for hairpin.", 428 priv->config.log_hp_size); 429 rte_errno = ERANGE; 430 return -rte_errno; 431 } 432 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 433 } else { 434 attr.wq_attr.log_hairpin_data_sz = 435 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 436 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 437 } 438 /* Set the packets number to the maximum value for performance. */ 439 attr.wq_attr.log_hairpin_num_packets = 440 attr.wq_attr.log_hairpin_data_sz - 441 MLX5_HAIRPIN_QUEUE_STRIDE; 442 attr.counter_set_id = priv->counter_set_id; 443 rxq_ctrl->rxq.delay_drop = priv->config.hp_delay_drop; 444 attr.delay_drop_en = priv->config.hp_delay_drop; 445 tmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->cdev->ctx, &attr, 446 rxq_ctrl->socket); 447 if (!tmpl->rq) { 448 DRV_LOG(ERR, 449 "Port %u Rx hairpin queue %u can't create rq object.", 450 priv->dev_data->port_id, idx); 451 rte_errno = errno; 452 return -rte_errno; 453 } 454 priv->dev_data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN; 455 return 0; 456 } 457 458 /** 459 * Create the Rx queue DevX object. 460 * 461 * @param rxq 462 * Pointer to Rx queue. 463 * 464 * @return 465 * 0 on success, a negative errno value otherwise and rte_errno is set. 466 */ 467 static int 468 mlx5_rxq_devx_obj_new(struct mlx5_rxq_priv *rxq) 469 { 470 struct mlx5_priv *priv = rxq->priv; 471 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 472 struct mlx5_rxq_data *rxq_data = &rxq_ctrl->rxq; 473 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 474 int ret = 0; 475 476 MLX5_ASSERT(rxq_data); 477 MLX5_ASSERT(tmpl); 478 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) 479 return mlx5_rxq_obj_hairpin_new(rxq); 480 tmpl->rxq_ctrl = rxq_ctrl; 481 if (rxq_ctrl->irq && !rxq_ctrl->started) { 482 int devx_ev_flag = 483 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA; 484 485 tmpl->devx_channel = mlx5_os_devx_create_event_channel 486 (priv->sh->cdev->ctx, 487 devx_ev_flag); 488 if (!tmpl->devx_channel) { 489 rte_errno = errno; 490 DRV_LOG(ERR, "Failed to create event channel %d.", 491 rte_errno); 492 goto error; 493 } 494 tmpl->fd = mlx5_os_get_devx_channel_fd(tmpl->devx_channel); 495 } 496 /* Create CQ using DevX API. */ 497 ret = mlx5_rxq_create_devx_cq_resources(rxq); 498 if (ret) { 499 DRV_LOG(ERR, "Failed to create CQ."); 500 goto error; 501 } 502 rxq_data->delay_drop = priv->config.std_delay_drop; 503 /* Create RQ using DevX API. */ 504 ret = mlx5_rxq_create_devx_rq_resources(rxq); 505 if (ret) { 506 DRV_LOG(ERR, "Port %u Rx queue %u RQ creation failure.", 507 priv->dev_data->port_id, rxq->idx); 508 rte_errno = ENOMEM; 509 goto error; 510 } 511 /* Change queue state to ready. */ 512 ret = mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RST2RDY); 513 if (ret) 514 goto error; 515 if (!rxq_data->shared) { 516 rxq_data->wqes = (void *)(uintptr_t)rxq->devx_rq.wq.umem_buf; 517 rxq_data->rq_db = (uint32_t *)(uintptr_t)rxq->devx_rq.wq.db_rec; 518 } else if (!rxq_ctrl->started) { 519 rxq_data->wqes = (void *)(uintptr_t)tmpl->devx_rmp.wq.umem_buf; 520 rxq_data->rq_db = 521 (uint32_t *)(uintptr_t)tmpl->devx_rmp.wq.db_rec; 522 } 523 if (!rxq_ctrl->started) { 524 mlx5_rxq_initialize(rxq_data); 525 rxq_ctrl->wqn = rxq->devx_rq.rq->id; 526 } 527 priv->dev_data->rx_queue_state[rxq->idx] = RTE_ETH_QUEUE_STATE_STARTED; 528 return 0; 529 error: 530 ret = rte_errno; /* Save rte_errno before cleanup. */ 531 mlx5_rxq_devx_obj_release(rxq); 532 rte_errno = ret; /* Restore rte_errno. */ 533 return -rte_errno; 534 } 535 536 /** 537 * Prepare RQT attribute structure for DevX RQT API. 538 * 539 * @param dev 540 * Pointer to Ethernet device. 541 * @param log_n 542 * Log of number of queues in the array. 543 * @param queues 544 * List of RX queue indices or NULL, in which case 545 * the attribute will be filled by drop queue ID. 546 * @param queues_n 547 * Size of @p queues array or 0 if it is NULL. 548 * @param ind_tbl 549 * DevX indirection table object. 550 * 551 * @return 552 * The RQT attr object initialized, NULL otherwise and rte_errno is set. 553 */ 554 static struct mlx5_devx_rqt_attr * 555 mlx5_devx_ind_table_create_rqt_attr(struct rte_eth_dev *dev, 556 const unsigned int log_n, 557 const uint16_t *queues, 558 const uint32_t queues_n) 559 { 560 struct mlx5_priv *priv = dev->data->dev_private; 561 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 562 const unsigned int rqt_n = 1 << log_n; 563 unsigned int i, j; 564 565 rqt_attr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rqt_attr) + 566 rqt_n * sizeof(uint32_t), 0, SOCKET_ID_ANY); 567 if (!rqt_attr) { 568 DRV_LOG(ERR, "Port %u cannot allocate RQT resources.", 569 dev->data->port_id); 570 rte_errno = ENOMEM; 571 return NULL; 572 } 573 rqt_attr->rqt_max_size = priv->config.ind_table_max_size; 574 rqt_attr->rqt_actual_size = rqt_n; 575 if (queues == NULL) { 576 for (i = 0; i < rqt_n; i++) 577 rqt_attr->rq_list[i] = 578 priv->drop_queue.rxq->devx_rq.rq->id; 579 return rqt_attr; 580 } 581 for (i = 0; i != queues_n; ++i) { 582 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, queues[i]); 583 584 MLX5_ASSERT(rxq != NULL); 585 if (rxq->ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) 586 rqt_attr->rq_list[i] = rxq->ctrl->obj->rq->id; 587 else 588 rqt_attr->rq_list[i] = rxq->devx_rq.rq->id; 589 } 590 MLX5_ASSERT(i > 0); 591 for (j = 0; i != rqt_n; ++j, ++i) 592 rqt_attr->rq_list[i] = rqt_attr->rq_list[j]; 593 return rqt_attr; 594 } 595 596 /** 597 * Create RQT using DevX API as a filed of indirection table. 598 * 599 * @param dev 600 * Pointer to Ethernet device. 601 * @param log_n 602 * Log of number of queues in the array. 603 * @param ind_tbl 604 * DevX indirection table object. 605 * 606 * @return 607 * 0 on success, a negative errno value otherwise and rte_errno is set. 608 */ 609 static int 610 mlx5_devx_ind_table_new(struct rte_eth_dev *dev, const unsigned int log_n, 611 struct mlx5_ind_table_obj *ind_tbl) 612 { 613 struct mlx5_priv *priv = dev->data->dev_private; 614 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 615 const uint16_t *queues = dev->data->dev_started ? ind_tbl->queues : 616 NULL; 617 618 MLX5_ASSERT(ind_tbl); 619 rqt_attr = mlx5_devx_ind_table_create_rqt_attr(dev, log_n, queues, 620 ind_tbl->queues_n); 621 if (!rqt_attr) 622 return -rte_errno; 623 ind_tbl->rqt = mlx5_devx_cmd_create_rqt(priv->sh->cdev->ctx, rqt_attr); 624 mlx5_free(rqt_attr); 625 if (!ind_tbl->rqt) { 626 DRV_LOG(ERR, "Port %u cannot create DevX RQT.", 627 dev->data->port_id); 628 rte_errno = errno; 629 return -rte_errno; 630 } 631 return 0; 632 } 633 634 /** 635 * Modify RQT using DevX API as a filed of indirection table. 636 * 637 * @param dev 638 * Pointer to Ethernet device. 639 * @param log_n 640 * Log of number of queues in the array. 641 * @param ind_tbl 642 * DevX indirection table object. 643 * 644 * @return 645 * 0 on success, a negative errno value otherwise and rte_errno is set. 646 */ 647 static int 648 mlx5_devx_ind_table_modify(struct rte_eth_dev *dev, const unsigned int log_n, 649 const uint16_t *queues, const uint32_t queues_n, 650 struct mlx5_ind_table_obj *ind_tbl) 651 { 652 int ret = 0; 653 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 654 655 MLX5_ASSERT(ind_tbl); 656 rqt_attr = mlx5_devx_ind_table_create_rqt_attr(dev, log_n, 657 queues, 658 queues_n); 659 if (!rqt_attr) 660 return -rte_errno; 661 ret = mlx5_devx_cmd_modify_rqt(ind_tbl->rqt, rqt_attr); 662 mlx5_free(rqt_attr); 663 if (ret) 664 DRV_LOG(ERR, "Port %u cannot modify DevX RQT.", 665 dev->data->port_id); 666 return ret; 667 } 668 669 /** 670 * Destroy the DevX RQT object. 671 * 672 * @param ind_table 673 * Indirection table to release. 674 */ 675 static void 676 mlx5_devx_ind_table_destroy(struct mlx5_ind_table_obj *ind_tbl) 677 { 678 claim_zero(mlx5_devx_cmd_destroy(ind_tbl->rqt)); 679 } 680 681 /** 682 * Set TIR attribute struct with relevant input values. 683 * 684 * @param[in] dev 685 * Pointer to Ethernet device. 686 * @param[in] rss_key 687 * RSS key for the Rx hash queue. 688 * @param[in] hash_fields 689 * Verbs protocol hash field to make the RSS on. 690 * @param[in] ind_tbl 691 * Indirection table for TIR. If table queues array is NULL, 692 * a TIR for drop queue is assumed. 693 * @param[in] tunnel 694 * Tunnel type. 695 * @param[out] tir_attr 696 * Parameters structure for TIR creation/modification. 697 * 698 * @return 699 * The Verbs/DevX object initialised index, 0 otherwise and rte_errno is set. 700 */ 701 static void 702 mlx5_devx_tir_attr_set(struct rte_eth_dev *dev, const uint8_t *rss_key, 703 uint64_t hash_fields, 704 const struct mlx5_ind_table_obj *ind_tbl, 705 int tunnel, struct mlx5_devx_tir_attr *tir_attr) 706 { 707 struct mlx5_priv *priv = dev->data->dev_private; 708 enum mlx5_rxq_type rxq_obj_type; 709 bool lro = true; 710 uint32_t i; 711 712 /* NULL queues designate drop queue. */ 713 if (ind_tbl->queues != NULL) { 714 struct mlx5_rxq_ctrl *rxq_ctrl = 715 mlx5_rxq_ctrl_get(dev, ind_tbl->queues[0]); 716 rxq_obj_type = rxq_ctrl != NULL ? rxq_ctrl->type : 717 MLX5_RXQ_TYPE_STANDARD; 718 719 /* Enable TIR LRO only if all the queues were configured for. */ 720 for (i = 0; i < ind_tbl->queues_n; ++i) { 721 struct mlx5_rxq_data *rxq_i = 722 mlx5_rxq_data_get(dev, ind_tbl->queues[i]); 723 724 if (rxq_i != NULL && !rxq_i->lro) { 725 lro = false; 726 break; 727 } 728 } 729 } else { 730 rxq_obj_type = priv->drop_queue.rxq->ctrl->type; 731 } 732 memset(tir_attr, 0, sizeof(*tir_attr)); 733 tir_attr->disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT; 734 tir_attr->rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ; 735 tir_attr->tunneled_offload_en = !!tunnel; 736 /* If needed, translate hash_fields bitmap to PRM format. */ 737 if (hash_fields) { 738 struct mlx5_rx_hash_field_select *rx_hash_field_select = 739 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 740 hash_fields & IBV_RX_HASH_INNER ? 741 &tir_attr->rx_hash_field_selector_inner : 742 #endif 743 &tir_attr->rx_hash_field_selector_outer; 744 /* 1 bit: 0: IPv4, 1: IPv6. */ 745 rx_hash_field_select->l3_prot_type = 746 !!(hash_fields & MLX5_IPV6_IBV_RX_HASH); 747 /* 1 bit: 0: TCP, 1: UDP. */ 748 rx_hash_field_select->l4_prot_type = 749 !!(hash_fields & MLX5_UDP_IBV_RX_HASH); 750 /* Bitmask which sets which fields to use in RX Hash. */ 751 rx_hash_field_select->selected_fields = 752 ((!!(hash_fields & MLX5_L3_SRC_IBV_RX_HASH)) << 753 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) | 754 (!!(hash_fields & MLX5_L3_DST_IBV_RX_HASH)) << 755 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP | 756 (!!(hash_fields & MLX5_L4_SRC_IBV_RX_HASH)) << 757 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT | 758 (!!(hash_fields & MLX5_L4_DST_IBV_RX_HASH)) << 759 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT; 760 } 761 if (rxq_obj_type == MLX5_RXQ_TYPE_HAIRPIN) 762 tir_attr->transport_domain = priv->sh->td->id; 763 else 764 tir_attr->transport_domain = priv->sh->tdn; 765 memcpy(tir_attr->rx_hash_toeplitz_key, rss_key, MLX5_RSS_HASH_KEY_LEN); 766 tir_attr->indirect_table = ind_tbl->rqt->id; 767 if (dev->data->dev_conf.lpbk_mode) 768 tir_attr->self_lb_block = 769 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST; 770 if (lro) { 771 tir_attr->lro_timeout_period_usecs = priv->config.lro.timeout; 772 tir_attr->lro_max_msg_sz = priv->max_lro_msg_size; 773 tir_attr->lro_enable_mask = 774 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | 775 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO; 776 } 777 } 778 779 /** 780 * Create an Rx Hash queue. 781 * 782 * @param dev 783 * Pointer to Ethernet device. 784 * @param hrxq 785 * Pointer to Rx Hash queue. 786 * @param tunnel 787 * Tunnel type. 788 * 789 * @return 790 * 0 on success, a negative errno value otherwise and rte_errno is set. 791 */ 792 static int 793 mlx5_devx_hrxq_new(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq, 794 int tunnel __rte_unused) 795 { 796 struct mlx5_priv *priv = dev->data->dev_private; 797 struct mlx5_devx_tir_attr tir_attr = {0}; 798 int err; 799 800 mlx5_devx_tir_attr_set(dev, hrxq->rss_key, hrxq->hash_fields, 801 hrxq->ind_table, tunnel, &tir_attr); 802 hrxq->tir = mlx5_devx_cmd_create_tir(priv->sh->cdev->ctx, &tir_attr); 803 if (!hrxq->tir) { 804 DRV_LOG(ERR, "Port %u cannot create DevX TIR.", 805 dev->data->port_id); 806 rte_errno = errno; 807 goto error; 808 } 809 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 810 if (mlx5_flow_os_create_flow_action_dest_devx_tir(hrxq->tir, 811 &hrxq->action)) { 812 rte_errno = errno; 813 goto error; 814 } 815 #endif 816 return 0; 817 error: 818 err = rte_errno; /* Save rte_errno before cleanup. */ 819 if (hrxq->tir) 820 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 821 rte_errno = err; /* Restore rte_errno. */ 822 return -rte_errno; 823 } 824 825 /** 826 * Destroy a DevX TIR object. 827 * 828 * @param hrxq 829 * Hash Rx queue to release its tir. 830 */ 831 static void 832 mlx5_devx_tir_destroy(struct mlx5_hrxq *hrxq) 833 { 834 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 835 } 836 837 /** 838 * Modify an Rx Hash queue configuration. 839 * 840 * @param dev 841 * Pointer to Ethernet device. 842 * @param hrxq 843 * Hash Rx queue to modify. 844 * @param rss_key 845 * RSS key for the Rx hash queue. 846 * @param hash_fields 847 * Verbs protocol hash field to make the RSS on. 848 * @param[in] ind_tbl 849 * Indirection table for TIR. 850 * 851 * @return 852 * 0 on success, a negative errno value otherwise and rte_errno is set. 853 */ 854 static int 855 mlx5_devx_hrxq_modify(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq, 856 const uint8_t *rss_key, 857 uint64_t hash_fields, 858 const struct mlx5_ind_table_obj *ind_tbl) 859 { 860 struct mlx5_devx_modify_tir_attr modify_tir = {0}; 861 862 /* 863 * untested for modification fields: 864 * - rx_hash_symmetric not set in hrxq_new(), 865 * - rx_hash_fn set hard-coded in hrxq_new(), 866 * - lro_xxx not set after rxq setup 867 */ 868 if (ind_tbl != hrxq->ind_table) 869 modify_tir.modify_bitmask |= 870 MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_INDIRECT_TABLE; 871 if (hash_fields != hrxq->hash_fields || 872 memcmp(hrxq->rss_key, rss_key, MLX5_RSS_HASH_KEY_LEN)) 873 modify_tir.modify_bitmask |= 874 MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_HASH; 875 mlx5_devx_tir_attr_set(dev, rss_key, hash_fields, ind_tbl, 876 0, /* N/A - tunnel modification unsupported */ 877 &modify_tir.tir); 878 modify_tir.tirn = hrxq->tir->id; 879 if (mlx5_devx_cmd_modify_tir(hrxq->tir, &modify_tir)) { 880 DRV_LOG(ERR, "port %u cannot modify DevX TIR", 881 dev->data->port_id); 882 rte_errno = errno; 883 return -rte_errno; 884 } 885 return 0; 886 } 887 888 /** 889 * Create a DevX drop Rx queue. 890 * 891 * @param dev 892 * Pointer to Ethernet device. 893 * 894 * @return 895 * 0 on success, a negative errno value otherwise and rte_errno is set. 896 */ 897 static int 898 mlx5_rxq_devx_obj_drop_create(struct rte_eth_dev *dev) 899 { 900 struct mlx5_priv *priv = dev->data->dev_private; 901 int socket_id = dev->device->numa_node; 902 struct mlx5_rxq_priv *rxq; 903 struct mlx5_rxq_ctrl *rxq_ctrl = NULL; 904 struct mlx5_rxq_obj *rxq_obj = NULL; 905 int ret; 906 907 /* 908 * Initialize dummy control structures. 909 * They are required to hold pointers for cleanup 910 * and are only accessible via drop queue DevX objects. 911 */ 912 rxq = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq), 0, socket_id); 913 if (rxq == NULL) { 914 DRV_LOG(ERR, "Port %u could not allocate drop queue private", 915 dev->data->port_id); 916 rte_errno = ENOMEM; 917 goto error; 918 } 919 rxq_ctrl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq_ctrl), 920 0, socket_id); 921 if (rxq_ctrl == NULL) { 922 DRV_LOG(ERR, "Port %u could not allocate drop queue control", 923 dev->data->port_id); 924 rte_errno = ENOMEM; 925 goto error; 926 } 927 rxq_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq_obj), 0, socket_id); 928 if (rxq_obj == NULL) { 929 DRV_LOG(ERR, "Port %u could not allocate drop queue object", 930 dev->data->port_id); 931 rte_errno = ENOMEM; 932 goto error; 933 } 934 rxq_obj->rxq_ctrl = rxq_ctrl; 935 rxq_ctrl->type = MLX5_RXQ_TYPE_STANDARD; 936 rxq_ctrl->sh = priv->sh; 937 rxq_ctrl->obj = rxq_obj; 938 rxq->ctrl = rxq_ctrl; 939 rxq->priv = priv; 940 LIST_INSERT_HEAD(&rxq_ctrl->owners, rxq, owner_entry); 941 /* Create CQ using DevX API. */ 942 ret = mlx5_rxq_create_devx_cq_resources(rxq); 943 if (ret != 0) { 944 DRV_LOG(ERR, "Port %u drop queue CQ creation failed.", 945 dev->data->port_id); 946 goto error; 947 } 948 rxq_ctrl->rxq.delay_drop = 0; 949 /* Create RQ using DevX API. */ 950 ret = mlx5_rxq_create_devx_rq_resources(rxq); 951 if (ret != 0) { 952 DRV_LOG(ERR, "Port %u drop queue RQ creation failed.", 953 dev->data->port_id); 954 rte_errno = ENOMEM; 955 goto error; 956 } 957 /* Change queue state to ready. */ 958 ret = mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RST2RDY); 959 if (ret != 0) 960 goto error; 961 /* Initialize drop queue. */ 962 priv->drop_queue.rxq = rxq; 963 return 0; 964 error: 965 ret = rte_errno; /* Save rte_errno before cleanup. */ 966 if (rxq != NULL && rxq->devx_rq.rq != NULL) 967 mlx5_devx_rq_destroy(&rxq->devx_rq); 968 if (rxq_obj != NULL) { 969 if (rxq_obj->cq_obj.cq != NULL) 970 mlx5_devx_cq_destroy(&rxq_obj->cq_obj); 971 if (rxq_obj->devx_channel) 972 mlx5_os_devx_destroy_event_channel 973 (rxq_obj->devx_channel); 974 mlx5_free(rxq_obj); 975 } 976 if (rxq_ctrl != NULL) 977 mlx5_free(rxq_ctrl); 978 if (rxq != NULL) 979 mlx5_free(rxq); 980 rte_errno = ret; /* Restore rte_errno. */ 981 return -rte_errno; 982 } 983 984 /** 985 * Release drop Rx queue resources. 986 * 987 * @param dev 988 * Pointer to Ethernet device. 989 */ 990 static void 991 mlx5_rxq_devx_obj_drop_release(struct rte_eth_dev *dev) 992 { 993 struct mlx5_priv *priv = dev->data->dev_private; 994 struct mlx5_rxq_priv *rxq = priv->drop_queue.rxq; 995 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 996 997 mlx5_rxq_devx_obj_release(rxq); 998 mlx5_free(rxq_ctrl->obj); 999 mlx5_free(rxq_ctrl); 1000 mlx5_free(rxq); 1001 priv->drop_queue.rxq = NULL; 1002 } 1003 1004 /** 1005 * Release a drop hash Rx queue. 1006 * 1007 * @param dev 1008 * Pointer to Ethernet device. 1009 */ 1010 static void 1011 mlx5_devx_drop_action_destroy(struct rte_eth_dev *dev) 1012 { 1013 struct mlx5_priv *priv = dev->data->dev_private; 1014 struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq; 1015 1016 if (hrxq->tir != NULL) 1017 mlx5_devx_tir_destroy(hrxq); 1018 if (hrxq->ind_table->ind_table != NULL) 1019 mlx5_devx_ind_table_destroy(hrxq->ind_table); 1020 if (priv->drop_queue.rxq->devx_rq.rq != NULL) 1021 mlx5_rxq_devx_obj_drop_release(dev); 1022 } 1023 1024 /** 1025 * Create a DevX drop action for Rx Hash queue. 1026 * 1027 * @param dev 1028 * Pointer to Ethernet device. 1029 * 1030 * @return 1031 * 0 on success, a negative errno value otherwise and rte_errno is set. 1032 */ 1033 static int 1034 mlx5_devx_drop_action_create(struct rte_eth_dev *dev) 1035 { 1036 struct mlx5_priv *priv = dev->data->dev_private; 1037 struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq; 1038 int ret; 1039 1040 ret = mlx5_rxq_devx_obj_drop_create(dev); 1041 if (ret != 0) { 1042 DRV_LOG(ERR, "Cannot create drop RX queue"); 1043 return ret; 1044 } 1045 /* hrxq->ind_table queues are NULL, drop RX queue ID will be used */ 1046 ret = mlx5_devx_ind_table_new(dev, 0, hrxq->ind_table); 1047 if (ret != 0) { 1048 DRV_LOG(ERR, "Cannot create drop hash RX queue indirection table"); 1049 goto error; 1050 } 1051 ret = mlx5_devx_hrxq_new(dev, hrxq, /* tunnel */ false); 1052 if (ret != 0) { 1053 DRV_LOG(ERR, "Cannot create drop hash RX queue"); 1054 goto error; 1055 } 1056 return 0; 1057 error: 1058 mlx5_devx_drop_action_destroy(dev); 1059 return ret; 1060 } 1061 1062 /** 1063 * Select TXQ TIS number. 1064 * 1065 * @param dev 1066 * Pointer to Ethernet device. 1067 * @param queue_idx 1068 * Queue index in DPDK Tx queue array. 1069 * 1070 * @return 1071 * > 0 on success, a negative errno value otherwise. 1072 */ 1073 static uint32_t 1074 mlx5_get_txq_tis_num(struct rte_eth_dev *dev, uint16_t queue_idx) 1075 { 1076 struct mlx5_priv *priv = dev->data->dev_private; 1077 int tis_idx; 1078 1079 if (priv->sh->bond.n_port && priv->sh->lag.affinity_mode == 1080 MLX5_LAG_MODE_TIS) { 1081 tis_idx = (priv->lag_affinity_idx + queue_idx) % 1082 priv->sh->bond.n_port; 1083 DRV_LOG(INFO, "port %d txq %d gets affinity %d and maps to PF %d.", 1084 dev->data->port_id, queue_idx, tis_idx + 1, 1085 priv->sh->lag.tx_remap_affinity[tis_idx]); 1086 } else { 1087 tis_idx = 0; 1088 } 1089 MLX5_ASSERT(priv->sh->tis[tis_idx]); 1090 return priv->sh->tis[tis_idx]->id; 1091 } 1092 1093 /** 1094 * Create the Tx hairpin queue object. 1095 * 1096 * @param dev 1097 * Pointer to Ethernet device. 1098 * @param idx 1099 * Queue index in DPDK Tx queue array. 1100 * 1101 * @return 1102 * 0 on success, a negative errno value otherwise and rte_errno is set. 1103 */ 1104 static int 1105 mlx5_txq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) 1106 { 1107 struct mlx5_priv *priv = dev->data->dev_private; 1108 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1109 struct mlx5_txq_ctrl *txq_ctrl = 1110 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1111 struct mlx5_devx_create_sq_attr attr = { 0 }; 1112 struct mlx5_txq_obj *tmpl = txq_ctrl->obj; 1113 uint32_t max_wq_data; 1114 1115 MLX5_ASSERT(txq_data); 1116 MLX5_ASSERT(tmpl); 1117 tmpl->txq_ctrl = txq_ctrl; 1118 attr.hairpin = 1; 1119 attr.tis_lst_sz = 1; 1120 max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; 1121 /* Jumbo frames > 9KB should be supported, and more packets. */ 1122 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 1123 if (priv->config.log_hp_size > max_wq_data) { 1124 DRV_LOG(ERR, "Total data size %u power of 2 is " 1125 "too large for hairpin.", 1126 priv->config.log_hp_size); 1127 rte_errno = ERANGE; 1128 return -rte_errno; 1129 } 1130 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 1131 } else { 1132 attr.wq_attr.log_hairpin_data_sz = 1133 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 1134 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 1135 } 1136 /* Set the packets number to the maximum value for performance. */ 1137 attr.wq_attr.log_hairpin_num_packets = 1138 attr.wq_attr.log_hairpin_data_sz - 1139 MLX5_HAIRPIN_QUEUE_STRIDE; 1140 1141 attr.tis_num = mlx5_get_txq_tis_num(dev, idx); 1142 tmpl->sq = mlx5_devx_cmd_create_sq(priv->sh->cdev->ctx, &attr); 1143 if (!tmpl->sq) { 1144 DRV_LOG(ERR, 1145 "Port %u tx hairpin queue %u can't create SQ object.", 1146 dev->data->port_id, idx); 1147 rte_errno = errno; 1148 return -rte_errno; 1149 } 1150 return 0; 1151 } 1152 1153 #if defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) || !defined(HAVE_INFINIBAND_VERBS_H) 1154 /** 1155 * Destroy the Tx queue DevX object. 1156 * 1157 * @param txq_obj 1158 * Txq object to destroy. 1159 */ 1160 static void 1161 mlx5_txq_release_devx_resources(struct mlx5_txq_obj *txq_obj) 1162 { 1163 mlx5_devx_sq_destroy(&txq_obj->sq_obj); 1164 memset(&txq_obj->sq_obj, 0, sizeof(txq_obj->sq_obj)); 1165 mlx5_devx_cq_destroy(&txq_obj->cq_obj); 1166 memset(&txq_obj->cq_obj, 0, sizeof(txq_obj->cq_obj)); 1167 } 1168 1169 /** 1170 * Create a SQ object and its resources using DevX. 1171 * 1172 * @param dev 1173 * Pointer to Ethernet device. 1174 * @param idx 1175 * Queue index in DPDK Tx queue array. 1176 * @param[in] log_desc_n 1177 * Log of number of descriptors in queue. 1178 * 1179 * @return 1180 * 0 on success, a negative errno value otherwise and rte_errno is set. 1181 */ 1182 static int 1183 mlx5_txq_create_devx_sq_resources(struct rte_eth_dev *dev, uint16_t idx, 1184 uint16_t log_desc_n) 1185 { 1186 struct mlx5_priv *priv = dev->data->dev_private; 1187 struct mlx5_common_device *cdev = priv->sh->cdev; 1188 struct mlx5_uar *uar = &priv->sh->tx_uar; 1189 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1190 struct mlx5_txq_ctrl *txq_ctrl = 1191 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1192 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1193 struct mlx5_devx_create_sq_attr sq_attr = { 1194 .flush_in_error_en = 1, 1195 .allow_multi_pkt_send_wqe = !!priv->config.mps, 1196 .min_wqe_inline_mode = priv->config.hca_attr.vport_inline_mode, 1197 .allow_swp = !!priv->config.swp, 1198 .cqn = txq_obj->cq_obj.cq->id, 1199 .tis_lst_sz = 1, 1200 .wq_attr = (struct mlx5_devx_wq_attr){ 1201 .pd = cdev->pdn, 1202 .uar_page = mlx5_os_get_devx_uar_page_id(uar->obj), 1203 }, 1204 .ts_format = 1205 mlx5_ts_format_conv(cdev->config.hca_attr.sq_ts_format), 1206 .tis_num = mlx5_get_txq_tis_num(dev, idx), 1207 }; 1208 1209 /* Create Send Queue object with DevX. */ 1210 return mlx5_devx_sq_create(cdev->ctx, &txq_obj->sq_obj, 1211 log_desc_n, &sq_attr, priv->sh->numa_node); 1212 } 1213 #endif 1214 1215 /** 1216 * Create the Tx queue DevX object. 1217 * 1218 * @param dev 1219 * Pointer to Ethernet device. 1220 * @param idx 1221 * Queue index in DPDK Tx queue array. 1222 * 1223 * @return 1224 * 0 on success, a negative errno value otherwise and rte_errno is set. 1225 */ 1226 int 1227 mlx5_txq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx) 1228 { 1229 struct mlx5_priv *priv = dev->data->dev_private; 1230 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1231 struct mlx5_txq_ctrl *txq_ctrl = 1232 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1233 1234 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) 1235 return mlx5_txq_obj_hairpin_new(dev, idx); 1236 #if !defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) && defined(HAVE_INFINIBAND_VERBS_H) 1237 DRV_LOG(ERR, "Port %u Tx queue %u cannot create with DevX, no UAR.", 1238 dev->data->port_id, idx); 1239 rte_errno = ENOMEM; 1240 return -rte_errno; 1241 #else 1242 struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv)); 1243 struct mlx5_dev_ctx_shared *sh = priv->sh; 1244 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1245 struct mlx5_devx_cq_attr cq_attr = { 1246 .uar_page_id = mlx5_os_get_devx_uar_page_id(sh->tx_uar.obj), 1247 }; 1248 uint32_t cqe_n, log_desc_n; 1249 uint32_t wqe_n, wqe_size; 1250 int ret = 0; 1251 1252 MLX5_ASSERT(txq_data); 1253 MLX5_ASSERT(txq_obj); 1254 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1255 MLX5_ASSERT(ppriv); 1256 txq_obj->txq_ctrl = txq_ctrl; 1257 txq_obj->dev = dev; 1258 cqe_n = (1UL << txq_data->elts_n) / MLX5_TX_COMP_THRESH + 1259 1 + MLX5_TX_COMP_THRESH_INLINE_DIV; 1260 log_desc_n = log2above(cqe_n); 1261 cqe_n = 1UL << log_desc_n; 1262 if (cqe_n > UINT16_MAX) { 1263 DRV_LOG(ERR, "Port %u Tx queue %u requests to many CQEs %u.", 1264 dev->data->port_id, txq_data->idx, cqe_n); 1265 rte_errno = EINVAL; 1266 return 0; 1267 } 1268 /* Create completion queue object with DevX. */ 1269 ret = mlx5_devx_cq_create(sh->cdev->ctx, &txq_obj->cq_obj, log_desc_n, 1270 &cq_attr, priv->sh->numa_node); 1271 if (ret) { 1272 DRV_LOG(ERR, "Port %u Tx queue %u CQ creation failure.", 1273 dev->data->port_id, idx); 1274 goto error; 1275 } 1276 txq_data->cqe_n = log_desc_n; 1277 txq_data->cqe_s = cqe_n; 1278 txq_data->cqe_m = txq_data->cqe_s - 1; 1279 txq_data->cqes = txq_obj->cq_obj.cqes; 1280 txq_data->cq_ci = 0; 1281 txq_data->cq_pi = 0; 1282 txq_data->cq_db = txq_obj->cq_obj.db_rec; 1283 *txq_data->cq_db = 0; 1284 /* 1285 * Adjust the amount of WQEs depending on inline settings. 1286 * The number of descriptors should be enough to handle 1287 * the specified number of packets. If queue is being created 1288 * with Verbs the rdma-core does queue size adjustment 1289 * internally in the mlx5_calc_sq_size(), we do the same 1290 * for the queue being created with DevX at this point. 1291 */ 1292 wqe_size = txq_data->tso_en ? 1293 RTE_ALIGN(txq_ctrl->max_tso_header, MLX5_WSEG_SIZE) : 0; 1294 wqe_size += sizeof(struct mlx5_wqe_cseg) + 1295 sizeof(struct mlx5_wqe_eseg) + 1296 sizeof(struct mlx5_wqe_dseg); 1297 if (txq_data->inlen_send) 1298 wqe_size = RTE_MAX(wqe_size, sizeof(struct mlx5_wqe_cseg) + 1299 sizeof(struct mlx5_wqe_eseg) + 1300 RTE_ALIGN(txq_data->inlen_send + 1301 sizeof(uint32_t), 1302 MLX5_WSEG_SIZE)); 1303 wqe_size = RTE_ALIGN(wqe_size, MLX5_WQE_SIZE) / MLX5_WQE_SIZE; 1304 /* Create Send Queue object with DevX. */ 1305 wqe_n = RTE_MIN((1UL << txq_data->elts_n) * wqe_size, 1306 (uint32_t)priv->sh->device_attr.max_qp_wr); 1307 log_desc_n = log2above(wqe_n); 1308 ret = mlx5_txq_create_devx_sq_resources(dev, idx, log_desc_n); 1309 if (ret) { 1310 DRV_LOG(ERR, "Port %u Tx queue %u SQ creation failure.", 1311 dev->data->port_id, idx); 1312 rte_errno = errno; 1313 goto error; 1314 } 1315 /* Create the Work Queue. */ 1316 txq_data->wqe_n = log_desc_n; 1317 txq_data->wqe_s = 1 << txq_data->wqe_n; 1318 txq_data->wqe_m = txq_data->wqe_s - 1; 1319 txq_data->wqes = (struct mlx5_wqe *)(uintptr_t)txq_obj->sq_obj.wqes; 1320 txq_data->wqes_end = txq_data->wqes + txq_data->wqe_s; 1321 txq_data->wqe_ci = 0; 1322 txq_data->wqe_pi = 0; 1323 txq_data->wqe_comp = 0; 1324 txq_data->wqe_thres = txq_data->wqe_s / MLX5_TX_COMP_THRESH_INLINE_DIV; 1325 txq_data->qp_db = &txq_obj->sq_obj.db_rec[MLX5_SND_DBR]; 1326 *txq_data->qp_db = 0; 1327 txq_data->qp_num_8s = txq_obj->sq_obj.sq->id << 8; 1328 txq_data->db_heu = sh->cdev->config.dbnc == MLX5_TXDB_HEURISTIC; 1329 txq_data->db_nc = sh->tx_uar.dbnc; 1330 /* Change Send Queue state to Ready-to-Send. */ 1331 ret = mlx5_txq_devx_modify(txq_obj, MLX5_TXQ_MOD_RST2RDY, 0); 1332 if (ret) { 1333 rte_errno = errno; 1334 DRV_LOG(ERR, 1335 "Port %u Tx queue %u SQ state to SQC_STATE_RDY failed.", 1336 dev->data->port_id, idx); 1337 goto error; 1338 } 1339 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 1340 /* 1341 * If using DevX need to query and store TIS transport domain value. 1342 * This is done once per port. 1343 * Will use this value on Rx, when creating matching TIR. 1344 */ 1345 if (!priv->sh->tdn) 1346 priv->sh->tdn = priv->sh->td->id; 1347 #endif 1348 txq_ctrl->uar_mmap_offset = 1349 mlx5_os_get_devx_uar_mmap_offset(sh->tx_uar.obj); 1350 ppriv->uar_table[txq_data->idx] = sh->tx_uar.bf_db; 1351 dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED; 1352 return 0; 1353 error: 1354 ret = rte_errno; /* Save rte_errno before cleanup. */ 1355 mlx5_txq_release_devx_resources(txq_obj); 1356 rte_errno = ret; /* Restore rte_errno. */ 1357 return -rte_errno; 1358 #endif 1359 } 1360 1361 /** 1362 * Release an Tx DevX queue object. 1363 * 1364 * @param txq_obj 1365 * DevX Tx queue object. 1366 */ 1367 void 1368 mlx5_txq_devx_obj_release(struct mlx5_txq_obj *txq_obj) 1369 { 1370 MLX5_ASSERT(txq_obj); 1371 if (txq_obj->txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 1372 if (txq_obj->tis) 1373 claim_zero(mlx5_devx_cmd_destroy(txq_obj->tis)); 1374 #if defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) || !defined(HAVE_INFINIBAND_VERBS_H) 1375 } else { 1376 mlx5_txq_release_devx_resources(txq_obj); 1377 #endif 1378 } 1379 } 1380 1381 struct mlx5_obj_ops devx_obj_ops = { 1382 .rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_rq_vlan_strip, 1383 .rxq_obj_new = mlx5_rxq_devx_obj_new, 1384 .rxq_event_get = mlx5_rx_devx_get_event, 1385 .rxq_obj_modify = mlx5_devx_modify_rq, 1386 .rxq_obj_release = mlx5_rxq_devx_obj_release, 1387 .ind_table_new = mlx5_devx_ind_table_new, 1388 .ind_table_modify = mlx5_devx_ind_table_modify, 1389 .ind_table_destroy = mlx5_devx_ind_table_destroy, 1390 .hrxq_new = mlx5_devx_hrxq_new, 1391 .hrxq_destroy = mlx5_devx_tir_destroy, 1392 .hrxq_modify = mlx5_devx_hrxq_modify, 1393 .drop_action_create = mlx5_devx_drop_action_create, 1394 .drop_action_destroy = mlx5_devx_drop_action_destroy, 1395 .txq_obj_new = mlx5_txq_devx_obj_new, 1396 .txq_obj_modify = mlx5_txq_devx_modify, 1397 .txq_obj_release = mlx5_txq_devx_obj_release, 1398 .lb_dummy_queue_create = NULL, 1399 .lb_dummy_queue_release = NULL, 1400 }; 1401