1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <stddef.h> 6 #include <errno.h> 7 #include <stdbool.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <sys/queue.h> 11 12 #include <rte_malloc.h> 13 #include <rte_common.h> 14 #include <rte_eal_paging.h> 15 16 #include <mlx5_glue.h> 17 #include <mlx5_devx_cmds.h> 18 #include <mlx5_common_devx.h> 19 #include <mlx5_malloc.h> 20 21 #include "mlx5.h" 22 #include "mlx5_common_os.h" 23 #include "mlx5_tx.h" 24 #include "mlx5_rx.h" 25 #include "mlx5_utils.h" 26 #include "mlx5_devx.h" 27 #include "mlx5_flow.h" 28 #include "mlx5_flow_os.h" 29 30 /** 31 * Modify RQ vlan stripping offload 32 * 33 * @param rxq 34 * Rx queue. 35 * @param on 36 * Enable/disable VLAN stripping. 37 * 38 * @return 39 * 0 on success, non-0 otherwise 40 */ 41 static int 42 mlx5_rxq_obj_modify_rq_vlan_strip(struct mlx5_rxq_priv *rxq, int on) 43 { 44 struct mlx5_devx_modify_rq_attr rq_attr; 45 46 memset(&rq_attr, 0, sizeof(rq_attr)); 47 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 48 rq_attr.state = MLX5_RQC_STATE_RDY; 49 rq_attr.vsd = (on ? 0 : 1); 50 rq_attr.modify_bitmask = MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD; 51 return mlx5_devx_cmd_modify_rq(rxq->devx_rq.rq, &rq_attr); 52 } 53 54 /** 55 * Modify RQ using DevX API. 56 * 57 * @param rxq 58 * DevX rx queue. 59 * @param type 60 * Type of change queue state. 61 * 62 * @return 63 * 0 on success, a negative errno value otherwise and rte_errno is set. 64 */ 65 static int 66 mlx5_devx_modify_rq(struct mlx5_rxq_priv *rxq, uint8_t type) 67 { 68 struct mlx5_devx_modify_rq_attr rq_attr; 69 70 memset(&rq_attr, 0, sizeof(rq_attr)); 71 switch (type) { 72 case MLX5_RXQ_MOD_ERR2RST: 73 rq_attr.rq_state = MLX5_RQC_STATE_ERR; 74 rq_attr.state = MLX5_RQC_STATE_RST; 75 break; 76 case MLX5_RXQ_MOD_RST2RDY: 77 rq_attr.rq_state = MLX5_RQC_STATE_RST; 78 rq_attr.state = MLX5_RQC_STATE_RDY; 79 break; 80 case MLX5_RXQ_MOD_RDY2ERR: 81 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 82 rq_attr.state = MLX5_RQC_STATE_ERR; 83 break; 84 case MLX5_RXQ_MOD_RDY2RST: 85 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 86 rq_attr.state = MLX5_RQC_STATE_RST; 87 break; 88 default: 89 break; 90 } 91 if (rxq->ctrl->is_hairpin) 92 return mlx5_devx_cmd_modify_rq(rxq->ctrl->obj->rq, &rq_attr); 93 return mlx5_devx_cmd_modify_rq(rxq->devx_rq.rq, &rq_attr); 94 } 95 96 /** 97 * Modify SQ using DevX API. 98 * 99 * @param txq_obj 100 * DevX Tx queue object. 101 * @param type 102 * Type of change queue state. 103 * @param dev_port 104 * Unnecessary. 105 * 106 * @return 107 * 0 on success, a negative errno value otherwise and rte_errno is set. 108 */ 109 int 110 mlx5_txq_devx_modify(struct mlx5_txq_obj *obj, enum mlx5_txq_modify_type type, 111 uint8_t dev_port) 112 { 113 struct mlx5_devx_modify_sq_attr msq_attr = { 0 }; 114 int ret; 115 116 if (type != MLX5_TXQ_MOD_RST2RDY) { 117 /* Change queue state to reset. */ 118 if (type == MLX5_TXQ_MOD_ERR2RDY) 119 msq_attr.sq_state = MLX5_SQC_STATE_ERR; 120 else 121 msq_attr.sq_state = MLX5_SQC_STATE_RDY; 122 msq_attr.state = MLX5_SQC_STATE_RST; 123 ret = mlx5_devx_cmd_modify_sq(obj->sq_obj.sq, &msq_attr); 124 if (ret) { 125 DRV_LOG(ERR, "Cannot change the Tx SQ state to RESET" 126 " %s", strerror(errno)); 127 rte_errno = errno; 128 return ret; 129 } 130 } 131 if (type != MLX5_TXQ_MOD_RDY2RST) { 132 /* Change queue state to ready. */ 133 msq_attr.sq_state = MLX5_SQC_STATE_RST; 134 msq_attr.state = MLX5_SQC_STATE_RDY; 135 ret = mlx5_devx_cmd_modify_sq(obj->sq_obj.sq, &msq_attr); 136 if (ret) { 137 DRV_LOG(ERR, "Cannot change the Tx SQ state to READY" 138 " %s", strerror(errno)); 139 rte_errno = errno; 140 return ret; 141 } 142 } 143 /* 144 * The dev_port variable is relevant only in Verbs API, and there is a 145 * pointer that points to this function and a parallel function in verbs 146 * intermittently, so they should have the same parameters. 147 */ 148 (void)dev_port; 149 return 0; 150 } 151 152 /** 153 * Release an Rx DevX queue object. 154 * 155 * @param rxq 156 * DevX Rx queue. 157 */ 158 static void 159 mlx5_rxq_devx_obj_release(struct mlx5_rxq_priv *rxq) 160 { 161 struct mlx5_rxq_obj *rxq_obj = rxq->ctrl->obj; 162 163 if (rxq_obj == NULL) 164 return; 165 if (rxq_obj->rxq_ctrl->is_hairpin) { 166 if (rxq_obj->rq == NULL) 167 return; 168 mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RDY2RST); 169 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 170 } else { 171 if (rxq->devx_rq.rq == NULL) 172 return; 173 mlx5_devx_rq_destroy(&rxq->devx_rq); 174 if (rxq->devx_rq.rmp != NULL && rxq->devx_rq.rmp->ref_cnt > 0) 175 return; 176 mlx5_devx_cq_destroy(&rxq_obj->cq_obj); 177 memset(&rxq_obj->cq_obj, 0, sizeof(rxq_obj->cq_obj)); 178 if (rxq_obj->devx_channel) { 179 mlx5_os_devx_destroy_event_channel 180 (rxq_obj->devx_channel); 181 rxq_obj->devx_channel = NULL; 182 } 183 } 184 rxq->ctrl->started = false; 185 } 186 187 /** 188 * Get event for an Rx DevX queue object. 189 * 190 * @param rxq_obj 191 * DevX Rx queue object. 192 * 193 * @return 194 * 0 on success, a negative errno value otherwise and rte_errno is set. 195 */ 196 static int 197 mlx5_rx_devx_get_event(struct mlx5_rxq_obj *rxq_obj) 198 { 199 #ifdef HAVE_IBV_DEVX_EVENT 200 union { 201 struct mlx5dv_devx_async_event_hdr event_resp; 202 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128]; 203 } out; 204 int ret = mlx5_glue->devx_get_event(rxq_obj->devx_channel, 205 &out.event_resp, 206 sizeof(out.buf)); 207 208 if (ret < 0) { 209 rte_errno = errno; 210 return -rte_errno; 211 } 212 if (out.event_resp.cookie != (uint64_t)(uintptr_t)rxq_obj->cq_obj.cq) { 213 rte_errno = EINVAL; 214 return -rte_errno; 215 } 216 return 0; 217 #else 218 (void)rxq_obj; 219 rte_errno = ENOTSUP; 220 return -rte_errno; 221 #endif /* HAVE_IBV_DEVX_EVENT */ 222 } 223 224 /** 225 * Create a RQ object using DevX. 226 * 227 * @param rxq 228 * Pointer to Rx queue. 229 * 230 * @return 231 * 0 on success, a negative errno value otherwise and rte_errno is set. 232 */ 233 static int 234 mlx5_rxq_create_devx_rq_resources(struct mlx5_rxq_priv *rxq) 235 { 236 struct mlx5_priv *priv = rxq->priv; 237 struct mlx5_common_device *cdev = priv->sh->cdev; 238 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 239 struct mlx5_rxq_data *rxq_data = &rxq->ctrl->rxq; 240 struct mlx5_devx_create_rq_attr rq_attr = { 0 }; 241 uint16_t log_desc_n = rxq_data->elts_n - rxq_data->sges_n; 242 uint32_t wqe_size, log_wqe_size; 243 244 /* Fill RQ attributes. */ 245 rq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE; 246 rq_attr.flush_in_error_en = 1; 247 rq_attr.vsd = (rxq_data->vlan_strip) ? 0 : 1; 248 rq_attr.cqn = rxq_ctrl->obj->cq_obj.cq->id; 249 rq_attr.scatter_fcs = (rxq_data->crc_present) ? 1 : 0; 250 rq_attr.ts_format = 251 mlx5_ts_format_conv(cdev->config.hca_attr.rq_ts_format); 252 /* Fill WQ attributes for this RQ. */ 253 if (mlx5_rxq_mprq_enabled(rxq_data)) { 254 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ; 255 /* 256 * Number of strides in each WQE: 257 * 512*2^single_wqe_log_num_of_strides. 258 */ 259 rq_attr.wq_attr.single_wqe_log_num_of_strides = 260 rxq_data->log_strd_num - 261 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES; 262 /* Stride size = (2^single_stride_log_num_of_bytes)*64B. */ 263 rq_attr.wq_attr.single_stride_log_num_of_bytes = 264 rxq_data->log_strd_sz - 265 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES; 266 wqe_size = sizeof(struct mlx5_wqe_mprq); 267 } else { 268 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 269 wqe_size = sizeof(struct mlx5_wqe_data_seg); 270 } 271 log_wqe_size = log2above(wqe_size) + rxq_data->sges_n; 272 wqe_size = 1 << log_wqe_size; /* round up power of two.*/ 273 rq_attr.wq_attr.log_wq_stride = log_wqe_size; 274 rq_attr.wq_attr.log_wq_sz = log_desc_n; 275 rq_attr.wq_attr.end_padding_mode = priv->config.hw_padding ? 276 MLX5_WQ_END_PAD_MODE_ALIGN : 277 MLX5_WQ_END_PAD_MODE_NONE; 278 rq_attr.wq_attr.pd = cdev->pdn; 279 rq_attr.counter_set_id = priv->counter_set_id; 280 rq_attr.delay_drop_en = rxq_data->delay_drop; 281 rq_attr.user_index = rte_cpu_to_be_16(priv->dev_data->port_id); 282 if (rxq_data->shared) /* Create RMP based RQ. */ 283 rxq->devx_rq.rmp = &rxq_ctrl->obj->devx_rmp; 284 /* Create RQ using DevX API. */ 285 return mlx5_devx_rq_create(cdev->ctx, &rxq->devx_rq, wqe_size, 286 log_desc_n, &rq_attr, rxq_ctrl->socket); 287 } 288 289 /** 290 * Create a DevX CQ object for an Rx queue. 291 * 292 * @param rxq 293 * Pointer to Rx queue. 294 * 295 * @return 296 * 0 on success, a negative errno value otherwise and rte_errno is set. 297 */ 298 static int 299 mlx5_rxq_create_devx_cq_resources(struct mlx5_rxq_priv *rxq) 300 { 301 struct mlx5_devx_cq *cq_obj = 0; 302 struct mlx5_devx_cq_attr cq_attr = { 0 }; 303 struct mlx5_priv *priv = rxq->priv; 304 struct mlx5_dev_ctx_shared *sh = priv->sh; 305 uint16_t port_id = priv->dev_data->port_id; 306 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 307 struct mlx5_rxq_data *rxq_data = &rxq_ctrl->rxq; 308 unsigned int cqe_n = mlx5_rxq_cqe_num(rxq_data); 309 uint32_t log_cqe_n; 310 uint16_t event_nums[1] = { 0 }; 311 int ret = 0; 312 313 if (rxq_ctrl->started) 314 return 0; 315 if (priv->config.cqe_comp && !rxq_data->hw_timestamp && 316 !rxq_data->lro) { 317 cq_attr.cqe_comp_en = 1u; 318 rxq_data->mcqe_format = priv->config.cqe_comp_fmt; 319 rxq_data->byte_mask = UINT32_MAX; 320 switch (priv->config.cqe_comp_fmt) { 321 case MLX5_CQE_RESP_FORMAT_HASH: 322 /* fallthrough */ 323 case MLX5_CQE_RESP_FORMAT_CSUM: 324 /* 325 * Select CSUM miniCQE format only for non-vectorized 326 * MPRQ Rx burst, use HASH miniCQE format for others. 327 */ 328 if (mlx5_rxq_check_vec_support(rxq_data) < 0 && 329 mlx5_rxq_mprq_enabled(rxq_data)) 330 cq_attr.mini_cqe_res_format = 331 MLX5_CQE_RESP_FORMAT_CSUM_STRIDX; 332 else 333 cq_attr.mini_cqe_res_format = 334 MLX5_CQE_RESP_FORMAT_HASH; 335 rxq_data->mcqe_format = cq_attr.mini_cqe_res_format; 336 break; 337 case MLX5_CQE_RESP_FORMAT_FTAG_STRIDX: 338 rxq_data->byte_mask = MLX5_LEN_WITH_MARK_MASK; 339 /* fallthrough */ 340 case MLX5_CQE_RESP_FORMAT_CSUM_STRIDX: 341 cq_attr.mini_cqe_res_format = priv->config.cqe_comp_fmt; 342 break; 343 case MLX5_CQE_RESP_FORMAT_L34H_STRIDX: 344 cq_attr.mini_cqe_res_format = 0; 345 cq_attr.mini_cqe_res_format_ext = 1; 346 break; 347 } 348 DRV_LOG(DEBUG, 349 "Port %u Rx CQE compression is enabled, format %d.", 350 port_id, priv->config.cqe_comp_fmt); 351 /* 352 * For vectorized Rx, it must not be doubled in order to 353 * make cq_ci and rq_ci aligned. 354 */ 355 if (mlx5_rxq_check_vec_support(rxq_data) < 0) 356 cqe_n *= 2; 357 } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) { 358 DRV_LOG(DEBUG, 359 "Port %u Rx CQE compression is disabled for HW timestamp.", 360 port_id); 361 } else if (priv->config.cqe_comp && rxq_data->lro) { 362 DRV_LOG(DEBUG, 363 "Port %u Rx CQE compression is disabled for LRO.", 364 port_id); 365 } 366 cq_attr.uar_page_id = mlx5_os_get_devx_uar_page_id(sh->rx_uar.obj); 367 log_cqe_n = log2above(cqe_n); 368 /* Create CQ using DevX API. */ 369 ret = mlx5_devx_cq_create(sh->cdev->ctx, &rxq_ctrl->obj->cq_obj, 370 log_cqe_n, &cq_attr, sh->numa_node); 371 if (ret) 372 return ret; 373 cq_obj = &rxq_ctrl->obj->cq_obj; 374 rxq_data->cqes = (volatile struct mlx5_cqe (*)[]) 375 (uintptr_t)cq_obj->cqes; 376 rxq_data->cq_db = cq_obj->db_rec; 377 rxq_data->uar_data = sh->rx_uar.cq_db; 378 rxq_data->cqe_n = log_cqe_n; 379 rxq_data->cqn = cq_obj->cq->id; 380 rxq_data->cq_ci = 0; 381 if (rxq_ctrl->obj->devx_channel) { 382 ret = mlx5_os_devx_subscribe_devx_event 383 (rxq_ctrl->obj->devx_channel, 384 cq_obj->cq->obj, 385 sizeof(event_nums), 386 event_nums, 387 (uint64_t)(uintptr_t)cq_obj->cq); 388 if (ret) { 389 DRV_LOG(ERR, "Fail to subscribe CQ to event channel."); 390 ret = errno; 391 mlx5_devx_cq_destroy(cq_obj); 392 memset(cq_obj, 0, sizeof(*cq_obj)); 393 rte_errno = ret; 394 return -ret; 395 } 396 } 397 return 0; 398 } 399 400 /** 401 * Create the Rx hairpin queue object. 402 * 403 * @param rxq 404 * Pointer to Rx queue. 405 * 406 * @return 407 * 0 on success, a negative errno value otherwise and rte_errno is set. 408 */ 409 static int 410 mlx5_rxq_obj_hairpin_new(struct mlx5_rxq_priv *rxq) 411 { 412 uint16_t idx = rxq->idx; 413 struct mlx5_priv *priv = rxq->priv; 414 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 415 struct mlx5_devx_create_rq_attr attr = { 0 }; 416 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 417 uint32_t max_wq_data; 418 419 MLX5_ASSERT(rxq != NULL && rxq->ctrl != NULL && tmpl != NULL); 420 tmpl->rxq_ctrl = rxq_ctrl; 421 attr.hairpin = 1; 422 max_wq_data = 423 priv->sh->cdev->config.hca_attr.log_max_hairpin_wq_data_sz; 424 /* Jumbo frames > 9KB should be supported, and more packets. */ 425 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 426 if (priv->config.log_hp_size > max_wq_data) { 427 DRV_LOG(ERR, "Total data size %u power of 2 is " 428 "too large for hairpin.", 429 priv->config.log_hp_size); 430 rte_errno = ERANGE; 431 return -rte_errno; 432 } 433 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 434 } else { 435 attr.wq_attr.log_hairpin_data_sz = 436 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 437 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 438 } 439 /* Set the packets number to the maximum value for performance. */ 440 attr.wq_attr.log_hairpin_num_packets = 441 attr.wq_attr.log_hairpin_data_sz - 442 MLX5_HAIRPIN_QUEUE_STRIDE; 443 attr.counter_set_id = priv->counter_set_id; 444 rxq_ctrl->rxq.delay_drop = priv->config.hp_delay_drop; 445 attr.delay_drop_en = priv->config.hp_delay_drop; 446 tmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->cdev->ctx, &attr, 447 rxq_ctrl->socket); 448 if (!tmpl->rq) { 449 DRV_LOG(ERR, 450 "Port %u Rx hairpin queue %u can't create rq object.", 451 priv->dev_data->port_id, idx); 452 rte_errno = errno; 453 return -rte_errno; 454 } 455 priv->dev_data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN; 456 return 0; 457 } 458 459 /** 460 * Create the Rx queue DevX object. 461 * 462 * @param rxq 463 * Pointer to Rx queue. 464 * 465 * @return 466 * 0 on success, a negative errno value otherwise and rte_errno is set. 467 */ 468 static int 469 mlx5_rxq_devx_obj_new(struct mlx5_rxq_priv *rxq) 470 { 471 struct mlx5_priv *priv = rxq->priv; 472 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 473 struct mlx5_rxq_data *rxq_data = &rxq_ctrl->rxq; 474 struct mlx5_rxq_obj *tmpl = rxq_ctrl->obj; 475 int ret = 0; 476 477 MLX5_ASSERT(rxq_data); 478 MLX5_ASSERT(tmpl); 479 if (rxq_ctrl->is_hairpin) 480 return mlx5_rxq_obj_hairpin_new(rxq); 481 tmpl->rxq_ctrl = rxq_ctrl; 482 if (rxq_ctrl->irq && !rxq_ctrl->started) { 483 int devx_ev_flag = 484 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA; 485 486 tmpl->devx_channel = mlx5_os_devx_create_event_channel 487 (priv->sh->cdev->ctx, 488 devx_ev_flag); 489 if (!tmpl->devx_channel) { 490 rte_errno = errno; 491 DRV_LOG(ERR, "Failed to create event channel %d.", 492 rte_errno); 493 goto error; 494 } 495 tmpl->fd = mlx5_os_get_devx_channel_fd(tmpl->devx_channel); 496 } 497 /* Create CQ using DevX API. */ 498 ret = mlx5_rxq_create_devx_cq_resources(rxq); 499 if (ret) { 500 DRV_LOG(ERR, "Failed to create CQ."); 501 goto error; 502 } 503 rxq_data->delay_drop = priv->config.std_delay_drop; 504 /* Create RQ using DevX API. */ 505 ret = mlx5_rxq_create_devx_rq_resources(rxq); 506 if (ret) { 507 DRV_LOG(ERR, "Port %u Rx queue %u RQ creation failure.", 508 priv->dev_data->port_id, rxq->idx); 509 rte_errno = ENOMEM; 510 goto error; 511 } 512 /* Change queue state to ready. */ 513 ret = mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RST2RDY); 514 if (ret) 515 goto error; 516 if (!rxq_data->shared) { 517 rxq_data->wqes = (void *)(uintptr_t)rxq->devx_rq.wq.umem_buf; 518 rxq_data->rq_db = (uint32_t *)(uintptr_t)rxq->devx_rq.wq.db_rec; 519 } else if (!rxq_ctrl->started) { 520 rxq_data->wqes = (void *)(uintptr_t)tmpl->devx_rmp.wq.umem_buf; 521 rxq_data->rq_db = 522 (uint32_t *)(uintptr_t)tmpl->devx_rmp.wq.db_rec; 523 } 524 if (!rxq_ctrl->started) { 525 mlx5_rxq_initialize(rxq_data); 526 rxq_ctrl->wqn = rxq->devx_rq.rq->id; 527 } 528 priv->dev_data->rx_queue_state[rxq->idx] = RTE_ETH_QUEUE_STATE_STARTED; 529 return 0; 530 error: 531 ret = rte_errno; /* Save rte_errno before cleanup. */ 532 mlx5_rxq_devx_obj_release(rxq); 533 rte_errno = ret; /* Restore rte_errno. */ 534 return -rte_errno; 535 } 536 537 /** 538 * Prepare RQT attribute structure for DevX RQT API. 539 * 540 * @param dev 541 * Pointer to Ethernet device. 542 * @param log_n 543 * Log of number of queues in the array. 544 * @param queues 545 * List of RX queue indices or NULL, in which case 546 * the attribute will be filled by drop queue ID. 547 * @param queues_n 548 * Size of @p queues array or 0 if it is NULL. 549 * @param ind_tbl 550 * DevX indirection table object. 551 * 552 * @return 553 * The RQT attr object initialized, NULL otherwise and rte_errno is set. 554 */ 555 static struct mlx5_devx_rqt_attr * 556 mlx5_devx_ind_table_create_rqt_attr(struct rte_eth_dev *dev, 557 const unsigned int log_n, 558 const uint16_t *queues, 559 const uint32_t queues_n) 560 { 561 struct mlx5_priv *priv = dev->data->dev_private; 562 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 563 const unsigned int rqt_n = 1 << log_n; 564 unsigned int i, j; 565 566 rqt_attr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rqt_attr) + 567 rqt_n * sizeof(uint32_t), 0, SOCKET_ID_ANY); 568 if (!rqt_attr) { 569 DRV_LOG(ERR, "Port %u cannot allocate RQT resources.", 570 dev->data->port_id); 571 rte_errno = ENOMEM; 572 return NULL; 573 } 574 rqt_attr->rqt_max_size = priv->sh->dev_cap.ind_table_max_size; 575 rqt_attr->rqt_actual_size = rqt_n; 576 if (queues == NULL) { 577 for (i = 0; i < rqt_n; i++) 578 rqt_attr->rq_list[i] = 579 priv->drop_queue.rxq->devx_rq.rq->id; 580 return rqt_attr; 581 } 582 for (i = 0; i != queues_n; ++i) { 583 if (mlx5_is_external_rxq(dev, queues[i])) { 584 struct mlx5_external_rxq *ext_rxq = 585 mlx5_ext_rxq_get(dev, queues[i]); 586 587 rqt_attr->rq_list[i] = ext_rxq->hw_id; 588 } else { 589 struct mlx5_rxq_priv *rxq = 590 mlx5_rxq_get(dev, queues[i]); 591 592 MLX5_ASSERT(rxq != NULL); 593 if (rxq->ctrl->is_hairpin) 594 rqt_attr->rq_list[i] = rxq->ctrl->obj->rq->id; 595 else 596 rqt_attr->rq_list[i] = rxq->devx_rq.rq->id; 597 } 598 } 599 MLX5_ASSERT(i > 0); 600 for (j = 0; i != rqt_n; ++j, ++i) 601 rqt_attr->rq_list[i] = rqt_attr->rq_list[j]; 602 return rqt_attr; 603 } 604 605 /** 606 * Create RQT using DevX API as a filed of indirection table. 607 * 608 * @param dev 609 * Pointer to Ethernet device. 610 * @param log_n 611 * Log of number of queues in the array. 612 * @param ind_tbl 613 * DevX indirection table object. 614 * 615 * @return 616 * 0 on success, a negative errno value otherwise and rte_errno is set. 617 */ 618 static int 619 mlx5_devx_ind_table_new(struct rte_eth_dev *dev, const unsigned int log_n, 620 struct mlx5_ind_table_obj *ind_tbl) 621 { 622 struct mlx5_priv *priv = dev->data->dev_private; 623 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 624 const uint16_t *queues = dev->data->dev_started ? ind_tbl->queues : 625 NULL; 626 627 MLX5_ASSERT(ind_tbl); 628 rqt_attr = mlx5_devx_ind_table_create_rqt_attr(dev, log_n, queues, 629 ind_tbl->queues_n); 630 if (!rqt_attr) 631 return -rte_errno; 632 ind_tbl->rqt = mlx5_devx_cmd_create_rqt(priv->sh->cdev->ctx, rqt_attr); 633 mlx5_free(rqt_attr); 634 if (!ind_tbl->rqt) { 635 DRV_LOG(ERR, "Port %u cannot create DevX RQT.", 636 dev->data->port_id); 637 rte_errno = errno; 638 return -rte_errno; 639 } 640 return 0; 641 } 642 643 /** 644 * Modify RQT using DevX API as a filed of indirection table. 645 * 646 * @param dev 647 * Pointer to Ethernet device. 648 * @param log_n 649 * Log of number of queues in the array. 650 * @param ind_tbl 651 * DevX indirection table object. 652 * 653 * @return 654 * 0 on success, a negative errno value otherwise and rte_errno is set. 655 */ 656 static int 657 mlx5_devx_ind_table_modify(struct rte_eth_dev *dev, const unsigned int log_n, 658 const uint16_t *queues, const uint32_t queues_n, 659 struct mlx5_ind_table_obj *ind_tbl) 660 { 661 int ret = 0; 662 struct mlx5_devx_rqt_attr *rqt_attr = NULL; 663 664 MLX5_ASSERT(ind_tbl); 665 rqt_attr = mlx5_devx_ind_table_create_rqt_attr(dev, log_n, 666 queues, 667 queues_n); 668 if (!rqt_attr) 669 return -rte_errno; 670 ret = mlx5_devx_cmd_modify_rqt(ind_tbl->rqt, rqt_attr); 671 mlx5_free(rqt_attr); 672 if (ret) 673 DRV_LOG(ERR, "Port %u cannot modify DevX RQT.", 674 dev->data->port_id); 675 return ret; 676 } 677 678 /** 679 * Destroy the DevX RQT object. 680 * 681 * @param ind_table 682 * Indirection table to release. 683 */ 684 static void 685 mlx5_devx_ind_table_destroy(struct mlx5_ind_table_obj *ind_tbl) 686 { 687 claim_zero(mlx5_devx_cmd_destroy(ind_tbl->rqt)); 688 } 689 690 /** 691 * Set TIR attribute struct with relevant input values. 692 * 693 * @param[in] dev 694 * Pointer to Ethernet device. 695 * @param[in] rss_key 696 * RSS key for the Rx hash queue. 697 * @param[in] hash_fields 698 * Verbs protocol hash field to make the RSS on. 699 * @param[in] ind_tbl 700 * Indirection table for TIR. If table queues array is NULL, 701 * a TIR for drop queue is assumed. 702 * @param[in] tunnel 703 * Tunnel type. 704 * @param[out] tir_attr 705 * Parameters structure for TIR creation/modification. 706 * 707 * @return 708 * The Verbs/DevX object initialised index, 0 otherwise and rte_errno is set. 709 */ 710 static void 711 mlx5_devx_tir_attr_set(struct rte_eth_dev *dev, const uint8_t *rss_key, 712 uint64_t hash_fields, 713 const struct mlx5_ind_table_obj *ind_tbl, 714 int tunnel, struct mlx5_devx_tir_attr *tir_attr) 715 { 716 struct mlx5_priv *priv = dev->data->dev_private; 717 bool is_hairpin; 718 bool lro = false; 719 uint32_t i; 720 721 /* NULL queues designate drop queue. */ 722 if (ind_tbl->queues == NULL) { 723 is_hairpin = priv->drop_queue.rxq->ctrl->is_hairpin; 724 } else if (mlx5_is_external_rxq(dev, ind_tbl->queues[0])) { 725 /* External RxQ supports neither Hairpin nor LRO. */ 726 is_hairpin = false; 727 } else { 728 is_hairpin = mlx5_rxq_is_hairpin(dev, ind_tbl->queues[0]); 729 lro = true; 730 /* Enable TIR LRO only if all the queues were configured for. */ 731 for (i = 0; i < ind_tbl->queues_n; ++i) { 732 struct mlx5_rxq_data *rxq_i = 733 mlx5_rxq_data_get(dev, ind_tbl->queues[i]); 734 735 if (rxq_i != NULL && !rxq_i->lro) { 736 lro = false; 737 break; 738 } 739 } 740 } 741 memset(tir_attr, 0, sizeof(*tir_attr)); 742 tir_attr->disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT; 743 tir_attr->rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ; 744 tir_attr->tunneled_offload_en = !!tunnel; 745 /* If needed, translate hash_fields bitmap to PRM format. */ 746 if (hash_fields) { 747 struct mlx5_rx_hash_field_select *rx_hash_field_select = 748 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT 749 hash_fields & IBV_RX_HASH_INNER ? 750 &tir_attr->rx_hash_field_selector_inner : 751 #endif 752 &tir_attr->rx_hash_field_selector_outer; 753 /* 1 bit: 0: IPv4, 1: IPv6. */ 754 rx_hash_field_select->l3_prot_type = 755 !!(hash_fields & MLX5_IPV6_IBV_RX_HASH); 756 /* 1 bit: 0: TCP, 1: UDP. */ 757 rx_hash_field_select->l4_prot_type = 758 !!(hash_fields & MLX5_UDP_IBV_RX_HASH); 759 /* Bitmask which sets which fields to use in RX Hash. */ 760 rx_hash_field_select->selected_fields = 761 ((!!(hash_fields & MLX5_L3_SRC_IBV_RX_HASH)) << 762 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) | 763 (!!(hash_fields & MLX5_L3_DST_IBV_RX_HASH)) << 764 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP | 765 (!!(hash_fields & MLX5_L4_SRC_IBV_RX_HASH)) << 766 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT | 767 (!!(hash_fields & MLX5_L4_DST_IBV_RX_HASH)) << 768 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT | 769 (!!(hash_fields & IBV_RX_HASH_IPSEC_SPI)) << 770 MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_IPSEC_SPI; 771 } 772 if (is_hairpin) 773 tir_attr->transport_domain = priv->sh->td->id; 774 else 775 tir_attr->transport_domain = priv->sh->tdn; 776 memcpy(tir_attr->rx_hash_toeplitz_key, rss_key, MLX5_RSS_HASH_KEY_LEN); 777 tir_attr->indirect_table = ind_tbl->rqt->id; 778 if (dev->data->dev_conf.lpbk_mode) 779 tir_attr->self_lb_block = MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST; 780 if (lro) { 781 MLX5_ASSERT(priv->sh->dev_cap.lro_supported); 782 tir_attr->lro_timeout_period_usecs = priv->config.lro_timeout; 783 tir_attr->lro_max_msg_sz = priv->max_lro_msg_size; 784 tir_attr->lro_enable_mask = 785 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | 786 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO; 787 } 788 } 789 790 /** 791 * Create an Rx Hash queue. 792 * 793 * @param dev 794 * Pointer to Ethernet device. 795 * @param hrxq 796 * Pointer to Rx Hash queue. 797 * @param tunnel 798 * Tunnel type. 799 * 800 * @return 801 * 0 on success, a negative errno value otherwise and rte_errno is set. 802 */ 803 static int 804 mlx5_devx_hrxq_new(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq, 805 int tunnel __rte_unused) 806 { 807 struct mlx5_priv *priv = dev->data->dev_private; 808 struct mlx5_devx_tir_attr tir_attr = {0}; 809 int err; 810 811 mlx5_devx_tir_attr_set(dev, hrxq->rss_key, hrxq->hash_fields, 812 hrxq->ind_table, tunnel, &tir_attr); 813 hrxq->tir = mlx5_devx_cmd_create_tir(priv->sh->cdev->ctx, &tir_attr); 814 if (!hrxq->tir) { 815 DRV_LOG(ERR, "Port %u cannot create DevX TIR.", 816 dev->data->port_id); 817 rte_errno = errno; 818 goto error; 819 } 820 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) 821 if (hrxq->hws_flags) { 822 hrxq->action = mlx5dr_action_create_dest_tir 823 (priv->dr_ctx, 824 (struct mlx5dr_devx_obj *)hrxq->tir, hrxq->hws_flags); 825 if (!hrxq->action) 826 goto error; 827 return 0; 828 } 829 if (mlx5_flow_os_create_flow_action_dest_devx_tir(hrxq->tir, 830 &hrxq->action)) { 831 rte_errno = errno; 832 goto error; 833 } 834 #endif 835 return 0; 836 error: 837 err = rte_errno; /* Save rte_errno before cleanup. */ 838 if (hrxq->tir) 839 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 840 rte_errno = err; /* Restore rte_errno. */ 841 return -rte_errno; 842 } 843 844 /** 845 * Destroy a DevX TIR object. 846 * 847 * @param hrxq 848 * Hash Rx queue to release its tir. 849 */ 850 static void 851 mlx5_devx_tir_destroy(struct mlx5_hrxq *hrxq) 852 { 853 claim_zero(mlx5_devx_cmd_destroy(hrxq->tir)); 854 } 855 856 /** 857 * Modify an Rx Hash queue configuration. 858 * 859 * @param dev 860 * Pointer to Ethernet device. 861 * @param hrxq 862 * Hash Rx queue to modify. 863 * @param rss_key 864 * RSS key for the Rx hash queue. 865 * @param hash_fields 866 * Verbs protocol hash field to make the RSS on. 867 * @param[in] ind_tbl 868 * Indirection table for TIR. 869 * 870 * @return 871 * 0 on success, a negative errno value otherwise and rte_errno is set. 872 */ 873 static int 874 mlx5_devx_hrxq_modify(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq, 875 const uint8_t *rss_key, 876 uint64_t hash_fields, 877 const struct mlx5_ind_table_obj *ind_tbl) 878 { 879 struct mlx5_devx_modify_tir_attr modify_tir = {0}; 880 881 /* 882 * untested for modification fields: 883 * - rx_hash_symmetric not set in hrxq_new(), 884 * - rx_hash_fn set hard-coded in hrxq_new(), 885 * - lro_xxx not set after rxq setup 886 */ 887 if (ind_tbl != hrxq->ind_table) 888 modify_tir.modify_bitmask |= 889 MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_INDIRECT_TABLE; 890 if (hash_fields != hrxq->hash_fields || 891 memcmp(hrxq->rss_key, rss_key, MLX5_RSS_HASH_KEY_LEN)) 892 modify_tir.modify_bitmask |= 893 MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_HASH; 894 mlx5_devx_tir_attr_set(dev, rss_key, hash_fields, ind_tbl, 895 0, /* N/A - tunnel modification unsupported */ 896 &modify_tir.tir); 897 modify_tir.tirn = hrxq->tir->id; 898 if (mlx5_devx_cmd_modify_tir(hrxq->tir, &modify_tir)) { 899 DRV_LOG(ERR, "port %u cannot modify DevX TIR", 900 dev->data->port_id); 901 rte_errno = errno; 902 return -rte_errno; 903 } 904 return 0; 905 } 906 907 /** 908 * Create a DevX drop Rx queue. 909 * 910 * @param dev 911 * Pointer to Ethernet device. 912 * 913 * @return 914 * 0 on success, a negative errno value otherwise and rte_errno is set. 915 */ 916 static int 917 mlx5_rxq_devx_obj_drop_create(struct rte_eth_dev *dev) 918 { 919 struct mlx5_priv *priv = dev->data->dev_private; 920 int socket_id = dev->device->numa_node; 921 struct mlx5_rxq_priv *rxq; 922 struct mlx5_rxq_ctrl *rxq_ctrl = NULL; 923 struct mlx5_rxq_obj *rxq_obj = NULL; 924 int ret; 925 926 /* 927 * Initialize dummy control structures. 928 * They are required to hold pointers for cleanup 929 * and are only accessible via drop queue DevX objects. 930 */ 931 rxq = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq), 0, socket_id); 932 if (rxq == NULL) { 933 DRV_LOG(ERR, "Port %u could not allocate drop queue private", 934 dev->data->port_id); 935 rte_errno = ENOMEM; 936 goto error; 937 } 938 rxq_ctrl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq_ctrl), 939 0, socket_id); 940 if (rxq_ctrl == NULL) { 941 DRV_LOG(ERR, "Port %u could not allocate drop queue control", 942 dev->data->port_id); 943 rte_errno = ENOMEM; 944 goto error; 945 } 946 rxq_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rxq_obj), 0, socket_id); 947 if (rxq_obj == NULL) { 948 DRV_LOG(ERR, "Port %u could not allocate drop queue object", 949 dev->data->port_id); 950 rte_errno = ENOMEM; 951 goto error; 952 } 953 /* set the CPU socket ID where the rxq_ctrl was allocated */ 954 rxq_ctrl->socket = socket_id; 955 rxq_obj->rxq_ctrl = rxq_ctrl; 956 rxq_ctrl->is_hairpin = false; 957 rxq_ctrl->sh = priv->sh; 958 rxq_ctrl->obj = rxq_obj; 959 rxq->ctrl = rxq_ctrl; 960 rxq->priv = priv; 961 LIST_INSERT_HEAD(&rxq_ctrl->owners, rxq, owner_entry); 962 /* Create CQ using DevX API. */ 963 ret = mlx5_rxq_create_devx_cq_resources(rxq); 964 if (ret != 0) { 965 DRV_LOG(ERR, "Port %u drop queue CQ creation failed.", 966 dev->data->port_id); 967 goto error; 968 } 969 rxq_ctrl->rxq.delay_drop = 0; 970 /* Create RQ using DevX API. */ 971 ret = mlx5_rxq_create_devx_rq_resources(rxq); 972 if (ret != 0) { 973 DRV_LOG(ERR, "Port %u drop queue RQ creation failed.", 974 dev->data->port_id); 975 rte_errno = ENOMEM; 976 goto error; 977 } 978 /* Change queue state to ready. */ 979 ret = mlx5_devx_modify_rq(rxq, MLX5_RXQ_MOD_RST2RDY); 980 if (ret != 0) 981 goto error; 982 /* Initialize drop queue. */ 983 priv->drop_queue.rxq = rxq; 984 return 0; 985 error: 986 ret = rte_errno; /* Save rte_errno before cleanup. */ 987 if (rxq != NULL && rxq->devx_rq.rq != NULL) 988 mlx5_devx_rq_destroy(&rxq->devx_rq); 989 if (rxq_obj != NULL) { 990 if (rxq_obj->cq_obj.cq != NULL) 991 mlx5_devx_cq_destroy(&rxq_obj->cq_obj); 992 if (rxq_obj->devx_channel) 993 mlx5_os_devx_destroy_event_channel 994 (rxq_obj->devx_channel); 995 mlx5_free(rxq_obj); 996 } 997 if (rxq_ctrl != NULL) 998 mlx5_free(rxq_ctrl); 999 if (rxq != NULL) 1000 mlx5_free(rxq); 1001 rte_errno = ret; /* Restore rte_errno. */ 1002 return -rte_errno; 1003 } 1004 1005 /** 1006 * Release drop Rx queue resources. 1007 * 1008 * @param dev 1009 * Pointer to Ethernet device. 1010 */ 1011 static void 1012 mlx5_rxq_devx_obj_drop_release(struct rte_eth_dev *dev) 1013 { 1014 struct mlx5_priv *priv = dev->data->dev_private; 1015 struct mlx5_rxq_priv *rxq = priv->drop_queue.rxq; 1016 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 1017 1018 mlx5_rxq_devx_obj_release(rxq); 1019 mlx5_free(rxq_ctrl->obj); 1020 mlx5_free(rxq_ctrl); 1021 mlx5_free(rxq); 1022 priv->drop_queue.rxq = NULL; 1023 } 1024 1025 /** 1026 * Release a drop hash Rx queue. 1027 * 1028 * @param dev 1029 * Pointer to Ethernet device. 1030 */ 1031 static void 1032 mlx5_devx_drop_action_destroy(struct rte_eth_dev *dev) 1033 { 1034 struct mlx5_priv *priv = dev->data->dev_private; 1035 struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq; 1036 1037 if (hrxq->tir != NULL) 1038 mlx5_devx_tir_destroy(hrxq); 1039 if (hrxq->ind_table->ind_table != NULL) 1040 mlx5_devx_ind_table_destroy(hrxq->ind_table); 1041 if (priv->drop_queue.rxq->devx_rq.rq != NULL) 1042 mlx5_rxq_devx_obj_drop_release(dev); 1043 } 1044 1045 /** 1046 * Create a DevX drop action for Rx Hash queue. 1047 * 1048 * @param dev 1049 * Pointer to Ethernet device. 1050 * 1051 * @return 1052 * 0 on success, a negative errno value otherwise and rte_errno is set. 1053 */ 1054 static int 1055 mlx5_devx_drop_action_create(struct rte_eth_dev *dev) 1056 { 1057 struct mlx5_priv *priv = dev->data->dev_private; 1058 struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq; 1059 int ret; 1060 1061 ret = mlx5_rxq_devx_obj_drop_create(dev); 1062 if (ret != 0) { 1063 DRV_LOG(ERR, "Cannot create drop RX queue"); 1064 return ret; 1065 } 1066 if (priv->sh->config.dv_flow_en == 2) 1067 return 0; 1068 /* hrxq->ind_table queues are NULL, drop RX queue ID will be used */ 1069 ret = mlx5_devx_ind_table_new(dev, 0, hrxq->ind_table); 1070 if (ret != 0) { 1071 DRV_LOG(ERR, "Cannot create drop hash RX queue indirection table"); 1072 goto error; 1073 } 1074 ret = mlx5_devx_hrxq_new(dev, hrxq, /* tunnel */ false); 1075 if (ret != 0) { 1076 DRV_LOG(ERR, "Cannot create drop hash RX queue"); 1077 goto error; 1078 } 1079 return 0; 1080 error: 1081 mlx5_devx_drop_action_destroy(dev); 1082 return ret; 1083 } 1084 1085 /** 1086 * Select TXQ TIS number. 1087 * 1088 * @param dev 1089 * Pointer to Ethernet device. 1090 * @param queue_idx 1091 * Queue index in DPDK Tx queue array. 1092 * 1093 * @return 1094 * > 0 on success, a negative errno value otherwise. 1095 */ 1096 static uint32_t 1097 mlx5_get_txq_tis_num(struct rte_eth_dev *dev, uint16_t queue_idx) 1098 { 1099 struct mlx5_priv *priv = dev->data->dev_private; 1100 int tis_idx; 1101 1102 if (priv->sh->bond.n_port && priv->sh->lag.affinity_mode == 1103 MLX5_LAG_MODE_TIS) { 1104 tis_idx = (priv->lag_affinity_idx + queue_idx) % 1105 priv->sh->bond.n_port; 1106 DRV_LOG(INFO, "port %d txq %d gets affinity %d and maps to PF %d.", 1107 dev->data->port_id, queue_idx, tis_idx + 1, 1108 priv->sh->lag.tx_remap_affinity[tis_idx]); 1109 } else { 1110 tis_idx = 0; 1111 } 1112 MLX5_ASSERT(priv->sh->tis[tis_idx]); 1113 return priv->sh->tis[tis_idx]->id; 1114 } 1115 1116 /** 1117 * Create the Tx hairpin queue object. 1118 * 1119 * @param dev 1120 * Pointer to Ethernet device. 1121 * @param idx 1122 * Queue index in DPDK Tx queue array. 1123 * 1124 * @return 1125 * 0 on success, a negative errno value otherwise and rte_errno is set. 1126 */ 1127 static int 1128 mlx5_txq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) 1129 { 1130 struct mlx5_priv *priv = dev->data->dev_private; 1131 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1132 struct mlx5_txq_ctrl *txq_ctrl = 1133 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1134 struct mlx5_devx_create_sq_attr attr = { 0 }; 1135 struct mlx5_txq_obj *tmpl = txq_ctrl->obj; 1136 uint32_t max_wq_data; 1137 1138 MLX5_ASSERT(txq_data); 1139 MLX5_ASSERT(tmpl); 1140 tmpl->txq_ctrl = txq_ctrl; 1141 attr.hairpin = 1; 1142 attr.tis_lst_sz = 1; 1143 max_wq_data = 1144 priv->sh->cdev->config.hca_attr.log_max_hairpin_wq_data_sz; 1145 /* Jumbo frames > 9KB should be supported, and more packets. */ 1146 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 1147 if (priv->config.log_hp_size > max_wq_data) { 1148 DRV_LOG(ERR, "Total data size %u power of 2 is " 1149 "too large for hairpin.", 1150 priv->config.log_hp_size); 1151 rte_errno = ERANGE; 1152 return -rte_errno; 1153 } 1154 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 1155 } else { 1156 attr.wq_attr.log_hairpin_data_sz = 1157 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 1158 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 1159 } 1160 /* Set the packets number to the maximum value for performance. */ 1161 attr.wq_attr.log_hairpin_num_packets = 1162 attr.wq_attr.log_hairpin_data_sz - 1163 MLX5_HAIRPIN_QUEUE_STRIDE; 1164 1165 attr.tis_num = mlx5_get_txq_tis_num(dev, idx); 1166 tmpl->sq = mlx5_devx_cmd_create_sq(priv->sh->cdev->ctx, &attr); 1167 if (!tmpl->sq) { 1168 DRV_LOG(ERR, 1169 "Port %u tx hairpin queue %u can't create SQ object.", 1170 dev->data->port_id, idx); 1171 rte_errno = errno; 1172 return -rte_errno; 1173 } 1174 return 0; 1175 } 1176 1177 #if defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) || !defined(HAVE_INFINIBAND_VERBS_H) 1178 /** 1179 * Destroy the Tx queue DevX object. 1180 * 1181 * @param txq_obj 1182 * Txq object to destroy. 1183 */ 1184 static void 1185 mlx5_txq_release_devx_resources(struct mlx5_txq_obj *txq_obj) 1186 { 1187 mlx5_devx_sq_destroy(&txq_obj->sq_obj); 1188 memset(&txq_obj->sq_obj, 0, sizeof(txq_obj->sq_obj)); 1189 mlx5_devx_cq_destroy(&txq_obj->cq_obj); 1190 memset(&txq_obj->cq_obj, 0, sizeof(txq_obj->cq_obj)); 1191 } 1192 1193 /** 1194 * Create a SQ object and its resources using DevX. 1195 * 1196 * @param dev 1197 * Pointer to Ethernet device. 1198 * @param idx 1199 * Queue index in DPDK Tx queue array. 1200 * @param[in] log_desc_n 1201 * Log of number of descriptors in queue. 1202 * 1203 * @return 1204 * 0 on success, a negative errno value otherwise and rte_errno is set. 1205 */ 1206 static int 1207 mlx5_txq_create_devx_sq_resources(struct rte_eth_dev *dev, uint16_t idx, 1208 uint16_t log_desc_n) 1209 { 1210 struct mlx5_priv *priv = dev->data->dev_private; 1211 struct mlx5_common_device *cdev = priv->sh->cdev; 1212 struct mlx5_uar *uar = &priv->sh->tx_uar; 1213 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1214 struct mlx5_txq_ctrl *txq_ctrl = 1215 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1216 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1217 struct mlx5_devx_create_sq_attr sq_attr = { 1218 .flush_in_error_en = 1, 1219 .allow_multi_pkt_send_wqe = !!priv->config.mps, 1220 .min_wqe_inline_mode = cdev->config.hca_attr.vport_inline_mode, 1221 .allow_swp = !!priv->sh->dev_cap.swp, 1222 .cqn = txq_obj->cq_obj.cq->id, 1223 .tis_lst_sz = 1, 1224 .wq_attr = (struct mlx5_devx_wq_attr){ 1225 .pd = cdev->pdn, 1226 .uar_page = mlx5_os_get_devx_uar_page_id(uar->obj), 1227 }, 1228 .ts_format = 1229 mlx5_ts_format_conv(cdev->config.hca_attr.sq_ts_format), 1230 .tis_num = mlx5_get_txq_tis_num(dev, idx), 1231 }; 1232 1233 /* Create Send Queue object with DevX. */ 1234 return mlx5_devx_sq_create(cdev->ctx, &txq_obj->sq_obj, 1235 log_desc_n, &sq_attr, priv->sh->numa_node); 1236 } 1237 #endif 1238 1239 /** 1240 * Create the Tx queue DevX object. 1241 * 1242 * @param dev 1243 * Pointer to Ethernet device. 1244 * @param idx 1245 * Queue index in DPDK Tx queue array. 1246 * 1247 * @return 1248 * 0 on success, a negative errno value otherwise and rte_errno is set. 1249 */ 1250 int 1251 mlx5_txq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx) 1252 { 1253 struct mlx5_priv *priv = dev->data->dev_private; 1254 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 1255 struct mlx5_txq_ctrl *txq_ctrl = 1256 container_of(txq_data, struct mlx5_txq_ctrl, txq); 1257 1258 if (txq_ctrl->is_hairpin) 1259 return mlx5_txq_obj_hairpin_new(dev, idx); 1260 #if !defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) && defined(HAVE_INFINIBAND_VERBS_H) 1261 DRV_LOG(ERR, "Port %u Tx queue %u cannot create with DevX, no UAR.", 1262 dev->data->port_id, idx); 1263 rte_errno = ENOMEM; 1264 return -rte_errno; 1265 #else 1266 struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv)); 1267 struct mlx5_dev_ctx_shared *sh = priv->sh; 1268 struct mlx5_txq_obj *txq_obj = txq_ctrl->obj; 1269 struct mlx5_devx_cq_attr cq_attr = { 1270 .uar_page_id = mlx5_os_get_devx_uar_page_id(sh->tx_uar.obj), 1271 }; 1272 uint32_t cqe_n, log_desc_n; 1273 uint32_t wqe_n, wqe_size; 1274 int ret = 0; 1275 1276 MLX5_ASSERT(txq_data); 1277 MLX5_ASSERT(txq_obj); 1278 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 1279 MLX5_ASSERT(ppriv); 1280 txq_obj->txq_ctrl = txq_ctrl; 1281 txq_obj->dev = dev; 1282 cqe_n = (1UL << txq_data->elts_n) / MLX5_TX_COMP_THRESH + 1283 1 + MLX5_TX_COMP_THRESH_INLINE_DIV; 1284 log_desc_n = log2above(cqe_n); 1285 cqe_n = 1UL << log_desc_n; 1286 if (cqe_n > UINT16_MAX) { 1287 DRV_LOG(ERR, "Port %u Tx queue %u requests to many CQEs %u.", 1288 dev->data->port_id, txq_data->idx, cqe_n); 1289 rte_errno = EINVAL; 1290 return 0; 1291 } 1292 /* Create completion queue object with DevX. */ 1293 ret = mlx5_devx_cq_create(sh->cdev->ctx, &txq_obj->cq_obj, log_desc_n, 1294 &cq_attr, priv->sh->numa_node); 1295 if (ret) { 1296 DRV_LOG(ERR, "Port %u Tx queue %u CQ creation failure.", 1297 dev->data->port_id, idx); 1298 goto error; 1299 } 1300 txq_data->cqe_n = log_desc_n; 1301 txq_data->cqe_s = cqe_n; 1302 txq_data->cqe_m = txq_data->cqe_s - 1; 1303 txq_data->cqes = txq_obj->cq_obj.cqes; 1304 txq_data->cq_ci = 0; 1305 txq_data->cq_pi = 0; 1306 txq_data->cq_db = txq_obj->cq_obj.db_rec; 1307 *txq_data->cq_db = 0; 1308 /* 1309 * Adjust the amount of WQEs depending on inline settings. 1310 * The number of descriptors should be enough to handle 1311 * the specified number of packets. If queue is being created 1312 * with Verbs the rdma-core does queue size adjustment 1313 * internally in the mlx5_calc_sq_size(), we do the same 1314 * for the queue being created with DevX at this point. 1315 */ 1316 wqe_size = txq_data->tso_en ? 1317 RTE_ALIGN(txq_ctrl->max_tso_header, MLX5_WSEG_SIZE) : 0; 1318 wqe_size += sizeof(struct mlx5_wqe_cseg) + 1319 sizeof(struct mlx5_wqe_eseg) + 1320 sizeof(struct mlx5_wqe_dseg); 1321 if (txq_data->inlen_send) 1322 wqe_size = RTE_MAX(wqe_size, sizeof(struct mlx5_wqe_cseg) + 1323 sizeof(struct mlx5_wqe_eseg) + 1324 RTE_ALIGN(txq_data->inlen_send + 1325 sizeof(uint32_t), 1326 MLX5_WSEG_SIZE)); 1327 wqe_size = RTE_ALIGN(wqe_size, MLX5_WQE_SIZE) / MLX5_WQE_SIZE; 1328 /* Create Send Queue object with DevX. */ 1329 wqe_n = RTE_MIN((1UL << txq_data->elts_n) * wqe_size, 1330 (uint32_t)priv->sh->dev_cap.max_qp_wr); 1331 log_desc_n = log2above(wqe_n); 1332 ret = mlx5_txq_create_devx_sq_resources(dev, idx, log_desc_n); 1333 if (ret) { 1334 DRV_LOG(ERR, "Port %u Tx queue %u SQ creation failure.", 1335 dev->data->port_id, idx); 1336 rte_errno = errno; 1337 goto error; 1338 } 1339 /* Create the Work Queue. */ 1340 txq_data->wqe_n = log_desc_n; 1341 txq_data->wqe_s = 1 << txq_data->wqe_n; 1342 txq_data->wqe_m = txq_data->wqe_s - 1; 1343 txq_data->wqes = (struct mlx5_wqe *)(uintptr_t)txq_obj->sq_obj.wqes; 1344 txq_data->wqes_end = txq_data->wqes + txq_data->wqe_s; 1345 txq_data->wqe_ci = 0; 1346 txq_data->wqe_pi = 0; 1347 txq_data->wqe_comp = 0; 1348 txq_data->wqe_thres = txq_data->wqe_s / MLX5_TX_COMP_THRESH_INLINE_DIV; 1349 txq_data->qp_db = &txq_obj->sq_obj.db_rec[MLX5_SND_DBR]; 1350 *txq_data->qp_db = 0; 1351 txq_data->qp_num_8s = txq_obj->sq_obj.sq->id << 8; 1352 txq_data->db_heu = sh->cdev->config.dbnc == MLX5_SQ_DB_HEURISTIC; 1353 txq_data->db_nc = sh->tx_uar.dbnc; 1354 txq_data->wait_on_time = !!(!sh->config.tx_pp && 1355 sh->cdev->config.hca_attr.wait_on_time); 1356 /* Change Send Queue state to Ready-to-Send. */ 1357 ret = mlx5_txq_devx_modify(txq_obj, MLX5_TXQ_MOD_RST2RDY, 0); 1358 if (ret) { 1359 rte_errno = errno; 1360 DRV_LOG(ERR, 1361 "Port %u Tx queue %u SQ state to SQC_STATE_RDY failed.", 1362 dev->data->port_id, idx); 1363 goto error; 1364 } 1365 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 1366 /* 1367 * If using DevX need to query and store TIS transport domain value. 1368 * This is done once per port. 1369 * Will use this value on Rx, when creating matching TIR. 1370 */ 1371 if (!priv->sh->tdn) 1372 priv->sh->tdn = priv->sh->td->id; 1373 #endif 1374 txq_ctrl->uar_mmap_offset = 1375 mlx5_os_get_devx_uar_mmap_offset(sh->tx_uar.obj); 1376 ppriv->uar_table[txq_data->idx] = sh->tx_uar.bf_db; 1377 dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED; 1378 return 0; 1379 error: 1380 ret = rte_errno; /* Save rte_errno before cleanup. */ 1381 mlx5_txq_release_devx_resources(txq_obj); 1382 rte_errno = ret; /* Restore rte_errno. */ 1383 return -rte_errno; 1384 #endif 1385 } 1386 1387 /** 1388 * Release an Tx DevX queue object. 1389 * 1390 * @param txq_obj 1391 * DevX Tx queue object. 1392 */ 1393 void 1394 mlx5_txq_devx_obj_release(struct mlx5_txq_obj *txq_obj) 1395 { 1396 MLX5_ASSERT(txq_obj); 1397 if (txq_obj->txq_ctrl->is_hairpin) { 1398 if (txq_obj->tis) 1399 claim_zero(mlx5_devx_cmd_destroy(txq_obj->tis)); 1400 #if defined(HAVE_MLX5DV_DEVX_UAR_OFFSET) || !defined(HAVE_INFINIBAND_VERBS_H) 1401 } else { 1402 mlx5_txq_release_devx_resources(txq_obj); 1403 #endif 1404 } 1405 } 1406 1407 struct mlx5_obj_ops devx_obj_ops = { 1408 .rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_rq_vlan_strip, 1409 .rxq_obj_new = mlx5_rxq_devx_obj_new, 1410 .rxq_event_get = mlx5_rx_devx_get_event, 1411 .rxq_obj_modify = mlx5_devx_modify_rq, 1412 .rxq_obj_release = mlx5_rxq_devx_obj_release, 1413 .ind_table_new = mlx5_devx_ind_table_new, 1414 .ind_table_modify = mlx5_devx_ind_table_modify, 1415 .ind_table_destroy = mlx5_devx_ind_table_destroy, 1416 .hrxq_new = mlx5_devx_hrxq_new, 1417 .hrxq_destroy = mlx5_devx_tir_destroy, 1418 .hrxq_modify = mlx5_devx_hrxq_modify, 1419 .drop_action_create = mlx5_devx_drop_action_create, 1420 .drop_action_destroy = mlx5_devx_drop_action_destroy, 1421 .txq_obj_new = mlx5_txq_devx_obj_new, 1422 .txq_obj_modify = mlx5_txq_devx_modify, 1423 .txq_obj_release = mlx5_txq_devx_obj_release, 1424 .lb_dummy_queue_create = NULL, 1425 .lb_dummy_queue_release = NULL, 1426 }; 1427