1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2020 Mellanox Technologies, Ltd 3 */ 4 5 #include <stddef.h> 6 #include <errno.h> 7 #include <string.h> 8 #include <stdint.h> 9 #include <sys/queue.h> 10 11 #include <rte_malloc.h> 12 #include <rte_common.h> 13 #include <rte_eal_paging.h> 14 15 #include <mlx5_glue.h> 16 #include <mlx5_devx_cmds.h> 17 #include <mlx5_malloc.h> 18 19 #include "mlx5.h" 20 #include "mlx5_common_os.h" 21 #include "mlx5_rxtx.h" 22 #include "mlx5_utils.h" 23 #include "mlx5_devx.h" 24 25 /** 26 * Modify RQ vlan stripping offload 27 * 28 * @param rxq_obj 29 * Rx queue object. 30 * 31 * @return 0 on success, non-0 otherwise 32 */ 33 static int 34 mlx5_rxq_obj_modify_rq_vlan_strip(struct mlx5_rxq_obj *rxq_obj, int on) 35 { 36 struct mlx5_devx_modify_rq_attr rq_attr; 37 38 memset(&rq_attr, 0, sizeof(rq_attr)); 39 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 40 rq_attr.state = MLX5_RQC_STATE_RDY; 41 rq_attr.vsd = (on ? 0 : 1); 42 rq_attr.modify_bitmask = MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD; 43 return mlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr); 44 } 45 46 /** 47 * Release the resources allocated for an RQ DevX object. 48 * 49 * @param rxq_ctrl 50 * DevX Rx queue object. 51 */ 52 static void 53 rxq_release_devx_rq_resources(struct mlx5_rxq_ctrl *rxq_ctrl) 54 { 55 if (rxq_ctrl->rxq.wqes) { 56 mlx5_free((void *)(uintptr_t)rxq_ctrl->rxq.wqes); 57 rxq_ctrl->rxq.wqes = NULL; 58 } 59 if (rxq_ctrl->wq_umem) { 60 mlx5_glue->devx_umem_dereg(rxq_ctrl->wq_umem); 61 rxq_ctrl->wq_umem = NULL; 62 } 63 } 64 65 /** 66 * Release the resources allocated for the Rx CQ DevX object. 67 * 68 * @param rxq_ctrl 69 * DevX Rx queue object. 70 */ 71 static void 72 rxq_release_devx_cq_resources(struct mlx5_rxq_ctrl *rxq_ctrl) 73 { 74 if (rxq_ctrl->rxq.cqes) { 75 rte_free((void *)(uintptr_t)rxq_ctrl->rxq.cqes); 76 rxq_ctrl->rxq.cqes = NULL; 77 } 78 if (rxq_ctrl->cq_umem) { 79 mlx5_glue->devx_umem_dereg(rxq_ctrl->cq_umem); 80 rxq_ctrl->cq_umem = NULL; 81 } 82 } 83 84 /** 85 * Release an Rx hairpin related resources. 86 * 87 * @param rxq_obj 88 * Hairpin Rx queue object. 89 */ 90 static void 91 mlx5_rxq_obj_hairpin_release(struct mlx5_rxq_obj *rxq_obj) 92 { 93 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 94 95 MLX5_ASSERT(rxq_obj); 96 rq_attr.state = MLX5_RQC_STATE_RST; 97 rq_attr.rq_state = MLX5_RQC_STATE_RDY; 98 mlx5_devx_cmd_modify_rq(rxq_obj->rq, &rq_attr); 99 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 100 } 101 102 /** 103 * Release an Rx DevX queue object. 104 * 105 * @param rxq_obj 106 * DevX Rx queue object. 107 */ 108 static void 109 mlx5_rxq_devx_obj_release(struct mlx5_rxq_obj *rxq_obj) 110 { 111 struct mlx5_priv *priv = rxq_obj->rxq_ctrl->priv; 112 113 MLX5_ASSERT(rxq_obj); 114 MLX5_ASSERT(rxq_obj->rq); 115 if (rxq_obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN) { 116 mlx5_rxq_obj_hairpin_release(rxq_obj); 117 } else { 118 MLX5_ASSERT(rxq_obj->devx_cq); 119 rxq_free_elts(rxq_obj->rxq_ctrl); 120 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->rq)); 121 claim_zero(mlx5_devx_cmd_destroy(rxq_obj->devx_cq)); 122 claim_zero(mlx5_release_dbr(&priv->dbrpgs, 123 rxq_obj->rxq_ctrl->rq_dbr_umem_id, 124 rxq_obj->rxq_ctrl->rq_dbr_offset)); 125 claim_zero(mlx5_release_dbr(&priv->dbrpgs, 126 rxq_obj->rxq_ctrl->cq_dbr_umem_id, 127 rxq_obj->rxq_ctrl->cq_dbr_offset)); 128 if (rxq_obj->devx_channel) 129 mlx5_glue->devx_destroy_event_channel 130 (rxq_obj->devx_channel); 131 rxq_release_devx_rq_resources(rxq_obj->rxq_ctrl); 132 rxq_release_devx_cq_resources(rxq_obj->rxq_ctrl); 133 } 134 LIST_REMOVE(rxq_obj, next); 135 mlx5_free(rxq_obj); 136 } 137 138 /** 139 * Fill common fields of create RQ attributes structure. 140 * 141 * @param rxq_data 142 * Pointer to Rx queue data. 143 * @param cqn 144 * CQ number to use with this RQ. 145 * @param rq_attr 146 * RQ attributes structure to fill.. 147 */ 148 static void 149 mlx5_devx_create_rq_attr_fill(struct mlx5_rxq_data *rxq_data, uint32_t cqn, 150 struct mlx5_devx_create_rq_attr *rq_attr) 151 { 152 rq_attr->state = MLX5_RQC_STATE_RST; 153 rq_attr->vsd = (rxq_data->vlan_strip) ? 0 : 1; 154 rq_attr->cqn = cqn; 155 rq_attr->scatter_fcs = (rxq_data->crc_present) ? 1 : 0; 156 } 157 158 /** 159 * Fill common fields of DevX WQ attributes structure. 160 * 161 * @param priv 162 * Pointer to device private data. 163 * @param rxq_ctrl 164 * Pointer to Rx queue control structure. 165 * @param wq_attr 166 * WQ attributes structure to fill.. 167 */ 168 static void 169 mlx5_devx_wq_attr_fill(struct mlx5_priv *priv, struct mlx5_rxq_ctrl *rxq_ctrl, 170 struct mlx5_devx_wq_attr *wq_attr) 171 { 172 wq_attr->end_padding_mode = priv->config.cqe_pad ? 173 MLX5_WQ_END_PAD_MODE_ALIGN : 174 MLX5_WQ_END_PAD_MODE_NONE; 175 wq_attr->pd = priv->sh->pdn; 176 wq_attr->dbr_addr = rxq_ctrl->rq_dbr_offset; 177 wq_attr->dbr_umem_id = rxq_ctrl->rq_dbr_umem_id; 178 wq_attr->dbr_umem_valid = 1; 179 wq_attr->wq_umem_id = mlx5_os_get_umem_id(rxq_ctrl->wq_umem); 180 wq_attr->wq_umem_valid = 1; 181 } 182 183 /** 184 * Create a RQ object using DevX. 185 * 186 * @param dev 187 * Pointer to Ethernet device. 188 * @param idx 189 * Queue index in DPDK Rx queue array. 190 * @param cqn 191 * CQ number to use with this RQ. 192 * 193 * @return 194 * The DevX object initialized, NULL otherwise and rte_errno is set. 195 */ 196 static struct mlx5_devx_obj * 197 mlx5_devx_rq_new(struct rte_eth_dev *dev, uint16_t idx, uint32_t cqn) 198 { 199 struct mlx5_priv *priv = dev->data->dev_private; 200 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 201 struct mlx5_rxq_ctrl *rxq_ctrl = 202 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 203 struct mlx5_devx_create_rq_attr rq_attr = { 0 }; 204 uint32_t wqe_n = 1 << (rxq_data->elts_n - rxq_data->sges_n); 205 uint32_t wq_size = 0; 206 uint32_t wqe_size = 0; 207 uint32_t log_wqe_size = 0; 208 void *buf = NULL; 209 struct mlx5_devx_obj *rq; 210 211 /* Fill RQ attributes. */ 212 rq_attr.mem_rq_type = MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE; 213 rq_attr.flush_in_error_en = 1; 214 mlx5_devx_create_rq_attr_fill(rxq_data, cqn, &rq_attr); 215 /* Fill WQ attributes for this RQ. */ 216 if (mlx5_rxq_mprq_enabled(rxq_data)) { 217 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ; 218 /* 219 * Number of strides in each WQE: 220 * 512*2^single_wqe_log_num_of_strides. 221 */ 222 rq_attr.wq_attr.single_wqe_log_num_of_strides = 223 rxq_data->strd_num_n - 224 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES; 225 /* Stride size = (2^single_stride_log_num_of_bytes)*64B. */ 226 rq_attr.wq_attr.single_stride_log_num_of_bytes = 227 rxq_data->strd_sz_n - 228 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES; 229 wqe_size = sizeof(struct mlx5_wqe_mprq); 230 } else { 231 rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC; 232 wqe_size = sizeof(struct mlx5_wqe_data_seg); 233 } 234 log_wqe_size = log2above(wqe_size) + rxq_data->sges_n; 235 rq_attr.wq_attr.log_wq_stride = log_wqe_size; 236 rq_attr.wq_attr.log_wq_sz = rxq_data->elts_n - rxq_data->sges_n; 237 /* Calculate and allocate WQ memory space. */ 238 wqe_size = 1 << log_wqe_size; /* round up power of two.*/ 239 wq_size = wqe_n * wqe_size; 240 size_t alignment = MLX5_WQE_BUF_ALIGNMENT; 241 if (alignment == (size_t)-1) { 242 DRV_LOG(ERR, "Failed to get mem page size"); 243 rte_errno = ENOMEM; 244 return NULL; 245 } 246 buf = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, wq_size, 247 alignment, rxq_ctrl->socket); 248 if (!buf) 249 return NULL; 250 rxq_data->wqes = buf; 251 rxq_ctrl->wq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, 252 buf, wq_size, 0); 253 if (!rxq_ctrl->wq_umem) { 254 mlx5_free(buf); 255 return NULL; 256 } 257 mlx5_devx_wq_attr_fill(priv, rxq_ctrl, &rq_attr.wq_attr); 258 rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &rq_attr, rxq_ctrl->socket); 259 if (!rq) 260 rxq_release_devx_rq_resources(rxq_ctrl); 261 return rq; 262 } 263 264 /** 265 * Create a DevX CQ object for an Rx queue. 266 * 267 * @param dev 268 * Pointer to Ethernet device. 269 * @param cqe_n 270 * Number of CQEs in CQ. 271 * @param idx 272 * Queue index in DPDK Rx queue array. 273 * @param rxq_obj 274 * Pointer to Rx queue object data. 275 * 276 * @return 277 * The DevX object initialized, NULL otherwise and rte_errno is set. 278 */ 279 static struct mlx5_devx_obj * 280 mlx5_devx_cq_new(struct rte_eth_dev *dev, unsigned int cqe_n, uint16_t idx, 281 struct mlx5_rxq_obj *rxq_obj) 282 { 283 struct mlx5_devx_obj *cq_obj = 0; 284 struct mlx5_devx_cq_attr cq_attr = { 0 }; 285 struct mlx5_priv *priv = dev->data->dev_private; 286 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 287 struct mlx5_rxq_ctrl *rxq_ctrl = 288 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 289 size_t page_size = rte_mem_page_size(); 290 uint32_t lcore = (uint32_t)rte_lcore_to_cpu_id(-1); 291 uint32_t eqn = 0; 292 void *buf = NULL; 293 uint16_t event_nums[1] = {0}; 294 uint32_t log_cqe_n; 295 uint32_t cq_size; 296 int ret = 0; 297 298 if (page_size == (size_t)-1) { 299 DRV_LOG(ERR, "Failed to get page_size."); 300 goto error; 301 } 302 if (priv->config.cqe_comp && !rxq_data->hw_timestamp && 303 !rxq_data->lro) { 304 cq_attr.cqe_comp_en = MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE; 305 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT 306 cq_attr.mini_cqe_res_format = 307 mlx5_rxq_mprq_enabled(rxq_data) ? 308 MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX : 309 MLX5DV_CQE_RES_FORMAT_HASH; 310 #else 311 cq_attr.mini_cqe_res_format = MLX5DV_CQE_RES_FORMAT_HASH; 312 #endif 313 /* 314 * For vectorized Rx, it must not be doubled in order to 315 * make cq_ci and rq_ci aligned. 316 */ 317 if (mlx5_rxq_check_vec_support(rxq_data) < 0) 318 cqe_n *= 2; 319 } else if (priv->config.cqe_comp && rxq_data->hw_timestamp) { 320 DRV_LOG(DEBUG, 321 "Port %u Rx CQE compression is disabled for HW" 322 " timestamp.", 323 dev->data->port_id); 324 } else if (priv->config.cqe_comp && rxq_data->lro) { 325 DRV_LOG(DEBUG, 326 "Port %u Rx CQE compression is disabled for LRO.", 327 dev->data->port_id); 328 } 329 #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD 330 if (priv->config.cqe_pad) 331 cq_attr.cqe_size = MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD; 332 #endif 333 log_cqe_n = log2above(cqe_n); 334 cq_size = sizeof(struct mlx5_cqe) * (1 << log_cqe_n); 335 /* Query the EQN for this core. */ 336 if (mlx5_glue->devx_query_eqn(priv->sh->ctx, lcore, &eqn)) { 337 DRV_LOG(ERR, "Failed to query EQN for CQ."); 338 goto error; 339 } 340 cq_attr.eqn = eqn; 341 buf = rte_calloc_socket(__func__, 1, cq_size, page_size, 342 rxq_ctrl->socket); 343 if (!buf) { 344 DRV_LOG(ERR, "Failed to allocate memory for CQ."); 345 goto error; 346 } 347 rxq_data->cqes = (volatile struct mlx5_cqe (*)[])(uintptr_t)buf; 348 rxq_ctrl->cq_umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, buf, 349 cq_size, 350 IBV_ACCESS_LOCAL_WRITE); 351 if (!rxq_ctrl->cq_umem) { 352 DRV_LOG(ERR, "Failed to register umem for CQ."); 353 goto error; 354 } 355 cq_attr.uar_page_id = 356 mlx5_os_get_devx_uar_page_id(priv->sh->devx_rx_uar); 357 cq_attr.q_umem_id = mlx5_os_get_umem_id(rxq_ctrl->cq_umem); 358 cq_attr.q_umem_valid = 1; 359 cq_attr.log_cq_size = log_cqe_n; 360 cq_attr.log_page_size = rte_log2_u32(page_size); 361 cq_attr.db_umem_offset = rxq_ctrl->cq_dbr_offset; 362 cq_attr.db_umem_id = rxq_ctrl->cq_dbr_umem_id; 363 cq_attr.db_umem_valid = 1; 364 cq_obj = mlx5_devx_cmd_create_cq(priv->sh->ctx, &cq_attr); 365 if (!cq_obj) 366 goto error; 367 rxq_data->cqe_n = log_cqe_n; 368 rxq_data->cqn = cq_obj->id; 369 if (rxq_obj->devx_channel) { 370 ret = mlx5_glue->devx_subscribe_devx_event 371 (rxq_obj->devx_channel, 372 cq_obj->obj, 373 sizeof(event_nums), 374 event_nums, 375 (uint64_t)(uintptr_t)cq_obj); 376 if (ret) { 377 DRV_LOG(ERR, "Fail to subscribe CQ to event channel."); 378 rte_errno = errno; 379 goto error; 380 } 381 } 382 /* Initialise CQ to 1's to mark HW ownership for all CQEs. */ 383 memset((void *)(uintptr_t)rxq_data->cqes, 0xFF, cq_size); 384 return cq_obj; 385 error: 386 if (cq_obj) 387 mlx5_devx_cmd_destroy(cq_obj); 388 rxq_release_devx_cq_resources(rxq_ctrl); 389 return NULL; 390 } 391 392 /** 393 * Create the Rx hairpin queue object. 394 * 395 * @param dev 396 * Pointer to Ethernet device. 397 * @param idx 398 * Queue index in DPDK Rx queue array. 399 * 400 * @return 401 * The hairpin DevX object initialized, NULL otherwise and rte_errno is set. 402 */ 403 static struct mlx5_rxq_obj * 404 mlx5_rxq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx) 405 { 406 struct mlx5_priv *priv = dev->data->dev_private; 407 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 408 struct mlx5_rxq_ctrl *rxq_ctrl = 409 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 410 struct mlx5_devx_create_rq_attr attr = { 0 }; 411 struct mlx5_rxq_obj *tmpl = NULL; 412 uint32_t max_wq_data; 413 414 MLX5_ASSERT(rxq_data); 415 MLX5_ASSERT(!rxq_ctrl->obj); 416 tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0, 417 rxq_ctrl->socket); 418 if (!tmpl) { 419 DRV_LOG(ERR, "port %u Rx queue %u cannot allocate resources", 420 dev->data->port_id, rxq_data->idx); 421 rte_errno = ENOMEM; 422 return NULL; 423 } 424 tmpl->type = MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN; 425 tmpl->rxq_ctrl = rxq_ctrl; 426 attr.hairpin = 1; 427 max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz; 428 /* Jumbo frames > 9KB should be supported, and more packets. */ 429 if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) { 430 if (priv->config.log_hp_size > max_wq_data) { 431 DRV_LOG(ERR, "Total data size %u power of 2 is " 432 "too large for hairpin.", 433 priv->config.log_hp_size); 434 mlx5_free(tmpl); 435 rte_errno = ERANGE; 436 return NULL; 437 } 438 attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size; 439 } else { 440 attr.wq_attr.log_hairpin_data_sz = 441 (max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ? 442 max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE; 443 } 444 /* Set the packets number to the maximum value for performance. */ 445 attr.wq_attr.log_hairpin_num_packets = 446 attr.wq_attr.log_hairpin_data_sz - 447 MLX5_HAIRPIN_QUEUE_STRIDE; 448 tmpl->rq = mlx5_devx_cmd_create_rq(priv->sh->ctx, &attr, 449 rxq_ctrl->socket); 450 if (!tmpl->rq) { 451 DRV_LOG(ERR, 452 "Port %u Rx hairpin queue %u can't create rq object.", 453 dev->data->port_id, idx); 454 mlx5_free(tmpl); 455 rte_errno = errno; 456 return NULL; 457 } 458 DRV_LOG(DEBUG, "port %u rxq %u updated with %p", dev->data->port_id, 459 idx, (void *)&tmpl); 460 LIST_INSERT_HEAD(&priv->rxqsobj, tmpl, next); 461 dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN; 462 return tmpl; 463 } 464 465 /** 466 * Create the Rx queue DevX object. 467 * 468 * @param dev 469 * Pointer to Ethernet device. 470 * @param idx 471 * Queue index in DPDK Rx queue array. 472 * 473 * @return 474 * The DevX object initialized, NULL otherwise and rte_errno is set. 475 */ 476 static struct mlx5_rxq_obj * 477 mlx5_rxq_devx_obj_new(struct rte_eth_dev *dev, uint16_t idx) 478 { 479 struct mlx5_priv *priv = dev->data->dev_private; 480 struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; 481 struct mlx5_rxq_ctrl *rxq_ctrl = 482 container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); 483 unsigned int cqe_n; 484 unsigned int wqe_n = 1 << rxq_data->elts_n; 485 struct mlx5_rxq_obj *tmpl = NULL; 486 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 487 struct mlx5_devx_dbr_page *cq_dbr_page = NULL; 488 struct mlx5_devx_dbr_page *rq_dbr_page = NULL; 489 int64_t dbr_offset; 490 int ret = 0; 491 492 MLX5_ASSERT(rxq_data); 493 MLX5_ASSERT(!rxq_ctrl->obj); 494 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) 495 return mlx5_rxq_obj_hairpin_new(dev, idx); 496 tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0, 497 rxq_ctrl->socket); 498 if (!tmpl) { 499 DRV_LOG(ERR, "port %u Rx queue %u cannot allocate resources", 500 dev->data->port_id, rxq_data->idx); 501 rte_errno = ENOMEM; 502 goto error; 503 } 504 tmpl->type = MLX5_RXQ_OBJ_TYPE_DEVX_RQ; 505 tmpl->rxq_ctrl = rxq_ctrl; 506 if (rxq_ctrl->irq) { 507 int devx_ev_flag = 508 MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA; 509 510 tmpl->devx_channel = mlx5_glue->devx_create_event_channel 511 (priv->sh->ctx, 512 devx_ev_flag); 513 if (!tmpl->devx_channel) { 514 rte_errno = errno; 515 DRV_LOG(ERR, "Failed to create event channel %d.", 516 rte_errno); 517 goto error; 518 } 519 tmpl->fd = mlx5_os_get_devx_channel_fd(tmpl->devx_channel); 520 } 521 if (mlx5_rxq_mprq_enabled(rxq_data)) 522 cqe_n = wqe_n * (1 << rxq_data->strd_num_n) - 1; 523 else 524 cqe_n = wqe_n - 1; 525 DRV_LOG(DEBUG, "port %u device_attr.max_qp_wr is %d", 526 dev->data->port_id, priv->sh->device_attr.max_qp_wr); 527 DRV_LOG(DEBUG, "port %u device_attr.max_sge is %d", 528 dev->data->port_id, priv->sh->device_attr.max_sge); 529 /* Allocate CQ door-bell. */ 530 dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs, &cq_dbr_page); 531 if (dbr_offset < 0) { 532 DRV_LOG(ERR, "Failed to allocate CQ door-bell."); 533 goto error; 534 } 535 rxq_ctrl->cq_dbr_offset = dbr_offset; 536 rxq_ctrl->cq_dbr_umem_id = mlx5_os_get_umem_id(cq_dbr_page->umem); 537 rxq_data->cq_db = (uint32_t *)((uintptr_t)cq_dbr_page->dbrs + 538 (uintptr_t)rxq_ctrl->cq_dbr_offset); 539 rxq_data->cq_uar = 540 mlx5_os_get_devx_uar_base_addr(priv->sh->devx_rx_uar); 541 /* Create CQ using DevX API. */ 542 tmpl->devx_cq = mlx5_devx_cq_new(dev, cqe_n, idx, tmpl); 543 if (!tmpl->devx_cq) { 544 DRV_LOG(ERR, "Failed to create CQ."); 545 goto error; 546 } 547 /* Allocate RQ door-bell. */ 548 dbr_offset = mlx5_get_dbr(priv->sh->ctx, &priv->dbrpgs, &rq_dbr_page); 549 if (dbr_offset < 0) { 550 DRV_LOG(ERR, "Failed to allocate RQ door-bell."); 551 goto error; 552 } 553 rxq_ctrl->rq_dbr_offset = dbr_offset; 554 rxq_ctrl->rq_dbr_umem_id = mlx5_os_get_umem_id(rq_dbr_page->umem); 555 rxq_data->rq_db = (uint32_t *)((uintptr_t)rq_dbr_page->dbrs + 556 (uintptr_t)rxq_ctrl->rq_dbr_offset); 557 /* Create RQ using DevX API. */ 558 tmpl->rq = mlx5_devx_rq_new(dev, idx, tmpl->devx_cq->id); 559 if (!tmpl->rq) { 560 DRV_LOG(ERR, "Port %u Rx queue %u RQ creation failure.", 561 dev->data->port_id, idx); 562 rte_errno = ENOMEM; 563 goto error; 564 } 565 /* Change queue state to ready. */ 566 rq_attr.rq_state = MLX5_RQC_STATE_RST; 567 rq_attr.state = MLX5_RQC_STATE_RDY; 568 ret = mlx5_devx_cmd_modify_rq(tmpl->rq, &rq_attr); 569 if (ret) 570 goto error; 571 rxq_data->cq_arm_sn = 0; 572 mlx5_rxq_initialize(rxq_data); 573 rxq_data->cq_ci = 0; 574 DRV_LOG(DEBUG, "port %u rxq %u updated with %p", dev->data->port_id, 575 idx, (void *)&tmpl); 576 LIST_INSERT_HEAD(&priv->rxqsobj, tmpl, next); 577 dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED; 578 rxq_ctrl->wqn = tmpl->rq->id; 579 return tmpl; 580 error: 581 if (tmpl) { 582 ret = rte_errno; /* Save rte_errno before cleanup. */ 583 if (tmpl->rq) 584 claim_zero(mlx5_devx_cmd_destroy(tmpl->rq)); 585 if (tmpl->devx_cq) 586 claim_zero(mlx5_devx_cmd_destroy(tmpl->devx_cq)); 587 if (tmpl->devx_channel) 588 mlx5_glue->devx_destroy_event_channel 589 (tmpl->devx_channel); 590 mlx5_free(tmpl); 591 rte_errno = ret; /* Restore rte_errno. */ 592 } 593 if (rq_dbr_page) 594 claim_zero(mlx5_release_dbr(&priv->dbrpgs, 595 rxq_ctrl->rq_dbr_umem_id, 596 rxq_ctrl->rq_dbr_offset)); 597 if (cq_dbr_page) 598 claim_zero(mlx5_release_dbr(&priv->dbrpgs, 599 rxq_ctrl->cq_dbr_umem_id, 600 rxq_ctrl->cq_dbr_offset)); 601 rxq_release_devx_rq_resources(rxq_ctrl); 602 rxq_release_devx_cq_resources(rxq_ctrl); 603 return NULL; 604 } 605 606 struct mlx5_obj_ops devx_obj_ops = { 607 .rxq_obj_modify_vlan_strip = mlx5_rxq_obj_modify_rq_vlan_strip, 608 .rxq_obj_new = mlx5_rxq_devx_obj_new, 609 .rxq_obj_release = mlx5_rxq_devx_obj_release, 610 }; 611