1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2022 Microsoft Corporation 3 */ 4 #include <ethdev_driver.h> 5 6 #include <infiniband/verbs.h> 7 #include <infiniband/manadv.h> 8 9 #include "mana.h" 10 11 static uint8_t mana_rss_hash_key_default[TOEPLITZ_HASH_KEY_SIZE_IN_BYTES] = { 12 0x2c, 0xc6, 0x81, 0xd1, 13 0x5b, 0xdb, 0xf4, 0xf7, 14 0xfc, 0xa2, 0x83, 0x19, 15 0xdb, 0x1a, 0x3e, 0x94, 16 0x6b, 0x9e, 0x38, 0xd9, 17 0x2c, 0x9c, 0x03, 0xd1, 18 0xad, 0x99, 0x44, 0xa7, 19 0xd9, 0x56, 0x3d, 0x59, 20 0x06, 0x3c, 0x25, 0xf3, 21 0xfc, 0x1f, 0xdc, 0x2a, 22 }; 23 24 int 25 mana_rq_ring_doorbell(struct mana_rxq *rxq) 26 { 27 struct mana_priv *priv = rxq->priv; 28 int ret; 29 void *db_page = priv->db_page; 30 31 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 32 struct rte_eth_dev *dev = 33 &rte_eth_devices[priv->dev_data->port_id]; 34 struct mana_process_priv *process_priv = dev->process_private; 35 36 db_page = process_priv->db_page; 37 } 38 39 /* Hardware Spec specifies that software client should set 0 for 40 * wqe_cnt for Receive Queues. 41 */ 42 #ifdef RTE_ARCH_32 43 ret = mana_ring_short_doorbell(db_page, GDMA_QUEUE_RECEIVE, 44 rxq->gdma_rq.id, 45 rxq->wqe_cnt_to_short_db * 46 GDMA_WQE_ALIGNMENT_UNIT_SIZE, 47 0); 48 #else 49 ret = mana_ring_doorbell(db_page, GDMA_QUEUE_RECEIVE, 50 rxq->gdma_rq.id, 51 rxq->gdma_rq.head * GDMA_WQE_ALIGNMENT_UNIT_SIZE, 52 0); 53 #endif 54 55 if (ret) 56 DP_LOG(ERR, "failed to ring RX doorbell ret %d", ret); 57 58 return ret; 59 } 60 61 static int 62 mana_post_rx_wqe(struct mana_rxq *rxq, struct rte_mbuf *mbuf) 63 { 64 struct gdma_sgl_element sgl[1]; 65 struct gdma_work_request request; 66 uint32_t wqe_size_in_bu; 67 struct mana_priv *priv = rxq->priv; 68 int ret; 69 struct mana_mr_cache *mr; 70 71 mr = mana_alloc_pmd_mr(&rxq->mr_btree, priv, mbuf); 72 if (!mr) { 73 DP_LOG(ERR, "failed to register RX MR"); 74 rte_pktmbuf_free(mbuf); 75 return -ENOMEM; 76 } 77 78 request.gdma_header.struct_size = sizeof(request); 79 80 sgl[0].address = rte_cpu_to_le_64(rte_pktmbuf_mtod(mbuf, uint64_t)); 81 sgl[0].memory_key = mr->lkey; 82 sgl[0].size = 83 rte_pktmbuf_data_room_size(rxq->mp) - 84 RTE_PKTMBUF_HEADROOM; 85 86 request.sgl = sgl; 87 request.num_sgl_elements = 1; 88 request.inline_oob_data = NULL; 89 request.inline_oob_size_in_bytes = 0; 90 request.flags = 0; 91 request.client_data_unit = NOT_USING_CLIENT_DATA_UNIT; 92 93 ret = gdma_post_work_request(&rxq->gdma_rq, &request, &wqe_size_in_bu); 94 if (!ret) { 95 struct mana_rxq_desc *desc = 96 &rxq->desc_ring[rxq->desc_ring_head]; 97 98 /* update queue for tracking pending packets */ 99 desc->pkt = mbuf; 100 desc->wqe_size_in_bu = wqe_size_in_bu; 101 #ifdef RTE_ARCH_32 102 rxq->wqe_cnt_to_short_db += wqe_size_in_bu; 103 #endif 104 rxq->desc_ring_head = (rxq->desc_ring_head + 1) % rxq->num_desc; 105 } else { 106 DP_LOG(DEBUG, "failed to post recv ret %d", ret); 107 return ret; 108 } 109 110 return 0; 111 } 112 113 /* 114 * Post work requests for a Rx queue. 115 */ 116 #define MANA_MBUF_BULK 32u 117 static int 118 mana_alloc_and_post_rx_wqes(struct mana_rxq *rxq, uint32_t count) 119 { 120 int ret; 121 uint32_t i, batch_count; 122 struct rte_mbuf *mbufs[MANA_MBUF_BULK]; 123 124 more_mbufs: 125 batch_count = RTE_MIN(count, MANA_MBUF_BULK); 126 ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, batch_count); 127 if (ret) { 128 DP_LOG(ERR, "failed to allocate mbufs for RX"); 129 rxq->stats.nombuf += count; 130 131 /* Bail out to ring doorbell for posted packets */ 132 goto out; 133 } 134 135 #ifdef RTE_ARCH_32 136 rxq->wqe_cnt_to_short_db = 0; 137 #endif 138 for (i = 0; i < batch_count; i++) { 139 ret = mana_post_rx_wqe(rxq, mbufs[i]); 140 if (ret) { 141 DP_LOG(ERR, "failed to post RX ret = %d", ret); 142 143 /* Free the remaining mbufs that are not posted */ 144 rte_pktmbuf_free_bulk(&mbufs[i], batch_count - i); 145 goto out; 146 } 147 148 #ifdef RTE_ARCH_32 149 if (rxq->wqe_cnt_to_short_db > RX_WQE_SHORT_DB_THRESHOLD) { 150 mana_rq_ring_doorbell(rxq); 151 rxq->wqe_cnt_to_short_db = 0; 152 } 153 #endif 154 } 155 156 count -= batch_count; 157 if (count > 0) 158 goto more_mbufs; 159 160 out: 161 mana_rq_ring_doorbell(rxq); 162 return ret; 163 } 164 165 int 166 mana_stop_rx_queues(struct rte_eth_dev *dev) 167 { 168 struct mana_priv *priv = dev->data->dev_private; 169 int ret, i; 170 171 for (i = 0; i < priv->num_queues; i++) 172 if (dev->data->rx_queue_state[i] == RTE_ETH_QUEUE_STATE_STOPPED) 173 return -EINVAL; 174 175 if (priv->rwq_qp) { 176 ret = ibv_destroy_qp(priv->rwq_qp); 177 if (ret) 178 DRV_LOG(ERR, "rx_queue destroy_qp failed %d", ret); 179 priv->rwq_qp = NULL; 180 } 181 182 if (priv->ind_table) { 183 ret = ibv_destroy_rwq_ind_table(priv->ind_table); 184 if (ret) 185 DRV_LOG(ERR, "destroy rwq ind table failed %d", ret); 186 priv->ind_table = NULL; 187 } 188 189 for (i = 0; i < priv->num_queues; i++) { 190 struct mana_rxq *rxq = dev->data->rx_queues[i]; 191 192 if (rxq->wq) { 193 ret = ibv_destroy_wq(rxq->wq); 194 if (ret) 195 DRV_LOG(ERR, 196 "rx_queue destroy_wq failed %d", ret); 197 rxq->wq = NULL; 198 } 199 200 if (rxq->cq) { 201 ret = ibv_destroy_cq(rxq->cq); 202 if (ret) 203 DRV_LOG(ERR, 204 "rx_queue destroy_cq failed %d", ret); 205 rxq->cq = NULL; 206 207 if (rxq->channel) { 208 ret = ibv_destroy_comp_channel(rxq->channel); 209 if (ret) 210 DRV_LOG(ERR, "failed destroy comp %d", 211 ret); 212 rxq->channel = NULL; 213 } 214 } 215 216 /* Drain and free posted WQEs */ 217 while (rxq->desc_ring_tail != rxq->desc_ring_head) { 218 struct mana_rxq_desc *desc = 219 &rxq->desc_ring[rxq->desc_ring_tail]; 220 221 rte_pktmbuf_free(desc->pkt); 222 223 rxq->desc_ring_tail = 224 (rxq->desc_ring_tail + 1) % rxq->num_desc; 225 } 226 rxq->desc_ring_head = 0; 227 rxq->desc_ring_tail = 0; 228 229 memset(&rxq->gdma_rq, 0, sizeof(rxq->gdma_rq)); 230 memset(&rxq->gdma_cq, 0, sizeof(rxq->gdma_cq)); 231 232 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 233 } 234 235 return 0; 236 } 237 238 int 239 mana_start_rx_queues(struct rte_eth_dev *dev) 240 { 241 struct mana_priv *priv = dev->data->dev_private; 242 int ret, i; 243 struct ibv_wq *ind_tbl[priv->num_queues]; 244 245 DRV_LOG(INFO, "start rx queues"); 246 247 for (i = 0; i < priv->num_queues; i++) 248 if (dev->data->rx_queue_state[i] == RTE_ETH_QUEUE_STATE_STARTED) 249 return -EINVAL; 250 251 for (i = 0; i < priv->num_queues; i++) { 252 struct mana_rxq *rxq = dev->data->rx_queues[i]; 253 struct ibv_wq_init_attr wq_attr = {}; 254 255 manadv_set_context_attr(priv->ib_ctx, 256 MANADV_CTX_ATTR_BUF_ALLOCATORS, 257 (void *)((uintptr_t)&(struct manadv_ctx_allocators){ 258 .alloc = &mana_alloc_verbs_buf, 259 .free = &mana_free_verbs_buf, 260 .data = (void *)(uintptr_t)rxq->socket, 261 })); 262 263 if (dev->data->dev_conf.intr_conf.rxq) { 264 rxq->channel = ibv_create_comp_channel(priv->ib_ctx); 265 if (!rxq->channel) { 266 ret = -errno; 267 DRV_LOG(ERR, "Queue %d comp channel failed", i); 268 goto fail; 269 } 270 271 ret = mana_fd_set_non_blocking(rxq->channel->fd); 272 if (ret) { 273 DRV_LOG(ERR, "Failed to set comp non-blocking"); 274 goto fail; 275 } 276 } 277 278 rxq->cq = ibv_create_cq(priv->ib_ctx, rxq->num_desc, 279 NULL, rxq->channel, 280 rxq->channel ? i : 0); 281 if (!rxq->cq) { 282 ret = -errno; 283 DRV_LOG(ERR, "failed to create rx cq queue %d", i); 284 goto fail; 285 } 286 287 wq_attr.wq_type = IBV_WQT_RQ; 288 wq_attr.max_wr = rxq->num_desc; 289 wq_attr.max_sge = 1; 290 wq_attr.pd = priv->ib_parent_pd; 291 wq_attr.cq = rxq->cq; 292 293 rxq->wq = ibv_create_wq(priv->ib_ctx, &wq_attr); 294 if (!rxq->wq) { 295 ret = -errno; 296 DRV_LOG(ERR, "failed to create rx wq %d", i); 297 goto fail; 298 } 299 300 ind_tbl[i] = rxq->wq; 301 } 302 303 struct ibv_rwq_ind_table_init_attr ind_table_attr = { 304 .log_ind_tbl_size = rte_log2_u32(RTE_DIM(ind_tbl)), 305 .ind_tbl = ind_tbl, 306 .comp_mask = 0, 307 }; 308 309 priv->ind_table = ibv_create_rwq_ind_table(priv->ib_ctx, 310 &ind_table_attr); 311 if (!priv->ind_table) { 312 ret = -errno; 313 DRV_LOG(ERR, "failed to create ind_table ret %d", ret); 314 goto fail; 315 } 316 317 DRV_LOG(INFO, "ind_table handle %d num %d", 318 priv->ind_table->ind_tbl_handle, 319 priv->ind_table->ind_tbl_num); 320 321 struct ibv_qp_init_attr_ex qp_attr_ex = { 322 .comp_mask = IBV_QP_INIT_ATTR_PD | 323 IBV_QP_INIT_ATTR_RX_HASH | 324 IBV_QP_INIT_ATTR_IND_TABLE, 325 .qp_type = IBV_QPT_RAW_PACKET, 326 .pd = priv->ib_parent_pd, 327 .rwq_ind_tbl = priv->ind_table, 328 .rx_hash_conf = { 329 .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ, 330 .rx_hash_key_len = TOEPLITZ_HASH_KEY_SIZE_IN_BYTES, 331 .rx_hash_key = mana_rss_hash_key_default, 332 .rx_hash_fields_mask = 333 IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4, 334 }, 335 336 }; 337 338 /* overwrite default if rss key is set */ 339 if (priv->rss_conf.rss_key_len && priv->rss_conf.rss_key) 340 qp_attr_ex.rx_hash_conf.rx_hash_key = 341 priv->rss_conf.rss_key; 342 343 /* overwrite default if rss hash fields are set */ 344 if (priv->rss_conf.rss_hf) { 345 qp_attr_ex.rx_hash_conf.rx_hash_fields_mask = 0; 346 347 if (priv->rss_conf.rss_hf & RTE_ETH_RSS_IPV4) 348 qp_attr_ex.rx_hash_conf.rx_hash_fields_mask |= 349 IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4; 350 351 if (priv->rss_conf.rss_hf & RTE_ETH_RSS_IPV6) 352 qp_attr_ex.rx_hash_conf.rx_hash_fields_mask |= 353 IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_SRC_IPV6; 354 355 if (priv->rss_conf.rss_hf & 356 (RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_NONFRAG_IPV6_TCP)) 357 qp_attr_ex.rx_hash_conf.rx_hash_fields_mask |= 358 IBV_RX_HASH_SRC_PORT_TCP | 359 IBV_RX_HASH_DST_PORT_TCP; 360 361 if (priv->rss_conf.rss_hf & 362 (RTE_ETH_RSS_NONFRAG_IPV4_UDP | RTE_ETH_RSS_NONFRAG_IPV6_UDP)) 363 qp_attr_ex.rx_hash_conf.rx_hash_fields_mask |= 364 IBV_RX_HASH_SRC_PORT_UDP | 365 IBV_RX_HASH_DST_PORT_UDP; 366 } 367 368 priv->rwq_qp = ibv_create_qp_ex(priv->ib_ctx, &qp_attr_ex); 369 if (!priv->rwq_qp) { 370 ret = -errno; 371 DRV_LOG(ERR, "rx ibv_create_qp_ex failed"); 372 goto fail; 373 } 374 375 for (i = 0; i < priv->num_queues; i++) { 376 struct mana_rxq *rxq = dev->data->rx_queues[i]; 377 struct manadv_obj obj = {}; 378 struct manadv_cq dv_cq; 379 struct manadv_rwq dv_wq; 380 381 obj.cq.in = rxq->cq; 382 obj.cq.out = &dv_cq; 383 obj.rwq.in = rxq->wq; 384 obj.rwq.out = &dv_wq; 385 ret = manadv_init_obj(&obj, MANADV_OBJ_CQ | MANADV_OBJ_RWQ); 386 if (ret) { 387 DRV_LOG(ERR, "manadv_init_obj failed ret %d", ret); 388 goto fail; 389 } 390 391 rxq->gdma_cq.buffer = obj.cq.out->buf; 392 rxq->gdma_cq.count = obj.cq.out->count; 393 rxq->gdma_cq.size = rxq->gdma_cq.count * COMP_ENTRY_SIZE; 394 rxq->gdma_cq.id = obj.cq.out->cq_id; 395 396 /* CQ head starts with count */ 397 rxq->gdma_cq.head = rxq->gdma_cq.count; 398 399 DRV_LOG(INFO, "rxq cq id %u buf %p count %u size %u", 400 rxq->gdma_cq.id, rxq->gdma_cq.buffer, 401 rxq->gdma_cq.count, rxq->gdma_cq.size); 402 403 priv->db_page = obj.rwq.out->db_page; 404 405 rxq->gdma_rq.buffer = obj.rwq.out->buf; 406 rxq->gdma_rq.count = obj.rwq.out->count; 407 rxq->gdma_rq.size = obj.rwq.out->size; 408 rxq->gdma_rq.id = obj.rwq.out->wq_id; 409 410 DRV_LOG(INFO, "rxq rq id %u buf %p count %u size %u", 411 rxq->gdma_rq.id, rxq->gdma_rq.buffer, 412 rxq->gdma_rq.count, rxq->gdma_rq.size); 413 414 rxq->comp_buf_len = 0; 415 rxq->comp_buf_idx = 0; 416 rxq->backlog_idx = 0; 417 } 418 419 for (i = 0; i < priv->num_queues; i++) { 420 struct mana_rxq *rxq = dev->data->rx_queues[i]; 421 422 ret = mana_alloc_and_post_rx_wqes(rxq, rxq->num_desc); 423 if (ret) 424 goto fail; 425 } 426 427 for (i = 0; i < priv->num_queues; i++) 428 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 429 430 return 0; 431 432 fail: 433 mana_stop_rx_queues(dev); 434 return ret; 435 } 436 437 uint16_t 438 mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 439 { 440 uint16_t pkt_received = 0; 441 uint16_t wqe_consumed = 0; 442 struct mana_rxq *rxq = dpdk_rxq; 443 struct mana_priv *priv = rxq->priv; 444 struct rte_mbuf *mbuf; 445 int ret; 446 uint32_t pkt_idx = rxq->backlog_idx; 447 uint32_t pkt_len; 448 uint32_t i; 449 int polled = 0; 450 451 #ifdef RTE_ARCH_32 452 rxq->wqe_cnt_to_short_db = 0; 453 #endif 454 455 repoll: 456 /* Polling on new completions if we have no backlog */ 457 if (rxq->comp_buf_idx == rxq->comp_buf_len) { 458 RTE_ASSERT(!pkt_idx); 459 rxq->comp_buf_len = 460 gdma_poll_completion_queue(&rxq->gdma_cq, 461 rxq->gdma_comp_buf, pkts_n); 462 rxq->comp_buf_idx = 0; 463 polled = 1; 464 } 465 466 i = rxq->comp_buf_idx; 467 while (i < rxq->comp_buf_len) { 468 struct mana_rx_comp_oob *oob = (struct mana_rx_comp_oob *) 469 rxq->gdma_comp_buf[i].cqe_data; 470 struct mana_rxq_desc *desc = 471 &rxq->desc_ring[rxq->desc_ring_tail]; 472 473 mbuf = desc->pkt; 474 475 switch (oob->cqe_hdr.cqe_type) { 476 case CQE_RX_OKAY: 477 case CQE_RX_COALESCED_4: 478 /* Proceed to process mbuf */ 479 break; 480 481 case CQE_RX_TRUNCATED: 482 default: 483 DP_LOG(ERR, "RX CQE type %d client %d vendor %d", 484 oob->cqe_hdr.cqe_type, oob->cqe_hdr.client_type, 485 oob->cqe_hdr.vendor_err); 486 487 rxq->stats.errors++; 488 rte_pktmbuf_free(mbuf); 489 490 i++; 491 goto drop; 492 } 493 494 DP_LOG(DEBUG, "mana_rx_comp_oob type %d rxq %p", 495 oob->cqe_hdr.cqe_type, rxq); 496 497 pkt_len = oob->packet_info[pkt_idx].packet_length; 498 if (!pkt_len) { 499 /* Move on to the next completion */ 500 pkt_idx = 0; 501 i++; 502 continue; 503 } 504 505 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 506 mbuf->nb_segs = 1; 507 mbuf->next = NULL; 508 mbuf->data_len = pkt_len; 509 mbuf->pkt_len = pkt_len; 510 mbuf->port = priv->port_id; 511 512 if (oob->rx_ip_header_checksum_succeeded) 513 mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; 514 515 if (oob->rx_ip_header_checksum_failed) 516 mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; 517 518 if (oob->rx_outer_ip_header_checksum_failed) 519 mbuf->ol_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 520 521 if (oob->rx_tcp_checksum_succeeded || 522 oob->rx_udp_checksum_succeeded) 523 mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; 524 525 if (oob->rx_tcp_checksum_failed || 526 oob->rx_udp_checksum_failed) 527 mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; 528 529 if (oob->rx_hash_type == MANA_HASH_L3 || 530 oob->rx_hash_type == MANA_HASH_L4) { 531 mbuf->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; 532 mbuf->hash.rss = oob->packet_info[pkt_idx].packet_hash; 533 } 534 535 pkts[pkt_received++] = mbuf; 536 rxq->stats.packets++; 537 rxq->stats.bytes += mbuf->data_len; 538 539 pkt_idx++; 540 /* Move on the next completion if all packets are processed */ 541 if (pkt_idx >= RX_COM_OOB_NUM_PACKETINFO_SEGMENTS) { 542 pkt_idx = 0; 543 i++; 544 } 545 546 drop: 547 rxq->desc_ring_tail++; 548 if (rxq->desc_ring_tail >= rxq->num_desc) 549 rxq->desc_ring_tail = 0; 550 551 rxq->gdma_rq.tail += desc->wqe_size_in_bu; 552 553 /* Record the number of the RX WQE we need to post to replenish 554 * consumed RX requests 555 */ 556 wqe_consumed++; 557 if (pkt_received == pkts_n) 558 break; 559 560 #ifdef RTE_ARCH_32 561 /* Always post WQE as soon as it's consumed for short DB */ 562 ret = mana_alloc_and_post_rx_wqes(rxq, wqe_consumed); 563 if (ret) { 564 DRV_LOG(ERR, "failed to post %d WQEs, ret %d", 565 wqe_consumed, ret); 566 return pkt_received; 567 } 568 wqe_consumed = 0; 569 570 /* Ring short doorbell if approaching the wqe increment 571 * limit. 572 */ 573 if (rxq->wqe_cnt_to_short_db > RX_WQE_SHORT_DB_THRESHOLD) { 574 mana_rq_ring_doorbell(rxq); 575 rxq->wqe_cnt_to_short_db = 0; 576 } 577 #endif 578 } 579 580 rxq->backlog_idx = pkt_idx; 581 rxq->comp_buf_idx = i; 582 583 /* If all CQEs are processed but there are more packets to read, poll the 584 * completion queue again because we may have not polled on the completion 585 * queue due to CQE not fully processed in the previous rx_burst 586 */ 587 if (pkt_received < pkts_n && !polled) { 588 polled = 1; 589 goto repoll; 590 } 591 592 if (wqe_consumed) { 593 ret = mana_alloc_and_post_rx_wqes(rxq, wqe_consumed); 594 if (ret) 595 DRV_LOG(ERR, "failed to post %d WQEs, ret %d", 596 wqe_consumed, ret); 597 } 598 599 return pkt_received; 600 } 601 602 #ifdef RTE_ARCH_32 603 static int 604 mana_arm_cq(struct mana_rxq *rxq __rte_unused, uint8_t arm __rte_unused) 605 { 606 DP_LOG(ERR, "Do not support in 32 bit"); 607 608 return -ENODEV; 609 } 610 #else 611 static int 612 mana_arm_cq(struct mana_rxq *rxq, uint8_t arm) 613 { 614 struct mana_priv *priv = rxq->priv; 615 uint32_t head = rxq->gdma_cq.head % 616 (rxq->gdma_cq.count << COMPLETION_QUEUE_ENTRY_OWNER_BITS_SIZE); 617 618 DP_LOG(DEBUG, "Ringing completion queue ID %u head %u arm %d", 619 rxq->gdma_cq.id, head, arm); 620 621 return mana_ring_doorbell(priv->db_page, GDMA_QUEUE_COMPLETION, 622 rxq->gdma_cq.id, head, arm); 623 } 624 #endif 625 626 int 627 mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) 628 { 629 struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id]; 630 631 return mana_arm_cq(rxq, 1); 632 } 633 634 int 635 mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) 636 { 637 struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id]; 638 struct ibv_cq *ev_cq; 639 void *ev_ctx; 640 int ret; 641 642 ret = ibv_get_cq_event(rxq->channel, &ev_cq, &ev_ctx); 643 if (ret) 644 ret = errno; 645 else if (ev_cq != rxq->cq) 646 ret = EINVAL; 647 648 if (ret) { 649 if (ret != EAGAIN) 650 DP_LOG(ERR, "Can't disable RX intr queue %d", 651 rx_queue_id); 652 } else { 653 ibv_ack_cq_events(rxq->cq, 1); 654 } 655 656 return -ret; 657 } 658