1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2022 Microsoft Corporation
3 */
4 #include <ethdev_driver.h>
5
6 #include <infiniband/verbs.h>
7 #include <infiniband/manadv.h>
8
9 #include "mana.h"
10
11 static uint8_t mana_rss_hash_key_default[TOEPLITZ_HASH_KEY_SIZE_IN_BYTES] = {
12 0x2c, 0xc6, 0x81, 0xd1,
13 0x5b, 0xdb, 0xf4, 0xf7,
14 0xfc, 0xa2, 0x83, 0x19,
15 0xdb, 0x1a, 0x3e, 0x94,
16 0x6b, 0x9e, 0x38, 0xd9,
17 0x2c, 0x9c, 0x03, 0xd1,
18 0xad, 0x99, 0x44, 0xa7,
19 0xd9, 0x56, 0x3d, 0x59,
20 0x06, 0x3c, 0x25, 0xf3,
21 0xfc, 0x1f, 0xdc, 0x2a,
22 };
23
24 int
mana_rq_ring_doorbell(struct mana_rxq * rxq)25 mana_rq_ring_doorbell(struct mana_rxq *rxq)
26 {
27 struct mana_priv *priv = rxq->priv;
28 int ret;
29 void *db_page = priv->db_page;
30
31 if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
32 struct rte_eth_dev *dev =
33 &rte_eth_devices[priv->dev_data->port_id];
34 struct mana_process_priv *process_priv = dev->process_private;
35
36 db_page = process_priv->db_page;
37 }
38
39 /* Hardware Spec specifies that software client should set 0 for
40 * wqe_cnt for Receive Queues.
41 */
42 #ifdef RTE_ARCH_32
43 ret = mana_ring_short_doorbell(db_page, GDMA_QUEUE_RECEIVE,
44 rxq->gdma_rq.id,
45 rxq->wqe_cnt_to_short_db *
46 GDMA_WQE_ALIGNMENT_UNIT_SIZE,
47 0);
48 #else
49 ret = mana_ring_doorbell(db_page, GDMA_QUEUE_RECEIVE,
50 rxq->gdma_rq.id,
51 rxq->gdma_rq.head * GDMA_WQE_ALIGNMENT_UNIT_SIZE,
52 0);
53 #endif
54
55 if (ret)
56 DP_LOG(ERR, "failed to ring RX doorbell ret %d", ret);
57
58 return ret;
59 }
60
61 static int
mana_post_rx_wqe(struct mana_rxq * rxq,struct rte_mbuf * mbuf)62 mana_post_rx_wqe(struct mana_rxq *rxq, struct rte_mbuf *mbuf)
63 {
64 struct gdma_sgl_element sgl[1];
65 struct gdma_work_request request;
66 uint32_t wqe_size_in_bu;
67 struct mana_priv *priv = rxq->priv;
68 int ret;
69 struct mana_mr_cache *mr;
70
71 mr = mana_alloc_pmd_mr(&rxq->mr_btree, priv, mbuf);
72 if (!mr) {
73 DP_LOG(ERR, "failed to register RX MR");
74 rte_pktmbuf_free(mbuf);
75 return -ENOMEM;
76 }
77
78 request.gdma_header.struct_size = sizeof(request);
79
80 sgl[0].address = rte_cpu_to_le_64(rte_pktmbuf_mtod(mbuf, uint64_t));
81 sgl[0].memory_key = mr->lkey;
82 sgl[0].size =
83 rte_pktmbuf_data_room_size(rxq->mp) -
84 RTE_PKTMBUF_HEADROOM;
85
86 request.sgl = sgl;
87 request.num_sgl_elements = 1;
88 request.inline_oob_data = NULL;
89 request.inline_oob_size_in_bytes = 0;
90 request.flags = 0;
91 request.client_data_unit = NOT_USING_CLIENT_DATA_UNIT;
92
93 ret = gdma_post_work_request(&rxq->gdma_rq, &request, &wqe_size_in_bu);
94 if (!ret) {
95 struct mana_rxq_desc *desc =
96 &rxq->desc_ring[rxq->desc_ring_head];
97
98 /* update queue for tracking pending packets */
99 desc->pkt = mbuf;
100 desc->wqe_size_in_bu = wqe_size_in_bu;
101 #ifdef RTE_ARCH_32
102 rxq->wqe_cnt_to_short_db += wqe_size_in_bu;
103 #endif
104 rxq->desc_ring_head = (rxq->desc_ring_head + 1) % rxq->num_desc;
105 } else {
106 DP_LOG(DEBUG, "failed to post recv ret %d", ret);
107 return ret;
108 }
109
110 return 0;
111 }
112
113 /*
114 * Post work requests for a Rx queue.
115 */
116 #define MANA_MBUF_BULK 32u
117 static int
mana_alloc_and_post_rx_wqes(struct mana_rxq * rxq,uint32_t count)118 mana_alloc_and_post_rx_wqes(struct mana_rxq *rxq, uint32_t count)
119 {
120 int ret;
121 uint32_t i, batch_count;
122 struct rte_mbuf *mbufs[MANA_MBUF_BULK];
123
124 more_mbufs:
125 batch_count = RTE_MIN(count, MANA_MBUF_BULK);
126 ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, batch_count);
127 if (ret) {
128 DP_LOG(ERR, "failed to allocate mbufs for RX");
129 rxq->stats.nombuf += count;
130
131 /* Bail out to ring doorbell for posted packets */
132 goto out;
133 }
134
135 #ifdef RTE_ARCH_32
136 rxq->wqe_cnt_to_short_db = 0;
137 #endif
138 for (i = 0; i < batch_count; i++) {
139 ret = mana_post_rx_wqe(rxq, mbufs[i]);
140 if (ret) {
141 DP_LOG(ERR, "failed to post RX ret = %d", ret);
142
143 /* Free the remaining mbufs that are not posted */
144 rte_pktmbuf_free_bulk(&mbufs[i], batch_count - i);
145 goto out;
146 }
147
148 #ifdef RTE_ARCH_32
149 if (rxq->wqe_cnt_to_short_db > RX_WQE_SHORT_DB_THRESHOLD) {
150 mana_rq_ring_doorbell(rxq);
151 rxq->wqe_cnt_to_short_db = 0;
152 }
153 #endif
154 }
155
156 count -= batch_count;
157 if (count > 0)
158 goto more_mbufs;
159
160 out:
161 mana_rq_ring_doorbell(rxq);
162 return ret;
163 }
164
165 int
mana_stop_rx_queues(struct rte_eth_dev * dev)166 mana_stop_rx_queues(struct rte_eth_dev *dev)
167 {
168 struct mana_priv *priv = dev->data->dev_private;
169 int ret, i;
170
171 for (i = 0; i < priv->num_queues; i++)
172 if (dev->data->rx_queue_state[i] == RTE_ETH_QUEUE_STATE_STOPPED)
173 return -EINVAL;
174
175 if (priv->rwq_qp) {
176 ret = ibv_destroy_qp(priv->rwq_qp);
177 if (ret)
178 DRV_LOG(ERR, "rx_queue destroy_qp failed %d", ret);
179 priv->rwq_qp = NULL;
180 }
181
182 if (priv->ind_table) {
183 ret = ibv_destroy_rwq_ind_table(priv->ind_table);
184 if (ret)
185 DRV_LOG(ERR, "destroy rwq ind table failed %d", ret);
186 priv->ind_table = NULL;
187 }
188
189 for (i = 0; i < priv->num_queues; i++) {
190 struct mana_rxq *rxq = dev->data->rx_queues[i];
191
192 if (rxq->wq) {
193 ret = ibv_destroy_wq(rxq->wq);
194 if (ret)
195 DRV_LOG(ERR,
196 "rx_queue destroy_wq failed %d", ret);
197 rxq->wq = NULL;
198 }
199
200 if (rxq->cq) {
201 ret = ibv_destroy_cq(rxq->cq);
202 if (ret)
203 DRV_LOG(ERR,
204 "rx_queue destroy_cq failed %d", ret);
205 rxq->cq = NULL;
206
207 if (rxq->channel) {
208 ret = ibv_destroy_comp_channel(rxq->channel);
209 if (ret)
210 DRV_LOG(ERR, "failed destroy comp %d",
211 ret);
212 rxq->channel = NULL;
213 }
214 }
215
216 /* Drain and free posted WQEs */
217 while (rxq->desc_ring_tail != rxq->desc_ring_head) {
218 struct mana_rxq_desc *desc =
219 &rxq->desc_ring[rxq->desc_ring_tail];
220
221 rte_pktmbuf_free(desc->pkt);
222
223 rxq->desc_ring_tail =
224 (rxq->desc_ring_tail + 1) % rxq->num_desc;
225 }
226 rxq->desc_ring_head = 0;
227 rxq->desc_ring_tail = 0;
228
229 memset(&rxq->gdma_rq, 0, sizeof(rxq->gdma_rq));
230 memset(&rxq->gdma_cq, 0, sizeof(rxq->gdma_cq));
231
232 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
233 }
234
235 return 0;
236 }
237
238 int
mana_start_rx_queues(struct rte_eth_dev * dev)239 mana_start_rx_queues(struct rte_eth_dev *dev)
240 {
241 struct mana_priv *priv = dev->data->dev_private;
242 int ret, i;
243 struct ibv_wq *ind_tbl[priv->num_queues];
244
245 DRV_LOG(INFO, "start rx queues");
246
247 for (i = 0; i < priv->num_queues; i++)
248 if (dev->data->rx_queue_state[i] == RTE_ETH_QUEUE_STATE_STARTED)
249 return -EINVAL;
250
251 for (i = 0; i < priv->num_queues; i++) {
252 struct mana_rxq *rxq = dev->data->rx_queues[i];
253 struct ibv_wq_init_attr wq_attr = {};
254
255 manadv_set_context_attr(priv->ib_ctx,
256 MANADV_CTX_ATTR_BUF_ALLOCATORS,
257 (void *)((uintptr_t)&(struct manadv_ctx_allocators){
258 .alloc = &mana_alloc_verbs_buf,
259 .free = &mana_free_verbs_buf,
260 .data = (void *)(uintptr_t)rxq->socket,
261 }));
262
263 if (dev->data->dev_conf.intr_conf.rxq) {
264 rxq->channel = ibv_create_comp_channel(priv->ib_ctx);
265 if (!rxq->channel) {
266 ret = -errno;
267 DRV_LOG(ERR, "Queue %d comp channel failed", i);
268 goto fail;
269 }
270
271 ret = mana_fd_set_non_blocking(rxq->channel->fd);
272 if (ret) {
273 DRV_LOG(ERR, "Failed to set comp non-blocking");
274 goto fail;
275 }
276 }
277
278 rxq->cq = ibv_create_cq(priv->ib_ctx, rxq->num_desc,
279 NULL, rxq->channel,
280 rxq->channel ? i : 0);
281 if (!rxq->cq) {
282 ret = -errno;
283 DRV_LOG(ERR, "failed to create rx cq queue %d", i);
284 goto fail;
285 }
286
287 wq_attr.wq_type = IBV_WQT_RQ;
288 wq_attr.max_wr = rxq->num_desc;
289 wq_attr.max_sge = 1;
290 wq_attr.pd = priv->ib_parent_pd;
291 wq_attr.cq = rxq->cq;
292
293 rxq->wq = ibv_create_wq(priv->ib_ctx, &wq_attr);
294 if (!rxq->wq) {
295 ret = -errno;
296 DRV_LOG(ERR, "failed to create rx wq %d", i);
297 goto fail;
298 }
299
300 ind_tbl[i] = rxq->wq;
301 }
302
303 struct ibv_rwq_ind_table_init_attr ind_table_attr = {
304 .log_ind_tbl_size = rte_log2_u32(RTE_DIM(ind_tbl)),
305 .ind_tbl = ind_tbl,
306 .comp_mask = 0,
307 };
308
309 priv->ind_table = ibv_create_rwq_ind_table(priv->ib_ctx,
310 &ind_table_attr);
311 if (!priv->ind_table) {
312 ret = -errno;
313 DRV_LOG(ERR, "failed to create ind_table ret %d", ret);
314 goto fail;
315 }
316
317 DRV_LOG(INFO, "ind_table handle %d num %d",
318 priv->ind_table->ind_tbl_handle,
319 priv->ind_table->ind_tbl_num);
320
321 struct ibv_qp_init_attr_ex qp_attr_ex = {
322 .comp_mask = IBV_QP_INIT_ATTR_PD |
323 IBV_QP_INIT_ATTR_RX_HASH |
324 IBV_QP_INIT_ATTR_IND_TABLE,
325 .qp_type = IBV_QPT_RAW_PACKET,
326 .pd = priv->ib_parent_pd,
327 .rwq_ind_tbl = priv->ind_table,
328 .rx_hash_conf = {
329 .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ,
330 .rx_hash_key_len = TOEPLITZ_HASH_KEY_SIZE_IN_BYTES,
331 .rx_hash_key = mana_rss_hash_key_default,
332 .rx_hash_fields_mask =
333 IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4,
334 },
335
336 };
337
338 /* overwrite default if rss key is set */
339 if (priv->rss_conf.rss_key_len && priv->rss_conf.rss_key)
340 qp_attr_ex.rx_hash_conf.rx_hash_key =
341 priv->rss_conf.rss_key;
342
343 /* overwrite default if rss hash fields are set */
344 if (priv->rss_conf.rss_hf) {
345 qp_attr_ex.rx_hash_conf.rx_hash_fields_mask = 0;
346
347 if (priv->rss_conf.rss_hf & RTE_ETH_RSS_IPV4)
348 qp_attr_ex.rx_hash_conf.rx_hash_fields_mask |=
349 IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4;
350
351 if (priv->rss_conf.rss_hf & RTE_ETH_RSS_IPV6)
352 qp_attr_ex.rx_hash_conf.rx_hash_fields_mask |=
353 IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_SRC_IPV6;
354
355 if (priv->rss_conf.rss_hf &
356 (RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_NONFRAG_IPV6_TCP))
357 qp_attr_ex.rx_hash_conf.rx_hash_fields_mask |=
358 IBV_RX_HASH_SRC_PORT_TCP |
359 IBV_RX_HASH_DST_PORT_TCP;
360
361 if (priv->rss_conf.rss_hf &
362 (RTE_ETH_RSS_NONFRAG_IPV4_UDP | RTE_ETH_RSS_NONFRAG_IPV6_UDP))
363 qp_attr_ex.rx_hash_conf.rx_hash_fields_mask |=
364 IBV_RX_HASH_SRC_PORT_UDP |
365 IBV_RX_HASH_DST_PORT_UDP;
366 }
367
368 priv->rwq_qp = ibv_create_qp_ex(priv->ib_ctx, &qp_attr_ex);
369 if (!priv->rwq_qp) {
370 ret = -errno;
371 DRV_LOG(ERR, "rx ibv_create_qp_ex failed");
372 goto fail;
373 }
374
375 for (i = 0; i < priv->num_queues; i++) {
376 struct mana_rxq *rxq = dev->data->rx_queues[i];
377 struct manadv_obj obj = {};
378 struct manadv_cq dv_cq;
379 struct manadv_rwq dv_wq;
380
381 obj.cq.in = rxq->cq;
382 obj.cq.out = &dv_cq;
383 obj.rwq.in = rxq->wq;
384 obj.rwq.out = &dv_wq;
385 ret = manadv_init_obj(&obj, MANADV_OBJ_CQ | MANADV_OBJ_RWQ);
386 if (ret) {
387 DRV_LOG(ERR, "manadv_init_obj failed ret %d", ret);
388 goto fail;
389 }
390
391 rxq->gdma_cq.buffer = obj.cq.out->buf;
392 rxq->gdma_cq.count = obj.cq.out->count;
393 rxq->gdma_cq.size = rxq->gdma_cq.count * COMP_ENTRY_SIZE;
394 rxq->gdma_cq.id = obj.cq.out->cq_id;
395
396 /* CQ head starts with count */
397 rxq->gdma_cq.head = rxq->gdma_cq.count;
398
399 DRV_LOG(INFO, "rxq cq id %u buf %p count %u size %u",
400 rxq->gdma_cq.id, rxq->gdma_cq.buffer,
401 rxq->gdma_cq.count, rxq->gdma_cq.size);
402
403 priv->db_page = obj.rwq.out->db_page;
404
405 rxq->gdma_rq.buffer = obj.rwq.out->buf;
406 rxq->gdma_rq.count = obj.rwq.out->count;
407 rxq->gdma_rq.size = obj.rwq.out->size;
408 rxq->gdma_rq.id = obj.rwq.out->wq_id;
409
410 DRV_LOG(INFO, "rxq rq id %u buf %p count %u size %u",
411 rxq->gdma_rq.id, rxq->gdma_rq.buffer,
412 rxq->gdma_rq.count, rxq->gdma_rq.size);
413
414 rxq->comp_buf_len = 0;
415 rxq->comp_buf_idx = 0;
416 rxq->backlog_idx = 0;
417 }
418
419 for (i = 0; i < priv->num_queues; i++) {
420 struct mana_rxq *rxq = dev->data->rx_queues[i];
421
422 ret = mana_alloc_and_post_rx_wqes(rxq, rxq->num_desc);
423 if (ret)
424 goto fail;
425 }
426
427 for (i = 0; i < priv->num_queues; i++)
428 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
429
430 return 0;
431
432 fail:
433 mana_stop_rx_queues(dev);
434 return ret;
435 }
436
437 uint16_t
mana_rx_burst(void * dpdk_rxq,struct rte_mbuf ** pkts,uint16_t pkts_n)438 mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
439 {
440 uint16_t pkt_received = 0;
441 uint16_t wqe_consumed = 0;
442 struct mana_rxq *rxq = dpdk_rxq;
443 struct mana_priv *priv = rxq->priv;
444 struct rte_mbuf *mbuf;
445 int ret;
446 uint32_t pkt_idx = rxq->backlog_idx;
447 uint32_t pkt_len;
448 uint32_t i;
449 int polled = 0;
450
451 #ifdef RTE_ARCH_32
452 rxq->wqe_cnt_to_short_db = 0;
453 #endif
454
455 repoll:
456 /* Polling on new completions if we have no backlog */
457 if (rxq->comp_buf_idx == rxq->comp_buf_len) {
458 RTE_ASSERT(!pkt_idx);
459 rxq->comp_buf_len =
460 gdma_poll_completion_queue(&rxq->gdma_cq,
461 rxq->gdma_comp_buf, pkts_n);
462 rxq->comp_buf_idx = 0;
463 polled = 1;
464 }
465
466 i = rxq->comp_buf_idx;
467 while (i < rxq->comp_buf_len) {
468 struct mana_rx_comp_oob *oob = (struct mana_rx_comp_oob *)
469 rxq->gdma_comp_buf[i].cqe_data;
470 struct mana_rxq_desc *desc =
471 &rxq->desc_ring[rxq->desc_ring_tail];
472
473 mbuf = desc->pkt;
474
475 switch (oob->cqe_hdr.cqe_type) {
476 case CQE_RX_OKAY:
477 case CQE_RX_COALESCED_4:
478 /* Proceed to process mbuf */
479 break;
480
481 case CQE_RX_TRUNCATED:
482 default:
483 DP_LOG(ERR, "RX CQE type %d client %d vendor %d",
484 oob->cqe_hdr.cqe_type, oob->cqe_hdr.client_type,
485 oob->cqe_hdr.vendor_err);
486
487 rxq->stats.errors++;
488 rte_pktmbuf_free(mbuf);
489
490 i++;
491 goto drop;
492 }
493
494 DP_LOG(DEBUG, "mana_rx_comp_oob type %d rxq %p",
495 oob->cqe_hdr.cqe_type, rxq);
496
497 pkt_len = oob->packet_info[pkt_idx].packet_length;
498 if (!pkt_len) {
499 /* Move on to the next completion */
500 pkt_idx = 0;
501 i++;
502 continue;
503 }
504
505 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
506 mbuf->nb_segs = 1;
507 mbuf->next = NULL;
508 mbuf->data_len = pkt_len;
509 mbuf->pkt_len = pkt_len;
510 mbuf->port = priv->port_id;
511
512 if (oob->rx_ip_header_checksum_succeeded)
513 mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD;
514
515 if (oob->rx_ip_header_checksum_failed)
516 mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD;
517
518 if (oob->rx_outer_ip_header_checksum_failed)
519 mbuf->ol_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD;
520
521 if (oob->rx_tcp_checksum_succeeded ||
522 oob->rx_udp_checksum_succeeded)
523 mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD;
524
525 if (oob->rx_tcp_checksum_failed ||
526 oob->rx_udp_checksum_failed)
527 mbuf->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD;
528
529 if (oob->rx_hash_type == MANA_HASH_L3 ||
530 oob->rx_hash_type == MANA_HASH_L4) {
531 mbuf->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
532 mbuf->hash.rss = oob->packet_info[pkt_idx].packet_hash;
533 }
534
535 pkt_idx++;
536 /* Move on the next completion if all packets are processed */
537 if (pkt_idx >= RX_COM_OOB_NUM_PACKETINFO_SEGMENTS) {
538 pkt_idx = 0;
539 i++;
540 }
541
542 if (oob->rx_vlan_tag_present) {
543 mbuf->ol_flags |=
544 RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
545 mbuf->vlan_tci = oob->rx_vlan_id;
546
547 if (!priv->vlan_strip && rte_vlan_insert(&mbuf)) {
548 DRV_LOG(ERR, "vlan insert failed");
549 rxq->stats.errors++;
550 rte_pktmbuf_free(mbuf);
551
552 goto drop;
553 }
554 }
555
556 pkts[pkt_received++] = mbuf;
557 rxq->stats.packets++;
558 rxq->stats.bytes += mbuf->data_len;
559
560 drop:
561 rxq->desc_ring_tail++;
562 if (rxq->desc_ring_tail >= rxq->num_desc)
563 rxq->desc_ring_tail = 0;
564
565 rxq->gdma_rq.tail += desc->wqe_size_in_bu;
566
567 /* Record the number of the RX WQE we need to post to replenish
568 * consumed RX requests
569 */
570 wqe_consumed++;
571 if (pkt_received == pkts_n)
572 break;
573
574 #ifdef RTE_ARCH_32
575 /* Always post WQE as soon as it's consumed for short DB */
576 ret = mana_alloc_and_post_rx_wqes(rxq, wqe_consumed);
577 if (ret) {
578 DRV_LOG(ERR, "failed to post %d WQEs, ret %d",
579 wqe_consumed, ret);
580 return pkt_received;
581 }
582 wqe_consumed = 0;
583
584 /* Ring short doorbell if approaching the wqe increment
585 * limit.
586 */
587 if (rxq->wqe_cnt_to_short_db > RX_WQE_SHORT_DB_THRESHOLD) {
588 mana_rq_ring_doorbell(rxq);
589 rxq->wqe_cnt_to_short_db = 0;
590 }
591 #endif
592 }
593
594 rxq->backlog_idx = pkt_idx;
595 rxq->comp_buf_idx = i;
596
597 /* If all CQEs are processed but there are more packets to read, poll the
598 * completion queue again because we may have not polled on the completion
599 * queue due to CQE not fully processed in the previous rx_burst
600 */
601 if (pkt_received < pkts_n && !polled) {
602 polled = 1;
603 goto repoll;
604 }
605
606 if (wqe_consumed) {
607 ret = mana_alloc_and_post_rx_wqes(rxq, wqe_consumed);
608 if (ret)
609 DRV_LOG(ERR, "failed to post %d WQEs, ret %d",
610 wqe_consumed, ret);
611 }
612
613 return pkt_received;
614 }
615
616 #ifdef RTE_ARCH_32
617 static int
mana_arm_cq(struct mana_rxq * rxq __rte_unused,uint8_t arm __rte_unused)618 mana_arm_cq(struct mana_rxq *rxq __rte_unused, uint8_t arm __rte_unused)
619 {
620 DP_LOG(ERR, "Do not support in 32 bit");
621
622 return -ENODEV;
623 }
624 #else
625 static int
mana_arm_cq(struct mana_rxq * rxq,uint8_t arm)626 mana_arm_cq(struct mana_rxq *rxq, uint8_t arm)
627 {
628 struct mana_priv *priv = rxq->priv;
629 uint32_t head = rxq->gdma_cq.head %
630 (rxq->gdma_cq.count << COMPLETION_QUEUE_ENTRY_OWNER_BITS_SIZE);
631
632 DP_LOG(DEBUG, "Ringing completion queue ID %u head %u arm %d",
633 rxq->gdma_cq.id, head, arm);
634
635 return mana_ring_doorbell(priv->db_page, GDMA_QUEUE_COMPLETION,
636 rxq->gdma_cq.id, head, arm);
637 }
638 #endif
639
640 int
mana_rx_intr_enable(struct rte_eth_dev * dev,uint16_t rx_queue_id)641 mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
642 {
643 struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id];
644
645 return mana_arm_cq(rxq, 1);
646 }
647
648 int
mana_rx_intr_disable(struct rte_eth_dev * dev,uint16_t rx_queue_id)649 mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
650 {
651 struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id];
652 struct ibv_cq *ev_cq;
653 void *ev_ctx;
654 int ret;
655
656 ret = ibv_get_cq_event(rxq->channel, &ev_cq, &ev_ctx);
657 if (ret)
658 ret = errno;
659 else if (ev_cq != rxq->cq)
660 ret = EINVAL;
661
662 if (ret) {
663 if (ret != EAGAIN)
664 DP_LOG(ERR, "Can't disable RX intr queue %d",
665 rx_queue_id);
666 } else {
667 ibv_ack_cq_events(rxq->cq, 1);
668 }
669
670 return -ret;
671 }
672