Lines Matching +full:supports +full:- +full:cqe
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
104 ifmr->ifm_status = IFM_AVALID;
105 ifmr->ifm_active = IFM_ETHER;
107 if (!apc->port_is_up) {
109 mana_dbg(NULL, "Port %u link is down\n", apc->port_idx);
113 ifmr->ifm_status |= IFM_ACTIVE;
114 ifmr->ifm_active |= IFM_100G_DR | IFM_FDX;
123 struct mana_port_stats *stats = &apc->port_stats;
127 return (counter_u64_fetch(stats->rx_packets));
129 return (counter_u64_fetch(stats->tx_packets));
131 return (counter_u64_fetch(stats->rx_bytes));
133 return (counter_u64_fetch(stats->tx_bytes));
135 return (counter_u64_fetch(stats->rx_drops));
137 return (counter_u64_fetch(stats->tx_drops));
155 if (apc->port_is_up)
177 new_mtu = ifr->ifr_mtu;
184 new_mtu, MAX_FRAME_SIZE - 18, MIN_FRAME_SIZE - 18);
188 if (apc->port_is_up)
191 apc->frame_size = new_mtu + 18;
203 if (!apc->port_is_up)
210 if (apc->port_is_up)
224 mask = (ifr->ifr_reqcap & if_getcapabilities(ifp)) ^
237 "Also disabled tso4 due to -txcsum.\n");
251 "Also disabled tso6 due to -txcsum6.\n");
296 rc = ifmedia_ioctl(ifp, ifr, &apc->media, command);
301 ifrk->ifrk_func = RSS_FUNC_TOEPLITZ;
302 ifrk->ifrk_keylen = MANA_HASH_KEY_SIZE;
303 memcpy(ifrk->ifrk_key, apc->hashkey, MANA_HASH_KEY_SIZE);
308 ifrh->ifrh_func = RSS_FUNC_TOEPLITZ;
309 ifrh->ifrh_types =
354 struct gdma_dev *gd = apc->ac->gdma_dev;
359 err = bus_dmamap_load_mbuf_sg(apc->tx_buf_tag, tx_info->dma_map,
364 counter_u64_add(tx_stats->collapse, 1);
367 counter_u64_add(tx_stats->collapse_err, 1);
376 err = bus_dmamap_load_mbuf_sg(apc->tx_buf_tag,
377 tx_info->dma_map, m, segs, &nsegs, BUS_DMA_NOWAIT);
381 tp->wqe_req.sgl[i].address = segs[i].ds_addr;
382 tp->wqe_req.sgl[i].mem_key = gd->gpa_mkey;
383 tp->wqe_req.sgl[i].size = segs[i].ds_len;
385 tp->wqe_req.num_sge = nsegs;
387 tx_info->mbuf = *m_head;
389 bus_dmamap_sync(apc->tx_buf_tag, tx_info->dma_map,
400 bus_dmamap_sync(apc->tx_buf_tag, tx_info->dma_map,
402 bus_dmamap_unload(apc->tx_buf_tag, tx_info->dma_map);
403 if (tx_info->mbuf) {
404 m_freem(tx_info->mbuf);
405 tx_info->mbuf = NULL;
419 mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rxq->datasize);
427 mlen = rxq->datasize;
430 mbuf->m_pkthdr.len = mbuf->m_len = mlen;
432 if (rx_oob->mbuf) {
433 mbuf = rx_oob->mbuf;
434 mlen = rx_oob->mbuf->m_pkthdr.len;
440 err = bus_dmamap_load_mbuf_sg(apc->rx_buf_tag, rx_oob->dma_map,
446 counter_u64_add(rxq->stats.dma_mapping_err, 1);
450 bus_dmamap_sync(apc->rx_buf_tag, rx_oob->dma_map,
453 rx_oob->mbuf = mbuf;
454 rx_oob->num_sge = 1;
455 rx_oob->sgl[0].address = segs[0].ds_addr;
456 rx_oob->sgl[0].size = mlen;
457 rx_oob->sgl[0].mem_key = apc->ac->gdma_dev->gpa_mkey;
470 bus_dmamap_sync(apc->rx_buf_tag, rx_oob->dma_map,
472 bus_dmamap_unload(apc->rx_buf_tag, rx_oob->dma_map);
474 if (free_mbuf && rx_oob->mbuf) {
475 m_freem(rx_oob->mbuf);
476 rx_oob->mbuf = NULL;
482 #define MANA_L3_PROTO(_mbuf) ((_mbuf)->m_pkthdr.PH_loc.sixteen[0])
483 #define MANA_L4_PROTO(_mbuf) ((_mbuf)->m_pkthdr.PH_loc.sixteen[1])
492 if_t ndev = txq->ndev;
495 struct mana_port_stats *port_stats = &apc->port_stats;
496 struct gdma_dev *gd = apc->ac->gdma_dev;
506 gdma_sq = txq->gdma_sq;
507 cq = &apc->tx_qp[txq->idx].tx_cq;
508 tx_stats = &txq->stats;
512 next_to_use = txq->next_to_use;
514 while ((mbuf = drbr_peek(ndev, txq->txq_br)) != NULL) {
515 if (!apc->port_is_up ||
517 drbr_putback(ndev, txq->txq_br, mbuf);
523 if_setdrvflagbits(apc->ndev, IFF_DRV_OACTIVE, 0);
524 counter_u64_add(tx_stats->stop, 1);
525 uint64_t stops = counter_u64_fetch(tx_stats->stop);
526 uint64_t wakeups = counter_u64_fetch(tx_stats->wakeup);
529 stops > wakeups && txq->alt_txq_idx == txq->idx) {
530 txq->alt_txq_idx =
531 (txq->idx + (stops / wakeups))
532 % apc->num_queues;
533 counter_u64_add(tx_stats->alt_chg, 1);
536 drbr_putback(ndev, txq->txq_br, mbuf);
538 taskqueue_enqueue(cq->cleanup_tq, &cq->cleanup_task);
542 tx_info = &txq->tx_buf_info[next_to_use];
552 counter_u64_add(tx_stats->dma_mapping_err, 1);
557 drbr_advance(ndev, txq->txq_br);
561 pkg.tx_oob.s_oob.vcq_num = cq->gdma_id;
562 pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame;
564 if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) {
565 pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset;
568 pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset;
583 if (mbuf->m_pkthdr.csum_flags & CSUM_TSO) {
593 pkg.tx_oob.s_oob.trans_off = mbuf->m_pkthdr.l3hlen;
595 pkg.wqe_req.client_data_unit = mbuf->m_pkthdr.tso_segsz;
597 } else if (mbuf->m_pkthdr.csum_flags &
609 mbuf->m_pkthdr.l3hlen;
613 } else if (mbuf->m_pkthdr.csum_flags & CSUM_IP) {
623 len = mbuf->m_pkthdr.len;
626 (struct gdma_posted_wqe_info *)&tx_info->wqe_inf);
633 drbr_advance(ndev, txq->txq_br);
640 (void)atomic_inc_return(&txq->pending_sends);
642 drbr_advance(ndev, txq->txq_br);
644 mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq);
650 txq->tso_pkts++;
651 txq->tso_bytes += len;
656 counter_u64_add_protected(tx_stats->packets, packets);
657 counter_u64_add_protected(port_stats->tx_packets, packets);
658 counter_u64_add_protected(tx_stats->bytes, bytes);
659 counter_u64_add_protected(port_stats->tx_bytes, bytes);
662 txq->next_to_use = next_to_use;
669 if_t ndev = txq->ndev;
672 while (!drbr_empty(ndev, txq->txq_br) && apc->port_is_up &&
674 mtx_lock(&txq->txq_mtx);
676 mtx_unlock(&txq->txq_mtx);
682 if (unlikely((m)->m_len < (len))) { \
700 if (eh->evl_encap_proto == ntohs(ETHERTYPE_VLAN)) {
701 etype = ntohs(eh->evl_proto);
704 etype = ntohs(eh->evl_encap_proto);
714 iphlen = ip->ip_hl << 2;
715 mbuf->m_pkthdr.l3hlen = ehlen + iphlen;
720 ip->ip_len = 0;
721 ip->ip_sum = 0;
722 th->th_sum = in_pseudo(ip->ip_src.s_addr,
723 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
729 if (ip6->ip6_nxt != IPPROTO_TCP) {
735 mbuf->m_pkthdr.l3hlen = ehlen + sizeof(*ip6);
739 ip6->ip6_plen = 0;
740 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
765 if (eh->evl_encap_proto == ntohs(ETHERTYPE_VLAN)) {
766 etype = ntohs(eh->evl_proto);
769 etype = ntohs(eh->evl_encap_proto);
781 iphlen = ip->ip_hl << 2;
782 mbuf->m_pkthdr.l3hlen = ehlen + iphlen;
784 MANA_L4_PROTO(mbuf) = ip->ip_p;
789 mbuf->m_pkthdr.l3hlen = ehlen + sizeof(*ip6);
791 MANA_L4_PROTO(mbuf) = ip6->ip6_nxt;
810 if (unlikely((!apc->port_is_up) ||
814 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
818 counter_u64_add_protected(apc->port_stats.tx_drops, 1);
826 counter_u64_add_protected(apc->port_stats.tx_drops, 1);
833 uint32_t hash = m->m_pkthdr.flowid;
834 txq_id = apc->indir_table[(hash) & MANA_INDIRECT_TABLE_MASK] %
835 apc->num_queues;
837 txq_id = m->m_pkthdr.flowid % apc->num_queues;
840 if (apc->enable_tx_altq)
841 txq_id = apc->tx_qp[txq_id].txq.alt_txq_idx;
843 txq = &apc->tx_qp[txq_id].txq;
845 is_drbr_empty = drbr_empty(ifp, txq->txq_br);
846 err = drbr_enqueue(ifp, txq->txq_br, m);
850 taskqueue_enqueue(txq->enqueue_tq, &txq->enqueue_task);
854 if (is_drbr_empty && mtx_trylock(&txq->txq_mtx)) {
856 mtx_unlock(&txq->txq_mtx);
858 taskqueue_enqueue(txq->enqueue_tq, &txq->enqueue_task);
867 bus_dma_tag_destroy(apc->tx_buf_tag);
868 bus_dma_tag_destroy(apc->rx_buf_tag);
869 apc->rx_buf_tag = NULL;
871 free(apc->rxqs, M_DEVBUF);
872 apc->rxqs = NULL;
874 mana_free_counters((counter_u64_t *)&apc->port_stats,
881 device_t dev = apc->ac->gdma_dev->gdma_context->dev;
898 &apc->tx_buf_tag);
915 &apc->rx_buf_tag);
921 apc->rxqs = mallocarray(apc->num_queues, sizeof(struct mana_rxq *),
931 struct gdma_context *gc = ac->gdma_dev->gdma_context;
934 device_t dev = gc->dev;
938 req->dev_id = gc->mana.dev_id;
939 req->activity_id = atomic_inc_return(&activity_id);
945 if (err || resp->status) {
947 err, resp->status);
951 if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 ||
952 req->activity_id != resp->activity_id) {
955 req->dev_id.as_uint32, resp->dev_id.as_uint32,
956 req->activity_id, resp->activity_id);
968 if (resp_hdr->response.msg_type != expected_code)
971 if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1)
974 if (resp_hdr->response.msg_size < min_size)
985 struct gdma_context *gc = ac->gdma_dev->gdma_context;
988 device_t dev = gc->dev;
1034 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1051 apc->port_handle = resp.vport;
1052 memcpy(apc->mac_addr, resp.mac_addr, ETHER_ADDR_LEN);
1060 apc->vport_use_count--;
1061 if (apc->vport_use_count < 0) {
1064 apc->vport_use_count);
1079 * For Ethernet usage, the hardware supports only one active user on a
1094 if (apc->vport_use_count > 0) {
1097 apc->vport_use_count++;
1101 req.vport = apc->port_handle;
1105 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1108 if_printf(apc->ndev, "Failed to configure vPort: %d\n", err);
1115 if_printf(apc->ndev, "Failed to configure vPort: %d, 0x%x\n",
1123 apc->tx_shortform_allowed = resp.short_form_allowed;
1124 apc->tx_vp_offset = resp.tx_vport_offset;
1126 if_printf(apc->ndev, "Configured vPort %ju PD %u DB %u\n",
1127 apc->port_handle, protection_dom_id, doorbell_pg_id);
1145 if_t ndev = apc->ndev;
1153 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size,
1156 req->vport = apc->port_handle;
1157 req->num_indir_entries = num_entries;
1158 req->indir_tab_offset = sizeof(*req);
1159 req->rx_enable = rx;
1160 req->rss_enable = apc->rss_state;
1161 req->update_default_rxobj = update_default_rxobj;
1162 req->update_hashkey = update_key;
1163 req->update_indir_tab = update_tab;
1164 req->default_rxobj = apc->default_rxobj;
1167 memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE);
1171 memcpy(req_indir_tab, apc->rxobj_table,
1172 req->num_indir_entries * sizeof(mana_handle_t));
1175 err = mana_send_request(apc->ac, req, req_buf_size, &resp,
1196 apc->port_handle, num_entries);
1212 if_t ndev = apc->ndev;
1219 req.wq_gdma_region = wq_spec->gdma_region;
1220 req.cq_gdma_region = cq_spec->gdma_region;
1221 req.wq_size = wq_spec->queue_size;
1222 req.cq_size = cq_spec->queue_size;
1223 req.cq_moderation_ctx_id = cq_spec->modr_ctx_id;
1224 req.cq_parent_qid = cq_spec->attached_eq;
1226 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1250 wq_spec->queue_index = resp.wq_id;
1251 cq_spec->queue_index = resp.cq_id;
1264 if_t ndev = apc->ndev;
1272 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1289 struct gdma_context *gc = ac->gdma_dev->gdma_context;
1293 if (!ac->eqs)
1296 for (i = 0; i < gc->max_num_queues; i++) {
1297 eq = ac->eqs[i].eq;
1304 free(ac->eqs, M_DEVBUF);
1305 ac->eqs = NULL;
1311 struct gdma_dev *gd = ac->gdma_dev;
1312 struct gdma_context *gc = gd->gdma_context;
1317 ac->eqs = mallocarray(gc->max_num_queues, sizeof(struct mana_eq),
1324 spec.eq.context = ac->eqs;
1327 for (i = 0; i < gc->max_num_queues; i++) {
1328 err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq);
1346 init_completion(&rxq->fence_event);
1350 req.wq_obj_handle = rxq->rxobj;
1352 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1355 if_printf(apc->ndev, "Failed to fence RQ %u: %d\n",
1356 rxq->rxq_idx, err);
1362 if_printf(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n",
1363 rxq->rxq_idx, err, resp.hdr.status);
1370 if (wait_for_completion_timeout(&rxq->fence_event, 10 * hz)) {
1371 if_printf(apc->ndev, "Failed to fence RQ %u: timed out\n",
1372 rxq->rxq_idx);
1386 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
1387 rxq = apc->rxqs[rxq_idx];
1402 used_space_old = wq->head - wq->tail;
1403 used_space_new = wq->head - (wq->tail + num_units);
1412 wq->tail += num_units;
1419 struct gdma_comp *completions = cq->gdma_comp_buf;
1424 struct mana_txq *txq = cq->txq;
1429 int txq_idx = txq->idx;
1437 ndev = txq->ndev;
1440 comp_read = mana_gd_poll_cq(cq->gdma_cq, completions,
1446 next_to_complete = txq->next_to_complete;
1457 if (cqe_oob->cqe_hdr.client_type !=
1460 "WARNING: Invalid CQE client type %u\n",
1461 cqe_oob->cqe_hdr.client_type);
1465 switch (cqe_oob->cqe_hdr.cqe_type) {
1480 "TX: txq %d CQE error %d, ntc = %d, "
1482 txq_idx, cqe_oob->cqe_hdr.cqe_type,
1483 next_to_complete, txq->pending_sends);
1484 counter_u64_add(txq->stats.cqe_err, 1);
1488 /* If the CQE type is unknown, log a debug msg,
1492 "ERROR: TX: Unknown CQE type %d\n",
1493 cqe_oob->cqe_hdr.cqe_type);
1494 counter_u64_add(txq->stats.cqe_unknown_type, 1);
1497 if (txq->gdma_txq_id != completions[i].wq_num) {
1501 txq->gdma_txq_id, completions[i].wq_num);
1505 tx_info = &txq->tx_buf_info[next_to_complete];
1506 if (!tx_info->mbuf) {
1511 txq_idx, next_to_complete, txq->next_to_use,
1512 txq->pending_sends, pkt_transmitted, sa_drop,
1517 wqe_info = &tx_info->wqe_inf;
1518 wqe_unit_cnt += wqe_info->wqe_size_in_bu;
1529 txq->next_to_complete = next_to_complete;
1537 mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt);
1542 gdma_wq = txq->gdma_sq;
1550 /* Ensure checking txq_full before apc->port_is_up. */
1553 if (txq_full && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) {
1554 /* Grab the txq lock and re-test */
1555 mtx_lock(&txq->txq_mtx);
1559 apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) {
1561 if_setdrvflagbits(apc->ndev, IFF_DRV_RUNNING,
1563 counter_u64_add(txq->stats.wakeup, 1);
1564 if (txq->alt_txq_idx != txq->idx) {
1565 uint64_t stops = counter_u64_fetch(txq->stats.stop);
1566 uint64_t wakeups = counter_u64_fetch(txq->stats.wakeup);
1569 txq->alt_txq_idx = txq->idx;
1570 counter_u64_add(txq->stats.alt_reset, 1);
1575 taskqueue_enqueue(txq->enqueue_tq, &txq->enqueue_task);
1577 mtx_unlock(&txq->txq_mtx);
1580 if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0)
1583 txq->idx, txq->pending_sends);
1585 cq->work_done = pkt_transmitted;
1595 curr_index = rxq->buf_index++;
1596 if (rxq->buf_index == rxq->num_rx_buf)
1597 rxq->buf_index = 0;
1599 recv_buf_oob = &rxq->rx_oobs[curr_index];
1601 err = mana_gd_post_work_request(rxq->gdma_rq, &recv_buf_oob->wqe_req,
1602 &recv_buf_oob->wqe_inf);
1605 rxq->rxq_idx, err);
1609 if (recv_buf_oob->wqe_inf.wqe_size_in_bu != 1) {
1611 rxq->rxq_idx, recv_buf_oob->wqe_inf.wqe_size_in_bu);
1616 mana_rx_mbuf(struct mbuf *mbuf, struct mana_rxcomp_oob *cqe,
1619 struct mana_stats *rx_stats = &rxq->stats;
1620 if_t ndev = rxq->ndev;
1621 uint32_t pkt_len = cqe->ppi[0].pkt_len;
1622 uint16_t rxq_idx = rxq->rxq_idx;
1628 rxq->rx_cq.work_done++;
1634 mbuf->m_flags |= M_PKTHDR;
1635 mbuf->m_pkthdr.len = pkt_len;
1636 mbuf->m_len = pkt_len;
1637 mbuf->m_pkthdr.rcvif = ndev;
1641 (cqe->rx_iphdr_csum_succeed)) {
1642 mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
1643 mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID;
1644 if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed) {
1645 mbuf->m_pkthdr.csum_flags |=
1647 mbuf->m_pkthdr.csum_data = 0xffff;
1649 if (cqe->rx_tcp_csum_succeed)
1654 if (cqe->rx_hashtype != 0) {
1655 mbuf->m_pkthdr.flowid = cqe->ppi[0].pkt_hash;
1657 uint16_t hashtype = cqe->rx_hashtype;
1694 mbuf->m_pkthdr.flowid = rxq_idx;
1700 rxq->lro_tried++;
1701 if (rxq->lro.lro_cnt != 0 &&
1702 tcp_lro_rx(&rxq->lro, mbuf, 0) == 0)
1705 rxq->lro_failed++;
1712 counter_u64_add_protected(rx_stats->packets, 1);
1713 counter_u64_add_protected(apc->port_stats.rx_packets, 1);
1714 counter_u64_add_protected(rx_stats->bytes, pkt_len);
1715 counter_u64_add_protected(apc->port_stats.rx_bytes, pkt_len);
1721 struct gdma_comp *cqe)
1723 struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data;
1725 if_t ndev = rxq->ndev;
1731 switch (oob->cqe_hdr.cqe_type) {
1737 counter_u64_add(apc->port_stats.rx_drops, 1);
1738 rxbuf_oob = &rxq->rx_oobs[rxq->buf_index];
1747 complete(&rxq->fence_event);
1751 if_printf(ndev, "Unknown RX CQE type = %d\n",
1752 oob->cqe_hdr.cqe_type);
1756 if (oob->cqe_hdr.cqe_type != CQE_RX_OKAY)
1759 pktlen = oob->ppi[0].pkt_len;
1764 rxq->gdma_id, cq->gdma_id, rxq->rxobj);
1768 curr = rxq->buf_index;
1769 rxbuf_oob = &rxq->rx_oobs[curr];
1770 if (rxbuf_oob->wqe_inf.wqe_size_in_bu != 1) {
1773 rxbuf_oob->wqe_inf.wqe_size_in_bu);
1778 old_mbuf = rxbuf_oob->mbuf;
1789 counter_u64_add(rxq->stats.mbuf_alloc_fail, 1);
1791 * Failed to load new mbuf, rxbuf_oob->mbuf is still
1802 mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu);
1810 struct gdma_comp *comp = cq->gdma_comp_buf;
1813 comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER);
1821 "WARNING: CQE not for receive queue\n");
1825 /* verify recv cqe references the right rxq */
1826 if (comp[i].wq_num != cq->rxq->gdma_id) {
1828 "WARNING: Received CQE %d not for "
1830 comp[i].wq_num, cq->rxq->gdma_id);
1834 mana_process_rx_cqe(cq->rxq, cq, &comp[i]);
1839 cq->rxq->gdma_rq->gdma_dev->gdma_context;
1841 mana_gd_wq_ring_doorbell(gc, cq->rxq->gdma_rq);
1844 tcp_lro_flush_all(&cq->rxq->lro);
1853 KASSERT(cq->gdma_cq == gdma_queue,
1854 ("cq do not match %p, %p", cq->gdma_cq, gdma_queue));
1856 if (cq->type == MANA_CQ_TYPE_RX) {
1862 if (cq->work_done < cq->budget && cq->do_not_ring_db == false)
1880 cq->work_done = 0;
1881 if (cq->type == MANA_CQ_TYPE_RX) {
1882 cq->budget = MANA_RX_BUDGET;
1884 cq->budget = MANA_TX_BUDGET;
1892 if (i == (MANA_POLL_BUDGET - 1))
1893 cq->budget = CQE_POLLING_BUFFER + 1;
1895 mana_cq_handler(cq, cq->gdma_cq);
1897 if (cq->work_done < cq->budget)
1900 cq->work_done = 0;
1909 taskqueue_enqueue(cq->cleanup_tq, &cq->cleanup_task);
1915 struct gdma_dev *gd = apc->ac->gdma_dev;
1917 if (!cq->gdma_cq)
1921 if (cq->cleanup_tq) {
1922 while (taskqueue_cancel(cq->cleanup_tq,
1923 &cq->cleanup_task, NULL)) {
1924 taskqueue_drain(cq->cleanup_tq,
1925 &cq->cleanup_task);
1928 taskqueue_free(cq->cleanup_tq);
1931 mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq);
1937 struct gdma_dev *gd = apc->ac->gdma_dev;
1942 if (!txq->gdma_sq)
1945 if ((pending_sends = atomic_read(&txq->pending_sends)) > 0) {
1951 if (txq->next_to_use != txq->next_to_complete) {
1955 txq->next_to_use, txq->next_to_complete);
1959 if (txq->txq_br) {
1960 mtx_lock(&txq->txq_mtx);
1961 drbr_flush(apc->ndev, txq->txq_br);
1962 mtx_unlock(&txq->txq_mtx);
1963 buf_ring_free(txq->txq_br, M_DEVBUF);
1967 if (txq->enqueue_tq) {
1968 while (taskqueue_cancel(txq->enqueue_tq,
1969 &txq->enqueue_task, NULL)) {
1970 taskqueue_drain(txq->enqueue_tq,
1971 &txq->enqueue_task);
1974 taskqueue_free(txq->enqueue_tq);
1977 if (txq->tx_buf_info) {
1978 /* Free all mbufs which are still in-flight */
1980 txbuf_info = &txq->tx_buf_info[i];
1981 if (txbuf_info->mbuf) {
1986 free(txq->tx_buf_info, M_DEVBUF);
1989 mana_free_counters((counter_u64_t *)&txq->stats,
1990 sizeof(txq->stats));
1992 mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq);
1994 mtx_destroy(&txq->txq_mtx);
2002 if (!apc->tx_qp)
2005 for (i = 0; i < apc->num_queues; i++) {
2006 mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object);
2008 mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq);
2010 mana_deinit_txq(apc, &apc->tx_qp[i].txq);
2013 free(apc->tx_qp, M_DEVBUF);
2014 apc->tx_qp = NULL;
2020 struct mana_context *ac = apc->ac;
2021 struct gdma_dev *gd = ac->gdma_dev;
2033 apc->tx_qp = mallocarray(apc->num_queues, sizeof(struct mana_tx_qp),
2048 gc = gd->gdma_context;
2050 for (i = 0; i < apc->num_queues; i++) {
2051 apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE;
2054 txq = &apc->tx_qp[i].txq;
2056 txq->ndev = net;
2057 txq->vp_offset = apc->tx_vp_offset;
2058 txq->idx = i;
2059 txq->alt_txq_idx = i;
2065 err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq);
2070 cq = &apc->tx_qp[i].tx_cq;
2071 cq->type = MANA_CQ_TYPE_TX;
2073 cq->txq = txq;
2080 spec.cq.parent_eq = ac->eqs[i].eq;
2082 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq);
2089 wq_spec.gdma_region = txq->gdma_sq->mem_info.dma_region_handle;
2090 wq_spec.queue_size = txq->gdma_sq->queue_size;
2092 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle;
2093 cq_spec.queue_size = cq->gdma_cq->queue_size;
2095 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
2097 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ,
2098 &wq_spec, &cq_spec, &apc->tx_qp[i].tx_object);
2103 txq->gdma_sq->id = wq_spec.queue_index;
2104 cq->gdma_cq->id = cq_spec.queue_index;
2106 txq->gdma_sq->mem_info.dma_region_handle =
2108 cq->gdma_cq->mem_info.dma_region_handle =
2111 txq->gdma_txq_id = txq->gdma_sq->id;
2113 cq->gdma_id = cq->gdma_cq->id;
2117 i, txq->gdma_txq_id, cq->gdma_id);
2119 if (cq->gdma_id >= gc->max_num_cqs) {
2120 if_printf(net, "CQ id %u too large.\n", cq->gdma_id);
2125 gc->cq_table[cq->gdma_id] = cq->gdma_cq;
2128 txq->tx_buf_info = malloc(MAX_SEND_BUFFERS_PER_QUEUE *
2132 snprintf(txq->txq_mtx_name, nitems(txq->txq_mtx_name),
2134 mtx_init(&txq->txq_mtx, txq->txq_mtx_name, NULL, MTX_DEF);
2136 txq->txq_br = buf_ring_alloc(4 * MAX_SEND_BUFFERS_PER_QUEUE,
2137 M_DEVBUF, M_WAITOK, &txq->txq_mtx);
2140 TASK_INIT(&txq->enqueue_task, 0, mana_xmit_taskfunc, txq);
2141 txq->enqueue_tq = taskqueue_create_fast("mana_tx_enque",
2142 M_NOWAIT, taskqueue_thread_enqueue, &txq->enqueue_tq);
2143 if (unlikely(txq->enqueue_tq == NULL)) {
2149 taskqueue_start_threads(&txq->enqueue_tq, 1, PI_NET,
2150 "mana txq p%u-tx%d", apc->port_idx, i);
2152 mana_alloc_counters((counter_u64_t *)&txq->stats,
2153 sizeof(txq->stats));
2156 cq->do_not_ring_db = false;
2158 NET_TASK_INIT(&cq->cleanup_task, 0, mana_poll, cq);
2159 cq->cleanup_tq =
2162 &cq->cleanup_tq);
2164 if (apc->last_tx_cq_bind_cpu < 0)
2165 apc->last_tx_cq_bind_cpu = CPU_FIRST();
2166 cq->cpu = apc->last_tx_cq_bind_cpu;
2167 apc->last_tx_cq_bind_cpu = CPU_NEXT(apc->last_tx_cq_bind_cpu);
2169 if (apc->bind_cleanup_thread_cpu) {
2171 CPU_SETOF(cq->cpu, &cpu_mask);
2172 taskqueue_start_threads_cpuset(&cq->cleanup_tq,
2174 "mana cq p%u-tx%u-cpu%d",
2175 apc->port_idx, txq->idx, cq->cpu);
2177 taskqueue_start_threads(&cq->cleanup_tq, 1,
2178 PI_NET, "mana cq p%u-tx%u",
2179 apc->port_idx, txq->idx);
2182 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT);
2195 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
2209 mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj);
2211 mana_deinit_cq(apc, &rxq->rx_cq);
2213 mana_free_counters((counter_u64_t *)&rxq->stats,
2214 sizeof(rxq->stats));
2217 tcp_lro_free(&rxq->lro);
2219 for (i = 0; i < rxq->num_rx_buf; i++) {
2220 rx_oob = &rxq->rx_oobs[i];
2222 if (rx_oob->mbuf)
2225 bus_dmamap_destroy(apc->rx_buf_tag, rx_oob->dma_map);
2228 if (rxq->gdma_rq)
2229 mana_gd_destroy_queue(gc, rxq->gdma_rq);
2245 if (rxq->datasize == 0 || rxq->datasize > PAGE_SIZE) {
2247 "WARNING: Invalid rxq datasize %u\n", rxq->datasize);
2253 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) {
2254 rx_oob = &rxq->rx_oobs[buf_idx];
2257 err = bus_dmamap_create(apc->rx_buf_tag, 0,
2258 &rx_oob->dma_map);
2271 bus_dmamap_destroy(apc->rx_buf_tag, rx_oob->dma_map);
2275 rx_oob->wqe_req.sgl = rx_oob->sgl;
2276 rx_oob->wqe_req.num_sge = rx_oob->num_sge;
2277 rx_oob->wqe_req.inline_oob_size = 0;
2278 rx_oob->wqe_req.inline_oob_data = NULL;
2279 rx_oob->wqe_req.flags = 0;
2280 rx_oob->wqe_req.client_data_unit = 0;
2283 MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32);
2297 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) {
2298 rx_oob = &rxq->rx_oobs[buf_idx];
2300 err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req,
2301 &rx_oob->wqe_inf);
2313 struct gdma_dev *gd = apc->ac->gdma_dev;
2323 gc = gd->gdma_context;
2328 rxq->ndev = ndev;
2329 rxq->num_rx_buf = RX_BUFFERS_PER_QUEUE;
2330 rxq->rxq_idx = rxq_idx;
2335 rxq->datasize = ALIGN(apc->frame_size, MCLBYTES);
2336 if (rxq->datasize > MAX_FRAME_SIZE)
2337 rxq->datasize = MAX_FRAME_SIZE;
2340 rxq_idx, rxq->datasize);
2342 rxq->rxobj = INVALID_MANA_HANDLE;
2350 err = tcp_lro_init(&rxq->lro);
2355 rxq->lro.ifp = ndev;
2359 mana_alloc_counters((counter_u64_t *)&rxq->stats,
2360 sizeof(rxq->stats));
2370 err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq);
2375 cq = &rxq->rx_cq;
2376 cq->type = MANA_CQ_TYPE_RX;
2377 cq->rxq = rxq;
2384 spec.cq.parent_eq = eq->eq;
2386 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq);
2392 wq_spec.gdma_region = rxq->gdma_rq->mem_info.dma_region_handle;
2393 wq_spec.queue_size = rxq->gdma_rq->queue_size;
2395 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle;
2396 cq_spec.queue_size = cq->gdma_cq->queue_size;
2398 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
2400 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ,
2401 &wq_spec, &cq_spec, &rxq->rxobj);
2405 rxq->gdma_rq->id = wq_spec.queue_index;
2406 cq->gdma_cq->id = cq_spec.queue_index;
2408 rxq->gdma_rq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION;
2409 cq->gdma_cq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION;
2411 rxq->gdma_id = rxq->gdma_rq->id;
2412 cq->gdma_id = cq->gdma_cq->id;
2418 if (cq->gdma_id >= gc->max_num_cqs) {
2423 gc->cq_table[cq->gdma_id] = cq->gdma_cq;
2426 cq->do_not_ring_db = false;
2428 NET_TASK_INIT(&cq->cleanup_task, 0, mana_poll, cq);
2429 cq->cleanup_tq =
2432 &cq->cleanup_tq);
2434 if (apc->last_rx_cq_bind_cpu < 0)
2435 apc->last_rx_cq_bind_cpu = CPU_FIRST();
2436 cq->cpu = apc->last_rx_cq_bind_cpu;
2437 apc->last_rx_cq_bind_cpu = CPU_NEXT(apc->last_rx_cq_bind_cpu);
2439 if (apc->bind_cleanup_thread_cpu) {
2441 CPU_SETOF(cq->cpu, &cpu_mask);
2442 taskqueue_start_threads_cpuset(&cq->cleanup_tq,
2444 "mana cq p%u-rx%u-cpu%d",
2445 apc->port_idx, rxq->rxq_idx, cq->cpu);
2447 taskqueue_start_threads(&cq->cleanup_tq, 1,
2448 PI_NET, "mana cq p%u-rx%u",
2449 apc->port_idx, rxq->rxq_idx);
2452 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT);
2470 struct mana_context *ac = apc->ac;
2475 for (i = 0; i < apc->num_queues; i++) {
2476 rxq = mana_create_rxq(apc, i, &ac->eqs[i], ndev);
2482 apc->rxqs[i] = rxq;
2485 apc->default_rxobj = apc->rxqs[0]->rxobj;
2496 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
2497 rxq = apc->rxqs[rxq_idx];
2502 apc->rxqs[rxq_idx] = NULL;
2513 struct gdma_dev *gd = apc->ac->gdma_dev;
2516 apc->default_rxobj = INVALID_MANA_HANDLE;
2518 err = mana_cfg_vport(apc, gd->pdid, gd->doorbell);
2531 apc->indir_table[i] = i % apc->num_queues;
2543 queue_idx = apc->indir_table[i];
2544 apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj;
2562 int port_idx = apc->port_idx;
2579 if (apc->max_queues > max_queues)
2580 apc->max_queues = max_queues;
2582 if (apc->num_queues > apc->max_queues)
2583 apc->num_queues = apc->max_queues;
2588 bus_dma_tag_destroy(apc->rx_buf_tag);
2589 apc->rx_buf_tag = NULL;
2590 free(apc->rxqs, M_DEVBUF);
2591 apc->rxqs = NULL;
2609 apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE;
2631 err = mana_alloc_queues(apc->ndev);
2640 apc->port_is_up = true;
2645 if_link_state_change(apc->ndev, LINK_STATE_UP);
2646 if_setdrvflagbits(apc->ndev, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
2658 if (!apc->port_is_up) {
2671 if (apc->port_is_up)
2674 /* No packet can be transmitted now since apc->port_is_up is false.
2675 * There is still a tiny chance that mana_poll_tx_cq() can re-enable
2676 * a txq because it may not timely see apc->port_is_up being cleared
2678 * new packets due to apc->port_is_up being false.
2680 * Drain all the in-flight TX packets
2682 for (i = 0; i < apc->num_queues; i++) {
2683 txq = &apc->tx_qp[i].txq;
2685 struct mana_cq *tx_cq = &apc->tx_qp[i].tx_cq;
2686 struct mana_cq *rx_cq = &(apc->rxqs[i]->rx_cq);
2688 tx_cq->do_not_ring_db = true;
2689 rx_cq->do_not_ring_db = true;
2692 taskqueue_enqueue(tx_cq->cleanup_tq, &tx_cq->cleanup_task);
2694 while (atomic_read(&txq->pending_sends) > 0)
2702 apc->rss_state = TRI_STATE_FALSE;
2719 apc->port_st_save = apc->port_is_up;
2720 apc->port_is_up = false;
2725 if (apc->port_st_save) {
2726 if_setdrvflagbits(apc->ndev, IFF_DRV_OACTIVE,
2728 if_link_state_change(apc->ndev, LINK_STATE_DOWN);
2732 err = mana_dealloc_queues(apc->ndev);
2734 if_printf(apc->ndev,
2770 struct gdma_context *gc = ac->gdma_dev->gdma_context;
2776 ndev = if_alloc_dev(IFT_ETHER, gc->dev);
2780 apc->ac = ac;
2781 apc->ndev = ndev;
2782 apc->max_queues = gc->max_num_queues;
2783 apc->num_queues = min_t(unsigned int,
2784 gc->max_num_queues, MANA_MAX_NUM_QUEUES);
2785 apc->port_handle = INVALID_MANA_HANDLE;
2786 apc->port_idx = port_idx;
2787 apc->frame_size = DEFAULT_FRAME_SIZE;
2788 apc->last_tx_cq_bind_cpu = -1;
2789 apc->last_rx_cq_bind_cpu = -1;
2790 apc->vport_use_count = 0;
2794 if_initname(ndev, device_get_name(gc->dev), port_idx);
2795 if_setdev(ndev,gc->dev);
2808 mana_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE);
2824 if_sethwtsomax(ndev, MANA_TSO_MAX_SZ -
2839 ifmedia_init(&apc->media, IFM_IMASK,
2841 ifmedia_add(&apc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2842 ifmedia_set(&apc->media, IFM_ETHER | IFM_AUTO);
2844 ether_ifattach(ndev, apc->mac_addr);
2847 mana_alloc_counters((counter_u64_t *)&apc->port_stats,
2866 struct gdma_context *gc = gd->gdma_context;
2867 device_t dev = gc->dev;
2880 ac->gdma_dev = gd;
2881 ac->num_ports = 1;
2882 gd->driver_data = ac;
2889 MANA_MICRO_VERSION, &ac->num_ports);
2893 if (ac->num_ports > MAX_PORTS_IN_MANA_DEV)
2894 ac->num_ports = MAX_PORTS_IN_MANA_DEV;
2896 for (i = 0; i < ac->num_ports; i++) {
2897 err = mana_probe_port(ac, i, &ac->ports[i]);
2915 struct gdma_context *gc = gd->gdma_context;
2916 struct mana_context *ac = gd->driver_data;
2917 device_t dev = gc->dev;
2921 for (i = 0; i < ac->num_ports; i++) {
2922 ndev = ac->ports[i];
2938 gd->driver_data = NULL;
2939 gd->gdma_context = NULL;