1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Hisilicon Limited. 3 */ 4 5 #include <rte_io.h> 6 #include <rte_ethdev_driver.h> 7 8 #include "hns3_ethdev.h" 9 #include "hns3_rxtx.h" 10 #include "hns3_rxtx_vec.h" 11 12 #if defined RTE_ARCH_ARM64 13 #include "hns3_rxtx_vec_neon.h" 14 #endif 15 16 int 17 hns3_tx_check_vec_support(struct rte_eth_dev *dev) 18 { 19 struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode; 20 21 /* Only support DEV_TX_OFFLOAD_MBUF_FAST_FREE */ 22 if (txmode->offloads != DEV_TX_OFFLOAD_MBUF_FAST_FREE) 23 return -ENOTSUP; 24 25 return 0; 26 } 27 28 uint16_t 29 hns3_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 30 { 31 struct hns3_tx_queue *txq = (struct hns3_tx_queue *)tx_queue; 32 uint16_t nb_tx = 0; 33 34 while (nb_pkts) { 35 uint16_t ret, new_burst; 36 37 new_burst = RTE_MIN(nb_pkts, txq->tx_rs_thresh); 38 ret = hns3_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], 39 new_burst); 40 nb_tx += ret; 41 nb_pkts -= ret; 42 if (ret < new_burst) 43 break; 44 } 45 46 return nb_tx; 47 } 48 49 static inline void 50 hns3_rxq_rearm_mbuf(struct hns3_rx_queue *rxq) 51 { 52 #define REARM_LOOP_STEP_NUM 4 53 struct hns3_entry *rxep = &rxq->sw_ring[rxq->rx_rearm_start]; 54 struct hns3_desc *rxdp = rxq->rx_ring + rxq->rx_rearm_start; 55 uint64_t dma_addr; 56 int i; 57 58 if (unlikely(rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep, 59 HNS3_DEFAULT_RXQ_REARM_THRESH) < 0)) { 60 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++; 61 return; 62 } 63 64 for (i = 0; i < HNS3_DEFAULT_RXQ_REARM_THRESH; i += REARM_LOOP_STEP_NUM, 65 rxep += REARM_LOOP_STEP_NUM, rxdp += REARM_LOOP_STEP_NUM) { 66 if (likely(i < 67 HNS3_DEFAULT_RXQ_REARM_THRESH - REARM_LOOP_STEP_NUM)) { 68 rte_prefetch_non_temporal(rxep[4].mbuf); 69 rte_prefetch_non_temporal(rxep[5].mbuf); 70 rte_prefetch_non_temporal(rxep[6].mbuf); 71 rte_prefetch_non_temporal(rxep[7].mbuf); 72 } 73 74 dma_addr = rte_mbuf_data_iova_default(rxep[0].mbuf); 75 rxdp[0].addr = rte_cpu_to_le_64(dma_addr); 76 rxdp[0].rx.bd_base_info = 0; 77 78 dma_addr = rte_mbuf_data_iova_default(rxep[1].mbuf); 79 rxdp[1].addr = rte_cpu_to_le_64(dma_addr); 80 rxdp[1].rx.bd_base_info = 0; 81 82 dma_addr = rte_mbuf_data_iova_default(rxep[2].mbuf); 83 rxdp[2].addr = rte_cpu_to_le_64(dma_addr); 84 rxdp[2].rx.bd_base_info = 0; 85 86 dma_addr = rte_mbuf_data_iova_default(rxep[3].mbuf); 87 rxdp[3].addr = rte_cpu_to_le_64(dma_addr); 88 rxdp[3].rx.bd_base_info = 0; 89 } 90 91 rxq->rx_rearm_start += HNS3_DEFAULT_RXQ_REARM_THRESH; 92 if (rxq->rx_rearm_start >= rxq->nb_rx_desc) 93 rxq->rx_rearm_start = 0; 94 95 rxq->rx_rearm_nb -= HNS3_DEFAULT_RXQ_REARM_THRESH; 96 97 hns3_write_reg_opt(rxq->io_head_reg, HNS3_DEFAULT_RXQ_REARM_THRESH); 98 } 99 100 uint16_t 101 hns3_recv_pkts_vec(void *__restrict rx_queue, 102 struct rte_mbuf **__restrict rx_pkts, 103 uint16_t nb_pkts) 104 { 105 struct hns3_rx_queue *rxq = rx_queue; 106 struct hns3_desc *rxdp = &rxq->rx_ring[rxq->next_to_use]; 107 uint64_t bd_err_mask; /* bit mask indicate whick pkts is error */ 108 uint16_t nb_rx; 109 110 nb_pkts = RTE_MIN(nb_pkts, HNS3_DEFAULT_RX_BURST); 111 nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, HNS3_DEFAULT_DESCS_PER_LOOP); 112 113 rte_prefetch_non_temporal(rxdp); 114 115 if (rxq->rx_rearm_nb > HNS3_DEFAULT_RXQ_REARM_THRESH) 116 hns3_rxq_rearm_mbuf(rxq); 117 118 if (unlikely(!(rxdp->rx.bd_base_info & 119 rte_cpu_to_le_32(1u << HNS3_RXD_VLD_B)))) 120 return 0; 121 122 rte_prefetch0(rxq->sw_ring[rxq->next_to_use + 0].mbuf); 123 rte_prefetch0(rxq->sw_ring[rxq->next_to_use + 1].mbuf); 124 rte_prefetch0(rxq->sw_ring[rxq->next_to_use + 2].mbuf); 125 rte_prefetch0(rxq->sw_ring[rxq->next_to_use + 3].mbuf); 126 127 bd_err_mask = 0; 128 nb_rx = hns3_recv_burst_vec(rxq, rx_pkts, nb_pkts, &bd_err_mask); 129 if (unlikely(bd_err_mask)) 130 nb_rx = hns3_rx_reassemble_pkts(rx_pkts, nb_rx, bd_err_mask); 131 132 return nb_rx; 133 } 134 135 static void 136 hns3_rxq_vec_setup_rearm_data(struct hns3_rx_queue *rxq) 137 { 138 uintptr_t p; 139 struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */ 140 141 mb_def.nb_segs = 1; 142 mb_def.data_off = RTE_PKTMBUF_HEADROOM; 143 mb_def.port = rxq->port_id; 144 rte_mbuf_refcnt_set(&mb_def, 1); 145 146 /* prevent compiler reordering: rearm_data covers previous fields */ 147 rte_compiler_barrier(); 148 p = (uintptr_t)&mb_def.rearm_data; 149 rxq->mbuf_initializer = *(uint64_t *)p; 150 } 151 152 void 153 hns3_rxq_vec_setup(struct hns3_rx_queue *rxq) 154 { 155 struct hns3_entry *sw_ring = &rxq->sw_ring[rxq->nb_rx_desc]; 156 unsigned int i; 157 158 memset(&rxq->rx_ring[rxq->nb_rx_desc], 0, 159 sizeof(struct hns3_desc) * HNS3_DEFAULT_RX_BURST); 160 161 memset(&rxq->fake_mbuf, 0, sizeof(rxq->fake_mbuf)); 162 for (i = 0; i < HNS3_DEFAULT_RX_BURST; i++) 163 sw_ring[i].mbuf = &rxq->fake_mbuf; 164 165 hns3_rxq_vec_setup_rearm_data(rxq); 166 167 memset(rxq->offset_table, 0, sizeof(rxq->offset_table)); 168 } 169 170 #ifndef RTE_LIBRTE_IEEE1588 171 static int 172 hns3_rxq_vec_check(struct hns3_rx_queue *rxq, void *arg) 173 { 174 uint32_t min_vec_bds = HNS3_DEFAULT_RXQ_REARM_THRESH + 175 HNS3_DEFAULT_RX_BURST; 176 177 if (rxq->nb_rx_desc < min_vec_bds) 178 return -ENOTSUP; 179 180 if (rxq->nb_rx_desc % HNS3_DEFAULT_RXQ_REARM_THRESH) 181 return -ENOTSUP; 182 183 RTE_SET_USED(arg); 184 return 0; 185 } 186 #endif 187 188 int 189 hns3_rx_check_vec_support(struct rte_eth_dev *dev) 190 { 191 #ifndef RTE_LIBRTE_IEEE1588 192 struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf; 193 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 194 uint64_t offloads_mask = DEV_RX_OFFLOAD_TCP_LRO | 195 DEV_RX_OFFLOAD_VLAN; 196 197 if (dev->data->scattered_rx) 198 return -ENOTSUP; 199 200 if (fconf->mode != RTE_FDIR_MODE_NONE) 201 return -ENOTSUP; 202 203 if (rxmode->offloads & offloads_mask) 204 return -ENOTSUP; 205 206 if (hns3_rxq_iterate(dev, hns3_rxq_vec_check, NULL) != 0) 207 return -ENOTSUP; 208 209 return 0; 210 #else 211 RTE_SET_USED(dev); 212 return -ENOTSUP; 213 #endif 214 } 215