1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Arm Corporation 3 */ 4 5 #include <stdlib.h> 6 #include <stdint.h> 7 #include <stdio.h> 8 #include <string.h> 9 #include <errno.h> 10 11 #include <rte_net.h> 12 #include <rte_vect.h> 13 14 #include "virtio_ethdev.h" 15 #include "virtio.h" 16 #include "virtio_rxtx_packed.h" 17 #include "virtqueue.h" 18 19 static inline int 20 virtqueue_enqueue_batch_packed_vec(struct virtnet_tx *txvq, 21 struct rte_mbuf **tx_pkts) 22 { 23 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 24 uint16_t head_size = vq->hw->vtnet_hdr_size; 25 uint16_t idx = vq->vq_avail_idx; 26 struct virtio_net_hdr *hdr; 27 struct vq_desc_extra *dxp; 28 struct vring_packed_desc *p_desc; 29 uint16_t i; 30 31 if (idx & PACKED_BATCH_MASK) 32 return -1; 33 34 if (unlikely((idx + PACKED_BATCH_SIZE) > vq->vq_nentries)) 35 return -1; 36 37 /* Map four refcnt and nb_segs from mbufs to one NEON register. */ 38 uint8x16_t ref_seg_msk = { 39 2, 3, 4, 5, 40 10, 11, 12, 13, 41 18, 19, 20, 21, 42 26, 27, 28, 29 43 }; 44 45 /* Map four data_off from mbufs to one NEON register. */ 46 uint8x8_t data_msk = { 47 0, 1, 48 8, 9, 49 16, 17, 50 24, 25 51 }; 52 53 uint16x8_t net_hdr_msk = { 54 0xFFFF, 0xFFFF, 55 0, 0, 0, 0 56 }; 57 58 uint16x4_t pkts[PACKED_BATCH_SIZE]; 59 uint8x16x2_t mbuf; 60 /* Load four mbufs rearm data. */ 61 RTE_BUILD_BUG_ON(REFCNT_BITS_OFFSET >= 64); 62 pkts[0] = vld1_u16((uint16_t *)&tx_pkts[0]->rearm_data); 63 pkts[1] = vld1_u16((uint16_t *)&tx_pkts[1]->rearm_data); 64 pkts[2] = vld1_u16((uint16_t *)&tx_pkts[2]->rearm_data); 65 pkts[3] = vld1_u16((uint16_t *)&tx_pkts[3]->rearm_data); 66 67 mbuf.val[0] = vreinterpretq_u8_u16(vcombine_u16(pkts[0], pkts[1])); 68 mbuf.val[1] = vreinterpretq_u8_u16(vcombine_u16(pkts[2], pkts[3])); 69 70 /* refcnt = 1 and nb_segs = 1 */ 71 uint32x4_t def_ref_seg = vdupq_n_u32(0x10001); 72 /* Check refcnt and nb_segs. */ 73 uint32x4_t ref_seg = vreinterpretq_u32_u8(vqtbl2q_u8(mbuf, ref_seg_msk)); 74 uint64x2_t cmp1 = vreinterpretq_u64_u32(~vceqq_u32(ref_seg, def_ref_seg)); 75 if (unlikely(vgetq_lane_u64(cmp1, 0) || vgetq_lane_u64(cmp1, 1))) 76 return -1; 77 78 /* Check headroom is enough. */ 79 uint16x4_t head_rooms = vdup_n_u16(head_size); 80 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) != 81 offsetof(struct rte_mbuf, rearm_data)); 82 uint16x4_t data_offset = vreinterpret_u16_u8(vqtbl2_u8(mbuf, data_msk)); 83 uint64x1_t cmp2 = vreinterpret_u64_u16(vclt_u16(data_offset, head_rooms)); 84 if (unlikely(vget_lane_u64(cmp2, 0))) 85 return -1; 86 87 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 88 dxp = &vq->vq_descx[idx + i]; 89 dxp->ndescs = 1; 90 dxp->cookie = tx_pkts[i]; 91 } 92 93 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 94 tx_pkts[i]->data_off -= head_size; 95 tx_pkts[i]->data_len += head_size; 96 } 97 98 uint64x2x2_t desc[PACKED_BATCH_SIZE / 2]; 99 uint64x2_t base_addr0 = { 100 tx_pkts[0]->buf_iova + tx_pkts[0]->data_off, 101 tx_pkts[1]->buf_iova + tx_pkts[1]->data_off 102 }; 103 uint64x2_t base_addr1 = { 104 tx_pkts[2]->buf_iova + tx_pkts[2]->data_off, 105 tx_pkts[3]->buf_iova + tx_pkts[3]->data_off 106 }; 107 108 desc[0].val[0] = base_addr0; 109 desc[1].val[0] = base_addr1; 110 111 uint64_t flags = (uint64_t)vq->vq_packed.cached_flags << FLAGS_LEN_BITS_OFFSET; 112 uint64x2_t tx_desc0 = { 113 flags | (uint64_t)idx << ID_BITS_OFFSET | tx_pkts[0]->data_len, 114 flags | (uint64_t)(idx + 1) << ID_BITS_OFFSET | tx_pkts[1]->data_len 115 }; 116 117 uint64x2_t tx_desc1 = { 118 flags | (uint64_t)(idx + 2) << ID_BITS_OFFSET | tx_pkts[2]->data_len, 119 flags | (uint64_t)(idx + 3) << ID_BITS_OFFSET | tx_pkts[3]->data_len 120 }; 121 122 desc[0].val[1] = tx_desc0; 123 desc[1].val[1] = tx_desc1; 124 125 if (!vq->hw->has_tx_offload) { 126 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 127 hdr = rte_pktmbuf_mtod_offset(tx_pkts[i], 128 struct virtio_net_hdr *, -head_size); 129 /* Clear net hdr. */ 130 uint16x8_t v_hdr = vld1q_u16((void *)hdr); 131 vst1q_u16((void *)hdr, vandq_u16(v_hdr, net_hdr_msk)); 132 } 133 } else { 134 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 135 hdr = rte_pktmbuf_mtod_offset(tx_pkts[i], 136 struct virtio_net_hdr *, -head_size); 137 virtqueue_xmit_offload(hdr, tx_pkts[i], true); 138 } 139 } 140 141 /* Enqueue packet buffers. */ 142 p_desc = &vq->vq_packed.ring.desc[idx]; 143 vst2q_u64((uint64_t *)p_desc, desc[0]); 144 vst2q_u64((uint64_t *)(p_desc + 2), desc[1]); 145 146 virtio_update_batch_stats(&txvq->stats, tx_pkts[0]->pkt_len, 147 tx_pkts[1]->pkt_len, tx_pkts[2]->pkt_len, 148 tx_pkts[3]->pkt_len); 149 150 vq->vq_avail_idx += PACKED_BATCH_SIZE; 151 vq->vq_free_cnt -= PACKED_BATCH_SIZE; 152 153 if (vq->vq_avail_idx >= vq->vq_nentries) { 154 vq->vq_avail_idx -= vq->vq_nentries; 155 vq->vq_packed.cached_flags ^= 156 VRING_PACKED_DESC_F_AVAIL_USED; 157 } 158 159 return 0; 160 } 161 162 static inline int 163 virtqueue_dequeue_batch_packed_vec(struct virtnet_rx *rxvq, 164 struct rte_mbuf **rx_pkts) 165 { 166 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 167 struct virtio_hw *hw = vq->hw; 168 uint16_t head_size = hw->vtnet_hdr_size; 169 uint16_t id = vq->vq_used_cons_idx; 170 struct vring_packed_desc *p_desc; 171 uint16_t i; 172 173 if (id & PACKED_BATCH_MASK) 174 return -1; 175 176 if (unlikely((id + PACKED_BATCH_SIZE) > vq->vq_nentries)) 177 return -1; 178 179 /* Map packed descriptor to mbuf fields. */ 180 uint8x16_t shuf_msk1 = { 181 0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */ 182 0, 1, /* octet 1~0, low 16 bits pkt_len */ 183 0xFF, 0xFF, /* skip high 16 bits of pkt_len, zero out */ 184 0, 1, /* octet 1~0, 16 bits data_len */ 185 0xFF, 0xFF, /* vlan tci set as unknown */ 186 0xFF, 0xFF, 0xFF, 0xFF 187 }; 188 189 uint8x16_t shuf_msk2 = { 190 0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */ 191 8, 9, /* octet 9~8, low 16 bits pkt_len */ 192 0xFF, 0xFF, /* skip high 16 bits of pkt_len, zero out */ 193 8, 9, /* octet 9~8, 16 bits data_len */ 194 0xFF, 0xFF, /* vlan tci set as unknown */ 195 0xFF, 0xFF, 0xFF, 0xFF 196 }; 197 198 /* Subtract the header length. */ 199 uint16x8_t len_adjust = { 200 0, 0, /* ignore pkt_type field */ 201 head_size, /* sub head_size on pkt_len */ 202 0, /* ignore high 16 bits of pkt_len */ 203 head_size, /* sub head_size on data_len */ 204 0, 0, 0 /* ignore non-length fields */ 205 }; 206 207 uint64x2_t desc[PACKED_BATCH_SIZE / 2]; 208 uint64x2x2_t mbp[PACKED_BATCH_SIZE / 2]; 209 uint64x2_t pkt_mb[PACKED_BATCH_SIZE]; 210 211 p_desc = &vq->vq_packed.ring.desc[id]; 212 /* Load high 64 bits of packed descriptor 0,1. */ 213 desc[0] = vld2q_u64((uint64_t *)(p_desc)).val[1]; 214 /* Load high 64 bits of packed descriptor 2,3. */ 215 desc[1] = vld2q_u64((uint64_t *)(p_desc + 2)).val[1]; 216 217 /* Only care avail/used bits. */ 218 uint32x4_t v_mask = vdupq_n_u32(PACKED_FLAGS_MASK); 219 /* Extract high 32 bits of packed descriptor (id, flags). */ 220 uint32x4_t v_desc = vuzp2q_u32(vreinterpretq_u32_u64(desc[0]), 221 vreinterpretq_u32_u64(desc[1])); 222 uint32x4_t v_flag = vandq_u32(v_desc, v_mask); 223 224 uint32x4_t v_used_flag = vdupq_n_u32(0); 225 if (vq->vq_packed.used_wrap_counter) 226 v_used_flag = vdupq_n_u32(PACKED_FLAGS_MASK); 227 228 uint64x2_t desc_stats = vreinterpretq_u64_u32(~vceqq_u32(v_flag, v_used_flag)); 229 230 /* Check all descs are used. */ 231 if (unlikely(vgetq_lane_u64(desc_stats, 0) || vgetq_lane_u64(desc_stats, 1))) 232 return -1; 233 234 /* Load 2 mbuf pointers per time. */ 235 mbp[0] = vld2q_u64((uint64_t *)&vq->vq_descx[id]); 236 vst1q_u64((uint64_t *)&rx_pkts[0], mbp[0].val[0]); 237 238 mbp[1] = vld2q_u64((uint64_t *)&vq->vq_descx[id + 2]); 239 vst1q_u64((uint64_t *)&rx_pkts[2], mbp[1].val[0]); 240 241 /** 242 * Update data length and packet length for descriptor. 243 * structure of pkt_mb: 244 * -------------------------------------------------------------------- 245 * |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci| 246 * -------------------------------------------------------------------- 247 */ 248 pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8( 249 vreinterpretq_u8_u64(desc[0]), shuf_msk1)); 250 pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8( 251 vreinterpretq_u8_u64(desc[0]), shuf_msk2)); 252 pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8( 253 vreinterpretq_u8_u64(desc[1]), shuf_msk1)); 254 pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8( 255 vreinterpretq_u8_u64(desc[1]), shuf_msk2)); 256 257 pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16( 258 vreinterpretq_u16_u64(pkt_mb[0]), len_adjust)); 259 pkt_mb[1] = vreinterpretq_u64_u16(vsubq_u16( 260 vreinterpretq_u16_u64(pkt_mb[1]), len_adjust)); 261 pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16( 262 vreinterpretq_u16_u64(pkt_mb[2]), len_adjust)); 263 pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16( 264 vreinterpretq_u16_u64(pkt_mb[3]), len_adjust)); 265 266 vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1, pkt_mb[0]); 267 vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1, pkt_mb[1]); 268 vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1, pkt_mb[2]); 269 vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1, pkt_mb[3]); 270 271 if (hw->has_rx_offload) { 272 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 273 char *addr = (char *)rx_pkts[i]->buf_addr + 274 RTE_PKTMBUF_HEADROOM - head_size; 275 virtio_vec_rx_offload(rx_pkts[i], 276 (struct virtio_net_hdr *)addr); 277 } 278 } 279 280 virtio_update_batch_stats(&rxvq->stats, rx_pkts[0]->pkt_len, 281 rx_pkts[1]->pkt_len, rx_pkts[2]->pkt_len, 282 rx_pkts[3]->pkt_len); 283 284 vq->vq_free_cnt += PACKED_BATCH_SIZE; 285 286 vq->vq_used_cons_idx += PACKED_BATCH_SIZE; 287 if (vq->vq_used_cons_idx >= vq->vq_nentries) { 288 vq->vq_used_cons_idx -= vq->vq_nentries; 289 vq->vq_packed.used_wrap_counter ^= 1; 290 } 291 292 return 0; 293 } 294