1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #ifndef _VIRTQUEUE_H_ 6 #define _VIRTQUEUE_H_ 7 8 #include <stdint.h> 9 10 #include <rte_atomic.h> 11 #include <rte_memory.h> 12 #include <rte_mempool.h> 13 14 #include "virtio_pci.h" 15 #include "virtio_ring.h" 16 #include "virtio_logs.h" 17 #include "virtio_rxtx.h" 18 19 struct rte_mbuf; 20 21 /* 22 * Per virtio_config.h in Linux. 23 * For virtio_pci on SMP, we don't need to order with respect to MMIO 24 * accesses through relaxed memory I/O windows, so smp_mb() et al are 25 * sufficient. 26 * 27 */ 28 #define virtio_mb() rte_smp_mb() 29 #define virtio_rmb() rte_smp_rmb() 30 #define virtio_wmb() rte_smp_wmb() 31 32 #ifdef RTE_PMD_PACKET_PREFETCH 33 #define rte_packet_prefetch(p) rte_prefetch1(p) 34 #else 35 #define rte_packet_prefetch(p) do {} while(0) 36 #endif 37 38 #define VIRTQUEUE_MAX_NAME_SZ 32 39 40 #ifdef RTE_VIRTIO_USER 41 /** 42 * Return the physical address (or virtual address in case of 43 * virtio-user) of mbuf data buffer. 44 * 45 * The address is firstly casted to the word size (sizeof(uintptr_t)) 46 * before casting it to uint64_t. This is to make it work with different 47 * combination of word size (64 bit and 32 bit) and virtio device 48 * (virtio-pci and virtio-user). 49 */ 50 #define VIRTIO_MBUF_ADDR(mb, vq) \ 51 ((uint64_t)(*(uintptr_t *)((uintptr_t)(mb) + (vq)->offset))) 52 #else 53 #define VIRTIO_MBUF_ADDR(mb, vq) ((mb)->buf_iova) 54 #endif 55 56 /** 57 * Return the physical address (or virtual address in case of 58 * virtio-user) of mbuf data buffer, taking care of mbuf data offset 59 */ 60 #define VIRTIO_MBUF_DATA_DMA_ADDR(mb, vq) \ 61 (VIRTIO_MBUF_ADDR(mb, vq) + (mb)->data_off) 62 63 #define VTNET_SQ_RQ_QUEUE_IDX 0 64 #define VTNET_SQ_TQ_QUEUE_IDX 1 65 #define VTNET_SQ_CQ_QUEUE_IDX 2 66 67 enum { VTNET_RQ = 0, VTNET_TQ = 1, VTNET_CQ = 2 }; 68 /** 69 * The maximum virtqueue size is 2^15. Use that value as the end of 70 * descriptor chain terminator since it will never be a valid index 71 * in the descriptor table. This is used to verify we are correctly 72 * handling vq_free_cnt. 73 */ 74 #define VQ_RING_DESC_CHAIN_END 32768 75 76 /** 77 * Control the RX mode, ie. promiscuous, allmulti, etc... 78 * All commands require an "out" sg entry containing a 1 byte 79 * state value, zero = disable, non-zero = enable. Commands 80 * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature. 81 * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA. 82 */ 83 #define VIRTIO_NET_CTRL_RX 0 84 #define VIRTIO_NET_CTRL_RX_PROMISC 0 85 #define VIRTIO_NET_CTRL_RX_ALLMULTI 1 86 #define VIRTIO_NET_CTRL_RX_ALLUNI 2 87 #define VIRTIO_NET_CTRL_RX_NOMULTI 3 88 #define VIRTIO_NET_CTRL_RX_NOUNI 4 89 #define VIRTIO_NET_CTRL_RX_NOBCAST 5 90 91 /** 92 * Control the MAC 93 * 94 * The MAC filter table is managed by the hypervisor, the guest should 95 * assume the size is infinite. Filtering should be considered 96 * non-perfect, ie. based on hypervisor resources, the guest may 97 * received packets from sources not specified in the filter list. 98 * 99 * In addition to the class/cmd header, the TABLE_SET command requires 100 * two out scatterlists. Each contains a 4 byte count of entries followed 101 * by a concatenated byte stream of the ETH_ALEN MAC addresses. The 102 * first sg list contains unicast addresses, the second is for multicast. 103 * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature 104 * is available. 105 * 106 * The ADDR_SET command requests one out scatterlist, it contains a 107 * 6 bytes MAC address. This functionality is present if the 108 * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available. 109 */ 110 struct virtio_net_ctrl_mac { 111 uint32_t entries; 112 uint8_t macs[][ETHER_ADDR_LEN]; 113 } __attribute__((__packed__)); 114 115 #define VIRTIO_NET_CTRL_MAC 1 116 #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 117 #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 118 119 /** 120 * Control VLAN filtering 121 * 122 * The VLAN filter table is controlled via a simple ADD/DEL interface. 123 * VLAN IDs not added may be filtered by the hypervisor. Del is the 124 * opposite of add. Both commands expect an out entry containing a 2 125 * byte VLAN ID. VLAN filtering is available with the 126 * VIRTIO_NET_F_CTRL_VLAN feature bit. 127 */ 128 #define VIRTIO_NET_CTRL_VLAN 2 129 #define VIRTIO_NET_CTRL_VLAN_ADD 0 130 #define VIRTIO_NET_CTRL_VLAN_DEL 1 131 132 struct virtio_net_ctrl_hdr { 133 uint8_t class; 134 uint8_t cmd; 135 } __attribute__((packed)); 136 137 typedef uint8_t virtio_net_ctrl_ack; 138 139 #define VIRTIO_NET_OK 0 140 #define VIRTIO_NET_ERR 1 141 142 #define VIRTIO_MAX_CTRL_DATA 2048 143 144 struct virtio_pmd_ctrl { 145 struct virtio_net_ctrl_hdr hdr; 146 virtio_net_ctrl_ack status; 147 uint8_t data[VIRTIO_MAX_CTRL_DATA]; 148 }; 149 150 struct vq_desc_extra { 151 void *cookie; 152 uint16_t ndescs; 153 }; 154 155 struct virtqueue { 156 struct virtio_hw *hw; /**< virtio_hw structure pointer. */ 157 struct vring vq_ring; /**< vring keeping desc, used and avail */ 158 /** 159 * Last consumed descriptor in the used table, 160 * trails vq_ring.used->idx. 161 */ 162 uint16_t vq_used_cons_idx; 163 uint16_t vq_nentries; /**< vring desc numbers */ 164 uint16_t vq_free_cnt; /**< num of desc available */ 165 uint16_t vq_avail_idx; /**< sync until needed */ 166 uint16_t vq_free_thresh; /**< free threshold */ 167 168 void *vq_ring_virt_mem; /**< linear address of vring*/ 169 unsigned int vq_ring_size; 170 171 union { 172 struct virtnet_rx rxq; 173 struct virtnet_tx txq; 174 struct virtnet_ctl cq; 175 }; 176 177 rte_iova_t vq_ring_mem; /**< physical address of vring, 178 * or virtual address for virtio_user. */ 179 180 /** 181 * Head of the free chain in the descriptor table. If 182 * there are no free descriptors, this will be set to 183 * VQ_RING_DESC_CHAIN_END. 184 */ 185 uint16_t vq_desc_head_idx; 186 uint16_t vq_desc_tail_idx; 187 uint16_t vq_queue_index; /**< PCI queue index */ 188 uint16_t offset; /**< relative offset to obtain addr in mbuf */ 189 uint16_t *notify_addr; 190 struct rte_mbuf **sw_ring; /**< RX software ring. */ 191 struct vq_desc_extra vq_descx[0]; 192 }; 193 194 /* If multiqueue is provided by host, then we suppport it. */ 195 #define VIRTIO_NET_CTRL_MQ 4 196 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 197 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 198 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 199 200 /** 201 * This is the first element of the scatter-gather list. If you don't 202 * specify GSO or CSUM features, you can simply ignore the header. 203 */ 204 struct virtio_net_hdr { 205 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /**< Use csum_start,csum_offset*/ 206 #define VIRTIO_NET_HDR_F_DATA_VALID 2 /**< Checksum is valid */ 207 uint8_t flags; 208 #define VIRTIO_NET_HDR_GSO_NONE 0 /**< Not a GSO frame */ 209 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /**< GSO frame, IPv4 TCP (TSO) */ 210 #define VIRTIO_NET_HDR_GSO_UDP 3 /**< GSO frame, IPv4 UDP (UFO) */ 211 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /**< GSO frame, IPv6 TCP */ 212 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /**< TCP has ECN set */ 213 uint8_t gso_type; 214 uint16_t hdr_len; /**< Ethernet + IP + tcp/udp hdrs */ 215 uint16_t gso_size; /**< Bytes to append to hdr_len per frame */ 216 uint16_t csum_start; /**< Position to start checksumming from */ 217 uint16_t csum_offset; /**< Offset after that to place checksum */ 218 }; 219 220 /** 221 * This is the version of the header to use when the MRG_RXBUF 222 * feature has been negotiated. 223 */ 224 struct virtio_net_hdr_mrg_rxbuf { 225 struct virtio_net_hdr hdr; 226 uint16_t num_buffers; /**< Number of merged rx buffers */ 227 }; 228 229 /* Region reserved to allow for transmit header and indirect ring */ 230 #define VIRTIO_MAX_TX_INDIRECT 8 231 struct virtio_tx_region { 232 struct virtio_net_hdr_mrg_rxbuf tx_hdr; 233 struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT] 234 __attribute__((__aligned__(16))); 235 }; 236 237 /* Chain all the descriptors in the ring with an END */ 238 static inline void 239 vring_desc_init(struct vring_desc *dp, uint16_t n) 240 { 241 uint16_t i; 242 243 for (i = 0; i < n - 1; i++) 244 dp[i].next = (uint16_t)(i + 1); 245 dp[i].next = VQ_RING_DESC_CHAIN_END; 246 } 247 248 /** 249 * Tell the backend not to interrupt us. 250 */ 251 static inline void 252 virtqueue_disable_intr(struct virtqueue *vq) 253 { 254 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 255 } 256 257 /** 258 * Tell the backend to interrupt us. 259 */ 260 static inline void 261 virtqueue_enable_intr(struct virtqueue *vq) 262 { 263 vq->vq_ring.avail->flags &= (~VRING_AVAIL_F_NO_INTERRUPT); 264 } 265 266 /** 267 * Dump virtqueue internal structures, for debug purpose only. 268 */ 269 void virtqueue_dump(struct virtqueue *vq); 270 /** 271 * Get all mbufs to be freed. 272 */ 273 struct rte_mbuf *virtqueue_detatch_unused(struct virtqueue *vq); 274 275 /* Flush the elements in the used ring. */ 276 void virtqueue_rxvq_flush(struct virtqueue *vq); 277 278 static inline int 279 virtqueue_full(const struct virtqueue *vq) 280 { 281 return vq->vq_free_cnt == 0; 282 } 283 284 #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx)) 285 286 void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx); 287 288 static inline void 289 vq_update_avail_idx(struct virtqueue *vq) 290 { 291 virtio_wmb(); 292 vq->vq_ring.avail->idx = vq->vq_avail_idx; 293 } 294 295 static inline void 296 vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx) 297 { 298 uint16_t avail_idx; 299 /* 300 * Place the head of the descriptor chain into the next slot and make 301 * it usable to the host. The chain is made available now rather than 302 * deferring to virtqueue_notify() in the hopes that if the host is 303 * currently running on another CPU, we can keep it processing the new 304 * descriptor. 305 */ 306 avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1)); 307 if (unlikely(vq->vq_ring.avail->ring[avail_idx] != desc_idx)) 308 vq->vq_ring.avail->ring[avail_idx] = desc_idx; 309 vq->vq_avail_idx++; 310 } 311 312 static inline int 313 virtqueue_kick_prepare(struct virtqueue *vq) 314 { 315 return !(vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY); 316 } 317 318 static inline void 319 virtqueue_notify(struct virtqueue *vq) 320 { 321 /* 322 * Ensure updated avail->idx is visible to host. 323 * For virtio on IA, the notificaiton is through io port operation 324 * which is a serialization instruction itself. 325 */ 326 VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq); 327 } 328 329 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 330 #define VIRTQUEUE_DUMP(vq) do { \ 331 uint16_t used_idx, nused; \ 332 used_idx = (vq)->vq_ring.used->idx; \ 333 nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \ 334 PMD_INIT_LOG(DEBUG, \ 335 "VQ: - size=%d; free=%d; used=%d; desc_head_idx=%d;" \ 336 " avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \ 337 " avail.flags=0x%x; used.flags=0x%x", \ 338 (vq)->vq_nentries, (vq)->vq_free_cnt, nused, \ 339 (vq)->vq_desc_head_idx, (vq)->vq_ring.avail->idx, \ 340 (vq)->vq_used_cons_idx, (vq)->vq_ring.used->idx, \ 341 (vq)->vq_ring.avail->flags, (vq)->vq_ring.used->flags); \ 342 } while (0) 343 #else 344 #define VIRTQUEUE_DUMP(vq) do { } while (0) 345 #endif 346 347 #endif /* _VIRTQUEUE_H_ */ 348