1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #ifndef _VIRTQUEUE_H_ 6 #define _VIRTQUEUE_H_ 7 8 #include <stdint.h> 9 10 #include <rte_atomic.h> 11 #include <rte_memory.h> 12 #include <rte_mempool.h> 13 14 #include "virtio_pci.h" 15 #include "virtio_ring.h" 16 #include "virtio_logs.h" 17 #include "virtio_rxtx.h" 18 19 struct rte_mbuf; 20 21 /* 22 * Per virtio_config.h in Linux. 23 * For virtio_pci on SMP, we don't need to order with respect to MMIO 24 * accesses through relaxed memory I/O windows, so smp_mb() et al are 25 * sufficient. 26 * 27 */ 28 #define virtio_mb() rte_smp_mb() 29 #define virtio_rmb() rte_smp_rmb() 30 #define virtio_wmb() rte_smp_wmb() 31 32 #ifdef RTE_PMD_PACKET_PREFETCH 33 #define rte_packet_prefetch(p) rte_prefetch1(p) 34 #else 35 #define rte_packet_prefetch(p) do {} while(0) 36 #endif 37 38 #define VIRTQUEUE_MAX_NAME_SZ 32 39 40 #ifdef RTE_VIRTIO_USER 41 /** 42 * Return the physical address (or virtual address in case of 43 * virtio-user) of mbuf data buffer. 44 * 45 * The address is firstly casted to the word size (sizeof(uintptr_t)) 46 * before casting it to uint64_t. This is to make it work with different 47 * combination of word size (64 bit and 32 bit) and virtio device 48 * (virtio-pci and virtio-user). 49 */ 50 #define VIRTIO_MBUF_ADDR(mb, vq) \ 51 ((uint64_t)(*(uintptr_t *)((uintptr_t)(mb) + (vq)->offset))) 52 #else 53 #define VIRTIO_MBUF_ADDR(mb, vq) ((mb)->buf_iova) 54 #endif 55 56 /** 57 * Return the physical address (or virtual address in case of 58 * virtio-user) of mbuf data buffer, taking care of mbuf data offset 59 */ 60 #define VIRTIO_MBUF_DATA_DMA_ADDR(mb, vq) \ 61 (VIRTIO_MBUF_ADDR(mb, vq) + (mb)->data_off) 62 63 #define VTNET_SQ_RQ_QUEUE_IDX 0 64 #define VTNET_SQ_TQ_QUEUE_IDX 1 65 #define VTNET_SQ_CQ_QUEUE_IDX 2 66 67 enum { VTNET_RQ = 0, VTNET_TQ = 1, VTNET_CQ = 2 }; 68 /** 69 * The maximum virtqueue size is 2^15. Use that value as the end of 70 * descriptor chain terminator since it will never be a valid index 71 * in the descriptor table. This is used to verify we are correctly 72 * handling vq_free_cnt. 73 */ 74 #define VQ_RING_DESC_CHAIN_END 32768 75 76 /** 77 * Control the RX mode, ie. promiscuous, allmulti, etc... 78 * All commands require an "out" sg entry containing a 1 byte 79 * state value, zero = disable, non-zero = enable. Commands 80 * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature. 81 * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA. 82 */ 83 #define VIRTIO_NET_CTRL_RX 0 84 #define VIRTIO_NET_CTRL_RX_PROMISC 0 85 #define VIRTIO_NET_CTRL_RX_ALLMULTI 1 86 #define VIRTIO_NET_CTRL_RX_ALLUNI 2 87 #define VIRTIO_NET_CTRL_RX_NOMULTI 3 88 #define VIRTIO_NET_CTRL_RX_NOUNI 4 89 #define VIRTIO_NET_CTRL_RX_NOBCAST 5 90 91 /** 92 * Control the MAC 93 * 94 * The MAC filter table is managed by the hypervisor, the guest should 95 * assume the size is infinite. Filtering should be considered 96 * non-perfect, ie. based on hypervisor resources, the guest may 97 * received packets from sources not specified in the filter list. 98 * 99 * In addition to the class/cmd header, the TABLE_SET command requires 100 * two out scatterlists. Each contains a 4 byte count of entries followed 101 * by a concatenated byte stream of the ETH_ALEN MAC addresses. The 102 * first sg list contains unicast addresses, the second is for multicast. 103 * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature 104 * is available. 105 * 106 * The ADDR_SET command requests one out scatterlist, it contains a 107 * 6 bytes MAC address. This functionality is present if the 108 * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available. 109 */ 110 struct virtio_net_ctrl_mac { 111 uint32_t entries; 112 uint8_t macs[][ETHER_ADDR_LEN]; 113 } __attribute__((__packed__)); 114 115 #define VIRTIO_NET_CTRL_MAC 1 116 #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 117 #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 118 119 /** 120 * Control VLAN filtering 121 * 122 * The VLAN filter table is controlled via a simple ADD/DEL interface. 123 * VLAN IDs not added may be filtered by the hypervisor. Del is the 124 * opposite of add. Both commands expect an out entry containing a 2 125 * byte VLAN ID. VLAN filtering is available with the 126 * VIRTIO_NET_F_CTRL_VLAN feature bit. 127 */ 128 #define VIRTIO_NET_CTRL_VLAN 2 129 #define VIRTIO_NET_CTRL_VLAN_ADD 0 130 #define VIRTIO_NET_CTRL_VLAN_DEL 1 131 132 /* 133 * Control link announce acknowledgement 134 * 135 * The command VIRTIO_NET_CTRL_ANNOUNCE_ACK is used to indicate that 136 * driver has recevied the notification; device would clear the 137 * VIRTIO_NET_S_ANNOUNCE bit in the status field after it receives 138 * this command. 139 */ 140 #define VIRTIO_NET_CTRL_ANNOUNCE 3 141 #define VIRTIO_NET_CTRL_ANNOUNCE_ACK 0 142 143 struct virtio_net_ctrl_hdr { 144 uint8_t class; 145 uint8_t cmd; 146 } __attribute__((packed)); 147 148 typedef uint8_t virtio_net_ctrl_ack; 149 150 #define VIRTIO_NET_OK 0 151 #define VIRTIO_NET_ERR 1 152 153 #define VIRTIO_MAX_CTRL_DATA 2048 154 155 struct virtio_pmd_ctrl { 156 struct virtio_net_ctrl_hdr hdr; 157 virtio_net_ctrl_ack status; 158 uint8_t data[VIRTIO_MAX_CTRL_DATA]; 159 }; 160 161 struct vq_desc_extra { 162 void *cookie; 163 uint16_t ndescs; 164 }; 165 166 struct virtqueue { 167 struct virtio_hw *hw; /**< virtio_hw structure pointer. */ 168 struct vring vq_ring; /**< vring keeping desc, used and avail */ 169 /** 170 * Last consumed descriptor in the used table, 171 * trails vq_ring.used->idx. 172 */ 173 uint16_t vq_used_cons_idx; 174 uint16_t vq_nentries; /**< vring desc numbers */ 175 uint16_t vq_free_cnt; /**< num of desc available */ 176 uint16_t vq_avail_idx; /**< sync until needed */ 177 uint16_t vq_free_thresh; /**< free threshold */ 178 179 void *vq_ring_virt_mem; /**< linear address of vring*/ 180 unsigned int vq_ring_size; 181 182 union { 183 struct virtnet_rx rxq; 184 struct virtnet_tx txq; 185 struct virtnet_ctl cq; 186 }; 187 188 rte_iova_t vq_ring_mem; /**< physical address of vring, 189 * or virtual address for virtio_user. */ 190 191 /** 192 * Head of the free chain in the descriptor table. If 193 * there are no free descriptors, this will be set to 194 * VQ_RING_DESC_CHAIN_END. 195 */ 196 uint16_t vq_desc_head_idx; 197 uint16_t vq_desc_tail_idx; 198 uint16_t vq_queue_index; /**< PCI queue index */ 199 uint16_t offset; /**< relative offset to obtain addr in mbuf */ 200 uint16_t *notify_addr; 201 struct rte_mbuf **sw_ring; /**< RX software ring. */ 202 struct vq_desc_extra vq_descx[0]; 203 }; 204 205 /* If multiqueue is provided by host, then we suppport it. */ 206 #define VIRTIO_NET_CTRL_MQ 4 207 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 208 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 209 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 210 211 /** 212 * This is the first element of the scatter-gather list. If you don't 213 * specify GSO or CSUM features, you can simply ignore the header. 214 */ 215 struct virtio_net_hdr { 216 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /**< Use csum_start,csum_offset*/ 217 #define VIRTIO_NET_HDR_F_DATA_VALID 2 /**< Checksum is valid */ 218 uint8_t flags; 219 #define VIRTIO_NET_HDR_GSO_NONE 0 /**< Not a GSO frame */ 220 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /**< GSO frame, IPv4 TCP (TSO) */ 221 #define VIRTIO_NET_HDR_GSO_UDP 3 /**< GSO frame, IPv4 UDP (UFO) */ 222 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /**< GSO frame, IPv6 TCP */ 223 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /**< TCP has ECN set */ 224 uint8_t gso_type; 225 uint16_t hdr_len; /**< Ethernet + IP + tcp/udp hdrs */ 226 uint16_t gso_size; /**< Bytes to append to hdr_len per frame */ 227 uint16_t csum_start; /**< Position to start checksumming from */ 228 uint16_t csum_offset; /**< Offset after that to place checksum */ 229 }; 230 231 /** 232 * This is the version of the header to use when the MRG_RXBUF 233 * feature has been negotiated. 234 */ 235 struct virtio_net_hdr_mrg_rxbuf { 236 struct virtio_net_hdr hdr; 237 uint16_t num_buffers; /**< Number of merged rx buffers */ 238 }; 239 240 /* Region reserved to allow for transmit header and indirect ring */ 241 #define VIRTIO_MAX_TX_INDIRECT 8 242 struct virtio_tx_region { 243 struct virtio_net_hdr_mrg_rxbuf tx_hdr; 244 struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT] 245 __attribute__((__aligned__(16))); 246 }; 247 248 /* Chain all the descriptors in the ring with an END */ 249 static inline void 250 vring_desc_init(struct vring_desc *dp, uint16_t n) 251 { 252 uint16_t i; 253 254 for (i = 0; i < n - 1; i++) 255 dp[i].next = (uint16_t)(i + 1); 256 dp[i].next = VQ_RING_DESC_CHAIN_END; 257 } 258 259 /** 260 * Tell the backend not to interrupt us. 261 */ 262 static inline void 263 virtqueue_disable_intr(struct virtqueue *vq) 264 { 265 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 266 } 267 268 /** 269 * Tell the backend to interrupt us. 270 */ 271 static inline void 272 virtqueue_enable_intr(struct virtqueue *vq) 273 { 274 vq->vq_ring.avail->flags &= (~VRING_AVAIL_F_NO_INTERRUPT); 275 } 276 277 /** 278 * Dump virtqueue internal structures, for debug purpose only. 279 */ 280 void virtqueue_dump(struct virtqueue *vq); 281 /** 282 * Get all mbufs to be freed. 283 */ 284 struct rte_mbuf *virtqueue_detach_unused(struct virtqueue *vq); 285 286 /* Flush the elements in the used ring. */ 287 void virtqueue_rxvq_flush(struct virtqueue *vq); 288 289 static inline int 290 virtqueue_full(const struct virtqueue *vq) 291 { 292 return vq->vq_free_cnt == 0; 293 } 294 295 static inline int 296 virtio_get_queue_type(struct virtio_hw *hw, uint16_t vtpci_queue_idx) 297 { 298 if (vtpci_queue_idx == hw->max_queue_pairs * 2) 299 return VTNET_CQ; 300 else if (vtpci_queue_idx % 2 == 0) 301 return VTNET_RQ; 302 else 303 return VTNET_TQ; 304 } 305 306 #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx)) 307 308 void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx); 309 310 static inline void 311 vq_update_avail_idx(struct virtqueue *vq) 312 { 313 virtio_wmb(); 314 vq->vq_ring.avail->idx = vq->vq_avail_idx; 315 } 316 317 static inline void 318 vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx) 319 { 320 uint16_t avail_idx; 321 /* 322 * Place the head of the descriptor chain into the next slot and make 323 * it usable to the host. The chain is made available now rather than 324 * deferring to virtqueue_notify() in the hopes that if the host is 325 * currently running on another CPU, we can keep it processing the new 326 * descriptor. 327 */ 328 avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1)); 329 if (unlikely(vq->vq_ring.avail->ring[avail_idx] != desc_idx)) 330 vq->vq_ring.avail->ring[avail_idx] = desc_idx; 331 vq->vq_avail_idx++; 332 } 333 334 static inline int 335 virtqueue_kick_prepare(struct virtqueue *vq) 336 { 337 return !(vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY); 338 } 339 340 static inline void 341 virtqueue_notify(struct virtqueue *vq) 342 { 343 /* 344 * Ensure updated avail->idx is visible to host. 345 * For virtio on IA, the notificaiton is through io port operation 346 * which is a serialization instruction itself. 347 */ 348 VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq); 349 } 350 351 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 352 #define VIRTQUEUE_DUMP(vq) do { \ 353 uint16_t used_idx, nused; \ 354 used_idx = (vq)->vq_ring.used->idx; \ 355 nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \ 356 PMD_INIT_LOG(DEBUG, \ 357 "VQ: - size=%d; free=%d; used=%d; desc_head_idx=%d;" \ 358 " avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \ 359 " avail.flags=0x%x; used.flags=0x%x", \ 360 (vq)->vq_nentries, (vq)->vq_free_cnt, nused, \ 361 (vq)->vq_desc_head_idx, (vq)->vq_ring.avail->idx, \ 362 (vq)->vq_used_cons_idx, (vq)->vq_ring.used->idx, \ 363 (vq)->vq_ring.avail->flags, (vq)->vq_ring.used->flags); \ 364 } while (0) 365 #else 366 #define VIRTQUEUE_DUMP(vq) do { } while (0) 367 #endif 368 369 #endif /* _VIRTQUEUE_H_ */ 370