1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef _VIRTQUEUE_H_ 35 #define _VIRTQUEUE_H_ 36 37 #include <stdint.h> 38 39 #include <rte_atomic.h> 40 #include <rte_memory.h> 41 #include <rte_memzone.h> 42 #include <rte_mempool.h> 43 44 #include "virtio_pci.h" 45 #include "virtio_ring.h" 46 #include "virtio_logs.h" 47 #include "virtio_rxtx.h" 48 49 struct rte_mbuf; 50 51 /* 52 * Per virtio_config.h in Linux. 53 * For virtio_pci on SMP, we don't need to order with respect to MMIO 54 * accesses through relaxed memory I/O windows, so smp_mb() et al are 55 * sufficient. 56 * 57 */ 58 #define virtio_mb() rte_smp_mb() 59 #define virtio_rmb() rte_smp_rmb() 60 #define virtio_wmb() rte_smp_wmb() 61 62 #ifdef RTE_PMD_PACKET_PREFETCH 63 #define rte_packet_prefetch(p) rte_prefetch1(p) 64 #else 65 #define rte_packet_prefetch(p) do {} while(0) 66 #endif 67 68 #define VIRTQUEUE_MAX_NAME_SZ 32 69 70 #ifdef RTE_VIRTIO_USER 71 /** 72 * Return the physical address (or virtual address in case of 73 * virtio-user) of mbuf data buffer. 74 * 75 * The address is firstly casted to the word size (sizeof(uintptr_t)) 76 * before casting it to uint64_t. This is to make it work with different 77 * combination of word size (64 bit and 32 bit) and virtio device 78 * (virtio-pci and virtio-user). 79 */ 80 #define VIRTIO_MBUF_ADDR(mb, vq) \ 81 ((uint64_t)(*(uintptr_t *)((uintptr_t)(mb) + (vq)->offset))) 82 #else 83 #define VIRTIO_MBUF_ADDR(mb, vq) ((mb)->buf_physaddr) 84 #endif 85 86 /** 87 * Return the physical address (or virtual address in case of 88 * virtio-user) of mbuf data buffer, taking care of mbuf data offset 89 */ 90 #define VIRTIO_MBUF_DATA_DMA_ADDR(mb, vq) \ 91 (VIRTIO_MBUF_ADDR(mb, vq) + (mb)->data_off) 92 93 #define VTNET_SQ_RQ_QUEUE_IDX 0 94 #define VTNET_SQ_TQ_QUEUE_IDX 1 95 #define VTNET_SQ_CQ_QUEUE_IDX 2 96 97 enum { VTNET_RQ = 0, VTNET_TQ = 1, VTNET_CQ = 2 }; 98 /** 99 * The maximum virtqueue size is 2^15. Use that value as the end of 100 * descriptor chain terminator since it will never be a valid index 101 * in the descriptor table. This is used to verify we are correctly 102 * handling vq_free_cnt. 103 */ 104 #define VQ_RING_DESC_CHAIN_END 32768 105 106 /** 107 * Control the RX mode, ie. promiscuous, allmulti, etc... 108 * All commands require an "out" sg entry containing a 1 byte 109 * state value, zero = disable, non-zero = enable. Commands 110 * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature. 111 * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA. 112 */ 113 #define VIRTIO_NET_CTRL_RX 0 114 #define VIRTIO_NET_CTRL_RX_PROMISC 0 115 #define VIRTIO_NET_CTRL_RX_ALLMULTI 1 116 #define VIRTIO_NET_CTRL_RX_ALLUNI 2 117 #define VIRTIO_NET_CTRL_RX_NOMULTI 3 118 #define VIRTIO_NET_CTRL_RX_NOUNI 4 119 #define VIRTIO_NET_CTRL_RX_NOBCAST 5 120 121 /** 122 * Control the MAC 123 * 124 * The MAC filter table is managed by the hypervisor, the guest should 125 * assume the size is infinite. Filtering should be considered 126 * non-perfect, ie. based on hypervisor resources, the guest may 127 * received packets from sources not specified in the filter list. 128 * 129 * In addition to the class/cmd header, the TABLE_SET command requires 130 * two out scatterlists. Each contains a 4 byte count of entries followed 131 * by a concatenated byte stream of the ETH_ALEN MAC addresses. The 132 * first sg list contains unicast addresses, the second is for multicast. 133 * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature 134 * is available. 135 * 136 * The ADDR_SET command requests one out scatterlist, it contains a 137 * 6 bytes MAC address. This functionality is present if the 138 * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available. 139 */ 140 struct virtio_net_ctrl_mac { 141 uint32_t entries; 142 uint8_t macs[][ETHER_ADDR_LEN]; 143 } __attribute__((__packed__)); 144 145 #define VIRTIO_NET_CTRL_MAC 1 146 #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 147 #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 148 149 /** 150 * Control VLAN filtering 151 * 152 * The VLAN filter table is controlled via a simple ADD/DEL interface. 153 * VLAN IDs not added may be filtered by the hypervisor. Del is the 154 * opposite of add. Both commands expect an out entry containing a 2 155 * byte VLAN ID. VLAN filtering is available with the 156 * VIRTIO_NET_F_CTRL_VLAN feature bit. 157 */ 158 #define VIRTIO_NET_CTRL_VLAN 2 159 #define VIRTIO_NET_CTRL_VLAN_ADD 0 160 #define VIRTIO_NET_CTRL_VLAN_DEL 1 161 162 struct virtio_net_ctrl_hdr { 163 uint8_t class; 164 uint8_t cmd; 165 } __attribute__((packed)); 166 167 typedef uint8_t virtio_net_ctrl_ack; 168 169 #define VIRTIO_NET_OK 0 170 #define VIRTIO_NET_ERR 1 171 172 #define VIRTIO_MAX_CTRL_DATA 2048 173 174 struct virtio_pmd_ctrl { 175 struct virtio_net_ctrl_hdr hdr; 176 virtio_net_ctrl_ack status; 177 uint8_t data[VIRTIO_MAX_CTRL_DATA]; 178 }; 179 180 struct vq_desc_extra { 181 void *cookie; 182 uint16_t ndescs; 183 }; 184 185 struct virtqueue { 186 struct virtio_hw *hw; /**< virtio_hw structure pointer. */ 187 struct vring vq_ring; /**< vring keeping desc, used and avail */ 188 /** 189 * Last consumed descriptor in the used table, 190 * trails vq_ring.used->idx. 191 */ 192 uint16_t vq_used_cons_idx; 193 uint16_t vq_nentries; /**< vring desc numbers */ 194 uint16_t vq_free_cnt; /**< num of desc available */ 195 uint16_t vq_avail_idx; /**< sync until needed */ 196 uint16_t vq_free_thresh; /**< free threshold */ 197 198 void *vq_ring_virt_mem; /**< linear address of vring*/ 199 unsigned int vq_ring_size; 200 201 union { 202 struct virtnet_rx rxq; 203 struct virtnet_tx txq; 204 struct virtnet_ctl cq; 205 }; 206 207 phys_addr_t vq_ring_mem; /**< physical address of vring, 208 * or virtual address for virtio_user. */ 209 210 /** 211 * Head of the free chain in the descriptor table. If 212 * there are no free descriptors, this will be set to 213 * VQ_RING_DESC_CHAIN_END. 214 */ 215 uint16_t vq_desc_head_idx; 216 uint16_t vq_desc_tail_idx; 217 uint16_t vq_queue_index; /**< PCI queue index */ 218 uint16_t offset; /**< relative offset to obtain addr in mbuf */ 219 uint16_t *notify_addr; 220 struct rte_mbuf **sw_ring; /**< RX software ring. */ 221 struct vq_desc_extra vq_descx[0]; 222 }; 223 224 /* If multiqueue is provided by host, then we suppport it. */ 225 #define VIRTIO_NET_CTRL_MQ 4 226 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 227 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 228 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 229 230 #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 231 232 /** 233 * This is the first element of the scatter-gather list. If you don't 234 * specify GSO or CSUM features, you can simply ignore the header. 235 */ 236 struct virtio_net_hdr { 237 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /**< Use csum_start,csum_offset*/ 238 #define VIRTIO_NET_HDR_F_DATA_VALID 2 /**< Checksum is valid */ 239 uint8_t flags; 240 #define VIRTIO_NET_HDR_GSO_NONE 0 /**< Not a GSO frame */ 241 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /**< GSO frame, IPv4 TCP (TSO) */ 242 #define VIRTIO_NET_HDR_GSO_UDP 3 /**< GSO frame, IPv4 UDP (UFO) */ 243 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /**< GSO frame, IPv6 TCP */ 244 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /**< TCP has ECN set */ 245 uint8_t gso_type; 246 uint16_t hdr_len; /**< Ethernet + IP + tcp/udp hdrs */ 247 uint16_t gso_size; /**< Bytes to append to hdr_len per frame */ 248 uint16_t csum_start; /**< Position to start checksumming from */ 249 uint16_t csum_offset; /**< Offset after that to place checksum */ 250 }; 251 252 /** 253 * This is the version of the header to use when the MRG_RXBUF 254 * feature has been negotiated. 255 */ 256 struct virtio_net_hdr_mrg_rxbuf { 257 struct virtio_net_hdr hdr; 258 uint16_t num_buffers; /**< Number of merged rx buffers */ 259 }; 260 261 /* Region reserved to allow for transmit header and indirect ring */ 262 #define VIRTIO_MAX_TX_INDIRECT 8 263 struct virtio_tx_region { 264 struct virtio_net_hdr_mrg_rxbuf tx_hdr; 265 struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT] 266 __attribute__((__aligned__(16))); 267 }; 268 269 /* Chain all the descriptors in the ring with an END */ 270 static inline void 271 vring_desc_init(struct vring_desc *dp, uint16_t n) 272 { 273 uint16_t i; 274 275 for (i = 0; i < n - 1; i++) 276 dp[i].next = (uint16_t)(i + 1); 277 dp[i].next = VQ_RING_DESC_CHAIN_END; 278 } 279 280 /** 281 * Tell the backend not to interrupt us. 282 */ 283 static inline void 284 virtqueue_disable_intr(struct virtqueue *vq) 285 { 286 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 287 } 288 289 /** 290 * Tell the backend to interrupt us. 291 */ 292 static inline void 293 virtqueue_enable_intr(struct virtqueue *vq) 294 { 295 vq->vq_ring.avail->flags &= (~VRING_AVAIL_F_NO_INTERRUPT); 296 } 297 298 /** 299 * Dump virtqueue internal structures, for debug purpose only. 300 */ 301 void virtqueue_dump(struct virtqueue *vq); 302 /** 303 * Get all mbufs to be freed. 304 */ 305 struct rte_mbuf *virtqueue_detatch_unused(struct virtqueue *vq); 306 307 static inline int 308 virtqueue_full(const struct virtqueue *vq) 309 { 310 return vq->vq_free_cnt == 0; 311 } 312 313 #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx)) 314 315 static inline void 316 vq_update_avail_idx(struct virtqueue *vq) 317 { 318 virtio_wmb(); 319 vq->vq_ring.avail->idx = vq->vq_avail_idx; 320 } 321 322 static inline void 323 vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx) 324 { 325 uint16_t avail_idx; 326 /* 327 * Place the head of the descriptor chain into the next slot and make 328 * it usable to the host. The chain is made available now rather than 329 * deferring to virtqueue_notify() in the hopes that if the host is 330 * currently running on another CPU, we can keep it processing the new 331 * descriptor. 332 */ 333 avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1)); 334 if (unlikely(vq->vq_ring.avail->ring[avail_idx] != desc_idx)) 335 vq->vq_ring.avail->ring[avail_idx] = desc_idx; 336 vq->vq_avail_idx++; 337 } 338 339 static inline int 340 virtqueue_kick_prepare(struct virtqueue *vq) 341 { 342 return !(vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY); 343 } 344 345 static inline void 346 virtqueue_notify(struct virtqueue *vq) 347 { 348 /* 349 * Ensure updated avail->idx is visible to host. 350 * For virtio on IA, the notificaiton is through io port operation 351 * which is a serialization instruction itself. 352 */ 353 VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq); 354 } 355 356 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 357 #define VIRTQUEUE_DUMP(vq) do { \ 358 uint16_t used_idx, nused; \ 359 used_idx = (vq)->vq_ring.used->idx; \ 360 nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \ 361 PMD_INIT_LOG(DEBUG, \ 362 "VQ: - size=%d; free=%d; used=%d; desc_head_idx=%d;" \ 363 " avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \ 364 " avail.flags=0x%x; used.flags=0x%x", \ 365 (vq)->vq_nentries, (vq)->vq_free_cnt, nused, \ 366 (vq)->vq_desc_head_idx, (vq)->vq_ring.avail->idx, \ 367 (vq)->vq_used_cons_idx, (vq)->vq_ring.used->idx, \ 368 (vq)->vq_ring.avail->flags, (vq)->vq_ring.used->flags); \ 369 } while (0) 370 #else 371 #define VIRTQUEUE_DUMP(vq) do { } while (0) 372 #endif 373 374 #endif /* _VIRTQUEUE_H_ */ 375