1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #ifndef _VIRTQUEUE_H_ 6 #define _VIRTQUEUE_H_ 7 8 #include <stdint.h> 9 10 #include <rte_atomic.h> 11 #include <rte_memory.h> 12 #include <rte_mempool.h> 13 14 #include "virtio_pci.h" 15 #include "virtio_ring.h" 16 #include "virtio_logs.h" 17 #include "virtio_rxtx.h" 18 19 struct rte_mbuf; 20 21 /* 22 * Per virtio_config.h in Linux. 23 * For virtio_pci on SMP, we don't need to order with respect to MMIO 24 * accesses through relaxed memory I/O windows, so smp_mb() et al are 25 * sufficient. 26 * 27 */ 28 #define virtio_mb() rte_smp_mb() 29 #define virtio_rmb() rte_smp_rmb() 30 #define virtio_wmb() rte_smp_wmb() 31 32 #ifdef RTE_PMD_PACKET_PREFETCH 33 #define rte_packet_prefetch(p) rte_prefetch1(p) 34 #else 35 #define rte_packet_prefetch(p) do {} while(0) 36 #endif 37 38 #define VIRTQUEUE_MAX_NAME_SZ 32 39 40 #ifdef RTE_VIRTIO_USER 41 /** 42 * Return the physical address (or virtual address in case of 43 * virtio-user) of mbuf data buffer. 44 * 45 * The address is firstly casted to the word size (sizeof(uintptr_t)) 46 * before casting it to uint64_t. This is to make it work with different 47 * combination of word size (64 bit and 32 bit) and virtio device 48 * (virtio-pci and virtio-user). 49 */ 50 #define VIRTIO_MBUF_ADDR(mb, vq) \ 51 ((uint64_t)(*(uintptr_t *)((uintptr_t)(mb) + (vq)->offset))) 52 #else 53 #define VIRTIO_MBUF_ADDR(mb, vq) ((mb)->buf_iova) 54 #endif 55 56 /** 57 * Return the physical address (or virtual address in case of 58 * virtio-user) of mbuf data buffer, taking care of mbuf data offset 59 */ 60 #define VIRTIO_MBUF_DATA_DMA_ADDR(mb, vq) \ 61 (VIRTIO_MBUF_ADDR(mb, vq) + (mb)->data_off) 62 63 #define VTNET_SQ_RQ_QUEUE_IDX 0 64 #define VTNET_SQ_TQ_QUEUE_IDX 1 65 #define VTNET_SQ_CQ_QUEUE_IDX 2 66 67 enum { VTNET_RQ = 0, VTNET_TQ = 1, VTNET_CQ = 2 }; 68 /** 69 * The maximum virtqueue size is 2^15. Use that value as the end of 70 * descriptor chain terminator since it will never be a valid index 71 * in the descriptor table. This is used to verify we are correctly 72 * handling vq_free_cnt. 73 */ 74 #define VQ_RING_DESC_CHAIN_END 32768 75 76 /** 77 * Control the RX mode, ie. promiscuous, allmulti, etc... 78 * All commands require an "out" sg entry containing a 1 byte 79 * state value, zero = disable, non-zero = enable. Commands 80 * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature. 81 * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA. 82 */ 83 #define VIRTIO_NET_CTRL_RX 0 84 #define VIRTIO_NET_CTRL_RX_PROMISC 0 85 #define VIRTIO_NET_CTRL_RX_ALLMULTI 1 86 #define VIRTIO_NET_CTRL_RX_ALLUNI 2 87 #define VIRTIO_NET_CTRL_RX_NOMULTI 3 88 #define VIRTIO_NET_CTRL_RX_NOUNI 4 89 #define VIRTIO_NET_CTRL_RX_NOBCAST 5 90 91 /** 92 * Control the MAC 93 * 94 * The MAC filter table is managed by the hypervisor, the guest should 95 * assume the size is infinite. Filtering should be considered 96 * non-perfect, ie. based on hypervisor resources, the guest may 97 * received packets from sources not specified in the filter list. 98 * 99 * In addition to the class/cmd header, the TABLE_SET command requires 100 * two out scatterlists. Each contains a 4 byte count of entries followed 101 * by a concatenated byte stream of the ETH_ALEN MAC addresses. The 102 * first sg list contains unicast addresses, the second is for multicast. 103 * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature 104 * is available. 105 * 106 * The ADDR_SET command requests one out scatterlist, it contains a 107 * 6 bytes MAC address. This functionality is present if the 108 * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available. 109 */ 110 struct virtio_net_ctrl_mac { 111 uint32_t entries; 112 uint8_t macs[][ETHER_ADDR_LEN]; 113 } __attribute__((__packed__)); 114 115 #define VIRTIO_NET_CTRL_MAC 1 116 #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 117 #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 118 119 /** 120 * Control VLAN filtering 121 * 122 * The VLAN filter table is controlled via a simple ADD/DEL interface. 123 * VLAN IDs not added may be filtered by the hypervisor. Del is the 124 * opposite of add. Both commands expect an out entry containing a 2 125 * byte VLAN ID. VLAN filtering is available with the 126 * VIRTIO_NET_F_CTRL_VLAN feature bit. 127 */ 128 #define VIRTIO_NET_CTRL_VLAN 2 129 #define VIRTIO_NET_CTRL_VLAN_ADD 0 130 #define VIRTIO_NET_CTRL_VLAN_DEL 1 131 132 /* 133 * Control link announce acknowledgement 134 * 135 * The command VIRTIO_NET_CTRL_ANNOUNCE_ACK is used to indicate that 136 * driver has recevied the notification; device would clear the 137 * VIRTIO_NET_S_ANNOUNCE bit in the status field after it receives 138 * this command. 139 */ 140 #define VIRTIO_NET_CTRL_ANNOUNCE 3 141 #define VIRTIO_NET_CTRL_ANNOUNCE_ACK 0 142 143 struct virtio_net_ctrl_hdr { 144 uint8_t class; 145 uint8_t cmd; 146 } __attribute__((packed)); 147 148 typedef uint8_t virtio_net_ctrl_ack; 149 150 #define VIRTIO_NET_OK 0 151 #define VIRTIO_NET_ERR 1 152 153 #define VIRTIO_MAX_CTRL_DATA 2048 154 155 struct virtio_pmd_ctrl { 156 struct virtio_net_ctrl_hdr hdr; 157 virtio_net_ctrl_ack status; 158 uint8_t data[VIRTIO_MAX_CTRL_DATA]; 159 }; 160 161 struct vq_desc_extra { 162 void *cookie; 163 uint16_t ndescs; 164 uint16_t next; 165 }; 166 167 struct virtqueue { 168 struct virtio_hw *hw; /**< virtio_hw structure pointer. */ 169 struct vring vq_ring; /**< vring keeping desc, used and avail */ 170 struct vring_packed ring_packed; /**< vring keeping descs */ 171 bool avail_wrap_counter; 172 bool used_wrap_counter; 173 uint16_t event_flags_shadow; 174 uint16_t avail_used_flags; 175 /** 176 * Last consumed descriptor in the used table, 177 * trails vq_ring.used->idx. 178 */ 179 uint16_t vq_used_cons_idx; 180 uint16_t vq_nentries; /**< vring desc numbers */ 181 uint16_t vq_free_cnt; /**< num of desc available */ 182 uint16_t vq_avail_idx; /**< sync until needed */ 183 uint16_t vq_free_thresh; /**< free threshold */ 184 185 void *vq_ring_virt_mem; /**< linear address of vring*/ 186 unsigned int vq_ring_size; 187 188 union { 189 struct virtnet_rx rxq; 190 struct virtnet_tx txq; 191 struct virtnet_ctl cq; 192 }; 193 194 rte_iova_t vq_ring_mem; /**< physical address of vring, 195 * or virtual address for virtio_user. */ 196 197 /** 198 * Head of the free chain in the descriptor table. If 199 * there are no free descriptors, this will be set to 200 * VQ_RING_DESC_CHAIN_END. 201 */ 202 uint16_t vq_desc_head_idx; 203 uint16_t vq_desc_tail_idx; 204 uint16_t vq_queue_index; /**< PCI queue index */ 205 uint16_t offset; /**< relative offset to obtain addr in mbuf */ 206 uint16_t *notify_addr; 207 struct rte_mbuf **sw_ring; /**< RX software ring. */ 208 struct vq_desc_extra vq_descx[0]; 209 }; 210 211 /* If multiqueue is provided by host, then we suppport it. */ 212 #define VIRTIO_NET_CTRL_MQ 4 213 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 214 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 215 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 216 217 /** 218 * This is the first element of the scatter-gather list. If you don't 219 * specify GSO or CSUM features, you can simply ignore the header. 220 */ 221 struct virtio_net_hdr { 222 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /**< Use csum_start,csum_offset*/ 223 #define VIRTIO_NET_HDR_F_DATA_VALID 2 /**< Checksum is valid */ 224 uint8_t flags; 225 #define VIRTIO_NET_HDR_GSO_NONE 0 /**< Not a GSO frame */ 226 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /**< GSO frame, IPv4 TCP (TSO) */ 227 #define VIRTIO_NET_HDR_GSO_UDP 3 /**< GSO frame, IPv4 UDP (UFO) */ 228 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /**< GSO frame, IPv6 TCP */ 229 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /**< TCP has ECN set */ 230 uint8_t gso_type; 231 uint16_t hdr_len; /**< Ethernet + IP + tcp/udp hdrs */ 232 uint16_t gso_size; /**< Bytes to append to hdr_len per frame */ 233 uint16_t csum_start; /**< Position to start checksumming from */ 234 uint16_t csum_offset; /**< Offset after that to place checksum */ 235 }; 236 237 /** 238 * This is the version of the header to use when the MRG_RXBUF 239 * feature has been negotiated. 240 */ 241 struct virtio_net_hdr_mrg_rxbuf { 242 struct virtio_net_hdr hdr; 243 uint16_t num_buffers; /**< Number of merged rx buffers */ 244 }; 245 246 /* Region reserved to allow for transmit header and indirect ring */ 247 #define VIRTIO_MAX_TX_INDIRECT 8 248 struct virtio_tx_region { 249 struct virtio_net_hdr_mrg_rxbuf tx_hdr; 250 union { 251 struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT] 252 __attribute__((__aligned__(16))); 253 struct vring_packed_desc tx_indir_pq[VIRTIO_MAX_TX_INDIRECT] 254 __attribute__((__aligned__(16))); 255 }; 256 }; 257 258 static inline int 259 desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq) 260 { 261 uint16_t used, avail, flags; 262 263 flags = desc->flags; 264 used = !!(flags & VRING_DESC_F_USED(1)); 265 avail = !!(flags & VRING_DESC_F_AVAIL(1)); 266 267 return avail == used && used == vq->used_wrap_counter; 268 } 269 270 271 static inline void 272 vring_desc_init_packed(struct virtqueue *vq, int n) 273 { 274 int i; 275 for (i = 0; i < n - 1; i++) { 276 vq->ring_packed.desc_packed[i].id = i; 277 vq->vq_descx[i].next = i + 1; 278 } 279 vq->ring_packed.desc_packed[i].id = i; 280 vq->vq_descx[i].next = VQ_RING_DESC_CHAIN_END; 281 } 282 283 /* Chain all the descriptors in the ring with an END */ 284 static inline void 285 vring_desc_init_split(struct vring_desc *dp, uint16_t n) 286 { 287 uint16_t i; 288 289 for (i = 0; i < n - 1; i++) 290 dp[i].next = (uint16_t)(i + 1); 291 dp[i].next = VQ_RING_DESC_CHAIN_END; 292 } 293 294 /** 295 * Tell the backend not to interrupt us. 296 */ 297 static inline void 298 virtqueue_disable_intr_packed(struct virtqueue *vq) 299 { 300 uint16_t *event_flags = &vq->ring_packed.driver_event->desc_event_flags; 301 302 *event_flags = RING_EVENT_FLAGS_DISABLE; 303 } 304 305 306 /** 307 * Tell the backend not to interrupt us. 308 */ 309 static inline void 310 virtqueue_disable_intr(struct virtqueue *vq) 311 { 312 if (vtpci_packed_queue(vq->hw)) 313 virtqueue_disable_intr_packed(vq); 314 else 315 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 316 } 317 318 /** 319 * Tell the backend to interrupt. Implementation for packed virtqueues. 320 */ 321 static inline void 322 virtqueue_enable_intr_packed(struct virtqueue *vq) 323 { 324 uint16_t *event_flags = &vq->ring_packed.driver_event->desc_event_flags; 325 326 327 if (vq->event_flags_shadow == RING_EVENT_FLAGS_DISABLE) { 328 virtio_wmb(); 329 vq->event_flags_shadow = RING_EVENT_FLAGS_ENABLE; 330 *event_flags = vq->event_flags_shadow; 331 } 332 } 333 334 /** 335 * Tell the backend to interrupt. Implementation for split virtqueues. 336 */ 337 static inline void 338 virtqueue_enable_intr_split(struct virtqueue *vq) 339 { 340 vq->vq_ring.avail->flags &= (~VRING_AVAIL_F_NO_INTERRUPT); 341 } 342 343 /** 344 * Tell the backend to interrupt us. 345 */ 346 static inline void 347 virtqueue_enable_intr(struct virtqueue *vq) 348 { 349 if (vtpci_packed_queue(vq->hw)) 350 virtqueue_enable_intr_packed(vq); 351 else 352 virtqueue_enable_intr_split(vq); 353 } 354 355 /** 356 * Dump virtqueue internal structures, for debug purpose only. 357 */ 358 void virtqueue_dump(struct virtqueue *vq); 359 /** 360 * Get all mbufs to be freed. 361 */ 362 struct rte_mbuf *virtqueue_detach_unused(struct virtqueue *vq); 363 364 /* Flush the elements in the used ring. */ 365 void virtqueue_rxvq_flush(struct virtqueue *vq); 366 367 static inline int 368 virtqueue_full(const struct virtqueue *vq) 369 { 370 return vq->vq_free_cnt == 0; 371 } 372 373 static inline int 374 virtio_get_queue_type(struct virtio_hw *hw, uint16_t vtpci_queue_idx) 375 { 376 if (vtpci_queue_idx == hw->max_queue_pairs * 2) 377 return VTNET_CQ; 378 else if (vtpci_queue_idx % 2 == 0) 379 return VTNET_RQ; 380 else 381 return VTNET_TQ; 382 } 383 384 #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx)) 385 386 void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx); 387 void vq_ring_free_chain_packed(struct virtqueue *vq, uint16_t used_idx); 388 void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, 389 uint16_t num); 390 391 static inline void 392 vq_update_avail_idx(struct virtqueue *vq) 393 { 394 virtio_wmb(); 395 vq->vq_ring.avail->idx = vq->vq_avail_idx; 396 } 397 398 static inline void 399 vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx) 400 { 401 uint16_t avail_idx; 402 /* 403 * Place the head of the descriptor chain into the next slot and make 404 * it usable to the host. The chain is made available now rather than 405 * deferring to virtqueue_notify() in the hopes that if the host is 406 * currently running on another CPU, we can keep it processing the new 407 * descriptor. 408 */ 409 avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1)); 410 if (unlikely(vq->vq_ring.avail->ring[avail_idx] != desc_idx)) 411 vq->vq_ring.avail->ring[avail_idx] = desc_idx; 412 vq->vq_avail_idx++; 413 } 414 415 static inline int 416 virtqueue_kick_prepare(struct virtqueue *vq) 417 { 418 return !(vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY); 419 } 420 421 static inline int 422 virtqueue_kick_prepare_packed(struct virtqueue *vq) 423 { 424 uint16_t flags; 425 426 virtio_mb(); 427 flags = vq->ring_packed.device_event->desc_event_flags; 428 429 return flags != RING_EVENT_FLAGS_DISABLE; 430 } 431 432 static inline void 433 virtqueue_notify(struct virtqueue *vq) 434 { 435 /* 436 * Ensure updated avail->idx is visible to host. 437 * For virtio on IA, the notificaiton is through io port operation 438 * which is a serialization instruction itself. 439 */ 440 VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq); 441 } 442 443 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 444 #define VIRTQUEUE_DUMP(vq) do { \ 445 uint16_t used_idx, nused; \ 446 used_idx = (vq)->vq_ring.used->idx; \ 447 nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \ 448 if (vtpci_packed_queue((vq)->hw)) { \ 449 PMD_INIT_LOG(DEBUG, \ 450 "VQ: - size=%d; free=%d; used_cons_idx=%d; avail_idx=%d;" \ 451 "VQ: - avail_wrap_counter=%d; used_wrap_counter=%d", \ 452 (vq)->vq_nentries, (vq)->vq_free_cnt, (vq)->vq_used_cons_idx, \ 453 (vq)->vq_avail_idx, (vq)->avail_wrap_counter, \ 454 (vq)->used_wrap_counter); \ 455 break; \ 456 } \ 457 PMD_INIT_LOG(DEBUG, \ 458 "VQ: - size=%d; free=%d; used=%d; desc_head_idx=%d;" \ 459 " avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \ 460 " avail.flags=0x%x; used.flags=0x%x", \ 461 (vq)->vq_nentries, (vq)->vq_free_cnt, nused, \ 462 (vq)->vq_desc_head_idx, (vq)->vq_ring.avail->idx, \ 463 (vq)->vq_used_cons_idx, (vq)->vq_ring.used->idx, \ 464 (vq)->vq_ring.avail->flags, (vq)->vq_ring.used->flags); \ 465 } while (0) 466 #else 467 #define VIRTQUEUE_DUMP(vq) do { } while (0) 468 #endif 469 470 #endif /* _VIRTQUEUE_H_ */ 471