1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #ifndef _VIRTQUEUE_H_ 6 #define _VIRTQUEUE_H_ 7 8 #include <stdint.h> 9 10 #include <rte_atomic.h> 11 #include <rte_memory.h> 12 #include <rte_mempool.h> 13 #include <rte_net.h> 14 15 #include "virtio.h" 16 #include "virtio_ring.h" 17 #include "virtio_logs.h" 18 #include "virtio_rxtx.h" 19 20 struct rte_mbuf; 21 22 #define DEFAULT_TX_FREE_THRESH 32 23 #define DEFAULT_RX_FREE_THRESH 32 24 25 #define VIRTIO_MBUF_BURST_SZ 64 26 /* 27 * Per virtio_ring.h in Linux. 28 * For virtio_pci on SMP, we don't need to order with respect to MMIO 29 * accesses through relaxed memory I/O windows, so thread_fence is 30 * sufficient. 31 * 32 * For using virtio to talk to real devices (eg. vDPA) we do need real 33 * barriers. 34 */ 35 static inline void 36 virtio_mb(uint8_t weak_barriers) 37 { 38 if (weak_barriers) 39 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 40 else 41 rte_mb(); 42 } 43 44 static inline void 45 virtio_rmb(uint8_t weak_barriers) 46 { 47 if (weak_barriers) 48 rte_atomic_thread_fence(__ATOMIC_ACQUIRE); 49 else 50 rte_io_rmb(); 51 } 52 53 static inline void 54 virtio_wmb(uint8_t weak_barriers) 55 { 56 if (weak_barriers) 57 rte_atomic_thread_fence(__ATOMIC_RELEASE); 58 else 59 rte_io_wmb(); 60 } 61 62 static inline uint16_t 63 virtqueue_fetch_flags_packed(struct vring_packed_desc *dp, 64 uint8_t weak_barriers) 65 { 66 uint16_t flags; 67 68 if (weak_barriers) { 69 /* x86 prefers to using rte_io_rmb over __atomic_load_n as it reports 70 * a better perf(~1.5%), which comes from the saved branch by the compiler. 71 * The if and else branch are identical on the platforms except Arm. 72 */ 73 #ifdef RTE_ARCH_ARM 74 flags = __atomic_load_n(&dp->flags, __ATOMIC_ACQUIRE); 75 #else 76 flags = dp->flags; 77 rte_io_rmb(); 78 #endif 79 } else { 80 flags = dp->flags; 81 rte_io_rmb(); 82 } 83 84 return flags; 85 } 86 87 static inline void 88 virtqueue_store_flags_packed(struct vring_packed_desc *dp, 89 uint16_t flags, uint8_t weak_barriers) 90 { 91 if (weak_barriers) { 92 /* x86 prefers to using rte_io_wmb over __atomic_store_n as it reports 93 * a better perf(~1.5%), which comes from the saved branch by the compiler. 94 * The if and else branch are identical on the platforms except Arm. 95 */ 96 #ifdef RTE_ARCH_ARM 97 __atomic_store_n(&dp->flags, flags, __ATOMIC_RELEASE); 98 #else 99 rte_io_wmb(); 100 dp->flags = flags; 101 #endif 102 } else { 103 rte_io_wmb(); 104 dp->flags = flags; 105 } 106 } 107 108 #ifdef RTE_PMD_PACKET_PREFETCH 109 #define rte_packet_prefetch(p) rte_prefetch1(p) 110 #else 111 #define rte_packet_prefetch(p) do {} while(0) 112 #endif 113 114 #define VIRTQUEUE_MAX_NAME_SZ 32 115 116 /** 117 * Return the IOVA (or virtual address in case of virtio-user) of mbuf 118 * data buffer. 119 * 120 * The address is firstly casted to the word size (sizeof(uintptr_t)) 121 * before casting it to uint64_t. This is to make it work with different 122 * combination of word size (64 bit and 32 bit) and virtio device 123 * (virtio-pci and virtio-user). 124 */ 125 #define VIRTIO_MBUF_ADDR(mb, vq) \ 126 ((uint64_t)(*(uintptr_t *)((uintptr_t)(mb) + (vq)->mbuf_addr_offset))) 127 128 /** 129 * Return the physical address (or virtual address in case of 130 * virtio-user) of mbuf data buffer, taking care of mbuf data offset 131 */ 132 #define VIRTIO_MBUF_DATA_DMA_ADDR(mb, vq) \ 133 (VIRTIO_MBUF_ADDR(mb, vq) + (mb)->data_off) 134 135 #define VTNET_SQ_RQ_QUEUE_IDX 0 136 #define VTNET_SQ_TQ_QUEUE_IDX 1 137 #define VTNET_SQ_CQ_QUEUE_IDX 2 138 139 enum { VTNET_RQ = 0, VTNET_TQ = 1, VTNET_CQ = 2 }; 140 /** 141 * The maximum virtqueue size is 2^15. Use that value as the end of 142 * descriptor chain terminator since it will never be a valid index 143 * in the descriptor table. This is used to verify we are correctly 144 * handling vq_free_cnt. 145 */ 146 #define VQ_RING_DESC_CHAIN_END 32768 147 148 /** 149 * Control the RX mode, ie. promiscuous, allmulti, etc... 150 * All commands require an "out" sg entry containing a 1 byte 151 * state value, zero = disable, non-zero = enable. Commands 152 * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature. 153 * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA. 154 */ 155 #define VIRTIO_NET_CTRL_RX 0 156 #define VIRTIO_NET_CTRL_RX_PROMISC 0 157 #define VIRTIO_NET_CTRL_RX_ALLMULTI 1 158 #define VIRTIO_NET_CTRL_RX_ALLUNI 2 159 #define VIRTIO_NET_CTRL_RX_NOMULTI 3 160 #define VIRTIO_NET_CTRL_RX_NOUNI 4 161 #define VIRTIO_NET_CTRL_RX_NOBCAST 5 162 163 /** 164 * Control the MAC 165 * 166 * The MAC filter table is managed by the hypervisor, the guest should 167 * assume the size is infinite. Filtering should be considered 168 * non-perfect, ie. based on hypervisor resources, the guest may 169 * received packets from sources not specified in the filter list. 170 * 171 * In addition to the class/cmd header, the TABLE_SET command requires 172 * two out scatterlists. Each contains a 4 byte count of entries followed 173 * by a concatenated byte stream of the ETH_ALEN MAC addresses. The 174 * first sg list contains unicast addresses, the second is for multicast. 175 * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature 176 * is available. 177 * 178 * The ADDR_SET command requests one out scatterlist, it contains a 179 * 6 bytes MAC address. This functionality is present if the 180 * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available. 181 */ 182 struct virtio_net_ctrl_mac { 183 uint32_t entries; 184 uint8_t macs[][RTE_ETHER_ADDR_LEN]; 185 } __rte_packed; 186 187 #define VIRTIO_NET_CTRL_MAC 1 188 #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 189 #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 190 191 /** 192 * Control VLAN filtering 193 * 194 * The VLAN filter table is controlled via a simple ADD/DEL interface. 195 * VLAN IDs not added may be filtered by the hypervisor. Del is the 196 * opposite of add. Both commands expect an out entry containing a 2 197 * byte VLAN ID. VLAN filtering is available with the 198 * VIRTIO_NET_F_CTRL_VLAN feature bit. 199 */ 200 #define VIRTIO_NET_CTRL_VLAN 2 201 #define VIRTIO_NET_CTRL_VLAN_ADD 0 202 #define VIRTIO_NET_CTRL_VLAN_DEL 1 203 204 /** 205 * RSS control 206 * 207 * The RSS feature configuration message is sent by the driver when 208 * VIRTIO_NET_F_RSS has been negotiated. It provides the device with 209 * hash types to use, hash key and indirection table. In this 210 * implementation, the driver only supports fixed key length (40B) 211 * and indirection table size (128 entries). 212 */ 213 #define VIRTIO_NET_RSS_RETA_SIZE 128 214 #define VIRTIO_NET_RSS_KEY_SIZE 40 215 216 struct virtio_net_ctrl_rss { 217 uint32_t hash_types; 218 uint16_t indirection_table_mask; 219 uint16_t unclassified_queue; 220 uint16_t indirection_table[VIRTIO_NET_RSS_RETA_SIZE]; 221 uint16_t max_tx_vq; 222 uint8_t hash_key_length; 223 uint8_t hash_key_data[VIRTIO_NET_RSS_KEY_SIZE]; 224 }; 225 226 /* 227 * Control link announce acknowledgement 228 * 229 * The command VIRTIO_NET_CTRL_ANNOUNCE_ACK is used to indicate that 230 * driver has recevied the notification; device would clear the 231 * VIRTIO_NET_S_ANNOUNCE bit in the status field after it receives 232 * this command. 233 */ 234 #define VIRTIO_NET_CTRL_ANNOUNCE 3 235 #define VIRTIO_NET_CTRL_ANNOUNCE_ACK 0 236 237 struct virtio_net_ctrl_hdr { 238 uint8_t class; 239 uint8_t cmd; 240 } __rte_packed; 241 242 typedef uint8_t virtio_net_ctrl_ack; 243 244 #define VIRTIO_NET_OK 0 245 #define VIRTIO_NET_ERR 1 246 247 #define VIRTIO_MAX_CTRL_DATA 2048 248 249 struct virtio_pmd_ctrl { 250 struct virtio_net_ctrl_hdr hdr; 251 virtio_net_ctrl_ack status; 252 uint8_t data[VIRTIO_MAX_CTRL_DATA]; 253 }; 254 255 struct vq_desc_extra { 256 void *cookie; 257 uint16_t ndescs; 258 uint16_t next; 259 }; 260 261 #define virtnet_rxq_to_vq(rxvq) container_of(rxvq, struct virtqueue, rxq) 262 #define virtnet_txq_to_vq(txvq) container_of(txvq, struct virtqueue, txq) 263 #define virtnet_cq_to_vq(cvq) container_of(cvq, struct virtqueue, cq) 264 265 struct virtqueue { 266 struct virtio_hw *hw; /**< virtio_hw structure pointer. */ 267 union { 268 struct { 269 /**< vring keeping desc, used and avail */ 270 struct vring ring; 271 } vq_split; 272 273 struct { 274 /**< vring keeping descs and events */ 275 struct vring_packed ring; 276 bool used_wrap_counter; 277 uint16_t cached_flags; /**< cached flags for descs */ 278 uint16_t event_flags_shadow; 279 } vq_packed; 280 }; 281 282 uint16_t vq_used_cons_idx; /**< last consumed descriptor */ 283 uint16_t vq_nentries; /**< vring desc numbers */ 284 uint16_t vq_free_cnt; /**< num of desc available */ 285 uint16_t vq_avail_idx; /**< sync until needed */ 286 uint16_t vq_free_thresh; /**< free threshold */ 287 288 /** 289 * Head of the free chain in the descriptor table. If 290 * there are no free descriptors, this will be set to 291 * VQ_RING_DESC_CHAIN_END. 292 */ 293 uint16_t vq_desc_head_idx; 294 uint16_t vq_desc_tail_idx; 295 uint16_t vq_queue_index; /**< PCI queue index */ 296 297 void *vq_ring_virt_mem; /**< linear address of vring*/ 298 unsigned int vq_ring_size; 299 uint16_t mbuf_addr_offset; 300 301 union { 302 struct virtnet_rx rxq; 303 struct virtnet_tx txq; 304 struct virtnet_ctl cq; 305 }; 306 307 rte_iova_t vq_ring_mem; /**< physical address of vring, 308 * or virtual address for virtio_user. */ 309 310 uint16_t *notify_addr; 311 struct rte_mbuf **sw_ring; /**< RX software ring. */ 312 struct vq_desc_extra vq_descx[0]; 313 }; 314 315 /* If multiqueue is provided by host, then we suppport it. */ 316 #define VIRTIO_NET_CTRL_MQ 4 317 318 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 319 #define VIRTIO_NET_CTRL_MQ_RSS_CONFIG 1 320 321 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 322 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 323 324 /** 325 * This is the first element of the scatter-gather list. If you don't 326 * specify GSO or CSUM features, you can simply ignore the header. 327 */ 328 struct virtio_net_hdr { 329 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /**< Use csum_start,csum_offset*/ 330 #define VIRTIO_NET_HDR_F_DATA_VALID 2 /**< Checksum is valid */ 331 uint8_t flags; 332 #define VIRTIO_NET_HDR_GSO_NONE 0 /**< Not a GSO frame */ 333 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /**< GSO frame, IPv4 TCP (TSO) */ 334 #define VIRTIO_NET_HDR_GSO_UDP 3 /**< GSO frame, IPv4 UDP (UFO) */ 335 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /**< GSO frame, IPv6 TCP */ 336 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /**< TCP has ECN set */ 337 uint8_t gso_type; 338 uint16_t hdr_len; /**< Ethernet + IP + tcp/udp hdrs */ 339 uint16_t gso_size; /**< Bytes to append to hdr_len per frame */ 340 uint16_t csum_start; /**< Position to start checksumming from */ 341 uint16_t csum_offset; /**< Offset after that to place checksum */ 342 }; 343 344 /** 345 * This is the version of the header to use when the MRG_RXBUF 346 * feature has been negotiated. 347 */ 348 struct virtio_net_hdr_mrg_rxbuf { 349 struct virtio_net_hdr hdr; 350 uint16_t num_buffers; /**< Number of merged rx buffers */ 351 }; 352 353 /* Region reserved to allow for transmit header and indirect ring */ 354 #define VIRTIO_MAX_TX_INDIRECT 8 355 struct virtio_tx_region { 356 struct virtio_net_hdr_mrg_rxbuf tx_hdr; 357 union { 358 struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT]; 359 struct vring_packed_desc 360 tx_packed_indir[VIRTIO_MAX_TX_INDIRECT]; 361 } __rte_aligned(16); 362 }; 363 364 static inline int 365 desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq) 366 { 367 uint16_t used, avail, flags; 368 369 flags = virtqueue_fetch_flags_packed(desc, vq->hw->weak_barriers); 370 used = !!(flags & VRING_PACKED_DESC_F_USED); 371 avail = !!(flags & VRING_PACKED_DESC_F_AVAIL); 372 373 return avail == used && used == vq->vq_packed.used_wrap_counter; 374 } 375 376 static inline void 377 vring_desc_init_packed(struct virtqueue *vq, int n) 378 { 379 int i; 380 for (i = 0; i < n - 1; i++) { 381 vq->vq_packed.ring.desc[i].id = i; 382 vq->vq_descx[i].next = i + 1; 383 } 384 vq->vq_packed.ring.desc[i].id = i; 385 vq->vq_descx[i].next = VQ_RING_DESC_CHAIN_END; 386 } 387 388 /* Chain all the descriptors in the ring with an END */ 389 static inline void 390 vring_desc_init_split(struct vring_desc *dp, uint16_t n) 391 { 392 uint16_t i; 393 394 for (i = 0; i < n - 1; i++) 395 dp[i].next = (uint16_t)(i + 1); 396 dp[i].next = VQ_RING_DESC_CHAIN_END; 397 } 398 399 static inline void 400 vring_desc_init_indirect_packed(struct vring_packed_desc *dp, int n) 401 { 402 int i; 403 for (i = 0; i < n; i++) { 404 dp[i].id = (uint16_t)i; 405 dp[i].flags = VRING_DESC_F_WRITE; 406 } 407 } 408 409 /** 410 * Tell the backend not to interrupt us. Implementation for packed virtqueues. 411 */ 412 static inline void 413 virtqueue_disable_intr_packed(struct virtqueue *vq) 414 { 415 if (vq->vq_packed.event_flags_shadow != RING_EVENT_FLAGS_DISABLE) { 416 vq->vq_packed.event_flags_shadow = RING_EVENT_FLAGS_DISABLE; 417 vq->vq_packed.ring.driver->desc_event_flags = 418 vq->vq_packed.event_flags_shadow; 419 } 420 } 421 422 /** 423 * Tell the backend not to interrupt us. Implementation for split virtqueues. 424 */ 425 static inline void 426 virtqueue_disable_intr_split(struct virtqueue *vq) 427 { 428 vq->vq_split.ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 429 } 430 431 /** 432 * Tell the backend not to interrupt us. 433 */ 434 static inline void 435 virtqueue_disable_intr(struct virtqueue *vq) 436 { 437 if (virtio_with_packed_queue(vq->hw)) 438 virtqueue_disable_intr_packed(vq); 439 else 440 virtqueue_disable_intr_split(vq); 441 } 442 443 /** 444 * Tell the backend to interrupt. Implementation for packed virtqueues. 445 */ 446 static inline void 447 virtqueue_enable_intr_packed(struct virtqueue *vq) 448 { 449 if (vq->vq_packed.event_flags_shadow == RING_EVENT_FLAGS_DISABLE) { 450 vq->vq_packed.event_flags_shadow = RING_EVENT_FLAGS_ENABLE; 451 vq->vq_packed.ring.driver->desc_event_flags = 452 vq->vq_packed.event_flags_shadow; 453 } 454 } 455 456 /** 457 * Tell the backend to interrupt. Implementation for split virtqueues. 458 */ 459 static inline void 460 virtqueue_enable_intr_split(struct virtqueue *vq) 461 { 462 vq->vq_split.ring.avail->flags &= (~VRING_AVAIL_F_NO_INTERRUPT); 463 } 464 465 /** 466 * Tell the backend to interrupt us. 467 */ 468 static inline void 469 virtqueue_enable_intr(struct virtqueue *vq) 470 { 471 if (virtio_with_packed_queue(vq->hw)) 472 virtqueue_enable_intr_packed(vq); 473 else 474 virtqueue_enable_intr_split(vq); 475 } 476 477 /** 478 * Dump virtqueue internal structures, for debug purpose only. 479 */ 480 void virtqueue_dump(struct virtqueue *vq); 481 /** 482 * Get all mbufs to be freed. 483 */ 484 struct rte_mbuf *virtqueue_detach_unused(struct virtqueue *vq); 485 486 /* Flush the elements in the used ring. */ 487 void virtqueue_rxvq_flush(struct virtqueue *vq); 488 489 int virtqueue_rxvq_reset_packed(struct virtqueue *vq); 490 491 int virtqueue_txvq_reset_packed(struct virtqueue *vq); 492 493 static inline int 494 virtqueue_full(const struct virtqueue *vq) 495 { 496 return vq->vq_free_cnt == 0; 497 } 498 499 static inline int 500 virtio_get_queue_type(struct virtio_hw *hw, uint16_t vq_idx) 501 { 502 if (vq_idx == hw->max_queue_pairs * 2) 503 return VTNET_CQ; 504 else if (vq_idx % 2 == 0) 505 return VTNET_RQ; 506 else 507 return VTNET_TQ; 508 } 509 510 /* virtqueue_nused has load-acquire or rte_io_rmb insed */ 511 static inline uint16_t 512 virtqueue_nused(const struct virtqueue *vq) 513 { 514 uint16_t idx; 515 516 if (vq->hw->weak_barriers) { 517 /** 518 * x86 prefers to using rte_smp_rmb over __atomic_load_n as it 519 * reports a slightly better perf, which comes from the saved 520 * branch by the compiler. 521 * The if and else branches are identical with the smp and io 522 * barriers both defined as compiler barriers on x86. 523 */ 524 #ifdef RTE_ARCH_X86_64 525 idx = vq->vq_split.ring.used->idx; 526 rte_smp_rmb(); 527 #else 528 idx = __atomic_load_n(&(vq)->vq_split.ring.used->idx, 529 __ATOMIC_ACQUIRE); 530 #endif 531 } else { 532 idx = vq->vq_split.ring.used->idx; 533 rte_io_rmb(); 534 } 535 return idx - vq->vq_used_cons_idx; 536 } 537 538 void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx); 539 void vq_ring_free_chain_packed(struct virtqueue *vq, uint16_t used_idx); 540 void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, 541 uint16_t num); 542 543 static inline void 544 vq_update_avail_idx(struct virtqueue *vq) 545 { 546 if (vq->hw->weak_barriers) { 547 /* x86 prefers to using rte_smp_wmb over __atomic_store_n as 548 * it reports a slightly better perf, which comes from the 549 * saved branch by the compiler. 550 * The if and else branches are identical with the smp and 551 * io barriers both defined as compiler barriers on x86. 552 */ 553 #ifdef RTE_ARCH_X86_64 554 rte_smp_wmb(); 555 vq->vq_split.ring.avail->idx = vq->vq_avail_idx; 556 #else 557 __atomic_store_n(&vq->vq_split.ring.avail->idx, 558 vq->vq_avail_idx, __ATOMIC_RELEASE); 559 #endif 560 } else { 561 rte_io_wmb(); 562 vq->vq_split.ring.avail->idx = vq->vq_avail_idx; 563 } 564 } 565 566 static inline void 567 vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx) 568 { 569 uint16_t avail_idx; 570 /* 571 * Place the head of the descriptor chain into the next slot and make 572 * it usable to the host. The chain is made available now rather than 573 * deferring to virtqueue_notify() in the hopes that if the host is 574 * currently running on another CPU, we can keep it processing the new 575 * descriptor. 576 */ 577 avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1)); 578 if (unlikely(vq->vq_split.ring.avail->ring[avail_idx] != desc_idx)) 579 vq->vq_split.ring.avail->ring[avail_idx] = desc_idx; 580 vq->vq_avail_idx++; 581 } 582 583 static inline int 584 virtqueue_kick_prepare(struct virtqueue *vq) 585 { 586 /* 587 * Ensure updated avail->idx is visible to vhost before reading 588 * the used->flags. 589 */ 590 virtio_mb(vq->hw->weak_barriers); 591 return !(vq->vq_split.ring.used->flags & VRING_USED_F_NO_NOTIFY); 592 } 593 594 static inline int 595 virtqueue_kick_prepare_packed(struct virtqueue *vq) 596 { 597 uint16_t flags; 598 599 /* 600 * Ensure updated data is visible to vhost before reading the flags. 601 */ 602 virtio_mb(vq->hw->weak_barriers); 603 flags = vq->vq_packed.ring.device->desc_event_flags; 604 605 return flags != RING_EVENT_FLAGS_DISABLE; 606 } 607 608 /* 609 * virtqueue_kick_prepare*() or the virtio_wmb() should be called 610 * before this function to be sure that all the data is visible to vhost. 611 */ 612 static inline void 613 virtqueue_notify(struct virtqueue *vq) 614 { 615 VIRTIO_OPS(vq->hw)->notify_queue(vq->hw, vq); 616 } 617 618 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP 619 #define VIRTQUEUE_DUMP(vq) do { \ 620 uint16_t used_idx, nused; \ 621 used_idx = __atomic_load_n(&(vq)->vq_split.ring.used->idx, \ 622 __ATOMIC_RELAXED); \ 623 nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \ 624 if (virtio_with_packed_queue((vq)->hw)) { \ 625 PMD_INIT_LOG(DEBUG, \ 626 "VQ: - size=%d; free=%d; used_cons_idx=%d; avail_idx=%d;" \ 627 " cached_flags=0x%x; used_wrap_counter=%d", \ 628 (vq)->vq_nentries, (vq)->vq_free_cnt, (vq)->vq_used_cons_idx, \ 629 (vq)->vq_avail_idx, (vq)->vq_packed.cached_flags, \ 630 (vq)->vq_packed.used_wrap_counter); \ 631 break; \ 632 } \ 633 PMD_INIT_LOG(DEBUG, \ 634 "VQ: - size=%d; free=%d; used=%d; desc_head_idx=%d;" \ 635 " avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \ 636 " avail.flags=0x%x; used.flags=0x%x", \ 637 (vq)->vq_nentries, (vq)->vq_free_cnt, nused, (vq)->vq_desc_head_idx, \ 638 (vq)->vq_split.ring.avail->idx, (vq)->vq_used_cons_idx, \ 639 __atomic_load_n(&(vq)->vq_split.ring.used->idx, __ATOMIC_RELAXED), \ 640 (vq)->vq_split.ring.avail->flags, (vq)->vq_split.ring.used->flags); \ 641 } while (0) 642 #else 643 #define VIRTQUEUE_DUMP(vq) do { } while (0) 644 #endif 645 646 /* avoid write operation when necessary, to lessen cache issues */ 647 #define ASSIGN_UNLESS_EQUAL(var, val) do { \ 648 typeof(var) *const var_ = &(var); \ 649 typeof(val) const val_ = (val); \ 650 if (*var_ != val_) \ 651 *var_ = val_; \ 652 } while (0) 653 654 #define virtqueue_clear_net_hdr(hdr) do { \ 655 typeof(hdr) hdr_ = (hdr); \ 656 ASSIGN_UNLESS_EQUAL((hdr_)->csum_start, 0); \ 657 ASSIGN_UNLESS_EQUAL((hdr_)->csum_offset, 0); \ 658 ASSIGN_UNLESS_EQUAL((hdr_)->flags, 0); \ 659 ASSIGN_UNLESS_EQUAL((hdr_)->gso_type, 0); \ 660 ASSIGN_UNLESS_EQUAL((hdr_)->gso_size, 0); \ 661 ASSIGN_UNLESS_EQUAL((hdr_)->hdr_len, 0); \ 662 } while (0) 663 664 static inline void 665 virtqueue_xmit_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *cookie) 666 { 667 uint64_t csum_l4 = cookie->ol_flags & RTE_MBUF_F_TX_L4_MASK; 668 uint16_t o_l23_len = (cookie->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) ? 669 cookie->outer_l2_len + cookie->outer_l3_len : 0; 670 671 if (cookie->ol_flags & RTE_MBUF_F_TX_TCP_SEG) 672 csum_l4 |= RTE_MBUF_F_TX_TCP_CKSUM; 673 674 switch (csum_l4) { 675 case RTE_MBUF_F_TX_UDP_CKSUM: 676 hdr->csum_start = o_l23_len + cookie->l2_len + cookie->l3_len; 677 hdr->csum_offset = offsetof(struct rte_udp_hdr, dgram_cksum); 678 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 679 break; 680 681 case RTE_MBUF_F_TX_TCP_CKSUM: 682 hdr->csum_start = o_l23_len + cookie->l2_len + cookie->l3_len; 683 hdr->csum_offset = offsetof(struct rte_tcp_hdr, cksum); 684 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 685 break; 686 687 default: 688 ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0); 689 ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0); 690 ASSIGN_UNLESS_EQUAL(hdr->flags, 0); 691 break; 692 } 693 694 /* TCP Segmentation Offload */ 695 if (cookie->ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 696 hdr->gso_type = (cookie->ol_flags & RTE_MBUF_F_TX_IPV6) ? 697 VIRTIO_NET_HDR_GSO_TCPV6 : 698 VIRTIO_NET_HDR_GSO_TCPV4; 699 hdr->gso_size = cookie->tso_segsz; 700 hdr->hdr_len = o_l23_len + cookie->l2_len + cookie->l3_len + 701 cookie->l4_len; 702 } else { 703 ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0); 704 ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0); 705 ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0); 706 } 707 } 708 709 static inline void 710 virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie, 711 uint16_t needed, int use_indirect, int can_push, 712 int in_order) 713 { 714 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr; 715 struct vq_desc_extra *dxp; 716 struct virtqueue *vq = virtnet_txq_to_vq(txvq); 717 struct vring_packed_desc *start_dp, *head_dp; 718 uint16_t idx, id, head_idx, head_flags; 719 int16_t head_size = vq->hw->vtnet_hdr_size; 720 struct virtio_net_hdr *hdr; 721 uint16_t prev; 722 bool prepend_header = false; 723 uint16_t seg_num = cookie->nb_segs; 724 725 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; 726 727 dxp = &vq->vq_descx[id]; 728 dxp->ndescs = needed; 729 dxp->cookie = cookie; 730 731 head_idx = vq->vq_avail_idx; 732 idx = head_idx; 733 prev = head_idx; 734 start_dp = vq->vq_packed.ring.desc; 735 736 head_dp = &vq->vq_packed.ring.desc[idx]; 737 head_flags = cookie->next ? VRING_DESC_F_NEXT : 0; 738 head_flags |= vq->vq_packed.cached_flags; 739 740 if (can_push) { 741 /* prepend cannot fail, checked by caller */ 742 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *, 743 -head_size); 744 prepend_header = true; 745 746 /* if offload disabled, it is not zeroed below, do it now */ 747 if (!vq->hw->has_tx_offload) 748 virtqueue_clear_net_hdr(hdr); 749 } else if (use_indirect) { 750 /* setup tx ring slot to point to indirect 751 * descriptor list stored in reserved region. 752 * 753 * the first slot in indirect ring is already preset 754 * to point to the header in reserved region 755 */ 756 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 757 RTE_PTR_DIFF(&txr[idx].tx_packed_indir, txr); 758 start_dp[idx].len = (seg_num + 1) * 759 sizeof(struct vring_packed_desc); 760 /* Packed descriptor id needs to be restored when inorder. */ 761 if (in_order) 762 start_dp[idx].id = idx; 763 /* reset flags for indirect desc */ 764 head_flags = VRING_DESC_F_INDIRECT; 765 head_flags |= vq->vq_packed.cached_flags; 766 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 767 768 /* loop below will fill in rest of the indirect elements */ 769 start_dp = txr[idx].tx_packed_indir; 770 idx = 1; 771 } else { 772 /* setup first tx ring slot to point to header 773 * stored in reserved region. 774 */ 775 start_dp[idx].addr = txvq->virtio_net_hdr_mem + 776 RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); 777 start_dp[idx].len = vq->hw->vtnet_hdr_size; 778 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr; 779 idx++; 780 if (idx >= vq->vq_nentries) { 781 idx -= vq->vq_nentries; 782 vq->vq_packed.cached_flags ^= 783 VRING_PACKED_DESC_F_AVAIL_USED; 784 } 785 } 786 787 if (vq->hw->has_tx_offload) 788 virtqueue_xmit_offload(hdr, cookie); 789 790 do { 791 uint16_t flags; 792 793 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq); 794 start_dp[idx].len = cookie->data_len; 795 if (prepend_header) { 796 start_dp[idx].addr -= head_size; 797 start_dp[idx].len += head_size; 798 prepend_header = false; 799 } 800 801 if (likely(idx != head_idx)) { 802 flags = cookie->next ? VRING_DESC_F_NEXT : 0; 803 flags |= vq->vq_packed.cached_flags; 804 start_dp[idx].flags = flags; 805 } 806 prev = idx; 807 idx++; 808 if (idx >= vq->vq_nentries) { 809 idx -= vq->vq_nentries; 810 vq->vq_packed.cached_flags ^= 811 VRING_PACKED_DESC_F_AVAIL_USED; 812 } 813 } while ((cookie = cookie->next) != NULL); 814 815 start_dp[prev].id = id; 816 817 if (use_indirect) { 818 idx = head_idx; 819 if (++idx >= vq->vq_nentries) { 820 idx -= vq->vq_nentries; 821 vq->vq_packed.cached_flags ^= 822 VRING_PACKED_DESC_F_AVAIL_USED; 823 } 824 } 825 826 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); 827 vq->vq_avail_idx = idx; 828 829 if (!in_order) { 830 vq->vq_desc_head_idx = dxp->next; 831 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) 832 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END; 833 } 834 835 virtqueue_store_flags_packed(head_dp, head_flags, 836 vq->hw->weak_barriers); 837 } 838 839 static void 840 vq_ring_free_id_packed(struct virtqueue *vq, uint16_t id) 841 { 842 struct vq_desc_extra *dxp; 843 844 dxp = &vq->vq_descx[id]; 845 vq->vq_free_cnt += dxp->ndescs; 846 847 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) 848 vq->vq_desc_head_idx = id; 849 else 850 vq->vq_descx[vq->vq_desc_tail_idx].next = id; 851 852 vq->vq_desc_tail_idx = id; 853 dxp->next = VQ_RING_DESC_CHAIN_END; 854 } 855 856 static void 857 virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, uint16_t num) 858 { 859 uint16_t used_idx, id, curr_id, free_cnt = 0; 860 uint16_t size = vq->vq_nentries; 861 struct vring_packed_desc *desc = vq->vq_packed.ring.desc; 862 struct vq_desc_extra *dxp; 863 int nb = num; 864 865 used_idx = vq->vq_used_cons_idx; 866 /* desc_is_used has a load-acquire or rte_io_rmb inside 867 * and wait for used desc in virtqueue. 868 */ 869 while (nb > 0 && desc_is_used(&desc[used_idx], vq)) { 870 id = desc[used_idx].id; 871 do { 872 curr_id = used_idx; 873 dxp = &vq->vq_descx[used_idx]; 874 used_idx += dxp->ndescs; 875 free_cnt += dxp->ndescs; 876 nb -= dxp->ndescs; 877 if (used_idx >= size) { 878 used_idx -= size; 879 vq->vq_packed.used_wrap_counter ^= 1; 880 } 881 if (dxp->cookie != NULL) { 882 rte_pktmbuf_free(dxp->cookie); 883 dxp->cookie = NULL; 884 } 885 } while (curr_id != id); 886 } 887 vq->vq_used_cons_idx = used_idx; 888 vq->vq_free_cnt += free_cnt; 889 } 890 891 static void 892 virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, uint16_t num) 893 { 894 uint16_t used_idx, id; 895 uint16_t size = vq->vq_nentries; 896 struct vring_packed_desc *desc = vq->vq_packed.ring.desc; 897 struct vq_desc_extra *dxp; 898 899 used_idx = vq->vq_used_cons_idx; 900 /* desc_is_used has a load-acquire or rte_io_rmb inside 901 * and wait for used desc in virtqueue. 902 */ 903 while (num-- && desc_is_used(&desc[used_idx], vq)) { 904 id = desc[used_idx].id; 905 dxp = &vq->vq_descx[id]; 906 vq->vq_used_cons_idx += dxp->ndescs; 907 if (vq->vq_used_cons_idx >= size) { 908 vq->vq_used_cons_idx -= size; 909 vq->vq_packed.used_wrap_counter ^= 1; 910 } 911 vq_ring_free_id_packed(vq, id); 912 if (dxp->cookie != NULL) { 913 rte_pktmbuf_free(dxp->cookie); 914 dxp->cookie = NULL; 915 } 916 used_idx = vq->vq_used_cons_idx; 917 } 918 } 919 920 /* Cleanup from completed transmits. */ 921 static inline void 922 virtio_xmit_cleanup_packed(struct virtqueue *vq, uint16_t num, int in_order) 923 { 924 if (in_order) 925 virtio_xmit_cleanup_inorder_packed(vq, num); 926 else 927 virtio_xmit_cleanup_normal_packed(vq, num); 928 } 929 930 static inline void 931 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num) 932 { 933 uint16_t i, used_idx, desc_idx; 934 for (i = 0; i < num; i++) { 935 struct vring_used_elem *uep; 936 struct vq_desc_extra *dxp; 937 938 used_idx = (uint16_t)(vq->vq_used_cons_idx & 939 (vq->vq_nentries - 1)); 940 uep = &vq->vq_split.ring.used->ring[used_idx]; 941 942 desc_idx = (uint16_t)uep->id; 943 dxp = &vq->vq_descx[desc_idx]; 944 vq->vq_used_cons_idx++; 945 vq_ring_free_chain(vq, desc_idx); 946 947 if (dxp->cookie != NULL) { 948 rte_pktmbuf_free(dxp->cookie); 949 dxp->cookie = NULL; 950 } 951 } 952 } 953 954 /* Cleanup from completed inorder transmits. */ 955 static __rte_always_inline void 956 virtio_xmit_cleanup_inorder(struct virtqueue *vq, uint16_t num) 957 { 958 uint16_t i, idx = vq->vq_used_cons_idx; 959 int16_t free_cnt = 0; 960 struct vq_desc_extra *dxp = NULL; 961 962 if (unlikely(num == 0)) 963 return; 964 965 for (i = 0; i < num; i++) { 966 dxp = &vq->vq_descx[idx++ & (vq->vq_nentries - 1)]; 967 free_cnt += dxp->ndescs; 968 if (dxp->cookie != NULL) { 969 rte_pktmbuf_free(dxp->cookie); 970 dxp->cookie = NULL; 971 } 972 } 973 974 vq->vq_free_cnt += free_cnt; 975 vq->vq_used_cons_idx = idx; 976 } 977 #endif /* _VIRTQUEUE_H_ */ 978