1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <linux/vhost.h> 6 #include <linux/virtio_net.h> 7 #include <stdint.h> 8 #include <stdlib.h> 9 #include <pthread.h> 10 #ifdef RTE_LIBRTE_VHOST_NUMA 11 #include <numa.h> 12 #include <numaif.h> 13 #endif 14 15 #include <rte_errno.h> 16 #include <rte_log.h> 17 #include <rte_memory.h> 18 #include <rte_malloc.h> 19 #include <rte_vhost.h> 20 21 #include "iotlb.h" 22 #include "vhost.h" 23 #include "vhost_user.h" 24 25 struct virtio_net *vhost_devices[RTE_MAX_VHOST_DEVICE]; 26 pthread_mutex_t vhost_dev_lock = PTHREAD_MUTEX_INITIALIZER; 27 pthread_mutex_t vhost_dma_lock = PTHREAD_MUTEX_INITIALIZER; 28 29 struct vhost_vq_stats_name_off { 30 char name[RTE_VHOST_STATS_NAME_SIZE]; 31 unsigned int offset; 32 }; 33 34 static const struct vhost_vq_stats_name_off vhost_vq_stat_strings[] = { 35 {"good_packets", offsetof(struct vhost_virtqueue, stats.packets)}, 36 {"good_bytes", offsetof(struct vhost_virtqueue, stats.bytes)}, 37 {"multicast_packets", offsetof(struct vhost_virtqueue, stats.multicast)}, 38 {"broadcast_packets", offsetof(struct vhost_virtqueue, stats.broadcast)}, 39 {"undersize_packets", offsetof(struct vhost_virtqueue, stats.size_bins[0])}, 40 {"size_64_packets", offsetof(struct vhost_virtqueue, stats.size_bins[1])}, 41 {"size_65_127_packets", offsetof(struct vhost_virtqueue, stats.size_bins[2])}, 42 {"size_128_255_packets", offsetof(struct vhost_virtqueue, stats.size_bins[3])}, 43 {"size_256_511_packets", offsetof(struct vhost_virtqueue, stats.size_bins[4])}, 44 {"size_512_1023_packets", offsetof(struct vhost_virtqueue, stats.size_bins[5])}, 45 {"size_1024_1518_packets", offsetof(struct vhost_virtqueue, stats.size_bins[6])}, 46 {"size_1519_max_packets", offsetof(struct vhost_virtqueue, stats.size_bins[7])}, 47 {"guest_notifications", offsetof(struct vhost_virtqueue, stats.guest_notifications)}, 48 {"guest_notifications_offloaded", offsetof(struct vhost_virtqueue, 49 stats.guest_notifications_offloaded)}, 50 {"guest_notifications_error", offsetof(struct vhost_virtqueue, 51 stats.guest_notifications_error)}, 52 {"guest_notifications_suppressed", offsetof(struct vhost_virtqueue, 53 stats.guest_notifications_suppressed)}, 54 {"iotlb_hits", offsetof(struct vhost_virtqueue, stats.iotlb_hits)}, 55 {"iotlb_misses", offsetof(struct vhost_virtqueue, stats.iotlb_misses)}, 56 {"inflight_submitted", offsetof(struct vhost_virtqueue, stats.inflight_submitted)}, 57 {"inflight_completed", offsetof(struct vhost_virtqueue, stats.inflight_completed)}, 58 }; 59 60 #define VHOST_NB_VQ_STATS RTE_DIM(vhost_vq_stat_strings) 61 62 static int 63 vhost_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm) 64 { 65 return dev->backend_ops->iotlb_miss(dev, iova, perm); 66 } 67 68 uint64_t 69 __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, 70 uint64_t iova, uint64_t *size, uint8_t perm) 71 { 72 uint64_t vva, tmp_size; 73 74 if (unlikely(!*size)) 75 return 0; 76 77 tmp_size = *size; 78 79 vva = vhost_user_iotlb_cache_find(dev, iova, &tmp_size, perm); 80 if (tmp_size == *size) { 81 if (dev->flags & VIRTIO_DEV_STATS_ENABLED) 82 vq->stats.iotlb_hits++; 83 return vva; 84 } 85 86 if (dev->flags & VIRTIO_DEV_STATS_ENABLED) 87 vq->stats.iotlb_misses++; 88 89 iova += tmp_size; 90 91 if (!vhost_user_iotlb_pending_miss(dev, iova, perm)) { 92 /* 93 * iotlb_lock is read-locked for a full burst, 94 * but it only protects the iotlb cache. 95 * In case of IOTLB miss, we might block on the socket, 96 * which could cause a deadlock with QEMU if an IOTLB update 97 * is being handled. We can safely unlock here to avoid it. 98 */ 99 vhost_user_iotlb_rd_unlock(vq); 100 101 vhost_user_iotlb_pending_insert(dev, iova, perm); 102 if (vhost_iotlb_miss(dev, iova, perm)) { 103 VHOST_LOG_DATA(dev->ifname, ERR, 104 "IOTLB miss req failed for IOVA 0x%" PRIx64 "\n", 105 iova); 106 vhost_user_iotlb_pending_remove(dev, iova, 1, perm); 107 } 108 109 vhost_user_iotlb_rd_lock(vq); 110 } 111 112 tmp_size = *size; 113 /* Retry in case of VDUSE, as it is synchronous */ 114 vva = vhost_user_iotlb_cache_find(dev, iova, &tmp_size, perm); 115 if (tmp_size == *size) 116 return vva; 117 118 return 0; 119 } 120 121 #define VHOST_LOG_PAGE 4096 122 123 /* 124 * Atomically set a bit in memory. 125 */ 126 static __rte_always_inline void 127 vhost_set_bit(unsigned int nr, volatile uint8_t *addr) 128 { 129 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) 130 /* 131 * __sync_ built-ins are deprecated, but rte_atomic_ ones 132 * are sub-optimized in older GCC versions. 133 */ 134 __sync_fetch_and_or_1(addr, (1U << nr)); 135 #else 136 rte_atomic_fetch_or_explicit((volatile uint8_t __rte_atomic *)addr, (1U << nr), 137 rte_memory_order_relaxed); 138 #endif 139 } 140 141 static __rte_always_inline void 142 vhost_log_page(uint8_t *log_base, uint64_t page) 143 { 144 vhost_set_bit(page % 8, &log_base[page / 8]); 145 } 146 147 void 148 __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) 149 { 150 uint64_t page; 151 152 if (unlikely(!dev->log_base || !len)) 153 return; 154 155 if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) 156 return; 157 158 /* To make sure guest memory updates are committed before logging */ 159 rte_atomic_thread_fence(rte_memory_order_release); 160 161 page = addr / VHOST_LOG_PAGE; 162 while (page * VHOST_LOG_PAGE < addr + len) { 163 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 164 page += 1; 165 } 166 } 167 168 void 169 __vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, 170 uint64_t iova, uint64_t len) 171 { 172 uint64_t hva, gpa, map_len; 173 map_len = len; 174 175 hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); 176 if (map_len != len) { 177 VHOST_LOG_DATA(dev->ifname, ERR, 178 "failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n", 179 iova); 180 return; 181 } 182 183 gpa = hva_to_gpa(dev, hva, len); 184 if (gpa) 185 __vhost_log_write(dev, gpa, len); 186 } 187 188 void 189 __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) 190 { 191 unsigned long *log_base; 192 int i; 193 194 if (unlikely(!dev->log_base)) 195 return; 196 197 /* No cache, nothing to sync */ 198 if (unlikely(!vq->log_cache)) 199 return; 200 201 rte_atomic_thread_fence(rte_memory_order_release); 202 203 log_base = (unsigned long *)(uintptr_t)dev->log_base; 204 205 for (i = 0; i < vq->log_cache_nb_elem; i++) { 206 struct log_cache_entry *elem = vq->log_cache + i; 207 208 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) 209 /* 210 * '__sync' builtins are deprecated, but 'rte_atomic' ones 211 * are sub-optimized in older GCC versions. 212 */ 213 __sync_fetch_and_or(log_base + elem->offset, elem->val); 214 #else 215 rte_atomic_fetch_or_explicit( 216 (unsigned long __rte_atomic *)(log_base + elem->offset), 217 elem->val, rte_memory_order_relaxed); 218 #endif 219 } 220 221 rte_atomic_thread_fence(rte_memory_order_release); 222 223 vq->log_cache_nb_elem = 0; 224 } 225 226 static __rte_always_inline void 227 vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, 228 uint64_t page) 229 { 230 uint32_t bit_nr = page % (sizeof(unsigned long) << 3); 231 uint32_t offset = page / (sizeof(unsigned long) << 3); 232 int i; 233 234 if (unlikely(!vq->log_cache)) { 235 /* No logging cache allocated, write dirty log map directly */ 236 rte_atomic_thread_fence(rte_memory_order_release); 237 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 238 239 return; 240 } 241 242 for (i = 0; i < vq->log_cache_nb_elem; i++) { 243 struct log_cache_entry *elem = vq->log_cache + i; 244 245 if (elem->offset == offset) { 246 elem->val |= (1UL << bit_nr); 247 return; 248 } 249 } 250 251 if (unlikely(i >= VHOST_LOG_CACHE_NR)) { 252 /* 253 * No more room for a new log cache entry, 254 * so write the dirty log map directly. 255 */ 256 rte_atomic_thread_fence(rte_memory_order_release); 257 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 258 259 return; 260 } 261 262 vq->log_cache[i].offset = offset; 263 vq->log_cache[i].val = (1UL << bit_nr); 264 vq->log_cache_nb_elem++; 265 } 266 267 void 268 __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, 269 uint64_t addr, uint64_t len) 270 { 271 uint64_t page; 272 273 if (unlikely(!dev->log_base || !len)) 274 return; 275 276 if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) 277 return; 278 279 page = addr / VHOST_LOG_PAGE; 280 while (page * VHOST_LOG_PAGE < addr + len) { 281 vhost_log_cache_page(dev, vq, page); 282 page += 1; 283 } 284 } 285 286 void 287 __vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, 288 uint64_t iova, uint64_t len) 289 { 290 uint64_t hva, gpa, map_len; 291 map_len = len; 292 293 hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); 294 if (map_len != len) { 295 VHOST_LOG_DATA(dev->ifname, ERR, 296 "failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n", 297 iova); 298 return; 299 } 300 301 gpa = hva_to_gpa(dev, hva, len); 302 if (gpa) 303 __vhost_log_cache_write(dev, vq, gpa, len); 304 } 305 306 void * 307 vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, 308 uint64_t desc_addr, uint64_t desc_len) 309 { 310 void *idesc; 311 uint64_t src, dst; 312 uint64_t len, remain = desc_len; 313 314 idesc = rte_malloc_socket(__func__, desc_len, 0, vq->numa_node); 315 if (unlikely(!idesc)) 316 return NULL; 317 318 dst = (uint64_t)(uintptr_t)idesc; 319 320 while (remain) { 321 len = remain; 322 src = vhost_iova_to_vva(dev, vq, desc_addr, &len, 323 VHOST_ACCESS_RO); 324 if (unlikely(!src || !len)) { 325 rte_free(idesc); 326 return NULL; 327 } 328 329 rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); 330 331 remain -= len; 332 dst += len; 333 desc_addr += len; 334 } 335 336 return idesc; 337 } 338 339 void 340 cleanup_vq(struct vhost_virtqueue *vq, int destroy) 341 { 342 if ((vq->callfd >= 0) && (destroy != 0)) 343 close(vq->callfd); 344 if (vq->kickfd >= 0) 345 close(vq->kickfd); 346 } 347 348 void 349 cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq) 350 { 351 if (!(dev->protocol_features & 352 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))) 353 return; 354 355 if (vq_is_packed(dev)) { 356 if (vq->inflight_packed) 357 vq->inflight_packed = NULL; 358 } else { 359 if (vq->inflight_split) 360 vq->inflight_split = NULL; 361 } 362 363 if (vq->resubmit_inflight) { 364 if (vq->resubmit_inflight->resubmit_list) { 365 rte_free(vq->resubmit_inflight->resubmit_list); 366 vq->resubmit_inflight->resubmit_list = NULL; 367 } 368 rte_free(vq->resubmit_inflight); 369 vq->resubmit_inflight = NULL; 370 } 371 } 372 373 /* 374 * Unmap any memory, close any file descriptors and 375 * free any memory owned by a device. 376 */ 377 void 378 cleanup_device(struct virtio_net *dev, int destroy) 379 { 380 uint32_t i; 381 382 vhost_backend_cleanup(dev); 383 384 for (i = 0; i < dev->nr_vring; i++) { 385 cleanup_vq(dev->virtqueue[i], destroy); 386 cleanup_vq_inflight(dev, dev->virtqueue[i]); 387 } 388 } 389 390 static void 391 vhost_free_async_mem(struct vhost_virtqueue *vq) 392 __rte_exclusive_locks_required(&vq->access_lock) 393 { 394 if (!vq->async) 395 return; 396 397 rte_free(vq->async->pkts_info); 398 rte_free(vq->async->pkts_cmpl_flag); 399 400 rte_free(vq->async->buffers_packed); 401 vq->async->buffers_packed = NULL; 402 rte_free(vq->async->descs_split); 403 vq->async->descs_split = NULL; 404 405 rte_free(vq->async); 406 vq->async = NULL; 407 } 408 409 void 410 free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq) 411 { 412 if (vq_is_packed(dev)) 413 rte_free(vq->shadow_used_packed); 414 else 415 rte_free(vq->shadow_used_split); 416 417 rte_rwlock_write_lock(&vq->access_lock); 418 vhost_free_async_mem(vq); 419 rte_rwlock_write_unlock(&vq->access_lock); 420 rte_free(vq->batch_copy_elems); 421 rte_free(vq->log_cache); 422 rte_free(vq); 423 } 424 425 /* 426 * Release virtqueues and device memory. 427 */ 428 static void 429 free_device(struct virtio_net *dev) 430 { 431 uint32_t i; 432 433 for (i = 0; i < dev->nr_vring; i++) 434 free_vq(dev, dev->virtqueue[i]); 435 436 rte_free(dev); 437 } 438 439 static __rte_always_inline int 440 log_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) 441 __rte_shared_locks_required(&vq->iotlb_lock) 442 { 443 if (likely(!(vq->ring_addrs.flags & (1 << VHOST_VRING_F_LOG)))) 444 return 0; 445 446 vq->log_guest_addr = translate_log_addr(dev, vq, 447 vq->ring_addrs.log_guest_addr); 448 if (vq->log_guest_addr == 0) 449 return -1; 450 451 return 0; 452 } 453 454 /* 455 * Converts vring log address to GPA 456 * If IOMMU is enabled, the log address is IOVA 457 * If IOMMU not enabled, the log address is already GPA 458 */ 459 uint64_t 460 translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq, 461 uint64_t log_addr) 462 { 463 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) { 464 const uint64_t exp_size = sizeof(uint64_t); 465 uint64_t hva, gpa; 466 uint64_t size = exp_size; 467 468 hva = vhost_iova_to_vva(dev, vq, log_addr, 469 &size, VHOST_ACCESS_RW); 470 471 if (size != exp_size) 472 return 0; 473 474 gpa = hva_to_gpa(dev, hva, exp_size); 475 if (!gpa) { 476 VHOST_LOG_DATA(dev->ifname, ERR, 477 "failed to find GPA for log_addr: 0x%" 478 PRIx64 " hva: 0x%" PRIx64 "\n", 479 log_addr, hva); 480 return 0; 481 } 482 return gpa; 483 484 } else 485 return log_addr; 486 } 487 488 static int 489 vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq) 490 __rte_shared_locks_required(&vq->iotlb_lock) 491 { 492 uint64_t req_size, size; 493 494 req_size = sizeof(struct vring_desc) * vq->size; 495 size = req_size; 496 vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq, 497 vq->ring_addrs.desc_user_addr, 498 &size, VHOST_ACCESS_RW); 499 if (!vq->desc || size != req_size) 500 return -1; 501 502 req_size = sizeof(struct vring_avail); 503 req_size += sizeof(uint16_t) * vq->size; 504 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 505 req_size += sizeof(uint16_t); 506 size = req_size; 507 vq->avail = (struct vring_avail *)(uintptr_t)vhost_iova_to_vva(dev, vq, 508 vq->ring_addrs.avail_user_addr, 509 &size, VHOST_ACCESS_RW); 510 if (!vq->avail || size != req_size) 511 return -1; 512 513 req_size = sizeof(struct vring_used); 514 req_size += sizeof(struct vring_used_elem) * vq->size; 515 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 516 req_size += sizeof(uint16_t); 517 size = req_size; 518 vq->used = (struct vring_used *)(uintptr_t)vhost_iova_to_vva(dev, vq, 519 vq->ring_addrs.used_user_addr, 520 &size, VHOST_ACCESS_RW); 521 if (!vq->used || size != req_size) 522 return -1; 523 524 return 0; 525 } 526 527 static int 528 vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) 529 __rte_shared_locks_required(&vq->iotlb_lock) 530 { 531 uint64_t req_size, size; 532 533 req_size = sizeof(struct vring_packed_desc) * vq->size; 534 size = req_size; 535 vq->desc_packed = (struct vring_packed_desc *)(uintptr_t) 536 vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr, 537 &size, VHOST_ACCESS_RW); 538 if (!vq->desc_packed || size != req_size) 539 return -1; 540 541 req_size = sizeof(struct vring_packed_desc_event); 542 size = req_size; 543 vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t) 544 vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr, 545 &size, VHOST_ACCESS_RW); 546 if (!vq->driver_event || size != req_size) 547 return -1; 548 549 req_size = sizeof(struct vring_packed_desc_event); 550 size = req_size; 551 vq->device_event = (struct vring_packed_desc_event *)(uintptr_t) 552 vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr, 553 &size, VHOST_ACCESS_RW); 554 if (!vq->device_event || size != req_size) 555 return -1; 556 557 return 0; 558 } 559 560 int 561 vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) 562 { 563 564 if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) 565 return -1; 566 567 if (vq_is_packed(dev)) { 568 if (vring_translate_packed(dev, vq) < 0) 569 return -1; 570 } else { 571 if (vring_translate_split(dev, vq) < 0) 572 return -1; 573 } 574 575 if (log_translate(dev, vq) < 0) 576 return -1; 577 578 vq->access_ok = true; 579 580 return 0; 581 } 582 583 void 584 vring_invalidate(struct virtio_net *dev __rte_unused, struct vhost_virtqueue *vq) 585 { 586 vhost_user_iotlb_wr_lock(vq); 587 588 vq->access_ok = false; 589 vq->desc = NULL; 590 vq->avail = NULL; 591 vq->used = NULL; 592 vq->log_guest_addr = 0; 593 594 vhost_user_iotlb_wr_unlock(vq); 595 } 596 597 static void 598 init_vring_queue(struct virtio_net *dev __rte_unused, struct vhost_virtqueue *vq, 599 uint32_t vring_idx) 600 { 601 int numa_node = SOCKET_ID_ANY; 602 603 memset(vq, 0, sizeof(struct vhost_virtqueue)); 604 605 vq->index = vring_idx; 606 vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; 607 vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; 608 vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF; 609 610 #ifdef RTE_LIBRTE_VHOST_NUMA 611 if (get_mempolicy(&numa_node, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR)) { 612 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to query numa node: %s\n", 613 rte_strerror(errno)); 614 numa_node = SOCKET_ID_ANY; 615 } 616 #endif 617 vq->numa_node = numa_node; 618 } 619 620 static void 621 reset_vring_queue(struct virtio_net *dev, struct vhost_virtqueue *vq) 622 { 623 int callfd; 624 625 callfd = vq->callfd; 626 init_vring_queue(dev, vq, vq->index); 627 vq->callfd = callfd; 628 } 629 630 int 631 alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx) 632 { 633 struct vhost_virtqueue *vq; 634 uint32_t i; 635 636 /* Also allocate holes, if any, up to requested vring index. */ 637 for (i = 0; i <= vring_idx; i++) { 638 if (dev->virtqueue[i]) 639 continue; 640 641 vq = rte_zmalloc(NULL, sizeof(struct vhost_virtqueue), 0); 642 if (vq == NULL) { 643 VHOST_LOG_CONFIG(dev->ifname, ERR, 644 "failed to allocate memory for vring %u.\n", 645 i); 646 return -1; 647 } 648 649 dev->virtqueue[i] = vq; 650 init_vring_queue(dev, vq, i); 651 rte_rwlock_init(&vq->access_lock); 652 rte_rwlock_init(&vq->iotlb_lock); 653 vq->avail_wrap_counter = 1; 654 vq->used_wrap_counter = 1; 655 vq->signalled_used_valid = false; 656 } 657 658 dev->nr_vring = RTE_MAX(dev->nr_vring, vring_idx + 1); 659 660 return 0; 661 } 662 663 /* 664 * Reset some variables in device structure, while keeping few 665 * others untouched, such as vid, ifname, nr_vring: they 666 * should be same unless the device is removed. 667 */ 668 void 669 reset_device(struct virtio_net *dev) 670 { 671 uint32_t i; 672 673 dev->features = 0; 674 dev->protocol_features = 0; 675 dev->flags &= VIRTIO_DEV_BUILTIN_VIRTIO_NET; 676 677 for (i = 0; i < dev->nr_vring; i++) { 678 struct vhost_virtqueue *vq = dev->virtqueue[i]; 679 680 if (!vq) { 681 VHOST_LOG_CONFIG(dev->ifname, ERR, 682 "failed to reset vring, virtqueue not allocated (%d)\n", i); 683 continue; 684 } 685 reset_vring_queue(dev, vq); 686 } 687 } 688 689 /* 690 * Invoked when there is a new vhost-user connection established (when 691 * there is a new virtio device being attached). 692 */ 693 int 694 vhost_new_device(struct vhost_backend_ops *ops) 695 { 696 struct virtio_net *dev; 697 int i; 698 699 if (ops == NULL) { 700 VHOST_LOG_CONFIG("device", ERR, "missing backend ops.\n"); 701 return -1; 702 } 703 704 if (ops->iotlb_miss == NULL) { 705 VHOST_LOG_CONFIG("device", ERR, "missing IOTLB miss backend op.\n"); 706 return -1; 707 } 708 709 if (ops->inject_irq == NULL) { 710 VHOST_LOG_CONFIG("device", ERR, "missing IRQ injection backend op.\n"); 711 return -1; 712 } 713 714 pthread_mutex_lock(&vhost_dev_lock); 715 for (i = 0; i < RTE_MAX_VHOST_DEVICE; i++) { 716 if (vhost_devices[i] == NULL) 717 break; 718 } 719 720 if (i == RTE_MAX_VHOST_DEVICE) { 721 VHOST_LOG_CONFIG("device", ERR, "failed to find a free slot for new device.\n"); 722 pthread_mutex_unlock(&vhost_dev_lock); 723 return -1; 724 } 725 726 dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0); 727 if (dev == NULL) { 728 VHOST_LOG_CONFIG("device", ERR, "failed to allocate memory for new device.\n"); 729 pthread_mutex_unlock(&vhost_dev_lock); 730 return -1; 731 } 732 733 vhost_devices[i] = dev; 734 pthread_mutex_unlock(&vhost_dev_lock); 735 736 dev->vid = i; 737 dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET; 738 dev->backend_req_fd = -1; 739 dev->postcopy_ufd = -1; 740 rte_spinlock_init(&dev->backend_req_lock); 741 dev->backend_ops = ops; 742 743 return i; 744 } 745 746 void 747 vhost_destroy_device_notify(struct virtio_net *dev) 748 { 749 struct rte_vdpa_device *vdpa_dev; 750 751 if (dev->flags & VIRTIO_DEV_RUNNING) { 752 vdpa_dev = dev->vdpa_dev; 753 if (vdpa_dev) 754 vdpa_dev->ops->dev_close(dev->vid); 755 dev->flags &= ~VIRTIO_DEV_RUNNING; 756 dev->notify_ops->destroy_device(dev->vid); 757 } 758 } 759 760 /* 761 * Invoked when there is the vhost-user connection is broken (when 762 * the virtio device is being detached). 763 */ 764 void 765 vhost_destroy_device(int vid) 766 { 767 struct virtio_net *dev = get_device(vid); 768 769 if (dev == NULL) 770 return; 771 772 vhost_destroy_device_notify(dev); 773 774 cleanup_device(dev, 1); 775 free_device(dev); 776 777 vhost_devices[vid] = NULL; 778 } 779 780 void 781 vhost_attach_vdpa_device(int vid, struct rte_vdpa_device *vdpa_dev) 782 { 783 struct virtio_net *dev = get_device(vid); 784 785 if (dev == NULL) 786 return; 787 788 dev->vdpa_dev = vdpa_dev; 789 } 790 791 void 792 vhost_set_ifname(int vid, const char *if_name, unsigned int if_len) 793 { 794 struct virtio_net *dev; 795 unsigned int len; 796 797 dev = get_device(vid); 798 if (dev == NULL) 799 return; 800 801 len = if_len > sizeof(dev->ifname) ? 802 sizeof(dev->ifname) : if_len; 803 804 strncpy(dev->ifname, if_name, len); 805 dev->ifname[sizeof(dev->ifname) - 1] = '\0'; 806 } 807 808 void 809 vhost_setup_virtio_net(int vid, bool enable, bool compliant_ol_flags, bool stats_enabled, 810 bool support_iommu) 811 { 812 struct virtio_net *dev = get_device(vid); 813 814 if (dev == NULL) 815 return; 816 817 if (enable) 818 dev->flags |= VIRTIO_DEV_BUILTIN_VIRTIO_NET; 819 else 820 dev->flags &= ~VIRTIO_DEV_BUILTIN_VIRTIO_NET; 821 if (!compliant_ol_flags) 822 dev->flags |= VIRTIO_DEV_LEGACY_OL_FLAGS; 823 else 824 dev->flags &= ~VIRTIO_DEV_LEGACY_OL_FLAGS; 825 if (stats_enabled) 826 dev->flags |= VIRTIO_DEV_STATS_ENABLED; 827 else 828 dev->flags &= ~VIRTIO_DEV_STATS_ENABLED; 829 if (support_iommu) 830 dev->flags |= VIRTIO_DEV_SUPPORT_IOMMU; 831 else 832 dev->flags &= ~VIRTIO_DEV_SUPPORT_IOMMU; 833 834 if (vhost_user_iotlb_init(dev) < 0) 835 VHOST_LOG_CONFIG("device", ERR, "failed to init IOTLB\n"); 836 837 } 838 839 void 840 vhost_enable_extbuf(int vid) 841 { 842 struct virtio_net *dev = get_device(vid); 843 844 if (dev == NULL) 845 return; 846 847 dev->extbuf = 1; 848 } 849 850 void 851 vhost_enable_linearbuf(int vid) 852 { 853 struct virtio_net *dev = get_device(vid); 854 855 if (dev == NULL) 856 return; 857 858 dev->linearbuf = 1; 859 } 860 861 int 862 rte_vhost_get_mtu(int vid, uint16_t *mtu) 863 { 864 struct virtio_net *dev = get_device(vid); 865 866 if (dev == NULL || mtu == NULL) 867 return -ENODEV; 868 869 if (!(dev->flags & VIRTIO_DEV_READY)) 870 return -EAGAIN; 871 872 if (!(dev->features & (1ULL << VIRTIO_NET_F_MTU))) 873 return -ENOTSUP; 874 875 *mtu = dev->mtu; 876 877 return 0; 878 } 879 880 int 881 rte_vhost_get_numa_node(int vid) 882 { 883 #ifdef RTE_LIBRTE_VHOST_NUMA 884 struct virtio_net *dev = get_device(vid); 885 int numa_node; 886 int ret; 887 888 if (dev == NULL || numa_available() != 0) 889 return -1; 890 891 ret = get_mempolicy(&numa_node, NULL, 0, dev, 892 MPOL_F_NODE | MPOL_F_ADDR); 893 if (ret < 0) { 894 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to query numa node: %s\n", 895 rte_strerror(errno)); 896 return -1; 897 } 898 899 return numa_node; 900 #else 901 RTE_SET_USED(vid); 902 return -1; 903 #endif 904 } 905 906 uint16_t 907 rte_vhost_get_vring_num(int vid) 908 { 909 struct virtio_net *dev = get_device(vid); 910 911 if (dev == NULL) 912 return 0; 913 914 return dev->nr_vring; 915 } 916 917 int 918 rte_vhost_get_ifname(int vid, char *buf, size_t len) 919 { 920 struct virtio_net *dev = get_device(vid); 921 922 if (dev == NULL || buf == NULL) 923 return -1; 924 925 len = RTE_MIN(len, sizeof(dev->ifname)); 926 927 strncpy(buf, dev->ifname, len); 928 buf[len - 1] = '\0'; 929 930 return 0; 931 } 932 933 int 934 rte_vhost_get_negotiated_features(int vid, uint64_t *features) 935 { 936 struct virtio_net *dev; 937 938 dev = get_device(vid); 939 if (dev == NULL || features == NULL) 940 return -1; 941 942 *features = dev->features; 943 return 0; 944 } 945 946 int 947 rte_vhost_get_negotiated_protocol_features(int vid, 948 uint64_t *protocol_features) 949 { 950 struct virtio_net *dev; 951 952 dev = get_device(vid); 953 if (dev == NULL || protocol_features == NULL) 954 return -1; 955 956 *protocol_features = dev->protocol_features; 957 return 0; 958 } 959 960 int 961 rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem) 962 { 963 struct virtio_net *dev; 964 struct rte_vhost_memory *m; 965 size_t size; 966 967 dev = get_device(vid); 968 if (dev == NULL || mem == NULL) 969 return -1; 970 971 size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region); 972 m = malloc(sizeof(struct rte_vhost_memory) + size); 973 if (!m) 974 return -1; 975 976 m->nregions = dev->mem->nregions; 977 memcpy(m->regions, dev->mem->regions, size); 978 *mem = m; 979 980 return 0; 981 } 982 983 int 984 rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, 985 struct rte_vhost_vring *vring) 986 { 987 struct virtio_net *dev; 988 struct vhost_virtqueue *vq; 989 990 dev = get_device(vid); 991 if (dev == NULL || vring == NULL) 992 return -1; 993 994 if (vring_idx >= VHOST_MAX_VRING) 995 return -1; 996 997 vq = dev->virtqueue[vring_idx]; 998 if (!vq) 999 return -1; 1000 1001 if (vq_is_packed(dev)) { 1002 vring->desc_packed = vq->desc_packed; 1003 vring->driver_event = vq->driver_event; 1004 vring->device_event = vq->device_event; 1005 } else { 1006 vring->desc = vq->desc; 1007 vring->avail = vq->avail; 1008 vring->used = vq->used; 1009 } 1010 vring->log_guest_addr = vq->log_guest_addr; 1011 1012 vring->callfd = vq->callfd; 1013 vring->kickfd = vq->kickfd; 1014 vring->size = vq->size; 1015 1016 return 0; 1017 } 1018 1019 int 1020 rte_vhost_get_vhost_ring_inflight(int vid, uint16_t vring_idx, 1021 struct rte_vhost_ring_inflight *vring) 1022 { 1023 struct virtio_net *dev; 1024 struct vhost_virtqueue *vq; 1025 1026 dev = get_device(vid); 1027 if (unlikely(!dev)) 1028 return -1; 1029 1030 if (vring_idx >= VHOST_MAX_VRING) 1031 return -1; 1032 1033 vq = dev->virtqueue[vring_idx]; 1034 if (unlikely(!vq)) 1035 return -1; 1036 1037 if (vq_is_packed(dev)) { 1038 if (unlikely(!vq->inflight_packed)) 1039 return -1; 1040 1041 vring->inflight_packed = vq->inflight_packed; 1042 } else { 1043 if (unlikely(!vq->inflight_split)) 1044 return -1; 1045 1046 vring->inflight_split = vq->inflight_split; 1047 } 1048 1049 vring->resubmit_inflight = vq->resubmit_inflight; 1050 1051 return 0; 1052 } 1053 1054 int 1055 rte_vhost_set_inflight_desc_split(int vid, uint16_t vring_idx, 1056 uint16_t idx) 1057 { 1058 struct vhost_virtqueue *vq; 1059 struct virtio_net *dev; 1060 1061 dev = get_device(vid); 1062 if (unlikely(!dev)) 1063 return -1; 1064 1065 if (unlikely(!(dev->protocol_features & 1066 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1067 return 0; 1068 1069 if (unlikely(vq_is_packed(dev))) 1070 return -1; 1071 1072 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1073 return -1; 1074 1075 vq = dev->virtqueue[vring_idx]; 1076 if (unlikely(!vq)) 1077 return -1; 1078 1079 if (unlikely(!vq->inflight_split)) 1080 return -1; 1081 1082 if (unlikely(idx >= vq->size)) 1083 return -1; 1084 1085 vq->inflight_split->desc[idx].counter = vq->global_counter++; 1086 vq->inflight_split->desc[idx].inflight = 1; 1087 return 0; 1088 } 1089 1090 int 1091 rte_vhost_set_inflight_desc_packed(int vid, uint16_t vring_idx, 1092 uint16_t head, uint16_t last, 1093 uint16_t *inflight_entry) 1094 { 1095 struct rte_vhost_inflight_info_packed *inflight_info; 1096 struct virtio_net *dev; 1097 struct vhost_virtqueue *vq; 1098 struct vring_packed_desc *desc; 1099 uint16_t old_free_head, free_head; 1100 1101 dev = get_device(vid); 1102 if (unlikely(!dev)) 1103 return -1; 1104 1105 if (unlikely(!(dev->protocol_features & 1106 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1107 return 0; 1108 1109 if (unlikely(!vq_is_packed(dev))) 1110 return -1; 1111 1112 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1113 return -1; 1114 1115 vq = dev->virtqueue[vring_idx]; 1116 if (unlikely(!vq)) 1117 return -1; 1118 1119 inflight_info = vq->inflight_packed; 1120 if (unlikely(!inflight_info)) 1121 return -1; 1122 1123 if (unlikely(head >= vq->size)) 1124 return -1; 1125 1126 desc = vq->desc_packed; 1127 old_free_head = inflight_info->old_free_head; 1128 if (unlikely(old_free_head >= vq->size)) 1129 return -1; 1130 1131 free_head = old_free_head; 1132 1133 /* init header descriptor */ 1134 inflight_info->desc[old_free_head].num = 0; 1135 inflight_info->desc[old_free_head].counter = vq->global_counter++; 1136 inflight_info->desc[old_free_head].inflight = 1; 1137 1138 /* save desc entry in flight entry */ 1139 while (head != ((last + 1) % vq->size)) { 1140 inflight_info->desc[old_free_head].num++; 1141 inflight_info->desc[free_head].addr = desc[head].addr; 1142 inflight_info->desc[free_head].len = desc[head].len; 1143 inflight_info->desc[free_head].flags = desc[head].flags; 1144 inflight_info->desc[free_head].id = desc[head].id; 1145 1146 inflight_info->desc[old_free_head].last = free_head; 1147 free_head = inflight_info->desc[free_head].next; 1148 inflight_info->free_head = free_head; 1149 head = (head + 1) % vq->size; 1150 } 1151 1152 inflight_info->old_free_head = free_head; 1153 *inflight_entry = old_free_head; 1154 1155 return 0; 1156 } 1157 1158 int 1159 rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx, 1160 uint16_t last_used_idx, uint16_t idx) 1161 { 1162 struct virtio_net *dev; 1163 struct vhost_virtqueue *vq; 1164 1165 dev = get_device(vid); 1166 if (unlikely(!dev)) 1167 return -1; 1168 1169 if (unlikely(!(dev->protocol_features & 1170 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1171 return 0; 1172 1173 if (unlikely(vq_is_packed(dev))) 1174 return -1; 1175 1176 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1177 return -1; 1178 1179 vq = dev->virtqueue[vring_idx]; 1180 if (unlikely(!vq)) 1181 return -1; 1182 1183 if (unlikely(!vq->inflight_split)) 1184 return -1; 1185 1186 if (unlikely(idx >= vq->size)) 1187 return -1; 1188 1189 rte_atomic_thread_fence(rte_memory_order_seq_cst); 1190 1191 vq->inflight_split->desc[idx].inflight = 0; 1192 1193 rte_atomic_thread_fence(rte_memory_order_seq_cst); 1194 1195 vq->inflight_split->used_idx = last_used_idx; 1196 return 0; 1197 } 1198 1199 int 1200 rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx, 1201 uint16_t head) 1202 { 1203 struct rte_vhost_inflight_info_packed *inflight_info; 1204 struct virtio_net *dev; 1205 struct vhost_virtqueue *vq; 1206 1207 dev = get_device(vid); 1208 if (unlikely(!dev)) 1209 return -1; 1210 1211 if (unlikely(!(dev->protocol_features & 1212 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1213 return 0; 1214 1215 if (unlikely(!vq_is_packed(dev))) 1216 return -1; 1217 1218 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1219 return -1; 1220 1221 vq = dev->virtqueue[vring_idx]; 1222 if (unlikely(!vq)) 1223 return -1; 1224 1225 inflight_info = vq->inflight_packed; 1226 if (unlikely(!inflight_info)) 1227 return -1; 1228 1229 if (unlikely(head >= vq->size)) 1230 return -1; 1231 1232 rte_atomic_thread_fence(rte_memory_order_seq_cst); 1233 1234 inflight_info->desc[head].inflight = 0; 1235 1236 rte_atomic_thread_fence(rte_memory_order_seq_cst); 1237 1238 inflight_info->old_free_head = inflight_info->free_head; 1239 inflight_info->old_used_idx = inflight_info->used_idx; 1240 inflight_info->old_used_wrap_counter = inflight_info->used_wrap_counter; 1241 1242 return 0; 1243 } 1244 1245 int 1246 rte_vhost_set_last_inflight_io_split(int vid, uint16_t vring_idx, 1247 uint16_t idx) 1248 { 1249 struct virtio_net *dev; 1250 struct vhost_virtqueue *vq; 1251 1252 dev = get_device(vid); 1253 if (unlikely(!dev)) 1254 return -1; 1255 1256 if (unlikely(!(dev->protocol_features & 1257 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1258 return 0; 1259 1260 if (unlikely(vq_is_packed(dev))) 1261 return -1; 1262 1263 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1264 return -1; 1265 1266 vq = dev->virtqueue[vring_idx]; 1267 if (unlikely(!vq)) 1268 return -1; 1269 1270 if (unlikely(!vq->inflight_split)) 1271 return -1; 1272 1273 if (unlikely(idx >= vq->size)) 1274 return -1; 1275 1276 vq->inflight_split->last_inflight_io = idx; 1277 return 0; 1278 } 1279 1280 int 1281 rte_vhost_set_last_inflight_io_packed(int vid, uint16_t vring_idx, 1282 uint16_t head) 1283 { 1284 struct rte_vhost_inflight_info_packed *inflight_info; 1285 struct virtio_net *dev; 1286 struct vhost_virtqueue *vq; 1287 uint16_t last; 1288 1289 dev = get_device(vid); 1290 if (unlikely(!dev)) 1291 return -1; 1292 1293 if (unlikely(!(dev->protocol_features & 1294 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1295 return 0; 1296 1297 if (unlikely(!vq_is_packed(dev))) 1298 return -1; 1299 1300 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1301 return -1; 1302 1303 vq = dev->virtqueue[vring_idx]; 1304 if (unlikely(!vq)) 1305 return -1; 1306 1307 inflight_info = vq->inflight_packed; 1308 if (unlikely(!inflight_info)) 1309 return -1; 1310 1311 if (unlikely(head >= vq->size)) 1312 return -1; 1313 1314 last = inflight_info->desc[head].last; 1315 if (unlikely(last >= vq->size)) 1316 return -1; 1317 1318 inflight_info->desc[last].next = inflight_info->free_head; 1319 inflight_info->free_head = head; 1320 inflight_info->used_idx += inflight_info->desc[head].num; 1321 if (inflight_info->used_idx >= inflight_info->desc_num) { 1322 inflight_info->used_idx -= inflight_info->desc_num; 1323 inflight_info->used_wrap_counter = 1324 !inflight_info->used_wrap_counter; 1325 } 1326 1327 return 0; 1328 } 1329 1330 int 1331 rte_vhost_vring_call(int vid, uint16_t vring_idx) 1332 { 1333 struct virtio_net *dev; 1334 struct vhost_virtqueue *vq; 1335 int ret = 0; 1336 1337 dev = get_device(vid); 1338 if (!dev) 1339 return -1; 1340 1341 if (vring_idx >= VHOST_MAX_VRING) 1342 return -1; 1343 1344 vq = dev->virtqueue[vring_idx]; 1345 if (!vq) 1346 return -1; 1347 1348 rte_rwlock_read_lock(&vq->access_lock); 1349 1350 if (unlikely(!vq->access_ok)) { 1351 ret = -1; 1352 goto out_unlock; 1353 } 1354 1355 if (vq_is_packed(dev)) 1356 vhost_vring_call_packed(dev, vq); 1357 else 1358 vhost_vring_call_split(dev, vq); 1359 1360 out_unlock: 1361 rte_rwlock_read_unlock(&vq->access_lock); 1362 1363 return ret; 1364 } 1365 1366 int 1367 rte_vhost_vring_call_nonblock(int vid, uint16_t vring_idx) 1368 { 1369 struct virtio_net *dev; 1370 struct vhost_virtqueue *vq; 1371 int ret = 0; 1372 1373 dev = get_device(vid); 1374 if (!dev) 1375 return -1; 1376 1377 if (vring_idx >= VHOST_MAX_VRING) 1378 return -1; 1379 1380 vq = dev->virtqueue[vring_idx]; 1381 if (!vq) 1382 return -1; 1383 1384 if (rte_rwlock_read_trylock(&vq->access_lock)) 1385 return -EAGAIN; 1386 1387 if (unlikely(!vq->access_ok)) { 1388 ret = -1; 1389 goto out_unlock; 1390 } 1391 1392 if (vq_is_packed(dev)) 1393 vhost_vring_call_packed(dev, vq); 1394 else 1395 vhost_vring_call_split(dev, vq); 1396 1397 out_unlock: 1398 rte_rwlock_read_unlock(&vq->access_lock); 1399 1400 return ret; 1401 } 1402 1403 uint16_t 1404 rte_vhost_avail_entries(int vid, uint16_t queue_id) 1405 { 1406 struct virtio_net *dev; 1407 struct vhost_virtqueue *vq; 1408 uint16_t ret = 0; 1409 1410 dev = get_device(vid); 1411 if (!dev) 1412 return 0; 1413 1414 if (queue_id >= VHOST_MAX_VRING) 1415 return 0; 1416 1417 vq = dev->virtqueue[queue_id]; 1418 if (!vq) 1419 return 0; 1420 1421 rte_rwlock_write_lock(&vq->access_lock); 1422 1423 if (unlikely(!vq->access_ok)) 1424 goto out; 1425 1426 if (unlikely(!vq->enabled)) 1427 goto out; 1428 1429 ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; 1430 1431 out: 1432 rte_rwlock_write_unlock(&vq->access_lock); 1433 return ret; 1434 } 1435 1436 static inline int 1437 vhost_enable_notify_split(struct virtio_net *dev, 1438 struct vhost_virtqueue *vq, int enable) 1439 { 1440 if (vq->used == NULL) 1441 return -1; 1442 1443 if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { 1444 if (enable) 1445 vq->used->flags &= ~VRING_USED_F_NO_NOTIFY; 1446 else 1447 vq->used->flags |= VRING_USED_F_NO_NOTIFY; 1448 } else { 1449 if (enable) 1450 vhost_avail_event(vq) = vq->last_avail_idx; 1451 } 1452 return 0; 1453 } 1454 1455 static inline int 1456 vhost_enable_notify_packed(struct virtio_net *dev, 1457 struct vhost_virtqueue *vq, int enable) 1458 { 1459 uint16_t flags; 1460 1461 if (vq->device_event == NULL) 1462 return -1; 1463 1464 if (!enable) { 1465 vq->device_event->flags = VRING_EVENT_F_DISABLE; 1466 return 0; 1467 } 1468 1469 flags = VRING_EVENT_F_ENABLE; 1470 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { 1471 flags = VRING_EVENT_F_DESC; 1472 vq->device_event->off_wrap = vq->last_avail_idx | 1473 vq->avail_wrap_counter << 15; 1474 } 1475 1476 rte_atomic_thread_fence(rte_memory_order_release); 1477 1478 vq->device_event->flags = flags; 1479 return 0; 1480 } 1481 1482 int 1483 vhost_enable_guest_notification(struct virtio_net *dev, 1484 struct vhost_virtqueue *vq, int enable) 1485 { 1486 /* 1487 * If the virtqueue is not ready yet, it will be applied 1488 * when it will become ready. 1489 */ 1490 if (!vq->ready) 1491 return 0; 1492 1493 if (vq_is_packed(dev)) 1494 return vhost_enable_notify_packed(dev, vq, enable); 1495 else 1496 return vhost_enable_notify_split(dev, vq, enable); 1497 } 1498 1499 int 1500 rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) 1501 { 1502 struct virtio_net *dev = get_device(vid); 1503 struct vhost_virtqueue *vq; 1504 int ret; 1505 1506 if (!dev) 1507 return -1; 1508 1509 if (queue_id >= VHOST_MAX_VRING) 1510 return -1; 1511 1512 vq = dev->virtqueue[queue_id]; 1513 if (!vq) 1514 return -1; 1515 1516 rte_rwlock_write_lock(&vq->access_lock); 1517 1518 if (unlikely(!vq->access_ok)) { 1519 ret = -1; 1520 goto out_unlock; 1521 } 1522 1523 vq->notif_enable = enable; 1524 ret = vhost_enable_guest_notification(dev, vq, enable); 1525 1526 out_unlock: 1527 rte_rwlock_write_unlock(&vq->access_lock); 1528 1529 return ret; 1530 } 1531 1532 void 1533 rte_vhost_notify_guest(int vid, uint16_t queue_id) 1534 { 1535 struct virtio_net *dev = get_device(vid); 1536 struct vhost_virtqueue *vq; 1537 1538 if (!dev || queue_id >= VHOST_MAX_VRING) 1539 return; 1540 1541 vq = dev->virtqueue[queue_id]; 1542 if (!vq) 1543 return; 1544 1545 rte_rwlock_read_lock(&vq->access_lock); 1546 1547 if (unlikely(!vq->access_ok)) 1548 goto out_unlock; 1549 1550 rte_atomic_store_explicit(&vq->irq_pending, false, rte_memory_order_release); 1551 1552 if (dev->backend_ops->inject_irq(dev, vq)) { 1553 if (dev->flags & VIRTIO_DEV_STATS_ENABLED) 1554 rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications_error, 1555 1, rte_memory_order_relaxed); 1556 } else { 1557 if (dev->flags & VIRTIO_DEV_STATS_ENABLED) 1558 rte_atomic_fetch_add_explicit(&vq->stats.guest_notifications, 1559 1, rte_memory_order_relaxed); 1560 if (dev->notify_ops->guest_notified) 1561 dev->notify_ops->guest_notified(dev->vid); 1562 } 1563 1564 out_unlock: 1565 rte_rwlock_read_unlock(&vq->access_lock); 1566 } 1567 1568 void 1569 rte_vhost_log_write(int vid, uint64_t addr, uint64_t len) 1570 { 1571 struct virtio_net *dev = get_device(vid); 1572 1573 if (dev == NULL) 1574 return; 1575 1576 vhost_log_write(dev, addr, len); 1577 } 1578 1579 void 1580 rte_vhost_log_used_vring(int vid, uint16_t vring_idx, 1581 uint64_t offset, uint64_t len) 1582 { 1583 struct virtio_net *dev; 1584 struct vhost_virtqueue *vq; 1585 1586 dev = get_device(vid); 1587 if (dev == NULL) 1588 return; 1589 1590 if (vring_idx >= VHOST_MAX_VRING) 1591 return; 1592 vq = dev->virtqueue[vring_idx]; 1593 if (!vq) 1594 return; 1595 1596 vhost_log_used_vring(dev, vq, offset, len); 1597 } 1598 1599 uint32_t 1600 rte_vhost_rx_queue_count(int vid, uint16_t qid) 1601 { 1602 struct virtio_net *dev; 1603 struct vhost_virtqueue *vq; 1604 uint32_t ret = 0; 1605 1606 dev = get_device(vid); 1607 if (dev == NULL) 1608 return 0; 1609 1610 if (unlikely(qid >= dev->nr_vring || (qid & 1) == 0)) { 1611 VHOST_LOG_DATA(dev->ifname, ERR, 1612 "%s: invalid virtqueue idx %d.\n", 1613 __func__, qid); 1614 return 0; 1615 } 1616 1617 vq = dev->virtqueue[qid]; 1618 if (vq == NULL) 1619 return 0; 1620 1621 rte_rwlock_write_lock(&vq->access_lock); 1622 1623 if (unlikely(!vq->access_ok)) 1624 goto out; 1625 1626 if (unlikely(!vq->enabled)) 1627 goto out; 1628 1629 ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx; 1630 1631 out: 1632 rte_rwlock_write_unlock(&vq->access_lock); 1633 return ret; 1634 } 1635 1636 struct rte_vdpa_device * 1637 rte_vhost_get_vdpa_device(int vid) 1638 { 1639 struct virtio_net *dev = get_device(vid); 1640 1641 if (dev == NULL) 1642 return NULL; 1643 1644 return dev->vdpa_dev; 1645 } 1646 1647 int 1648 rte_vhost_get_log_base(int vid, uint64_t *log_base, 1649 uint64_t *log_size) 1650 { 1651 struct virtio_net *dev = get_device(vid); 1652 1653 if (dev == NULL || log_base == NULL || log_size == NULL) 1654 return -1; 1655 1656 *log_base = dev->log_base; 1657 *log_size = dev->log_size; 1658 1659 return 0; 1660 } 1661 1662 int 1663 rte_vhost_get_vring_base(int vid, uint16_t queue_id, 1664 uint16_t *last_avail_idx, uint16_t *last_used_idx) 1665 { 1666 struct vhost_virtqueue *vq; 1667 struct virtio_net *dev = get_device(vid); 1668 1669 if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) 1670 return -1; 1671 1672 if (queue_id >= VHOST_MAX_VRING) 1673 return -1; 1674 1675 vq = dev->virtqueue[queue_id]; 1676 if (!vq) 1677 return -1; 1678 1679 if (vq_is_packed(dev)) { 1680 *last_avail_idx = (vq->avail_wrap_counter << 15) | 1681 vq->last_avail_idx; 1682 *last_used_idx = (vq->used_wrap_counter << 15) | 1683 vq->last_used_idx; 1684 } else { 1685 *last_avail_idx = vq->last_avail_idx; 1686 *last_used_idx = vq->last_used_idx; 1687 } 1688 1689 return 0; 1690 } 1691 1692 int 1693 rte_vhost_set_vring_base(int vid, uint16_t queue_id, 1694 uint16_t last_avail_idx, uint16_t last_used_idx) 1695 { 1696 struct vhost_virtqueue *vq; 1697 struct virtio_net *dev = get_device(vid); 1698 1699 if (!dev) 1700 return -1; 1701 1702 if (queue_id >= VHOST_MAX_VRING) 1703 return -1; 1704 1705 vq = dev->virtqueue[queue_id]; 1706 if (!vq) 1707 return -1; 1708 1709 if (vq_is_packed(dev)) { 1710 vq->last_avail_idx = last_avail_idx & 0x7fff; 1711 vq->avail_wrap_counter = !!(last_avail_idx & (1 << 15)); 1712 vq->last_used_idx = last_used_idx & 0x7fff; 1713 vq->used_wrap_counter = !!(last_used_idx & (1 << 15)); 1714 } else { 1715 vq->last_avail_idx = last_avail_idx; 1716 vq->last_used_idx = last_used_idx; 1717 } 1718 1719 return 0; 1720 } 1721 1722 int 1723 rte_vhost_get_vring_base_from_inflight(int vid, 1724 uint16_t queue_id, 1725 uint16_t *last_avail_idx, 1726 uint16_t *last_used_idx) 1727 { 1728 struct rte_vhost_inflight_info_packed *inflight_info; 1729 struct vhost_virtqueue *vq; 1730 struct virtio_net *dev = get_device(vid); 1731 1732 if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) 1733 return -1; 1734 1735 if (queue_id >= VHOST_MAX_VRING) 1736 return -1; 1737 1738 vq = dev->virtqueue[queue_id]; 1739 if (!vq) 1740 return -1; 1741 1742 if (!vq_is_packed(dev)) 1743 return -1; 1744 1745 inflight_info = vq->inflight_packed; 1746 if (!inflight_info) 1747 return -1; 1748 1749 *last_avail_idx = (inflight_info->old_used_wrap_counter << 15) | 1750 inflight_info->old_used_idx; 1751 *last_used_idx = *last_avail_idx; 1752 1753 return 0; 1754 } 1755 1756 int 1757 rte_vhost_extern_callback_register(int vid, 1758 struct rte_vhost_user_extern_ops const * const ops, void *ctx) 1759 { 1760 struct virtio_net *dev = get_device(vid); 1761 1762 if (dev == NULL || ops == NULL) 1763 return -1; 1764 1765 dev->extern_ops = *ops; 1766 dev->extern_data = ctx; 1767 return 0; 1768 } 1769 1770 static __rte_always_inline int 1771 async_channel_register(struct virtio_net *dev, struct vhost_virtqueue *vq) 1772 __rte_exclusive_locks_required(&vq->access_lock) 1773 { 1774 struct vhost_async *async; 1775 int node = vq->numa_node; 1776 1777 if (unlikely(vq->async)) { 1778 VHOST_LOG_CONFIG(dev->ifname, ERR, 1779 "async register failed: already registered (qid: %d)\n", 1780 vq->index); 1781 return -1; 1782 } 1783 1784 async = rte_zmalloc_socket(NULL, sizeof(struct vhost_async), 0, node); 1785 if (!async) { 1786 VHOST_LOG_CONFIG(dev->ifname, ERR, 1787 "failed to allocate async metadata (qid: %d)\n", 1788 vq->index); 1789 return -1; 1790 } 1791 1792 async->pkts_info = rte_malloc_socket(NULL, vq->size * sizeof(struct async_inflight_info), 1793 RTE_CACHE_LINE_SIZE, node); 1794 if (!async->pkts_info) { 1795 VHOST_LOG_CONFIG(dev->ifname, ERR, 1796 "failed to allocate async_pkts_info (qid: %d)\n", 1797 vq->index); 1798 goto out_free_async; 1799 } 1800 1801 async->pkts_cmpl_flag = rte_zmalloc_socket(NULL, vq->size * sizeof(bool), 1802 RTE_CACHE_LINE_SIZE, node); 1803 if (!async->pkts_cmpl_flag) { 1804 VHOST_LOG_CONFIG(dev->ifname, ERR, 1805 "failed to allocate async pkts_cmpl_flag (qid: %d)\n", 1806 vq->index); 1807 goto out_free_async; 1808 } 1809 1810 if (vq_is_packed(dev)) { 1811 async->buffers_packed = rte_malloc_socket(NULL, 1812 vq->size * sizeof(struct vring_used_elem_packed), 1813 RTE_CACHE_LINE_SIZE, node); 1814 if (!async->buffers_packed) { 1815 VHOST_LOG_CONFIG(dev->ifname, ERR, 1816 "failed to allocate async buffers (qid: %d)\n", 1817 vq->index); 1818 goto out_free_inflight; 1819 } 1820 } else { 1821 async->descs_split = rte_malloc_socket(NULL, 1822 vq->size * sizeof(struct vring_used_elem), 1823 RTE_CACHE_LINE_SIZE, node); 1824 if (!async->descs_split) { 1825 VHOST_LOG_CONFIG(dev->ifname, ERR, 1826 "failed to allocate async descs (qid: %d)\n", 1827 vq->index); 1828 goto out_free_inflight; 1829 } 1830 } 1831 1832 vq->async = async; 1833 1834 return 0; 1835 out_free_inflight: 1836 rte_free(async->pkts_info); 1837 out_free_async: 1838 rte_free(async); 1839 1840 return -1; 1841 } 1842 1843 int 1844 rte_vhost_async_channel_register(int vid, uint16_t queue_id) 1845 { 1846 struct vhost_virtqueue *vq; 1847 struct virtio_net *dev = get_device(vid); 1848 int ret; 1849 1850 if (dev == NULL) 1851 return -1; 1852 1853 if (queue_id >= VHOST_MAX_VRING) 1854 return -1; 1855 1856 vq = dev->virtqueue[queue_id]; 1857 1858 if (unlikely(vq == NULL || !dev->async_copy || dev->vdpa_dev != NULL)) 1859 return -1; 1860 1861 rte_rwlock_write_lock(&vq->access_lock); 1862 1863 if (unlikely(!vq->access_ok)) { 1864 ret = -1; 1865 goto out_unlock; 1866 } 1867 1868 ret = async_channel_register(dev, vq); 1869 1870 out_unlock: 1871 rte_rwlock_write_unlock(&vq->access_lock); 1872 1873 return ret; 1874 } 1875 1876 int 1877 rte_vhost_async_channel_register_thread_unsafe(int vid, uint16_t queue_id) 1878 { 1879 struct vhost_virtqueue *vq; 1880 struct virtio_net *dev = get_device(vid); 1881 1882 if (dev == NULL) 1883 return -1; 1884 1885 if (queue_id >= VHOST_MAX_VRING) 1886 return -1; 1887 1888 vq = dev->virtqueue[queue_id]; 1889 1890 if (unlikely(vq == NULL || !dev->async_copy || dev->vdpa_dev != NULL)) 1891 return -1; 1892 1893 vq_assert_lock(dev, vq); 1894 1895 return async_channel_register(dev, vq); 1896 } 1897 1898 int 1899 rte_vhost_async_channel_unregister(int vid, uint16_t queue_id) 1900 { 1901 struct vhost_virtqueue *vq; 1902 struct virtio_net *dev = get_device(vid); 1903 int ret = -1; 1904 1905 if (dev == NULL) 1906 return ret; 1907 1908 if (queue_id >= VHOST_MAX_VRING) 1909 return ret; 1910 1911 vq = dev->virtqueue[queue_id]; 1912 1913 if (vq == NULL) 1914 return ret; 1915 1916 if (rte_rwlock_write_trylock(&vq->access_lock)) { 1917 VHOST_LOG_CONFIG(dev->ifname, ERR, 1918 "failed to unregister async channel, virtqueue busy.\n"); 1919 return ret; 1920 } 1921 1922 if (unlikely(!vq->access_ok)) { 1923 ret = -1; 1924 goto out_unlock; 1925 } 1926 1927 if (!vq->async) { 1928 ret = 0; 1929 } else if (vq->async->pkts_inflight_n) { 1930 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to unregister async channel.\n"); 1931 VHOST_LOG_CONFIG(dev->ifname, ERR, 1932 "inflight packets must be completed before unregistration.\n"); 1933 } else { 1934 vhost_free_async_mem(vq); 1935 ret = 0; 1936 } 1937 1938 out_unlock: 1939 rte_rwlock_write_unlock(&vq->access_lock); 1940 1941 return ret; 1942 } 1943 1944 int 1945 rte_vhost_async_channel_unregister_thread_unsafe(int vid, uint16_t queue_id) 1946 { 1947 struct vhost_virtqueue *vq; 1948 struct virtio_net *dev = get_device(vid); 1949 1950 if (dev == NULL) 1951 return -1; 1952 1953 if (queue_id >= VHOST_MAX_VRING) 1954 return -1; 1955 1956 vq = dev->virtqueue[queue_id]; 1957 1958 if (vq == NULL) 1959 return -1; 1960 1961 vq_assert_lock(dev, vq); 1962 1963 if (!vq->async) 1964 return 0; 1965 1966 if (vq->async->pkts_inflight_n) { 1967 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to unregister async channel.\n"); 1968 VHOST_LOG_CONFIG(dev->ifname, ERR, 1969 "inflight packets must be completed before unregistration.\n"); 1970 return -1; 1971 } 1972 1973 vhost_free_async_mem(vq); 1974 1975 return 0; 1976 } 1977 1978 int 1979 rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id) 1980 { 1981 struct rte_dma_info info; 1982 void *pkts_cmpl_flag_addr; 1983 uint16_t max_desc; 1984 1985 pthread_mutex_lock(&vhost_dma_lock); 1986 1987 if (!rte_dma_is_valid(dma_id)) { 1988 VHOST_LOG_CONFIG("dma", ERR, "DMA %d is not found.\n", dma_id); 1989 goto error; 1990 } 1991 1992 if (rte_dma_info_get(dma_id, &info) != 0) { 1993 VHOST_LOG_CONFIG("dma", ERR, "Fail to get DMA %d information.\n", dma_id); 1994 goto error; 1995 } 1996 1997 if (vchan_id >= info.max_vchans) { 1998 VHOST_LOG_CONFIG("dma", ERR, "Invalid DMA %d vChannel %u.\n", dma_id, vchan_id); 1999 goto error; 2000 } 2001 2002 if (!dma_copy_track[dma_id].vchans) { 2003 struct async_dma_vchan_info *vchans; 2004 2005 vchans = rte_zmalloc(NULL, sizeof(struct async_dma_vchan_info) * info.max_vchans, 2006 RTE_CACHE_LINE_SIZE); 2007 if (vchans == NULL) { 2008 VHOST_LOG_CONFIG("dma", ERR, 2009 "Failed to allocate vchans for DMA %d vChannel %u.\n", 2010 dma_id, vchan_id); 2011 goto error; 2012 } 2013 2014 dma_copy_track[dma_id].vchans = vchans; 2015 } 2016 2017 if (dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr) { 2018 VHOST_LOG_CONFIG("dma", INFO, "DMA %d vChannel %u already registered.\n", 2019 dma_id, vchan_id); 2020 pthread_mutex_unlock(&vhost_dma_lock); 2021 return 0; 2022 } 2023 2024 max_desc = info.max_desc; 2025 if (!rte_is_power_of_2(max_desc)) 2026 max_desc = rte_align32pow2(max_desc); 2027 2028 pkts_cmpl_flag_addr = rte_zmalloc(NULL, sizeof(bool *) * max_desc, RTE_CACHE_LINE_SIZE); 2029 if (!pkts_cmpl_flag_addr) { 2030 VHOST_LOG_CONFIG("dma", ERR, 2031 "Failed to allocate pkts_cmpl_flag_addr for DMA %d vChannel %u.\n", 2032 dma_id, vchan_id); 2033 2034 if (dma_copy_track[dma_id].nr_vchans == 0) { 2035 rte_free(dma_copy_track[dma_id].vchans); 2036 dma_copy_track[dma_id].vchans = NULL; 2037 } 2038 goto error; 2039 } 2040 2041 dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr = pkts_cmpl_flag_addr; 2042 dma_copy_track[dma_id].vchans[vchan_id].ring_size = max_desc; 2043 dma_copy_track[dma_id].vchans[vchan_id].ring_mask = max_desc - 1; 2044 dma_copy_track[dma_id].nr_vchans++; 2045 2046 pthread_mutex_unlock(&vhost_dma_lock); 2047 return 0; 2048 2049 error: 2050 pthread_mutex_unlock(&vhost_dma_lock); 2051 return -1; 2052 } 2053 2054 int 2055 rte_vhost_async_get_inflight(int vid, uint16_t queue_id) 2056 { 2057 struct vhost_virtqueue *vq; 2058 struct virtio_net *dev = get_device(vid); 2059 int ret = -1; 2060 2061 if (dev == NULL) 2062 return ret; 2063 2064 if (queue_id >= VHOST_MAX_VRING) 2065 return ret; 2066 2067 vq = dev->virtqueue[queue_id]; 2068 2069 if (vq == NULL) 2070 return ret; 2071 2072 if (rte_rwlock_write_trylock(&vq->access_lock)) { 2073 VHOST_LOG_CONFIG(dev->ifname, DEBUG, 2074 "failed to check in-flight packets. virtqueue busy.\n"); 2075 return ret; 2076 } 2077 2078 if (unlikely(!vq->access_ok)) { 2079 ret = -1; 2080 goto out_unlock; 2081 } 2082 2083 if (vq->async) 2084 ret = vq->async->pkts_inflight_n; 2085 2086 out_unlock: 2087 rte_rwlock_write_unlock(&vq->access_lock); 2088 2089 return ret; 2090 } 2091 2092 int 2093 rte_vhost_async_get_inflight_thread_unsafe(int vid, uint16_t queue_id) 2094 { 2095 struct vhost_virtqueue *vq; 2096 struct virtio_net *dev = get_device(vid); 2097 int ret = -1; 2098 2099 if (dev == NULL) 2100 return ret; 2101 2102 if (queue_id >= VHOST_MAX_VRING) 2103 return ret; 2104 2105 vq = dev->virtqueue[queue_id]; 2106 2107 if (vq == NULL) 2108 return ret; 2109 2110 vq_assert_lock(dev, vq); 2111 2112 if (!vq->async) 2113 return ret; 2114 2115 ret = vq->async->pkts_inflight_n; 2116 2117 return ret; 2118 } 2119 2120 int 2121 rte_vhost_get_monitor_addr(int vid, uint16_t queue_id, 2122 struct rte_vhost_power_monitor_cond *pmc) 2123 { 2124 struct virtio_net *dev = get_device(vid); 2125 struct vhost_virtqueue *vq; 2126 int ret = 0; 2127 2128 if (dev == NULL) 2129 return -1; 2130 if (queue_id >= VHOST_MAX_VRING) 2131 return -1; 2132 2133 vq = dev->virtqueue[queue_id]; 2134 if (vq == NULL) 2135 return -1; 2136 2137 rte_rwlock_read_lock(&vq->access_lock); 2138 2139 if (unlikely(!vq->access_ok)) { 2140 ret = -1; 2141 goto out_unlock; 2142 } 2143 2144 if (vq_is_packed(dev)) { 2145 struct vring_packed_desc *desc; 2146 desc = vq->desc_packed; 2147 pmc->addr = &desc[vq->last_avail_idx].flags; 2148 if (vq->avail_wrap_counter) 2149 pmc->val = VRING_DESC_F_AVAIL; 2150 else 2151 pmc->val = VRING_DESC_F_USED; 2152 pmc->mask = VRING_DESC_F_AVAIL | VRING_DESC_F_USED; 2153 pmc->size = sizeof(desc[vq->last_avail_idx].flags); 2154 pmc->match = 1; 2155 } else { 2156 pmc->addr = &vq->avail->idx; 2157 pmc->val = vq->last_avail_idx & (vq->size - 1); 2158 pmc->mask = vq->size - 1; 2159 pmc->size = sizeof(vq->avail->idx); 2160 pmc->match = 0; 2161 } 2162 2163 out_unlock: 2164 rte_rwlock_read_unlock(&vq->access_lock); 2165 2166 return ret; 2167 } 2168 2169 2170 int 2171 rte_vhost_vring_stats_get_names(int vid, uint16_t queue_id, 2172 struct rte_vhost_stat_name *name, unsigned int size) 2173 { 2174 struct virtio_net *dev = get_device(vid); 2175 unsigned int i; 2176 2177 if (dev == NULL) 2178 return -1; 2179 2180 if (queue_id >= dev->nr_vring) 2181 return -1; 2182 2183 if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED)) 2184 return -1; 2185 2186 if (name == NULL || size < VHOST_NB_VQ_STATS) 2187 return VHOST_NB_VQ_STATS; 2188 2189 for (i = 0; i < VHOST_NB_VQ_STATS; i++) 2190 snprintf(name[i].name, sizeof(name[i].name), "%s_q%u_%s", 2191 (queue_id & 1) ? "rx" : "tx", 2192 queue_id / 2, vhost_vq_stat_strings[i].name); 2193 2194 return VHOST_NB_VQ_STATS; 2195 } 2196 2197 int 2198 rte_vhost_vring_stats_get(int vid, uint16_t queue_id, 2199 struct rte_vhost_stat *stats, unsigned int n) 2200 { 2201 struct virtio_net *dev = get_device(vid); 2202 struct vhost_virtqueue *vq; 2203 unsigned int i; 2204 int ret = VHOST_NB_VQ_STATS; 2205 2206 if (dev == NULL) 2207 return -1; 2208 2209 if (queue_id >= dev->nr_vring) 2210 return -1; 2211 2212 if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED)) 2213 return -1; 2214 2215 if (stats == NULL || n < VHOST_NB_VQ_STATS) 2216 return VHOST_NB_VQ_STATS; 2217 2218 vq = dev->virtqueue[queue_id]; 2219 2220 rte_rwlock_write_lock(&vq->access_lock); 2221 2222 if (unlikely(!vq->access_ok)) { 2223 ret = -1; 2224 goto out_unlock; 2225 } 2226 2227 for (i = 0; i < VHOST_NB_VQ_STATS; i++) { 2228 /* 2229 * No need to the read atomic counters as such, due to the 2230 * above write access_lock preventing them to be updated. 2231 */ 2232 stats[i].value = 2233 *(uint64_t *)(((char *)vq) + vhost_vq_stat_strings[i].offset); 2234 stats[i].id = i; 2235 } 2236 2237 out_unlock: 2238 rte_rwlock_write_unlock(&vq->access_lock); 2239 2240 return ret; 2241 } 2242 2243 int rte_vhost_vring_stats_reset(int vid, uint16_t queue_id) 2244 { 2245 struct virtio_net *dev = get_device(vid); 2246 struct vhost_virtqueue *vq; 2247 int ret = 0; 2248 2249 if (dev == NULL) 2250 return -1; 2251 2252 if (queue_id >= dev->nr_vring) 2253 return -1; 2254 2255 if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED)) 2256 return -1; 2257 2258 vq = dev->virtqueue[queue_id]; 2259 2260 rte_rwlock_write_lock(&vq->access_lock); 2261 2262 if (unlikely(!vq->access_ok)) { 2263 ret = -1; 2264 goto out_unlock; 2265 } 2266 /* 2267 * No need to the reset atomic counters as such, due to the 2268 * above write access_lock preventing them to be updated. 2269 */ 2270 memset(&vq->stats, 0, sizeof(vq->stats)); 2271 2272 out_unlock: 2273 rte_rwlock_write_unlock(&vq->access_lock); 2274 2275 return ret; 2276 } 2277 2278 int 2279 rte_vhost_async_dma_unconfigure(int16_t dma_id, uint16_t vchan_id) 2280 { 2281 struct rte_dma_info info; 2282 struct rte_dma_stats stats = { 0 }; 2283 2284 pthread_mutex_lock(&vhost_dma_lock); 2285 2286 if (!rte_dma_is_valid(dma_id)) { 2287 VHOST_LOG_CONFIG("dma", ERR, "DMA %d is not found.\n", dma_id); 2288 goto error; 2289 } 2290 2291 if (rte_dma_info_get(dma_id, &info) != 0) { 2292 VHOST_LOG_CONFIG("dma", ERR, "Fail to get DMA %d information.\n", dma_id); 2293 goto error; 2294 } 2295 2296 if (vchan_id >= info.max_vchans || !dma_copy_track[dma_id].vchans || 2297 !dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr) { 2298 VHOST_LOG_CONFIG("dma", ERR, "Invalid channel %d:%u.\n", dma_id, vchan_id); 2299 goto error; 2300 } 2301 2302 if (rte_dma_stats_get(dma_id, vchan_id, &stats) != 0) { 2303 VHOST_LOG_CONFIG("dma", ERR, 2304 "Failed to get stats for DMA %d vChannel %u.\n", dma_id, vchan_id); 2305 goto error; 2306 } 2307 2308 if (stats.submitted - stats.completed != 0) { 2309 VHOST_LOG_CONFIG("dma", ERR, 2310 "Do not unconfigure when there are inflight packets.\n"); 2311 goto error; 2312 } 2313 2314 rte_free(dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr); 2315 dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr = NULL; 2316 dma_copy_track[dma_id].nr_vchans--; 2317 2318 if (dma_copy_track[dma_id].nr_vchans == 0) { 2319 rte_free(dma_copy_track[dma_id].vchans); 2320 dma_copy_track[dma_id].vchans = NULL; 2321 } 2322 2323 pthread_mutex_unlock(&vhost_dma_lock); 2324 return 0; 2325 2326 error: 2327 pthread_mutex_unlock(&vhost_dma_lock); 2328 return -1; 2329 } 2330 2331 RTE_LOG_REGISTER_SUFFIX(vhost_config_log_level, config, INFO); 2332 RTE_LOG_REGISTER_SUFFIX(vhost_data_log_level, data, WARNING); 2333