1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <linux/vhost.h> 6 #include <linux/virtio_net.h> 7 #include <stdint.h> 8 #include <stdlib.h> 9 #ifdef RTE_LIBRTE_VHOST_NUMA 10 #include <numa.h> 11 #include <numaif.h> 12 #endif 13 14 #include <rte_errno.h> 15 #include <rte_log.h> 16 #include <rte_memory.h> 17 #include <rte_malloc.h> 18 #include <rte_vhost.h> 19 20 #include "iotlb.h" 21 #include "vhost.h" 22 #include "vhost_user.h" 23 24 struct virtio_net *vhost_devices[RTE_MAX_VHOST_DEVICE]; 25 pthread_mutex_t vhost_dev_lock = PTHREAD_MUTEX_INITIALIZER; 26 pthread_mutex_t vhost_dma_lock = PTHREAD_MUTEX_INITIALIZER; 27 28 struct vhost_vq_stats_name_off { 29 char name[RTE_VHOST_STATS_NAME_SIZE]; 30 unsigned int offset; 31 }; 32 33 static const struct vhost_vq_stats_name_off vhost_vq_stat_strings[] = { 34 {"good_packets", offsetof(struct vhost_virtqueue, stats.packets)}, 35 {"good_bytes", offsetof(struct vhost_virtqueue, stats.bytes)}, 36 {"multicast_packets", offsetof(struct vhost_virtqueue, stats.multicast)}, 37 {"broadcast_packets", offsetof(struct vhost_virtqueue, stats.broadcast)}, 38 {"undersize_packets", offsetof(struct vhost_virtqueue, stats.size_bins[0])}, 39 {"size_64_packets", offsetof(struct vhost_virtqueue, stats.size_bins[1])}, 40 {"size_65_127_packets", offsetof(struct vhost_virtqueue, stats.size_bins[2])}, 41 {"size_128_255_packets", offsetof(struct vhost_virtqueue, stats.size_bins[3])}, 42 {"size_256_511_packets", offsetof(struct vhost_virtqueue, stats.size_bins[4])}, 43 {"size_512_1023_packets", offsetof(struct vhost_virtqueue, stats.size_bins[5])}, 44 {"size_1024_1518_packets", offsetof(struct vhost_virtqueue, stats.size_bins[6])}, 45 {"size_1519_max_packets", offsetof(struct vhost_virtqueue, stats.size_bins[7])}, 46 {"guest_notifications", offsetof(struct vhost_virtqueue, stats.guest_notifications)}, 47 {"iotlb_hits", offsetof(struct vhost_virtqueue, stats.iotlb_hits)}, 48 {"iotlb_misses", offsetof(struct vhost_virtqueue, stats.iotlb_misses)}, 49 {"inflight_submitted", offsetof(struct vhost_virtqueue, stats.inflight_submitted)}, 50 {"inflight_completed", offsetof(struct vhost_virtqueue, stats.inflight_completed)}, 51 }; 52 53 #define VHOST_NB_VQ_STATS RTE_DIM(vhost_vq_stat_strings) 54 55 uint64_t 56 __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, 57 uint64_t iova, uint64_t *size, uint8_t perm) 58 { 59 uint64_t vva, tmp_size; 60 61 if (unlikely(!*size)) 62 return 0; 63 64 tmp_size = *size; 65 66 vva = vhost_user_iotlb_cache_find(vq, iova, &tmp_size, perm); 67 if (tmp_size == *size) { 68 if (dev->flags & VIRTIO_DEV_STATS_ENABLED) 69 vq->stats.iotlb_hits++; 70 return vva; 71 } 72 73 if (dev->flags & VIRTIO_DEV_STATS_ENABLED) 74 vq->stats.iotlb_misses++; 75 76 iova += tmp_size; 77 78 if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) { 79 /* 80 * iotlb_lock is read-locked for a full burst, 81 * but it only protects the iotlb cache. 82 * In case of IOTLB miss, we might block on the socket, 83 * which could cause a deadlock with QEMU if an IOTLB update 84 * is being handled. We can safely unlock here to avoid it. 85 */ 86 vhost_user_iotlb_rd_unlock(vq); 87 88 vhost_user_iotlb_pending_insert(dev, vq, iova, perm); 89 if (vhost_user_iotlb_miss(dev, iova, perm)) { 90 VHOST_LOG_DATA(dev->ifname, ERR, 91 "IOTLB miss req failed for IOVA 0x%" PRIx64 "\n", 92 iova); 93 vhost_user_iotlb_pending_remove(vq, iova, 1, perm); 94 } 95 96 vhost_user_iotlb_rd_lock(vq); 97 } 98 99 return 0; 100 } 101 102 #define VHOST_LOG_PAGE 4096 103 104 /* 105 * Atomically set a bit in memory. 106 */ 107 static __rte_always_inline void 108 vhost_set_bit(unsigned int nr, volatile uint8_t *addr) 109 { 110 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) 111 /* 112 * __sync_ built-ins are deprecated, but __atomic_ ones 113 * are sub-optimized in older GCC versions. 114 */ 115 __sync_fetch_and_or_1(addr, (1U << nr)); 116 #else 117 __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED); 118 #endif 119 } 120 121 static __rte_always_inline void 122 vhost_log_page(uint8_t *log_base, uint64_t page) 123 { 124 vhost_set_bit(page % 8, &log_base[page / 8]); 125 } 126 127 void 128 __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) 129 { 130 uint64_t page; 131 132 if (unlikely(!dev->log_base || !len)) 133 return; 134 135 if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) 136 return; 137 138 /* To make sure guest memory updates are committed before logging */ 139 rte_atomic_thread_fence(__ATOMIC_RELEASE); 140 141 page = addr / VHOST_LOG_PAGE; 142 while (page * VHOST_LOG_PAGE < addr + len) { 143 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 144 page += 1; 145 } 146 } 147 148 void 149 __vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, 150 uint64_t iova, uint64_t len) 151 { 152 uint64_t hva, gpa, map_len; 153 map_len = len; 154 155 hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); 156 if (map_len != len) { 157 VHOST_LOG_DATA(dev->ifname, ERR, 158 "failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n", 159 iova); 160 return; 161 } 162 163 gpa = hva_to_gpa(dev, hva, len); 164 if (gpa) 165 __vhost_log_write(dev, gpa, len); 166 } 167 168 void 169 __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) 170 { 171 unsigned long *log_base; 172 int i; 173 174 if (unlikely(!dev->log_base)) 175 return; 176 177 /* No cache, nothing to sync */ 178 if (unlikely(!vq->log_cache)) 179 return; 180 181 rte_atomic_thread_fence(__ATOMIC_RELEASE); 182 183 log_base = (unsigned long *)(uintptr_t)dev->log_base; 184 185 for (i = 0; i < vq->log_cache_nb_elem; i++) { 186 struct log_cache_entry *elem = vq->log_cache + i; 187 188 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) 189 /* 190 * '__sync' builtins are deprecated, but '__atomic' ones 191 * are sub-optimized in older GCC versions. 192 */ 193 __sync_fetch_and_or(log_base + elem->offset, elem->val); 194 #else 195 __atomic_fetch_or(log_base + elem->offset, elem->val, 196 __ATOMIC_RELAXED); 197 #endif 198 } 199 200 rte_atomic_thread_fence(__ATOMIC_RELEASE); 201 202 vq->log_cache_nb_elem = 0; 203 } 204 205 static __rte_always_inline void 206 vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, 207 uint64_t page) 208 { 209 uint32_t bit_nr = page % (sizeof(unsigned long) << 3); 210 uint32_t offset = page / (sizeof(unsigned long) << 3); 211 int i; 212 213 if (unlikely(!vq->log_cache)) { 214 /* No logging cache allocated, write dirty log map directly */ 215 rte_atomic_thread_fence(__ATOMIC_RELEASE); 216 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 217 218 return; 219 } 220 221 for (i = 0; i < vq->log_cache_nb_elem; i++) { 222 struct log_cache_entry *elem = vq->log_cache + i; 223 224 if (elem->offset == offset) { 225 elem->val |= (1UL << bit_nr); 226 return; 227 } 228 } 229 230 if (unlikely(i >= VHOST_LOG_CACHE_NR)) { 231 /* 232 * No more room for a new log cache entry, 233 * so write the dirty log map directly. 234 */ 235 rte_atomic_thread_fence(__ATOMIC_RELEASE); 236 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 237 238 return; 239 } 240 241 vq->log_cache[i].offset = offset; 242 vq->log_cache[i].val = (1UL << bit_nr); 243 vq->log_cache_nb_elem++; 244 } 245 246 void 247 __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, 248 uint64_t addr, uint64_t len) 249 { 250 uint64_t page; 251 252 if (unlikely(!dev->log_base || !len)) 253 return; 254 255 if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) 256 return; 257 258 page = addr / VHOST_LOG_PAGE; 259 while (page * VHOST_LOG_PAGE < addr + len) { 260 vhost_log_cache_page(dev, vq, page); 261 page += 1; 262 } 263 } 264 265 void 266 __vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, 267 uint64_t iova, uint64_t len) 268 { 269 uint64_t hva, gpa, map_len; 270 map_len = len; 271 272 hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); 273 if (map_len != len) { 274 VHOST_LOG_DATA(dev->ifname, ERR, 275 "failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n", 276 iova); 277 return; 278 } 279 280 gpa = hva_to_gpa(dev, hva, len); 281 if (gpa) 282 __vhost_log_cache_write(dev, vq, gpa, len); 283 } 284 285 void * 286 vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, 287 uint64_t desc_addr, uint64_t desc_len) 288 { 289 void *idesc; 290 uint64_t src, dst; 291 uint64_t len, remain = desc_len; 292 293 idesc = rte_malloc_socket(__func__, desc_len, 0, vq->numa_node); 294 if (unlikely(!idesc)) 295 return NULL; 296 297 dst = (uint64_t)(uintptr_t)idesc; 298 299 while (remain) { 300 len = remain; 301 src = vhost_iova_to_vva(dev, vq, desc_addr, &len, 302 VHOST_ACCESS_RO); 303 if (unlikely(!src || !len)) { 304 rte_free(idesc); 305 return NULL; 306 } 307 308 rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); 309 310 remain -= len; 311 dst += len; 312 desc_addr += len; 313 } 314 315 return idesc; 316 } 317 318 void 319 cleanup_vq(struct vhost_virtqueue *vq, int destroy) 320 { 321 if ((vq->callfd >= 0) && (destroy != 0)) 322 close(vq->callfd); 323 if (vq->kickfd >= 0) 324 close(vq->kickfd); 325 } 326 327 void 328 cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq) 329 { 330 if (!(dev->protocol_features & 331 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))) 332 return; 333 334 if (vq_is_packed(dev)) { 335 if (vq->inflight_packed) 336 vq->inflight_packed = NULL; 337 } else { 338 if (vq->inflight_split) 339 vq->inflight_split = NULL; 340 } 341 342 if (vq->resubmit_inflight) { 343 if (vq->resubmit_inflight->resubmit_list) { 344 rte_free(vq->resubmit_inflight->resubmit_list); 345 vq->resubmit_inflight->resubmit_list = NULL; 346 } 347 rte_free(vq->resubmit_inflight); 348 vq->resubmit_inflight = NULL; 349 } 350 } 351 352 /* 353 * Unmap any memory, close any file descriptors and 354 * free any memory owned by a device. 355 */ 356 void 357 cleanup_device(struct virtio_net *dev, int destroy) 358 { 359 uint32_t i; 360 361 vhost_backend_cleanup(dev); 362 363 for (i = 0; i < dev->nr_vring; i++) { 364 cleanup_vq(dev->virtqueue[i], destroy); 365 cleanup_vq_inflight(dev, dev->virtqueue[i]); 366 } 367 } 368 369 static void 370 vhost_free_async_mem(struct vhost_virtqueue *vq) 371 __rte_exclusive_locks_required(&vq->access_lock) 372 { 373 if (!vq->async) 374 return; 375 376 rte_free(vq->async->pkts_info); 377 rte_free(vq->async->pkts_cmpl_flag); 378 379 rte_free(vq->async->buffers_packed); 380 vq->async->buffers_packed = NULL; 381 rte_free(vq->async->descs_split); 382 vq->async->descs_split = NULL; 383 384 rte_free(vq->async); 385 vq->async = NULL; 386 } 387 388 void 389 free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq) 390 { 391 if (vq_is_packed(dev)) 392 rte_free(vq->shadow_used_packed); 393 else 394 rte_free(vq->shadow_used_split); 395 396 rte_spinlock_lock(&vq->access_lock); 397 vhost_free_async_mem(vq); 398 rte_spinlock_unlock(&vq->access_lock); 399 rte_free(vq->batch_copy_elems); 400 vhost_user_iotlb_destroy(vq); 401 rte_free(vq->log_cache); 402 rte_free(vq); 403 } 404 405 /* 406 * Release virtqueues and device memory. 407 */ 408 static void 409 free_device(struct virtio_net *dev) 410 { 411 uint32_t i; 412 413 for (i = 0; i < dev->nr_vring; i++) 414 free_vq(dev, dev->virtqueue[i]); 415 416 rte_free(dev); 417 } 418 419 static __rte_always_inline int 420 log_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) 421 __rte_shared_locks_required(&vq->iotlb_lock) 422 { 423 if (likely(!(vq->ring_addrs.flags & (1 << VHOST_VRING_F_LOG)))) 424 return 0; 425 426 vq->log_guest_addr = translate_log_addr(dev, vq, 427 vq->ring_addrs.log_guest_addr); 428 if (vq->log_guest_addr == 0) 429 return -1; 430 431 return 0; 432 } 433 434 /* 435 * Converts vring log address to GPA 436 * If IOMMU is enabled, the log address is IOVA 437 * If IOMMU not enabled, the log address is already GPA 438 */ 439 uint64_t 440 translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq, 441 uint64_t log_addr) 442 { 443 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) { 444 const uint64_t exp_size = sizeof(uint64_t); 445 uint64_t hva, gpa; 446 uint64_t size = exp_size; 447 448 hva = vhost_iova_to_vva(dev, vq, log_addr, 449 &size, VHOST_ACCESS_RW); 450 451 if (size != exp_size) 452 return 0; 453 454 gpa = hva_to_gpa(dev, hva, exp_size); 455 if (!gpa) { 456 VHOST_LOG_DATA(dev->ifname, ERR, 457 "failed to find GPA for log_addr: 0x%" 458 PRIx64 " hva: 0x%" PRIx64 "\n", 459 log_addr, hva); 460 return 0; 461 } 462 return gpa; 463 464 } else 465 return log_addr; 466 } 467 468 static int 469 vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq) 470 __rte_shared_locks_required(&vq->iotlb_lock) 471 { 472 uint64_t req_size, size; 473 474 req_size = sizeof(struct vring_desc) * vq->size; 475 size = req_size; 476 vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq, 477 vq->ring_addrs.desc_user_addr, 478 &size, VHOST_ACCESS_RW); 479 if (!vq->desc || size != req_size) 480 return -1; 481 482 req_size = sizeof(struct vring_avail); 483 req_size += sizeof(uint16_t) * vq->size; 484 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 485 req_size += sizeof(uint16_t); 486 size = req_size; 487 vq->avail = (struct vring_avail *)(uintptr_t)vhost_iova_to_vva(dev, vq, 488 vq->ring_addrs.avail_user_addr, 489 &size, VHOST_ACCESS_RW); 490 if (!vq->avail || size != req_size) 491 return -1; 492 493 req_size = sizeof(struct vring_used); 494 req_size += sizeof(struct vring_used_elem) * vq->size; 495 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 496 req_size += sizeof(uint16_t); 497 size = req_size; 498 vq->used = (struct vring_used *)(uintptr_t)vhost_iova_to_vva(dev, vq, 499 vq->ring_addrs.used_user_addr, 500 &size, VHOST_ACCESS_RW); 501 if (!vq->used || size != req_size) 502 return -1; 503 504 return 0; 505 } 506 507 static int 508 vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) 509 __rte_shared_locks_required(&vq->iotlb_lock) 510 { 511 uint64_t req_size, size; 512 513 req_size = sizeof(struct vring_packed_desc) * vq->size; 514 size = req_size; 515 vq->desc_packed = (struct vring_packed_desc *)(uintptr_t) 516 vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr, 517 &size, VHOST_ACCESS_RW); 518 if (!vq->desc_packed || size != req_size) 519 return -1; 520 521 req_size = sizeof(struct vring_packed_desc_event); 522 size = req_size; 523 vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t) 524 vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr, 525 &size, VHOST_ACCESS_RW); 526 if (!vq->driver_event || size != req_size) 527 return -1; 528 529 req_size = sizeof(struct vring_packed_desc_event); 530 size = req_size; 531 vq->device_event = (struct vring_packed_desc_event *)(uintptr_t) 532 vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr, 533 &size, VHOST_ACCESS_RW); 534 if (!vq->device_event || size != req_size) 535 return -1; 536 537 return 0; 538 } 539 540 int 541 vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) 542 { 543 544 if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) 545 return -1; 546 547 if (vq_is_packed(dev)) { 548 if (vring_translate_packed(dev, vq) < 0) 549 return -1; 550 } else { 551 if (vring_translate_split(dev, vq) < 0) 552 return -1; 553 } 554 555 if (log_translate(dev, vq) < 0) 556 return -1; 557 558 vq->access_ok = true; 559 560 return 0; 561 } 562 563 void 564 vring_invalidate(struct virtio_net *dev __rte_unused, struct vhost_virtqueue *vq) 565 { 566 vhost_user_iotlb_wr_lock(vq); 567 568 vq->access_ok = false; 569 vq->desc = NULL; 570 vq->avail = NULL; 571 vq->used = NULL; 572 vq->log_guest_addr = 0; 573 574 vhost_user_iotlb_wr_unlock(vq); 575 } 576 577 static void 578 init_vring_queue(struct virtio_net *dev, struct vhost_virtqueue *vq, 579 uint32_t vring_idx) 580 { 581 int numa_node = SOCKET_ID_ANY; 582 583 memset(vq, 0, sizeof(struct vhost_virtqueue)); 584 585 vq->index = vring_idx; 586 vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; 587 vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; 588 vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF; 589 590 #ifdef RTE_LIBRTE_VHOST_NUMA 591 if (get_mempolicy(&numa_node, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR)) { 592 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to query numa node: %s\n", 593 rte_strerror(errno)); 594 numa_node = SOCKET_ID_ANY; 595 } 596 #endif 597 vq->numa_node = numa_node; 598 599 vhost_user_iotlb_init(dev, vq); 600 } 601 602 static void 603 reset_vring_queue(struct virtio_net *dev, struct vhost_virtqueue *vq) 604 { 605 int callfd; 606 607 callfd = vq->callfd; 608 init_vring_queue(dev, vq, vq->index); 609 vq->callfd = callfd; 610 } 611 612 int 613 alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx) 614 { 615 struct vhost_virtqueue *vq; 616 uint32_t i; 617 618 /* Also allocate holes, if any, up to requested vring index. */ 619 for (i = 0; i <= vring_idx; i++) { 620 if (dev->virtqueue[i]) 621 continue; 622 623 vq = rte_zmalloc(NULL, sizeof(struct vhost_virtqueue), 0); 624 if (vq == NULL) { 625 VHOST_LOG_CONFIG(dev->ifname, ERR, 626 "failed to allocate memory for vring %u.\n", 627 i); 628 return -1; 629 } 630 631 dev->virtqueue[i] = vq; 632 init_vring_queue(dev, vq, i); 633 rte_spinlock_init(&vq->access_lock); 634 vq->avail_wrap_counter = 1; 635 vq->used_wrap_counter = 1; 636 vq->signalled_used_valid = false; 637 } 638 639 dev->nr_vring = RTE_MAX(dev->nr_vring, vring_idx + 1); 640 641 return 0; 642 } 643 644 /* 645 * Reset some variables in device structure, while keeping few 646 * others untouched, such as vid, ifname, nr_vring: they 647 * should be same unless the device is removed. 648 */ 649 void 650 reset_device(struct virtio_net *dev) 651 { 652 uint32_t i; 653 654 dev->features = 0; 655 dev->protocol_features = 0; 656 dev->flags &= VIRTIO_DEV_BUILTIN_VIRTIO_NET; 657 658 for (i = 0; i < dev->nr_vring; i++) { 659 struct vhost_virtqueue *vq = dev->virtqueue[i]; 660 661 if (!vq) { 662 VHOST_LOG_CONFIG(dev->ifname, ERR, 663 "failed to reset vring, virtqueue not allocated (%d)\n", i); 664 continue; 665 } 666 reset_vring_queue(dev, vq); 667 } 668 } 669 670 /* 671 * Invoked when there is a new vhost-user connection established (when 672 * there is a new virtio device being attached). 673 */ 674 int 675 vhost_new_device(void) 676 { 677 struct virtio_net *dev; 678 int i; 679 680 pthread_mutex_lock(&vhost_dev_lock); 681 for (i = 0; i < RTE_MAX_VHOST_DEVICE; i++) { 682 if (vhost_devices[i] == NULL) 683 break; 684 } 685 686 if (i == RTE_MAX_VHOST_DEVICE) { 687 VHOST_LOG_CONFIG("device", ERR, "failed to find a free slot for new device.\n"); 688 pthread_mutex_unlock(&vhost_dev_lock); 689 return -1; 690 } 691 692 dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0); 693 if (dev == NULL) { 694 VHOST_LOG_CONFIG("device", ERR, "failed to allocate memory for new device.\n"); 695 pthread_mutex_unlock(&vhost_dev_lock); 696 return -1; 697 } 698 699 vhost_devices[i] = dev; 700 pthread_mutex_unlock(&vhost_dev_lock); 701 702 dev->vid = i; 703 dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET; 704 dev->slave_req_fd = -1; 705 dev->postcopy_ufd = -1; 706 rte_spinlock_init(&dev->slave_req_lock); 707 708 return i; 709 } 710 711 void 712 vhost_destroy_device_notify(struct virtio_net *dev) 713 { 714 struct rte_vdpa_device *vdpa_dev; 715 716 if (dev->flags & VIRTIO_DEV_RUNNING) { 717 vdpa_dev = dev->vdpa_dev; 718 if (vdpa_dev) 719 vdpa_dev->ops->dev_close(dev->vid); 720 dev->flags &= ~VIRTIO_DEV_RUNNING; 721 dev->notify_ops->destroy_device(dev->vid); 722 } 723 } 724 725 /* 726 * Invoked when there is the vhost-user connection is broken (when 727 * the virtio device is being detached). 728 */ 729 void 730 vhost_destroy_device(int vid) 731 { 732 struct virtio_net *dev = get_device(vid); 733 734 if (dev == NULL) 735 return; 736 737 vhost_destroy_device_notify(dev); 738 739 cleanup_device(dev, 1); 740 free_device(dev); 741 742 vhost_devices[vid] = NULL; 743 } 744 745 void 746 vhost_attach_vdpa_device(int vid, struct rte_vdpa_device *vdpa_dev) 747 { 748 struct virtio_net *dev = get_device(vid); 749 750 if (dev == NULL) 751 return; 752 753 dev->vdpa_dev = vdpa_dev; 754 } 755 756 void 757 vhost_set_ifname(int vid, const char *if_name, unsigned int if_len) 758 { 759 struct virtio_net *dev; 760 unsigned int len; 761 762 dev = get_device(vid); 763 if (dev == NULL) 764 return; 765 766 len = if_len > sizeof(dev->ifname) ? 767 sizeof(dev->ifname) : if_len; 768 769 strncpy(dev->ifname, if_name, len); 770 dev->ifname[sizeof(dev->ifname) - 1] = '\0'; 771 } 772 773 void 774 vhost_setup_virtio_net(int vid, bool enable, bool compliant_ol_flags, bool stats_enabled, 775 bool support_iommu) 776 { 777 struct virtio_net *dev = get_device(vid); 778 779 if (dev == NULL) 780 return; 781 782 if (enable) 783 dev->flags |= VIRTIO_DEV_BUILTIN_VIRTIO_NET; 784 else 785 dev->flags &= ~VIRTIO_DEV_BUILTIN_VIRTIO_NET; 786 if (!compliant_ol_flags) 787 dev->flags |= VIRTIO_DEV_LEGACY_OL_FLAGS; 788 else 789 dev->flags &= ~VIRTIO_DEV_LEGACY_OL_FLAGS; 790 if (stats_enabled) 791 dev->flags |= VIRTIO_DEV_STATS_ENABLED; 792 else 793 dev->flags &= ~VIRTIO_DEV_STATS_ENABLED; 794 if (support_iommu) 795 dev->flags |= VIRTIO_DEV_SUPPORT_IOMMU; 796 else 797 dev->flags &= ~VIRTIO_DEV_SUPPORT_IOMMU; 798 } 799 800 void 801 vhost_enable_extbuf(int vid) 802 { 803 struct virtio_net *dev = get_device(vid); 804 805 if (dev == NULL) 806 return; 807 808 dev->extbuf = 1; 809 } 810 811 void 812 vhost_enable_linearbuf(int vid) 813 { 814 struct virtio_net *dev = get_device(vid); 815 816 if (dev == NULL) 817 return; 818 819 dev->linearbuf = 1; 820 } 821 822 int 823 rte_vhost_get_mtu(int vid, uint16_t *mtu) 824 { 825 struct virtio_net *dev = get_device(vid); 826 827 if (dev == NULL || mtu == NULL) 828 return -ENODEV; 829 830 if (!(dev->flags & VIRTIO_DEV_READY)) 831 return -EAGAIN; 832 833 if (!(dev->features & (1ULL << VIRTIO_NET_F_MTU))) 834 return -ENOTSUP; 835 836 *mtu = dev->mtu; 837 838 return 0; 839 } 840 841 int 842 rte_vhost_get_numa_node(int vid) 843 { 844 #ifdef RTE_LIBRTE_VHOST_NUMA 845 struct virtio_net *dev = get_device(vid); 846 int numa_node; 847 int ret; 848 849 if (dev == NULL || numa_available() != 0) 850 return -1; 851 852 ret = get_mempolicy(&numa_node, NULL, 0, dev, 853 MPOL_F_NODE | MPOL_F_ADDR); 854 if (ret < 0) { 855 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to query numa node: %s\n", 856 rte_strerror(errno)); 857 return -1; 858 } 859 860 return numa_node; 861 #else 862 RTE_SET_USED(vid); 863 return -1; 864 #endif 865 } 866 867 uint16_t 868 rte_vhost_get_vring_num(int vid) 869 { 870 struct virtio_net *dev = get_device(vid); 871 872 if (dev == NULL) 873 return 0; 874 875 return dev->nr_vring; 876 } 877 878 int 879 rte_vhost_get_ifname(int vid, char *buf, size_t len) 880 { 881 struct virtio_net *dev = get_device(vid); 882 883 if (dev == NULL || buf == NULL) 884 return -1; 885 886 len = RTE_MIN(len, sizeof(dev->ifname)); 887 888 strncpy(buf, dev->ifname, len); 889 buf[len - 1] = '\0'; 890 891 return 0; 892 } 893 894 int 895 rte_vhost_get_negotiated_features(int vid, uint64_t *features) 896 { 897 struct virtio_net *dev; 898 899 dev = get_device(vid); 900 if (dev == NULL || features == NULL) 901 return -1; 902 903 *features = dev->features; 904 return 0; 905 } 906 907 int 908 rte_vhost_get_negotiated_protocol_features(int vid, 909 uint64_t *protocol_features) 910 { 911 struct virtio_net *dev; 912 913 dev = get_device(vid); 914 if (dev == NULL || protocol_features == NULL) 915 return -1; 916 917 *protocol_features = dev->protocol_features; 918 return 0; 919 } 920 921 int 922 rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem) 923 { 924 struct virtio_net *dev; 925 struct rte_vhost_memory *m; 926 size_t size; 927 928 dev = get_device(vid); 929 if (dev == NULL || mem == NULL) 930 return -1; 931 932 size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region); 933 m = malloc(sizeof(struct rte_vhost_memory) + size); 934 if (!m) 935 return -1; 936 937 m->nregions = dev->mem->nregions; 938 memcpy(m->regions, dev->mem->regions, size); 939 *mem = m; 940 941 return 0; 942 } 943 944 int 945 rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, 946 struct rte_vhost_vring *vring) 947 { 948 struct virtio_net *dev; 949 struct vhost_virtqueue *vq; 950 951 dev = get_device(vid); 952 if (dev == NULL || vring == NULL) 953 return -1; 954 955 if (vring_idx >= VHOST_MAX_VRING) 956 return -1; 957 958 vq = dev->virtqueue[vring_idx]; 959 if (!vq) 960 return -1; 961 962 if (vq_is_packed(dev)) { 963 vring->desc_packed = vq->desc_packed; 964 vring->driver_event = vq->driver_event; 965 vring->device_event = vq->device_event; 966 } else { 967 vring->desc = vq->desc; 968 vring->avail = vq->avail; 969 vring->used = vq->used; 970 } 971 vring->log_guest_addr = vq->log_guest_addr; 972 973 vring->callfd = vq->callfd; 974 vring->kickfd = vq->kickfd; 975 vring->size = vq->size; 976 977 return 0; 978 } 979 980 int 981 rte_vhost_get_vhost_ring_inflight(int vid, uint16_t vring_idx, 982 struct rte_vhost_ring_inflight *vring) 983 { 984 struct virtio_net *dev; 985 struct vhost_virtqueue *vq; 986 987 dev = get_device(vid); 988 if (unlikely(!dev)) 989 return -1; 990 991 if (vring_idx >= VHOST_MAX_VRING) 992 return -1; 993 994 vq = dev->virtqueue[vring_idx]; 995 if (unlikely(!vq)) 996 return -1; 997 998 if (vq_is_packed(dev)) { 999 if (unlikely(!vq->inflight_packed)) 1000 return -1; 1001 1002 vring->inflight_packed = vq->inflight_packed; 1003 } else { 1004 if (unlikely(!vq->inflight_split)) 1005 return -1; 1006 1007 vring->inflight_split = vq->inflight_split; 1008 } 1009 1010 vring->resubmit_inflight = vq->resubmit_inflight; 1011 1012 return 0; 1013 } 1014 1015 int 1016 rte_vhost_set_inflight_desc_split(int vid, uint16_t vring_idx, 1017 uint16_t idx) 1018 { 1019 struct vhost_virtqueue *vq; 1020 struct virtio_net *dev; 1021 1022 dev = get_device(vid); 1023 if (unlikely(!dev)) 1024 return -1; 1025 1026 if (unlikely(!(dev->protocol_features & 1027 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1028 return 0; 1029 1030 if (unlikely(vq_is_packed(dev))) 1031 return -1; 1032 1033 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1034 return -1; 1035 1036 vq = dev->virtqueue[vring_idx]; 1037 if (unlikely(!vq)) 1038 return -1; 1039 1040 if (unlikely(!vq->inflight_split)) 1041 return -1; 1042 1043 if (unlikely(idx >= vq->size)) 1044 return -1; 1045 1046 vq->inflight_split->desc[idx].counter = vq->global_counter++; 1047 vq->inflight_split->desc[idx].inflight = 1; 1048 return 0; 1049 } 1050 1051 int 1052 rte_vhost_set_inflight_desc_packed(int vid, uint16_t vring_idx, 1053 uint16_t head, uint16_t last, 1054 uint16_t *inflight_entry) 1055 { 1056 struct rte_vhost_inflight_info_packed *inflight_info; 1057 struct virtio_net *dev; 1058 struct vhost_virtqueue *vq; 1059 struct vring_packed_desc *desc; 1060 uint16_t old_free_head, free_head; 1061 1062 dev = get_device(vid); 1063 if (unlikely(!dev)) 1064 return -1; 1065 1066 if (unlikely(!(dev->protocol_features & 1067 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1068 return 0; 1069 1070 if (unlikely(!vq_is_packed(dev))) 1071 return -1; 1072 1073 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1074 return -1; 1075 1076 vq = dev->virtqueue[vring_idx]; 1077 if (unlikely(!vq)) 1078 return -1; 1079 1080 inflight_info = vq->inflight_packed; 1081 if (unlikely(!inflight_info)) 1082 return -1; 1083 1084 if (unlikely(head >= vq->size)) 1085 return -1; 1086 1087 desc = vq->desc_packed; 1088 old_free_head = inflight_info->old_free_head; 1089 if (unlikely(old_free_head >= vq->size)) 1090 return -1; 1091 1092 free_head = old_free_head; 1093 1094 /* init header descriptor */ 1095 inflight_info->desc[old_free_head].num = 0; 1096 inflight_info->desc[old_free_head].counter = vq->global_counter++; 1097 inflight_info->desc[old_free_head].inflight = 1; 1098 1099 /* save desc entry in flight entry */ 1100 while (head != ((last + 1) % vq->size)) { 1101 inflight_info->desc[old_free_head].num++; 1102 inflight_info->desc[free_head].addr = desc[head].addr; 1103 inflight_info->desc[free_head].len = desc[head].len; 1104 inflight_info->desc[free_head].flags = desc[head].flags; 1105 inflight_info->desc[free_head].id = desc[head].id; 1106 1107 inflight_info->desc[old_free_head].last = free_head; 1108 free_head = inflight_info->desc[free_head].next; 1109 inflight_info->free_head = free_head; 1110 head = (head + 1) % vq->size; 1111 } 1112 1113 inflight_info->old_free_head = free_head; 1114 *inflight_entry = old_free_head; 1115 1116 return 0; 1117 } 1118 1119 int 1120 rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx, 1121 uint16_t last_used_idx, uint16_t idx) 1122 { 1123 struct virtio_net *dev; 1124 struct vhost_virtqueue *vq; 1125 1126 dev = get_device(vid); 1127 if (unlikely(!dev)) 1128 return -1; 1129 1130 if (unlikely(!(dev->protocol_features & 1131 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1132 return 0; 1133 1134 if (unlikely(vq_is_packed(dev))) 1135 return -1; 1136 1137 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1138 return -1; 1139 1140 vq = dev->virtqueue[vring_idx]; 1141 if (unlikely(!vq)) 1142 return -1; 1143 1144 if (unlikely(!vq->inflight_split)) 1145 return -1; 1146 1147 if (unlikely(idx >= vq->size)) 1148 return -1; 1149 1150 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1151 1152 vq->inflight_split->desc[idx].inflight = 0; 1153 1154 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1155 1156 vq->inflight_split->used_idx = last_used_idx; 1157 return 0; 1158 } 1159 1160 int 1161 rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx, 1162 uint16_t head) 1163 { 1164 struct rte_vhost_inflight_info_packed *inflight_info; 1165 struct virtio_net *dev; 1166 struct vhost_virtqueue *vq; 1167 1168 dev = get_device(vid); 1169 if (unlikely(!dev)) 1170 return -1; 1171 1172 if (unlikely(!(dev->protocol_features & 1173 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1174 return 0; 1175 1176 if (unlikely(!vq_is_packed(dev))) 1177 return -1; 1178 1179 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1180 return -1; 1181 1182 vq = dev->virtqueue[vring_idx]; 1183 if (unlikely(!vq)) 1184 return -1; 1185 1186 inflight_info = vq->inflight_packed; 1187 if (unlikely(!inflight_info)) 1188 return -1; 1189 1190 if (unlikely(head >= vq->size)) 1191 return -1; 1192 1193 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1194 1195 inflight_info->desc[head].inflight = 0; 1196 1197 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1198 1199 inflight_info->old_free_head = inflight_info->free_head; 1200 inflight_info->old_used_idx = inflight_info->used_idx; 1201 inflight_info->old_used_wrap_counter = inflight_info->used_wrap_counter; 1202 1203 return 0; 1204 } 1205 1206 int 1207 rte_vhost_set_last_inflight_io_split(int vid, uint16_t vring_idx, 1208 uint16_t idx) 1209 { 1210 struct virtio_net *dev; 1211 struct vhost_virtqueue *vq; 1212 1213 dev = get_device(vid); 1214 if (unlikely(!dev)) 1215 return -1; 1216 1217 if (unlikely(!(dev->protocol_features & 1218 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1219 return 0; 1220 1221 if (unlikely(vq_is_packed(dev))) 1222 return -1; 1223 1224 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1225 return -1; 1226 1227 vq = dev->virtqueue[vring_idx]; 1228 if (unlikely(!vq)) 1229 return -1; 1230 1231 if (unlikely(!vq->inflight_split)) 1232 return -1; 1233 1234 if (unlikely(idx >= vq->size)) 1235 return -1; 1236 1237 vq->inflight_split->last_inflight_io = idx; 1238 return 0; 1239 } 1240 1241 int 1242 rte_vhost_set_last_inflight_io_packed(int vid, uint16_t vring_idx, 1243 uint16_t head) 1244 { 1245 struct rte_vhost_inflight_info_packed *inflight_info; 1246 struct virtio_net *dev; 1247 struct vhost_virtqueue *vq; 1248 uint16_t last; 1249 1250 dev = get_device(vid); 1251 if (unlikely(!dev)) 1252 return -1; 1253 1254 if (unlikely(!(dev->protocol_features & 1255 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1256 return 0; 1257 1258 if (unlikely(!vq_is_packed(dev))) 1259 return -1; 1260 1261 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1262 return -1; 1263 1264 vq = dev->virtqueue[vring_idx]; 1265 if (unlikely(!vq)) 1266 return -1; 1267 1268 inflight_info = vq->inflight_packed; 1269 if (unlikely(!inflight_info)) 1270 return -1; 1271 1272 if (unlikely(head >= vq->size)) 1273 return -1; 1274 1275 last = inflight_info->desc[head].last; 1276 if (unlikely(last >= vq->size)) 1277 return -1; 1278 1279 inflight_info->desc[last].next = inflight_info->free_head; 1280 inflight_info->free_head = head; 1281 inflight_info->used_idx += inflight_info->desc[head].num; 1282 if (inflight_info->used_idx >= inflight_info->desc_num) { 1283 inflight_info->used_idx -= inflight_info->desc_num; 1284 inflight_info->used_wrap_counter = 1285 !inflight_info->used_wrap_counter; 1286 } 1287 1288 return 0; 1289 } 1290 1291 int 1292 rte_vhost_vring_call(int vid, uint16_t vring_idx) 1293 { 1294 struct virtio_net *dev; 1295 struct vhost_virtqueue *vq; 1296 1297 dev = get_device(vid); 1298 if (!dev) 1299 return -1; 1300 1301 if (vring_idx >= VHOST_MAX_VRING) 1302 return -1; 1303 1304 vq = dev->virtqueue[vring_idx]; 1305 if (!vq) 1306 return -1; 1307 1308 rte_spinlock_lock(&vq->access_lock); 1309 1310 if (vq_is_packed(dev)) 1311 vhost_vring_call_packed(dev, vq); 1312 else 1313 vhost_vring_call_split(dev, vq); 1314 1315 rte_spinlock_unlock(&vq->access_lock); 1316 1317 return 0; 1318 } 1319 1320 int 1321 rte_vhost_vring_call_nonblock(int vid, uint16_t vring_idx) 1322 { 1323 struct virtio_net *dev; 1324 struct vhost_virtqueue *vq; 1325 1326 dev = get_device(vid); 1327 if (!dev) 1328 return -1; 1329 1330 if (vring_idx >= VHOST_MAX_VRING) 1331 return -1; 1332 1333 vq = dev->virtqueue[vring_idx]; 1334 if (!vq) 1335 return -1; 1336 1337 if (!rte_spinlock_trylock(&vq->access_lock)) 1338 return -EAGAIN; 1339 1340 if (vq_is_packed(dev)) 1341 vhost_vring_call_packed(dev, vq); 1342 else 1343 vhost_vring_call_split(dev, vq); 1344 1345 rte_spinlock_unlock(&vq->access_lock); 1346 1347 return 0; 1348 } 1349 1350 uint16_t 1351 rte_vhost_avail_entries(int vid, uint16_t queue_id) 1352 { 1353 struct virtio_net *dev; 1354 struct vhost_virtqueue *vq; 1355 uint16_t ret = 0; 1356 1357 dev = get_device(vid); 1358 if (!dev) 1359 return 0; 1360 1361 if (queue_id >= VHOST_MAX_VRING) 1362 return 0; 1363 1364 vq = dev->virtqueue[queue_id]; 1365 if (!vq) 1366 return 0; 1367 1368 rte_spinlock_lock(&vq->access_lock); 1369 1370 if (unlikely(!vq->enabled || vq->avail == NULL)) 1371 goto out; 1372 1373 ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; 1374 1375 out: 1376 rte_spinlock_unlock(&vq->access_lock); 1377 return ret; 1378 } 1379 1380 static inline int 1381 vhost_enable_notify_split(struct virtio_net *dev, 1382 struct vhost_virtqueue *vq, int enable) 1383 { 1384 if (vq->used == NULL) 1385 return -1; 1386 1387 if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { 1388 if (enable) 1389 vq->used->flags &= ~VRING_USED_F_NO_NOTIFY; 1390 else 1391 vq->used->flags |= VRING_USED_F_NO_NOTIFY; 1392 } else { 1393 if (enable) 1394 vhost_avail_event(vq) = vq->last_avail_idx; 1395 } 1396 return 0; 1397 } 1398 1399 static inline int 1400 vhost_enable_notify_packed(struct virtio_net *dev, 1401 struct vhost_virtqueue *vq, int enable) 1402 { 1403 uint16_t flags; 1404 1405 if (vq->device_event == NULL) 1406 return -1; 1407 1408 if (!enable) { 1409 vq->device_event->flags = VRING_EVENT_F_DISABLE; 1410 return 0; 1411 } 1412 1413 flags = VRING_EVENT_F_ENABLE; 1414 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { 1415 flags = VRING_EVENT_F_DESC; 1416 vq->device_event->off_wrap = vq->last_avail_idx | 1417 vq->avail_wrap_counter << 15; 1418 } 1419 1420 rte_atomic_thread_fence(__ATOMIC_RELEASE); 1421 1422 vq->device_event->flags = flags; 1423 return 0; 1424 } 1425 1426 int 1427 vhost_enable_guest_notification(struct virtio_net *dev, 1428 struct vhost_virtqueue *vq, int enable) 1429 { 1430 /* 1431 * If the virtqueue is not ready yet, it will be applied 1432 * when it will become ready. 1433 */ 1434 if (!vq->ready) 1435 return 0; 1436 1437 if (vq_is_packed(dev)) 1438 return vhost_enable_notify_packed(dev, vq, enable); 1439 else 1440 return vhost_enable_notify_split(dev, vq, enable); 1441 } 1442 1443 int 1444 rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) 1445 { 1446 struct virtio_net *dev = get_device(vid); 1447 struct vhost_virtqueue *vq; 1448 int ret; 1449 1450 if (!dev) 1451 return -1; 1452 1453 if (queue_id >= VHOST_MAX_VRING) 1454 return -1; 1455 1456 vq = dev->virtqueue[queue_id]; 1457 if (!vq) 1458 return -1; 1459 1460 rte_spinlock_lock(&vq->access_lock); 1461 1462 vq->notif_enable = enable; 1463 ret = vhost_enable_guest_notification(dev, vq, enable); 1464 1465 rte_spinlock_unlock(&vq->access_lock); 1466 1467 return ret; 1468 } 1469 1470 void 1471 rte_vhost_log_write(int vid, uint64_t addr, uint64_t len) 1472 { 1473 struct virtio_net *dev = get_device(vid); 1474 1475 if (dev == NULL) 1476 return; 1477 1478 vhost_log_write(dev, addr, len); 1479 } 1480 1481 void 1482 rte_vhost_log_used_vring(int vid, uint16_t vring_idx, 1483 uint64_t offset, uint64_t len) 1484 { 1485 struct virtio_net *dev; 1486 struct vhost_virtqueue *vq; 1487 1488 dev = get_device(vid); 1489 if (dev == NULL) 1490 return; 1491 1492 if (vring_idx >= VHOST_MAX_VRING) 1493 return; 1494 vq = dev->virtqueue[vring_idx]; 1495 if (!vq) 1496 return; 1497 1498 vhost_log_used_vring(dev, vq, offset, len); 1499 } 1500 1501 uint32_t 1502 rte_vhost_rx_queue_count(int vid, uint16_t qid) 1503 { 1504 struct virtio_net *dev; 1505 struct vhost_virtqueue *vq; 1506 uint32_t ret = 0; 1507 1508 dev = get_device(vid); 1509 if (dev == NULL) 1510 return 0; 1511 1512 if (unlikely(qid >= dev->nr_vring || (qid & 1) == 0)) { 1513 VHOST_LOG_DATA(dev->ifname, ERR, 1514 "%s: invalid virtqueue idx %d.\n", 1515 __func__, qid); 1516 return 0; 1517 } 1518 1519 vq = dev->virtqueue[qid]; 1520 if (vq == NULL) 1521 return 0; 1522 1523 rte_spinlock_lock(&vq->access_lock); 1524 1525 if (unlikely(!vq->enabled || vq->avail == NULL)) 1526 goto out; 1527 1528 ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx; 1529 1530 out: 1531 rte_spinlock_unlock(&vq->access_lock); 1532 return ret; 1533 } 1534 1535 struct rte_vdpa_device * 1536 rte_vhost_get_vdpa_device(int vid) 1537 { 1538 struct virtio_net *dev = get_device(vid); 1539 1540 if (dev == NULL) 1541 return NULL; 1542 1543 return dev->vdpa_dev; 1544 } 1545 1546 int 1547 rte_vhost_get_log_base(int vid, uint64_t *log_base, 1548 uint64_t *log_size) 1549 { 1550 struct virtio_net *dev = get_device(vid); 1551 1552 if (dev == NULL || log_base == NULL || log_size == NULL) 1553 return -1; 1554 1555 *log_base = dev->log_base; 1556 *log_size = dev->log_size; 1557 1558 return 0; 1559 } 1560 1561 int 1562 rte_vhost_get_vring_base(int vid, uint16_t queue_id, 1563 uint16_t *last_avail_idx, uint16_t *last_used_idx) 1564 { 1565 struct vhost_virtqueue *vq; 1566 struct virtio_net *dev = get_device(vid); 1567 1568 if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) 1569 return -1; 1570 1571 if (queue_id >= VHOST_MAX_VRING) 1572 return -1; 1573 1574 vq = dev->virtqueue[queue_id]; 1575 if (!vq) 1576 return -1; 1577 1578 if (vq_is_packed(dev)) { 1579 *last_avail_idx = (vq->avail_wrap_counter << 15) | 1580 vq->last_avail_idx; 1581 *last_used_idx = (vq->used_wrap_counter << 15) | 1582 vq->last_used_idx; 1583 } else { 1584 *last_avail_idx = vq->last_avail_idx; 1585 *last_used_idx = vq->last_used_idx; 1586 } 1587 1588 return 0; 1589 } 1590 1591 int 1592 rte_vhost_set_vring_base(int vid, uint16_t queue_id, 1593 uint16_t last_avail_idx, uint16_t last_used_idx) 1594 { 1595 struct vhost_virtqueue *vq; 1596 struct virtio_net *dev = get_device(vid); 1597 1598 if (!dev) 1599 return -1; 1600 1601 if (queue_id >= VHOST_MAX_VRING) 1602 return -1; 1603 1604 vq = dev->virtqueue[queue_id]; 1605 if (!vq) 1606 return -1; 1607 1608 if (vq_is_packed(dev)) { 1609 vq->last_avail_idx = last_avail_idx & 0x7fff; 1610 vq->avail_wrap_counter = !!(last_avail_idx & (1 << 15)); 1611 vq->last_used_idx = last_used_idx & 0x7fff; 1612 vq->used_wrap_counter = !!(last_used_idx & (1 << 15)); 1613 } else { 1614 vq->last_avail_idx = last_avail_idx; 1615 vq->last_used_idx = last_used_idx; 1616 } 1617 1618 return 0; 1619 } 1620 1621 int 1622 rte_vhost_get_vring_base_from_inflight(int vid, 1623 uint16_t queue_id, 1624 uint16_t *last_avail_idx, 1625 uint16_t *last_used_idx) 1626 { 1627 struct rte_vhost_inflight_info_packed *inflight_info; 1628 struct vhost_virtqueue *vq; 1629 struct virtio_net *dev = get_device(vid); 1630 1631 if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) 1632 return -1; 1633 1634 if (queue_id >= VHOST_MAX_VRING) 1635 return -1; 1636 1637 vq = dev->virtqueue[queue_id]; 1638 if (!vq) 1639 return -1; 1640 1641 if (!vq_is_packed(dev)) 1642 return -1; 1643 1644 inflight_info = vq->inflight_packed; 1645 if (!inflight_info) 1646 return -1; 1647 1648 *last_avail_idx = (inflight_info->old_used_wrap_counter << 15) | 1649 inflight_info->old_used_idx; 1650 *last_used_idx = *last_avail_idx; 1651 1652 return 0; 1653 } 1654 1655 int 1656 rte_vhost_extern_callback_register(int vid, 1657 struct rte_vhost_user_extern_ops const * const ops, void *ctx) 1658 { 1659 struct virtio_net *dev = get_device(vid); 1660 1661 if (dev == NULL || ops == NULL) 1662 return -1; 1663 1664 dev->extern_ops = *ops; 1665 dev->extern_data = ctx; 1666 return 0; 1667 } 1668 1669 static __rte_always_inline int 1670 async_channel_register(struct virtio_net *dev, struct vhost_virtqueue *vq) 1671 __rte_exclusive_locks_required(&vq->access_lock) 1672 { 1673 struct vhost_async *async; 1674 int node = vq->numa_node; 1675 1676 if (unlikely(vq->async)) { 1677 VHOST_LOG_CONFIG(dev->ifname, ERR, 1678 "async register failed: already registered (qid: %d)\n", 1679 vq->index); 1680 return -1; 1681 } 1682 1683 async = rte_zmalloc_socket(NULL, sizeof(struct vhost_async), 0, node); 1684 if (!async) { 1685 VHOST_LOG_CONFIG(dev->ifname, ERR, 1686 "failed to allocate async metadata (qid: %d)\n", 1687 vq->index); 1688 return -1; 1689 } 1690 1691 async->pkts_info = rte_malloc_socket(NULL, vq->size * sizeof(struct async_inflight_info), 1692 RTE_CACHE_LINE_SIZE, node); 1693 if (!async->pkts_info) { 1694 VHOST_LOG_CONFIG(dev->ifname, ERR, 1695 "failed to allocate async_pkts_info (qid: %d)\n", 1696 vq->index); 1697 goto out_free_async; 1698 } 1699 1700 async->pkts_cmpl_flag = rte_zmalloc_socket(NULL, vq->size * sizeof(bool), 1701 RTE_CACHE_LINE_SIZE, node); 1702 if (!async->pkts_cmpl_flag) { 1703 VHOST_LOG_CONFIG(dev->ifname, ERR, 1704 "failed to allocate async pkts_cmpl_flag (qid: %d)\n", 1705 vq->index); 1706 goto out_free_async; 1707 } 1708 1709 if (vq_is_packed(dev)) { 1710 async->buffers_packed = rte_malloc_socket(NULL, 1711 vq->size * sizeof(struct vring_used_elem_packed), 1712 RTE_CACHE_LINE_SIZE, node); 1713 if (!async->buffers_packed) { 1714 VHOST_LOG_CONFIG(dev->ifname, ERR, 1715 "failed to allocate async buffers (qid: %d)\n", 1716 vq->index); 1717 goto out_free_inflight; 1718 } 1719 } else { 1720 async->descs_split = rte_malloc_socket(NULL, 1721 vq->size * sizeof(struct vring_used_elem), 1722 RTE_CACHE_LINE_SIZE, node); 1723 if (!async->descs_split) { 1724 VHOST_LOG_CONFIG(dev->ifname, ERR, 1725 "failed to allocate async descs (qid: %d)\n", 1726 vq->index); 1727 goto out_free_inflight; 1728 } 1729 } 1730 1731 vq->async = async; 1732 1733 return 0; 1734 out_free_inflight: 1735 rte_free(async->pkts_info); 1736 out_free_async: 1737 rte_free(async); 1738 1739 return -1; 1740 } 1741 1742 int 1743 rte_vhost_async_channel_register(int vid, uint16_t queue_id) 1744 { 1745 struct vhost_virtqueue *vq; 1746 struct virtio_net *dev = get_device(vid); 1747 int ret; 1748 1749 if (dev == NULL) 1750 return -1; 1751 1752 if (queue_id >= VHOST_MAX_VRING) 1753 return -1; 1754 1755 vq = dev->virtqueue[queue_id]; 1756 1757 if (unlikely(vq == NULL || !dev->async_copy)) 1758 return -1; 1759 1760 rte_spinlock_lock(&vq->access_lock); 1761 ret = async_channel_register(dev, vq); 1762 rte_spinlock_unlock(&vq->access_lock); 1763 1764 return ret; 1765 } 1766 1767 int 1768 rte_vhost_async_channel_register_thread_unsafe(int vid, uint16_t queue_id) 1769 { 1770 struct vhost_virtqueue *vq; 1771 struct virtio_net *dev = get_device(vid); 1772 1773 if (dev == NULL) 1774 return -1; 1775 1776 if (queue_id >= VHOST_MAX_VRING) 1777 return -1; 1778 1779 vq = dev->virtqueue[queue_id]; 1780 1781 if (unlikely(vq == NULL || !dev->async_copy)) 1782 return -1; 1783 1784 vq_assert_lock(dev, vq); 1785 1786 return async_channel_register(dev, vq); 1787 } 1788 1789 int 1790 rte_vhost_async_channel_unregister(int vid, uint16_t queue_id) 1791 { 1792 struct vhost_virtqueue *vq; 1793 struct virtio_net *dev = get_device(vid); 1794 int ret = -1; 1795 1796 if (dev == NULL) 1797 return ret; 1798 1799 if (queue_id >= VHOST_MAX_VRING) 1800 return ret; 1801 1802 vq = dev->virtqueue[queue_id]; 1803 1804 if (vq == NULL) 1805 return ret; 1806 1807 if (!rte_spinlock_trylock(&vq->access_lock)) { 1808 VHOST_LOG_CONFIG(dev->ifname, ERR, 1809 "failed to unregister async channel, virtqueue busy.\n"); 1810 return ret; 1811 } 1812 1813 if (!vq->async) { 1814 ret = 0; 1815 } else if (vq->async->pkts_inflight_n) { 1816 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to unregister async channel.\n"); 1817 VHOST_LOG_CONFIG(dev->ifname, ERR, 1818 "inflight packets must be completed before unregistration.\n"); 1819 } else { 1820 vhost_free_async_mem(vq); 1821 ret = 0; 1822 } 1823 1824 rte_spinlock_unlock(&vq->access_lock); 1825 1826 return ret; 1827 } 1828 1829 int 1830 rte_vhost_async_channel_unregister_thread_unsafe(int vid, uint16_t queue_id) 1831 { 1832 struct vhost_virtqueue *vq; 1833 struct virtio_net *dev = get_device(vid); 1834 1835 if (dev == NULL) 1836 return -1; 1837 1838 if (queue_id >= VHOST_MAX_VRING) 1839 return -1; 1840 1841 vq = dev->virtqueue[queue_id]; 1842 1843 if (vq == NULL) 1844 return -1; 1845 1846 vq_assert_lock(dev, vq); 1847 1848 if (!vq->async) 1849 return 0; 1850 1851 if (vq->async->pkts_inflight_n) { 1852 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to unregister async channel.\n"); 1853 VHOST_LOG_CONFIG(dev->ifname, ERR, 1854 "inflight packets must be completed before unregistration.\n"); 1855 return -1; 1856 } 1857 1858 vhost_free_async_mem(vq); 1859 1860 return 0; 1861 } 1862 1863 int 1864 rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id) 1865 { 1866 struct rte_dma_info info; 1867 void *pkts_cmpl_flag_addr; 1868 uint16_t max_desc; 1869 1870 pthread_mutex_lock(&vhost_dma_lock); 1871 1872 if (!rte_dma_is_valid(dma_id)) { 1873 VHOST_LOG_CONFIG("dma", ERR, "DMA %d is not found.\n", dma_id); 1874 goto error; 1875 } 1876 1877 if (rte_dma_info_get(dma_id, &info) != 0) { 1878 VHOST_LOG_CONFIG("dma", ERR, "Fail to get DMA %d information.\n", dma_id); 1879 goto error; 1880 } 1881 1882 if (vchan_id >= info.max_vchans) { 1883 VHOST_LOG_CONFIG("dma", ERR, "Invalid DMA %d vChannel %u.\n", dma_id, vchan_id); 1884 goto error; 1885 } 1886 1887 if (!dma_copy_track[dma_id].vchans) { 1888 struct async_dma_vchan_info *vchans; 1889 1890 vchans = rte_zmalloc(NULL, sizeof(struct async_dma_vchan_info) * info.max_vchans, 1891 RTE_CACHE_LINE_SIZE); 1892 if (vchans == NULL) { 1893 VHOST_LOG_CONFIG("dma", ERR, 1894 "Failed to allocate vchans for DMA %d vChannel %u.\n", 1895 dma_id, vchan_id); 1896 goto error; 1897 } 1898 1899 dma_copy_track[dma_id].vchans = vchans; 1900 } 1901 1902 if (dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr) { 1903 VHOST_LOG_CONFIG("dma", INFO, "DMA %d vChannel %u already registered.\n", 1904 dma_id, vchan_id); 1905 pthread_mutex_unlock(&vhost_dma_lock); 1906 return 0; 1907 } 1908 1909 max_desc = info.max_desc; 1910 if (!rte_is_power_of_2(max_desc)) 1911 max_desc = rte_align32pow2(max_desc); 1912 1913 pkts_cmpl_flag_addr = rte_zmalloc(NULL, sizeof(bool *) * max_desc, RTE_CACHE_LINE_SIZE); 1914 if (!pkts_cmpl_flag_addr) { 1915 VHOST_LOG_CONFIG("dma", ERR, 1916 "Failed to allocate pkts_cmpl_flag_addr for DMA %d vChannel %u.\n", 1917 dma_id, vchan_id); 1918 1919 if (dma_copy_track[dma_id].nr_vchans == 0) { 1920 rte_free(dma_copy_track[dma_id].vchans); 1921 dma_copy_track[dma_id].vchans = NULL; 1922 } 1923 goto error; 1924 } 1925 1926 dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr = pkts_cmpl_flag_addr; 1927 dma_copy_track[dma_id].vchans[vchan_id].ring_size = max_desc; 1928 dma_copy_track[dma_id].vchans[vchan_id].ring_mask = max_desc - 1; 1929 dma_copy_track[dma_id].nr_vchans++; 1930 1931 pthread_mutex_unlock(&vhost_dma_lock); 1932 return 0; 1933 1934 error: 1935 pthread_mutex_unlock(&vhost_dma_lock); 1936 return -1; 1937 } 1938 1939 int 1940 rte_vhost_async_get_inflight(int vid, uint16_t queue_id) 1941 { 1942 struct vhost_virtqueue *vq; 1943 struct virtio_net *dev = get_device(vid); 1944 int ret = -1; 1945 1946 if (dev == NULL) 1947 return ret; 1948 1949 if (queue_id >= VHOST_MAX_VRING) 1950 return ret; 1951 1952 vq = dev->virtqueue[queue_id]; 1953 1954 if (vq == NULL) 1955 return ret; 1956 1957 if (!rte_spinlock_trylock(&vq->access_lock)) { 1958 VHOST_LOG_CONFIG(dev->ifname, DEBUG, 1959 "failed to check in-flight packets. virtqueue busy.\n"); 1960 return ret; 1961 } 1962 1963 if (vq->async) 1964 ret = vq->async->pkts_inflight_n; 1965 1966 rte_spinlock_unlock(&vq->access_lock); 1967 1968 return ret; 1969 } 1970 1971 int 1972 rte_vhost_async_get_inflight_thread_unsafe(int vid, uint16_t queue_id) 1973 { 1974 struct vhost_virtqueue *vq; 1975 struct virtio_net *dev = get_device(vid); 1976 int ret = -1; 1977 1978 if (dev == NULL) 1979 return ret; 1980 1981 if (queue_id >= VHOST_MAX_VRING) 1982 return ret; 1983 1984 vq = dev->virtqueue[queue_id]; 1985 1986 if (vq == NULL) 1987 return ret; 1988 1989 vq_assert_lock(dev, vq); 1990 1991 if (!vq->async) 1992 return ret; 1993 1994 ret = vq->async->pkts_inflight_n; 1995 1996 return ret; 1997 } 1998 1999 int 2000 rte_vhost_get_monitor_addr(int vid, uint16_t queue_id, 2001 struct rte_vhost_power_monitor_cond *pmc) 2002 { 2003 struct virtio_net *dev = get_device(vid); 2004 struct vhost_virtqueue *vq; 2005 2006 if (dev == NULL) 2007 return -1; 2008 if (queue_id >= VHOST_MAX_VRING) 2009 return -1; 2010 2011 vq = dev->virtqueue[queue_id]; 2012 if (vq == NULL) 2013 return -1; 2014 2015 if (vq_is_packed(dev)) { 2016 struct vring_packed_desc *desc; 2017 desc = vq->desc_packed; 2018 pmc->addr = &desc[vq->last_avail_idx].flags; 2019 if (vq->avail_wrap_counter) 2020 pmc->val = VRING_DESC_F_AVAIL; 2021 else 2022 pmc->val = VRING_DESC_F_USED; 2023 pmc->mask = VRING_DESC_F_AVAIL | VRING_DESC_F_USED; 2024 pmc->size = sizeof(desc[vq->last_avail_idx].flags); 2025 pmc->match = 1; 2026 } else { 2027 pmc->addr = &vq->avail->idx; 2028 pmc->val = vq->last_avail_idx & (vq->size - 1); 2029 pmc->mask = vq->size - 1; 2030 pmc->size = sizeof(vq->avail->idx); 2031 pmc->match = 0; 2032 } 2033 2034 return 0; 2035 } 2036 2037 2038 int 2039 rte_vhost_vring_stats_get_names(int vid, uint16_t queue_id, 2040 struct rte_vhost_stat_name *name, unsigned int size) 2041 { 2042 struct virtio_net *dev = get_device(vid); 2043 unsigned int i; 2044 2045 if (dev == NULL) 2046 return -1; 2047 2048 if (queue_id >= dev->nr_vring) 2049 return -1; 2050 2051 if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED)) 2052 return -1; 2053 2054 if (name == NULL || size < VHOST_NB_VQ_STATS) 2055 return VHOST_NB_VQ_STATS; 2056 2057 for (i = 0; i < VHOST_NB_VQ_STATS; i++) 2058 snprintf(name[i].name, sizeof(name[i].name), "%s_q%u_%s", 2059 (queue_id & 1) ? "rx" : "tx", 2060 queue_id / 2, vhost_vq_stat_strings[i].name); 2061 2062 return VHOST_NB_VQ_STATS; 2063 } 2064 2065 int 2066 rte_vhost_vring_stats_get(int vid, uint16_t queue_id, 2067 struct rte_vhost_stat *stats, unsigned int n) 2068 { 2069 struct virtio_net *dev = get_device(vid); 2070 struct vhost_virtqueue *vq; 2071 unsigned int i; 2072 2073 if (dev == NULL) 2074 return -1; 2075 2076 if (queue_id >= dev->nr_vring) 2077 return -1; 2078 2079 if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED)) 2080 return -1; 2081 2082 if (stats == NULL || n < VHOST_NB_VQ_STATS) 2083 return VHOST_NB_VQ_STATS; 2084 2085 vq = dev->virtqueue[queue_id]; 2086 2087 rte_spinlock_lock(&vq->access_lock); 2088 for (i = 0; i < VHOST_NB_VQ_STATS; i++) { 2089 stats[i].value = 2090 *(uint64_t *)(((char *)vq) + vhost_vq_stat_strings[i].offset); 2091 stats[i].id = i; 2092 } 2093 rte_spinlock_unlock(&vq->access_lock); 2094 2095 return VHOST_NB_VQ_STATS; 2096 } 2097 2098 int rte_vhost_vring_stats_reset(int vid, uint16_t queue_id) 2099 { 2100 struct virtio_net *dev = get_device(vid); 2101 struct vhost_virtqueue *vq; 2102 2103 if (dev == NULL) 2104 return -1; 2105 2106 if (queue_id >= dev->nr_vring) 2107 return -1; 2108 2109 if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED)) 2110 return -1; 2111 2112 vq = dev->virtqueue[queue_id]; 2113 2114 rte_spinlock_lock(&vq->access_lock); 2115 memset(&vq->stats, 0, sizeof(vq->stats)); 2116 rte_spinlock_unlock(&vq->access_lock); 2117 2118 return 0; 2119 } 2120 2121 int 2122 rte_vhost_async_dma_unconfigure(int16_t dma_id, uint16_t vchan_id) 2123 { 2124 struct rte_dma_info info; 2125 struct rte_dma_stats stats = { 0 }; 2126 2127 pthread_mutex_lock(&vhost_dma_lock); 2128 2129 if (!rte_dma_is_valid(dma_id)) { 2130 VHOST_LOG_CONFIG("dma", ERR, "DMA %d is not found.\n", dma_id); 2131 goto error; 2132 } 2133 2134 if (rte_dma_info_get(dma_id, &info) != 0) { 2135 VHOST_LOG_CONFIG("dma", ERR, "Fail to get DMA %d information.\n", dma_id); 2136 goto error; 2137 } 2138 2139 if (vchan_id >= info.max_vchans || !dma_copy_track[dma_id].vchans || 2140 !dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr) { 2141 VHOST_LOG_CONFIG("dma", ERR, "Invalid channel %d:%u.\n", dma_id, vchan_id); 2142 goto error; 2143 } 2144 2145 if (rte_dma_stats_get(dma_id, vchan_id, &stats) != 0) { 2146 VHOST_LOG_CONFIG("dma", ERR, 2147 "Failed to get stats for DMA %d vChannel %u.\n", dma_id, vchan_id); 2148 goto error; 2149 } 2150 2151 if (stats.submitted - stats.completed != 0) { 2152 VHOST_LOG_CONFIG("dma", ERR, 2153 "Do not unconfigure when there are inflight packets.\n"); 2154 goto error; 2155 } 2156 2157 rte_free(dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr); 2158 dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr = NULL; 2159 dma_copy_track[dma_id].nr_vchans--; 2160 2161 if (dma_copy_track[dma_id].nr_vchans == 0) { 2162 rte_free(dma_copy_track[dma_id].vchans); 2163 dma_copy_track[dma_id].vchans = NULL; 2164 } 2165 2166 pthread_mutex_unlock(&vhost_dma_lock); 2167 return 0; 2168 2169 error: 2170 pthread_mutex_unlock(&vhost_dma_lock); 2171 return -1; 2172 } 2173 2174 RTE_LOG_REGISTER_SUFFIX(vhost_config_log_level, config, INFO); 2175 RTE_LOG_REGISTER_SUFFIX(vhost_data_log_level, data, WARNING); 2176