1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <linux/vhost.h> 6 #include <linux/virtio_net.h> 7 #include <stdint.h> 8 #include <stdlib.h> 9 #ifdef RTE_LIBRTE_VHOST_NUMA 10 #include <numa.h> 11 #include <numaif.h> 12 #endif 13 14 #include <rte_errno.h> 15 #include <rte_log.h> 16 #include <rte_memory.h> 17 #include <rte_malloc.h> 18 #include <rte_vhost.h> 19 20 #include "iotlb.h" 21 #include "vhost.h" 22 #include "vhost_user.h" 23 24 struct virtio_net *vhost_devices[RTE_MAX_VHOST_DEVICE]; 25 pthread_mutex_t vhost_dev_lock = PTHREAD_MUTEX_INITIALIZER; 26 27 struct vhost_vq_stats_name_off { 28 char name[RTE_VHOST_STATS_NAME_SIZE]; 29 unsigned int offset; 30 }; 31 32 static const struct vhost_vq_stats_name_off vhost_vq_stat_strings[] = { 33 {"good_packets", offsetof(struct vhost_virtqueue, stats.packets)}, 34 {"good_bytes", offsetof(struct vhost_virtqueue, stats.bytes)}, 35 {"multicast_packets", offsetof(struct vhost_virtqueue, stats.multicast)}, 36 {"broadcast_packets", offsetof(struct vhost_virtqueue, stats.broadcast)}, 37 {"undersize_packets", offsetof(struct vhost_virtqueue, stats.size_bins[0])}, 38 {"size_64_packets", offsetof(struct vhost_virtqueue, stats.size_bins[1])}, 39 {"size_65_127_packets", offsetof(struct vhost_virtqueue, stats.size_bins[2])}, 40 {"size_128_255_packets", offsetof(struct vhost_virtqueue, stats.size_bins[3])}, 41 {"size_256_511_packets", offsetof(struct vhost_virtqueue, stats.size_bins[4])}, 42 {"size_512_1023_packets", offsetof(struct vhost_virtqueue, stats.size_bins[5])}, 43 {"size_1024_1518_packets", offsetof(struct vhost_virtqueue, stats.size_bins[6])}, 44 {"size_1519_max_packets", offsetof(struct vhost_virtqueue, stats.size_bins[7])}, 45 {"guest_notifications", offsetof(struct vhost_virtqueue, stats.guest_notifications)}, 46 {"iotlb_hits", offsetof(struct vhost_virtqueue, stats.iotlb_hits)}, 47 {"iotlb_misses", offsetof(struct vhost_virtqueue, stats.iotlb_misses)}, 48 {"inflight_submitted", offsetof(struct vhost_virtqueue, stats.inflight_submitted)}, 49 {"inflight_completed", offsetof(struct vhost_virtqueue, stats.inflight_completed)}, 50 }; 51 52 #define VHOST_NB_VQ_STATS RTE_DIM(vhost_vq_stat_strings) 53 54 /* Called with iotlb_lock read-locked */ 55 uint64_t 56 __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, 57 uint64_t iova, uint64_t *size, uint8_t perm) 58 { 59 uint64_t vva, tmp_size; 60 61 if (unlikely(!*size)) 62 return 0; 63 64 tmp_size = *size; 65 66 vva = vhost_user_iotlb_cache_find(vq, iova, &tmp_size, perm); 67 if (tmp_size == *size) { 68 if (dev->flags & VIRTIO_DEV_STATS_ENABLED) 69 vq->stats.iotlb_hits++; 70 return vva; 71 } 72 73 if (dev->flags & VIRTIO_DEV_STATS_ENABLED) 74 vq->stats.iotlb_misses++; 75 76 iova += tmp_size; 77 78 if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) { 79 /* 80 * iotlb_lock is read-locked for a full burst, 81 * but it only protects the iotlb cache. 82 * In case of IOTLB miss, we might block on the socket, 83 * which could cause a deadlock with QEMU if an IOTLB update 84 * is being handled. We can safely unlock here to avoid it. 85 */ 86 vhost_user_iotlb_rd_unlock(vq); 87 88 vhost_user_iotlb_pending_insert(dev, vq, iova, perm); 89 if (vhost_user_iotlb_miss(dev, iova, perm)) { 90 VHOST_LOG_DATA(dev->ifname, ERR, 91 "IOTLB miss req failed for IOVA 0x%" PRIx64 "\n", 92 iova); 93 vhost_user_iotlb_pending_remove(vq, iova, 1, perm); 94 } 95 96 vhost_user_iotlb_rd_lock(vq); 97 } 98 99 return 0; 100 } 101 102 #define VHOST_LOG_PAGE 4096 103 104 /* 105 * Atomically set a bit in memory. 106 */ 107 static __rte_always_inline void 108 vhost_set_bit(unsigned int nr, volatile uint8_t *addr) 109 { 110 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) 111 /* 112 * __sync_ built-ins are deprecated, but __atomic_ ones 113 * are sub-optimized in older GCC versions. 114 */ 115 __sync_fetch_and_or_1(addr, (1U << nr)); 116 #else 117 __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED); 118 #endif 119 } 120 121 static __rte_always_inline void 122 vhost_log_page(uint8_t *log_base, uint64_t page) 123 { 124 vhost_set_bit(page % 8, &log_base[page / 8]); 125 } 126 127 void 128 __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) 129 { 130 uint64_t page; 131 132 if (unlikely(!dev->log_base || !len)) 133 return; 134 135 if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) 136 return; 137 138 /* To make sure guest memory updates are committed before logging */ 139 rte_atomic_thread_fence(__ATOMIC_RELEASE); 140 141 page = addr / VHOST_LOG_PAGE; 142 while (page * VHOST_LOG_PAGE < addr + len) { 143 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 144 page += 1; 145 } 146 } 147 148 void 149 __vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, 150 uint64_t iova, uint64_t len) 151 { 152 uint64_t hva, gpa, map_len; 153 map_len = len; 154 155 hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); 156 if (map_len != len) { 157 VHOST_LOG_DATA(dev->ifname, ERR, 158 "failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n", 159 iova); 160 return; 161 } 162 163 gpa = hva_to_gpa(dev, hva, len); 164 if (gpa) 165 __vhost_log_write(dev, gpa, len); 166 } 167 168 void 169 __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) 170 { 171 unsigned long *log_base; 172 int i; 173 174 if (unlikely(!dev->log_base)) 175 return; 176 177 /* No cache, nothing to sync */ 178 if (unlikely(!vq->log_cache)) 179 return; 180 181 rte_atomic_thread_fence(__ATOMIC_RELEASE); 182 183 log_base = (unsigned long *)(uintptr_t)dev->log_base; 184 185 for (i = 0; i < vq->log_cache_nb_elem; i++) { 186 struct log_cache_entry *elem = vq->log_cache + i; 187 188 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) 189 /* 190 * '__sync' builtins are deprecated, but '__atomic' ones 191 * are sub-optimized in older GCC versions. 192 */ 193 __sync_fetch_and_or(log_base + elem->offset, elem->val); 194 #else 195 __atomic_fetch_or(log_base + elem->offset, elem->val, 196 __ATOMIC_RELAXED); 197 #endif 198 } 199 200 rte_atomic_thread_fence(__ATOMIC_RELEASE); 201 202 vq->log_cache_nb_elem = 0; 203 } 204 205 static __rte_always_inline void 206 vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, 207 uint64_t page) 208 { 209 uint32_t bit_nr = page % (sizeof(unsigned long) << 3); 210 uint32_t offset = page / (sizeof(unsigned long) << 3); 211 int i; 212 213 if (unlikely(!vq->log_cache)) { 214 /* No logging cache allocated, write dirty log map directly */ 215 rte_atomic_thread_fence(__ATOMIC_RELEASE); 216 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 217 218 return; 219 } 220 221 for (i = 0; i < vq->log_cache_nb_elem; i++) { 222 struct log_cache_entry *elem = vq->log_cache + i; 223 224 if (elem->offset == offset) { 225 elem->val |= (1UL << bit_nr); 226 return; 227 } 228 } 229 230 if (unlikely(i >= VHOST_LOG_CACHE_NR)) { 231 /* 232 * No more room for a new log cache entry, 233 * so write the dirty log map directly. 234 */ 235 rte_atomic_thread_fence(__ATOMIC_RELEASE); 236 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 237 238 return; 239 } 240 241 vq->log_cache[i].offset = offset; 242 vq->log_cache[i].val = (1UL << bit_nr); 243 vq->log_cache_nb_elem++; 244 } 245 246 void 247 __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, 248 uint64_t addr, uint64_t len) 249 { 250 uint64_t page; 251 252 if (unlikely(!dev->log_base || !len)) 253 return; 254 255 if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) 256 return; 257 258 page = addr / VHOST_LOG_PAGE; 259 while (page * VHOST_LOG_PAGE < addr + len) { 260 vhost_log_cache_page(dev, vq, page); 261 page += 1; 262 } 263 } 264 265 void 266 __vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, 267 uint64_t iova, uint64_t len) 268 { 269 uint64_t hva, gpa, map_len; 270 map_len = len; 271 272 hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); 273 if (map_len != len) { 274 VHOST_LOG_DATA(dev->ifname, ERR, 275 "failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n", 276 iova); 277 return; 278 } 279 280 gpa = hva_to_gpa(dev, hva, len); 281 if (gpa) 282 __vhost_log_cache_write(dev, vq, gpa, len); 283 } 284 285 void * 286 vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, 287 uint64_t desc_addr, uint64_t desc_len) 288 { 289 void *idesc; 290 uint64_t src, dst; 291 uint64_t len, remain = desc_len; 292 293 idesc = rte_malloc_socket(__func__, desc_len, 0, vq->numa_node); 294 if (unlikely(!idesc)) 295 return NULL; 296 297 dst = (uint64_t)(uintptr_t)idesc; 298 299 while (remain) { 300 len = remain; 301 src = vhost_iova_to_vva(dev, vq, desc_addr, &len, 302 VHOST_ACCESS_RO); 303 if (unlikely(!src || !len)) { 304 rte_free(idesc); 305 return NULL; 306 } 307 308 rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); 309 310 remain -= len; 311 dst += len; 312 desc_addr += len; 313 } 314 315 return idesc; 316 } 317 318 void 319 cleanup_vq(struct vhost_virtqueue *vq, int destroy) 320 { 321 if ((vq->callfd >= 0) && (destroy != 0)) 322 close(vq->callfd); 323 if (vq->kickfd >= 0) 324 close(vq->kickfd); 325 } 326 327 void 328 cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq) 329 { 330 if (!(dev->protocol_features & 331 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))) 332 return; 333 334 if (vq_is_packed(dev)) { 335 if (vq->inflight_packed) 336 vq->inflight_packed = NULL; 337 } else { 338 if (vq->inflight_split) 339 vq->inflight_split = NULL; 340 } 341 342 if (vq->resubmit_inflight) { 343 if (vq->resubmit_inflight->resubmit_list) { 344 rte_free(vq->resubmit_inflight->resubmit_list); 345 vq->resubmit_inflight->resubmit_list = NULL; 346 } 347 rte_free(vq->resubmit_inflight); 348 vq->resubmit_inflight = NULL; 349 } 350 } 351 352 /* 353 * Unmap any memory, close any file descriptors and 354 * free any memory owned by a device. 355 */ 356 void 357 cleanup_device(struct virtio_net *dev, int destroy) 358 { 359 uint32_t i; 360 361 vhost_backend_cleanup(dev); 362 363 for (i = 0; i < dev->nr_vring; i++) { 364 cleanup_vq(dev->virtqueue[i], destroy); 365 cleanup_vq_inflight(dev, dev->virtqueue[i]); 366 } 367 } 368 369 static void 370 vhost_free_async_mem(struct vhost_virtqueue *vq) 371 { 372 if (!vq->async) 373 return; 374 375 rte_free(vq->async->pkts_info); 376 rte_free(vq->async->pkts_cmpl_flag); 377 378 rte_free(vq->async->buffers_packed); 379 vq->async->buffers_packed = NULL; 380 rte_free(vq->async->descs_split); 381 vq->async->descs_split = NULL; 382 383 rte_free(vq->async); 384 vq->async = NULL; 385 } 386 387 void 388 free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq) 389 { 390 if (vq_is_packed(dev)) 391 rte_free(vq->shadow_used_packed); 392 else 393 rte_free(vq->shadow_used_split); 394 395 vhost_free_async_mem(vq); 396 rte_free(vq->batch_copy_elems); 397 vhost_user_iotlb_destroy(vq); 398 rte_free(vq->log_cache); 399 rte_free(vq); 400 } 401 402 /* 403 * Release virtqueues and device memory. 404 */ 405 static void 406 free_device(struct virtio_net *dev) 407 { 408 uint32_t i; 409 410 for (i = 0; i < dev->nr_vring; i++) 411 free_vq(dev, dev->virtqueue[i]); 412 413 rte_free(dev); 414 } 415 416 static __rte_always_inline int 417 log_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) 418 { 419 if (likely(!(vq->ring_addrs.flags & (1 << VHOST_VRING_F_LOG)))) 420 return 0; 421 422 vq->log_guest_addr = translate_log_addr(dev, vq, 423 vq->ring_addrs.log_guest_addr); 424 if (vq->log_guest_addr == 0) 425 return -1; 426 427 return 0; 428 } 429 430 /* 431 * Converts vring log address to GPA 432 * If IOMMU is enabled, the log address is IOVA 433 * If IOMMU not enabled, the log address is already GPA 434 * 435 * Caller should have iotlb_lock read-locked 436 */ 437 uint64_t 438 translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq, 439 uint64_t log_addr) 440 { 441 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) { 442 const uint64_t exp_size = sizeof(uint64_t); 443 uint64_t hva, gpa; 444 uint64_t size = exp_size; 445 446 hva = vhost_iova_to_vva(dev, vq, log_addr, 447 &size, VHOST_ACCESS_RW); 448 449 if (size != exp_size) 450 return 0; 451 452 gpa = hva_to_gpa(dev, hva, exp_size); 453 if (!gpa) { 454 VHOST_LOG_DATA(dev->ifname, ERR, 455 "failed to find GPA for log_addr: 0x%" 456 PRIx64 " hva: 0x%" PRIx64 "\n", 457 log_addr, hva); 458 return 0; 459 } 460 return gpa; 461 462 } else 463 return log_addr; 464 } 465 466 /* Caller should have iotlb_lock read-locked */ 467 static int 468 vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq) 469 { 470 uint64_t req_size, size; 471 472 req_size = sizeof(struct vring_desc) * vq->size; 473 size = req_size; 474 vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq, 475 vq->ring_addrs.desc_user_addr, 476 &size, VHOST_ACCESS_RW); 477 if (!vq->desc || size != req_size) 478 return -1; 479 480 req_size = sizeof(struct vring_avail); 481 req_size += sizeof(uint16_t) * vq->size; 482 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 483 req_size += sizeof(uint16_t); 484 size = req_size; 485 vq->avail = (struct vring_avail *)(uintptr_t)vhost_iova_to_vva(dev, vq, 486 vq->ring_addrs.avail_user_addr, 487 &size, VHOST_ACCESS_RW); 488 if (!vq->avail || size != req_size) 489 return -1; 490 491 req_size = sizeof(struct vring_used); 492 req_size += sizeof(struct vring_used_elem) * vq->size; 493 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 494 req_size += sizeof(uint16_t); 495 size = req_size; 496 vq->used = (struct vring_used *)(uintptr_t)vhost_iova_to_vva(dev, vq, 497 vq->ring_addrs.used_user_addr, 498 &size, VHOST_ACCESS_RW); 499 if (!vq->used || size != req_size) 500 return -1; 501 502 return 0; 503 } 504 505 /* Caller should have iotlb_lock read-locked */ 506 static int 507 vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) 508 { 509 uint64_t req_size, size; 510 511 req_size = sizeof(struct vring_packed_desc) * vq->size; 512 size = req_size; 513 vq->desc_packed = (struct vring_packed_desc *)(uintptr_t) 514 vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr, 515 &size, VHOST_ACCESS_RW); 516 if (!vq->desc_packed || size != req_size) 517 return -1; 518 519 req_size = sizeof(struct vring_packed_desc_event); 520 size = req_size; 521 vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t) 522 vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr, 523 &size, VHOST_ACCESS_RW); 524 if (!vq->driver_event || size != req_size) 525 return -1; 526 527 req_size = sizeof(struct vring_packed_desc_event); 528 size = req_size; 529 vq->device_event = (struct vring_packed_desc_event *)(uintptr_t) 530 vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr, 531 &size, VHOST_ACCESS_RW); 532 if (!vq->device_event || size != req_size) 533 return -1; 534 535 return 0; 536 } 537 538 int 539 vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) 540 { 541 542 if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) 543 return -1; 544 545 if (vq_is_packed(dev)) { 546 if (vring_translate_packed(dev, vq) < 0) 547 return -1; 548 } else { 549 if (vring_translate_split(dev, vq) < 0) 550 return -1; 551 } 552 553 if (log_translate(dev, vq) < 0) 554 return -1; 555 556 vq->access_ok = true; 557 558 return 0; 559 } 560 561 void 562 vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq) 563 { 564 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) 565 vhost_user_iotlb_wr_lock(vq); 566 567 vq->access_ok = false; 568 vq->desc = NULL; 569 vq->avail = NULL; 570 vq->used = NULL; 571 vq->log_guest_addr = 0; 572 573 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) 574 vhost_user_iotlb_wr_unlock(vq); 575 } 576 577 static void 578 init_vring_queue(struct virtio_net *dev, struct vhost_virtqueue *vq, 579 uint32_t vring_idx) 580 { 581 int numa_node = SOCKET_ID_ANY; 582 583 memset(vq, 0, sizeof(struct vhost_virtqueue)); 584 585 vq->index = vring_idx; 586 vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; 587 vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; 588 vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF; 589 590 #ifdef RTE_LIBRTE_VHOST_NUMA 591 if (get_mempolicy(&numa_node, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR)) { 592 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to query numa node: %s\n", 593 rte_strerror(errno)); 594 numa_node = SOCKET_ID_ANY; 595 } 596 #endif 597 vq->numa_node = numa_node; 598 599 vhost_user_iotlb_init(dev, vq); 600 } 601 602 static void 603 reset_vring_queue(struct virtio_net *dev, struct vhost_virtqueue *vq) 604 { 605 int callfd; 606 607 callfd = vq->callfd; 608 init_vring_queue(dev, vq, vq->index); 609 vq->callfd = callfd; 610 } 611 612 int 613 alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx) 614 { 615 struct vhost_virtqueue *vq; 616 uint32_t i; 617 618 /* Also allocate holes, if any, up to requested vring index. */ 619 for (i = 0; i <= vring_idx; i++) { 620 if (dev->virtqueue[i]) 621 continue; 622 623 vq = rte_zmalloc(NULL, sizeof(struct vhost_virtqueue), 0); 624 if (vq == NULL) { 625 VHOST_LOG_CONFIG(dev->ifname, ERR, 626 "failed to allocate memory for vring %u.\n", 627 i); 628 return -1; 629 } 630 631 dev->virtqueue[i] = vq; 632 init_vring_queue(dev, vq, i); 633 rte_spinlock_init(&vq->access_lock); 634 vq->avail_wrap_counter = 1; 635 vq->used_wrap_counter = 1; 636 vq->signalled_used_valid = false; 637 } 638 639 dev->nr_vring = RTE_MAX(dev->nr_vring, vring_idx + 1); 640 641 return 0; 642 } 643 644 /* 645 * Reset some variables in device structure, while keeping few 646 * others untouched, such as vid, ifname, nr_vring: they 647 * should be same unless the device is removed. 648 */ 649 void 650 reset_device(struct virtio_net *dev) 651 { 652 uint32_t i; 653 654 dev->features = 0; 655 dev->protocol_features = 0; 656 dev->flags &= VIRTIO_DEV_BUILTIN_VIRTIO_NET; 657 658 for (i = 0; i < dev->nr_vring; i++) { 659 struct vhost_virtqueue *vq = dev->virtqueue[i]; 660 661 if (!vq) { 662 VHOST_LOG_CONFIG(dev->ifname, ERR, 663 "failed to reset vring, virtqueue not allocated (%d)\n", i); 664 continue; 665 } 666 reset_vring_queue(dev, vq); 667 } 668 } 669 670 /* 671 * Invoked when there is a new vhost-user connection established (when 672 * there is a new virtio device being attached). 673 */ 674 int 675 vhost_new_device(void) 676 { 677 struct virtio_net *dev; 678 int i; 679 680 pthread_mutex_lock(&vhost_dev_lock); 681 for (i = 0; i < RTE_MAX_VHOST_DEVICE; i++) { 682 if (vhost_devices[i] == NULL) 683 break; 684 } 685 686 if (i == RTE_MAX_VHOST_DEVICE) { 687 VHOST_LOG_CONFIG("device", ERR, "failed to find a free slot for new device.\n"); 688 pthread_mutex_unlock(&vhost_dev_lock); 689 return -1; 690 } 691 692 dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0); 693 if (dev == NULL) { 694 VHOST_LOG_CONFIG("device", ERR, "failed to allocate memory for new device.\n"); 695 pthread_mutex_unlock(&vhost_dev_lock); 696 return -1; 697 } 698 699 vhost_devices[i] = dev; 700 pthread_mutex_unlock(&vhost_dev_lock); 701 702 dev->vid = i; 703 dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET; 704 dev->slave_req_fd = -1; 705 dev->postcopy_ufd = -1; 706 rte_spinlock_init(&dev->slave_req_lock); 707 708 return i; 709 } 710 711 void 712 vhost_destroy_device_notify(struct virtio_net *dev) 713 { 714 struct rte_vdpa_device *vdpa_dev; 715 716 if (dev->flags & VIRTIO_DEV_RUNNING) { 717 vdpa_dev = dev->vdpa_dev; 718 if (vdpa_dev) 719 vdpa_dev->ops->dev_close(dev->vid); 720 dev->flags &= ~VIRTIO_DEV_RUNNING; 721 dev->notify_ops->destroy_device(dev->vid); 722 } 723 } 724 725 /* 726 * Invoked when there is the vhost-user connection is broken (when 727 * the virtio device is being detached). 728 */ 729 void 730 vhost_destroy_device(int vid) 731 { 732 struct virtio_net *dev = get_device(vid); 733 734 if (dev == NULL) 735 return; 736 737 vhost_destroy_device_notify(dev); 738 739 cleanup_device(dev, 1); 740 free_device(dev); 741 742 vhost_devices[vid] = NULL; 743 } 744 745 void 746 vhost_attach_vdpa_device(int vid, struct rte_vdpa_device *vdpa_dev) 747 { 748 struct virtio_net *dev = get_device(vid); 749 750 if (dev == NULL) 751 return; 752 753 dev->vdpa_dev = vdpa_dev; 754 } 755 756 void 757 vhost_set_ifname(int vid, const char *if_name, unsigned int if_len) 758 { 759 struct virtio_net *dev; 760 unsigned int len; 761 762 dev = get_device(vid); 763 if (dev == NULL) 764 return; 765 766 len = if_len > sizeof(dev->ifname) ? 767 sizeof(dev->ifname) : if_len; 768 769 strncpy(dev->ifname, if_name, len); 770 dev->ifname[sizeof(dev->ifname) - 1] = '\0'; 771 } 772 773 void 774 vhost_setup_virtio_net(int vid, bool enable, bool compliant_ol_flags, bool stats_enabled) 775 { 776 struct virtio_net *dev = get_device(vid); 777 778 if (dev == NULL) 779 return; 780 781 if (enable) 782 dev->flags |= VIRTIO_DEV_BUILTIN_VIRTIO_NET; 783 else 784 dev->flags &= ~VIRTIO_DEV_BUILTIN_VIRTIO_NET; 785 if (!compliant_ol_flags) 786 dev->flags |= VIRTIO_DEV_LEGACY_OL_FLAGS; 787 else 788 dev->flags &= ~VIRTIO_DEV_LEGACY_OL_FLAGS; 789 if (stats_enabled) 790 dev->flags |= VIRTIO_DEV_STATS_ENABLED; 791 else 792 dev->flags &= ~VIRTIO_DEV_STATS_ENABLED; 793 } 794 795 void 796 vhost_enable_extbuf(int vid) 797 { 798 struct virtio_net *dev = get_device(vid); 799 800 if (dev == NULL) 801 return; 802 803 dev->extbuf = 1; 804 } 805 806 void 807 vhost_enable_linearbuf(int vid) 808 { 809 struct virtio_net *dev = get_device(vid); 810 811 if (dev == NULL) 812 return; 813 814 dev->linearbuf = 1; 815 } 816 817 int 818 rte_vhost_get_mtu(int vid, uint16_t *mtu) 819 { 820 struct virtio_net *dev = get_device(vid); 821 822 if (dev == NULL || mtu == NULL) 823 return -ENODEV; 824 825 if (!(dev->flags & VIRTIO_DEV_READY)) 826 return -EAGAIN; 827 828 if (!(dev->features & (1ULL << VIRTIO_NET_F_MTU))) 829 return -ENOTSUP; 830 831 *mtu = dev->mtu; 832 833 return 0; 834 } 835 836 int 837 rte_vhost_get_numa_node(int vid) 838 { 839 #ifdef RTE_LIBRTE_VHOST_NUMA 840 struct virtio_net *dev = get_device(vid); 841 int numa_node; 842 int ret; 843 844 if (dev == NULL || numa_available() != 0) 845 return -1; 846 847 ret = get_mempolicy(&numa_node, NULL, 0, dev, 848 MPOL_F_NODE | MPOL_F_ADDR); 849 if (ret < 0) { 850 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to query numa node: %s\n", 851 rte_strerror(errno)); 852 return -1; 853 } 854 855 return numa_node; 856 #else 857 RTE_SET_USED(vid); 858 return -1; 859 #endif 860 } 861 862 uint32_t 863 rte_vhost_get_queue_num(int vid) 864 { 865 struct virtio_net *dev = get_device(vid); 866 867 if (dev == NULL) 868 return 0; 869 870 return dev->nr_vring / 2; 871 } 872 873 uint16_t 874 rte_vhost_get_vring_num(int vid) 875 { 876 struct virtio_net *dev = get_device(vid); 877 878 if (dev == NULL) 879 return 0; 880 881 return dev->nr_vring; 882 } 883 884 int 885 rte_vhost_get_ifname(int vid, char *buf, size_t len) 886 { 887 struct virtio_net *dev = get_device(vid); 888 889 if (dev == NULL || buf == NULL) 890 return -1; 891 892 len = RTE_MIN(len, sizeof(dev->ifname)); 893 894 strncpy(buf, dev->ifname, len); 895 buf[len - 1] = '\0'; 896 897 return 0; 898 } 899 900 int 901 rte_vhost_get_negotiated_features(int vid, uint64_t *features) 902 { 903 struct virtio_net *dev; 904 905 dev = get_device(vid); 906 if (dev == NULL || features == NULL) 907 return -1; 908 909 *features = dev->features; 910 return 0; 911 } 912 913 int 914 rte_vhost_get_negotiated_protocol_features(int vid, 915 uint64_t *protocol_features) 916 { 917 struct virtio_net *dev; 918 919 dev = get_device(vid); 920 if (dev == NULL || protocol_features == NULL) 921 return -1; 922 923 *protocol_features = dev->protocol_features; 924 return 0; 925 } 926 927 int 928 rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem) 929 { 930 struct virtio_net *dev; 931 struct rte_vhost_memory *m; 932 size_t size; 933 934 dev = get_device(vid); 935 if (dev == NULL || mem == NULL) 936 return -1; 937 938 size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region); 939 m = malloc(sizeof(struct rte_vhost_memory) + size); 940 if (!m) 941 return -1; 942 943 m->nregions = dev->mem->nregions; 944 memcpy(m->regions, dev->mem->regions, size); 945 *mem = m; 946 947 return 0; 948 } 949 950 int 951 rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, 952 struct rte_vhost_vring *vring) 953 { 954 struct virtio_net *dev; 955 struct vhost_virtqueue *vq; 956 957 dev = get_device(vid); 958 if (dev == NULL || vring == NULL) 959 return -1; 960 961 if (vring_idx >= VHOST_MAX_VRING) 962 return -1; 963 964 vq = dev->virtqueue[vring_idx]; 965 if (!vq) 966 return -1; 967 968 if (vq_is_packed(dev)) { 969 vring->desc_packed = vq->desc_packed; 970 vring->driver_event = vq->driver_event; 971 vring->device_event = vq->device_event; 972 } else { 973 vring->desc = vq->desc; 974 vring->avail = vq->avail; 975 vring->used = vq->used; 976 } 977 vring->log_guest_addr = vq->log_guest_addr; 978 979 vring->callfd = vq->callfd; 980 vring->kickfd = vq->kickfd; 981 vring->size = vq->size; 982 983 return 0; 984 } 985 986 int 987 rte_vhost_get_vhost_ring_inflight(int vid, uint16_t vring_idx, 988 struct rte_vhost_ring_inflight *vring) 989 { 990 struct virtio_net *dev; 991 struct vhost_virtqueue *vq; 992 993 dev = get_device(vid); 994 if (unlikely(!dev)) 995 return -1; 996 997 if (vring_idx >= VHOST_MAX_VRING) 998 return -1; 999 1000 vq = dev->virtqueue[vring_idx]; 1001 if (unlikely(!vq)) 1002 return -1; 1003 1004 if (vq_is_packed(dev)) { 1005 if (unlikely(!vq->inflight_packed)) 1006 return -1; 1007 1008 vring->inflight_packed = vq->inflight_packed; 1009 } else { 1010 if (unlikely(!vq->inflight_split)) 1011 return -1; 1012 1013 vring->inflight_split = vq->inflight_split; 1014 } 1015 1016 vring->resubmit_inflight = vq->resubmit_inflight; 1017 1018 return 0; 1019 } 1020 1021 int 1022 rte_vhost_set_inflight_desc_split(int vid, uint16_t vring_idx, 1023 uint16_t idx) 1024 { 1025 struct vhost_virtqueue *vq; 1026 struct virtio_net *dev; 1027 1028 dev = get_device(vid); 1029 if (unlikely(!dev)) 1030 return -1; 1031 1032 if (unlikely(!(dev->protocol_features & 1033 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1034 return 0; 1035 1036 if (unlikely(vq_is_packed(dev))) 1037 return -1; 1038 1039 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1040 return -1; 1041 1042 vq = dev->virtqueue[vring_idx]; 1043 if (unlikely(!vq)) 1044 return -1; 1045 1046 if (unlikely(!vq->inflight_split)) 1047 return -1; 1048 1049 if (unlikely(idx >= vq->size)) 1050 return -1; 1051 1052 vq->inflight_split->desc[idx].counter = vq->global_counter++; 1053 vq->inflight_split->desc[idx].inflight = 1; 1054 return 0; 1055 } 1056 1057 int 1058 rte_vhost_set_inflight_desc_packed(int vid, uint16_t vring_idx, 1059 uint16_t head, uint16_t last, 1060 uint16_t *inflight_entry) 1061 { 1062 struct rte_vhost_inflight_info_packed *inflight_info; 1063 struct virtio_net *dev; 1064 struct vhost_virtqueue *vq; 1065 struct vring_packed_desc *desc; 1066 uint16_t old_free_head, free_head; 1067 1068 dev = get_device(vid); 1069 if (unlikely(!dev)) 1070 return -1; 1071 1072 if (unlikely(!(dev->protocol_features & 1073 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1074 return 0; 1075 1076 if (unlikely(!vq_is_packed(dev))) 1077 return -1; 1078 1079 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1080 return -1; 1081 1082 vq = dev->virtqueue[vring_idx]; 1083 if (unlikely(!vq)) 1084 return -1; 1085 1086 inflight_info = vq->inflight_packed; 1087 if (unlikely(!inflight_info)) 1088 return -1; 1089 1090 if (unlikely(head >= vq->size)) 1091 return -1; 1092 1093 desc = vq->desc_packed; 1094 old_free_head = inflight_info->old_free_head; 1095 if (unlikely(old_free_head >= vq->size)) 1096 return -1; 1097 1098 free_head = old_free_head; 1099 1100 /* init header descriptor */ 1101 inflight_info->desc[old_free_head].num = 0; 1102 inflight_info->desc[old_free_head].counter = vq->global_counter++; 1103 inflight_info->desc[old_free_head].inflight = 1; 1104 1105 /* save desc entry in flight entry */ 1106 while (head != ((last + 1) % vq->size)) { 1107 inflight_info->desc[old_free_head].num++; 1108 inflight_info->desc[free_head].addr = desc[head].addr; 1109 inflight_info->desc[free_head].len = desc[head].len; 1110 inflight_info->desc[free_head].flags = desc[head].flags; 1111 inflight_info->desc[free_head].id = desc[head].id; 1112 1113 inflight_info->desc[old_free_head].last = free_head; 1114 free_head = inflight_info->desc[free_head].next; 1115 inflight_info->free_head = free_head; 1116 head = (head + 1) % vq->size; 1117 } 1118 1119 inflight_info->old_free_head = free_head; 1120 *inflight_entry = old_free_head; 1121 1122 return 0; 1123 } 1124 1125 int 1126 rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx, 1127 uint16_t last_used_idx, uint16_t idx) 1128 { 1129 struct virtio_net *dev; 1130 struct vhost_virtqueue *vq; 1131 1132 dev = get_device(vid); 1133 if (unlikely(!dev)) 1134 return -1; 1135 1136 if (unlikely(!(dev->protocol_features & 1137 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1138 return 0; 1139 1140 if (unlikely(vq_is_packed(dev))) 1141 return -1; 1142 1143 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1144 return -1; 1145 1146 vq = dev->virtqueue[vring_idx]; 1147 if (unlikely(!vq)) 1148 return -1; 1149 1150 if (unlikely(!vq->inflight_split)) 1151 return -1; 1152 1153 if (unlikely(idx >= vq->size)) 1154 return -1; 1155 1156 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1157 1158 vq->inflight_split->desc[idx].inflight = 0; 1159 1160 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1161 1162 vq->inflight_split->used_idx = last_used_idx; 1163 return 0; 1164 } 1165 1166 int 1167 rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx, 1168 uint16_t head) 1169 { 1170 struct rte_vhost_inflight_info_packed *inflight_info; 1171 struct virtio_net *dev; 1172 struct vhost_virtqueue *vq; 1173 1174 dev = get_device(vid); 1175 if (unlikely(!dev)) 1176 return -1; 1177 1178 if (unlikely(!(dev->protocol_features & 1179 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1180 return 0; 1181 1182 if (unlikely(!vq_is_packed(dev))) 1183 return -1; 1184 1185 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1186 return -1; 1187 1188 vq = dev->virtqueue[vring_idx]; 1189 if (unlikely(!vq)) 1190 return -1; 1191 1192 inflight_info = vq->inflight_packed; 1193 if (unlikely(!inflight_info)) 1194 return -1; 1195 1196 if (unlikely(head >= vq->size)) 1197 return -1; 1198 1199 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1200 1201 inflight_info->desc[head].inflight = 0; 1202 1203 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1204 1205 inflight_info->old_free_head = inflight_info->free_head; 1206 inflight_info->old_used_idx = inflight_info->used_idx; 1207 inflight_info->old_used_wrap_counter = inflight_info->used_wrap_counter; 1208 1209 return 0; 1210 } 1211 1212 int 1213 rte_vhost_set_last_inflight_io_split(int vid, uint16_t vring_idx, 1214 uint16_t idx) 1215 { 1216 struct virtio_net *dev; 1217 struct vhost_virtqueue *vq; 1218 1219 dev = get_device(vid); 1220 if (unlikely(!dev)) 1221 return -1; 1222 1223 if (unlikely(!(dev->protocol_features & 1224 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1225 return 0; 1226 1227 if (unlikely(vq_is_packed(dev))) 1228 return -1; 1229 1230 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1231 return -1; 1232 1233 vq = dev->virtqueue[vring_idx]; 1234 if (unlikely(!vq)) 1235 return -1; 1236 1237 if (unlikely(!vq->inflight_split)) 1238 return -1; 1239 1240 if (unlikely(idx >= vq->size)) 1241 return -1; 1242 1243 vq->inflight_split->last_inflight_io = idx; 1244 return 0; 1245 } 1246 1247 int 1248 rte_vhost_set_last_inflight_io_packed(int vid, uint16_t vring_idx, 1249 uint16_t head) 1250 { 1251 struct rte_vhost_inflight_info_packed *inflight_info; 1252 struct virtio_net *dev; 1253 struct vhost_virtqueue *vq; 1254 uint16_t last; 1255 1256 dev = get_device(vid); 1257 if (unlikely(!dev)) 1258 return -1; 1259 1260 if (unlikely(!(dev->protocol_features & 1261 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1262 return 0; 1263 1264 if (unlikely(!vq_is_packed(dev))) 1265 return -1; 1266 1267 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1268 return -1; 1269 1270 vq = dev->virtqueue[vring_idx]; 1271 if (unlikely(!vq)) 1272 return -1; 1273 1274 inflight_info = vq->inflight_packed; 1275 if (unlikely(!inflight_info)) 1276 return -1; 1277 1278 if (unlikely(head >= vq->size)) 1279 return -1; 1280 1281 last = inflight_info->desc[head].last; 1282 if (unlikely(last >= vq->size)) 1283 return -1; 1284 1285 inflight_info->desc[last].next = inflight_info->free_head; 1286 inflight_info->free_head = head; 1287 inflight_info->used_idx += inflight_info->desc[head].num; 1288 if (inflight_info->used_idx >= inflight_info->desc_num) { 1289 inflight_info->used_idx -= inflight_info->desc_num; 1290 inflight_info->used_wrap_counter = 1291 !inflight_info->used_wrap_counter; 1292 } 1293 1294 return 0; 1295 } 1296 1297 int 1298 rte_vhost_vring_call(int vid, uint16_t vring_idx) 1299 { 1300 struct virtio_net *dev; 1301 struct vhost_virtqueue *vq; 1302 1303 dev = get_device(vid); 1304 if (!dev) 1305 return -1; 1306 1307 if (vring_idx >= VHOST_MAX_VRING) 1308 return -1; 1309 1310 vq = dev->virtqueue[vring_idx]; 1311 if (!vq) 1312 return -1; 1313 1314 rte_spinlock_lock(&vq->access_lock); 1315 1316 if (vq_is_packed(dev)) 1317 vhost_vring_call_packed(dev, vq); 1318 else 1319 vhost_vring_call_split(dev, vq); 1320 1321 rte_spinlock_unlock(&vq->access_lock); 1322 1323 return 0; 1324 } 1325 1326 uint16_t 1327 rte_vhost_avail_entries(int vid, uint16_t queue_id) 1328 { 1329 struct virtio_net *dev; 1330 struct vhost_virtqueue *vq; 1331 uint16_t ret = 0; 1332 1333 dev = get_device(vid); 1334 if (!dev) 1335 return 0; 1336 1337 if (queue_id >= VHOST_MAX_VRING) 1338 return 0; 1339 1340 vq = dev->virtqueue[queue_id]; 1341 if (!vq) 1342 return 0; 1343 1344 rte_spinlock_lock(&vq->access_lock); 1345 1346 if (unlikely(!vq->enabled || vq->avail == NULL)) 1347 goto out; 1348 1349 ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; 1350 1351 out: 1352 rte_spinlock_unlock(&vq->access_lock); 1353 return ret; 1354 } 1355 1356 static inline int 1357 vhost_enable_notify_split(struct virtio_net *dev, 1358 struct vhost_virtqueue *vq, int enable) 1359 { 1360 if (vq->used == NULL) 1361 return -1; 1362 1363 if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { 1364 if (enable) 1365 vq->used->flags &= ~VRING_USED_F_NO_NOTIFY; 1366 else 1367 vq->used->flags |= VRING_USED_F_NO_NOTIFY; 1368 } else { 1369 if (enable) 1370 vhost_avail_event(vq) = vq->last_avail_idx; 1371 } 1372 return 0; 1373 } 1374 1375 static inline int 1376 vhost_enable_notify_packed(struct virtio_net *dev, 1377 struct vhost_virtqueue *vq, int enable) 1378 { 1379 uint16_t flags; 1380 1381 if (vq->device_event == NULL) 1382 return -1; 1383 1384 if (!enable) { 1385 vq->device_event->flags = VRING_EVENT_F_DISABLE; 1386 return 0; 1387 } 1388 1389 flags = VRING_EVENT_F_ENABLE; 1390 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { 1391 flags = VRING_EVENT_F_DESC; 1392 vq->device_event->off_wrap = vq->last_avail_idx | 1393 vq->avail_wrap_counter << 15; 1394 } 1395 1396 rte_atomic_thread_fence(__ATOMIC_RELEASE); 1397 1398 vq->device_event->flags = flags; 1399 return 0; 1400 } 1401 1402 int 1403 vhost_enable_guest_notification(struct virtio_net *dev, 1404 struct vhost_virtqueue *vq, int enable) 1405 { 1406 /* 1407 * If the virtqueue is not ready yet, it will be applied 1408 * when it will become ready. 1409 */ 1410 if (!vq->ready) 1411 return 0; 1412 1413 if (vq_is_packed(dev)) 1414 return vhost_enable_notify_packed(dev, vq, enable); 1415 else 1416 return vhost_enable_notify_split(dev, vq, enable); 1417 } 1418 1419 int 1420 rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) 1421 { 1422 struct virtio_net *dev = get_device(vid); 1423 struct vhost_virtqueue *vq; 1424 int ret; 1425 1426 if (!dev) 1427 return -1; 1428 1429 if (queue_id >= VHOST_MAX_VRING) 1430 return -1; 1431 1432 vq = dev->virtqueue[queue_id]; 1433 if (!vq) 1434 return -1; 1435 1436 rte_spinlock_lock(&vq->access_lock); 1437 1438 vq->notif_enable = enable; 1439 ret = vhost_enable_guest_notification(dev, vq, enable); 1440 1441 rte_spinlock_unlock(&vq->access_lock); 1442 1443 return ret; 1444 } 1445 1446 void 1447 rte_vhost_log_write(int vid, uint64_t addr, uint64_t len) 1448 { 1449 struct virtio_net *dev = get_device(vid); 1450 1451 if (dev == NULL) 1452 return; 1453 1454 vhost_log_write(dev, addr, len); 1455 } 1456 1457 void 1458 rte_vhost_log_used_vring(int vid, uint16_t vring_idx, 1459 uint64_t offset, uint64_t len) 1460 { 1461 struct virtio_net *dev; 1462 struct vhost_virtqueue *vq; 1463 1464 dev = get_device(vid); 1465 if (dev == NULL) 1466 return; 1467 1468 if (vring_idx >= VHOST_MAX_VRING) 1469 return; 1470 vq = dev->virtqueue[vring_idx]; 1471 if (!vq) 1472 return; 1473 1474 vhost_log_used_vring(dev, vq, offset, len); 1475 } 1476 1477 uint32_t 1478 rte_vhost_rx_queue_count(int vid, uint16_t qid) 1479 { 1480 struct virtio_net *dev; 1481 struct vhost_virtqueue *vq; 1482 uint32_t ret = 0; 1483 1484 dev = get_device(vid); 1485 if (dev == NULL) 1486 return 0; 1487 1488 if (unlikely(qid >= dev->nr_vring || (qid & 1) == 0)) { 1489 VHOST_LOG_DATA(dev->ifname, ERR, 1490 "%s: invalid virtqueue idx %d.\n", 1491 __func__, qid); 1492 return 0; 1493 } 1494 1495 vq = dev->virtqueue[qid]; 1496 if (vq == NULL) 1497 return 0; 1498 1499 rte_spinlock_lock(&vq->access_lock); 1500 1501 if (unlikely(!vq->enabled || vq->avail == NULL)) 1502 goto out; 1503 1504 ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx; 1505 1506 out: 1507 rte_spinlock_unlock(&vq->access_lock); 1508 return ret; 1509 } 1510 1511 struct rte_vdpa_device * 1512 rte_vhost_get_vdpa_device(int vid) 1513 { 1514 struct virtio_net *dev = get_device(vid); 1515 1516 if (dev == NULL) 1517 return NULL; 1518 1519 return dev->vdpa_dev; 1520 } 1521 1522 int 1523 rte_vhost_get_log_base(int vid, uint64_t *log_base, 1524 uint64_t *log_size) 1525 { 1526 struct virtio_net *dev = get_device(vid); 1527 1528 if (dev == NULL || log_base == NULL || log_size == NULL) 1529 return -1; 1530 1531 *log_base = dev->log_base; 1532 *log_size = dev->log_size; 1533 1534 return 0; 1535 } 1536 1537 int 1538 rte_vhost_get_vring_base(int vid, uint16_t queue_id, 1539 uint16_t *last_avail_idx, uint16_t *last_used_idx) 1540 { 1541 struct vhost_virtqueue *vq; 1542 struct virtio_net *dev = get_device(vid); 1543 1544 if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) 1545 return -1; 1546 1547 if (queue_id >= VHOST_MAX_VRING) 1548 return -1; 1549 1550 vq = dev->virtqueue[queue_id]; 1551 if (!vq) 1552 return -1; 1553 1554 if (vq_is_packed(dev)) { 1555 *last_avail_idx = (vq->avail_wrap_counter << 15) | 1556 vq->last_avail_idx; 1557 *last_used_idx = (vq->used_wrap_counter << 15) | 1558 vq->last_used_idx; 1559 } else { 1560 *last_avail_idx = vq->last_avail_idx; 1561 *last_used_idx = vq->last_used_idx; 1562 } 1563 1564 return 0; 1565 } 1566 1567 int 1568 rte_vhost_set_vring_base(int vid, uint16_t queue_id, 1569 uint16_t last_avail_idx, uint16_t last_used_idx) 1570 { 1571 struct vhost_virtqueue *vq; 1572 struct virtio_net *dev = get_device(vid); 1573 1574 if (!dev) 1575 return -1; 1576 1577 if (queue_id >= VHOST_MAX_VRING) 1578 return -1; 1579 1580 vq = dev->virtqueue[queue_id]; 1581 if (!vq) 1582 return -1; 1583 1584 if (vq_is_packed(dev)) { 1585 vq->last_avail_idx = last_avail_idx & 0x7fff; 1586 vq->avail_wrap_counter = !!(last_avail_idx & (1 << 15)); 1587 vq->last_used_idx = last_used_idx & 0x7fff; 1588 vq->used_wrap_counter = !!(last_used_idx & (1 << 15)); 1589 } else { 1590 vq->last_avail_idx = last_avail_idx; 1591 vq->last_used_idx = last_used_idx; 1592 } 1593 1594 return 0; 1595 } 1596 1597 int 1598 rte_vhost_get_vring_base_from_inflight(int vid, 1599 uint16_t queue_id, 1600 uint16_t *last_avail_idx, 1601 uint16_t *last_used_idx) 1602 { 1603 struct rte_vhost_inflight_info_packed *inflight_info; 1604 struct vhost_virtqueue *vq; 1605 struct virtio_net *dev = get_device(vid); 1606 1607 if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) 1608 return -1; 1609 1610 if (queue_id >= VHOST_MAX_VRING) 1611 return -1; 1612 1613 vq = dev->virtqueue[queue_id]; 1614 if (!vq) 1615 return -1; 1616 1617 if (!vq_is_packed(dev)) 1618 return -1; 1619 1620 inflight_info = vq->inflight_packed; 1621 if (!inflight_info) 1622 return -1; 1623 1624 *last_avail_idx = (inflight_info->old_used_wrap_counter << 15) | 1625 inflight_info->old_used_idx; 1626 *last_used_idx = *last_avail_idx; 1627 1628 return 0; 1629 } 1630 1631 int 1632 rte_vhost_extern_callback_register(int vid, 1633 struct rte_vhost_user_extern_ops const * const ops, void *ctx) 1634 { 1635 struct virtio_net *dev = get_device(vid); 1636 1637 if (dev == NULL || ops == NULL) 1638 return -1; 1639 1640 dev->extern_ops = *ops; 1641 dev->extern_data = ctx; 1642 return 0; 1643 } 1644 1645 static __rte_always_inline int 1646 async_channel_register(struct virtio_net *dev, struct vhost_virtqueue *vq) 1647 { 1648 struct vhost_async *async; 1649 int node = vq->numa_node; 1650 1651 if (unlikely(vq->async)) { 1652 VHOST_LOG_CONFIG(dev->ifname, ERR, 1653 "async register failed: already registered (qid: %d)\n", 1654 vq->index); 1655 return -1; 1656 } 1657 1658 async = rte_zmalloc_socket(NULL, sizeof(struct vhost_async), 0, node); 1659 if (!async) { 1660 VHOST_LOG_CONFIG(dev->ifname, ERR, 1661 "failed to allocate async metadata (qid: %d)\n", 1662 vq->index); 1663 return -1; 1664 } 1665 1666 async->pkts_info = rte_malloc_socket(NULL, vq->size * sizeof(struct async_inflight_info), 1667 RTE_CACHE_LINE_SIZE, node); 1668 if (!async->pkts_info) { 1669 VHOST_LOG_CONFIG(dev->ifname, ERR, 1670 "failed to allocate async_pkts_info (qid: %d)\n", 1671 vq->index); 1672 goto out_free_async; 1673 } 1674 1675 async->pkts_cmpl_flag = rte_zmalloc_socket(NULL, vq->size * sizeof(bool), 1676 RTE_CACHE_LINE_SIZE, node); 1677 if (!async->pkts_cmpl_flag) { 1678 VHOST_LOG_CONFIG(dev->ifname, ERR, 1679 "failed to allocate async pkts_cmpl_flag (qid: %d)\n", 1680 vq->index); 1681 goto out_free_async; 1682 } 1683 1684 if (vq_is_packed(dev)) { 1685 async->buffers_packed = rte_malloc_socket(NULL, 1686 vq->size * sizeof(struct vring_used_elem_packed), 1687 RTE_CACHE_LINE_SIZE, node); 1688 if (!async->buffers_packed) { 1689 VHOST_LOG_CONFIG(dev->ifname, ERR, 1690 "failed to allocate async buffers (qid: %d)\n", 1691 vq->index); 1692 goto out_free_inflight; 1693 } 1694 } else { 1695 async->descs_split = rte_malloc_socket(NULL, 1696 vq->size * sizeof(struct vring_used_elem), 1697 RTE_CACHE_LINE_SIZE, node); 1698 if (!async->descs_split) { 1699 VHOST_LOG_CONFIG(dev->ifname, ERR, 1700 "failed to allocate async descs (qid: %d)\n", 1701 vq->index); 1702 goto out_free_inflight; 1703 } 1704 } 1705 1706 vq->async = async; 1707 1708 return 0; 1709 out_free_inflight: 1710 rte_free(async->pkts_info); 1711 out_free_async: 1712 rte_free(async); 1713 1714 return -1; 1715 } 1716 1717 int 1718 rte_vhost_async_channel_register(int vid, uint16_t queue_id) 1719 { 1720 struct vhost_virtqueue *vq; 1721 struct virtio_net *dev = get_device(vid); 1722 int ret; 1723 1724 if (dev == NULL) 1725 return -1; 1726 1727 if (queue_id >= VHOST_MAX_VRING) 1728 return -1; 1729 1730 vq = dev->virtqueue[queue_id]; 1731 1732 if (unlikely(vq == NULL || !dev->async_copy)) 1733 return -1; 1734 1735 rte_spinlock_lock(&vq->access_lock); 1736 ret = async_channel_register(dev, vq); 1737 rte_spinlock_unlock(&vq->access_lock); 1738 1739 return ret; 1740 } 1741 1742 int 1743 rte_vhost_async_channel_register_thread_unsafe(int vid, uint16_t queue_id) 1744 { 1745 struct vhost_virtqueue *vq; 1746 struct virtio_net *dev = get_device(vid); 1747 1748 if (dev == NULL) 1749 return -1; 1750 1751 if (queue_id >= VHOST_MAX_VRING) 1752 return -1; 1753 1754 vq = dev->virtqueue[queue_id]; 1755 1756 if (unlikely(vq == NULL || !dev->async_copy)) 1757 return -1; 1758 1759 if (unlikely(!rte_spinlock_is_locked(&vq->access_lock))) { 1760 VHOST_LOG_CONFIG(dev->ifname, ERR, "%s() called without access lock taken.\n", 1761 __func__); 1762 return -1; 1763 } 1764 1765 return async_channel_register(dev, vq); 1766 } 1767 1768 int 1769 rte_vhost_async_channel_unregister(int vid, uint16_t queue_id) 1770 { 1771 struct vhost_virtqueue *vq; 1772 struct virtio_net *dev = get_device(vid); 1773 int ret = -1; 1774 1775 if (dev == NULL) 1776 return ret; 1777 1778 if (queue_id >= VHOST_MAX_VRING) 1779 return ret; 1780 1781 vq = dev->virtqueue[queue_id]; 1782 1783 if (vq == NULL) 1784 return ret; 1785 1786 if (!rte_spinlock_trylock(&vq->access_lock)) { 1787 VHOST_LOG_CONFIG(dev->ifname, ERR, 1788 "failed to unregister async channel, virtqueue busy.\n"); 1789 return ret; 1790 } 1791 1792 if (!vq->async) { 1793 ret = 0; 1794 } else if (vq->async->pkts_inflight_n) { 1795 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to unregister async channel.\n"); 1796 VHOST_LOG_CONFIG(dev->ifname, ERR, 1797 "inflight packets must be completed before unregistration.\n"); 1798 } else { 1799 vhost_free_async_mem(vq); 1800 ret = 0; 1801 } 1802 1803 rte_spinlock_unlock(&vq->access_lock); 1804 1805 return ret; 1806 } 1807 1808 int 1809 rte_vhost_async_channel_unregister_thread_unsafe(int vid, uint16_t queue_id) 1810 { 1811 struct vhost_virtqueue *vq; 1812 struct virtio_net *dev = get_device(vid); 1813 1814 if (dev == NULL) 1815 return -1; 1816 1817 if (queue_id >= VHOST_MAX_VRING) 1818 return -1; 1819 1820 vq = dev->virtqueue[queue_id]; 1821 1822 if (vq == NULL) 1823 return -1; 1824 1825 if (unlikely(!rte_spinlock_is_locked(&vq->access_lock))) { 1826 VHOST_LOG_CONFIG(dev->ifname, ERR, "%s() called without access lock taken.\n", 1827 __func__); 1828 return -1; 1829 } 1830 1831 if (!vq->async) 1832 return 0; 1833 1834 if (vq->async->pkts_inflight_n) { 1835 VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to unregister async channel.\n"); 1836 VHOST_LOG_CONFIG(dev->ifname, ERR, 1837 "inflight packets must be completed before unregistration.\n"); 1838 return -1; 1839 } 1840 1841 vhost_free_async_mem(vq); 1842 1843 return 0; 1844 } 1845 1846 int 1847 rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id) 1848 { 1849 struct rte_dma_info info; 1850 void *pkts_cmpl_flag_addr; 1851 uint16_t max_desc; 1852 1853 if (!rte_dma_is_valid(dma_id)) { 1854 VHOST_LOG_CONFIG("dma", ERR, "DMA %d is not found.\n", dma_id); 1855 return -1; 1856 } 1857 1858 if (rte_dma_info_get(dma_id, &info) != 0) { 1859 VHOST_LOG_CONFIG("dma", ERR, "Fail to get DMA %d information.\n", dma_id); 1860 return -1; 1861 } 1862 1863 if (vchan_id >= info.max_vchans) { 1864 VHOST_LOG_CONFIG("dma", ERR, "Invalid DMA %d vChannel %u.\n", dma_id, vchan_id); 1865 return -1; 1866 } 1867 1868 if (!dma_copy_track[dma_id].vchans) { 1869 struct async_dma_vchan_info *vchans; 1870 1871 vchans = rte_zmalloc(NULL, sizeof(struct async_dma_vchan_info) * info.max_vchans, 1872 RTE_CACHE_LINE_SIZE); 1873 if (vchans == NULL) { 1874 VHOST_LOG_CONFIG("dma", ERR, 1875 "Failed to allocate vchans for DMA %d vChannel %u.\n", 1876 dma_id, vchan_id); 1877 return -1; 1878 } 1879 1880 dma_copy_track[dma_id].vchans = vchans; 1881 } 1882 1883 if (dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr) { 1884 VHOST_LOG_CONFIG("dma", INFO, "DMA %d vChannel %u already registered.\n", 1885 dma_id, vchan_id); 1886 return 0; 1887 } 1888 1889 max_desc = info.max_desc; 1890 if (!rte_is_power_of_2(max_desc)) 1891 max_desc = rte_align32pow2(max_desc); 1892 1893 pkts_cmpl_flag_addr = rte_zmalloc(NULL, sizeof(bool *) * max_desc, RTE_CACHE_LINE_SIZE); 1894 if (!pkts_cmpl_flag_addr) { 1895 VHOST_LOG_CONFIG("dma", ERR, 1896 "Failed to allocate pkts_cmpl_flag_addr for DMA %d vChannel %u.\n", 1897 dma_id, vchan_id); 1898 1899 if (dma_copy_track[dma_id].nr_vchans == 0) { 1900 rte_free(dma_copy_track[dma_id].vchans); 1901 dma_copy_track[dma_id].vchans = NULL; 1902 } 1903 return -1; 1904 } 1905 1906 dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr = pkts_cmpl_flag_addr; 1907 dma_copy_track[dma_id].vchans[vchan_id].ring_size = max_desc; 1908 dma_copy_track[dma_id].vchans[vchan_id].ring_mask = max_desc - 1; 1909 dma_copy_track[dma_id].nr_vchans++; 1910 1911 return 0; 1912 } 1913 1914 int 1915 rte_vhost_async_get_inflight(int vid, uint16_t queue_id) 1916 { 1917 struct vhost_virtqueue *vq; 1918 struct virtio_net *dev = get_device(vid); 1919 int ret = -1; 1920 1921 if (dev == NULL) 1922 return ret; 1923 1924 if (queue_id >= VHOST_MAX_VRING) 1925 return ret; 1926 1927 vq = dev->virtqueue[queue_id]; 1928 1929 if (vq == NULL) 1930 return ret; 1931 1932 if (!rte_spinlock_trylock(&vq->access_lock)) { 1933 VHOST_LOG_CONFIG(dev->ifname, DEBUG, 1934 "failed to check in-flight packets. virtqueue busy.\n"); 1935 return ret; 1936 } 1937 1938 if (vq->async) 1939 ret = vq->async->pkts_inflight_n; 1940 1941 rte_spinlock_unlock(&vq->access_lock); 1942 1943 return ret; 1944 } 1945 1946 int 1947 rte_vhost_async_get_inflight_thread_unsafe(int vid, uint16_t queue_id) 1948 { 1949 struct vhost_virtqueue *vq; 1950 struct virtio_net *dev = get_device(vid); 1951 int ret = -1; 1952 1953 if (dev == NULL) 1954 return ret; 1955 1956 if (queue_id >= VHOST_MAX_VRING) 1957 return ret; 1958 1959 vq = dev->virtqueue[queue_id]; 1960 1961 if (vq == NULL) 1962 return ret; 1963 1964 if (unlikely(!rte_spinlock_is_locked(&vq->access_lock))) { 1965 VHOST_LOG_CONFIG(dev->ifname, ERR, "%s() called without access lock taken.\n", 1966 __func__); 1967 return -1; 1968 } 1969 1970 if (!vq->async) 1971 return ret; 1972 1973 ret = vq->async->pkts_inflight_n; 1974 1975 return ret; 1976 } 1977 1978 int 1979 rte_vhost_get_monitor_addr(int vid, uint16_t queue_id, 1980 struct rte_vhost_power_monitor_cond *pmc) 1981 { 1982 struct virtio_net *dev = get_device(vid); 1983 struct vhost_virtqueue *vq; 1984 1985 if (dev == NULL) 1986 return -1; 1987 if (queue_id >= VHOST_MAX_VRING) 1988 return -1; 1989 1990 vq = dev->virtqueue[queue_id]; 1991 if (vq == NULL) 1992 return -1; 1993 1994 if (vq_is_packed(dev)) { 1995 struct vring_packed_desc *desc; 1996 desc = vq->desc_packed; 1997 pmc->addr = &desc[vq->last_avail_idx].flags; 1998 if (vq->avail_wrap_counter) 1999 pmc->val = VRING_DESC_F_AVAIL; 2000 else 2001 pmc->val = VRING_DESC_F_USED; 2002 pmc->mask = VRING_DESC_F_AVAIL | VRING_DESC_F_USED; 2003 pmc->size = sizeof(desc[vq->last_avail_idx].flags); 2004 pmc->match = 1; 2005 } else { 2006 pmc->addr = &vq->avail->idx; 2007 pmc->val = vq->last_avail_idx & (vq->size - 1); 2008 pmc->mask = vq->size - 1; 2009 pmc->size = sizeof(vq->avail->idx); 2010 pmc->match = 0; 2011 } 2012 2013 return 0; 2014 } 2015 2016 2017 int 2018 rte_vhost_vring_stats_get_names(int vid, uint16_t queue_id, 2019 struct rte_vhost_stat_name *name, unsigned int size) 2020 { 2021 struct virtio_net *dev = get_device(vid); 2022 unsigned int i; 2023 2024 if (dev == NULL) 2025 return -1; 2026 2027 if (queue_id >= dev->nr_vring) 2028 return -1; 2029 2030 if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED)) 2031 return -1; 2032 2033 if (name == NULL || size < VHOST_NB_VQ_STATS) 2034 return VHOST_NB_VQ_STATS; 2035 2036 for (i = 0; i < VHOST_NB_VQ_STATS; i++) 2037 snprintf(name[i].name, sizeof(name[i].name), "%s_q%u_%s", 2038 (queue_id & 1) ? "rx" : "tx", 2039 queue_id / 2, vhost_vq_stat_strings[i].name); 2040 2041 return VHOST_NB_VQ_STATS; 2042 } 2043 2044 int 2045 rte_vhost_vring_stats_get(int vid, uint16_t queue_id, 2046 struct rte_vhost_stat *stats, unsigned int n) 2047 { 2048 struct virtio_net *dev = get_device(vid); 2049 struct vhost_virtqueue *vq; 2050 unsigned int i; 2051 2052 if (dev == NULL) 2053 return -1; 2054 2055 if (queue_id >= dev->nr_vring) 2056 return -1; 2057 2058 if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED)) 2059 return -1; 2060 2061 if (stats == NULL || n < VHOST_NB_VQ_STATS) 2062 return VHOST_NB_VQ_STATS; 2063 2064 vq = dev->virtqueue[queue_id]; 2065 2066 rte_spinlock_lock(&vq->access_lock); 2067 for (i = 0; i < VHOST_NB_VQ_STATS; i++) { 2068 stats[i].value = 2069 *(uint64_t *)(((char *)vq) + vhost_vq_stat_strings[i].offset); 2070 stats[i].id = i; 2071 } 2072 rte_spinlock_unlock(&vq->access_lock); 2073 2074 return VHOST_NB_VQ_STATS; 2075 } 2076 2077 int rte_vhost_vring_stats_reset(int vid, uint16_t queue_id) 2078 { 2079 struct virtio_net *dev = get_device(vid); 2080 struct vhost_virtqueue *vq; 2081 2082 if (dev == NULL) 2083 return -1; 2084 2085 if (queue_id >= dev->nr_vring) 2086 return -1; 2087 2088 if (!(dev->flags & VIRTIO_DEV_STATS_ENABLED)) 2089 return -1; 2090 2091 vq = dev->virtqueue[queue_id]; 2092 2093 rte_spinlock_lock(&vq->access_lock); 2094 memset(&vq->stats, 0, sizeof(vq->stats)); 2095 rte_spinlock_unlock(&vq->access_lock); 2096 2097 return 0; 2098 } 2099 2100 RTE_LOG_REGISTER_SUFFIX(vhost_config_log_level, config, INFO); 2101 RTE_LOG_REGISTER_SUFFIX(vhost_data_log_level, data, WARNING); 2102