1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <linux/vhost.h> 6 #include <linux/virtio_net.h> 7 #include <stdint.h> 8 #include <stdlib.h> 9 #ifdef RTE_LIBRTE_VHOST_NUMA 10 #include <numa.h> 11 #include <numaif.h> 12 #endif 13 14 #include <rte_errno.h> 15 #include <rte_log.h> 16 #include <rte_memory.h> 17 #include <rte_malloc.h> 18 #include <rte_vhost.h> 19 20 #include "iotlb.h" 21 #include "vhost.h" 22 #include "vhost_user.h" 23 24 struct virtio_net *vhost_devices[RTE_MAX_VHOST_DEVICE]; 25 pthread_mutex_t vhost_dev_lock = PTHREAD_MUTEX_INITIALIZER; 26 27 /* Called with iotlb_lock read-locked */ 28 uint64_t 29 __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, 30 uint64_t iova, uint64_t *size, uint8_t perm) 31 { 32 uint64_t vva, tmp_size; 33 34 if (unlikely(!*size)) 35 return 0; 36 37 tmp_size = *size; 38 39 vva = vhost_user_iotlb_cache_find(vq, iova, &tmp_size, perm); 40 if (tmp_size == *size) 41 return vva; 42 43 iova += tmp_size; 44 45 if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) { 46 /* 47 * iotlb_lock is read-locked for a full burst, 48 * but it only protects the iotlb cache. 49 * In case of IOTLB miss, we might block on the socket, 50 * which could cause a deadlock with QEMU if an IOTLB update 51 * is being handled. We can safely unlock here to avoid it. 52 */ 53 vhost_user_iotlb_rd_unlock(vq); 54 55 vhost_user_iotlb_pending_insert(dev, vq, iova, perm); 56 if (vhost_user_iotlb_miss(dev, iova, perm)) { 57 VHOST_LOG_DATA(ERR, "(%s) IOTLB miss req failed for IOVA 0x%" PRIx64 "\n", 58 dev->ifname, iova); 59 vhost_user_iotlb_pending_remove(vq, iova, 1, perm); 60 } 61 62 vhost_user_iotlb_rd_lock(vq); 63 } 64 65 return 0; 66 } 67 68 #define VHOST_LOG_PAGE 4096 69 70 /* 71 * Atomically set a bit in memory. 72 */ 73 static __rte_always_inline void 74 vhost_set_bit(unsigned int nr, volatile uint8_t *addr) 75 { 76 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) 77 /* 78 * __sync_ built-ins are deprecated, but __atomic_ ones 79 * are sub-optimized in older GCC versions. 80 */ 81 __sync_fetch_and_or_1(addr, (1U << nr)); 82 #else 83 __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED); 84 #endif 85 } 86 87 static __rte_always_inline void 88 vhost_log_page(uint8_t *log_base, uint64_t page) 89 { 90 vhost_set_bit(page % 8, &log_base[page / 8]); 91 } 92 93 void 94 __vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) 95 { 96 uint64_t page; 97 98 if (unlikely(!dev->log_base || !len)) 99 return; 100 101 if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) 102 return; 103 104 /* To make sure guest memory updates are committed before logging */ 105 rte_atomic_thread_fence(__ATOMIC_RELEASE); 106 107 page = addr / VHOST_LOG_PAGE; 108 while (page * VHOST_LOG_PAGE < addr + len) { 109 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 110 page += 1; 111 } 112 } 113 114 void 115 __vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, 116 uint64_t iova, uint64_t len) 117 { 118 uint64_t hva, gpa, map_len; 119 map_len = len; 120 121 hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); 122 if (map_len != len) { 123 VHOST_LOG_DATA(ERR, 124 "(%s) failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n", 125 dev->ifname, iova); 126 return; 127 } 128 129 gpa = hva_to_gpa(dev, hva, len); 130 if (gpa) 131 __vhost_log_write(dev, gpa, len); 132 } 133 134 void 135 __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) 136 { 137 unsigned long *log_base; 138 int i; 139 140 if (unlikely(!dev->log_base)) 141 return; 142 143 /* No cache, nothing to sync */ 144 if (unlikely(!vq->log_cache)) 145 return; 146 147 rte_atomic_thread_fence(__ATOMIC_RELEASE); 148 149 log_base = (unsigned long *)(uintptr_t)dev->log_base; 150 151 for (i = 0; i < vq->log_cache_nb_elem; i++) { 152 struct log_cache_entry *elem = vq->log_cache + i; 153 154 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) 155 /* 156 * '__sync' builtins are deprecated, but '__atomic' ones 157 * are sub-optimized in older GCC versions. 158 */ 159 __sync_fetch_and_or(log_base + elem->offset, elem->val); 160 #else 161 __atomic_fetch_or(log_base + elem->offset, elem->val, 162 __ATOMIC_RELAXED); 163 #endif 164 } 165 166 rte_atomic_thread_fence(__ATOMIC_RELEASE); 167 168 vq->log_cache_nb_elem = 0; 169 } 170 171 static __rte_always_inline void 172 vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, 173 uint64_t page) 174 { 175 uint32_t bit_nr = page % (sizeof(unsigned long) << 3); 176 uint32_t offset = page / (sizeof(unsigned long) << 3); 177 int i; 178 179 if (unlikely(!vq->log_cache)) { 180 /* No logging cache allocated, write dirty log map directly */ 181 rte_atomic_thread_fence(__ATOMIC_RELEASE); 182 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 183 184 return; 185 } 186 187 for (i = 0; i < vq->log_cache_nb_elem; i++) { 188 struct log_cache_entry *elem = vq->log_cache + i; 189 190 if (elem->offset == offset) { 191 elem->val |= (1UL << bit_nr); 192 return; 193 } 194 } 195 196 if (unlikely(i >= VHOST_LOG_CACHE_NR)) { 197 /* 198 * No more room for a new log cache entry, 199 * so write the dirty log map directly. 200 */ 201 rte_atomic_thread_fence(__ATOMIC_RELEASE); 202 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 203 204 return; 205 } 206 207 vq->log_cache[i].offset = offset; 208 vq->log_cache[i].val = (1UL << bit_nr); 209 vq->log_cache_nb_elem++; 210 } 211 212 void 213 __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, 214 uint64_t addr, uint64_t len) 215 { 216 uint64_t page; 217 218 if (unlikely(!dev->log_base || !len)) 219 return; 220 221 if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) 222 return; 223 224 page = addr / VHOST_LOG_PAGE; 225 while (page * VHOST_LOG_PAGE < addr + len) { 226 vhost_log_cache_page(dev, vq, page); 227 page += 1; 228 } 229 } 230 231 void 232 __vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, 233 uint64_t iova, uint64_t len) 234 { 235 uint64_t hva, gpa, map_len; 236 map_len = len; 237 238 hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); 239 if (map_len != len) { 240 VHOST_LOG_DATA(ERR, 241 "(%s) failed to write log for IOVA 0x%" PRIx64 ". No IOTLB entry found\n", 242 dev->ifname, iova); 243 return; 244 } 245 246 gpa = hva_to_gpa(dev, hva, len); 247 if (gpa) 248 __vhost_log_cache_write(dev, vq, gpa, len); 249 } 250 251 void * 252 vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, 253 uint64_t desc_addr, uint64_t desc_len) 254 { 255 void *idesc; 256 uint64_t src, dst; 257 uint64_t len, remain = desc_len; 258 259 idesc = rte_malloc_socket(__func__, desc_len, 0, vq->numa_node); 260 if (unlikely(!idesc)) 261 return NULL; 262 263 dst = (uint64_t)(uintptr_t)idesc; 264 265 while (remain) { 266 len = remain; 267 src = vhost_iova_to_vva(dev, vq, desc_addr, &len, 268 VHOST_ACCESS_RO); 269 if (unlikely(!src || !len)) { 270 rte_free(idesc); 271 return NULL; 272 } 273 274 rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); 275 276 remain -= len; 277 dst += len; 278 desc_addr += len; 279 } 280 281 return idesc; 282 } 283 284 void 285 cleanup_vq(struct vhost_virtqueue *vq, int destroy) 286 { 287 if ((vq->callfd >= 0) && (destroy != 0)) 288 close(vq->callfd); 289 if (vq->kickfd >= 0) 290 close(vq->kickfd); 291 } 292 293 void 294 cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq) 295 { 296 if (!(dev->protocol_features & 297 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD))) 298 return; 299 300 if (vq_is_packed(dev)) { 301 if (vq->inflight_packed) 302 vq->inflight_packed = NULL; 303 } else { 304 if (vq->inflight_split) 305 vq->inflight_split = NULL; 306 } 307 308 if (vq->resubmit_inflight) { 309 if (vq->resubmit_inflight->resubmit_list) { 310 rte_free(vq->resubmit_inflight->resubmit_list); 311 vq->resubmit_inflight->resubmit_list = NULL; 312 } 313 rte_free(vq->resubmit_inflight); 314 vq->resubmit_inflight = NULL; 315 } 316 } 317 318 /* 319 * Unmap any memory, close any file descriptors and 320 * free any memory owned by a device. 321 */ 322 void 323 cleanup_device(struct virtio_net *dev, int destroy) 324 { 325 uint32_t i; 326 327 vhost_backend_cleanup(dev); 328 329 for (i = 0; i < dev->nr_vring; i++) { 330 cleanup_vq(dev->virtqueue[i], destroy); 331 cleanup_vq_inflight(dev, dev->virtqueue[i]); 332 } 333 } 334 335 static void 336 vhost_free_async_mem(struct vhost_virtqueue *vq) 337 { 338 if (!vq->async) 339 return; 340 341 rte_free(vq->async->pkts_info); 342 rte_free(vq->async->pkts_cmpl_flag); 343 344 rte_free(vq->async->buffers_packed); 345 vq->async->buffers_packed = NULL; 346 rte_free(vq->async->descs_split); 347 vq->async->descs_split = NULL; 348 349 rte_free(vq->async); 350 vq->async = NULL; 351 } 352 353 void 354 free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq) 355 { 356 if (vq_is_packed(dev)) 357 rte_free(vq->shadow_used_packed); 358 else 359 rte_free(vq->shadow_used_split); 360 361 vhost_free_async_mem(vq); 362 rte_free(vq->batch_copy_elems); 363 rte_mempool_free(vq->iotlb_pool); 364 rte_free(vq->log_cache); 365 rte_free(vq); 366 } 367 368 /* 369 * Release virtqueues and device memory. 370 */ 371 static void 372 free_device(struct virtio_net *dev) 373 { 374 uint32_t i; 375 376 for (i = 0; i < dev->nr_vring; i++) 377 free_vq(dev, dev->virtqueue[i]); 378 379 rte_free(dev); 380 } 381 382 static __rte_always_inline int 383 log_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) 384 { 385 if (likely(!(vq->ring_addrs.flags & (1 << VHOST_VRING_F_LOG)))) 386 return 0; 387 388 vq->log_guest_addr = translate_log_addr(dev, vq, 389 vq->ring_addrs.log_guest_addr); 390 if (vq->log_guest_addr == 0) 391 return -1; 392 393 return 0; 394 } 395 396 /* 397 * Converts vring log address to GPA 398 * If IOMMU is enabled, the log address is IOVA 399 * If IOMMU not enabled, the log address is already GPA 400 * 401 * Caller should have iotlb_lock read-locked 402 */ 403 uint64_t 404 translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq, 405 uint64_t log_addr) 406 { 407 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) { 408 const uint64_t exp_size = sizeof(uint64_t); 409 uint64_t hva, gpa; 410 uint64_t size = exp_size; 411 412 hva = vhost_iova_to_vva(dev, vq, log_addr, 413 &size, VHOST_ACCESS_RW); 414 415 if (size != exp_size) 416 return 0; 417 418 gpa = hva_to_gpa(dev, hva, exp_size); 419 if (!gpa) { 420 VHOST_LOG_DATA(ERR, 421 "(%s) failed to find GPA for log_addr: 0x%" 422 PRIx64 " hva: 0x%" PRIx64 "\n", 423 dev->ifname, log_addr, hva); 424 return 0; 425 } 426 return gpa; 427 428 } else 429 return log_addr; 430 } 431 432 /* Caller should have iotlb_lock read-locked */ 433 static int 434 vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq) 435 { 436 uint64_t req_size, size; 437 438 req_size = sizeof(struct vring_desc) * vq->size; 439 size = req_size; 440 vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq, 441 vq->ring_addrs.desc_user_addr, 442 &size, VHOST_ACCESS_RW); 443 if (!vq->desc || size != req_size) 444 return -1; 445 446 req_size = sizeof(struct vring_avail); 447 req_size += sizeof(uint16_t) * vq->size; 448 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 449 req_size += sizeof(uint16_t); 450 size = req_size; 451 vq->avail = (struct vring_avail *)(uintptr_t)vhost_iova_to_vva(dev, vq, 452 vq->ring_addrs.avail_user_addr, 453 &size, VHOST_ACCESS_RW); 454 if (!vq->avail || size != req_size) 455 return -1; 456 457 req_size = sizeof(struct vring_used); 458 req_size += sizeof(struct vring_used_elem) * vq->size; 459 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 460 req_size += sizeof(uint16_t); 461 size = req_size; 462 vq->used = (struct vring_used *)(uintptr_t)vhost_iova_to_vva(dev, vq, 463 vq->ring_addrs.used_user_addr, 464 &size, VHOST_ACCESS_RW); 465 if (!vq->used || size != req_size) 466 return -1; 467 468 return 0; 469 } 470 471 /* Caller should have iotlb_lock read-locked */ 472 static int 473 vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) 474 { 475 uint64_t req_size, size; 476 477 req_size = sizeof(struct vring_packed_desc) * vq->size; 478 size = req_size; 479 vq->desc_packed = (struct vring_packed_desc *)(uintptr_t) 480 vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr, 481 &size, VHOST_ACCESS_RW); 482 if (!vq->desc_packed || size != req_size) 483 return -1; 484 485 req_size = sizeof(struct vring_packed_desc_event); 486 size = req_size; 487 vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t) 488 vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr, 489 &size, VHOST_ACCESS_RW); 490 if (!vq->driver_event || size != req_size) 491 return -1; 492 493 req_size = sizeof(struct vring_packed_desc_event); 494 size = req_size; 495 vq->device_event = (struct vring_packed_desc_event *)(uintptr_t) 496 vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr, 497 &size, VHOST_ACCESS_RW); 498 if (!vq->device_event || size != req_size) 499 return -1; 500 501 return 0; 502 } 503 504 int 505 vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) 506 { 507 508 if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) 509 return -1; 510 511 if (vq_is_packed(dev)) { 512 if (vring_translate_packed(dev, vq) < 0) 513 return -1; 514 } else { 515 if (vring_translate_split(dev, vq) < 0) 516 return -1; 517 } 518 519 if (log_translate(dev, vq) < 0) 520 return -1; 521 522 vq->access_ok = true; 523 524 return 0; 525 } 526 527 void 528 vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq) 529 { 530 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) 531 vhost_user_iotlb_wr_lock(vq); 532 533 vq->access_ok = false; 534 vq->desc = NULL; 535 vq->avail = NULL; 536 vq->used = NULL; 537 vq->log_guest_addr = 0; 538 539 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) 540 vhost_user_iotlb_wr_unlock(vq); 541 } 542 543 static void 544 init_vring_queue(struct virtio_net *dev, uint32_t vring_idx) 545 { 546 struct vhost_virtqueue *vq; 547 int numa_node = SOCKET_ID_ANY; 548 549 if (vring_idx >= VHOST_MAX_VRING) { 550 VHOST_LOG_CONFIG(ERR, "(%s) failed to init vring, out of bound (%d)\n", 551 dev->ifname, vring_idx); 552 return; 553 } 554 555 vq = dev->virtqueue[vring_idx]; 556 if (!vq) { 557 VHOST_LOG_CONFIG(ERR, "(%s) virtqueue not allocated (%d)\n", 558 dev->ifname, vring_idx); 559 return; 560 } 561 562 memset(vq, 0, sizeof(struct vhost_virtqueue)); 563 564 vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; 565 vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; 566 vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF; 567 568 #ifdef RTE_LIBRTE_VHOST_NUMA 569 if (get_mempolicy(&numa_node, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR)) { 570 VHOST_LOG_CONFIG(ERR, "(%s) failed to query numa node: %s\n", 571 dev->ifname, rte_strerror(errno)); 572 numa_node = SOCKET_ID_ANY; 573 } 574 #endif 575 vq->numa_node = numa_node; 576 577 vhost_user_iotlb_init(dev, vring_idx); 578 } 579 580 static void 581 reset_vring_queue(struct virtio_net *dev, uint32_t vring_idx) 582 { 583 struct vhost_virtqueue *vq; 584 int callfd; 585 586 if (vring_idx >= VHOST_MAX_VRING) { 587 VHOST_LOG_CONFIG(ERR, 588 "(%s) failed to reset vring, out of bound (%d)\n", 589 dev->ifname, vring_idx); 590 return; 591 } 592 593 vq = dev->virtqueue[vring_idx]; 594 if (!vq) { 595 VHOST_LOG_CONFIG(ERR, "(%s) failed to reset vring, virtqueue not allocated (%d)\n", 596 dev->ifname, vring_idx); 597 return; 598 } 599 600 callfd = vq->callfd; 601 init_vring_queue(dev, vring_idx); 602 vq->callfd = callfd; 603 } 604 605 int 606 alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx) 607 { 608 struct vhost_virtqueue *vq; 609 uint32_t i; 610 611 /* Also allocate holes, if any, up to requested vring index. */ 612 for (i = 0; i <= vring_idx; i++) { 613 if (dev->virtqueue[i]) 614 continue; 615 616 vq = rte_zmalloc(NULL, sizeof(struct vhost_virtqueue), 0); 617 if (vq == NULL) { 618 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate memory for vring %u.\n", 619 dev->ifname, i); 620 return -1; 621 } 622 623 dev->virtqueue[i] = vq; 624 init_vring_queue(dev, i); 625 rte_spinlock_init(&vq->access_lock); 626 vq->avail_wrap_counter = 1; 627 vq->used_wrap_counter = 1; 628 vq->signalled_used_valid = false; 629 } 630 631 dev->nr_vring = RTE_MAX(dev->nr_vring, vring_idx + 1); 632 633 return 0; 634 } 635 636 /* 637 * Reset some variables in device structure, while keeping few 638 * others untouched, such as vid, ifname, nr_vring: they 639 * should be same unless the device is removed. 640 */ 641 void 642 reset_device(struct virtio_net *dev) 643 { 644 uint32_t i; 645 646 dev->features = 0; 647 dev->protocol_features = 0; 648 dev->flags &= VIRTIO_DEV_BUILTIN_VIRTIO_NET; 649 650 for (i = 0; i < dev->nr_vring; i++) 651 reset_vring_queue(dev, i); 652 } 653 654 /* 655 * Invoked when there is a new vhost-user connection established (when 656 * there is a new virtio device being attached). 657 */ 658 int 659 vhost_new_device(void) 660 { 661 struct virtio_net *dev; 662 int i; 663 664 pthread_mutex_lock(&vhost_dev_lock); 665 for (i = 0; i < RTE_MAX_VHOST_DEVICE; i++) { 666 if (vhost_devices[i] == NULL) 667 break; 668 } 669 670 if (i == RTE_MAX_VHOST_DEVICE) { 671 VHOST_LOG_CONFIG(ERR, "failed to find a free slot for new device.\n"); 672 pthread_mutex_unlock(&vhost_dev_lock); 673 return -1; 674 } 675 676 dev = rte_zmalloc(NULL, sizeof(struct virtio_net), 0); 677 if (dev == NULL) { 678 VHOST_LOG_CONFIG(ERR, "failed to allocate memory for new device.\n"); 679 pthread_mutex_unlock(&vhost_dev_lock); 680 return -1; 681 } 682 683 vhost_devices[i] = dev; 684 pthread_mutex_unlock(&vhost_dev_lock); 685 686 dev->vid = i; 687 dev->flags = VIRTIO_DEV_BUILTIN_VIRTIO_NET; 688 dev->slave_req_fd = -1; 689 dev->postcopy_ufd = -1; 690 rte_spinlock_init(&dev->slave_req_lock); 691 692 return i; 693 } 694 695 void 696 vhost_destroy_device_notify(struct virtio_net *dev) 697 { 698 struct rte_vdpa_device *vdpa_dev; 699 700 if (dev->flags & VIRTIO_DEV_RUNNING) { 701 vdpa_dev = dev->vdpa_dev; 702 if (vdpa_dev) 703 vdpa_dev->ops->dev_close(dev->vid); 704 dev->flags &= ~VIRTIO_DEV_RUNNING; 705 dev->notify_ops->destroy_device(dev->vid); 706 } 707 } 708 709 /* 710 * Invoked when there is the vhost-user connection is broken (when 711 * the virtio device is being detached). 712 */ 713 void 714 vhost_destroy_device(int vid) 715 { 716 struct virtio_net *dev = get_device(vid); 717 718 if (dev == NULL) 719 return; 720 721 vhost_destroy_device_notify(dev); 722 723 cleanup_device(dev, 1); 724 free_device(dev); 725 726 vhost_devices[vid] = NULL; 727 } 728 729 void 730 vhost_attach_vdpa_device(int vid, struct rte_vdpa_device *vdpa_dev) 731 { 732 struct virtio_net *dev = get_device(vid); 733 734 if (dev == NULL) 735 return; 736 737 dev->vdpa_dev = vdpa_dev; 738 } 739 740 void 741 vhost_set_ifname(int vid, const char *if_name, unsigned int if_len) 742 { 743 struct virtio_net *dev; 744 unsigned int len; 745 746 dev = get_device(vid); 747 if (dev == NULL) 748 return; 749 750 len = if_len > sizeof(dev->ifname) ? 751 sizeof(dev->ifname) : if_len; 752 753 strncpy(dev->ifname, if_name, len); 754 dev->ifname[sizeof(dev->ifname) - 1] = '\0'; 755 } 756 757 void 758 vhost_setup_virtio_net(int vid, bool enable, bool compliant_ol_flags) 759 { 760 struct virtio_net *dev = get_device(vid); 761 762 if (dev == NULL) 763 return; 764 765 if (enable) 766 dev->flags |= VIRTIO_DEV_BUILTIN_VIRTIO_NET; 767 else 768 dev->flags &= ~VIRTIO_DEV_BUILTIN_VIRTIO_NET; 769 if (!compliant_ol_flags) 770 dev->flags |= VIRTIO_DEV_LEGACY_OL_FLAGS; 771 else 772 dev->flags &= ~VIRTIO_DEV_LEGACY_OL_FLAGS; 773 } 774 775 void 776 vhost_enable_extbuf(int vid) 777 { 778 struct virtio_net *dev = get_device(vid); 779 780 if (dev == NULL) 781 return; 782 783 dev->extbuf = 1; 784 } 785 786 void 787 vhost_enable_linearbuf(int vid) 788 { 789 struct virtio_net *dev = get_device(vid); 790 791 if (dev == NULL) 792 return; 793 794 dev->linearbuf = 1; 795 } 796 797 int 798 rte_vhost_get_mtu(int vid, uint16_t *mtu) 799 { 800 struct virtio_net *dev = get_device(vid); 801 802 if (dev == NULL || mtu == NULL) 803 return -ENODEV; 804 805 if (!(dev->flags & VIRTIO_DEV_READY)) 806 return -EAGAIN; 807 808 if (!(dev->features & (1ULL << VIRTIO_NET_F_MTU))) 809 return -ENOTSUP; 810 811 *mtu = dev->mtu; 812 813 return 0; 814 } 815 816 int 817 rte_vhost_get_numa_node(int vid) 818 { 819 #ifdef RTE_LIBRTE_VHOST_NUMA 820 struct virtio_net *dev = get_device(vid); 821 int numa_node; 822 int ret; 823 824 if (dev == NULL || numa_available() != 0) 825 return -1; 826 827 ret = get_mempolicy(&numa_node, NULL, 0, dev, 828 MPOL_F_NODE | MPOL_F_ADDR); 829 if (ret < 0) { 830 VHOST_LOG_CONFIG(ERR, "(%s) failed to query numa node: %s\n", 831 dev->ifname, rte_strerror(errno)); 832 return -1; 833 } 834 835 return numa_node; 836 #else 837 RTE_SET_USED(vid); 838 return -1; 839 #endif 840 } 841 842 uint32_t 843 rte_vhost_get_queue_num(int vid) 844 { 845 struct virtio_net *dev = get_device(vid); 846 847 if (dev == NULL) 848 return 0; 849 850 return dev->nr_vring / 2; 851 } 852 853 uint16_t 854 rte_vhost_get_vring_num(int vid) 855 { 856 struct virtio_net *dev = get_device(vid); 857 858 if (dev == NULL) 859 return 0; 860 861 return dev->nr_vring; 862 } 863 864 int 865 rte_vhost_get_ifname(int vid, char *buf, size_t len) 866 { 867 struct virtio_net *dev = get_device(vid); 868 869 if (dev == NULL || buf == NULL) 870 return -1; 871 872 len = RTE_MIN(len, sizeof(dev->ifname)); 873 874 strncpy(buf, dev->ifname, len); 875 buf[len - 1] = '\0'; 876 877 return 0; 878 } 879 880 int 881 rte_vhost_get_negotiated_features(int vid, uint64_t *features) 882 { 883 struct virtio_net *dev; 884 885 dev = get_device(vid); 886 if (dev == NULL || features == NULL) 887 return -1; 888 889 *features = dev->features; 890 return 0; 891 } 892 893 int 894 rte_vhost_get_negotiated_protocol_features(int vid, 895 uint64_t *protocol_features) 896 { 897 struct virtio_net *dev; 898 899 dev = get_device(vid); 900 if (dev == NULL || protocol_features == NULL) 901 return -1; 902 903 *protocol_features = dev->protocol_features; 904 return 0; 905 } 906 907 int 908 rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem) 909 { 910 struct virtio_net *dev; 911 struct rte_vhost_memory *m; 912 size_t size; 913 914 dev = get_device(vid); 915 if (dev == NULL || mem == NULL) 916 return -1; 917 918 size = dev->mem->nregions * sizeof(struct rte_vhost_mem_region); 919 m = malloc(sizeof(struct rte_vhost_memory) + size); 920 if (!m) 921 return -1; 922 923 m->nregions = dev->mem->nregions; 924 memcpy(m->regions, dev->mem->regions, size); 925 *mem = m; 926 927 return 0; 928 } 929 930 int 931 rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx, 932 struct rte_vhost_vring *vring) 933 { 934 struct virtio_net *dev; 935 struct vhost_virtqueue *vq; 936 937 dev = get_device(vid); 938 if (dev == NULL || vring == NULL) 939 return -1; 940 941 if (vring_idx >= VHOST_MAX_VRING) 942 return -1; 943 944 vq = dev->virtqueue[vring_idx]; 945 if (!vq) 946 return -1; 947 948 if (vq_is_packed(dev)) { 949 vring->desc_packed = vq->desc_packed; 950 vring->driver_event = vq->driver_event; 951 vring->device_event = vq->device_event; 952 } else { 953 vring->desc = vq->desc; 954 vring->avail = vq->avail; 955 vring->used = vq->used; 956 } 957 vring->log_guest_addr = vq->log_guest_addr; 958 959 vring->callfd = vq->callfd; 960 vring->kickfd = vq->kickfd; 961 vring->size = vq->size; 962 963 return 0; 964 } 965 966 int 967 rte_vhost_get_vhost_ring_inflight(int vid, uint16_t vring_idx, 968 struct rte_vhost_ring_inflight *vring) 969 { 970 struct virtio_net *dev; 971 struct vhost_virtqueue *vq; 972 973 dev = get_device(vid); 974 if (unlikely(!dev)) 975 return -1; 976 977 if (vring_idx >= VHOST_MAX_VRING) 978 return -1; 979 980 vq = dev->virtqueue[vring_idx]; 981 if (unlikely(!vq)) 982 return -1; 983 984 if (vq_is_packed(dev)) { 985 if (unlikely(!vq->inflight_packed)) 986 return -1; 987 988 vring->inflight_packed = vq->inflight_packed; 989 } else { 990 if (unlikely(!vq->inflight_split)) 991 return -1; 992 993 vring->inflight_split = vq->inflight_split; 994 } 995 996 vring->resubmit_inflight = vq->resubmit_inflight; 997 998 return 0; 999 } 1000 1001 int 1002 rte_vhost_set_inflight_desc_split(int vid, uint16_t vring_idx, 1003 uint16_t idx) 1004 { 1005 struct vhost_virtqueue *vq; 1006 struct virtio_net *dev; 1007 1008 dev = get_device(vid); 1009 if (unlikely(!dev)) 1010 return -1; 1011 1012 if (unlikely(!(dev->protocol_features & 1013 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1014 return 0; 1015 1016 if (unlikely(vq_is_packed(dev))) 1017 return -1; 1018 1019 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1020 return -1; 1021 1022 vq = dev->virtqueue[vring_idx]; 1023 if (unlikely(!vq)) 1024 return -1; 1025 1026 if (unlikely(!vq->inflight_split)) 1027 return -1; 1028 1029 if (unlikely(idx >= vq->size)) 1030 return -1; 1031 1032 vq->inflight_split->desc[idx].counter = vq->global_counter++; 1033 vq->inflight_split->desc[idx].inflight = 1; 1034 return 0; 1035 } 1036 1037 int 1038 rte_vhost_set_inflight_desc_packed(int vid, uint16_t vring_idx, 1039 uint16_t head, uint16_t last, 1040 uint16_t *inflight_entry) 1041 { 1042 struct rte_vhost_inflight_info_packed *inflight_info; 1043 struct virtio_net *dev; 1044 struct vhost_virtqueue *vq; 1045 struct vring_packed_desc *desc; 1046 uint16_t old_free_head, free_head; 1047 1048 dev = get_device(vid); 1049 if (unlikely(!dev)) 1050 return -1; 1051 1052 if (unlikely(!(dev->protocol_features & 1053 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1054 return 0; 1055 1056 if (unlikely(!vq_is_packed(dev))) 1057 return -1; 1058 1059 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1060 return -1; 1061 1062 vq = dev->virtqueue[vring_idx]; 1063 if (unlikely(!vq)) 1064 return -1; 1065 1066 inflight_info = vq->inflight_packed; 1067 if (unlikely(!inflight_info)) 1068 return -1; 1069 1070 if (unlikely(head >= vq->size)) 1071 return -1; 1072 1073 desc = vq->desc_packed; 1074 old_free_head = inflight_info->old_free_head; 1075 if (unlikely(old_free_head >= vq->size)) 1076 return -1; 1077 1078 free_head = old_free_head; 1079 1080 /* init header descriptor */ 1081 inflight_info->desc[old_free_head].num = 0; 1082 inflight_info->desc[old_free_head].counter = vq->global_counter++; 1083 inflight_info->desc[old_free_head].inflight = 1; 1084 1085 /* save desc entry in flight entry */ 1086 while (head != ((last + 1) % vq->size)) { 1087 inflight_info->desc[old_free_head].num++; 1088 inflight_info->desc[free_head].addr = desc[head].addr; 1089 inflight_info->desc[free_head].len = desc[head].len; 1090 inflight_info->desc[free_head].flags = desc[head].flags; 1091 inflight_info->desc[free_head].id = desc[head].id; 1092 1093 inflight_info->desc[old_free_head].last = free_head; 1094 free_head = inflight_info->desc[free_head].next; 1095 inflight_info->free_head = free_head; 1096 head = (head + 1) % vq->size; 1097 } 1098 1099 inflight_info->old_free_head = free_head; 1100 *inflight_entry = old_free_head; 1101 1102 return 0; 1103 } 1104 1105 int 1106 rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx, 1107 uint16_t last_used_idx, uint16_t idx) 1108 { 1109 struct virtio_net *dev; 1110 struct vhost_virtqueue *vq; 1111 1112 dev = get_device(vid); 1113 if (unlikely(!dev)) 1114 return -1; 1115 1116 if (unlikely(!(dev->protocol_features & 1117 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1118 return 0; 1119 1120 if (unlikely(vq_is_packed(dev))) 1121 return -1; 1122 1123 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1124 return -1; 1125 1126 vq = dev->virtqueue[vring_idx]; 1127 if (unlikely(!vq)) 1128 return -1; 1129 1130 if (unlikely(!vq->inflight_split)) 1131 return -1; 1132 1133 if (unlikely(idx >= vq->size)) 1134 return -1; 1135 1136 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1137 1138 vq->inflight_split->desc[idx].inflight = 0; 1139 1140 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1141 1142 vq->inflight_split->used_idx = last_used_idx; 1143 return 0; 1144 } 1145 1146 int 1147 rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx, 1148 uint16_t head) 1149 { 1150 struct rte_vhost_inflight_info_packed *inflight_info; 1151 struct virtio_net *dev; 1152 struct vhost_virtqueue *vq; 1153 1154 dev = get_device(vid); 1155 if (unlikely(!dev)) 1156 return -1; 1157 1158 if (unlikely(!(dev->protocol_features & 1159 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1160 return 0; 1161 1162 if (unlikely(!vq_is_packed(dev))) 1163 return -1; 1164 1165 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1166 return -1; 1167 1168 vq = dev->virtqueue[vring_idx]; 1169 if (unlikely(!vq)) 1170 return -1; 1171 1172 inflight_info = vq->inflight_packed; 1173 if (unlikely(!inflight_info)) 1174 return -1; 1175 1176 if (unlikely(head >= vq->size)) 1177 return -1; 1178 1179 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1180 1181 inflight_info->desc[head].inflight = 0; 1182 1183 rte_atomic_thread_fence(__ATOMIC_SEQ_CST); 1184 1185 inflight_info->old_free_head = inflight_info->free_head; 1186 inflight_info->old_used_idx = inflight_info->used_idx; 1187 inflight_info->old_used_wrap_counter = inflight_info->used_wrap_counter; 1188 1189 return 0; 1190 } 1191 1192 int 1193 rte_vhost_set_last_inflight_io_split(int vid, uint16_t vring_idx, 1194 uint16_t idx) 1195 { 1196 struct virtio_net *dev; 1197 struct vhost_virtqueue *vq; 1198 1199 dev = get_device(vid); 1200 if (unlikely(!dev)) 1201 return -1; 1202 1203 if (unlikely(!(dev->protocol_features & 1204 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1205 return 0; 1206 1207 if (unlikely(vq_is_packed(dev))) 1208 return -1; 1209 1210 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1211 return -1; 1212 1213 vq = dev->virtqueue[vring_idx]; 1214 if (unlikely(!vq)) 1215 return -1; 1216 1217 if (unlikely(!vq->inflight_split)) 1218 return -1; 1219 1220 if (unlikely(idx >= vq->size)) 1221 return -1; 1222 1223 vq->inflight_split->last_inflight_io = idx; 1224 return 0; 1225 } 1226 1227 int 1228 rte_vhost_set_last_inflight_io_packed(int vid, uint16_t vring_idx, 1229 uint16_t head) 1230 { 1231 struct rte_vhost_inflight_info_packed *inflight_info; 1232 struct virtio_net *dev; 1233 struct vhost_virtqueue *vq; 1234 uint16_t last; 1235 1236 dev = get_device(vid); 1237 if (unlikely(!dev)) 1238 return -1; 1239 1240 if (unlikely(!(dev->protocol_features & 1241 (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))) 1242 return 0; 1243 1244 if (unlikely(!vq_is_packed(dev))) 1245 return -1; 1246 1247 if (unlikely(vring_idx >= VHOST_MAX_VRING)) 1248 return -1; 1249 1250 vq = dev->virtqueue[vring_idx]; 1251 if (unlikely(!vq)) 1252 return -1; 1253 1254 inflight_info = vq->inflight_packed; 1255 if (unlikely(!inflight_info)) 1256 return -1; 1257 1258 if (unlikely(head >= vq->size)) 1259 return -1; 1260 1261 last = inflight_info->desc[head].last; 1262 if (unlikely(last >= vq->size)) 1263 return -1; 1264 1265 inflight_info->desc[last].next = inflight_info->free_head; 1266 inflight_info->free_head = head; 1267 inflight_info->used_idx += inflight_info->desc[head].num; 1268 if (inflight_info->used_idx >= inflight_info->desc_num) { 1269 inflight_info->used_idx -= inflight_info->desc_num; 1270 inflight_info->used_wrap_counter = 1271 !inflight_info->used_wrap_counter; 1272 } 1273 1274 return 0; 1275 } 1276 1277 int 1278 rte_vhost_vring_call(int vid, uint16_t vring_idx) 1279 { 1280 struct virtio_net *dev; 1281 struct vhost_virtqueue *vq; 1282 1283 dev = get_device(vid); 1284 if (!dev) 1285 return -1; 1286 1287 if (vring_idx >= VHOST_MAX_VRING) 1288 return -1; 1289 1290 vq = dev->virtqueue[vring_idx]; 1291 if (!vq) 1292 return -1; 1293 1294 if (vq_is_packed(dev)) 1295 vhost_vring_call_packed(dev, vq); 1296 else 1297 vhost_vring_call_split(dev, vq); 1298 1299 return 0; 1300 } 1301 1302 uint16_t 1303 rte_vhost_avail_entries(int vid, uint16_t queue_id) 1304 { 1305 struct virtio_net *dev; 1306 struct vhost_virtqueue *vq; 1307 uint16_t ret = 0; 1308 1309 dev = get_device(vid); 1310 if (!dev) 1311 return 0; 1312 1313 if (queue_id >= VHOST_MAX_VRING) 1314 return 0; 1315 1316 vq = dev->virtqueue[queue_id]; 1317 if (!vq) 1318 return 0; 1319 1320 rte_spinlock_lock(&vq->access_lock); 1321 1322 if (unlikely(!vq->enabled || vq->avail == NULL)) 1323 goto out; 1324 1325 ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; 1326 1327 out: 1328 rte_spinlock_unlock(&vq->access_lock); 1329 return ret; 1330 } 1331 1332 static inline int 1333 vhost_enable_notify_split(struct virtio_net *dev, 1334 struct vhost_virtqueue *vq, int enable) 1335 { 1336 if (vq->used == NULL) 1337 return -1; 1338 1339 if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { 1340 if (enable) 1341 vq->used->flags &= ~VRING_USED_F_NO_NOTIFY; 1342 else 1343 vq->used->flags |= VRING_USED_F_NO_NOTIFY; 1344 } else { 1345 if (enable) 1346 vhost_avail_event(vq) = vq->last_avail_idx; 1347 } 1348 return 0; 1349 } 1350 1351 static inline int 1352 vhost_enable_notify_packed(struct virtio_net *dev, 1353 struct vhost_virtqueue *vq, int enable) 1354 { 1355 uint16_t flags; 1356 1357 if (vq->device_event == NULL) 1358 return -1; 1359 1360 if (!enable) { 1361 vq->device_event->flags = VRING_EVENT_F_DISABLE; 1362 return 0; 1363 } 1364 1365 flags = VRING_EVENT_F_ENABLE; 1366 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { 1367 flags = VRING_EVENT_F_DESC; 1368 vq->device_event->off_wrap = vq->last_avail_idx | 1369 vq->avail_wrap_counter << 15; 1370 } 1371 1372 rte_atomic_thread_fence(__ATOMIC_RELEASE); 1373 1374 vq->device_event->flags = flags; 1375 return 0; 1376 } 1377 1378 int 1379 vhost_enable_guest_notification(struct virtio_net *dev, 1380 struct vhost_virtqueue *vq, int enable) 1381 { 1382 /* 1383 * If the virtqueue is not ready yet, it will be applied 1384 * when it will become ready. 1385 */ 1386 if (!vq->ready) 1387 return 0; 1388 1389 if (vq_is_packed(dev)) 1390 return vhost_enable_notify_packed(dev, vq, enable); 1391 else 1392 return vhost_enable_notify_split(dev, vq, enable); 1393 } 1394 1395 int 1396 rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) 1397 { 1398 struct virtio_net *dev = get_device(vid); 1399 struct vhost_virtqueue *vq; 1400 int ret; 1401 1402 if (!dev) 1403 return -1; 1404 1405 if (queue_id >= VHOST_MAX_VRING) 1406 return -1; 1407 1408 vq = dev->virtqueue[queue_id]; 1409 if (!vq) 1410 return -1; 1411 1412 rte_spinlock_lock(&vq->access_lock); 1413 1414 vq->notif_enable = enable; 1415 ret = vhost_enable_guest_notification(dev, vq, enable); 1416 1417 rte_spinlock_unlock(&vq->access_lock); 1418 1419 return ret; 1420 } 1421 1422 void 1423 rte_vhost_log_write(int vid, uint64_t addr, uint64_t len) 1424 { 1425 struct virtio_net *dev = get_device(vid); 1426 1427 if (dev == NULL) 1428 return; 1429 1430 vhost_log_write(dev, addr, len); 1431 } 1432 1433 void 1434 rte_vhost_log_used_vring(int vid, uint16_t vring_idx, 1435 uint64_t offset, uint64_t len) 1436 { 1437 struct virtio_net *dev; 1438 struct vhost_virtqueue *vq; 1439 1440 dev = get_device(vid); 1441 if (dev == NULL) 1442 return; 1443 1444 if (vring_idx >= VHOST_MAX_VRING) 1445 return; 1446 vq = dev->virtqueue[vring_idx]; 1447 if (!vq) 1448 return; 1449 1450 vhost_log_used_vring(dev, vq, offset, len); 1451 } 1452 1453 uint32_t 1454 rte_vhost_rx_queue_count(int vid, uint16_t qid) 1455 { 1456 struct virtio_net *dev; 1457 struct vhost_virtqueue *vq; 1458 uint32_t ret = 0; 1459 1460 dev = get_device(vid); 1461 if (dev == NULL) 1462 return 0; 1463 1464 if (unlikely(qid >= dev->nr_vring || (qid & 1) == 0)) { 1465 VHOST_LOG_DATA(ERR, "(%s) %s: invalid virtqueue idx %d.\n", 1466 dev->ifname, __func__, qid); 1467 return 0; 1468 } 1469 1470 vq = dev->virtqueue[qid]; 1471 if (vq == NULL) 1472 return 0; 1473 1474 rte_spinlock_lock(&vq->access_lock); 1475 1476 if (unlikely(!vq->enabled || vq->avail == NULL)) 1477 goto out; 1478 1479 ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx; 1480 1481 out: 1482 rte_spinlock_unlock(&vq->access_lock); 1483 return ret; 1484 } 1485 1486 struct rte_vdpa_device * 1487 rte_vhost_get_vdpa_device(int vid) 1488 { 1489 struct virtio_net *dev = get_device(vid); 1490 1491 if (dev == NULL) 1492 return NULL; 1493 1494 return dev->vdpa_dev; 1495 } 1496 1497 int 1498 rte_vhost_get_log_base(int vid, uint64_t *log_base, 1499 uint64_t *log_size) 1500 { 1501 struct virtio_net *dev = get_device(vid); 1502 1503 if (dev == NULL || log_base == NULL || log_size == NULL) 1504 return -1; 1505 1506 *log_base = dev->log_base; 1507 *log_size = dev->log_size; 1508 1509 return 0; 1510 } 1511 1512 int 1513 rte_vhost_get_vring_base(int vid, uint16_t queue_id, 1514 uint16_t *last_avail_idx, uint16_t *last_used_idx) 1515 { 1516 struct vhost_virtqueue *vq; 1517 struct virtio_net *dev = get_device(vid); 1518 1519 if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) 1520 return -1; 1521 1522 if (queue_id >= VHOST_MAX_VRING) 1523 return -1; 1524 1525 vq = dev->virtqueue[queue_id]; 1526 if (!vq) 1527 return -1; 1528 1529 if (vq_is_packed(dev)) { 1530 *last_avail_idx = (vq->avail_wrap_counter << 15) | 1531 vq->last_avail_idx; 1532 *last_used_idx = (vq->used_wrap_counter << 15) | 1533 vq->last_used_idx; 1534 } else { 1535 *last_avail_idx = vq->last_avail_idx; 1536 *last_used_idx = vq->last_used_idx; 1537 } 1538 1539 return 0; 1540 } 1541 1542 int 1543 rte_vhost_set_vring_base(int vid, uint16_t queue_id, 1544 uint16_t last_avail_idx, uint16_t last_used_idx) 1545 { 1546 struct vhost_virtqueue *vq; 1547 struct virtio_net *dev = get_device(vid); 1548 1549 if (!dev) 1550 return -1; 1551 1552 if (queue_id >= VHOST_MAX_VRING) 1553 return -1; 1554 1555 vq = dev->virtqueue[queue_id]; 1556 if (!vq) 1557 return -1; 1558 1559 if (vq_is_packed(dev)) { 1560 vq->last_avail_idx = last_avail_idx & 0x7fff; 1561 vq->avail_wrap_counter = !!(last_avail_idx & (1 << 15)); 1562 vq->last_used_idx = last_used_idx & 0x7fff; 1563 vq->used_wrap_counter = !!(last_used_idx & (1 << 15)); 1564 } else { 1565 vq->last_avail_idx = last_avail_idx; 1566 vq->last_used_idx = last_used_idx; 1567 } 1568 1569 return 0; 1570 } 1571 1572 int 1573 rte_vhost_get_vring_base_from_inflight(int vid, 1574 uint16_t queue_id, 1575 uint16_t *last_avail_idx, 1576 uint16_t *last_used_idx) 1577 { 1578 struct rte_vhost_inflight_info_packed *inflight_info; 1579 struct vhost_virtqueue *vq; 1580 struct virtio_net *dev = get_device(vid); 1581 1582 if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL) 1583 return -1; 1584 1585 if (queue_id >= VHOST_MAX_VRING) 1586 return -1; 1587 1588 vq = dev->virtqueue[queue_id]; 1589 if (!vq) 1590 return -1; 1591 1592 if (!vq_is_packed(dev)) 1593 return -1; 1594 1595 inflight_info = vq->inflight_packed; 1596 if (!inflight_info) 1597 return -1; 1598 1599 *last_avail_idx = (inflight_info->old_used_wrap_counter << 15) | 1600 inflight_info->old_used_idx; 1601 *last_used_idx = *last_avail_idx; 1602 1603 return 0; 1604 } 1605 1606 int 1607 rte_vhost_extern_callback_register(int vid, 1608 struct rte_vhost_user_extern_ops const * const ops, void *ctx) 1609 { 1610 struct virtio_net *dev = get_device(vid); 1611 1612 if (dev == NULL || ops == NULL) 1613 return -1; 1614 1615 dev->extern_ops = *ops; 1616 dev->extern_data = ctx; 1617 return 0; 1618 } 1619 1620 static __rte_always_inline int 1621 async_channel_register(int vid, uint16_t queue_id) 1622 { 1623 struct virtio_net *dev = get_device(vid); 1624 struct vhost_virtqueue *vq = dev->virtqueue[queue_id]; 1625 struct vhost_async *async; 1626 int node = vq->numa_node; 1627 1628 if (unlikely(vq->async)) { 1629 VHOST_LOG_CONFIG(ERR, 1630 "(%s) async register failed: already registered (qid: %d)\n", 1631 dev->ifname, queue_id); 1632 return -1; 1633 } 1634 1635 async = rte_zmalloc_socket(NULL, sizeof(struct vhost_async), 0, node); 1636 if (!async) { 1637 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async metadata (qid: %d)\n", 1638 dev->ifname, queue_id); 1639 return -1; 1640 } 1641 1642 async->pkts_info = rte_malloc_socket(NULL, vq->size * sizeof(struct async_inflight_info), 1643 RTE_CACHE_LINE_SIZE, node); 1644 if (!async->pkts_info) { 1645 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async_pkts_info (qid: %d)\n", 1646 dev->ifname, queue_id); 1647 goto out_free_async; 1648 } 1649 1650 async->pkts_cmpl_flag = rte_zmalloc_socket(NULL, vq->size * sizeof(bool), 1651 RTE_CACHE_LINE_SIZE, node); 1652 if (!async->pkts_cmpl_flag) { 1653 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async pkts_cmpl_flag (qid: %d)\n", 1654 dev->ifname, queue_id); 1655 goto out_free_async; 1656 } 1657 1658 if (vq_is_packed(dev)) { 1659 async->buffers_packed = rte_malloc_socket(NULL, 1660 vq->size * sizeof(struct vring_used_elem_packed), 1661 RTE_CACHE_LINE_SIZE, node); 1662 if (!async->buffers_packed) { 1663 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async buffers (qid: %d)\n", 1664 dev->ifname, queue_id); 1665 goto out_free_inflight; 1666 } 1667 } else { 1668 async->descs_split = rte_malloc_socket(NULL, 1669 vq->size * sizeof(struct vring_used_elem), 1670 RTE_CACHE_LINE_SIZE, node); 1671 if (!async->descs_split) { 1672 VHOST_LOG_CONFIG(ERR, "(%s) failed to allocate async descs (qid: %d)\n", 1673 dev->ifname, queue_id); 1674 goto out_free_inflight; 1675 } 1676 } 1677 1678 vq->async = async; 1679 1680 return 0; 1681 out_free_inflight: 1682 rte_free(async->pkts_info); 1683 out_free_async: 1684 rte_free(async); 1685 1686 return -1; 1687 } 1688 1689 int 1690 rte_vhost_async_channel_register(int vid, uint16_t queue_id) 1691 { 1692 struct vhost_virtqueue *vq; 1693 struct virtio_net *dev = get_device(vid); 1694 int ret; 1695 1696 if (dev == NULL) 1697 return -1; 1698 1699 if (queue_id >= VHOST_MAX_VRING) 1700 return -1; 1701 1702 vq = dev->virtqueue[queue_id]; 1703 1704 if (unlikely(vq == NULL || !dev->async_copy)) 1705 return -1; 1706 1707 rte_spinlock_lock(&vq->access_lock); 1708 ret = async_channel_register(vid, queue_id); 1709 rte_spinlock_unlock(&vq->access_lock); 1710 1711 return ret; 1712 } 1713 1714 int 1715 rte_vhost_async_channel_register_thread_unsafe(int vid, uint16_t queue_id) 1716 { 1717 struct vhost_virtqueue *vq; 1718 struct virtio_net *dev = get_device(vid); 1719 1720 if (dev == NULL) 1721 return -1; 1722 1723 if (queue_id >= VHOST_MAX_VRING) 1724 return -1; 1725 1726 vq = dev->virtqueue[queue_id]; 1727 1728 if (unlikely(vq == NULL || !dev->async_copy)) 1729 return -1; 1730 1731 return async_channel_register(vid, queue_id); 1732 } 1733 1734 int 1735 rte_vhost_async_channel_unregister(int vid, uint16_t queue_id) 1736 { 1737 struct vhost_virtqueue *vq; 1738 struct virtio_net *dev = get_device(vid); 1739 int ret = -1; 1740 1741 if (dev == NULL) 1742 return ret; 1743 1744 if (queue_id >= VHOST_MAX_VRING) 1745 return ret; 1746 1747 vq = dev->virtqueue[queue_id]; 1748 1749 if (vq == NULL) 1750 return ret; 1751 1752 ret = 0; 1753 1754 if (!vq->async) 1755 return ret; 1756 1757 if (!rte_spinlock_trylock(&vq->access_lock)) { 1758 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel, virtqueue busy.\n", 1759 dev->ifname); 1760 return -1; 1761 } 1762 1763 if (vq->async->pkts_inflight_n) { 1764 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel.\n", dev->ifname); 1765 VHOST_LOG_CONFIG(ERR, "(%s) inflight packets must be completed before unregistration.\n", 1766 dev->ifname); 1767 ret = -1; 1768 goto out; 1769 } 1770 1771 vhost_free_async_mem(vq); 1772 out: 1773 rte_spinlock_unlock(&vq->access_lock); 1774 1775 return ret; 1776 } 1777 1778 int 1779 rte_vhost_async_channel_unregister_thread_unsafe(int vid, uint16_t queue_id) 1780 { 1781 struct vhost_virtqueue *vq; 1782 struct virtio_net *dev = get_device(vid); 1783 1784 if (dev == NULL) 1785 return -1; 1786 1787 if (queue_id >= VHOST_MAX_VRING) 1788 return -1; 1789 1790 vq = dev->virtqueue[queue_id]; 1791 1792 if (vq == NULL) 1793 return -1; 1794 1795 if (!vq->async) 1796 return 0; 1797 1798 if (vq->async->pkts_inflight_n) { 1799 VHOST_LOG_CONFIG(ERR, "(%s) failed to unregister async channel.\n", dev->ifname); 1800 VHOST_LOG_CONFIG(ERR, "(%s) inflight packets must be completed before unregistration.\n", 1801 dev->ifname); 1802 return -1; 1803 } 1804 1805 vhost_free_async_mem(vq); 1806 1807 return 0; 1808 } 1809 1810 int 1811 rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id) 1812 { 1813 struct rte_dma_info info; 1814 void *pkts_cmpl_flag_addr; 1815 uint16_t max_desc; 1816 1817 if (!rte_dma_is_valid(dma_id)) { 1818 VHOST_LOG_CONFIG(ERR, "DMA %d is not found.\n", dma_id); 1819 return -1; 1820 } 1821 1822 rte_dma_info_get(dma_id, &info); 1823 if (vchan_id >= info.max_vchans) { 1824 VHOST_LOG_CONFIG(ERR, "Invalid DMA %d vChannel %u.\n", dma_id, vchan_id); 1825 return -1; 1826 } 1827 1828 if (!dma_copy_track[dma_id].vchans) { 1829 struct async_dma_vchan_info *vchans; 1830 1831 vchans = rte_zmalloc(NULL, sizeof(struct async_dma_vchan_info) * info.max_vchans, 1832 RTE_CACHE_LINE_SIZE); 1833 if (vchans == NULL) { 1834 VHOST_LOG_CONFIG(ERR, "Failed to allocate vchans for DMA %d vChannel %u.\n", 1835 dma_id, vchan_id); 1836 return -1; 1837 } 1838 1839 dma_copy_track[dma_id].vchans = vchans; 1840 } 1841 1842 if (dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr) { 1843 VHOST_LOG_CONFIG(INFO, "DMA %d vChannel %u already registered.\n", dma_id, 1844 vchan_id); 1845 return 0; 1846 } 1847 1848 max_desc = info.max_desc; 1849 if (!rte_is_power_of_2(max_desc)) 1850 max_desc = rte_align32pow2(max_desc); 1851 1852 pkts_cmpl_flag_addr = rte_zmalloc(NULL, sizeof(bool *) * max_desc, RTE_CACHE_LINE_SIZE); 1853 if (!pkts_cmpl_flag_addr) { 1854 VHOST_LOG_CONFIG(ERR, "Failed to allocate pkts_cmpl_flag_addr for DMA %d " 1855 "vChannel %u.\n", dma_id, vchan_id); 1856 1857 if (dma_copy_track[dma_id].nr_vchans == 0) { 1858 rte_free(dma_copy_track[dma_id].vchans); 1859 dma_copy_track[dma_id].vchans = NULL; 1860 } 1861 return -1; 1862 } 1863 1864 dma_copy_track[dma_id].vchans[vchan_id].pkts_cmpl_flag_addr = pkts_cmpl_flag_addr; 1865 dma_copy_track[dma_id].vchans[vchan_id].ring_size = max_desc; 1866 dma_copy_track[dma_id].vchans[vchan_id].ring_mask = max_desc - 1; 1867 dma_copy_track[dma_id].nr_vchans++; 1868 1869 return 0; 1870 } 1871 1872 int 1873 rte_vhost_async_get_inflight(int vid, uint16_t queue_id) 1874 { 1875 struct vhost_virtqueue *vq; 1876 struct virtio_net *dev = get_device(vid); 1877 int ret = -1; 1878 1879 if (dev == NULL) 1880 return ret; 1881 1882 if (queue_id >= VHOST_MAX_VRING) 1883 return ret; 1884 1885 vq = dev->virtqueue[queue_id]; 1886 1887 if (vq == NULL) 1888 return ret; 1889 1890 if (!vq->async) 1891 return ret; 1892 1893 if (!rte_spinlock_trylock(&vq->access_lock)) { 1894 VHOST_LOG_CONFIG(DEBUG, 1895 "(%s) failed to check in-flight packets. virtqueue busy.\n", 1896 dev->ifname); 1897 return ret; 1898 } 1899 1900 ret = vq->async->pkts_inflight_n; 1901 rte_spinlock_unlock(&vq->access_lock); 1902 1903 return ret; 1904 } 1905 1906 int 1907 rte_vhost_get_monitor_addr(int vid, uint16_t queue_id, 1908 struct rte_vhost_power_monitor_cond *pmc) 1909 { 1910 struct virtio_net *dev = get_device(vid); 1911 struct vhost_virtqueue *vq; 1912 1913 if (dev == NULL) 1914 return -1; 1915 if (queue_id >= VHOST_MAX_VRING) 1916 return -1; 1917 1918 vq = dev->virtqueue[queue_id]; 1919 if (vq == NULL) 1920 return -1; 1921 1922 if (vq_is_packed(dev)) { 1923 struct vring_packed_desc *desc; 1924 desc = vq->desc_packed; 1925 pmc->addr = &desc[vq->last_avail_idx].flags; 1926 if (vq->avail_wrap_counter) 1927 pmc->val = VRING_DESC_F_AVAIL; 1928 else 1929 pmc->val = VRING_DESC_F_USED; 1930 pmc->mask = VRING_DESC_F_AVAIL | VRING_DESC_F_USED; 1931 pmc->size = sizeof(desc[vq->last_avail_idx].flags); 1932 pmc->match = 1; 1933 } else { 1934 pmc->addr = &vq->avail->idx; 1935 pmc->val = vq->last_avail_idx & (vq->size - 1); 1936 pmc->mask = vq->size - 1; 1937 pmc->size = sizeof(vq->avail->idx); 1938 pmc->match = 0; 1939 } 1940 1941 return 0; 1942 } 1943 1944 RTE_LOG_REGISTER_SUFFIX(vhost_config_log_level, config, INFO); 1945 RTE_LOG_REGISTER_SUFFIX(vhost_data_log_level, data, WARNING); 1946