1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <fcntl.h> 9 #include <string.h> 10 #include <errno.h> 11 #include <sys/mman.h> 12 #include <unistd.h> 13 #include <sys/eventfd.h> 14 #include <sys/types.h> 15 #include <sys/stat.h> 16 17 #include <rte_alarm.h> 18 #include <rte_string_fns.h> 19 #include <rte_eal_memconfig.h> 20 #include <rte_malloc.h> 21 #include <rte_io.h> 22 23 #include "vhost.h" 24 #include "virtio.h" 25 #include "virtio_user_dev.h" 26 #include "../virtio_ethdev.h" 27 28 #define VIRTIO_USER_MEM_EVENT_CLB_NAME "virtio_user_mem_event_clb" 29 30 const char * const virtio_user_backend_strings[] = { 31 [VIRTIO_USER_BACKEND_UNKNOWN] = "VIRTIO_USER_BACKEND_UNKNOWN", 32 [VIRTIO_USER_BACKEND_VHOST_USER] = "VHOST_USER", 33 [VIRTIO_USER_BACKEND_VHOST_KERNEL] = "VHOST_NET", 34 [VIRTIO_USER_BACKEND_VHOST_VDPA] = "VHOST_VDPA", 35 }; 36 37 static int 38 virtio_user_uninit_notify_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 39 { 40 if (dev->kickfds[queue_sel] >= 0) { 41 close(dev->kickfds[queue_sel]); 42 dev->kickfds[queue_sel] = -1; 43 } 44 45 if (dev->callfds[queue_sel] >= 0) { 46 close(dev->callfds[queue_sel]); 47 dev->callfds[queue_sel] = -1; 48 } 49 50 return 0; 51 } 52 53 static int 54 virtio_user_init_notify_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 55 { 56 /* May use invalid flag, but some backend uses kickfd and 57 * callfd as criteria to judge if dev is alive. so finally we 58 * use real event_fd. 59 */ 60 dev->callfds[queue_sel] = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 61 if (dev->callfds[queue_sel] < 0) { 62 PMD_DRV_LOG(ERR, "(%s) Failed to setup callfd for queue %u: %s", 63 dev->path, queue_sel, strerror(errno)); 64 return -1; 65 } 66 dev->kickfds[queue_sel] = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 67 if (dev->kickfds[queue_sel] < 0) { 68 PMD_DRV_LOG(ERR, "(%s) Failed to setup kickfd for queue %u: %s", 69 dev->path, queue_sel, strerror(errno)); 70 return -1; 71 } 72 73 return 0; 74 } 75 76 static int 77 virtio_user_destroy_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 78 { 79 struct vhost_vring_state state; 80 int ret; 81 82 state.index = queue_sel; 83 ret = dev->ops->get_vring_base(dev, &state); 84 if (ret < 0) { 85 PMD_DRV_LOG(ERR, "(%s) Failed to destroy queue %u", dev->path, queue_sel); 86 return -1; 87 } 88 89 return 0; 90 } 91 92 static int 93 virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 94 { 95 /* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come 96 * firstly because vhost depends on this msg to allocate virtqueue 97 * pair. 98 */ 99 struct vhost_vring_file file; 100 int ret; 101 102 file.index = queue_sel; 103 file.fd = dev->callfds[queue_sel]; 104 ret = dev->ops->set_vring_call(dev, &file); 105 if (ret < 0) { 106 PMD_INIT_LOG(ERR, "(%s) Failed to create queue %u", dev->path, queue_sel); 107 return -1; 108 } 109 110 return 0; 111 } 112 113 static int 114 virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 115 { 116 int ret; 117 struct vhost_vring_file file; 118 struct vhost_vring_state state; 119 struct vring *vring = &dev->vrings.split[queue_sel]; 120 struct vring_packed *pq_vring = &dev->vrings.packed[queue_sel]; 121 uint64_t desc_addr, avail_addr, used_addr; 122 struct vhost_vring_addr addr = { 123 .index = queue_sel, 124 .log_guest_addr = 0, 125 .flags = 0, /* disable log */ 126 }; 127 128 if (queue_sel == dev->max_queue_pairs * 2) { 129 if (!dev->scvq) { 130 PMD_INIT_LOG(ERR, "(%s) Shadow control queue expected but missing", 131 dev->path); 132 goto err; 133 } 134 135 /* Use shadow control queue information */ 136 vring = &dev->scvq->vq_split.ring; 137 pq_vring = &dev->scvq->vq_packed.ring; 138 } 139 140 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) { 141 desc_addr = pq_vring->desc_iova; 142 avail_addr = desc_addr + pq_vring->num * sizeof(struct vring_packed_desc); 143 used_addr = RTE_ALIGN_CEIL(avail_addr + sizeof(struct vring_packed_desc_event), 144 VIRTIO_VRING_ALIGN); 145 146 addr.desc_user_addr = desc_addr; 147 addr.avail_user_addr = avail_addr; 148 addr.used_user_addr = used_addr; 149 } else { 150 desc_addr = vring->desc_iova; 151 avail_addr = desc_addr + vring->num * sizeof(struct vring_desc); 152 used_addr = RTE_ALIGN_CEIL((uintptr_t)(&vring->avail->ring[vring->num]), 153 VIRTIO_VRING_ALIGN); 154 155 addr.desc_user_addr = desc_addr; 156 addr.avail_user_addr = avail_addr; 157 addr.used_user_addr = used_addr; 158 } 159 160 state.index = queue_sel; 161 state.num = vring->num; 162 ret = dev->ops->set_vring_num(dev, &state); 163 if (ret < 0) 164 goto err; 165 166 state.index = queue_sel; 167 state.num = 0; /* no reservation */ 168 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) 169 state.num |= (1 << 15); 170 ret = dev->ops->set_vring_base(dev, &state); 171 if (ret < 0) 172 goto err; 173 174 ret = dev->ops->set_vring_addr(dev, &addr); 175 if (ret < 0) 176 goto err; 177 178 /* Of all per virtqueue MSGs, make sure VHOST_USER_SET_VRING_KICK comes 179 * lastly because vhost depends on this msg to judge if 180 * virtio is ready. 181 */ 182 file.index = queue_sel; 183 file.fd = dev->kickfds[queue_sel]; 184 ret = dev->ops->set_vring_kick(dev, &file); 185 if (ret < 0) 186 goto err; 187 188 return 0; 189 err: 190 PMD_INIT_LOG(ERR, "(%s) Failed to kick queue %u", dev->path, queue_sel); 191 192 return -1; 193 } 194 195 static int 196 virtio_user_foreach_queue(struct virtio_user_dev *dev, 197 int (*fn)(struct virtio_user_dev *, uint32_t)) 198 { 199 uint32_t i, nr_vq; 200 201 nr_vq = dev->max_queue_pairs * 2; 202 if (dev->hw_cvq) 203 nr_vq++; 204 205 for (i = 0; i < nr_vq; i++) 206 if (fn(dev, i) < 0) 207 return -1; 208 209 return 0; 210 } 211 212 int 213 virtio_user_dev_set_features(struct virtio_user_dev *dev) 214 { 215 uint64_t features; 216 int ret = -1; 217 218 pthread_mutex_lock(&dev->mutex); 219 220 /* Step 0: tell vhost to create queues */ 221 if (virtio_user_foreach_queue(dev, virtio_user_create_queue) < 0) 222 goto error; 223 224 features = dev->features; 225 226 /* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init */ 227 features &= ~(1ull << VIRTIO_NET_F_MAC); 228 /* Strip VIRTIO_NET_F_CTRL_VQ if the devices does not really support control VQ */ 229 if (!dev->hw_cvq) 230 features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ); 231 features &= ~(1ull << VIRTIO_NET_F_STATUS); 232 ret = dev->ops->set_features(dev, features); 233 if (ret < 0) 234 goto error; 235 PMD_DRV_LOG(INFO, "(%s) set features: 0x%" PRIx64, dev->path, features); 236 error: 237 pthread_mutex_unlock(&dev->mutex); 238 239 return ret; 240 } 241 242 int 243 virtio_user_start_device(struct virtio_user_dev *dev) 244 { 245 int ret; 246 247 /* 248 * XXX workaround! 249 * 250 * We need to make sure that the locks will be 251 * taken in the correct order to avoid deadlocks. 252 * 253 * Before releasing this lock, this thread should 254 * not trigger any memory hotplug events. 255 * 256 * This is a temporary workaround, and should be 257 * replaced when we get proper supports from the 258 * memory subsystem in the future. 259 */ 260 rte_mcfg_mem_read_lock(); 261 pthread_mutex_lock(&dev->mutex); 262 263 /* Step 2: share memory regions */ 264 ret = dev->ops->set_memory_table(dev); 265 if (ret < 0) 266 goto error; 267 268 /* Step 3: kick queues */ 269 ret = virtio_user_foreach_queue(dev, virtio_user_kick_queue); 270 if (ret < 0) 271 goto error; 272 273 /* Step 4: enable queues 274 * we enable the 1st queue pair by default. 275 */ 276 ret = dev->ops->enable_qp(dev, 0, 1); 277 if (ret < 0) 278 goto error; 279 280 if (dev->scvq) { 281 ret = dev->ops->cvq_enable(dev, 1); 282 if (ret < 0) 283 goto error; 284 } 285 286 dev->started = true; 287 288 pthread_mutex_unlock(&dev->mutex); 289 rte_mcfg_mem_read_unlock(); 290 291 return 0; 292 error: 293 pthread_mutex_unlock(&dev->mutex); 294 rte_mcfg_mem_read_unlock(); 295 296 PMD_INIT_LOG(ERR, "(%s) Failed to start device", dev->path); 297 298 /* TODO: free resource here or caller to check */ 299 return -1; 300 } 301 302 int virtio_user_stop_device(struct virtio_user_dev *dev) 303 { 304 uint32_t i; 305 int ret; 306 307 pthread_mutex_lock(&dev->mutex); 308 if (!dev->started) 309 goto out; 310 311 for (i = 0; i < dev->max_queue_pairs; ++i) { 312 ret = dev->ops->enable_qp(dev, i, 0); 313 if (ret < 0) 314 goto err; 315 } 316 317 if (dev->scvq) { 318 ret = dev->ops->cvq_enable(dev, 0); 319 if (ret < 0) 320 goto err; 321 } 322 323 /* Stop the backend. */ 324 if (virtio_user_foreach_queue(dev, virtio_user_destroy_queue) < 0) 325 goto err; 326 327 dev->started = false; 328 329 out: 330 pthread_mutex_unlock(&dev->mutex); 331 332 return 0; 333 err: 334 pthread_mutex_unlock(&dev->mutex); 335 336 PMD_INIT_LOG(ERR, "(%s) Failed to stop device", dev->path); 337 338 return -1; 339 } 340 341 static int 342 virtio_user_dev_init_max_queue_pairs(struct virtio_user_dev *dev, uint32_t user_max_qp) 343 { 344 int ret; 345 346 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MQ))) { 347 dev->max_queue_pairs = 1; 348 return 0; 349 } 350 351 if (!dev->ops->get_config) { 352 dev->max_queue_pairs = user_max_qp; 353 return 0; 354 } 355 356 ret = dev->ops->get_config(dev, (uint8_t *)&dev->max_queue_pairs, 357 offsetof(struct virtio_net_config, max_virtqueue_pairs), 358 sizeof(uint16_t)); 359 if (ret) { 360 /* 361 * We need to know the max queue pair from the device so that 362 * the control queue gets the right index. 363 */ 364 dev->max_queue_pairs = 1; 365 PMD_DRV_LOG(ERR, "(%s) Failed to get max queue pairs from device", dev->path); 366 367 return ret; 368 } 369 370 return 0; 371 } 372 373 int 374 virtio_user_dev_get_rss_config(struct virtio_user_dev *dev, void *dst, size_t offset, int length) 375 { 376 int ret = 0; 377 378 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_RSS))) 379 return -ENOTSUP; 380 381 if (!dev->ops->get_config) 382 return -ENOTSUP; 383 384 ret = dev->ops->get_config(dev, dst, offset, length); 385 if (ret) 386 PMD_DRV_LOG(ERR, "(%s) Failed to get rss config in device", dev->path); 387 388 return ret; 389 } 390 391 int 392 virtio_user_dev_set_mac(struct virtio_user_dev *dev) 393 { 394 int ret = 0; 395 396 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MAC))) 397 return -ENOTSUP; 398 399 if (!dev->ops->set_config) 400 return -ENOTSUP; 401 402 ret = dev->ops->set_config(dev, dev->mac_addr, 403 offsetof(struct virtio_net_config, mac), 404 RTE_ETHER_ADDR_LEN); 405 if (ret) 406 PMD_DRV_LOG(ERR, "(%s) Failed to set MAC address in device", dev->path); 407 408 return ret; 409 } 410 411 int 412 virtio_user_dev_get_mac(struct virtio_user_dev *dev) 413 { 414 int ret = 0; 415 416 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MAC))) 417 return -ENOTSUP; 418 419 if (!dev->ops->get_config) 420 return -ENOTSUP; 421 422 ret = dev->ops->get_config(dev, dev->mac_addr, 423 offsetof(struct virtio_net_config, mac), 424 RTE_ETHER_ADDR_LEN); 425 if (ret) 426 PMD_DRV_LOG(ERR, "(%s) Failed to get MAC address from device", dev->path); 427 428 return ret; 429 } 430 431 static void 432 virtio_user_dev_init_mac(struct virtio_user_dev *dev, const char *mac) 433 { 434 struct rte_ether_addr cmdline_mac; 435 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 436 int ret; 437 438 if (mac && rte_ether_unformat_addr(mac, &cmdline_mac) == 0) { 439 /* 440 * MAC address was passed from command-line, try to store 441 * it in the device if it supports it. Otherwise try to use 442 * the device one. 443 */ 444 memcpy(dev->mac_addr, &cmdline_mac, RTE_ETHER_ADDR_LEN); 445 dev->mac_specified = 1; 446 447 /* Setting MAC may fail, continue to get the device one in this case */ 448 virtio_user_dev_set_mac(dev); 449 ret = virtio_user_dev_get_mac(dev); 450 if (ret == -ENOTSUP) 451 goto out; 452 453 if (memcmp(&cmdline_mac, dev->mac_addr, RTE_ETHER_ADDR_LEN)) 454 PMD_DRV_LOG(INFO, "(%s) Device MAC update failed", dev->path); 455 } else { 456 ret = virtio_user_dev_get_mac(dev); 457 if (ret) { 458 PMD_DRV_LOG(ERR, "(%s) No valid MAC in devargs or device, use random", 459 dev->path); 460 return; 461 } 462 463 dev->mac_specified = 1; 464 } 465 out: 466 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, 467 (struct rte_ether_addr *)dev->mac_addr); 468 PMD_DRV_LOG(INFO, "(%s) MAC %s specified", dev->path, buf); 469 } 470 471 static int 472 virtio_user_dev_init_notify(struct virtio_user_dev *dev) 473 { 474 475 if (virtio_user_foreach_queue(dev, virtio_user_init_notify_queue) < 0) 476 goto err; 477 478 if (dev->device_features & (1ULL << VIRTIO_F_NOTIFICATION_DATA)) 479 if (dev->ops->map_notification_area && 480 dev->ops->map_notification_area(dev)) 481 goto err; 482 483 return 0; 484 err: 485 virtio_user_foreach_queue(dev, virtio_user_uninit_notify_queue); 486 487 return -1; 488 } 489 490 static void 491 virtio_user_dev_uninit_notify(struct virtio_user_dev *dev) 492 { 493 virtio_user_foreach_queue(dev, virtio_user_uninit_notify_queue); 494 495 if (dev->ops->unmap_notification_area && dev->notify_area) 496 dev->ops->unmap_notification_area(dev); 497 } 498 499 static int 500 virtio_user_fill_intr_handle(struct virtio_user_dev *dev) 501 { 502 uint32_t i; 503 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 504 505 if (eth_dev->intr_handle == NULL) { 506 eth_dev->intr_handle = 507 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE); 508 if (eth_dev->intr_handle == NULL) { 509 PMD_DRV_LOG(ERR, "(%s) failed to allocate intr_handle", dev->path); 510 return -1; 511 } 512 } 513 514 for (i = 0; i < dev->max_queue_pairs; ++i) { 515 if (rte_intr_efds_index_set(eth_dev->intr_handle, i, 516 dev->callfds[2 * i + VTNET_SQ_RQ_QUEUE_IDX])) 517 return -rte_errno; 518 } 519 520 if (rte_intr_nb_efd_set(eth_dev->intr_handle, dev->max_queue_pairs)) 521 return -rte_errno; 522 523 if (rte_intr_max_intr_set(eth_dev->intr_handle, 524 dev->max_queue_pairs + 1)) 525 return -rte_errno; 526 527 if (rte_intr_type_set(eth_dev->intr_handle, RTE_INTR_HANDLE_VDEV)) 528 return -rte_errno; 529 530 /* For virtio vdev, no need to read counter for clean */ 531 if (rte_intr_efd_counter_size_set(eth_dev->intr_handle, 0)) 532 return -rte_errno; 533 534 if (rte_intr_fd_set(eth_dev->intr_handle, dev->ops->get_intr_fd(dev))) 535 return -rte_errno; 536 537 return 0; 538 } 539 540 static void 541 virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused, 542 const void *addr, 543 size_t len __rte_unused, 544 void *arg) 545 { 546 struct virtio_user_dev *dev = arg; 547 struct rte_memseg_list *msl; 548 uint16_t i; 549 int ret = 0; 550 551 /* ignore externally allocated memory */ 552 msl = rte_mem_virt2memseg_list(addr); 553 if (msl->external) 554 return; 555 556 pthread_mutex_lock(&dev->mutex); 557 558 if (dev->started == false) 559 goto exit; 560 561 /* Step 1: pause the active queues */ 562 for (i = 0; i < dev->queue_pairs; i++) { 563 ret = dev->ops->enable_qp(dev, i, 0); 564 if (ret < 0) 565 goto exit; 566 } 567 568 /* Step 2: update memory regions */ 569 ret = dev->ops->set_memory_table(dev); 570 if (ret < 0) 571 goto exit; 572 573 /* Step 3: resume the active queues */ 574 for (i = 0; i < dev->queue_pairs; i++) { 575 ret = dev->ops->enable_qp(dev, i, 1); 576 if (ret < 0) 577 goto exit; 578 } 579 580 exit: 581 pthread_mutex_unlock(&dev->mutex); 582 583 if (ret < 0) 584 PMD_DRV_LOG(ERR, "(%s) Failed to update memory table", dev->path); 585 } 586 587 static int 588 virtio_user_dev_setup(struct virtio_user_dev *dev) 589 { 590 if (dev->is_server) { 591 if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) { 592 PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!"); 593 return -1; 594 } 595 } 596 597 switch (dev->backend_type) { 598 case VIRTIO_USER_BACKEND_VHOST_USER: 599 dev->ops = &virtio_ops_user; 600 break; 601 case VIRTIO_USER_BACKEND_VHOST_KERNEL: 602 dev->ops = &virtio_ops_kernel; 603 break; 604 case VIRTIO_USER_BACKEND_VHOST_VDPA: 605 dev->ops = &virtio_ops_vdpa; 606 break; 607 default: 608 PMD_DRV_LOG(ERR, "(%s) Unknown backend type", dev->path); 609 return -1; 610 } 611 612 if (dev->ops->setup(dev) < 0) { 613 PMD_INIT_LOG(ERR, "(%s) Failed to setup backend", dev->path); 614 return -1; 615 } 616 617 return 0; 618 } 619 620 static int 621 virtio_user_alloc_vrings(struct virtio_user_dev *dev) 622 { 623 int i, size, nr_vrings; 624 bool packed_ring = !!(dev->device_features & (1ull << VIRTIO_F_RING_PACKED)); 625 626 nr_vrings = dev->max_queue_pairs * 2; 627 if (dev->frontend_features & (1ull << VIRTIO_NET_F_CTRL_VQ)) 628 nr_vrings++; 629 630 dev->callfds = rte_zmalloc("virtio_user_dev", nr_vrings * sizeof(*dev->callfds), 0); 631 if (!dev->callfds) { 632 PMD_INIT_LOG(ERR, "(%s) Failed to alloc callfds", dev->path); 633 return -1; 634 } 635 636 dev->kickfds = rte_zmalloc("virtio_user_dev", nr_vrings * sizeof(*dev->kickfds), 0); 637 if (!dev->kickfds) { 638 PMD_INIT_LOG(ERR, "(%s) Failed to alloc kickfds", dev->path); 639 goto free_callfds; 640 } 641 642 for (i = 0; i < nr_vrings; i++) { 643 dev->callfds[i] = -1; 644 dev->kickfds[i] = -1; 645 } 646 647 if (packed_ring) 648 size = sizeof(*dev->vrings.packed); 649 else 650 size = sizeof(*dev->vrings.split); 651 dev->vrings.ptr = rte_zmalloc("virtio_user_dev", nr_vrings * size, 0); 652 if (!dev->vrings.ptr) { 653 PMD_INIT_LOG(ERR, "(%s) Failed to alloc vrings metadata", dev->path); 654 goto free_kickfds; 655 } 656 657 if (packed_ring) { 658 dev->packed_queues = rte_zmalloc("virtio_user_dev", 659 nr_vrings * sizeof(*dev->packed_queues), 0); 660 if (!dev->packed_queues) { 661 PMD_INIT_LOG(ERR, "(%s) Failed to alloc packed queues metadata", 662 dev->path); 663 goto free_vrings; 664 } 665 } 666 667 dev->qp_enabled = rte_zmalloc("virtio_user_dev", 668 dev->max_queue_pairs * sizeof(*dev->qp_enabled), 0); 669 if (!dev->qp_enabled) { 670 PMD_INIT_LOG(ERR, "(%s) Failed to alloc QP enable states", dev->path); 671 goto free_packed_queues; 672 } 673 674 return 0; 675 676 free_packed_queues: 677 rte_free(dev->packed_queues); 678 dev->packed_queues = NULL; 679 free_vrings: 680 rte_free(dev->vrings.ptr); 681 dev->vrings.ptr = NULL; 682 free_kickfds: 683 rte_free(dev->kickfds); 684 dev->kickfds = NULL; 685 free_callfds: 686 rte_free(dev->callfds); 687 dev->callfds = NULL; 688 689 return -1; 690 } 691 692 static void 693 virtio_user_free_vrings(struct virtio_user_dev *dev) 694 { 695 rte_free(dev->qp_enabled); 696 dev->qp_enabled = NULL; 697 rte_free(dev->packed_queues); 698 dev->packed_queues = NULL; 699 rte_free(dev->vrings.ptr); 700 dev->vrings.ptr = NULL; 701 rte_free(dev->kickfds); 702 dev->kickfds = NULL; 703 rte_free(dev->callfds); 704 dev->callfds = NULL; 705 } 706 707 /* Use below macro to filter features from vhost backend */ 708 #define VIRTIO_USER_SUPPORTED_FEATURES \ 709 (1ULL << VIRTIO_NET_F_MAC | \ 710 1ULL << VIRTIO_NET_F_STATUS | \ 711 1ULL << VIRTIO_NET_F_MQ | \ 712 1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR | \ 713 1ULL << VIRTIO_NET_F_CTRL_VQ | \ 714 1ULL << VIRTIO_NET_F_CTRL_RX | \ 715 1ULL << VIRTIO_NET_F_CTRL_VLAN | \ 716 1ULL << VIRTIO_NET_F_CSUM | \ 717 1ULL << VIRTIO_NET_F_HOST_TSO4 | \ 718 1ULL << VIRTIO_NET_F_HOST_TSO6 | \ 719 1ULL << VIRTIO_NET_F_MRG_RXBUF | \ 720 1ULL << VIRTIO_RING_F_INDIRECT_DESC | \ 721 1ULL << VIRTIO_NET_F_GUEST_CSUM | \ 722 1ULL << VIRTIO_NET_F_GUEST_TSO4 | \ 723 1ULL << VIRTIO_NET_F_GUEST_TSO6 | \ 724 1ULL << VIRTIO_F_IN_ORDER | \ 725 1ULL << VIRTIO_F_VERSION_1 | \ 726 1ULL << VIRTIO_F_RING_PACKED | \ 727 1ULL << VIRTIO_F_NOTIFICATION_DATA | \ 728 1ULL << VIRTIO_F_ORDER_PLATFORM | \ 729 1ULL << VIRTIO_NET_F_RSS) 730 731 int 732 virtio_user_dev_init(struct virtio_user_dev *dev, char *path, uint16_t queues, 733 int cq, int queue_size, const char *mac, char **ifname, 734 int server, int mrg_rxbuf, int in_order, int packed_vq, 735 enum virtio_user_backend_type backend_type) 736 { 737 uint64_t backend_features; 738 739 pthread_mutex_init(&dev->mutex, NULL); 740 strlcpy(dev->path, path, PATH_MAX); 741 742 dev->started = 0; 743 dev->queue_pairs = 1; /* mq disabled by default */ 744 dev->max_queue_pairs = queues; /* initialize to user requested value for kernel backend */ 745 dev->queue_size = queue_size; 746 dev->is_server = server; 747 dev->mac_specified = 0; 748 dev->frontend_features = 0; 749 dev->unsupported_features = 0; 750 dev->backend_type = backend_type; 751 dev->ifname = *ifname; 752 753 if (virtio_user_dev_setup(dev) < 0) { 754 PMD_INIT_LOG(ERR, "(%s) backend set up fails", dev->path); 755 return -1; 756 } 757 758 if (dev->ops->set_owner(dev) < 0) { 759 PMD_INIT_LOG(ERR, "(%s) Failed to set backend owner", dev->path); 760 goto destroy; 761 } 762 763 if (dev->ops->get_backend_features(&backend_features) < 0) { 764 PMD_INIT_LOG(ERR, "(%s) Failed to get backend features", dev->path); 765 goto destroy; 766 } 767 768 dev->unsupported_features = ~(VIRTIO_USER_SUPPORTED_FEATURES | backend_features); 769 770 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 771 PMD_INIT_LOG(ERR, "(%s) Failed to get device features", dev->path); 772 goto destroy; 773 } 774 775 virtio_user_dev_init_mac(dev, mac); 776 777 if (virtio_user_dev_init_max_queue_pairs(dev, queues)) 778 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); 779 780 if (dev->max_queue_pairs > 1 || dev->hw_cvq) 781 cq = 1; 782 783 if (!mrg_rxbuf) 784 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF); 785 786 if (!in_order) 787 dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER); 788 789 if (!packed_vq) 790 dev->unsupported_features |= (1ull << VIRTIO_F_RING_PACKED); 791 792 if (dev->mac_specified) 793 dev->frontend_features |= (1ull << VIRTIO_NET_F_MAC); 794 else 795 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC); 796 797 if (cq) { 798 /* Except for vDPA, the device does not really need to know 799 * anything about CQ, so if necessary, we just claim to support 800 * control queue. 801 */ 802 dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 803 } else { 804 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 805 /* Also disable features that depend on VIRTIO_NET_F_CTRL_VQ */ 806 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX); 807 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN); 808 dev->unsupported_features |= 809 (1ull << VIRTIO_NET_F_GUEST_ANNOUNCE); 810 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); 811 dev->unsupported_features |= 812 (1ull << VIRTIO_NET_F_CTRL_MAC_ADDR); 813 } 814 815 /* The backend will not report this feature, we add it explicitly */ 816 if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) 817 dev->frontend_features |= (1ull << VIRTIO_NET_F_STATUS); 818 819 dev->frontend_features &= ~dev->unsupported_features; 820 dev->device_features &= ~dev->unsupported_features; 821 822 if (virtio_user_alloc_vrings(dev) < 0) { 823 PMD_INIT_LOG(ERR, "(%s) Failed to allocate vring metadata", dev->path); 824 goto destroy; 825 } 826 827 if (virtio_user_dev_init_notify(dev) < 0) { 828 PMD_INIT_LOG(ERR, "(%s) Failed to init notifiers", dev->path); 829 goto free_vrings; 830 } 831 832 if (virtio_user_fill_intr_handle(dev) < 0) { 833 PMD_INIT_LOG(ERR, "(%s) Failed to init interrupt handler", dev->path); 834 goto notify_uninit; 835 } 836 837 if (rte_mem_event_callback_register(VIRTIO_USER_MEM_EVENT_CLB_NAME, 838 virtio_user_mem_event_cb, dev)) { 839 if (rte_errno != ENOTSUP) { 840 PMD_INIT_LOG(ERR, "(%s) Failed to register mem event callback", 841 dev->path); 842 goto notify_uninit; 843 } 844 } 845 846 *ifname = NULL; 847 return 0; 848 849 notify_uninit: 850 virtio_user_dev_uninit_notify(dev); 851 free_vrings: 852 virtio_user_free_vrings(dev); 853 destroy: 854 dev->ops->destroy(dev); 855 856 return -1; 857 } 858 859 void 860 virtio_user_dev_uninit(struct virtio_user_dev *dev) 861 { 862 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 863 864 rte_intr_instance_free(eth_dev->intr_handle); 865 eth_dev->intr_handle = NULL; 866 867 virtio_user_stop_device(dev); 868 869 rte_mem_event_callback_unregister(VIRTIO_USER_MEM_EVENT_CLB_NAME, dev); 870 871 virtio_user_dev_uninit_notify(dev); 872 873 virtio_user_free_vrings(dev); 874 875 free(dev->ifname); 876 877 if (dev->is_server) 878 unlink(dev->path); 879 880 dev->ops->destroy(dev); 881 } 882 883 static uint8_t 884 virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs) 885 { 886 uint16_t i; 887 uint8_t ret = 0; 888 889 if (q_pairs > dev->max_queue_pairs) { 890 PMD_INIT_LOG(ERR, "(%s) multi-q config %u, but only %u supported", 891 dev->path, q_pairs, dev->max_queue_pairs); 892 return -1; 893 } 894 895 for (i = 0; i < q_pairs; ++i) 896 ret |= dev->ops->enable_qp(dev, i, 1); 897 for (i = q_pairs; i < dev->max_queue_pairs; ++i) 898 ret |= dev->ops->enable_qp(dev, i, 0); 899 900 dev->queue_pairs = q_pairs; 901 902 return ret; 903 } 904 905 #define CVQ_MAX_DATA_DESCS 32 906 907 static inline void * 908 virtio_user_iova2virt(struct virtio_user_dev *dev, rte_iova_t iova) 909 { 910 if (rte_eal_iova_mode() == RTE_IOVA_VA || dev->hw.use_va) 911 return (void *)(uintptr_t)iova; 912 else 913 return rte_mem_iova2virt(iova); 914 } 915 916 static uint32_t 917 virtio_user_handle_ctrl_msg_split(struct virtio_user_dev *dev, struct vring *vring, 918 uint16_t idx_hdr) 919 { 920 struct virtio_net_ctrl_hdr *hdr; 921 virtio_net_ctrl_ack status = ~0; 922 uint16_t i, idx_data, idx_status; 923 uint32_t n_descs = 0; 924 int dlen[CVQ_MAX_DATA_DESCS], nb_dlen = 0; 925 926 /* locate desc for header, data, and status */ 927 idx_data = vring->desc[idx_hdr].next; 928 n_descs++; 929 930 i = idx_data; 931 while (vring->desc[i].flags == VRING_DESC_F_NEXT) { 932 dlen[nb_dlen++] = vring->desc[i].len; 933 i = vring->desc[i].next; 934 n_descs++; 935 } 936 937 /* locate desc for status */ 938 idx_status = i; 939 n_descs++; 940 941 hdr = virtio_user_iova2virt(dev, vring->desc[idx_hdr].addr); 942 if (hdr->class == VIRTIO_NET_CTRL_MQ && 943 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 944 uint16_t queues, *addr; 945 946 addr = virtio_user_iova2virt(dev, vring->desc[idx_data].addr); 947 queues = *addr; 948 status = virtio_user_handle_mq(dev, queues); 949 } else if (hdr->class == VIRTIO_NET_CTRL_MQ && hdr->cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) { 950 struct virtio_net_ctrl_rss *rss; 951 952 rss = virtio_user_iova2virt(dev, vring->desc[idx_data].addr); 953 status = virtio_user_handle_mq(dev, rss->max_tx_vq); 954 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 955 hdr->class == VIRTIO_NET_CTRL_MAC || 956 hdr->class == VIRTIO_NET_CTRL_VLAN) { 957 status = 0; 958 } 959 960 if (!status && dev->scvq) 961 status = virtio_send_command(&dev->scvq->cq, 962 (struct virtio_pmd_ctrl *)hdr, dlen, nb_dlen); 963 964 /* Update status */ 965 *(virtio_net_ctrl_ack *)virtio_user_iova2virt(dev, vring->desc[idx_status].addr) = status; 966 967 return n_descs; 968 } 969 970 static inline int 971 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) 972 { 973 uint16_t flags = rte_atomic_load_explicit(&desc->flags, rte_memory_order_acquire); 974 975 return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) && 976 wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED); 977 } 978 979 static uint32_t 980 virtio_user_handle_ctrl_msg_packed(struct virtio_user_dev *dev, 981 struct vring_packed *vring, 982 uint16_t idx_hdr) 983 { 984 struct virtio_net_ctrl_hdr *hdr; 985 virtio_net_ctrl_ack status = ~0; 986 uint16_t idx_data, idx_status; 987 /* initialize to one, header is first */ 988 uint32_t n_descs = 1; 989 int dlen[CVQ_MAX_DATA_DESCS], nb_dlen = 0; 990 991 /* locate desc for header, data, and status */ 992 idx_data = idx_hdr + 1; 993 if (idx_data >= dev->queue_size) 994 idx_data -= dev->queue_size; 995 996 n_descs++; 997 998 idx_status = idx_data; 999 while (vring->desc[idx_status].flags & VRING_DESC_F_NEXT) { 1000 dlen[nb_dlen++] = vring->desc[idx_status].len; 1001 idx_status++; 1002 if (idx_status >= dev->queue_size) 1003 idx_status -= dev->queue_size; 1004 n_descs++; 1005 } 1006 1007 hdr = virtio_user_iova2virt(dev, vring->desc[idx_hdr].addr); 1008 if (hdr->class == VIRTIO_NET_CTRL_MQ && 1009 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 1010 uint16_t queues, *addr; 1011 1012 addr = virtio_user_iova2virt(dev, vring->desc[idx_data].addr); 1013 queues = *addr; 1014 status = virtio_user_handle_mq(dev, queues); 1015 } else if (hdr->class == VIRTIO_NET_CTRL_MQ && hdr->cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) { 1016 struct virtio_net_ctrl_rss *rss; 1017 1018 rss = virtio_user_iova2virt(dev, vring->desc[idx_data].addr); 1019 status = virtio_user_handle_mq(dev, rss->max_tx_vq); 1020 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 1021 hdr->class == VIRTIO_NET_CTRL_MAC || 1022 hdr->class == VIRTIO_NET_CTRL_VLAN) { 1023 status = 0; 1024 } 1025 1026 if (!status && dev->scvq) 1027 status = virtio_send_command(&dev->scvq->cq, 1028 (struct virtio_pmd_ctrl *)hdr, dlen, nb_dlen); 1029 1030 /* Update status */ 1031 *(virtio_net_ctrl_ack *)virtio_user_iova2virt(dev, vring->desc[idx_status].addr) = status; 1032 1033 /* Update used descriptor */ 1034 vring->desc[idx_hdr].id = vring->desc[idx_status].id; 1035 vring->desc[idx_hdr].len = sizeof(status); 1036 1037 return n_descs; 1038 } 1039 1040 static void 1041 virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx) 1042 { 1043 struct virtio_user_queue *vq = &dev->packed_queues[queue_idx]; 1044 struct vring_packed *vring = &dev->vrings.packed[queue_idx]; 1045 uint16_t n_descs, flags; 1046 1047 /* Perform a load-acquire barrier in desc_is_avail to 1048 * enforce the ordering between desc flags and desc 1049 * content. 1050 */ 1051 while (desc_is_avail(&vring->desc[vq->used_idx], 1052 vq->used_wrap_counter)) { 1053 1054 n_descs = virtio_user_handle_ctrl_msg_packed(dev, vring, 1055 vq->used_idx); 1056 1057 flags = VRING_DESC_F_WRITE; 1058 if (vq->used_wrap_counter) 1059 flags |= VRING_PACKED_DESC_F_AVAIL_USED; 1060 1061 rte_atomic_store_explicit(&vring->desc[vq->used_idx].flags, flags, 1062 rte_memory_order_release); 1063 1064 vq->used_idx += n_descs; 1065 if (vq->used_idx >= dev->queue_size) { 1066 vq->used_idx -= dev->queue_size; 1067 vq->used_wrap_counter ^= 1; 1068 } 1069 } 1070 } 1071 1072 static void 1073 virtio_user_handle_cq_split(struct virtio_user_dev *dev, uint16_t queue_idx) 1074 { 1075 uint16_t avail_idx, desc_idx; 1076 struct vring_used_elem *uep; 1077 uint32_t n_descs; 1078 struct vring *vring = &dev->vrings.split[queue_idx]; 1079 1080 /* Consume avail ring, using used ring idx as first one */ 1081 while (rte_atomic_load_explicit(&vring->used->idx, rte_memory_order_relaxed) 1082 != vring->avail->idx) { 1083 avail_idx = rte_atomic_load_explicit(&vring->used->idx, rte_memory_order_relaxed) 1084 & (vring->num - 1); 1085 desc_idx = vring->avail->ring[avail_idx]; 1086 1087 n_descs = virtio_user_handle_ctrl_msg_split(dev, vring, desc_idx); 1088 1089 /* Update used ring */ 1090 uep = &vring->used->ring[avail_idx]; 1091 uep->id = desc_idx; 1092 uep->len = n_descs; 1093 1094 rte_atomic_fetch_add_explicit(&vring->used->idx, 1, rte_memory_order_relaxed); 1095 } 1096 } 1097 1098 void 1099 virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx) 1100 { 1101 if (virtio_with_packed_queue(&dev->hw)) 1102 virtio_user_handle_cq_packed(dev, queue_idx); 1103 else 1104 virtio_user_handle_cq_split(dev, queue_idx); 1105 } 1106 1107 static void 1108 virtio_user_control_queue_notify(struct virtqueue *vq, void *cookie) 1109 { 1110 struct virtio_user_dev *dev = cookie; 1111 uint64_t notify_data = 1; 1112 1113 if (!dev->notify_area) { 1114 if (write(dev->kickfds[vq->vq_queue_index], ¬ify_data, sizeof(notify_data)) < 0) 1115 PMD_DRV_LOG(ERR, "failed to kick backend: %s", 1116 strerror(errno)); 1117 return; 1118 } else if (!virtio_with_feature(&dev->hw, VIRTIO_F_NOTIFICATION_DATA)) { 1119 rte_write16(vq->vq_queue_index, vq->notify_addr); 1120 return; 1121 } 1122 1123 if (virtio_with_packed_queue(&dev->hw)) { 1124 /* Bit[0:15]: vq queue index 1125 * Bit[16:30]: avail index 1126 * Bit[31]: avail wrap counter 1127 */ 1128 notify_data = ((uint32_t)(!!(vq->vq_packed.cached_flags & 1129 VRING_PACKED_DESC_F_AVAIL)) << 31) | 1130 ((uint32_t)vq->vq_avail_idx << 16) | 1131 vq->vq_queue_index; 1132 } else { 1133 /* Bit[0:15]: vq queue index 1134 * Bit[16:31]: avail index 1135 */ 1136 notify_data = ((uint32_t)vq->vq_avail_idx << 16) | 1137 vq->vq_queue_index; 1138 } 1139 rte_write32(notify_data, vq->notify_addr); 1140 } 1141 1142 int 1143 virtio_user_dev_create_shadow_cvq(struct virtio_user_dev *dev, struct virtqueue *vq) 1144 { 1145 char name[VIRTQUEUE_MAX_NAME_SZ]; 1146 struct virtqueue *scvq; 1147 1148 snprintf(name, sizeof(name), "port%d_shadow_cvq", vq->hw->port_id); 1149 scvq = virtqueue_alloc(&dev->hw, vq->vq_queue_index, vq->vq_nentries, 1150 VTNET_CQ, SOCKET_ID_ANY, name); 1151 if (!scvq) { 1152 PMD_INIT_LOG(ERR, "(%s) Failed to alloc shadow control vq", dev->path); 1153 return -ENOMEM; 1154 } 1155 1156 scvq->cq.notify_queue = &virtio_user_control_queue_notify; 1157 scvq->cq.notify_cookie = dev; 1158 scvq->notify_addr = vq->notify_addr; 1159 dev->scvq = scvq; 1160 1161 return 0; 1162 } 1163 1164 void 1165 virtio_user_dev_destroy_shadow_cvq(struct virtio_user_dev *dev) 1166 { 1167 if (!dev->scvq) 1168 return; 1169 1170 virtqueue_free(dev->scvq); 1171 dev->scvq = NULL; 1172 } 1173 1174 int 1175 virtio_user_dev_set_status(struct virtio_user_dev *dev, uint8_t status) 1176 { 1177 int ret; 1178 1179 pthread_mutex_lock(&dev->mutex); 1180 dev->status = status; 1181 ret = dev->ops->set_status(dev, status); 1182 if (ret && ret != -ENOTSUP) 1183 PMD_INIT_LOG(ERR, "(%s) Failed to set backend status", dev->path); 1184 1185 pthread_mutex_unlock(&dev->mutex); 1186 return ret; 1187 } 1188 1189 int 1190 virtio_user_dev_update_status(struct virtio_user_dev *dev) 1191 { 1192 int ret; 1193 uint8_t status; 1194 1195 pthread_mutex_lock(&dev->mutex); 1196 1197 ret = dev->ops->get_status(dev, &status); 1198 if (!ret) { 1199 dev->status = status; 1200 PMD_INIT_LOG(DEBUG, "Updated Device Status(0x%08x):", 1201 dev->status); 1202 PMD_INIT_LOG(DEBUG, "\t-RESET: %u", 1203 (dev->status == VIRTIO_CONFIG_STATUS_RESET)); 1204 PMD_INIT_LOG(DEBUG, "\t-ACKNOWLEDGE: %u", 1205 !!(dev->status & VIRTIO_CONFIG_STATUS_ACK)); 1206 PMD_INIT_LOG(DEBUG, "\t-DRIVER: %u", 1207 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER)); 1208 PMD_INIT_LOG(DEBUG, "\t-DRIVER_OK: %u", 1209 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK)); 1210 PMD_INIT_LOG(DEBUG, "\t-FEATURES_OK: %u", 1211 !!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK)); 1212 PMD_INIT_LOG(DEBUG, "\t-DEVICE_NEED_RESET: %u", 1213 !!(dev->status & VIRTIO_CONFIG_STATUS_DEV_NEED_RESET)); 1214 PMD_INIT_LOG(DEBUG, "\t-FAILED: %u", 1215 !!(dev->status & VIRTIO_CONFIG_STATUS_FAILED)); 1216 } else if (ret != -ENOTSUP) { 1217 PMD_INIT_LOG(ERR, "(%s) Failed to get backend status", dev->path); 1218 } 1219 1220 pthread_mutex_unlock(&dev->mutex); 1221 return ret; 1222 } 1223 1224 int 1225 virtio_user_dev_update_link_state(struct virtio_user_dev *dev) 1226 { 1227 if (dev->ops->update_link_state) 1228 return dev->ops->update_link_state(dev); 1229 1230 return 0; 1231 } 1232 1233 static void 1234 virtio_user_dev_reset_queues_packed(struct rte_eth_dev *eth_dev) 1235 { 1236 struct virtio_user_dev *dev = eth_dev->data->dev_private; 1237 struct virtio_hw *hw = &dev->hw; 1238 struct virtnet_rx *rxvq; 1239 struct virtnet_tx *txvq; 1240 uint16_t i; 1241 1242 /* Add lock to avoid queue contention. */ 1243 rte_spinlock_lock(&hw->state_lock); 1244 hw->started = 0; 1245 1246 /* 1247 * Waiting for datapath to complete before resetting queues. 1248 * 1 ms should be enough for the ongoing Tx/Rx function to finish. 1249 */ 1250 rte_delay_ms(1); 1251 1252 /* Vring reset for each Tx queue and Rx queue. */ 1253 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 1254 rxvq = eth_dev->data->rx_queues[i]; 1255 virtqueue_rxvq_reset_packed(virtnet_rxq_to_vq(rxvq)); 1256 virtio_dev_rx_queue_setup_finish(eth_dev, i); 1257 } 1258 1259 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 1260 txvq = eth_dev->data->tx_queues[i]; 1261 virtqueue_txvq_reset_packed(virtnet_txq_to_vq(txvq)); 1262 } 1263 1264 hw->started = 1; 1265 rte_spinlock_unlock(&hw->state_lock); 1266 } 1267 1268 void 1269 virtio_user_dev_delayed_disconnect_handler(void *param) 1270 { 1271 struct virtio_user_dev *dev = param; 1272 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1273 1274 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 1275 PMD_DRV_LOG(ERR, "interrupt disable failed"); 1276 return; 1277 } 1278 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d", 1279 rte_intr_fd_get(eth_dev->intr_handle)); 1280 if (rte_intr_callback_unregister(eth_dev->intr_handle, 1281 virtio_interrupt_handler, 1282 eth_dev) != 1) 1283 PMD_DRV_LOG(ERR, "interrupt unregister failed"); 1284 1285 if (dev->is_server) { 1286 if (dev->ops->server_disconnect) 1287 dev->ops->server_disconnect(dev); 1288 1289 rte_intr_fd_set(eth_dev->intr_handle, 1290 dev->ops->get_intr_fd(dev)); 1291 1292 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d", 1293 rte_intr_fd_get(eth_dev->intr_handle)); 1294 1295 if (rte_intr_callback_register(eth_dev->intr_handle, 1296 virtio_interrupt_handler, 1297 eth_dev)) 1298 PMD_DRV_LOG(ERR, "interrupt register failed"); 1299 1300 if (rte_intr_enable(eth_dev->intr_handle) < 0) { 1301 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1302 return; 1303 } 1304 } 1305 } 1306 1307 static void 1308 virtio_user_dev_delayed_intr_reconfig_handler(void *param) 1309 { 1310 struct virtio_user_dev *dev = param; 1311 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1312 1313 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d", 1314 rte_intr_fd_get(eth_dev->intr_handle)); 1315 1316 if (rte_intr_callback_unregister(eth_dev->intr_handle, 1317 virtio_interrupt_handler, 1318 eth_dev) != 1) 1319 PMD_DRV_LOG(ERR, "interrupt unregister failed"); 1320 1321 rte_intr_fd_set(eth_dev->intr_handle, dev->ops->get_intr_fd(dev)); 1322 1323 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d", 1324 rte_intr_fd_get(eth_dev->intr_handle)); 1325 1326 if (rte_intr_callback_register(eth_dev->intr_handle, 1327 virtio_interrupt_handler, eth_dev)) 1328 PMD_DRV_LOG(ERR, "interrupt register failed"); 1329 1330 if (rte_intr_enable(eth_dev->intr_handle) < 0) 1331 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1332 } 1333 1334 int 1335 virtio_user_dev_server_reconnect(struct virtio_user_dev *dev) 1336 { 1337 int ret, old_status; 1338 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1339 struct virtio_hw *hw = &dev->hw; 1340 1341 if (!dev->ops->server_reconnect) { 1342 PMD_DRV_LOG(ERR, "(%s) Missing server reconnect callback", dev->path); 1343 return -1; 1344 } 1345 1346 if (dev->ops->server_reconnect(dev)) { 1347 PMD_DRV_LOG(ERR, "(%s) Reconnect callback call failed", dev->path); 1348 return -1; 1349 } 1350 1351 old_status = dev->status; 1352 1353 virtio_reset(hw); 1354 1355 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 1356 1357 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 1358 1359 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 1360 PMD_INIT_LOG(ERR, "get_features failed: %s", 1361 strerror(errno)); 1362 return -1; 1363 } 1364 1365 /* unmask vhost-user unsupported features */ 1366 dev->device_features &= ~(dev->unsupported_features); 1367 1368 dev->features &= (dev->device_features | dev->frontend_features); 1369 1370 /* For packed ring, resetting queues is required in reconnection. */ 1371 if (virtio_with_packed_queue(hw) && 1372 (old_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) { 1373 PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped" 1374 " when packed ring reconnecting."); 1375 virtio_user_dev_reset_queues_packed(eth_dev); 1376 } 1377 1378 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); 1379 1380 /* Start the device */ 1381 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK); 1382 if (!dev->started) 1383 return -1; 1384 1385 if (dev->queue_pairs > 1) { 1386 ret = virtio_user_handle_mq(dev, dev->queue_pairs); 1387 if (ret != 0) { 1388 PMD_INIT_LOG(ERR, "Fails to enable multi-queue pairs!"); 1389 return -1; 1390 } 1391 } 1392 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { 1393 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 1394 PMD_DRV_LOG(ERR, "interrupt disable failed"); 1395 return -1; 1396 } 1397 /* 1398 * This function can be called from the interrupt handler, so 1399 * we can't unregister interrupt handler here. Setting 1400 * alarm to do that later. 1401 */ 1402 rte_eal_alarm_set(1, 1403 virtio_user_dev_delayed_intr_reconfig_handler, 1404 (void *)dev); 1405 } 1406 PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!"); 1407 return 0; 1408 } 1409