1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <fcntl.h> 9 #include <string.h> 10 #include <errno.h> 11 #include <sys/mman.h> 12 #include <unistd.h> 13 #include <sys/eventfd.h> 14 #include <sys/types.h> 15 #include <sys/stat.h> 16 17 #include <rte_alarm.h> 18 #include <rte_string_fns.h> 19 #include <rte_eal_memconfig.h> 20 #include <rte_malloc.h> 21 #include <rte_io.h> 22 23 #include "vhost.h" 24 #include "virtio.h" 25 #include "virtio_user_dev.h" 26 #include "../virtio_ethdev.h" 27 28 #define VIRTIO_USER_MEM_EVENT_CLB_NAME "virtio_user_mem_event_clb" 29 30 const char * const virtio_user_backend_strings[] = { 31 [VIRTIO_USER_BACKEND_UNKNOWN] = "VIRTIO_USER_BACKEND_UNKNOWN", 32 [VIRTIO_USER_BACKEND_VHOST_USER] = "VHOST_USER", 33 [VIRTIO_USER_BACKEND_VHOST_KERNEL] = "VHOST_NET", 34 [VIRTIO_USER_BACKEND_VHOST_VDPA] = "VHOST_VDPA", 35 }; 36 37 static int 38 virtio_user_uninit_notify_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 39 { 40 if (dev->kickfds[queue_sel] >= 0) { 41 close(dev->kickfds[queue_sel]); 42 dev->kickfds[queue_sel] = -1; 43 } 44 45 if (dev->callfds[queue_sel] >= 0) { 46 close(dev->callfds[queue_sel]); 47 dev->callfds[queue_sel] = -1; 48 } 49 50 return 0; 51 } 52 53 static int 54 virtio_user_init_notify_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 55 { 56 /* May use invalid flag, but some backend uses kickfd and 57 * callfd as criteria to judge if dev is alive. so finally we 58 * use real event_fd. 59 */ 60 dev->callfds[queue_sel] = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 61 if (dev->callfds[queue_sel] < 0) { 62 PMD_DRV_LOG(ERR, "(%s) Failed to setup callfd for queue %u: %s", 63 dev->path, queue_sel, strerror(errno)); 64 return -1; 65 } 66 dev->kickfds[queue_sel] = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 67 if (dev->kickfds[queue_sel] < 0) { 68 PMD_DRV_LOG(ERR, "(%s) Failed to setup kickfd for queue %u: %s", 69 dev->path, queue_sel, strerror(errno)); 70 return -1; 71 } 72 73 return 0; 74 } 75 76 static int 77 virtio_user_destroy_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 78 { 79 struct vhost_vring_state state; 80 int ret; 81 82 state.index = queue_sel; 83 ret = dev->ops->get_vring_base(dev, &state); 84 if (ret < 0) { 85 PMD_DRV_LOG(ERR, "(%s) Failed to destroy queue %u", dev->path, queue_sel); 86 return -1; 87 } 88 89 return 0; 90 } 91 92 static int 93 virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 94 { 95 /* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come 96 * firstly because vhost depends on this msg to allocate virtqueue 97 * pair. 98 */ 99 struct vhost_vring_file file; 100 int ret; 101 102 file.index = queue_sel; 103 file.fd = dev->callfds[queue_sel]; 104 ret = dev->ops->set_vring_call(dev, &file); 105 if (ret < 0) { 106 PMD_INIT_LOG(ERR, "(%s) Failed to create queue %u", dev->path, queue_sel); 107 return -1; 108 } 109 110 return 0; 111 } 112 113 static int 114 virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 115 { 116 int ret; 117 struct vhost_vring_file file; 118 struct vhost_vring_state state; 119 struct vring *vring = &dev->vrings.split[queue_sel]; 120 struct vring_packed *pq_vring = &dev->vrings.packed[queue_sel]; 121 struct vhost_vring_addr addr = { 122 .index = queue_sel, 123 .log_guest_addr = 0, 124 .flags = 0, /* disable log */ 125 }; 126 127 if (queue_sel == dev->max_queue_pairs * 2) { 128 if (!dev->scvq) { 129 PMD_INIT_LOG(ERR, "(%s) Shadow control queue expected but missing", 130 dev->path); 131 goto err; 132 } 133 134 /* Use shadow control queue information */ 135 vring = &dev->scvq->vq_split.ring; 136 pq_vring = &dev->scvq->vq_packed.ring; 137 } 138 139 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) { 140 addr.desc_user_addr = 141 (uint64_t)(uintptr_t)pq_vring->desc; 142 addr.avail_user_addr = 143 (uint64_t)(uintptr_t)pq_vring->driver; 144 addr.used_user_addr = 145 (uint64_t)(uintptr_t)pq_vring->device; 146 } else { 147 addr.desc_user_addr = (uint64_t)(uintptr_t)vring->desc; 148 addr.avail_user_addr = (uint64_t)(uintptr_t)vring->avail; 149 addr.used_user_addr = (uint64_t)(uintptr_t)vring->used; 150 } 151 152 state.index = queue_sel; 153 state.num = vring->num; 154 ret = dev->ops->set_vring_num(dev, &state); 155 if (ret < 0) 156 goto err; 157 158 state.index = queue_sel; 159 state.num = 0; /* no reservation */ 160 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) 161 state.num |= (1 << 15); 162 ret = dev->ops->set_vring_base(dev, &state); 163 if (ret < 0) 164 goto err; 165 166 ret = dev->ops->set_vring_addr(dev, &addr); 167 if (ret < 0) 168 goto err; 169 170 /* Of all per virtqueue MSGs, make sure VHOST_USER_SET_VRING_KICK comes 171 * lastly because vhost depends on this msg to judge if 172 * virtio is ready. 173 */ 174 file.index = queue_sel; 175 file.fd = dev->kickfds[queue_sel]; 176 ret = dev->ops->set_vring_kick(dev, &file); 177 if (ret < 0) 178 goto err; 179 180 return 0; 181 err: 182 PMD_INIT_LOG(ERR, "(%s) Failed to kick queue %u", dev->path, queue_sel); 183 184 return -1; 185 } 186 187 static int 188 virtio_user_foreach_queue(struct virtio_user_dev *dev, 189 int (*fn)(struct virtio_user_dev *, uint32_t)) 190 { 191 uint32_t i, nr_vq; 192 193 nr_vq = dev->max_queue_pairs * 2; 194 if (dev->hw_cvq) 195 nr_vq++; 196 197 for (i = 0; i < nr_vq; i++) 198 if (fn(dev, i) < 0) 199 return -1; 200 201 return 0; 202 } 203 204 int 205 virtio_user_dev_set_features(struct virtio_user_dev *dev) 206 { 207 uint64_t features; 208 int ret = -1; 209 210 pthread_mutex_lock(&dev->mutex); 211 212 /* Step 0: tell vhost to create queues */ 213 if (virtio_user_foreach_queue(dev, virtio_user_create_queue) < 0) 214 goto error; 215 216 features = dev->features; 217 218 /* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init */ 219 features &= ~(1ull << VIRTIO_NET_F_MAC); 220 /* Strip VIRTIO_NET_F_CTRL_VQ if the devices does not really support control VQ */ 221 if (!dev->hw_cvq) 222 features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ); 223 features &= ~(1ull << VIRTIO_NET_F_STATUS); 224 ret = dev->ops->set_features(dev, features); 225 if (ret < 0) 226 goto error; 227 PMD_DRV_LOG(INFO, "(%s) set features: 0x%" PRIx64, dev->path, features); 228 error: 229 pthread_mutex_unlock(&dev->mutex); 230 231 return ret; 232 } 233 234 int 235 virtio_user_start_device(struct virtio_user_dev *dev) 236 { 237 int ret; 238 239 /* 240 * XXX workaround! 241 * 242 * We need to make sure that the locks will be 243 * taken in the correct order to avoid deadlocks. 244 * 245 * Before releasing this lock, this thread should 246 * not trigger any memory hotplug events. 247 * 248 * This is a temporary workaround, and should be 249 * replaced when we get proper supports from the 250 * memory subsystem in the future. 251 */ 252 rte_mcfg_mem_read_lock(); 253 pthread_mutex_lock(&dev->mutex); 254 255 /* Step 2: share memory regions */ 256 ret = dev->ops->set_memory_table(dev); 257 if (ret < 0) 258 goto error; 259 260 /* Step 3: kick queues */ 261 ret = virtio_user_foreach_queue(dev, virtio_user_kick_queue); 262 if (ret < 0) 263 goto error; 264 265 /* Step 4: enable queues 266 * we enable the 1st queue pair by default. 267 */ 268 ret = dev->ops->enable_qp(dev, 0, 1); 269 if (ret < 0) 270 goto error; 271 272 if (dev->scvq) { 273 ret = dev->ops->cvq_enable(dev, 1); 274 if (ret < 0) 275 goto error; 276 } 277 278 dev->started = true; 279 280 pthread_mutex_unlock(&dev->mutex); 281 rte_mcfg_mem_read_unlock(); 282 283 return 0; 284 error: 285 pthread_mutex_unlock(&dev->mutex); 286 rte_mcfg_mem_read_unlock(); 287 288 PMD_INIT_LOG(ERR, "(%s) Failed to start device", dev->path); 289 290 /* TODO: free resource here or caller to check */ 291 return -1; 292 } 293 294 int virtio_user_stop_device(struct virtio_user_dev *dev) 295 { 296 uint32_t i; 297 int ret; 298 299 pthread_mutex_lock(&dev->mutex); 300 if (!dev->started) 301 goto out; 302 303 for (i = 0; i < dev->max_queue_pairs; ++i) { 304 ret = dev->ops->enable_qp(dev, i, 0); 305 if (ret < 0) 306 goto err; 307 } 308 309 if (dev->scvq) { 310 ret = dev->ops->cvq_enable(dev, 0); 311 if (ret < 0) 312 goto err; 313 } 314 315 /* Stop the backend. */ 316 if (virtio_user_foreach_queue(dev, virtio_user_destroy_queue) < 0) 317 goto err; 318 319 dev->started = false; 320 321 out: 322 pthread_mutex_unlock(&dev->mutex); 323 324 return 0; 325 err: 326 pthread_mutex_unlock(&dev->mutex); 327 328 PMD_INIT_LOG(ERR, "(%s) Failed to stop device", dev->path); 329 330 return -1; 331 } 332 333 static int 334 virtio_user_dev_init_max_queue_pairs(struct virtio_user_dev *dev, uint32_t user_max_qp) 335 { 336 int ret; 337 338 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MQ))) { 339 dev->max_queue_pairs = 1; 340 return 0; 341 } 342 343 if (!dev->ops->get_config) { 344 dev->max_queue_pairs = user_max_qp; 345 return 0; 346 } 347 348 ret = dev->ops->get_config(dev, (uint8_t *)&dev->max_queue_pairs, 349 offsetof(struct virtio_net_config, max_virtqueue_pairs), 350 sizeof(uint16_t)); 351 if (ret) { 352 /* 353 * We need to know the max queue pair from the device so that 354 * the control queue gets the right index. 355 */ 356 dev->max_queue_pairs = 1; 357 PMD_DRV_LOG(ERR, "(%s) Failed to get max queue pairs from device", dev->path); 358 359 return ret; 360 } 361 362 return 0; 363 } 364 365 int 366 virtio_user_dev_get_rss_config(struct virtio_user_dev *dev, void *dst, size_t offset, int length) 367 { 368 int ret = 0; 369 370 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_RSS))) 371 return -ENOTSUP; 372 373 if (!dev->ops->get_config) 374 return -ENOTSUP; 375 376 ret = dev->ops->get_config(dev, dst, offset, length); 377 if (ret) 378 PMD_DRV_LOG(ERR, "(%s) Failed to get rss config in device", dev->path); 379 380 return ret; 381 } 382 383 int 384 virtio_user_dev_set_mac(struct virtio_user_dev *dev) 385 { 386 int ret = 0; 387 388 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MAC))) 389 return -ENOTSUP; 390 391 if (!dev->ops->set_config) 392 return -ENOTSUP; 393 394 ret = dev->ops->set_config(dev, dev->mac_addr, 395 offsetof(struct virtio_net_config, mac), 396 RTE_ETHER_ADDR_LEN); 397 if (ret) 398 PMD_DRV_LOG(ERR, "(%s) Failed to set MAC address in device", dev->path); 399 400 return ret; 401 } 402 403 int 404 virtio_user_dev_get_mac(struct virtio_user_dev *dev) 405 { 406 int ret = 0; 407 408 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MAC))) 409 return -ENOTSUP; 410 411 if (!dev->ops->get_config) 412 return -ENOTSUP; 413 414 ret = dev->ops->get_config(dev, dev->mac_addr, 415 offsetof(struct virtio_net_config, mac), 416 RTE_ETHER_ADDR_LEN); 417 if (ret) 418 PMD_DRV_LOG(ERR, "(%s) Failed to get MAC address from device", dev->path); 419 420 return ret; 421 } 422 423 static void 424 virtio_user_dev_init_mac(struct virtio_user_dev *dev, const char *mac) 425 { 426 struct rte_ether_addr cmdline_mac; 427 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 428 int ret; 429 430 if (mac && rte_ether_unformat_addr(mac, &cmdline_mac) == 0) { 431 /* 432 * MAC address was passed from command-line, try to store 433 * it in the device if it supports it. Otherwise try to use 434 * the device one. 435 */ 436 memcpy(dev->mac_addr, &cmdline_mac, RTE_ETHER_ADDR_LEN); 437 dev->mac_specified = 1; 438 439 /* Setting MAC may fail, continue to get the device one in this case */ 440 virtio_user_dev_set_mac(dev); 441 ret = virtio_user_dev_get_mac(dev); 442 if (ret == -ENOTSUP) 443 goto out; 444 445 if (memcmp(&cmdline_mac, dev->mac_addr, RTE_ETHER_ADDR_LEN)) 446 PMD_DRV_LOG(INFO, "(%s) Device MAC update failed", dev->path); 447 } else { 448 ret = virtio_user_dev_get_mac(dev); 449 if (ret) { 450 PMD_DRV_LOG(ERR, "(%s) No valid MAC in devargs or device, use random", 451 dev->path); 452 return; 453 } 454 455 dev->mac_specified = 1; 456 } 457 out: 458 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, 459 (struct rte_ether_addr *)dev->mac_addr); 460 PMD_DRV_LOG(INFO, "(%s) MAC %s specified", dev->path, buf); 461 } 462 463 static int 464 virtio_user_dev_init_notify(struct virtio_user_dev *dev) 465 { 466 467 if (virtio_user_foreach_queue(dev, virtio_user_init_notify_queue) < 0) 468 goto err; 469 470 if (dev->device_features & (1ULL << VIRTIO_F_NOTIFICATION_DATA)) 471 if (dev->ops->map_notification_area && 472 dev->ops->map_notification_area(dev)) 473 goto err; 474 475 return 0; 476 err: 477 virtio_user_foreach_queue(dev, virtio_user_uninit_notify_queue); 478 479 return -1; 480 } 481 482 static void 483 virtio_user_dev_uninit_notify(struct virtio_user_dev *dev) 484 { 485 virtio_user_foreach_queue(dev, virtio_user_uninit_notify_queue); 486 487 if (dev->ops->unmap_notification_area && dev->notify_area) 488 dev->ops->unmap_notification_area(dev); 489 } 490 491 static int 492 virtio_user_fill_intr_handle(struct virtio_user_dev *dev) 493 { 494 uint32_t i; 495 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 496 497 if (eth_dev->intr_handle == NULL) { 498 eth_dev->intr_handle = 499 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE); 500 if (eth_dev->intr_handle == NULL) { 501 PMD_DRV_LOG(ERR, "(%s) failed to allocate intr_handle", dev->path); 502 return -1; 503 } 504 } 505 506 for (i = 0; i < dev->max_queue_pairs; ++i) { 507 if (rte_intr_efds_index_set(eth_dev->intr_handle, i, 508 dev->callfds[2 * i + VTNET_SQ_RQ_QUEUE_IDX])) 509 return -rte_errno; 510 } 511 512 if (rte_intr_nb_efd_set(eth_dev->intr_handle, dev->max_queue_pairs)) 513 return -rte_errno; 514 515 if (rte_intr_max_intr_set(eth_dev->intr_handle, 516 dev->max_queue_pairs + 1)) 517 return -rte_errno; 518 519 if (rte_intr_type_set(eth_dev->intr_handle, RTE_INTR_HANDLE_VDEV)) 520 return -rte_errno; 521 522 /* For virtio vdev, no need to read counter for clean */ 523 if (rte_intr_efd_counter_size_set(eth_dev->intr_handle, 0)) 524 return -rte_errno; 525 526 if (rte_intr_fd_set(eth_dev->intr_handle, dev->ops->get_intr_fd(dev))) 527 return -rte_errno; 528 529 return 0; 530 } 531 532 static void 533 virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused, 534 const void *addr, 535 size_t len __rte_unused, 536 void *arg) 537 { 538 struct virtio_user_dev *dev = arg; 539 struct rte_memseg_list *msl; 540 uint16_t i; 541 int ret = 0; 542 543 /* ignore externally allocated memory */ 544 msl = rte_mem_virt2memseg_list(addr); 545 if (msl->external) 546 return; 547 548 pthread_mutex_lock(&dev->mutex); 549 550 if (dev->started == false) 551 goto exit; 552 553 /* Step 1: pause the active queues */ 554 for (i = 0; i < dev->queue_pairs; i++) { 555 ret = dev->ops->enable_qp(dev, i, 0); 556 if (ret < 0) 557 goto exit; 558 } 559 560 /* Step 2: update memory regions */ 561 ret = dev->ops->set_memory_table(dev); 562 if (ret < 0) 563 goto exit; 564 565 /* Step 3: resume the active queues */ 566 for (i = 0; i < dev->queue_pairs; i++) { 567 ret = dev->ops->enable_qp(dev, i, 1); 568 if (ret < 0) 569 goto exit; 570 } 571 572 exit: 573 pthread_mutex_unlock(&dev->mutex); 574 575 if (ret < 0) 576 PMD_DRV_LOG(ERR, "(%s) Failed to update memory table", dev->path); 577 } 578 579 static int 580 virtio_user_dev_setup(struct virtio_user_dev *dev) 581 { 582 if (dev->is_server) { 583 if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) { 584 PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!"); 585 return -1; 586 } 587 } 588 589 switch (dev->backend_type) { 590 case VIRTIO_USER_BACKEND_VHOST_USER: 591 dev->ops = &virtio_ops_user; 592 break; 593 case VIRTIO_USER_BACKEND_VHOST_KERNEL: 594 dev->ops = &virtio_ops_kernel; 595 break; 596 case VIRTIO_USER_BACKEND_VHOST_VDPA: 597 dev->ops = &virtio_ops_vdpa; 598 break; 599 default: 600 PMD_DRV_LOG(ERR, "(%s) Unknown backend type", dev->path); 601 return -1; 602 } 603 604 if (dev->ops->setup(dev) < 0) { 605 PMD_INIT_LOG(ERR, "(%s) Failed to setup backend", dev->path); 606 return -1; 607 } 608 609 return 0; 610 } 611 612 static int 613 virtio_user_alloc_vrings(struct virtio_user_dev *dev) 614 { 615 int i, size, nr_vrings; 616 bool packed_ring = !!(dev->device_features & (1ull << VIRTIO_F_RING_PACKED)); 617 618 nr_vrings = dev->max_queue_pairs * 2; 619 if (dev->device_features & (1ull << VIRTIO_NET_F_CTRL_VQ)) 620 nr_vrings++; 621 622 dev->callfds = rte_zmalloc("virtio_user_dev", nr_vrings * sizeof(*dev->callfds), 0); 623 if (!dev->callfds) { 624 PMD_INIT_LOG(ERR, "(%s) Failed to alloc callfds", dev->path); 625 return -1; 626 } 627 628 dev->kickfds = rte_zmalloc("virtio_user_dev", nr_vrings * sizeof(*dev->kickfds), 0); 629 if (!dev->kickfds) { 630 PMD_INIT_LOG(ERR, "(%s) Failed to alloc kickfds", dev->path); 631 goto free_callfds; 632 } 633 634 for (i = 0; i < nr_vrings; i++) { 635 dev->callfds[i] = -1; 636 dev->kickfds[i] = -1; 637 } 638 639 if (packed_ring) 640 size = sizeof(*dev->vrings.packed); 641 else 642 size = sizeof(*dev->vrings.split); 643 dev->vrings.ptr = rte_zmalloc("virtio_user_dev", nr_vrings * size, 0); 644 if (!dev->vrings.ptr) { 645 PMD_INIT_LOG(ERR, "(%s) Failed to alloc vrings metadata", dev->path); 646 goto free_kickfds; 647 } 648 649 if (packed_ring) { 650 dev->packed_queues = rte_zmalloc("virtio_user_dev", 651 nr_vrings * sizeof(*dev->packed_queues), 0); 652 if (!dev->packed_queues) { 653 PMD_INIT_LOG(ERR, "(%s) Failed to alloc packed queues metadata", 654 dev->path); 655 goto free_vrings; 656 } 657 } 658 659 dev->qp_enabled = rte_zmalloc("virtio_user_dev", 660 dev->max_queue_pairs * sizeof(*dev->qp_enabled), 0); 661 if (!dev->qp_enabled) { 662 PMD_INIT_LOG(ERR, "(%s) Failed to alloc QP enable states", dev->path); 663 goto free_packed_queues; 664 } 665 666 return 0; 667 668 free_packed_queues: 669 rte_free(dev->packed_queues); 670 dev->packed_queues = NULL; 671 free_vrings: 672 rte_free(dev->vrings.ptr); 673 dev->vrings.ptr = NULL; 674 free_kickfds: 675 rte_free(dev->kickfds); 676 dev->kickfds = NULL; 677 free_callfds: 678 rte_free(dev->callfds); 679 dev->callfds = NULL; 680 681 return -1; 682 } 683 684 static void 685 virtio_user_free_vrings(struct virtio_user_dev *dev) 686 { 687 rte_free(dev->qp_enabled); 688 dev->qp_enabled = NULL; 689 rte_free(dev->packed_queues); 690 dev->packed_queues = NULL; 691 rte_free(dev->vrings.ptr); 692 dev->vrings.ptr = NULL; 693 rte_free(dev->kickfds); 694 dev->kickfds = NULL; 695 rte_free(dev->callfds); 696 dev->callfds = NULL; 697 } 698 699 /* Use below macro to filter features from vhost backend */ 700 #define VIRTIO_USER_SUPPORTED_FEATURES \ 701 (1ULL << VIRTIO_NET_F_MAC | \ 702 1ULL << VIRTIO_NET_F_STATUS | \ 703 1ULL << VIRTIO_NET_F_MQ | \ 704 1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR | \ 705 1ULL << VIRTIO_NET_F_CTRL_VQ | \ 706 1ULL << VIRTIO_NET_F_CTRL_RX | \ 707 1ULL << VIRTIO_NET_F_CTRL_VLAN | \ 708 1ULL << VIRTIO_NET_F_CSUM | \ 709 1ULL << VIRTIO_NET_F_HOST_TSO4 | \ 710 1ULL << VIRTIO_NET_F_HOST_TSO6 | \ 711 1ULL << VIRTIO_NET_F_MRG_RXBUF | \ 712 1ULL << VIRTIO_RING_F_INDIRECT_DESC | \ 713 1ULL << VIRTIO_NET_F_GUEST_CSUM | \ 714 1ULL << VIRTIO_NET_F_GUEST_TSO4 | \ 715 1ULL << VIRTIO_NET_F_GUEST_TSO6 | \ 716 1ULL << VIRTIO_F_IN_ORDER | \ 717 1ULL << VIRTIO_F_VERSION_1 | \ 718 1ULL << VIRTIO_F_RING_PACKED | \ 719 1ULL << VIRTIO_F_NOTIFICATION_DATA | \ 720 1ULL << VIRTIO_F_ORDER_PLATFORM | \ 721 1ULL << VIRTIO_NET_F_RSS) 722 723 int 724 virtio_user_dev_init(struct virtio_user_dev *dev, char *path, uint16_t queues, 725 int cq, int queue_size, const char *mac, char **ifname, 726 int server, int mrg_rxbuf, int in_order, int packed_vq, 727 enum virtio_user_backend_type backend_type) 728 { 729 uint64_t backend_features; 730 731 pthread_mutex_init(&dev->mutex, NULL); 732 strlcpy(dev->path, path, PATH_MAX); 733 734 dev->started = 0; 735 dev->queue_pairs = 1; /* mq disabled by default */ 736 dev->max_queue_pairs = queues; /* initialize to user requested value for kernel backend */ 737 dev->queue_size = queue_size; 738 dev->is_server = server; 739 dev->mac_specified = 0; 740 dev->frontend_features = 0; 741 dev->unsupported_features = 0; 742 dev->backend_type = backend_type; 743 dev->ifname = *ifname; 744 745 if (virtio_user_dev_setup(dev) < 0) { 746 PMD_INIT_LOG(ERR, "(%s) backend set up fails", dev->path); 747 return -1; 748 } 749 750 if (dev->ops->set_owner(dev) < 0) { 751 PMD_INIT_LOG(ERR, "(%s) Failed to set backend owner", dev->path); 752 goto destroy; 753 } 754 755 if (dev->ops->get_backend_features(&backend_features) < 0) { 756 PMD_INIT_LOG(ERR, "(%s) Failed to get backend features", dev->path); 757 goto destroy; 758 } 759 760 dev->unsupported_features = ~(VIRTIO_USER_SUPPORTED_FEATURES | backend_features); 761 762 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 763 PMD_INIT_LOG(ERR, "(%s) Failed to get device features", dev->path); 764 goto destroy; 765 } 766 767 virtio_user_dev_init_mac(dev, mac); 768 769 if (virtio_user_dev_init_max_queue_pairs(dev, queues)) 770 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); 771 772 if (dev->max_queue_pairs > 1 || dev->hw_cvq) 773 cq = 1; 774 775 if (!mrg_rxbuf) 776 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF); 777 778 if (!in_order) 779 dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER); 780 781 if (!packed_vq) 782 dev->unsupported_features |= (1ull << VIRTIO_F_RING_PACKED); 783 784 if (dev->mac_specified) 785 dev->frontend_features |= (1ull << VIRTIO_NET_F_MAC); 786 else 787 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC); 788 789 if (cq) { 790 /* Except for vDPA, the device does not really need to know 791 * anything about CQ, so if necessary, we just claim to support 792 * control queue. 793 */ 794 dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 795 } else { 796 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 797 /* Also disable features that depend on VIRTIO_NET_F_CTRL_VQ */ 798 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX); 799 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN); 800 dev->unsupported_features |= 801 (1ull << VIRTIO_NET_F_GUEST_ANNOUNCE); 802 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); 803 dev->unsupported_features |= 804 (1ull << VIRTIO_NET_F_CTRL_MAC_ADDR); 805 } 806 807 /* The backend will not report this feature, we add it explicitly */ 808 if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) 809 dev->frontend_features |= (1ull << VIRTIO_NET_F_STATUS); 810 811 dev->frontend_features &= ~dev->unsupported_features; 812 dev->device_features &= ~dev->unsupported_features; 813 814 if (virtio_user_alloc_vrings(dev) < 0) { 815 PMD_INIT_LOG(ERR, "(%s) Failed to allocate vring metadata", dev->path); 816 goto destroy; 817 } 818 819 if (virtio_user_dev_init_notify(dev) < 0) { 820 PMD_INIT_LOG(ERR, "(%s) Failed to init notifiers", dev->path); 821 goto free_vrings; 822 } 823 824 if (virtio_user_fill_intr_handle(dev) < 0) { 825 PMD_INIT_LOG(ERR, "(%s) Failed to init interrupt handler", dev->path); 826 goto notify_uninit; 827 } 828 829 if (rte_mem_event_callback_register(VIRTIO_USER_MEM_EVENT_CLB_NAME, 830 virtio_user_mem_event_cb, dev)) { 831 if (rte_errno != ENOTSUP) { 832 PMD_INIT_LOG(ERR, "(%s) Failed to register mem event callback", 833 dev->path); 834 goto notify_uninit; 835 } 836 } 837 838 *ifname = NULL; 839 return 0; 840 841 notify_uninit: 842 virtio_user_dev_uninit_notify(dev); 843 free_vrings: 844 virtio_user_free_vrings(dev); 845 destroy: 846 dev->ops->destroy(dev); 847 848 return -1; 849 } 850 851 void 852 virtio_user_dev_uninit(struct virtio_user_dev *dev) 853 { 854 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 855 856 rte_intr_instance_free(eth_dev->intr_handle); 857 eth_dev->intr_handle = NULL; 858 859 virtio_user_stop_device(dev); 860 861 rte_mem_event_callback_unregister(VIRTIO_USER_MEM_EVENT_CLB_NAME, dev); 862 863 virtio_user_dev_uninit_notify(dev); 864 865 virtio_user_free_vrings(dev); 866 867 free(dev->ifname); 868 869 if (dev->is_server) 870 unlink(dev->path); 871 872 dev->ops->destroy(dev); 873 } 874 875 static uint8_t 876 virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs) 877 { 878 uint16_t i; 879 uint8_t ret = 0; 880 881 if (q_pairs > dev->max_queue_pairs) { 882 PMD_INIT_LOG(ERR, "(%s) multi-q config %u, but only %u supported", 883 dev->path, q_pairs, dev->max_queue_pairs); 884 return -1; 885 } 886 887 for (i = 0; i < q_pairs; ++i) 888 ret |= dev->ops->enable_qp(dev, i, 1); 889 for (i = q_pairs; i < dev->max_queue_pairs; ++i) 890 ret |= dev->ops->enable_qp(dev, i, 0); 891 892 dev->queue_pairs = q_pairs; 893 894 return ret; 895 } 896 897 #define CVQ_MAX_DATA_DESCS 32 898 899 static uint32_t 900 virtio_user_handle_ctrl_msg_split(struct virtio_user_dev *dev, struct vring *vring, 901 uint16_t idx_hdr) 902 { 903 struct virtio_net_ctrl_hdr *hdr; 904 virtio_net_ctrl_ack status = ~0; 905 uint16_t i, idx_data, idx_status; 906 uint32_t n_descs = 0; 907 int dlen[CVQ_MAX_DATA_DESCS], nb_dlen = 0; 908 909 /* locate desc for header, data, and status */ 910 idx_data = vring->desc[idx_hdr].next; 911 n_descs++; 912 913 i = idx_data; 914 while (vring->desc[i].flags == VRING_DESC_F_NEXT) { 915 dlen[nb_dlen++] = vring->desc[i].len; 916 i = vring->desc[i].next; 917 n_descs++; 918 } 919 920 /* locate desc for status */ 921 idx_status = i; 922 n_descs++; 923 924 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 925 if (hdr->class == VIRTIO_NET_CTRL_MQ && 926 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 927 uint16_t queues; 928 929 queues = *(uint16_t *)(uintptr_t)vring->desc[idx_data].addr; 930 status = virtio_user_handle_mq(dev, queues); 931 } else if (hdr->class == VIRTIO_NET_CTRL_MQ && hdr->cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) { 932 struct virtio_net_ctrl_rss *rss; 933 934 rss = (struct virtio_net_ctrl_rss *)(uintptr_t)vring->desc[idx_data].addr; 935 status = virtio_user_handle_mq(dev, rss->max_tx_vq); 936 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 937 hdr->class == VIRTIO_NET_CTRL_MAC || 938 hdr->class == VIRTIO_NET_CTRL_VLAN) { 939 status = 0; 940 } 941 942 if (!status && dev->scvq) 943 status = virtio_send_command(&dev->scvq->cq, 944 (struct virtio_pmd_ctrl *)hdr, dlen, nb_dlen); 945 946 /* Update status */ 947 *(virtio_net_ctrl_ack *)(uintptr_t)vring->desc[idx_status].addr = status; 948 949 return n_descs; 950 } 951 952 static inline int 953 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) 954 { 955 uint16_t flags = rte_atomic_load_explicit(&desc->flags, rte_memory_order_acquire); 956 957 return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) && 958 wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED); 959 } 960 961 static uint32_t 962 virtio_user_handle_ctrl_msg_packed(struct virtio_user_dev *dev, 963 struct vring_packed *vring, 964 uint16_t idx_hdr) 965 { 966 struct virtio_net_ctrl_hdr *hdr; 967 virtio_net_ctrl_ack status = ~0; 968 uint16_t idx_data, idx_status; 969 /* initialize to one, header is first */ 970 uint32_t n_descs = 1; 971 int dlen[CVQ_MAX_DATA_DESCS], nb_dlen = 0; 972 973 /* locate desc for header, data, and status */ 974 idx_data = idx_hdr + 1; 975 if (idx_data >= dev->queue_size) 976 idx_data -= dev->queue_size; 977 978 n_descs++; 979 980 idx_status = idx_data; 981 while (vring->desc[idx_status].flags & VRING_DESC_F_NEXT) { 982 dlen[nb_dlen++] = vring->desc[idx_status].len; 983 idx_status++; 984 if (idx_status >= dev->queue_size) 985 idx_status -= dev->queue_size; 986 n_descs++; 987 } 988 989 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 990 if (hdr->class == VIRTIO_NET_CTRL_MQ && 991 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 992 uint16_t queues; 993 994 queues = *(uint16_t *)(uintptr_t) 995 vring->desc[idx_data].addr; 996 status = virtio_user_handle_mq(dev, queues); 997 } else if (hdr->class == VIRTIO_NET_CTRL_MQ && hdr->cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) { 998 struct virtio_net_ctrl_rss *rss; 999 1000 rss = (struct virtio_net_ctrl_rss *)(uintptr_t)vring->desc[idx_data].addr; 1001 status = virtio_user_handle_mq(dev, rss->max_tx_vq); 1002 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 1003 hdr->class == VIRTIO_NET_CTRL_MAC || 1004 hdr->class == VIRTIO_NET_CTRL_VLAN) { 1005 status = 0; 1006 } 1007 1008 if (!status && dev->scvq) 1009 status = virtio_send_command(&dev->scvq->cq, 1010 (struct virtio_pmd_ctrl *)hdr, dlen, nb_dlen); 1011 1012 /* Update status */ 1013 *(virtio_net_ctrl_ack *)(uintptr_t) 1014 vring->desc[idx_status].addr = status; 1015 1016 /* Update used descriptor */ 1017 vring->desc[idx_hdr].id = vring->desc[idx_status].id; 1018 vring->desc[idx_hdr].len = sizeof(status); 1019 1020 return n_descs; 1021 } 1022 1023 static void 1024 virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx) 1025 { 1026 struct virtio_user_queue *vq = &dev->packed_queues[queue_idx]; 1027 struct vring_packed *vring = &dev->vrings.packed[queue_idx]; 1028 uint16_t n_descs, flags; 1029 1030 /* Perform a load-acquire barrier in desc_is_avail to 1031 * enforce the ordering between desc flags and desc 1032 * content. 1033 */ 1034 while (desc_is_avail(&vring->desc[vq->used_idx], 1035 vq->used_wrap_counter)) { 1036 1037 n_descs = virtio_user_handle_ctrl_msg_packed(dev, vring, 1038 vq->used_idx); 1039 1040 flags = VRING_DESC_F_WRITE; 1041 if (vq->used_wrap_counter) 1042 flags |= VRING_PACKED_DESC_F_AVAIL_USED; 1043 1044 rte_atomic_store_explicit(&vring->desc[vq->used_idx].flags, flags, 1045 rte_memory_order_release); 1046 1047 vq->used_idx += n_descs; 1048 if (vq->used_idx >= dev->queue_size) { 1049 vq->used_idx -= dev->queue_size; 1050 vq->used_wrap_counter ^= 1; 1051 } 1052 } 1053 } 1054 1055 static void 1056 virtio_user_handle_cq_split(struct virtio_user_dev *dev, uint16_t queue_idx) 1057 { 1058 uint16_t avail_idx, desc_idx; 1059 struct vring_used_elem *uep; 1060 uint32_t n_descs; 1061 struct vring *vring = &dev->vrings.split[queue_idx]; 1062 1063 /* Consume avail ring, using used ring idx as first one */ 1064 while (rte_atomic_load_explicit(&vring->used->idx, rte_memory_order_relaxed) 1065 != vring->avail->idx) { 1066 avail_idx = rte_atomic_load_explicit(&vring->used->idx, rte_memory_order_relaxed) 1067 & (vring->num - 1); 1068 desc_idx = vring->avail->ring[avail_idx]; 1069 1070 n_descs = virtio_user_handle_ctrl_msg_split(dev, vring, desc_idx); 1071 1072 /* Update used ring */ 1073 uep = &vring->used->ring[avail_idx]; 1074 uep->id = desc_idx; 1075 uep->len = n_descs; 1076 1077 rte_atomic_fetch_add_explicit(&vring->used->idx, 1, rte_memory_order_relaxed); 1078 } 1079 } 1080 1081 void 1082 virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx) 1083 { 1084 if (virtio_with_packed_queue(&dev->hw)) 1085 virtio_user_handle_cq_packed(dev, queue_idx); 1086 else 1087 virtio_user_handle_cq_split(dev, queue_idx); 1088 } 1089 1090 static void 1091 virtio_user_control_queue_notify(struct virtqueue *vq, void *cookie) 1092 { 1093 struct virtio_user_dev *dev = cookie; 1094 uint64_t notify_data = 1; 1095 1096 if (!dev->notify_area) { 1097 if (write(dev->kickfds[vq->vq_queue_index], ¬ify_data, sizeof(notify_data)) < 0) 1098 PMD_DRV_LOG(ERR, "failed to kick backend: %s", 1099 strerror(errno)); 1100 return; 1101 } else if (!virtio_with_feature(&dev->hw, VIRTIO_F_NOTIFICATION_DATA)) { 1102 rte_write16(vq->vq_queue_index, vq->notify_addr); 1103 return; 1104 } 1105 1106 if (virtio_with_packed_queue(&dev->hw)) { 1107 /* Bit[0:15]: vq queue index 1108 * Bit[16:30]: avail index 1109 * Bit[31]: avail wrap counter 1110 */ 1111 notify_data = ((uint32_t)(!!(vq->vq_packed.cached_flags & 1112 VRING_PACKED_DESC_F_AVAIL)) << 31) | 1113 ((uint32_t)vq->vq_avail_idx << 16) | 1114 vq->vq_queue_index; 1115 } else { 1116 /* Bit[0:15]: vq queue index 1117 * Bit[16:31]: avail index 1118 */ 1119 notify_data = ((uint32_t)vq->vq_avail_idx << 16) | 1120 vq->vq_queue_index; 1121 } 1122 rte_write32(notify_data, vq->notify_addr); 1123 } 1124 1125 int 1126 virtio_user_dev_create_shadow_cvq(struct virtio_user_dev *dev, struct virtqueue *vq) 1127 { 1128 char name[VIRTQUEUE_MAX_NAME_SZ]; 1129 struct virtqueue *scvq; 1130 1131 snprintf(name, sizeof(name), "port%d_shadow_cvq", vq->hw->port_id); 1132 scvq = virtqueue_alloc(&dev->hw, vq->vq_queue_index, vq->vq_nentries, 1133 VTNET_CQ, SOCKET_ID_ANY, name); 1134 if (!scvq) { 1135 PMD_INIT_LOG(ERR, "(%s) Failed to alloc shadow control vq\n", dev->path); 1136 return -ENOMEM; 1137 } 1138 1139 scvq->cq.notify_queue = &virtio_user_control_queue_notify; 1140 scvq->cq.notify_cookie = dev; 1141 scvq->notify_addr = vq->notify_addr; 1142 dev->scvq = scvq; 1143 1144 return 0; 1145 } 1146 1147 void 1148 virtio_user_dev_destroy_shadow_cvq(struct virtio_user_dev *dev) 1149 { 1150 if (!dev->scvq) 1151 return; 1152 1153 virtqueue_free(dev->scvq); 1154 dev->scvq = NULL; 1155 } 1156 1157 int 1158 virtio_user_dev_set_status(struct virtio_user_dev *dev, uint8_t status) 1159 { 1160 int ret; 1161 1162 pthread_mutex_lock(&dev->mutex); 1163 dev->status = status; 1164 ret = dev->ops->set_status(dev, status); 1165 if (ret && ret != -ENOTSUP) 1166 PMD_INIT_LOG(ERR, "(%s) Failed to set backend status", dev->path); 1167 1168 pthread_mutex_unlock(&dev->mutex); 1169 return ret; 1170 } 1171 1172 int 1173 virtio_user_dev_update_status(struct virtio_user_dev *dev) 1174 { 1175 int ret; 1176 uint8_t status; 1177 1178 pthread_mutex_lock(&dev->mutex); 1179 1180 ret = dev->ops->get_status(dev, &status); 1181 if (!ret) { 1182 dev->status = status; 1183 PMD_INIT_LOG(DEBUG, "Updated Device Status(0x%08x):\n" 1184 "\t-RESET: %u\n" 1185 "\t-ACKNOWLEDGE: %u\n" 1186 "\t-DRIVER: %u\n" 1187 "\t-DRIVER_OK: %u\n" 1188 "\t-FEATURES_OK: %u\n" 1189 "\t-DEVICE_NEED_RESET: %u\n" 1190 "\t-FAILED: %u", 1191 dev->status, 1192 (dev->status == VIRTIO_CONFIG_STATUS_RESET), 1193 !!(dev->status & VIRTIO_CONFIG_STATUS_ACK), 1194 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER), 1195 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK), 1196 !!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK), 1197 !!(dev->status & VIRTIO_CONFIG_STATUS_DEV_NEED_RESET), 1198 !!(dev->status & VIRTIO_CONFIG_STATUS_FAILED)); 1199 } else if (ret != -ENOTSUP) { 1200 PMD_INIT_LOG(ERR, "(%s) Failed to get backend status", dev->path); 1201 } 1202 1203 pthread_mutex_unlock(&dev->mutex); 1204 return ret; 1205 } 1206 1207 int 1208 virtio_user_dev_update_link_state(struct virtio_user_dev *dev) 1209 { 1210 if (dev->ops->update_link_state) 1211 return dev->ops->update_link_state(dev); 1212 1213 return 0; 1214 } 1215 1216 static void 1217 virtio_user_dev_reset_queues_packed(struct rte_eth_dev *eth_dev) 1218 { 1219 struct virtio_user_dev *dev = eth_dev->data->dev_private; 1220 struct virtio_hw *hw = &dev->hw; 1221 struct virtnet_rx *rxvq; 1222 struct virtnet_tx *txvq; 1223 uint16_t i; 1224 1225 /* Add lock to avoid queue contention. */ 1226 rte_spinlock_lock(&hw->state_lock); 1227 hw->started = 0; 1228 1229 /* 1230 * Waiting for datapath to complete before resetting queues. 1231 * 1 ms should be enough for the ongoing Tx/Rx function to finish. 1232 */ 1233 rte_delay_ms(1); 1234 1235 /* Vring reset for each Tx queue and Rx queue. */ 1236 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 1237 rxvq = eth_dev->data->rx_queues[i]; 1238 virtqueue_rxvq_reset_packed(virtnet_rxq_to_vq(rxvq)); 1239 virtio_dev_rx_queue_setup_finish(eth_dev, i); 1240 } 1241 1242 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 1243 txvq = eth_dev->data->tx_queues[i]; 1244 virtqueue_txvq_reset_packed(virtnet_txq_to_vq(txvq)); 1245 } 1246 1247 hw->started = 1; 1248 rte_spinlock_unlock(&hw->state_lock); 1249 } 1250 1251 void 1252 virtio_user_dev_delayed_disconnect_handler(void *param) 1253 { 1254 struct virtio_user_dev *dev = param; 1255 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1256 1257 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 1258 PMD_DRV_LOG(ERR, "interrupt disable failed"); 1259 return; 1260 } 1261 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d", 1262 rte_intr_fd_get(eth_dev->intr_handle)); 1263 if (rte_intr_callback_unregister(eth_dev->intr_handle, 1264 virtio_interrupt_handler, 1265 eth_dev) != 1) 1266 PMD_DRV_LOG(ERR, "interrupt unregister failed"); 1267 1268 if (dev->is_server) { 1269 if (dev->ops->server_disconnect) 1270 dev->ops->server_disconnect(dev); 1271 1272 rte_intr_fd_set(eth_dev->intr_handle, 1273 dev->ops->get_intr_fd(dev)); 1274 1275 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d", 1276 rte_intr_fd_get(eth_dev->intr_handle)); 1277 1278 if (rte_intr_callback_register(eth_dev->intr_handle, 1279 virtio_interrupt_handler, 1280 eth_dev)) 1281 PMD_DRV_LOG(ERR, "interrupt register failed"); 1282 1283 if (rte_intr_enable(eth_dev->intr_handle) < 0) { 1284 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1285 return; 1286 } 1287 } 1288 } 1289 1290 static void 1291 virtio_user_dev_delayed_intr_reconfig_handler(void *param) 1292 { 1293 struct virtio_user_dev *dev = param; 1294 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1295 1296 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d", 1297 rte_intr_fd_get(eth_dev->intr_handle)); 1298 1299 if (rte_intr_callback_unregister(eth_dev->intr_handle, 1300 virtio_interrupt_handler, 1301 eth_dev) != 1) 1302 PMD_DRV_LOG(ERR, "interrupt unregister failed"); 1303 1304 rte_intr_fd_set(eth_dev->intr_handle, dev->ops->get_intr_fd(dev)); 1305 1306 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d", 1307 rte_intr_fd_get(eth_dev->intr_handle)); 1308 1309 if (rte_intr_callback_register(eth_dev->intr_handle, 1310 virtio_interrupt_handler, eth_dev)) 1311 PMD_DRV_LOG(ERR, "interrupt register failed"); 1312 1313 if (rte_intr_enable(eth_dev->intr_handle) < 0) 1314 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1315 } 1316 1317 int 1318 virtio_user_dev_server_reconnect(struct virtio_user_dev *dev) 1319 { 1320 int ret, old_status; 1321 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1322 struct virtio_hw *hw = &dev->hw; 1323 1324 if (!dev->ops->server_reconnect) { 1325 PMD_DRV_LOG(ERR, "(%s) Missing server reconnect callback", dev->path); 1326 return -1; 1327 } 1328 1329 if (dev->ops->server_reconnect(dev)) { 1330 PMD_DRV_LOG(ERR, "(%s) Reconnect callback call failed", dev->path); 1331 return -1; 1332 } 1333 1334 old_status = dev->status; 1335 1336 virtio_reset(hw); 1337 1338 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 1339 1340 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 1341 1342 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 1343 PMD_INIT_LOG(ERR, "get_features failed: %s", 1344 strerror(errno)); 1345 return -1; 1346 } 1347 1348 /* unmask vhost-user unsupported features */ 1349 dev->device_features &= ~(dev->unsupported_features); 1350 1351 dev->features &= (dev->device_features | dev->frontend_features); 1352 1353 /* For packed ring, resetting queues is required in reconnection. */ 1354 if (virtio_with_packed_queue(hw) && 1355 (old_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) { 1356 PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped" 1357 " when packed ring reconnecting."); 1358 virtio_user_dev_reset_queues_packed(eth_dev); 1359 } 1360 1361 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); 1362 1363 /* Start the device */ 1364 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK); 1365 if (!dev->started) 1366 return -1; 1367 1368 if (dev->queue_pairs > 1) { 1369 ret = virtio_user_handle_mq(dev, dev->queue_pairs); 1370 if (ret != 0) { 1371 PMD_INIT_LOG(ERR, "Fails to enable multi-queue pairs!"); 1372 return -1; 1373 } 1374 } 1375 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { 1376 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 1377 PMD_DRV_LOG(ERR, "interrupt disable failed"); 1378 return -1; 1379 } 1380 /* 1381 * This function can be called from the interrupt handler, so 1382 * we can't unregister interrupt handler here. Setting 1383 * alarm to do that later. 1384 */ 1385 rte_eal_alarm_set(1, 1386 virtio_user_dev_delayed_intr_reconfig_handler, 1387 (void *)dev); 1388 } 1389 PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!"); 1390 return 0; 1391 } 1392