1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <fcntl.h> 9 #include <string.h> 10 #include <errno.h> 11 #include <sys/mman.h> 12 #include <unistd.h> 13 #include <sys/eventfd.h> 14 #include <sys/types.h> 15 #include <sys/stat.h> 16 17 #include <rte_alarm.h> 18 #include <rte_string_fns.h> 19 #include <rte_eal_memconfig.h> 20 #include <rte_malloc.h> 21 #include <rte_io.h> 22 23 #include "vhost.h" 24 #include "virtio_user_dev.h" 25 #include "../virtio_ethdev.h" 26 27 #define VIRTIO_USER_MEM_EVENT_CLB_NAME "virtio_user_mem_event_clb" 28 29 const char * const virtio_user_backend_strings[] = { 30 [VIRTIO_USER_BACKEND_UNKNOWN] = "VIRTIO_USER_BACKEND_UNKNOWN", 31 [VIRTIO_USER_BACKEND_VHOST_USER] = "VHOST_USER", 32 [VIRTIO_USER_BACKEND_VHOST_KERNEL] = "VHOST_NET", 33 [VIRTIO_USER_BACKEND_VHOST_VDPA] = "VHOST_VDPA", 34 }; 35 36 static int 37 virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 38 { 39 /* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come 40 * firstly because vhost depends on this msg to allocate virtqueue 41 * pair. 42 */ 43 struct vhost_vring_file file; 44 int ret; 45 46 file.index = queue_sel; 47 file.fd = dev->callfds[queue_sel]; 48 ret = dev->ops->set_vring_call(dev, &file); 49 if (ret < 0) { 50 PMD_INIT_LOG(ERR, "(%s) Failed to create queue %u", dev->path, queue_sel); 51 return -1; 52 } 53 54 return 0; 55 } 56 57 static int 58 virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 59 { 60 int ret; 61 struct vhost_vring_file file; 62 struct vhost_vring_state state; 63 struct vring *vring = &dev->vrings.split[queue_sel]; 64 struct vring_packed *pq_vring = &dev->vrings.packed[queue_sel]; 65 struct vhost_vring_addr addr = { 66 .index = queue_sel, 67 .log_guest_addr = 0, 68 .flags = 0, /* disable log */ 69 }; 70 71 if (queue_sel == dev->max_queue_pairs * 2) { 72 if (!dev->scvq) { 73 PMD_INIT_LOG(ERR, "(%s) Shadow control queue expected but missing", 74 dev->path); 75 goto err; 76 } 77 78 /* Use shadow control queue information */ 79 vring = &dev->scvq->vq_split.ring; 80 pq_vring = &dev->scvq->vq_packed.ring; 81 } 82 83 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) { 84 addr.desc_user_addr = 85 (uint64_t)(uintptr_t)pq_vring->desc; 86 addr.avail_user_addr = 87 (uint64_t)(uintptr_t)pq_vring->driver; 88 addr.used_user_addr = 89 (uint64_t)(uintptr_t)pq_vring->device; 90 } else { 91 addr.desc_user_addr = (uint64_t)(uintptr_t)vring->desc; 92 addr.avail_user_addr = (uint64_t)(uintptr_t)vring->avail; 93 addr.used_user_addr = (uint64_t)(uintptr_t)vring->used; 94 } 95 96 state.index = queue_sel; 97 state.num = vring->num; 98 ret = dev->ops->set_vring_num(dev, &state); 99 if (ret < 0) 100 goto err; 101 102 state.index = queue_sel; 103 state.num = 0; /* no reservation */ 104 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) 105 state.num |= (1 << 15); 106 ret = dev->ops->set_vring_base(dev, &state); 107 if (ret < 0) 108 goto err; 109 110 ret = dev->ops->set_vring_addr(dev, &addr); 111 if (ret < 0) 112 goto err; 113 114 /* Of all per virtqueue MSGs, make sure VHOST_USER_SET_VRING_KICK comes 115 * lastly because vhost depends on this msg to judge if 116 * virtio is ready. 117 */ 118 file.index = queue_sel; 119 file.fd = dev->kickfds[queue_sel]; 120 ret = dev->ops->set_vring_kick(dev, &file); 121 if (ret < 0) 122 goto err; 123 124 return 0; 125 err: 126 PMD_INIT_LOG(ERR, "(%s) Failed to kick queue %u", dev->path, queue_sel); 127 128 return -1; 129 } 130 131 static int 132 virtio_user_queue_setup(struct virtio_user_dev *dev, 133 int (*fn)(struct virtio_user_dev *, uint32_t)) 134 { 135 uint32_t i, nr_vq; 136 137 nr_vq = dev->max_queue_pairs * 2; 138 if (dev->hw_cvq) 139 nr_vq++; 140 141 for (i = 0; i < nr_vq; i++) { 142 if (fn(dev, i) < 0) { 143 PMD_DRV_LOG(ERR, "(%s) setup VQ %u failed", dev->path, i); 144 return -1; 145 } 146 } 147 148 return 0; 149 } 150 151 int 152 virtio_user_dev_set_features(struct virtio_user_dev *dev) 153 { 154 uint64_t features; 155 int ret = -1; 156 157 pthread_mutex_lock(&dev->mutex); 158 159 /* Step 0: tell vhost to create queues */ 160 if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0) 161 goto error; 162 163 features = dev->features; 164 165 /* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init */ 166 features &= ~(1ull << VIRTIO_NET_F_MAC); 167 /* Strip VIRTIO_NET_F_CTRL_VQ if the devices does not really support control VQ */ 168 if (!dev->hw_cvq) 169 features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ); 170 features &= ~(1ull << VIRTIO_NET_F_STATUS); 171 ret = dev->ops->set_features(dev, features); 172 if (ret < 0) 173 goto error; 174 PMD_DRV_LOG(INFO, "(%s) set features: 0x%" PRIx64, dev->path, features); 175 error: 176 pthread_mutex_unlock(&dev->mutex); 177 178 return ret; 179 } 180 181 int 182 virtio_user_start_device(struct virtio_user_dev *dev) 183 { 184 int ret; 185 186 /* 187 * XXX workaround! 188 * 189 * We need to make sure that the locks will be 190 * taken in the correct order to avoid deadlocks. 191 * 192 * Before releasing this lock, this thread should 193 * not trigger any memory hotplug events. 194 * 195 * This is a temporary workaround, and should be 196 * replaced when we get proper supports from the 197 * memory subsystem in the future. 198 */ 199 rte_mcfg_mem_read_lock(); 200 pthread_mutex_lock(&dev->mutex); 201 202 /* Step 2: share memory regions */ 203 ret = dev->ops->set_memory_table(dev); 204 if (ret < 0) 205 goto error; 206 207 /* Step 3: kick queues */ 208 ret = virtio_user_queue_setup(dev, virtio_user_kick_queue); 209 if (ret < 0) 210 goto error; 211 212 /* Step 4: enable queues 213 * we enable the 1st queue pair by default. 214 */ 215 ret = dev->ops->enable_qp(dev, 0, 1); 216 if (ret < 0) 217 goto error; 218 219 if (dev->scvq) { 220 ret = dev->ops->cvq_enable(dev, 1); 221 if (ret < 0) 222 goto error; 223 } 224 225 dev->started = true; 226 227 pthread_mutex_unlock(&dev->mutex); 228 rte_mcfg_mem_read_unlock(); 229 230 return 0; 231 error: 232 pthread_mutex_unlock(&dev->mutex); 233 rte_mcfg_mem_read_unlock(); 234 235 PMD_INIT_LOG(ERR, "(%s) Failed to start device", dev->path); 236 237 /* TODO: free resource here or caller to check */ 238 return -1; 239 } 240 241 int virtio_user_stop_device(struct virtio_user_dev *dev) 242 { 243 struct vhost_vring_state state; 244 uint32_t i; 245 int ret; 246 247 pthread_mutex_lock(&dev->mutex); 248 if (!dev->started) 249 goto out; 250 251 for (i = 0; i < dev->max_queue_pairs; ++i) { 252 ret = dev->ops->enable_qp(dev, i, 0); 253 if (ret < 0) 254 goto err; 255 } 256 257 if (dev->scvq) { 258 ret = dev->ops->cvq_enable(dev, 0); 259 if (ret < 0) 260 goto err; 261 } 262 263 /* Stop the backend. */ 264 for (i = 0; i < dev->max_queue_pairs * 2; ++i) { 265 state.index = i; 266 ret = dev->ops->get_vring_base(dev, &state); 267 if (ret < 0) { 268 PMD_DRV_LOG(ERR, "(%s) get_vring_base failed, index=%u", dev->path, i); 269 goto err; 270 } 271 } 272 273 dev->started = false; 274 275 out: 276 pthread_mutex_unlock(&dev->mutex); 277 278 return 0; 279 err: 280 pthread_mutex_unlock(&dev->mutex); 281 282 PMD_INIT_LOG(ERR, "(%s) Failed to stop device", dev->path); 283 284 return -1; 285 } 286 287 static int 288 virtio_user_dev_init_max_queue_pairs(struct virtio_user_dev *dev, uint32_t user_max_qp) 289 { 290 int ret; 291 292 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MQ))) { 293 dev->max_queue_pairs = 1; 294 return 0; 295 } 296 297 if (!dev->ops->get_config) { 298 dev->max_queue_pairs = user_max_qp; 299 return 0; 300 } 301 302 ret = dev->ops->get_config(dev, (uint8_t *)&dev->max_queue_pairs, 303 offsetof(struct virtio_net_config, max_virtqueue_pairs), 304 sizeof(uint16_t)); 305 if (ret) { 306 /* 307 * We need to know the max queue pair from the device so that 308 * the control queue gets the right index. 309 */ 310 dev->max_queue_pairs = 1; 311 PMD_DRV_LOG(ERR, "(%s) Failed to get max queue pairs from device", dev->path); 312 313 return ret; 314 } 315 316 return 0; 317 } 318 319 int 320 virtio_user_dev_get_rss_config(struct virtio_user_dev *dev, void *dst, size_t offset, int length) 321 { 322 int ret = 0; 323 324 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_RSS))) 325 return -ENOTSUP; 326 327 if (!dev->ops->get_config) 328 return -ENOTSUP; 329 330 ret = dev->ops->get_config(dev, dst, offset, length); 331 if (ret) 332 PMD_DRV_LOG(ERR, "(%s) Failed to get rss config in device", dev->path); 333 334 return ret; 335 } 336 337 int 338 virtio_user_dev_set_mac(struct virtio_user_dev *dev) 339 { 340 int ret = 0; 341 342 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MAC))) 343 return -ENOTSUP; 344 345 if (!dev->ops->set_config) 346 return -ENOTSUP; 347 348 ret = dev->ops->set_config(dev, dev->mac_addr, 349 offsetof(struct virtio_net_config, mac), 350 RTE_ETHER_ADDR_LEN); 351 if (ret) 352 PMD_DRV_LOG(ERR, "(%s) Failed to set MAC address in device", dev->path); 353 354 return ret; 355 } 356 357 int 358 virtio_user_dev_get_mac(struct virtio_user_dev *dev) 359 { 360 int ret = 0; 361 362 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MAC))) 363 return -ENOTSUP; 364 365 if (!dev->ops->get_config) 366 return -ENOTSUP; 367 368 ret = dev->ops->get_config(dev, dev->mac_addr, 369 offsetof(struct virtio_net_config, mac), 370 RTE_ETHER_ADDR_LEN); 371 if (ret) 372 PMD_DRV_LOG(ERR, "(%s) Failed to get MAC address from device", dev->path); 373 374 return ret; 375 } 376 377 static void 378 virtio_user_dev_init_mac(struct virtio_user_dev *dev, const char *mac) 379 { 380 struct rte_ether_addr cmdline_mac; 381 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 382 int ret; 383 384 if (mac && rte_ether_unformat_addr(mac, &cmdline_mac) == 0) { 385 /* 386 * MAC address was passed from command-line, try to store 387 * it in the device if it supports it. Otherwise try to use 388 * the device one. 389 */ 390 memcpy(dev->mac_addr, &cmdline_mac, RTE_ETHER_ADDR_LEN); 391 dev->mac_specified = 1; 392 393 /* Setting MAC may fail, continue to get the device one in this case */ 394 virtio_user_dev_set_mac(dev); 395 ret = virtio_user_dev_get_mac(dev); 396 if (ret == -ENOTSUP) 397 goto out; 398 399 if (memcmp(&cmdline_mac, dev->mac_addr, RTE_ETHER_ADDR_LEN)) 400 PMD_DRV_LOG(INFO, "(%s) Device MAC update failed", dev->path); 401 } else { 402 ret = virtio_user_dev_get_mac(dev); 403 if (ret) { 404 PMD_DRV_LOG(ERR, "(%s) No valid MAC in devargs or device, use random", 405 dev->path); 406 return; 407 } 408 409 dev->mac_specified = 1; 410 } 411 out: 412 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, 413 (struct rte_ether_addr *)dev->mac_addr); 414 PMD_DRV_LOG(INFO, "(%s) MAC %s specified", dev->path, buf); 415 } 416 417 static int 418 virtio_user_dev_init_notify(struct virtio_user_dev *dev) 419 { 420 uint32_t i, j, nr_vq; 421 int callfd; 422 int kickfd; 423 424 nr_vq = dev->max_queue_pairs * 2; 425 if (dev->hw_cvq) 426 nr_vq++; 427 428 for (i = 0; i < nr_vq; i++) { 429 /* May use invalid flag, but some backend uses kickfd and 430 * callfd as criteria to judge if dev is alive. so finally we 431 * use real event_fd. 432 */ 433 callfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 434 if (callfd < 0) { 435 PMD_DRV_LOG(ERR, "(%s) callfd error, %s", dev->path, strerror(errno)); 436 goto err; 437 } 438 kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 439 if (kickfd < 0) { 440 close(callfd); 441 PMD_DRV_LOG(ERR, "(%s) kickfd error, %s", dev->path, strerror(errno)); 442 goto err; 443 } 444 dev->callfds[i] = callfd; 445 dev->kickfds[i] = kickfd; 446 } 447 448 if (dev->device_features & (1ULL << VIRTIO_F_NOTIFICATION_DATA)) 449 if (dev->ops->map_notification_area && 450 dev->ops->map_notification_area(dev)) 451 goto err; 452 453 return 0; 454 err: 455 for (j = 0; j < i; j++) { 456 if (dev->kickfds[j] >= 0) { 457 close(dev->kickfds[j]); 458 dev->kickfds[j] = -1; 459 } 460 if (dev->callfds[j] >= 0) { 461 close(dev->callfds[j]); 462 dev->callfds[j] = -1; 463 } 464 } 465 466 return -1; 467 } 468 469 static void 470 virtio_user_dev_uninit_notify(struct virtio_user_dev *dev) 471 { 472 uint32_t i; 473 474 for (i = 0; i < dev->max_queue_pairs * 2; ++i) { 475 if (dev->kickfds[i] >= 0) { 476 close(dev->kickfds[i]); 477 dev->kickfds[i] = -1; 478 } 479 if (dev->callfds[i] >= 0) { 480 close(dev->callfds[i]); 481 dev->callfds[i] = -1; 482 } 483 } 484 if (dev->ops->unmap_notification_area && dev->notify_area) 485 dev->ops->unmap_notification_area(dev); 486 } 487 488 static int 489 virtio_user_fill_intr_handle(struct virtio_user_dev *dev) 490 { 491 uint32_t i; 492 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 493 494 if (eth_dev->intr_handle == NULL) { 495 eth_dev->intr_handle = 496 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE); 497 if (eth_dev->intr_handle == NULL) { 498 PMD_DRV_LOG(ERR, "(%s) failed to allocate intr_handle", dev->path); 499 return -1; 500 } 501 } 502 503 for (i = 0; i < dev->max_queue_pairs; ++i) { 504 if (rte_intr_efds_index_set(eth_dev->intr_handle, i, 505 dev->callfds[2 * i + VTNET_SQ_RQ_QUEUE_IDX])) 506 return -rte_errno; 507 } 508 509 if (rte_intr_nb_efd_set(eth_dev->intr_handle, dev->max_queue_pairs)) 510 return -rte_errno; 511 512 if (rte_intr_max_intr_set(eth_dev->intr_handle, 513 dev->max_queue_pairs + 1)) 514 return -rte_errno; 515 516 if (rte_intr_type_set(eth_dev->intr_handle, RTE_INTR_HANDLE_VDEV)) 517 return -rte_errno; 518 519 /* For virtio vdev, no need to read counter for clean */ 520 if (rte_intr_efd_counter_size_set(eth_dev->intr_handle, 0)) 521 return -rte_errno; 522 523 if (rte_intr_fd_set(eth_dev->intr_handle, dev->ops->get_intr_fd(dev))) 524 return -rte_errno; 525 526 return 0; 527 } 528 529 static void 530 virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused, 531 const void *addr, 532 size_t len __rte_unused, 533 void *arg) 534 { 535 struct virtio_user_dev *dev = arg; 536 struct rte_memseg_list *msl; 537 uint16_t i; 538 int ret = 0; 539 540 /* ignore externally allocated memory */ 541 msl = rte_mem_virt2memseg_list(addr); 542 if (msl->external) 543 return; 544 545 pthread_mutex_lock(&dev->mutex); 546 547 if (dev->started == false) 548 goto exit; 549 550 /* Step 1: pause the active queues */ 551 for (i = 0; i < dev->queue_pairs; i++) { 552 ret = dev->ops->enable_qp(dev, i, 0); 553 if (ret < 0) 554 goto exit; 555 } 556 557 /* Step 2: update memory regions */ 558 ret = dev->ops->set_memory_table(dev); 559 if (ret < 0) 560 goto exit; 561 562 /* Step 3: resume the active queues */ 563 for (i = 0; i < dev->queue_pairs; i++) { 564 ret = dev->ops->enable_qp(dev, i, 1); 565 if (ret < 0) 566 goto exit; 567 } 568 569 exit: 570 pthread_mutex_unlock(&dev->mutex); 571 572 if (ret < 0) 573 PMD_DRV_LOG(ERR, "(%s) Failed to update memory table", dev->path); 574 } 575 576 static int 577 virtio_user_dev_setup(struct virtio_user_dev *dev) 578 { 579 if (dev->is_server) { 580 if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) { 581 PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!"); 582 return -1; 583 } 584 } 585 586 switch (dev->backend_type) { 587 case VIRTIO_USER_BACKEND_VHOST_USER: 588 dev->ops = &virtio_ops_user; 589 break; 590 case VIRTIO_USER_BACKEND_VHOST_KERNEL: 591 dev->ops = &virtio_ops_kernel; 592 break; 593 case VIRTIO_USER_BACKEND_VHOST_VDPA: 594 dev->ops = &virtio_ops_vdpa; 595 break; 596 default: 597 PMD_DRV_LOG(ERR, "(%s) Unknown backend type", dev->path); 598 return -1; 599 } 600 601 if (dev->ops->setup(dev) < 0) { 602 PMD_INIT_LOG(ERR, "(%s) Failed to setup backend", dev->path); 603 return -1; 604 } 605 606 return 0; 607 } 608 609 static int 610 virtio_user_alloc_vrings(struct virtio_user_dev *dev) 611 { 612 int i, size, nr_vrings; 613 bool packed_ring = !!(dev->device_features & (1ull << VIRTIO_F_RING_PACKED)); 614 615 nr_vrings = dev->max_queue_pairs * 2; 616 if (dev->device_features & (1ull << VIRTIO_NET_F_MQ)) 617 nr_vrings++; 618 619 dev->callfds = rte_zmalloc("virtio_user_dev", nr_vrings * sizeof(*dev->callfds), 0); 620 if (!dev->callfds) { 621 PMD_INIT_LOG(ERR, "(%s) Failed to alloc callfds", dev->path); 622 return -1; 623 } 624 625 dev->kickfds = rte_zmalloc("virtio_user_dev", nr_vrings * sizeof(*dev->kickfds), 0); 626 if (!dev->kickfds) { 627 PMD_INIT_LOG(ERR, "(%s) Failed to alloc kickfds", dev->path); 628 goto free_callfds; 629 } 630 631 for (i = 0; i < nr_vrings; i++) { 632 dev->callfds[i] = -1; 633 dev->kickfds[i] = -1; 634 } 635 636 if (packed_ring) 637 size = sizeof(*dev->vrings.packed); 638 else 639 size = sizeof(*dev->vrings.split); 640 dev->vrings.ptr = rte_zmalloc("virtio_user_dev", nr_vrings * size, 0); 641 if (!dev->vrings.ptr) { 642 PMD_INIT_LOG(ERR, "(%s) Failed to alloc vrings metadata", dev->path); 643 goto free_kickfds; 644 } 645 646 if (packed_ring) { 647 dev->packed_queues = rte_zmalloc("virtio_user_dev", 648 nr_vrings * sizeof(*dev->packed_queues), 0); 649 if (!dev->packed_queues) { 650 PMD_INIT_LOG(ERR, "(%s) Failed to alloc packed queues metadata", 651 dev->path); 652 goto free_vrings; 653 } 654 } 655 656 dev->qp_enabled = rte_zmalloc("virtio_user_dev", 657 dev->max_queue_pairs * sizeof(*dev->qp_enabled), 0); 658 if (!dev->qp_enabled) { 659 PMD_INIT_LOG(ERR, "(%s) Failed to alloc QP enable states", dev->path); 660 goto free_packed_queues; 661 } 662 663 return 0; 664 665 free_packed_queues: 666 rte_free(dev->packed_queues); 667 dev->packed_queues = NULL; 668 free_vrings: 669 rte_free(dev->vrings.ptr); 670 dev->vrings.ptr = NULL; 671 free_kickfds: 672 rte_free(dev->kickfds); 673 dev->kickfds = NULL; 674 free_callfds: 675 rte_free(dev->callfds); 676 dev->callfds = NULL; 677 678 return -1; 679 } 680 681 static void 682 virtio_user_free_vrings(struct virtio_user_dev *dev) 683 { 684 rte_free(dev->qp_enabled); 685 dev->qp_enabled = NULL; 686 rte_free(dev->packed_queues); 687 dev->packed_queues = NULL; 688 rte_free(dev->vrings.ptr); 689 dev->vrings.ptr = NULL; 690 rte_free(dev->kickfds); 691 dev->kickfds = NULL; 692 rte_free(dev->callfds); 693 dev->callfds = NULL; 694 } 695 696 /* Use below macro to filter features from vhost backend */ 697 #define VIRTIO_USER_SUPPORTED_FEATURES \ 698 (1ULL << VIRTIO_NET_F_MAC | \ 699 1ULL << VIRTIO_NET_F_STATUS | \ 700 1ULL << VIRTIO_NET_F_MQ | \ 701 1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR | \ 702 1ULL << VIRTIO_NET_F_CTRL_VQ | \ 703 1ULL << VIRTIO_NET_F_CTRL_RX | \ 704 1ULL << VIRTIO_NET_F_CTRL_VLAN | \ 705 1ULL << VIRTIO_NET_F_CSUM | \ 706 1ULL << VIRTIO_NET_F_HOST_TSO4 | \ 707 1ULL << VIRTIO_NET_F_HOST_TSO6 | \ 708 1ULL << VIRTIO_NET_F_MRG_RXBUF | \ 709 1ULL << VIRTIO_RING_F_INDIRECT_DESC | \ 710 1ULL << VIRTIO_NET_F_GUEST_CSUM | \ 711 1ULL << VIRTIO_NET_F_GUEST_TSO4 | \ 712 1ULL << VIRTIO_NET_F_GUEST_TSO6 | \ 713 1ULL << VIRTIO_F_IN_ORDER | \ 714 1ULL << VIRTIO_F_VERSION_1 | \ 715 1ULL << VIRTIO_F_RING_PACKED | \ 716 1ULL << VIRTIO_F_NOTIFICATION_DATA | \ 717 1ULL << VIRTIO_NET_F_RSS) 718 719 int 720 virtio_user_dev_init(struct virtio_user_dev *dev, char *path, uint16_t queues, 721 int cq, int queue_size, const char *mac, char **ifname, 722 int server, int mrg_rxbuf, int in_order, int packed_vq, 723 enum virtio_user_backend_type backend_type) 724 { 725 uint64_t backend_features; 726 727 pthread_mutex_init(&dev->mutex, NULL); 728 strlcpy(dev->path, path, PATH_MAX); 729 730 dev->started = 0; 731 dev->queue_pairs = 1; /* mq disabled by default */ 732 dev->max_queue_pairs = queues; /* initialize to user requested value for kernel backend */ 733 dev->queue_size = queue_size; 734 dev->is_server = server; 735 dev->mac_specified = 0; 736 dev->frontend_features = 0; 737 dev->unsupported_features = 0; 738 dev->backend_type = backend_type; 739 dev->ifname = *ifname; 740 741 if (virtio_user_dev_setup(dev) < 0) { 742 PMD_INIT_LOG(ERR, "(%s) backend set up fails", dev->path); 743 return -1; 744 } 745 746 if (dev->ops->set_owner(dev) < 0) { 747 PMD_INIT_LOG(ERR, "(%s) Failed to set backend owner", dev->path); 748 goto destroy; 749 } 750 751 if (dev->ops->get_backend_features(&backend_features) < 0) { 752 PMD_INIT_LOG(ERR, "(%s) Failed to get backend features", dev->path); 753 goto destroy; 754 } 755 756 dev->unsupported_features = ~(VIRTIO_USER_SUPPORTED_FEATURES | backend_features); 757 758 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 759 PMD_INIT_LOG(ERR, "(%s) Failed to get device features", dev->path); 760 goto destroy; 761 } 762 763 virtio_user_dev_init_mac(dev, mac); 764 765 if (virtio_user_dev_init_max_queue_pairs(dev, queues)) 766 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); 767 768 if (dev->max_queue_pairs > 1 || dev->hw_cvq) 769 cq = 1; 770 771 if (!mrg_rxbuf) 772 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF); 773 774 if (!in_order) 775 dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER); 776 777 if (!packed_vq) 778 dev->unsupported_features |= (1ull << VIRTIO_F_RING_PACKED); 779 780 if (dev->mac_specified) 781 dev->frontend_features |= (1ull << VIRTIO_NET_F_MAC); 782 else 783 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC); 784 785 if (cq) { 786 /* Except for vDPA, the device does not really need to know 787 * anything about CQ, so if necessary, we just claim to support 788 * control queue. 789 */ 790 dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 791 } else { 792 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 793 /* Also disable features that depend on VIRTIO_NET_F_CTRL_VQ */ 794 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX); 795 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN); 796 dev->unsupported_features |= 797 (1ull << VIRTIO_NET_F_GUEST_ANNOUNCE); 798 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); 799 dev->unsupported_features |= 800 (1ull << VIRTIO_NET_F_CTRL_MAC_ADDR); 801 } 802 803 /* The backend will not report this feature, we add it explicitly */ 804 if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) 805 dev->frontend_features |= (1ull << VIRTIO_NET_F_STATUS); 806 807 dev->frontend_features &= ~dev->unsupported_features; 808 dev->device_features &= ~dev->unsupported_features; 809 810 if (virtio_user_alloc_vrings(dev) < 0) { 811 PMD_INIT_LOG(ERR, "(%s) Failed to allocate vring metadata", dev->path); 812 goto destroy; 813 } 814 815 if (virtio_user_dev_init_notify(dev) < 0) { 816 PMD_INIT_LOG(ERR, "(%s) Failed to init notifiers", dev->path); 817 goto free_vrings; 818 } 819 820 if (virtio_user_fill_intr_handle(dev) < 0) { 821 PMD_INIT_LOG(ERR, "(%s) Failed to init interrupt handler", dev->path); 822 goto notify_uninit; 823 } 824 825 if (rte_mem_event_callback_register(VIRTIO_USER_MEM_EVENT_CLB_NAME, 826 virtio_user_mem_event_cb, dev)) { 827 if (rte_errno != ENOTSUP) { 828 PMD_INIT_LOG(ERR, "(%s) Failed to register mem event callback", 829 dev->path); 830 goto notify_uninit; 831 } 832 } 833 834 *ifname = NULL; 835 return 0; 836 837 notify_uninit: 838 virtio_user_dev_uninit_notify(dev); 839 free_vrings: 840 virtio_user_free_vrings(dev); 841 destroy: 842 dev->ops->destroy(dev); 843 844 return -1; 845 } 846 847 void 848 virtio_user_dev_uninit(struct virtio_user_dev *dev) 849 { 850 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 851 852 rte_intr_instance_free(eth_dev->intr_handle); 853 eth_dev->intr_handle = NULL; 854 855 virtio_user_stop_device(dev); 856 857 rte_mem_event_callback_unregister(VIRTIO_USER_MEM_EVENT_CLB_NAME, dev); 858 859 virtio_user_dev_uninit_notify(dev); 860 861 virtio_user_free_vrings(dev); 862 863 free(dev->ifname); 864 865 if (dev->is_server) 866 unlink(dev->path); 867 868 dev->ops->destroy(dev); 869 } 870 871 static uint8_t 872 virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs) 873 { 874 uint16_t i; 875 uint8_t ret = 0; 876 877 if (q_pairs > dev->max_queue_pairs) { 878 PMD_INIT_LOG(ERR, "(%s) multi-q config %u, but only %u supported", 879 dev->path, q_pairs, dev->max_queue_pairs); 880 return -1; 881 } 882 883 for (i = 0; i < q_pairs; ++i) 884 ret |= dev->ops->enable_qp(dev, i, 1); 885 for (i = q_pairs; i < dev->max_queue_pairs; ++i) 886 ret |= dev->ops->enable_qp(dev, i, 0); 887 888 dev->queue_pairs = q_pairs; 889 890 return ret; 891 } 892 893 #define CVQ_MAX_DATA_DESCS 32 894 895 static uint32_t 896 virtio_user_handle_ctrl_msg_split(struct virtio_user_dev *dev, struct vring *vring, 897 uint16_t idx_hdr) 898 { 899 struct virtio_net_ctrl_hdr *hdr; 900 virtio_net_ctrl_ack status = ~0; 901 uint16_t i, idx_data, idx_status; 902 uint32_t n_descs = 0; 903 int dlen[CVQ_MAX_DATA_DESCS], nb_dlen = 0; 904 905 /* locate desc for header, data, and status */ 906 idx_data = vring->desc[idx_hdr].next; 907 n_descs++; 908 909 i = idx_data; 910 while (vring->desc[i].flags == VRING_DESC_F_NEXT) { 911 dlen[nb_dlen++] = vring->desc[i].len; 912 i = vring->desc[i].next; 913 n_descs++; 914 } 915 916 /* locate desc for status */ 917 idx_status = i; 918 n_descs++; 919 920 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 921 if (hdr->class == VIRTIO_NET_CTRL_MQ && 922 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 923 uint16_t queues; 924 925 queues = *(uint16_t *)(uintptr_t)vring->desc[idx_data].addr; 926 status = virtio_user_handle_mq(dev, queues); 927 } else if (hdr->class == VIRTIO_NET_CTRL_MQ && hdr->cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) { 928 struct virtio_net_ctrl_rss *rss; 929 930 rss = (struct virtio_net_ctrl_rss *)(uintptr_t)vring->desc[idx_data].addr; 931 status = virtio_user_handle_mq(dev, rss->max_tx_vq); 932 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 933 hdr->class == VIRTIO_NET_CTRL_MAC || 934 hdr->class == VIRTIO_NET_CTRL_VLAN) { 935 status = 0; 936 } 937 938 if (!status && dev->scvq) 939 status = virtio_send_command(&dev->scvq->cq, 940 (struct virtio_pmd_ctrl *)hdr, dlen, nb_dlen); 941 942 /* Update status */ 943 *(virtio_net_ctrl_ack *)(uintptr_t)vring->desc[idx_status].addr = status; 944 945 return n_descs; 946 } 947 948 static inline int 949 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) 950 { 951 uint16_t flags = rte_atomic_load_explicit(&desc->flags, rte_memory_order_acquire); 952 953 return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) && 954 wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED); 955 } 956 957 static uint32_t 958 virtio_user_handle_ctrl_msg_packed(struct virtio_user_dev *dev, 959 struct vring_packed *vring, 960 uint16_t idx_hdr) 961 { 962 struct virtio_net_ctrl_hdr *hdr; 963 virtio_net_ctrl_ack status = ~0; 964 uint16_t idx_data, idx_status; 965 /* initialize to one, header is first */ 966 uint32_t n_descs = 1; 967 int dlen[CVQ_MAX_DATA_DESCS], nb_dlen = 0; 968 969 /* locate desc for header, data, and status */ 970 idx_data = idx_hdr + 1; 971 if (idx_data >= dev->queue_size) 972 idx_data -= dev->queue_size; 973 974 n_descs++; 975 976 idx_status = idx_data; 977 while (vring->desc[idx_status].flags & VRING_DESC_F_NEXT) { 978 dlen[nb_dlen++] = vring->desc[idx_status].len; 979 idx_status++; 980 if (idx_status >= dev->queue_size) 981 idx_status -= dev->queue_size; 982 n_descs++; 983 } 984 985 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 986 if (hdr->class == VIRTIO_NET_CTRL_MQ && 987 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 988 uint16_t queues; 989 990 queues = *(uint16_t *)(uintptr_t) 991 vring->desc[idx_data].addr; 992 status = virtio_user_handle_mq(dev, queues); 993 } else if (hdr->class == VIRTIO_NET_CTRL_MQ && hdr->cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) { 994 struct virtio_net_ctrl_rss *rss; 995 996 rss = (struct virtio_net_ctrl_rss *)(uintptr_t)vring->desc[idx_data].addr; 997 status = virtio_user_handle_mq(dev, rss->max_tx_vq); 998 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 999 hdr->class == VIRTIO_NET_CTRL_MAC || 1000 hdr->class == VIRTIO_NET_CTRL_VLAN) { 1001 status = 0; 1002 } 1003 1004 if (!status && dev->scvq) 1005 status = virtio_send_command(&dev->scvq->cq, 1006 (struct virtio_pmd_ctrl *)hdr, dlen, nb_dlen); 1007 1008 /* Update status */ 1009 *(virtio_net_ctrl_ack *)(uintptr_t) 1010 vring->desc[idx_status].addr = status; 1011 1012 /* Update used descriptor */ 1013 vring->desc[idx_hdr].id = vring->desc[idx_status].id; 1014 vring->desc[idx_hdr].len = sizeof(status); 1015 1016 return n_descs; 1017 } 1018 1019 static void 1020 virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx) 1021 { 1022 struct virtio_user_queue *vq = &dev->packed_queues[queue_idx]; 1023 struct vring_packed *vring = &dev->vrings.packed[queue_idx]; 1024 uint16_t n_descs, flags; 1025 1026 /* Perform a load-acquire barrier in desc_is_avail to 1027 * enforce the ordering between desc flags and desc 1028 * content. 1029 */ 1030 while (desc_is_avail(&vring->desc[vq->used_idx], 1031 vq->used_wrap_counter)) { 1032 1033 n_descs = virtio_user_handle_ctrl_msg_packed(dev, vring, 1034 vq->used_idx); 1035 1036 flags = VRING_DESC_F_WRITE; 1037 if (vq->used_wrap_counter) 1038 flags |= VRING_PACKED_DESC_F_AVAIL_USED; 1039 1040 rte_atomic_store_explicit(&vring->desc[vq->used_idx].flags, flags, 1041 rte_memory_order_release); 1042 1043 vq->used_idx += n_descs; 1044 if (vq->used_idx >= dev->queue_size) { 1045 vq->used_idx -= dev->queue_size; 1046 vq->used_wrap_counter ^= 1; 1047 } 1048 } 1049 } 1050 1051 static void 1052 virtio_user_handle_cq_split(struct virtio_user_dev *dev, uint16_t queue_idx) 1053 { 1054 uint16_t avail_idx, desc_idx; 1055 struct vring_used_elem *uep; 1056 uint32_t n_descs; 1057 struct vring *vring = &dev->vrings.split[queue_idx]; 1058 1059 /* Consume avail ring, using used ring idx as first one */ 1060 while (rte_atomic_load_explicit(&vring->used->idx, rte_memory_order_relaxed) 1061 != vring->avail->idx) { 1062 avail_idx = rte_atomic_load_explicit(&vring->used->idx, rte_memory_order_relaxed) 1063 & (vring->num - 1); 1064 desc_idx = vring->avail->ring[avail_idx]; 1065 1066 n_descs = virtio_user_handle_ctrl_msg_split(dev, vring, desc_idx); 1067 1068 /* Update used ring */ 1069 uep = &vring->used->ring[avail_idx]; 1070 uep->id = desc_idx; 1071 uep->len = n_descs; 1072 1073 rte_atomic_fetch_add_explicit(&vring->used->idx, 1, rte_memory_order_relaxed); 1074 } 1075 } 1076 1077 void 1078 virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx) 1079 { 1080 if (virtio_with_packed_queue(&dev->hw)) 1081 virtio_user_handle_cq_packed(dev, queue_idx); 1082 else 1083 virtio_user_handle_cq_split(dev, queue_idx); 1084 } 1085 1086 static void 1087 virtio_user_control_queue_notify(struct virtqueue *vq, void *cookie) 1088 { 1089 struct virtio_user_dev *dev = cookie; 1090 uint64_t notify_data = 1; 1091 1092 if (!dev->notify_area) { 1093 if (write(dev->kickfds[vq->vq_queue_index], ¬ify_data, sizeof(notify_data)) < 0) 1094 PMD_DRV_LOG(ERR, "failed to kick backend: %s", 1095 strerror(errno)); 1096 return; 1097 } else if (!virtio_with_feature(&dev->hw, VIRTIO_F_NOTIFICATION_DATA)) { 1098 rte_write16(vq->vq_queue_index, vq->notify_addr); 1099 return; 1100 } 1101 1102 if (virtio_with_packed_queue(&dev->hw)) { 1103 /* Bit[0:15]: vq queue index 1104 * Bit[16:30]: avail index 1105 * Bit[31]: avail wrap counter 1106 */ 1107 notify_data = ((uint32_t)(!!(vq->vq_packed.cached_flags & 1108 VRING_PACKED_DESC_F_AVAIL)) << 31) | 1109 ((uint32_t)vq->vq_avail_idx << 16) | 1110 vq->vq_queue_index; 1111 } else { 1112 /* Bit[0:15]: vq queue index 1113 * Bit[16:31]: avail index 1114 */ 1115 notify_data = ((uint32_t)vq->vq_avail_idx << 16) | 1116 vq->vq_queue_index; 1117 } 1118 rte_write32(notify_data, vq->notify_addr); 1119 } 1120 1121 int 1122 virtio_user_dev_create_shadow_cvq(struct virtio_user_dev *dev, struct virtqueue *vq) 1123 { 1124 char name[VIRTQUEUE_MAX_NAME_SZ]; 1125 struct virtqueue *scvq; 1126 1127 snprintf(name, sizeof(name), "port%d_shadow_cvq", vq->hw->port_id); 1128 scvq = virtqueue_alloc(&dev->hw, vq->vq_queue_index, vq->vq_nentries, 1129 VTNET_CQ, SOCKET_ID_ANY, name); 1130 if (!scvq) { 1131 PMD_INIT_LOG(ERR, "(%s) Failed to alloc shadow control vq\n", dev->path); 1132 return -ENOMEM; 1133 } 1134 1135 scvq->cq.notify_queue = &virtio_user_control_queue_notify; 1136 scvq->cq.notify_cookie = dev; 1137 scvq->notify_addr = vq->notify_addr; 1138 dev->scvq = scvq; 1139 1140 return 0; 1141 } 1142 1143 void 1144 virtio_user_dev_destroy_shadow_cvq(struct virtio_user_dev *dev) 1145 { 1146 if (!dev->scvq) 1147 return; 1148 1149 virtqueue_free(dev->scvq); 1150 dev->scvq = NULL; 1151 } 1152 1153 int 1154 virtio_user_dev_set_status(struct virtio_user_dev *dev, uint8_t status) 1155 { 1156 int ret; 1157 1158 pthread_mutex_lock(&dev->mutex); 1159 dev->status = status; 1160 ret = dev->ops->set_status(dev, status); 1161 if (ret && ret != -ENOTSUP) 1162 PMD_INIT_LOG(ERR, "(%s) Failed to set backend status", dev->path); 1163 1164 pthread_mutex_unlock(&dev->mutex); 1165 return ret; 1166 } 1167 1168 int 1169 virtio_user_dev_update_status(struct virtio_user_dev *dev) 1170 { 1171 int ret; 1172 uint8_t status; 1173 1174 pthread_mutex_lock(&dev->mutex); 1175 1176 ret = dev->ops->get_status(dev, &status); 1177 if (!ret) { 1178 dev->status = status; 1179 PMD_INIT_LOG(DEBUG, "Updated Device Status(0x%08x):\n" 1180 "\t-RESET: %u\n" 1181 "\t-ACKNOWLEDGE: %u\n" 1182 "\t-DRIVER: %u\n" 1183 "\t-DRIVER_OK: %u\n" 1184 "\t-FEATURES_OK: %u\n" 1185 "\t-DEVICE_NEED_RESET: %u\n" 1186 "\t-FAILED: %u", 1187 dev->status, 1188 (dev->status == VIRTIO_CONFIG_STATUS_RESET), 1189 !!(dev->status & VIRTIO_CONFIG_STATUS_ACK), 1190 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER), 1191 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK), 1192 !!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK), 1193 !!(dev->status & VIRTIO_CONFIG_STATUS_DEV_NEED_RESET), 1194 !!(dev->status & VIRTIO_CONFIG_STATUS_FAILED)); 1195 } else if (ret != -ENOTSUP) { 1196 PMD_INIT_LOG(ERR, "(%s) Failed to get backend status", dev->path); 1197 } 1198 1199 pthread_mutex_unlock(&dev->mutex); 1200 return ret; 1201 } 1202 1203 int 1204 virtio_user_dev_update_link_state(struct virtio_user_dev *dev) 1205 { 1206 if (dev->ops->update_link_state) 1207 return dev->ops->update_link_state(dev); 1208 1209 return 0; 1210 } 1211 1212 static void 1213 virtio_user_dev_reset_queues_packed(struct rte_eth_dev *eth_dev) 1214 { 1215 struct virtio_user_dev *dev = eth_dev->data->dev_private; 1216 struct virtio_hw *hw = &dev->hw; 1217 struct virtnet_rx *rxvq; 1218 struct virtnet_tx *txvq; 1219 uint16_t i; 1220 1221 /* Add lock to avoid queue contention. */ 1222 rte_spinlock_lock(&hw->state_lock); 1223 hw->started = 0; 1224 1225 /* 1226 * Waiting for datapath to complete before resetting queues. 1227 * 1 ms should be enough for the ongoing Tx/Rx function to finish. 1228 */ 1229 rte_delay_ms(1); 1230 1231 /* Vring reset for each Tx queue and Rx queue. */ 1232 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 1233 rxvq = eth_dev->data->rx_queues[i]; 1234 virtqueue_rxvq_reset_packed(virtnet_rxq_to_vq(rxvq)); 1235 virtio_dev_rx_queue_setup_finish(eth_dev, i); 1236 } 1237 1238 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 1239 txvq = eth_dev->data->tx_queues[i]; 1240 virtqueue_txvq_reset_packed(virtnet_txq_to_vq(txvq)); 1241 } 1242 1243 hw->started = 1; 1244 rte_spinlock_unlock(&hw->state_lock); 1245 } 1246 1247 void 1248 virtio_user_dev_delayed_disconnect_handler(void *param) 1249 { 1250 struct virtio_user_dev *dev = param; 1251 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1252 1253 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 1254 PMD_DRV_LOG(ERR, "interrupt disable failed"); 1255 return; 1256 } 1257 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d", 1258 rte_intr_fd_get(eth_dev->intr_handle)); 1259 if (rte_intr_callback_unregister(eth_dev->intr_handle, 1260 virtio_interrupt_handler, 1261 eth_dev) != 1) 1262 PMD_DRV_LOG(ERR, "interrupt unregister failed"); 1263 1264 if (dev->is_server) { 1265 if (dev->ops->server_disconnect) 1266 dev->ops->server_disconnect(dev); 1267 1268 rte_intr_fd_set(eth_dev->intr_handle, 1269 dev->ops->get_intr_fd(dev)); 1270 1271 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d", 1272 rte_intr_fd_get(eth_dev->intr_handle)); 1273 1274 if (rte_intr_callback_register(eth_dev->intr_handle, 1275 virtio_interrupt_handler, 1276 eth_dev)) 1277 PMD_DRV_LOG(ERR, "interrupt register failed"); 1278 1279 if (rte_intr_enable(eth_dev->intr_handle) < 0) { 1280 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1281 return; 1282 } 1283 } 1284 } 1285 1286 static void 1287 virtio_user_dev_delayed_intr_reconfig_handler(void *param) 1288 { 1289 struct virtio_user_dev *dev = param; 1290 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1291 1292 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d", 1293 rte_intr_fd_get(eth_dev->intr_handle)); 1294 1295 if (rte_intr_callback_unregister(eth_dev->intr_handle, 1296 virtio_interrupt_handler, 1297 eth_dev) != 1) 1298 PMD_DRV_LOG(ERR, "interrupt unregister failed"); 1299 1300 rte_intr_fd_set(eth_dev->intr_handle, dev->ops->get_intr_fd(dev)); 1301 1302 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d", 1303 rte_intr_fd_get(eth_dev->intr_handle)); 1304 1305 if (rte_intr_callback_register(eth_dev->intr_handle, 1306 virtio_interrupt_handler, eth_dev)) 1307 PMD_DRV_LOG(ERR, "interrupt register failed"); 1308 1309 if (rte_intr_enable(eth_dev->intr_handle) < 0) 1310 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1311 } 1312 1313 int 1314 virtio_user_dev_server_reconnect(struct virtio_user_dev *dev) 1315 { 1316 int ret, old_status; 1317 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1318 struct virtio_hw *hw = &dev->hw; 1319 1320 if (!dev->ops->server_reconnect) { 1321 PMD_DRV_LOG(ERR, "(%s) Missing server reconnect callback", dev->path); 1322 return -1; 1323 } 1324 1325 if (dev->ops->server_reconnect(dev)) { 1326 PMD_DRV_LOG(ERR, "(%s) Reconnect callback call failed", dev->path); 1327 return -1; 1328 } 1329 1330 old_status = dev->status; 1331 1332 virtio_reset(hw); 1333 1334 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 1335 1336 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 1337 1338 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 1339 PMD_INIT_LOG(ERR, "get_features failed: %s", 1340 strerror(errno)); 1341 return -1; 1342 } 1343 1344 /* unmask vhost-user unsupported features */ 1345 dev->device_features &= ~(dev->unsupported_features); 1346 1347 dev->features &= (dev->device_features | dev->frontend_features); 1348 1349 /* For packed ring, resetting queues is required in reconnection. */ 1350 if (virtio_with_packed_queue(hw) && 1351 (old_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) { 1352 PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped" 1353 " when packed ring reconnecting."); 1354 virtio_user_dev_reset_queues_packed(eth_dev); 1355 } 1356 1357 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); 1358 1359 /* Start the device */ 1360 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK); 1361 if (!dev->started) 1362 return -1; 1363 1364 if (dev->queue_pairs > 1) { 1365 ret = virtio_user_handle_mq(dev, dev->queue_pairs); 1366 if (ret != 0) { 1367 PMD_INIT_LOG(ERR, "Fails to enable multi-queue pairs!"); 1368 return -1; 1369 } 1370 } 1371 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { 1372 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 1373 PMD_DRV_LOG(ERR, "interrupt disable failed"); 1374 return -1; 1375 } 1376 /* 1377 * This function can be called from the interrupt handler, so 1378 * we can't unregister interrupt handler here. Setting 1379 * alarm to do that later. 1380 */ 1381 rte_eal_alarm_set(1, 1382 virtio_user_dev_delayed_intr_reconfig_handler, 1383 (void *)dev); 1384 } 1385 PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!"); 1386 return 0; 1387 } 1388