1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <fcntl.h> 9 #include <string.h> 10 #include <errno.h> 11 #include <sys/mman.h> 12 #include <unistd.h> 13 #include <sys/eventfd.h> 14 #include <sys/types.h> 15 #include <sys/stat.h> 16 17 #include <rte_alarm.h> 18 #include <rte_string_fns.h> 19 #include <rte_eal_memconfig.h> 20 21 #include "vhost.h" 22 #include "virtio_user_dev.h" 23 #include "../virtio_ethdev.h" 24 25 #define VIRTIO_USER_MEM_EVENT_CLB_NAME "virtio_user_mem_event_clb" 26 27 const char * const virtio_user_backend_strings[] = { 28 [VIRTIO_USER_BACKEND_UNKNOWN] = "VIRTIO_USER_BACKEND_UNKNOWN", 29 [VIRTIO_USER_BACKEND_VHOST_USER] = "VHOST_USER", 30 [VIRTIO_USER_BACKEND_VHOST_KERNEL] = "VHOST_NET", 31 [VIRTIO_USER_BACKEND_VHOST_VDPA] = "VHOST_VDPA", 32 }; 33 34 static int 35 virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 36 { 37 /* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come 38 * firstly because vhost depends on this msg to allocate virtqueue 39 * pair. 40 */ 41 struct vhost_vring_file file; 42 int ret; 43 44 file.index = queue_sel; 45 file.fd = dev->callfds[queue_sel]; 46 ret = dev->ops->set_vring_call(dev, &file); 47 if (ret < 0) { 48 PMD_INIT_LOG(ERR, "(%s) Failed to create queue %u", dev->path, queue_sel); 49 return -1; 50 } 51 52 return 0; 53 } 54 55 static int 56 virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 57 { 58 int ret; 59 struct vhost_vring_file file; 60 struct vhost_vring_state state; 61 struct vring *vring = &dev->vrings[queue_sel]; 62 struct vring_packed *pq_vring = &dev->packed_vrings[queue_sel]; 63 struct vhost_vring_addr addr = { 64 .index = queue_sel, 65 .log_guest_addr = 0, 66 .flags = 0, /* disable log */ 67 }; 68 69 if (queue_sel == dev->max_queue_pairs * 2) { 70 if (!dev->scvq) { 71 PMD_INIT_LOG(ERR, "(%s) Shadow control queue expected but missing", 72 dev->path); 73 goto err; 74 } 75 76 /* Use shadow control queue information */ 77 vring = &dev->scvq->vq_split.ring; 78 pq_vring = &dev->scvq->vq_packed.ring; 79 } 80 81 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) { 82 addr.desc_user_addr = 83 (uint64_t)(uintptr_t)pq_vring->desc; 84 addr.avail_user_addr = 85 (uint64_t)(uintptr_t)pq_vring->driver; 86 addr.used_user_addr = 87 (uint64_t)(uintptr_t)pq_vring->device; 88 } else { 89 addr.desc_user_addr = (uint64_t)(uintptr_t)vring->desc; 90 addr.avail_user_addr = (uint64_t)(uintptr_t)vring->avail; 91 addr.used_user_addr = (uint64_t)(uintptr_t)vring->used; 92 } 93 94 state.index = queue_sel; 95 state.num = vring->num; 96 ret = dev->ops->set_vring_num(dev, &state); 97 if (ret < 0) 98 goto err; 99 100 state.index = queue_sel; 101 state.num = 0; /* no reservation */ 102 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) 103 state.num |= (1 << 15); 104 ret = dev->ops->set_vring_base(dev, &state); 105 if (ret < 0) 106 goto err; 107 108 ret = dev->ops->set_vring_addr(dev, &addr); 109 if (ret < 0) 110 goto err; 111 112 /* Of all per virtqueue MSGs, make sure VHOST_USER_SET_VRING_KICK comes 113 * lastly because vhost depends on this msg to judge if 114 * virtio is ready. 115 */ 116 file.index = queue_sel; 117 file.fd = dev->kickfds[queue_sel]; 118 ret = dev->ops->set_vring_kick(dev, &file); 119 if (ret < 0) 120 goto err; 121 122 return 0; 123 err: 124 PMD_INIT_LOG(ERR, "(%s) Failed to kick queue %u", dev->path, queue_sel); 125 126 return -1; 127 } 128 129 static int 130 virtio_user_queue_setup(struct virtio_user_dev *dev, 131 int (*fn)(struct virtio_user_dev *, uint32_t)) 132 { 133 uint32_t i, nr_vq; 134 135 nr_vq = dev->max_queue_pairs * 2; 136 if (dev->hw_cvq) 137 nr_vq++; 138 139 for (i = 0; i < nr_vq; i++) { 140 if (fn(dev, i) < 0) { 141 PMD_DRV_LOG(ERR, "(%s) setup VQ %u failed", dev->path, i); 142 return -1; 143 } 144 } 145 146 return 0; 147 } 148 149 int 150 virtio_user_dev_set_features(struct virtio_user_dev *dev) 151 { 152 uint64_t features; 153 int ret = -1; 154 155 pthread_mutex_lock(&dev->mutex); 156 157 /* Step 0: tell vhost to create queues */ 158 if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0) 159 goto error; 160 161 features = dev->features; 162 163 /* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init */ 164 features &= ~(1ull << VIRTIO_NET_F_MAC); 165 /* Strip VIRTIO_NET_F_CTRL_VQ if the devices does not really support control VQ */ 166 if (!dev->hw_cvq) 167 features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ); 168 features &= ~(1ull << VIRTIO_NET_F_STATUS); 169 ret = dev->ops->set_features(dev, features); 170 if (ret < 0) 171 goto error; 172 PMD_DRV_LOG(INFO, "(%s) set features: 0x%" PRIx64, dev->path, features); 173 error: 174 pthread_mutex_unlock(&dev->mutex); 175 176 return ret; 177 } 178 179 int 180 virtio_user_start_device(struct virtio_user_dev *dev) 181 { 182 int ret; 183 184 /* 185 * XXX workaround! 186 * 187 * We need to make sure that the locks will be 188 * taken in the correct order to avoid deadlocks. 189 * 190 * Before releasing this lock, this thread should 191 * not trigger any memory hotplug events. 192 * 193 * This is a temporary workaround, and should be 194 * replaced when we get proper supports from the 195 * memory subsystem in the future. 196 */ 197 rte_mcfg_mem_read_lock(); 198 pthread_mutex_lock(&dev->mutex); 199 200 /* Step 2: share memory regions */ 201 ret = dev->ops->set_memory_table(dev); 202 if (ret < 0) 203 goto error; 204 205 /* Step 3: kick queues */ 206 ret = virtio_user_queue_setup(dev, virtio_user_kick_queue); 207 if (ret < 0) 208 goto error; 209 210 /* Step 4: enable queues 211 * we enable the 1st queue pair by default. 212 */ 213 ret = dev->ops->enable_qp(dev, 0, 1); 214 if (ret < 0) 215 goto error; 216 217 dev->started = true; 218 219 pthread_mutex_unlock(&dev->mutex); 220 rte_mcfg_mem_read_unlock(); 221 222 return 0; 223 error: 224 pthread_mutex_unlock(&dev->mutex); 225 rte_mcfg_mem_read_unlock(); 226 227 PMD_INIT_LOG(ERR, "(%s) Failed to start device", dev->path); 228 229 /* TODO: free resource here or caller to check */ 230 return -1; 231 } 232 233 int virtio_user_stop_device(struct virtio_user_dev *dev) 234 { 235 struct vhost_vring_state state; 236 uint32_t i; 237 int ret; 238 239 pthread_mutex_lock(&dev->mutex); 240 if (!dev->started) 241 goto out; 242 243 for (i = 0; i < dev->max_queue_pairs; ++i) { 244 ret = dev->ops->enable_qp(dev, i, 0); 245 if (ret < 0) 246 goto err; 247 } 248 249 /* Stop the backend. */ 250 for (i = 0; i < dev->max_queue_pairs * 2; ++i) { 251 state.index = i; 252 ret = dev->ops->get_vring_base(dev, &state); 253 if (ret < 0) { 254 PMD_DRV_LOG(ERR, "(%s) get_vring_base failed, index=%u", dev->path, i); 255 goto err; 256 } 257 } 258 259 dev->started = false; 260 261 out: 262 pthread_mutex_unlock(&dev->mutex); 263 264 return 0; 265 err: 266 pthread_mutex_unlock(&dev->mutex); 267 268 PMD_INIT_LOG(ERR, "(%s) Failed to stop device", dev->path); 269 270 return -1; 271 } 272 273 static int 274 virtio_user_dev_init_max_queue_pairs(struct virtio_user_dev *dev, uint32_t user_max_qp) 275 { 276 int ret; 277 278 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MQ))) { 279 dev->max_queue_pairs = 1; 280 return 0; 281 } 282 283 if (!dev->ops->get_config) { 284 dev->max_queue_pairs = user_max_qp; 285 return 0; 286 } 287 288 ret = dev->ops->get_config(dev, (uint8_t *)&dev->max_queue_pairs, 289 offsetof(struct virtio_net_config, max_virtqueue_pairs), 290 sizeof(uint16_t)); 291 if (ret) { 292 /* 293 * We need to know the max queue pair from the device so that 294 * the control queue gets the right index. 295 */ 296 dev->max_queue_pairs = 1; 297 PMD_DRV_LOG(ERR, "(%s) Failed to get max queue pairs from device", dev->path); 298 299 return ret; 300 } 301 302 if (dev->max_queue_pairs > VIRTIO_MAX_VIRTQUEUE_PAIRS) { 303 /* 304 * If the device supports control queue, the control queue 305 * index is max_virtqueue_pairs * 2. Disable MQ if it happens. 306 */ 307 PMD_DRV_LOG(ERR, "(%s) Device advertises too many queues (%u, max supported %u)", 308 dev->path, dev->max_queue_pairs, VIRTIO_MAX_VIRTQUEUE_PAIRS); 309 dev->max_queue_pairs = 1; 310 311 return -1; 312 } 313 314 return 0; 315 } 316 317 int 318 virtio_user_dev_set_mac(struct virtio_user_dev *dev) 319 { 320 int ret = 0; 321 322 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MAC))) 323 return -ENOTSUP; 324 325 if (!dev->ops->set_config) 326 return -ENOTSUP; 327 328 ret = dev->ops->set_config(dev, dev->mac_addr, 329 offsetof(struct virtio_net_config, mac), 330 RTE_ETHER_ADDR_LEN); 331 if (ret) 332 PMD_DRV_LOG(ERR, "(%s) Failed to set MAC address in device", dev->path); 333 334 return ret; 335 } 336 337 int 338 virtio_user_dev_get_mac(struct virtio_user_dev *dev) 339 { 340 int ret = 0; 341 342 if (!(dev->device_features & (1ULL << VIRTIO_NET_F_MAC))) 343 return -ENOTSUP; 344 345 if (!dev->ops->get_config) 346 return -ENOTSUP; 347 348 ret = dev->ops->get_config(dev, dev->mac_addr, 349 offsetof(struct virtio_net_config, mac), 350 RTE_ETHER_ADDR_LEN); 351 if (ret) 352 PMD_DRV_LOG(ERR, "(%s) Failed to get MAC address from device", dev->path); 353 354 return ret; 355 } 356 357 static void 358 virtio_user_dev_init_mac(struct virtio_user_dev *dev, const char *mac) 359 { 360 struct rte_ether_addr cmdline_mac; 361 char buf[RTE_ETHER_ADDR_FMT_SIZE]; 362 int ret; 363 364 if (mac && rte_ether_unformat_addr(mac, &cmdline_mac) == 0) { 365 /* 366 * MAC address was passed from command-line, try to store 367 * it in the device if it supports it. Otherwise try to use 368 * the device one. 369 */ 370 memcpy(dev->mac_addr, &cmdline_mac, RTE_ETHER_ADDR_LEN); 371 dev->mac_specified = 1; 372 373 /* Setting MAC may fail, continue to get the device one in this case */ 374 virtio_user_dev_set_mac(dev); 375 ret = virtio_user_dev_get_mac(dev); 376 if (ret == -ENOTSUP) 377 goto out; 378 379 if (memcmp(&cmdline_mac, dev->mac_addr, RTE_ETHER_ADDR_LEN)) 380 PMD_DRV_LOG(INFO, "(%s) Device MAC update failed", dev->path); 381 } else { 382 ret = virtio_user_dev_get_mac(dev); 383 if (ret) { 384 PMD_DRV_LOG(ERR, "(%s) No valid MAC in devargs or device, use random", 385 dev->path); 386 return; 387 } 388 389 dev->mac_specified = 1; 390 } 391 out: 392 rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, 393 (struct rte_ether_addr *)dev->mac_addr); 394 PMD_DRV_LOG(INFO, "(%s) MAC %s specified", dev->path, buf); 395 } 396 397 static int 398 virtio_user_dev_init_notify(struct virtio_user_dev *dev) 399 { 400 uint32_t i, j, nr_vq; 401 int callfd; 402 int kickfd; 403 404 nr_vq = dev->max_queue_pairs * 2; 405 if (dev->hw_cvq) 406 nr_vq++; 407 408 for (i = 0; i < nr_vq; i++) { 409 /* May use invalid flag, but some backend uses kickfd and 410 * callfd as criteria to judge if dev is alive. so finally we 411 * use real event_fd. 412 */ 413 callfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 414 if (callfd < 0) { 415 PMD_DRV_LOG(ERR, "(%s) callfd error, %s", dev->path, strerror(errno)); 416 goto err; 417 } 418 kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 419 if (kickfd < 0) { 420 close(callfd); 421 PMD_DRV_LOG(ERR, "(%s) kickfd error, %s", dev->path, strerror(errno)); 422 goto err; 423 } 424 dev->callfds[i] = callfd; 425 dev->kickfds[i] = kickfd; 426 } 427 428 return 0; 429 err: 430 for (j = 0; j < i; j++) { 431 if (dev->kickfds[j] >= 0) { 432 close(dev->kickfds[j]); 433 dev->kickfds[j] = -1; 434 } 435 if (dev->callfds[j] >= 0) { 436 close(dev->callfds[j]); 437 dev->callfds[j] = -1; 438 } 439 } 440 441 return -1; 442 } 443 444 static void 445 virtio_user_dev_uninit_notify(struct virtio_user_dev *dev) 446 { 447 uint32_t i; 448 449 for (i = 0; i < dev->max_queue_pairs * 2; ++i) { 450 if (dev->kickfds[i] >= 0) { 451 close(dev->kickfds[i]); 452 dev->kickfds[i] = -1; 453 } 454 if (dev->callfds[i] >= 0) { 455 close(dev->callfds[i]); 456 dev->callfds[i] = -1; 457 } 458 } 459 } 460 461 static int 462 virtio_user_fill_intr_handle(struct virtio_user_dev *dev) 463 { 464 uint32_t i; 465 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 466 467 if (eth_dev->intr_handle == NULL) { 468 eth_dev->intr_handle = 469 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE); 470 if (eth_dev->intr_handle == NULL) { 471 PMD_DRV_LOG(ERR, "(%s) failed to allocate intr_handle", dev->path); 472 return -1; 473 } 474 } 475 476 for (i = 0; i < dev->max_queue_pairs; ++i) { 477 if (rte_intr_efds_index_set(eth_dev->intr_handle, i, 478 dev->callfds[2 * i + VTNET_SQ_RQ_QUEUE_IDX])) 479 return -rte_errno; 480 } 481 482 if (rte_intr_nb_efd_set(eth_dev->intr_handle, dev->max_queue_pairs)) 483 return -rte_errno; 484 485 if (rte_intr_max_intr_set(eth_dev->intr_handle, 486 dev->max_queue_pairs + 1)) 487 return -rte_errno; 488 489 if (rte_intr_type_set(eth_dev->intr_handle, RTE_INTR_HANDLE_VDEV)) 490 return -rte_errno; 491 492 /* For virtio vdev, no need to read counter for clean */ 493 if (rte_intr_efd_counter_size_set(eth_dev->intr_handle, 0)) 494 return -rte_errno; 495 496 if (rte_intr_fd_set(eth_dev->intr_handle, dev->ops->get_intr_fd(dev))) 497 return -rte_errno; 498 499 return 0; 500 } 501 502 static void 503 virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused, 504 const void *addr, 505 size_t len __rte_unused, 506 void *arg) 507 { 508 struct virtio_user_dev *dev = arg; 509 struct rte_memseg_list *msl; 510 uint16_t i; 511 int ret = 0; 512 513 /* ignore externally allocated memory */ 514 msl = rte_mem_virt2memseg_list(addr); 515 if (msl->external) 516 return; 517 518 pthread_mutex_lock(&dev->mutex); 519 520 if (dev->started == false) 521 goto exit; 522 523 /* Step 1: pause the active queues */ 524 for (i = 0; i < dev->queue_pairs; i++) { 525 ret = dev->ops->enable_qp(dev, i, 0); 526 if (ret < 0) 527 goto exit; 528 } 529 530 /* Step 2: update memory regions */ 531 ret = dev->ops->set_memory_table(dev); 532 if (ret < 0) 533 goto exit; 534 535 /* Step 3: resume the active queues */ 536 for (i = 0; i < dev->queue_pairs; i++) { 537 ret = dev->ops->enable_qp(dev, i, 1); 538 if (ret < 0) 539 goto exit; 540 } 541 542 exit: 543 pthread_mutex_unlock(&dev->mutex); 544 545 if (ret < 0) 546 PMD_DRV_LOG(ERR, "(%s) Failed to update memory table", dev->path); 547 } 548 549 static int 550 virtio_user_dev_setup(struct virtio_user_dev *dev) 551 { 552 if (dev->is_server) { 553 if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) { 554 PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!"); 555 return -1; 556 } 557 } 558 559 switch (dev->backend_type) { 560 case VIRTIO_USER_BACKEND_VHOST_USER: 561 dev->ops = &virtio_ops_user; 562 break; 563 case VIRTIO_USER_BACKEND_VHOST_KERNEL: 564 dev->ops = &virtio_ops_kernel; 565 break; 566 case VIRTIO_USER_BACKEND_VHOST_VDPA: 567 dev->ops = &virtio_ops_vdpa; 568 break; 569 default: 570 PMD_DRV_LOG(ERR, "(%s) Unknown backend type", dev->path); 571 return -1; 572 } 573 574 if (dev->ops->setup(dev) < 0) { 575 PMD_INIT_LOG(ERR, "(%s) Failed to setup backend", dev->path); 576 return -1; 577 } 578 579 return 0; 580 } 581 582 /* Use below macro to filter features from vhost backend */ 583 #define VIRTIO_USER_SUPPORTED_FEATURES \ 584 (1ULL << VIRTIO_NET_F_MAC | \ 585 1ULL << VIRTIO_NET_F_STATUS | \ 586 1ULL << VIRTIO_NET_F_MQ | \ 587 1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR | \ 588 1ULL << VIRTIO_NET_F_CTRL_VQ | \ 589 1ULL << VIRTIO_NET_F_CTRL_RX | \ 590 1ULL << VIRTIO_NET_F_CTRL_VLAN | \ 591 1ULL << VIRTIO_NET_F_CSUM | \ 592 1ULL << VIRTIO_NET_F_HOST_TSO4 | \ 593 1ULL << VIRTIO_NET_F_HOST_TSO6 | \ 594 1ULL << VIRTIO_NET_F_MRG_RXBUF | \ 595 1ULL << VIRTIO_RING_F_INDIRECT_DESC | \ 596 1ULL << VIRTIO_NET_F_GUEST_CSUM | \ 597 1ULL << VIRTIO_NET_F_GUEST_TSO4 | \ 598 1ULL << VIRTIO_NET_F_GUEST_TSO6 | \ 599 1ULL << VIRTIO_F_IN_ORDER | \ 600 1ULL << VIRTIO_F_VERSION_1 | \ 601 1ULL << VIRTIO_F_RING_PACKED) 602 603 int 604 virtio_user_dev_init(struct virtio_user_dev *dev, char *path, uint16_t queues, 605 int cq, int queue_size, const char *mac, char **ifname, 606 int server, int mrg_rxbuf, int in_order, int packed_vq, 607 enum virtio_user_backend_type backend_type) 608 { 609 uint64_t backend_features; 610 int i; 611 612 pthread_mutex_init(&dev->mutex, NULL); 613 strlcpy(dev->path, path, PATH_MAX); 614 615 for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; i++) { 616 dev->kickfds[i] = -1; 617 dev->callfds[i] = -1; 618 } 619 620 dev->started = 0; 621 dev->queue_pairs = 1; /* mq disabled by default */ 622 dev->queue_size = queue_size; 623 dev->is_server = server; 624 dev->mac_specified = 0; 625 dev->frontend_features = 0; 626 dev->unsupported_features = 0; 627 dev->backend_type = backend_type; 628 629 if (*ifname) { 630 dev->ifname = *ifname; 631 *ifname = NULL; 632 } 633 634 if (virtio_user_dev_setup(dev) < 0) { 635 PMD_INIT_LOG(ERR, "(%s) backend set up fails", dev->path); 636 return -1; 637 } 638 639 if (dev->ops->set_owner(dev) < 0) { 640 PMD_INIT_LOG(ERR, "(%s) Failed to set backend owner", dev->path); 641 goto destroy; 642 } 643 644 if (dev->ops->get_backend_features(&backend_features) < 0) { 645 PMD_INIT_LOG(ERR, "(%s) Failed to get backend features", dev->path); 646 goto destroy; 647 } 648 649 dev->unsupported_features = ~(VIRTIO_USER_SUPPORTED_FEATURES | backend_features); 650 651 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 652 PMD_INIT_LOG(ERR, "(%s) Failed to get device features", dev->path); 653 goto destroy; 654 } 655 656 virtio_user_dev_init_mac(dev, mac); 657 658 if (virtio_user_dev_init_max_queue_pairs(dev, queues)) 659 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); 660 661 if (dev->max_queue_pairs > 1) 662 cq = 1; 663 664 if (virtio_user_dev_init_notify(dev) < 0) { 665 PMD_INIT_LOG(ERR, "(%s) Failed to init notifiers", dev->path); 666 goto destroy; 667 } 668 669 if (virtio_user_fill_intr_handle(dev) < 0) { 670 PMD_INIT_LOG(ERR, "(%s) Failed to init interrupt handler", dev->path); 671 goto notify_uninit; 672 } 673 674 if (!mrg_rxbuf) 675 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF); 676 677 if (!in_order) 678 dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER); 679 680 if (!packed_vq) 681 dev->unsupported_features |= (1ull << VIRTIO_F_RING_PACKED); 682 683 if (dev->mac_specified) 684 dev->frontend_features |= (1ull << VIRTIO_NET_F_MAC); 685 else 686 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC); 687 688 if (cq) { 689 /* device does not really need to know anything about CQ, 690 * so if necessary, we just claim to support CQ 691 */ 692 dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 693 } else { 694 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 695 /* Also disable features that depend on VIRTIO_NET_F_CTRL_VQ */ 696 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX); 697 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN); 698 dev->unsupported_features |= 699 (1ull << VIRTIO_NET_F_GUEST_ANNOUNCE); 700 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); 701 dev->unsupported_features |= 702 (1ull << VIRTIO_NET_F_CTRL_MAC_ADDR); 703 } 704 705 /* The backend will not report this feature, we add it explicitly */ 706 if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) 707 dev->frontend_features |= (1ull << VIRTIO_NET_F_STATUS); 708 709 dev->frontend_features &= ~dev->unsupported_features; 710 dev->device_features &= ~dev->unsupported_features; 711 712 if (rte_mem_event_callback_register(VIRTIO_USER_MEM_EVENT_CLB_NAME, 713 virtio_user_mem_event_cb, dev)) { 714 if (rte_errno != ENOTSUP) { 715 PMD_INIT_LOG(ERR, "(%s) Failed to register mem event callback", 716 dev->path); 717 goto notify_uninit; 718 } 719 } 720 721 return 0; 722 723 notify_uninit: 724 virtio_user_dev_uninit_notify(dev); 725 destroy: 726 dev->ops->destroy(dev); 727 728 return -1; 729 } 730 731 void 732 virtio_user_dev_uninit(struct virtio_user_dev *dev) 733 { 734 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 735 736 rte_intr_instance_free(eth_dev->intr_handle); 737 eth_dev->intr_handle = NULL; 738 739 virtio_user_stop_device(dev); 740 741 rte_mem_event_callback_unregister(VIRTIO_USER_MEM_EVENT_CLB_NAME, dev); 742 743 virtio_user_dev_uninit_notify(dev); 744 745 free(dev->ifname); 746 747 if (dev->is_server) 748 unlink(dev->path); 749 750 dev->ops->destroy(dev); 751 } 752 753 uint8_t 754 virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs) 755 { 756 uint16_t i; 757 uint8_t ret = 0; 758 759 if (q_pairs > dev->max_queue_pairs) { 760 PMD_INIT_LOG(ERR, "(%s) multi-q config %u, but only %u supported", 761 dev->path, q_pairs, dev->max_queue_pairs); 762 return -1; 763 } 764 765 for (i = 0; i < q_pairs; ++i) 766 ret |= dev->ops->enable_qp(dev, i, 1); 767 for (i = q_pairs; i < dev->max_queue_pairs; ++i) 768 ret |= dev->ops->enable_qp(dev, i, 0); 769 770 if (dev->scvq) 771 ret |= dev->ops->cvq_enable(dev, 1); 772 773 dev->queue_pairs = q_pairs; 774 775 return ret; 776 } 777 778 static uint32_t 779 virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring, 780 uint16_t idx_hdr) 781 { 782 struct virtio_net_ctrl_hdr *hdr; 783 virtio_net_ctrl_ack status = ~0; 784 uint16_t i, idx_data, idx_status; 785 uint32_t n_descs = 0; 786 787 /* locate desc for header, data, and status */ 788 idx_data = vring->desc[idx_hdr].next; 789 n_descs++; 790 791 i = idx_data; 792 while (vring->desc[i].flags == VRING_DESC_F_NEXT) { 793 i = vring->desc[i].next; 794 n_descs++; 795 } 796 797 /* locate desc for status */ 798 idx_status = i; 799 n_descs++; 800 801 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 802 if (hdr->class == VIRTIO_NET_CTRL_MQ && 803 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 804 uint16_t queues; 805 806 queues = *(uint16_t *)(uintptr_t)vring->desc[idx_data].addr; 807 status = virtio_user_handle_mq(dev, queues); 808 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 809 hdr->class == VIRTIO_NET_CTRL_MAC || 810 hdr->class == VIRTIO_NET_CTRL_VLAN) { 811 status = 0; 812 } 813 814 /* Update status */ 815 *(virtio_net_ctrl_ack *)(uintptr_t)vring->desc[idx_status].addr = status; 816 817 return n_descs; 818 } 819 820 static inline int 821 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) 822 { 823 uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE); 824 825 return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) && 826 wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED); 827 } 828 829 static uint32_t 830 virtio_user_handle_ctrl_msg_packed(struct virtio_user_dev *dev, 831 struct vring_packed *vring, 832 uint16_t idx_hdr) 833 { 834 struct virtio_net_ctrl_hdr *hdr; 835 virtio_net_ctrl_ack status = ~0; 836 uint16_t idx_data, idx_status; 837 /* initialize to one, header is first */ 838 uint32_t n_descs = 1; 839 840 /* locate desc for header, data, and status */ 841 idx_data = idx_hdr + 1; 842 if (idx_data >= dev->queue_size) 843 idx_data -= dev->queue_size; 844 845 n_descs++; 846 847 idx_status = idx_data; 848 while (vring->desc[idx_status].flags & VRING_DESC_F_NEXT) { 849 idx_status++; 850 if (idx_status >= dev->queue_size) 851 idx_status -= dev->queue_size; 852 n_descs++; 853 } 854 855 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 856 if (hdr->class == VIRTIO_NET_CTRL_MQ && 857 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 858 uint16_t queues; 859 860 queues = *(uint16_t *)(uintptr_t) 861 vring->desc[idx_data].addr; 862 status = virtio_user_handle_mq(dev, queues); 863 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 864 hdr->class == VIRTIO_NET_CTRL_MAC || 865 hdr->class == VIRTIO_NET_CTRL_VLAN) { 866 status = 0; 867 } 868 869 /* Update status */ 870 *(virtio_net_ctrl_ack *)(uintptr_t) 871 vring->desc[idx_status].addr = status; 872 873 /* Update used descriptor */ 874 vring->desc[idx_hdr].id = vring->desc[idx_status].id; 875 vring->desc[idx_hdr].len = sizeof(status); 876 877 return n_descs; 878 } 879 880 void 881 virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx) 882 { 883 struct virtio_user_queue *vq = &dev->packed_queues[queue_idx]; 884 struct vring_packed *vring = &dev->packed_vrings[queue_idx]; 885 uint16_t n_descs, flags; 886 887 /* Perform a load-acquire barrier in desc_is_avail to 888 * enforce the ordering between desc flags and desc 889 * content. 890 */ 891 while (desc_is_avail(&vring->desc[vq->used_idx], 892 vq->used_wrap_counter)) { 893 894 n_descs = virtio_user_handle_ctrl_msg_packed(dev, vring, 895 vq->used_idx); 896 897 flags = VRING_DESC_F_WRITE; 898 if (vq->used_wrap_counter) 899 flags |= VRING_PACKED_DESC_F_AVAIL_USED; 900 901 __atomic_store_n(&vring->desc[vq->used_idx].flags, flags, 902 __ATOMIC_RELEASE); 903 904 vq->used_idx += n_descs; 905 if (vq->used_idx >= dev->queue_size) { 906 vq->used_idx -= dev->queue_size; 907 vq->used_wrap_counter ^= 1; 908 } 909 } 910 } 911 912 void 913 virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx) 914 { 915 uint16_t avail_idx, desc_idx; 916 struct vring_used_elem *uep; 917 uint32_t n_descs; 918 struct vring *vring = &dev->vrings[queue_idx]; 919 920 /* Consume avail ring, using used ring idx as first one */ 921 while (__atomic_load_n(&vring->used->idx, __ATOMIC_RELAXED) 922 != vring->avail->idx) { 923 avail_idx = __atomic_load_n(&vring->used->idx, __ATOMIC_RELAXED) 924 & (vring->num - 1); 925 desc_idx = vring->avail->ring[avail_idx]; 926 927 n_descs = virtio_user_handle_ctrl_msg(dev, vring, desc_idx); 928 929 /* Update used ring */ 930 uep = &vring->used->ring[avail_idx]; 931 uep->id = desc_idx; 932 uep->len = n_descs; 933 934 __atomic_add_fetch(&vring->used->idx, 1, __ATOMIC_RELAXED); 935 } 936 } 937 938 static void 939 virtio_user_control_queue_notify(struct virtqueue *vq, void *cookie) 940 { 941 struct virtio_user_dev *dev = cookie; 942 uint64_t buf = 1; 943 944 if (write(dev->kickfds[vq->vq_queue_index], &buf, sizeof(buf)) < 0) 945 PMD_DRV_LOG(ERR, "failed to kick backend: %s", 946 strerror(errno)); 947 } 948 949 int 950 virtio_user_dev_create_shadow_cvq(struct virtio_user_dev *dev, struct virtqueue *vq) 951 { 952 char name[VIRTQUEUE_MAX_NAME_SZ]; 953 struct virtqueue *scvq; 954 955 snprintf(name, sizeof(name), "port%d_shadow_cvq", vq->hw->port_id); 956 scvq = virtqueue_alloc(&dev->hw, vq->vq_queue_index, vq->vq_nentries, 957 VTNET_CQ, SOCKET_ID_ANY, name); 958 if (!scvq) { 959 PMD_INIT_LOG(ERR, "(%s) Failed to alloc shadow control vq\n", dev->path); 960 return -ENOMEM; 961 } 962 963 scvq->cq.notify_queue = &virtio_user_control_queue_notify; 964 scvq->cq.notify_cookie = dev; 965 dev->scvq = scvq; 966 967 return 0; 968 } 969 970 void 971 virtio_user_dev_destroy_shadow_cvq(struct virtio_user_dev *dev) 972 { 973 if (!dev->scvq) 974 return; 975 976 virtqueue_free(dev->scvq); 977 dev->scvq = NULL; 978 } 979 980 int 981 virtio_user_dev_set_status(struct virtio_user_dev *dev, uint8_t status) 982 { 983 int ret; 984 985 pthread_mutex_lock(&dev->mutex); 986 dev->status = status; 987 ret = dev->ops->set_status(dev, status); 988 if (ret && ret != -ENOTSUP) 989 PMD_INIT_LOG(ERR, "(%s) Failed to set backend status", dev->path); 990 991 pthread_mutex_unlock(&dev->mutex); 992 return ret; 993 } 994 995 int 996 virtio_user_dev_update_status(struct virtio_user_dev *dev) 997 { 998 int ret; 999 uint8_t status; 1000 1001 pthread_mutex_lock(&dev->mutex); 1002 1003 ret = dev->ops->get_status(dev, &status); 1004 if (!ret) { 1005 dev->status = status; 1006 PMD_INIT_LOG(DEBUG, "Updated Device Status(0x%08x):\n" 1007 "\t-RESET: %u\n" 1008 "\t-ACKNOWLEDGE: %u\n" 1009 "\t-DRIVER: %u\n" 1010 "\t-DRIVER_OK: %u\n" 1011 "\t-FEATURES_OK: %u\n" 1012 "\t-DEVICE_NEED_RESET: %u\n" 1013 "\t-FAILED: %u", 1014 dev->status, 1015 (dev->status == VIRTIO_CONFIG_STATUS_RESET), 1016 !!(dev->status & VIRTIO_CONFIG_STATUS_ACK), 1017 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER), 1018 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK), 1019 !!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK), 1020 !!(dev->status & VIRTIO_CONFIG_STATUS_DEV_NEED_RESET), 1021 !!(dev->status & VIRTIO_CONFIG_STATUS_FAILED)); 1022 } else if (ret != -ENOTSUP) { 1023 PMD_INIT_LOG(ERR, "(%s) Failed to get backend status", dev->path); 1024 } 1025 1026 pthread_mutex_unlock(&dev->mutex); 1027 return ret; 1028 } 1029 1030 int 1031 virtio_user_dev_update_link_state(struct virtio_user_dev *dev) 1032 { 1033 if (dev->ops->update_link_state) 1034 return dev->ops->update_link_state(dev); 1035 1036 return 0; 1037 } 1038 1039 static void 1040 virtio_user_dev_reset_queues_packed(struct rte_eth_dev *eth_dev) 1041 { 1042 struct virtio_user_dev *dev = eth_dev->data->dev_private; 1043 struct virtio_hw *hw = &dev->hw; 1044 struct virtnet_rx *rxvq; 1045 struct virtnet_tx *txvq; 1046 uint16_t i; 1047 1048 /* Add lock to avoid queue contention. */ 1049 rte_spinlock_lock(&hw->state_lock); 1050 hw->started = 0; 1051 1052 /* 1053 * Waiting for datapath to complete before resetting queues. 1054 * 1 ms should be enough for the ongoing Tx/Rx function to finish. 1055 */ 1056 rte_delay_ms(1); 1057 1058 /* Vring reset for each Tx queue and Rx queue. */ 1059 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 1060 rxvq = eth_dev->data->rx_queues[i]; 1061 virtqueue_rxvq_reset_packed(virtnet_rxq_to_vq(rxvq)); 1062 virtio_dev_rx_queue_setup_finish(eth_dev, i); 1063 } 1064 1065 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 1066 txvq = eth_dev->data->tx_queues[i]; 1067 virtqueue_txvq_reset_packed(virtnet_txq_to_vq(txvq)); 1068 } 1069 1070 hw->started = 1; 1071 rte_spinlock_unlock(&hw->state_lock); 1072 } 1073 1074 void 1075 virtio_user_dev_delayed_disconnect_handler(void *param) 1076 { 1077 struct virtio_user_dev *dev = param; 1078 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1079 1080 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 1081 PMD_DRV_LOG(ERR, "interrupt disable failed"); 1082 return; 1083 } 1084 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d", 1085 rte_intr_fd_get(eth_dev->intr_handle)); 1086 if (rte_intr_callback_unregister(eth_dev->intr_handle, 1087 virtio_interrupt_handler, 1088 eth_dev) != 1) 1089 PMD_DRV_LOG(ERR, "interrupt unregister failed"); 1090 1091 if (dev->is_server) { 1092 if (dev->ops->server_disconnect) 1093 dev->ops->server_disconnect(dev); 1094 1095 rte_intr_fd_set(eth_dev->intr_handle, 1096 dev->ops->get_intr_fd(dev)); 1097 1098 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d", 1099 rte_intr_fd_get(eth_dev->intr_handle)); 1100 1101 if (rte_intr_callback_register(eth_dev->intr_handle, 1102 virtio_interrupt_handler, 1103 eth_dev)) 1104 PMD_DRV_LOG(ERR, "interrupt register failed"); 1105 1106 if (rte_intr_enable(eth_dev->intr_handle) < 0) { 1107 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1108 return; 1109 } 1110 } 1111 } 1112 1113 static void 1114 virtio_user_dev_delayed_intr_reconfig_handler(void *param) 1115 { 1116 struct virtio_user_dev *dev = param; 1117 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1118 1119 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d", 1120 rte_intr_fd_get(eth_dev->intr_handle)); 1121 1122 if (rte_intr_callback_unregister(eth_dev->intr_handle, 1123 virtio_interrupt_handler, 1124 eth_dev) != 1) 1125 PMD_DRV_LOG(ERR, "interrupt unregister failed"); 1126 1127 rte_intr_fd_set(eth_dev->intr_handle, dev->ops->get_intr_fd(dev)); 1128 1129 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d", 1130 rte_intr_fd_get(eth_dev->intr_handle)); 1131 1132 if (rte_intr_callback_register(eth_dev->intr_handle, 1133 virtio_interrupt_handler, eth_dev)) 1134 PMD_DRV_LOG(ERR, "interrupt register failed"); 1135 1136 if (rte_intr_enable(eth_dev->intr_handle) < 0) 1137 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1138 } 1139 1140 int 1141 virtio_user_dev_server_reconnect(struct virtio_user_dev *dev) 1142 { 1143 int ret, old_status; 1144 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 1145 struct virtio_hw *hw = &dev->hw; 1146 1147 if (!dev->ops->server_reconnect) { 1148 PMD_DRV_LOG(ERR, "(%s) Missing server reconnect callback", dev->path); 1149 return -1; 1150 } 1151 1152 if (dev->ops->server_reconnect(dev)) { 1153 PMD_DRV_LOG(ERR, "(%s) Reconnect callback call failed", dev->path); 1154 return -1; 1155 } 1156 1157 old_status = dev->status; 1158 1159 virtio_reset(hw); 1160 1161 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 1162 1163 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 1164 1165 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 1166 PMD_INIT_LOG(ERR, "get_features failed: %s", 1167 strerror(errno)); 1168 return -1; 1169 } 1170 1171 /* unmask vhost-user unsupported features */ 1172 dev->device_features &= ~(dev->unsupported_features); 1173 1174 dev->features &= (dev->device_features | dev->frontend_features); 1175 1176 /* For packed ring, resetting queues is required in reconnection. */ 1177 if (virtio_with_packed_queue(hw) && 1178 (old_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) { 1179 PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped" 1180 " when packed ring reconnecting."); 1181 virtio_user_dev_reset_queues_packed(eth_dev); 1182 } 1183 1184 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); 1185 1186 /* Start the device */ 1187 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK); 1188 if (!dev->started) 1189 return -1; 1190 1191 if (dev->queue_pairs > 1) { 1192 ret = virtio_user_handle_mq(dev, dev->queue_pairs); 1193 if (ret != 0) { 1194 PMD_INIT_LOG(ERR, "Fails to enable multi-queue pairs!"); 1195 return -1; 1196 } 1197 } 1198 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { 1199 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 1200 PMD_DRV_LOG(ERR, "interrupt disable failed"); 1201 return -1; 1202 } 1203 /* 1204 * This function can be called from the interrupt handler, so 1205 * we can't unregister interrupt handler here. Setting 1206 * alarm to do that later. 1207 */ 1208 rte_eal_alarm_set(1, 1209 virtio_user_dev_delayed_intr_reconfig_handler, 1210 (void *)dev); 1211 } 1212 PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!"); 1213 return 0; 1214 } 1215