1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <fcntl.h> 8 #include <string.h> 9 #include <errno.h> 10 #include <sys/mman.h> 11 #include <unistd.h> 12 #include <sys/eventfd.h> 13 #include <sys/types.h> 14 #include <sys/stat.h> 15 16 #include <rte_alarm.h> 17 #include <rte_string_fns.h> 18 #include <rte_eal_memconfig.h> 19 20 #include "vhost.h" 21 #include "virtio_user_dev.h" 22 #include "../virtio_ethdev.h" 23 24 #define VIRTIO_USER_MEM_EVENT_CLB_NAME "virtio_user_mem_event_clb" 25 26 const char * const virtio_user_backend_strings[] = { 27 [VIRTIO_USER_BACKEND_UNKNOWN] = "VIRTIO_USER_BACKEND_UNKNOWN", 28 [VIRTIO_USER_BACKEND_VHOST_USER] = "VHOST_USER", 29 [VIRTIO_USER_BACKEND_VHOST_KERNEL] = "VHOST_NET", 30 [VIRTIO_USER_BACKEND_VHOST_VDPA] = "VHOST_VDPA", 31 }; 32 33 static int 34 virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 35 { 36 /* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come 37 * firstly because vhost depends on this msg to allocate virtqueue 38 * pair. 39 */ 40 struct vhost_vring_file file; 41 int ret; 42 43 file.index = queue_sel; 44 file.fd = dev->callfds[queue_sel]; 45 ret = dev->ops->set_vring_call(dev, &file); 46 if (ret < 0) { 47 PMD_INIT_LOG(ERR, "(%s) Failed to create queue %u\n", dev->path, queue_sel); 48 return -1; 49 } 50 51 return 0; 52 } 53 54 static int 55 virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 56 { 57 int ret; 58 struct vhost_vring_file file; 59 struct vhost_vring_state state; 60 struct vring *vring = &dev->vrings[queue_sel]; 61 struct vring_packed *pq_vring = &dev->packed_vrings[queue_sel]; 62 struct vhost_vring_addr addr = { 63 .index = queue_sel, 64 .log_guest_addr = 0, 65 .flags = 0, /* disable log */ 66 }; 67 68 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) { 69 addr.desc_user_addr = 70 (uint64_t)(uintptr_t)pq_vring->desc; 71 addr.avail_user_addr = 72 (uint64_t)(uintptr_t)pq_vring->driver; 73 addr.used_user_addr = 74 (uint64_t)(uintptr_t)pq_vring->device; 75 } else { 76 addr.desc_user_addr = (uint64_t)(uintptr_t)vring->desc; 77 addr.avail_user_addr = (uint64_t)(uintptr_t)vring->avail; 78 addr.used_user_addr = (uint64_t)(uintptr_t)vring->used; 79 } 80 81 state.index = queue_sel; 82 state.num = vring->num; 83 ret = dev->ops->set_vring_num(dev, &state); 84 if (ret < 0) 85 goto err; 86 87 state.index = queue_sel; 88 state.num = 0; /* no reservation */ 89 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) 90 state.num |= (1 << 15); 91 ret = dev->ops->set_vring_base(dev, &state); 92 if (ret < 0) 93 goto err; 94 95 ret = dev->ops->set_vring_addr(dev, &addr); 96 if (ret < 0) 97 goto err; 98 99 /* Of all per virtqueue MSGs, make sure VHOST_USER_SET_VRING_KICK comes 100 * lastly because vhost depends on this msg to judge if 101 * virtio is ready. 102 */ 103 file.index = queue_sel; 104 file.fd = dev->kickfds[queue_sel]; 105 ret = dev->ops->set_vring_kick(dev, &file); 106 if (ret < 0) 107 goto err; 108 109 return 0; 110 err: 111 PMD_INIT_LOG(ERR, "(%s) Failed to kick queue %u\n", dev->path, queue_sel); 112 113 return -1; 114 } 115 116 static int 117 virtio_user_queue_setup(struct virtio_user_dev *dev, 118 int (*fn)(struct virtio_user_dev *, uint32_t)) 119 { 120 uint32_t i, queue_sel; 121 122 for (i = 0; i < dev->max_queue_pairs; ++i) { 123 queue_sel = 2 * i + VTNET_SQ_RQ_QUEUE_IDX; 124 if (fn(dev, queue_sel) < 0) { 125 PMD_DRV_LOG(ERR, "(%s) setup rx vq %u failed", dev->path, i); 126 return -1; 127 } 128 } 129 for (i = 0; i < dev->max_queue_pairs; ++i) { 130 queue_sel = 2 * i + VTNET_SQ_TQ_QUEUE_IDX; 131 if (fn(dev, queue_sel) < 0) { 132 PMD_DRV_LOG(INFO, "(%s) setup tx vq %u failed", dev->path, i); 133 return -1; 134 } 135 } 136 137 return 0; 138 } 139 140 int 141 virtio_user_dev_set_features(struct virtio_user_dev *dev) 142 { 143 uint64_t features; 144 int ret = -1; 145 146 pthread_mutex_lock(&dev->mutex); 147 148 /* Step 0: tell vhost to create queues */ 149 if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0) 150 goto error; 151 152 features = dev->features; 153 154 /* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init */ 155 features &= ~(1ull << VIRTIO_NET_F_MAC); 156 /* Strip VIRTIO_NET_F_CTRL_VQ, as devices do not really need to know */ 157 features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ); 158 features &= ~(1ull << VIRTIO_NET_F_STATUS); 159 ret = dev->ops->set_features(dev, features); 160 if (ret < 0) 161 goto error; 162 PMD_DRV_LOG(INFO, "(%s) set features: 0x%" PRIx64, dev->path, features); 163 error: 164 pthread_mutex_unlock(&dev->mutex); 165 166 return ret; 167 } 168 169 int 170 virtio_user_start_device(struct virtio_user_dev *dev) 171 { 172 int ret; 173 174 /* 175 * XXX workaround! 176 * 177 * We need to make sure that the locks will be 178 * taken in the correct order to avoid deadlocks. 179 * 180 * Before releasing this lock, this thread should 181 * not trigger any memory hotplug events. 182 * 183 * This is a temporary workaround, and should be 184 * replaced when we get proper supports from the 185 * memory subsystem in the future. 186 */ 187 rte_mcfg_mem_read_lock(); 188 pthread_mutex_lock(&dev->mutex); 189 190 /* Step 2: share memory regions */ 191 ret = dev->ops->set_memory_table(dev); 192 if (ret < 0) 193 goto error; 194 195 /* Step 3: kick queues */ 196 ret = virtio_user_queue_setup(dev, virtio_user_kick_queue); 197 if (ret < 0) 198 goto error; 199 200 /* Step 4: enable queues 201 * we enable the 1st queue pair by default. 202 */ 203 ret = dev->ops->enable_qp(dev, 0, 1); 204 if (ret < 0) 205 goto error; 206 207 dev->started = true; 208 209 pthread_mutex_unlock(&dev->mutex); 210 rte_mcfg_mem_read_unlock(); 211 212 return 0; 213 error: 214 pthread_mutex_unlock(&dev->mutex); 215 rte_mcfg_mem_read_unlock(); 216 217 PMD_INIT_LOG(ERR, "(%s) Failed to start device\n", dev->path); 218 219 /* TODO: free resource here or caller to check */ 220 return -1; 221 } 222 223 int virtio_user_stop_device(struct virtio_user_dev *dev) 224 { 225 struct vhost_vring_state state; 226 uint32_t i; 227 int ret; 228 229 pthread_mutex_lock(&dev->mutex); 230 if (!dev->started) 231 goto out; 232 233 for (i = 0; i < dev->max_queue_pairs; ++i) { 234 ret = dev->ops->enable_qp(dev, i, 0); 235 if (ret < 0) 236 goto err; 237 } 238 239 /* Stop the backend. */ 240 for (i = 0; i < dev->max_queue_pairs * 2; ++i) { 241 state.index = i; 242 ret = dev->ops->get_vring_base(dev, &state); 243 if (ret < 0) { 244 PMD_DRV_LOG(ERR, "(%s) get_vring_base failed, index=%u", dev->path, i); 245 goto err; 246 } 247 } 248 249 dev->started = false; 250 251 out: 252 pthread_mutex_unlock(&dev->mutex); 253 254 return 0; 255 err: 256 pthread_mutex_unlock(&dev->mutex); 257 258 PMD_INIT_LOG(ERR, "(%s) Failed to stop device\n", dev->path); 259 260 return -1; 261 } 262 263 static inline void 264 parse_mac(struct virtio_user_dev *dev, const char *mac) 265 { 266 struct rte_ether_addr tmp; 267 268 if (!mac) 269 return; 270 271 if (rte_ether_unformat_addr(mac, &tmp) == 0) { 272 memcpy(dev->mac_addr, &tmp, RTE_ETHER_ADDR_LEN); 273 dev->mac_specified = 1; 274 } else { 275 /* ignore the wrong mac, use random mac */ 276 PMD_DRV_LOG(ERR, "wrong format of mac: %s", mac); 277 } 278 } 279 280 static int 281 virtio_user_dev_init_notify(struct virtio_user_dev *dev) 282 { 283 uint32_t i, j; 284 int callfd; 285 int kickfd; 286 287 for (i = 0; i < dev->max_queue_pairs * 2; i++) { 288 /* May use invalid flag, but some backend uses kickfd and 289 * callfd as criteria to judge if dev is alive. so finally we 290 * use real event_fd. 291 */ 292 callfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 293 if (callfd < 0) { 294 PMD_DRV_LOG(ERR, "(%s) callfd error, %s", dev->path, strerror(errno)); 295 goto err; 296 } 297 kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 298 if (kickfd < 0) { 299 close(callfd); 300 PMD_DRV_LOG(ERR, "(%s) kickfd error, %s", dev->path, strerror(errno)); 301 goto err; 302 } 303 dev->callfds[i] = callfd; 304 dev->kickfds[i] = kickfd; 305 } 306 307 return 0; 308 err: 309 for (j = 0; j < i; j++) { 310 if (dev->kickfds[j] >= 0) { 311 close(dev->kickfds[j]); 312 dev->kickfds[j] = -1; 313 } 314 if (dev->callfds[j] >= 0) { 315 close(dev->callfds[j]); 316 dev->callfds[j] = -1; 317 } 318 } 319 320 return -1; 321 } 322 323 static void 324 virtio_user_dev_uninit_notify(struct virtio_user_dev *dev) 325 { 326 uint32_t i; 327 328 for (i = 0; i < dev->max_queue_pairs * 2; ++i) { 329 if (dev->kickfds[i] >= 0) { 330 close(dev->kickfds[i]); 331 dev->kickfds[i] = -1; 332 } 333 if (dev->callfds[i] >= 0) { 334 close(dev->callfds[i]); 335 dev->callfds[i] = -1; 336 } 337 } 338 } 339 340 static int 341 virtio_user_fill_intr_handle(struct virtio_user_dev *dev) 342 { 343 uint32_t i; 344 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 345 346 if (!eth_dev->intr_handle) { 347 eth_dev->intr_handle = malloc(sizeof(*eth_dev->intr_handle)); 348 if (!eth_dev->intr_handle) { 349 PMD_DRV_LOG(ERR, "(%s) failed to allocate intr_handle", dev->path); 350 return -1; 351 } 352 memset(eth_dev->intr_handle, 0, sizeof(*eth_dev->intr_handle)); 353 } 354 355 for (i = 0; i < dev->max_queue_pairs; ++i) 356 eth_dev->intr_handle->efds[i] = dev->callfds[i]; 357 eth_dev->intr_handle->nb_efd = dev->max_queue_pairs; 358 eth_dev->intr_handle->max_intr = dev->max_queue_pairs + 1; 359 eth_dev->intr_handle->type = RTE_INTR_HANDLE_VDEV; 360 /* For virtio vdev, no need to read counter for clean */ 361 eth_dev->intr_handle->efd_counter_size = 0; 362 eth_dev->intr_handle->fd = dev->ops->get_intr_fd(dev); 363 364 return 0; 365 } 366 367 static void 368 virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused, 369 const void *addr, 370 size_t len __rte_unused, 371 void *arg) 372 { 373 struct virtio_user_dev *dev = arg; 374 struct rte_memseg_list *msl; 375 uint16_t i; 376 int ret = 0; 377 378 /* ignore externally allocated memory */ 379 msl = rte_mem_virt2memseg_list(addr); 380 if (msl->external) 381 return; 382 383 pthread_mutex_lock(&dev->mutex); 384 385 if (dev->started == false) 386 goto exit; 387 388 /* Step 1: pause the active queues */ 389 for (i = 0; i < dev->queue_pairs; i++) { 390 ret = dev->ops->enable_qp(dev, i, 0); 391 if (ret < 0) 392 goto exit; 393 } 394 395 /* Step 2: update memory regions */ 396 ret = dev->ops->set_memory_table(dev); 397 if (ret < 0) 398 goto exit; 399 400 /* Step 3: resume the active queues */ 401 for (i = 0; i < dev->queue_pairs; i++) { 402 ret = dev->ops->enable_qp(dev, i, 1); 403 if (ret < 0) 404 goto exit; 405 } 406 407 exit: 408 pthread_mutex_unlock(&dev->mutex); 409 410 if (ret < 0) 411 PMD_DRV_LOG(ERR, "(%s) Failed to update memory table\n", dev->path); 412 } 413 414 static int 415 virtio_user_dev_setup(struct virtio_user_dev *dev) 416 { 417 if (dev->is_server) { 418 if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) { 419 PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!"); 420 return -1; 421 } 422 } 423 424 switch (dev->backend_type) { 425 case VIRTIO_USER_BACKEND_VHOST_USER: 426 dev->ops = &virtio_ops_user; 427 break; 428 case VIRTIO_USER_BACKEND_VHOST_KERNEL: 429 dev->ops = &virtio_ops_kernel; 430 break; 431 case VIRTIO_USER_BACKEND_VHOST_VDPA: 432 dev->ops = &virtio_ops_vdpa; 433 break; 434 default: 435 PMD_DRV_LOG(ERR, "(%s) Unknown backend type", dev->path); 436 return -1; 437 } 438 439 if (dev->ops->setup(dev) < 0) { 440 PMD_INIT_LOG(ERR, "(%s) Failed to setup backend\n", dev->path); 441 return -1; 442 } 443 444 if (virtio_user_dev_init_notify(dev) < 0) { 445 PMD_INIT_LOG(ERR, "(%s) Failed to init notifiers\n", dev->path); 446 goto destroy; 447 } 448 449 if (virtio_user_fill_intr_handle(dev) < 0) { 450 PMD_INIT_LOG(ERR, "(%s) Failed to init interrupt handler\n", dev->path); 451 goto uninit; 452 } 453 454 return 0; 455 456 uninit: 457 virtio_user_dev_uninit_notify(dev); 458 destroy: 459 dev->ops->destroy(dev); 460 461 return -1; 462 } 463 464 /* Use below macro to filter features from vhost backend */ 465 #define VIRTIO_USER_SUPPORTED_FEATURES \ 466 (1ULL << VIRTIO_NET_F_MAC | \ 467 1ULL << VIRTIO_NET_F_STATUS | \ 468 1ULL << VIRTIO_NET_F_MQ | \ 469 1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR | \ 470 1ULL << VIRTIO_NET_F_CTRL_VQ | \ 471 1ULL << VIRTIO_NET_F_CTRL_RX | \ 472 1ULL << VIRTIO_NET_F_CTRL_VLAN | \ 473 1ULL << VIRTIO_NET_F_CSUM | \ 474 1ULL << VIRTIO_NET_F_HOST_TSO4 | \ 475 1ULL << VIRTIO_NET_F_HOST_TSO6 | \ 476 1ULL << VIRTIO_NET_F_MRG_RXBUF | \ 477 1ULL << VIRTIO_RING_F_INDIRECT_DESC | \ 478 1ULL << VIRTIO_NET_F_GUEST_CSUM | \ 479 1ULL << VIRTIO_NET_F_GUEST_TSO4 | \ 480 1ULL << VIRTIO_NET_F_GUEST_TSO6 | \ 481 1ULL << VIRTIO_F_IN_ORDER | \ 482 1ULL << VIRTIO_F_VERSION_1 | \ 483 1ULL << VIRTIO_F_RING_PACKED) 484 485 int 486 virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues, 487 int cq, int queue_size, const char *mac, char **ifname, 488 int server, int mrg_rxbuf, int in_order, int packed_vq, 489 enum virtio_user_backend_type backend_type) 490 { 491 uint64_t backend_features; 492 int i; 493 494 pthread_mutex_init(&dev->mutex, NULL); 495 strlcpy(dev->path, path, PATH_MAX); 496 497 for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; i++) { 498 dev->kickfds[i] = -1; 499 dev->callfds[i] = -1; 500 } 501 502 dev->started = 0; 503 dev->max_queue_pairs = queues; 504 dev->queue_pairs = 1; /* mq disabled by default */ 505 dev->queue_size = queue_size; 506 dev->is_server = server; 507 dev->mac_specified = 0; 508 dev->frontend_features = 0; 509 dev->unsupported_features = 0; 510 dev->backend_type = backend_type; 511 512 parse_mac(dev, mac); 513 514 if (*ifname) { 515 dev->ifname = *ifname; 516 *ifname = NULL; 517 } 518 519 if (virtio_user_dev_setup(dev) < 0) { 520 PMD_INIT_LOG(ERR, "(%s) backend set up fails", dev->path); 521 return -1; 522 } 523 524 if (dev->ops->set_owner(dev) < 0) { 525 PMD_INIT_LOG(ERR, "(%s) Failed to set backend owner", dev->path); 526 return -1; 527 } 528 529 if (dev->ops->get_backend_features(&backend_features) < 0) { 530 PMD_INIT_LOG(ERR, "(%s) Failed to get backend features", dev->path); 531 return -1; 532 } 533 534 dev->unsupported_features = ~(VIRTIO_USER_SUPPORTED_FEATURES | backend_features); 535 536 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 537 PMD_INIT_LOG(ERR, "(%s) Failed to get device features", dev->path); 538 return -1; 539 } 540 541 if (!mrg_rxbuf) 542 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF); 543 544 if (!in_order) 545 dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER); 546 547 if (!packed_vq) 548 dev->unsupported_features |= (1ull << VIRTIO_F_RING_PACKED); 549 550 if (dev->mac_specified) 551 dev->frontend_features |= (1ull << VIRTIO_NET_F_MAC); 552 else 553 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC); 554 555 if (cq) { 556 /* device does not really need to know anything about CQ, 557 * so if necessary, we just claim to support CQ 558 */ 559 dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 560 } else { 561 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 562 /* Also disable features that depend on VIRTIO_NET_F_CTRL_VQ */ 563 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX); 564 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN); 565 dev->unsupported_features |= 566 (1ull << VIRTIO_NET_F_GUEST_ANNOUNCE); 567 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); 568 dev->unsupported_features |= 569 (1ull << VIRTIO_NET_F_CTRL_MAC_ADDR); 570 } 571 572 /* The backend will not report this feature, we add it explicitly */ 573 if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) 574 dev->frontend_features |= (1ull << VIRTIO_NET_F_STATUS); 575 576 /* 577 * Device features = 578 * (frontend_features | backend_features) & ~unsupported_features; 579 */ 580 dev->device_features |= dev->frontend_features; 581 dev->device_features &= ~dev->unsupported_features; 582 583 if (rte_mem_event_callback_register(VIRTIO_USER_MEM_EVENT_CLB_NAME, 584 virtio_user_mem_event_cb, dev)) { 585 if (rte_errno != ENOTSUP) { 586 PMD_INIT_LOG(ERR, "(%s) Failed to register mem event callback\n", 587 dev->path); 588 return -1; 589 } 590 } 591 592 return 0; 593 } 594 595 void 596 virtio_user_dev_uninit(struct virtio_user_dev *dev) 597 { 598 virtio_user_stop_device(dev); 599 600 rte_mem_event_callback_unregister(VIRTIO_USER_MEM_EVENT_CLB_NAME, dev); 601 602 virtio_user_dev_uninit_notify(dev); 603 604 free(dev->ifname); 605 606 if (dev->is_server) 607 unlink(dev->path); 608 609 dev->ops->destroy(dev); 610 } 611 612 uint8_t 613 virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs) 614 { 615 uint16_t i; 616 uint8_t ret = 0; 617 618 if (q_pairs > dev->max_queue_pairs) { 619 PMD_INIT_LOG(ERR, "(%s) multi-q config %u, but only %u supported", 620 dev->path, q_pairs, dev->max_queue_pairs); 621 return -1; 622 } 623 624 for (i = 0; i < q_pairs; ++i) 625 ret |= dev->ops->enable_qp(dev, i, 1); 626 for (i = q_pairs; i < dev->max_queue_pairs; ++i) 627 ret |= dev->ops->enable_qp(dev, i, 0); 628 629 dev->queue_pairs = q_pairs; 630 631 return ret; 632 } 633 634 static uint32_t 635 virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring, 636 uint16_t idx_hdr) 637 { 638 struct virtio_net_ctrl_hdr *hdr; 639 virtio_net_ctrl_ack status = ~0; 640 uint16_t i, idx_data, idx_status; 641 uint32_t n_descs = 0; 642 643 /* locate desc for header, data, and status */ 644 idx_data = vring->desc[idx_hdr].next; 645 n_descs++; 646 647 i = idx_data; 648 while (vring->desc[i].flags == VRING_DESC_F_NEXT) { 649 i = vring->desc[i].next; 650 n_descs++; 651 } 652 653 /* locate desc for status */ 654 idx_status = i; 655 n_descs++; 656 657 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 658 if (hdr->class == VIRTIO_NET_CTRL_MQ && 659 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 660 uint16_t queues; 661 662 queues = *(uint16_t *)(uintptr_t)vring->desc[idx_data].addr; 663 status = virtio_user_handle_mq(dev, queues); 664 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 665 hdr->class == VIRTIO_NET_CTRL_MAC || 666 hdr->class == VIRTIO_NET_CTRL_VLAN) { 667 status = 0; 668 } 669 670 /* Update status */ 671 *(virtio_net_ctrl_ack *)(uintptr_t)vring->desc[idx_status].addr = status; 672 673 return n_descs; 674 } 675 676 static inline int 677 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) 678 { 679 uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE); 680 681 return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) && 682 wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED); 683 } 684 685 static uint32_t 686 virtio_user_handle_ctrl_msg_packed(struct virtio_user_dev *dev, 687 struct vring_packed *vring, 688 uint16_t idx_hdr) 689 { 690 struct virtio_net_ctrl_hdr *hdr; 691 virtio_net_ctrl_ack status = ~0; 692 uint16_t idx_data, idx_status; 693 /* initialize to one, header is first */ 694 uint32_t n_descs = 1; 695 696 /* locate desc for header, data, and status */ 697 idx_data = idx_hdr + 1; 698 if (idx_data >= dev->queue_size) 699 idx_data -= dev->queue_size; 700 701 n_descs++; 702 703 idx_status = idx_data; 704 while (vring->desc[idx_status].flags & VRING_DESC_F_NEXT) { 705 idx_status++; 706 if (idx_status >= dev->queue_size) 707 idx_status -= dev->queue_size; 708 n_descs++; 709 } 710 711 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 712 if (hdr->class == VIRTIO_NET_CTRL_MQ && 713 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 714 uint16_t queues; 715 716 queues = *(uint16_t *)(uintptr_t) 717 vring->desc[idx_data].addr; 718 status = virtio_user_handle_mq(dev, queues); 719 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 720 hdr->class == VIRTIO_NET_CTRL_MAC || 721 hdr->class == VIRTIO_NET_CTRL_VLAN) { 722 status = 0; 723 } 724 725 /* Update status */ 726 *(virtio_net_ctrl_ack *)(uintptr_t) 727 vring->desc[idx_status].addr = status; 728 729 /* Update used descriptor */ 730 vring->desc[idx_hdr].id = vring->desc[idx_status].id; 731 vring->desc[idx_hdr].len = sizeof(status); 732 733 return n_descs; 734 } 735 736 void 737 virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx) 738 { 739 struct virtio_user_queue *vq = &dev->packed_queues[queue_idx]; 740 struct vring_packed *vring = &dev->packed_vrings[queue_idx]; 741 uint16_t n_descs, flags; 742 743 /* Perform a load-acquire barrier in desc_is_avail to 744 * enforce the ordering between desc flags and desc 745 * content. 746 */ 747 while (desc_is_avail(&vring->desc[vq->used_idx], 748 vq->used_wrap_counter)) { 749 750 n_descs = virtio_user_handle_ctrl_msg_packed(dev, vring, 751 vq->used_idx); 752 753 flags = VRING_DESC_F_WRITE; 754 if (vq->used_wrap_counter) 755 flags |= VRING_PACKED_DESC_F_AVAIL_USED; 756 757 __atomic_store_n(&vring->desc[vq->used_idx].flags, flags, 758 __ATOMIC_RELEASE); 759 760 vq->used_idx += n_descs; 761 if (vq->used_idx >= dev->queue_size) { 762 vq->used_idx -= dev->queue_size; 763 vq->used_wrap_counter ^= 1; 764 } 765 } 766 } 767 768 void 769 virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx) 770 { 771 uint16_t avail_idx, desc_idx; 772 struct vring_used_elem *uep; 773 uint32_t n_descs; 774 struct vring *vring = &dev->vrings[queue_idx]; 775 776 /* Consume avail ring, using used ring idx as first one */ 777 while (__atomic_load_n(&vring->used->idx, __ATOMIC_RELAXED) 778 != vring->avail->idx) { 779 avail_idx = __atomic_load_n(&vring->used->idx, __ATOMIC_RELAXED) 780 & (vring->num - 1); 781 desc_idx = vring->avail->ring[avail_idx]; 782 783 n_descs = virtio_user_handle_ctrl_msg(dev, vring, desc_idx); 784 785 /* Update used ring */ 786 uep = &vring->used->ring[avail_idx]; 787 uep->id = desc_idx; 788 uep->len = n_descs; 789 790 __atomic_add_fetch(&vring->used->idx, 1, __ATOMIC_RELAXED); 791 } 792 } 793 794 int 795 virtio_user_dev_set_status(struct virtio_user_dev *dev, uint8_t status) 796 { 797 int ret; 798 799 pthread_mutex_lock(&dev->mutex); 800 dev->status = status; 801 ret = dev->ops->set_status(dev, status); 802 if (ret && ret != -ENOTSUP) 803 PMD_INIT_LOG(ERR, "(%s) Failed to set backend status\n", dev->path); 804 805 pthread_mutex_unlock(&dev->mutex); 806 return ret; 807 } 808 809 int 810 virtio_user_dev_update_status(struct virtio_user_dev *dev) 811 { 812 int ret; 813 uint8_t status; 814 815 pthread_mutex_lock(&dev->mutex); 816 817 ret = dev->ops->get_status(dev, &status); 818 if (!ret) { 819 dev->status = status; 820 PMD_INIT_LOG(DEBUG, "Updated Device Status(0x%08x):\n" 821 "\t-RESET: %u\n" 822 "\t-ACKNOWLEDGE: %u\n" 823 "\t-DRIVER: %u\n" 824 "\t-DRIVER_OK: %u\n" 825 "\t-FEATURES_OK: %u\n" 826 "\t-DEVICE_NEED_RESET: %u\n" 827 "\t-FAILED: %u\n", 828 dev->status, 829 (dev->status == VIRTIO_CONFIG_STATUS_RESET), 830 !!(dev->status & VIRTIO_CONFIG_STATUS_ACK), 831 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER), 832 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK), 833 !!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK), 834 !!(dev->status & VIRTIO_CONFIG_STATUS_DEV_NEED_RESET), 835 !!(dev->status & VIRTIO_CONFIG_STATUS_FAILED)); 836 } else if (ret != -ENOTSUP) { 837 PMD_INIT_LOG(ERR, "(%s) Failed to get backend status\n", dev->path); 838 } 839 840 pthread_mutex_unlock(&dev->mutex); 841 return ret; 842 } 843 844 int 845 virtio_user_dev_update_link_state(struct virtio_user_dev *dev) 846 { 847 if (dev->ops->update_link_state) 848 return dev->ops->update_link_state(dev); 849 850 return 0; 851 } 852 853 static void 854 virtio_user_dev_reset_queues_packed(struct rte_eth_dev *eth_dev) 855 { 856 struct virtio_user_dev *dev = eth_dev->data->dev_private; 857 struct virtio_hw *hw = &dev->hw; 858 struct virtnet_rx *rxvq; 859 struct virtnet_tx *txvq; 860 uint16_t i; 861 862 /* Add lock to avoid queue contention. */ 863 rte_spinlock_lock(&hw->state_lock); 864 hw->started = 0; 865 866 /* 867 * Waiting for datapath to complete before resetting queues. 868 * 1 ms should be enough for the ongoing Tx/Rx function to finish. 869 */ 870 rte_delay_ms(1); 871 872 /* Vring reset for each Tx queue and Rx queue. */ 873 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 874 rxvq = eth_dev->data->rx_queues[i]; 875 virtqueue_rxvq_reset_packed(virtnet_rxq_to_vq(rxvq)); 876 virtio_dev_rx_queue_setup_finish(eth_dev, i); 877 } 878 879 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 880 txvq = eth_dev->data->tx_queues[i]; 881 virtqueue_txvq_reset_packed(virtnet_txq_to_vq(txvq)); 882 } 883 884 hw->started = 1; 885 rte_spinlock_unlock(&hw->state_lock); 886 } 887 888 void 889 virtio_user_dev_delayed_disconnect_handler(void *param) 890 { 891 struct virtio_user_dev *dev = param; 892 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 893 894 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 895 PMD_DRV_LOG(ERR, "interrupt disable failed"); 896 return; 897 } 898 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d", 899 eth_dev->intr_handle->fd); 900 if (rte_intr_callback_unregister(eth_dev->intr_handle, 901 virtio_interrupt_handler, 902 eth_dev) != 1) 903 PMD_DRV_LOG(ERR, "interrupt unregister failed"); 904 905 if (dev->is_server) { 906 if (dev->ops->server_disconnect) 907 dev->ops->server_disconnect(dev); 908 909 eth_dev->intr_handle->fd = dev->ops->get_intr_fd(dev); 910 911 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d", 912 eth_dev->intr_handle->fd); 913 914 if (rte_intr_callback_register(eth_dev->intr_handle, 915 virtio_interrupt_handler, 916 eth_dev)) 917 PMD_DRV_LOG(ERR, "interrupt register failed"); 918 919 if (rte_intr_enable(eth_dev->intr_handle) < 0) { 920 PMD_DRV_LOG(ERR, "interrupt enable failed"); 921 return; 922 } 923 } 924 } 925 926 static void 927 virtio_user_dev_delayed_intr_reconfig_handler(void *param) 928 { 929 struct virtio_user_dev *dev = param; 930 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 931 932 PMD_DRV_LOG(DEBUG, "Unregistering intr fd: %d", 933 eth_dev->intr_handle->fd); 934 935 if (rte_intr_callback_unregister(eth_dev->intr_handle, 936 virtio_interrupt_handler, 937 eth_dev) != 1) 938 PMD_DRV_LOG(ERR, "interrupt unregister failed"); 939 940 eth_dev->intr_handle->fd = dev->ops->get_intr_fd(dev); 941 942 PMD_DRV_LOG(DEBUG, "Registering intr fd: %d", eth_dev->intr_handle->fd); 943 944 if (rte_intr_callback_register(eth_dev->intr_handle, 945 virtio_interrupt_handler, eth_dev)) 946 PMD_DRV_LOG(ERR, "interrupt register failed"); 947 948 if (rte_intr_enable(eth_dev->intr_handle) < 0) 949 PMD_DRV_LOG(ERR, "interrupt enable failed"); 950 } 951 952 int 953 virtio_user_dev_server_reconnect(struct virtio_user_dev *dev) 954 { 955 int ret, old_status; 956 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 957 struct virtio_hw *hw = &dev->hw; 958 959 if (!dev->ops->server_reconnect) { 960 PMD_DRV_LOG(ERR, "(%s) Missing server reconnect callback", dev->path); 961 return -1; 962 } 963 964 if (dev->ops->server_reconnect(dev)) { 965 PMD_DRV_LOG(ERR, "(%s) Reconnect callback call failed", dev->path); 966 return -1; 967 } 968 969 old_status = dev->status; 970 971 virtio_reset(hw); 972 973 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 974 975 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 976 977 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 978 PMD_INIT_LOG(ERR, "get_features failed: %s", 979 strerror(errno)); 980 return -1; 981 } 982 983 dev->device_features |= dev->frontend_features; 984 985 /* unmask vhost-user unsupported features */ 986 dev->device_features &= ~(dev->unsupported_features); 987 988 dev->features &= dev->device_features; 989 990 /* For packed ring, resetting queues is required in reconnection. */ 991 if (virtio_with_packed_queue(hw) && 992 (old_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) { 993 PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped" 994 " when packed ring reconnecting."); 995 virtio_user_dev_reset_queues_packed(eth_dev); 996 } 997 998 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); 999 1000 /* Start the device */ 1001 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK); 1002 if (!dev->started) 1003 return -1; 1004 1005 if (dev->queue_pairs > 1) { 1006 ret = virtio_user_handle_mq(dev, dev->queue_pairs); 1007 if (ret != 0) { 1008 PMD_INIT_LOG(ERR, "Fails to enable multi-queue pairs!"); 1009 return -1; 1010 } 1011 } 1012 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { 1013 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 1014 PMD_DRV_LOG(ERR, "interrupt disable failed"); 1015 return -1; 1016 } 1017 /* 1018 * This function can be called from the interrupt handler, so 1019 * we can't unregister interrupt handler here. Setting 1020 * alarm to do that later. 1021 */ 1022 rte_eal_alarm_set(1, 1023 virtio_user_dev_delayed_intr_reconfig_handler, 1024 (void *)dev); 1025 } 1026 PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!"); 1027 return 0; 1028 } 1029