1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdio.h> 7 #include <fcntl.h> 8 #include <string.h> 9 #include <errno.h> 10 #include <sys/mman.h> 11 #include <unistd.h> 12 #include <sys/eventfd.h> 13 #include <sys/types.h> 14 #include <sys/stat.h> 15 16 #include <rte_string_fns.h> 17 #include <rte_eal_memconfig.h> 18 19 #include "vhost.h" 20 #include "virtio_user_dev.h" 21 #include "../virtio_ethdev.h" 22 23 #define VIRTIO_USER_MEM_EVENT_CLB_NAME "virtio_user_mem_event_clb" 24 25 const char * const virtio_user_backend_strings[] = { 26 [VIRTIO_USER_BACKEND_UNKNOWN] = "VIRTIO_USER_BACKEND_UNKNOWN", 27 [VIRTIO_USER_BACKEND_VHOST_USER] = "VHOST_USER", 28 [VIRTIO_USER_BACKEND_VHOST_KERNEL] = "VHOST_NET", 29 [VIRTIO_USER_BACKEND_VHOST_VDPA] = "VHOST_VDPA", 30 }; 31 32 static int 33 virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 34 { 35 /* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come 36 * firstly because vhost depends on this msg to allocate virtqueue 37 * pair. 38 */ 39 struct vhost_vring_file file; 40 int ret; 41 42 file.index = queue_sel; 43 file.fd = dev->callfds[queue_sel]; 44 ret = dev->ops->set_vring_call(dev, &file); 45 if (ret < 0) { 46 PMD_INIT_LOG(ERR, "(%s) Failed to create queue %u\n", dev->path, queue_sel); 47 return -1; 48 } 49 50 return 0; 51 } 52 53 static int 54 virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 55 { 56 int ret; 57 struct vhost_vring_file file; 58 struct vhost_vring_state state; 59 struct vring *vring = &dev->vrings[queue_sel]; 60 struct vring_packed *pq_vring = &dev->packed_vrings[queue_sel]; 61 struct vhost_vring_addr addr = { 62 .index = queue_sel, 63 .log_guest_addr = 0, 64 .flags = 0, /* disable log */ 65 }; 66 67 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) { 68 addr.desc_user_addr = 69 (uint64_t)(uintptr_t)pq_vring->desc; 70 addr.avail_user_addr = 71 (uint64_t)(uintptr_t)pq_vring->driver; 72 addr.used_user_addr = 73 (uint64_t)(uintptr_t)pq_vring->device; 74 } else { 75 addr.desc_user_addr = (uint64_t)(uintptr_t)vring->desc; 76 addr.avail_user_addr = (uint64_t)(uintptr_t)vring->avail; 77 addr.used_user_addr = (uint64_t)(uintptr_t)vring->used; 78 } 79 80 state.index = queue_sel; 81 state.num = vring->num; 82 ret = dev->ops->set_vring_num(dev, &state); 83 if (ret < 0) 84 goto err; 85 86 state.index = queue_sel; 87 state.num = 0; /* no reservation */ 88 if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) 89 state.num |= (1 << 15); 90 ret = dev->ops->set_vring_base(dev, &state); 91 if (ret < 0) 92 goto err; 93 94 ret = dev->ops->set_vring_addr(dev, &addr); 95 if (ret < 0) 96 goto err; 97 98 /* Of all per virtqueue MSGs, make sure VHOST_USER_SET_VRING_KICK comes 99 * lastly because vhost depends on this msg to judge if 100 * virtio is ready. 101 */ 102 file.index = queue_sel; 103 file.fd = dev->kickfds[queue_sel]; 104 ret = dev->ops->set_vring_kick(dev, &file); 105 if (ret < 0) 106 goto err; 107 108 return 0; 109 err: 110 PMD_INIT_LOG(ERR, "(%s) Failed to kick queue %u\n", dev->path, queue_sel); 111 112 return -1; 113 } 114 115 static int 116 virtio_user_queue_setup(struct virtio_user_dev *dev, 117 int (*fn)(struct virtio_user_dev *, uint32_t)) 118 { 119 uint32_t i, queue_sel; 120 121 for (i = 0; i < dev->max_queue_pairs; ++i) { 122 queue_sel = 2 * i + VTNET_SQ_RQ_QUEUE_IDX; 123 if (fn(dev, queue_sel) < 0) { 124 PMD_DRV_LOG(ERR, "(%s) setup rx vq %u failed", dev->path, i); 125 return -1; 126 } 127 } 128 for (i = 0; i < dev->max_queue_pairs; ++i) { 129 queue_sel = 2 * i + VTNET_SQ_TQ_QUEUE_IDX; 130 if (fn(dev, queue_sel) < 0) { 131 PMD_DRV_LOG(INFO, "(%s) setup tx vq %u failed", dev->path, i); 132 return -1; 133 } 134 } 135 136 return 0; 137 } 138 139 int 140 virtio_user_dev_set_features(struct virtio_user_dev *dev) 141 { 142 uint64_t features; 143 int ret = -1; 144 145 pthread_mutex_lock(&dev->mutex); 146 147 /* Step 0: tell vhost to create queues */ 148 if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0) 149 goto error; 150 151 features = dev->features; 152 153 /* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init */ 154 features &= ~(1ull << VIRTIO_NET_F_MAC); 155 /* Strip VIRTIO_NET_F_CTRL_VQ, as devices do not really need to know */ 156 features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ); 157 features &= ~(1ull << VIRTIO_NET_F_STATUS); 158 ret = dev->ops->set_features(dev, features); 159 if (ret < 0) 160 goto error; 161 PMD_DRV_LOG(INFO, "(%s) set features: 0x%" PRIx64, dev->path, features); 162 error: 163 pthread_mutex_unlock(&dev->mutex); 164 165 return ret; 166 } 167 168 int 169 virtio_user_start_device(struct virtio_user_dev *dev) 170 { 171 int ret; 172 173 /* 174 * XXX workaround! 175 * 176 * We need to make sure that the locks will be 177 * taken in the correct order to avoid deadlocks. 178 * 179 * Before releasing this lock, this thread should 180 * not trigger any memory hotplug events. 181 * 182 * This is a temporary workaround, and should be 183 * replaced when we get proper supports from the 184 * memory subsystem in the future. 185 */ 186 rte_mcfg_mem_read_lock(); 187 pthread_mutex_lock(&dev->mutex); 188 189 /* Step 2: share memory regions */ 190 ret = dev->ops->set_memory_table(dev); 191 if (ret < 0) 192 goto error; 193 194 /* Step 3: kick queues */ 195 ret = virtio_user_queue_setup(dev, virtio_user_kick_queue); 196 if (ret < 0) 197 goto error; 198 199 /* Step 4: enable queues 200 * we enable the 1st queue pair by default. 201 */ 202 ret = dev->ops->enable_qp(dev, 0, 1); 203 if (ret < 0) 204 goto error; 205 206 dev->started = true; 207 208 pthread_mutex_unlock(&dev->mutex); 209 rte_mcfg_mem_read_unlock(); 210 211 return 0; 212 error: 213 pthread_mutex_unlock(&dev->mutex); 214 rte_mcfg_mem_read_unlock(); 215 216 PMD_INIT_LOG(ERR, "(%s) Failed to start device\n", dev->path); 217 218 /* TODO: free resource here or caller to check */ 219 return -1; 220 } 221 222 int virtio_user_stop_device(struct virtio_user_dev *dev) 223 { 224 struct vhost_vring_state state; 225 uint32_t i; 226 int ret; 227 228 pthread_mutex_lock(&dev->mutex); 229 if (!dev->started) 230 goto out; 231 232 for (i = 0; i < dev->max_queue_pairs; ++i) { 233 ret = dev->ops->enable_qp(dev, i, 0); 234 if (ret < 0) 235 goto err; 236 } 237 238 /* Stop the backend. */ 239 for (i = 0; i < dev->max_queue_pairs * 2; ++i) { 240 state.index = i; 241 ret = dev->ops->get_vring_base(dev, &state); 242 if (ret < 0) { 243 PMD_DRV_LOG(ERR, "(%s) get_vring_base failed, index=%u", dev->path, i); 244 goto err; 245 } 246 } 247 248 dev->started = false; 249 250 out: 251 pthread_mutex_unlock(&dev->mutex); 252 253 return 0; 254 err: 255 pthread_mutex_unlock(&dev->mutex); 256 257 PMD_INIT_LOG(ERR, "(%s) Failed to stop device\n", dev->path); 258 259 return -1; 260 } 261 262 static inline void 263 parse_mac(struct virtio_user_dev *dev, const char *mac) 264 { 265 struct rte_ether_addr tmp; 266 267 if (!mac) 268 return; 269 270 if (rte_ether_unformat_addr(mac, &tmp) == 0) { 271 memcpy(dev->mac_addr, &tmp, RTE_ETHER_ADDR_LEN); 272 dev->mac_specified = 1; 273 } else { 274 /* ignore the wrong mac, use random mac */ 275 PMD_DRV_LOG(ERR, "wrong format of mac: %s", mac); 276 } 277 } 278 279 static int 280 virtio_user_dev_init_notify(struct virtio_user_dev *dev) 281 { 282 uint32_t i, j; 283 int callfd; 284 int kickfd; 285 286 for (i = 0; i < dev->max_queue_pairs * 2; i++) { 287 /* May use invalid flag, but some backend uses kickfd and 288 * callfd as criteria to judge if dev is alive. so finally we 289 * use real event_fd. 290 */ 291 callfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 292 if (callfd < 0) { 293 PMD_DRV_LOG(ERR, "(%s) callfd error, %s", dev->path, strerror(errno)); 294 goto err; 295 } 296 kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 297 if (kickfd < 0) { 298 close(callfd); 299 PMD_DRV_LOG(ERR, "(%s) kickfd error, %s", dev->path, strerror(errno)); 300 goto err; 301 } 302 dev->callfds[i] = callfd; 303 dev->kickfds[i] = kickfd; 304 } 305 306 return 0; 307 err: 308 for (j = 0; j < i; j++) { 309 if (dev->kickfds[j] >= 0) { 310 close(dev->kickfds[j]); 311 dev->kickfds[j] = -1; 312 } 313 if (dev->callfds[j] >= 0) { 314 close(dev->callfds[j]); 315 dev->callfds[j] = -1; 316 } 317 } 318 319 return -1; 320 } 321 322 static void 323 virtio_user_dev_uninit_notify(struct virtio_user_dev *dev) 324 { 325 uint32_t i; 326 327 for (i = 0; i < dev->max_queue_pairs * 2; ++i) { 328 if (dev->kickfds[i] >= 0) { 329 close(dev->kickfds[i]); 330 dev->kickfds[i] = -1; 331 } 332 if (dev->callfds[i] >= 0) { 333 close(dev->callfds[i]); 334 dev->callfds[i] = -1; 335 } 336 } 337 } 338 339 static int 340 virtio_user_fill_intr_handle(struct virtio_user_dev *dev) 341 { 342 uint32_t i; 343 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 344 345 if (!eth_dev->intr_handle) { 346 eth_dev->intr_handle = malloc(sizeof(*eth_dev->intr_handle)); 347 if (!eth_dev->intr_handle) { 348 PMD_DRV_LOG(ERR, "(%s) failed to allocate intr_handle", dev->path); 349 return -1; 350 } 351 memset(eth_dev->intr_handle, 0, sizeof(*eth_dev->intr_handle)); 352 } 353 354 for (i = 0; i < dev->max_queue_pairs; ++i) 355 eth_dev->intr_handle->efds[i] = dev->callfds[i]; 356 eth_dev->intr_handle->nb_efd = dev->max_queue_pairs; 357 eth_dev->intr_handle->max_intr = dev->max_queue_pairs + 1; 358 eth_dev->intr_handle->type = RTE_INTR_HANDLE_VDEV; 359 /* For virtio vdev, no need to read counter for clean */ 360 eth_dev->intr_handle->efd_counter_size = 0; 361 eth_dev->intr_handle->fd = dev->ops->get_intr_fd(dev); 362 363 return 0; 364 } 365 366 static void 367 virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused, 368 const void *addr, 369 size_t len __rte_unused, 370 void *arg) 371 { 372 struct virtio_user_dev *dev = arg; 373 struct rte_memseg_list *msl; 374 uint16_t i; 375 int ret = 0; 376 377 /* ignore externally allocated memory */ 378 msl = rte_mem_virt2memseg_list(addr); 379 if (msl->external) 380 return; 381 382 pthread_mutex_lock(&dev->mutex); 383 384 if (dev->started == false) 385 goto exit; 386 387 /* Step 1: pause the active queues */ 388 for (i = 0; i < dev->queue_pairs; i++) { 389 ret = dev->ops->enable_qp(dev, i, 0); 390 if (ret < 0) 391 goto exit; 392 } 393 394 /* Step 2: update memory regions */ 395 ret = dev->ops->set_memory_table(dev); 396 if (ret < 0) 397 goto exit; 398 399 /* Step 3: resume the active queues */ 400 for (i = 0; i < dev->queue_pairs; i++) { 401 ret = dev->ops->enable_qp(dev, i, 1); 402 if (ret < 0) 403 goto exit; 404 } 405 406 exit: 407 pthread_mutex_unlock(&dev->mutex); 408 409 if (ret < 0) 410 PMD_DRV_LOG(ERR, "(%s) Failed to update memory table\n", dev->path); 411 } 412 413 static int 414 virtio_user_dev_setup(struct virtio_user_dev *dev) 415 { 416 if (dev->is_server) { 417 if (dev->backend_type != VIRTIO_USER_BACKEND_VHOST_USER) { 418 PMD_DRV_LOG(ERR, "Server mode only supports vhost-user!"); 419 return -1; 420 } 421 } 422 423 switch (dev->backend_type) { 424 case VIRTIO_USER_BACKEND_VHOST_USER: 425 dev->ops = &virtio_ops_user; 426 break; 427 case VIRTIO_USER_BACKEND_VHOST_KERNEL: 428 dev->ops = &virtio_ops_kernel; 429 break; 430 case VIRTIO_USER_BACKEND_VHOST_VDPA: 431 dev->ops = &virtio_ops_vdpa; 432 break; 433 default: 434 PMD_DRV_LOG(ERR, "(%s) Unknown backend type", dev->path); 435 return -1; 436 } 437 438 if (dev->ops->setup(dev) < 0) { 439 PMD_INIT_LOG(ERR, "(%s) Failed to setup backend\n", dev->path); 440 return -1; 441 } 442 443 if (virtio_user_dev_init_notify(dev) < 0) { 444 PMD_INIT_LOG(ERR, "(%s) Failed to init notifiers\n", dev->path); 445 goto destroy; 446 } 447 448 if (virtio_user_fill_intr_handle(dev) < 0) { 449 PMD_INIT_LOG(ERR, "(%s) Failed to init interrupt handler\n", dev->path); 450 goto uninit; 451 } 452 453 return 0; 454 455 uninit: 456 virtio_user_dev_uninit_notify(dev); 457 destroy: 458 dev->ops->destroy(dev); 459 460 return -1; 461 } 462 463 /* Use below macro to filter features from vhost backend */ 464 #define VIRTIO_USER_SUPPORTED_FEATURES \ 465 (1ULL << VIRTIO_NET_F_MAC | \ 466 1ULL << VIRTIO_NET_F_STATUS | \ 467 1ULL << VIRTIO_NET_F_MQ | \ 468 1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR | \ 469 1ULL << VIRTIO_NET_F_CTRL_VQ | \ 470 1ULL << VIRTIO_NET_F_CTRL_RX | \ 471 1ULL << VIRTIO_NET_F_CTRL_VLAN | \ 472 1ULL << VIRTIO_NET_F_CSUM | \ 473 1ULL << VIRTIO_NET_F_HOST_TSO4 | \ 474 1ULL << VIRTIO_NET_F_HOST_TSO6 | \ 475 1ULL << VIRTIO_NET_F_MRG_RXBUF | \ 476 1ULL << VIRTIO_RING_F_INDIRECT_DESC | \ 477 1ULL << VIRTIO_NET_F_GUEST_CSUM | \ 478 1ULL << VIRTIO_NET_F_GUEST_TSO4 | \ 479 1ULL << VIRTIO_NET_F_GUEST_TSO6 | \ 480 1ULL << VIRTIO_F_IN_ORDER | \ 481 1ULL << VIRTIO_F_VERSION_1 | \ 482 1ULL << VIRTIO_F_RING_PACKED) 483 484 int 485 virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues, 486 int cq, int queue_size, const char *mac, char **ifname, 487 int server, int mrg_rxbuf, int in_order, int packed_vq, 488 enum virtio_user_backend_type backend_type) 489 { 490 uint64_t backend_features; 491 int i; 492 493 pthread_mutex_init(&dev->mutex, NULL); 494 strlcpy(dev->path, path, PATH_MAX); 495 496 for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; i++) { 497 dev->kickfds[i] = -1; 498 dev->callfds[i] = -1; 499 } 500 501 dev->started = 0; 502 dev->max_queue_pairs = queues; 503 dev->queue_pairs = 1; /* mq disabled by default */ 504 dev->queue_size = queue_size; 505 dev->is_server = server; 506 dev->mac_specified = 0; 507 dev->frontend_features = 0; 508 dev->unsupported_features = 0; 509 dev->backend_type = backend_type; 510 511 parse_mac(dev, mac); 512 513 if (*ifname) { 514 dev->ifname = *ifname; 515 *ifname = NULL; 516 } 517 518 if (virtio_user_dev_setup(dev) < 0) { 519 PMD_INIT_LOG(ERR, "(%s) backend set up fails", dev->path); 520 return -1; 521 } 522 523 if (dev->ops->set_owner(dev) < 0) { 524 PMD_INIT_LOG(ERR, "(%s) Failed to set backend owner", dev->path); 525 return -1; 526 } 527 528 if (dev->ops->get_backend_features(&backend_features) < 0) { 529 PMD_INIT_LOG(ERR, "(%s) Failed to get backend features", dev->path); 530 return -1; 531 } 532 533 dev->unsupported_features = ~(VIRTIO_USER_SUPPORTED_FEATURES | backend_features); 534 535 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 536 PMD_INIT_LOG(ERR, "(%s) Failed to get device features", dev->path); 537 return -1; 538 } 539 540 if (!mrg_rxbuf) 541 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF); 542 543 if (!in_order) 544 dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER); 545 546 if (!packed_vq) 547 dev->unsupported_features |= (1ull << VIRTIO_F_RING_PACKED); 548 549 if (dev->mac_specified) 550 dev->frontend_features |= (1ull << VIRTIO_NET_F_MAC); 551 else 552 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC); 553 554 if (cq) { 555 /* device does not really need to know anything about CQ, 556 * so if necessary, we just claim to support CQ 557 */ 558 dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 559 } else { 560 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 561 /* Also disable features that depend on VIRTIO_NET_F_CTRL_VQ */ 562 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX); 563 dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN); 564 dev->unsupported_features |= 565 (1ull << VIRTIO_NET_F_GUEST_ANNOUNCE); 566 dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); 567 dev->unsupported_features |= 568 (1ull << VIRTIO_NET_F_CTRL_MAC_ADDR); 569 } 570 571 /* The backend will not report this feature, we add it explicitly */ 572 if (dev->backend_type == VIRTIO_USER_BACKEND_VHOST_USER) 573 dev->frontend_features |= (1ull << VIRTIO_NET_F_STATUS); 574 575 /* 576 * Device features = 577 * (frontend_features | backend_features) & ~unsupported_features; 578 */ 579 dev->device_features |= dev->frontend_features; 580 dev->device_features &= ~dev->unsupported_features; 581 582 if (rte_mem_event_callback_register(VIRTIO_USER_MEM_EVENT_CLB_NAME, 583 virtio_user_mem_event_cb, dev)) { 584 if (rte_errno != ENOTSUP) { 585 PMD_INIT_LOG(ERR, "(%s) Failed to register mem event callback\n", 586 dev->path); 587 return -1; 588 } 589 } 590 591 return 0; 592 } 593 594 void 595 virtio_user_dev_uninit(struct virtio_user_dev *dev) 596 { 597 virtio_user_stop_device(dev); 598 599 rte_mem_event_callback_unregister(VIRTIO_USER_MEM_EVENT_CLB_NAME, dev); 600 601 virtio_user_dev_uninit_notify(dev); 602 603 free(dev->ifname); 604 605 if (dev->is_server) 606 unlink(dev->path); 607 608 dev->ops->destroy(dev); 609 } 610 611 uint8_t 612 virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs) 613 { 614 uint16_t i; 615 uint8_t ret = 0; 616 617 if (q_pairs > dev->max_queue_pairs) { 618 PMD_INIT_LOG(ERR, "(%s) multi-q config %u, but only %u supported", 619 dev->path, q_pairs, dev->max_queue_pairs); 620 return -1; 621 } 622 623 for (i = 0; i < q_pairs; ++i) 624 ret |= dev->ops->enable_qp(dev, i, 1); 625 for (i = q_pairs; i < dev->max_queue_pairs; ++i) 626 ret |= dev->ops->enable_qp(dev, i, 0); 627 628 dev->queue_pairs = q_pairs; 629 630 return ret; 631 } 632 633 static uint32_t 634 virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring, 635 uint16_t idx_hdr) 636 { 637 struct virtio_net_ctrl_hdr *hdr; 638 virtio_net_ctrl_ack status = ~0; 639 uint16_t i, idx_data, idx_status; 640 uint32_t n_descs = 0; 641 642 /* locate desc for header, data, and status */ 643 idx_data = vring->desc[idx_hdr].next; 644 n_descs++; 645 646 i = idx_data; 647 while (vring->desc[i].flags == VRING_DESC_F_NEXT) { 648 i = vring->desc[i].next; 649 n_descs++; 650 } 651 652 /* locate desc for status */ 653 idx_status = i; 654 n_descs++; 655 656 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 657 if (hdr->class == VIRTIO_NET_CTRL_MQ && 658 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 659 uint16_t queues; 660 661 queues = *(uint16_t *)(uintptr_t)vring->desc[idx_data].addr; 662 status = virtio_user_handle_mq(dev, queues); 663 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 664 hdr->class == VIRTIO_NET_CTRL_MAC || 665 hdr->class == VIRTIO_NET_CTRL_VLAN) { 666 status = 0; 667 } 668 669 /* Update status */ 670 *(virtio_net_ctrl_ack *)(uintptr_t)vring->desc[idx_status].addr = status; 671 672 return n_descs; 673 } 674 675 static inline int 676 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) 677 { 678 uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE); 679 680 return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) && 681 wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED); 682 } 683 684 static uint32_t 685 virtio_user_handle_ctrl_msg_packed(struct virtio_user_dev *dev, 686 struct vring_packed *vring, 687 uint16_t idx_hdr) 688 { 689 struct virtio_net_ctrl_hdr *hdr; 690 virtio_net_ctrl_ack status = ~0; 691 uint16_t idx_data, idx_status; 692 /* initialize to one, header is first */ 693 uint32_t n_descs = 1; 694 695 /* locate desc for header, data, and status */ 696 idx_data = idx_hdr + 1; 697 if (idx_data >= dev->queue_size) 698 idx_data -= dev->queue_size; 699 700 n_descs++; 701 702 idx_status = idx_data; 703 while (vring->desc[idx_status].flags & VRING_DESC_F_NEXT) { 704 idx_status++; 705 if (idx_status >= dev->queue_size) 706 idx_status -= dev->queue_size; 707 n_descs++; 708 } 709 710 hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 711 if (hdr->class == VIRTIO_NET_CTRL_MQ && 712 hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 713 uint16_t queues; 714 715 queues = *(uint16_t *)(uintptr_t) 716 vring->desc[idx_data].addr; 717 status = virtio_user_handle_mq(dev, queues); 718 } else if (hdr->class == VIRTIO_NET_CTRL_RX || 719 hdr->class == VIRTIO_NET_CTRL_MAC || 720 hdr->class == VIRTIO_NET_CTRL_VLAN) { 721 status = 0; 722 } 723 724 /* Update status */ 725 *(virtio_net_ctrl_ack *)(uintptr_t) 726 vring->desc[idx_status].addr = status; 727 728 /* Update used descriptor */ 729 vring->desc[idx_hdr].id = vring->desc[idx_status].id; 730 vring->desc[idx_hdr].len = sizeof(status); 731 732 return n_descs; 733 } 734 735 void 736 virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx) 737 { 738 struct virtio_user_queue *vq = &dev->packed_queues[queue_idx]; 739 struct vring_packed *vring = &dev->packed_vrings[queue_idx]; 740 uint16_t n_descs, flags; 741 742 /* Perform a load-acquire barrier in desc_is_avail to 743 * enforce the ordering between desc flags and desc 744 * content. 745 */ 746 while (desc_is_avail(&vring->desc[vq->used_idx], 747 vq->used_wrap_counter)) { 748 749 n_descs = virtio_user_handle_ctrl_msg_packed(dev, vring, 750 vq->used_idx); 751 752 flags = VRING_DESC_F_WRITE; 753 if (vq->used_wrap_counter) 754 flags |= VRING_PACKED_DESC_F_AVAIL_USED; 755 756 __atomic_store_n(&vring->desc[vq->used_idx].flags, flags, 757 __ATOMIC_RELEASE); 758 759 vq->used_idx += n_descs; 760 if (vq->used_idx >= dev->queue_size) { 761 vq->used_idx -= dev->queue_size; 762 vq->used_wrap_counter ^= 1; 763 } 764 } 765 } 766 767 void 768 virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx) 769 { 770 uint16_t avail_idx, desc_idx; 771 struct vring_used_elem *uep; 772 uint32_t n_descs; 773 struct vring *vring = &dev->vrings[queue_idx]; 774 775 /* Consume avail ring, using used ring idx as first one */ 776 while (__atomic_load_n(&vring->used->idx, __ATOMIC_RELAXED) 777 != vring->avail->idx) { 778 avail_idx = __atomic_load_n(&vring->used->idx, __ATOMIC_RELAXED) 779 & (vring->num - 1); 780 desc_idx = vring->avail->ring[avail_idx]; 781 782 n_descs = virtio_user_handle_ctrl_msg(dev, vring, desc_idx); 783 784 /* Update used ring */ 785 uep = &vring->used->ring[avail_idx]; 786 uep->id = desc_idx; 787 uep->len = n_descs; 788 789 __atomic_add_fetch(&vring->used->idx, 1, __ATOMIC_RELAXED); 790 } 791 } 792 793 int 794 virtio_user_dev_set_status(struct virtio_user_dev *dev, uint8_t status) 795 { 796 int ret; 797 798 pthread_mutex_lock(&dev->mutex); 799 dev->status = status; 800 ret = dev->ops->set_status(dev, status); 801 if (ret && ret != -ENOTSUP) 802 PMD_INIT_LOG(ERR, "(%s) Failed to set backend status\n", dev->path); 803 804 pthread_mutex_unlock(&dev->mutex); 805 return ret; 806 } 807 808 int 809 virtio_user_dev_update_status(struct virtio_user_dev *dev) 810 { 811 int ret; 812 uint8_t status; 813 814 pthread_mutex_lock(&dev->mutex); 815 816 ret = dev->ops->get_status(dev, &status); 817 if (!ret) { 818 dev->status = status; 819 PMD_INIT_LOG(DEBUG, "Updated Device Status(0x%08x):\n" 820 "\t-RESET: %u\n" 821 "\t-ACKNOWLEDGE: %u\n" 822 "\t-DRIVER: %u\n" 823 "\t-DRIVER_OK: %u\n" 824 "\t-FEATURES_OK: %u\n" 825 "\t-DEVICE_NEED_RESET: %u\n" 826 "\t-FAILED: %u\n", 827 dev->status, 828 (dev->status == VIRTIO_CONFIG_STATUS_RESET), 829 !!(dev->status & VIRTIO_CONFIG_STATUS_ACK), 830 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER), 831 !!(dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK), 832 !!(dev->status & VIRTIO_CONFIG_STATUS_FEATURES_OK), 833 !!(dev->status & VIRTIO_CONFIG_STATUS_DEV_NEED_RESET), 834 !!(dev->status & VIRTIO_CONFIG_STATUS_FAILED)); 835 } else if (ret != -ENOTSUP) { 836 PMD_INIT_LOG(ERR, "(%s) Failed to get backend status\n", dev->path); 837 } 838 839 pthread_mutex_unlock(&dev->mutex); 840 return ret; 841 } 842 843 int 844 virtio_user_dev_update_link_state(struct virtio_user_dev *dev) 845 { 846 if (dev->ops->update_link_state) 847 return dev->ops->update_link_state(dev); 848 849 return 0; 850 } 851 852 static void 853 virtio_user_dev_reset_queues_packed(struct rte_eth_dev *eth_dev) 854 { 855 struct virtio_user_dev *dev = eth_dev->data->dev_private; 856 struct virtio_hw *hw = &dev->hw; 857 struct virtnet_rx *rxvq; 858 struct virtnet_tx *txvq; 859 uint16_t i; 860 861 /* Add lock to avoid queue contention. */ 862 rte_spinlock_lock(&hw->state_lock); 863 hw->started = 0; 864 865 /* 866 * Waiting for datapath to complete before resetting queues. 867 * 1 ms should be enough for the ongoing Tx/Rx function to finish. 868 */ 869 rte_delay_ms(1); 870 871 /* Vring reset for each Tx queue and Rx queue. */ 872 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 873 rxvq = eth_dev->data->rx_queues[i]; 874 virtqueue_rxvq_reset_packed(virtnet_rxq_to_vq(rxvq)); 875 virtio_dev_rx_queue_setup_finish(eth_dev, i); 876 } 877 878 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 879 txvq = eth_dev->data->tx_queues[i]; 880 virtqueue_txvq_reset_packed(virtnet_txq_to_vq(txvq)); 881 } 882 883 hw->started = 1; 884 rte_spinlock_unlock(&hw->state_lock); 885 } 886 887 void 888 virtio_user_dev_delayed_handler(void *param) 889 { 890 struct virtio_user_dev *dev = param; 891 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 892 893 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 894 PMD_DRV_LOG(ERR, "interrupt disable failed"); 895 return; 896 } 897 rte_intr_callback_unregister(eth_dev->intr_handle, 898 virtio_interrupt_handler, eth_dev); 899 if (dev->is_server) { 900 if (dev->ops->server_disconnect) 901 dev->ops->server_disconnect(dev); 902 eth_dev->intr_handle->fd = dev->ops->get_intr_fd(dev); 903 rte_intr_callback_register(eth_dev->intr_handle, 904 virtio_interrupt_handler, eth_dev); 905 if (rte_intr_enable(eth_dev->intr_handle) < 0) { 906 PMD_DRV_LOG(ERR, "interrupt enable failed"); 907 return; 908 } 909 } 910 } 911 912 int 913 virtio_user_dev_server_reconnect(struct virtio_user_dev *dev) 914 { 915 int ret, old_status; 916 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->hw.port_id]; 917 struct virtio_hw *hw = &dev->hw; 918 919 if (!dev->ops->server_reconnect) { 920 PMD_DRV_LOG(ERR, "(%s) Missing server reconnect callback", dev->path); 921 return -1; 922 } 923 924 if (dev->ops->server_reconnect(dev)) { 925 PMD_DRV_LOG(ERR, "(%s) Reconnect callback call failed", dev->path); 926 return -1; 927 } 928 929 old_status = dev->status; 930 931 virtio_reset(hw); 932 933 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 934 935 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 936 937 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 938 PMD_INIT_LOG(ERR, "get_features failed: %s", 939 strerror(errno)); 940 return -1; 941 } 942 943 dev->device_features |= dev->frontend_features; 944 945 /* unmask vhost-user unsupported features */ 946 dev->device_features &= ~(dev->unsupported_features); 947 948 dev->features &= dev->device_features; 949 950 /* For packed ring, resetting queues is required in reconnection. */ 951 if (virtio_with_packed_queue(hw) && 952 (old_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) { 953 PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped" 954 " when packed ring reconnecting."); 955 virtio_user_dev_reset_queues_packed(eth_dev); 956 } 957 958 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); 959 960 /* Start the device */ 961 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK); 962 if (!dev->started) 963 return -1; 964 965 if (dev->queue_pairs > 1) { 966 ret = virtio_user_handle_mq(dev, dev->queue_pairs); 967 if (ret != 0) { 968 PMD_INIT_LOG(ERR, "Fails to enable multi-queue pairs!"); 969 return -1; 970 } 971 } 972 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { 973 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 974 PMD_DRV_LOG(ERR, "interrupt disable failed"); 975 return -1; 976 } 977 rte_intr_callback_unregister(eth_dev->intr_handle, 978 virtio_interrupt_handler, 979 eth_dev); 980 981 eth_dev->intr_handle->fd = dev->ops->get_intr_fd(dev); 982 rte_intr_callback_register(eth_dev->intr_handle, 983 virtio_interrupt_handler, eth_dev); 984 985 if (rte_intr_enable(eth_dev->intr_handle) < 0) { 986 PMD_DRV_LOG(ERR, "interrupt enable failed"); 987 return -1; 988 } 989 } 990 PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!"); 991 return 0; 992 } 993