1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <sys/types.h> 7 #include <unistd.h> 8 #include <fcntl.h> 9 #include <linux/major.h> 10 #include <sys/stat.h> 11 #include <sys/sysmacros.h> 12 #include <sys/socket.h> 13 14 #include <rte_malloc.h> 15 #include <rte_kvargs.h> 16 #include <ethdev_vdev.h> 17 #include <rte_bus_vdev.h> 18 #include <rte_alarm.h> 19 #include <rte_cycles.h> 20 21 #include "virtio_ethdev.h" 22 #include "virtio_logs.h" 23 #include "virtio.h" 24 #include "virtqueue.h" 25 #include "virtio_rxtx.h" 26 #include "virtio_user/virtio_user_dev.h" 27 #include "virtio_user/vhost.h" 28 29 #define virtio_user_get_dev(hwp) container_of(hwp, struct virtio_user_dev, hw) 30 31 static void 32 virtio_user_reset_queues_packed(struct rte_eth_dev *eth_dev) 33 { 34 struct virtio_user_dev *dev = eth_dev->data->dev_private; 35 struct virtio_hw *hw = &dev->hw; 36 struct virtnet_rx *rxvq; 37 struct virtnet_tx *txvq; 38 uint16_t i; 39 40 /* Add lock to avoid queue contention. */ 41 rte_spinlock_lock(&hw->state_lock); 42 hw->started = 0; 43 44 /* 45 * Waitting for datapath to complete before resetting queues. 46 * 1 ms should be enough for the ongoing Tx/Rx function to finish. 47 */ 48 rte_delay_ms(1); 49 50 /* Vring reset for each Tx queue and Rx queue. */ 51 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 52 rxvq = eth_dev->data->rx_queues[i]; 53 virtqueue_rxvq_reset_packed(rxvq->vq); 54 virtio_dev_rx_queue_setup_finish(eth_dev, i); 55 } 56 57 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 58 txvq = eth_dev->data->tx_queues[i]; 59 virtqueue_txvq_reset_packed(txvq->vq); 60 } 61 62 hw->started = 1; 63 rte_spinlock_unlock(&hw->state_lock); 64 } 65 66 67 static int 68 virtio_user_server_reconnect(struct virtio_user_dev *dev) 69 { 70 int ret, connectfd, old_status; 71 struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id]; 72 struct virtio_hw *hw = &dev->hw; 73 74 connectfd = accept(dev->listenfd, NULL, NULL); 75 if (connectfd < 0) 76 return -1; 77 78 dev->vhostfd = connectfd; 79 old_status = dev->status; 80 81 virtio_reset(hw); 82 83 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 84 85 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 86 87 if (dev->ops->get_features(dev, &dev->device_features) < 0) { 88 PMD_INIT_LOG(ERR, "get_features failed: %s", 89 strerror(errno)); 90 return -1; 91 } 92 93 dev->device_features |= dev->frontend_features; 94 95 /* umask vhost-user unsupported features */ 96 dev->device_features &= ~(dev->unsupported_features); 97 98 dev->features &= dev->device_features; 99 100 /* For packed ring, resetting queues is required in reconnection. */ 101 if (virtio_with_packed_queue(hw) && 102 (old_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) { 103 PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped" 104 " when packed ring reconnecting."); 105 virtio_user_reset_queues_packed(eth_dev); 106 } 107 108 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); 109 110 /* Start the device */ 111 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK); 112 if (!dev->started) 113 return -1; 114 115 if (dev->queue_pairs > 1) { 116 ret = virtio_user_handle_mq(dev, dev->queue_pairs); 117 if (ret != 0) { 118 PMD_INIT_LOG(ERR, "Fails to enable multi-queue pairs!"); 119 return -1; 120 } 121 } 122 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { 123 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 124 PMD_DRV_LOG(ERR, "interrupt disable failed"); 125 return -1; 126 } 127 rte_intr_callback_unregister(eth_dev->intr_handle, 128 virtio_interrupt_handler, 129 eth_dev); 130 eth_dev->intr_handle->fd = connectfd; 131 rte_intr_callback_register(eth_dev->intr_handle, 132 virtio_interrupt_handler, eth_dev); 133 134 if (rte_intr_enable(eth_dev->intr_handle) < 0) { 135 PMD_DRV_LOG(ERR, "interrupt enable failed"); 136 return -1; 137 } 138 } 139 PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!"); 140 return 0; 141 } 142 143 static void 144 virtio_user_delayed_handler(void *param) 145 { 146 struct virtio_hw *hw = (struct virtio_hw *)param; 147 struct rte_eth_dev *eth_dev = &rte_eth_devices[hw->port_id]; 148 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 149 150 if (rte_intr_disable(eth_dev->intr_handle) < 0) { 151 PMD_DRV_LOG(ERR, "interrupt disable failed"); 152 return; 153 } 154 rte_intr_callback_unregister(eth_dev->intr_handle, 155 virtio_interrupt_handler, eth_dev); 156 if (dev->is_server) { 157 if (dev->vhostfd >= 0) { 158 close(dev->vhostfd); 159 dev->vhostfd = -1; 160 } 161 eth_dev->intr_handle->fd = dev->listenfd; 162 rte_intr_callback_register(eth_dev->intr_handle, 163 virtio_interrupt_handler, eth_dev); 164 if (rte_intr_enable(eth_dev->intr_handle) < 0) { 165 PMD_DRV_LOG(ERR, "interrupt enable failed"); 166 return; 167 } 168 } 169 } 170 171 static void 172 virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, 173 void *dst, int length) 174 { 175 int i; 176 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 177 178 if (offset == offsetof(struct virtio_net_config, mac) && 179 length == RTE_ETHER_ADDR_LEN) { 180 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) 181 ((uint8_t *)dst)[i] = dev->mac_addr[i]; 182 return; 183 } 184 185 if (offset == offsetof(struct virtio_net_config, status)) { 186 char buf[128]; 187 188 if (dev->vhostfd >= 0) { 189 int r; 190 int flags; 191 192 flags = fcntl(dev->vhostfd, F_GETFL); 193 if (fcntl(dev->vhostfd, F_SETFL, 194 flags | O_NONBLOCK) == -1) { 195 PMD_DRV_LOG(ERR, "error setting O_NONBLOCK flag"); 196 return; 197 } 198 r = recv(dev->vhostfd, buf, 128, MSG_PEEK); 199 if (r == 0 || (r < 0 && errno != EAGAIN)) { 200 dev->net_status &= (~VIRTIO_NET_S_LINK_UP); 201 PMD_DRV_LOG(ERR, "virtio-user port %u is down", 202 hw->port_id); 203 204 /* This function could be called in the process 205 * of interrupt handling, callback cannot be 206 * unregistered here, set an alarm to do it. 207 */ 208 rte_eal_alarm_set(1, 209 virtio_user_delayed_handler, 210 (void *)hw); 211 } else { 212 dev->net_status |= VIRTIO_NET_S_LINK_UP; 213 } 214 if (fcntl(dev->vhostfd, F_SETFL, 215 flags & ~O_NONBLOCK) == -1) { 216 PMD_DRV_LOG(ERR, "error clearing O_NONBLOCK flag"); 217 return; 218 } 219 } else if (dev->is_server) { 220 dev->net_status &= (~VIRTIO_NET_S_LINK_UP); 221 if (virtio_user_server_reconnect(dev) >= 0) 222 dev->net_status |= VIRTIO_NET_S_LINK_UP; 223 } 224 225 *(uint16_t *)dst = dev->net_status; 226 } 227 228 if (offset == offsetof(struct virtio_net_config, max_virtqueue_pairs)) 229 *(uint16_t *)dst = dev->max_queue_pairs; 230 } 231 232 static void 233 virtio_user_write_dev_config(struct virtio_hw *hw, size_t offset, 234 const void *src, int length) 235 { 236 int i; 237 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 238 239 if ((offset == offsetof(struct virtio_net_config, mac)) && 240 (length == RTE_ETHER_ADDR_LEN)) 241 for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) 242 dev->mac_addr[i] = ((const uint8_t *)src)[i]; 243 else 244 PMD_DRV_LOG(ERR, "not supported offset=%zu, len=%d", 245 offset, length); 246 } 247 248 static void 249 virtio_user_reset(struct virtio_hw *hw) 250 { 251 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 252 253 if (dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK) 254 virtio_user_stop_device(dev); 255 } 256 257 static void 258 virtio_user_set_status(struct virtio_hw *hw, uint8_t status) 259 { 260 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 261 uint8_t old_status = dev->status; 262 263 if (status & VIRTIO_CONFIG_STATUS_FEATURES_OK && 264 ~old_status & VIRTIO_CONFIG_STATUS_FEATURES_OK) 265 virtio_user_dev_set_features(dev); 266 if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK) 267 virtio_user_start_device(dev); 268 else if (status == VIRTIO_CONFIG_STATUS_RESET) 269 virtio_user_reset(hw); 270 271 virtio_user_dev_set_status(dev, status); 272 } 273 274 static uint8_t 275 virtio_user_get_status(struct virtio_hw *hw) 276 { 277 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 278 279 virtio_user_dev_update_status(dev); 280 281 return dev->status; 282 } 283 284 static uint64_t 285 virtio_user_get_features(struct virtio_hw *hw) 286 { 287 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 288 289 /* unmask feature bits defined in vhost user protocol */ 290 return dev->device_features & VIRTIO_PMD_SUPPORTED_GUEST_FEATURES; 291 } 292 293 static void 294 virtio_user_set_features(struct virtio_hw *hw, uint64_t features) 295 { 296 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 297 298 dev->features = features & dev->device_features; 299 } 300 301 static int 302 virtio_user_features_ok(struct virtio_hw *hw __rte_unused) 303 { 304 return 0; 305 } 306 307 static uint8_t 308 virtio_user_get_isr(struct virtio_hw *hw __rte_unused) 309 { 310 /* rxq interrupts and config interrupt are separated in virtio-user, 311 * here we only report config change. 312 */ 313 return VIRTIO_ISR_CONFIG; 314 } 315 316 static uint16_t 317 virtio_user_set_config_irq(struct virtio_hw *hw __rte_unused, 318 uint16_t vec __rte_unused) 319 { 320 return 0; 321 } 322 323 static uint16_t 324 virtio_user_set_queue_irq(struct virtio_hw *hw __rte_unused, 325 struct virtqueue *vq __rte_unused, 326 uint16_t vec) 327 { 328 /* pretend we have done that */ 329 return vec; 330 } 331 332 /* This function is to get the queue size, aka, number of descs, of a specified 333 * queue. Different with the VHOST_USER_GET_QUEUE_NUM, which is used to get the 334 * max supported queues. 335 */ 336 static uint16_t 337 virtio_user_get_queue_num(struct virtio_hw *hw, uint16_t queue_id __rte_unused) 338 { 339 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 340 341 /* Currently, each queue has same queue size */ 342 return dev->queue_size; 343 } 344 345 static void 346 virtio_user_setup_queue_packed(struct virtqueue *vq, 347 struct virtio_user_dev *dev) 348 { 349 uint16_t queue_idx = vq->vq_queue_index; 350 struct vring_packed *vring; 351 uint64_t desc_addr; 352 uint64_t avail_addr; 353 uint64_t used_addr; 354 uint16_t i; 355 356 vring = &dev->packed_vrings[queue_idx]; 357 desc_addr = (uintptr_t)vq->vq_ring_virt_mem; 358 avail_addr = desc_addr + vq->vq_nentries * 359 sizeof(struct vring_packed_desc); 360 used_addr = RTE_ALIGN_CEIL(avail_addr + 361 sizeof(struct vring_packed_desc_event), 362 VIRTIO_VRING_ALIGN); 363 vring->num = vq->vq_nentries; 364 vring->desc = (void *)(uintptr_t)desc_addr; 365 vring->driver = (void *)(uintptr_t)avail_addr; 366 vring->device = (void *)(uintptr_t)used_addr; 367 dev->packed_queues[queue_idx].avail_wrap_counter = true; 368 dev->packed_queues[queue_idx].used_wrap_counter = true; 369 370 for (i = 0; i < vring->num; i++) 371 vring->desc[i].flags = 0; 372 } 373 374 static void 375 virtio_user_setup_queue_split(struct virtqueue *vq, struct virtio_user_dev *dev) 376 { 377 uint16_t queue_idx = vq->vq_queue_index; 378 uint64_t desc_addr, avail_addr, used_addr; 379 380 desc_addr = (uintptr_t)vq->vq_ring_virt_mem; 381 avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc); 382 used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail, 383 ring[vq->vq_nentries]), 384 VIRTIO_VRING_ALIGN); 385 386 dev->vrings[queue_idx].num = vq->vq_nentries; 387 dev->vrings[queue_idx].desc = (void *)(uintptr_t)desc_addr; 388 dev->vrings[queue_idx].avail = (void *)(uintptr_t)avail_addr; 389 dev->vrings[queue_idx].used = (void *)(uintptr_t)used_addr; 390 } 391 392 static int 393 virtio_user_setup_queue(struct virtio_hw *hw, struct virtqueue *vq) 394 { 395 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 396 397 if (virtio_with_packed_queue(hw)) 398 virtio_user_setup_queue_packed(vq, dev); 399 else 400 virtio_user_setup_queue_split(vq, dev); 401 402 return 0; 403 } 404 405 static void 406 virtio_user_del_queue(struct virtio_hw *hw, struct virtqueue *vq) 407 { 408 /* For legacy devices, write 0 to VIRTIO_PCI_QUEUE_PFN port, QEMU 409 * correspondingly stops the ioeventfds, and reset the status of 410 * the device. 411 * For modern devices, set queue desc, avail, used in PCI bar to 0, 412 * not see any more behavior in QEMU. 413 * 414 * Here we just care about what information to deliver to vhost-user 415 * or vhost-kernel. So we just close ioeventfd for now. 416 */ 417 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 418 419 close(dev->callfds[vq->vq_queue_index]); 420 close(dev->kickfds[vq->vq_queue_index]); 421 } 422 423 static void 424 virtio_user_notify_queue(struct virtio_hw *hw, struct virtqueue *vq) 425 { 426 uint64_t buf = 1; 427 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 428 429 if (hw->cvq && (hw->cvq->vq == vq)) { 430 if (virtio_with_packed_queue(vq->hw)) 431 virtio_user_handle_cq_packed(dev, vq->vq_queue_index); 432 else 433 virtio_user_handle_cq(dev, vq->vq_queue_index); 434 return; 435 } 436 437 if (write(dev->kickfds[vq->vq_queue_index], &buf, sizeof(buf)) < 0) 438 PMD_DRV_LOG(ERR, "failed to kick backend: %s", 439 strerror(errno)); 440 } 441 442 static int 443 virtio_user_dev_close(struct virtio_hw *hw) 444 { 445 struct virtio_user_dev *dev = virtio_user_get_dev(hw); 446 447 virtio_user_dev_uninit(dev); 448 449 return 0; 450 } 451 452 const struct virtio_ops virtio_user_ops = { 453 .read_dev_cfg = virtio_user_read_dev_config, 454 .write_dev_cfg = virtio_user_write_dev_config, 455 .get_status = virtio_user_get_status, 456 .set_status = virtio_user_set_status, 457 .get_features = virtio_user_get_features, 458 .set_features = virtio_user_set_features, 459 .features_ok = virtio_user_features_ok, 460 .get_isr = virtio_user_get_isr, 461 .set_config_irq = virtio_user_set_config_irq, 462 .set_queue_irq = virtio_user_set_queue_irq, 463 .get_queue_num = virtio_user_get_queue_num, 464 .setup_queue = virtio_user_setup_queue, 465 .del_queue = virtio_user_del_queue, 466 .notify_queue = virtio_user_notify_queue, 467 .dev_close = virtio_user_dev_close, 468 }; 469 470 static const char *valid_args[] = { 471 #define VIRTIO_USER_ARG_QUEUES_NUM "queues" 472 VIRTIO_USER_ARG_QUEUES_NUM, 473 #define VIRTIO_USER_ARG_CQ_NUM "cq" 474 VIRTIO_USER_ARG_CQ_NUM, 475 #define VIRTIO_USER_ARG_MAC "mac" 476 VIRTIO_USER_ARG_MAC, 477 #define VIRTIO_USER_ARG_PATH "path" 478 VIRTIO_USER_ARG_PATH, 479 #define VIRTIO_USER_ARG_QUEUE_SIZE "queue_size" 480 VIRTIO_USER_ARG_QUEUE_SIZE, 481 #define VIRTIO_USER_ARG_INTERFACE_NAME "iface" 482 VIRTIO_USER_ARG_INTERFACE_NAME, 483 #define VIRTIO_USER_ARG_SERVER_MODE "server" 484 VIRTIO_USER_ARG_SERVER_MODE, 485 #define VIRTIO_USER_ARG_MRG_RXBUF "mrg_rxbuf" 486 VIRTIO_USER_ARG_MRG_RXBUF, 487 #define VIRTIO_USER_ARG_IN_ORDER "in_order" 488 VIRTIO_USER_ARG_IN_ORDER, 489 #define VIRTIO_USER_ARG_PACKED_VQ "packed_vq" 490 VIRTIO_USER_ARG_PACKED_VQ, 491 #define VIRTIO_USER_ARG_SPEED "speed" 492 VIRTIO_USER_ARG_SPEED, 493 #define VIRTIO_USER_ARG_VECTORIZED "vectorized" 494 VIRTIO_USER_ARG_VECTORIZED, 495 NULL 496 }; 497 498 #define VIRTIO_USER_DEF_CQ_EN 0 499 #define VIRTIO_USER_DEF_Q_NUM 1 500 #define VIRTIO_USER_DEF_Q_SZ 256 501 #define VIRTIO_USER_DEF_SERVER_MODE 0 502 503 static int 504 get_string_arg(const char *key __rte_unused, 505 const char *value, void *extra_args) 506 { 507 if (!value || !extra_args) 508 return -EINVAL; 509 510 *(char **)extra_args = strdup(value); 511 512 if (!*(char **)extra_args) 513 return -ENOMEM; 514 515 return 0; 516 } 517 518 static int 519 get_integer_arg(const char *key __rte_unused, 520 const char *value, void *extra_args) 521 { 522 uint64_t integer = 0; 523 if (!value || !extra_args) 524 return -EINVAL; 525 errno = 0; 526 integer = strtoull(value, NULL, 0); 527 /* extra_args keeps default value, it should be replaced 528 * only in case of successful parsing of the 'value' arg 529 */ 530 if (errno == 0) 531 *(uint64_t *)extra_args = integer; 532 return -errno; 533 } 534 535 static uint32_t 536 vdpa_dynamic_major_num(void) 537 { 538 FILE *fp; 539 char *line = NULL; 540 size_t size; 541 char name[11]; 542 bool found = false; 543 uint32_t num; 544 545 fp = fopen("/proc/devices", "r"); 546 if (fp == NULL) { 547 PMD_INIT_LOG(ERR, "Cannot open /proc/devices: %s", 548 strerror(errno)); 549 return UNNAMED_MAJOR; 550 } 551 552 while (getline(&line, &size, fp) > 0) { 553 char *stripped = line + strspn(line, " "); 554 if ((sscanf(stripped, "%u %10s", &num, name) == 2) && 555 (strncmp(name, "vhost-vdpa", 10) == 0)) { 556 found = true; 557 break; 558 } 559 } 560 fclose(fp); 561 return found ? num : UNNAMED_MAJOR; 562 } 563 564 static enum virtio_user_backend_type 565 virtio_user_backend_type(const char *path) 566 { 567 struct stat sb; 568 569 if (stat(path, &sb) == -1) { 570 if (errno == ENOENT) 571 return VIRTIO_USER_BACKEND_VHOST_USER; 572 573 PMD_INIT_LOG(ERR, "Stat fails: %s (%s)\n", path, 574 strerror(errno)); 575 return VIRTIO_USER_BACKEND_UNKNOWN; 576 } 577 578 if (S_ISSOCK(sb.st_mode)) { 579 return VIRTIO_USER_BACKEND_VHOST_USER; 580 } else if (S_ISCHR(sb.st_mode)) { 581 if (major(sb.st_rdev) == MISC_MAJOR) 582 return VIRTIO_USER_BACKEND_VHOST_KERNEL; 583 if (major(sb.st_rdev) == vdpa_dynamic_major_num()) 584 return VIRTIO_USER_BACKEND_VHOST_VDPA; 585 } 586 return VIRTIO_USER_BACKEND_UNKNOWN; 587 } 588 589 static struct rte_eth_dev * 590 virtio_user_eth_dev_alloc(struct rte_vdev_device *vdev) 591 { 592 struct rte_eth_dev *eth_dev; 593 struct rte_eth_dev_data *data; 594 struct virtio_hw *hw; 595 struct virtio_user_dev *dev; 596 597 eth_dev = rte_eth_vdev_allocate(vdev, sizeof(*dev)); 598 if (!eth_dev) { 599 PMD_INIT_LOG(ERR, "cannot alloc rte_eth_dev"); 600 return NULL; 601 } 602 603 data = eth_dev->data; 604 dev = eth_dev->data->dev_private; 605 hw = &dev->hw; 606 607 hw->port_id = data->port_id; 608 dev->port_id = data->port_id; 609 VIRTIO_OPS(hw) = &virtio_user_ops; 610 611 hw->intr_lsc = 1; 612 hw->use_vec_rx = 0; 613 hw->use_vec_tx = 0; 614 hw->use_inorder_rx = 0; 615 hw->use_inorder_tx = 0; 616 617 return eth_dev; 618 } 619 620 static void 621 virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev) 622 { 623 rte_eth_dev_release_port(eth_dev); 624 } 625 626 /* Dev initialization routine. Invoked once for each virtio vdev at 627 * EAL init time, see rte_bus_probe(). 628 * Returns 0 on success. 629 */ 630 static int 631 virtio_user_pmd_probe(struct rte_vdev_device *vdev) 632 { 633 struct rte_kvargs *kvlist = NULL; 634 struct rte_eth_dev *eth_dev; 635 struct virtio_hw *hw; 636 struct virtio_user_dev *dev; 637 enum virtio_user_backend_type backend_type = VIRTIO_USER_BACKEND_UNKNOWN; 638 uint64_t queues = VIRTIO_USER_DEF_Q_NUM; 639 uint64_t cq = VIRTIO_USER_DEF_CQ_EN; 640 uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ; 641 uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE; 642 uint64_t mrg_rxbuf = 1; 643 uint64_t in_order = 1; 644 uint64_t packed_vq = 0; 645 uint64_t vectorized = 0; 646 char *path = NULL; 647 char *ifname = NULL; 648 char *mac_addr = NULL; 649 int ret = -1; 650 651 RTE_BUILD_BUG_ON(offsetof(struct virtio_user_dev, hw) != 0); 652 653 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 654 const char *name = rte_vdev_device_name(vdev); 655 eth_dev = rte_eth_dev_attach_secondary(name); 656 if (!eth_dev) { 657 PMD_INIT_LOG(ERR, "Failed to probe %s", name); 658 return -1; 659 } 660 661 dev = eth_dev->data->dev_private; 662 hw = &dev->hw; 663 VIRTIO_OPS(hw) = &virtio_user_ops; 664 665 if (eth_virtio_dev_init(eth_dev) < 0) { 666 PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails"); 667 rte_eth_dev_release_port(eth_dev); 668 return -1; 669 } 670 671 eth_dev->dev_ops = &virtio_user_secondary_eth_dev_ops; 672 eth_dev->device = &vdev->device; 673 rte_eth_dev_probing_finish(eth_dev); 674 return 0; 675 } 676 677 kvlist = rte_kvargs_parse(rte_vdev_device_args(vdev), valid_args); 678 if (!kvlist) { 679 PMD_INIT_LOG(ERR, "error when parsing param"); 680 goto end; 681 } 682 683 if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PATH) == 1) { 684 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PATH, 685 &get_string_arg, &path) < 0) { 686 PMD_INIT_LOG(ERR, "error to parse %s", 687 VIRTIO_USER_ARG_PATH); 688 goto end; 689 } 690 } else { 691 PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user", 692 VIRTIO_USER_ARG_PATH); 693 goto end; 694 } 695 696 backend_type = virtio_user_backend_type(path); 697 if (backend_type == VIRTIO_USER_BACKEND_UNKNOWN) { 698 PMD_INIT_LOG(ERR, 699 "unable to determine backend type for path %s", 700 path); 701 goto end; 702 } 703 PMD_INIT_LOG(INFO, "Backend type detected: %s", 704 virtio_user_backend_strings[backend_type]); 705 706 if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME) == 1) { 707 if (backend_type != VIRTIO_USER_BACKEND_VHOST_KERNEL) { 708 PMD_INIT_LOG(ERR, 709 "arg %s applies only to vhost-kernel backend", 710 VIRTIO_USER_ARG_INTERFACE_NAME); 711 goto end; 712 } 713 714 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME, 715 &get_string_arg, &ifname) < 0) { 716 PMD_INIT_LOG(ERR, "error to parse %s", 717 VIRTIO_USER_ARG_INTERFACE_NAME); 718 goto end; 719 } 720 } 721 722 if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MAC) == 1) { 723 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MAC, 724 &get_string_arg, &mac_addr) < 0) { 725 PMD_INIT_LOG(ERR, "error to parse %s", 726 VIRTIO_USER_ARG_MAC); 727 goto end; 728 } 729 } 730 731 if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE) == 1) { 732 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE, 733 &get_integer_arg, &queue_size) < 0) { 734 PMD_INIT_LOG(ERR, "error to parse %s", 735 VIRTIO_USER_ARG_QUEUE_SIZE); 736 goto end; 737 } 738 } 739 740 if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUES_NUM) == 1) { 741 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUES_NUM, 742 &get_integer_arg, &queues) < 0) { 743 PMD_INIT_LOG(ERR, "error to parse %s", 744 VIRTIO_USER_ARG_QUEUES_NUM); 745 goto end; 746 } 747 } 748 749 if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) { 750 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE, 751 &get_integer_arg, &server_mode) < 0) { 752 PMD_INIT_LOG(ERR, "error to parse %s", 753 VIRTIO_USER_ARG_SERVER_MODE); 754 goto end; 755 } 756 } 757 758 if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) { 759 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM, 760 &get_integer_arg, &cq) < 0) { 761 PMD_INIT_LOG(ERR, "error to parse %s", 762 VIRTIO_USER_ARG_CQ_NUM); 763 goto end; 764 } 765 } else if (queues > 1) { 766 cq = 1; 767 } 768 769 if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PACKED_VQ) == 1) { 770 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PACKED_VQ, 771 &get_integer_arg, &packed_vq) < 0) { 772 PMD_INIT_LOG(ERR, "error to parse %s", 773 VIRTIO_USER_ARG_PACKED_VQ); 774 goto end; 775 } 776 } 777 778 if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_VECTORIZED) == 1) { 779 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_VECTORIZED, 780 &get_integer_arg, &vectorized) < 0) { 781 PMD_INIT_LOG(ERR, "error to parse %s", 782 VIRTIO_USER_ARG_VECTORIZED); 783 goto end; 784 } 785 } 786 787 if (queues > 1 && cq == 0) { 788 PMD_INIT_LOG(ERR, "multi-q requires ctrl-q"); 789 goto end; 790 } 791 792 if (queues > VIRTIO_MAX_VIRTQUEUE_PAIRS) { 793 PMD_INIT_LOG(ERR, "arg %s %" PRIu64 " exceeds the limit %u", 794 VIRTIO_USER_ARG_QUEUES_NUM, queues, 795 VIRTIO_MAX_VIRTQUEUE_PAIRS); 796 goto end; 797 } 798 799 if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MRG_RXBUF) == 1) { 800 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MRG_RXBUF, 801 &get_integer_arg, &mrg_rxbuf) < 0) { 802 PMD_INIT_LOG(ERR, "error to parse %s", 803 VIRTIO_USER_ARG_MRG_RXBUF); 804 goto end; 805 } 806 } 807 808 if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_IN_ORDER) == 1) { 809 if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_IN_ORDER, 810 &get_integer_arg, &in_order) < 0) { 811 PMD_INIT_LOG(ERR, "error to parse %s", 812 VIRTIO_USER_ARG_IN_ORDER); 813 goto end; 814 } 815 } 816 817 eth_dev = virtio_user_eth_dev_alloc(vdev); 818 if (!eth_dev) { 819 PMD_INIT_LOG(ERR, "virtio_user fails to alloc device"); 820 goto end; 821 } 822 823 dev = eth_dev->data->dev_private; 824 hw = &dev->hw; 825 if (virtio_user_dev_init(dev, path, queues, cq, 826 queue_size, mac_addr, &ifname, server_mode, 827 mrg_rxbuf, in_order, packed_vq, backend_type) < 0) { 828 PMD_INIT_LOG(ERR, "virtio_user_dev_init fails"); 829 virtio_user_eth_dev_free(eth_dev); 830 goto end; 831 } 832 833 /* previously called by pci probing for physical dev */ 834 if (eth_virtio_dev_init(eth_dev) < 0) { 835 PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails"); 836 virtio_user_eth_dev_free(eth_dev); 837 goto end; 838 } 839 840 if (vectorized) { 841 if (packed_vq) { 842 #if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM) 843 hw->use_vec_rx = 1; 844 hw->use_vec_tx = 1; 845 #else 846 PMD_INIT_LOG(INFO, 847 "building environment do not support packed ring vectorized"); 848 #endif 849 } else { 850 hw->use_vec_rx = 1; 851 } 852 } 853 854 rte_eth_dev_probing_finish(eth_dev); 855 ret = 0; 856 857 end: 858 if (kvlist) 859 rte_kvargs_free(kvlist); 860 if (path) 861 free(path); 862 if (mac_addr) 863 free(mac_addr); 864 if (ifname) 865 free(ifname); 866 return ret; 867 } 868 869 static int 870 virtio_user_pmd_remove(struct rte_vdev_device *vdev) 871 { 872 const char *name; 873 struct rte_eth_dev *eth_dev; 874 875 if (!vdev) 876 return -EINVAL; 877 878 name = rte_vdev_device_name(vdev); 879 PMD_DRV_LOG(INFO, "Un-Initializing %s", name); 880 eth_dev = rte_eth_dev_allocated(name); 881 /* Port has already been released by close. */ 882 if (!eth_dev) 883 return 0; 884 885 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 886 return rte_eth_dev_release_port(eth_dev); 887 888 /* make sure the device is stopped, queues freed */ 889 return rte_eth_dev_close(eth_dev->data->port_id); 890 } 891 892 static int virtio_user_pmd_dma_map(struct rte_vdev_device *vdev, void *addr, 893 uint64_t iova, size_t len) 894 { 895 const char *name; 896 struct rte_eth_dev *eth_dev; 897 struct virtio_user_dev *dev; 898 899 if (!vdev) 900 return -EINVAL; 901 902 name = rte_vdev_device_name(vdev); 903 eth_dev = rte_eth_dev_allocated(name); 904 /* Port has already been released by close. */ 905 if (!eth_dev) 906 return 0; 907 908 dev = eth_dev->data->dev_private; 909 910 if (dev->ops->dma_map) 911 return dev->ops->dma_map(dev, addr, iova, len); 912 913 return 0; 914 } 915 916 static int virtio_user_pmd_dma_unmap(struct rte_vdev_device *vdev, void *addr, 917 uint64_t iova, size_t len) 918 { 919 const char *name; 920 struct rte_eth_dev *eth_dev; 921 struct virtio_user_dev *dev; 922 923 if (!vdev) 924 return -EINVAL; 925 926 name = rte_vdev_device_name(vdev); 927 eth_dev = rte_eth_dev_allocated(name); 928 /* Port has already been released by close. */ 929 if (!eth_dev) 930 return 0; 931 932 dev = eth_dev->data->dev_private; 933 934 if (dev->ops->dma_unmap) 935 return dev->ops->dma_unmap(dev, addr, iova, len); 936 937 return 0; 938 } 939 940 static struct rte_vdev_driver virtio_user_driver = { 941 .probe = virtio_user_pmd_probe, 942 .remove = virtio_user_pmd_remove, 943 .dma_map = virtio_user_pmd_dma_map, 944 .dma_unmap = virtio_user_pmd_dma_unmap, 945 .drv_flags = RTE_VDEV_DRV_NEED_IOVA_AS_VA, 946 }; 947 948 RTE_PMD_REGISTER_VDEV(net_virtio_user, virtio_user_driver); 949 RTE_PMD_REGISTER_ALIAS(net_virtio_user, virtio_user); 950 RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user, 951 "path=<path> " 952 "mac=<mac addr> " 953 "cq=<int> " 954 "queue_size=<int> " 955 "queues=<int> " 956 "iface=<string> " 957 "server=<0|1> " 958 "mrg_rxbuf=<0|1> " 959 "in_order=<0|1> " 960 "packed_vq=<0|1> " 961 "speed=<int> " 962 "vectorized=<0|1>"); 963