1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <arpa/inet.h> 35 #include <getopt.h> 36 #include <linux/if_ether.h> 37 #include <linux/if_vlan.h> 38 #include <linux/virtio_net.h> 39 #include <linux/virtio_ring.h> 40 #include <signal.h> 41 #include <stdint.h> 42 #include <sys/eventfd.h> 43 #include <sys/param.h> 44 #include <unistd.h> 45 46 #include <rte_atomic.h> 47 #include <rte_cycles.h> 48 #include <rte_ethdev.h> 49 #include <rte_log.h> 50 #include <rte_string_fns.h> 51 #include <rte_malloc.h> 52 #include <rte_virtio_net.h> 53 #include <rte_ip.h> 54 #include <rte_tcp.h> 55 56 #include "main.h" 57 58 #ifndef MAX_QUEUES 59 #define MAX_QUEUES 128 60 #endif 61 62 /* the maximum number of external ports supported */ 63 #define MAX_SUP_PORTS 1 64 65 /* 66 * Calculate the number of buffers needed per port 67 */ 68 #define NUM_MBUFS_PER_PORT ((MAX_QUEUES*RTE_TEST_RX_DESC_DEFAULT) + \ 69 (num_switching_cores*MAX_PKT_BURST) + \ 70 (num_switching_cores*RTE_TEST_TX_DESC_DEFAULT) +\ 71 ((num_switching_cores+1)*MBUF_CACHE_SIZE)) 72 73 #define MBUF_CACHE_SIZE 128 74 #define MBUF_DATA_SIZE RTE_MBUF_DEFAULT_BUF_SIZE 75 76 #define MAX_PKT_BURST 32 /* Max burst size for RX/TX */ 77 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 78 79 #define BURST_RX_WAIT_US 15 /* Defines how long we wait between retries on RX */ 80 #define BURST_RX_RETRIES 4 /* Number of retries on RX. */ 81 82 #define JUMBO_FRAME_MAX_SIZE 0x2600 83 84 /* State of virtio device. */ 85 #define DEVICE_MAC_LEARNING 0 86 #define DEVICE_RX 1 87 #define DEVICE_SAFE_REMOVE 2 88 89 /* Configurable number of RX/TX ring descriptors */ 90 #define RTE_TEST_RX_DESC_DEFAULT 1024 91 #define RTE_TEST_TX_DESC_DEFAULT 512 92 93 #define INVALID_PORT_ID 0xFF 94 95 /* Max number of devices. Limited by vmdq. */ 96 #define MAX_DEVICES 64 97 98 /* Size of buffers used for snprintfs. */ 99 #define MAX_PRINT_BUFF 6072 100 101 /* Maximum character device basename size. */ 102 #define MAX_BASENAME_SZ 10 103 104 /* Maximum long option length for option parsing. */ 105 #define MAX_LONG_OPT_SZ 64 106 107 /* Used to compare MAC addresses. */ 108 #define MAC_ADDR_CMP 0xFFFFFFFFFFFFULL 109 110 /* mask of enabled ports */ 111 static uint32_t enabled_port_mask = 0; 112 113 /* Promiscuous mode */ 114 static uint32_t promiscuous; 115 116 /*Number of switching cores enabled*/ 117 static uint32_t num_switching_cores = 0; 118 119 /* number of devices/queues to support*/ 120 static uint32_t num_queues = 0; 121 static uint32_t num_devices; 122 123 static struct rte_mempool *mbuf_pool; 124 static int mergeable; 125 126 /* Do vlan strip on host, enabled on default */ 127 static uint32_t vlan_strip = 1; 128 129 /* Enable VM2VM communications. If this is disabled then the MAC address compare is skipped. */ 130 typedef enum { 131 VM2VM_DISABLED = 0, 132 VM2VM_SOFTWARE = 1, 133 VM2VM_HARDWARE = 2, 134 VM2VM_LAST 135 } vm2vm_type; 136 static vm2vm_type vm2vm_mode = VM2VM_SOFTWARE; 137 138 /* Enable stats. */ 139 static uint32_t enable_stats = 0; 140 /* Enable retries on RX. */ 141 static uint32_t enable_retry = 1; 142 143 /* Disable TX checksum offload */ 144 static uint32_t enable_tx_csum; 145 146 /* Disable TSO offload */ 147 static uint32_t enable_tso; 148 149 /* Specify timeout (in useconds) between retries on RX. */ 150 static uint32_t burst_rx_delay_time = BURST_RX_WAIT_US; 151 /* Specify the number of retries on RX. */ 152 static uint32_t burst_rx_retry_num = BURST_RX_RETRIES; 153 154 /* Character device basename. Can be set by user. */ 155 static char dev_basename[MAX_BASENAME_SZ] = "vhost-net"; 156 157 /* empty vmdq configuration structure. Filled in programatically */ 158 static struct rte_eth_conf vmdq_conf_default = { 159 .rxmode = { 160 .mq_mode = ETH_MQ_RX_VMDQ_ONLY, 161 .split_hdr_size = 0, 162 .header_split = 0, /**< Header Split disabled */ 163 .hw_ip_checksum = 0, /**< IP checksum offload disabled */ 164 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 165 /* 166 * It is necessary for 1G NIC such as I350, 167 * this fixes bug of ipv4 forwarding in guest can't 168 * forward pakets from one virtio dev to another virtio dev. 169 */ 170 .hw_vlan_strip = 1, /**< VLAN strip enabled. */ 171 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 172 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 173 }, 174 175 .txmode = { 176 .mq_mode = ETH_MQ_TX_NONE, 177 }, 178 .rx_adv_conf = { 179 /* 180 * should be overridden separately in code with 181 * appropriate values 182 */ 183 .vmdq_rx_conf = { 184 .nb_queue_pools = ETH_8_POOLS, 185 .enable_default_pool = 0, 186 .default_pool = 0, 187 .nb_pool_maps = 0, 188 .pool_map = {{0, 0},}, 189 }, 190 }, 191 }; 192 193 static unsigned lcore_ids[RTE_MAX_LCORE]; 194 static uint8_t ports[RTE_MAX_ETHPORTS]; 195 static unsigned num_ports = 0; /**< The number of ports specified in command line */ 196 static uint16_t num_pf_queues, num_vmdq_queues; 197 static uint16_t vmdq_pool_base, vmdq_queue_base; 198 static uint16_t queues_per_pool; 199 200 const uint16_t vlan_tags[] = { 201 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 202 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 203 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 204 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 205 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 206 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 207 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 208 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, 209 }; 210 211 /* ethernet addresses of ports */ 212 static struct ether_addr vmdq_ports_eth_addr[RTE_MAX_ETHPORTS]; 213 214 static struct vhost_dev_tailq_list vhost_dev_list = 215 TAILQ_HEAD_INITIALIZER(vhost_dev_list); 216 217 static struct lcore_info lcore_info[RTE_MAX_LCORE]; 218 219 /* Used for queueing bursts of TX packets. */ 220 struct mbuf_table { 221 unsigned len; 222 unsigned txq_id; 223 struct rte_mbuf *m_table[MAX_PKT_BURST]; 224 }; 225 226 /* TX queue for each data core. */ 227 struct mbuf_table lcore_tx_queue[RTE_MAX_LCORE]; 228 229 #define VLAN_HLEN 4 230 231 /* Per-device statistics struct */ 232 struct device_statistics { 233 uint64_t tx_total; 234 rte_atomic64_t rx_total_atomic; 235 uint64_t tx; 236 rte_atomic64_t rx_atomic; 237 } __rte_cache_aligned; 238 struct device_statistics dev_statistics[MAX_DEVICES]; 239 240 /* 241 * Builds up the correct configuration for VMDQ VLAN pool map 242 * according to the pool & queue limits. 243 */ 244 static inline int 245 get_eth_conf(struct rte_eth_conf *eth_conf, uint32_t num_devices) 246 { 247 struct rte_eth_vmdq_rx_conf conf; 248 struct rte_eth_vmdq_rx_conf *def_conf = 249 &vmdq_conf_default.rx_adv_conf.vmdq_rx_conf; 250 unsigned i; 251 252 memset(&conf, 0, sizeof(conf)); 253 conf.nb_queue_pools = (enum rte_eth_nb_pools)num_devices; 254 conf.nb_pool_maps = num_devices; 255 conf.enable_loop_back = def_conf->enable_loop_back; 256 conf.rx_mode = def_conf->rx_mode; 257 258 for (i = 0; i < conf.nb_pool_maps; i++) { 259 conf.pool_map[i].vlan_id = vlan_tags[ i ]; 260 conf.pool_map[i].pools = (1UL << i); 261 } 262 263 (void)(rte_memcpy(eth_conf, &vmdq_conf_default, sizeof(*eth_conf))); 264 (void)(rte_memcpy(ð_conf->rx_adv_conf.vmdq_rx_conf, &conf, 265 sizeof(eth_conf->rx_adv_conf.vmdq_rx_conf))); 266 return 0; 267 } 268 269 /* 270 * Validate the device number according to the max pool number gotten form 271 * dev_info. If the device number is invalid, give the error message and 272 * return -1. Each device must have its own pool. 273 */ 274 static inline int 275 validate_num_devices(uint32_t max_nb_devices) 276 { 277 if (num_devices > max_nb_devices) { 278 RTE_LOG(ERR, VHOST_PORT, "invalid number of devices\n"); 279 return -1; 280 } 281 return 0; 282 } 283 284 /* 285 * Initialises a given port using global settings and with the rx buffers 286 * coming from the mbuf_pool passed as parameter 287 */ 288 static inline int 289 port_init(uint8_t port) 290 { 291 struct rte_eth_dev_info dev_info; 292 struct rte_eth_conf port_conf; 293 struct rte_eth_rxconf *rxconf; 294 struct rte_eth_txconf *txconf; 295 int16_t rx_rings, tx_rings; 296 uint16_t rx_ring_size, tx_ring_size; 297 int retval; 298 uint16_t q; 299 300 /* The max pool number from dev_info will be used to validate the pool number specified in cmd line */ 301 rte_eth_dev_info_get (port, &dev_info); 302 303 if (dev_info.max_rx_queues > MAX_QUEUES) { 304 rte_exit(EXIT_FAILURE, 305 "please define MAX_QUEUES no less than %u in %s\n", 306 dev_info.max_rx_queues, __FILE__); 307 } 308 309 rxconf = &dev_info.default_rxconf; 310 txconf = &dev_info.default_txconf; 311 rxconf->rx_drop_en = 1; 312 313 /* Enable vlan offload */ 314 txconf->txq_flags &= ~ETH_TXQ_FLAGS_NOVLANOFFL; 315 316 /*configure the number of supported virtio devices based on VMDQ limits */ 317 num_devices = dev_info.max_vmdq_pools; 318 319 rx_ring_size = RTE_TEST_RX_DESC_DEFAULT; 320 tx_ring_size = RTE_TEST_TX_DESC_DEFAULT; 321 tx_rings = (uint16_t)rte_lcore_count(); 322 323 retval = validate_num_devices(MAX_DEVICES); 324 if (retval < 0) 325 return retval; 326 327 /* Get port configuration. */ 328 retval = get_eth_conf(&port_conf, num_devices); 329 if (retval < 0) 330 return retval; 331 /* NIC queues are divided into pf queues and vmdq queues. */ 332 num_pf_queues = dev_info.max_rx_queues - dev_info.vmdq_queue_num; 333 queues_per_pool = dev_info.vmdq_queue_num / dev_info.max_vmdq_pools; 334 num_vmdq_queues = num_devices * queues_per_pool; 335 num_queues = num_pf_queues + num_vmdq_queues; 336 vmdq_queue_base = dev_info.vmdq_queue_base; 337 vmdq_pool_base = dev_info.vmdq_pool_base; 338 printf("pf queue num: %u, configured vmdq pool num: %u, each vmdq pool has %u queues\n", 339 num_pf_queues, num_devices, queues_per_pool); 340 341 if (port >= rte_eth_dev_count()) return -1; 342 343 if (enable_tx_csum == 0) 344 rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_CSUM); 345 346 if (enable_tso == 0) { 347 rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_HOST_TSO4); 348 rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_HOST_TSO6); 349 } 350 351 rx_rings = (uint16_t)dev_info.max_rx_queues; 352 /* Configure ethernet device. */ 353 retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); 354 if (retval != 0) 355 return retval; 356 357 /* Setup the queues. */ 358 for (q = 0; q < rx_rings; q ++) { 359 retval = rte_eth_rx_queue_setup(port, q, rx_ring_size, 360 rte_eth_dev_socket_id(port), 361 rxconf, 362 mbuf_pool); 363 if (retval < 0) 364 return retval; 365 } 366 for (q = 0; q < tx_rings; q ++) { 367 retval = rte_eth_tx_queue_setup(port, q, tx_ring_size, 368 rte_eth_dev_socket_id(port), 369 txconf); 370 if (retval < 0) 371 return retval; 372 } 373 374 /* Start the device. */ 375 retval = rte_eth_dev_start(port); 376 if (retval < 0) { 377 RTE_LOG(ERR, VHOST_DATA, "Failed to start the device.\n"); 378 return retval; 379 } 380 381 if (promiscuous) 382 rte_eth_promiscuous_enable(port); 383 384 rte_eth_macaddr_get(port, &vmdq_ports_eth_addr[port]); 385 RTE_LOG(INFO, VHOST_PORT, "Max virtio devices supported: %u\n", num_devices); 386 RTE_LOG(INFO, VHOST_PORT, "Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8 387 " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n", 388 (unsigned)port, 389 vmdq_ports_eth_addr[port].addr_bytes[0], 390 vmdq_ports_eth_addr[port].addr_bytes[1], 391 vmdq_ports_eth_addr[port].addr_bytes[2], 392 vmdq_ports_eth_addr[port].addr_bytes[3], 393 vmdq_ports_eth_addr[port].addr_bytes[4], 394 vmdq_ports_eth_addr[port].addr_bytes[5]); 395 396 return 0; 397 } 398 399 /* 400 * Set character device basename. 401 */ 402 static int 403 us_vhost_parse_basename(const char *q_arg) 404 { 405 /* parse number string */ 406 407 if (strnlen(q_arg, MAX_BASENAME_SZ) > MAX_BASENAME_SZ) 408 return -1; 409 else 410 snprintf((char*)&dev_basename, MAX_BASENAME_SZ, "%s", q_arg); 411 412 return 0; 413 } 414 415 /* 416 * Parse the portmask provided at run time. 417 */ 418 static int 419 parse_portmask(const char *portmask) 420 { 421 char *end = NULL; 422 unsigned long pm; 423 424 errno = 0; 425 426 /* parse hexadecimal string */ 427 pm = strtoul(portmask, &end, 16); 428 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0)) 429 return -1; 430 431 if (pm == 0) 432 return -1; 433 434 return pm; 435 436 } 437 438 /* 439 * Parse num options at run time. 440 */ 441 static int 442 parse_num_opt(const char *q_arg, uint32_t max_valid_value) 443 { 444 char *end = NULL; 445 unsigned long num; 446 447 errno = 0; 448 449 /* parse unsigned int string */ 450 num = strtoul(q_arg, &end, 10); 451 if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0') || (errno != 0)) 452 return -1; 453 454 if (num > max_valid_value) 455 return -1; 456 457 return num; 458 459 } 460 461 /* 462 * Display usage 463 */ 464 static void 465 us_vhost_usage(const char *prgname) 466 { 467 RTE_LOG(INFO, VHOST_CONFIG, "%s [EAL options] -- -p PORTMASK\n" 468 " --vm2vm [0|1|2]\n" 469 " --rx_retry [0|1] --mergeable [0|1] --stats [0-N]\n" 470 " --dev-basename <name>\n" 471 " --nb-devices ND\n" 472 " -p PORTMASK: Set mask for ports to be used by application\n" 473 " --vm2vm [0|1|2]: disable/software(default)/hardware vm2vm comms\n" 474 " --rx-retry [0|1]: disable/enable(default) retries on rx. Enable retry if destintation queue is full\n" 475 " --rx-retry-delay [0-N]: timeout(in usecond) between retries on RX. This makes effect only if retries on rx enabled\n" 476 " --rx-retry-num [0-N]: the number of retries on rx. This makes effect only if retries on rx enabled\n" 477 " --mergeable [0|1]: disable(default)/enable RX mergeable buffers\n" 478 " --vlan-strip [0|1]: disable/enable(default) RX VLAN strip on host\n" 479 " --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n" 480 " --dev-basename: The basename to be used for the character device.\n" 481 " --tx-csum [0|1] disable/enable TX checksum offload.\n" 482 " --tso [0|1] disable/enable TCP segment offload.\n", 483 prgname); 484 } 485 486 /* 487 * Parse the arguments given in the command line of the application. 488 */ 489 static int 490 us_vhost_parse_args(int argc, char **argv) 491 { 492 int opt, ret; 493 int option_index; 494 unsigned i; 495 const char *prgname = argv[0]; 496 static struct option long_option[] = { 497 {"vm2vm", required_argument, NULL, 0}, 498 {"rx-retry", required_argument, NULL, 0}, 499 {"rx-retry-delay", required_argument, NULL, 0}, 500 {"rx-retry-num", required_argument, NULL, 0}, 501 {"mergeable", required_argument, NULL, 0}, 502 {"vlan-strip", required_argument, NULL, 0}, 503 {"stats", required_argument, NULL, 0}, 504 {"dev-basename", required_argument, NULL, 0}, 505 {"tx-csum", required_argument, NULL, 0}, 506 {"tso", required_argument, NULL, 0}, 507 {NULL, 0, 0, 0}, 508 }; 509 510 /* Parse command line */ 511 while ((opt = getopt_long(argc, argv, "p:P", 512 long_option, &option_index)) != EOF) { 513 switch (opt) { 514 /* Portmask */ 515 case 'p': 516 enabled_port_mask = parse_portmask(optarg); 517 if (enabled_port_mask == 0) { 518 RTE_LOG(INFO, VHOST_CONFIG, "Invalid portmask\n"); 519 us_vhost_usage(prgname); 520 return -1; 521 } 522 break; 523 524 case 'P': 525 promiscuous = 1; 526 vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.rx_mode = 527 ETH_VMDQ_ACCEPT_BROADCAST | 528 ETH_VMDQ_ACCEPT_MULTICAST; 529 rte_vhost_feature_enable(1ULL << VIRTIO_NET_F_CTRL_RX); 530 531 break; 532 533 case 0: 534 /* Enable/disable vm2vm comms. */ 535 if (!strncmp(long_option[option_index].name, "vm2vm", 536 MAX_LONG_OPT_SZ)) { 537 ret = parse_num_opt(optarg, (VM2VM_LAST - 1)); 538 if (ret == -1) { 539 RTE_LOG(INFO, VHOST_CONFIG, 540 "Invalid argument for " 541 "vm2vm [0|1|2]\n"); 542 us_vhost_usage(prgname); 543 return -1; 544 } else { 545 vm2vm_mode = (vm2vm_type)ret; 546 } 547 } 548 549 /* Enable/disable retries on RX. */ 550 if (!strncmp(long_option[option_index].name, "rx-retry", MAX_LONG_OPT_SZ)) { 551 ret = parse_num_opt(optarg, 1); 552 if (ret == -1) { 553 RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry [0|1]\n"); 554 us_vhost_usage(prgname); 555 return -1; 556 } else { 557 enable_retry = ret; 558 } 559 } 560 561 /* Enable/disable TX checksum offload. */ 562 if (!strncmp(long_option[option_index].name, "tx-csum", MAX_LONG_OPT_SZ)) { 563 ret = parse_num_opt(optarg, 1); 564 if (ret == -1) { 565 RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for tx-csum [0|1]\n"); 566 us_vhost_usage(prgname); 567 return -1; 568 } else 569 enable_tx_csum = ret; 570 } 571 572 /* Enable/disable TSO offload. */ 573 if (!strncmp(long_option[option_index].name, "tso", MAX_LONG_OPT_SZ)) { 574 ret = parse_num_opt(optarg, 1); 575 if (ret == -1) { 576 RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for tso [0|1]\n"); 577 us_vhost_usage(prgname); 578 return -1; 579 } else 580 enable_tso = ret; 581 } 582 583 /* Specify the retries delay time (in useconds) on RX. */ 584 if (!strncmp(long_option[option_index].name, "rx-retry-delay", MAX_LONG_OPT_SZ)) { 585 ret = parse_num_opt(optarg, INT32_MAX); 586 if (ret == -1) { 587 RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-delay [0-N]\n"); 588 us_vhost_usage(prgname); 589 return -1; 590 } else { 591 burst_rx_delay_time = ret; 592 } 593 } 594 595 /* Specify the retries number on RX. */ 596 if (!strncmp(long_option[option_index].name, "rx-retry-num", MAX_LONG_OPT_SZ)) { 597 ret = parse_num_opt(optarg, INT32_MAX); 598 if (ret == -1) { 599 RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for rx-retry-num [0-N]\n"); 600 us_vhost_usage(prgname); 601 return -1; 602 } else { 603 burst_rx_retry_num = ret; 604 } 605 } 606 607 /* Enable/disable RX mergeable buffers. */ 608 if (!strncmp(long_option[option_index].name, "mergeable", MAX_LONG_OPT_SZ)) { 609 ret = parse_num_opt(optarg, 1); 610 if (ret == -1) { 611 RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for mergeable [0|1]\n"); 612 us_vhost_usage(prgname); 613 return -1; 614 } else { 615 mergeable = !!ret; 616 if (ret) { 617 vmdq_conf_default.rxmode.jumbo_frame = 1; 618 vmdq_conf_default.rxmode.max_rx_pkt_len 619 = JUMBO_FRAME_MAX_SIZE; 620 } 621 } 622 } 623 624 /* Enable/disable RX VLAN strip on host. */ 625 if (!strncmp(long_option[option_index].name, 626 "vlan-strip", MAX_LONG_OPT_SZ)) { 627 ret = parse_num_opt(optarg, 1); 628 if (ret == -1) { 629 RTE_LOG(INFO, VHOST_CONFIG, 630 "Invalid argument for VLAN strip [0|1]\n"); 631 us_vhost_usage(prgname); 632 return -1; 633 } else { 634 vlan_strip = !!ret; 635 vmdq_conf_default.rxmode.hw_vlan_strip = 636 vlan_strip; 637 } 638 } 639 640 /* Enable/disable stats. */ 641 if (!strncmp(long_option[option_index].name, "stats", MAX_LONG_OPT_SZ)) { 642 ret = parse_num_opt(optarg, INT32_MAX); 643 if (ret == -1) { 644 RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for stats [0..N]\n"); 645 us_vhost_usage(prgname); 646 return -1; 647 } else { 648 enable_stats = ret; 649 } 650 } 651 652 /* Set character device basename. */ 653 if (!strncmp(long_option[option_index].name, "dev-basename", MAX_LONG_OPT_SZ)) { 654 if (us_vhost_parse_basename(optarg) == -1) { 655 RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for character device basename (Max %d characters)\n", MAX_BASENAME_SZ); 656 us_vhost_usage(prgname); 657 return -1; 658 } 659 } 660 661 break; 662 663 /* Invalid option - print options. */ 664 default: 665 us_vhost_usage(prgname); 666 return -1; 667 } 668 } 669 670 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 671 if (enabled_port_mask & (1 << i)) 672 ports[num_ports++] = (uint8_t)i; 673 } 674 675 if ((num_ports == 0) || (num_ports > MAX_SUP_PORTS)) { 676 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u," 677 "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS); 678 return -1; 679 } 680 681 return 0; 682 } 683 684 /* 685 * Update the global var NUM_PORTS and array PORTS according to system ports number 686 * and return valid ports number 687 */ 688 static unsigned check_ports_num(unsigned nb_ports) 689 { 690 unsigned valid_num_ports = num_ports; 691 unsigned portid; 692 693 if (num_ports > nb_ports) { 694 RTE_LOG(INFO, VHOST_PORT, "\nSpecified port number(%u) exceeds total system port number(%u)\n", 695 num_ports, nb_ports); 696 num_ports = nb_ports; 697 } 698 699 for (portid = 0; portid < num_ports; portid ++) { 700 if (ports[portid] >= nb_ports) { 701 RTE_LOG(INFO, VHOST_PORT, "\nSpecified port ID(%u) exceeds max system port ID(%u)\n", 702 ports[portid], (nb_ports - 1)); 703 ports[portid] = INVALID_PORT_ID; 704 valid_num_ports--; 705 } 706 } 707 return valid_num_ports; 708 } 709 710 /* 711 * Compares a packet destination MAC address to a device MAC address. 712 */ 713 static inline int __attribute__((always_inline)) 714 ether_addr_cmp(struct ether_addr *ea, struct ether_addr *eb) 715 { 716 return ((*(uint64_t *)ea ^ *(uint64_t *)eb) & MAC_ADDR_CMP) == 0; 717 } 718 719 static inline struct vhost_dev *__attribute__((always_inline)) 720 find_vhost_dev(struct ether_addr *mac) 721 { 722 struct vhost_dev *vdev; 723 724 TAILQ_FOREACH(vdev, &vhost_dev_list, next) { 725 if (vdev->ready == DEVICE_RX && 726 ether_addr_cmp(mac, &vdev->mac_address)) 727 return vdev; 728 } 729 730 return NULL; 731 } 732 733 /* 734 * This function learns the MAC address of the device and registers this along with a 735 * vlan tag to a VMDQ. 736 */ 737 static int 738 link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m) 739 { 740 struct ether_hdr *pkt_hdr; 741 struct virtio_net *dev = vdev->dev; 742 int i, ret; 743 744 /* Learn MAC address of guest device from packet */ 745 pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); 746 747 if (find_vhost_dev(&pkt_hdr->s_addr)) { 748 RTE_LOG(ERR, VHOST_DATA, 749 "Device (%" PRIu64 ") is using a registered MAC!\n", 750 dev->device_fh); 751 return -1; 752 } 753 754 for (i = 0; i < ETHER_ADDR_LEN; i++) 755 vdev->mac_address.addr_bytes[i] = pkt_hdr->s_addr.addr_bytes[i]; 756 757 /* vlan_tag currently uses the device_id. */ 758 vdev->vlan_tag = vlan_tags[dev->device_fh]; 759 760 /* Print out VMDQ registration info. */ 761 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") MAC_ADDRESS %02x:%02x:%02x:%02x:%02x:%02x and VLAN_TAG %d registered\n", 762 dev->device_fh, 763 vdev->mac_address.addr_bytes[0], vdev->mac_address.addr_bytes[1], 764 vdev->mac_address.addr_bytes[2], vdev->mac_address.addr_bytes[3], 765 vdev->mac_address.addr_bytes[4], vdev->mac_address.addr_bytes[5], 766 vdev->vlan_tag); 767 768 /* Register the MAC address. */ 769 ret = rte_eth_dev_mac_addr_add(ports[0], &vdev->mac_address, 770 (uint32_t)dev->device_fh + vmdq_pool_base); 771 if (ret) 772 RTE_LOG(ERR, VHOST_DATA, "(%"PRIu64") Failed to add device MAC address to VMDQ\n", 773 dev->device_fh); 774 775 /* Enable stripping of the vlan tag as we handle routing. */ 776 if (vlan_strip) 777 rte_eth_dev_set_vlan_strip_on_queue(ports[0], 778 (uint16_t)vdev->vmdq_rx_q, 1); 779 780 /* Set device as ready for RX. */ 781 vdev->ready = DEVICE_RX; 782 783 return 0; 784 } 785 786 /* 787 * Removes MAC address and vlan tag from VMDQ. Ensures that nothing is adding buffers to the RX 788 * queue before disabling RX on the device. 789 */ 790 static inline void 791 unlink_vmdq(struct vhost_dev *vdev) 792 { 793 unsigned i = 0; 794 unsigned rx_count; 795 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 796 797 if (vdev->ready == DEVICE_RX) { 798 /*clear MAC and VLAN settings*/ 799 rte_eth_dev_mac_addr_remove(ports[0], &vdev->mac_address); 800 for (i = 0; i < 6; i++) 801 vdev->mac_address.addr_bytes[i] = 0; 802 803 vdev->vlan_tag = 0; 804 805 /*Clear out the receive buffers*/ 806 rx_count = rte_eth_rx_burst(ports[0], 807 (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST); 808 809 while (rx_count) { 810 for (i = 0; i < rx_count; i++) 811 rte_pktmbuf_free(pkts_burst[i]); 812 813 rx_count = rte_eth_rx_burst(ports[0], 814 (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST); 815 } 816 817 vdev->ready = DEVICE_MAC_LEARNING; 818 } 819 } 820 821 /* 822 * Check if the packet destination MAC address is for a local device. If so then put 823 * the packet on that devices RX queue. If not then return. 824 */ 825 static inline int __attribute__((always_inline)) 826 virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m) 827 { 828 struct ether_hdr *pkt_hdr; 829 uint64_t ret = 0; 830 struct vhost_dev *dst_vdev; 831 uint64_t fh; 832 833 pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); 834 835 dst_vdev = find_vhost_dev(&pkt_hdr->d_addr); 836 if (!dst_vdev) 837 return -1; 838 839 fh = dst_vdev->dev->device_fh; 840 if (fh == vdev->dev->device_fh) { 841 RTE_LOG(DEBUG, VHOST_DATA, 842 "(%" PRIu64 ") TX: src and dst MAC is same. " 843 "Dropping packet.\n", fh); 844 return 0; 845 } 846 847 RTE_LOG(DEBUG, VHOST_DATA, 848 "(%" PRIu64 ") TX: MAC address is local\n", fh); 849 850 if (unlikely(dst_vdev->remove)) { 851 RTE_LOG(DEBUG, VHOST_DATA, "(%" PRIu64 ") " 852 "Device is marked for removal\n", fh); 853 return 0; 854 } 855 856 /* send the packet to the local virtio device */ 857 ret = rte_vhost_enqueue_burst(dst_vdev->dev, VIRTIO_RXQ, &m, 1); 858 if (enable_stats) { 859 rte_atomic64_inc(&dev_statistics[fh].rx_total_atomic); 860 rte_atomic64_add(&dev_statistics[fh].rx_atomic, ret); 861 dev_statistics[vdev->dev->device_fh].tx_total++; 862 dev_statistics[vdev->dev->device_fh].tx += ret; 863 } 864 865 return 0; 866 } 867 868 /* 869 * Check if the destination MAC of a packet is one local VM, 870 * and get its vlan tag, and offset if it is. 871 */ 872 static inline int __attribute__((always_inline)) 873 find_local_dest(struct virtio_net *dev, struct rte_mbuf *m, 874 uint32_t *offset, uint16_t *vlan_tag) 875 { 876 struct vhost_dev *dst_vdev; 877 struct ether_hdr *pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); 878 879 dst_vdev = find_vhost_dev(&pkt_hdr->d_addr); 880 if (!dst_vdev) 881 return 0; 882 883 if (dst_vdev->dev->device_fh == dev->device_fh) { 884 RTE_LOG(DEBUG, VHOST_DATA, 885 "(%" PRIu64 ") TX: src and dst MAC is same. " 886 " Dropping packet.\n", dst_vdev->dev->device_fh); 887 return -1; 888 } 889 890 /* 891 * HW vlan strip will reduce the packet length 892 * by minus length of vlan tag, so need restore 893 * the packet length by plus it. 894 */ 895 *offset = VLAN_HLEN; 896 *vlan_tag = vlan_tags[(uint16_t)dst_vdev->dev->device_fh]; 897 898 RTE_LOG(DEBUG, VHOST_DATA, 899 "(%" PRIu64 ") TX: pkt to local VM device id: (%" PRIu64 ") " 900 "vlan tag: %u.\n", 901 dev->device_fh, dst_vdev->dev->device_fh, *vlan_tag); 902 903 return 0; 904 } 905 906 static uint16_t 907 get_psd_sum(void *l3_hdr, uint64_t ol_flags) 908 { 909 if (ol_flags & PKT_TX_IPV4) 910 return rte_ipv4_phdr_cksum(l3_hdr, ol_flags); 911 else /* assume ethertype == ETHER_TYPE_IPv6 */ 912 return rte_ipv6_phdr_cksum(l3_hdr, ol_flags); 913 } 914 915 static void virtio_tx_offload(struct rte_mbuf *m) 916 { 917 void *l3_hdr; 918 struct ipv4_hdr *ipv4_hdr = NULL; 919 struct tcp_hdr *tcp_hdr = NULL; 920 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); 921 922 l3_hdr = (char *)eth_hdr + m->l2_len; 923 924 if (m->ol_flags & PKT_TX_IPV4) { 925 ipv4_hdr = l3_hdr; 926 ipv4_hdr->hdr_checksum = 0; 927 m->ol_flags |= PKT_TX_IP_CKSUM; 928 } 929 930 tcp_hdr = (struct tcp_hdr *)((char *)l3_hdr + m->l3_len); 931 tcp_hdr->cksum = get_psd_sum(l3_hdr, m->ol_flags); 932 } 933 934 /* 935 * This function routes the TX packet to the correct interface. This may be a local device 936 * or the physical port. 937 */ 938 static inline void __attribute__((always_inline)) 939 virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag) 940 { 941 struct mbuf_table *tx_q; 942 struct rte_mbuf **m_table; 943 unsigned len, ret, offset = 0; 944 const uint16_t lcore_id = rte_lcore_id(); 945 struct virtio_net *dev = vdev->dev; 946 struct ether_hdr *nh; 947 948 /*check if destination is local VM*/ 949 if ((vm2vm_mode == VM2VM_SOFTWARE) && (virtio_tx_local(vdev, m) == 0)) { 950 rte_pktmbuf_free(m); 951 return; 952 } 953 954 if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) { 955 if (unlikely(find_local_dest(dev, m, &offset, &vlan_tag) != 0)) { 956 rte_pktmbuf_free(m); 957 return; 958 } 959 } 960 961 RTE_LOG(DEBUG, VHOST_DATA, "(%" PRIu64 ") TX: " 962 "MAC address is external\n", dev->device_fh); 963 964 /*Add packet to the port tx queue*/ 965 tx_q = &lcore_tx_queue[lcore_id]; 966 len = tx_q->len; 967 968 nh = rte_pktmbuf_mtod(m, struct ether_hdr *); 969 if (unlikely(nh->ether_type == rte_cpu_to_be_16(ETHER_TYPE_VLAN))) { 970 /* Guest has inserted the vlan tag. */ 971 struct vlan_hdr *vh = (struct vlan_hdr *) (nh + 1); 972 uint16_t vlan_tag_be = rte_cpu_to_be_16(vlan_tag); 973 if ((vm2vm_mode == VM2VM_HARDWARE) && 974 (vh->vlan_tci != vlan_tag_be)) 975 vh->vlan_tci = vlan_tag_be; 976 } else { 977 m->ol_flags |= PKT_TX_VLAN_PKT; 978 979 /* 980 * Find the right seg to adjust the data len when offset is 981 * bigger than tail room size. 982 */ 983 if (unlikely(vm2vm_mode == VM2VM_HARDWARE)) { 984 if (likely(offset <= rte_pktmbuf_tailroom(m))) 985 m->data_len += offset; 986 else { 987 struct rte_mbuf *seg = m; 988 989 while ((seg->next != NULL) && 990 (offset > rte_pktmbuf_tailroom(seg))) 991 seg = seg->next; 992 993 seg->data_len += offset; 994 } 995 m->pkt_len += offset; 996 } 997 998 m->vlan_tci = vlan_tag; 999 } 1000 1001 if (m->ol_flags & PKT_TX_TCP_SEG) 1002 virtio_tx_offload(m); 1003 1004 tx_q->m_table[len] = m; 1005 len++; 1006 if (enable_stats) { 1007 dev_statistics[dev->device_fh].tx_total++; 1008 dev_statistics[dev->device_fh].tx++; 1009 } 1010 1011 if (unlikely(len == MAX_PKT_BURST)) { 1012 m_table = (struct rte_mbuf **)tx_q->m_table; 1013 ret = rte_eth_tx_burst(ports[0], (uint16_t)tx_q->txq_id, m_table, (uint16_t) len); 1014 /* Free any buffers not handled by TX and update the port stats. */ 1015 if (unlikely(ret < len)) { 1016 do { 1017 rte_pktmbuf_free(m_table[ret]); 1018 } while (++ret < len); 1019 } 1020 1021 len = 0; 1022 } 1023 1024 tx_q->len = len; 1025 return; 1026 } 1027 /* 1028 * This function is called by each data core. It handles all RX/TX registered with the 1029 * core. For TX the specific lcore linked list is used. For RX, MAC addresses are compared 1030 * with all devices in the main linked list. 1031 */ 1032 static int 1033 switch_worker(__attribute__((unused)) void *arg) 1034 { 1035 struct virtio_net *dev = NULL; 1036 struct vhost_dev *vdev = NULL; 1037 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1038 struct mbuf_table *tx_q; 1039 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; 1040 uint64_t prev_tsc, diff_tsc, cur_tsc, ret_count = 0; 1041 unsigned ret, i; 1042 const uint16_t lcore_id = rte_lcore_id(); 1043 const uint16_t num_cores = (uint16_t)rte_lcore_count(); 1044 uint16_t rx_count = 0; 1045 uint16_t tx_count; 1046 uint32_t retry = 0; 1047 1048 RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started\n", lcore_id); 1049 prev_tsc = 0; 1050 1051 tx_q = &lcore_tx_queue[lcore_id]; 1052 for (i = 0; i < num_cores; i ++) { 1053 if (lcore_ids[i] == lcore_id) { 1054 tx_q->txq_id = i; 1055 break; 1056 } 1057 } 1058 1059 while(1) { 1060 cur_tsc = rte_rdtsc(); 1061 /* 1062 * TX burst queue drain 1063 */ 1064 diff_tsc = cur_tsc - prev_tsc; 1065 if (unlikely(diff_tsc > drain_tsc)) { 1066 1067 if (tx_q->len) { 1068 RTE_LOG(DEBUG, VHOST_DATA, 1069 "TX queue drained after timeout with burst size %u\n", 1070 tx_q->len); 1071 1072 /*Tx any packets in the queue*/ 1073 ret = rte_eth_tx_burst(ports[0], (uint16_t)tx_q->txq_id, 1074 (struct rte_mbuf **)tx_q->m_table, 1075 (uint16_t)tx_q->len); 1076 if (unlikely(ret < tx_q->len)) { 1077 do { 1078 rte_pktmbuf_free(tx_q->m_table[ret]); 1079 } while (++ret < tx_q->len); 1080 } 1081 1082 tx_q->len = 0; 1083 } 1084 1085 prev_tsc = cur_tsc; 1086 1087 } 1088 1089 /* 1090 * Inform the configuration core that we have exited the 1091 * linked list and that no devices are in use if requested. 1092 */ 1093 if (lcore_info[lcore_id].dev_removal_flag == REQUEST_DEV_REMOVAL) 1094 lcore_info[lcore_id].dev_removal_flag = ACK_DEV_REMOVAL; 1095 1096 /* 1097 * Process devices 1098 */ 1099 TAILQ_FOREACH(vdev, &lcore_info[lcore_id].vdev_list, next) { 1100 uint64_t fh; 1101 1102 dev = vdev->dev; 1103 fh = dev->device_fh; 1104 1105 if (unlikely(vdev->remove)) { 1106 unlink_vmdq(vdev); 1107 vdev->ready = DEVICE_SAFE_REMOVE; 1108 continue; 1109 } 1110 1111 if (likely(vdev->ready == DEVICE_RX)) { 1112 /*Handle guest RX*/ 1113 rx_count = rte_eth_rx_burst(ports[0], 1114 vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST); 1115 1116 if (rx_count) { 1117 /* 1118 * Retry is enabled and the queue is full then we wait and retry to avoid packet loss 1119 * Here MAX_PKT_BURST must be less than virtio queue size 1120 */ 1121 if (enable_retry && unlikely(rx_count > rte_vring_available_entries(dev, VIRTIO_RXQ))) { 1122 for (retry = 0; retry < burst_rx_retry_num; retry++) { 1123 rte_delay_us(burst_rx_delay_time); 1124 if (rx_count <= rte_vring_available_entries(dev, VIRTIO_RXQ)) 1125 break; 1126 } 1127 } 1128 ret_count = rte_vhost_enqueue_burst(dev, VIRTIO_RXQ, pkts_burst, rx_count); 1129 if (enable_stats) { 1130 rte_atomic64_add( 1131 &dev_statistics[fh].rx_total_atomic, 1132 rx_count); 1133 rte_atomic64_add( 1134 &dev_statistics[fh].rx_atomic, 1135 ret_count); 1136 } 1137 while (likely(rx_count)) { 1138 rx_count--; 1139 rte_pktmbuf_free(pkts_burst[rx_count]); 1140 } 1141 1142 } 1143 } 1144 1145 if (likely(!vdev->remove)) { 1146 /* Handle guest TX*/ 1147 tx_count = rte_vhost_dequeue_burst(dev, VIRTIO_TXQ, mbuf_pool, pkts_burst, MAX_PKT_BURST); 1148 /* If this is the first received packet we need to learn the MAC and setup VMDQ */ 1149 if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && tx_count) { 1150 if (vdev->remove || (link_vmdq(vdev, pkts_burst[0]) == -1)) { 1151 while (tx_count) 1152 rte_pktmbuf_free(pkts_burst[--tx_count]); 1153 } 1154 } 1155 for (i = 0; i < tx_count; ++i) { 1156 virtio_tx_route(vdev, pkts_burst[i], 1157 vlan_tags[(uint16_t)dev->device_fh]); 1158 } 1159 } 1160 } 1161 } 1162 1163 return 0; 1164 } 1165 1166 /* 1167 * Remove a device from the specific data core linked list and from the 1168 * main linked list. Synchonization occurs through the use of the 1169 * lcore dev_removal_flag. Device is made volatile here to avoid re-ordering 1170 * of dev->remove=1 which can cause an infinite loop in the rte_pause loop. 1171 */ 1172 static void 1173 destroy_device (volatile struct virtio_net *dev) 1174 { 1175 struct vhost_dev *vdev; 1176 int lcore; 1177 1178 dev->flags &= ~VIRTIO_DEV_RUNNING; 1179 1180 vdev = (struct vhost_dev *)dev->priv; 1181 /*set the remove flag. */ 1182 vdev->remove = 1; 1183 while(vdev->ready != DEVICE_SAFE_REMOVE) { 1184 rte_pause(); 1185 } 1186 1187 TAILQ_REMOVE(&lcore_info[vdev->coreid].vdev_list, vdev, next); 1188 TAILQ_REMOVE(&vhost_dev_list, vdev, next); 1189 1190 /* Set the dev_removal_flag on each lcore. */ 1191 RTE_LCORE_FOREACH_SLAVE(lcore) 1192 lcore_info[lcore].dev_removal_flag = REQUEST_DEV_REMOVAL; 1193 1194 /* 1195 * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL 1196 * we can be sure that they can no longer access the device removed 1197 * from the linked lists and that the devices are no longer in use. 1198 */ 1199 RTE_LCORE_FOREACH_SLAVE(lcore) { 1200 while (lcore_info[lcore].dev_removal_flag != ACK_DEV_REMOVAL) 1201 rte_pause(); 1202 } 1203 1204 lcore_info[vdev->coreid].device_num--; 1205 1206 RTE_LOG(INFO, VHOST_DATA, 1207 "(%" PRIu64 ") Device has been removed from data core\n", 1208 dev->device_fh); 1209 1210 rte_free(vdev); 1211 } 1212 1213 /* 1214 * A new device is added to a data core. First the device is added to the main linked list 1215 * and the allocated to a specific data core. 1216 */ 1217 static int 1218 new_device (struct virtio_net *dev) 1219 { 1220 int lcore, core_add = 0; 1221 uint32_t device_num_min = num_devices; 1222 struct vhost_dev *vdev; 1223 1224 vdev = rte_zmalloc("vhost device", sizeof(*vdev), RTE_CACHE_LINE_SIZE); 1225 if (vdev == NULL) { 1226 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Couldn't allocate memory for vhost dev\n", 1227 dev->device_fh); 1228 return -1; 1229 } 1230 vdev->dev = dev; 1231 dev->priv = vdev; 1232 1233 TAILQ_INSERT_TAIL(&vhost_dev_list, vdev, next); 1234 vdev->vmdq_rx_q 1235 = dev->device_fh * queues_per_pool + vmdq_queue_base; 1236 1237 /*reset ready flag*/ 1238 vdev->ready = DEVICE_MAC_LEARNING; 1239 vdev->remove = 0; 1240 1241 /* Find a suitable lcore to add the device. */ 1242 RTE_LCORE_FOREACH_SLAVE(lcore) { 1243 if (lcore_info[lcore].device_num < device_num_min) { 1244 device_num_min = lcore_info[lcore].device_num; 1245 core_add = lcore; 1246 } 1247 } 1248 vdev->coreid = core_add; 1249 1250 TAILQ_INSERT_TAIL(&lcore_info[vdev->coreid].vdev_list, vdev, next); 1251 lcore_info[vdev->coreid].device_num++; 1252 1253 /* Initialize device stats */ 1254 memset(&dev_statistics[dev->device_fh], 0, sizeof(struct device_statistics)); 1255 1256 /* Disable notifications. */ 1257 rte_vhost_enable_guest_notification(dev, VIRTIO_RXQ, 0); 1258 rte_vhost_enable_guest_notification(dev, VIRTIO_TXQ, 0); 1259 dev->flags |= VIRTIO_DEV_RUNNING; 1260 1261 RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been added to data core %d\n", dev->device_fh, vdev->coreid); 1262 1263 return 0; 1264 } 1265 1266 /* 1267 * These callback allow devices to be added to the data core when configuration 1268 * has been fully complete. 1269 */ 1270 static const struct virtio_net_device_ops virtio_net_device_ops = 1271 { 1272 .new_device = new_device, 1273 .destroy_device = destroy_device, 1274 }; 1275 1276 /* 1277 * This is a thread will wake up after a period to print stats if the user has 1278 * enabled them. 1279 */ 1280 static void 1281 print_stats(void) 1282 { 1283 struct vhost_dev *vdev; 1284 uint64_t tx_dropped, rx_dropped; 1285 uint64_t tx, tx_total, rx, rx_total; 1286 uint32_t device_fh; 1287 const char clr[] = { 27, '[', '2', 'J', '\0' }; 1288 const char top_left[] = { 27, '[', '1', ';', '1', 'H','\0' }; 1289 1290 while(1) { 1291 sleep(enable_stats); 1292 1293 /* Clear screen and move to top left */ 1294 printf("%s%s", clr, top_left); 1295 1296 printf("\nDevice statistics ===================================="); 1297 1298 TAILQ_FOREACH(vdev, &vhost_dev_list, next) { 1299 device_fh = vdev->dev->device_fh; 1300 tx_total = dev_statistics[device_fh].tx_total; 1301 tx = dev_statistics[device_fh].tx; 1302 tx_dropped = tx_total - tx; 1303 rx_total = rte_atomic64_read( 1304 &dev_statistics[device_fh].rx_total_atomic); 1305 rx = rte_atomic64_read( 1306 &dev_statistics[device_fh].rx_atomic); 1307 rx_dropped = rx_total - rx; 1308 1309 printf("\nStatistics for device %"PRIu32" ------------------------------" 1310 "\nTX total: %"PRIu64"" 1311 "\nTX dropped: %"PRIu64"" 1312 "\nTX successful: %"PRIu64"" 1313 "\nRX total: %"PRIu64"" 1314 "\nRX dropped: %"PRIu64"" 1315 "\nRX successful: %"PRIu64"", 1316 device_fh, 1317 tx_total, 1318 tx_dropped, 1319 tx, 1320 rx_total, 1321 rx_dropped, 1322 rx); 1323 } 1324 printf("\n======================================================\n"); 1325 } 1326 } 1327 1328 /* When we receive a INT signal, unregister vhost driver */ 1329 static void 1330 sigint_handler(__rte_unused int signum) 1331 { 1332 /* Unregister vhost driver. */ 1333 int ret = rte_vhost_driver_unregister((char *)&dev_basename); 1334 if (ret != 0) 1335 rte_exit(EXIT_FAILURE, "vhost driver unregister failure.\n"); 1336 exit(0); 1337 } 1338 1339 /* 1340 * Main function, does initialisation and calls the per-lcore functions. The CUSE 1341 * device is also registered here to handle the IOCTLs. 1342 */ 1343 int 1344 main(int argc, char *argv[]) 1345 { 1346 unsigned lcore_id, core_id = 0; 1347 unsigned nb_ports, valid_num_ports; 1348 int ret; 1349 uint8_t portid; 1350 static pthread_t tid; 1351 char thread_name[RTE_MAX_THREAD_NAME_LEN]; 1352 1353 signal(SIGINT, sigint_handler); 1354 1355 /* init EAL */ 1356 ret = rte_eal_init(argc, argv); 1357 if (ret < 0) 1358 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 1359 argc -= ret; 1360 argv += ret; 1361 1362 /* parse app arguments */ 1363 ret = us_vhost_parse_args(argc, argv); 1364 if (ret < 0) 1365 rte_exit(EXIT_FAILURE, "Invalid argument\n"); 1366 1367 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id ++) 1368 TAILQ_INIT(&lcore_info[lcore_id].vdev_list); 1369 1370 if (rte_lcore_is_enabled(lcore_id)) 1371 lcore_ids[core_id ++] = lcore_id; 1372 1373 if (rte_lcore_count() > RTE_MAX_LCORE) 1374 rte_exit(EXIT_FAILURE,"Not enough cores\n"); 1375 1376 /*set the number of swithcing cores available*/ 1377 num_switching_cores = rte_lcore_count()-1; 1378 1379 /* Get the number of physical ports. */ 1380 nb_ports = rte_eth_dev_count(); 1381 if (nb_ports > RTE_MAX_ETHPORTS) 1382 nb_ports = RTE_MAX_ETHPORTS; 1383 1384 /* 1385 * Update the global var NUM_PORTS and global array PORTS 1386 * and get value of var VALID_NUM_PORTS according to system ports number 1387 */ 1388 valid_num_ports = check_ports_num(nb_ports); 1389 1390 if ((valid_num_ports == 0) || (valid_num_ports > MAX_SUP_PORTS)) { 1391 RTE_LOG(INFO, VHOST_PORT, "Current enabled port number is %u," 1392 "but only %u port can be enabled\n",num_ports, MAX_SUP_PORTS); 1393 return -1; 1394 } 1395 1396 /* Create the mbuf pool. */ 1397 mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", 1398 NUM_MBUFS_PER_PORT * valid_num_ports, MBUF_CACHE_SIZE, 1399 0, MBUF_DATA_SIZE, rte_socket_id()); 1400 if (mbuf_pool == NULL) 1401 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 1402 1403 if (vm2vm_mode == VM2VM_HARDWARE) { 1404 /* Enable VT loop back to let L2 switch to do it. */ 1405 vmdq_conf_default.rx_adv_conf.vmdq_rx_conf.enable_loop_back = 1; 1406 RTE_LOG(DEBUG, VHOST_CONFIG, 1407 "Enable loop back for L2 switch in vmdq.\n"); 1408 } 1409 1410 /* initialize all ports */ 1411 for (portid = 0; portid < nb_ports; portid++) { 1412 /* skip ports that are not enabled */ 1413 if ((enabled_port_mask & (1 << portid)) == 0) { 1414 RTE_LOG(INFO, VHOST_PORT, 1415 "Skipping disabled port %d\n", portid); 1416 continue; 1417 } 1418 if (port_init(portid) != 0) 1419 rte_exit(EXIT_FAILURE, 1420 "Cannot initialize network ports\n"); 1421 } 1422 1423 /* Initialize device stats */ 1424 memset(&dev_statistics, 0, sizeof(dev_statistics)); 1425 1426 /* Enable stats if the user option is set. */ 1427 if (enable_stats) { 1428 ret = pthread_create(&tid, NULL, (void *)print_stats, NULL); 1429 if (ret != 0) 1430 rte_exit(EXIT_FAILURE, 1431 "Cannot create print-stats thread\n"); 1432 1433 /* Set thread_name for aid in debugging. */ 1434 snprintf(thread_name, RTE_MAX_THREAD_NAME_LEN, "print-stats"); 1435 ret = rte_thread_setname(tid, thread_name); 1436 if (ret != 0) 1437 RTE_LOG(ERR, VHOST_CONFIG, 1438 "Cannot set print-stats name\n"); 1439 } 1440 1441 /* Launch all data cores. */ 1442 RTE_LCORE_FOREACH_SLAVE(lcore_id) 1443 rte_eal_remote_launch(switch_worker, NULL, lcore_id); 1444 1445 if (mergeable == 0) 1446 rte_vhost_feature_disable(1ULL << VIRTIO_NET_F_MRG_RXBUF); 1447 1448 /* Register vhost(cuse or user) driver to handle vhost messages. */ 1449 ret = rte_vhost_driver_register((char *)&dev_basename); 1450 if (ret != 0) 1451 rte_exit(EXIT_FAILURE, "vhost driver register failure.\n"); 1452 1453 rte_vhost_driver_callback_register(&virtio_net_device_ops); 1454 1455 /* Start CUSE session. */ 1456 rte_vhost_driver_session_start(); 1457 return 0; 1458 1459 } 1460