1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdint.h> 35 #include <string.h> 36 #include <stdio.h> 37 #include <errno.h> 38 #include <unistd.h> 39 #ifdef RTE_EXEC_ENV_LINUXAPP 40 #include <dirent.h> 41 #include <fcntl.h> 42 #endif 43 44 #include <rte_ethdev.h> 45 #include <rte_memcpy.h> 46 #include <rte_string_fns.h> 47 #include <rte_memzone.h> 48 #include <rte_malloc.h> 49 #include <rte_atomic.h> 50 #include <rte_branch_prediction.h> 51 #include <rte_pci.h> 52 #include <rte_ether.h> 53 #include <rte_common.h> 54 #include <rte_errno.h> 55 56 #include <rte_memory.h> 57 #include <rte_eal.h> 58 #include <rte_dev.h> 59 60 #include "virtio_ethdev.h" 61 #include "virtio_pci.h" 62 #include "virtio_logs.h" 63 #include "virtqueue.h" 64 65 66 static int eth_virtio_dev_init(struct rte_eth_dev *eth_dev); 67 static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev); 68 static int virtio_dev_configure(struct rte_eth_dev *dev); 69 static int virtio_dev_start(struct rte_eth_dev *dev); 70 static void virtio_dev_stop(struct rte_eth_dev *dev); 71 static void virtio_dev_promiscuous_enable(struct rte_eth_dev *dev); 72 static void virtio_dev_promiscuous_disable(struct rte_eth_dev *dev); 73 static void virtio_dev_allmulticast_enable(struct rte_eth_dev *dev); 74 static void virtio_dev_allmulticast_disable(struct rte_eth_dev *dev); 75 static void virtio_dev_info_get(struct rte_eth_dev *dev, 76 struct rte_eth_dev_info *dev_info); 77 static int virtio_dev_link_update(struct rte_eth_dev *dev, 78 __rte_unused int wait_to_complete); 79 80 static void virtio_set_hwaddr(struct virtio_hw *hw); 81 static void virtio_get_hwaddr(struct virtio_hw *hw); 82 83 static void virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 84 static void virtio_dev_stats_reset(struct rte_eth_dev *dev); 85 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev); 86 static int virtio_vlan_filter_set(struct rte_eth_dev *dev, 87 uint16_t vlan_id, int on); 88 static void virtio_mac_addr_add(struct rte_eth_dev *dev, 89 struct ether_addr *mac_addr, 90 uint32_t index, uint32_t vmdq __rte_unused); 91 static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index); 92 static void virtio_mac_addr_set(struct rte_eth_dev *dev, 93 struct ether_addr *mac_addr); 94 95 static int virtio_dev_queue_stats_mapping_set( 96 __rte_unused struct rte_eth_dev *eth_dev, 97 __rte_unused uint16_t queue_id, 98 __rte_unused uint8_t stat_idx, 99 __rte_unused uint8_t is_rx); 100 101 /* 102 * The set of PCI devices this driver supports 103 */ 104 static const struct rte_pci_id pci_id_virtio_map[] = { 105 106 #define RTE_PCI_DEV_ID_DECL_VIRTIO(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, 107 #include "rte_pci_dev_ids.h" 108 109 { .vendor_id = 0, /* sentinel */ }, 110 }; 111 112 static int 113 virtio_send_command(struct virtqueue *vq, struct virtio_pmd_ctrl *ctrl, 114 int *dlen, int pkt_num) 115 { 116 uint32_t head, i; 117 int k, sum = 0; 118 virtio_net_ctrl_ack status = ~0; 119 struct virtio_pmd_ctrl result; 120 121 ctrl->status = status; 122 123 if (!(vq && vq->hw->cvq)) { 124 PMD_INIT_LOG(ERR, 125 "%s(): Control queue is not supported.", 126 __func__); 127 return -1; 128 } 129 head = vq->vq_desc_head_idx; 130 131 PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, " 132 "vq->hw->cvq = %p vq = %p", 133 vq->vq_desc_head_idx, status, vq->hw->cvq, vq); 134 135 if ((vq->vq_free_cnt < ((uint32_t)pkt_num + 2)) || (pkt_num < 1)) 136 return -1; 137 138 memcpy(vq->virtio_net_hdr_mz->addr, ctrl, 139 sizeof(struct virtio_pmd_ctrl)); 140 141 /* 142 * Format is enforced in qemu code: 143 * One TX packet for header; 144 * At least one TX packet per argument; 145 * One RX packet for ACK. 146 */ 147 vq->vq_ring.desc[head].flags = VRING_DESC_F_NEXT; 148 vq->vq_ring.desc[head].addr = vq->virtio_net_hdr_mz->phys_addr; 149 vq->vq_ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr); 150 vq->vq_free_cnt--; 151 i = vq->vq_ring.desc[head].next; 152 153 for (k = 0; k < pkt_num; k++) { 154 vq->vq_ring.desc[i].flags = VRING_DESC_F_NEXT; 155 vq->vq_ring.desc[i].addr = vq->virtio_net_hdr_mz->phys_addr 156 + sizeof(struct virtio_net_ctrl_hdr) 157 + sizeof(ctrl->status) + sizeof(uint8_t)*sum; 158 vq->vq_ring.desc[i].len = dlen[k]; 159 sum += dlen[k]; 160 vq->vq_free_cnt--; 161 i = vq->vq_ring.desc[i].next; 162 } 163 164 vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE; 165 vq->vq_ring.desc[i].addr = vq->virtio_net_hdr_mz->phys_addr 166 + sizeof(struct virtio_net_ctrl_hdr); 167 vq->vq_ring.desc[i].len = sizeof(ctrl->status); 168 vq->vq_free_cnt--; 169 170 vq->vq_desc_head_idx = vq->vq_ring.desc[i].next; 171 172 vq_update_avail_ring(vq, head); 173 vq_update_avail_idx(vq); 174 175 PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index); 176 177 virtqueue_notify(vq); 178 179 rte_rmb(); 180 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) { 181 rte_rmb(); 182 usleep(100); 183 } 184 185 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) { 186 uint32_t idx, desc_idx, used_idx; 187 struct vring_used_elem *uep; 188 189 used_idx = (uint32_t)(vq->vq_used_cons_idx 190 & (vq->vq_nentries - 1)); 191 uep = &vq->vq_ring.used->ring[used_idx]; 192 idx = (uint32_t) uep->id; 193 desc_idx = idx; 194 195 while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) { 196 desc_idx = vq->vq_ring.desc[desc_idx].next; 197 vq->vq_free_cnt++; 198 } 199 200 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx; 201 vq->vq_desc_head_idx = idx; 202 203 vq->vq_used_cons_idx++; 204 vq->vq_free_cnt++; 205 } 206 207 PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d", 208 vq->vq_free_cnt, vq->vq_desc_head_idx); 209 210 memcpy(&result, vq->virtio_net_hdr_mz->addr, 211 sizeof(struct virtio_pmd_ctrl)); 212 213 return result.status; 214 } 215 216 static int 217 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues) 218 { 219 struct virtio_hw *hw = dev->data->dev_private; 220 struct virtio_pmd_ctrl ctrl; 221 int dlen[1]; 222 int ret; 223 224 ctrl.hdr.class = VIRTIO_NET_CTRL_MQ; 225 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; 226 memcpy(ctrl.data, &nb_queues, sizeof(uint16_t)); 227 228 dlen[0] = sizeof(uint16_t); 229 230 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 231 if (ret) { 232 PMD_INIT_LOG(ERR, "Multiqueue configured but send command " 233 "failed, this is too late now..."); 234 return -EINVAL; 235 } 236 237 return 0; 238 } 239 240 void 241 virtio_dev_queue_release(struct virtqueue *vq) { 242 struct virtio_hw *hw; 243 244 if (vq) { 245 hw = vq->hw; 246 /* Select and deactivate the queue */ 247 VIRTIO_WRITE_REG_2(hw, VIRTIO_PCI_QUEUE_SEL, vq->queue_id); 248 VIRTIO_WRITE_REG_4(hw, VIRTIO_PCI_QUEUE_PFN, 0); 249 250 rte_free(vq); 251 vq = NULL; 252 } 253 } 254 255 int virtio_dev_queue_setup(struct rte_eth_dev *dev, 256 int queue_type, 257 uint16_t queue_idx, 258 uint16_t vtpci_queue_idx, 259 uint16_t nb_desc, 260 unsigned int socket_id, 261 struct virtqueue **pvq) 262 { 263 char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 264 const struct rte_memzone *mz; 265 unsigned int vq_size, size; 266 struct virtio_hw *hw = dev->data->dev_private; 267 struct virtqueue *vq = NULL; 268 269 /* Write the virtqueue index to the Queue Select Field */ 270 VIRTIO_WRITE_REG_2(hw, VIRTIO_PCI_QUEUE_SEL, vtpci_queue_idx); 271 PMD_INIT_LOG(DEBUG, "selecting queue: %u", vtpci_queue_idx); 272 273 /* 274 * Read the virtqueue size from the Queue Size field 275 * Always power of 2 and if 0 virtqueue does not exist 276 */ 277 vq_size = VIRTIO_READ_REG_2(hw, VIRTIO_PCI_QUEUE_NUM); 278 PMD_INIT_LOG(DEBUG, "vq_size: %u nb_desc:%u", vq_size, nb_desc); 279 if (vq_size == 0) { 280 PMD_INIT_LOG(ERR, "%s: virtqueue does not exist", __func__); 281 return -EINVAL; 282 } 283 284 if (!rte_is_power_of_2(vq_size)) { 285 PMD_INIT_LOG(ERR, "%s: virtqueue size is not powerof 2", __func__); 286 return -EINVAL; 287 } 288 289 if (queue_type == VTNET_RQ) { 290 snprintf(vq_name, sizeof(vq_name), "port%d_rvq%d", 291 dev->data->port_id, queue_idx); 292 vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) + 293 vq_size * sizeof(struct vq_desc_extra), RTE_CACHE_LINE_SIZE); 294 } else if (queue_type == VTNET_TQ) { 295 snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d", 296 dev->data->port_id, queue_idx); 297 vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) + 298 vq_size * sizeof(struct vq_desc_extra), RTE_CACHE_LINE_SIZE); 299 } else if (queue_type == VTNET_CQ) { 300 snprintf(vq_name, sizeof(vq_name), "port%d_cvq", 301 dev->data->port_id); 302 vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) + 303 vq_size * sizeof(struct vq_desc_extra), 304 RTE_CACHE_LINE_SIZE); 305 } 306 if (vq == NULL) { 307 PMD_INIT_LOG(ERR, "%s: Can not allocate virtqueue", __func__); 308 return (-ENOMEM); 309 } 310 311 vq->hw = hw; 312 vq->port_id = dev->data->port_id; 313 vq->queue_id = queue_idx; 314 vq->vq_queue_index = vtpci_queue_idx; 315 vq->vq_nentries = vq_size; 316 317 if (nb_desc == 0 || nb_desc > vq_size) 318 nb_desc = vq_size; 319 vq->vq_free_cnt = nb_desc; 320 321 /* 322 * Reserve a memzone for vring elements 323 */ 324 size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN); 325 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN); 326 PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d", size, vq->vq_ring_size); 327 328 mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, 329 socket_id, 0, VIRTIO_PCI_VRING_ALIGN); 330 if (mz == NULL) { 331 if (rte_errno == EEXIST) 332 mz = rte_memzone_lookup(vq_name); 333 if (mz == NULL) { 334 rte_free(vq); 335 return -ENOMEM; 336 } 337 } 338 339 /* 340 * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, 341 * and only accepts 32 bit page frame number. 342 * Check if the allocated physical memory exceeds 16TB. 343 */ 344 if ((mz->phys_addr + vq->vq_ring_size - 1) >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { 345 PMD_INIT_LOG(ERR, "vring address shouldn't be above 16TB!"); 346 rte_free(vq); 347 return -ENOMEM; 348 } 349 350 memset(mz->addr, 0, sizeof(mz->len)); 351 vq->mz = mz; 352 vq->vq_ring_mem = mz->phys_addr; 353 vq->vq_ring_virt_mem = mz->addr; 354 PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%"PRIx64, (uint64_t)mz->phys_addr); 355 PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: 0x%"PRIx64, (uint64_t)(uintptr_t)mz->addr); 356 vq->virtio_net_hdr_mz = NULL; 357 vq->virtio_net_hdr_mem = 0; 358 359 if (queue_type == VTNET_TQ) { 360 /* 361 * For each xmit packet, allocate a virtio_net_hdr 362 */ 363 snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d_hdrzone", 364 dev->data->port_id, queue_idx); 365 vq->virtio_net_hdr_mz = rte_memzone_reserve_aligned(vq_name, 366 vq_size * hw->vtnet_hdr_size, 367 socket_id, 0, RTE_CACHE_LINE_SIZE); 368 if (vq->virtio_net_hdr_mz == NULL) { 369 if (rte_errno == EEXIST) 370 vq->virtio_net_hdr_mz = 371 rte_memzone_lookup(vq_name); 372 if (vq->virtio_net_hdr_mz == NULL) { 373 rte_free(vq); 374 return -ENOMEM; 375 } 376 } 377 vq->virtio_net_hdr_mem = 378 vq->virtio_net_hdr_mz->phys_addr; 379 memset(vq->virtio_net_hdr_mz->addr, 0, 380 vq_size * hw->vtnet_hdr_size); 381 } else if (queue_type == VTNET_CQ) { 382 /* Allocate a page for control vq command, data and status */ 383 snprintf(vq_name, sizeof(vq_name), "port%d_cvq_hdrzone", 384 dev->data->port_id); 385 vq->virtio_net_hdr_mz = rte_memzone_reserve_aligned(vq_name, 386 PAGE_SIZE, socket_id, 0, RTE_CACHE_LINE_SIZE); 387 if (vq->virtio_net_hdr_mz == NULL) { 388 if (rte_errno == EEXIST) 389 vq->virtio_net_hdr_mz = 390 rte_memzone_lookup(vq_name); 391 if (vq->virtio_net_hdr_mz == NULL) { 392 rte_free(vq); 393 return -ENOMEM; 394 } 395 } 396 vq->virtio_net_hdr_mem = 397 vq->virtio_net_hdr_mz->phys_addr; 398 memset(vq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE); 399 } 400 401 /* 402 * Set guest physical address of the virtqueue 403 * in VIRTIO_PCI_QUEUE_PFN config register of device 404 */ 405 VIRTIO_WRITE_REG_4(hw, VIRTIO_PCI_QUEUE_PFN, 406 mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 407 *pvq = vq; 408 return 0; 409 } 410 411 static int 412 virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx, 413 uint32_t socket_id) 414 { 415 struct virtqueue *vq; 416 int ret; 417 struct virtio_hw *hw = dev->data->dev_private; 418 419 PMD_INIT_FUNC_TRACE(); 420 ret = virtio_dev_queue_setup(dev, VTNET_CQ, VTNET_SQ_CQ_QUEUE_IDX, 421 vtpci_queue_idx, 0, socket_id, &vq); 422 if (ret < 0) { 423 PMD_INIT_LOG(ERR, "control vq initialization failed"); 424 return ret; 425 } 426 427 hw->cvq = vq; 428 return 0; 429 } 430 431 static void 432 virtio_free_queues(struct rte_eth_dev *dev) 433 { 434 unsigned int i; 435 436 for (i = 0; i < dev->data->nb_rx_queues; i++) 437 virtio_dev_rx_queue_release(dev->data->rx_queues[i]); 438 439 dev->data->nb_rx_queues = 0; 440 441 for (i = 0; i < dev->data->nb_tx_queues; i++) 442 virtio_dev_tx_queue_release(dev->data->tx_queues[i]); 443 444 dev->data->nb_tx_queues = 0; 445 } 446 447 static void 448 virtio_dev_close(struct rte_eth_dev *dev) 449 { 450 struct virtio_hw *hw = dev->data->dev_private; 451 struct rte_pci_device *pci_dev = dev->pci_dev; 452 453 PMD_INIT_LOG(DEBUG, "virtio_dev_close"); 454 455 /* reset the NIC */ 456 if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) 457 vtpci_irq_config(hw, VIRTIO_MSI_NO_VECTOR); 458 vtpci_reset(hw); 459 hw->started = 0; 460 virtio_dev_free_mbufs(dev); 461 virtio_free_queues(dev); 462 } 463 464 static void 465 virtio_dev_promiscuous_enable(struct rte_eth_dev *dev) 466 { 467 struct virtio_hw *hw = dev->data->dev_private; 468 struct virtio_pmd_ctrl ctrl; 469 int dlen[1]; 470 int ret; 471 472 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 473 PMD_INIT_LOG(INFO, "host does not support rx control\n"); 474 return; 475 } 476 477 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 478 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC; 479 ctrl.data[0] = 1; 480 dlen[0] = 1; 481 482 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 483 if (ret) 484 PMD_INIT_LOG(ERR, "Failed to enable promisc"); 485 } 486 487 static void 488 virtio_dev_promiscuous_disable(struct rte_eth_dev *dev) 489 { 490 struct virtio_hw *hw = dev->data->dev_private; 491 struct virtio_pmd_ctrl ctrl; 492 int dlen[1]; 493 int ret; 494 495 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 496 PMD_INIT_LOG(INFO, "host does not support rx control\n"); 497 return; 498 } 499 500 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 501 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC; 502 ctrl.data[0] = 0; 503 dlen[0] = 1; 504 505 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 506 if (ret) 507 PMD_INIT_LOG(ERR, "Failed to disable promisc"); 508 } 509 510 static void 511 virtio_dev_allmulticast_enable(struct rte_eth_dev *dev) 512 { 513 struct virtio_hw *hw = dev->data->dev_private; 514 struct virtio_pmd_ctrl ctrl; 515 int dlen[1]; 516 int ret; 517 518 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 519 PMD_INIT_LOG(INFO, "host does not support rx control\n"); 520 return; 521 } 522 523 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 524 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI; 525 ctrl.data[0] = 1; 526 dlen[0] = 1; 527 528 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 529 if (ret) 530 PMD_INIT_LOG(ERR, "Failed to enable allmulticast"); 531 } 532 533 static void 534 virtio_dev_allmulticast_disable(struct rte_eth_dev *dev) 535 { 536 struct virtio_hw *hw = dev->data->dev_private; 537 struct virtio_pmd_ctrl ctrl; 538 int dlen[1]; 539 int ret; 540 541 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 542 PMD_INIT_LOG(INFO, "host does not support rx control\n"); 543 return; 544 } 545 546 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 547 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI; 548 ctrl.data[0] = 0; 549 dlen[0] = 1; 550 551 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 552 if (ret) 553 PMD_INIT_LOG(ERR, "Failed to disable allmulticast"); 554 } 555 556 /* 557 * dev_ops for virtio, bare necessities for basic operation 558 */ 559 static const struct eth_dev_ops virtio_eth_dev_ops = { 560 .dev_configure = virtio_dev_configure, 561 .dev_start = virtio_dev_start, 562 .dev_stop = virtio_dev_stop, 563 .dev_close = virtio_dev_close, 564 .promiscuous_enable = virtio_dev_promiscuous_enable, 565 .promiscuous_disable = virtio_dev_promiscuous_disable, 566 .allmulticast_enable = virtio_dev_allmulticast_enable, 567 .allmulticast_disable = virtio_dev_allmulticast_disable, 568 569 .dev_infos_get = virtio_dev_info_get, 570 .stats_get = virtio_dev_stats_get, 571 .stats_reset = virtio_dev_stats_reset, 572 .link_update = virtio_dev_link_update, 573 .rx_queue_setup = virtio_dev_rx_queue_setup, 574 .rx_queue_release = virtio_dev_rx_queue_release, 575 .tx_queue_setup = virtio_dev_tx_queue_setup, 576 .tx_queue_release = virtio_dev_tx_queue_release, 577 /* collect stats per queue */ 578 .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set, 579 .vlan_filter_set = virtio_vlan_filter_set, 580 .mac_addr_add = virtio_mac_addr_add, 581 .mac_addr_remove = virtio_mac_addr_remove, 582 .mac_addr_set = virtio_mac_addr_set, 583 }; 584 585 static inline int 586 virtio_dev_atomic_read_link_status(struct rte_eth_dev *dev, 587 struct rte_eth_link *link) 588 { 589 struct rte_eth_link *dst = link; 590 struct rte_eth_link *src = &(dev->data->dev_link); 591 592 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 593 *(uint64_t *)src) == 0) 594 return -1; 595 596 return 0; 597 } 598 599 /** 600 * Atomically writes the link status information into global 601 * structure rte_eth_dev. 602 * 603 * @param dev 604 * - Pointer to the structure rte_eth_dev to read from. 605 * - Pointer to the buffer to be saved with the link status. 606 * 607 * @return 608 * - On success, zero. 609 * - On failure, negative value. 610 */ 611 static inline int 612 virtio_dev_atomic_write_link_status(struct rte_eth_dev *dev, 613 struct rte_eth_link *link) 614 { 615 struct rte_eth_link *dst = &(dev->data->dev_link); 616 struct rte_eth_link *src = link; 617 618 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 619 *(uint64_t *)src) == 0) 620 return -1; 621 622 return 0; 623 } 624 625 static void 626 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 627 { 628 unsigned i; 629 630 for (i = 0; i < dev->data->nb_tx_queues; i++) { 631 const struct virtqueue *txvq = dev->data->tx_queues[i]; 632 if (txvq == NULL) 633 continue; 634 635 stats->opackets += txvq->packets; 636 stats->obytes += txvq->bytes; 637 stats->oerrors += txvq->errors; 638 639 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 640 stats->q_opackets[i] = txvq->packets; 641 stats->q_obytes[i] = txvq->bytes; 642 } 643 } 644 645 for (i = 0; i < dev->data->nb_rx_queues; i++) { 646 const struct virtqueue *rxvq = dev->data->rx_queues[i]; 647 if (rxvq == NULL) 648 continue; 649 650 stats->ipackets += rxvq->packets; 651 stats->ibytes += rxvq->bytes; 652 stats->ierrors += rxvq->errors; 653 654 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 655 stats->q_ipackets[i] = rxvq->packets; 656 stats->q_ibytes[i] = rxvq->bytes; 657 } 658 } 659 660 stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed; 661 } 662 663 static void 664 virtio_dev_stats_reset(struct rte_eth_dev *dev) 665 { 666 unsigned int i; 667 668 for (i = 0; i < dev->data->nb_tx_queues; i++) { 669 struct virtqueue *txvq = dev->data->tx_queues[i]; 670 if (txvq == NULL) 671 continue; 672 673 txvq->packets = 0; 674 txvq->bytes = 0; 675 txvq->errors = 0; 676 } 677 678 for (i = 0; i < dev->data->nb_rx_queues; i++) { 679 struct virtqueue *rxvq = dev->data->rx_queues[i]; 680 if (rxvq == NULL) 681 continue; 682 683 rxvq->packets = 0; 684 rxvq->bytes = 0; 685 rxvq->errors = 0; 686 } 687 688 dev->data->rx_mbuf_alloc_failed = 0; 689 } 690 691 static void 692 virtio_set_hwaddr(struct virtio_hw *hw) 693 { 694 vtpci_write_dev_config(hw, 695 offsetof(struct virtio_net_config, mac), 696 &hw->mac_addr, ETHER_ADDR_LEN); 697 } 698 699 static void 700 virtio_get_hwaddr(struct virtio_hw *hw) 701 { 702 if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) { 703 vtpci_read_dev_config(hw, 704 offsetof(struct virtio_net_config, mac), 705 &hw->mac_addr, ETHER_ADDR_LEN); 706 } else { 707 eth_random_addr(&hw->mac_addr[0]); 708 virtio_set_hwaddr(hw); 709 } 710 } 711 712 static void 713 virtio_mac_table_set(struct virtio_hw *hw, 714 const struct virtio_net_ctrl_mac *uc, 715 const struct virtio_net_ctrl_mac *mc) 716 { 717 struct virtio_pmd_ctrl ctrl; 718 int err, len[2]; 719 720 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 721 PMD_DRV_LOG(INFO, "host does not support mac table\n"); 722 return; 723 } 724 725 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC; 726 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; 727 728 len[0] = uc->entries * ETHER_ADDR_LEN + sizeof(uc->entries); 729 memcpy(ctrl.data, uc, len[0]); 730 731 len[1] = mc->entries * ETHER_ADDR_LEN + sizeof(mc->entries); 732 memcpy(ctrl.data + len[0], mc, len[1]); 733 734 err = virtio_send_command(hw->cvq, &ctrl, len, 2); 735 if (err != 0) 736 PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err); 737 } 738 739 static void 740 virtio_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 741 uint32_t index, uint32_t vmdq __rte_unused) 742 { 743 struct virtio_hw *hw = dev->data->dev_private; 744 const struct ether_addr *addrs = dev->data->mac_addrs; 745 unsigned int i; 746 struct virtio_net_ctrl_mac *uc, *mc; 747 748 if (index >= VIRTIO_MAX_MAC_ADDRS) { 749 PMD_DRV_LOG(ERR, "mac address index %u out of range", index); 750 return; 751 } 752 753 uc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(uc->entries)); 754 uc->entries = 0; 755 mc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(mc->entries)); 756 mc->entries = 0; 757 758 for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) { 759 const struct ether_addr *addr 760 = (i == index) ? mac_addr : addrs + i; 761 struct virtio_net_ctrl_mac *tbl 762 = is_multicast_ether_addr(addr) ? mc : uc; 763 764 memcpy(&tbl->macs[tbl->entries++], addr, ETHER_ADDR_LEN); 765 } 766 767 virtio_mac_table_set(hw, uc, mc); 768 } 769 770 static void 771 virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) 772 { 773 struct virtio_hw *hw = dev->data->dev_private; 774 struct ether_addr *addrs = dev->data->mac_addrs; 775 struct virtio_net_ctrl_mac *uc, *mc; 776 unsigned int i; 777 778 if (index >= VIRTIO_MAX_MAC_ADDRS) { 779 PMD_DRV_LOG(ERR, "mac address index %u out of range", index); 780 return; 781 } 782 783 uc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(uc->entries)); 784 uc->entries = 0; 785 mc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(mc->entries)); 786 mc->entries = 0; 787 788 for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) { 789 struct virtio_net_ctrl_mac *tbl; 790 791 if (i == index || is_zero_ether_addr(addrs + i)) 792 continue; 793 794 tbl = is_multicast_ether_addr(addrs + i) ? mc : uc; 795 memcpy(&tbl->macs[tbl->entries++], addrs + i, ETHER_ADDR_LEN); 796 } 797 798 virtio_mac_table_set(hw, uc, mc); 799 } 800 801 static void 802 virtio_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) 803 { 804 struct virtio_hw *hw = dev->data->dev_private; 805 806 memcpy(hw->mac_addr, mac_addr, ETHER_ADDR_LEN); 807 808 /* Use atomic update if available */ 809 if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 810 struct virtio_pmd_ctrl ctrl; 811 int len = ETHER_ADDR_LEN; 812 813 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC; 814 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET; 815 816 memcpy(ctrl.data, mac_addr, ETHER_ADDR_LEN); 817 virtio_send_command(hw->cvq, &ctrl, &len, 1); 818 } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) 819 virtio_set_hwaddr(hw); 820 } 821 822 static int 823 virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 824 { 825 struct virtio_hw *hw = dev->data->dev_private; 826 struct virtio_pmd_ctrl ctrl; 827 int len; 828 829 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) 830 return -ENOTSUP; 831 832 ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN; 833 ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL; 834 memcpy(ctrl.data, &vlan_id, sizeof(vlan_id)); 835 len = sizeof(vlan_id); 836 837 return virtio_send_command(hw->cvq, &ctrl, &len, 1); 838 } 839 840 static void 841 virtio_negotiate_features(struct virtio_hw *hw) 842 { 843 uint32_t host_features; 844 845 /* Prepare guest_features: feature that driver wants to support */ 846 hw->guest_features = VIRTIO_PMD_GUEST_FEATURES; 847 PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %x", 848 hw->guest_features); 849 850 /* Read device(host) feature bits */ 851 host_features = VIRTIO_READ_REG_4(hw, VIRTIO_PCI_HOST_FEATURES); 852 PMD_INIT_LOG(DEBUG, "host_features before negotiate = %x", 853 host_features); 854 855 /* 856 * Negotiate features: Subset of device feature bits are written back 857 * guest feature bits. 858 */ 859 hw->guest_features = vtpci_negotiate_features(hw, host_features); 860 PMD_INIT_LOG(DEBUG, "features after negotiate = %x", 861 hw->guest_features); 862 } 863 864 #ifdef RTE_EXEC_ENV_LINUXAPP 865 static int 866 parse_sysfs_value(const char *filename, unsigned long *val) 867 { 868 FILE *f; 869 char buf[BUFSIZ]; 870 char *end = NULL; 871 872 f = fopen(filename, "r"); 873 if (f == NULL) { 874 PMD_INIT_LOG(ERR, "%s(): cannot open sysfs value %s", 875 __func__, filename); 876 return -1; 877 } 878 879 if (fgets(buf, sizeof(buf), f) == NULL) { 880 PMD_INIT_LOG(ERR, "%s(): cannot read sysfs value %s", 881 __func__, filename); 882 fclose(f); 883 return -1; 884 } 885 *val = strtoul(buf, &end, 0); 886 if ((buf[0] == '\0') || (end == NULL) || (*end != '\n')) { 887 PMD_INIT_LOG(ERR, "%s(): cannot parse sysfs value %s", 888 __func__, filename); 889 fclose(f); 890 return -1; 891 } 892 fclose(f); 893 return 0; 894 } 895 896 static int get_uio_dev(struct rte_pci_addr *loc, char *buf, unsigned int buflen, 897 unsigned int *uio_num) 898 { 899 struct dirent *e; 900 DIR *dir; 901 char dirname[PATH_MAX]; 902 903 /* depending on kernel version, uio can be located in uio/uioX 904 * or uio:uioX */ 905 snprintf(dirname, sizeof(dirname), 906 SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/uio", 907 loc->domain, loc->bus, loc->devid, loc->function); 908 dir = opendir(dirname); 909 if (dir == NULL) { 910 /* retry with the parent directory */ 911 snprintf(dirname, sizeof(dirname), 912 SYSFS_PCI_DEVICES "/" PCI_PRI_FMT, 913 loc->domain, loc->bus, loc->devid, loc->function); 914 dir = opendir(dirname); 915 916 if (dir == NULL) { 917 PMD_INIT_LOG(ERR, "Cannot opendir %s", dirname); 918 return -1; 919 } 920 } 921 922 /* take the first file starting with "uio" */ 923 while ((e = readdir(dir)) != NULL) { 924 /* format could be uio%d ...*/ 925 int shortprefix_len = sizeof("uio") - 1; 926 /* ... or uio:uio%d */ 927 int longprefix_len = sizeof("uio:uio") - 1; 928 char *endptr; 929 930 if (strncmp(e->d_name, "uio", 3) != 0) 931 continue; 932 933 /* first try uio%d */ 934 errno = 0; 935 *uio_num = strtoull(e->d_name + shortprefix_len, &endptr, 10); 936 if (errno == 0 && endptr != (e->d_name + shortprefix_len)) { 937 snprintf(buf, buflen, "%s/uio%u", dirname, *uio_num); 938 break; 939 } 940 941 /* then try uio:uio%d */ 942 errno = 0; 943 *uio_num = strtoull(e->d_name + longprefix_len, &endptr, 10); 944 if (errno == 0 && endptr != (e->d_name + longprefix_len)) { 945 snprintf(buf, buflen, "%s/uio:uio%u", dirname, 946 *uio_num); 947 break; 948 } 949 } 950 closedir(dir); 951 952 /* No uio resource found */ 953 if (e == NULL) { 954 PMD_INIT_LOG(ERR, "Could not find uio resource"); 955 return -1; 956 } 957 958 return 0; 959 } 960 961 static int 962 virtio_has_msix(const struct rte_pci_addr *loc) 963 { 964 DIR *d; 965 char dirname[PATH_MAX]; 966 967 snprintf(dirname, sizeof(dirname), 968 SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/msi_irqs", 969 loc->domain, loc->bus, loc->devid, loc->function); 970 971 d = opendir(dirname); 972 if (d) 973 closedir(d); 974 975 return (d != NULL); 976 } 977 978 /* Extract I/O port numbers from sysfs */ 979 static int virtio_resource_init_by_uio(struct rte_pci_device *pci_dev) 980 { 981 char dirname[PATH_MAX]; 982 char filename[PATH_MAX]; 983 unsigned long start, size; 984 unsigned int uio_num; 985 986 if (get_uio_dev(&pci_dev->addr, dirname, sizeof(dirname), &uio_num) < 0) 987 return -1; 988 989 /* get portio size */ 990 snprintf(filename, sizeof(filename), 991 "%s/portio/port0/size", dirname); 992 if (parse_sysfs_value(filename, &size) < 0) { 993 PMD_INIT_LOG(ERR, "%s(): cannot parse size", 994 __func__); 995 return -1; 996 } 997 998 /* get portio start */ 999 snprintf(filename, sizeof(filename), 1000 "%s/portio/port0/start", dirname); 1001 if (parse_sysfs_value(filename, &start) < 0) { 1002 PMD_INIT_LOG(ERR, "%s(): cannot parse portio start", 1003 __func__); 1004 return -1; 1005 } 1006 pci_dev->mem_resource[0].addr = (void *)(uintptr_t)start; 1007 pci_dev->mem_resource[0].len = (uint64_t)size; 1008 PMD_INIT_LOG(DEBUG, 1009 "PCI Port IO found start=0x%lx with size=0x%lx", 1010 start, size); 1011 1012 /* save fd */ 1013 memset(dirname, 0, sizeof(dirname)); 1014 snprintf(dirname, sizeof(dirname), "/dev/uio%u", uio_num); 1015 pci_dev->intr_handle.fd = open(dirname, O_RDWR); 1016 if (pci_dev->intr_handle.fd < 0) { 1017 PMD_INIT_LOG(ERR, "Cannot open %s: %s\n", 1018 dirname, strerror(errno)); 1019 return -1; 1020 } 1021 1022 pci_dev->intr_handle.type = RTE_INTR_HANDLE_UIO; 1023 pci_dev->driver->drv_flags |= RTE_PCI_DRV_INTR_LSC; 1024 1025 return 0; 1026 } 1027 1028 /* Extract port I/O numbers from proc/ioports */ 1029 static int virtio_resource_init_by_ioports(struct rte_pci_device *pci_dev) 1030 { 1031 uint16_t start, end; 1032 int size; 1033 FILE *fp; 1034 char *line = NULL; 1035 char pci_id[16]; 1036 int found = 0; 1037 size_t linesz; 1038 1039 snprintf(pci_id, sizeof(pci_id), PCI_PRI_FMT, 1040 pci_dev->addr.domain, 1041 pci_dev->addr.bus, 1042 pci_dev->addr.devid, 1043 pci_dev->addr.function); 1044 1045 fp = fopen("/proc/ioports", "r"); 1046 if (fp == NULL) { 1047 PMD_INIT_LOG(ERR, "%s(): can't open ioports", __func__); 1048 return -1; 1049 } 1050 1051 while (getdelim(&line, &linesz, '\n', fp) > 0) { 1052 char *ptr = line; 1053 char *left; 1054 int n; 1055 1056 n = strcspn(ptr, ":"); 1057 ptr[n] = 0; 1058 left = &ptr[n+1]; 1059 1060 while (*left && isspace(*left)) 1061 left++; 1062 1063 if (!strncmp(left, pci_id, strlen(pci_id))) { 1064 found = 1; 1065 1066 while (*ptr && isspace(*ptr)) 1067 ptr++; 1068 1069 sscanf(ptr, "%04hx-%04hx", &start, &end); 1070 size = end - start + 1; 1071 1072 break; 1073 } 1074 } 1075 1076 free(line); 1077 fclose(fp); 1078 1079 if (!found) 1080 return -1; 1081 1082 pci_dev->mem_resource[0].addr = (void *)(uintptr_t)(uint32_t)start; 1083 pci_dev->mem_resource[0].len = (uint64_t)size; 1084 PMD_INIT_LOG(DEBUG, 1085 "PCI Port IO found start=0x%x with size=0x%x", 1086 start, size); 1087 1088 /* can't support lsc interrupt without uio */ 1089 pci_dev->driver->drv_flags &= ~RTE_PCI_DRV_INTR_LSC; 1090 1091 return 0; 1092 } 1093 1094 /* Extract I/O port numbers from sysfs */ 1095 static int virtio_resource_init(struct rte_pci_device *pci_dev) 1096 { 1097 if (virtio_resource_init_by_uio(pci_dev) == 0) 1098 return 0; 1099 else 1100 return virtio_resource_init_by_ioports(pci_dev); 1101 } 1102 1103 #else 1104 static int 1105 virtio_has_msix(const struct rte_pci_addr *loc __rte_unused) 1106 { 1107 /* nic_uio does not enable interrupts, return 0 (false). */ 1108 return 0; 1109 } 1110 1111 static int virtio_resource_init(struct rte_pci_device *pci_dev __rte_unused) 1112 { 1113 /* no setup required */ 1114 return 0; 1115 } 1116 #endif 1117 1118 /* 1119 * Process Virtio Config changed interrupt and call the callback 1120 * if link state changed. 1121 */ 1122 static void 1123 virtio_interrupt_handler(__rte_unused struct rte_intr_handle *handle, 1124 void *param) 1125 { 1126 struct rte_eth_dev *dev = param; 1127 struct virtio_hw *hw = dev->data->dev_private; 1128 uint8_t isr; 1129 1130 /* Read interrupt status which clears interrupt */ 1131 isr = vtpci_isr(hw); 1132 PMD_DRV_LOG(INFO, "interrupt status = %#x", isr); 1133 1134 if (rte_intr_enable(&dev->pci_dev->intr_handle) < 0) 1135 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1136 1137 if (isr & VIRTIO_PCI_ISR_CONFIG) { 1138 if (virtio_dev_link_update(dev, 0) == 0) 1139 _rte_eth_dev_callback_process(dev, 1140 RTE_ETH_EVENT_INTR_LSC); 1141 } 1142 1143 } 1144 1145 static void 1146 rx_func_get(struct rte_eth_dev *eth_dev) 1147 { 1148 struct virtio_hw *hw = eth_dev->data->dev_private; 1149 if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) 1150 eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts; 1151 else 1152 eth_dev->rx_pkt_burst = &virtio_recv_pkts; 1153 } 1154 1155 /* 1156 * This function is based on probe() function in virtio_pci.c 1157 * It returns 0 on success. 1158 */ 1159 static int 1160 eth_virtio_dev_init(struct rte_eth_dev *eth_dev) 1161 { 1162 struct virtio_hw *hw = eth_dev->data->dev_private; 1163 struct virtio_net_config *config; 1164 struct virtio_net_config local_config; 1165 uint32_t offset_conf = sizeof(config->mac); 1166 struct rte_pci_device *pci_dev; 1167 1168 RTE_BUILD_BUG_ON(RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr)); 1169 1170 eth_dev->dev_ops = &virtio_eth_dev_ops; 1171 eth_dev->tx_pkt_burst = &virtio_xmit_pkts; 1172 1173 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1174 rx_func_get(eth_dev); 1175 return 0; 1176 } 1177 1178 /* Allocate memory for storing MAC addresses */ 1179 eth_dev->data->mac_addrs = rte_zmalloc("virtio", ETHER_ADDR_LEN, 0); 1180 if (eth_dev->data->mac_addrs == NULL) { 1181 PMD_INIT_LOG(ERR, 1182 "Failed to allocate %d bytes needed to store MAC addresses", 1183 ETHER_ADDR_LEN); 1184 return -ENOMEM; 1185 } 1186 1187 pci_dev = eth_dev->pci_dev; 1188 if (virtio_resource_init(pci_dev) < 0) 1189 return -1; 1190 1191 hw->use_msix = virtio_has_msix(&pci_dev->addr); 1192 hw->io_base = (uint32_t)(uintptr_t)pci_dev->mem_resource[0].addr; 1193 1194 /* Reset the device although not necessary at startup */ 1195 vtpci_reset(hw); 1196 1197 /* Tell the host we've noticed this device. */ 1198 vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 1199 1200 /* Tell the host we've known how to drive the device. */ 1201 vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 1202 virtio_negotiate_features(hw); 1203 1204 /* If host does not support status then disable LSC */ 1205 if (!vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) 1206 pci_dev->driver->drv_flags &= ~RTE_PCI_DRV_INTR_LSC; 1207 1208 rx_func_get(eth_dev); 1209 1210 /* Setting up rx_header size for the device */ 1211 if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) 1212 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); 1213 else 1214 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr); 1215 1216 /* Copy the permanent MAC address to: virtio_hw */ 1217 virtio_get_hwaddr(hw); 1218 ether_addr_copy((struct ether_addr *) hw->mac_addr, 1219 ð_dev->data->mac_addrs[0]); 1220 PMD_INIT_LOG(DEBUG, 1221 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X", 1222 hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2], 1223 hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]); 1224 1225 if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) { 1226 config = &local_config; 1227 1228 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { 1229 offset_conf += sizeof(config->status); 1230 } else { 1231 PMD_INIT_LOG(DEBUG, 1232 "VIRTIO_NET_F_STATUS is not supported"); 1233 config->status = 0; 1234 } 1235 1236 if (vtpci_with_feature(hw, VIRTIO_NET_F_MQ)) { 1237 offset_conf += sizeof(config->max_virtqueue_pairs); 1238 } else { 1239 PMD_INIT_LOG(DEBUG, 1240 "VIRTIO_NET_F_MQ is not supported"); 1241 config->max_virtqueue_pairs = 1; 1242 } 1243 1244 vtpci_read_dev_config(hw, 0, (uint8_t *)config, offset_conf); 1245 1246 hw->max_rx_queues = 1247 (VIRTIO_MAX_RX_QUEUES < config->max_virtqueue_pairs) ? 1248 VIRTIO_MAX_RX_QUEUES : config->max_virtqueue_pairs; 1249 hw->max_tx_queues = 1250 (VIRTIO_MAX_TX_QUEUES < config->max_virtqueue_pairs) ? 1251 VIRTIO_MAX_TX_QUEUES : config->max_virtqueue_pairs; 1252 1253 virtio_dev_cq_queue_setup(eth_dev, 1254 config->max_virtqueue_pairs * 2, 1255 SOCKET_ID_ANY); 1256 1257 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d", 1258 config->max_virtqueue_pairs); 1259 PMD_INIT_LOG(DEBUG, "config->status=%d", config->status); 1260 PMD_INIT_LOG(DEBUG, 1261 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X", 1262 config->mac[0], config->mac[1], 1263 config->mac[2], config->mac[3], 1264 config->mac[4], config->mac[5]); 1265 } else { 1266 hw->max_rx_queues = 1; 1267 hw->max_tx_queues = 1; 1268 } 1269 1270 eth_dev->data->nb_rx_queues = hw->max_rx_queues; 1271 eth_dev->data->nb_tx_queues = hw->max_tx_queues; 1272 1273 PMD_INIT_LOG(DEBUG, "hw->max_rx_queues=%d hw->max_tx_queues=%d", 1274 hw->max_rx_queues, hw->max_tx_queues); 1275 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x", 1276 eth_dev->data->port_id, pci_dev->id.vendor_id, 1277 pci_dev->id.device_id); 1278 1279 /* Setup interrupt callback */ 1280 if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) 1281 rte_intr_callback_register(&pci_dev->intr_handle, 1282 virtio_interrupt_handler, eth_dev); 1283 1284 virtio_dev_cq_start(eth_dev); 1285 1286 return 0; 1287 } 1288 1289 static int 1290 eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev) 1291 { 1292 struct rte_pci_device *pci_dev; 1293 struct virtio_hw *hw = eth_dev->data->dev_private; 1294 1295 PMD_INIT_FUNC_TRACE(); 1296 1297 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 1298 return -EPERM; 1299 1300 if (hw->started == 1) { 1301 virtio_dev_stop(eth_dev); 1302 virtio_dev_close(eth_dev); 1303 } 1304 pci_dev = eth_dev->pci_dev; 1305 1306 eth_dev->dev_ops = NULL; 1307 eth_dev->tx_pkt_burst = NULL; 1308 eth_dev->rx_pkt_burst = NULL; 1309 1310 virtio_dev_queue_release(hw->cvq); 1311 1312 rte_free(eth_dev->data->mac_addrs); 1313 eth_dev->data->mac_addrs = NULL; 1314 1315 /* reset interrupt callback */ 1316 if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) 1317 rte_intr_callback_unregister(&pci_dev->intr_handle, 1318 virtio_interrupt_handler, 1319 eth_dev); 1320 1321 PMD_INIT_LOG(DEBUG, "dev_uninit completed"); 1322 1323 return 0; 1324 } 1325 1326 static struct eth_driver rte_virtio_pmd = { 1327 .pci_drv = { 1328 .name = "rte_virtio_pmd", 1329 .id_table = pci_id_virtio_map, 1330 .drv_flags = RTE_PCI_DRV_DETACHABLE, 1331 }, 1332 .eth_dev_init = eth_virtio_dev_init, 1333 .eth_dev_uninit = eth_virtio_dev_uninit, 1334 .dev_private_size = sizeof(struct virtio_hw), 1335 }; 1336 1337 /* 1338 * Driver initialization routine. 1339 * Invoked once at EAL init time. 1340 * Register itself as the [Poll Mode] Driver of PCI virtio devices. 1341 * Returns 0 on success. 1342 */ 1343 static int 1344 rte_virtio_pmd_init(const char *name __rte_unused, 1345 const char *param __rte_unused) 1346 { 1347 if (rte_eal_iopl_init() != 0) { 1348 PMD_INIT_LOG(ERR, "IOPL call failed - cannot use virtio PMD"); 1349 return -1; 1350 } 1351 1352 rte_eth_driver_register(&rte_virtio_pmd); 1353 return 0; 1354 } 1355 1356 /* 1357 * Configure virtio device 1358 * It returns 0 on success. 1359 */ 1360 static int 1361 virtio_dev_configure(struct rte_eth_dev *dev) 1362 { 1363 const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 1364 struct virtio_hw *hw = dev->data->dev_private; 1365 struct rte_pci_device *pci_dev = dev->pci_dev; 1366 1367 PMD_INIT_LOG(DEBUG, "configure"); 1368 1369 if (rxmode->hw_ip_checksum) { 1370 PMD_DRV_LOG(ERR, "HW IP checksum not supported"); 1371 return (-EINVAL); 1372 } 1373 1374 hw->vlan_strip = rxmode->hw_vlan_strip; 1375 1376 if (rxmode->hw_vlan_filter 1377 && !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { 1378 PMD_DRV_LOG(NOTICE, 1379 "vlan filtering not available on this host"); 1380 return -ENOTSUP; 1381 } 1382 1383 if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) 1384 if (vtpci_irq_config(hw, 0) == VIRTIO_MSI_NO_VECTOR) { 1385 PMD_DRV_LOG(ERR, "failed to set config vector"); 1386 return -EBUSY; 1387 } 1388 1389 return 0; 1390 } 1391 1392 1393 static int 1394 virtio_dev_start(struct rte_eth_dev *dev) 1395 { 1396 uint16_t nb_queues, i; 1397 struct virtio_hw *hw = dev->data->dev_private; 1398 struct rte_pci_device *pci_dev = dev->pci_dev; 1399 1400 /* check if lsc interrupt feature is enabled */ 1401 if (dev->data->dev_conf.intr_conf.lsc) { 1402 if (!(pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC)) { 1403 PMD_DRV_LOG(ERR, "link status not supported by host"); 1404 return -ENOTSUP; 1405 } 1406 1407 if (rte_intr_enable(&dev->pci_dev->intr_handle) < 0) { 1408 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1409 return -EIO; 1410 } 1411 } 1412 1413 /* Initialize Link state */ 1414 virtio_dev_link_update(dev, 0); 1415 1416 /* On restart after stop do not touch queues */ 1417 if (hw->started) 1418 return 0; 1419 1420 /* Do final configuration before rx/tx engine starts */ 1421 virtio_dev_rxtx_start(dev); 1422 vtpci_reinit_complete(hw); 1423 1424 hw->started = 1; 1425 1426 /*Notify the backend 1427 *Otherwise the tap backend might already stop its queue due to fullness. 1428 *vhost backend will have no chance to be waked up 1429 */ 1430 nb_queues = dev->data->nb_rx_queues; 1431 if (nb_queues > 1) { 1432 if (virtio_set_multiple_queues(dev, nb_queues) != 0) 1433 return -EINVAL; 1434 } 1435 1436 PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues); 1437 1438 for (i = 0; i < nb_queues; i++) 1439 virtqueue_notify(dev->data->rx_queues[i]); 1440 1441 PMD_INIT_LOG(DEBUG, "Notified backend at initialization"); 1442 1443 for (i = 0; i < dev->data->nb_rx_queues; i++) 1444 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); 1445 1446 for (i = 0; i < dev->data->nb_tx_queues; i++) 1447 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); 1448 1449 return 0; 1450 } 1451 1452 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) 1453 { 1454 struct rte_mbuf *buf; 1455 int i, mbuf_num = 0; 1456 1457 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1458 PMD_INIT_LOG(DEBUG, 1459 "Before freeing rxq[%d] used and unused buf", i); 1460 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); 1461 1462 PMD_INIT_LOG(DEBUG, "rx_queues[%d]=%p", 1463 i, dev->data->rx_queues[i]); 1464 while ((buf = (struct rte_mbuf *)virtqueue_detatch_unused( 1465 dev->data->rx_queues[i])) != NULL) { 1466 rte_pktmbuf_free(buf); 1467 mbuf_num++; 1468 } 1469 1470 PMD_INIT_LOG(DEBUG, "free %d mbufs", mbuf_num); 1471 PMD_INIT_LOG(DEBUG, 1472 "After freeing rxq[%d] used and unused buf", i); 1473 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); 1474 } 1475 1476 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1477 PMD_INIT_LOG(DEBUG, 1478 "Before freeing txq[%d] used and unused bufs", 1479 i); 1480 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); 1481 1482 mbuf_num = 0; 1483 while ((buf = (struct rte_mbuf *)virtqueue_detatch_unused( 1484 dev->data->tx_queues[i])) != NULL) { 1485 rte_pktmbuf_free(buf); 1486 1487 mbuf_num++; 1488 } 1489 1490 PMD_INIT_LOG(DEBUG, "free %d mbufs", mbuf_num); 1491 PMD_INIT_LOG(DEBUG, 1492 "After freeing txq[%d] used and unused buf", i); 1493 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); 1494 } 1495 } 1496 1497 /* 1498 * Stop device: disable interrupt and mark link down 1499 */ 1500 static void 1501 virtio_dev_stop(struct rte_eth_dev *dev) 1502 { 1503 struct rte_eth_link link; 1504 1505 PMD_INIT_LOG(DEBUG, "stop"); 1506 1507 if (dev->data->dev_conf.intr_conf.lsc) 1508 rte_intr_disable(&dev->pci_dev->intr_handle); 1509 1510 memset(&link, 0, sizeof(link)); 1511 virtio_dev_atomic_write_link_status(dev, &link); 1512 } 1513 1514 static int 1515 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) 1516 { 1517 struct rte_eth_link link, old; 1518 uint16_t status; 1519 struct virtio_hw *hw = dev->data->dev_private; 1520 memset(&link, 0, sizeof(link)); 1521 virtio_dev_atomic_read_link_status(dev, &link); 1522 old = link; 1523 link.link_duplex = FULL_DUPLEX; 1524 link.link_speed = SPEED_10G; 1525 1526 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { 1527 PMD_INIT_LOG(DEBUG, "Get link status from hw"); 1528 vtpci_read_dev_config(hw, 1529 offsetof(struct virtio_net_config, status), 1530 &status, sizeof(status)); 1531 if ((status & VIRTIO_NET_S_LINK_UP) == 0) { 1532 link.link_status = 0; 1533 PMD_INIT_LOG(DEBUG, "Port %d is down", 1534 dev->data->port_id); 1535 } else { 1536 link.link_status = 1; 1537 PMD_INIT_LOG(DEBUG, "Port %d is up", 1538 dev->data->port_id); 1539 } 1540 } else { 1541 link.link_status = 1; /* Link up */ 1542 } 1543 virtio_dev_atomic_write_link_status(dev, &link); 1544 1545 return (old.link_status == link.link_status) ? -1 : 0; 1546 } 1547 1548 static void 1549 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 1550 { 1551 struct virtio_hw *hw = dev->data->dev_private; 1552 1553 dev_info->driver_name = dev->driver->pci_drv.name; 1554 dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues; 1555 dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues; 1556 dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE; 1557 dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN; 1558 dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS; 1559 dev_info->default_txconf = (struct rte_eth_txconf) { 1560 .txq_flags = ETH_TXQ_FLAGS_NOOFFLOADS 1561 }; 1562 } 1563 1564 /* 1565 * It enables testpmd to collect per queue stats. 1566 */ 1567 static int 1568 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev, 1569 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx, 1570 __rte_unused uint8_t is_rx) 1571 { 1572 return 0; 1573 } 1574 1575 static struct rte_driver rte_virtio_driver = { 1576 .type = PMD_PDEV, 1577 .init = rte_virtio_pmd_init, 1578 }; 1579 1580 PMD_REGISTER_DRIVER(rte_virtio_driver); 1581