1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <string.h> 7 #include <stdio.h> 8 #include <errno.h> 9 #include <unistd.h> 10 11 #include <rte_ethdev_driver.h> 12 #include <rte_ethdev_pci.h> 13 #include <rte_memcpy.h> 14 #include <rte_string_fns.h> 15 #include <rte_memzone.h> 16 #include <rte_malloc.h> 17 #include <rte_branch_prediction.h> 18 #include <rte_pci.h> 19 #include <rte_bus_pci.h> 20 #include <rte_ether.h> 21 #include <rte_ip.h> 22 #include <rte_arp.h> 23 #include <rte_common.h> 24 #include <rte_errno.h> 25 #include <rte_cpuflags.h> 26 27 #include <rte_memory.h> 28 #include <rte_eal.h> 29 #include <rte_dev.h> 30 #include <rte_cycles.h> 31 #include <rte_kvargs.h> 32 33 #include "virtio_ethdev.h" 34 #include "virtio_pci.h" 35 #include "virtio_logs.h" 36 #include "virtqueue.h" 37 #include "virtio_rxtx.h" 38 #include "virtio_user/virtio_user_dev.h" 39 40 static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev); 41 static int virtio_dev_configure(struct rte_eth_dev *dev); 42 static int virtio_dev_start(struct rte_eth_dev *dev); 43 static void virtio_dev_stop(struct rte_eth_dev *dev); 44 static void virtio_dev_promiscuous_enable(struct rte_eth_dev *dev); 45 static void virtio_dev_promiscuous_disable(struct rte_eth_dev *dev); 46 static void virtio_dev_allmulticast_enable(struct rte_eth_dev *dev); 47 static void virtio_dev_allmulticast_disable(struct rte_eth_dev *dev); 48 static void virtio_dev_info_get(struct rte_eth_dev *dev, 49 struct rte_eth_dev_info *dev_info); 50 static int virtio_dev_link_update(struct rte_eth_dev *dev, 51 int wait_to_complete); 52 static int virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); 53 54 static void virtio_set_hwaddr(struct virtio_hw *hw); 55 static void virtio_get_hwaddr(struct virtio_hw *hw); 56 57 static int virtio_dev_stats_get(struct rte_eth_dev *dev, 58 struct rte_eth_stats *stats); 59 static int virtio_dev_xstats_get(struct rte_eth_dev *dev, 60 struct rte_eth_xstat *xstats, unsigned n); 61 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev, 62 struct rte_eth_xstat_name *xstats_names, 63 unsigned limit); 64 static void virtio_dev_stats_reset(struct rte_eth_dev *dev); 65 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev); 66 static int virtio_vlan_filter_set(struct rte_eth_dev *dev, 67 uint16_t vlan_id, int on); 68 static int virtio_mac_addr_add(struct rte_eth_dev *dev, 69 struct rte_ether_addr *mac_addr, 70 uint32_t index, uint32_t vmdq); 71 static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index); 72 static int virtio_mac_addr_set(struct rte_eth_dev *dev, 73 struct rte_ether_addr *mac_addr); 74 75 static int virtio_intr_disable(struct rte_eth_dev *dev); 76 77 static int virtio_dev_queue_stats_mapping_set( 78 struct rte_eth_dev *eth_dev, 79 uint16_t queue_id, 80 uint8_t stat_idx, 81 uint8_t is_rx); 82 83 int virtio_logtype_init; 84 int virtio_logtype_driver; 85 86 static void virtio_notify_peers(struct rte_eth_dev *dev); 87 static void virtio_ack_link_announce(struct rte_eth_dev *dev); 88 89 /* 90 * The set of PCI devices this driver supports 91 */ 92 static const struct rte_pci_id pci_id_virtio_map[] = { 93 { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_LEGACY_DEVICEID_NET) }, 94 { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_MODERN_DEVICEID_NET) }, 95 { .vendor_id = 0, /* sentinel */ }, 96 }; 97 98 struct rte_virtio_xstats_name_off { 99 char name[RTE_ETH_XSTATS_NAME_SIZE]; 100 unsigned offset; 101 }; 102 103 /* [rt]x_qX_ is prepended to the name string here */ 104 static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = { 105 {"good_packets", offsetof(struct virtnet_rx, stats.packets)}, 106 {"good_bytes", offsetof(struct virtnet_rx, stats.bytes)}, 107 {"errors", offsetof(struct virtnet_rx, stats.errors)}, 108 {"multicast_packets", offsetof(struct virtnet_rx, stats.multicast)}, 109 {"broadcast_packets", offsetof(struct virtnet_rx, stats.broadcast)}, 110 {"undersize_packets", offsetof(struct virtnet_rx, stats.size_bins[0])}, 111 {"size_64_packets", offsetof(struct virtnet_rx, stats.size_bins[1])}, 112 {"size_65_127_packets", offsetof(struct virtnet_rx, stats.size_bins[2])}, 113 {"size_128_255_packets", offsetof(struct virtnet_rx, stats.size_bins[3])}, 114 {"size_256_511_packets", offsetof(struct virtnet_rx, stats.size_bins[4])}, 115 {"size_512_1023_packets", offsetof(struct virtnet_rx, stats.size_bins[5])}, 116 {"size_1024_1518_packets", offsetof(struct virtnet_rx, stats.size_bins[6])}, 117 {"size_1519_max_packets", offsetof(struct virtnet_rx, stats.size_bins[7])}, 118 }; 119 120 /* [rt]x_qX_ is prepended to the name string here */ 121 static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = { 122 {"good_packets", offsetof(struct virtnet_tx, stats.packets)}, 123 {"good_bytes", offsetof(struct virtnet_tx, stats.bytes)}, 124 {"errors", offsetof(struct virtnet_tx, stats.errors)}, 125 {"multicast_packets", offsetof(struct virtnet_tx, stats.multicast)}, 126 {"broadcast_packets", offsetof(struct virtnet_tx, stats.broadcast)}, 127 {"undersize_packets", offsetof(struct virtnet_tx, stats.size_bins[0])}, 128 {"size_64_packets", offsetof(struct virtnet_tx, stats.size_bins[1])}, 129 {"size_65_127_packets", offsetof(struct virtnet_tx, stats.size_bins[2])}, 130 {"size_128_255_packets", offsetof(struct virtnet_tx, stats.size_bins[3])}, 131 {"size_256_511_packets", offsetof(struct virtnet_tx, stats.size_bins[4])}, 132 {"size_512_1023_packets", offsetof(struct virtnet_tx, stats.size_bins[5])}, 133 {"size_1024_1518_packets", offsetof(struct virtnet_tx, stats.size_bins[6])}, 134 {"size_1519_max_packets", offsetof(struct virtnet_tx, stats.size_bins[7])}, 135 }; 136 137 #define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \ 138 sizeof(rte_virtio_rxq_stat_strings[0])) 139 #define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \ 140 sizeof(rte_virtio_txq_stat_strings[0])) 141 142 struct virtio_hw_internal virtio_hw_internal[RTE_MAX_ETHPORTS]; 143 144 static struct virtio_pmd_ctrl * 145 virtio_send_command_packed(struct virtnet_ctl *cvq, 146 struct virtio_pmd_ctrl *ctrl, 147 int *dlen, int pkt_num) 148 { 149 struct virtqueue *vq = cvq->vq; 150 int head; 151 struct vring_packed_desc *desc = vq->vq_packed.ring.desc; 152 struct virtio_pmd_ctrl *result; 153 uint16_t flags; 154 int sum = 0; 155 int nb_descs = 0; 156 int k; 157 158 /* 159 * Format is enforced in qemu code: 160 * One TX packet for header; 161 * At least one TX packet per argument; 162 * One RX packet for ACK. 163 */ 164 head = vq->vq_avail_idx; 165 flags = vq->vq_packed.cached_flags; 166 desc[head].addr = cvq->virtio_net_hdr_mem; 167 desc[head].len = sizeof(struct virtio_net_ctrl_hdr); 168 vq->vq_free_cnt--; 169 nb_descs++; 170 if (++vq->vq_avail_idx >= vq->vq_nentries) { 171 vq->vq_avail_idx -= vq->vq_nentries; 172 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; 173 } 174 175 for (k = 0; k < pkt_num; k++) { 176 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem 177 + sizeof(struct virtio_net_ctrl_hdr) 178 + sizeof(ctrl->status) + sizeof(uint8_t) * sum; 179 desc[vq->vq_avail_idx].len = dlen[k]; 180 desc[vq->vq_avail_idx].flags = VRING_DESC_F_NEXT | 181 vq->vq_packed.cached_flags; 182 sum += dlen[k]; 183 vq->vq_free_cnt--; 184 nb_descs++; 185 if (++vq->vq_avail_idx >= vq->vq_nentries) { 186 vq->vq_avail_idx -= vq->vq_nentries; 187 vq->vq_packed.cached_flags ^= 188 VRING_PACKED_DESC_F_AVAIL_USED; 189 } 190 } 191 192 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem 193 + sizeof(struct virtio_net_ctrl_hdr); 194 desc[vq->vq_avail_idx].len = sizeof(ctrl->status); 195 desc[vq->vq_avail_idx].flags = VRING_DESC_F_WRITE | 196 vq->vq_packed.cached_flags; 197 vq->vq_free_cnt--; 198 nb_descs++; 199 if (++vq->vq_avail_idx >= vq->vq_nentries) { 200 vq->vq_avail_idx -= vq->vq_nentries; 201 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; 202 } 203 204 virtio_wmb(vq->hw->weak_barriers); 205 desc[head].flags = VRING_DESC_F_NEXT | flags; 206 207 virtio_wmb(vq->hw->weak_barriers); 208 virtqueue_notify(vq); 209 210 /* wait for used descriptors in virtqueue */ 211 while (!desc_is_used(&desc[head], vq)) 212 usleep(100); 213 214 virtio_rmb(vq->hw->weak_barriers); 215 216 /* now get used descriptors */ 217 vq->vq_free_cnt += nb_descs; 218 vq->vq_used_cons_idx += nb_descs; 219 if (vq->vq_used_cons_idx >= vq->vq_nentries) { 220 vq->vq_used_cons_idx -= vq->vq_nentries; 221 vq->vq_packed.used_wrap_counter ^= 1; 222 } 223 224 PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\n" 225 "vq->vq_avail_idx=%d\n" 226 "vq->vq_used_cons_idx=%d\n" 227 "vq->vq_packed.cached_flags=0x%x\n" 228 "vq->vq_packed.used_wrap_counter=%d\n", 229 vq->vq_free_cnt, 230 vq->vq_avail_idx, 231 vq->vq_used_cons_idx, 232 vq->vq_packed.cached_flags, 233 vq->vq_packed.used_wrap_counter); 234 235 result = cvq->virtio_net_hdr_mz->addr; 236 return result; 237 } 238 239 static struct virtio_pmd_ctrl * 240 virtio_send_command_split(struct virtnet_ctl *cvq, 241 struct virtio_pmd_ctrl *ctrl, 242 int *dlen, int pkt_num) 243 { 244 struct virtio_pmd_ctrl *result; 245 struct virtqueue *vq = cvq->vq; 246 uint32_t head, i; 247 int k, sum = 0; 248 249 head = vq->vq_desc_head_idx; 250 251 /* 252 * Format is enforced in qemu code: 253 * One TX packet for header; 254 * At least one TX packet per argument; 255 * One RX packet for ACK. 256 */ 257 vq->vq_split.ring.desc[head].flags = VRING_DESC_F_NEXT; 258 vq->vq_split.ring.desc[head].addr = cvq->virtio_net_hdr_mem; 259 vq->vq_split.ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr); 260 vq->vq_free_cnt--; 261 i = vq->vq_split.ring.desc[head].next; 262 263 for (k = 0; k < pkt_num; k++) { 264 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_NEXT; 265 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem 266 + sizeof(struct virtio_net_ctrl_hdr) 267 + sizeof(ctrl->status) + sizeof(uint8_t)*sum; 268 vq->vq_split.ring.desc[i].len = dlen[k]; 269 sum += dlen[k]; 270 vq->vq_free_cnt--; 271 i = vq->vq_split.ring.desc[i].next; 272 } 273 274 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_WRITE; 275 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem 276 + sizeof(struct virtio_net_ctrl_hdr); 277 vq->vq_split.ring.desc[i].len = sizeof(ctrl->status); 278 vq->vq_free_cnt--; 279 280 vq->vq_desc_head_idx = vq->vq_split.ring.desc[i].next; 281 282 vq_update_avail_ring(vq, head); 283 vq_update_avail_idx(vq); 284 285 PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index); 286 287 virtqueue_notify(vq); 288 289 rte_rmb(); 290 while (VIRTQUEUE_NUSED(vq) == 0) { 291 rte_rmb(); 292 usleep(100); 293 } 294 295 while (VIRTQUEUE_NUSED(vq)) { 296 uint32_t idx, desc_idx, used_idx; 297 struct vring_used_elem *uep; 298 299 used_idx = (uint32_t)(vq->vq_used_cons_idx 300 & (vq->vq_nentries - 1)); 301 uep = &vq->vq_split.ring.used->ring[used_idx]; 302 idx = (uint32_t) uep->id; 303 desc_idx = idx; 304 305 while (vq->vq_split.ring.desc[desc_idx].flags & 306 VRING_DESC_F_NEXT) { 307 desc_idx = vq->vq_split.ring.desc[desc_idx].next; 308 vq->vq_free_cnt++; 309 } 310 311 vq->vq_split.ring.desc[desc_idx].next = vq->vq_desc_head_idx; 312 vq->vq_desc_head_idx = idx; 313 314 vq->vq_used_cons_idx++; 315 vq->vq_free_cnt++; 316 } 317 318 PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d", 319 vq->vq_free_cnt, vq->vq_desc_head_idx); 320 321 result = cvq->virtio_net_hdr_mz->addr; 322 return result; 323 } 324 325 static int 326 virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl, 327 int *dlen, int pkt_num) 328 { 329 virtio_net_ctrl_ack status = ~0; 330 struct virtio_pmd_ctrl *result; 331 struct virtqueue *vq; 332 333 ctrl->status = status; 334 335 if (!cvq || !cvq->vq) { 336 PMD_INIT_LOG(ERR, "Control queue is not supported."); 337 return -1; 338 } 339 340 rte_spinlock_lock(&cvq->lock); 341 vq = cvq->vq; 342 343 PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, " 344 "vq->hw->cvq = %p vq = %p", 345 vq->vq_desc_head_idx, status, vq->hw->cvq, vq); 346 347 if (vq->vq_free_cnt < pkt_num + 2 || pkt_num < 1) { 348 rte_spinlock_unlock(&cvq->lock); 349 return -1; 350 } 351 352 memcpy(cvq->virtio_net_hdr_mz->addr, ctrl, 353 sizeof(struct virtio_pmd_ctrl)); 354 355 if (vtpci_packed_queue(vq->hw)) 356 result = virtio_send_command_packed(cvq, ctrl, dlen, pkt_num); 357 else 358 result = virtio_send_command_split(cvq, ctrl, dlen, pkt_num); 359 360 rte_spinlock_unlock(&cvq->lock); 361 return result->status; 362 } 363 364 static int 365 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues) 366 { 367 struct virtio_hw *hw = dev->data->dev_private; 368 struct virtio_pmd_ctrl ctrl; 369 int dlen[1]; 370 int ret; 371 372 ctrl.hdr.class = VIRTIO_NET_CTRL_MQ; 373 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; 374 memcpy(ctrl.data, &nb_queues, sizeof(uint16_t)); 375 376 dlen[0] = sizeof(uint16_t); 377 378 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 379 if (ret) { 380 PMD_INIT_LOG(ERR, "Multiqueue configured but send command " 381 "failed, this is too late now..."); 382 return -EINVAL; 383 } 384 385 return 0; 386 } 387 388 static void 389 virtio_dev_queue_release(void *queue __rte_unused) 390 { 391 /* do nothing */ 392 } 393 394 static uint16_t 395 virtio_get_nr_vq(struct virtio_hw *hw) 396 { 397 uint16_t nr_vq = hw->max_queue_pairs * 2; 398 399 if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) 400 nr_vq += 1; 401 402 return nr_vq; 403 } 404 405 static void 406 virtio_init_vring(struct virtqueue *vq) 407 { 408 int size = vq->vq_nentries; 409 uint8_t *ring_mem = vq->vq_ring_virt_mem; 410 411 PMD_INIT_FUNC_TRACE(); 412 413 memset(ring_mem, 0, vq->vq_ring_size); 414 415 vq->vq_used_cons_idx = 0; 416 vq->vq_desc_head_idx = 0; 417 vq->vq_avail_idx = 0; 418 vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); 419 vq->vq_free_cnt = vq->vq_nentries; 420 memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries); 421 if (vtpci_packed_queue(vq->hw)) { 422 vring_init_packed(&vq->vq_packed.ring, ring_mem, 423 VIRTIO_PCI_VRING_ALIGN, size); 424 vring_desc_init_packed(vq, size); 425 } else { 426 struct vring *vr = &vq->vq_split.ring; 427 428 vring_init_split(vr, ring_mem, VIRTIO_PCI_VRING_ALIGN, size); 429 vring_desc_init_split(vr->desc, size); 430 } 431 /* 432 * Disable device(host) interrupting guest 433 */ 434 virtqueue_disable_intr(vq); 435 } 436 437 static int 438 virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx) 439 { 440 char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 441 char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ]; 442 const struct rte_memzone *mz = NULL, *hdr_mz = NULL; 443 unsigned int vq_size, size; 444 struct virtio_hw *hw = dev->data->dev_private; 445 struct virtnet_rx *rxvq = NULL; 446 struct virtnet_tx *txvq = NULL; 447 struct virtnet_ctl *cvq = NULL; 448 struct virtqueue *vq; 449 size_t sz_hdr_mz = 0; 450 void *sw_ring = NULL; 451 int queue_type = virtio_get_queue_type(hw, vtpci_queue_idx); 452 int ret; 453 int numa_node = dev->device->numa_node; 454 455 PMD_INIT_LOG(INFO, "setting up queue: %u on NUMA node %d", 456 vtpci_queue_idx, numa_node); 457 458 /* 459 * Read the virtqueue size from the Queue Size field 460 * Always power of 2 and if 0 virtqueue does not exist 461 */ 462 vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx); 463 PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size); 464 if (vq_size == 0) { 465 PMD_INIT_LOG(ERR, "virtqueue does not exist"); 466 return -EINVAL; 467 } 468 469 if (!rte_is_power_of_2(vq_size)) { 470 PMD_INIT_LOG(ERR, "virtqueue size is not powerof 2"); 471 return -EINVAL; 472 } 473 474 snprintf(vq_name, sizeof(vq_name), "port%d_vq%d", 475 dev->data->port_id, vtpci_queue_idx); 476 477 size = RTE_ALIGN_CEIL(sizeof(*vq) + 478 vq_size * sizeof(struct vq_desc_extra), 479 RTE_CACHE_LINE_SIZE); 480 if (queue_type == VTNET_TQ) { 481 /* 482 * For each xmit packet, allocate a virtio_net_hdr 483 * and indirect ring elements 484 */ 485 sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region); 486 } else if (queue_type == VTNET_CQ) { 487 /* Allocate a page for control vq command, data and status */ 488 sz_hdr_mz = PAGE_SIZE; 489 } 490 491 vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE, 492 numa_node); 493 if (vq == NULL) { 494 PMD_INIT_LOG(ERR, "can not allocate vq"); 495 return -ENOMEM; 496 } 497 hw->vqs[vtpci_queue_idx] = vq; 498 499 vq->hw = hw; 500 vq->vq_queue_index = vtpci_queue_idx; 501 vq->vq_nentries = vq_size; 502 if (vtpci_packed_queue(hw)) { 503 vq->vq_packed.used_wrap_counter = 1; 504 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL; 505 vq->vq_packed.event_flags_shadow = 0; 506 if (queue_type == VTNET_RQ) 507 vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE; 508 } 509 510 /* 511 * Reserve a memzone for vring elements 512 */ 513 size = vring_size(hw, vq_size, VIRTIO_PCI_VRING_ALIGN); 514 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN); 515 PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d", 516 size, vq->vq_ring_size); 517 518 mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, 519 numa_node, RTE_MEMZONE_IOVA_CONTIG, 520 VIRTIO_PCI_VRING_ALIGN); 521 if (mz == NULL) { 522 if (rte_errno == EEXIST) 523 mz = rte_memzone_lookup(vq_name); 524 if (mz == NULL) { 525 ret = -ENOMEM; 526 goto fail_q_alloc; 527 } 528 } 529 530 memset(mz->addr, 0, mz->len); 531 532 vq->vq_ring_mem = mz->iova; 533 vq->vq_ring_virt_mem = mz->addr; 534 PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%" PRIx64, 535 (uint64_t)mz->iova); 536 PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: 0x%" PRIx64, 537 (uint64_t)(uintptr_t)mz->addr); 538 539 virtio_init_vring(vq); 540 541 if (sz_hdr_mz) { 542 snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr", 543 dev->data->port_id, vtpci_queue_idx); 544 hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz, 545 numa_node, RTE_MEMZONE_IOVA_CONTIG, 546 RTE_CACHE_LINE_SIZE); 547 if (hdr_mz == NULL) { 548 if (rte_errno == EEXIST) 549 hdr_mz = rte_memzone_lookup(vq_hdr_name); 550 if (hdr_mz == NULL) { 551 ret = -ENOMEM; 552 goto fail_q_alloc; 553 } 554 } 555 } 556 557 if (queue_type == VTNET_RQ) { 558 size_t sz_sw = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) * 559 sizeof(vq->sw_ring[0]); 560 561 sw_ring = rte_zmalloc_socket("sw_ring", sz_sw, 562 RTE_CACHE_LINE_SIZE, numa_node); 563 if (!sw_ring) { 564 PMD_INIT_LOG(ERR, "can not allocate RX soft ring"); 565 ret = -ENOMEM; 566 goto fail_q_alloc; 567 } 568 569 vq->sw_ring = sw_ring; 570 rxvq = &vq->rxq; 571 rxvq->vq = vq; 572 rxvq->port_id = dev->data->port_id; 573 rxvq->mz = mz; 574 } else if (queue_type == VTNET_TQ) { 575 txvq = &vq->txq; 576 txvq->vq = vq; 577 txvq->port_id = dev->data->port_id; 578 txvq->mz = mz; 579 txvq->virtio_net_hdr_mz = hdr_mz; 580 txvq->virtio_net_hdr_mem = hdr_mz->iova; 581 } else if (queue_type == VTNET_CQ) { 582 cvq = &vq->cq; 583 cvq->vq = vq; 584 cvq->mz = mz; 585 cvq->virtio_net_hdr_mz = hdr_mz; 586 cvq->virtio_net_hdr_mem = hdr_mz->iova; 587 memset(cvq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE); 588 589 hw->cvq = cvq; 590 } 591 592 /* For virtio_user case (that is when hw->dev is NULL), we use 593 * virtual address. And we need properly set _offset_, please see 594 * VIRTIO_MBUF_DATA_DMA_ADDR in virtqueue.h for more information. 595 */ 596 if (!hw->virtio_user_dev) 597 vq->offset = offsetof(struct rte_mbuf, buf_iova); 598 else { 599 vq->vq_ring_mem = (uintptr_t)mz->addr; 600 vq->offset = offsetof(struct rte_mbuf, buf_addr); 601 if (queue_type == VTNET_TQ) 602 txvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr; 603 else if (queue_type == VTNET_CQ) 604 cvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr; 605 } 606 607 if (queue_type == VTNET_TQ) { 608 struct virtio_tx_region *txr; 609 unsigned int i; 610 611 txr = hdr_mz->addr; 612 memset(txr, 0, vq_size * sizeof(*txr)); 613 for (i = 0; i < vq_size; i++) { 614 struct vring_desc *start_dp = txr[i].tx_indir; 615 616 /* first indirect descriptor is always the tx header */ 617 if (!vtpci_packed_queue(hw)) { 618 vring_desc_init_split(start_dp, 619 RTE_DIM(txr[i].tx_indir)); 620 start_dp->addr = txvq->virtio_net_hdr_mem 621 + i * sizeof(*txr) 622 + offsetof(struct virtio_tx_region, 623 tx_hdr); 624 start_dp->len = hw->vtnet_hdr_size; 625 start_dp->flags = VRING_DESC_F_NEXT; 626 } 627 } 628 } 629 630 if (VTPCI_OPS(hw)->setup_queue(hw, vq) < 0) { 631 PMD_INIT_LOG(ERR, "setup_queue failed"); 632 return -EINVAL; 633 } 634 635 return 0; 636 637 fail_q_alloc: 638 rte_free(sw_ring); 639 rte_memzone_free(hdr_mz); 640 rte_memzone_free(mz); 641 rte_free(vq); 642 643 return ret; 644 } 645 646 static void 647 virtio_free_queues(struct virtio_hw *hw) 648 { 649 uint16_t nr_vq = virtio_get_nr_vq(hw); 650 struct virtqueue *vq; 651 int queue_type; 652 uint16_t i; 653 654 if (hw->vqs == NULL) 655 return; 656 657 for (i = 0; i < nr_vq; i++) { 658 vq = hw->vqs[i]; 659 if (!vq) 660 continue; 661 662 queue_type = virtio_get_queue_type(hw, i); 663 if (queue_type == VTNET_RQ) { 664 rte_free(vq->sw_ring); 665 rte_memzone_free(vq->rxq.mz); 666 } else if (queue_type == VTNET_TQ) { 667 rte_memzone_free(vq->txq.mz); 668 rte_memzone_free(vq->txq.virtio_net_hdr_mz); 669 } else { 670 rte_memzone_free(vq->cq.mz); 671 rte_memzone_free(vq->cq.virtio_net_hdr_mz); 672 } 673 674 rte_free(vq); 675 hw->vqs[i] = NULL; 676 } 677 678 rte_free(hw->vqs); 679 hw->vqs = NULL; 680 } 681 682 static int 683 virtio_alloc_queues(struct rte_eth_dev *dev) 684 { 685 struct virtio_hw *hw = dev->data->dev_private; 686 uint16_t nr_vq = virtio_get_nr_vq(hw); 687 uint16_t i; 688 int ret; 689 690 hw->vqs = rte_zmalloc(NULL, sizeof(struct virtqueue *) * nr_vq, 0); 691 if (!hw->vqs) { 692 PMD_INIT_LOG(ERR, "failed to allocate vqs"); 693 return -ENOMEM; 694 } 695 696 for (i = 0; i < nr_vq; i++) { 697 ret = virtio_init_queue(dev, i); 698 if (ret < 0) { 699 virtio_free_queues(hw); 700 return ret; 701 } 702 } 703 704 return 0; 705 } 706 707 static void virtio_queues_unbind_intr(struct rte_eth_dev *dev); 708 709 static void 710 virtio_dev_close(struct rte_eth_dev *dev) 711 { 712 struct virtio_hw *hw = dev->data->dev_private; 713 struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf; 714 715 PMD_INIT_LOG(DEBUG, "virtio_dev_close"); 716 717 if (!hw->opened) 718 return; 719 hw->opened = false; 720 721 /* reset the NIC */ 722 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) 723 VTPCI_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR); 724 if (intr_conf->rxq) 725 virtio_queues_unbind_intr(dev); 726 727 if (intr_conf->lsc || intr_conf->rxq) { 728 virtio_intr_disable(dev); 729 rte_intr_efd_disable(dev->intr_handle); 730 rte_free(dev->intr_handle->intr_vec); 731 dev->intr_handle->intr_vec = NULL; 732 } 733 734 vtpci_reset(hw); 735 virtio_dev_free_mbufs(dev); 736 virtio_free_queues(hw); 737 738 #ifdef RTE_VIRTIO_USER 739 if (hw->virtio_user_dev) 740 virtio_user_dev_uninit(hw->virtio_user_dev); 741 else 742 #endif 743 if (dev->device) { 744 rte_pci_unmap_device(RTE_ETH_DEV_TO_PCI(dev)); 745 if (!hw->modern) 746 rte_pci_ioport_unmap(VTPCI_IO(hw)); 747 } 748 } 749 750 static void 751 virtio_dev_promiscuous_enable(struct rte_eth_dev *dev) 752 { 753 struct virtio_hw *hw = dev->data->dev_private; 754 struct virtio_pmd_ctrl ctrl; 755 int dlen[1]; 756 int ret; 757 758 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 759 PMD_INIT_LOG(INFO, "host does not support rx control"); 760 return; 761 } 762 763 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 764 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC; 765 ctrl.data[0] = 1; 766 dlen[0] = 1; 767 768 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 769 if (ret) 770 PMD_INIT_LOG(ERR, "Failed to enable promisc"); 771 } 772 773 static void 774 virtio_dev_promiscuous_disable(struct rte_eth_dev *dev) 775 { 776 struct virtio_hw *hw = dev->data->dev_private; 777 struct virtio_pmd_ctrl ctrl; 778 int dlen[1]; 779 int ret; 780 781 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 782 PMD_INIT_LOG(INFO, "host does not support rx control"); 783 return; 784 } 785 786 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 787 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC; 788 ctrl.data[0] = 0; 789 dlen[0] = 1; 790 791 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 792 if (ret) 793 PMD_INIT_LOG(ERR, "Failed to disable promisc"); 794 } 795 796 static void 797 virtio_dev_allmulticast_enable(struct rte_eth_dev *dev) 798 { 799 struct virtio_hw *hw = dev->data->dev_private; 800 struct virtio_pmd_ctrl ctrl; 801 int dlen[1]; 802 int ret; 803 804 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 805 PMD_INIT_LOG(INFO, "host does not support rx control"); 806 return; 807 } 808 809 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 810 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI; 811 ctrl.data[0] = 1; 812 dlen[0] = 1; 813 814 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 815 if (ret) 816 PMD_INIT_LOG(ERR, "Failed to enable allmulticast"); 817 } 818 819 static void 820 virtio_dev_allmulticast_disable(struct rte_eth_dev *dev) 821 { 822 struct virtio_hw *hw = dev->data->dev_private; 823 struct virtio_pmd_ctrl ctrl; 824 int dlen[1]; 825 int ret; 826 827 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 828 PMD_INIT_LOG(INFO, "host does not support rx control"); 829 return; 830 } 831 832 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 833 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI; 834 ctrl.data[0] = 0; 835 dlen[0] = 1; 836 837 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 838 if (ret) 839 PMD_INIT_LOG(ERR, "Failed to disable allmulticast"); 840 } 841 842 #define VLAN_TAG_LEN 4 /* 802.3ac tag (not DMA'd) */ 843 static int 844 virtio_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 845 { 846 struct virtio_hw *hw = dev->data->dev_private; 847 uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN + 848 hw->vtnet_hdr_size; 849 uint32_t frame_size = mtu + ether_hdr_len; 850 uint32_t max_frame_size = hw->max_mtu + ether_hdr_len; 851 852 max_frame_size = RTE_MIN(max_frame_size, VIRTIO_MAX_RX_PKTLEN); 853 854 if (mtu < RTE_ETHER_MIN_MTU || frame_size > max_frame_size) { 855 PMD_INIT_LOG(ERR, "MTU should be between %d and %d", 856 RTE_ETHER_MIN_MTU, max_frame_size - ether_hdr_len); 857 return -EINVAL; 858 } 859 return 0; 860 } 861 862 static int 863 virtio_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 864 { 865 struct virtio_hw *hw = dev->data->dev_private; 866 struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id]; 867 struct virtqueue *vq = rxvq->vq; 868 869 virtqueue_enable_intr(vq); 870 virtio_mb(hw->weak_barriers); 871 return 0; 872 } 873 874 static int 875 virtio_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 876 { 877 struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id]; 878 struct virtqueue *vq = rxvq->vq; 879 880 virtqueue_disable_intr(vq); 881 return 0; 882 } 883 884 /* 885 * dev_ops for virtio, bare necessities for basic operation 886 */ 887 static const struct eth_dev_ops virtio_eth_dev_ops = { 888 .dev_configure = virtio_dev_configure, 889 .dev_start = virtio_dev_start, 890 .dev_stop = virtio_dev_stop, 891 .dev_close = virtio_dev_close, 892 .promiscuous_enable = virtio_dev_promiscuous_enable, 893 .promiscuous_disable = virtio_dev_promiscuous_disable, 894 .allmulticast_enable = virtio_dev_allmulticast_enable, 895 .allmulticast_disable = virtio_dev_allmulticast_disable, 896 .mtu_set = virtio_mtu_set, 897 .dev_infos_get = virtio_dev_info_get, 898 .stats_get = virtio_dev_stats_get, 899 .xstats_get = virtio_dev_xstats_get, 900 .xstats_get_names = virtio_dev_xstats_get_names, 901 .stats_reset = virtio_dev_stats_reset, 902 .xstats_reset = virtio_dev_stats_reset, 903 .link_update = virtio_dev_link_update, 904 .vlan_offload_set = virtio_dev_vlan_offload_set, 905 .rx_queue_setup = virtio_dev_rx_queue_setup, 906 .rx_queue_intr_enable = virtio_dev_rx_queue_intr_enable, 907 .rx_queue_intr_disable = virtio_dev_rx_queue_intr_disable, 908 .rx_queue_release = virtio_dev_queue_release, 909 .rx_descriptor_done = virtio_dev_rx_queue_done, 910 .tx_queue_setup = virtio_dev_tx_queue_setup, 911 .tx_queue_release = virtio_dev_queue_release, 912 /* collect stats per queue */ 913 .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set, 914 .vlan_filter_set = virtio_vlan_filter_set, 915 .mac_addr_add = virtio_mac_addr_add, 916 .mac_addr_remove = virtio_mac_addr_remove, 917 .mac_addr_set = virtio_mac_addr_set, 918 }; 919 920 /* 921 * dev_ops for virtio-user in secondary processes, as we just have 922 * some limited supports currently. 923 */ 924 const struct eth_dev_ops virtio_user_secondary_eth_dev_ops = { 925 .dev_infos_get = virtio_dev_info_get, 926 .stats_get = virtio_dev_stats_get, 927 .xstats_get = virtio_dev_xstats_get, 928 .xstats_get_names = virtio_dev_xstats_get_names, 929 .stats_reset = virtio_dev_stats_reset, 930 .xstats_reset = virtio_dev_stats_reset, 931 /* collect stats per queue */ 932 .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set, 933 }; 934 935 static void 936 virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 937 { 938 unsigned i; 939 940 for (i = 0; i < dev->data->nb_tx_queues; i++) { 941 const struct virtnet_tx *txvq = dev->data->tx_queues[i]; 942 if (txvq == NULL) 943 continue; 944 945 stats->opackets += txvq->stats.packets; 946 stats->obytes += txvq->stats.bytes; 947 stats->oerrors += txvq->stats.errors; 948 949 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 950 stats->q_opackets[i] = txvq->stats.packets; 951 stats->q_obytes[i] = txvq->stats.bytes; 952 } 953 } 954 955 for (i = 0; i < dev->data->nb_rx_queues; i++) { 956 const struct virtnet_rx *rxvq = dev->data->rx_queues[i]; 957 if (rxvq == NULL) 958 continue; 959 960 stats->ipackets += rxvq->stats.packets; 961 stats->ibytes += rxvq->stats.bytes; 962 stats->ierrors += rxvq->stats.errors; 963 964 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 965 stats->q_ipackets[i] = rxvq->stats.packets; 966 stats->q_ibytes[i] = rxvq->stats.bytes; 967 } 968 } 969 970 stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed; 971 } 972 973 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev, 974 struct rte_eth_xstat_name *xstats_names, 975 __rte_unused unsigned limit) 976 { 977 unsigned i; 978 unsigned count = 0; 979 unsigned t; 980 981 unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS + 982 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS; 983 984 if (xstats_names != NULL) { 985 /* Note: limit checked in rte_eth_xstats_names() */ 986 987 for (i = 0; i < dev->data->nb_rx_queues; i++) { 988 struct virtnet_rx *rxvq = dev->data->rx_queues[i]; 989 if (rxvq == NULL) 990 continue; 991 for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) { 992 snprintf(xstats_names[count].name, 993 sizeof(xstats_names[count].name), 994 "rx_q%u_%s", i, 995 rte_virtio_rxq_stat_strings[t].name); 996 count++; 997 } 998 } 999 1000 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1001 struct virtnet_tx *txvq = dev->data->tx_queues[i]; 1002 if (txvq == NULL) 1003 continue; 1004 for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) { 1005 snprintf(xstats_names[count].name, 1006 sizeof(xstats_names[count].name), 1007 "tx_q%u_%s", i, 1008 rte_virtio_txq_stat_strings[t].name); 1009 count++; 1010 } 1011 } 1012 return count; 1013 } 1014 return nstats; 1015 } 1016 1017 static int 1018 virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1019 unsigned n) 1020 { 1021 unsigned i; 1022 unsigned count = 0; 1023 1024 unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS + 1025 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS; 1026 1027 if (n < nstats) 1028 return nstats; 1029 1030 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1031 struct virtnet_rx *rxvq = dev->data->rx_queues[i]; 1032 1033 if (rxvq == NULL) 1034 continue; 1035 1036 unsigned t; 1037 1038 for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) { 1039 xstats[count].value = *(uint64_t *)(((char *)rxvq) + 1040 rte_virtio_rxq_stat_strings[t].offset); 1041 xstats[count].id = count; 1042 count++; 1043 } 1044 } 1045 1046 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1047 struct virtnet_tx *txvq = dev->data->tx_queues[i]; 1048 1049 if (txvq == NULL) 1050 continue; 1051 1052 unsigned t; 1053 1054 for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) { 1055 xstats[count].value = *(uint64_t *)(((char *)txvq) + 1056 rte_virtio_txq_stat_strings[t].offset); 1057 xstats[count].id = count; 1058 count++; 1059 } 1060 } 1061 1062 return count; 1063 } 1064 1065 static int 1066 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1067 { 1068 virtio_update_stats(dev, stats); 1069 1070 return 0; 1071 } 1072 1073 static void 1074 virtio_dev_stats_reset(struct rte_eth_dev *dev) 1075 { 1076 unsigned int i; 1077 1078 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1079 struct virtnet_tx *txvq = dev->data->tx_queues[i]; 1080 if (txvq == NULL) 1081 continue; 1082 1083 txvq->stats.packets = 0; 1084 txvq->stats.bytes = 0; 1085 txvq->stats.errors = 0; 1086 txvq->stats.multicast = 0; 1087 txvq->stats.broadcast = 0; 1088 memset(txvq->stats.size_bins, 0, 1089 sizeof(txvq->stats.size_bins[0]) * 8); 1090 } 1091 1092 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1093 struct virtnet_rx *rxvq = dev->data->rx_queues[i]; 1094 if (rxvq == NULL) 1095 continue; 1096 1097 rxvq->stats.packets = 0; 1098 rxvq->stats.bytes = 0; 1099 rxvq->stats.errors = 0; 1100 rxvq->stats.multicast = 0; 1101 rxvq->stats.broadcast = 0; 1102 memset(rxvq->stats.size_bins, 0, 1103 sizeof(rxvq->stats.size_bins[0]) * 8); 1104 } 1105 } 1106 1107 static void 1108 virtio_set_hwaddr(struct virtio_hw *hw) 1109 { 1110 vtpci_write_dev_config(hw, 1111 offsetof(struct virtio_net_config, mac), 1112 &hw->mac_addr, RTE_ETHER_ADDR_LEN); 1113 } 1114 1115 static void 1116 virtio_get_hwaddr(struct virtio_hw *hw) 1117 { 1118 if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) { 1119 vtpci_read_dev_config(hw, 1120 offsetof(struct virtio_net_config, mac), 1121 &hw->mac_addr, RTE_ETHER_ADDR_LEN); 1122 } else { 1123 rte_eth_random_addr(&hw->mac_addr[0]); 1124 virtio_set_hwaddr(hw); 1125 } 1126 } 1127 1128 static int 1129 virtio_mac_table_set(struct virtio_hw *hw, 1130 const struct virtio_net_ctrl_mac *uc, 1131 const struct virtio_net_ctrl_mac *mc) 1132 { 1133 struct virtio_pmd_ctrl ctrl; 1134 int err, len[2]; 1135 1136 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 1137 PMD_DRV_LOG(INFO, "host does not support mac table"); 1138 return -1; 1139 } 1140 1141 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC; 1142 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; 1143 1144 len[0] = uc->entries * RTE_ETHER_ADDR_LEN + sizeof(uc->entries); 1145 memcpy(ctrl.data, uc, len[0]); 1146 1147 len[1] = mc->entries * RTE_ETHER_ADDR_LEN + sizeof(mc->entries); 1148 memcpy(ctrl.data + len[0], mc, len[1]); 1149 1150 err = virtio_send_command(hw->cvq, &ctrl, len, 2); 1151 if (err != 0) 1152 PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err); 1153 return err; 1154 } 1155 1156 static int 1157 virtio_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 1158 uint32_t index, uint32_t vmdq __rte_unused) 1159 { 1160 struct virtio_hw *hw = dev->data->dev_private; 1161 const struct rte_ether_addr *addrs = dev->data->mac_addrs; 1162 unsigned int i; 1163 struct virtio_net_ctrl_mac *uc, *mc; 1164 1165 if (index >= VIRTIO_MAX_MAC_ADDRS) { 1166 PMD_DRV_LOG(ERR, "mac address index %u out of range", index); 1167 return -EINVAL; 1168 } 1169 1170 uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN + 1171 sizeof(uc->entries)); 1172 uc->entries = 0; 1173 mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN + 1174 sizeof(mc->entries)); 1175 mc->entries = 0; 1176 1177 for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) { 1178 const struct rte_ether_addr *addr 1179 = (i == index) ? mac_addr : addrs + i; 1180 struct virtio_net_ctrl_mac *tbl 1181 = rte_is_multicast_ether_addr(addr) ? mc : uc; 1182 1183 memcpy(&tbl->macs[tbl->entries++], addr, RTE_ETHER_ADDR_LEN); 1184 } 1185 1186 return virtio_mac_table_set(hw, uc, mc); 1187 } 1188 1189 static void 1190 virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) 1191 { 1192 struct virtio_hw *hw = dev->data->dev_private; 1193 struct rte_ether_addr *addrs = dev->data->mac_addrs; 1194 struct virtio_net_ctrl_mac *uc, *mc; 1195 unsigned int i; 1196 1197 if (index >= VIRTIO_MAX_MAC_ADDRS) { 1198 PMD_DRV_LOG(ERR, "mac address index %u out of range", index); 1199 return; 1200 } 1201 1202 uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN + 1203 sizeof(uc->entries)); 1204 uc->entries = 0; 1205 mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN + 1206 sizeof(mc->entries)); 1207 mc->entries = 0; 1208 1209 for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) { 1210 struct virtio_net_ctrl_mac *tbl; 1211 1212 if (i == index || rte_is_zero_ether_addr(addrs + i)) 1213 continue; 1214 1215 tbl = rte_is_multicast_ether_addr(addrs + i) ? mc : uc; 1216 memcpy(&tbl->macs[tbl->entries++], addrs + i, 1217 RTE_ETHER_ADDR_LEN); 1218 } 1219 1220 virtio_mac_table_set(hw, uc, mc); 1221 } 1222 1223 static int 1224 virtio_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) 1225 { 1226 struct virtio_hw *hw = dev->data->dev_private; 1227 1228 memcpy(hw->mac_addr, mac_addr, RTE_ETHER_ADDR_LEN); 1229 1230 /* Use atomic update if available */ 1231 if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 1232 struct virtio_pmd_ctrl ctrl; 1233 int len = RTE_ETHER_ADDR_LEN; 1234 1235 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC; 1236 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET; 1237 1238 memcpy(ctrl.data, mac_addr, RTE_ETHER_ADDR_LEN); 1239 return virtio_send_command(hw->cvq, &ctrl, &len, 1); 1240 } 1241 1242 if (!vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) 1243 return -ENOTSUP; 1244 1245 virtio_set_hwaddr(hw); 1246 return 0; 1247 } 1248 1249 static int 1250 virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1251 { 1252 struct virtio_hw *hw = dev->data->dev_private; 1253 struct virtio_pmd_ctrl ctrl; 1254 int len; 1255 1256 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) 1257 return -ENOTSUP; 1258 1259 ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN; 1260 ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL; 1261 memcpy(ctrl.data, &vlan_id, sizeof(vlan_id)); 1262 len = sizeof(vlan_id); 1263 1264 return virtio_send_command(hw->cvq, &ctrl, &len, 1); 1265 } 1266 1267 static int 1268 virtio_intr_enable(struct rte_eth_dev *dev) 1269 { 1270 struct virtio_hw *hw = dev->data->dev_private; 1271 1272 if (rte_intr_enable(dev->intr_handle) < 0) 1273 return -1; 1274 1275 if (!hw->virtio_user_dev) 1276 hw->use_msix = vtpci_msix_detect(RTE_ETH_DEV_TO_PCI(dev)); 1277 1278 return 0; 1279 } 1280 1281 static int 1282 virtio_intr_disable(struct rte_eth_dev *dev) 1283 { 1284 struct virtio_hw *hw = dev->data->dev_private; 1285 1286 if (rte_intr_disable(dev->intr_handle) < 0) 1287 return -1; 1288 1289 if (!hw->virtio_user_dev) 1290 hw->use_msix = vtpci_msix_detect(RTE_ETH_DEV_TO_PCI(dev)); 1291 1292 return 0; 1293 } 1294 1295 static int 1296 virtio_negotiate_features(struct virtio_hw *hw, uint64_t req_features) 1297 { 1298 uint64_t host_features; 1299 1300 /* Prepare guest_features: feature that driver wants to support */ 1301 PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64, 1302 req_features); 1303 1304 /* Read device(host) feature bits */ 1305 host_features = VTPCI_OPS(hw)->get_features(hw); 1306 PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64, 1307 host_features); 1308 1309 /* If supported, ensure MTU value is valid before acknowledging it. */ 1310 if (host_features & req_features & (1ULL << VIRTIO_NET_F_MTU)) { 1311 struct virtio_net_config config; 1312 1313 vtpci_read_dev_config(hw, 1314 offsetof(struct virtio_net_config, mtu), 1315 &config.mtu, sizeof(config.mtu)); 1316 1317 if (config.mtu < RTE_ETHER_MIN_MTU) 1318 req_features &= ~(1ULL << VIRTIO_NET_F_MTU); 1319 } 1320 1321 /* 1322 * Negotiate features: Subset of device feature bits are written back 1323 * guest feature bits. 1324 */ 1325 hw->guest_features = req_features; 1326 hw->guest_features = vtpci_negotiate_features(hw, host_features); 1327 PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64, 1328 hw->guest_features); 1329 1330 if (hw->modern) { 1331 if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) { 1332 PMD_INIT_LOG(ERR, 1333 "VIRTIO_F_VERSION_1 features is not enabled."); 1334 return -1; 1335 } 1336 vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); 1337 if (!(vtpci_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) { 1338 PMD_INIT_LOG(ERR, 1339 "failed to set FEATURES_OK status!"); 1340 return -1; 1341 } 1342 } 1343 1344 hw->req_guest_features = req_features; 1345 1346 return 0; 1347 } 1348 1349 int 1350 virtio_dev_pause(struct rte_eth_dev *dev) 1351 { 1352 struct virtio_hw *hw = dev->data->dev_private; 1353 1354 rte_spinlock_lock(&hw->state_lock); 1355 1356 if (hw->started == 0) { 1357 /* Device is just stopped. */ 1358 rte_spinlock_unlock(&hw->state_lock); 1359 return -1; 1360 } 1361 hw->started = 0; 1362 /* 1363 * Prevent the worker threads from touching queues to avoid contention, 1364 * 1 ms should be enough for the ongoing Tx function to finish. 1365 */ 1366 rte_delay_ms(1); 1367 return 0; 1368 } 1369 1370 /* 1371 * Recover hw state to let the worker threads continue. 1372 */ 1373 void 1374 virtio_dev_resume(struct rte_eth_dev *dev) 1375 { 1376 struct virtio_hw *hw = dev->data->dev_private; 1377 1378 hw->started = 1; 1379 rte_spinlock_unlock(&hw->state_lock); 1380 } 1381 1382 /* 1383 * Should be called only after device is paused. 1384 */ 1385 int 1386 virtio_inject_pkts(struct rte_eth_dev *dev, struct rte_mbuf **tx_pkts, 1387 int nb_pkts) 1388 { 1389 struct virtio_hw *hw = dev->data->dev_private; 1390 struct virtnet_tx *txvq = dev->data->tx_queues[0]; 1391 int ret; 1392 1393 hw->inject_pkts = tx_pkts; 1394 ret = dev->tx_pkt_burst(txvq, tx_pkts, nb_pkts); 1395 hw->inject_pkts = NULL; 1396 1397 return ret; 1398 } 1399 1400 static void 1401 virtio_notify_peers(struct rte_eth_dev *dev) 1402 { 1403 struct virtio_hw *hw = dev->data->dev_private; 1404 struct virtnet_rx *rxvq; 1405 struct rte_mbuf *rarp_mbuf; 1406 1407 if (!dev->data->rx_queues) 1408 return; 1409 1410 rxvq = dev->data->rx_queues[0]; 1411 if (!rxvq) 1412 return; 1413 1414 rarp_mbuf = rte_net_make_rarp_packet(rxvq->mpool, 1415 (struct rte_ether_addr *)hw->mac_addr); 1416 if (rarp_mbuf == NULL) { 1417 PMD_DRV_LOG(ERR, "failed to make RARP packet."); 1418 return; 1419 } 1420 1421 /* If virtio port just stopped, no need to send RARP */ 1422 if (virtio_dev_pause(dev) < 0) { 1423 rte_pktmbuf_free(rarp_mbuf); 1424 return; 1425 } 1426 1427 virtio_inject_pkts(dev, &rarp_mbuf, 1); 1428 virtio_dev_resume(dev); 1429 } 1430 1431 static void 1432 virtio_ack_link_announce(struct rte_eth_dev *dev) 1433 { 1434 struct virtio_hw *hw = dev->data->dev_private; 1435 struct virtio_pmd_ctrl ctrl; 1436 1437 ctrl.hdr.class = VIRTIO_NET_CTRL_ANNOUNCE; 1438 ctrl.hdr.cmd = VIRTIO_NET_CTRL_ANNOUNCE_ACK; 1439 1440 virtio_send_command(hw->cvq, &ctrl, NULL, 0); 1441 } 1442 1443 /* 1444 * Process virtio config changed interrupt. Call the callback 1445 * if link state changed, generate gratuitous RARP packet if 1446 * the status indicates an ANNOUNCE. 1447 */ 1448 void 1449 virtio_interrupt_handler(void *param) 1450 { 1451 struct rte_eth_dev *dev = param; 1452 struct virtio_hw *hw = dev->data->dev_private; 1453 uint8_t isr; 1454 uint16_t status; 1455 1456 /* Read interrupt status which clears interrupt */ 1457 isr = vtpci_isr(hw); 1458 PMD_DRV_LOG(INFO, "interrupt status = %#x", isr); 1459 1460 if (virtio_intr_enable(dev) < 0) 1461 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1462 1463 if (isr & VIRTIO_PCI_ISR_CONFIG) { 1464 if (virtio_dev_link_update(dev, 0) == 0) 1465 _rte_eth_dev_callback_process(dev, 1466 RTE_ETH_EVENT_INTR_LSC, 1467 NULL); 1468 1469 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { 1470 vtpci_read_dev_config(hw, 1471 offsetof(struct virtio_net_config, status), 1472 &status, sizeof(status)); 1473 if (status & VIRTIO_NET_S_ANNOUNCE) { 1474 virtio_notify_peers(dev); 1475 if (hw->cvq) 1476 virtio_ack_link_announce(dev); 1477 } 1478 } 1479 } 1480 } 1481 1482 /* set rx and tx handlers according to what is supported */ 1483 static void 1484 set_rxtx_funcs(struct rte_eth_dev *eth_dev) 1485 { 1486 struct virtio_hw *hw = eth_dev->data->dev_private; 1487 1488 eth_dev->tx_pkt_prepare = virtio_xmit_pkts_prepare; 1489 if (vtpci_packed_queue(hw)) { 1490 PMD_INIT_LOG(INFO, 1491 "virtio: using packed ring %s Tx path on port %u", 1492 hw->use_inorder_tx ? "inorder" : "standard", 1493 eth_dev->data->port_id); 1494 eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed; 1495 } else { 1496 if (hw->use_inorder_tx) { 1497 PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on port %u", 1498 eth_dev->data->port_id); 1499 eth_dev->tx_pkt_burst = virtio_xmit_pkts_inorder; 1500 } else { 1501 PMD_INIT_LOG(INFO, "virtio: using standard Tx path on port %u", 1502 eth_dev->data->port_id); 1503 eth_dev->tx_pkt_burst = virtio_xmit_pkts; 1504 } 1505 } 1506 1507 if (vtpci_packed_queue(hw)) { 1508 if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 1509 PMD_INIT_LOG(INFO, 1510 "virtio: using packed ring mergeable buffer Rx path on port %u", 1511 eth_dev->data->port_id); 1512 eth_dev->rx_pkt_burst = 1513 &virtio_recv_mergeable_pkts_packed; 1514 } else { 1515 PMD_INIT_LOG(INFO, 1516 "virtio: using packed ring standard Rx path on port %u", 1517 eth_dev->data->port_id); 1518 eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed; 1519 } 1520 } else { 1521 if (hw->use_simple_rx) { 1522 PMD_INIT_LOG(INFO, "virtio: using simple Rx path on port %u", 1523 eth_dev->data->port_id); 1524 eth_dev->rx_pkt_burst = virtio_recv_pkts_vec; 1525 } else if (hw->use_inorder_rx) { 1526 PMD_INIT_LOG(INFO, 1527 "virtio: using inorder Rx path on port %u", 1528 eth_dev->data->port_id); 1529 eth_dev->rx_pkt_burst = &virtio_recv_pkts_inorder; 1530 } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 1531 PMD_INIT_LOG(INFO, 1532 "virtio: using mergeable buffer Rx path on port %u", 1533 eth_dev->data->port_id); 1534 eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts; 1535 } else { 1536 PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u", 1537 eth_dev->data->port_id); 1538 eth_dev->rx_pkt_burst = &virtio_recv_pkts; 1539 } 1540 } 1541 1542 } 1543 1544 /* Only support 1:1 queue/interrupt mapping so far. 1545 * TODO: support n:1 queue/interrupt mapping when there are limited number of 1546 * interrupt vectors (<N+1). 1547 */ 1548 static int 1549 virtio_queues_bind_intr(struct rte_eth_dev *dev) 1550 { 1551 uint32_t i; 1552 struct virtio_hw *hw = dev->data->dev_private; 1553 1554 PMD_INIT_LOG(INFO, "queue/interrupt binding"); 1555 for (i = 0; i < dev->data->nb_rx_queues; ++i) { 1556 dev->intr_handle->intr_vec[i] = i + 1; 1557 if (VTPCI_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) == 1558 VIRTIO_MSI_NO_VECTOR) { 1559 PMD_DRV_LOG(ERR, "failed to set queue vector"); 1560 return -EBUSY; 1561 } 1562 } 1563 1564 return 0; 1565 } 1566 1567 static void 1568 virtio_queues_unbind_intr(struct rte_eth_dev *dev) 1569 { 1570 uint32_t i; 1571 struct virtio_hw *hw = dev->data->dev_private; 1572 1573 PMD_INIT_LOG(INFO, "queue/interrupt unbinding"); 1574 for (i = 0; i < dev->data->nb_rx_queues; ++i) 1575 VTPCI_OPS(hw)->set_queue_irq(hw, 1576 hw->vqs[i * VTNET_CQ], 1577 VIRTIO_MSI_NO_VECTOR); 1578 } 1579 1580 static int 1581 virtio_configure_intr(struct rte_eth_dev *dev) 1582 { 1583 struct virtio_hw *hw = dev->data->dev_private; 1584 1585 if (!rte_intr_cap_multiple(dev->intr_handle)) { 1586 PMD_INIT_LOG(ERR, "Multiple intr vector not supported"); 1587 return -ENOTSUP; 1588 } 1589 1590 if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) { 1591 PMD_INIT_LOG(ERR, "Fail to create eventfd"); 1592 return -1; 1593 } 1594 1595 if (!dev->intr_handle->intr_vec) { 1596 dev->intr_handle->intr_vec = 1597 rte_zmalloc("intr_vec", 1598 hw->max_queue_pairs * sizeof(int), 0); 1599 if (!dev->intr_handle->intr_vec) { 1600 PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors", 1601 hw->max_queue_pairs); 1602 return -ENOMEM; 1603 } 1604 } 1605 1606 /* Re-register callback to update max_intr */ 1607 rte_intr_callback_unregister(dev->intr_handle, 1608 virtio_interrupt_handler, 1609 dev); 1610 rte_intr_callback_register(dev->intr_handle, 1611 virtio_interrupt_handler, 1612 dev); 1613 1614 /* DO NOT try to remove this! This function will enable msix, or QEMU 1615 * will encounter SIGSEGV when DRIVER_OK is sent. 1616 * And for legacy devices, this should be done before queue/vec binding 1617 * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR 1618 * (22) will be ignored. 1619 */ 1620 if (virtio_intr_enable(dev) < 0) { 1621 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1622 return -1; 1623 } 1624 1625 if (virtio_queues_bind_intr(dev) < 0) { 1626 PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt"); 1627 return -1; 1628 } 1629 1630 return 0; 1631 } 1632 1633 /* reset device and renegotiate features if needed */ 1634 static int 1635 virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) 1636 { 1637 struct virtio_hw *hw = eth_dev->data->dev_private; 1638 struct virtio_net_config *config; 1639 struct virtio_net_config local_config; 1640 struct rte_pci_device *pci_dev = NULL; 1641 int ret; 1642 1643 /* Reset the device although not necessary at startup */ 1644 vtpci_reset(hw); 1645 1646 if (hw->vqs) { 1647 virtio_dev_free_mbufs(eth_dev); 1648 virtio_free_queues(hw); 1649 } 1650 1651 /* Tell the host we've noticed this device. */ 1652 vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 1653 1654 /* Tell the host we've known how to drive the device. */ 1655 vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 1656 if (virtio_negotiate_features(hw, req_features) < 0) 1657 return -1; 1658 1659 hw->weak_barriers = !vtpci_with_feature(hw, VIRTIO_F_ORDER_PLATFORM); 1660 1661 if (!hw->virtio_user_dev) 1662 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1663 1664 /* If host does not support both status and MSI-X then disable LSC */ 1665 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS) && 1666 hw->use_msix != VIRTIO_MSIX_NONE) 1667 eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC; 1668 else 1669 eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC; 1670 1671 /* Setting up rx_header size for the device */ 1672 if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) || 1673 vtpci_with_feature(hw, VIRTIO_F_VERSION_1) || 1674 vtpci_with_feature(hw, VIRTIO_F_RING_PACKED)) 1675 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); 1676 else 1677 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr); 1678 1679 /* Copy the permanent MAC address to: virtio_hw */ 1680 virtio_get_hwaddr(hw); 1681 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr, 1682 ð_dev->data->mac_addrs[0]); 1683 PMD_INIT_LOG(DEBUG, 1684 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X", 1685 hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2], 1686 hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]); 1687 1688 if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) { 1689 config = &local_config; 1690 1691 vtpci_read_dev_config(hw, 1692 offsetof(struct virtio_net_config, mac), 1693 &config->mac, sizeof(config->mac)); 1694 1695 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { 1696 vtpci_read_dev_config(hw, 1697 offsetof(struct virtio_net_config, status), 1698 &config->status, sizeof(config->status)); 1699 } else { 1700 PMD_INIT_LOG(DEBUG, 1701 "VIRTIO_NET_F_STATUS is not supported"); 1702 config->status = 0; 1703 } 1704 1705 if (vtpci_with_feature(hw, VIRTIO_NET_F_MQ)) { 1706 vtpci_read_dev_config(hw, 1707 offsetof(struct virtio_net_config, max_virtqueue_pairs), 1708 &config->max_virtqueue_pairs, 1709 sizeof(config->max_virtqueue_pairs)); 1710 } else { 1711 PMD_INIT_LOG(DEBUG, 1712 "VIRTIO_NET_F_MQ is not supported"); 1713 config->max_virtqueue_pairs = 1; 1714 } 1715 1716 hw->max_queue_pairs = config->max_virtqueue_pairs; 1717 1718 if (vtpci_with_feature(hw, VIRTIO_NET_F_MTU)) { 1719 vtpci_read_dev_config(hw, 1720 offsetof(struct virtio_net_config, mtu), 1721 &config->mtu, 1722 sizeof(config->mtu)); 1723 1724 /* 1725 * MTU value has already been checked at negotiation 1726 * time, but check again in case it has changed since 1727 * then, which should not happen. 1728 */ 1729 if (config->mtu < RTE_ETHER_MIN_MTU) { 1730 PMD_INIT_LOG(ERR, "invalid max MTU value (%u)", 1731 config->mtu); 1732 return -1; 1733 } 1734 1735 hw->max_mtu = config->mtu; 1736 /* Set initial MTU to maximum one supported by vhost */ 1737 eth_dev->data->mtu = config->mtu; 1738 1739 } else { 1740 hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN - 1741 VLAN_TAG_LEN - hw->vtnet_hdr_size; 1742 } 1743 1744 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d", 1745 config->max_virtqueue_pairs); 1746 PMD_INIT_LOG(DEBUG, "config->status=%d", config->status); 1747 PMD_INIT_LOG(DEBUG, 1748 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X", 1749 config->mac[0], config->mac[1], 1750 config->mac[2], config->mac[3], 1751 config->mac[4], config->mac[5]); 1752 } else { 1753 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=1"); 1754 hw->max_queue_pairs = 1; 1755 hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN - 1756 VLAN_TAG_LEN - hw->vtnet_hdr_size; 1757 } 1758 1759 ret = virtio_alloc_queues(eth_dev); 1760 if (ret < 0) 1761 return ret; 1762 1763 if (eth_dev->data->dev_conf.intr_conf.rxq) { 1764 if (virtio_configure_intr(eth_dev) < 0) { 1765 PMD_INIT_LOG(ERR, "failed to configure interrupt"); 1766 virtio_free_queues(hw); 1767 return -1; 1768 } 1769 } 1770 1771 vtpci_reinit_complete(hw); 1772 1773 if (pci_dev) 1774 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x", 1775 eth_dev->data->port_id, pci_dev->id.vendor_id, 1776 pci_dev->id.device_id); 1777 1778 return 0; 1779 } 1780 1781 /* 1782 * Remap the PCI device again (IO port map for legacy device and 1783 * memory map for modern device), so that the secondary process 1784 * could have the PCI initiated correctly. 1785 */ 1786 static int 1787 virtio_remap_pci(struct rte_pci_device *pci_dev, struct virtio_hw *hw) 1788 { 1789 if (hw->modern) { 1790 /* 1791 * We don't have to re-parse the PCI config space, since 1792 * rte_pci_map_device() makes sure the mapped address 1793 * in secondary process would equal to the one mapped in 1794 * the primary process: error will be returned if that 1795 * requirement is not met. 1796 * 1797 * That said, we could simply reuse all cap pointers 1798 * (such as dev_cfg, common_cfg, etc.) parsed from the 1799 * primary process, which is stored in shared memory. 1800 */ 1801 if (rte_pci_map_device(pci_dev)) { 1802 PMD_INIT_LOG(DEBUG, "failed to map pci device!"); 1803 return -1; 1804 } 1805 } else { 1806 if (rte_pci_ioport_map(pci_dev, 0, VTPCI_IO(hw)) < 0) 1807 return -1; 1808 } 1809 1810 return 0; 1811 } 1812 1813 static void 1814 virtio_set_vtpci_ops(struct virtio_hw *hw) 1815 { 1816 #ifdef RTE_VIRTIO_USER 1817 if (hw->virtio_user_dev) 1818 VTPCI_OPS(hw) = &virtio_user_ops; 1819 else 1820 #endif 1821 if (hw->modern) 1822 VTPCI_OPS(hw) = &modern_ops; 1823 else 1824 VTPCI_OPS(hw) = &legacy_ops; 1825 } 1826 1827 /* 1828 * This function is based on probe() function in virtio_pci.c 1829 * It returns 0 on success. 1830 */ 1831 int 1832 eth_virtio_dev_init(struct rte_eth_dev *eth_dev) 1833 { 1834 struct virtio_hw *hw = eth_dev->data->dev_private; 1835 int ret; 1836 1837 RTE_BUILD_BUG_ON(RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr_mrg_rxbuf)); 1838 1839 eth_dev->dev_ops = &virtio_eth_dev_ops; 1840 1841 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1842 if (!hw->virtio_user_dev) { 1843 ret = virtio_remap_pci(RTE_ETH_DEV_TO_PCI(eth_dev), hw); 1844 if (ret) 1845 return ret; 1846 } 1847 1848 virtio_set_vtpci_ops(hw); 1849 set_rxtx_funcs(eth_dev); 1850 1851 return 0; 1852 } 1853 1854 /* 1855 * Pass the information to the rte_eth_dev_close() that it should also 1856 * release the private port resources. 1857 */ 1858 eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 1859 1860 /* Allocate memory for storing MAC addresses */ 1861 eth_dev->data->mac_addrs = rte_zmalloc("virtio", 1862 VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN, 0); 1863 if (eth_dev->data->mac_addrs == NULL) { 1864 PMD_INIT_LOG(ERR, 1865 "Failed to allocate %d bytes needed to store MAC addresses", 1866 VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN); 1867 return -ENOMEM; 1868 } 1869 1870 hw->port_id = eth_dev->data->port_id; 1871 /* For virtio_user case the hw->virtio_user_dev is populated by 1872 * virtio_user_eth_dev_alloc() before eth_virtio_dev_init() is called. 1873 */ 1874 if (!hw->virtio_user_dev) { 1875 ret = vtpci_init(RTE_ETH_DEV_TO_PCI(eth_dev), hw); 1876 if (ret) 1877 goto err_vtpci_init; 1878 } 1879 1880 /* reset device and negotiate default features */ 1881 ret = virtio_init_device(eth_dev, VIRTIO_PMD_DEFAULT_GUEST_FEATURES); 1882 if (ret < 0) 1883 goto err_virtio_init; 1884 1885 hw->opened = true; 1886 1887 return 0; 1888 1889 err_virtio_init: 1890 if (!hw->virtio_user_dev) { 1891 rte_pci_unmap_device(RTE_ETH_DEV_TO_PCI(eth_dev)); 1892 if (!hw->modern) 1893 rte_pci_ioport_unmap(VTPCI_IO(hw)); 1894 } 1895 err_vtpci_init: 1896 rte_free(eth_dev->data->mac_addrs); 1897 eth_dev->data->mac_addrs = NULL; 1898 return ret; 1899 } 1900 1901 static int 1902 eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev) 1903 { 1904 PMD_INIT_FUNC_TRACE(); 1905 1906 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 1907 return 0; 1908 1909 virtio_dev_stop(eth_dev); 1910 virtio_dev_close(eth_dev); 1911 1912 eth_dev->dev_ops = NULL; 1913 eth_dev->tx_pkt_burst = NULL; 1914 eth_dev->rx_pkt_burst = NULL; 1915 1916 PMD_INIT_LOG(DEBUG, "dev_uninit completed"); 1917 1918 return 0; 1919 } 1920 1921 static int vdpa_check_handler(__rte_unused const char *key, 1922 const char *value, __rte_unused void *opaque) 1923 { 1924 if (strcmp(value, "1")) 1925 return -1; 1926 1927 return 0; 1928 } 1929 1930 static int 1931 vdpa_mode_selected(struct rte_devargs *devargs) 1932 { 1933 struct rte_kvargs *kvlist; 1934 const char *key = "vdpa"; 1935 int ret = 0; 1936 1937 if (devargs == NULL) 1938 return 0; 1939 1940 kvlist = rte_kvargs_parse(devargs->args, NULL); 1941 if (kvlist == NULL) 1942 return 0; 1943 1944 if (!rte_kvargs_count(kvlist, key)) 1945 goto exit; 1946 1947 /* vdpa mode selected when there's a key-value pair: vdpa=1 */ 1948 if (rte_kvargs_process(kvlist, key, 1949 vdpa_check_handler, NULL) < 0) { 1950 goto exit; 1951 } 1952 ret = 1; 1953 1954 exit: 1955 rte_kvargs_free(kvlist); 1956 return ret; 1957 } 1958 1959 static int eth_virtio_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1960 struct rte_pci_device *pci_dev) 1961 { 1962 if (rte_eal_iopl_init() != 0) { 1963 PMD_INIT_LOG(ERR, "IOPL call failed - cannot use virtio PMD"); 1964 return 1; 1965 } 1966 1967 /* virtio pmd skips probe if device needs to work in vdpa mode */ 1968 if (vdpa_mode_selected(pci_dev->device.devargs)) 1969 return 1; 1970 1971 return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct virtio_hw), 1972 eth_virtio_dev_init); 1973 } 1974 1975 static int eth_virtio_pci_remove(struct rte_pci_device *pci_dev) 1976 { 1977 int ret; 1978 1979 ret = rte_eth_dev_pci_generic_remove(pci_dev, eth_virtio_dev_uninit); 1980 /* Port has already been released by close. */ 1981 if (ret == -ENODEV) 1982 ret = 0; 1983 return ret; 1984 } 1985 1986 static struct rte_pci_driver rte_virtio_pmd = { 1987 .driver = { 1988 .name = "net_virtio", 1989 }, 1990 .id_table = pci_id_virtio_map, 1991 .drv_flags = 0, 1992 .probe = eth_virtio_pci_probe, 1993 .remove = eth_virtio_pci_remove, 1994 }; 1995 1996 RTE_INIT(rte_virtio_pmd_init) 1997 { 1998 rte_eal_iopl_init(); 1999 rte_pci_register(&rte_virtio_pmd); 2000 } 2001 2002 static bool 2003 rx_offload_enabled(struct virtio_hw *hw) 2004 { 2005 return vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) || 2006 vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || 2007 vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6); 2008 } 2009 2010 static bool 2011 tx_offload_enabled(struct virtio_hw *hw) 2012 { 2013 return vtpci_with_feature(hw, VIRTIO_NET_F_CSUM) || 2014 vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) || 2015 vtpci_with_feature(hw, VIRTIO_NET_F_HOST_TSO6); 2016 } 2017 2018 /* 2019 * Configure virtio device 2020 * It returns 0 on success. 2021 */ 2022 static int 2023 virtio_dev_configure(struct rte_eth_dev *dev) 2024 { 2025 const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 2026 const struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode; 2027 struct virtio_hw *hw = dev->data->dev_private; 2028 uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN + 2029 hw->vtnet_hdr_size; 2030 uint64_t rx_offloads = rxmode->offloads; 2031 uint64_t tx_offloads = txmode->offloads; 2032 uint64_t req_features; 2033 int ret; 2034 2035 PMD_INIT_LOG(DEBUG, "configure"); 2036 req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES; 2037 2038 if (dev->data->dev_conf.intr_conf.rxq) { 2039 ret = virtio_init_device(dev, hw->req_guest_features); 2040 if (ret < 0) 2041 return ret; 2042 } 2043 2044 if (rxmode->max_rx_pkt_len > hw->max_mtu + ether_hdr_len) 2045 req_features &= ~(1ULL << VIRTIO_NET_F_MTU); 2046 2047 if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM | 2048 DEV_RX_OFFLOAD_TCP_CKSUM)) 2049 req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM); 2050 2051 if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) 2052 req_features |= 2053 (1ULL << VIRTIO_NET_F_GUEST_TSO4) | 2054 (1ULL << VIRTIO_NET_F_GUEST_TSO6); 2055 2056 if (tx_offloads & (DEV_TX_OFFLOAD_UDP_CKSUM | 2057 DEV_TX_OFFLOAD_TCP_CKSUM)) 2058 req_features |= (1ULL << VIRTIO_NET_F_CSUM); 2059 2060 if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) 2061 req_features |= 2062 (1ULL << VIRTIO_NET_F_HOST_TSO4) | 2063 (1ULL << VIRTIO_NET_F_HOST_TSO6); 2064 2065 /* if request features changed, reinit the device */ 2066 if (req_features != hw->req_guest_features) { 2067 ret = virtio_init_device(dev, req_features); 2068 if (ret < 0) 2069 return ret; 2070 } 2071 2072 if ((rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM | 2073 DEV_RX_OFFLOAD_TCP_CKSUM)) && 2074 !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) { 2075 PMD_DRV_LOG(ERR, 2076 "rx checksum not available on this host"); 2077 return -ENOTSUP; 2078 } 2079 2080 if ((rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) && 2081 (!vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || 2082 !vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6))) { 2083 PMD_DRV_LOG(ERR, 2084 "Large Receive Offload not available on this host"); 2085 return -ENOTSUP; 2086 } 2087 2088 /* start control queue */ 2089 if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) 2090 virtio_dev_cq_start(dev); 2091 2092 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2093 hw->vlan_strip = 1; 2094 2095 if ((rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER) 2096 && !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { 2097 PMD_DRV_LOG(ERR, 2098 "vlan filtering not available on this host"); 2099 return -ENOTSUP; 2100 } 2101 2102 hw->has_tx_offload = tx_offload_enabled(hw); 2103 hw->has_rx_offload = rx_offload_enabled(hw); 2104 2105 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) 2106 /* Enable vector (0) for Link State Intrerrupt */ 2107 if (VTPCI_OPS(hw)->set_config_irq(hw, 0) == 2108 VIRTIO_MSI_NO_VECTOR) { 2109 PMD_DRV_LOG(ERR, "failed to set config vector"); 2110 return -EBUSY; 2111 } 2112 2113 rte_spinlock_init(&hw->state_lock); 2114 2115 hw->use_simple_rx = 1; 2116 2117 if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) { 2118 hw->use_inorder_tx = 1; 2119 hw->use_inorder_rx = 1; 2120 hw->use_simple_rx = 0; 2121 } 2122 2123 if (vtpci_packed_queue(hw)) { 2124 hw->use_simple_rx = 0; 2125 hw->use_inorder_rx = 0; 2126 } 2127 2128 #if defined RTE_ARCH_ARM64 || defined RTE_ARCH_ARM 2129 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) { 2130 hw->use_simple_rx = 0; 2131 } 2132 #endif 2133 if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 2134 hw->use_simple_rx = 0; 2135 } 2136 2137 if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM | 2138 DEV_RX_OFFLOAD_TCP_CKSUM | 2139 DEV_RX_OFFLOAD_TCP_LRO | 2140 DEV_RX_OFFLOAD_VLAN_STRIP)) 2141 hw->use_simple_rx = 0; 2142 2143 return 0; 2144 } 2145 2146 2147 static int 2148 virtio_dev_start(struct rte_eth_dev *dev) 2149 { 2150 uint16_t nb_queues, i; 2151 struct virtnet_rx *rxvq; 2152 struct virtnet_tx *txvq __rte_unused; 2153 struct virtio_hw *hw = dev->data->dev_private; 2154 int ret; 2155 2156 /* Finish the initialization of the queues */ 2157 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2158 ret = virtio_dev_rx_queue_setup_finish(dev, i); 2159 if (ret < 0) 2160 return ret; 2161 } 2162 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2163 ret = virtio_dev_tx_queue_setup_finish(dev, i); 2164 if (ret < 0) 2165 return ret; 2166 } 2167 2168 /* check if lsc interrupt feature is enabled */ 2169 if (dev->data->dev_conf.intr_conf.lsc) { 2170 if (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) { 2171 PMD_DRV_LOG(ERR, "link status not supported by host"); 2172 return -ENOTSUP; 2173 } 2174 } 2175 2176 /* Enable uio/vfio intr/eventfd mapping: althrough we already did that 2177 * in device configure, but it could be unmapped when device is 2178 * stopped. 2179 */ 2180 if (dev->data->dev_conf.intr_conf.lsc || 2181 dev->data->dev_conf.intr_conf.rxq) { 2182 virtio_intr_disable(dev); 2183 2184 /* Setup interrupt callback */ 2185 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) 2186 rte_intr_callback_register(dev->intr_handle, 2187 virtio_interrupt_handler, 2188 dev); 2189 2190 if (virtio_intr_enable(dev) < 0) { 2191 PMD_DRV_LOG(ERR, "interrupt enable failed"); 2192 return -EIO; 2193 } 2194 } 2195 2196 /*Notify the backend 2197 *Otherwise the tap backend might already stop its queue due to fullness. 2198 *vhost backend will have no chance to be waked up 2199 */ 2200 nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues); 2201 if (hw->max_queue_pairs > 1) { 2202 if (virtio_set_multiple_queues(dev, nb_queues) != 0) 2203 return -EINVAL; 2204 } 2205 2206 PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues); 2207 2208 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2209 rxvq = dev->data->rx_queues[i]; 2210 /* Flush the old packets */ 2211 virtqueue_rxvq_flush(rxvq->vq); 2212 virtqueue_notify(rxvq->vq); 2213 } 2214 2215 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2216 txvq = dev->data->tx_queues[i]; 2217 virtqueue_notify(txvq->vq); 2218 } 2219 2220 PMD_INIT_LOG(DEBUG, "Notified backend at initialization"); 2221 2222 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2223 rxvq = dev->data->rx_queues[i]; 2224 VIRTQUEUE_DUMP(rxvq->vq); 2225 } 2226 2227 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2228 txvq = dev->data->tx_queues[i]; 2229 VIRTQUEUE_DUMP(txvq->vq); 2230 } 2231 2232 set_rxtx_funcs(dev); 2233 hw->started = true; 2234 2235 /* Initialize Link state */ 2236 virtio_dev_link_update(dev, 0); 2237 2238 return 0; 2239 } 2240 2241 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) 2242 { 2243 struct virtio_hw *hw = dev->data->dev_private; 2244 uint16_t nr_vq = virtio_get_nr_vq(hw); 2245 const char *type __rte_unused; 2246 unsigned int i, mbuf_num = 0; 2247 struct virtqueue *vq; 2248 struct rte_mbuf *buf; 2249 int queue_type; 2250 2251 if (hw->vqs == NULL) 2252 return; 2253 2254 for (i = 0; i < nr_vq; i++) { 2255 vq = hw->vqs[i]; 2256 if (!vq) 2257 continue; 2258 2259 queue_type = virtio_get_queue_type(hw, i); 2260 if (queue_type == VTNET_RQ) 2261 type = "rxq"; 2262 else if (queue_type == VTNET_TQ) 2263 type = "txq"; 2264 else 2265 continue; 2266 2267 PMD_INIT_LOG(DEBUG, 2268 "Before freeing %s[%d] used and unused buf", 2269 type, i); 2270 VIRTQUEUE_DUMP(vq); 2271 2272 while ((buf = virtqueue_detach_unused(vq)) != NULL) { 2273 rte_pktmbuf_free(buf); 2274 mbuf_num++; 2275 } 2276 2277 PMD_INIT_LOG(DEBUG, 2278 "After freeing %s[%d] used and unused buf", 2279 type, i); 2280 VIRTQUEUE_DUMP(vq); 2281 } 2282 2283 PMD_INIT_LOG(DEBUG, "%d mbufs freed", mbuf_num); 2284 } 2285 2286 /* 2287 * Stop device: disable interrupt and mark link down 2288 */ 2289 static void 2290 virtio_dev_stop(struct rte_eth_dev *dev) 2291 { 2292 struct virtio_hw *hw = dev->data->dev_private; 2293 struct rte_eth_link link; 2294 struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf; 2295 2296 PMD_INIT_LOG(DEBUG, "stop"); 2297 2298 rte_spinlock_lock(&hw->state_lock); 2299 if (!hw->started) 2300 goto out_unlock; 2301 hw->started = false; 2302 2303 if (intr_conf->lsc || intr_conf->rxq) { 2304 virtio_intr_disable(dev); 2305 2306 /* Reset interrupt callback */ 2307 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { 2308 rte_intr_callback_unregister(dev->intr_handle, 2309 virtio_interrupt_handler, 2310 dev); 2311 } 2312 } 2313 2314 memset(&link, 0, sizeof(link)); 2315 rte_eth_linkstatus_set(dev, &link); 2316 out_unlock: 2317 rte_spinlock_unlock(&hw->state_lock); 2318 } 2319 2320 static int 2321 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) 2322 { 2323 struct rte_eth_link link; 2324 uint16_t status; 2325 struct virtio_hw *hw = dev->data->dev_private; 2326 2327 memset(&link, 0, sizeof(link)); 2328 link.link_duplex = ETH_LINK_FULL_DUPLEX; 2329 link.link_speed = ETH_SPEED_NUM_10G; 2330 link.link_autoneg = ETH_LINK_FIXED; 2331 2332 if (!hw->started) { 2333 link.link_status = ETH_LINK_DOWN; 2334 } else if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { 2335 PMD_INIT_LOG(DEBUG, "Get link status from hw"); 2336 vtpci_read_dev_config(hw, 2337 offsetof(struct virtio_net_config, status), 2338 &status, sizeof(status)); 2339 if ((status & VIRTIO_NET_S_LINK_UP) == 0) { 2340 link.link_status = ETH_LINK_DOWN; 2341 PMD_INIT_LOG(DEBUG, "Port %d is down", 2342 dev->data->port_id); 2343 } else { 2344 link.link_status = ETH_LINK_UP; 2345 PMD_INIT_LOG(DEBUG, "Port %d is up", 2346 dev->data->port_id); 2347 } 2348 } else { 2349 link.link_status = ETH_LINK_UP; 2350 } 2351 2352 return rte_eth_linkstatus_set(dev, &link); 2353 } 2354 2355 static int 2356 virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2357 { 2358 const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 2359 struct virtio_hw *hw = dev->data->dev_private; 2360 uint64_t offloads = rxmode->offloads; 2361 2362 if (mask & ETH_VLAN_FILTER_MASK) { 2363 if ((offloads & DEV_RX_OFFLOAD_VLAN_FILTER) && 2364 !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { 2365 2366 PMD_DRV_LOG(NOTICE, 2367 "vlan filtering not available on this host"); 2368 2369 return -ENOTSUP; 2370 } 2371 } 2372 2373 if (mask & ETH_VLAN_STRIP_MASK) 2374 hw->vlan_strip = !!(offloads & DEV_RX_OFFLOAD_VLAN_STRIP); 2375 2376 return 0; 2377 } 2378 2379 static void 2380 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2381 { 2382 uint64_t tso_mask, host_features; 2383 struct virtio_hw *hw = dev->data->dev_private; 2384 2385 dev_info->speed_capa = ETH_LINK_SPEED_10G; /* fake value */ 2386 2387 dev_info->max_rx_queues = 2388 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES); 2389 dev_info->max_tx_queues = 2390 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_TX_QUEUES); 2391 dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE; 2392 dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN; 2393 dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS; 2394 2395 host_features = VTPCI_OPS(hw)->get_features(hw); 2396 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; 2397 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME; 2398 if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) { 2399 dev_info->rx_offload_capa |= 2400 DEV_RX_OFFLOAD_TCP_CKSUM | 2401 DEV_RX_OFFLOAD_UDP_CKSUM; 2402 } 2403 if (host_features & (1ULL << VIRTIO_NET_F_CTRL_VLAN)) 2404 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_VLAN_FILTER; 2405 tso_mask = (1ULL << VIRTIO_NET_F_GUEST_TSO4) | 2406 (1ULL << VIRTIO_NET_F_GUEST_TSO6); 2407 if ((host_features & tso_mask) == tso_mask) 2408 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO; 2409 2410 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS | 2411 DEV_TX_OFFLOAD_VLAN_INSERT; 2412 if (host_features & (1ULL << VIRTIO_NET_F_CSUM)) { 2413 dev_info->tx_offload_capa |= 2414 DEV_TX_OFFLOAD_UDP_CKSUM | 2415 DEV_TX_OFFLOAD_TCP_CKSUM; 2416 } 2417 tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) | 2418 (1ULL << VIRTIO_NET_F_HOST_TSO6); 2419 if ((host_features & tso_mask) == tso_mask) 2420 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 2421 } 2422 2423 /* 2424 * It enables testpmd to collect per queue stats. 2425 */ 2426 static int 2427 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev, 2428 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx, 2429 __rte_unused uint8_t is_rx) 2430 { 2431 return 0; 2432 } 2433 2434 RTE_PMD_EXPORT_NAME(net_virtio, __COUNTER__); 2435 RTE_PMD_REGISTER_PCI_TABLE(net_virtio, pci_id_virtio_map); 2436 RTE_PMD_REGISTER_KMOD_DEP(net_virtio, "* igb_uio | uio_pci_generic | vfio-pci"); 2437 2438 RTE_INIT(virtio_init_log) 2439 { 2440 virtio_logtype_init = rte_log_register("pmd.net.virtio.init"); 2441 if (virtio_logtype_init >= 0) 2442 rte_log_set_level(virtio_logtype_init, RTE_LOG_NOTICE); 2443 virtio_logtype_driver = rte_log_register("pmd.net.virtio.driver"); 2444 if (virtio_logtype_driver >= 0) 2445 rte_log_set_level(virtio_logtype_driver, RTE_LOG_NOTICE); 2446 } 2447