15566a3e3SBruce Richardson /* SPDX-License-Identifier: BSD-3-Clause 25566a3e3SBruce Richardson * Copyright(c) 2010-2016 Intel Corporation 3e9efa4d9SJianfeng Tan */ 4e9efa4d9SJianfeng Tan 5e9efa4d9SJianfeng Tan #include <stdint.h> 672b452c5SDmitry Kozlyuk #include <stdlib.h> 7e9efa4d9SJianfeng Tan #include <sys/types.h> 8e9efa4d9SJianfeng Tan #include <unistd.h> 9ef53b603SJianfeng Tan #include <fcntl.h> 107d62bf6fSMaxime Coquelin #include <linux/major.h> 11f908b22eSAdrian Moreno #include <sys/stat.h> 127d62bf6fSMaxime Coquelin #include <sys/sysmacros.h> 13ef53b603SJianfeng Tan #include <sys/socket.h> 14e9efa4d9SJianfeng Tan 15ce2eabddSJianfeng Tan #include <rte_malloc.h> 16ce2eabddSJianfeng Tan #include <rte_kvargs.h> 17df96fd0dSBruce Richardson #include <ethdev_vdev.h> 184851ef2bSDavid Marchand #include <bus_vdev_driver.h> 19ef53b603SJianfeng Tan #include <rte_alarm.h> 206ebbf410SXuan Ding #include <rte_cycles.h> 210fd27826SSrujana Challa #include <rte_io.h> 22ce2eabddSJianfeng Tan 23ce2eabddSJianfeng Tan #include "virtio_ethdev.h" 24e9efa4d9SJianfeng Tan #include "virtio_logs.h" 25f8b60756SMaxime Coquelin #include "virtio.h" 26e9efa4d9SJianfeng Tan #include "virtqueue.h" 271b69528eSJianfeng Tan #include "virtio_rxtx.h" 28e9efa4d9SJianfeng Tan #include "virtio_user/virtio_user_dev.h" 297f468b2eSTiwei Bie #include "virtio_user/vhost.h" 30e9efa4d9SJianfeng Tan 3165832c82SMaxime Coquelin #define virtio_user_get_dev(hwp) container_of(hwp, struct virtio_user_dev, hw) 32e9efa4d9SJianfeng Tan 336ebbf410SXuan Ding static void 34e9efa4d9SJianfeng Tan virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset, 35e9efa4d9SJianfeng Tan void *dst, int length) 36e9efa4d9SJianfeng Tan { 37e9efa4d9SJianfeng Tan int i; 38e9efa4d9SJianfeng Tan struct virtio_user_dev *dev = virtio_user_get_dev(hw); 39e9efa4d9SJianfeng Tan 40e9efa4d9SJianfeng Tan if (offset == offsetof(struct virtio_net_config, mac) && 4135b2d13fSOlivier Matz length == RTE_ETHER_ADDR_LEN) { 4235b2d13fSOlivier Matz for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) 43e9efa4d9SJianfeng Tan ((uint8_t *)dst)[i] = dev->mac_addr[i]; 44e9efa4d9SJianfeng Tan return; 45e9efa4d9SJianfeng Tan } 46e9efa4d9SJianfeng Tan 47ef53b603SJianfeng Tan if (offset == offsetof(struct virtio_net_config, status)) { 4894973531SMaxime Coquelin virtio_user_dev_update_link_state(dev); 49bd8f50a4SZhiyong Yang 50d0131e49SXiao Wang *(uint16_t *)dst = dev->net_status; 51ef53b603SJianfeng Tan } 52e9efa4d9SJianfeng Tan 53e9efa4d9SJianfeng Tan if (offset == offsetof(struct virtio_net_config, max_virtqueue_pairs)) 54e9efa4d9SJianfeng Tan *(uint16_t *)dst = dev->max_queue_pairs; 55dbce501eSSrujana Challa 56dbce501eSSrujana Challa if (offset >= offsetof(struct virtio_net_config, rss_max_key_size)) 57dbce501eSSrujana Challa virtio_user_dev_get_rss_config(dev, dst, offset, length); 58e9efa4d9SJianfeng Tan } 59e9efa4d9SJianfeng Tan 60e9efa4d9SJianfeng Tan static void 61e9efa4d9SJianfeng Tan virtio_user_write_dev_config(struct virtio_hw *hw, size_t offset, 62e9efa4d9SJianfeng Tan const void *src, int length) 63e9efa4d9SJianfeng Tan { 64e9efa4d9SJianfeng Tan int i; 65e9efa4d9SJianfeng Tan struct virtio_user_dev *dev = virtio_user_get_dev(hw); 66e9efa4d9SJianfeng Tan 67e9efa4d9SJianfeng Tan if ((offset == offsetof(struct virtio_net_config, mac)) && 68c995b005SMaxime Coquelin (length == RTE_ETHER_ADDR_LEN)) { 6935b2d13fSOlivier Matz for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) 70e9efa4d9SJianfeng Tan dev->mac_addr[i] = ((const uint8_t *)src)[i]; 71c995b005SMaxime Coquelin virtio_user_dev_set_mac(dev); 72c995b005SMaxime Coquelin virtio_user_dev_get_mac(dev); 73c995b005SMaxime Coquelin } else { 74f2462150SFerruh Yigit PMD_DRV_LOG(ERR, "not supported offset=%zu, len=%d", 75e9efa4d9SJianfeng Tan offset, length); 76e9efa4d9SJianfeng Tan } 77c995b005SMaxime Coquelin } 78e9efa4d9SJianfeng Tan 79e9efa4d9SJianfeng Tan static void 80c12a26eeSJianfeng Tan virtio_user_reset(struct virtio_hw *hw) 81c12a26eeSJianfeng Tan { 82c12a26eeSJianfeng Tan struct virtio_user_dev *dev = virtio_user_get_dev(hw); 83c12a26eeSJianfeng Tan 84c12a26eeSJianfeng Tan if (dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK) 85c12a26eeSJianfeng Tan virtio_user_stop_device(dev); 86c12a26eeSJianfeng Tan } 87c12a26eeSJianfeng Tan 88c12a26eeSJianfeng Tan static void 89e9efa4d9SJianfeng Tan virtio_user_set_status(struct virtio_hw *hw, uint8_t status) 90e9efa4d9SJianfeng Tan { 91e9efa4d9SJianfeng Tan struct virtio_user_dev *dev = virtio_user_get_dev(hw); 92844e4683SMaxime Coquelin uint8_t old_status = dev->status; 93e9efa4d9SJianfeng Tan 94844e4683SMaxime Coquelin if (status & VIRTIO_CONFIG_STATUS_FEATURES_OK && 95844e4683SMaxime Coquelin ~old_status & VIRTIO_CONFIG_STATUS_FEATURES_OK) 96844e4683SMaxime Coquelin virtio_user_dev_set_features(dev); 97c1a6f787SMaxime Coquelin 98c1a6f787SMaxime Coquelin if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK) { 99c1a6f787SMaxime Coquelin if (virtio_user_start_device(dev)) { 100c1a6f787SMaxime Coquelin virtio_user_dev_update_status(dev); 101c1a6f787SMaxime Coquelin return; 102c1a6f787SMaxime Coquelin } 103c1a6f787SMaxime Coquelin } else if (status == VIRTIO_CONFIG_STATUS_RESET) { 104c12a26eeSJianfeng Tan virtio_user_reset(hw); 105c1a6f787SMaxime Coquelin } 106d7e10ea9SAdrian Moreno 107d7e10ea9SAdrian Moreno virtio_user_dev_set_status(dev, status); 108e9efa4d9SJianfeng Tan } 109e9efa4d9SJianfeng Tan 110e9efa4d9SJianfeng Tan static uint8_t 111e9efa4d9SJianfeng Tan virtio_user_get_status(struct virtio_hw *hw) 112e9efa4d9SJianfeng Tan { 113e9efa4d9SJianfeng Tan struct virtio_user_dev *dev = virtio_user_get_dev(hw); 114e9efa4d9SJianfeng Tan 115d7e10ea9SAdrian Moreno virtio_user_dev_update_status(dev); 1160b0dc66cSAdrian Moreno 117e9efa4d9SJianfeng Tan return dev->status; 118e9efa4d9SJianfeng Tan } 119e9efa4d9SJianfeng Tan 120e9efa4d9SJianfeng Tan static uint64_t 121e9efa4d9SJianfeng Tan virtio_user_get_features(struct virtio_hw *hw) 122e9efa4d9SJianfeng Tan { 123e9efa4d9SJianfeng Tan struct virtio_user_dev *dev = virtio_user_get_dev(hw); 124e9efa4d9SJianfeng Tan 125142678d4SJianfeng Tan /* unmask feature bits defined in vhost user protocol */ 126f078c2f0SMaxime Coquelin return (dev->device_features | dev->frontend_features) & 127f078c2f0SMaxime Coquelin VIRTIO_PMD_SUPPORTED_GUEST_FEATURES; 128e9efa4d9SJianfeng Tan } 129e9efa4d9SJianfeng Tan 130e9efa4d9SJianfeng Tan static void 131e9efa4d9SJianfeng Tan virtio_user_set_features(struct virtio_hw *hw, uint64_t features) 132e9efa4d9SJianfeng Tan { 133e9efa4d9SJianfeng Tan struct virtio_user_dev *dev = virtio_user_get_dev(hw); 134e9efa4d9SJianfeng Tan 135f078c2f0SMaxime Coquelin dev->features = features & (dev->device_features | dev->frontend_features); 136e9efa4d9SJianfeng Tan } 137e9efa4d9SJianfeng Tan 138cbb135b3SMaxime Coquelin static int 139cbb135b3SMaxime Coquelin virtio_user_features_ok(struct virtio_hw *hw __rte_unused) 140cbb135b3SMaxime Coquelin { 141cbb135b3SMaxime Coquelin return 0; 142cbb135b3SMaxime Coquelin } 143cbb135b3SMaxime Coquelin 144e9efa4d9SJianfeng Tan static uint8_t 145e9efa4d9SJianfeng Tan virtio_user_get_isr(struct virtio_hw *hw __rte_unused) 146e9efa4d9SJianfeng Tan { 14735c4f855SJianfeng Tan /* rxq interrupts and config interrupt are separated in virtio-user, 14835c4f855SJianfeng Tan * here we only report config change. 149e9efa4d9SJianfeng Tan */ 1506a504290SMaxime Coquelin return VIRTIO_ISR_CONFIG; 151e9efa4d9SJianfeng Tan } 152e9efa4d9SJianfeng Tan 153e9efa4d9SJianfeng Tan static uint16_t 154e9efa4d9SJianfeng Tan virtio_user_set_config_irq(struct virtio_hw *hw __rte_unused, 155e9efa4d9SJianfeng Tan uint16_t vec __rte_unused) 156e9efa4d9SJianfeng Tan { 15735c4f855SJianfeng Tan return 0; 158e9efa4d9SJianfeng Tan } 159e9efa4d9SJianfeng Tan 1603d4fb6fdSJianfeng Tan static uint16_t 1613d4fb6fdSJianfeng Tan virtio_user_set_queue_irq(struct virtio_hw *hw __rte_unused, 1623d4fb6fdSJianfeng Tan struct virtqueue *vq __rte_unused, 1633d4fb6fdSJianfeng Tan uint16_t vec) 1643d4fb6fdSJianfeng Tan { 1653d4fb6fdSJianfeng Tan /* pretend we have done that */ 1663d4fb6fdSJianfeng Tan return vec; 1673d4fb6fdSJianfeng Tan } 1683d4fb6fdSJianfeng Tan 169e9efa4d9SJianfeng Tan /* This function is to get the queue size, aka, number of descs, of a specified 170e9efa4d9SJianfeng Tan * queue. Different with the VHOST_USER_GET_QUEUE_NUM, which is used to get the 171e9efa4d9SJianfeng Tan * max supported queues. 172e9efa4d9SJianfeng Tan */ 173e9efa4d9SJianfeng Tan static uint16_t 174e9efa4d9SJianfeng Tan virtio_user_get_queue_num(struct virtio_hw *hw, uint16_t queue_id __rte_unused) 175e9efa4d9SJianfeng Tan { 176e9efa4d9SJianfeng Tan struct virtio_user_dev *dev = virtio_user_get_dev(hw); 177e9efa4d9SJianfeng Tan 178e9efa4d9SJianfeng Tan /* Currently, each queue has same queue size */ 179e9efa4d9SJianfeng Tan return dev->queue_size; 180e9efa4d9SJianfeng Tan } 181e9efa4d9SJianfeng Tan 18248a44640SJens Freimann static void 18348a44640SJens Freimann virtio_user_setup_queue_packed(struct virtqueue *vq, 18448a44640SJens Freimann struct virtio_user_dev *dev) 185e9efa4d9SJianfeng Tan { 18648a44640SJens Freimann uint16_t queue_idx = vq->vq_queue_index; 18748a44640SJens Freimann struct vring_packed *vring; 18848a44640SJens Freimann uint64_t desc_addr; 18948a44640SJens Freimann uint64_t avail_addr; 19048a44640SJens Freimann uint64_t used_addr; 19148a44640SJens Freimann uint16_t i; 19248a44640SJens Freimann 1936fdf32d1SMaxime Coquelin vring = &dev->vrings.packed[queue_idx]; 19448a44640SJens Freimann desc_addr = (uintptr_t)vq->vq_ring_virt_mem; 19548a44640SJens Freimann avail_addr = desc_addr + vq->vq_nentries * 19648a44640SJens Freimann sizeof(struct vring_packed_desc); 19748a44640SJens Freimann used_addr = RTE_ALIGN_CEIL(avail_addr + 19848a44640SJens Freimann sizeof(struct vring_packed_desc_event), 199df968842SMaxime Coquelin VIRTIO_VRING_ALIGN); 20048a44640SJens Freimann vring->num = vq->vq_nentries; 201666ef294SSrujana Challa vring->desc_iova = vq->vq_ring_mem; 2024cdc4d98STiwei Bie vring->desc = (void *)(uintptr_t)desc_addr; 2034cdc4d98STiwei Bie vring->driver = (void *)(uintptr_t)avail_addr; 2044cdc4d98STiwei Bie vring->device = (void *)(uintptr_t)used_addr; 20548a44640SJens Freimann dev->packed_queues[queue_idx].avail_wrap_counter = true; 20648a44640SJens Freimann dev->packed_queues[queue_idx].used_wrap_counter = true; 207*ff11fc60SKommula Shiva Shankar dev->packed_queues[queue_idx].used_idx = 0; 20848a44640SJens Freimann 20945c224e7STiwei Bie for (i = 0; i < vring->num; i++) 2104cdc4d98STiwei Bie vring->desc[i].flags = 0; 21148a44640SJens Freimann } 21248a44640SJens Freimann 21348a44640SJens Freimann static void 21448a44640SJens Freimann virtio_user_setup_queue_split(struct virtqueue *vq, struct virtio_user_dev *dev) 21548a44640SJens Freimann { 216e9efa4d9SJianfeng Tan uint16_t queue_idx = vq->vq_queue_index; 217e9efa4d9SJianfeng Tan uint64_t desc_addr, avail_addr, used_addr; 218e9efa4d9SJianfeng Tan 219e9efa4d9SJianfeng Tan desc_addr = (uintptr_t)vq->vq_ring_virt_mem; 220e9efa4d9SJianfeng Tan avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc); 221e9efa4d9SJianfeng Tan used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail, 222e9efa4d9SJianfeng Tan ring[vq->vq_nentries]), 223df968842SMaxime Coquelin VIRTIO_VRING_ALIGN); 224e9efa4d9SJianfeng Tan 2256fdf32d1SMaxime Coquelin dev->vrings.split[queue_idx].num = vq->vq_nentries; 226666ef294SSrujana Challa dev->vrings.split[queue_idx].desc_iova = vq->vq_ring_mem; 2276fdf32d1SMaxime Coquelin dev->vrings.split[queue_idx].desc = (void *)(uintptr_t)desc_addr; 2286fdf32d1SMaxime Coquelin dev->vrings.split[queue_idx].avail = (void *)(uintptr_t)avail_addr; 2296fdf32d1SMaxime Coquelin dev->vrings.split[queue_idx].used = (void *)(uintptr_t)used_addr; 23048a44640SJens Freimann } 23148a44640SJens Freimann 23248a44640SJens Freimann static int 23348a44640SJens Freimann virtio_user_setup_queue(struct virtio_hw *hw, struct virtqueue *vq) 23448a44640SJens Freimann { 23548a44640SJens Freimann struct virtio_user_dev *dev = virtio_user_get_dev(hw); 23648a44640SJens Freimann 237b4f9a45aSMaxime Coquelin if (virtio_with_packed_queue(hw)) 23848a44640SJens Freimann virtio_user_setup_queue_packed(vq, dev); 23948a44640SJens Freimann else 24048a44640SJens Freimann virtio_user_setup_queue_split(vq, dev); 241e9efa4d9SJianfeng Tan 2420fd27826SSrujana Challa if (dev->notify_area) 2430fd27826SSrujana Challa vq->notify_addr = dev->notify_area[vq->vq_queue_index]; 2440fd27826SSrujana Challa 24566b45ceaSMaxime Coquelin if (dev->hw_cvq && hw->cvq && (virtnet_cq_to_vq(hw->cvq) == vq)) 24666b45ceaSMaxime Coquelin return virtio_user_dev_create_shadow_cvq(dev, vq); 24766b45ceaSMaxime Coquelin 248e9efa4d9SJianfeng Tan return 0; 249e9efa4d9SJianfeng Tan } 250e9efa4d9SJianfeng Tan 251e9efa4d9SJianfeng Tan static void 252e9efa4d9SJianfeng Tan virtio_user_del_queue(struct virtio_hw *hw, struct virtqueue *vq) 253e9efa4d9SJianfeng Tan { 254e9efa4d9SJianfeng Tan /* For legacy devices, write 0 to VIRTIO_PCI_QUEUE_PFN port, QEMU 255e9efa4d9SJianfeng Tan * correspondingly stops the ioeventfds, and reset the status of 256e9efa4d9SJianfeng Tan * the device. 257e9efa4d9SJianfeng Tan * For modern devices, set queue desc, avail, used in PCI bar to 0, 258e9efa4d9SJianfeng Tan * not see any more behavior in QEMU. 259e9efa4d9SJianfeng Tan * 260e9efa4d9SJianfeng Tan * Here we just care about what information to deliver to vhost-user 261e9efa4d9SJianfeng Tan * or vhost-kernel. So we just close ioeventfd for now. 262e9efa4d9SJianfeng Tan */ 263e9efa4d9SJianfeng Tan struct virtio_user_dev *dev = virtio_user_get_dev(hw); 264e9efa4d9SJianfeng Tan 265e9efa4d9SJianfeng Tan close(dev->callfds[vq->vq_queue_index]); 266e9efa4d9SJianfeng Tan close(dev->kickfds[vq->vq_queue_index]); 26766b45ceaSMaxime Coquelin 26866b45ceaSMaxime Coquelin if (hw->cvq && (virtnet_cq_to_vq(hw->cvq) == vq)) 26966b45ceaSMaxime Coquelin virtio_user_dev_destroy_shadow_cvq(dev); 270e9efa4d9SJianfeng Tan } 271e9efa4d9SJianfeng Tan 272e9efa4d9SJianfeng Tan static void 273e9efa4d9SJianfeng Tan virtio_user_notify_queue(struct virtio_hw *hw, struct virtqueue *vq) 274e9efa4d9SJianfeng Tan { 275e9efa4d9SJianfeng Tan struct virtio_user_dev *dev = virtio_user_get_dev(hw); 2760fd27826SSrujana Challa uint64_t notify_data = 1; 277e9efa4d9SJianfeng Tan 2783169550fSMaxime Coquelin if (hw->cvq && (virtnet_cq_to_vq(hw->cvq) == vq)) { 2791b69528eSJianfeng Tan virtio_user_handle_cq(dev, vq->vq_queue_index); 280fcdb603aSMaxime Coquelin 2811b69528eSJianfeng Tan return; 2821b69528eSJianfeng Tan } 2831b69528eSJianfeng Tan 2840fd27826SSrujana Challa if (!dev->notify_area) { 2850fd27826SSrujana Challa if (write(dev->kickfds[vq->vq_queue_index], ¬ify_data, 2860fd27826SSrujana Challa sizeof(notify_data)) < 0) 287f2462150SFerruh Yigit PMD_DRV_LOG(ERR, "failed to kick backend: %s", 288e9efa4d9SJianfeng Tan strerror(errno)); 2890fd27826SSrujana Challa return; 2900fd27826SSrujana Challa } else if (!virtio_with_feature(hw, VIRTIO_F_NOTIFICATION_DATA)) { 2910fd27826SSrujana Challa rte_write16(vq->vq_queue_index, vq->notify_addr); 2920fd27826SSrujana Challa return; 2930fd27826SSrujana Challa } 2940fd27826SSrujana Challa 2950fd27826SSrujana Challa if (virtio_with_packed_queue(hw)) { 2960fd27826SSrujana Challa /* Bit[0:15]: vq queue index 2970fd27826SSrujana Challa * Bit[16:30]: avail index 2980fd27826SSrujana Challa * Bit[31]: avail wrap counter 2990fd27826SSrujana Challa */ 3000fd27826SSrujana Challa notify_data = ((uint32_t)(!!(vq->vq_packed.cached_flags & 3010fd27826SSrujana Challa VRING_PACKED_DESC_F_AVAIL)) << 31) | 3020fd27826SSrujana Challa ((uint32_t)vq->vq_avail_idx << 16) | 3030fd27826SSrujana Challa vq->vq_queue_index; 3040fd27826SSrujana Challa } else { 3050fd27826SSrujana Challa /* Bit[0:15]: vq queue index 3060fd27826SSrujana Challa * Bit[16:31]: avail index 3070fd27826SSrujana Challa */ 3080fd27826SSrujana Challa notify_data = ((uint32_t)vq->vq_avail_idx << 16) | 3090fd27826SSrujana Challa vq->vq_queue_index; 3100fd27826SSrujana Challa } 3110fd27826SSrujana Challa rte_write32(notify_data, vq->notify_addr); 312e9efa4d9SJianfeng Tan } 313e9efa4d9SJianfeng Tan 314f12908c8SMaxime Coquelin static int 315f12908c8SMaxime Coquelin virtio_user_dev_close(struct virtio_hw *hw) 316f12908c8SMaxime Coquelin { 317f12908c8SMaxime Coquelin struct virtio_user_dev *dev = virtio_user_get_dev(hw); 318f12908c8SMaxime Coquelin 319f12908c8SMaxime Coquelin virtio_user_dev_uninit(dev); 320f12908c8SMaxime Coquelin 321f12908c8SMaxime Coquelin return 0; 322f12908c8SMaxime Coquelin } 323f12908c8SMaxime Coquelin 324f8b60756SMaxime Coquelin const struct virtio_ops virtio_user_ops = { 325e9efa4d9SJianfeng Tan .read_dev_cfg = virtio_user_read_dev_config, 326e9efa4d9SJianfeng Tan .write_dev_cfg = virtio_user_write_dev_config, 327e9efa4d9SJianfeng Tan .get_status = virtio_user_get_status, 328e9efa4d9SJianfeng Tan .set_status = virtio_user_set_status, 329e9efa4d9SJianfeng Tan .get_features = virtio_user_get_features, 330e9efa4d9SJianfeng Tan .set_features = virtio_user_set_features, 331cbb135b3SMaxime Coquelin .features_ok = virtio_user_features_ok, 332e9efa4d9SJianfeng Tan .get_isr = virtio_user_get_isr, 333e9efa4d9SJianfeng Tan .set_config_irq = virtio_user_set_config_irq, 3343d4fb6fdSJianfeng Tan .set_queue_irq = virtio_user_set_queue_irq, 335e9efa4d9SJianfeng Tan .get_queue_num = virtio_user_get_queue_num, 336e9efa4d9SJianfeng Tan .setup_queue = virtio_user_setup_queue, 337e9efa4d9SJianfeng Tan .del_queue = virtio_user_del_queue, 338e9efa4d9SJianfeng Tan .notify_queue = virtio_user_notify_queue, 339f12908c8SMaxime Coquelin .dev_close = virtio_user_dev_close, 340e9efa4d9SJianfeng Tan }; 341ce2eabddSJianfeng Tan 342ce2eabddSJianfeng Tan static const char *valid_args[] = { 343ce2eabddSJianfeng Tan #define VIRTIO_USER_ARG_QUEUES_NUM "queues" 344ce2eabddSJianfeng Tan VIRTIO_USER_ARG_QUEUES_NUM, 345ce2eabddSJianfeng Tan #define VIRTIO_USER_ARG_CQ_NUM "cq" 346ce2eabddSJianfeng Tan VIRTIO_USER_ARG_CQ_NUM, 347ce2eabddSJianfeng Tan #define VIRTIO_USER_ARG_MAC "mac" 348ce2eabddSJianfeng Tan VIRTIO_USER_ARG_MAC, 349ce2eabddSJianfeng Tan #define VIRTIO_USER_ARG_PATH "path" 350ce2eabddSJianfeng Tan VIRTIO_USER_ARG_PATH, 351ce2eabddSJianfeng Tan #define VIRTIO_USER_ARG_QUEUE_SIZE "queue_size" 352ce2eabddSJianfeng Tan VIRTIO_USER_ARG_QUEUE_SIZE, 3534214a1b4SWenfeng Liu #define VIRTIO_USER_ARG_INTERFACE_NAME "iface" 3544214a1b4SWenfeng Liu VIRTIO_USER_ARG_INTERFACE_NAME, 355bd8f50a4SZhiyong Yang #define VIRTIO_USER_ARG_SERVER_MODE "server" 356bd8f50a4SZhiyong Yang VIRTIO_USER_ARG_SERVER_MODE, 357488ed97aSMarvin Liu #define VIRTIO_USER_ARG_MRG_RXBUF "mrg_rxbuf" 358488ed97aSMarvin Liu VIRTIO_USER_ARG_MRG_RXBUF, 359488ed97aSMarvin Liu #define VIRTIO_USER_ARG_IN_ORDER "in_order" 360488ed97aSMarvin Liu VIRTIO_USER_ARG_IN_ORDER, 36134f3966cSYuanhan Liu #define VIRTIO_USER_ARG_PACKED_VQ "packed_vq" 36234f3966cSYuanhan Liu VIRTIO_USER_ARG_PACKED_VQ, 363b0db4beaSIvan Dyukov #define VIRTIO_USER_ARG_SPEED "speed" 364b0db4beaSIvan Dyukov VIRTIO_USER_ARG_SPEED, 3656b7eefbcSMarvin Liu #define VIRTIO_USER_ARG_VECTORIZED "vectorized" 3666b7eefbcSMarvin Liu VIRTIO_USER_ARG_VECTORIZED, 367ce2eabddSJianfeng Tan NULL 368ce2eabddSJianfeng Tan }; 369ce2eabddSJianfeng Tan 370ce2eabddSJianfeng Tan #define VIRTIO_USER_DEF_CQ_EN 0 371ce2eabddSJianfeng Tan #define VIRTIO_USER_DEF_Q_NUM 1 372ce2eabddSJianfeng Tan #define VIRTIO_USER_DEF_Q_SZ 256 373bd8f50a4SZhiyong Yang #define VIRTIO_USER_DEF_SERVER_MODE 0 374ce2eabddSJianfeng Tan 375ce2eabddSJianfeng Tan static int 376ce2eabddSJianfeng Tan get_string_arg(const char *key __rte_unused, 377ce2eabddSJianfeng Tan const char *value, void *extra_args) 378ce2eabddSJianfeng Tan { 379ce2eabddSJianfeng Tan if (!value || !extra_args) 380ce2eabddSJianfeng Tan return -EINVAL; 381ce2eabddSJianfeng Tan 382ce2eabddSJianfeng Tan *(char **)extra_args = strdup(value); 383ce2eabddSJianfeng Tan 3844214a1b4SWenfeng Liu if (!*(char **)extra_args) 3854214a1b4SWenfeng Liu return -ENOMEM; 3864214a1b4SWenfeng Liu 387ce2eabddSJianfeng Tan return 0; 388ce2eabddSJianfeng Tan } 389ce2eabddSJianfeng Tan 390ce2eabddSJianfeng Tan static int 391ce2eabddSJianfeng Tan get_integer_arg(const char *key __rte_unused, 392ce2eabddSJianfeng Tan const char *value, void *extra_args) 393ce2eabddSJianfeng Tan { 394bc5b6c11SIvan Dyukov uint64_t integer = 0; 395ce2eabddSJianfeng Tan if (!value || !extra_args) 396ce2eabddSJianfeng Tan return -EINVAL; 397bc5b6c11SIvan Dyukov errno = 0; 398bc5b6c11SIvan Dyukov integer = strtoull(value, NULL, 0); 399bc5b6c11SIvan Dyukov /* extra_args keeps default value, it should be replaced 400bc5b6c11SIvan Dyukov * only in case of successful parsing of the 'value' arg 401bc5b6c11SIvan Dyukov */ 402bc5b6c11SIvan Dyukov if (errno == 0) 403bc5b6c11SIvan Dyukov *(uint64_t *)extra_args = integer; 404bc5b6c11SIvan Dyukov return -errno; 405ce2eabddSJianfeng Tan } 406ce2eabddSJianfeng Tan 4077d62bf6fSMaxime Coquelin static uint32_t 4087d62bf6fSMaxime Coquelin vdpa_dynamic_major_num(void) 4097d62bf6fSMaxime Coquelin { 4107d62bf6fSMaxime Coquelin FILE *fp; 4117d62bf6fSMaxime Coquelin char *line = NULL; 4129f528374SChengwen Feng size_t size = 0; 4137d62bf6fSMaxime Coquelin char name[11]; 4147d62bf6fSMaxime Coquelin bool found = false; 4157d62bf6fSMaxime Coquelin uint32_t num; 4167d62bf6fSMaxime Coquelin 4177d62bf6fSMaxime Coquelin fp = fopen("/proc/devices", "r"); 4187d62bf6fSMaxime Coquelin if (fp == NULL) { 4197d62bf6fSMaxime Coquelin PMD_INIT_LOG(ERR, "Cannot open /proc/devices: %s", 4207d62bf6fSMaxime Coquelin strerror(errno)); 4217d62bf6fSMaxime Coquelin return UNNAMED_MAJOR; 4227d62bf6fSMaxime Coquelin } 4237d62bf6fSMaxime Coquelin 4247d62bf6fSMaxime Coquelin while (getline(&line, &size, fp) > 0) { 4257d62bf6fSMaxime Coquelin char *stripped = line + strspn(line, " "); 4267d62bf6fSMaxime Coquelin if ((sscanf(stripped, "%u %10s", &num, name) == 2) && 4277d62bf6fSMaxime Coquelin (strncmp(name, "vhost-vdpa", 10) == 0)) { 4287d62bf6fSMaxime Coquelin found = true; 4297d62bf6fSMaxime Coquelin break; 4307d62bf6fSMaxime Coquelin } 4317d62bf6fSMaxime Coquelin } 4329f528374SChengwen Feng free(line); 4337d62bf6fSMaxime Coquelin fclose(fp); 4347d62bf6fSMaxime Coquelin return found ? num : UNNAMED_MAJOR; 4357d62bf6fSMaxime Coquelin } 4367d62bf6fSMaxime Coquelin 437f908b22eSAdrian Moreno static enum virtio_user_backend_type 438f908b22eSAdrian Moreno virtio_user_backend_type(const char *path) 439f908b22eSAdrian Moreno { 440f908b22eSAdrian Moreno struct stat sb; 441f908b22eSAdrian Moreno 4427d62bf6fSMaxime Coquelin if (stat(path, &sb) == -1) { 443b0395dc8SAdrian Moreno if (errno == ENOENT) 444b0395dc8SAdrian Moreno return VIRTIO_USER_BACKEND_VHOST_USER; 445b0395dc8SAdrian Moreno 446f3854ebaSThomas Monjalon PMD_INIT_LOG(ERR, "Stat fails: %s (%s)", path, 4477d62bf6fSMaxime Coquelin strerror(errno)); 448f908b22eSAdrian Moreno return VIRTIO_USER_BACKEND_UNKNOWN; 4497d62bf6fSMaxime Coquelin } 450f908b22eSAdrian Moreno 4517d62bf6fSMaxime Coquelin if (S_ISSOCK(sb.st_mode)) { 4527d62bf6fSMaxime Coquelin return VIRTIO_USER_BACKEND_VHOST_USER; 4537d62bf6fSMaxime Coquelin } else if (S_ISCHR(sb.st_mode)) { 4547d62bf6fSMaxime Coquelin if (major(sb.st_rdev) == MISC_MAJOR) 4557d62bf6fSMaxime Coquelin return VIRTIO_USER_BACKEND_VHOST_KERNEL; 4567d62bf6fSMaxime Coquelin if (major(sb.st_rdev) == vdpa_dynamic_major_num()) 4577d62bf6fSMaxime Coquelin return VIRTIO_USER_BACKEND_VHOST_VDPA; 4587d62bf6fSMaxime Coquelin } 4597d62bf6fSMaxime Coquelin return VIRTIO_USER_BACKEND_UNKNOWN; 460f908b22eSAdrian Moreno } 461f908b22eSAdrian Moreno 462ce2eabddSJianfeng Tan static struct rte_eth_dev * 463050fe6e9SJan Blunck virtio_user_eth_dev_alloc(struct rte_vdev_device *vdev) 464ce2eabddSJianfeng Tan { 465ce2eabddSJianfeng Tan struct rte_eth_dev *eth_dev; 466ce2eabddSJianfeng Tan struct rte_eth_dev_data *data; 467ce2eabddSJianfeng Tan struct virtio_hw *hw; 468ce2eabddSJianfeng Tan struct virtio_user_dev *dev; 469ce2eabddSJianfeng Tan 47065832c82SMaxime Coquelin eth_dev = rte_eth_vdev_allocate(vdev, sizeof(*dev)); 471ce2eabddSJianfeng Tan if (!eth_dev) { 472ce2eabddSJianfeng Tan PMD_INIT_LOG(ERR, "cannot alloc rte_eth_dev"); 473ce2eabddSJianfeng Tan return NULL; 474ce2eabddSJianfeng Tan } 475ce2eabddSJianfeng Tan 476ce2eabddSJianfeng Tan data = eth_dev->data; 47765832c82SMaxime Coquelin dev = eth_dev->data->dev_private; 47865832c82SMaxime Coquelin hw = &dev->hw; 479ce2eabddSJianfeng Tan 480553f4593SYuanhan Liu hw->port_id = data->port_id; 481f8b60756SMaxime Coquelin VIRTIO_OPS(hw) = &virtio_user_ops; 4826a504290SMaxime Coquelin 4836a504290SMaxime Coquelin hw->intr_lsc = 1; 4844710e16aSMarvin Liu hw->use_vec_rx = 0; 48577d66da8SMarvin Liu hw->use_vec_tx = 0; 4869f233f54SMarvin Liu hw->use_inorder_rx = 0; 4879f233f54SMarvin Liu hw->use_inorder_tx = 0; 48865832c82SMaxime Coquelin 489ce2eabddSJianfeng Tan return eth_dev; 490ce2eabddSJianfeng Tan } 491ce2eabddSJianfeng Tan 492ca8326a9SJianfeng Tan static void 493ca8326a9SJianfeng Tan virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev) 494ca8326a9SJianfeng Tan { 495ca8326a9SJianfeng Tan rte_eth_dev_release_port(eth_dev); 496ca8326a9SJianfeng Tan } 497ca8326a9SJianfeng Tan 498ce2eabddSJianfeng Tan /* Dev initialization routine. Invoked once for each virtio vdev at 499c3b2fdfeSYong Wang * EAL init time, see rte_bus_probe(). 500ce2eabddSJianfeng Tan * Returns 0 on success. 501ce2eabddSJianfeng Tan */ 502ce2eabddSJianfeng Tan static int 50365832c82SMaxime Coquelin virtio_user_pmd_probe(struct rte_vdev_device *vdev) 504ce2eabddSJianfeng Tan { 50514f06474SJianfeng Tan struct rte_kvargs *kvlist = NULL; 506ce2eabddSJianfeng Tan struct rte_eth_dev *eth_dev; 507ce2eabddSJianfeng Tan struct virtio_hw *hw; 50865832c82SMaxime Coquelin struct virtio_user_dev *dev; 509f908b22eSAdrian Moreno enum virtio_user_backend_type backend_type = VIRTIO_USER_BACKEND_UNKNOWN; 510ce2eabddSJianfeng Tan uint64_t queues = VIRTIO_USER_DEF_Q_NUM; 511ce2eabddSJianfeng Tan uint64_t cq = VIRTIO_USER_DEF_CQ_EN; 512ce2eabddSJianfeng Tan uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ; 513bd8f50a4SZhiyong Yang uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE; 514488ed97aSMarvin Liu uint64_t mrg_rxbuf = 1; 515488ed97aSMarvin Liu uint64_t in_order = 1; 5169070f88bSTiwei Bie uint64_t packed_vq = 0; 5176b7eefbcSMarvin Liu uint64_t vectorized = 0; 518ce2eabddSJianfeng Tan char *path = NULL; 5194214a1b4SWenfeng Liu char *ifname = NULL; 520ce2eabddSJianfeng Tan char *mac_addr = NULL; 521ce2eabddSJianfeng Tan int ret = -1; 522ce2eabddSJianfeng Tan 52365832c82SMaxime Coquelin RTE_BUILD_BUG_ON(offsetof(struct virtio_user_dev, hw) != 0); 52465832c82SMaxime Coquelin 5251c8489daSTiwei Bie if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 52665832c82SMaxime Coquelin const char *name = rte_vdev_device_name(vdev); 5271c8489daSTiwei Bie eth_dev = rte_eth_dev_attach_secondary(name); 5281c8489daSTiwei Bie if (!eth_dev) { 52988fa5bb6SStephen Hemminger PMD_INIT_LOG(ERR, "Failed to probe %s", name); 5301c8489daSTiwei Bie return -1; 5311c8489daSTiwei Bie } 5321c8489daSTiwei Bie 533c4fa9cc7SMaxime Coquelin dev = eth_dev->data->dev_private; 534c4fa9cc7SMaxime Coquelin hw = &dev->hw; 535f8b60756SMaxime Coquelin VIRTIO_OPS(hw) = &virtio_user_ops; 536c4fa9cc7SMaxime Coquelin 5371c8489daSTiwei Bie if (eth_virtio_dev_init(eth_dev) < 0) { 5381c8489daSTiwei Bie PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails"); 5391c8489daSTiwei Bie rte_eth_dev_release_port(eth_dev); 5401c8489daSTiwei Bie return -1; 5411c8489daSTiwei Bie } 5421c8489daSTiwei Bie 5431c8489daSTiwei Bie eth_dev->dev_ops = &virtio_user_secondary_eth_dev_ops; 54465832c82SMaxime Coquelin eth_dev->device = &vdev->device; 5451c8489daSTiwei Bie rte_eth_dev_probing_finish(eth_dev); 5461c8489daSTiwei Bie return 0; 5471c8489daSTiwei Bie } 5481c8489daSTiwei Bie 54965832c82SMaxime Coquelin kvlist = rte_kvargs_parse(rte_vdev_device_args(vdev), valid_args); 550ce2eabddSJianfeng Tan if (!kvlist) { 551ce2eabddSJianfeng Tan PMD_INIT_LOG(ERR, "error when parsing param"); 552ce2eabddSJianfeng Tan goto end; 553ce2eabddSJianfeng Tan } 554ce2eabddSJianfeng Tan 5559cca159eSMaxime Coquelin if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PATH) == 1) { 556ca8326a9SJianfeng Tan if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PATH, 557ca8326a9SJianfeng Tan &get_string_arg, &path) < 0) { 558404bd6bfSJianfeng Tan PMD_INIT_LOG(ERR, "error to parse %s", 559404bd6bfSJianfeng Tan VIRTIO_USER_ARG_PATH); 560404bd6bfSJianfeng Tan goto end; 561404bd6bfSJianfeng Tan } 5629cca159eSMaxime Coquelin } else { 563f2462150SFerruh Yigit PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user", 5642dac0df4STiwei Bie VIRTIO_USER_ARG_PATH); 565ce2eabddSJianfeng Tan goto end; 566ce2eabddSJianfeng Tan } 567ce2eabddSJianfeng Tan 568f908b22eSAdrian Moreno backend_type = virtio_user_backend_type(path); 569f908b22eSAdrian Moreno if (backend_type == VIRTIO_USER_BACKEND_UNKNOWN) { 570f908b22eSAdrian Moreno PMD_INIT_LOG(ERR, 571f908b22eSAdrian Moreno "unable to determine backend type for path %s", 572f908b22eSAdrian Moreno path); 573f908b22eSAdrian Moreno goto end; 574f908b22eSAdrian Moreno } 575b0395dc8SAdrian Moreno PMD_INIT_LOG(INFO, "Backend type detected: %s", 576b0395dc8SAdrian Moreno virtio_user_backend_strings[backend_type]); 577f908b22eSAdrian Moreno 5784214a1b4SWenfeng Liu if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME) == 1) { 579f908b22eSAdrian Moreno if (backend_type != VIRTIO_USER_BACKEND_VHOST_KERNEL) { 5804214a1b4SWenfeng Liu PMD_INIT_LOG(ERR, 5814214a1b4SWenfeng Liu "arg %s applies only to vhost-kernel backend", 5824214a1b4SWenfeng Liu VIRTIO_USER_ARG_INTERFACE_NAME); 5834214a1b4SWenfeng Liu goto end; 5844214a1b4SWenfeng Liu } 5854214a1b4SWenfeng Liu 5864214a1b4SWenfeng Liu if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME, 5874214a1b4SWenfeng Liu &get_string_arg, &ifname) < 0) { 5884214a1b4SWenfeng Liu PMD_INIT_LOG(ERR, "error to parse %s", 5894214a1b4SWenfeng Liu VIRTIO_USER_ARG_INTERFACE_NAME); 5904214a1b4SWenfeng Liu goto end; 5914214a1b4SWenfeng Liu } 5924214a1b4SWenfeng Liu } 5934214a1b4SWenfeng Liu 594404bd6bfSJianfeng Tan if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MAC) == 1) { 595ca8326a9SJianfeng Tan if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MAC, 596ca8326a9SJianfeng Tan &get_string_arg, &mac_addr) < 0) { 597404bd6bfSJianfeng Tan PMD_INIT_LOG(ERR, "error to parse %s", 598404bd6bfSJianfeng Tan VIRTIO_USER_ARG_MAC); 599404bd6bfSJianfeng Tan goto end; 600404bd6bfSJianfeng Tan } 601404bd6bfSJianfeng Tan } 602ce2eabddSJianfeng Tan 603404bd6bfSJianfeng Tan if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE) == 1) { 604ca8326a9SJianfeng Tan if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE, 605ca8326a9SJianfeng Tan &get_integer_arg, &queue_size) < 0) { 606404bd6bfSJianfeng Tan PMD_INIT_LOG(ERR, "error to parse %s", 607404bd6bfSJianfeng Tan VIRTIO_USER_ARG_QUEUE_SIZE); 608404bd6bfSJianfeng Tan goto end; 609404bd6bfSJianfeng Tan } 610404bd6bfSJianfeng Tan } 611ce2eabddSJianfeng Tan 612404bd6bfSJianfeng Tan if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUES_NUM) == 1) { 613ca8326a9SJianfeng Tan if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUES_NUM, 614ca8326a9SJianfeng Tan &get_integer_arg, &queues) < 0) { 615404bd6bfSJianfeng Tan PMD_INIT_LOG(ERR, "error to parse %s", 616404bd6bfSJianfeng Tan VIRTIO_USER_ARG_QUEUES_NUM); 617404bd6bfSJianfeng Tan goto end; 618404bd6bfSJianfeng Tan } 619404bd6bfSJianfeng Tan } 620ce2eabddSJianfeng Tan 621bd8f50a4SZhiyong Yang if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) { 622bd8f50a4SZhiyong Yang if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE, 623bd8f50a4SZhiyong Yang &get_integer_arg, &server_mode) < 0) { 624bd8f50a4SZhiyong Yang PMD_INIT_LOG(ERR, "error to parse %s", 625bd8f50a4SZhiyong Yang VIRTIO_USER_ARG_SERVER_MODE); 626bd8f50a4SZhiyong Yang goto end; 627bd8f50a4SZhiyong Yang } 628bd8f50a4SZhiyong Yang } 629bd8f50a4SZhiyong Yang 630404bd6bfSJianfeng Tan if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) { 631ca8326a9SJianfeng Tan if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM, 632ca8326a9SJianfeng Tan &get_integer_arg, &cq) < 0) { 633404bd6bfSJianfeng Tan PMD_INIT_LOG(ERR, "error to parse %s", 634404bd6bfSJianfeng Tan VIRTIO_USER_ARG_CQ_NUM); 635404bd6bfSJianfeng Tan goto end; 636404bd6bfSJianfeng Tan } 637404bd6bfSJianfeng Tan } 6381b69528eSJianfeng Tan 63934f3966cSYuanhan Liu if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PACKED_VQ) == 1) { 64034f3966cSYuanhan Liu if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PACKED_VQ, 64134f3966cSYuanhan Liu &get_integer_arg, &packed_vq) < 0) { 64234f3966cSYuanhan Liu PMD_INIT_LOG(ERR, "error to parse %s", 64334f3966cSYuanhan Liu VIRTIO_USER_ARG_PACKED_VQ); 64434f3966cSYuanhan Liu goto end; 64534f3966cSYuanhan Liu } 64634f3966cSYuanhan Liu } 64734f3966cSYuanhan Liu 6486b7eefbcSMarvin Liu if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_VECTORIZED) == 1) { 6496b7eefbcSMarvin Liu if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_VECTORIZED, 6506b7eefbcSMarvin Liu &get_integer_arg, &vectorized) < 0) { 6516b7eefbcSMarvin Liu PMD_INIT_LOG(ERR, "error to parse %s", 6526b7eefbcSMarvin Liu VIRTIO_USER_ARG_VECTORIZED); 6536b7eefbcSMarvin Liu goto end; 6546b7eefbcSMarvin Liu } 6556b7eefbcSMarvin Liu } 6566b7eefbcSMarvin Liu 657488ed97aSMarvin Liu if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MRG_RXBUF) == 1) { 658488ed97aSMarvin Liu if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MRG_RXBUF, 659488ed97aSMarvin Liu &get_integer_arg, &mrg_rxbuf) < 0) { 660488ed97aSMarvin Liu PMD_INIT_LOG(ERR, "error to parse %s", 661488ed97aSMarvin Liu VIRTIO_USER_ARG_MRG_RXBUF); 662488ed97aSMarvin Liu goto end; 663488ed97aSMarvin Liu } 664488ed97aSMarvin Liu } 665488ed97aSMarvin Liu 666488ed97aSMarvin Liu if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_IN_ORDER) == 1) { 667488ed97aSMarvin Liu if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_IN_ORDER, 668488ed97aSMarvin Liu &get_integer_arg, &in_order) < 0) { 669488ed97aSMarvin Liu PMD_INIT_LOG(ERR, "error to parse %s", 670488ed97aSMarvin Liu VIRTIO_USER_ARG_IN_ORDER); 671488ed97aSMarvin Liu goto end; 672488ed97aSMarvin Liu } 673488ed97aSMarvin Liu } 674488ed97aSMarvin Liu 67565832c82SMaxime Coquelin eth_dev = virtio_user_eth_dev_alloc(vdev); 676ce2eabddSJianfeng Tan if (!eth_dev) { 677e8df94b8SJianfeng Tan PMD_INIT_LOG(ERR, "virtio_user fails to alloc device"); 678ce2eabddSJianfeng Tan goto end; 679ce2eabddSJianfeng Tan } 680ce2eabddSJianfeng Tan 68165832c82SMaxime Coquelin dev = eth_dev->data->dev_private; 68265832c82SMaxime Coquelin hw = &dev->hw; 68352901852SMaxime Coquelin if (virtio_user_dev_init(dev, path, (uint16_t)queues, cq, 6841c8489daSTiwei Bie queue_size, mac_addr, &ifname, server_mode, 685f908b22eSAdrian Moreno mrg_rxbuf, in_order, packed_vq, backend_type) < 0) { 686ca8326a9SJianfeng Tan PMD_INIT_LOG(ERR, "virtio_user_dev_init fails"); 687ca8326a9SJianfeng Tan virtio_user_eth_dev_free(eth_dev); 688ce2eabddSJianfeng Tan goto end; 689ca8326a9SJianfeng Tan } 690fbe90cddSThomas Monjalon 691ba55c94aSMaxime Coquelin /* 692ba55c94aSMaxime Coquelin * Virtio-user requires using virtual addresses for the descriptors 693ba55c94aSMaxime Coquelin * buffers, whatever other devices require 694ba55c94aSMaxime Coquelin */ 695666ef294SSrujana Challa if (backend_type == VIRTIO_USER_BACKEND_VHOST_VDPA) 696666ef294SSrujana Challa /* vDPA backend requires using IOVA for the buffers 697666ef294SSrujana Challa * to make it work in IOVA as PA mode also. 698666ef294SSrujana Challa */ 699666ef294SSrujana Challa hw->use_va = false; 700666ef294SSrujana Challa else 701ba55c94aSMaxime Coquelin hw->use_va = true; 702ba55c94aSMaxime Coquelin 70387db93e0SDavid Marchand /* previously called by pci probing for physical dev */ 704ce2eabddSJianfeng Tan if (eth_virtio_dev_init(eth_dev) < 0) { 705ce2eabddSJianfeng Tan PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails"); 706edca47a1SHarold Huang virtio_user_dev_uninit(dev); 707ca8326a9SJianfeng Tan virtio_user_eth_dev_free(eth_dev); 708ce2eabddSJianfeng Tan goto end; 709ce2eabddSJianfeng Tan } 710fbe90cddSThomas Monjalon 71177d66da8SMarvin Liu if (vectorized) { 71277d66da8SMarvin Liu if (packed_vq) { 7139ef38ddbSJoyce Kong #if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM) 7146b7eefbcSMarvin Liu hw->use_vec_rx = 1; 71577d66da8SMarvin Liu hw->use_vec_tx = 1; 71677d66da8SMarvin Liu #else 71777d66da8SMarvin Liu PMD_INIT_LOG(INFO, 71877d66da8SMarvin Liu "building environment do not support packed ring vectorized"); 71977d66da8SMarvin Liu #endif 72077d66da8SMarvin Liu } else { 72177d66da8SMarvin Liu hw->use_vec_rx = 1; 72277d66da8SMarvin Liu } 72377d66da8SMarvin Liu } 7246b7eefbcSMarvin Liu 725fbe90cddSThomas Monjalon rte_eth_dev_probing_finish(eth_dev); 726ce2eabddSJianfeng Tan ret = 0; 727ce2eabddSJianfeng Tan 728ce2eabddSJianfeng Tan end: 72914f06474SJianfeng Tan rte_kvargs_free(kvlist); 730ce2eabddSJianfeng Tan free(path); 731ce2eabddSJianfeng Tan free(mac_addr); 7324214a1b4SWenfeng Liu free(ifname); 733ce2eabddSJianfeng Tan return ret; 734ce2eabddSJianfeng Tan } 735ce2eabddSJianfeng Tan 736ce2eabddSJianfeng Tan static int 7375d2aa461SJan Blunck virtio_user_pmd_remove(struct rte_vdev_device *vdev) 738ce2eabddSJianfeng Tan { 7395d2aa461SJan Blunck const char *name; 740ce2eabddSJianfeng Tan struct rte_eth_dev *eth_dev; 741ce2eabddSJianfeng Tan 7425d2aa461SJan Blunck if (!vdev) 743ce2eabddSJianfeng Tan return -EINVAL; 744ce2eabddSJianfeng Tan 7455d2aa461SJan Blunck name = rte_vdev_device_name(vdev); 746f2462150SFerruh Yigit PMD_DRV_LOG(INFO, "Un-Initializing %s", name); 747ce2eabddSJianfeng Tan eth_dev = rte_eth_dev_allocated(name); 7487f468b2eSTiwei Bie /* Port has already been released by close. */ 749ce2eabddSJianfeng Tan if (!eth_dev) 7507f468b2eSTiwei Bie return 0; 751ce2eabddSJianfeng Tan 7521c8489daSTiwei Bie if (rte_eal_process_type() != RTE_PROC_PRIMARY) 7531c8489daSTiwei Bie return rte_eth_dev_release_port(eth_dev); 7541c8489daSTiwei Bie 755ce2eabddSJianfeng Tan /* make sure the device is stopped, queues freed */ 7568a5a0aadSThomas Monjalon return rte_eth_dev_close(eth_dev->data->port_id); 757ce2eabddSJianfeng Tan } 758ce2eabddSJianfeng Tan 75986e71eb2SMaxime Coquelin static int virtio_user_pmd_dma_map(struct rte_vdev_device *vdev, void *addr, 76086e71eb2SMaxime Coquelin uint64_t iova, size_t len) 76186e71eb2SMaxime Coquelin { 76286e71eb2SMaxime Coquelin const char *name; 76386e71eb2SMaxime Coquelin struct rte_eth_dev *eth_dev; 76486e71eb2SMaxime Coquelin struct virtio_user_dev *dev; 76586e71eb2SMaxime Coquelin 76686e71eb2SMaxime Coquelin if (!vdev) 76786e71eb2SMaxime Coquelin return -EINVAL; 76886e71eb2SMaxime Coquelin 76986e71eb2SMaxime Coquelin name = rte_vdev_device_name(vdev); 77086e71eb2SMaxime Coquelin eth_dev = rte_eth_dev_allocated(name); 77186e71eb2SMaxime Coquelin /* Port has already been released by close. */ 77286e71eb2SMaxime Coquelin if (!eth_dev) 77386e71eb2SMaxime Coquelin return 0; 77486e71eb2SMaxime Coquelin 77565832c82SMaxime Coquelin dev = eth_dev->data->dev_private; 77686e71eb2SMaxime Coquelin 77786e71eb2SMaxime Coquelin if (dev->ops->dma_map) 77886e71eb2SMaxime Coquelin return dev->ops->dma_map(dev, addr, iova, len); 77986e71eb2SMaxime Coquelin 78086e71eb2SMaxime Coquelin return 0; 78186e71eb2SMaxime Coquelin } 78286e71eb2SMaxime Coquelin 78386e71eb2SMaxime Coquelin static int virtio_user_pmd_dma_unmap(struct rte_vdev_device *vdev, void *addr, 78486e71eb2SMaxime Coquelin uint64_t iova, size_t len) 78586e71eb2SMaxime Coquelin { 78686e71eb2SMaxime Coquelin const char *name; 78786e71eb2SMaxime Coquelin struct rte_eth_dev *eth_dev; 78886e71eb2SMaxime Coquelin struct virtio_user_dev *dev; 78986e71eb2SMaxime Coquelin 79086e71eb2SMaxime Coquelin if (!vdev) 79186e71eb2SMaxime Coquelin return -EINVAL; 79286e71eb2SMaxime Coquelin 79386e71eb2SMaxime Coquelin name = rte_vdev_device_name(vdev); 79486e71eb2SMaxime Coquelin eth_dev = rte_eth_dev_allocated(name); 79586e71eb2SMaxime Coquelin /* Port has already been released by close. */ 79686e71eb2SMaxime Coquelin if (!eth_dev) 79786e71eb2SMaxime Coquelin return 0; 79886e71eb2SMaxime Coquelin 79965832c82SMaxime Coquelin dev = eth_dev->data->dev_private; 80086e71eb2SMaxime Coquelin 80186e71eb2SMaxime Coquelin if (dev->ops->dma_unmap) 80286e71eb2SMaxime Coquelin return dev->ops->dma_unmap(dev, addr, iova, len); 80386e71eb2SMaxime Coquelin 80486e71eb2SMaxime Coquelin return 0; 80586e71eb2SMaxime Coquelin } 80686e71eb2SMaxime Coquelin 807fe363dd4SJan Viktorin static struct rte_vdev_driver virtio_user_driver = { 80850a3345fSShreyansh Jain .probe = virtio_user_pmd_probe, 80950a3345fSShreyansh Jain .remove = virtio_user_pmd_remove, 81086e71eb2SMaxime Coquelin .dma_map = virtio_user_pmd_dma_map, 81186e71eb2SMaxime Coquelin .dma_unmap = virtio_user_pmd_dma_unmap, 812ce2eabddSJianfeng Tan }; 813ce2eabddSJianfeng Tan 81401f19227SShreyansh Jain RTE_PMD_REGISTER_VDEV(net_virtio_user, virtio_user_driver); 8159fa80cb2SJan Blunck RTE_PMD_REGISTER_ALIAS(net_virtio_user, virtio_user); 81601f19227SShreyansh Jain RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user, 81744e32a67SPablo de Lara "path=<path> " 81844e32a67SPablo de Lara "mac=<mac addr> " 81944e32a67SPablo de Lara "cq=<int> " 82044e32a67SPablo de Lara "queue_size=<int> " 8214214a1b4SWenfeng Liu "queues=<int> " 822488ed97aSMarvin Liu "iface=<string> " 82362758c76STiwei Bie "server=<0|1> " 824488ed97aSMarvin Liu "mrg_rxbuf=<0|1> " 8259070f88bSTiwei Bie "in_order=<0|1> " 826b0db4beaSIvan Dyukov "packed_vq=<0|1> " 8276b7eefbcSMarvin Liu "speed=<int> " 8286b7eefbcSMarvin Liu "vectorized=<0|1>"); 829