1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 /** 6 * @file 7 * 8 * Device specific vhost lib 9 */ 10 11 #include <sys/queue.h> 12 13 #include <rte_class.h> 14 #include <rte_malloc.h> 15 #include <rte_spinlock.h> 16 #include <rte_tailq.h> 17 18 #include "rte_vdpa.h" 19 #include "vdpa_driver.h" 20 #include "vhost.h" 21 22 /** Double linked list of vDPA devices. */ 23 TAILQ_HEAD(vdpa_device_list, rte_vdpa_device); 24 25 static struct vdpa_device_list vdpa_device_list = 26 TAILQ_HEAD_INITIALIZER(vdpa_device_list); 27 static rte_spinlock_t vdpa_device_list_lock = RTE_SPINLOCK_INITIALIZER; 28 29 30 /* Unsafe, needs to be called with vdpa_device_list_lock held */ 31 static struct rte_vdpa_device * 32 __vdpa_find_device_by_name(const char *name) 33 { 34 struct rte_vdpa_device *dev, *ret = NULL; 35 36 if (name == NULL) 37 return NULL; 38 39 TAILQ_FOREACH(dev, &vdpa_device_list, next) { 40 if (!strncmp(dev->device->name, name, RTE_DEV_NAME_MAX_LEN)) { 41 ret = dev; 42 break; 43 } 44 } 45 46 return ret; 47 } 48 49 struct rte_vdpa_device * 50 rte_vdpa_find_device_by_name(const char *name) 51 { 52 struct rte_vdpa_device *dev; 53 54 rte_spinlock_lock(&vdpa_device_list_lock); 55 dev = __vdpa_find_device_by_name(name); 56 rte_spinlock_unlock(&vdpa_device_list_lock); 57 58 return dev; 59 } 60 61 struct rte_device * 62 rte_vdpa_get_rte_device(struct rte_vdpa_device *vdpa_dev) 63 { 64 if (vdpa_dev == NULL) 65 return NULL; 66 67 return vdpa_dev->device; 68 } 69 70 struct rte_vdpa_device * 71 rte_vdpa_register_device(struct rte_device *rte_dev, 72 struct rte_vdpa_dev_ops *ops) 73 { 74 struct rte_vdpa_device *dev; 75 76 if (ops == NULL) 77 return NULL; 78 79 /* Check mandatory ops are implemented */ 80 if (!ops->get_queue_num || !ops->get_features || 81 !ops->get_protocol_features || !ops->dev_conf || 82 !ops->dev_close || !ops->set_vring_state || 83 !ops->set_features) { 84 VHOST_LOG_CONFIG(rte_dev->name, ERR, 85 "Some mandatory vDPA ops aren't implemented\n"); 86 return NULL; 87 } 88 89 rte_spinlock_lock(&vdpa_device_list_lock); 90 /* Check the device hasn't been register already */ 91 dev = __vdpa_find_device_by_name(rte_dev->name); 92 if (dev) { 93 dev = NULL; 94 goto out_unlock; 95 } 96 97 dev = rte_zmalloc(NULL, sizeof(*dev), 0); 98 if (!dev) 99 goto out_unlock; 100 101 dev->device = rte_dev; 102 dev->ops = ops; 103 TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next); 104 out_unlock: 105 rte_spinlock_unlock(&vdpa_device_list_lock); 106 107 return dev; 108 } 109 110 int 111 rte_vdpa_unregister_device(struct rte_vdpa_device *dev) 112 { 113 struct rte_vdpa_device *cur_dev, *tmp_dev; 114 int ret = -1; 115 116 rte_spinlock_lock(&vdpa_device_list_lock); 117 RTE_TAILQ_FOREACH_SAFE(cur_dev, &vdpa_device_list, next, tmp_dev) { 118 if (dev != cur_dev) 119 continue; 120 121 TAILQ_REMOVE(&vdpa_device_list, dev, next); 122 rte_free(dev); 123 ret = 0; 124 break; 125 } 126 rte_spinlock_unlock(&vdpa_device_list_lock); 127 128 return ret; 129 } 130 131 int 132 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m) 133 { 134 struct virtio_net *dev = get_device(vid); 135 uint16_t idx, idx_m, desc_id; 136 struct vhost_virtqueue *vq; 137 struct vring_desc desc; 138 struct vring_desc *desc_ring; 139 struct vring_desc *idesc = NULL; 140 struct vring *s_vring; 141 uint64_t dlen; 142 uint32_t nr_descs; 143 int ret; 144 145 if (!dev || !vring_m) 146 return -1; 147 148 if (qid >= dev->nr_vring) 149 return -1; 150 151 if (vq_is_packed(dev)) 152 return -1; 153 154 s_vring = (struct vring *)vring_m; 155 vq = dev->virtqueue[qid]; 156 idx = vq->used->idx; 157 idx_m = s_vring->used->idx; 158 ret = (uint16_t)(idx_m - idx); 159 160 while (idx != idx_m) { 161 /* copy used entry, used ring logging is not covered here */ 162 vq->used->ring[idx & (vq->size - 1)] = 163 s_vring->used->ring[idx & (vq->size - 1)]; 164 165 desc_id = vq->used->ring[idx & (vq->size - 1)].id; 166 desc_ring = vq->desc; 167 nr_descs = vq->size; 168 169 if (unlikely(desc_id >= vq->size)) 170 return -1; 171 172 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) { 173 dlen = vq->desc[desc_id].len; 174 nr_descs = dlen / sizeof(struct vring_desc); 175 if (unlikely(nr_descs > vq->size)) 176 return -1; 177 178 desc_ring = (struct vring_desc *)(uintptr_t) 179 vhost_iova_to_vva(dev, vq, 180 vq->desc[desc_id].addr, &dlen, 181 VHOST_ACCESS_RO); 182 if (unlikely(!desc_ring)) 183 return -1; 184 185 if (unlikely(dlen < vq->desc[desc_id].len)) { 186 idesc = vhost_alloc_copy_ind_table(dev, vq, 187 vq->desc[desc_id].addr, 188 vq->desc[desc_id].len); 189 if (unlikely(!idesc)) 190 return -1; 191 192 desc_ring = idesc; 193 } 194 195 desc_id = 0; 196 } 197 198 /* dirty page logging for DMA writeable buffer */ 199 do { 200 if (unlikely(desc_id >= vq->size)) 201 goto fail; 202 if (unlikely(nr_descs-- == 0)) 203 goto fail; 204 desc = desc_ring[desc_id]; 205 if (desc.flags & VRING_DESC_F_WRITE) 206 vhost_log_write_iova(dev, vq, desc.addr, 207 desc.len); 208 desc_id = desc.next; 209 } while (desc.flags & VRING_DESC_F_NEXT); 210 211 if (unlikely(idesc)) { 212 free_ind_table(idesc); 213 idesc = NULL; 214 } 215 216 idx++; 217 } 218 219 /* used idx is the synchronization point for the split vring */ 220 __atomic_store_n(&vq->used->idx, idx_m, __ATOMIC_RELEASE); 221 222 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 223 vring_used_event(s_vring) = idx_m; 224 225 return ret; 226 227 fail: 228 if (unlikely(idesc)) 229 free_ind_table(idesc); 230 return -1; 231 } 232 233 int 234 rte_vdpa_get_queue_num(struct rte_vdpa_device *dev, uint32_t *queue_num) 235 { 236 if (dev == NULL || dev->ops == NULL || dev->ops->get_queue_num == NULL) 237 return -1; 238 239 return dev->ops->get_queue_num(dev, queue_num); 240 } 241 242 int 243 rte_vdpa_get_features(struct rte_vdpa_device *dev, uint64_t *features) 244 { 245 if (dev == NULL || dev->ops == NULL || dev->ops->get_features == NULL) 246 return -1; 247 248 return dev->ops->get_features(dev, features); 249 } 250 251 int 252 rte_vdpa_get_protocol_features(struct rte_vdpa_device *dev, uint64_t *features) 253 { 254 if (dev == NULL || dev->ops == NULL || 255 dev->ops->get_protocol_features == NULL) 256 return -1; 257 258 return dev->ops->get_protocol_features(dev, features); 259 } 260 261 int 262 rte_vdpa_get_stats_names(struct rte_vdpa_device *dev, 263 struct rte_vdpa_stat_name *stats_names, 264 unsigned int size) 265 { 266 if (!dev) 267 return -EINVAL; 268 269 RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats_names, -ENOTSUP); 270 271 return dev->ops->get_stats_names(dev, stats_names, size); 272 } 273 274 int 275 rte_vdpa_get_stats(struct rte_vdpa_device *dev, uint16_t qid, 276 struct rte_vdpa_stat *stats, unsigned int n) 277 { 278 if (!dev || !stats || !n) 279 return -EINVAL; 280 281 RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats, -ENOTSUP); 282 283 return dev->ops->get_stats(dev, qid, stats, n); 284 } 285 286 int 287 rte_vdpa_reset_stats(struct rte_vdpa_device *dev, uint16_t qid) 288 { 289 if (!dev) 290 return -EINVAL; 291 292 RTE_FUNC_PTR_OR_ERR_RET(dev->ops->reset_stats, -ENOTSUP); 293 294 return dev->ops->reset_stats(dev, qid); 295 } 296 297 static int 298 vdpa_dev_match(struct rte_vdpa_device *dev, 299 const struct rte_device *rte_dev) 300 { 301 if (dev->device == rte_dev) 302 return 0; 303 304 return -1; 305 } 306 307 /* Generic rte_vdpa_dev comparison function. */ 308 typedef int (*rte_vdpa_cmp_t)(struct rte_vdpa_device *, 309 const struct rte_device *rte_dev); 310 311 static struct rte_vdpa_device * 312 vdpa_find_device(const struct rte_vdpa_device *start, rte_vdpa_cmp_t cmp, 313 struct rte_device *rte_dev) 314 { 315 struct rte_vdpa_device *dev; 316 317 rte_spinlock_lock(&vdpa_device_list_lock); 318 if (start == NULL) 319 dev = TAILQ_FIRST(&vdpa_device_list); 320 else 321 dev = TAILQ_NEXT(start, next); 322 323 while (dev != NULL) { 324 if (cmp(dev, rte_dev) == 0) 325 break; 326 327 dev = TAILQ_NEXT(dev, next); 328 } 329 rte_spinlock_unlock(&vdpa_device_list_lock); 330 331 return dev; 332 } 333 334 static void * 335 vdpa_dev_iterate(const void *start, 336 const char *str, 337 const struct rte_dev_iterator *it) 338 { 339 struct rte_vdpa_device *vdpa_dev = NULL; 340 341 RTE_SET_USED(str); 342 343 vdpa_dev = vdpa_find_device(start, vdpa_dev_match, it->device); 344 345 return vdpa_dev; 346 } 347 348 static struct rte_class rte_class_vdpa = { 349 .dev_iterate = vdpa_dev_iterate, 350 }; 351 352 RTE_REGISTER_CLASS(vdpa, rte_class_vdpa); 353