1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 /** 6 * @file 7 * 8 * Device specific vhost lib 9 */ 10 11 #include <sys/queue.h> 12 13 #include <dev_driver.h> 14 #include <rte_class.h> 15 #include <rte_malloc.h> 16 #include <rte_spinlock.h> 17 #include <rte_tailq.h> 18 19 #include "rte_vdpa.h" 20 #include "vdpa_driver.h" 21 #include "vhost.h" 22 23 /** Double linked list of vDPA devices. */ 24 TAILQ_HEAD(vdpa_device_list, rte_vdpa_device); 25 26 static struct vdpa_device_list vdpa_device_list = 27 TAILQ_HEAD_INITIALIZER(vdpa_device_list); 28 static rte_spinlock_t vdpa_device_list_lock = RTE_SPINLOCK_INITIALIZER; 29 30 31 /* Unsafe, needs to be called with vdpa_device_list_lock held */ 32 static struct rte_vdpa_device * 33 __vdpa_find_device_by_name(const char *name) 34 { 35 struct rte_vdpa_device *dev, *ret = NULL; 36 37 if (name == NULL) 38 return NULL; 39 40 TAILQ_FOREACH(dev, &vdpa_device_list, next) { 41 if (!strncmp(dev->device->name, name, RTE_DEV_NAME_MAX_LEN)) { 42 ret = dev; 43 break; 44 } 45 } 46 47 return ret; 48 } 49 50 struct rte_vdpa_device * 51 rte_vdpa_find_device_by_name(const char *name) 52 { 53 struct rte_vdpa_device *dev; 54 55 rte_spinlock_lock(&vdpa_device_list_lock); 56 dev = __vdpa_find_device_by_name(name); 57 rte_spinlock_unlock(&vdpa_device_list_lock); 58 59 return dev; 60 } 61 62 struct rte_device * 63 rte_vdpa_get_rte_device(struct rte_vdpa_device *vdpa_dev) 64 { 65 if (vdpa_dev == NULL) 66 return NULL; 67 68 return vdpa_dev->device; 69 } 70 71 struct rte_vdpa_device * 72 rte_vdpa_register_device(struct rte_device *rte_dev, 73 struct rte_vdpa_dev_ops *ops) 74 { 75 struct rte_vdpa_device *dev; 76 int ret = 0; 77 78 if (ops == NULL) 79 return NULL; 80 81 /* Check mandatory ops are implemented */ 82 if (!ops->get_queue_num || !ops->get_features || 83 !ops->get_protocol_features || !ops->dev_conf || 84 !ops->dev_close || !ops->set_vring_state || 85 !ops->set_features) { 86 VHOST_LOG_CONFIG(rte_dev->name, ERR, 87 "Some mandatory vDPA ops aren't implemented\n"); 88 return NULL; 89 } 90 91 rte_spinlock_lock(&vdpa_device_list_lock); 92 /* Check the device hasn't been register already */ 93 dev = __vdpa_find_device_by_name(rte_dev->name); 94 if (dev) { 95 dev = NULL; 96 goto out_unlock; 97 } 98 99 dev = rte_zmalloc(NULL, sizeof(*dev), 0); 100 if (!dev) 101 goto out_unlock; 102 103 dev->device = rte_dev; 104 dev->ops = ops; 105 106 if (ops->get_dev_type) { 107 ret = ops->get_dev_type(dev, &dev->type); 108 if (ret) { 109 VHOST_LOG_CONFIG(rte_dev->name, ERR, 110 "Failed to get vdpa dev type.\n"); 111 ret = -1; 112 goto out_unlock; 113 } 114 } else { 115 /** by default, we assume vdpa device is a net device */ 116 dev->type = RTE_VHOST_VDPA_DEVICE_TYPE_NET; 117 } 118 119 TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next); 120 out_unlock: 121 rte_spinlock_unlock(&vdpa_device_list_lock); 122 123 return dev; 124 } 125 126 int 127 rte_vdpa_unregister_device(struct rte_vdpa_device *dev) 128 { 129 struct rte_vdpa_device *cur_dev, *tmp_dev; 130 int ret = -1; 131 132 rte_spinlock_lock(&vdpa_device_list_lock); 133 RTE_TAILQ_FOREACH_SAFE(cur_dev, &vdpa_device_list, next, tmp_dev) { 134 if (dev != cur_dev) 135 continue; 136 137 TAILQ_REMOVE(&vdpa_device_list, dev, next); 138 rte_free(dev); 139 ret = 0; 140 break; 141 } 142 rte_spinlock_unlock(&vdpa_device_list_lock); 143 144 return ret; 145 } 146 147 int 148 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m) 149 { 150 struct virtio_net *dev = get_device(vid); 151 uint16_t idx, idx_m, desc_id; 152 struct vhost_virtqueue *vq; 153 struct vring_desc desc; 154 struct vring_desc *desc_ring; 155 struct vring_desc *idesc = NULL; 156 struct vring *s_vring; 157 uint64_t dlen; 158 uint32_t nr_descs; 159 int ret; 160 161 if (!dev || !vring_m) 162 return -1; 163 164 if (qid >= dev->nr_vring) 165 return -1; 166 167 if (vq_is_packed(dev)) 168 return -1; 169 170 s_vring = (struct vring *)vring_m; 171 vq = dev->virtqueue[qid]; 172 idx = vq->used->idx; 173 idx_m = s_vring->used->idx; 174 ret = (uint16_t)(idx_m - idx); 175 176 while (idx != idx_m) { 177 /* copy used entry, used ring logging is not covered here */ 178 vq->used->ring[idx & (vq->size - 1)] = 179 s_vring->used->ring[idx & (vq->size - 1)]; 180 181 desc_id = vq->used->ring[idx & (vq->size - 1)].id; 182 desc_ring = vq->desc; 183 nr_descs = vq->size; 184 185 if (unlikely(desc_id >= vq->size)) 186 return -1; 187 188 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) { 189 dlen = vq->desc[desc_id].len; 190 nr_descs = dlen / sizeof(struct vring_desc); 191 if (unlikely(nr_descs > vq->size)) 192 return -1; 193 194 desc_ring = (struct vring_desc *)(uintptr_t) 195 vhost_iova_to_vva(dev, vq, 196 vq->desc[desc_id].addr, &dlen, 197 VHOST_ACCESS_RO); 198 if (unlikely(!desc_ring)) 199 return -1; 200 201 if (unlikely(dlen < vq->desc[desc_id].len)) { 202 idesc = vhost_alloc_copy_ind_table(dev, vq, 203 vq->desc[desc_id].addr, 204 vq->desc[desc_id].len); 205 if (unlikely(!idesc)) 206 return -1; 207 208 desc_ring = idesc; 209 } 210 211 desc_id = 0; 212 } 213 214 /* dirty page logging for DMA writeable buffer */ 215 do { 216 if (unlikely(desc_id >= vq->size)) 217 goto fail; 218 if (unlikely(nr_descs-- == 0)) 219 goto fail; 220 desc = desc_ring[desc_id]; 221 if (desc.flags & VRING_DESC_F_WRITE) 222 vhost_log_write_iova(dev, vq, desc.addr, 223 desc.len); 224 desc_id = desc.next; 225 } while (desc.flags & VRING_DESC_F_NEXT); 226 227 if (unlikely(idesc)) { 228 free_ind_table(idesc); 229 idesc = NULL; 230 } 231 232 idx++; 233 } 234 235 /* used idx is the synchronization point for the split vring */ 236 __atomic_store_n(&vq->used->idx, idx_m, __ATOMIC_RELEASE); 237 238 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 239 vring_used_event(s_vring) = idx_m; 240 241 return ret; 242 243 fail: 244 if (unlikely(idesc)) 245 free_ind_table(idesc); 246 return -1; 247 } 248 249 int 250 rte_vdpa_get_queue_num(struct rte_vdpa_device *dev, uint32_t *queue_num) 251 { 252 if (dev == NULL || dev->ops == NULL || dev->ops->get_queue_num == NULL) 253 return -1; 254 255 return dev->ops->get_queue_num(dev, queue_num); 256 } 257 258 int 259 rte_vdpa_get_features(struct rte_vdpa_device *dev, uint64_t *features) 260 { 261 if (dev == NULL || dev->ops == NULL || dev->ops->get_features == NULL) 262 return -1; 263 264 return dev->ops->get_features(dev, features); 265 } 266 267 int 268 rte_vdpa_get_protocol_features(struct rte_vdpa_device *dev, uint64_t *features) 269 { 270 if (dev == NULL || dev->ops == NULL || 271 dev->ops->get_protocol_features == NULL) 272 return -1; 273 274 return dev->ops->get_protocol_features(dev, features); 275 } 276 277 int 278 rte_vdpa_get_stats_names(struct rte_vdpa_device *dev, 279 struct rte_vdpa_stat_name *stats_names, 280 unsigned int size) 281 { 282 if (!dev) 283 return -EINVAL; 284 285 if (dev->ops->get_stats_names == NULL) 286 return -ENOTSUP; 287 288 return dev->ops->get_stats_names(dev, stats_names, size); 289 } 290 291 int 292 rte_vdpa_get_stats(struct rte_vdpa_device *dev, uint16_t qid, 293 struct rte_vdpa_stat *stats, unsigned int n) 294 { 295 if (!dev || !stats || !n) 296 return -EINVAL; 297 298 if (dev->ops->get_stats == NULL) 299 return -ENOTSUP; 300 301 return dev->ops->get_stats(dev, qid, stats, n); 302 } 303 304 int 305 rte_vdpa_reset_stats(struct rte_vdpa_device *dev, uint16_t qid) 306 { 307 if (!dev) 308 return -EINVAL; 309 310 if (dev->ops->reset_stats == NULL) 311 return -ENOTSUP; 312 313 return dev->ops->reset_stats(dev, qid); 314 } 315 316 static int 317 vdpa_dev_match(struct rte_vdpa_device *dev, 318 const struct rte_device *rte_dev) 319 { 320 if (dev->device == rte_dev) 321 return 0; 322 323 return -1; 324 } 325 326 /* Generic rte_vdpa_dev comparison function. */ 327 typedef int (*rte_vdpa_cmp_t)(struct rte_vdpa_device *, 328 const struct rte_device *rte_dev); 329 330 static struct rte_vdpa_device * 331 vdpa_find_device(const struct rte_vdpa_device *start, rte_vdpa_cmp_t cmp, 332 struct rte_device *rte_dev) 333 { 334 struct rte_vdpa_device *dev; 335 336 rte_spinlock_lock(&vdpa_device_list_lock); 337 if (start == NULL) 338 dev = TAILQ_FIRST(&vdpa_device_list); 339 else 340 dev = TAILQ_NEXT(start, next); 341 342 while (dev != NULL) { 343 if (cmp(dev, rte_dev) == 0) 344 break; 345 346 dev = TAILQ_NEXT(dev, next); 347 } 348 rte_spinlock_unlock(&vdpa_device_list_lock); 349 350 return dev; 351 } 352 353 static void * 354 vdpa_dev_iterate(const void *start, 355 const char *str, 356 const struct rte_dev_iterator *it) 357 { 358 struct rte_vdpa_device *vdpa_dev = NULL; 359 360 RTE_SET_USED(str); 361 362 vdpa_dev = vdpa_find_device(start, vdpa_dev_match, it->device); 363 364 return vdpa_dev; 365 } 366 367 static struct rte_class rte_class_vdpa = { 368 .dev_iterate = vdpa_dev_iterate, 369 }; 370 371 RTE_REGISTER_CLASS(vdpa, rte_class_vdpa); 372