1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 /** 6 * @file 7 * 8 * Device specific vhost lib 9 */ 10 11 #include <sys/queue.h> 12 13 #include <dev_driver.h> 14 #include <rte_class.h> 15 #include <rte_malloc.h> 16 #include <rte_spinlock.h> 17 #include <rte_tailq.h> 18 19 #include "rte_vdpa.h" 20 #include "vdpa_driver.h" 21 #include "vhost.h" 22 #include "iotlb.h" 23 24 /** Double linked list of vDPA devices. */ 25 TAILQ_HEAD(vdpa_device_list, rte_vdpa_device); 26 27 static struct vdpa_device_list vdpa_device_list__ = 28 TAILQ_HEAD_INITIALIZER(vdpa_device_list__); 29 static rte_spinlock_t vdpa_device_list_lock = RTE_SPINLOCK_INITIALIZER; 30 static struct vdpa_device_list * const vdpa_device_list 31 __rte_guarded_by(&vdpa_device_list_lock) = &vdpa_device_list__; 32 33 static struct rte_vdpa_device * 34 __vdpa_find_device_by_name(const char *name) 35 __rte_exclusive_locks_required(&vdpa_device_list_lock) 36 { 37 struct rte_vdpa_device *dev, *ret = NULL; 38 39 if (name == NULL) 40 return NULL; 41 42 TAILQ_FOREACH(dev, vdpa_device_list, next) { 43 if (!strncmp(dev->device->name, name, RTE_DEV_NAME_MAX_LEN)) { 44 ret = dev; 45 break; 46 } 47 } 48 49 return ret; 50 } 51 52 struct rte_vdpa_device * 53 rte_vdpa_find_device_by_name(const char *name) 54 { 55 struct rte_vdpa_device *dev; 56 57 rte_spinlock_lock(&vdpa_device_list_lock); 58 dev = __vdpa_find_device_by_name(name); 59 rte_spinlock_unlock(&vdpa_device_list_lock); 60 61 return dev; 62 } 63 64 struct rte_device * 65 rte_vdpa_get_rte_device(struct rte_vdpa_device *vdpa_dev) 66 { 67 if (vdpa_dev == NULL) 68 return NULL; 69 70 return vdpa_dev->device; 71 } 72 73 struct rte_vdpa_device * 74 rte_vdpa_register_device(struct rte_device *rte_dev, 75 struct rte_vdpa_dev_ops *ops) 76 { 77 struct rte_vdpa_device *dev; 78 int ret = 0; 79 80 if (ops == NULL) 81 return NULL; 82 83 /* Check mandatory ops are implemented */ 84 if (!ops->get_queue_num || !ops->get_features || 85 !ops->get_protocol_features || !ops->dev_conf || 86 !ops->dev_close || !ops->set_vring_state || 87 !ops->set_features) { 88 VHOST_CONFIG_LOG(rte_dev->name, ERR, 89 "Some mandatory vDPA ops aren't implemented"); 90 return NULL; 91 } 92 93 rte_spinlock_lock(&vdpa_device_list_lock); 94 /* Check the device hasn't been register already */ 95 dev = __vdpa_find_device_by_name(rte_dev->name); 96 if (dev) { 97 dev = NULL; 98 goto out_unlock; 99 } 100 101 dev = rte_zmalloc(NULL, sizeof(*dev), 0); 102 if (!dev) 103 goto out_unlock; 104 105 dev->device = rte_dev; 106 dev->ops = ops; 107 108 if (ops->get_dev_type) { 109 ret = ops->get_dev_type(dev, &dev->type); 110 if (ret) { 111 VHOST_CONFIG_LOG(rte_dev->name, ERR, 112 "Failed to get vdpa dev type."); 113 ret = -1; 114 goto out_unlock; 115 } 116 } else { 117 /** by default, we assume vdpa device is a net device */ 118 dev->type = RTE_VHOST_VDPA_DEVICE_TYPE_NET; 119 } 120 121 TAILQ_INSERT_TAIL(vdpa_device_list, dev, next); 122 out_unlock: 123 rte_spinlock_unlock(&vdpa_device_list_lock); 124 125 return dev; 126 } 127 128 int 129 rte_vdpa_unregister_device(struct rte_vdpa_device *dev) 130 { 131 struct rte_vdpa_device *cur_dev, *tmp_dev; 132 int ret = -1; 133 134 rte_spinlock_lock(&vdpa_device_list_lock); 135 RTE_TAILQ_FOREACH_SAFE(cur_dev, vdpa_device_list, next, tmp_dev) { 136 if (dev != cur_dev) 137 continue; 138 139 TAILQ_REMOVE(vdpa_device_list, dev, next); 140 rte_free(dev); 141 ret = 0; 142 break; 143 } 144 rte_spinlock_unlock(&vdpa_device_list_lock); 145 146 return ret; 147 } 148 149 int 150 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m) 151 { 152 struct virtio_net *dev = get_device(vid); 153 uint16_t idx, idx_m, desc_id; 154 struct vhost_virtqueue *vq; 155 struct vring_desc desc; 156 struct vring_desc *desc_ring; 157 struct vring_desc *idesc = NULL; 158 struct vring *s_vring; 159 uint64_t dlen; 160 uint32_t nr_descs; 161 int ret; 162 163 if (!dev || !vring_m) 164 return -1; 165 166 if (qid >= dev->nr_vring) 167 return -1; 168 169 if (vq_is_packed(dev)) 170 return -1; 171 172 s_vring = (struct vring *)vring_m; 173 vq = dev->virtqueue[qid]; 174 idx = vq->used->idx; 175 idx_m = s_vring->used->idx; 176 ret = (uint16_t)(idx_m - idx); 177 vq->used->flags = s_vring->used->flags; 178 179 while (idx != idx_m) { 180 /* copy used entry, used ring logging is not covered here */ 181 vq->used->ring[idx & (vq->size - 1)] = 182 s_vring->used->ring[idx & (vq->size - 1)]; 183 184 desc_id = vq->used->ring[idx & (vq->size - 1)].id; 185 desc_ring = vq->desc; 186 nr_descs = vq->size; 187 188 if (unlikely(desc_id >= vq->size)) 189 return -1; 190 191 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) { 192 dlen = vq->desc[desc_id].len; 193 nr_descs = dlen / sizeof(struct vring_desc); 194 if (unlikely(nr_descs > vq->size)) 195 return -1; 196 197 vhost_user_iotlb_rd_lock(vq); 198 desc_ring = (struct vring_desc *)(uintptr_t) 199 vhost_iova_to_vva(dev, vq, 200 vq->desc[desc_id].addr, &dlen, 201 VHOST_ACCESS_RO); 202 vhost_user_iotlb_rd_unlock(vq); 203 if (unlikely(!desc_ring)) 204 return -1; 205 206 if (unlikely(dlen < vq->desc[desc_id].len)) { 207 vhost_user_iotlb_rd_lock(vq); 208 idesc = vhost_alloc_copy_ind_table(dev, vq, 209 vq->desc[desc_id].addr, 210 vq->desc[desc_id].len); 211 vhost_user_iotlb_rd_unlock(vq); 212 if (unlikely(!idesc)) 213 return -1; 214 215 desc_ring = idesc; 216 } 217 218 desc_id = 0; 219 } 220 221 /* dirty page logging for DMA writeable buffer */ 222 do { 223 if (unlikely(desc_id >= vq->size)) 224 goto fail; 225 if (unlikely(nr_descs-- == 0)) 226 goto fail; 227 desc = desc_ring[desc_id]; 228 if (desc.flags & VRING_DESC_F_WRITE) { 229 vhost_user_iotlb_rd_lock(vq); 230 vhost_log_write_iova(dev, vq, desc.addr, 231 desc.len); 232 vhost_user_iotlb_rd_unlock(vq); 233 } 234 desc_id = desc.next; 235 } while (desc.flags & VRING_DESC_F_NEXT); 236 237 if (unlikely(idesc)) { 238 free_ind_table(idesc); 239 idesc = NULL; 240 } 241 242 idx++; 243 } 244 245 /* used idx is the synchronization point for the split vring */ 246 rte_atomic_store_explicit((unsigned short __rte_atomic *)&vq->used->idx, 247 idx_m, rte_memory_order_release); 248 249 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 250 vring_used_event(s_vring) = idx_m; 251 252 return ret; 253 254 fail: 255 if (unlikely(idesc)) 256 free_ind_table(idesc); 257 return -1; 258 } 259 260 int 261 rte_vdpa_get_queue_num(struct rte_vdpa_device *dev, uint32_t *queue_num) 262 { 263 if (dev == NULL || dev->ops == NULL || dev->ops->get_queue_num == NULL) 264 return -1; 265 266 return dev->ops->get_queue_num(dev, queue_num); 267 } 268 269 int 270 rte_vdpa_get_features(struct rte_vdpa_device *dev, uint64_t *features) 271 { 272 if (dev == NULL || dev->ops == NULL || dev->ops->get_features == NULL) 273 return -1; 274 275 return dev->ops->get_features(dev, features); 276 } 277 278 int 279 rte_vdpa_get_protocol_features(struct rte_vdpa_device *dev, uint64_t *features) 280 { 281 if (dev == NULL || dev->ops == NULL || 282 dev->ops->get_protocol_features == NULL) 283 return -1; 284 285 return dev->ops->get_protocol_features(dev, features); 286 } 287 288 int 289 rte_vdpa_get_stats_names(struct rte_vdpa_device *dev, 290 struct rte_vdpa_stat_name *stats_names, 291 unsigned int size) 292 { 293 if (!dev) 294 return -EINVAL; 295 296 if (dev->ops->get_stats_names == NULL) 297 return -ENOTSUP; 298 299 return dev->ops->get_stats_names(dev, stats_names, size); 300 } 301 302 int 303 rte_vdpa_get_stats(struct rte_vdpa_device *dev, uint16_t qid, 304 struct rte_vdpa_stat *stats, unsigned int n) 305 { 306 if (!dev || !stats || !n) 307 return -EINVAL; 308 309 if (dev->ops->get_stats == NULL) 310 return -ENOTSUP; 311 312 return dev->ops->get_stats(dev, qid, stats, n); 313 } 314 315 int 316 rte_vdpa_reset_stats(struct rte_vdpa_device *dev, uint16_t qid) 317 { 318 if (!dev) 319 return -EINVAL; 320 321 if (dev->ops->reset_stats == NULL) 322 return -ENOTSUP; 323 324 return dev->ops->reset_stats(dev, qid); 325 } 326 327 static int 328 vdpa_dev_match(struct rte_vdpa_device *dev, 329 const struct rte_device *rte_dev) 330 { 331 if (dev->device == rte_dev) 332 return 0; 333 334 return -1; 335 } 336 337 /* Generic rte_vdpa_dev comparison function. */ 338 typedef int (*rte_vdpa_cmp_t)(struct rte_vdpa_device *, 339 const struct rte_device *rte_dev); 340 341 static struct rte_vdpa_device * 342 vdpa_find_device(const struct rte_vdpa_device *start, rte_vdpa_cmp_t cmp, 343 struct rte_device *rte_dev) 344 { 345 struct rte_vdpa_device *dev; 346 347 rte_spinlock_lock(&vdpa_device_list_lock); 348 if (start == NULL) 349 dev = TAILQ_FIRST(vdpa_device_list); 350 else 351 dev = TAILQ_NEXT(start, next); 352 353 while (dev != NULL) { 354 if (cmp(dev, rte_dev) == 0) 355 break; 356 357 dev = TAILQ_NEXT(dev, next); 358 } 359 rte_spinlock_unlock(&vdpa_device_list_lock); 360 361 return dev; 362 } 363 364 static void * 365 vdpa_dev_iterate(const void *start, 366 const char *str, 367 const struct rte_dev_iterator *it) 368 { 369 struct rte_vdpa_device *vdpa_dev = NULL; 370 371 RTE_SET_USED(str); 372 373 vdpa_dev = vdpa_find_device(start, vdpa_dev_match, it->device); 374 375 return vdpa_dev; 376 } 377 378 static struct rte_class rte_class_vdpa = { 379 .dev_iterate = vdpa_dev_iterate, 380 }; 381 382 RTE_REGISTER_CLASS(vdpa, rte_class_vdpa); 383