1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 /** 6 * @file 7 * 8 * Device specific vhost lib 9 */ 10 11 #include <sys/queue.h> 12 13 #include <dev_driver.h> 14 #include <rte_class.h> 15 #include <rte_malloc.h> 16 #include <rte_spinlock.h> 17 #include <rte_tailq.h> 18 19 #include "rte_vdpa.h" 20 #include "vdpa_driver.h" 21 #include "vhost.h" 22 #include "iotlb.h" 23 24 /** Double linked list of vDPA devices. */ 25 TAILQ_HEAD(vdpa_device_list, rte_vdpa_device); 26 27 static struct vdpa_device_list vdpa_device_list__ = 28 TAILQ_HEAD_INITIALIZER(vdpa_device_list__); 29 static rte_spinlock_t vdpa_device_list_lock = RTE_SPINLOCK_INITIALIZER; 30 static struct vdpa_device_list * const vdpa_device_list 31 __rte_guarded_by(&vdpa_device_list_lock) = &vdpa_device_list__; 32 33 static struct rte_vdpa_device * 34 __vdpa_find_device_by_name(const char *name) 35 __rte_exclusive_locks_required(&vdpa_device_list_lock) 36 { 37 struct rte_vdpa_device *dev, *ret = NULL; 38 39 if (name == NULL) 40 return NULL; 41 42 TAILQ_FOREACH(dev, vdpa_device_list, next) { 43 if (!strncmp(dev->device->name, name, RTE_DEV_NAME_MAX_LEN)) { 44 ret = dev; 45 break; 46 } 47 } 48 49 return ret; 50 } 51 52 struct rte_vdpa_device * 53 rte_vdpa_find_device_by_name(const char *name) 54 { 55 struct rte_vdpa_device *dev; 56 57 rte_spinlock_lock(&vdpa_device_list_lock); 58 dev = __vdpa_find_device_by_name(name); 59 rte_spinlock_unlock(&vdpa_device_list_lock); 60 61 return dev; 62 } 63 64 struct rte_device * 65 rte_vdpa_get_rte_device(struct rte_vdpa_device *vdpa_dev) 66 { 67 if (vdpa_dev == NULL) 68 return NULL; 69 70 return vdpa_dev->device; 71 } 72 73 struct rte_vdpa_device * 74 rte_vdpa_register_device(struct rte_device *rte_dev, 75 struct rte_vdpa_dev_ops *ops) 76 { 77 struct rte_vdpa_device *dev; 78 int ret = 0; 79 80 if (ops == NULL) 81 return NULL; 82 83 /* Check mandatory ops are implemented */ 84 if (!ops->get_queue_num || !ops->get_features || 85 !ops->get_protocol_features || !ops->dev_conf || 86 !ops->dev_close || !ops->set_vring_state || 87 !ops->set_features) { 88 VHOST_CONFIG_LOG(rte_dev->name, ERR, 89 "Some mandatory vDPA ops aren't implemented"); 90 return NULL; 91 } 92 93 rte_spinlock_lock(&vdpa_device_list_lock); 94 /* Check the device hasn't been register already */ 95 dev = __vdpa_find_device_by_name(rte_dev->name); 96 if (dev) { 97 dev = NULL; 98 goto out_unlock; 99 } 100 101 dev = rte_zmalloc(NULL, sizeof(*dev), 0); 102 if (!dev) 103 goto out_unlock; 104 105 dev->device = rte_dev; 106 dev->ops = ops; 107 108 if (ops->get_dev_type) { 109 ret = ops->get_dev_type(dev, &dev->type); 110 if (ret) { 111 VHOST_CONFIG_LOG(rte_dev->name, ERR, 112 "Failed to get vdpa dev type."); 113 ret = -1; 114 goto out_unlock; 115 } 116 } else { 117 /** by default, we assume vdpa device is a net device */ 118 dev->type = RTE_VHOST_VDPA_DEVICE_TYPE_NET; 119 } 120 121 TAILQ_INSERT_TAIL(vdpa_device_list, dev, next); 122 out_unlock: 123 rte_spinlock_unlock(&vdpa_device_list_lock); 124 125 return dev; 126 } 127 128 int 129 rte_vdpa_unregister_device(struct rte_vdpa_device *dev) 130 { 131 struct rte_vdpa_device *cur_dev, *tmp_dev; 132 int ret = -1; 133 134 rte_spinlock_lock(&vdpa_device_list_lock); 135 RTE_TAILQ_FOREACH_SAFE(cur_dev, vdpa_device_list, next, tmp_dev) { 136 if (dev != cur_dev) 137 continue; 138 139 TAILQ_REMOVE(vdpa_device_list, dev, next); 140 rte_free(dev); 141 ret = 0; 142 break; 143 } 144 rte_spinlock_unlock(&vdpa_device_list_lock); 145 146 return ret; 147 } 148 149 int 150 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m) 151 { 152 struct virtio_net *dev = get_device(vid); 153 uint16_t idx, idx_m, desc_id; 154 struct vhost_virtqueue *vq; 155 struct vring_desc desc; 156 struct vring_desc *desc_ring; 157 struct vring_desc *idesc = NULL; 158 struct vring *s_vring; 159 uint64_t dlen; 160 uint32_t nr_descs; 161 int ret; 162 163 if (!dev || !vring_m) 164 return -1; 165 166 if (qid >= dev->nr_vring) 167 return -1; 168 169 if (vq_is_packed(dev)) 170 return -1; 171 172 s_vring = (struct vring *)vring_m; 173 vq = dev->virtqueue[qid]; 174 idx = vq->used->idx; 175 idx_m = s_vring->used->idx; 176 ret = (uint16_t)(idx_m - idx); 177 178 while (idx != idx_m) { 179 /* copy used entry, used ring logging is not covered here */ 180 vq->used->ring[idx & (vq->size - 1)] = 181 s_vring->used->ring[idx & (vq->size - 1)]; 182 183 desc_id = vq->used->ring[idx & (vq->size - 1)].id; 184 desc_ring = vq->desc; 185 nr_descs = vq->size; 186 187 if (unlikely(desc_id >= vq->size)) 188 return -1; 189 190 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) { 191 dlen = vq->desc[desc_id].len; 192 nr_descs = dlen / sizeof(struct vring_desc); 193 if (unlikely(nr_descs > vq->size)) 194 return -1; 195 196 vhost_user_iotlb_rd_lock(vq); 197 desc_ring = (struct vring_desc *)(uintptr_t) 198 vhost_iova_to_vva(dev, vq, 199 vq->desc[desc_id].addr, &dlen, 200 VHOST_ACCESS_RO); 201 vhost_user_iotlb_rd_unlock(vq); 202 if (unlikely(!desc_ring)) 203 return -1; 204 205 if (unlikely(dlen < vq->desc[desc_id].len)) { 206 vhost_user_iotlb_rd_lock(vq); 207 idesc = vhost_alloc_copy_ind_table(dev, vq, 208 vq->desc[desc_id].addr, 209 vq->desc[desc_id].len); 210 vhost_user_iotlb_rd_unlock(vq); 211 if (unlikely(!idesc)) 212 return -1; 213 214 desc_ring = idesc; 215 } 216 217 desc_id = 0; 218 } 219 220 /* dirty page logging for DMA writeable buffer */ 221 do { 222 if (unlikely(desc_id >= vq->size)) 223 goto fail; 224 if (unlikely(nr_descs-- == 0)) 225 goto fail; 226 desc = desc_ring[desc_id]; 227 if (desc.flags & VRING_DESC_F_WRITE) { 228 vhost_user_iotlb_rd_lock(vq); 229 vhost_log_write_iova(dev, vq, desc.addr, 230 desc.len); 231 vhost_user_iotlb_rd_unlock(vq); 232 } 233 desc_id = desc.next; 234 } while (desc.flags & VRING_DESC_F_NEXT); 235 236 if (unlikely(idesc)) { 237 free_ind_table(idesc); 238 idesc = NULL; 239 } 240 241 idx++; 242 } 243 244 /* used idx is the synchronization point for the split vring */ 245 rte_atomic_store_explicit((unsigned short __rte_atomic *)&vq->used->idx, 246 idx_m, rte_memory_order_release); 247 248 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) 249 vring_used_event(s_vring) = idx_m; 250 251 return ret; 252 253 fail: 254 if (unlikely(idesc)) 255 free_ind_table(idesc); 256 return -1; 257 } 258 259 int 260 rte_vdpa_get_queue_num(struct rte_vdpa_device *dev, uint32_t *queue_num) 261 { 262 if (dev == NULL || dev->ops == NULL || dev->ops->get_queue_num == NULL) 263 return -1; 264 265 return dev->ops->get_queue_num(dev, queue_num); 266 } 267 268 int 269 rte_vdpa_get_features(struct rte_vdpa_device *dev, uint64_t *features) 270 { 271 if (dev == NULL || dev->ops == NULL || dev->ops->get_features == NULL) 272 return -1; 273 274 return dev->ops->get_features(dev, features); 275 } 276 277 int 278 rte_vdpa_get_protocol_features(struct rte_vdpa_device *dev, uint64_t *features) 279 { 280 if (dev == NULL || dev->ops == NULL || 281 dev->ops->get_protocol_features == NULL) 282 return -1; 283 284 return dev->ops->get_protocol_features(dev, features); 285 } 286 287 int 288 rte_vdpa_get_stats_names(struct rte_vdpa_device *dev, 289 struct rte_vdpa_stat_name *stats_names, 290 unsigned int size) 291 { 292 if (!dev) 293 return -EINVAL; 294 295 if (dev->ops->get_stats_names == NULL) 296 return -ENOTSUP; 297 298 return dev->ops->get_stats_names(dev, stats_names, size); 299 } 300 301 int 302 rte_vdpa_get_stats(struct rte_vdpa_device *dev, uint16_t qid, 303 struct rte_vdpa_stat *stats, unsigned int n) 304 { 305 if (!dev || !stats || !n) 306 return -EINVAL; 307 308 if (dev->ops->get_stats == NULL) 309 return -ENOTSUP; 310 311 return dev->ops->get_stats(dev, qid, stats, n); 312 } 313 314 int 315 rte_vdpa_reset_stats(struct rte_vdpa_device *dev, uint16_t qid) 316 { 317 if (!dev) 318 return -EINVAL; 319 320 if (dev->ops->reset_stats == NULL) 321 return -ENOTSUP; 322 323 return dev->ops->reset_stats(dev, qid); 324 } 325 326 static int 327 vdpa_dev_match(struct rte_vdpa_device *dev, 328 const struct rte_device *rte_dev) 329 { 330 if (dev->device == rte_dev) 331 return 0; 332 333 return -1; 334 } 335 336 /* Generic rte_vdpa_dev comparison function. */ 337 typedef int (*rte_vdpa_cmp_t)(struct rte_vdpa_device *, 338 const struct rte_device *rte_dev); 339 340 static struct rte_vdpa_device * 341 vdpa_find_device(const struct rte_vdpa_device *start, rte_vdpa_cmp_t cmp, 342 struct rte_device *rte_dev) 343 { 344 struct rte_vdpa_device *dev; 345 346 rte_spinlock_lock(&vdpa_device_list_lock); 347 if (start == NULL) 348 dev = TAILQ_FIRST(vdpa_device_list); 349 else 350 dev = TAILQ_NEXT(start, next); 351 352 while (dev != NULL) { 353 if (cmp(dev, rte_dev) == 0) 354 break; 355 356 dev = TAILQ_NEXT(dev, next); 357 } 358 rte_spinlock_unlock(&vdpa_device_list_lock); 359 360 return dev; 361 } 362 363 static void * 364 vdpa_dev_iterate(const void *start, 365 const char *str, 366 const struct rte_dev_iterator *it) 367 { 368 struct rte_vdpa_device *vdpa_dev = NULL; 369 370 RTE_SET_USED(str); 371 372 vdpa_dev = vdpa_find_device(start, vdpa_dev_match, it->device); 373 374 return vdpa_dev; 375 } 376 377 static struct rte_class rte_class_vdpa = { 378 .dev_iterate = vdpa_dev_iterate, 379 }; 380 381 RTE_REGISTER_CLASS(vdpa, rte_class_vdpa); 382