1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2017 Red Hat, Inc. 3 */ 4 5 #ifdef RTE_LIBRTE_VHOST_NUMA 6 #include <numaif.h> 7 #endif 8 9 #include <rte_tailq.h> 10 11 #include "iotlb.h" 12 #include "vhost.h" 13 14 struct vhost_iotlb_entry { 15 TAILQ_ENTRY(vhost_iotlb_entry) next; 16 SLIST_ENTRY(vhost_iotlb_entry) next_free; 17 18 uint64_t iova; 19 uint64_t uaddr; 20 uint64_t size; 21 uint8_t perm; 22 }; 23 24 #define IOTLB_CACHE_SIZE 2048 25 26 static struct vhost_iotlb_entry * 27 vhost_user_iotlb_pool_get(struct vhost_virtqueue *vq) 28 { 29 struct vhost_iotlb_entry *node; 30 31 rte_spinlock_lock(&vq->iotlb_free_lock); 32 node = SLIST_FIRST(&vq->iotlb_free_list); 33 if (node != NULL) 34 SLIST_REMOVE_HEAD(&vq->iotlb_free_list, next_free); 35 rte_spinlock_unlock(&vq->iotlb_free_lock); 36 return node; 37 } 38 39 static void 40 vhost_user_iotlb_pool_put(struct vhost_virtqueue *vq, 41 struct vhost_iotlb_entry *node) 42 { 43 rte_spinlock_lock(&vq->iotlb_free_lock); 44 SLIST_INSERT_HEAD(&vq->iotlb_free_list, node, next_free); 45 rte_spinlock_unlock(&vq->iotlb_free_lock); 46 } 47 48 static void 49 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq); 50 51 static void 52 vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq) 53 { 54 struct vhost_iotlb_entry *node, *temp_node; 55 56 rte_rwlock_write_lock(&vq->iotlb_pending_lock); 57 58 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) { 59 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next); 60 vhost_user_iotlb_pool_put(vq, node); 61 } 62 63 rte_rwlock_write_unlock(&vq->iotlb_pending_lock); 64 } 65 66 bool 67 vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova, 68 uint8_t perm) 69 { 70 struct vhost_iotlb_entry *node; 71 bool found = false; 72 73 rte_rwlock_read_lock(&vq->iotlb_pending_lock); 74 75 TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) { 76 if ((node->iova == iova) && (node->perm == perm)) { 77 found = true; 78 break; 79 } 80 } 81 82 rte_rwlock_read_unlock(&vq->iotlb_pending_lock); 83 84 return found; 85 } 86 87 void 88 vhost_user_iotlb_pending_insert(struct virtio_net *dev, struct vhost_virtqueue *vq, 89 uint64_t iova, uint8_t perm) 90 { 91 struct vhost_iotlb_entry *node; 92 93 node = vhost_user_iotlb_pool_get(vq); 94 if (node == NULL) { 95 VHOST_LOG_CONFIG(dev->ifname, DEBUG, 96 "IOTLB pool for vq %"PRIu32" empty, clear entries for pending insertion\n", 97 vq->index); 98 if (!TAILQ_EMPTY(&vq->iotlb_pending_list)) 99 vhost_user_iotlb_pending_remove_all(vq); 100 else 101 vhost_user_iotlb_cache_random_evict(vq); 102 node = vhost_user_iotlb_pool_get(vq); 103 if (node == NULL) { 104 VHOST_LOG_CONFIG(dev->ifname, ERR, 105 "IOTLB pool vq %"PRIu32" still empty, pending insertion failure\n", 106 vq->index); 107 return; 108 } 109 } 110 111 node->iova = iova; 112 node->perm = perm; 113 114 rte_rwlock_write_lock(&vq->iotlb_pending_lock); 115 116 TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next); 117 118 rte_rwlock_write_unlock(&vq->iotlb_pending_lock); 119 } 120 121 void 122 vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq, 123 uint64_t iova, uint64_t size, uint8_t perm) 124 { 125 struct vhost_iotlb_entry *node, *temp_node; 126 127 rte_rwlock_write_lock(&vq->iotlb_pending_lock); 128 129 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, 130 temp_node) { 131 if (node->iova < iova) 132 continue; 133 if (node->iova >= iova + size) 134 continue; 135 if ((node->perm & perm) != node->perm) 136 continue; 137 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next); 138 vhost_user_iotlb_pool_put(vq, node); 139 } 140 141 rte_rwlock_write_unlock(&vq->iotlb_pending_lock); 142 } 143 144 static void 145 vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq) 146 { 147 struct vhost_iotlb_entry *node, *temp_node; 148 149 rte_rwlock_write_lock(&vq->iotlb_lock); 150 151 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) { 152 TAILQ_REMOVE(&vq->iotlb_list, node, next); 153 vhost_user_iotlb_pool_put(vq, node); 154 } 155 156 vq->iotlb_cache_nr = 0; 157 158 rte_rwlock_write_unlock(&vq->iotlb_lock); 159 } 160 161 static void 162 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq) 163 { 164 struct vhost_iotlb_entry *node, *temp_node; 165 int entry_idx; 166 167 rte_rwlock_write_lock(&vq->iotlb_lock); 168 169 entry_idx = rte_rand() % vq->iotlb_cache_nr; 170 171 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) { 172 if (!entry_idx) { 173 TAILQ_REMOVE(&vq->iotlb_list, node, next); 174 vhost_user_iotlb_pool_put(vq, node); 175 vq->iotlb_cache_nr--; 176 break; 177 } 178 entry_idx--; 179 } 180 181 rte_rwlock_write_unlock(&vq->iotlb_lock); 182 } 183 184 void 185 vhost_user_iotlb_cache_insert(struct virtio_net *dev, struct vhost_virtqueue *vq, 186 uint64_t iova, uint64_t uaddr, 187 uint64_t size, uint8_t perm) 188 { 189 struct vhost_iotlb_entry *node, *new_node; 190 191 new_node = vhost_user_iotlb_pool_get(vq); 192 if (new_node == NULL) { 193 VHOST_LOG_CONFIG(dev->ifname, DEBUG, 194 "IOTLB pool vq %"PRIu32" empty, clear entries for cache insertion\n", 195 vq->index); 196 if (!TAILQ_EMPTY(&vq->iotlb_list)) 197 vhost_user_iotlb_cache_random_evict(vq); 198 else 199 vhost_user_iotlb_pending_remove_all(vq); 200 new_node = vhost_user_iotlb_pool_get(vq); 201 if (new_node == NULL) { 202 VHOST_LOG_CONFIG(dev->ifname, ERR, 203 "IOTLB pool vq %"PRIu32" still empty, cache insertion failed\n", 204 vq->index); 205 return; 206 } 207 } 208 209 new_node->iova = iova; 210 new_node->uaddr = uaddr; 211 new_node->size = size; 212 new_node->perm = perm; 213 214 rte_rwlock_write_lock(&vq->iotlb_lock); 215 216 TAILQ_FOREACH(node, &vq->iotlb_list, next) { 217 /* 218 * Entries must be invalidated before being updated. 219 * So if iova already in list, assume identical. 220 */ 221 if (node->iova == new_node->iova) { 222 vhost_user_iotlb_pool_put(vq, new_node); 223 goto unlock; 224 } else if (node->iova > new_node->iova) { 225 TAILQ_INSERT_BEFORE(node, new_node, next); 226 vq->iotlb_cache_nr++; 227 goto unlock; 228 } 229 } 230 231 TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next); 232 vq->iotlb_cache_nr++; 233 234 unlock: 235 vhost_user_iotlb_pending_remove(vq, iova, size, perm); 236 237 rte_rwlock_write_unlock(&vq->iotlb_lock); 238 239 } 240 241 void 242 vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq, 243 uint64_t iova, uint64_t size) 244 { 245 struct vhost_iotlb_entry *node, *temp_node; 246 247 if (unlikely(!size)) 248 return; 249 250 rte_rwlock_write_lock(&vq->iotlb_lock); 251 252 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) { 253 /* Sorted list */ 254 if (unlikely(iova + size < node->iova)) 255 break; 256 257 if (iova < node->iova + node->size) { 258 TAILQ_REMOVE(&vq->iotlb_list, node, next); 259 vhost_user_iotlb_pool_put(vq, node); 260 vq->iotlb_cache_nr--; 261 } 262 } 263 264 rte_rwlock_write_unlock(&vq->iotlb_lock); 265 } 266 267 uint64_t 268 vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova, 269 uint64_t *size, uint8_t perm) 270 { 271 struct vhost_iotlb_entry *node; 272 uint64_t offset, vva = 0, mapped = 0; 273 274 if (unlikely(!*size)) 275 goto out; 276 277 TAILQ_FOREACH(node, &vq->iotlb_list, next) { 278 /* List sorted by iova */ 279 if (unlikely(iova < node->iova)) 280 break; 281 282 if (iova >= node->iova + node->size) 283 continue; 284 285 if (unlikely((perm & node->perm) != perm)) { 286 vva = 0; 287 break; 288 } 289 290 offset = iova - node->iova; 291 if (!vva) 292 vva = node->uaddr + offset; 293 294 mapped += node->size - offset; 295 iova = node->iova + node->size; 296 297 if (mapped >= *size) 298 break; 299 } 300 301 out: 302 /* Only part of the requested chunk is mapped */ 303 if (unlikely(mapped < *size)) 304 *size = mapped; 305 306 return vva; 307 } 308 309 void 310 vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq) 311 { 312 vhost_user_iotlb_cache_remove_all(vq); 313 vhost_user_iotlb_pending_remove_all(vq); 314 } 315 316 int 317 vhost_user_iotlb_init(struct virtio_net *dev, struct vhost_virtqueue *vq) 318 { 319 unsigned int i; 320 int socket = 0; 321 322 if (vq->iotlb_pool) { 323 /* 324 * The cache has already been initialized, 325 * just drop all cached and pending entries. 326 */ 327 vhost_user_iotlb_flush_all(vq); 328 rte_free(vq->iotlb_pool); 329 } 330 331 #ifdef RTE_LIBRTE_VHOST_NUMA 332 if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0) 333 socket = 0; 334 #endif 335 336 rte_spinlock_init(&vq->iotlb_free_lock); 337 rte_rwlock_init(&vq->iotlb_lock); 338 rte_rwlock_init(&vq->iotlb_pending_lock); 339 340 SLIST_INIT(&vq->iotlb_free_list); 341 TAILQ_INIT(&vq->iotlb_list); 342 TAILQ_INIT(&vq->iotlb_pending_list); 343 344 vq->iotlb_pool = rte_calloc_socket("iotlb", IOTLB_CACHE_SIZE, 345 sizeof(struct vhost_iotlb_entry), 0, socket); 346 if (!vq->iotlb_pool) { 347 VHOST_LOG_CONFIG(dev->ifname, ERR, 348 "Failed to create IOTLB cache pool for vq %"PRIu32"\n", 349 vq->index); 350 return -1; 351 } 352 for (i = 0; i < IOTLB_CACHE_SIZE; i++) 353 vhost_user_iotlb_pool_put(vq, &vq->iotlb_pool[i]); 354 355 vq->iotlb_cache_nr = 0; 356 357 return 0; 358 } 359 360 void 361 vhost_user_iotlb_destroy(struct vhost_virtqueue *vq) 362 { 363 rte_free(vq->iotlb_pool); 364 } 365