1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2017 Red Hat, Inc. 3 */ 4 5 #ifdef RTE_LIBRTE_VHOST_NUMA 6 #include <numaif.h> 7 #endif 8 9 #include <rte_tailq.h> 10 11 #include "iotlb.h" 12 #include "vhost.h" 13 14 struct vhost_iotlb_entry { 15 TAILQ_ENTRY(vhost_iotlb_entry) next; 16 17 uint64_t iova; 18 uint64_t uaddr; 19 uint64_t size; 20 uint8_t perm; 21 }; 22 23 #define IOTLB_CACHE_SIZE 2048 24 25 static void 26 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq); 27 28 static void 29 vhost_user_iotlb_pending_remove_all(struct vhost_virtqueue *vq) 30 { 31 struct vhost_iotlb_entry *node, *temp_node; 32 33 rte_rwlock_write_lock(&vq->iotlb_pending_lock); 34 35 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, temp_node) { 36 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next); 37 rte_mempool_put(vq->iotlb_pool, node); 38 } 39 40 rte_rwlock_write_unlock(&vq->iotlb_pending_lock); 41 } 42 43 bool 44 vhost_user_iotlb_pending_miss(struct vhost_virtqueue *vq, uint64_t iova, 45 uint8_t perm) 46 { 47 struct vhost_iotlb_entry *node; 48 bool found = false; 49 50 rte_rwlock_read_lock(&vq->iotlb_pending_lock); 51 52 TAILQ_FOREACH(node, &vq->iotlb_pending_list, next) { 53 if ((node->iova == iova) && (node->perm == perm)) { 54 found = true; 55 break; 56 } 57 } 58 59 rte_rwlock_read_unlock(&vq->iotlb_pending_lock); 60 61 return found; 62 } 63 64 void 65 vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq, 66 uint64_t iova, uint8_t perm) 67 { 68 struct vhost_iotlb_entry *node; 69 int ret; 70 71 ret = rte_mempool_get(vq->iotlb_pool, (void **)&node); 72 if (ret) { 73 VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n"); 74 if (!TAILQ_EMPTY(&vq->iotlb_pending_list)) 75 vhost_user_iotlb_pending_remove_all(vq); 76 else 77 vhost_user_iotlb_cache_random_evict(vq); 78 ret = rte_mempool_get(vq->iotlb_pool, (void **)&node); 79 if (ret) { 80 VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n"); 81 return; 82 } 83 } 84 85 node->iova = iova; 86 node->perm = perm; 87 88 rte_rwlock_write_lock(&vq->iotlb_pending_lock); 89 90 TAILQ_INSERT_TAIL(&vq->iotlb_pending_list, node, next); 91 92 rte_rwlock_write_unlock(&vq->iotlb_pending_lock); 93 } 94 95 void 96 vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq, 97 uint64_t iova, uint64_t size, uint8_t perm) 98 { 99 struct vhost_iotlb_entry *node, *temp_node; 100 101 rte_rwlock_write_lock(&vq->iotlb_pending_lock); 102 103 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_pending_list, next, 104 temp_node) { 105 if (node->iova < iova) 106 continue; 107 if (node->iova >= iova + size) 108 continue; 109 if ((node->perm & perm) != node->perm) 110 continue; 111 TAILQ_REMOVE(&vq->iotlb_pending_list, node, next); 112 rte_mempool_put(vq->iotlb_pool, node); 113 } 114 115 rte_rwlock_write_unlock(&vq->iotlb_pending_lock); 116 } 117 118 static void 119 vhost_user_iotlb_cache_remove_all(struct vhost_virtqueue *vq) 120 { 121 struct vhost_iotlb_entry *node, *temp_node; 122 123 rte_rwlock_write_lock(&vq->iotlb_lock); 124 125 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) { 126 TAILQ_REMOVE(&vq->iotlb_list, node, next); 127 rte_mempool_put(vq->iotlb_pool, node); 128 } 129 130 vq->iotlb_cache_nr = 0; 131 132 rte_rwlock_write_unlock(&vq->iotlb_lock); 133 } 134 135 static void 136 vhost_user_iotlb_cache_random_evict(struct vhost_virtqueue *vq) 137 { 138 struct vhost_iotlb_entry *node, *temp_node; 139 int entry_idx; 140 141 rte_rwlock_write_lock(&vq->iotlb_lock); 142 143 entry_idx = rte_rand() % vq->iotlb_cache_nr; 144 145 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) { 146 if (!entry_idx) { 147 TAILQ_REMOVE(&vq->iotlb_list, node, next); 148 rte_mempool_put(vq->iotlb_pool, node); 149 vq->iotlb_cache_nr--; 150 break; 151 } 152 entry_idx--; 153 } 154 155 rte_rwlock_write_unlock(&vq->iotlb_lock); 156 } 157 158 void 159 vhost_user_iotlb_cache_insert(struct vhost_virtqueue *vq, uint64_t iova, 160 uint64_t uaddr, uint64_t size, uint8_t perm) 161 { 162 struct vhost_iotlb_entry *node, *new_node; 163 int ret; 164 165 ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node); 166 if (ret) { 167 VHOST_LOG_CONFIG(DEBUG, "IOTLB pool empty, clear entries\n"); 168 if (!TAILQ_EMPTY(&vq->iotlb_list)) 169 vhost_user_iotlb_cache_random_evict(vq); 170 else 171 vhost_user_iotlb_pending_remove_all(vq); 172 ret = rte_mempool_get(vq->iotlb_pool, (void **)&new_node); 173 if (ret) { 174 VHOST_LOG_CONFIG(ERR, "IOTLB pool still empty, failure\n"); 175 return; 176 } 177 } 178 179 new_node->iova = iova; 180 new_node->uaddr = uaddr; 181 new_node->size = size; 182 new_node->perm = perm; 183 184 rte_rwlock_write_lock(&vq->iotlb_lock); 185 186 TAILQ_FOREACH(node, &vq->iotlb_list, next) { 187 /* 188 * Entries must be invalidated before being updated. 189 * So if iova already in list, assume identical. 190 */ 191 if (node->iova == new_node->iova) { 192 rte_mempool_put(vq->iotlb_pool, new_node); 193 goto unlock; 194 } else if (node->iova > new_node->iova) { 195 TAILQ_INSERT_BEFORE(node, new_node, next); 196 vq->iotlb_cache_nr++; 197 goto unlock; 198 } 199 } 200 201 TAILQ_INSERT_TAIL(&vq->iotlb_list, new_node, next); 202 vq->iotlb_cache_nr++; 203 204 unlock: 205 vhost_user_iotlb_pending_remove(vq, iova, size, perm); 206 207 rte_rwlock_write_unlock(&vq->iotlb_lock); 208 209 } 210 211 void 212 vhost_user_iotlb_cache_remove(struct vhost_virtqueue *vq, 213 uint64_t iova, uint64_t size) 214 { 215 struct vhost_iotlb_entry *node, *temp_node; 216 217 if (unlikely(!size)) 218 return; 219 220 rte_rwlock_write_lock(&vq->iotlb_lock); 221 222 RTE_TAILQ_FOREACH_SAFE(node, &vq->iotlb_list, next, temp_node) { 223 /* Sorted list */ 224 if (unlikely(iova + size < node->iova)) 225 break; 226 227 if (iova < node->iova + node->size) { 228 TAILQ_REMOVE(&vq->iotlb_list, node, next); 229 rte_mempool_put(vq->iotlb_pool, node); 230 vq->iotlb_cache_nr--; 231 } 232 } 233 234 rte_rwlock_write_unlock(&vq->iotlb_lock); 235 } 236 237 uint64_t 238 vhost_user_iotlb_cache_find(struct vhost_virtqueue *vq, uint64_t iova, 239 uint64_t *size, uint8_t perm) 240 { 241 struct vhost_iotlb_entry *node; 242 uint64_t offset, vva = 0, mapped = 0; 243 244 if (unlikely(!*size)) 245 goto out; 246 247 TAILQ_FOREACH(node, &vq->iotlb_list, next) { 248 /* List sorted by iova */ 249 if (unlikely(iova < node->iova)) 250 break; 251 252 if (iova >= node->iova + node->size) 253 continue; 254 255 if (unlikely((perm & node->perm) != perm)) { 256 vva = 0; 257 break; 258 } 259 260 offset = iova - node->iova; 261 if (!vva) 262 vva = node->uaddr + offset; 263 264 mapped += node->size - offset; 265 iova = node->iova + node->size; 266 267 if (mapped >= *size) 268 break; 269 } 270 271 out: 272 /* Only part of the requested chunk is mapped */ 273 if (unlikely(mapped < *size)) 274 *size = mapped; 275 276 return vva; 277 } 278 279 void 280 vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq) 281 { 282 vhost_user_iotlb_cache_remove_all(vq); 283 vhost_user_iotlb_pending_remove_all(vq); 284 } 285 286 int 287 vhost_user_iotlb_init(struct virtio_net *dev, int vq_index) 288 { 289 char pool_name[RTE_MEMPOOL_NAMESIZE]; 290 struct vhost_virtqueue *vq = dev->virtqueue[vq_index]; 291 int socket = 0; 292 293 if (vq->iotlb_pool) { 294 /* 295 * The cache has already been initialized, 296 * just drop all cached and pending entries. 297 */ 298 vhost_user_iotlb_flush_all(vq); 299 } 300 301 #ifdef RTE_LIBRTE_VHOST_NUMA 302 if (get_mempolicy(&socket, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR) != 0) 303 socket = 0; 304 #endif 305 306 rte_rwlock_init(&vq->iotlb_lock); 307 rte_rwlock_init(&vq->iotlb_pending_lock); 308 309 TAILQ_INIT(&vq->iotlb_list); 310 TAILQ_INIT(&vq->iotlb_pending_list); 311 312 snprintf(pool_name, sizeof(pool_name), "iotlb_%u_%d_%d", 313 getpid(), dev->vid, vq_index); 314 VHOST_LOG_CONFIG(DEBUG, "IOTLB cache name: %s\n", pool_name); 315 316 /* If already created, free it and recreate */ 317 vq->iotlb_pool = rte_mempool_lookup(pool_name); 318 if (vq->iotlb_pool) 319 rte_mempool_free(vq->iotlb_pool); 320 321 vq->iotlb_pool = rte_mempool_create(pool_name, 322 IOTLB_CACHE_SIZE, sizeof(struct vhost_iotlb_entry), 0, 323 0, 0, NULL, NULL, NULL, socket, 324 RTE_MEMPOOL_F_NO_CACHE_ALIGN | 325 RTE_MEMPOOL_F_SP_PUT); 326 if (!vq->iotlb_pool) { 327 VHOST_LOG_CONFIG(ERR, 328 "Failed to create IOTLB cache pool (%s)\n", 329 pool_name); 330 return -1; 331 } 332 333 vq->iotlb_cache_nr = 0; 334 335 return 0; 336 } 337