57 vhost_user_iotlb_set_dump(struct virtio_net *dev, struct vhost_iotlb_entry *node)  in vhost_user_iotlb_set_dump()  argument
61 	start = node->uaddr + node->uoffset;  in vhost_user_iotlb_set_dump()
62 	mem_set_dump(dev, (void *)(uintptr_t)start, node->size, true, RTE_BIT64(node->page_shift));  in vhost_user_iotlb_set_dump()
66 vhost_user_iotlb_clear_dump(struct virtio_net *dev, struct vhost_iotlb_entry *node,  in vhost_user_iotlb_clear_dump()  argument
71 	start = node->uaddr + node->uoffset;  in vhost_user_iotlb_clear_dump()
72 	end = start + node->size;  in vhost_user_iotlb_clear_dump()
75 	if (vhost_user_iotlb_share_page(prev, node))  in vhost_user_iotlb_clear_dump()
76 		start = RTE_ALIGN_CEIL(start, RTE_BIT64(node->page_shift));  in vhost_user_iotlb_clear_dump()
79 	if (vhost_user_iotlb_share_page(node, next))  in vhost_user_iotlb_clear_dump()
80 		end = RTE_ALIGN_FLOOR(end, RTE_BIT64(node->page_shift));  in vhost_user_iotlb_clear_dump()
84 			RTE_BIT64(node->page_shift));  in vhost_user_iotlb_clear_dump()
90 	struct vhost_iotlb_entry *node;  in vhost_user_iotlb_pool_get()  local
93 	node = SLIST_FIRST(&dev->iotlb_free_list);  in vhost_user_iotlb_pool_get()
94 	if (node != NULL)  in vhost_user_iotlb_pool_get()
97 	return node;  in vhost_user_iotlb_pool_get()
101 vhost_user_iotlb_pool_put(struct virtio_net *dev, struct vhost_iotlb_entry *node)  in vhost_user_iotlb_pool_put()  argument
104 	SLIST_INSERT_HEAD(&dev->iotlb_free_list, node, next_free);  in vhost_user_iotlb_pool_put()
114 	struct vhost_iotlb_entry *node, *temp_node;  in vhost_user_iotlb_pending_remove_all()  local
118 	RTE_TAILQ_FOREACH_SAFE(node, &dev->iotlb_pending_list, next, temp_node) {  in vhost_user_iotlb_pending_remove_all()
119 		TAILQ_REMOVE(&dev->iotlb_pending_list, node, next);  in vhost_user_iotlb_pending_remove_all()
120 		vhost_user_iotlb_pool_put(dev, node);  in vhost_user_iotlb_pending_remove_all()
129 	struct vhost_iotlb_entry *node;  in vhost_user_iotlb_pending_miss()  local
134 	TAILQ_FOREACH(node, &dev->iotlb_pending_list, next) {  in vhost_user_iotlb_pending_miss()
135 		if ((node->iova == iova) && (node->perm == perm)) {  in vhost_user_iotlb_pending_miss()
149 	struct vhost_iotlb_entry *node;  in vhost_user_iotlb_pending_insert()  local
151 	node = vhost_user_iotlb_pool_get(dev);  in vhost_user_iotlb_pending_insert()
152 	if (node == NULL) {  in vhost_user_iotlb_pending_insert()
159 		node = vhost_user_iotlb_pool_get(dev);  in vhost_user_iotlb_pending_insert()
160 		if (node == NULL) {  in vhost_user_iotlb_pending_insert()
167 	node->iova = iova;  in vhost_user_iotlb_pending_insert()
168 	node->perm = perm;  in vhost_user_iotlb_pending_insert()
172 	TAILQ_INSERT_TAIL(&dev->iotlb_pending_list, node, next);  in vhost_user_iotlb_pending_insert()
180 	struct vhost_iotlb_entry *node, *temp_node;  in vhost_user_iotlb_pending_remove()  local
184 	RTE_TAILQ_FOREACH_SAFE(node, &dev->iotlb_pending_list, next,  in vhost_user_iotlb_pending_remove()
186 		if (node->iova < iova)  in vhost_user_iotlb_pending_remove()
188 		if (node->iova >= iova + size)  in vhost_user_iotlb_pending_remove()
190 		if ((node->perm & perm) != node->perm)  in vhost_user_iotlb_pending_remove()
192 		TAILQ_REMOVE(&dev->iotlb_pending_list, node, next);  in vhost_user_iotlb_pending_remove()
193 		vhost_user_iotlb_pool_put(dev, node);  in vhost_user_iotlb_pending_remove()
202 	struct vhost_iotlb_entry *node, *temp_node;  in vhost_user_iotlb_cache_remove_all()  local
206 	RTE_TAILQ_FOREACH_SAFE(node, &dev->iotlb_list, next, temp_node) {  in vhost_user_iotlb_cache_remove_all()
207 		vhost_user_iotlb_clear_dump(dev, node, NULL, NULL);  in vhost_user_iotlb_cache_remove_all()
209 		TAILQ_REMOVE(&dev->iotlb_list, node, next);  in vhost_user_iotlb_cache_remove_all()
210 		vhost_user_iotlb_remove_notify(dev, node);  in vhost_user_iotlb_cache_remove_all()
211 		vhost_user_iotlb_pool_put(dev, node);  in vhost_user_iotlb_cache_remove_all()
222 	struct vhost_iotlb_entry *node, *temp_node, *prev_node = NULL;  in vhost_user_iotlb_cache_random_evict()  local
229 	RTE_TAILQ_FOREACH_SAFE(node, &dev->iotlb_list, next, temp_node) {  in vhost_user_iotlb_cache_random_evict()
231 			struct vhost_iotlb_entry *next_node = RTE_TAILQ_NEXT(node, next);  in vhost_user_iotlb_cache_random_evict()
233 			vhost_user_iotlb_clear_dump(dev, node, prev_node, next_node);  in vhost_user_iotlb_cache_random_evict()
235 			TAILQ_REMOVE(&dev->iotlb_list, node, next);  in vhost_user_iotlb_cache_random_evict()
236 			vhost_user_iotlb_remove_notify(dev, node);  in vhost_user_iotlb_cache_random_evict()
237 			vhost_user_iotlb_pool_put(dev, node);  in vhost_user_iotlb_cache_random_evict()
241 		prev_node = node;  in vhost_user_iotlb_cache_random_evict()
252 	struct vhost_iotlb_entry *node, *new_node;  in vhost_user_iotlb_cache_insert()  local
279 	TAILQ_FOREACH(node, &dev->iotlb_list, next) {  in vhost_user_iotlb_cache_insert()
284 		if (node->iova == new_node->iova) {  in vhost_user_iotlb_cache_insert()
287 		} else if (node->iova > new_node->iova) {  in vhost_user_iotlb_cache_insert()
290 			TAILQ_INSERT_BEFORE(node, new_node, next);  in vhost_user_iotlb_cache_insert()
310 	struct vhost_iotlb_entry *node, *temp_node, *prev_node = NULL;  in vhost_user_iotlb_cache_remove()  local
317 	RTE_TAILQ_FOREACH_SAFE(node, &dev->iotlb_list, next, temp_node) {  in vhost_user_iotlb_cache_remove()
319 		if (unlikely(iova + size < node->iova))  in vhost_user_iotlb_cache_remove()
322 		if (iova < node->iova + node->size) {  in vhost_user_iotlb_cache_remove()
323 			struct vhost_iotlb_entry *next_node = RTE_TAILQ_NEXT(node, next);  in vhost_user_iotlb_cache_remove()
325 			vhost_user_iotlb_clear_dump(dev, node, prev_node, next_node);  in vhost_user_iotlb_cache_remove()
327 			TAILQ_REMOVE(&dev->iotlb_list, node, next);  in vhost_user_iotlb_cache_remove()
328 			vhost_user_iotlb_remove_notify(dev, node);  in vhost_user_iotlb_cache_remove()
329 			vhost_user_iotlb_pool_put(dev, node);  in vhost_user_iotlb_cache_remove()
332 			prev_node = node;  in vhost_user_iotlb_cache_remove()
342 	struct vhost_iotlb_entry *node;  in vhost_user_iotlb_cache_find()  local
348 	TAILQ_FOREACH(node, &dev->iotlb_list, next) {  in vhost_user_iotlb_cache_find()
350 		if (unlikely(iova < node->iova))  in vhost_user_iotlb_cache_find()
353 		if (iova >= node->iova + node->size)  in vhost_user_iotlb_cache_find()
356 		if (unlikely((perm & node->perm) != perm)) {  in vhost_user_iotlb_cache_find()
361 		offset = iova - node->iova;  in vhost_user_iotlb_cache_find()
363 			vva = node->uaddr + node->uoffset + offset;  in vhost_user_iotlb_cache_find()
365 		mapped += node->size - offset;  in vhost_user_iotlb_cache_find()
366 		iova = node->iova + node->size;  in vhost_user_iotlb_cache_find()