35 virtio_init_vring(struct virtqueue *vq)  in virtio_init_vring()  argument
37 	int size = vq->vq_nentries;  in virtio_init_vring()
38 	struct vring *vr = &vq->vq_ring;  in virtio_init_vring()
39 	uint8_t *ring_mem = vq->vq_ring_virt_mem;  in virtio_init_vring()
44 	memset(ring_mem, 0, vq->vq_ring_size);  in virtio_init_vring()
46 	vq->vq_used_cons_idx = 0;  in virtio_init_vring()
47 	vq->vq_desc_head_idx = 0;  in virtio_init_vring()
48 	vq->vq_avail_idx = 0;  in virtio_init_vring()
49 	vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);  in virtio_init_vring()
50 	vq->vq_free_cnt = vq->vq_nentries;  in virtio_init_vring()
51 	vq->req_start = VQ_RING_DESC_CHAIN_END;  in virtio_init_vring()
52 	vq->req_end = VQ_RING_DESC_CHAIN_END;  in virtio_init_vring()
53 	vq->reqs_finished = 0;  in virtio_init_vring()
54 	memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);  in virtio_init_vring()
63 	if (vq->vdev->negotiated_features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {  in virtio_init_vring()
64 		vring_used_event(&vq->vq_ring) = UINT16_MAX;  in virtio_init_vring()
66 		vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;  in virtio_init_vring()
74 	struct virtqueue *vq;  in virtio_init_queue()  local
96 	size = sizeof(*vq) + vq_size * sizeof(struct vq_desc_extra);  in virtio_init_queue()
98 	if (posix_memalign((void **)&vq, SPDK_CACHE_LINE_SIZE, size)) {  in virtio_init_queue()
102 	memset(vq, 0, size);  in virtio_init_queue()
103 	dev->vqs[vtpci_queue_idx] = vq;  in virtio_init_queue()
105 	vq->vdev = dev;  in virtio_init_queue()
106 	vq->vq_queue_index = vtpci_queue_idx;  in virtio_init_queue()
107 	vq->vq_nentries = vq_size;  in virtio_init_queue()
113 	vq->vq_ring_size = SPDK_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);  in virtio_init_queue()
115 		      size, vq->vq_ring_size);  in virtio_init_queue()
117 	vq->owner_thread = NULL;  in virtio_init_queue()
119 	rc = virtio_dev_backend_ops(dev)->setup_queue(dev, vq);  in virtio_init_queue()
122 		free(vq);  in virtio_init_queue()
128 		      vq->vq_ring_mem);  in virtio_init_queue()
130 		      (uint64_t)(uintptr_t)vq->vq_ring_virt_mem);  in virtio_init_queue()
132 	virtio_init_vring(vq);  in virtio_init_queue()
140 	struct virtqueue *vq;  in virtio_free_queues()  local
148 		vq = dev->vqs[i];  in virtio_free_queues()
149 		if (!vq) {  in virtio_free_queues()
153 		virtio_dev_backend_ops(dev)->del_queue(dev, vq);  in virtio_free_queues()
155 		free(vq);  in virtio_free_queues()
301 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)  in vq_ring_free_chain()  argument
307 	dp  = &vq->vq_ring.desc[desc_idx];  in vq_ring_free_chain()
308 	dxp = &vq->vq_descx[desc_idx];  in vq_ring_free_chain()
309 	vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);  in vq_ring_free_chain()
313 			dp = &vq->vq_ring.desc[dp->next];  in vq_ring_free_chain()
323 	if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {  in vq_ring_free_chain()
324 		vq->vq_desc_head_idx = desc_idx;  in vq_ring_free_chain()
326 		dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx];  in vq_ring_free_chain()
330 	vq->vq_desc_tail_idx = desc_idx_last;  in vq_ring_free_chain()
335 virtqueue_dequeue_burst_rx(struct virtqueue *vq, void **rx_pkts,  in virtqueue_dequeue_burst_rx()  argument
345 		used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));  in virtqueue_dequeue_burst_rx()
346 		uep = &vq->vq_ring.used->ring[used_idx];  in virtqueue_dequeue_burst_rx()
349 		cookie = vq->vq_descx[desc_idx].cookie;  in virtqueue_dequeue_burst_rx()
353 				     vq->vq_used_cons_idx);  in virtqueue_dequeue_burst_rx()
360 		vq->vq_used_cons_idx++;  in virtqueue_dequeue_burst_rx()
361 		vq_ring_free_chain(vq, desc_idx);  in virtqueue_dequeue_burst_rx()
362 		vq->vq_descx[desc_idx].cookie = NULL;  in virtqueue_dequeue_burst_rx()
369 finish_req(struct virtqueue *vq)  in finish_req()  argument
374 	desc = &vq->vq_ring.desc[vq->req_end];  in finish_req()
384 	avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1));  in finish_req()
385 	vq->vq_ring.avail->ring[avail_idx] = vq->req_start;  in finish_req()
386 	vq->vq_avail_idx++;  in finish_req()
387 	vq->req_end = VQ_RING_DESC_CHAIN_END;  in finish_req()
389 	vq->vq_ring.avail->idx = vq->vq_avail_idx;  in finish_req()
390 	vq->reqs_finished++;  in finish_req()
394 virtqueue_req_start(struct virtqueue *vq, void *cookie, int iovcnt)  in virtqueue_req_start()  argument
399 	if (2 * iovcnt > vq->vq_free_cnt) {  in virtqueue_req_start()
400 		return iovcnt > vq->vq_nentries ? -EINVAL : -ENOMEM;  in virtqueue_req_start()
403 	if (vq->req_end != VQ_RING_DESC_CHAIN_END) {  in virtqueue_req_start()
404 		finish_req(vq);  in virtqueue_req_start()
407 	vq->req_start = vq->vq_desc_head_idx;  in virtqueue_req_start()
408 	dxp = &vq->vq_descx[vq->req_start];  in virtqueue_req_start()
416 virtqueue_req_flush(struct virtqueue *vq)  in virtqueue_req_flush()  argument
420 	if (vq->req_end == VQ_RING_DESC_CHAIN_END) {  in virtqueue_req_flush()
425 	finish_req(vq);  in virtqueue_req_flush()
428 	reqs_finished = vq->reqs_finished;  in virtqueue_req_flush()
429 	vq->reqs_finished = 0;  in virtqueue_req_flush()
431 	if (vq->vdev->negotiated_features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {  in virtqueue_req_flush()
435 		vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx - vq->vq_nentries - 1;  in virtqueue_req_flush()
437 		if (!vring_need_event(vring_avail_event(&vq->vq_ring),  in virtqueue_req_flush()
438 				      vq->vq_avail_idx,  in virtqueue_req_flush()
439 				      vq->vq_avail_idx - reqs_finished)) {  in virtqueue_req_flush()
442 	} else if (vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) {  in virtqueue_req_flush()
446 	virtio_dev_backend_ops(vq->vdev)->notify_queue(vq->vdev, vq);  in virtqueue_req_flush()
451 virtqueue_req_abort(struct virtqueue *vq)  in virtqueue_req_abort()  argument
455 	if (vq->req_start == VQ_RING_DESC_CHAIN_END) {  in virtqueue_req_abort()
460 	desc = &vq->vq_ring.desc[vq->req_end];  in virtqueue_req_abort()
463 	vq_ring_free_chain(vq, vq->req_start);  in virtqueue_req_abort()
464 	vq->req_start = VQ_RING_DESC_CHAIN_END;  in virtqueue_req_abort()
468 virtqueue_req_add_iovs(struct virtqueue *vq, struct iovec *iovs, uint16_t iovcnt,  in virtqueue_req_add_iovs()  argument
478 	assert(vq->req_start != VQ_RING_DESC_CHAIN_END);  in virtqueue_req_add_iovs()
479 	assert(iovcnt <= vq->vq_free_cnt);  in virtqueue_req_add_iovs()
485 	prev_head = vq->req_end;  in virtqueue_req_add_iovs()
486 	new_head = vq->vq_desc_head_idx;  in virtqueue_req_add_iovs()
493 			desc = &vq->vq_ring.desc[new_head];  in virtqueue_req_add_iovs()
496 			if (!vq->vdev->is_hw) {  in virtqueue_req_add_iovs()
517 	dxp = &vq->vq_descx[vq->req_start];  in virtqueue_req_add_iovs()
520 	vq->req_end = prev_head;  in virtqueue_req_add_iovs()
521 	vq->vq_desc_head_idx = new_head;  in virtqueue_req_add_iovs()
522 	vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - used_desc_count);  in virtqueue_req_add_iovs()
523 	if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) {  in virtqueue_req_add_iovs()
524 		assert(vq->vq_free_cnt == 0);  in virtqueue_req_add_iovs()
525 		vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;  in virtqueue_req_add_iovs()
531 virtio_recv_pkts(struct virtqueue *vq, void **io, uint32_t *len, uint16_t nb_pkts)  in virtio_recv_pkts()  argument
535 	nb_used = vq->vq_ring.used->idx - vq->vq_used_cons_idx;  in virtio_recv_pkts()
540 		num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);  in virtio_recv_pkts()
543 	return virtqueue_dequeue_burst_rx(vq, io, len, num);  in virtio_recv_pkts()
549 	struct virtqueue *vq = NULL;  in virtio_dev_acquire_queue()  local
558 	vq = vdev->vqs[index];  in virtio_dev_acquire_queue()
559 	if (vq == NULL || vq->owner_thread != NULL) {  in virtio_dev_acquire_queue()
564 	vq->owner_thread = spdk_get_thread();  in virtio_dev_acquire_queue()
572 	struct virtqueue *vq = NULL;  in virtio_dev_find_and_acquire_queue()  local
577 		vq = vdev->vqs[i];  in virtio_dev_find_and_acquire_queue()
578 		if (vq != NULL && vq->owner_thread == NULL) {  in virtio_dev_find_and_acquire_queue()
583 	if (vq == NULL || i == vdev->max_queues) {  in virtio_dev_find_and_acquire_queue()
589 	vq->owner_thread = spdk_get_thread();  in virtio_dev_find_and_acquire_queue()
621 	struct virtqueue *vq = NULL;  in virtio_dev_release_queue()  local
630 	vq = vdev->vqs[index];  in virtio_dev_release_queue()
631 	if (vq == NULL) {  in virtio_dev_release_queue()
637 	assert(vq->owner_thread == spdk_get_thread());  in virtio_dev_release_queue()
638 	vq->owner_thread = NULL;  in virtio_dev_release_queue()