1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2015 Intel Corporation
3 */
4 #include <stdint.h>
5 #include <unistd.h>
6
7 #include <rte_eal_paging.h>
8 #include <rte_malloc.h>
9 #include <rte_mbuf.h>
10 #include <rte_memzone.h>
11
12 #include "virtqueue.h"
13 #include "virtio_logs.h"
14 #include "virtio.h"
15 #include "virtio_rxtx_simple.h"
16
17 /*
18 * Two types of mbuf to be cleaned:
19 * 1) mbuf that has been consumed by backend but not used by virtio.
20 * 2) mbuf that hasn't been consumed by backend.
21 */
22 struct rte_mbuf *
virtqueue_detach_unused(struct virtqueue * vq)23 virtqueue_detach_unused(struct virtqueue *vq)
24 {
25 struct rte_mbuf *cookie;
26 struct virtio_hw *hw;
27 uint16_t start, end;
28 int type, idx;
29
30 if (vq == NULL)
31 return NULL;
32
33 hw = vq->hw;
34 type = virtio_get_queue_type(hw, vq->vq_queue_index);
35 start = vq->vq_avail_idx & (vq->vq_nentries - 1);
36 end = (vq->vq_avail_idx + vq->vq_free_cnt) & (vq->vq_nentries - 1);
37
38 for (idx = 0; idx < vq->vq_nentries; idx++) {
39 if (hw->use_vec_rx && !virtio_with_packed_queue(hw) &&
40 type == VTNET_RQ) {
41 if (start <= end && idx >= start && idx < end)
42 continue;
43 if (start > end && (idx >= start || idx < end))
44 continue;
45 cookie = vq->rxq.sw_ring[idx];
46 if (cookie != NULL) {
47 vq->rxq.sw_ring[idx] = NULL;
48 return cookie;
49 }
50 } else {
51 cookie = vq->vq_descx[idx].cookie;
52 if (cookie != NULL) {
53 vq->vq_descx[idx].cookie = NULL;
54 return cookie;
55 }
56 }
57 }
58
59 return NULL;
60 }
61
62 /* Flush used descs */
63 static void
virtqueue_rxvq_flush_packed(struct virtqueue * vq)64 virtqueue_rxvq_flush_packed(struct virtqueue *vq)
65 {
66 struct vq_desc_extra *dxp;
67 uint16_t i;
68
69 struct vring_packed_desc *descs = vq->vq_packed.ring.desc;
70 int cnt = 0;
71
72 i = vq->vq_used_cons_idx;
73 while (desc_is_used(&descs[i], vq) && cnt++ < vq->vq_nentries) {
74 dxp = &vq->vq_descx[descs[i].id];
75 if (dxp->cookie != NULL) {
76 rte_pktmbuf_free(dxp->cookie);
77 dxp->cookie = NULL;
78 }
79 vq->vq_free_cnt++;
80 vq->vq_used_cons_idx++;
81 if (vq->vq_used_cons_idx >= vq->vq_nentries) {
82 vq->vq_used_cons_idx -= vq->vq_nentries;
83 vq->vq_packed.used_wrap_counter ^= 1;
84 }
85 i = vq->vq_used_cons_idx;
86 }
87 }
88
89 /* Flush the elements in the used ring. */
90 static void
virtqueue_rxvq_flush_split(struct virtqueue * vq)91 virtqueue_rxvq_flush_split(struct virtqueue *vq)
92 {
93 struct virtnet_rx *rxq = &vq->rxq;
94 struct virtio_hw *hw = vq->hw;
95 struct vring_used_elem *uep;
96 struct vq_desc_extra *dxp;
97 uint16_t used_idx, desc_idx;
98 uint16_t nb_used, i;
99
100 nb_used = virtqueue_nused(vq);
101
102 for (i = 0; i < nb_used; i++) {
103 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1);
104 uep = &vq->vq_split.ring.used->ring[used_idx];
105 if (hw->use_vec_rx) {
106 desc_idx = used_idx;
107 rte_pktmbuf_free(vq->rxq.sw_ring[desc_idx]);
108 vq->vq_free_cnt++;
109 } else if (hw->use_inorder_rx) {
110 desc_idx = (uint16_t)uep->id;
111 dxp = &vq->vq_descx[desc_idx];
112 if (dxp->cookie != NULL) {
113 rte_pktmbuf_free(dxp->cookie);
114 dxp->cookie = NULL;
115 }
116 vq_ring_free_inorder(vq, desc_idx, 1);
117 } else {
118 desc_idx = (uint16_t)uep->id;
119 dxp = &vq->vq_descx[desc_idx];
120 if (dxp->cookie != NULL) {
121 rte_pktmbuf_free(dxp->cookie);
122 dxp->cookie = NULL;
123 }
124 vq_ring_free_chain(vq, desc_idx);
125 }
126 vq->vq_used_cons_idx++;
127 }
128
129 if (hw->use_vec_rx) {
130 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {
131 virtio_rxq_rearm_vec(rxq);
132 if (virtqueue_kick_prepare(vq))
133 virtqueue_notify(vq);
134 }
135 }
136 }
137
138 /* Flush the elements in the used ring. */
139 void
virtqueue_rxvq_flush(struct virtqueue * vq)140 virtqueue_rxvq_flush(struct virtqueue *vq)
141 {
142 struct virtio_hw *hw = vq->hw;
143
144 if (virtio_with_packed_queue(hw))
145 virtqueue_rxvq_flush_packed(vq);
146 else
147 virtqueue_rxvq_flush_split(vq);
148 }
149
150 static void
virtqueue_txq_indirect_header_init_packed(struct virtqueue * vq,uint32_t idx)151 virtqueue_txq_indirect_header_init_packed(struct virtqueue *vq, uint32_t idx)
152 {
153 struct virtio_tx_region *txr;
154 struct vring_packed_desc *desc;
155 rte_iova_t hdr_mem;
156
157 txr = vq->txq.hdr_mz->addr;
158 hdr_mem = vq->txq.hdr_mem;
159 desc = txr[idx].tx_packed_indir;
160
161 vring_desc_init_indirect_packed(desc, RTE_DIM(txr[idx].tx_packed_indir));
162 desc->addr = hdr_mem + idx * sizeof(*txr) + offsetof(struct virtio_tx_region, tx_hdr);
163 desc->len = vq->hw->vtnet_hdr_size;
164 }
165
166 static void
virtqueue_txq_indirect_header_init_split(struct virtqueue * vq,uint32_t idx)167 virtqueue_txq_indirect_header_init_split(struct virtqueue *vq, uint32_t idx)
168 {
169 struct virtio_tx_region *txr;
170 struct vring_desc *desc;
171 rte_iova_t hdr_mem;
172
173 txr = vq->txq.hdr_mz->addr;
174 hdr_mem = vq->txq.hdr_mem;
175 desc = txr[idx].tx_indir;
176
177 vring_desc_init_split(desc, RTE_DIM(txr[idx].tx_indir));
178 desc->addr = hdr_mem + idx * sizeof(*txr) + offsetof(struct virtio_tx_region, tx_hdr);
179 desc->len = vq->hw->vtnet_hdr_size;
180 desc->flags = VRING_DESC_F_NEXT;
181 }
182
183 void
virtqueue_txq_indirect_headers_init(struct virtqueue * vq)184 virtqueue_txq_indirect_headers_init(struct virtqueue *vq)
185 {
186 uint32_t i;
187
188 if (!virtio_with_feature(vq->hw, VIRTIO_RING_F_INDIRECT_DESC))
189 return;
190
191 for (i = 0; i < vq->vq_nentries; i++)
192 if (virtio_with_packed_queue(vq->hw))
193 virtqueue_txq_indirect_header_init_packed(vq, i);
194 else
195 virtqueue_txq_indirect_header_init_split(vq, i);
196 }
197
198 int
virtqueue_rxvq_reset_packed(struct virtqueue * vq)199 virtqueue_rxvq_reset_packed(struct virtqueue *vq)
200 {
201 int size = vq->vq_nentries;
202 struct vq_desc_extra *dxp;
203 uint16_t desc_idx;
204
205 vq->vq_used_cons_idx = 0;
206 vq->vq_desc_head_idx = 0;
207 vq->vq_avail_idx = 0;
208 vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
209 vq->vq_free_cnt = vq->vq_nentries;
210
211 vq->vq_packed.used_wrap_counter = 1;
212 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL;
213 vq->vq_packed.event_flags_shadow = 0;
214 vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE;
215
216 memset(vq->mz->addr, 0, vq->mz->len);
217
218 for (desc_idx = 0; desc_idx < vq->vq_nentries; desc_idx++) {
219 dxp = &vq->vq_descx[desc_idx];
220 if (dxp->cookie != NULL) {
221 rte_pktmbuf_free(dxp->cookie);
222 dxp->cookie = NULL;
223 }
224 }
225
226 vring_desc_init_packed(vq, size);
227
228 virtqueue_disable_intr(vq);
229 return 0;
230 }
231
232 int
virtqueue_txvq_reset_packed(struct virtqueue * vq)233 virtqueue_txvq_reset_packed(struct virtqueue *vq)
234 {
235 int size = vq->vq_nentries;
236 struct vq_desc_extra *dxp;
237 uint16_t desc_idx;
238
239 vq->vq_used_cons_idx = 0;
240 vq->vq_desc_head_idx = 0;
241 vq->vq_avail_idx = 0;
242 vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
243 vq->vq_free_cnt = vq->vq_nentries;
244
245 vq->vq_packed.used_wrap_counter = 1;
246 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL;
247 vq->vq_packed.event_flags_shadow = 0;
248
249 memset(vq->mz->addr, 0, vq->mz->len);
250 memset(vq->txq.hdr_mz->addr, 0, vq->txq.hdr_mz->len);
251
252 for (desc_idx = 0; desc_idx < vq->vq_nentries; desc_idx++) {
253 dxp = &vq->vq_descx[desc_idx];
254 if (dxp->cookie != NULL) {
255 rte_pktmbuf_free(dxp->cookie);
256 dxp->cookie = NULL;
257 }
258 }
259
260 virtqueue_txq_indirect_headers_init(vq);
261 vring_desc_init_packed(vq, size);
262 virtqueue_disable_intr(vq);
263
264 return 0;
265 }
266
267
268 static void
virtio_init_vring(struct virtqueue * vq)269 virtio_init_vring(struct virtqueue *vq)
270 {
271 int size = vq->vq_nentries;
272 uint8_t *ring_mem = vq->vq_ring_virt_mem;
273
274 PMD_INIT_FUNC_TRACE();
275
276 memset(ring_mem, 0, vq->vq_ring_size);
277
278 vq->vq_used_cons_idx = 0;
279 vq->vq_desc_head_idx = 0;
280 vq->vq_avail_idx = 0;
281 vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
282 vq->vq_free_cnt = vq->vq_nentries;
283 memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
284 if (virtio_with_packed_queue(vq->hw)) {
285 vring_init_packed(&vq->vq_packed.ring, ring_mem, vq->vq_ring_mem,
286 VIRTIO_VRING_ALIGN, size);
287 vring_desc_init_packed(vq, size);
288 } else {
289 struct vring *vr = &vq->vq_split.ring;
290
291 vring_init_split(vr, ring_mem, vq->vq_ring_mem, VIRTIO_VRING_ALIGN, size);
292 vring_desc_init_split(vr->desc, size);
293 }
294 /*
295 * Disable device(host) interrupting guest
296 */
297 virtqueue_disable_intr(vq);
298 }
299
300 static int
virtio_alloc_queue_headers(struct virtqueue * vq,int numa_node,const char * name)301 virtio_alloc_queue_headers(struct virtqueue *vq, int numa_node, const char *name)
302 {
303 char hdr_name[VIRTQUEUE_MAX_NAME_SZ];
304 const struct rte_memzone **hdr_mz;
305 rte_iova_t *hdr_mem;
306 ssize_t size;
307 int queue_type;
308
309 queue_type = virtio_get_queue_type(vq->hw, vq->vq_queue_index);
310 switch (queue_type) {
311 case VTNET_TQ:
312 /*
313 * For each xmit packet, allocate a virtio_net_hdr
314 * and indirect ring elements
315 */
316 size = vq->vq_nentries * sizeof(struct virtio_tx_region);
317 hdr_mz = &vq->txq.hdr_mz;
318 hdr_mem = &vq->txq.hdr_mem;
319 break;
320 case VTNET_CQ:
321 /* Allocate a page for control vq command, data and status */
322 size = rte_mem_page_size();
323 hdr_mz = &vq->cq.hdr_mz;
324 hdr_mem = &vq->cq.hdr_mem;
325 break;
326 case VTNET_RQ:
327 /* fallthrough */
328 default:
329 return 0;
330 }
331
332 snprintf(hdr_name, sizeof(hdr_name), "%s_hdr", name);
333 *hdr_mz = rte_memzone_reserve_aligned(hdr_name, size, numa_node,
334 RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE);
335 if (*hdr_mz == NULL) {
336 if (rte_errno == EEXIST)
337 *hdr_mz = rte_memzone_lookup(hdr_name);
338 if (*hdr_mz == NULL)
339 return -ENOMEM;
340 }
341
342 memset((*hdr_mz)->addr, 0, size);
343
344 if (vq->hw->use_va)
345 *hdr_mem = (uintptr_t)(*hdr_mz)->addr;
346 else
347 *hdr_mem = (uintptr_t)(*hdr_mz)->iova;
348
349 return 0;
350 }
351
352 static void
virtio_free_queue_headers(struct virtqueue * vq)353 virtio_free_queue_headers(struct virtqueue *vq)
354 {
355 const struct rte_memzone **hdr_mz;
356 rte_iova_t *hdr_mem;
357 int queue_type;
358
359 queue_type = virtio_get_queue_type(vq->hw, vq->vq_queue_index);
360 switch (queue_type) {
361 case VTNET_TQ:
362 hdr_mz = &vq->txq.hdr_mz;
363 hdr_mem = &vq->txq.hdr_mem;
364 break;
365 case VTNET_CQ:
366 hdr_mz = &vq->cq.hdr_mz;
367 hdr_mem = &vq->cq.hdr_mem;
368 break;
369 case VTNET_RQ:
370 /* fallthrough */
371 default:
372 return;
373 }
374
375 rte_memzone_free(*hdr_mz);
376 *hdr_mz = NULL;
377 *hdr_mem = 0;
378 }
379
380 static int
virtio_rxq_sw_ring_alloc(struct virtqueue * vq,int numa_node)381 virtio_rxq_sw_ring_alloc(struct virtqueue *vq, int numa_node)
382 {
383 void *sw_ring;
384 struct rte_mbuf *mbuf;
385 size_t size;
386
387 /* SW ring is only used with vectorized datapath */
388 if (!vq->hw->use_vec_rx)
389 return 0;
390
391 size = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq->vq_nentries) * sizeof(vq->rxq.sw_ring[0]);
392
393 sw_ring = rte_zmalloc_socket("sw_ring", size, RTE_CACHE_LINE_SIZE, numa_node);
394 if (!sw_ring) {
395 PMD_INIT_LOG(ERR, "can not allocate RX soft ring");
396 return -ENOMEM;
397 }
398
399 mbuf = rte_zmalloc_socket("sw_ring", sizeof(*mbuf), RTE_CACHE_LINE_SIZE, numa_node);
400 if (!mbuf) {
401 PMD_INIT_LOG(ERR, "can not allocate fake mbuf");
402 rte_free(sw_ring);
403 return -ENOMEM;
404 }
405
406 vq->rxq.sw_ring = sw_ring;
407 vq->rxq.fake_mbuf = mbuf;
408
409 return 0;
410 }
411
412 static void
virtio_rxq_sw_ring_free(struct virtqueue * vq)413 virtio_rxq_sw_ring_free(struct virtqueue *vq)
414 {
415 rte_free(vq->rxq.fake_mbuf);
416 vq->rxq.fake_mbuf = NULL;
417 rte_free(vq->rxq.sw_ring);
418 vq->rxq.sw_ring = NULL;
419 }
420
421 struct virtqueue *
virtqueue_alloc(struct virtio_hw * hw,uint16_t index,uint16_t num,int type,int node,const char * name)422 virtqueue_alloc(struct virtio_hw *hw, uint16_t index, uint16_t num, int type,
423 int node, const char *name)
424 {
425 struct virtqueue *vq;
426 const struct rte_memzone *mz;
427 unsigned int size;
428
429 size = sizeof(*vq) + num * sizeof(struct vq_desc_extra);
430 size = RTE_ALIGN_CEIL(size, RTE_CACHE_LINE_SIZE);
431
432 vq = rte_zmalloc_socket(name, size, RTE_CACHE_LINE_SIZE, node);
433 if (vq == NULL) {
434 PMD_INIT_LOG(ERR, "can not allocate vq");
435 return NULL;
436 }
437
438 vq->hw = hw;
439 vq->vq_queue_index = index;
440 vq->vq_nentries = num;
441 if (virtio_with_packed_queue(hw)) {
442 vq->vq_packed.used_wrap_counter = 1;
443 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL;
444 vq->vq_packed.event_flags_shadow = 0;
445 if (type == VTNET_RQ)
446 vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE;
447 }
448
449 /*
450 * Reserve a memzone for vring elements
451 */
452 size = vring_size(hw, num, VIRTIO_VRING_ALIGN);
453 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_VRING_ALIGN);
454 PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d", size, vq->vq_ring_size);
455
456 mz = rte_memzone_reserve_aligned(name, vq->vq_ring_size, node,
457 RTE_MEMZONE_IOVA_CONTIG, VIRTIO_VRING_ALIGN);
458 if (mz == NULL) {
459 if (rte_errno == EEXIST)
460 mz = rte_memzone_lookup(name);
461 if (mz == NULL)
462 goto free_vq;
463 }
464
465 memset(mz->addr, 0, mz->len);
466 vq->mz = mz;
467 vq->vq_ring_virt_mem = mz->addr;
468
469 if (hw->use_va) {
470 vq->vq_ring_mem = (uintptr_t)mz->addr;
471 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_addr);
472 vq->mbuf_addr_mask = UINTPTR_MAX;
473 } else {
474 vq->vq_ring_mem = mz->iova;
475 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_iova);
476 vq->mbuf_addr_mask = UINT64_MAX;
477 }
478
479 PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%" PRIx64, vq->vq_ring_mem);
480 PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: %p", vq->vq_ring_virt_mem);
481
482 virtio_init_vring(vq);
483
484 if (virtio_alloc_queue_headers(vq, node, name)) {
485 PMD_INIT_LOG(ERR, "Failed to alloc queue headers");
486 goto free_mz;
487 }
488
489 switch (type) {
490 case VTNET_RQ:
491 if (virtio_rxq_sw_ring_alloc(vq, node))
492 goto free_hdr_mz;
493 break;
494 case VTNET_TQ:
495 virtqueue_txq_indirect_headers_init(vq);
496 break;
497 }
498
499 return vq;
500
501 free_hdr_mz:
502 virtio_free_queue_headers(vq);
503 free_mz:
504 rte_memzone_free(mz);
505 free_vq:
506 rte_free(vq);
507
508 return NULL;
509 }
510
511 void
virtqueue_free(struct virtqueue * vq)512 virtqueue_free(struct virtqueue *vq)
513 {
514 int type;
515
516 type = virtio_get_queue_type(vq->hw, vq->vq_queue_index);
517 switch (type) {
518 case VTNET_RQ:
519 virtio_rxq_sw_ring_free(vq);
520 break;
521 case VTNET_TQ:
522 case VTNET_CQ:
523 virtio_free_queue_headers(vq);
524 break;
525 }
526
527 rte_memzone_free(vq->mz);
528 rte_free(vq);
529 }
530