1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2015 Intel Corporation 3 */ 4 #include <stdint.h> 5 #include <unistd.h> 6 7 #include <rte_eal_paging.h> 8 #include <rte_malloc.h> 9 #include <rte_mbuf.h> 10 #include <rte_memzone.h> 11 12 #include "virtqueue.h" 13 #include "virtio_logs.h" 14 #include "virtio.h" 15 #include "virtio_rxtx_simple.h" 16 17 /* 18 * Two types of mbuf to be cleaned: 19 * 1) mbuf that has been consumed by backend but not used by virtio. 20 * 2) mbuf that hasn't been consumed by backend. 21 */ 22 struct rte_mbuf * 23 virtqueue_detach_unused(struct virtqueue *vq) 24 { 25 struct rte_mbuf *cookie; 26 struct virtio_hw *hw; 27 uint16_t start, end; 28 int type, idx; 29 30 if (vq == NULL) 31 return NULL; 32 33 hw = vq->hw; 34 type = virtio_get_queue_type(hw, vq->vq_queue_index); 35 start = vq->vq_avail_idx & (vq->vq_nentries - 1); 36 end = (vq->vq_avail_idx + vq->vq_free_cnt) & (vq->vq_nentries - 1); 37 38 for (idx = 0; idx < vq->vq_nentries; idx++) { 39 if (hw->use_vec_rx && !virtio_with_packed_queue(hw) && 40 type == VTNET_RQ) { 41 if (start <= end && idx >= start && idx < end) 42 continue; 43 if (start > end && (idx >= start || idx < end)) 44 continue; 45 cookie = vq->rxq.sw_ring[idx]; 46 if (cookie != NULL) { 47 vq->rxq.sw_ring[idx] = NULL; 48 return cookie; 49 } 50 } else { 51 cookie = vq->vq_descx[idx].cookie; 52 if (cookie != NULL) { 53 vq->vq_descx[idx].cookie = NULL; 54 return cookie; 55 } 56 } 57 } 58 59 return NULL; 60 } 61 62 /* Flush used descs */ 63 static void 64 virtqueue_rxvq_flush_packed(struct virtqueue *vq) 65 { 66 struct vq_desc_extra *dxp; 67 uint16_t i; 68 69 struct vring_packed_desc *descs = vq->vq_packed.ring.desc; 70 int cnt = 0; 71 72 i = vq->vq_used_cons_idx; 73 while (desc_is_used(&descs[i], vq) && cnt++ < vq->vq_nentries) { 74 dxp = &vq->vq_descx[descs[i].id]; 75 if (dxp->cookie != NULL) { 76 rte_pktmbuf_free(dxp->cookie); 77 dxp->cookie = NULL; 78 } 79 vq->vq_free_cnt++; 80 vq->vq_used_cons_idx++; 81 if (vq->vq_used_cons_idx >= vq->vq_nentries) { 82 vq->vq_used_cons_idx -= vq->vq_nentries; 83 vq->vq_packed.used_wrap_counter ^= 1; 84 } 85 i = vq->vq_used_cons_idx; 86 } 87 } 88 89 /* Flush the elements in the used ring. */ 90 static void 91 virtqueue_rxvq_flush_split(struct virtqueue *vq) 92 { 93 struct virtnet_rx *rxq = &vq->rxq; 94 struct virtio_hw *hw = vq->hw; 95 struct vring_used_elem *uep; 96 struct vq_desc_extra *dxp; 97 uint16_t used_idx, desc_idx; 98 uint16_t nb_used, i; 99 100 nb_used = virtqueue_nused(vq); 101 102 for (i = 0; i < nb_used; i++) { 103 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1); 104 uep = &vq->vq_split.ring.used->ring[used_idx]; 105 if (hw->use_vec_rx) { 106 desc_idx = used_idx; 107 rte_pktmbuf_free(vq->rxq.sw_ring[desc_idx]); 108 vq->vq_free_cnt++; 109 } else if (hw->use_inorder_rx) { 110 desc_idx = (uint16_t)uep->id; 111 dxp = &vq->vq_descx[desc_idx]; 112 if (dxp->cookie != NULL) { 113 rte_pktmbuf_free(dxp->cookie); 114 dxp->cookie = NULL; 115 } 116 vq_ring_free_inorder(vq, desc_idx, 1); 117 } else { 118 desc_idx = (uint16_t)uep->id; 119 dxp = &vq->vq_descx[desc_idx]; 120 if (dxp->cookie != NULL) { 121 rte_pktmbuf_free(dxp->cookie); 122 dxp->cookie = NULL; 123 } 124 vq_ring_free_chain(vq, desc_idx); 125 } 126 vq->vq_used_cons_idx++; 127 } 128 129 if (hw->use_vec_rx) { 130 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) { 131 virtio_rxq_rearm_vec(rxq); 132 if (virtqueue_kick_prepare(vq)) 133 virtqueue_notify(vq); 134 } 135 } 136 } 137 138 /* Flush the elements in the used ring. */ 139 void 140 virtqueue_rxvq_flush(struct virtqueue *vq) 141 { 142 struct virtio_hw *hw = vq->hw; 143 144 if (virtio_with_packed_queue(hw)) 145 virtqueue_rxvq_flush_packed(vq); 146 else 147 virtqueue_rxvq_flush_split(vq); 148 } 149 150 static void 151 virtqueue_txq_indirect_header_init_packed(struct virtqueue *vq, uint32_t idx) 152 { 153 struct virtio_tx_region *txr; 154 struct vring_packed_desc *desc; 155 rte_iova_t hdr_mem; 156 157 txr = vq->txq.hdr_mz->addr; 158 hdr_mem = vq->txq.hdr_mem; 159 desc = txr[idx].tx_packed_indir; 160 161 vring_desc_init_indirect_packed(desc, RTE_DIM(txr[idx].tx_packed_indir)); 162 desc->addr = hdr_mem + idx * sizeof(*txr) + offsetof(struct virtio_tx_region, tx_hdr); 163 desc->len = vq->hw->vtnet_hdr_size; 164 } 165 166 static void 167 virtqueue_txq_indirect_header_init_split(struct virtqueue *vq, uint32_t idx) 168 { 169 struct virtio_tx_region *txr; 170 struct vring_desc *desc; 171 rte_iova_t hdr_mem; 172 173 txr = vq->txq.hdr_mz->addr; 174 hdr_mem = vq->txq.hdr_mem; 175 desc = txr[idx].tx_indir; 176 177 vring_desc_init_split(desc, RTE_DIM(txr[idx].tx_indir)); 178 desc->addr = hdr_mem + idx * sizeof(*txr) + offsetof(struct virtio_tx_region, tx_hdr); 179 desc->len = vq->hw->vtnet_hdr_size; 180 desc->flags = VRING_DESC_F_NEXT; 181 } 182 183 void 184 virtqueue_txq_indirect_headers_init(struct virtqueue *vq) 185 { 186 uint32_t i; 187 188 if (!virtio_with_feature(vq->hw, VIRTIO_RING_F_INDIRECT_DESC)) 189 return; 190 191 for (i = 0; i < vq->vq_nentries; i++) 192 if (virtio_with_packed_queue(vq->hw)) 193 virtqueue_txq_indirect_header_init_packed(vq, i); 194 else 195 virtqueue_txq_indirect_header_init_split(vq, i); 196 } 197 198 int 199 virtqueue_rxvq_reset_packed(struct virtqueue *vq) 200 { 201 int size = vq->vq_nentries; 202 struct vq_desc_extra *dxp; 203 uint16_t desc_idx; 204 205 vq->vq_used_cons_idx = 0; 206 vq->vq_desc_head_idx = 0; 207 vq->vq_avail_idx = 0; 208 vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); 209 vq->vq_free_cnt = vq->vq_nentries; 210 211 vq->vq_packed.used_wrap_counter = 1; 212 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL; 213 vq->vq_packed.event_flags_shadow = 0; 214 vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE; 215 216 memset(vq->mz->addr, 0, vq->mz->len); 217 218 for (desc_idx = 0; desc_idx < vq->vq_nentries; desc_idx++) { 219 dxp = &vq->vq_descx[desc_idx]; 220 if (dxp->cookie != NULL) { 221 rte_pktmbuf_free(dxp->cookie); 222 dxp->cookie = NULL; 223 } 224 } 225 226 vring_desc_init_packed(vq, size); 227 228 virtqueue_disable_intr(vq); 229 return 0; 230 } 231 232 int 233 virtqueue_txvq_reset_packed(struct virtqueue *vq) 234 { 235 int size = vq->vq_nentries; 236 struct vq_desc_extra *dxp; 237 uint16_t desc_idx; 238 239 vq->vq_used_cons_idx = 0; 240 vq->vq_desc_head_idx = 0; 241 vq->vq_avail_idx = 0; 242 vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); 243 vq->vq_free_cnt = vq->vq_nentries; 244 245 vq->vq_packed.used_wrap_counter = 1; 246 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL; 247 vq->vq_packed.event_flags_shadow = 0; 248 249 memset(vq->mz->addr, 0, vq->mz->len); 250 memset(vq->txq.hdr_mz->addr, 0, vq->txq.hdr_mz->len); 251 252 for (desc_idx = 0; desc_idx < vq->vq_nentries; desc_idx++) { 253 dxp = &vq->vq_descx[desc_idx]; 254 if (dxp->cookie != NULL) { 255 rte_pktmbuf_free(dxp->cookie); 256 dxp->cookie = NULL; 257 } 258 } 259 260 virtqueue_txq_indirect_headers_init(vq); 261 vring_desc_init_packed(vq, size); 262 virtqueue_disable_intr(vq); 263 264 return 0; 265 } 266 267 268 static void 269 virtio_init_vring(struct virtqueue *vq) 270 { 271 int size = vq->vq_nentries; 272 uint8_t *ring_mem = vq->vq_ring_virt_mem; 273 274 PMD_INIT_FUNC_TRACE(); 275 276 memset(ring_mem, 0, vq->vq_ring_size); 277 278 vq->vq_used_cons_idx = 0; 279 vq->vq_desc_head_idx = 0; 280 vq->vq_avail_idx = 0; 281 vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); 282 vq->vq_free_cnt = vq->vq_nentries; 283 memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries); 284 if (virtio_with_packed_queue(vq->hw)) { 285 vring_init_packed(&vq->vq_packed.ring, ring_mem, 286 VIRTIO_VRING_ALIGN, size); 287 vring_desc_init_packed(vq, size); 288 } else { 289 struct vring *vr = &vq->vq_split.ring; 290 291 vring_init_split(vr, ring_mem, VIRTIO_VRING_ALIGN, size); 292 vring_desc_init_split(vr->desc, size); 293 } 294 /* 295 * Disable device(host) interrupting guest 296 */ 297 virtqueue_disable_intr(vq); 298 } 299 300 static int 301 virtio_alloc_queue_headers(struct virtqueue *vq, int numa_node, const char *name) 302 { 303 char hdr_name[VIRTQUEUE_MAX_NAME_SZ]; 304 const struct rte_memzone **hdr_mz; 305 rte_iova_t *hdr_mem; 306 ssize_t size; 307 int queue_type; 308 309 queue_type = virtio_get_queue_type(vq->hw, vq->vq_queue_index); 310 switch (queue_type) { 311 case VTNET_TQ: 312 /* 313 * For each xmit packet, allocate a virtio_net_hdr 314 * and indirect ring elements 315 */ 316 size = vq->vq_nentries * sizeof(struct virtio_tx_region); 317 hdr_mz = &vq->txq.hdr_mz; 318 hdr_mem = &vq->txq.hdr_mem; 319 break; 320 case VTNET_CQ: 321 /* Allocate a page for control vq command, data and status */ 322 size = rte_mem_page_size(); 323 hdr_mz = &vq->cq.hdr_mz; 324 hdr_mem = &vq->cq.hdr_mem; 325 break; 326 case VTNET_RQ: 327 /* fallthrough */ 328 default: 329 return 0; 330 } 331 332 snprintf(hdr_name, sizeof(hdr_name), "%s_hdr", name); 333 *hdr_mz = rte_memzone_reserve_aligned(hdr_name, size, numa_node, 334 RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE); 335 if (*hdr_mz == NULL) { 336 if (rte_errno == EEXIST) 337 *hdr_mz = rte_memzone_lookup(hdr_name); 338 if (*hdr_mz == NULL) 339 return -ENOMEM; 340 } 341 342 memset((*hdr_mz)->addr, 0, size); 343 344 if (vq->hw->use_va) 345 *hdr_mem = (uintptr_t)(*hdr_mz)->addr; 346 else 347 *hdr_mem = (uintptr_t)(*hdr_mz)->iova; 348 349 return 0; 350 } 351 352 static void 353 virtio_free_queue_headers(struct virtqueue *vq) 354 { 355 const struct rte_memzone **hdr_mz; 356 rte_iova_t *hdr_mem; 357 int queue_type; 358 359 queue_type = virtio_get_queue_type(vq->hw, vq->vq_queue_index); 360 switch (queue_type) { 361 case VTNET_TQ: 362 hdr_mz = &vq->txq.hdr_mz; 363 hdr_mem = &vq->txq.hdr_mem; 364 break; 365 case VTNET_CQ: 366 hdr_mz = &vq->cq.hdr_mz; 367 hdr_mem = &vq->cq.hdr_mem; 368 break; 369 case VTNET_RQ: 370 /* fallthrough */ 371 default: 372 return; 373 } 374 375 rte_memzone_free(*hdr_mz); 376 *hdr_mz = NULL; 377 *hdr_mem = 0; 378 } 379 380 static int 381 virtio_rxq_sw_ring_alloc(struct virtqueue *vq, int numa_node) 382 { 383 void *sw_ring; 384 struct rte_mbuf *mbuf; 385 size_t size; 386 387 /* SW ring is only used with vectorized datapath */ 388 if (!vq->hw->use_vec_rx) 389 return 0; 390 391 size = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq->vq_nentries) * sizeof(vq->rxq.sw_ring[0]); 392 393 sw_ring = rte_zmalloc_socket("sw_ring", size, RTE_CACHE_LINE_SIZE, numa_node); 394 if (!sw_ring) { 395 PMD_INIT_LOG(ERR, "can not allocate RX soft ring"); 396 return -ENOMEM; 397 } 398 399 mbuf = rte_zmalloc_socket("sw_ring", sizeof(*mbuf), RTE_CACHE_LINE_SIZE, numa_node); 400 if (!mbuf) { 401 PMD_INIT_LOG(ERR, "can not allocate fake mbuf"); 402 rte_free(sw_ring); 403 return -ENOMEM; 404 } 405 406 vq->rxq.sw_ring = sw_ring; 407 vq->rxq.fake_mbuf = mbuf; 408 409 return 0; 410 } 411 412 static void 413 virtio_rxq_sw_ring_free(struct virtqueue *vq) 414 { 415 rte_free(vq->rxq.fake_mbuf); 416 vq->rxq.fake_mbuf = NULL; 417 rte_free(vq->rxq.sw_ring); 418 vq->rxq.sw_ring = NULL; 419 } 420 421 struct virtqueue * 422 virtqueue_alloc(struct virtio_hw *hw, uint16_t index, uint16_t num, int type, 423 int node, const char *name) 424 { 425 struct virtqueue *vq; 426 const struct rte_memzone *mz; 427 unsigned int size; 428 429 size = sizeof(*vq) + num * sizeof(struct vq_desc_extra); 430 size = RTE_ALIGN_CEIL(size, RTE_CACHE_LINE_SIZE); 431 432 vq = rte_zmalloc_socket(name, size, RTE_CACHE_LINE_SIZE, node); 433 if (vq == NULL) { 434 PMD_INIT_LOG(ERR, "can not allocate vq"); 435 return NULL; 436 } 437 438 vq->hw = hw; 439 vq->vq_queue_index = index; 440 vq->vq_nentries = num; 441 if (virtio_with_packed_queue(hw)) { 442 vq->vq_packed.used_wrap_counter = 1; 443 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL; 444 vq->vq_packed.event_flags_shadow = 0; 445 if (type == VTNET_RQ) 446 vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE; 447 } 448 449 /* 450 * Reserve a memzone for vring elements 451 */ 452 size = vring_size(hw, num, VIRTIO_VRING_ALIGN); 453 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_VRING_ALIGN); 454 PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d", size, vq->vq_ring_size); 455 456 mz = rte_memzone_reserve_aligned(name, vq->vq_ring_size, node, 457 RTE_MEMZONE_IOVA_CONTIG, VIRTIO_VRING_ALIGN); 458 if (mz == NULL) { 459 if (rte_errno == EEXIST) 460 mz = rte_memzone_lookup(name); 461 if (mz == NULL) 462 goto free_vq; 463 } 464 465 memset(mz->addr, 0, mz->len); 466 vq->mz = mz; 467 vq->vq_ring_virt_mem = mz->addr; 468 469 if (hw->use_va) { 470 vq->vq_ring_mem = (uintptr_t)mz->addr; 471 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_addr); 472 } else { 473 vq->vq_ring_mem = mz->iova; 474 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_iova); 475 } 476 477 PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%" PRIx64, vq->vq_ring_mem); 478 PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: %p", vq->vq_ring_virt_mem); 479 480 virtio_init_vring(vq); 481 482 if (virtio_alloc_queue_headers(vq, node, name)) { 483 PMD_INIT_LOG(ERR, "Failed to alloc queue headers"); 484 goto free_mz; 485 } 486 487 switch (type) { 488 case VTNET_RQ: 489 if (virtio_rxq_sw_ring_alloc(vq, node)) 490 goto free_hdr_mz; 491 break; 492 case VTNET_TQ: 493 virtqueue_txq_indirect_headers_init(vq); 494 break; 495 } 496 497 return vq; 498 499 free_hdr_mz: 500 virtio_free_queue_headers(vq); 501 free_mz: 502 rte_memzone_free(mz); 503 free_vq: 504 rte_free(vq); 505 506 return NULL; 507 } 508 509 void 510 virtqueue_free(struct virtqueue *vq) 511 { 512 int type; 513 514 type = virtio_get_queue_type(vq->hw, vq->vq_queue_index); 515 switch (type) { 516 case VTNET_RQ: 517 virtio_rxq_sw_ring_free(vq); 518 break; 519 case VTNET_TQ: 520 case VTNET_CQ: 521 virtio_free_queue_headers(vq); 522 break; 523 } 524 525 rte_memzone_free(vq->mz); 526 rte_free(vq); 527 } 528