926 	struct vhost_dev *vdev;  in find_vhost_dev()  local
928 	TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {  in find_vhost_dev()
929 		if (vdev->ready == DEVICE_RX &&  in find_vhost_dev()
930 		    rte_is_same_ether_addr(mac, &vdev->mac_address))  in find_vhost_dev()
931 			return vdev;  in find_vhost_dev()
942 link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m)  in link_vmdq()  argument
953 			vdev->vid);  in link_vmdq()
958 		vdev->mac_address.addr_bytes[i] =  in link_vmdq()
962 	vdev->vlan_tag = vlan_tags[vdev->vid];  in link_vmdq()
967 		vdev->vid, RTE_ETHER_ADDR_BYTES(&vdev->mac_address),  in link_vmdq()
968 		vdev->vlan_tag);  in link_vmdq()
971 	ret = rte_eth_dev_mac_addr_add(ports[0], &vdev->mac_address,  in link_vmdq()
972 				(uint32_t)vdev->vid + vmdq_pool_base);  in link_vmdq()
976 			vdev->vid);  in link_vmdq()
978 	rte_eth_dev_set_vlan_strip_on_queue(ports[0], vdev->vmdq_rx_q, 1);  in link_vmdq()
981 	vdev->ready = DEVICE_RX;  in link_vmdq()
991 unlink_vmdq(struct vhost_dev *vdev)  in unlink_vmdq()  argument
997 	if (vdev->ready == DEVICE_RX) {  in unlink_vmdq()
999 		rte_eth_dev_mac_addr_remove(ports[0], &vdev->mac_address);  in unlink_vmdq()
1001 			vdev->mac_address.addr_bytes[i] = 0;  in unlink_vmdq()
1003 		vdev->vlan_tag = 0;  in unlink_vmdq()
1007 					(uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);  in unlink_vmdq()
1014 					(uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);  in unlink_vmdq()
1017 		vdev->ready = DEVICE_MAC_LEARNING;  in unlink_vmdq()
1029 complete_async_pkts(struct vhost_dev *vdev)  in complete_async_pkts()  argument
1033 	int16_t dma_id = dma_bind[vid2socketid[vdev->vid]].dmas[VIRTIO_RXQ].dev_id;  in complete_async_pkts()
1035 	complete_count = rte_vhost_poll_enqueue_completed(vdev->vid,  in complete_async_pkts()
1065 drain_vhost(struct vhost_dev *vdev)  in drain_vhost()  argument
1068 	uint32_t buff_idx = rte_lcore_id() * RTE_MAX_VHOST_DEVICE + vdev->vid;  in drain_vhost()
1072 	ret = vdev_queue_ops[vdev->vid].enqueue_pkt_burst(vdev, VIRTIO_RXQ, m, nr_xmit);  in drain_vhost()
1075 		rte_atomic_fetch_add_explicit(&vdev->stats.rx_total_atomic, nr_xmit,  in drain_vhost()
1077 		rte_atomic_fetch_add_explicit(&vdev->stats.rx_atomic, ret,  in drain_vhost()
1081 	if (!dma_bind[vid2socketid[vdev->vid]].dmas[VIRTIO_RXQ].async_enabled) {  in drain_vhost()
1095 	struct vhost_dev *vdev;  in drain_vhost_table()  local
1098 	TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {  in drain_vhost_table()
1099 		if (unlikely(vdev->remove == 1))  in drain_vhost_table()
1102 		vhost_txq = vhost_txbuff[lcore_id * RTE_MAX_VHOST_DEVICE + vdev->vid];  in drain_vhost_table()
1110 			drain_vhost(vdev);  in drain_vhost_table()
1122 virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)  in virtio_tx_local()  argument
1134 	if (vdev->vid == dst_vdev->vid) {  in virtio_tx_local()
1137 			vdev->vid);  in virtio_tx_local()
1154 		vdev->stats.tx_total++;  in virtio_tx_local()
1155 		vdev->stats.tx++;  in virtio_tx_local()
1171 find_local_dest(struct vhost_dev *vdev, struct rte_mbuf *m,  in find_local_dest()  argument
1182 	if (vdev->vid == dst_vdev->vid) {  in find_local_dest()
1185 			vdev->vid);  in find_local_dest()
1195 	*vlan_tag = vlan_tags[vdev->vid];  in find_local_dest()
1199 		vdev->vid, dst_vdev->vid, *vlan_tag);  in find_local_dest()
1252 virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, uint16_t vlan_tag)  in virtio_tx_route()  argument
1265 			if (vdev2 != vdev)  in virtio_tx_route()
1266 				sync_virtio_xmit(vdev2, vdev, m);  in virtio_tx_route()
1272 	if ((vm2vm_mode == VM2VM_SOFTWARE) && (virtio_tx_local(vdev, m) == 0))  in virtio_tx_route()
1276 		if (unlikely(find_local_dest(vdev, m, &offset,  in virtio_tx_route()
1284 		"(%d) TX: MAC address is external\n", vdev->vid);  in virtio_tx_route()
1329 		vdev->stats.tx_total++;  in virtio_tx_route()
1330 		vdev->stats.tx++;  in virtio_tx_route()
1380 drain_eth_rx(struct vhost_dev *vdev)  in drain_eth_rx()  argument
1385 	rx_count = rte_eth_rx_burst(ports[0], vdev->vmdq_rx_q,  in drain_eth_rx()
1391 	enqueue_count = vdev_queue_ops[vdev->vid].enqueue_pkt_burst(vdev,  in drain_eth_rx()
1400 			enqueue_count += vdev_queue_ops[vdev->vid].enqueue_pkt_burst(vdev,  in drain_eth_rx()
1407 		rte_atomic_fetch_add_explicit(&vdev->stats.rx_total_atomic, rx_count,  in drain_eth_rx()
1409 		rte_atomic_fetch_add_explicit(&vdev->stats.rx_atomic, enqueue_count,  in drain_eth_rx()
1413 	if (!dma_bind[vid2socketid[vdev->vid]].dmas[VIRTIO_RXQ].async_enabled) {  in drain_eth_rx()
1444 drain_virtio_tx(struct vhost_dev *vdev)  in drain_virtio_tx()  argument
1450 	count = vdev_queue_ops[vdev->vid].dequeue_pkt_burst(vdev,  in drain_virtio_tx()
1454 	if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && count) {  in drain_virtio_tx()
1455 		if (vdev->remove || link_vmdq(vdev, pkts[0]) == -1)  in drain_virtio_tx()
1460 		virtio_tx_route(vdev, pkts[i], vlan_tags[vdev->vid]);  in drain_virtio_tx()
1485 	struct vhost_dev *vdev;  in switch_worker()  local
1511 		TAILQ_FOREACH(vdev, &lcore_info[lcore_id].vdev_list,  in switch_worker()
1513 			if (unlikely(vdev->remove)) {  in switch_worker()
1514 				unlink_vmdq(vdev);  in switch_worker()
1515 				vdev->ready = DEVICE_SAFE_REMOVE;  in switch_worker()
1519 			if (likely(vdev->ready == DEVICE_RX))  in switch_worker()
1520 				drain_eth_rx(vdev);  in switch_worker()
1522 			if (likely(!vdev->remove))  in switch_worker()
1523 				drain_virtio_tx(vdev);  in switch_worker()
1531 vhost_clear_queue_thread_unsafe(struct vhost_dev *vdev, uint16_t queue_id)  in vhost_clear_queue_thread_unsafe()  argument
1536 	int16_t dma_id = dma_bind[vid2socketid[vdev->vid]].dmas[queue_id].dev_id;  in vhost_clear_queue_thread_unsafe()
1537 	pkts_inflight = rte_vhost_async_get_inflight_thread_unsafe(vdev->vid, queue_id);  in vhost_clear_queue_thread_unsafe()
1542 		n_pkt = rte_vhost_clear_queue_thread_unsafe(vdev->vid, queue_id, m_cpl,  in vhost_clear_queue_thread_unsafe()
1545 		pkts_inflight = rte_vhost_async_get_inflight_thread_unsafe(vdev->vid,  in vhost_clear_queue_thread_unsafe()
1551 vhost_clear_queue(struct vhost_dev *vdev, uint16_t queue_id)  in vhost_clear_queue()  argument
1556 	int16_t dma_id = dma_bind[vid2socketid[vdev->vid]].dmas[queue_id].dev_id;  in vhost_clear_queue()
1557 	pkts_inflight = rte_vhost_async_get_inflight(vdev->vid, queue_id);  in vhost_clear_queue()
1562 		n_pkt = rte_vhost_clear_queue(vdev->vid, queue_id, m_cpl,  in vhost_clear_queue()
1565 		pkts_inflight = rte_vhost_async_get_inflight(vdev->vid, queue_id);  in vhost_clear_queue()
1578 	struct vhost_dev *vdev = NULL;  in destroy_device()  local
1582 	TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {  in destroy_device()
1583 		if (vdev->vid == vid)  in destroy_device()
1586 	if (!vdev)  in destroy_device()
1589 	vdev->remove = 1;  in destroy_device()
1590 	while(vdev->ready != DEVICE_SAFE_REMOVE) {  in destroy_device()
1598 		vs_vhost_net_remove(vdev);  in destroy_device()
1600 	TAILQ_REMOVE(&lcore_info[vdev->coreid].vdev_list, vdev,  in destroy_device()
1602 	TAILQ_REMOVE(&vhost_dev_list, vdev, global_vdev_entry);  in destroy_device()
1619 	lcore_info[vdev->coreid].device_num--;  in destroy_device()
1623 		vdev->vid);  in destroy_device()
1626 		vhost_clear_queue(vdev, VIRTIO_RXQ);  in destroy_device()
1632 		vhost_clear_queue(vdev, VIRTIO_TXQ);  in destroy_device()
1637 	rte_free(vdev);  in destroy_device()
1709 	struct vhost_dev *vdev;  in new_device()  local
1712 	vdev = rte_zmalloc("vhost device", sizeof(*vdev), RTE_CACHE_LINE_SIZE);  in new_device()
1713 	if (vdev == NULL) {  in new_device()
1719 	vdev->vid = vid;  in new_device()
1746 		vs_vhost_net_setup(vdev);  in new_device()
1748 	TAILQ_INSERT_TAIL(&vhost_dev_list, vdev, global_vdev_entry);  in new_device()
1749 	vdev->vmdq_rx_q = vid * queues_per_pool + vmdq_queue_base;  in new_device()
1752 	vdev->ready = DEVICE_MAC_LEARNING;  in new_device()
1753 	vdev->remove = 0;  in new_device()
1762 	vdev->coreid = core_add;  in new_device()
1764 	TAILQ_INSERT_TAIL(&lcore_info[vdev->coreid].vdev_list, vdev,  in new_device()
1766 	lcore_info[vdev->coreid].device_num++;  in new_device()
1774 		vid, vdev->coreid);  in new_device()
1782 	struct vhost_dev *vdev = NULL;  in vring_state_changed()  local
1784 	TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {  in vring_state_changed()
1785 		if (vdev->vid == vid)  in vring_state_changed()
1788 	if (!vdev)  in vring_state_changed()
1793 			vhost_clear_queue_thread_unsafe(vdev, queue_id);  in vring_state_changed()
1817 	struct vhost_dev *vdev;  in print_stats()  local
1830 		TAILQ_FOREACH(vdev, &vhost_dev_list, global_vdev_entry) {  in print_stats()
1831 			tx_total   = vdev->stats.tx_total;  in print_stats()
1832 			tx         = vdev->stats.tx;  in print_stats()
1835 			rx_total = rte_atomic_load_explicit(&vdev->stats.rx_total_atomic,  in print_stats()
1837 			rx         = rte_atomic_load_explicit(&vdev->stats.rx_atomic,  in print_stats()
1849 				vdev->vid,  in print_stats()