xref: /dpdk/drivers/bus/vmbus/vmbus_bufring.c (revision 831dba47bd365b8a539dfb51fffdd01f8a436f6c)
1*831dba47SStephen Hemminger /* SPDX-License-Identifier: BSD-3-Clause
2*831dba47SStephen Hemminger  * Copyright (c) 2009-2012,2016 Microsoft Corp.
3*831dba47SStephen Hemminger  * Copyright (c) 2012 NetApp Inc.
4*831dba47SStephen Hemminger  * Copyright (c) 2012 Citrix Inc.
5*831dba47SStephen Hemminger  * All rights reserved.
6*831dba47SStephen Hemminger  */
7*831dba47SStephen Hemminger 
8*831dba47SStephen Hemminger #include <unistd.h>
9*831dba47SStephen Hemminger #include <stdint.h>
10*831dba47SStephen Hemminger #include <stdbool.h>
11*831dba47SStephen Hemminger #include <string.h>
12*831dba47SStephen Hemminger #include <sys/uio.h>
13*831dba47SStephen Hemminger 
14*831dba47SStephen Hemminger #include <rte_eal.h>
15*831dba47SStephen Hemminger #include <rte_tailq.h>
16*831dba47SStephen Hemminger #include <rte_log.h>
17*831dba47SStephen Hemminger #include <rte_malloc.h>
18*831dba47SStephen Hemminger #include <rte_bus.h>
19*831dba47SStephen Hemminger #include <rte_atomic.h>
20*831dba47SStephen Hemminger #include <rte_memory.h>
21*831dba47SStephen Hemminger #include <rte_pause.h>
22*831dba47SStephen Hemminger #include <rte_bus_vmbus.h>
23*831dba47SStephen Hemminger 
24*831dba47SStephen Hemminger #include "private.h"
25*831dba47SStephen Hemminger 
26*831dba47SStephen Hemminger /* Increase bufring index by inc with wraparound */
27*831dba47SStephen Hemminger static inline uint32_t vmbus_br_idxinc(uint32_t idx, uint32_t inc, uint32_t sz)
28*831dba47SStephen Hemminger {
29*831dba47SStephen Hemminger 	idx += inc;
30*831dba47SStephen Hemminger 	if (idx >= sz)
31*831dba47SStephen Hemminger 		idx -= sz;
32*831dba47SStephen Hemminger 
33*831dba47SStephen Hemminger 	return idx;
34*831dba47SStephen Hemminger }
35*831dba47SStephen Hemminger 
36*831dba47SStephen Hemminger void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen)
37*831dba47SStephen Hemminger {
38*831dba47SStephen Hemminger 	br->vbr = buf;
39*831dba47SStephen Hemminger 	br->windex = br->vbr->windex;
40*831dba47SStephen Hemminger 	br->dsize = blen - sizeof(struct vmbus_bufring);
41*831dba47SStephen Hemminger }
42*831dba47SStephen Hemminger 
43*831dba47SStephen Hemminger /*
44*831dba47SStephen Hemminger  * When we write to the ring buffer, check if the host needs to be
45*831dba47SStephen Hemminger  * signaled.
46*831dba47SStephen Hemminger  *
47*831dba47SStephen Hemminger  * The contract:
48*831dba47SStephen Hemminger  * - The host guarantees that while it is draining the TX bufring,
49*831dba47SStephen Hemminger  *   it will set the br_imask to indicate it does not need to be
50*831dba47SStephen Hemminger  *   interrupted when new data are added.
51*831dba47SStephen Hemminger  * - The host guarantees that it will completely drain the TX bufring
52*831dba47SStephen Hemminger  *   before exiting the read loop.  Further, once the TX bufring is
53*831dba47SStephen Hemminger  *   empty, it will clear the br_imask and re-check to see if new
54*831dba47SStephen Hemminger  *   data have arrived.
55*831dba47SStephen Hemminger  */
56*831dba47SStephen Hemminger static inline bool
57*831dba47SStephen Hemminger vmbus_txbr_need_signal(const struct vmbus_br *tbr, uint32_t old_windex)
58*831dba47SStephen Hemminger {
59*831dba47SStephen Hemminger 	rte_smp_mb();
60*831dba47SStephen Hemminger 	if (tbr->vbr->imask)
61*831dba47SStephen Hemminger 		return false;
62*831dba47SStephen Hemminger 
63*831dba47SStephen Hemminger 	rte_smp_rmb();
64*831dba47SStephen Hemminger 
65*831dba47SStephen Hemminger 	/*
66*831dba47SStephen Hemminger 	 * This is the only case we need to signal when the
67*831dba47SStephen Hemminger 	 * ring transitions from being empty to non-empty.
68*831dba47SStephen Hemminger 	 */
69*831dba47SStephen Hemminger 	return old_windex == tbr->vbr->rindex;
70*831dba47SStephen Hemminger }
71*831dba47SStephen Hemminger 
72*831dba47SStephen Hemminger static inline uint32_t
73*831dba47SStephen Hemminger vmbus_txbr_copyto(const struct vmbus_br *tbr, uint32_t windex,
74*831dba47SStephen Hemminger 		  const void *src0, uint32_t cplen)
75*831dba47SStephen Hemminger {
76*831dba47SStephen Hemminger 	uint8_t *br_data = tbr->vbr->data;
77*831dba47SStephen Hemminger 	uint32_t br_dsize = tbr->dsize;
78*831dba47SStephen Hemminger 	const uint8_t *src = src0;
79*831dba47SStephen Hemminger 
80*831dba47SStephen Hemminger 	/* XXX use double mapping like Linux kernel? */
81*831dba47SStephen Hemminger 	if (cplen > br_dsize - windex) {
82*831dba47SStephen Hemminger 		uint32_t fraglen = br_dsize - windex;
83*831dba47SStephen Hemminger 
84*831dba47SStephen Hemminger 		/* Wrap-around detected */
85*831dba47SStephen Hemminger 		memcpy(br_data + windex, src, fraglen);
86*831dba47SStephen Hemminger 		memcpy(br_data, src + fraglen, cplen - fraglen);
87*831dba47SStephen Hemminger 	} else {
88*831dba47SStephen Hemminger 		memcpy(br_data + windex, src, cplen);
89*831dba47SStephen Hemminger 	}
90*831dba47SStephen Hemminger 
91*831dba47SStephen Hemminger 	return vmbus_br_idxinc(windex, cplen, br_dsize);
92*831dba47SStephen Hemminger }
93*831dba47SStephen Hemminger 
94*831dba47SStephen Hemminger /*
95*831dba47SStephen Hemminger  * Write scattered channel packet to TX bufring.
96*831dba47SStephen Hemminger  *
97*831dba47SStephen Hemminger  * The offset of this channel packet is written as a 64bits value
98*831dba47SStephen Hemminger  * immediately after this channel packet.
99*831dba47SStephen Hemminger  *
100*831dba47SStephen Hemminger  * The write goes through three stages:
101*831dba47SStephen Hemminger  *  1. Reserve space in ring buffer for the new data.
102*831dba47SStephen Hemminger  *     Writer atomically moves priv_write_index.
103*831dba47SStephen Hemminger  *  2. Copy the new data into the ring.
104*831dba47SStephen Hemminger  *  3. Update the tail of the ring (visible to host) that indicates
105*831dba47SStephen Hemminger  *     next read location. Writer updates write_index
106*831dba47SStephen Hemminger  */
107*831dba47SStephen Hemminger int
108*831dba47SStephen Hemminger vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen,
109*831dba47SStephen Hemminger 		 bool *need_sig)
110*831dba47SStephen Hemminger {
111*831dba47SStephen Hemminger 	struct vmbus_bufring *vbr = tbr->vbr;
112*831dba47SStephen Hemminger 	uint32_t ring_size = tbr->dsize;
113*831dba47SStephen Hemminger 	uint32_t old_windex, next_windex, windex, total;
114*831dba47SStephen Hemminger 	uint64_t save_windex;
115*831dba47SStephen Hemminger 	int i;
116*831dba47SStephen Hemminger 
117*831dba47SStephen Hemminger 	total = 0;
118*831dba47SStephen Hemminger 	for (i = 0; i < iovlen; i++)
119*831dba47SStephen Hemminger 		total += iov[i].iov_len;
120*831dba47SStephen Hemminger 	total += sizeof(save_windex);
121*831dba47SStephen Hemminger 
122*831dba47SStephen Hemminger 	/* Reserve space in ring */
123*831dba47SStephen Hemminger 	do {
124*831dba47SStephen Hemminger 		uint32_t avail;
125*831dba47SStephen Hemminger 
126*831dba47SStephen Hemminger 		/* Get current free location */
127*831dba47SStephen Hemminger 		old_windex = tbr->windex;
128*831dba47SStephen Hemminger 
129*831dba47SStephen Hemminger 		/* Prevent compiler reordering this with calculation */
130*831dba47SStephen Hemminger 		rte_compiler_barrier();
131*831dba47SStephen Hemminger 
132*831dba47SStephen Hemminger 		avail = vmbus_br_availwrite(tbr, old_windex);
133*831dba47SStephen Hemminger 
134*831dba47SStephen Hemminger 		/* If not enough space in ring, then tell caller. */
135*831dba47SStephen Hemminger 		if (avail <= total)
136*831dba47SStephen Hemminger 			return -EAGAIN;
137*831dba47SStephen Hemminger 
138*831dba47SStephen Hemminger 		next_windex = vmbus_br_idxinc(old_windex, total, ring_size);
139*831dba47SStephen Hemminger 
140*831dba47SStephen Hemminger 		/* Atomic update of next write_index for other threads */
141*831dba47SStephen Hemminger 	} while (!rte_atomic32_cmpset(&tbr->windex, old_windex, next_windex));
142*831dba47SStephen Hemminger 
143*831dba47SStephen Hemminger 	/* Space from old..new is now reserved */
144*831dba47SStephen Hemminger 	windex = old_windex;
145*831dba47SStephen Hemminger 	for (i = 0; i < iovlen; i++) {
146*831dba47SStephen Hemminger 		windex = vmbus_txbr_copyto(tbr, windex,
147*831dba47SStephen Hemminger 					   iov[i].iov_base, iov[i].iov_len);
148*831dba47SStephen Hemminger 	}
149*831dba47SStephen Hemminger 
150*831dba47SStephen Hemminger 	/* Set the offset of the current channel packet. */
151*831dba47SStephen Hemminger 	save_windex = ((uint64_t)old_windex) << 32;
152*831dba47SStephen Hemminger 	windex = vmbus_txbr_copyto(tbr, windex, &save_windex,
153*831dba47SStephen Hemminger 				   sizeof(save_windex));
154*831dba47SStephen Hemminger 
155*831dba47SStephen Hemminger 	/* The region reserved should match region used */
156*831dba47SStephen Hemminger 	RTE_ASSERT(windex == next_windex);
157*831dba47SStephen Hemminger 
158*831dba47SStephen Hemminger 	/* Ensure that data is available before updating host index */
159*831dba47SStephen Hemminger 	rte_smp_wmb();
160*831dba47SStephen Hemminger 
161*831dba47SStephen Hemminger 	/* Checkin for our reservation. wait for our turn to update host */
162*831dba47SStephen Hemminger 	while (!rte_atomic32_cmpset(&vbr->windex, old_windex, next_windex))
163*831dba47SStephen Hemminger 		rte_pause();
164*831dba47SStephen Hemminger 
165*831dba47SStephen Hemminger 	/* If host had read all data before this, then need to signal */
166*831dba47SStephen Hemminger 	*need_sig |= vmbus_txbr_need_signal(tbr, old_windex);
167*831dba47SStephen Hemminger 	return 0;
168*831dba47SStephen Hemminger }
169*831dba47SStephen Hemminger 
170*831dba47SStephen Hemminger static inline uint32_t
171*831dba47SStephen Hemminger vmbus_rxbr_copyfrom(const struct vmbus_br *rbr, uint32_t rindex,
172*831dba47SStephen Hemminger 		    void *dst0, size_t cplen)
173*831dba47SStephen Hemminger {
174*831dba47SStephen Hemminger 	const uint8_t *br_data = rbr->vbr->data;
175*831dba47SStephen Hemminger 	uint32_t br_dsize = rbr->dsize;
176*831dba47SStephen Hemminger 	uint8_t *dst = dst0;
177*831dba47SStephen Hemminger 
178*831dba47SStephen Hemminger 	if (cplen > br_dsize - rindex) {
179*831dba47SStephen Hemminger 		uint32_t fraglen = br_dsize - rindex;
180*831dba47SStephen Hemminger 
181*831dba47SStephen Hemminger 		/* Wrap-around detected. */
182*831dba47SStephen Hemminger 		memcpy(dst, br_data + rindex, fraglen);
183*831dba47SStephen Hemminger 		memcpy(dst + fraglen, br_data, cplen - fraglen);
184*831dba47SStephen Hemminger 	} else {
185*831dba47SStephen Hemminger 		memcpy(dst, br_data + rindex, cplen);
186*831dba47SStephen Hemminger 	}
187*831dba47SStephen Hemminger 
188*831dba47SStephen Hemminger 	return vmbus_br_idxinc(rindex, cplen, br_dsize);
189*831dba47SStephen Hemminger }
190*831dba47SStephen Hemminger 
191*831dba47SStephen Hemminger /* Copy data from receive ring but don't change index */
192*831dba47SStephen Hemminger int
193*831dba47SStephen Hemminger vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen)
194*831dba47SStephen Hemminger {
195*831dba47SStephen Hemminger 	uint32_t avail;
196*831dba47SStephen Hemminger 
197*831dba47SStephen Hemminger 	/*
198*831dba47SStephen Hemminger 	 * The requested data and the 64bits channel packet
199*831dba47SStephen Hemminger 	 * offset should be there at least.
200*831dba47SStephen Hemminger 	 */
201*831dba47SStephen Hemminger 	avail = vmbus_br_availread(rbr);
202*831dba47SStephen Hemminger 	if (avail < dlen + sizeof(uint64_t))
203*831dba47SStephen Hemminger 		return -EAGAIN;
204*831dba47SStephen Hemminger 
205*831dba47SStephen Hemminger 	vmbus_rxbr_copyfrom(rbr, rbr->vbr->rindex, data, dlen);
206*831dba47SStephen Hemminger 	return 0;
207*831dba47SStephen Hemminger }
208*831dba47SStephen Hemminger 
209*831dba47SStephen Hemminger /*
210*831dba47SStephen Hemminger  * Copy data from receive ring and change index
211*831dba47SStephen Hemminger  * NOTE:
212*831dba47SStephen Hemminger  * We assume (dlen + skip) == sizeof(channel packet).
213*831dba47SStephen Hemminger  */
214*831dba47SStephen Hemminger int
215*831dba47SStephen Hemminger vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t skip)
216*831dba47SStephen Hemminger {
217*831dba47SStephen Hemminger 	struct vmbus_bufring *vbr = rbr->vbr;
218*831dba47SStephen Hemminger 	uint32_t br_dsize = rbr->dsize;
219*831dba47SStephen Hemminger 	uint32_t rindex;
220*831dba47SStephen Hemminger 
221*831dba47SStephen Hemminger 	if (vmbus_br_availread(rbr) < dlen + skip + sizeof(uint64_t))
222*831dba47SStephen Hemminger 		return -EAGAIN;
223*831dba47SStephen Hemminger 
224*831dba47SStephen Hemminger 	/*
225*831dba47SStephen Hemminger 	 * Copy channel packet from RX bufring.
226*831dba47SStephen Hemminger 	 */
227*831dba47SStephen Hemminger 	rindex = vmbus_br_idxinc(rbr->vbr->rindex, skip, br_dsize);
228*831dba47SStephen Hemminger 	rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen);
229*831dba47SStephen Hemminger 
230*831dba47SStephen Hemminger 	/*
231*831dba47SStephen Hemminger 	 * Discard this channel packet's 64bits offset, which is useless to us.
232*831dba47SStephen Hemminger 	 */
233*831dba47SStephen Hemminger 	rindex = vmbus_br_idxinc(rindex, sizeof(uint64_t), br_dsize);
234*831dba47SStephen Hemminger 
235*831dba47SStephen Hemminger 	/* Update the read index _after_ the channel packet is fetched.	 */
236*831dba47SStephen Hemminger 	rte_compiler_barrier();
237*831dba47SStephen Hemminger 
238*831dba47SStephen Hemminger 	vbr->rindex = rindex;
239*831dba47SStephen Hemminger 
240*831dba47SStephen Hemminger 	return 0;
241*831dba47SStephen Hemminger }
242