1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2009-2012,2016 Microsoft Corp.
3 * Copyright (c) 2012 NetApp Inc.
4 * Copyright (c) 2012 Citrix Inc.
5 * All rights reserved.
6 */
7
8 #include <unistd.h>
9 #include <stdint.h>
10 #include <stdbool.h>
11 #include <string.h>
12 #include <sys/uio.h>
13
14 #include <rte_eal.h>
15 #include <rte_tailq.h>
16 #include <rte_log.h>
17 #include <rte_malloc.h>
18 #include <rte_atomic.h>
19 #include <rte_memory.h>
20 #include <rte_pause.h>
21 #include <rte_bus_vmbus.h>
22
23 #include "private.h"
24
25 /* Increase bufring index by inc with wraparound */
vmbus_br_idxinc(uint32_t idx,uint32_t inc,uint32_t sz)26 static inline uint32_t vmbus_br_idxinc(uint32_t idx, uint32_t inc, uint32_t sz)
27 {
28 idx += inc;
29 if (idx >= sz)
30 idx -= sz;
31
32 return idx;
33 }
34
vmbus_br_setup(struct vmbus_br * br,void * buf,unsigned int blen)35 void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen)
36 {
37 br->vbr = buf;
38 br->windex = br->vbr->windex;
39 br->dsize = blen - sizeof(struct vmbus_bufring);
40 }
41
42 /*
43 * When we write to the ring buffer, check if the host needs to be
44 * signaled.
45 *
46 * The contract:
47 * - The host guarantees that while it is draining the TX bufring,
48 * it will set the br_imask to indicate it does not need to be
49 * interrupted when new data are added.
50 * - The host guarantees that it will completely drain the TX bufring
51 * before exiting the read loop. Further, once the TX bufring is
52 * empty, it will clear the br_imask and re-check to see if new
53 * data have arrived.
54 */
55 static inline bool
vmbus_txbr_need_signal(const struct vmbus_bufring * vbr,uint32_t old_windex)56 vmbus_txbr_need_signal(const struct vmbus_bufring *vbr, uint32_t old_windex)
57 {
58 rte_smp_mb();
59 if (vbr->imask)
60 return false;
61
62 rte_smp_rmb();
63
64 /*
65 * This is the only case we need to signal when the
66 * ring transitions from being empty to non-empty.
67 */
68 return old_windex == vbr->rindex;
69 }
70
71 static inline uint32_t
vmbus_txbr_copyto(const struct vmbus_br * tbr,uint32_t windex,const void * src0,uint32_t cplen)72 vmbus_txbr_copyto(const struct vmbus_br *tbr, uint32_t windex,
73 const void *src0, uint32_t cplen)
74 {
75 uint8_t *br_data = tbr->vbr->data;
76 uint32_t br_dsize = tbr->dsize;
77 const uint8_t *src = src0;
78
79 /* XXX use double mapping like Linux kernel? */
80 if (cplen > br_dsize - windex) {
81 uint32_t fraglen = br_dsize - windex;
82
83 /* Wrap-around detected */
84 memcpy(br_data + windex, src, fraglen);
85 memcpy(br_data, src + fraglen, cplen - fraglen);
86 } else {
87 memcpy(br_data + windex, src, cplen);
88 }
89
90 return vmbus_br_idxinc(windex, cplen, br_dsize);
91 }
92
93 /*
94 * Write scattered channel packet to TX bufring.
95 *
96 * The offset of this channel packet is written as a 64bits value
97 * immediately after this channel packet.
98 *
99 * The write goes through three stages:
100 * 1. Reserve space in ring buffer for the new data.
101 * Writer atomically moves priv_write_index.
102 * 2. Copy the new data into the ring.
103 * 3. Update the tail of the ring (visible to host) that indicates
104 * next read location. Writer updates write_index
105 */
106 int
vmbus_txbr_write(struct vmbus_br * tbr,const struct iovec iov[],int iovlen,bool * need_sig)107 vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen,
108 bool *need_sig)
109 {
110 struct vmbus_bufring *vbr = tbr->vbr;
111 uint32_t ring_size = tbr->dsize;
112 uint32_t old_windex, next_windex, windex, total;
113 uint64_t save_windex;
114 int i;
115
116 total = 0;
117 for (i = 0; i < iovlen; i++)
118 total += iov[i].iov_len;
119 total += sizeof(save_windex);
120
121 /* Reserve space in ring */
122 do {
123 uint32_t avail;
124
125 /* Get current free location */
126 old_windex = tbr->windex;
127
128 /* Prevent compiler reordering this with calculation */
129 rte_compiler_barrier();
130
131 avail = vmbus_br_availwrite(tbr, old_windex);
132
133 /* If not enough space in ring, then tell caller. */
134 if (avail <= total)
135 return -EAGAIN;
136
137 next_windex = vmbus_br_idxinc(old_windex, total, ring_size);
138
139 /* Atomic update of next write_index for other threads */
140 } while (!rte_atomic32_cmpset(&tbr->windex, old_windex, next_windex));
141
142 /* Space from old..new is now reserved */
143 windex = old_windex;
144 for (i = 0; i < iovlen; i++) {
145 windex = vmbus_txbr_copyto(tbr, windex,
146 iov[i].iov_base, iov[i].iov_len);
147 }
148
149 /* Set the offset of the current channel packet. */
150 save_windex = ((uint64_t)old_windex) << 32;
151 windex = vmbus_txbr_copyto(tbr, windex, &save_windex,
152 sizeof(save_windex));
153
154 /* The region reserved should match region used */
155 RTE_ASSERT(windex == next_windex);
156
157 /* Ensure that data is available before updating host index */
158 rte_smp_wmb();
159
160 /* Checkin for our reservation. wait for our turn to update host */
161 while (!rte_atomic32_cmpset(&vbr->windex, old_windex, next_windex))
162 rte_pause();
163
164 /* If host had read all data before this, then need to signal */
165 *need_sig |= vmbus_txbr_need_signal(vbr, old_windex);
166 return 0;
167 }
168
169 static inline uint32_t
vmbus_rxbr_copyfrom(const struct vmbus_br * rbr,uint32_t rindex,void * dst0,size_t cplen)170 vmbus_rxbr_copyfrom(const struct vmbus_br *rbr, uint32_t rindex,
171 void *dst0, size_t cplen)
172 {
173 const uint8_t *br_data = rbr->vbr->data;
174 uint32_t br_dsize = rbr->dsize;
175 uint8_t *dst = dst0;
176
177 if (cplen > br_dsize - rindex) {
178 uint32_t fraglen = br_dsize - rindex;
179
180 /* Wrap-around detected. */
181 memcpy(dst, br_data + rindex, fraglen);
182 memcpy(dst + fraglen, br_data, cplen - fraglen);
183 } else {
184 memcpy(dst, br_data + rindex, cplen);
185 }
186
187 return vmbus_br_idxinc(rindex, cplen, br_dsize);
188 }
189
190 /* Copy data from receive ring but don't change index */
191 int
vmbus_rxbr_peek(const struct vmbus_br * rbr,void * data,size_t dlen)192 vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen)
193 {
194 uint32_t avail;
195
196 /*
197 * The requested data and the 64bits channel packet
198 * offset should be there at least.
199 */
200 avail = vmbus_br_availread(rbr);
201 if (avail < dlen + sizeof(uint64_t))
202 return -EAGAIN;
203
204 vmbus_rxbr_copyfrom(rbr, rbr->vbr->rindex, data, dlen);
205 return 0;
206 }
207
208 /*
209 * Copy data from receive ring and change index
210 * NOTE:
211 * We assume (dlen + skip) == sizeof(channel packet).
212 */
213 int
vmbus_rxbr_read(struct vmbus_br * rbr,void * data,size_t dlen,size_t skip)214 vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t skip)
215 {
216 struct vmbus_bufring *vbr = rbr->vbr;
217 uint32_t br_dsize = rbr->dsize;
218 uint32_t rindex;
219
220 if (vmbus_br_availread(rbr) < dlen + skip + sizeof(uint64_t))
221 return -EAGAIN;
222
223 /* Record where host was when we started read (for debug) */
224 rbr->windex = rbr->vbr->windex;
225
226 /*
227 * Copy channel packet from RX bufring.
228 */
229 rindex = vmbus_br_idxinc(rbr->vbr->rindex, skip, br_dsize);
230 rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen);
231
232 /*
233 * Discard this channel packet's 64bits offset, which is useless to us.
234 */
235 rindex = vmbus_br_idxinc(rindex, sizeof(uint64_t), br_dsize);
236
237 /* Update the read index _after_ the channel packet is fetched. */
238 rte_compiler_barrier();
239
240 vbr->rindex = rindex;
241
242 return 0;
243 }
244