1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2009-2012,2016 Microsoft Corp. 3 * Copyright (c) 2012 NetApp Inc. 4 * Copyright (c) 2012 Citrix Inc. 5 * All rights reserved. 6 */ 7 8 #include <unistd.h> 9 #include <stdint.h> 10 #include <stdbool.h> 11 #include <string.h> 12 #include <sys/uio.h> 13 14 #include <rte_eal.h> 15 #include <rte_tailq.h> 16 #include <rte_log.h> 17 #include <rte_malloc.h> 18 #include <rte_bus.h> 19 #include <rte_atomic.h> 20 #include <rte_memory.h> 21 #include <rte_pause.h> 22 #include <rte_bus_vmbus.h> 23 24 #include "private.h" 25 26 /* Increase bufring index by inc with wraparound */ 27 static inline uint32_t vmbus_br_idxinc(uint32_t idx, uint32_t inc, uint32_t sz) 28 { 29 idx += inc; 30 if (idx >= sz) 31 idx -= sz; 32 33 return idx; 34 } 35 36 void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen) 37 { 38 br->vbr = buf; 39 br->windex = br->vbr->windex; 40 br->dsize = blen - sizeof(struct vmbus_bufring); 41 } 42 43 /* 44 * When we write to the ring buffer, check if the host needs to be 45 * signaled. 46 * 47 * The contract: 48 * - The host guarantees that while it is draining the TX bufring, 49 * it will set the br_imask to indicate it does not need to be 50 * interrupted when new data are added. 51 * - The host guarantees that it will completely drain the TX bufring 52 * before exiting the read loop. Further, once the TX bufring is 53 * empty, it will clear the br_imask and re-check to see if new 54 * data have arrived. 55 */ 56 static inline bool 57 vmbus_txbr_need_signal(const struct vmbus_bufring *vbr, uint32_t old_windex) 58 { 59 rte_smp_mb(); 60 if (vbr->imask) 61 return false; 62 63 rte_smp_rmb(); 64 65 /* 66 * This is the only case we need to signal when the 67 * ring transitions from being empty to non-empty. 68 */ 69 return old_windex == vbr->rindex; 70 } 71 72 static inline uint32_t 73 vmbus_txbr_copyto(const struct vmbus_br *tbr, uint32_t windex, 74 const void *src0, uint32_t cplen) 75 { 76 uint8_t *br_data = tbr->vbr->data; 77 uint32_t br_dsize = tbr->dsize; 78 const uint8_t *src = src0; 79 80 /* XXX use double mapping like Linux kernel? */ 81 if (cplen > br_dsize - windex) { 82 uint32_t fraglen = br_dsize - windex; 83 84 /* Wrap-around detected */ 85 memcpy(br_data + windex, src, fraglen); 86 memcpy(br_data, src + fraglen, cplen - fraglen); 87 } else { 88 memcpy(br_data + windex, src, cplen); 89 } 90 91 return vmbus_br_idxinc(windex, cplen, br_dsize); 92 } 93 94 /* 95 * Write scattered channel packet to TX bufring. 96 * 97 * The offset of this channel packet is written as a 64bits value 98 * immediately after this channel packet. 99 * 100 * The write goes through three stages: 101 * 1. Reserve space in ring buffer for the new data. 102 * Writer atomically moves priv_write_index. 103 * 2. Copy the new data into the ring. 104 * 3. Update the tail of the ring (visible to host) that indicates 105 * next read location. Writer updates write_index 106 */ 107 int 108 vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen, 109 bool *need_sig) 110 { 111 struct vmbus_bufring *vbr = tbr->vbr; 112 uint32_t ring_size = tbr->dsize; 113 uint32_t old_windex, next_windex, windex, total; 114 uint64_t save_windex; 115 int i; 116 117 total = 0; 118 for (i = 0; i < iovlen; i++) 119 total += iov[i].iov_len; 120 total += sizeof(save_windex); 121 122 /* Reserve space in ring */ 123 do { 124 uint32_t avail; 125 126 /* Get current free location */ 127 old_windex = tbr->windex; 128 129 /* Prevent compiler reordering this with calculation */ 130 rte_compiler_barrier(); 131 132 avail = vmbus_br_availwrite(tbr, old_windex); 133 134 /* If not enough space in ring, then tell caller. */ 135 if (avail <= total) 136 return -EAGAIN; 137 138 next_windex = vmbus_br_idxinc(old_windex, total, ring_size); 139 140 /* Atomic update of next write_index for other threads */ 141 } while (!rte_atomic32_cmpset(&tbr->windex, old_windex, next_windex)); 142 143 /* Space from old..new is now reserved */ 144 windex = old_windex; 145 for (i = 0; i < iovlen; i++) { 146 windex = vmbus_txbr_copyto(tbr, windex, 147 iov[i].iov_base, iov[i].iov_len); 148 } 149 150 /* Set the offset of the current channel packet. */ 151 save_windex = ((uint64_t)old_windex) << 32; 152 windex = vmbus_txbr_copyto(tbr, windex, &save_windex, 153 sizeof(save_windex)); 154 155 /* The region reserved should match region used */ 156 RTE_ASSERT(windex == next_windex); 157 158 /* Ensure that data is available before updating host index */ 159 rte_smp_wmb(); 160 161 /* Checkin for our reservation. wait for our turn to update host */ 162 while (!rte_atomic32_cmpset(&vbr->windex, old_windex, next_windex)) 163 rte_pause(); 164 165 /* If host had read all data before this, then need to signal */ 166 *need_sig |= vmbus_txbr_need_signal(vbr, old_windex); 167 return 0; 168 } 169 170 static inline uint32_t 171 vmbus_rxbr_copyfrom(const struct vmbus_br *rbr, uint32_t rindex, 172 void *dst0, size_t cplen) 173 { 174 const uint8_t *br_data = rbr->vbr->data; 175 uint32_t br_dsize = rbr->dsize; 176 uint8_t *dst = dst0; 177 178 if (cplen > br_dsize - rindex) { 179 uint32_t fraglen = br_dsize - rindex; 180 181 /* Wrap-around detected. */ 182 memcpy(dst, br_data + rindex, fraglen); 183 memcpy(dst + fraglen, br_data, cplen - fraglen); 184 } else { 185 memcpy(dst, br_data + rindex, cplen); 186 } 187 188 return vmbus_br_idxinc(rindex, cplen, br_dsize); 189 } 190 191 /* Copy data from receive ring but don't change index */ 192 int 193 vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen) 194 { 195 uint32_t avail; 196 197 /* 198 * The requested data and the 64bits channel packet 199 * offset should be there at least. 200 */ 201 avail = vmbus_br_availread(rbr); 202 if (avail < dlen + sizeof(uint64_t)) 203 return -EAGAIN; 204 205 vmbus_rxbr_copyfrom(rbr, rbr->vbr->rindex, data, dlen); 206 return 0; 207 } 208 209 /* 210 * Copy data from receive ring and change index 211 * NOTE: 212 * We assume (dlen + skip) == sizeof(channel packet). 213 */ 214 int 215 vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t skip) 216 { 217 struct vmbus_bufring *vbr = rbr->vbr; 218 uint32_t br_dsize = rbr->dsize; 219 uint32_t rindex; 220 221 if (vmbus_br_availread(rbr) < dlen + skip + sizeof(uint64_t)) 222 return -EAGAIN; 223 224 /* Record where host was when we started read (for debug) */ 225 rbr->windex = rbr->vbr->windex; 226 227 /* 228 * Copy channel packet from RX bufring. 229 */ 230 rindex = vmbus_br_idxinc(rbr->vbr->rindex, skip, br_dsize); 231 rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen); 232 233 /* 234 * Discard this channel packet's 64bits offset, which is useless to us. 235 */ 236 rindex = vmbus_br_idxinc(rindex, sizeof(uint64_t), br_dsize); 237 238 /* Update the read index _after_ the channel packet is fetched. */ 239 rte_compiler_barrier(); 240 241 vbr->rindex = rindex; 242 243 return 0; 244 } 245