1*831dba47SStephen Hemminger /* SPDX-License-Identifier: BSD-3-Clause 2*831dba47SStephen Hemminger * Copyright (c) 2009-2012,2016 Microsoft Corp. 3*831dba47SStephen Hemminger * Copyright (c) 2012 NetApp Inc. 4*831dba47SStephen Hemminger * Copyright (c) 2012 Citrix Inc. 5*831dba47SStephen Hemminger * All rights reserved. 6*831dba47SStephen Hemminger */ 7*831dba47SStephen Hemminger 8*831dba47SStephen Hemminger #include <unistd.h> 9*831dba47SStephen Hemminger #include <stdint.h> 10*831dba47SStephen Hemminger #include <stdbool.h> 11*831dba47SStephen Hemminger #include <string.h> 12*831dba47SStephen Hemminger #include <sys/uio.h> 13*831dba47SStephen Hemminger 14*831dba47SStephen Hemminger #include <rte_eal.h> 15*831dba47SStephen Hemminger #include <rte_tailq.h> 16*831dba47SStephen Hemminger #include <rte_log.h> 17*831dba47SStephen Hemminger #include <rte_malloc.h> 18*831dba47SStephen Hemminger #include <rte_bus.h> 19*831dba47SStephen Hemminger #include <rte_atomic.h> 20*831dba47SStephen Hemminger #include <rte_memory.h> 21*831dba47SStephen Hemminger #include <rte_pause.h> 22*831dba47SStephen Hemminger #include <rte_bus_vmbus.h> 23*831dba47SStephen Hemminger 24*831dba47SStephen Hemminger #include "private.h" 25*831dba47SStephen Hemminger 26*831dba47SStephen Hemminger /* Increase bufring index by inc with wraparound */ 27*831dba47SStephen Hemminger static inline uint32_t vmbus_br_idxinc(uint32_t idx, uint32_t inc, uint32_t sz) 28*831dba47SStephen Hemminger { 29*831dba47SStephen Hemminger idx += inc; 30*831dba47SStephen Hemminger if (idx >= sz) 31*831dba47SStephen Hemminger idx -= sz; 32*831dba47SStephen Hemminger 33*831dba47SStephen Hemminger return idx; 34*831dba47SStephen Hemminger } 35*831dba47SStephen Hemminger 36*831dba47SStephen Hemminger void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen) 37*831dba47SStephen Hemminger { 38*831dba47SStephen Hemminger br->vbr = buf; 39*831dba47SStephen Hemminger br->windex = br->vbr->windex; 40*831dba47SStephen Hemminger br->dsize = blen - sizeof(struct vmbus_bufring); 41*831dba47SStephen Hemminger } 42*831dba47SStephen Hemminger 43*831dba47SStephen Hemminger /* 44*831dba47SStephen Hemminger * When we write to the ring buffer, check if the host needs to be 45*831dba47SStephen Hemminger * signaled. 46*831dba47SStephen Hemminger * 47*831dba47SStephen Hemminger * The contract: 48*831dba47SStephen Hemminger * - The host guarantees that while it is draining the TX bufring, 49*831dba47SStephen Hemminger * it will set the br_imask to indicate it does not need to be 50*831dba47SStephen Hemminger * interrupted when new data are added. 51*831dba47SStephen Hemminger * - The host guarantees that it will completely drain the TX bufring 52*831dba47SStephen Hemminger * before exiting the read loop. Further, once the TX bufring is 53*831dba47SStephen Hemminger * empty, it will clear the br_imask and re-check to see if new 54*831dba47SStephen Hemminger * data have arrived. 55*831dba47SStephen Hemminger */ 56*831dba47SStephen Hemminger static inline bool 57*831dba47SStephen Hemminger vmbus_txbr_need_signal(const struct vmbus_br *tbr, uint32_t old_windex) 58*831dba47SStephen Hemminger { 59*831dba47SStephen Hemminger rte_smp_mb(); 60*831dba47SStephen Hemminger if (tbr->vbr->imask) 61*831dba47SStephen Hemminger return false; 62*831dba47SStephen Hemminger 63*831dba47SStephen Hemminger rte_smp_rmb(); 64*831dba47SStephen Hemminger 65*831dba47SStephen Hemminger /* 66*831dba47SStephen Hemminger * This is the only case we need to signal when the 67*831dba47SStephen Hemminger * ring transitions from being empty to non-empty. 68*831dba47SStephen Hemminger */ 69*831dba47SStephen Hemminger return old_windex == tbr->vbr->rindex; 70*831dba47SStephen Hemminger } 71*831dba47SStephen Hemminger 72*831dba47SStephen Hemminger static inline uint32_t 73*831dba47SStephen Hemminger vmbus_txbr_copyto(const struct vmbus_br *tbr, uint32_t windex, 74*831dba47SStephen Hemminger const void *src0, uint32_t cplen) 75*831dba47SStephen Hemminger { 76*831dba47SStephen Hemminger uint8_t *br_data = tbr->vbr->data; 77*831dba47SStephen Hemminger uint32_t br_dsize = tbr->dsize; 78*831dba47SStephen Hemminger const uint8_t *src = src0; 79*831dba47SStephen Hemminger 80*831dba47SStephen Hemminger /* XXX use double mapping like Linux kernel? */ 81*831dba47SStephen Hemminger if (cplen > br_dsize - windex) { 82*831dba47SStephen Hemminger uint32_t fraglen = br_dsize - windex; 83*831dba47SStephen Hemminger 84*831dba47SStephen Hemminger /* Wrap-around detected */ 85*831dba47SStephen Hemminger memcpy(br_data + windex, src, fraglen); 86*831dba47SStephen Hemminger memcpy(br_data, src + fraglen, cplen - fraglen); 87*831dba47SStephen Hemminger } else { 88*831dba47SStephen Hemminger memcpy(br_data + windex, src, cplen); 89*831dba47SStephen Hemminger } 90*831dba47SStephen Hemminger 91*831dba47SStephen Hemminger return vmbus_br_idxinc(windex, cplen, br_dsize); 92*831dba47SStephen Hemminger } 93*831dba47SStephen Hemminger 94*831dba47SStephen Hemminger /* 95*831dba47SStephen Hemminger * Write scattered channel packet to TX bufring. 96*831dba47SStephen Hemminger * 97*831dba47SStephen Hemminger * The offset of this channel packet is written as a 64bits value 98*831dba47SStephen Hemminger * immediately after this channel packet. 99*831dba47SStephen Hemminger * 100*831dba47SStephen Hemminger * The write goes through three stages: 101*831dba47SStephen Hemminger * 1. Reserve space in ring buffer for the new data. 102*831dba47SStephen Hemminger * Writer atomically moves priv_write_index. 103*831dba47SStephen Hemminger * 2. Copy the new data into the ring. 104*831dba47SStephen Hemminger * 3. Update the tail of the ring (visible to host) that indicates 105*831dba47SStephen Hemminger * next read location. Writer updates write_index 106*831dba47SStephen Hemminger */ 107*831dba47SStephen Hemminger int 108*831dba47SStephen Hemminger vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen, 109*831dba47SStephen Hemminger bool *need_sig) 110*831dba47SStephen Hemminger { 111*831dba47SStephen Hemminger struct vmbus_bufring *vbr = tbr->vbr; 112*831dba47SStephen Hemminger uint32_t ring_size = tbr->dsize; 113*831dba47SStephen Hemminger uint32_t old_windex, next_windex, windex, total; 114*831dba47SStephen Hemminger uint64_t save_windex; 115*831dba47SStephen Hemminger int i; 116*831dba47SStephen Hemminger 117*831dba47SStephen Hemminger total = 0; 118*831dba47SStephen Hemminger for (i = 0; i < iovlen; i++) 119*831dba47SStephen Hemminger total += iov[i].iov_len; 120*831dba47SStephen Hemminger total += sizeof(save_windex); 121*831dba47SStephen Hemminger 122*831dba47SStephen Hemminger /* Reserve space in ring */ 123*831dba47SStephen Hemminger do { 124*831dba47SStephen Hemminger uint32_t avail; 125*831dba47SStephen Hemminger 126*831dba47SStephen Hemminger /* Get current free location */ 127*831dba47SStephen Hemminger old_windex = tbr->windex; 128*831dba47SStephen Hemminger 129*831dba47SStephen Hemminger /* Prevent compiler reordering this with calculation */ 130*831dba47SStephen Hemminger rte_compiler_barrier(); 131*831dba47SStephen Hemminger 132*831dba47SStephen Hemminger avail = vmbus_br_availwrite(tbr, old_windex); 133*831dba47SStephen Hemminger 134*831dba47SStephen Hemminger /* If not enough space in ring, then tell caller. */ 135*831dba47SStephen Hemminger if (avail <= total) 136*831dba47SStephen Hemminger return -EAGAIN; 137*831dba47SStephen Hemminger 138*831dba47SStephen Hemminger next_windex = vmbus_br_idxinc(old_windex, total, ring_size); 139*831dba47SStephen Hemminger 140*831dba47SStephen Hemminger /* Atomic update of next write_index for other threads */ 141*831dba47SStephen Hemminger } while (!rte_atomic32_cmpset(&tbr->windex, old_windex, next_windex)); 142*831dba47SStephen Hemminger 143*831dba47SStephen Hemminger /* Space from old..new is now reserved */ 144*831dba47SStephen Hemminger windex = old_windex; 145*831dba47SStephen Hemminger for (i = 0; i < iovlen; i++) { 146*831dba47SStephen Hemminger windex = vmbus_txbr_copyto(tbr, windex, 147*831dba47SStephen Hemminger iov[i].iov_base, iov[i].iov_len); 148*831dba47SStephen Hemminger } 149*831dba47SStephen Hemminger 150*831dba47SStephen Hemminger /* Set the offset of the current channel packet. */ 151*831dba47SStephen Hemminger save_windex = ((uint64_t)old_windex) << 32; 152*831dba47SStephen Hemminger windex = vmbus_txbr_copyto(tbr, windex, &save_windex, 153*831dba47SStephen Hemminger sizeof(save_windex)); 154*831dba47SStephen Hemminger 155*831dba47SStephen Hemminger /* The region reserved should match region used */ 156*831dba47SStephen Hemminger RTE_ASSERT(windex == next_windex); 157*831dba47SStephen Hemminger 158*831dba47SStephen Hemminger /* Ensure that data is available before updating host index */ 159*831dba47SStephen Hemminger rte_smp_wmb(); 160*831dba47SStephen Hemminger 161*831dba47SStephen Hemminger /* Checkin for our reservation. wait for our turn to update host */ 162*831dba47SStephen Hemminger while (!rte_atomic32_cmpset(&vbr->windex, old_windex, next_windex)) 163*831dba47SStephen Hemminger rte_pause(); 164*831dba47SStephen Hemminger 165*831dba47SStephen Hemminger /* If host had read all data before this, then need to signal */ 166*831dba47SStephen Hemminger *need_sig |= vmbus_txbr_need_signal(tbr, old_windex); 167*831dba47SStephen Hemminger return 0; 168*831dba47SStephen Hemminger } 169*831dba47SStephen Hemminger 170*831dba47SStephen Hemminger static inline uint32_t 171*831dba47SStephen Hemminger vmbus_rxbr_copyfrom(const struct vmbus_br *rbr, uint32_t rindex, 172*831dba47SStephen Hemminger void *dst0, size_t cplen) 173*831dba47SStephen Hemminger { 174*831dba47SStephen Hemminger const uint8_t *br_data = rbr->vbr->data; 175*831dba47SStephen Hemminger uint32_t br_dsize = rbr->dsize; 176*831dba47SStephen Hemminger uint8_t *dst = dst0; 177*831dba47SStephen Hemminger 178*831dba47SStephen Hemminger if (cplen > br_dsize - rindex) { 179*831dba47SStephen Hemminger uint32_t fraglen = br_dsize - rindex; 180*831dba47SStephen Hemminger 181*831dba47SStephen Hemminger /* Wrap-around detected. */ 182*831dba47SStephen Hemminger memcpy(dst, br_data + rindex, fraglen); 183*831dba47SStephen Hemminger memcpy(dst + fraglen, br_data, cplen - fraglen); 184*831dba47SStephen Hemminger } else { 185*831dba47SStephen Hemminger memcpy(dst, br_data + rindex, cplen); 186*831dba47SStephen Hemminger } 187*831dba47SStephen Hemminger 188*831dba47SStephen Hemminger return vmbus_br_idxinc(rindex, cplen, br_dsize); 189*831dba47SStephen Hemminger } 190*831dba47SStephen Hemminger 191*831dba47SStephen Hemminger /* Copy data from receive ring but don't change index */ 192*831dba47SStephen Hemminger int 193*831dba47SStephen Hemminger vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen) 194*831dba47SStephen Hemminger { 195*831dba47SStephen Hemminger uint32_t avail; 196*831dba47SStephen Hemminger 197*831dba47SStephen Hemminger /* 198*831dba47SStephen Hemminger * The requested data and the 64bits channel packet 199*831dba47SStephen Hemminger * offset should be there at least. 200*831dba47SStephen Hemminger */ 201*831dba47SStephen Hemminger avail = vmbus_br_availread(rbr); 202*831dba47SStephen Hemminger if (avail < dlen + sizeof(uint64_t)) 203*831dba47SStephen Hemminger return -EAGAIN; 204*831dba47SStephen Hemminger 205*831dba47SStephen Hemminger vmbus_rxbr_copyfrom(rbr, rbr->vbr->rindex, data, dlen); 206*831dba47SStephen Hemminger return 0; 207*831dba47SStephen Hemminger } 208*831dba47SStephen Hemminger 209*831dba47SStephen Hemminger /* 210*831dba47SStephen Hemminger * Copy data from receive ring and change index 211*831dba47SStephen Hemminger * NOTE: 212*831dba47SStephen Hemminger * We assume (dlen + skip) == sizeof(channel packet). 213*831dba47SStephen Hemminger */ 214*831dba47SStephen Hemminger int 215*831dba47SStephen Hemminger vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t skip) 216*831dba47SStephen Hemminger { 217*831dba47SStephen Hemminger struct vmbus_bufring *vbr = rbr->vbr; 218*831dba47SStephen Hemminger uint32_t br_dsize = rbr->dsize; 219*831dba47SStephen Hemminger uint32_t rindex; 220*831dba47SStephen Hemminger 221*831dba47SStephen Hemminger if (vmbus_br_availread(rbr) < dlen + skip + sizeof(uint64_t)) 222*831dba47SStephen Hemminger return -EAGAIN; 223*831dba47SStephen Hemminger 224*831dba47SStephen Hemminger /* 225*831dba47SStephen Hemminger * Copy channel packet from RX bufring. 226*831dba47SStephen Hemminger */ 227*831dba47SStephen Hemminger rindex = vmbus_br_idxinc(rbr->vbr->rindex, skip, br_dsize); 228*831dba47SStephen Hemminger rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen); 229*831dba47SStephen Hemminger 230*831dba47SStephen Hemminger /* 231*831dba47SStephen Hemminger * Discard this channel packet's 64bits offset, which is useless to us. 232*831dba47SStephen Hemminger */ 233*831dba47SStephen Hemminger rindex = vmbus_br_idxinc(rindex, sizeof(uint64_t), br_dsize); 234*831dba47SStephen Hemminger 235*831dba47SStephen Hemminger /* Update the read index _after_ the channel packet is fetched. */ 236*831dba47SStephen Hemminger rte_compiler_barrier(); 237*831dba47SStephen Hemminger 238*831dba47SStephen Hemminger vbr->rindex = rindex; 239*831dba47SStephen Hemminger 240*831dba47SStephen Hemminger return 0; 241*831dba47SStephen Hemminger } 242