1 /* 2 * BSD LICENSE 3 * 4 * Copyright (C) Cavium, Inc. 2017. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Cavium, Inc nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <stdint.h> 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <unistd.h> 37 38 #include <rte_atomic.h> 39 #include <rte_common.h> 40 #include <rte_ethdev.h> 41 #include <rte_ether.h> 42 #include <rte_log.h> 43 #include <rte_mbuf.h> 44 #include <rte_prefetch.h> 45 46 #include "octeontx_ethdev.h" 47 #include "octeontx_rxtx.h" 48 #include "octeontx_logs.h" 49 50 /* Packet type table */ 51 #define PTYPE_SIZE OCCTX_PKI_LTYPE_LAST 52 53 static const uint32_t __rte_cache_aligned 54 ptype_table[PTYPE_SIZE][PTYPE_SIZE][PTYPE_SIZE] = { 55 [LC_NONE][LE_NONE][LF_NONE] = RTE_PTYPE_UNKNOWN, 56 [LC_NONE][LE_NONE][LF_IPSEC_ESP] = RTE_PTYPE_UNKNOWN, 57 [LC_NONE][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L4_FRAG, 58 [LC_NONE][LE_NONE][LF_IPCOMP] = RTE_PTYPE_UNKNOWN, 59 [LC_NONE][LE_NONE][LF_TCP] = RTE_PTYPE_L4_TCP, 60 [LC_NONE][LE_NONE][LF_UDP] = RTE_PTYPE_L4_UDP, 61 [LC_NONE][LE_NONE][LF_GRE] = RTE_PTYPE_TUNNEL_GRE, 62 [LC_NONE][LE_NONE][LF_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE, 63 [LC_NONE][LE_NONE][LF_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN, 64 [LC_NONE][LE_NONE][LF_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE, 65 66 [LC_IPV4][LE_NONE][LF_NONE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, 67 [LC_IPV4][LE_NONE][LF_IPSEC_ESP] = 68 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV4, 69 [LC_IPV4][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG, 70 [LC_IPV4][LE_NONE][LF_IPCOMP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN, 71 [LC_IPV4][LE_NONE][LF_TCP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, 72 [LC_IPV4][LE_NONE][LF_UDP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, 73 [LC_IPV4][LE_NONE][LF_GRE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GRE, 74 [LC_IPV4][LE_NONE][LF_UDP_GENEVE] = 75 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GENEVE, 76 [LC_IPV4][LE_NONE][LF_UDP_VXLAN] = 77 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_VXLAN, 78 [LC_IPV4][LE_NONE][LF_NVGRE] = 79 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE, 80 81 [LC_IPV4_OPT][LE_NONE][LF_NONE] = 82 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, 83 [LC_IPV4_OPT][LE_NONE][LF_IPSEC_ESP] = 84 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L3_IPV4, 85 [LC_IPV4_OPT][LE_NONE][LF_IPFRAG] = 86 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_FRAG, 87 [LC_IPV4_OPT][LE_NONE][LF_IPCOMP] = 88 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN, 89 [LC_IPV4_OPT][LE_NONE][LF_TCP] = 90 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, 91 [LC_IPV4_OPT][LE_NONE][LF_UDP] = 92 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, 93 [LC_IPV4_OPT][LE_NONE][LF_GRE] = 94 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GRE, 95 [LC_IPV4_OPT][LE_NONE][LF_UDP_GENEVE] = 96 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GENEVE, 97 [LC_IPV4_OPT][LE_NONE][LF_UDP_VXLAN] = 98 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_VXLAN, 99 [LC_IPV4_OPT][LE_NONE][LF_NVGRE] = 100 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_NVGRE, 101 102 [LC_IPV6][LE_NONE][LF_NONE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, 103 [LC_IPV6][LE_NONE][LF_IPSEC_ESP] = 104 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L3_IPV4, 105 [LC_IPV6][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG, 106 [LC_IPV6][LE_NONE][LF_IPCOMP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN, 107 [LC_IPV6][LE_NONE][LF_TCP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, 108 [LC_IPV6][LE_NONE][LF_UDP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, 109 [LC_IPV6][LE_NONE][LF_GRE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GRE, 110 [LC_IPV6][LE_NONE][LF_UDP_GENEVE] = 111 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GENEVE, 112 [LC_IPV6][LE_NONE][LF_UDP_VXLAN] = 113 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_VXLAN, 114 [LC_IPV6][LE_NONE][LF_NVGRE] = 115 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE, 116 [LC_IPV6_OPT][LE_NONE][LF_NONE] = 117 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, 118 [LC_IPV6_OPT][LE_NONE][LF_IPSEC_ESP] = 119 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L3_IPV4, 120 [LC_IPV6_OPT][LE_NONE][LF_IPFRAG] = 121 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_FRAG, 122 [LC_IPV6_OPT][LE_NONE][LF_IPCOMP] = 123 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN, 124 [LC_IPV6_OPT][LE_NONE][LF_TCP] = 125 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, 126 [LC_IPV6_OPT][LE_NONE][LF_UDP] = 127 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, 128 [LC_IPV6_OPT][LE_NONE][LF_GRE] = 129 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GRE, 130 [LC_IPV6_OPT][LE_NONE][LF_UDP_GENEVE] = 131 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GENEVE, 132 [LC_IPV6_OPT][LE_NONE][LF_UDP_VXLAN] = 133 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_VXLAN, 134 [LC_IPV6_OPT][LE_NONE][LF_NVGRE] = 135 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_NVGRE, 136 137 }; 138 139 static __rte_always_inline uint16_t __hot 140 __octeontx_xmit_pkts(void *lmtline_va, void *ioreg_va, int64_t *fc_status_va, 141 struct rte_mbuf *tx_pkt) 142 { 143 uint64_t cmd_buf[4]; 144 uint16_t gaura_id; 145 146 if (unlikely(*((volatile int64_t *)fc_status_va) < 0)) 147 return -ENOSPC; 148 149 /* Get the gaura Id */ 150 gaura_id = octeontx_fpa_bufpool_gpool((uintptr_t)tx_pkt->pool->pool_id); 151 152 /* Setup PKO_SEND_HDR_S */ 153 cmd_buf[0] = tx_pkt->data_len & 0xffff; 154 cmd_buf[1] = 0x0; 155 156 /* Set don't free bit if reference count > 1 */ 157 if (rte_mbuf_refcnt_read(tx_pkt) > 1) 158 cmd_buf[0] |= (1ULL << 58); /* SET DF */ 159 160 /* Setup PKO_SEND_GATHER_S */ 161 cmd_buf[(1 << 1) | 1] = rte_mbuf_data_dma_addr(tx_pkt); 162 cmd_buf[(1 << 1) | 0] = PKO_SEND_GATHER_SUBDC | 163 PKO_SEND_GATHER_LDTYPE(0x1ull) | 164 PKO_SEND_GATHER_GAUAR((long)gaura_id) | 165 tx_pkt->data_len; 166 167 octeontx_reg_lmtst(lmtline_va, ioreg_va, cmd_buf, PKO_CMD_SZ); 168 169 return 0; 170 } 171 172 uint16_t __hot 173 octeontx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 174 { 175 int count; 176 struct octeontx_txq *txq = tx_queue; 177 octeontx_dq_t *dq = &txq->dq; 178 int res; 179 180 count = 0; 181 182 while (count < nb_pkts) { 183 res = __octeontx_xmit_pkts(dq->lmtline_va, dq->ioreg_va, 184 dq->fc_status_va, 185 tx_pkts[count]); 186 if (res < 0) 187 break; 188 189 count++; 190 } 191 192 return count; /* return number of pkts transmitted */ 193 } 194 195 uint16_t __hot 196 octeontx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 197 { 198 struct rte_mbuf *mbuf; 199 struct octeontx_rxq *rxq; 200 struct rte_event ev; 201 octtx_wqe_t *wqe; 202 size_t count; 203 uint16_t valid_event; 204 205 rxq = rx_queue; 206 count = 0; 207 while (count < nb_pkts) { 208 valid_event = rte_event_dequeue_burst(rxq->evdev, 209 rxq->ev_ports, &ev, 210 1, 0); 211 if (!valid_event) 212 break; 213 214 wqe = (octtx_wqe_t *)(uintptr_t)ev.u64; 215 rte_prefetch_non_temporal(wqe); 216 217 /* Get mbuf from wqe */ 218 mbuf = (struct rte_mbuf *)((uintptr_t)wqe - 219 OCTTX_PACKET_WQE_SKIP); 220 mbuf->packet_type = 221 ptype_table[wqe->s.w2.lcty][wqe->s.w2.lety][wqe->s.w2.lfty]; 222 mbuf->data_off = RTE_PTR_DIFF(wqe->s.w3.addr, mbuf->buf_addr); 223 mbuf->pkt_len = wqe->s.w1.len; 224 mbuf->data_len = mbuf->pkt_len; 225 mbuf->nb_segs = 1; 226 mbuf->ol_flags = 0; 227 mbuf->port = rxq->port_id; 228 rte_mbuf_refcnt_set(mbuf, 1); 229 rx_pkts[count++] = mbuf; 230 } 231 232 return count; /* return number of pkts received */ 233 } 234