1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2020 Marvell International Ltd. 3 */ 4 5 #include <arpa/inet.h> 6 #include <sys/socket.h> 7 8 #include <rte_ethdev.h> 9 #include <rte_ether.h> 10 #include <rte_graph.h> 11 #include <rte_graph_worker.h> 12 #include <rte_ip.h> 13 #include <rte_lpm.h> 14 15 #include "rte_node_ip4_api.h" 16 17 #include "node_private.h" 18 19 #define IPV4_L3FWD_LPM_MAX_RULES 1024 20 #define IPV4_L3FWD_LPM_NUMBER_TBL8S (1 << 8) 21 22 /* IP4 Lookup global data struct */ 23 struct ip4_lookup_node_main { 24 struct rte_lpm *lpm_tbl[RTE_MAX_NUMA_NODES]; 25 }; 26 27 struct ip4_lookup_node_ctx { 28 /* Socket's LPM table */ 29 struct rte_lpm *lpm; 30 /* Dynamic offset to mbuf priv1 */ 31 int mbuf_priv1_off; 32 }; 33 34 int node_mbuf_priv1_dynfield_offset = -1; 35 36 static struct ip4_lookup_node_main ip4_lookup_nm; 37 38 #define IP4_LOOKUP_NODE_LPM(ctx) \ 39 (((struct ip4_lookup_node_ctx *)ctx)->lpm) 40 41 #define IP4_LOOKUP_NODE_PRIV1_OFF(ctx) \ 42 (((struct ip4_lookup_node_ctx *)ctx)->mbuf_priv1_off) 43 44 #if defined(__ARM_NEON) 45 #include "ip4_lookup_neon.h" 46 #elif defined(RTE_ARCH_X86) 47 #include "ip4_lookup_sse.h" 48 #endif 49 50 static uint16_t 51 ip4_lookup_node_process_scalar(struct rte_graph *graph, struct rte_node *node, 52 void **objs, uint16_t nb_objs) 53 { 54 struct rte_lpm *lpm = IP4_LOOKUP_NODE_LPM(node->ctx); 55 const int dyn = IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx); 56 struct rte_ipv4_hdr *ipv4_hdr; 57 void **to_next, **from; 58 uint16_t last_spec = 0; 59 struct rte_mbuf *mbuf; 60 rte_edge_t next_index; 61 uint16_t held = 0; 62 uint32_t drop_nh; 63 int i, rc; 64 65 /* Speculative next */ 66 next_index = RTE_NODE_IP4_LOOKUP_NEXT_REWRITE; 67 /* Drop node */ 68 drop_nh = ((uint32_t)RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP) << 16; 69 from = objs; 70 71 /* Get stream for the speculated next node */ 72 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs); 73 for (i = 0; i < nb_objs; i++) { 74 uint32_t next_hop; 75 uint16_t next; 76 77 mbuf = (struct rte_mbuf *)objs[i]; 78 79 /* Extract DIP of mbuf0 */ 80 ipv4_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_ipv4_hdr *, 81 sizeof(struct rte_ether_hdr)); 82 /* Extract cksum, ttl as ipv4 hdr is in cache */ 83 node_mbuf_priv1(mbuf, dyn)->cksum = ipv4_hdr->hdr_checksum; 84 node_mbuf_priv1(mbuf, dyn)->ttl = ipv4_hdr->time_to_live; 85 86 rc = rte_lpm_lookup(lpm, rte_be_to_cpu_32(ipv4_hdr->dst_addr), 87 &next_hop); 88 next_hop = (rc == 0) ? next_hop : drop_nh; 89 NODE_INCREMENT_XSTAT_ID(node, 0, rc != 0, 1); 90 91 node_mbuf_priv1(mbuf, dyn)->nh = (uint16_t)next_hop; 92 next_hop = next_hop >> 16; 93 next = (uint16_t)next_hop; 94 95 if (unlikely(next_index != next)) { 96 /* Copy things successfully speculated till now */ 97 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 98 from += last_spec; 99 to_next += last_spec; 100 held += last_spec; 101 last_spec = 0; 102 103 rte_node_enqueue_x1(graph, node, next, from[0]); 104 from += 1; 105 } else { 106 last_spec += 1; 107 } 108 } 109 110 /* !!! Home run !!! */ 111 if (likely(last_spec == nb_objs)) { 112 rte_node_next_stream_move(graph, node, next_index); 113 return nb_objs; 114 } 115 held += last_spec; 116 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 117 rte_node_next_stream_put(graph, node, next_index, held); 118 119 return nb_objs; 120 } 121 122 int 123 rte_node_ip4_route_add(uint32_t ip, uint8_t depth, uint16_t next_hop, 124 enum rte_node_ip4_lookup_next next_node) 125 { 126 char abuf[INET6_ADDRSTRLEN]; 127 struct in_addr in; 128 uint8_t socket; 129 uint32_t val; 130 int ret; 131 132 in.s_addr = htonl(ip); 133 inet_ntop(AF_INET, &in, abuf, sizeof(abuf)); 134 /* Embedded next node id into 24 bit next hop */ 135 val = ((next_node << 16) | next_hop) & ((1ull << 24) - 1); 136 node_dbg("ip4_lookup", "LPM: Adding route %s / %d nh (0x%x)", abuf, 137 depth, val); 138 139 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) { 140 if (!ip4_lookup_nm.lpm_tbl[socket]) 141 continue; 142 143 ret = rte_lpm_add(ip4_lookup_nm.lpm_tbl[socket], 144 ip, depth, val); 145 if (ret < 0) { 146 node_err("ip4_lookup", 147 "Unable to add entry %s / %d nh (%x) to LPM table on sock %d, rc=%d", 148 abuf, depth, val, socket, ret); 149 return ret; 150 } 151 } 152 153 return 0; 154 } 155 156 static int 157 setup_lpm(struct ip4_lookup_node_main *nm, int socket) 158 { 159 struct rte_lpm_config config_ipv4; 160 char s[RTE_LPM_NAMESIZE]; 161 162 /* One LPM table per socket */ 163 if (nm->lpm_tbl[socket]) 164 return 0; 165 166 /* create the LPM table */ 167 config_ipv4.max_rules = IPV4_L3FWD_LPM_MAX_RULES; 168 config_ipv4.number_tbl8s = IPV4_L3FWD_LPM_NUMBER_TBL8S; 169 config_ipv4.flags = 0; 170 snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socket); 171 nm->lpm_tbl[socket] = rte_lpm_create(s, socket, &config_ipv4); 172 if (nm->lpm_tbl[socket] == NULL) 173 return -rte_errno; 174 175 return 0; 176 } 177 178 static int 179 ip4_lookup_node_init(const struct rte_graph *graph, struct rte_node *node) 180 { 181 uint16_t socket, lcore_id; 182 static uint8_t init_once; 183 int rc; 184 185 RTE_SET_USED(graph); 186 RTE_BUILD_BUG_ON(sizeof(struct ip4_lookup_node_ctx) > RTE_NODE_CTX_SZ); 187 188 if (!init_once) { 189 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register( 190 &node_mbuf_priv1_dynfield_desc); 191 if (node_mbuf_priv1_dynfield_offset < 0) 192 return -rte_errno; 193 194 /* Setup LPM tables for all sockets */ 195 RTE_LCORE_FOREACH(lcore_id) 196 { 197 socket = rte_lcore_to_socket_id(lcore_id); 198 rc = setup_lpm(&ip4_lookup_nm, socket); 199 if (rc) { 200 node_err("ip4_lookup", 201 "Failed to setup lpm tbl for sock %u, rc=%d", 202 socket, rc); 203 return rc; 204 } 205 } 206 init_once = 1; 207 } 208 209 /* Update socket's LPM and mbuf dyn priv1 offset in node ctx */ 210 IP4_LOOKUP_NODE_LPM(node->ctx) = ip4_lookup_nm.lpm_tbl[graph->socket]; 211 IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset; 212 213 #if defined(__ARM_NEON) || defined(RTE_ARCH_X86) 214 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) 215 node->process = ip4_lookup_node_process_vec; 216 #endif 217 218 node_dbg("ip4_lookup", "Initialized ip4_lookup node"); 219 220 return 0; 221 } 222 223 static struct rte_node_xstats ip4_lookup_xstats = { 224 .nb_xstats = 1, 225 .xstat_desc = { 226 [0] = "ip4_lookup_error", 227 }, 228 }; 229 230 static struct rte_node_register ip4_lookup_node = { 231 .process = ip4_lookup_node_process_scalar, 232 .name = "ip4_lookup", 233 234 .init = ip4_lookup_node_init, 235 .xstats = &ip4_lookup_xstats, 236 237 .nb_edges = RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP + 1, 238 .next_nodes = { 239 [RTE_NODE_IP4_LOOKUP_NEXT_IP4_LOCAL] = "ip4_local", 240 [RTE_NODE_IP4_LOOKUP_NEXT_REWRITE] = "ip4_rewrite", 241 [RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP] = "pkt_drop", 242 }, 243 }; 244 245 RTE_NODE_REGISTER(ip4_lookup_node); 246