1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2020 Marvell International Ltd. 3 */ 4 5 #include <arpa/inet.h> 6 #include <sys/socket.h> 7 8 #include <rte_ethdev.h> 9 #include <rte_ether.h> 10 #include <rte_graph.h> 11 #include <rte_graph_worker.h> 12 #include <rte_ip.h> 13 #include <rte_lpm.h> 14 15 #include "rte_node_ip4_api.h" 16 17 #include "node_private.h" 18 19 #define IPV4_L3FWD_LPM_MAX_RULES 1024 20 #define IPV4_L3FWD_LPM_NUMBER_TBL8S (1 << 8) 21 22 /* IP4 Lookup global data struct */ 23 struct ip4_lookup_node_main { 24 struct rte_lpm *lpm_tbl[RTE_MAX_NUMA_NODES]; 25 }; 26 27 struct ip4_lookup_node_ctx { 28 /* Socket's LPM table */ 29 struct rte_lpm *lpm; 30 /* Dynamic offset to mbuf priv1 */ 31 int mbuf_priv1_off; 32 }; 33 34 int node_mbuf_priv1_dynfield_offset = -1; 35 36 static struct ip4_lookup_node_main ip4_lookup_nm; 37 38 #define IP4_LOOKUP_NODE_LPM(ctx) \ 39 (((struct ip4_lookup_node_ctx *)ctx)->lpm) 40 41 #define IP4_LOOKUP_NODE_PRIV1_OFF(ctx) \ 42 (((struct ip4_lookup_node_ctx *)ctx)->mbuf_priv1_off) 43 44 #if defined(__ARM_NEON) 45 #include "ip4_lookup_neon.h" 46 #elif defined(RTE_ARCH_X86) 47 #include "ip4_lookup_sse.h" 48 #endif 49 50 static uint16_t 51 ip4_lookup_node_process_scalar(struct rte_graph *graph, struct rte_node *node, 52 void **objs, uint16_t nb_objs) 53 { 54 struct rte_lpm *lpm = IP4_LOOKUP_NODE_LPM(node->ctx); 55 const int dyn = IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx); 56 struct rte_ipv4_hdr *ipv4_hdr; 57 void **to_next, **from; 58 uint16_t last_spec = 0; 59 struct rte_mbuf *mbuf; 60 rte_edge_t next_index; 61 uint16_t held = 0; 62 uint32_t drop_nh; 63 int i, rc; 64 65 /* Speculative next */ 66 next_index = RTE_NODE_IP4_LOOKUP_NEXT_REWRITE; 67 /* Drop node */ 68 drop_nh = ((uint32_t)RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP) << 16; 69 from = objs; 70 71 /* Get stream for the speculated next node */ 72 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs); 73 for (i = 0; i < nb_objs; i++) { 74 uint32_t next_hop; 75 uint16_t next; 76 77 mbuf = (struct rte_mbuf *)objs[i]; 78 79 /* Extract DIP of mbuf0 */ 80 ipv4_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_ipv4_hdr *, 81 sizeof(struct rte_ether_hdr)); 82 /* Extract cksum, ttl as ipv4 hdr is in cache */ 83 node_mbuf_priv1(mbuf, dyn)->cksum = ipv4_hdr->hdr_checksum; 84 node_mbuf_priv1(mbuf, dyn)->ttl = ipv4_hdr->time_to_live; 85 86 rc = rte_lpm_lookup(lpm, rte_be_to_cpu_32(ipv4_hdr->dst_addr), 87 &next_hop); 88 next_hop = (rc == 0) ? next_hop : drop_nh; 89 90 node_mbuf_priv1(mbuf, dyn)->nh = (uint16_t)next_hop; 91 next_hop = next_hop >> 16; 92 next = (uint16_t)next_hop; 93 94 if (unlikely(next_index != next)) { 95 /* Copy things successfully speculated till now */ 96 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 97 from += last_spec; 98 to_next += last_spec; 99 held += last_spec; 100 last_spec = 0; 101 102 rte_node_enqueue_x1(graph, node, next, from[0]); 103 from += 1; 104 } else { 105 last_spec += 1; 106 } 107 } 108 109 /* !!! Home run !!! */ 110 if (likely(last_spec == nb_objs)) { 111 rte_node_next_stream_move(graph, node, next_index); 112 return nb_objs; 113 } 114 held += last_spec; 115 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 116 rte_node_next_stream_put(graph, node, next_index, held); 117 118 return nb_objs; 119 } 120 121 int 122 rte_node_ip4_route_add(uint32_t ip, uint8_t depth, uint16_t next_hop, 123 enum rte_node_ip4_lookup_next next_node) 124 { 125 char abuf[INET6_ADDRSTRLEN]; 126 struct in_addr in; 127 uint8_t socket; 128 uint32_t val; 129 int ret; 130 131 in.s_addr = htonl(ip); 132 inet_ntop(AF_INET, &in, abuf, sizeof(abuf)); 133 /* Embedded next node id into 24 bit next hop */ 134 val = ((next_node << 16) | next_hop) & ((1ull << 24) - 1); 135 node_dbg("ip4_lookup", "LPM: Adding route %s / %d nh (0x%x)", abuf, 136 depth, val); 137 138 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) { 139 if (!ip4_lookup_nm.lpm_tbl[socket]) 140 continue; 141 142 ret = rte_lpm_add(ip4_lookup_nm.lpm_tbl[socket], 143 ip, depth, val); 144 if (ret < 0) { 145 node_err("ip4_lookup", 146 "Unable to add entry %s / %d nh (%x) to LPM table on sock %d, rc=%d", 147 abuf, depth, val, socket, ret); 148 return ret; 149 } 150 } 151 152 return 0; 153 } 154 155 static int 156 setup_lpm(struct ip4_lookup_node_main *nm, int socket) 157 { 158 struct rte_lpm_config config_ipv4; 159 char s[RTE_LPM_NAMESIZE]; 160 161 /* One LPM table per socket */ 162 if (nm->lpm_tbl[socket]) 163 return 0; 164 165 /* create the LPM table */ 166 config_ipv4.max_rules = IPV4_L3FWD_LPM_MAX_RULES; 167 config_ipv4.number_tbl8s = IPV4_L3FWD_LPM_NUMBER_TBL8S; 168 config_ipv4.flags = 0; 169 snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socket); 170 nm->lpm_tbl[socket] = rte_lpm_create(s, socket, &config_ipv4); 171 if (nm->lpm_tbl[socket] == NULL) 172 return -rte_errno; 173 174 return 0; 175 } 176 177 static int 178 ip4_lookup_node_init(const struct rte_graph *graph, struct rte_node *node) 179 { 180 uint16_t socket, lcore_id; 181 static uint8_t init_once; 182 int rc; 183 184 RTE_SET_USED(graph); 185 RTE_BUILD_BUG_ON(sizeof(struct ip4_lookup_node_ctx) > RTE_NODE_CTX_SZ); 186 187 if (!init_once) { 188 node_mbuf_priv1_dynfield_offset = rte_mbuf_dynfield_register( 189 &node_mbuf_priv1_dynfield_desc); 190 if (node_mbuf_priv1_dynfield_offset < 0) 191 return -rte_errno; 192 193 /* Setup LPM tables for all sockets */ 194 RTE_LCORE_FOREACH(lcore_id) 195 { 196 socket = rte_lcore_to_socket_id(lcore_id); 197 rc = setup_lpm(&ip4_lookup_nm, socket); 198 if (rc) { 199 node_err("ip4_lookup", 200 "Failed to setup lpm tbl for sock %u, rc=%d", 201 socket, rc); 202 return rc; 203 } 204 } 205 init_once = 1; 206 } 207 208 /* Update socket's LPM and mbuf dyn priv1 offset in node ctx */ 209 IP4_LOOKUP_NODE_LPM(node->ctx) = ip4_lookup_nm.lpm_tbl[graph->socket]; 210 IP4_LOOKUP_NODE_PRIV1_OFF(node->ctx) = node_mbuf_priv1_dynfield_offset; 211 212 #if defined(__ARM_NEON) || defined(RTE_ARCH_X86) 213 if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) 214 node->process = ip4_lookup_node_process_vec; 215 #endif 216 217 node_dbg("ip4_lookup", "Initialized ip4_lookup node"); 218 219 return 0; 220 } 221 222 static struct rte_node_register ip4_lookup_node = { 223 .process = ip4_lookup_node_process_scalar, 224 .name = "ip4_lookup", 225 226 .init = ip4_lookup_node_init, 227 228 .nb_edges = RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP + 1, 229 .next_nodes = { 230 [RTE_NODE_IP4_LOOKUP_NEXT_IP4_LOCAL] = "ip4_local", 231 [RTE_NODE_IP4_LOOKUP_NEXT_REWRITE] = "ip4_rewrite", 232 [RTE_NODE_IP4_LOOKUP_NEXT_PKT_DROP] = "pkt_drop", 233 }, 234 }; 235 236 RTE_NODE_REGISTER(ip4_lookup_node); 237