1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2023 Marvell International Ltd. 3 */ 4 5 #include <arpa/inet.h> 6 #include <sys/socket.h> 7 8 #include <rte_ethdev.h> 9 #include <rte_ether.h> 10 #include <rte_graph.h> 11 #include <rte_graph_worker.h> 12 #include <rte_ip.h> 13 #include <rte_lpm.h> 14 #include <rte_hash.h> 15 #include <rte_fbk_hash.h> 16 #include <rte_jhash.h> 17 #include <rte_hash_crc.h> 18 19 #include "rte_node_udp4_input_api.h" 20 21 #include "node_private.h" 22 23 #define UDP4_INPUT_HASH_TBL_SIZE 1024 24 25 #define UDP4_INPUT_NODE_HASH(ctx) \ 26 (((struct udp4_input_node_ctx *)ctx)->hash) 27 28 #define UDP4_INPUT_NODE_NEXT_INDEX(ctx) \ 29 (((struct udp4_input_node_ctx *)ctx)->next_index) 30 31 32 /* UDP4 input global data struct */ 33 struct udp4_input_node_main { 34 struct rte_hash *hash_tbl[RTE_MAX_NUMA_NODES]; 35 }; 36 37 static struct udp4_input_node_main udp4_input_nm; 38 39 struct udp4_input_node_ctx { 40 /* Socket's Hash table */ 41 struct rte_hash *hash; 42 /* Cached next index */ 43 uint16_t next_index; 44 }; 45 46 struct flow_key { 47 uint32_t prt_dst; 48 }; 49 50 static struct rte_hash_parameters udp4_params = { 51 .entries = UDP4_INPUT_HASH_TBL_SIZE, 52 .key_len = sizeof(uint32_t), 53 .hash_func = rte_jhash, 54 .hash_func_init_val = 0, 55 .socket_id = 0, 56 }; 57 58 int 59 rte_node_udp4_dst_port_add(uint32_t dst_port, rte_edge_t next_node) 60 { 61 uint8_t socket; 62 int rc; 63 64 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) { 65 if (!udp4_input_nm.hash_tbl[socket]) 66 continue; 67 68 rc = rte_hash_add_key_data(udp4_input_nm.hash_tbl[socket], 69 &dst_port, (void *)(uintptr_t)next_node); 70 if (rc < 0) { 71 node_err("udp4_lookup", "Failed to add key for sock %u, rc=%d", 72 socket, rc); 73 return rc; 74 } 75 } 76 return 0; 77 } 78 79 int 80 rte_node_udp4_usr_node_add(const char *usr_node) 81 { 82 const char *next_nodes = usr_node; 83 rte_node_t udp4_input_node_id, count; 84 85 udp4_input_node_id = rte_node_from_name("udp4_input"); 86 count = rte_node_edge_update(udp4_input_node_id, RTE_EDGE_ID_INVALID, 87 &next_nodes, 1); 88 if (count == 0) { 89 node_dbg("udp4_input", "Adding usr node as edge to udp4_input failed"); 90 return count; 91 } 92 count = rte_node_edge_count(udp4_input_node_id) - 1; 93 return count; 94 } 95 96 static int 97 setup_udp4_dstprt_hash(struct udp4_input_node_main *nm, int socket) 98 { 99 struct rte_hash_parameters *hash_udp4 = &udp4_params; 100 char s[RTE_HASH_NAMESIZE]; 101 102 /* One Hash table per socket */ 103 if (nm->hash_tbl[socket]) 104 return 0; 105 106 /* create Hash table */ 107 snprintf(s, sizeof(s), "UDP4_INPUT_HASH_%d", socket); 108 hash_udp4->name = s; 109 hash_udp4->socket_id = socket; 110 nm->hash_tbl[socket] = rte_hash_create(hash_udp4); 111 if (nm->hash_tbl[socket] == NULL) 112 return -rte_errno; 113 114 return 0; 115 } 116 117 static int 118 udp4_input_node_init(const struct rte_graph *graph, struct rte_node *node) 119 { 120 uint16_t socket, lcore_id; 121 static uint8_t init_once; 122 int rc; 123 124 RTE_SET_USED(graph); 125 RTE_BUILD_BUG_ON(sizeof(struct udp4_input_node_ctx) > RTE_NODE_CTX_SZ); 126 127 if (!init_once) { 128 129 /* Setup HASH tables for all sockets */ 130 RTE_LCORE_FOREACH(lcore_id) 131 { 132 socket = rte_lcore_to_socket_id(lcore_id); 133 rc = setup_udp4_dstprt_hash(&udp4_input_nm, socket); 134 if (rc) { 135 node_err("udp4_lookup", 136 "Failed to setup hash tbl for sock %u, rc=%d", 137 socket, rc); 138 return rc; 139 } 140 } 141 init_once = 1; 142 } 143 144 UDP4_INPUT_NODE_HASH(node->ctx) = udp4_input_nm.hash_tbl[graph->socket]; 145 146 node_dbg("udp4_input", "Initialized udp4_input node"); 147 return 0; 148 } 149 150 static uint16_t 151 udp4_input_node_process_scalar(struct rte_graph *graph, struct rte_node *node, 152 void **objs, uint16_t nb_objs) 153 { 154 struct rte_hash *hash_tbl_handle = UDP4_INPUT_NODE_HASH(node->ctx); 155 rte_edge_t next_index, udplookup_node; 156 struct rte_udp_hdr *pkt_udp_hdr; 157 uint16_t last_spec = 0; 158 void **to_next, **from; 159 struct rte_mbuf *mbuf; 160 uint16_t held = 0; 161 uint16_t next = 0; 162 int i, rc; 163 164 /* Speculative next */ 165 next_index = UDP4_INPUT_NODE_NEXT_INDEX(node->ctx); 166 167 from = objs; 168 169 to_next = rte_node_next_stream_get(graph, node, next_index, nb_objs); 170 for (i = 0; i < nb_objs; i++) { 171 struct flow_key key_port; 172 173 mbuf = (struct rte_mbuf *)objs[i]; 174 pkt_udp_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_udp_hdr *, 175 sizeof(struct rte_ether_hdr) + 176 sizeof(struct rte_ipv4_hdr)); 177 178 key_port.prt_dst = rte_cpu_to_be_16(pkt_udp_hdr->dst_port); 179 rc = rte_hash_lookup_data(hash_tbl_handle, 180 &key_port.prt_dst, 181 (void **)&udplookup_node); 182 next = (rc < 0) ? RTE_NODE_UDP4_INPUT_NEXT_PKT_DROP 183 : udplookup_node; 184 185 if (unlikely(next_index != next)) { 186 /* Copy things successfully speculated till now */ 187 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 188 from += last_spec; 189 to_next += last_spec; 190 held += last_spec; 191 last_spec = 0; 192 193 rte_node_enqueue_x1(graph, node, next, from[0]); 194 from += 1; 195 } else { 196 last_spec += 1; 197 } 198 } 199 /* !!! Home run !!! */ 200 if (likely(last_spec == nb_objs)) { 201 rte_node_next_stream_move(graph, node, next_index); 202 return nb_objs; 203 } 204 held += last_spec; 205 rte_memcpy(to_next, from, last_spec * sizeof(from[0])); 206 rte_node_next_stream_put(graph, node, next_index, held); 207 /* Save the last next used */ 208 UDP4_INPUT_NODE_NEXT_INDEX(node->ctx) = next; 209 210 return nb_objs; 211 } 212 213 static struct rte_node_register udp4_input_node = { 214 .process = udp4_input_node_process_scalar, 215 .name = "udp4_input", 216 217 .init = udp4_input_node_init, 218 219 .nb_edges = RTE_NODE_UDP4_INPUT_NEXT_PKT_DROP + 1, 220 .next_nodes = { 221 [RTE_NODE_UDP4_INPUT_NEXT_PKT_DROP] = "pkt_drop", 222 }, 223 }; 224 225 RTE_NODE_REGISTER(udp4_input_node); 226