1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2023 Marvell International Ltd. 3 */ 4 5 #include <fcntl.h> 6 #include <poll.h> 7 #include <stdlib.h> 8 #include <sys/ioctl.h> 9 #include <sys/socket.h> 10 #include <unistd.h> 11 12 #include <rte_debug.h> 13 #include <rte_ethdev.h> 14 #include <rte_graph.h> 15 #include <rte_graph_worker.h> 16 #include <rte_ip.h> 17 #include <rte_malloc.h> 18 #include <rte_mbuf.h> 19 #include <rte_mempool.h> 20 #include <rte_net.h> 21 22 #include "ethdev_rx_priv.h" 23 #include "kernel_rx_priv.h" 24 #include "node_private.h" 25 26 static inline struct rte_mbuf * 27 alloc_rx_mbuf(kernel_rx_node_ctx_t *ctx) 28 { 29 kernel_rx_info_t *rx = ctx->recv_info; 30 31 if (rx->idx >= rx->cnt) { 32 uint16_t cnt; 33 34 rx->idx = 0; 35 rx->cnt = 0; 36 37 cnt = rte_pktmbuf_alloc_bulk(ctx->pktmbuf_pool, rx->rx_bufs, KERN_RX_CACHE_COUNT); 38 if (cnt <= 0) 39 return NULL; 40 41 rx->cnt = cnt; 42 } 43 44 return rx->rx_bufs[rx->idx++]; 45 } 46 47 static inline void 48 mbuf_update(struct rte_mbuf **mbufs, uint16_t nb_pkts) 49 { 50 struct rte_net_hdr_lens hdr_lens; 51 struct rte_mbuf *m; 52 int i; 53 54 for (i = 0; i < nb_pkts; i++) { 55 m = mbufs[i]; 56 57 m->packet_type = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK); 58 59 m->ol_flags = 0; 60 m->tx_offload = 0; 61 62 m->l2_len = hdr_lens.l2_len; 63 m->l3_len = hdr_lens.l3_len; 64 m->l4_len = hdr_lens.l4_len; 65 } 66 } 67 68 static uint16_t 69 recv_pkt_parse(void **objs, uint16_t nb_pkts) 70 { 71 uint16_t pkts_left = nb_pkts; 72 struct rte_mbuf **pkts; 73 int i; 74 75 pkts = (struct rte_mbuf **)objs; 76 77 if (pkts_left >= 4) { 78 for (i = 0; i < 4; i++) 79 rte_prefetch0(rte_pktmbuf_mtod(pkts[i], void *)); 80 } 81 82 while (pkts_left >= 12) { 83 /* Prefetch next-next mbufs */ 84 rte_prefetch0(pkts[8]); 85 rte_prefetch0(pkts[9]); 86 rte_prefetch0(pkts[10]); 87 rte_prefetch0(pkts[11]); 88 89 /* Prefetch next mbuf data */ 90 rte_prefetch0(rte_pktmbuf_mtod(pkts[4], void *)); 91 rte_prefetch0(rte_pktmbuf_mtod(pkts[5], void *)); 92 rte_prefetch0(rte_pktmbuf_mtod(pkts[6], void *)); 93 rte_prefetch0(rte_pktmbuf_mtod(pkts[7], void *)); 94 95 /* Extract ptype of mbufs */ 96 mbuf_update(pkts, 4); 97 98 pkts += 4; 99 pkts_left -= 4; 100 } 101 102 if (pkts_left > 0) 103 mbuf_update(pkts, pkts_left); 104 105 return nb_pkts; 106 } 107 108 static uint16_t 109 kernel_rx_node_do(struct rte_graph *graph, struct rte_node *node, kernel_rx_node_ctx_t *ctx) 110 { 111 kernel_rx_info_t *rx; 112 uint16_t next_index; 113 int fd; 114 115 rx = ctx->recv_info; 116 next_index = rx->node_next; 117 118 fd = rx->sock; 119 if (fd > 0) { 120 struct rte_mbuf **mbufs; 121 uint16_t len = 0, count = 0; 122 int nb_cnt, i; 123 124 nb_cnt = (node->size >= RTE_GRAPH_BURST_SIZE) ? RTE_GRAPH_BURST_SIZE : node->size; 125 126 mbufs = (struct rte_mbuf **)node->objs; 127 for (i = 0; i < nb_cnt; i++) { 128 struct rte_mbuf *m = alloc_rx_mbuf(ctx); 129 130 if (!m) 131 break; 132 133 len = read(fd, rte_pktmbuf_mtod(m, char *), rte_pktmbuf_tailroom(m)); 134 if (len == 0 || len == 0xFFFF) { 135 rte_pktmbuf_free(m); 136 if (rx->idx <= 0) 137 node_dbg("kernel_rx", "rx_mbuf array is empty"); 138 rx->idx--; 139 break; 140 } 141 *mbufs++ = m; 142 143 m->port = node->id; 144 rte_pktmbuf_data_len(m) = len; 145 146 count++; 147 } 148 149 if (count) { 150 recv_pkt_parse(node->objs, count); 151 node->idx = count; 152 153 /* Enqueue to next node */ 154 rte_node_next_stream_move(graph, node, next_index); 155 } 156 157 return count; 158 } 159 160 return 0; 161 } 162 163 static uint16_t 164 kernel_rx_node_process(struct rte_graph *graph, struct rte_node *node, void **objs, 165 uint16_t nb_objs) 166 { 167 kernel_rx_node_ctx_t *ctx = (kernel_rx_node_ctx_t *)node->ctx; 168 int fd; 169 170 RTE_SET_USED(objs); 171 RTE_SET_USED(nb_objs); 172 173 if (!ctx) 174 return 0; 175 176 fd = ctx->recv_info->sock; 177 if (fd > 0) { 178 struct pollfd fds = {.fd = fd, .events = POLLIN}; 179 180 if (poll(&fds, 1, 0) > 0) { 181 if (fds.revents & POLLIN) 182 return kernel_rx_node_do(graph, node, ctx); 183 } 184 } 185 186 return 0; 187 } 188 189 static int 190 kernel_rx_node_init(const struct rte_graph *graph, struct rte_node *node) 191 { 192 struct kernel_rx_node_main *rx_node_main = kernel_rx_node_data_get(); 193 kernel_rx_node_ctx_t *ctx = (kernel_rx_node_ctx_t *)node->ctx; 194 kernel_rx_node_elem_t *elem = rx_node_main->head; 195 kernel_rx_info_t *recv_info; 196 int sock; 197 198 while (elem) { 199 if (elem->nid == node->id) { 200 /* Update node specific context */ 201 memcpy(ctx, &elem->ctx, sizeof(kernel_rx_node_ctx_t)); 202 break; 203 } 204 elem = elem->next; 205 } 206 207 RTE_VERIFY(elem != NULL); 208 209 if (ctx->pktmbuf_pool == NULL) { 210 node_err("kernel_rx", "Invalid mbuf pool on graph %s", graph->name); 211 return -EINVAL; 212 } 213 214 recv_info = rte_zmalloc_socket("kernel_rx_info", sizeof(kernel_rx_info_t), 215 RTE_CACHE_LINE_SIZE, graph->socket); 216 if (!recv_info) { 217 node_err("kernel_rx", "Kernel recv_info is NULL"); 218 return -ENOMEM; 219 } 220 221 sock = socket(AF_INET, SOCK_RAW, IPPROTO_RAW); 222 if (sock < 0) { 223 node_err("kernel_rx", "Unable to open RAW socket"); 224 return sock; 225 } 226 227 recv_info->sock = sock; 228 ctx->recv_info = recv_info; 229 230 return 0; 231 } 232 233 static void 234 kernel_rx_node_fini(const struct rte_graph *graph __rte_unused, struct rte_node *node) 235 { 236 kernel_rx_node_ctx_t *ctx = (kernel_rx_node_ctx_t *)node->ctx; 237 238 if (ctx->recv_info) { 239 close(ctx->recv_info->sock); 240 ctx->recv_info->sock = -1; 241 rte_free(ctx->recv_info); 242 } 243 244 ctx->recv_info = NULL; 245 } 246 247 struct kernel_rx_node_main * 248 kernel_rx_node_data_get(void) 249 { 250 static struct kernel_rx_node_main kernel_rx_main; 251 252 return &kernel_rx_main; 253 } 254 255 static struct rte_node_register kernel_rx_node_base = { 256 .process = kernel_rx_node_process, 257 .flags = RTE_NODE_SOURCE_F, 258 .name = "kernel_rx", 259 260 .init = kernel_rx_node_init, 261 .fini = kernel_rx_node_fini, 262 263 .nb_edges = KERNEL_RX_NEXT_MAX, 264 .next_nodes = { 265 [KERNEL_RX_NEXT_PKT_CLS] = "pkt_cls", 266 [KERNEL_RX_NEXT_IP4_LOOKUP] = "ip4_lookup", 267 }, 268 }; 269 270 struct rte_node_register * 271 kernel_rx_node_get(void) 272 { 273 return &kernel_rx_node_base; 274 } 275 276 RTE_NODE_REGISTER(kernel_rx_node_base); 277