1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2023 Marvell International Ltd.
3 */
4
5 #include <fcntl.h>
6 #include <poll.h>
7 #include <stdlib.h>
8 #include <sys/ioctl.h>
9 #include <sys/socket.h>
10 #include <unistd.h>
11
12 #include <rte_debug.h>
13 #include <rte_ethdev.h>
14 #include <rte_graph.h>
15 #include <rte_graph_worker.h>
16 #include <rte_ip.h>
17 #include <rte_malloc.h>
18 #include <rte_mbuf.h>
19 #include <rte_mempool.h>
20 #include <rte_net.h>
21
22 #include "ethdev_rx_priv.h"
23 #include "kernel_rx_priv.h"
24 #include "node_private.h"
25
26 static inline struct rte_mbuf *
alloc_rx_mbuf(kernel_rx_node_ctx_t * ctx)27 alloc_rx_mbuf(kernel_rx_node_ctx_t *ctx)
28 {
29 kernel_rx_info_t *rx = ctx->recv_info;
30
31 if (rx->idx >= rx->cnt) {
32 uint16_t cnt;
33
34 rx->idx = 0;
35 rx->cnt = 0;
36
37 cnt = rte_pktmbuf_alloc_bulk(ctx->pktmbuf_pool, rx->rx_bufs, KERN_RX_CACHE_COUNT);
38 if (cnt <= 0)
39 return NULL;
40
41 rx->cnt = cnt;
42 }
43
44 return rx->rx_bufs[rx->idx++];
45 }
46
47 static inline void
mbuf_update(struct rte_mbuf ** mbufs,uint16_t nb_pkts)48 mbuf_update(struct rte_mbuf **mbufs, uint16_t nb_pkts)
49 {
50 struct rte_net_hdr_lens hdr_lens;
51 struct rte_mbuf *m;
52 int i;
53
54 for (i = 0; i < nb_pkts; i++) {
55 m = mbufs[i];
56
57 m->packet_type = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK);
58
59 m->ol_flags = 0;
60 m->tx_offload = 0;
61
62 m->l2_len = hdr_lens.l2_len;
63 m->l3_len = hdr_lens.l3_len;
64 m->l4_len = hdr_lens.l4_len;
65 }
66 }
67
68 static uint16_t
recv_pkt_parse(void ** objs,uint16_t nb_pkts)69 recv_pkt_parse(void **objs, uint16_t nb_pkts)
70 {
71 uint16_t pkts_left = nb_pkts;
72 struct rte_mbuf **pkts;
73 int i;
74
75 pkts = (struct rte_mbuf **)objs;
76
77 if (pkts_left >= 4) {
78 for (i = 0; i < 4; i++)
79 rte_prefetch0(rte_pktmbuf_mtod(pkts[i], void *));
80 }
81
82 while (pkts_left >= 12) {
83 /* Prefetch next-next mbufs */
84 rte_prefetch0(pkts[8]);
85 rte_prefetch0(pkts[9]);
86 rte_prefetch0(pkts[10]);
87 rte_prefetch0(pkts[11]);
88
89 /* Prefetch next mbuf data */
90 rte_prefetch0(rte_pktmbuf_mtod(pkts[4], void *));
91 rte_prefetch0(rte_pktmbuf_mtod(pkts[5], void *));
92 rte_prefetch0(rte_pktmbuf_mtod(pkts[6], void *));
93 rte_prefetch0(rte_pktmbuf_mtod(pkts[7], void *));
94
95 /* Extract ptype of mbufs */
96 mbuf_update(pkts, 4);
97
98 pkts += 4;
99 pkts_left -= 4;
100 }
101
102 if (pkts_left > 0)
103 mbuf_update(pkts, pkts_left);
104
105 return nb_pkts;
106 }
107
108 static uint16_t
kernel_rx_node_do(struct rte_graph * graph,struct rte_node * node,kernel_rx_node_ctx_t * ctx)109 kernel_rx_node_do(struct rte_graph *graph, struct rte_node *node, kernel_rx_node_ctx_t *ctx)
110 {
111 kernel_rx_info_t *rx;
112 uint16_t next_index;
113 int fd;
114
115 rx = ctx->recv_info;
116 next_index = rx->node_next;
117
118 fd = rx->sock;
119 if (fd > 0) {
120 struct rte_mbuf **mbufs;
121 uint16_t len = 0, count = 0;
122 int nb_cnt, i;
123
124 nb_cnt = (node->size >= RTE_GRAPH_BURST_SIZE) ? RTE_GRAPH_BURST_SIZE : node->size;
125
126 mbufs = (struct rte_mbuf **)node->objs;
127 for (i = 0; i < nb_cnt; i++) {
128 struct rte_mbuf *m = alloc_rx_mbuf(ctx);
129
130 if (!m)
131 break;
132
133 len = read(fd, rte_pktmbuf_mtod(m, char *), rte_pktmbuf_tailroom(m));
134 if (len == 0 || len == 0xFFFF) {
135 rte_pktmbuf_free(m);
136 if (rx->idx <= 0)
137 node_dbg("kernel_rx", "rx_mbuf array is empty");
138 rx->idx--;
139 break;
140 }
141 *mbufs++ = m;
142
143 m->port = node->id;
144 rte_pktmbuf_data_len(m) = len;
145
146 count++;
147 }
148
149 if (count) {
150 recv_pkt_parse(node->objs, count);
151 node->idx = count;
152
153 /* Enqueue to next node */
154 rte_node_next_stream_move(graph, node, next_index);
155 }
156
157 return count;
158 }
159
160 return 0;
161 }
162
163 static uint16_t
kernel_rx_node_process(struct rte_graph * graph,struct rte_node * node,void ** objs,uint16_t nb_objs)164 kernel_rx_node_process(struct rte_graph *graph, struct rte_node *node, void **objs,
165 uint16_t nb_objs)
166 {
167 kernel_rx_node_ctx_t *ctx = (kernel_rx_node_ctx_t *)node->ctx;
168 int fd;
169
170 RTE_SET_USED(objs);
171 RTE_SET_USED(nb_objs);
172
173 if (!ctx)
174 return 0;
175
176 fd = ctx->recv_info->sock;
177 if (fd > 0) {
178 struct pollfd fds = {.fd = fd, .events = POLLIN};
179
180 if (poll(&fds, 1, 0) > 0) {
181 if (fds.revents & POLLIN)
182 return kernel_rx_node_do(graph, node, ctx);
183 }
184 }
185
186 return 0;
187 }
188
189 static int
kernel_rx_node_init(const struct rte_graph * graph,struct rte_node * node)190 kernel_rx_node_init(const struct rte_graph *graph, struct rte_node *node)
191 {
192 struct kernel_rx_node_main *rx_node_main = kernel_rx_node_data_get();
193 kernel_rx_node_ctx_t *ctx = (kernel_rx_node_ctx_t *)node->ctx;
194 kernel_rx_node_elem_t *elem = rx_node_main->head;
195 kernel_rx_info_t *recv_info;
196 int sock;
197
198 while (elem) {
199 if (elem->nid == node->id) {
200 /* Update node specific context */
201 memcpy(ctx, &elem->ctx, sizeof(kernel_rx_node_ctx_t));
202 break;
203 }
204 elem = elem->next;
205 }
206
207 RTE_VERIFY(elem != NULL);
208
209 if (ctx->pktmbuf_pool == NULL) {
210 node_err("kernel_rx", "Invalid mbuf pool on graph %s", graph->name);
211 return -EINVAL;
212 }
213
214 recv_info = rte_zmalloc_socket("kernel_rx_info", sizeof(kernel_rx_info_t),
215 RTE_CACHE_LINE_SIZE, graph->socket);
216 if (!recv_info) {
217 node_err("kernel_rx", "Kernel recv_info is NULL");
218 return -ENOMEM;
219 }
220
221 sock = socket(AF_INET, SOCK_RAW, IPPROTO_RAW);
222 if (sock < 0) {
223 node_err("kernel_rx", "Unable to open RAW socket");
224 return sock;
225 }
226
227 recv_info->sock = sock;
228 ctx->recv_info = recv_info;
229
230 return 0;
231 }
232
233 static void
kernel_rx_node_fini(const struct rte_graph * graph __rte_unused,struct rte_node * node)234 kernel_rx_node_fini(const struct rte_graph *graph __rte_unused, struct rte_node *node)
235 {
236 kernel_rx_node_ctx_t *ctx = (kernel_rx_node_ctx_t *)node->ctx;
237
238 if (ctx->recv_info) {
239 close(ctx->recv_info->sock);
240 ctx->recv_info->sock = -1;
241 rte_free(ctx->recv_info);
242 }
243
244 ctx->recv_info = NULL;
245 }
246
247 struct kernel_rx_node_main *
kernel_rx_node_data_get(void)248 kernel_rx_node_data_get(void)
249 {
250 static struct kernel_rx_node_main kernel_rx_main;
251
252 return &kernel_rx_main;
253 }
254
255 static struct rte_node_register kernel_rx_node_base = {
256 .process = kernel_rx_node_process,
257 .flags = RTE_NODE_SOURCE_F,
258 .name = "kernel_rx",
259
260 .init = kernel_rx_node_init,
261 .fini = kernel_rx_node_fini,
262
263 .nb_edges = KERNEL_RX_NEXT_MAX,
264 .next_nodes = {
265 [KERNEL_RX_NEXT_PKT_CLS] = "pkt_cls",
266 [KERNEL_RX_NEXT_IP4_LOOKUP] = "ip4_lookup",
267 },
268 };
269
270 struct rte_node_register *
kernel_rx_node_get(void)271 kernel_rx_node_get(void)
272 {
273 return &kernel_rx_node_base;
274 }
275
276 RTE_NODE_REGISTER(kernel_rx_node_base);
277