1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4
5 #include <stdio.h>
6 #include <stdint.h>
7
8 #include <rte_common.h>
9 #include <rte_log.h>
10 #include <rte_debug.h>
11 #include <rte_byteorder.h>
12
13 #include "bpf_impl.h"
14
15 #define BPF_JMP_UNC(ins) ((ins) += (ins)->off)
16
17 #define BPF_JMP_CND_REG(reg, ins, op, type) \
18 ((ins) += \
19 ((type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg]) ? \
20 (ins)->off : 0)
21
22 #define BPF_JMP_CND_IMM(reg, ins, op, type) \
23 ((ins) += \
24 ((type)(reg)[(ins)->dst_reg] op (type)(ins)->imm) ? \
25 (ins)->off : 0)
26
27 #define BPF_NEG_ALU(reg, ins, type) \
28 ((reg)[(ins)->dst_reg] = (type)(-(reg)[(ins)->dst_reg]))
29
30 #define EBPF_MOV_ALU_REG(reg, ins, type) \
31 ((reg)[(ins)->dst_reg] = (type)(reg)[(ins)->src_reg])
32
33 #define BPF_OP_ALU_REG(reg, ins, op, type) \
34 ((reg)[(ins)->dst_reg] = \
35 (type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg])
36
37 #define EBPF_MOV_ALU_IMM(reg, ins, type) \
38 ((reg)[(ins)->dst_reg] = (type)(ins)->imm)
39
40 #define BPF_OP_ALU_IMM(reg, ins, op, type) \
41 ((reg)[(ins)->dst_reg] = \
42 (type)(reg)[(ins)->dst_reg] op (type)(ins)->imm)
43
44 #define BPF_DIV_ZERO_CHECK(bpf, reg, ins, type) do { \
45 if ((type)(reg)[(ins)->src_reg] == 0) { \
46 RTE_BPF_LOG_LINE(ERR, \
47 "%s(%p): division by 0 at pc: %#zx;", \
48 __func__, bpf, \
49 (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \
50 return 0; \
51 } \
52 } while (0)
53
54 #define BPF_LD_REG(reg, ins, type) \
55 ((reg)[(ins)->dst_reg] = \
56 *(type *)(uintptr_t)((reg)[(ins)->src_reg] + (ins)->off))
57
58 #define BPF_ST_IMM(reg, ins, type) \
59 (*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
60 (type)(ins)->imm)
61
62 #define BPF_ST_REG(reg, ins, type) \
63 (*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
64 (type)(reg)[(ins)->src_reg])
65
66 #define BPF_ST_XADD_REG(reg, ins, tp) \
67 (rte_atomic##tp##_add((rte_atomic##tp##_t *) \
68 (uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \
69 reg[ins->src_reg]))
70
71 /* BPF_LD | BPF_ABS/BPF_IND */
72
73 #define NOP(x) (x)
74
75 #define BPF_LD_ABS(bpf, reg, ins, type, op) do { \
76 const type *p = bpf_ld_mbuf(bpf, reg, ins, (ins)->imm, sizeof(type)); \
77 if (p == NULL) \
78 return 0; \
79 reg[EBPF_REG_0] = op(p[0]); \
80 } while (0)
81
82 #define BPF_LD_IND(bpf, reg, ins, type, op) do { \
83 uint32_t ofs = reg[ins->src_reg] + (ins)->imm; \
84 const type *p = bpf_ld_mbuf(bpf, reg, ins, ofs, sizeof(type)); \
85 if (p == NULL) \
86 return 0; \
87 reg[EBPF_REG_0] = op(p[0]); \
88 } while (0)
89
90
91 static inline void
bpf_alu_be(uint64_t reg[EBPF_REG_NUM],const struct ebpf_insn * ins)92 bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
93 {
94 uint64_t *v;
95
96 v = reg + ins->dst_reg;
97 switch (ins->imm) {
98 case 16:
99 *v = rte_cpu_to_be_16(*v);
100 break;
101 case 32:
102 *v = rte_cpu_to_be_32(*v);
103 break;
104 case 64:
105 *v = rte_cpu_to_be_64(*v);
106 break;
107 }
108 }
109
110 static inline void
bpf_alu_le(uint64_t reg[EBPF_REG_NUM],const struct ebpf_insn * ins)111 bpf_alu_le(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
112 {
113 uint64_t *v;
114
115 v = reg + ins->dst_reg;
116 switch (ins->imm) {
117 case 16:
118 *v = rte_cpu_to_le_16(*v);
119 break;
120 case 32:
121 *v = rte_cpu_to_le_32(*v);
122 break;
123 case 64:
124 *v = rte_cpu_to_le_64(*v);
125 break;
126 }
127 }
128
129 static inline const void *
bpf_ld_mbuf(const struct rte_bpf * bpf,uint64_t reg[EBPF_REG_NUM],const struct ebpf_insn * ins,uint32_t off,uint32_t len)130 bpf_ld_mbuf(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM],
131 const struct ebpf_insn *ins, uint32_t off, uint32_t len)
132 {
133 const struct rte_mbuf *mb;
134 const void *p;
135
136 mb = (const struct rte_mbuf *)(uintptr_t)reg[EBPF_REG_6];
137 p = rte_pktmbuf_read(mb, off, len, reg + EBPF_REG_0);
138 if (p == NULL)
139 RTE_BPF_LOG_LINE(DEBUG, "%s(bpf=%p, mbuf=%p, ofs=%u, len=%u): "
140 "load beyond packet boundary at pc: %#zx;",
141 __func__, bpf, mb, off, len,
142 (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins);
143 return p;
144 }
145
146 static inline uint64_t
bpf_exec(const struct rte_bpf * bpf,uint64_t reg[EBPF_REG_NUM])147 bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
148 {
149 const struct ebpf_insn *ins;
150
151 for (ins = bpf->prm.ins; ; ins++) {
152 switch (ins->code) {
153 /* 32 bit ALU IMM operations */
154 case (BPF_ALU | BPF_ADD | BPF_K):
155 BPF_OP_ALU_IMM(reg, ins, +, uint32_t);
156 break;
157 case (BPF_ALU | BPF_SUB | BPF_K):
158 BPF_OP_ALU_IMM(reg, ins, -, uint32_t);
159 break;
160 case (BPF_ALU | BPF_AND | BPF_K):
161 BPF_OP_ALU_IMM(reg, ins, &, uint32_t);
162 break;
163 case (BPF_ALU | BPF_OR | BPF_K):
164 BPF_OP_ALU_IMM(reg, ins, |, uint32_t);
165 break;
166 case (BPF_ALU | BPF_LSH | BPF_K):
167 BPF_OP_ALU_IMM(reg, ins, <<, uint32_t);
168 break;
169 case (BPF_ALU | BPF_RSH | BPF_K):
170 BPF_OP_ALU_IMM(reg, ins, >>, uint32_t);
171 break;
172 case (BPF_ALU | BPF_XOR | BPF_K):
173 BPF_OP_ALU_IMM(reg, ins, ^, uint32_t);
174 break;
175 case (BPF_ALU | BPF_MUL | BPF_K):
176 BPF_OP_ALU_IMM(reg, ins, *, uint32_t);
177 break;
178 case (BPF_ALU | BPF_DIV | BPF_K):
179 BPF_OP_ALU_IMM(reg, ins, /, uint32_t);
180 break;
181 case (BPF_ALU | BPF_MOD | BPF_K):
182 BPF_OP_ALU_IMM(reg, ins, %, uint32_t);
183 break;
184 case (BPF_ALU | EBPF_MOV | BPF_K):
185 EBPF_MOV_ALU_IMM(reg, ins, uint32_t);
186 break;
187 /* 32 bit ALU REG operations */
188 case (BPF_ALU | BPF_ADD | BPF_X):
189 BPF_OP_ALU_REG(reg, ins, +, uint32_t);
190 break;
191 case (BPF_ALU | BPF_SUB | BPF_X):
192 BPF_OP_ALU_REG(reg, ins, -, uint32_t);
193 break;
194 case (BPF_ALU | BPF_AND | BPF_X):
195 BPF_OP_ALU_REG(reg, ins, &, uint32_t);
196 break;
197 case (BPF_ALU | BPF_OR | BPF_X):
198 BPF_OP_ALU_REG(reg, ins, |, uint32_t);
199 break;
200 case (BPF_ALU | BPF_LSH | BPF_X):
201 BPF_OP_ALU_REG(reg, ins, <<, uint32_t);
202 break;
203 case (BPF_ALU | BPF_RSH | BPF_X):
204 BPF_OP_ALU_REG(reg, ins, >>, uint32_t);
205 break;
206 case (BPF_ALU | BPF_XOR | BPF_X):
207 BPF_OP_ALU_REG(reg, ins, ^, uint32_t);
208 break;
209 case (BPF_ALU | BPF_MUL | BPF_X):
210 BPF_OP_ALU_REG(reg, ins, *, uint32_t);
211 break;
212 case (BPF_ALU | BPF_DIV | BPF_X):
213 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);
214 BPF_OP_ALU_REG(reg, ins, /, uint32_t);
215 break;
216 case (BPF_ALU | BPF_MOD | BPF_X):
217 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);
218 BPF_OP_ALU_REG(reg, ins, %, uint32_t);
219 break;
220 case (BPF_ALU | EBPF_MOV | BPF_X):
221 EBPF_MOV_ALU_REG(reg, ins, uint32_t);
222 break;
223 case (BPF_ALU | BPF_NEG):
224 BPF_NEG_ALU(reg, ins, uint32_t);
225 break;
226 case (BPF_ALU | EBPF_END | EBPF_TO_BE):
227 bpf_alu_be(reg, ins);
228 break;
229 case (BPF_ALU | EBPF_END | EBPF_TO_LE):
230 bpf_alu_le(reg, ins);
231 break;
232 /* 64 bit ALU IMM operations */
233 case (EBPF_ALU64 | BPF_ADD | BPF_K):
234 BPF_OP_ALU_IMM(reg, ins, +, uint64_t);
235 break;
236 case (EBPF_ALU64 | BPF_SUB | BPF_K):
237 BPF_OP_ALU_IMM(reg, ins, -, uint64_t);
238 break;
239 case (EBPF_ALU64 | BPF_AND | BPF_K):
240 BPF_OP_ALU_IMM(reg, ins, &, uint64_t);
241 break;
242 case (EBPF_ALU64 | BPF_OR | BPF_K):
243 BPF_OP_ALU_IMM(reg, ins, |, uint64_t);
244 break;
245 case (EBPF_ALU64 | BPF_LSH | BPF_K):
246 BPF_OP_ALU_IMM(reg, ins, <<, uint64_t);
247 break;
248 case (EBPF_ALU64 | BPF_RSH | BPF_K):
249 BPF_OP_ALU_IMM(reg, ins, >>, uint64_t);
250 break;
251 case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
252 BPF_OP_ALU_IMM(reg, ins, >>, int64_t);
253 break;
254 case (EBPF_ALU64 | BPF_XOR | BPF_K):
255 BPF_OP_ALU_IMM(reg, ins, ^, uint64_t);
256 break;
257 case (EBPF_ALU64 | BPF_MUL | BPF_K):
258 BPF_OP_ALU_IMM(reg, ins, *, uint64_t);
259 break;
260 case (EBPF_ALU64 | BPF_DIV | BPF_K):
261 BPF_OP_ALU_IMM(reg, ins, /, uint64_t);
262 break;
263 case (EBPF_ALU64 | BPF_MOD | BPF_K):
264 BPF_OP_ALU_IMM(reg, ins, %, uint64_t);
265 break;
266 case (EBPF_ALU64 | EBPF_MOV | BPF_K):
267 EBPF_MOV_ALU_IMM(reg, ins, uint64_t);
268 break;
269 /* 64 bit ALU REG operations */
270 case (EBPF_ALU64 | BPF_ADD | BPF_X):
271 BPF_OP_ALU_REG(reg, ins, +, uint64_t);
272 break;
273 case (EBPF_ALU64 | BPF_SUB | BPF_X):
274 BPF_OP_ALU_REG(reg, ins, -, uint64_t);
275 break;
276 case (EBPF_ALU64 | BPF_AND | BPF_X):
277 BPF_OP_ALU_REG(reg, ins, &, uint64_t);
278 break;
279 case (EBPF_ALU64 | BPF_OR | BPF_X):
280 BPF_OP_ALU_REG(reg, ins, |, uint64_t);
281 break;
282 case (EBPF_ALU64 | BPF_LSH | BPF_X):
283 BPF_OP_ALU_REG(reg, ins, <<, uint64_t);
284 break;
285 case (EBPF_ALU64 | BPF_RSH | BPF_X):
286 BPF_OP_ALU_REG(reg, ins, >>, uint64_t);
287 break;
288 case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
289 BPF_OP_ALU_REG(reg, ins, >>, int64_t);
290 break;
291 case (EBPF_ALU64 | BPF_XOR | BPF_X):
292 BPF_OP_ALU_REG(reg, ins, ^, uint64_t);
293 break;
294 case (EBPF_ALU64 | BPF_MUL | BPF_X):
295 BPF_OP_ALU_REG(reg, ins, *, uint64_t);
296 break;
297 case (EBPF_ALU64 | BPF_DIV | BPF_X):
298 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);
299 BPF_OP_ALU_REG(reg, ins, /, uint64_t);
300 break;
301 case (EBPF_ALU64 | BPF_MOD | BPF_X):
302 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);
303 BPF_OP_ALU_REG(reg, ins, %, uint64_t);
304 break;
305 case (EBPF_ALU64 | EBPF_MOV | BPF_X):
306 EBPF_MOV_ALU_REG(reg, ins, uint64_t);
307 break;
308 case (EBPF_ALU64 | BPF_NEG):
309 BPF_NEG_ALU(reg, ins, uint64_t);
310 break;
311 /* load instructions */
312 case (BPF_LDX | BPF_MEM | BPF_B):
313 BPF_LD_REG(reg, ins, uint8_t);
314 break;
315 case (BPF_LDX | BPF_MEM | BPF_H):
316 BPF_LD_REG(reg, ins, uint16_t);
317 break;
318 case (BPF_LDX | BPF_MEM | BPF_W):
319 BPF_LD_REG(reg, ins, uint32_t);
320 break;
321 case (BPF_LDX | BPF_MEM | EBPF_DW):
322 BPF_LD_REG(reg, ins, uint64_t);
323 break;
324 /* load 64 bit immediate value */
325 case (BPF_LD | BPF_IMM | EBPF_DW):
326 reg[ins->dst_reg] = (uint32_t)ins[0].imm |
327 (uint64_t)(uint32_t)ins[1].imm << 32;
328 ins++;
329 break;
330 /* load absolute instructions */
331 case (BPF_LD | BPF_ABS | BPF_B):
332 BPF_LD_ABS(bpf, reg, ins, uint8_t, NOP);
333 break;
334 case (BPF_LD | BPF_ABS | BPF_H):
335 BPF_LD_ABS(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);
336 break;
337 case (BPF_LD | BPF_ABS | BPF_W):
338 BPF_LD_ABS(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);
339 break;
340 /* load indirect instructions */
341 case (BPF_LD | BPF_IND | BPF_B):
342 BPF_LD_IND(bpf, reg, ins, uint8_t, NOP);
343 break;
344 case (BPF_LD | BPF_IND | BPF_H):
345 BPF_LD_IND(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);
346 break;
347 case (BPF_LD | BPF_IND | BPF_W):
348 BPF_LD_IND(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);
349 break;
350 /* store instructions */
351 case (BPF_STX | BPF_MEM | BPF_B):
352 BPF_ST_REG(reg, ins, uint8_t);
353 break;
354 case (BPF_STX | BPF_MEM | BPF_H):
355 BPF_ST_REG(reg, ins, uint16_t);
356 break;
357 case (BPF_STX | BPF_MEM | BPF_W):
358 BPF_ST_REG(reg, ins, uint32_t);
359 break;
360 case (BPF_STX | BPF_MEM | EBPF_DW):
361 BPF_ST_REG(reg, ins, uint64_t);
362 break;
363 case (BPF_ST | BPF_MEM | BPF_B):
364 BPF_ST_IMM(reg, ins, uint8_t);
365 break;
366 case (BPF_ST | BPF_MEM | BPF_H):
367 BPF_ST_IMM(reg, ins, uint16_t);
368 break;
369 case (BPF_ST | BPF_MEM | BPF_W):
370 BPF_ST_IMM(reg, ins, uint32_t);
371 break;
372 case (BPF_ST | BPF_MEM | EBPF_DW):
373 BPF_ST_IMM(reg, ins, uint64_t);
374 break;
375 /* atomic add instructions */
376 case (BPF_STX | EBPF_XADD | BPF_W):
377 BPF_ST_XADD_REG(reg, ins, 32);
378 break;
379 case (BPF_STX | EBPF_XADD | EBPF_DW):
380 BPF_ST_XADD_REG(reg, ins, 64);
381 break;
382 /* jump instructions */
383 case (BPF_JMP | BPF_JA):
384 BPF_JMP_UNC(ins);
385 break;
386 /* jump IMM instructions */
387 case (BPF_JMP | BPF_JEQ | BPF_K):
388 BPF_JMP_CND_IMM(reg, ins, ==, uint64_t);
389 break;
390 case (BPF_JMP | EBPF_JNE | BPF_K):
391 BPF_JMP_CND_IMM(reg, ins, !=, uint64_t);
392 break;
393 case (BPF_JMP | BPF_JGT | BPF_K):
394 BPF_JMP_CND_IMM(reg, ins, >, uint64_t);
395 break;
396 case (BPF_JMP | EBPF_JLT | BPF_K):
397 BPF_JMP_CND_IMM(reg, ins, <, uint64_t);
398 break;
399 case (BPF_JMP | BPF_JGE | BPF_K):
400 BPF_JMP_CND_IMM(reg, ins, >=, uint64_t);
401 break;
402 case (BPF_JMP | EBPF_JLE | BPF_K):
403 BPF_JMP_CND_IMM(reg, ins, <=, uint64_t);
404 break;
405 case (BPF_JMP | EBPF_JSGT | BPF_K):
406 BPF_JMP_CND_IMM(reg, ins, >, int64_t);
407 break;
408 case (BPF_JMP | EBPF_JSLT | BPF_K):
409 BPF_JMP_CND_IMM(reg, ins, <, int64_t);
410 break;
411 case (BPF_JMP | EBPF_JSGE | BPF_K):
412 BPF_JMP_CND_IMM(reg, ins, >=, int64_t);
413 break;
414 case (BPF_JMP | EBPF_JSLE | BPF_K):
415 BPF_JMP_CND_IMM(reg, ins, <=, int64_t);
416 break;
417 case (BPF_JMP | BPF_JSET | BPF_K):
418 BPF_JMP_CND_IMM(reg, ins, &, uint64_t);
419 break;
420 /* jump REG instructions */
421 case (BPF_JMP | BPF_JEQ | BPF_X):
422 BPF_JMP_CND_REG(reg, ins, ==, uint64_t);
423 break;
424 case (BPF_JMP | EBPF_JNE | BPF_X):
425 BPF_JMP_CND_REG(reg, ins, !=, uint64_t);
426 break;
427 case (BPF_JMP | BPF_JGT | BPF_X):
428 BPF_JMP_CND_REG(reg, ins, >, uint64_t);
429 break;
430 case (BPF_JMP | EBPF_JLT | BPF_X):
431 BPF_JMP_CND_REG(reg, ins, <, uint64_t);
432 break;
433 case (BPF_JMP | BPF_JGE | BPF_X):
434 BPF_JMP_CND_REG(reg, ins, >=, uint64_t);
435 break;
436 case (BPF_JMP | EBPF_JLE | BPF_X):
437 BPF_JMP_CND_REG(reg, ins, <=, uint64_t);
438 break;
439 case (BPF_JMP | EBPF_JSGT | BPF_X):
440 BPF_JMP_CND_REG(reg, ins, >, int64_t);
441 break;
442 case (BPF_JMP | EBPF_JSLT | BPF_X):
443 BPF_JMP_CND_REG(reg, ins, <, int64_t);
444 break;
445 case (BPF_JMP | EBPF_JSGE | BPF_X):
446 BPF_JMP_CND_REG(reg, ins, >=, int64_t);
447 break;
448 case (BPF_JMP | EBPF_JSLE | BPF_X):
449 BPF_JMP_CND_REG(reg, ins, <=, int64_t);
450 break;
451 case (BPF_JMP | BPF_JSET | BPF_X):
452 BPF_JMP_CND_REG(reg, ins, &, uint64_t);
453 break;
454 /* call instructions */
455 case (BPF_JMP | EBPF_CALL):
456 reg[EBPF_REG_0] = bpf->prm.xsym[ins->imm].func.val(
457 reg[EBPF_REG_1], reg[EBPF_REG_2],
458 reg[EBPF_REG_3], reg[EBPF_REG_4],
459 reg[EBPF_REG_5]);
460 break;
461 /* return instruction */
462 case (BPF_JMP | EBPF_EXIT):
463 return reg[EBPF_REG_0];
464 default:
465 RTE_BPF_LOG_LINE(ERR,
466 "%s(%p): invalid opcode %#x at pc: %#zx;",
467 __func__, bpf, ins->code,
468 (uintptr_t)ins - (uintptr_t)bpf->prm.ins);
469 return 0;
470 }
471 }
472
473 /* should never be reached */
474 RTE_VERIFY(0);
475 return 0;
476 }
477
478 uint32_t
rte_bpf_exec_burst(const struct rte_bpf * bpf,void * ctx[],uint64_t rc[],uint32_t num)479 rte_bpf_exec_burst(const struct rte_bpf *bpf, void *ctx[], uint64_t rc[],
480 uint32_t num)
481 {
482 uint32_t i;
483 uint64_t reg[EBPF_REG_NUM];
484 uint64_t stack[MAX_BPF_STACK_SIZE / sizeof(uint64_t)];
485
486 for (i = 0; i != num; i++) {
487
488 reg[EBPF_REG_1] = (uintptr_t)ctx[i];
489 reg[EBPF_REG_10] = (uintptr_t)(stack + RTE_DIM(stack));
490
491 rc[i] = bpf_exec(bpf, reg);
492 }
493
494 return i;
495 }
496
497 uint64_t
rte_bpf_exec(const struct rte_bpf * bpf,void * ctx)498 rte_bpf_exec(const struct rte_bpf *bpf, void *ctx)
499 {
500 uint64_t rc;
501
502 rte_bpf_exec_burst(bpf, &ctx, &rc, 1);
503 return rc;
504 }
505