1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4
5 #include <errno.h>
6 #include <stdint.h>
7
8 #include <rte_common.h>
9 #include <rte_log.h>
10 #include <rte_debug.h>
11
12 #include "bpf_impl.h"
13
14 #define GET_BPF_OP(op) (BPF_OP(op) >> 4)
15
16 enum {
17 RAX = 0, /* scratch, return value */
18 RCX = 1, /* scratch, 4th arg */
19 RDX = 2, /* scratch, 3rd arg */
20 RBX = 3, /* callee saved */
21 RSP = 4, /* stack pointer */
22 RBP = 5, /* frame pointer, callee saved */
23 RSI = 6, /* scratch, 2nd arg */
24 RDI = 7, /* scratch, 1st arg */
25 R8 = 8, /* scratch, 5th arg */
26 R9 = 9, /* scratch, 6th arg */
27 R10 = 10, /* scratch */
28 R11 = 11, /* scratch */
29 R12 = 12, /* callee saved */
30 R13 = 13, /* callee saved */
31 R14 = 14, /* callee saved */
32 R15 = 15, /* callee saved */
33 };
34
35 #define IS_EXT_REG(r) ((r) >= R8)
36
37 enum {
38 REX_PREFIX = 0x40, /* fixed value 0100 */
39 REX_W = 0x8, /* 64bit operand size */
40 REX_R = 0x4, /* extension of the ModRM.reg field */
41 REX_X = 0x2, /* extension of the SIB.index field */
42 REX_B = 0x1, /* extension of the ModRM.rm field */
43 };
44
45 enum {
46 MOD_INDIRECT = 0,
47 MOD_IDISP8 = 1,
48 MOD_IDISP32 = 2,
49 MOD_DIRECT = 3,
50 };
51
52 enum {
53 SIB_SCALE_1 = 0,
54 SIB_SCALE_2 = 1,
55 SIB_SCALE_4 = 2,
56 SIB_SCALE_8 = 3,
57 };
58
59 /*
60 * eBPF to x86_64 register mappings.
61 */
62 static const uint32_t ebpf2x86[] = {
63 [EBPF_REG_0] = RAX,
64 [EBPF_REG_1] = RDI,
65 [EBPF_REG_2] = RSI,
66 [EBPF_REG_3] = RDX,
67 [EBPF_REG_4] = RCX,
68 [EBPF_REG_5] = R8,
69 [EBPF_REG_6] = RBX,
70 [EBPF_REG_7] = R13,
71 [EBPF_REG_8] = R14,
72 [EBPF_REG_9] = R15,
73 [EBPF_REG_10] = RBP,
74 };
75
76 /*
77 * r10 and r11 are used as a scratch temporary registers.
78 */
79 enum {
80 REG_DIV_IMM = R9,
81 REG_TMP0 = R11,
82 REG_TMP1 = R10,
83 };
84
85 /* LD_ABS/LD_IMM offsets */
86 enum {
87 LDMB_FSP_OFS, /* fast-path */
88 LDMB_SLP_OFS, /* slow-path */
89 LDMB_FIN_OFS, /* final part */
90 LDMB_OFS_NUM
91 };
92
93 /*
94 * callee saved registers list.
95 * keep RBP as the last one.
96 */
97 static const uint32_t save_regs[] = {RBX, R12, R13, R14, R15, RBP};
98
99 struct bpf_jit_state {
100 uint32_t idx;
101 size_t sz;
102 struct {
103 uint32_t num;
104 int32_t off;
105 } exit;
106 struct {
107 uint32_t stack_ofs;
108 } ldmb;
109 uint32_t reguse;
110 int32_t *off;
111 uint8_t *ins;
112 };
113
114 #define INUSE(v, r) (((v) >> (r)) & 1)
115 #define USED(v, r) ((v) |= 1 << (r))
116
117 union bpf_jit_imm {
118 uint32_t u32;
119 uint8_t u8[4];
120 };
121
122 /*
123 * In many cases for imm8 we can produce shorter code.
124 */
125 static size_t
imm_size(int32_t v)126 imm_size(int32_t v)
127 {
128 if (v == (int8_t)v)
129 return sizeof(int8_t);
130 return sizeof(int32_t);
131 }
132
133 static void
emit_bytes(struct bpf_jit_state * st,const uint8_t ins[],uint32_t sz)134 emit_bytes(struct bpf_jit_state *st, const uint8_t ins[], uint32_t sz)
135 {
136 uint32_t i;
137
138 if (st->ins != NULL) {
139 for (i = 0; i != sz; i++)
140 st->ins[st->sz + i] = ins[i];
141 }
142 st->sz += sz;
143 }
144
145 static void
emit_imm(struct bpf_jit_state * st,const uint32_t imm,uint32_t sz)146 emit_imm(struct bpf_jit_state *st, const uint32_t imm, uint32_t sz)
147 {
148 union bpf_jit_imm v;
149
150 v.u32 = imm;
151 emit_bytes(st, v.u8, sz);
152 }
153
154 /*
155 * emit REX byte
156 */
157 static void
emit_rex(struct bpf_jit_state * st,uint32_t op,uint32_t reg,uint32_t rm)158 emit_rex(struct bpf_jit_state *st, uint32_t op, uint32_t reg, uint32_t rm)
159 {
160 uint8_t rex;
161
162 /* mark operand registers as used*/
163 USED(st->reguse, reg);
164 USED(st->reguse, rm);
165
166 rex = 0;
167 if (BPF_CLASS(op) == EBPF_ALU64 ||
168 op == (BPF_ST | BPF_MEM | EBPF_DW) ||
169 op == (BPF_STX | BPF_MEM | EBPF_DW) ||
170 op == (BPF_STX | EBPF_XADD | EBPF_DW) ||
171 op == (BPF_LD | BPF_IMM | EBPF_DW) ||
172 (BPF_CLASS(op) == BPF_LDX &&
173 BPF_MODE(op) == BPF_MEM &&
174 BPF_SIZE(op) != BPF_W))
175 rex |= REX_W;
176
177 if (IS_EXT_REG(reg))
178 rex |= REX_R;
179
180 if (IS_EXT_REG(rm))
181 rex |= REX_B;
182
183 /* store using SIL, DIL */
184 if (op == (BPF_STX | BPF_MEM | BPF_B) && (reg == RDI || reg == RSI))
185 rex |= REX_PREFIX;
186
187 if (rex != 0) {
188 rex |= REX_PREFIX;
189 emit_bytes(st, &rex, sizeof(rex));
190 }
191 }
192
193 /*
194 * emit MODRegRM byte
195 */
196 static void
emit_modregrm(struct bpf_jit_state * st,uint32_t mod,uint32_t reg,uint32_t rm)197 emit_modregrm(struct bpf_jit_state *st, uint32_t mod, uint32_t reg, uint32_t rm)
198 {
199 uint8_t v;
200
201 v = mod << 6 | (reg & 7) << 3 | (rm & 7);
202 emit_bytes(st, &v, sizeof(v));
203 }
204
205 /*
206 * emit SIB byte
207 */
208 static void
emit_sib(struct bpf_jit_state * st,uint32_t scale,uint32_t idx,uint32_t base)209 emit_sib(struct bpf_jit_state *st, uint32_t scale, uint32_t idx, uint32_t base)
210 {
211 uint8_t v;
212
213 v = scale << 6 | (idx & 7) << 3 | (base & 7);
214 emit_bytes(st, &v, sizeof(v));
215 }
216
217 /*
218 * emit OPCODE+REGIDX byte
219 */
220 static void
emit_opcode(struct bpf_jit_state * st,uint8_t ops,uint32_t reg)221 emit_opcode(struct bpf_jit_state *st, uint8_t ops, uint32_t reg)
222 {
223 uint8_t v;
224
225 v = ops | (reg & 7);
226 emit_bytes(st, &v, sizeof(v));
227 }
228
229
230 /*
231 * emit xchg %<sreg>, %<dreg>
232 */
233 static void
emit_xchg_reg(struct bpf_jit_state * st,uint32_t sreg,uint32_t dreg)234 emit_xchg_reg(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
235 {
236 const uint8_t ops = 0x87;
237
238 emit_rex(st, EBPF_ALU64, sreg, dreg);
239 emit_bytes(st, &ops, sizeof(ops));
240 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
241 }
242
243 /*
244 * emit neg %<dreg>
245 */
246 static void
emit_neg(struct bpf_jit_state * st,uint32_t op,uint32_t dreg)247 emit_neg(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
248 {
249 const uint8_t ops = 0xF7;
250 const uint8_t mods = 3;
251
252 emit_rex(st, op, 0, dreg);
253 emit_bytes(st, &ops, sizeof(ops));
254 emit_modregrm(st, MOD_DIRECT, mods, dreg);
255 }
256
257 /*
258 * emit mov %<sreg>, %<dreg>
259 */
260 static void
emit_mov_reg(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg)261 emit_mov_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
262 uint32_t dreg)
263 {
264 const uint8_t ops = 0x89;
265
266 /* if operands are 32-bit, then it can be used to clear upper 32-bit */
267 if (sreg != dreg || BPF_CLASS(op) == BPF_ALU) {
268 emit_rex(st, op, sreg, dreg);
269 emit_bytes(st, &ops, sizeof(ops));
270 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
271 }
272 }
273
274 /*
275 * emit movzwl %<sreg>, %<dreg>
276 */
277 static void
emit_movzwl(struct bpf_jit_state * st,uint32_t sreg,uint32_t dreg)278 emit_movzwl(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
279 {
280 static const uint8_t ops[] = {0x0F, 0xB7};
281
282 emit_rex(st, BPF_ALU, sreg, dreg);
283 emit_bytes(st, ops, sizeof(ops));
284 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
285 }
286
287 /*
288 * emit ror <imm8>, %<dreg>
289 */
290 static void
emit_ror_imm(struct bpf_jit_state * st,uint32_t dreg,uint32_t imm)291 emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
292 {
293 const uint8_t prfx = 0x66;
294 const uint8_t ops = 0xC1;
295 const uint8_t mods = 1;
296
297 emit_bytes(st, &prfx, sizeof(prfx));
298 emit_rex(st, BPF_ALU, 0, dreg);
299 emit_bytes(st, &ops, sizeof(ops));
300 emit_modregrm(st, MOD_DIRECT, mods, dreg);
301 emit_imm(st, imm, imm_size(imm));
302 }
303
304 /*
305 * emit bswap %<dreg>
306 */
307 static void
emit_be2le_48(struct bpf_jit_state * st,uint32_t dreg,uint32_t imm)308 emit_be2le_48(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
309 {
310 uint32_t rop;
311
312 const uint8_t ops = 0x0F;
313 const uint8_t mods = 1;
314
315 rop = (imm == 64) ? EBPF_ALU64 : BPF_ALU;
316 emit_rex(st, rop, 0, dreg);
317 emit_bytes(st, &ops, sizeof(ops));
318 emit_modregrm(st, MOD_DIRECT, mods, dreg);
319 }
320
321 static void
emit_be2le(struct bpf_jit_state * st,uint32_t dreg,uint32_t imm)322 emit_be2le(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
323 {
324 if (imm == 16) {
325 emit_ror_imm(st, dreg, 8);
326 emit_movzwl(st, dreg, dreg);
327 } else
328 emit_be2le_48(st, dreg, imm);
329 }
330
331 /*
332 * In general it is NOP for x86.
333 * Just clear the upper bits.
334 */
335 static void
emit_le2be(struct bpf_jit_state * st,uint32_t dreg,uint32_t imm)336 emit_le2be(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
337 {
338 if (imm == 16)
339 emit_movzwl(st, dreg, dreg);
340 else if (imm == 32)
341 emit_mov_reg(st, BPF_ALU | EBPF_MOV | BPF_X, dreg, dreg);
342 }
343
344 /*
345 * emit one of:
346 * add <imm>, %<dreg>
347 * and <imm>, %<dreg>
348 * or <imm>, %<dreg>
349 * sub <imm>, %<dreg>
350 * xor <imm>, %<dreg>
351 */
352 static void
emit_alu_imm(struct bpf_jit_state * st,uint32_t op,uint32_t dreg,uint32_t imm)353 emit_alu_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
354 {
355 uint8_t mod, opcode;
356 uint32_t bop, imsz;
357
358 const uint8_t op8 = 0x83;
359 const uint8_t op32 = 0x81;
360 static const uint8_t mods[] = {
361 [GET_BPF_OP(BPF_ADD)] = 0,
362 [GET_BPF_OP(BPF_AND)] = 4,
363 [GET_BPF_OP(BPF_OR)] = 1,
364 [GET_BPF_OP(BPF_SUB)] = 5,
365 [GET_BPF_OP(BPF_XOR)] = 6,
366 };
367
368 bop = GET_BPF_OP(op);
369 mod = mods[bop];
370
371 imsz = imm_size(imm);
372 opcode = (imsz == 1) ? op8 : op32;
373
374 emit_rex(st, op, 0, dreg);
375 emit_bytes(st, &opcode, sizeof(opcode));
376 emit_modregrm(st, MOD_DIRECT, mod, dreg);
377 emit_imm(st, imm, imsz);
378 }
379
380 /*
381 * emit one of:
382 * add %<sreg>, %<dreg>
383 * and %<sreg>, %<dreg>
384 * or %<sreg>, %<dreg>
385 * sub %<sreg>, %<dreg>
386 * xor %<sreg>, %<dreg>
387 */
388 static void
emit_alu_reg(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg)389 emit_alu_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
390 uint32_t dreg)
391 {
392 uint32_t bop;
393
394 static const uint8_t ops[] = {
395 [GET_BPF_OP(BPF_ADD)] = 0x01,
396 [GET_BPF_OP(BPF_AND)] = 0x21,
397 [GET_BPF_OP(BPF_OR)] = 0x09,
398 [GET_BPF_OP(BPF_SUB)] = 0x29,
399 [GET_BPF_OP(BPF_XOR)] = 0x31,
400 };
401
402 bop = GET_BPF_OP(op);
403
404 emit_rex(st, op, sreg, dreg);
405 emit_bytes(st, &ops[bop], sizeof(ops[bop]));
406 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
407 }
408
409 static void
emit_shift(struct bpf_jit_state * st,uint32_t op,uint32_t dreg)410 emit_shift(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
411 {
412 uint8_t mod;
413 uint32_t bop, opx;
414
415 static const uint8_t ops[] = {0xC1, 0xD3};
416 static const uint8_t mods[] = {
417 [GET_BPF_OP(BPF_LSH)] = 4,
418 [GET_BPF_OP(BPF_RSH)] = 5,
419 [GET_BPF_OP(EBPF_ARSH)] = 7,
420 };
421
422 bop = GET_BPF_OP(op);
423 mod = mods[bop];
424 opx = (BPF_SRC(op) == BPF_X);
425
426 emit_rex(st, op, 0, dreg);
427 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
428 emit_modregrm(st, MOD_DIRECT, mod, dreg);
429 }
430
431 /*
432 * emit one of:
433 * shl <imm>, %<dreg>
434 * shr <imm>, %<dreg>
435 * sar <imm>, %<dreg>
436 */
437 static void
emit_shift_imm(struct bpf_jit_state * st,uint32_t op,uint32_t dreg,uint32_t imm)438 emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
439 uint32_t imm)
440 {
441 emit_shift(st, op, dreg);
442 emit_imm(st, imm, imm_size(imm));
443 }
444
445 /*
446 * emit one of:
447 * shl %<dreg>
448 * shr %<dreg>
449 * sar %<dreg>
450 * note that rcx is implicitly used as a source register, so few extra
451 * instructions for register spillage might be necessary.
452 */
453 static void
emit_shift_reg(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg)454 emit_shift_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
455 uint32_t dreg)
456 {
457 if (sreg != RCX)
458 emit_xchg_reg(st, RCX, sreg);
459
460 emit_shift(st, op, (dreg == RCX) ? sreg : dreg);
461
462 if (sreg != RCX)
463 emit_xchg_reg(st, RCX, sreg);
464 }
465
466 /*
467 * emit mov <imm>, %<dreg>
468 */
469 static void
emit_mov_imm(struct bpf_jit_state * st,uint32_t op,uint32_t dreg,uint32_t imm)470 emit_mov_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
471 {
472 const uint8_t ops = 0xC7;
473
474 if (imm == 0) {
475 /* replace 'mov 0, %<dst>' with 'xor %<dst>, %<dst>' */
476 op = BPF_CLASS(op) | BPF_XOR | BPF_X;
477 emit_alu_reg(st, op, dreg, dreg);
478 return;
479 }
480
481 emit_rex(st, op, 0, dreg);
482 emit_bytes(st, &ops, sizeof(ops));
483 emit_modregrm(st, MOD_DIRECT, 0, dreg);
484 emit_imm(st, imm, sizeof(imm));
485 }
486
487 /*
488 * emit mov <imm64>, %<dreg>
489 */
490 static void
emit_ld_imm64(struct bpf_jit_state * st,uint32_t dreg,uint32_t imm0,uint32_t imm1)491 emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0,
492 uint32_t imm1)
493 {
494 uint32_t op;
495
496 const uint8_t ops = 0xB8;
497
498 op = (imm1 == 0) ? BPF_ALU : EBPF_ALU64;
499
500 emit_rex(st, op, 0, dreg);
501 emit_opcode(st, ops, dreg);
502
503 emit_imm(st, imm0, sizeof(imm0));
504 if (imm1 != 0)
505 emit_imm(st, imm1, sizeof(imm1));
506 }
507
508 /*
509 * note that rax:rdx are implicitly used as source/destination registers,
510 * so some reg spillage is necessary.
511 * emit:
512 * mov %rax, %r11
513 * mov %rdx, %r10
514 * mov %<dreg>, %rax
515 * either:
516 * mov %<sreg>, %rdx
517 * OR
518 * mov <imm>, %rdx
519 * mul %rdx
520 * mov %r10, %rdx
521 * mov %rax, %<dreg>
522 * mov %r11, %rax
523 */
524 static void
emit_mul(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg,uint32_t imm)525 emit_mul(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
526 uint32_t imm)
527 {
528 const uint8_t ops = 0xF7;
529 const uint8_t mods = 4;
530
531 /* save rax & rdx */
532 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
533 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
534
535 /* rax = dreg */
536 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
537
538 if (BPF_SRC(op) == BPF_X)
539 /* rdx = sreg */
540 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
541 sreg == RAX ? REG_TMP0 : sreg, RDX);
542 else
543 /* rdx = imm */
544 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, imm);
545
546 emit_rex(st, op, RAX, RDX);
547 emit_bytes(st, &ops, sizeof(ops));
548 emit_modregrm(st, MOD_DIRECT, mods, RDX);
549
550 if (dreg != RDX)
551 /* restore rdx */
552 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
553
554 if (dreg != RAX) {
555 /* dreg = rax */
556 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
557 /* restore rax */
558 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
559 }
560 }
561
562 /*
563 * emit mov <ofs>(%<sreg>), %<dreg>
564 * note that for non 64-bit ops, higher bits have to be cleared.
565 */
566 static void
emit_ld_reg(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg,int32_t ofs)567 emit_ld_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
568 int32_t ofs)
569 {
570 uint32_t mods, opsz;
571 const uint8_t op32 = 0x8B;
572 const uint8_t op16[] = {0x0F, 0xB7};
573 const uint8_t op8[] = {0x0F, 0xB6};
574
575 emit_rex(st, op, dreg, sreg);
576
577 opsz = BPF_SIZE(op);
578 if (opsz == BPF_B)
579 emit_bytes(st, op8, sizeof(op8));
580 else if (opsz == BPF_H)
581 emit_bytes(st, op16, sizeof(op16));
582 else
583 emit_bytes(st, &op32, sizeof(op32));
584
585 mods = (imm_size(ofs) == 1) ? MOD_IDISP8 : MOD_IDISP32;
586
587 emit_modregrm(st, mods, dreg, sreg);
588 if (sreg == RSP || sreg == R12)
589 emit_sib(st, SIB_SCALE_1, sreg, sreg);
590 emit_imm(st, ofs, imm_size(ofs));
591 }
592
593 /*
594 * emit one of:
595 * mov %<sreg>, <ofs>(%<dreg>)
596 * mov <imm>, <ofs>(%<dreg>)
597 */
598 static void
emit_st_common(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg,uint32_t imm,int32_t ofs)599 emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
600 uint32_t dreg, uint32_t imm, int32_t ofs)
601 {
602 uint32_t mods, imsz, opsz, opx;
603 const uint8_t prfx16 = 0x66;
604
605 /* 8 bit instruction opcodes */
606 static const uint8_t op8[] = {0xC6, 0x88};
607
608 /* 16/32/64 bit instruction opcodes */
609 static const uint8_t ops[] = {0xC7, 0x89};
610
611 /* is the instruction has immediate value or src reg? */
612 opx = (BPF_CLASS(op) == BPF_STX);
613
614 opsz = BPF_SIZE(op);
615 if (opsz == BPF_H)
616 emit_bytes(st, &prfx16, sizeof(prfx16));
617
618 emit_rex(st, op, sreg, dreg);
619
620 if (opsz == BPF_B)
621 emit_bytes(st, &op8[opx], sizeof(op8[opx]));
622 else
623 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
624
625 imsz = imm_size(ofs);
626 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
627
628 emit_modregrm(st, mods, sreg, dreg);
629
630 if (dreg == RSP || dreg == R12)
631 emit_sib(st, SIB_SCALE_1, dreg, dreg);
632
633 emit_imm(st, ofs, imsz);
634
635 if (opx == 0) {
636 imsz = RTE_MIN(bpf_size(opsz), sizeof(imm));
637 emit_imm(st, imm, imsz);
638 }
639 }
640
641 static void
emit_st_imm(struct bpf_jit_state * st,uint32_t op,uint32_t dreg,uint32_t imm,int32_t ofs)642 emit_st_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm,
643 int32_t ofs)
644 {
645 emit_st_common(st, op, 0, dreg, imm, ofs);
646 }
647
648 static void
emit_st_reg(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg,int32_t ofs)649 emit_st_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
650 int32_t ofs)
651 {
652 emit_st_common(st, op, sreg, dreg, 0, ofs);
653 }
654
655 /*
656 * emit lock add %<sreg>, <ofs>(%<dreg>)
657 */
658 static void
emit_st_xadd(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg,int32_t ofs)659 emit_st_xadd(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
660 uint32_t dreg, int32_t ofs)
661 {
662 uint32_t imsz, mods;
663
664 const uint8_t lck = 0xF0; /* lock prefix */
665 const uint8_t ops = 0x01; /* add opcode */
666
667 imsz = imm_size(ofs);
668 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
669
670 emit_bytes(st, &lck, sizeof(lck));
671 emit_rex(st, op, sreg, dreg);
672 emit_bytes(st, &ops, sizeof(ops));
673 emit_modregrm(st, mods, sreg, dreg);
674 emit_imm(st, ofs, imsz);
675 }
676
677 /*
678 * emit:
679 * mov <imm64>, (%rax)
680 * call *%rax
681 */
682 static void
emit_call(struct bpf_jit_state * st,uintptr_t trg)683 emit_call(struct bpf_jit_state *st, uintptr_t trg)
684 {
685 const uint8_t ops = 0xFF;
686 const uint8_t mods = 2;
687
688 emit_ld_imm64(st, RAX, trg, trg >> 32);
689 emit_bytes(st, &ops, sizeof(ops));
690 emit_modregrm(st, MOD_DIRECT, mods, RAX);
691 }
692
693 /*
694 * emit jmp <ofs>
695 * where 'ofs' is the target offset for the native code.
696 */
697 static void
emit_abs_jmp(struct bpf_jit_state * st,int32_t ofs)698 emit_abs_jmp(struct bpf_jit_state *st, int32_t ofs)
699 {
700 int32_t joff;
701 uint32_t imsz;
702
703 const uint8_t op8 = 0xEB;
704 const uint8_t op32 = 0xE9;
705
706 const int32_t sz8 = sizeof(op8) + sizeof(uint8_t);
707 const int32_t sz32 = sizeof(op32) + sizeof(uint32_t);
708
709 /* max possible jmp instruction size */
710 const int32_t iszm = RTE_MAX(sz8, sz32);
711
712 joff = ofs - st->sz;
713 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
714
715 if (imsz == 1) {
716 emit_bytes(st, &op8, sizeof(op8));
717 joff -= sz8;
718 } else {
719 emit_bytes(st, &op32, sizeof(op32));
720 joff -= sz32;
721 }
722
723 emit_imm(st, joff, imsz);
724 }
725
726 /*
727 * emit jmp <ofs>
728 * where 'ofs' is the target offset for the BPF bytecode.
729 */
730 static void
emit_jmp(struct bpf_jit_state * st,int32_t ofs)731 emit_jmp(struct bpf_jit_state *st, int32_t ofs)
732 {
733 emit_abs_jmp(st, st->off[st->idx + ofs]);
734 }
735
736 /*
737 * emit one of:
738 * cmovz %<sreg>, <%dreg>
739 * cmovne %<sreg>, <%dreg>
740 * cmova %<sreg>, <%dreg>
741 * cmovb %<sreg>, <%dreg>
742 * cmovae %<sreg>, <%dreg>
743 * cmovbe %<sreg>, <%dreg>
744 * cmovg %<sreg>, <%dreg>
745 * cmovl %<sreg>, <%dreg>
746 * cmovge %<sreg>, <%dreg>
747 * cmovle %<sreg>, <%dreg>
748 */
749 static void
emit_movcc_reg(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg)750 emit_movcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
751 uint32_t dreg)
752 {
753 uint32_t bop;
754
755 static const uint8_t ops[][2] = {
756 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x44}, /* CMOVZ */
757 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x45}, /* CMOVNE */
758 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x47}, /* CMOVA */
759 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x42}, /* CMOVB */
760 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x43}, /* CMOVAE */
761 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x46}, /* CMOVBE */
762 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x4F}, /* CMOVG */
763 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x4C}, /* CMOVL */
764 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x4D}, /* CMOVGE */
765 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x4E}, /* CMOVLE */
766 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x45}, /* CMOVNE */
767 };
768
769 bop = GET_BPF_OP(op);
770
771 emit_rex(st, op, dreg, sreg);
772 emit_bytes(st, ops[bop], sizeof(ops[bop]));
773 emit_modregrm(st, MOD_DIRECT, dreg, sreg);
774 }
775
776 /*
777 * emit one of:
778 * je <ofs>
779 * jne <ofs>
780 * ja <ofs>
781 * jb <ofs>
782 * jae <ofs>
783 * jbe <ofs>
784 * jg <ofs>
785 * jl <ofs>
786 * jge <ofs>
787 * jle <ofs>
788 * where 'ofs' is the target offset for the native code.
789 */
790 static void
emit_abs_jcc(struct bpf_jit_state * st,uint32_t op,int32_t ofs)791 emit_abs_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
792 {
793 uint32_t bop, imsz;
794 int32_t joff;
795
796 static const uint8_t op8[] = {
797 [GET_BPF_OP(BPF_JEQ)] = 0x74, /* JE */
798 [GET_BPF_OP(EBPF_JNE)] = 0x75, /* JNE */
799 [GET_BPF_OP(BPF_JGT)] = 0x77, /* JA */
800 [GET_BPF_OP(EBPF_JLT)] = 0x72, /* JB */
801 [GET_BPF_OP(BPF_JGE)] = 0x73, /* JAE */
802 [GET_BPF_OP(EBPF_JLE)] = 0x76, /* JBE */
803 [GET_BPF_OP(EBPF_JSGT)] = 0x7F, /* JG */
804 [GET_BPF_OP(EBPF_JSLT)] = 0x7C, /* JL */
805 [GET_BPF_OP(EBPF_JSGE)] = 0x7D, /*JGE */
806 [GET_BPF_OP(EBPF_JSLE)] = 0x7E, /* JLE */
807 [GET_BPF_OP(BPF_JSET)] = 0x75, /*JNE */
808 };
809
810 static const uint8_t op32[][2] = {
811 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x84}, /* JE */
812 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x85}, /* JNE */
813 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x87}, /* JA */
814 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x82}, /* JB */
815 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x83}, /* JAE */
816 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x86}, /* JBE */
817 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x8F}, /* JG */
818 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x8C}, /* JL */
819 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x8D}, /*JGE */
820 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x8E}, /* JLE */
821 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x85}, /*JNE */
822 };
823
824 const int32_t sz8 = sizeof(op8[0]) + sizeof(uint8_t);
825 const int32_t sz32 = sizeof(op32[0]) + sizeof(uint32_t);
826
827 /* max possible jcc instruction size */
828 const int32_t iszm = RTE_MAX(sz8, sz32);
829
830 joff = ofs - st->sz;
831 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
832
833 bop = GET_BPF_OP(op);
834
835 if (imsz == 1) {
836 emit_bytes(st, &op8[bop], sizeof(op8[bop]));
837 joff -= sz8;
838 } else {
839 emit_bytes(st, op32[bop], sizeof(op32[bop]));
840 joff -= sz32;
841 }
842
843 emit_imm(st, joff, imsz);
844 }
845
846 /*
847 * emit one of:
848 * je <ofs>
849 * jne <ofs>
850 * ja <ofs>
851 * jb <ofs>
852 * jae <ofs>
853 * jbe <ofs>
854 * jg <ofs>
855 * jl <ofs>
856 * jge <ofs>
857 * jle <ofs>
858 * where 'ofs' is the target offset for the BPF bytecode.
859 */
860 static void
emit_jcc(struct bpf_jit_state * st,uint32_t op,int32_t ofs)861 emit_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
862 {
863 emit_abs_jcc(st, op, st->off[st->idx + ofs]);
864 }
865
866
867 /*
868 * emit cmp <imm>, %<dreg>
869 */
870 static void
emit_cmp_imm(struct bpf_jit_state * st,uint32_t op,uint32_t dreg,uint32_t imm)871 emit_cmp_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
872 {
873 uint8_t ops;
874 uint32_t imsz;
875
876 const uint8_t op8 = 0x83;
877 const uint8_t op32 = 0x81;
878 const uint8_t mods = 7;
879
880 imsz = imm_size(imm);
881 ops = (imsz == 1) ? op8 : op32;
882
883 emit_rex(st, op, 0, dreg);
884 emit_bytes(st, &ops, sizeof(ops));
885 emit_modregrm(st, MOD_DIRECT, mods, dreg);
886 emit_imm(st, imm, imsz);
887 }
888
889 /*
890 * emit test <imm>, %<dreg>
891 */
892 static void
emit_tst_imm(struct bpf_jit_state * st,uint32_t op,uint32_t dreg,uint32_t imm)893 emit_tst_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
894 {
895 const uint8_t ops = 0xF7;
896 const uint8_t mods = 0;
897
898 emit_rex(st, op, 0, dreg);
899 emit_bytes(st, &ops, sizeof(ops));
900 emit_modregrm(st, MOD_DIRECT, mods, dreg);
901 emit_imm(st, imm, imm_size(imm));
902 }
903
904 static void
emit_jcc_imm(struct bpf_jit_state * st,uint32_t op,uint32_t dreg,uint32_t imm,int32_t ofs)905 emit_jcc_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
906 uint32_t imm, int32_t ofs)
907 {
908 if (BPF_OP(op) == BPF_JSET)
909 emit_tst_imm(st, EBPF_ALU64, dreg, imm);
910 else
911 emit_cmp_imm(st, EBPF_ALU64, dreg, imm);
912
913 emit_jcc(st, op, ofs);
914 }
915
916 /*
917 * emit test %<sreg>, %<dreg>
918 */
919 static void
emit_tst_reg(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg)920 emit_tst_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
921 uint32_t dreg)
922 {
923 const uint8_t ops = 0x85;
924
925 emit_rex(st, op, sreg, dreg);
926 emit_bytes(st, &ops, sizeof(ops));
927 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
928 }
929
930 /*
931 * emit cmp %<sreg>, %<dreg>
932 */
933 static void
emit_cmp_reg(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg)934 emit_cmp_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
935 uint32_t dreg)
936 {
937 const uint8_t ops = 0x39;
938
939 emit_rex(st, op, sreg, dreg);
940 emit_bytes(st, &ops, sizeof(ops));
941 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
942
943 }
944
945 static void
emit_jcc_reg(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg,int32_t ofs)946 emit_jcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
947 uint32_t dreg, int32_t ofs)
948 {
949 if (BPF_OP(op) == BPF_JSET)
950 emit_tst_reg(st, EBPF_ALU64, sreg, dreg);
951 else
952 emit_cmp_reg(st, EBPF_ALU64, sreg, dreg);
953
954 emit_jcc(st, op, ofs);
955 }
956
957 /*
958 * note that rax:rdx are implicitly used as source/destination registers,
959 * so some reg spillage is necessary.
960 * emit:
961 * mov %rax, %r11
962 * mov %rdx, %r10
963 * mov %<dreg>, %rax
964 * xor %rdx, %rdx
965 * for divisor as immediate value:
966 * mov <imm>, %r9
967 * div %<divisor_reg>
968 * mov %r10, %rdx
969 * mov %rax, %<dreg>
970 * mov %r11, %rax
971 * either:
972 * mov %rax, %<dreg>
973 * OR
974 * mov %rdx, %<dreg>
975 * mov %r11, %rax
976 * mov %r10, %rdx
977 */
978 static void
emit_div(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t dreg,uint32_t imm)979 emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
980 uint32_t imm)
981 {
982 uint32_t sr;
983
984 const uint8_t ops = 0xF7;
985 const uint8_t mods = 6;
986
987 if (BPF_SRC(op) == BPF_X) {
988
989 /* check that src divisor is not zero */
990 emit_tst_reg(st, BPF_CLASS(op), sreg, sreg);
991
992 /* exit with return value zero */
993 emit_movcc_reg(st, BPF_CLASS(op) | BPF_JEQ | BPF_X, sreg, RAX);
994 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
995 }
996
997 /* save rax & rdx */
998 if (dreg != RAX)
999 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
1000 if (dreg != RDX)
1001 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
1002
1003 /* fill rax & rdx */
1004 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
1005 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, 0);
1006
1007 if (BPF_SRC(op) == BPF_X) {
1008 sr = sreg;
1009 if (sr == RAX)
1010 sr = REG_TMP0;
1011 else if (sr == RDX)
1012 sr = REG_TMP1;
1013 } else {
1014 sr = REG_DIV_IMM;
1015 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, sr, imm);
1016 }
1017
1018 emit_rex(st, op, 0, sr);
1019 emit_bytes(st, &ops, sizeof(ops));
1020 emit_modregrm(st, MOD_DIRECT, mods, sr);
1021
1022 if (BPF_OP(op) == BPF_DIV)
1023 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
1024 else
1025 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, dreg);
1026
1027 if (dreg != RAX)
1028 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
1029 if (dreg != RDX)
1030 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
1031 }
1032
1033 /*
1034 * helper function, used by emit_ld_mbuf().
1035 * generates code for 'fast_path':
1036 * calculate load offset and check is it inside first packet segment.
1037 */
1038 static void
emit_ldmb_fast_path(struct bpf_jit_state * st,const uint32_t rg[EBPF_REG_7],uint32_t sreg,uint32_t mode,uint32_t sz,uint32_t imm,const int32_t ofs[LDMB_OFS_NUM])1039 emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
1040 uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm,
1041 const int32_t ofs[LDMB_OFS_NUM])
1042 {
1043 /* make R2 contain *off* value */
1044
1045 if (sreg != rg[EBPF_REG_2]) {
1046 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K,
1047 rg[EBPF_REG_2], imm);
1048 if (mode == BPF_IND)
1049 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
1050 sreg, rg[EBPF_REG_2]);
1051 } else
1052 /* BPF_IND with sreg == R2 */
1053 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K,
1054 rg[EBPF_REG_2], imm);
1055
1056 /* R3 = mbuf->data_len */
1057 emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
1058 rg[EBPF_REG_6], rg[EBPF_REG_3],
1059 offsetof(struct rte_mbuf, data_len));
1060
1061 /* R3 = R3 - R2 */
1062 emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X,
1063 rg[EBPF_REG_2], rg[EBPF_REG_3]);
1064
1065 /* JSLT R3, <sz> <slow_path> */
1066 emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz);
1067 emit_abs_jcc(st, BPF_JMP | EBPF_JSLT | BPF_K, ofs[LDMB_SLP_OFS]);
1068
1069 /* R3 = mbuf->data_off */
1070 emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
1071 rg[EBPF_REG_6], rg[EBPF_REG_3],
1072 offsetof(struct rte_mbuf, data_off));
1073
1074 /* R0 = mbuf->buf_addr */
1075 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
1076 rg[EBPF_REG_6], rg[EBPF_REG_0],
1077 offsetof(struct rte_mbuf, buf_addr));
1078
1079 /* R0 = R0 + R3 */
1080 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
1081 rg[EBPF_REG_3], rg[EBPF_REG_0]);
1082
1083 /* R0 = R0 + R2 */
1084 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
1085 rg[EBPF_REG_2], rg[EBPF_REG_0]);
1086
1087 /* JMP <fin_part> */
1088 emit_abs_jmp(st, ofs[LDMB_FIN_OFS]);
1089 }
1090
1091 /*
1092 * helper function, used by emit_ld_mbuf().
1093 * generates code for 'slow_path':
1094 * call __rte_pktmbuf_read() and check return value.
1095 */
1096 static void
emit_ldmb_slow_path(struct bpf_jit_state * st,const uint32_t rg[EBPF_REG_7],uint32_t sz)1097 emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
1098 uint32_t sz)
1099 {
1100 /* make R3 contain *len* value (1/2/4) */
1101
1102 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz);
1103
1104 /* make R4 contain (RBP - ldmb.stack_ofs) */
1105
1106 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]);
1107 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4],
1108 st->ldmb.stack_ofs);
1109
1110 /* make R1 contain mbuf ptr */
1111
1112 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
1113 rg[EBPF_REG_6], rg[EBPF_REG_1]);
1114
1115 /* call rte_pktmbuf_read */
1116 emit_call(st, (uintptr_t)__rte_pktmbuf_read);
1117
1118 /* check that return value (R0) is not zero */
1119 emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]);
1120 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
1121 }
1122
1123 /*
1124 * helper function, used by emit_ld_mbuf().
1125 * generates final part of code for BPF_ABS/BPF_IND load:
1126 * perform data load and endianness conversion.
1127 * expects dreg to contain valid data pointer.
1128 */
1129 static void
emit_ldmb_fin(struct bpf_jit_state * st,uint32_t dreg,uint32_t opsz,uint32_t sz)1130 emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz,
1131 uint32_t sz)
1132 {
1133 emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0);
1134 if (sz != sizeof(uint8_t))
1135 emit_be2le(st, dreg, sz * CHAR_BIT);
1136 }
1137
1138 /*
1139 * emit code for BPF_ABS/BPF_IND load.
1140 * generates the following construction:
1141 * fast_path:
1142 * off = ins->sreg + ins->imm
1143 * if (mbuf->data_len - off < ins->opsz)
1144 * goto slow_path;
1145 * ptr = mbuf->buf_addr + mbuf->data_off + off;
1146 * goto fin_part;
1147 * slow_path:
1148 * typeof(ins->opsz) buf; //allocate space on the stack
1149 * ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf);
1150 * if (ptr == NULL)
1151 * goto exit_label;
1152 * fin_part:
1153 * res = *(typeof(ins->opsz))ptr;
1154 * res = bswap(res);
1155 */
1156 static void
emit_ld_mbuf(struct bpf_jit_state * st,uint32_t op,uint32_t sreg,uint32_t imm)1157 emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t imm)
1158 {
1159 uint32_t i, mode, opsz, sz;
1160 uint32_t rg[EBPF_REG_7];
1161 int32_t ofs[LDMB_OFS_NUM];
1162
1163 mode = BPF_MODE(op);
1164 opsz = BPF_SIZE(op);
1165 sz = bpf_size(opsz);
1166
1167 for (i = 0; i != RTE_DIM(rg); i++)
1168 rg[i] = ebpf2x86[i];
1169
1170 /* fill with fake offsets */
1171 for (i = 0; i != RTE_DIM(ofs); i++)
1172 ofs[i] = st->sz + INT8_MAX;
1173
1174 /* dry run first to calculate jump offsets */
1175
1176 ofs[LDMB_FSP_OFS] = st->sz;
1177 emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
1178 ofs[LDMB_SLP_OFS] = st->sz;
1179 emit_ldmb_slow_path(st, rg, sz);
1180 ofs[LDMB_FIN_OFS] = st->sz;
1181 emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
1182
1183 RTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX);
1184
1185 /* reset dry-run code and do a proper run */
1186
1187 st->sz = ofs[LDMB_FSP_OFS];
1188 emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
1189 emit_ldmb_slow_path(st, rg, sz);
1190 emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
1191 }
1192
1193 static void
emit_prolog(struct bpf_jit_state * st,int32_t stack_size)1194 emit_prolog(struct bpf_jit_state *st, int32_t stack_size)
1195 {
1196 uint32_t i;
1197 int32_t spil, ofs;
1198
1199 spil = 0;
1200 for (i = 0; i != RTE_DIM(save_regs); i++)
1201 spil += INUSE(st->reguse, save_regs[i]);
1202
1203 /* we can avoid touching the stack at all */
1204 if (spil == 0)
1205 return;
1206
1207
1208 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP,
1209 spil * sizeof(uint64_t));
1210
1211 ofs = 0;
1212 for (i = 0; i != RTE_DIM(save_regs); i++) {
1213 if (INUSE(st->reguse, save_regs[i]) != 0) {
1214 emit_st_reg(st, BPF_STX | BPF_MEM | EBPF_DW,
1215 save_regs[i], RSP, ofs);
1216 ofs += sizeof(uint64_t);
1217 }
1218 }
1219
1220 if (INUSE(st->reguse, RBP) != 0) {
1221 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RSP, RBP);
1222 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP, stack_size);
1223 }
1224 }
1225
1226 /*
1227 * emit ret
1228 */
1229 static void
emit_ret(struct bpf_jit_state * st)1230 emit_ret(struct bpf_jit_state *st)
1231 {
1232 const uint8_t ops = 0xC3;
1233
1234 emit_bytes(st, &ops, sizeof(ops));
1235 }
1236
1237 static void
emit_epilog(struct bpf_jit_state * st)1238 emit_epilog(struct bpf_jit_state *st)
1239 {
1240 uint32_t i;
1241 int32_t spil, ofs;
1242
1243 /* if we already have an epilog generate a jump to it */
1244 if (st->exit.num++ != 0) {
1245 emit_abs_jmp(st, st->exit.off);
1246 return;
1247 }
1248
1249 /* store offset of epilog block */
1250 st->exit.off = st->sz;
1251
1252 spil = 0;
1253 for (i = 0; i != RTE_DIM(save_regs); i++)
1254 spil += INUSE(st->reguse, save_regs[i]);
1255
1256 if (spil != 0) {
1257
1258 if (INUSE(st->reguse, RBP) != 0)
1259 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
1260 RBP, RSP);
1261
1262 ofs = 0;
1263 for (i = 0; i != RTE_DIM(save_regs); i++) {
1264 if (INUSE(st->reguse, save_regs[i]) != 0) {
1265 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
1266 RSP, save_regs[i], ofs);
1267 ofs += sizeof(uint64_t);
1268 }
1269 }
1270
1271 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, RSP,
1272 spil * sizeof(uint64_t));
1273 }
1274
1275 emit_ret(st);
1276 }
1277
1278 /*
1279 * walk through bpf code and translate them x86_64 one.
1280 */
1281 static int
emit(struct bpf_jit_state * st,const struct rte_bpf * bpf)1282 emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
1283 {
1284 uint32_t i, dr, op, sr;
1285 const struct ebpf_insn *ins;
1286
1287 /* reset state fields */
1288 st->sz = 0;
1289 st->exit.num = 0;
1290 st->ldmb.stack_ofs = bpf->stack_sz;
1291
1292 emit_prolog(st, bpf->stack_sz);
1293
1294 for (i = 0; i != bpf->prm.nb_ins; i++) {
1295
1296 st->idx = i;
1297 st->off[i] = st->sz;
1298
1299 ins = bpf->prm.ins + i;
1300
1301 dr = ebpf2x86[ins->dst_reg];
1302 sr = ebpf2x86[ins->src_reg];
1303 op = ins->code;
1304
1305 switch (op) {
1306 /* 32 bit ALU IMM operations */
1307 case (BPF_ALU | BPF_ADD | BPF_K):
1308 case (BPF_ALU | BPF_SUB | BPF_K):
1309 case (BPF_ALU | BPF_AND | BPF_K):
1310 case (BPF_ALU | BPF_OR | BPF_K):
1311 case (BPF_ALU | BPF_XOR | BPF_K):
1312 emit_alu_imm(st, op, dr, ins->imm);
1313 break;
1314 case (BPF_ALU | BPF_LSH | BPF_K):
1315 case (BPF_ALU | BPF_RSH | BPF_K):
1316 emit_shift_imm(st, op, dr, ins->imm);
1317 break;
1318 case (BPF_ALU | EBPF_MOV | BPF_K):
1319 emit_mov_imm(st, op, dr, ins->imm);
1320 break;
1321 /* 32 bit ALU REG operations */
1322 case (BPF_ALU | BPF_ADD | BPF_X):
1323 case (BPF_ALU | BPF_SUB | BPF_X):
1324 case (BPF_ALU | BPF_AND | BPF_X):
1325 case (BPF_ALU | BPF_OR | BPF_X):
1326 case (BPF_ALU | BPF_XOR | BPF_X):
1327 emit_alu_reg(st, op, sr, dr);
1328 break;
1329 case (BPF_ALU | BPF_LSH | BPF_X):
1330 case (BPF_ALU | BPF_RSH | BPF_X):
1331 emit_shift_reg(st, op, sr, dr);
1332 break;
1333 case (BPF_ALU | EBPF_MOV | BPF_X):
1334 emit_mov_reg(st, op, sr, dr);
1335 break;
1336 case (BPF_ALU | BPF_NEG):
1337 emit_neg(st, op, dr);
1338 break;
1339 case (BPF_ALU | EBPF_END | EBPF_TO_BE):
1340 emit_be2le(st, dr, ins->imm);
1341 break;
1342 case (BPF_ALU | EBPF_END | EBPF_TO_LE):
1343 emit_le2be(st, dr, ins->imm);
1344 break;
1345 /* 64 bit ALU IMM operations */
1346 case (EBPF_ALU64 | BPF_ADD | BPF_K):
1347 case (EBPF_ALU64 | BPF_SUB | BPF_K):
1348 case (EBPF_ALU64 | BPF_AND | BPF_K):
1349 case (EBPF_ALU64 | BPF_OR | BPF_K):
1350 case (EBPF_ALU64 | BPF_XOR | BPF_K):
1351 emit_alu_imm(st, op, dr, ins->imm);
1352 break;
1353 case (EBPF_ALU64 | BPF_LSH | BPF_K):
1354 case (EBPF_ALU64 | BPF_RSH | BPF_K):
1355 case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
1356 emit_shift_imm(st, op, dr, ins->imm);
1357 break;
1358 case (EBPF_ALU64 | EBPF_MOV | BPF_K):
1359 emit_mov_imm(st, op, dr, ins->imm);
1360 break;
1361 /* 64 bit ALU REG operations */
1362 case (EBPF_ALU64 | BPF_ADD | BPF_X):
1363 case (EBPF_ALU64 | BPF_SUB | BPF_X):
1364 case (EBPF_ALU64 | BPF_AND | BPF_X):
1365 case (EBPF_ALU64 | BPF_OR | BPF_X):
1366 case (EBPF_ALU64 | BPF_XOR | BPF_X):
1367 emit_alu_reg(st, op, sr, dr);
1368 break;
1369 case (EBPF_ALU64 | BPF_LSH | BPF_X):
1370 case (EBPF_ALU64 | BPF_RSH | BPF_X):
1371 case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
1372 emit_shift_reg(st, op, sr, dr);
1373 break;
1374 case (EBPF_ALU64 | EBPF_MOV | BPF_X):
1375 emit_mov_reg(st, op, sr, dr);
1376 break;
1377 case (EBPF_ALU64 | BPF_NEG):
1378 emit_neg(st, op, dr);
1379 break;
1380 /* multiply instructions */
1381 case (BPF_ALU | BPF_MUL | BPF_K):
1382 case (BPF_ALU | BPF_MUL | BPF_X):
1383 case (EBPF_ALU64 | BPF_MUL | BPF_K):
1384 case (EBPF_ALU64 | BPF_MUL | BPF_X):
1385 emit_mul(st, op, sr, dr, ins->imm);
1386 break;
1387 /* divide instructions */
1388 case (BPF_ALU | BPF_DIV | BPF_K):
1389 case (BPF_ALU | BPF_MOD | BPF_K):
1390 case (BPF_ALU | BPF_DIV | BPF_X):
1391 case (BPF_ALU | BPF_MOD | BPF_X):
1392 case (EBPF_ALU64 | BPF_DIV | BPF_K):
1393 case (EBPF_ALU64 | BPF_MOD | BPF_K):
1394 case (EBPF_ALU64 | BPF_DIV | BPF_X):
1395 case (EBPF_ALU64 | BPF_MOD | BPF_X):
1396 emit_div(st, op, sr, dr, ins->imm);
1397 break;
1398 /* load instructions */
1399 case (BPF_LDX | BPF_MEM | BPF_B):
1400 case (BPF_LDX | BPF_MEM | BPF_H):
1401 case (BPF_LDX | BPF_MEM | BPF_W):
1402 case (BPF_LDX | BPF_MEM | EBPF_DW):
1403 emit_ld_reg(st, op, sr, dr, ins->off);
1404 break;
1405 /* load 64 bit immediate value */
1406 case (BPF_LD | BPF_IMM | EBPF_DW):
1407 emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);
1408 i++;
1409 break;
1410 /* load absolute/indirect instructions */
1411 case (BPF_LD | BPF_ABS | BPF_B):
1412 case (BPF_LD | BPF_ABS | BPF_H):
1413 case (BPF_LD | BPF_ABS | BPF_W):
1414 case (BPF_LD | BPF_IND | BPF_B):
1415 case (BPF_LD | BPF_IND | BPF_H):
1416 case (BPF_LD | BPF_IND | BPF_W):
1417 emit_ld_mbuf(st, op, sr, ins->imm);
1418 break;
1419 /* store instructions */
1420 case (BPF_STX | BPF_MEM | BPF_B):
1421 case (BPF_STX | BPF_MEM | BPF_H):
1422 case (BPF_STX | BPF_MEM | BPF_W):
1423 case (BPF_STX | BPF_MEM | EBPF_DW):
1424 emit_st_reg(st, op, sr, dr, ins->off);
1425 break;
1426 case (BPF_ST | BPF_MEM | BPF_B):
1427 case (BPF_ST | BPF_MEM | BPF_H):
1428 case (BPF_ST | BPF_MEM | BPF_W):
1429 case (BPF_ST | BPF_MEM | EBPF_DW):
1430 emit_st_imm(st, op, dr, ins->imm, ins->off);
1431 break;
1432 /* atomic add instructions */
1433 case (BPF_STX | EBPF_XADD | BPF_W):
1434 case (BPF_STX | EBPF_XADD | EBPF_DW):
1435 emit_st_xadd(st, op, sr, dr, ins->off);
1436 break;
1437 /* jump instructions */
1438 case (BPF_JMP | BPF_JA):
1439 emit_jmp(st, ins->off + 1);
1440 break;
1441 /* jump IMM instructions */
1442 case (BPF_JMP | BPF_JEQ | BPF_K):
1443 case (BPF_JMP | EBPF_JNE | BPF_K):
1444 case (BPF_JMP | BPF_JGT | BPF_K):
1445 case (BPF_JMP | EBPF_JLT | BPF_K):
1446 case (BPF_JMP | BPF_JGE | BPF_K):
1447 case (BPF_JMP | EBPF_JLE | BPF_K):
1448 case (BPF_JMP | EBPF_JSGT | BPF_K):
1449 case (BPF_JMP | EBPF_JSLT | BPF_K):
1450 case (BPF_JMP | EBPF_JSGE | BPF_K):
1451 case (BPF_JMP | EBPF_JSLE | BPF_K):
1452 case (BPF_JMP | BPF_JSET | BPF_K):
1453 emit_jcc_imm(st, op, dr, ins->imm, ins->off + 1);
1454 break;
1455 /* jump REG instructions */
1456 case (BPF_JMP | BPF_JEQ | BPF_X):
1457 case (BPF_JMP | EBPF_JNE | BPF_X):
1458 case (BPF_JMP | BPF_JGT | BPF_X):
1459 case (BPF_JMP | EBPF_JLT | BPF_X):
1460 case (BPF_JMP | BPF_JGE | BPF_X):
1461 case (BPF_JMP | EBPF_JLE | BPF_X):
1462 case (BPF_JMP | EBPF_JSGT | BPF_X):
1463 case (BPF_JMP | EBPF_JSLT | BPF_X):
1464 case (BPF_JMP | EBPF_JSGE | BPF_X):
1465 case (BPF_JMP | EBPF_JSLE | BPF_X):
1466 case (BPF_JMP | BPF_JSET | BPF_X):
1467 emit_jcc_reg(st, op, sr, dr, ins->off + 1);
1468 break;
1469 /* call instructions */
1470 case (BPF_JMP | EBPF_CALL):
1471 emit_call(st,
1472 (uintptr_t)bpf->prm.xsym[ins->imm].func.val);
1473 break;
1474 /* return instruction */
1475 case (BPF_JMP | EBPF_EXIT):
1476 emit_epilog(st);
1477 break;
1478 default:
1479 RTE_BPF_LOG_LINE(ERR,
1480 "%s(%p): invalid opcode %#x at pc: %u;",
1481 __func__, bpf, ins->code, i);
1482 return -EINVAL;
1483 }
1484 }
1485
1486 return 0;
1487 }
1488
1489 /*
1490 * produce a native ISA version of the given BPF code.
1491 */
1492 int
__rte_bpf_jit_x86(struct rte_bpf * bpf)1493 __rte_bpf_jit_x86(struct rte_bpf *bpf)
1494 {
1495 int32_t rc;
1496 uint32_t i;
1497 size_t sz;
1498 struct bpf_jit_state st;
1499
1500 /* init state */
1501 memset(&st, 0, sizeof(st));
1502 st.off = malloc(bpf->prm.nb_ins * sizeof(st.off[0]));
1503 if (st.off == NULL)
1504 return -ENOMEM;
1505
1506 /* fill with fake offsets */
1507 st.exit.off = INT32_MAX;
1508 for (i = 0; i != bpf->prm.nb_ins; i++)
1509 st.off[i] = INT32_MAX;
1510
1511 /*
1512 * dry runs, used to calculate total code size and valid jump offsets.
1513 * stop when we get minimal possible size
1514 */
1515 do {
1516 sz = st.sz;
1517 rc = emit(&st, bpf);
1518 } while (rc == 0 && sz != st.sz);
1519
1520 if (rc == 0) {
1521
1522 /* allocate memory needed */
1523 st.ins = mmap(NULL, st.sz, PROT_READ | PROT_WRITE,
1524 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1525 if (st.ins == MAP_FAILED)
1526 rc = -ENOMEM;
1527 else
1528 /* generate code */
1529 rc = emit(&st, bpf);
1530 }
1531
1532 if (rc == 0 && mprotect(st.ins, st.sz, PROT_READ | PROT_EXEC) != 0)
1533 rc = -ENOMEM;
1534
1535 if (rc != 0)
1536 munmap(st.ins, st.sz);
1537 else {
1538 bpf->jit.func = (void *)st.ins;
1539 bpf->jit.sz = st.sz;
1540 }
1541
1542 free(st.off);
1543 return rc;
1544 }
1545