1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <errno.h> 7 #include <stdint.h> 8 #include <inttypes.h> 9 10 #include <rte_common.h> 11 #include <rte_log.h> 12 #include <rte_debug.h> 13 #include <rte_memory.h> 14 #include <rte_eal.h> 15 #include <rte_byteorder.h> 16 17 #include "bpf_impl.h" 18 19 #define GET_BPF_OP(op) (BPF_OP(op) >> 4) 20 21 enum { 22 RAX = 0, /* scratch, return value */ 23 RCX = 1, /* scratch, 4th arg */ 24 RDX = 2, /* scratch, 3rd arg */ 25 RBX = 3, /* callee saved */ 26 RSP = 4, /* stack pointer */ 27 RBP = 5, /* frame pointer, callee saved */ 28 RSI = 6, /* scratch, 2nd arg */ 29 RDI = 7, /* scratch, 1st arg */ 30 R8 = 8, /* scratch, 5th arg */ 31 R9 = 9, /* scratch, 6th arg */ 32 R10 = 10, /* scratch */ 33 R11 = 11, /* scratch */ 34 R12 = 12, /* callee saved */ 35 R13 = 13, /* callee saved */ 36 R14 = 14, /* callee saved */ 37 R15 = 15, /* callee saved */ 38 }; 39 40 #define IS_EXT_REG(r) ((r) >= R8) 41 42 enum { 43 REX_PREFIX = 0x40, /* fixed value 0100 */ 44 REX_W = 0x8, /* 64bit operand size */ 45 REX_R = 0x4, /* extension of the ModRM.reg field */ 46 REX_X = 0x2, /* extension of the SIB.index field */ 47 REX_B = 0x1, /* extension of the ModRM.rm field */ 48 }; 49 50 enum { 51 MOD_INDIRECT = 0, 52 MOD_IDISP8 = 1, 53 MOD_IDISP32 = 2, 54 MOD_DIRECT = 3, 55 }; 56 57 enum { 58 SIB_SCALE_1 = 0, 59 SIB_SCALE_2 = 1, 60 SIB_SCALE_4 = 2, 61 SIB_SCALE_8 = 3, 62 }; 63 64 /* 65 * eBPF to x86_64 register mappings. 66 */ 67 static const uint32_t ebpf2x86[] = { 68 [EBPF_REG_0] = RAX, 69 [EBPF_REG_1] = RDI, 70 [EBPF_REG_2] = RSI, 71 [EBPF_REG_3] = RDX, 72 [EBPF_REG_4] = RCX, 73 [EBPF_REG_5] = R8, 74 [EBPF_REG_6] = RBX, 75 [EBPF_REG_7] = R13, 76 [EBPF_REG_8] = R14, 77 [EBPF_REG_9] = R15, 78 [EBPF_REG_10] = RBP, 79 }; 80 81 /* 82 * r10 and r11 are used as a scratch temporary registers. 83 */ 84 enum { 85 REG_DIV_IMM = R9, 86 REG_TMP0 = R11, 87 REG_TMP1 = R10, 88 }; 89 90 /* LD_ABS/LD_IMM offsets */ 91 enum { 92 LDMB_FSP_OFS, /* fast-path */ 93 LDMB_SLP_OFS, /* slow-path */ 94 LDMB_FIN_OFS, /* final part */ 95 LDMB_OFS_NUM 96 }; 97 98 /* 99 * callee saved registers list. 100 * keep RBP as the last one. 101 */ 102 static const uint32_t save_regs[] = {RBX, R12, R13, R14, R15, RBP}; 103 104 struct bpf_jit_state { 105 uint32_t idx; 106 size_t sz; 107 struct { 108 uint32_t num; 109 int32_t off; 110 } exit; 111 struct { 112 uint32_t stack_ofs; 113 } ldmb; 114 uint32_t reguse; 115 int32_t *off; 116 uint8_t *ins; 117 }; 118 119 #define INUSE(v, r) (((v) >> (r)) & 1) 120 #define USED(v, r) ((v) |= 1 << (r)) 121 122 union bpf_jit_imm { 123 uint32_t u32; 124 uint8_t u8[4]; 125 }; 126 127 /* 128 * In many cases for imm8 we can produce shorter code. 129 */ 130 static size_t 131 imm_size(int32_t v) 132 { 133 if (v == (int8_t)v) 134 return sizeof(int8_t); 135 return sizeof(int32_t); 136 } 137 138 static void 139 emit_bytes(struct bpf_jit_state *st, const uint8_t ins[], uint32_t sz) 140 { 141 uint32_t i; 142 143 if (st->ins != NULL) { 144 for (i = 0; i != sz; i++) 145 st->ins[st->sz + i] = ins[i]; 146 } 147 st->sz += sz; 148 } 149 150 static void 151 emit_imm(struct bpf_jit_state *st, const uint32_t imm, uint32_t sz) 152 { 153 union bpf_jit_imm v; 154 155 v.u32 = imm; 156 emit_bytes(st, v.u8, sz); 157 } 158 159 /* 160 * emit REX byte 161 */ 162 static void 163 emit_rex(struct bpf_jit_state *st, uint32_t op, uint32_t reg, uint32_t rm) 164 { 165 uint8_t rex; 166 167 /* mark operand registers as used*/ 168 USED(st->reguse, reg); 169 USED(st->reguse, rm); 170 171 rex = 0; 172 if (BPF_CLASS(op) == EBPF_ALU64 || 173 op == (BPF_ST | BPF_MEM | EBPF_DW) || 174 op == (BPF_STX | BPF_MEM | EBPF_DW) || 175 op == (BPF_STX | EBPF_XADD | EBPF_DW) || 176 op == (BPF_LD | BPF_IMM | EBPF_DW) || 177 (BPF_CLASS(op) == BPF_LDX && 178 BPF_MODE(op) == BPF_MEM && 179 BPF_SIZE(op) != BPF_W)) 180 rex |= REX_W; 181 182 if (IS_EXT_REG(reg)) 183 rex |= REX_R; 184 185 if (IS_EXT_REG(rm)) 186 rex |= REX_B; 187 188 /* store using SIL, DIL */ 189 if (op == (BPF_STX | BPF_MEM | BPF_B) && (reg == RDI || reg == RSI)) 190 rex |= REX_PREFIX; 191 192 if (rex != 0) { 193 rex |= REX_PREFIX; 194 emit_bytes(st, &rex, sizeof(rex)); 195 } 196 } 197 198 /* 199 * emit MODRegRM byte 200 */ 201 static void 202 emit_modregrm(struct bpf_jit_state *st, uint32_t mod, uint32_t reg, uint32_t rm) 203 { 204 uint8_t v; 205 206 v = mod << 6 | (reg & 7) << 3 | (rm & 7); 207 emit_bytes(st, &v, sizeof(v)); 208 } 209 210 /* 211 * emit SIB byte 212 */ 213 static void 214 emit_sib(struct bpf_jit_state *st, uint32_t scale, uint32_t idx, uint32_t base) 215 { 216 uint8_t v; 217 218 v = scale << 6 | (idx & 7) << 3 | (base & 7); 219 emit_bytes(st, &v, sizeof(v)); 220 } 221 222 /* 223 * emit OPCODE+REGIDX byte 224 */ 225 static void 226 emit_opcode(struct bpf_jit_state *st, uint8_t ops, uint32_t reg) 227 { 228 uint8_t v; 229 230 v = ops | (reg & 7); 231 emit_bytes(st, &v, sizeof(v)); 232 } 233 234 235 /* 236 * emit xchg %<sreg>, %<dreg> 237 */ 238 static void 239 emit_xchg_reg(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg) 240 { 241 const uint8_t ops = 0x87; 242 243 emit_rex(st, EBPF_ALU64, sreg, dreg); 244 emit_bytes(st, &ops, sizeof(ops)); 245 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 246 } 247 248 /* 249 * emit neg %<dreg> 250 */ 251 static void 252 emit_neg(struct bpf_jit_state *st, uint32_t op, uint32_t dreg) 253 { 254 const uint8_t ops = 0xF7; 255 const uint8_t mods = 3; 256 257 emit_rex(st, op, 0, dreg); 258 emit_bytes(st, &ops, sizeof(ops)); 259 emit_modregrm(st, MOD_DIRECT, mods, dreg); 260 } 261 262 /* 263 * emit mov %<sreg>, %<dreg> 264 */ 265 static void 266 emit_mov_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 267 uint32_t dreg) 268 { 269 const uint8_t ops = 0x89; 270 271 /* if operands are 32-bit, then it can be used to clear upper 32-bit */ 272 if (sreg != dreg || BPF_CLASS(op) == BPF_ALU) { 273 emit_rex(st, op, sreg, dreg); 274 emit_bytes(st, &ops, sizeof(ops)); 275 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 276 } 277 } 278 279 /* 280 * emit movzwl %<sreg>, %<dreg> 281 */ 282 static void 283 emit_movzwl(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg) 284 { 285 static const uint8_t ops[] = {0x0F, 0xB7}; 286 287 emit_rex(st, BPF_ALU, sreg, dreg); 288 emit_bytes(st, ops, sizeof(ops)); 289 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 290 } 291 292 /* 293 * emit ror <imm8>, %<dreg> 294 */ 295 static void 296 emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm) 297 { 298 const uint8_t prfx = 0x66; 299 const uint8_t ops = 0xC1; 300 const uint8_t mods = 1; 301 302 emit_bytes(st, &prfx, sizeof(prfx)); 303 emit_rex(st, BPF_ALU, 0, dreg); 304 emit_bytes(st, &ops, sizeof(ops)); 305 emit_modregrm(st, MOD_DIRECT, mods, dreg); 306 emit_imm(st, imm, imm_size(imm)); 307 } 308 309 /* 310 * emit bswap %<dreg> 311 */ 312 static void 313 emit_be2le_48(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm) 314 { 315 uint32_t rop; 316 317 const uint8_t ops = 0x0F; 318 const uint8_t mods = 1; 319 320 rop = (imm == 64) ? EBPF_ALU64 : BPF_ALU; 321 emit_rex(st, rop, 0, dreg); 322 emit_bytes(st, &ops, sizeof(ops)); 323 emit_modregrm(st, MOD_DIRECT, mods, dreg); 324 } 325 326 static void 327 emit_be2le(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm) 328 { 329 if (imm == 16) { 330 emit_ror_imm(st, dreg, 8); 331 emit_movzwl(st, dreg, dreg); 332 } else 333 emit_be2le_48(st, dreg, imm); 334 } 335 336 /* 337 * In general it is NOP for x86. 338 * Just clear the upper bits. 339 */ 340 static void 341 emit_le2be(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm) 342 { 343 if (imm == 16) 344 emit_movzwl(st, dreg, dreg); 345 else if (imm == 32) 346 emit_mov_reg(st, BPF_ALU | EBPF_MOV | BPF_X, dreg, dreg); 347 } 348 349 /* 350 * emit one of: 351 * add <imm>, %<dreg> 352 * and <imm>, %<dreg> 353 * or <imm>, %<dreg> 354 * sub <imm>, %<dreg> 355 * xor <imm>, %<dreg> 356 */ 357 static void 358 emit_alu_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm) 359 { 360 uint8_t mod, opcode; 361 uint32_t bop, imsz; 362 363 const uint8_t op8 = 0x83; 364 const uint8_t op32 = 0x81; 365 static const uint8_t mods[] = { 366 [GET_BPF_OP(BPF_ADD)] = 0, 367 [GET_BPF_OP(BPF_AND)] = 4, 368 [GET_BPF_OP(BPF_OR)] = 1, 369 [GET_BPF_OP(BPF_SUB)] = 5, 370 [GET_BPF_OP(BPF_XOR)] = 6, 371 }; 372 373 bop = GET_BPF_OP(op); 374 mod = mods[bop]; 375 376 imsz = imm_size(imm); 377 opcode = (imsz == 1) ? op8 : op32; 378 379 emit_rex(st, op, 0, dreg); 380 emit_bytes(st, &opcode, sizeof(opcode)); 381 emit_modregrm(st, MOD_DIRECT, mod, dreg); 382 emit_imm(st, imm, imsz); 383 } 384 385 /* 386 * emit one of: 387 * add %<sreg>, %<dreg> 388 * and %<sreg>, %<dreg> 389 * or %<sreg>, %<dreg> 390 * sub %<sreg>, %<dreg> 391 * xor %<sreg>, %<dreg> 392 */ 393 static void 394 emit_alu_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 395 uint32_t dreg) 396 { 397 uint32_t bop; 398 399 static const uint8_t ops[] = { 400 [GET_BPF_OP(BPF_ADD)] = 0x01, 401 [GET_BPF_OP(BPF_AND)] = 0x21, 402 [GET_BPF_OP(BPF_OR)] = 0x09, 403 [GET_BPF_OP(BPF_SUB)] = 0x29, 404 [GET_BPF_OP(BPF_XOR)] = 0x31, 405 }; 406 407 bop = GET_BPF_OP(op); 408 409 emit_rex(st, op, sreg, dreg); 410 emit_bytes(st, &ops[bop], sizeof(ops[bop])); 411 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 412 } 413 414 static void 415 emit_shift(struct bpf_jit_state *st, uint32_t op, uint32_t dreg) 416 { 417 uint8_t mod; 418 uint32_t bop, opx; 419 420 static const uint8_t ops[] = {0xC1, 0xD3}; 421 static const uint8_t mods[] = { 422 [GET_BPF_OP(BPF_LSH)] = 4, 423 [GET_BPF_OP(BPF_RSH)] = 5, 424 [GET_BPF_OP(EBPF_ARSH)] = 7, 425 }; 426 427 bop = GET_BPF_OP(op); 428 mod = mods[bop]; 429 opx = (BPF_SRC(op) == BPF_X); 430 431 emit_rex(st, op, 0, dreg); 432 emit_bytes(st, &ops[opx], sizeof(ops[opx])); 433 emit_modregrm(st, MOD_DIRECT, mod, dreg); 434 } 435 436 /* 437 * emit one of: 438 * shl <imm>, %<dreg> 439 * shr <imm>, %<dreg> 440 * sar <imm>, %<dreg> 441 */ 442 static void 443 emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, 444 uint32_t imm) 445 { 446 emit_shift(st, op, dreg); 447 emit_imm(st, imm, imm_size(imm)); 448 } 449 450 /* 451 * emit one of: 452 * shl %<dreg> 453 * shr %<dreg> 454 * sar %<dreg> 455 * note that rcx is implicitly used as a source register, so few extra 456 * instructions for register spillage might be necessary. 457 */ 458 static void 459 emit_shift_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 460 uint32_t dreg) 461 { 462 if (sreg != RCX) 463 emit_xchg_reg(st, RCX, sreg); 464 465 emit_shift(st, op, (dreg == RCX) ? sreg : dreg); 466 467 if (sreg != RCX) 468 emit_xchg_reg(st, RCX, sreg); 469 } 470 471 /* 472 * emit mov <imm>, %<dreg> 473 */ 474 static void 475 emit_mov_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm) 476 { 477 const uint8_t ops = 0xC7; 478 479 if (imm == 0) { 480 /* replace 'mov 0, %<dst>' with 'xor %<dst>, %<dst>' */ 481 op = BPF_CLASS(op) | BPF_XOR | BPF_X; 482 emit_alu_reg(st, op, dreg, dreg); 483 return; 484 } 485 486 emit_rex(st, op, 0, dreg); 487 emit_bytes(st, &ops, sizeof(ops)); 488 emit_modregrm(st, MOD_DIRECT, 0, dreg); 489 emit_imm(st, imm, sizeof(imm)); 490 } 491 492 /* 493 * emit mov <imm64>, %<dreg> 494 */ 495 static void 496 emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0, 497 uint32_t imm1) 498 { 499 uint32_t op; 500 501 const uint8_t ops = 0xB8; 502 503 op = (imm1 == 0) ? BPF_ALU : EBPF_ALU64; 504 505 emit_rex(st, op, 0, dreg); 506 emit_opcode(st, ops, dreg); 507 508 emit_imm(st, imm0, sizeof(imm0)); 509 if (imm1 != 0) 510 emit_imm(st, imm1, sizeof(imm1)); 511 } 512 513 /* 514 * note that rax:rdx are implicitly used as source/destination registers, 515 * so some reg spillage is necessary. 516 * emit: 517 * mov %rax, %r11 518 * mov %rdx, %r10 519 * mov %<dreg>, %rax 520 * either: 521 * mov %<sreg>, %rdx 522 * OR 523 * mov <imm>, %rdx 524 * mul %rdx 525 * mov %r10, %rdx 526 * mov %rax, %<dreg> 527 * mov %r11, %rax 528 */ 529 static void 530 emit_mul(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, 531 uint32_t imm) 532 { 533 const uint8_t ops = 0xF7; 534 const uint8_t mods = 4; 535 536 /* save rax & rdx */ 537 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0); 538 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1); 539 540 /* rax = dreg */ 541 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX); 542 543 if (BPF_SRC(op) == BPF_X) 544 /* rdx = sreg */ 545 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, 546 sreg == RAX ? REG_TMP0 : sreg, RDX); 547 else 548 /* rdx = imm */ 549 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, imm); 550 551 emit_rex(st, op, RAX, RDX); 552 emit_bytes(st, &ops, sizeof(ops)); 553 emit_modregrm(st, MOD_DIRECT, mods, RDX); 554 555 if (dreg != RDX) 556 /* restore rdx */ 557 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX); 558 559 if (dreg != RAX) { 560 /* dreg = rax */ 561 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg); 562 /* restore rax */ 563 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX); 564 } 565 } 566 567 /* 568 * emit mov <ofs>(%<sreg>), %<dreg> 569 * note that for non 64-bit ops, higher bits have to be cleared. 570 */ 571 static void 572 emit_ld_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, 573 int32_t ofs) 574 { 575 uint32_t mods, opsz; 576 const uint8_t op32 = 0x8B; 577 const uint8_t op16[] = {0x0F, 0xB7}; 578 const uint8_t op8[] = {0x0F, 0xB6}; 579 580 emit_rex(st, op, dreg, sreg); 581 582 opsz = BPF_SIZE(op); 583 if (opsz == BPF_B) 584 emit_bytes(st, op8, sizeof(op8)); 585 else if (opsz == BPF_H) 586 emit_bytes(st, op16, sizeof(op16)); 587 else 588 emit_bytes(st, &op32, sizeof(op32)); 589 590 mods = (imm_size(ofs) == 1) ? MOD_IDISP8 : MOD_IDISP32; 591 592 emit_modregrm(st, mods, dreg, sreg); 593 if (sreg == RSP || sreg == R12) 594 emit_sib(st, SIB_SCALE_1, sreg, sreg); 595 emit_imm(st, ofs, imm_size(ofs)); 596 } 597 598 /* 599 * emit one of: 600 * mov %<sreg>, <ofs>(%<dreg>) 601 * mov <imm>, <ofs>(%<dreg>) 602 */ 603 static void 604 emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 605 uint32_t dreg, uint32_t imm, int32_t ofs) 606 { 607 uint32_t mods, imsz, opsz, opx; 608 const uint8_t prfx16 = 0x66; 609 610 /* 8 bit instruction opcodes */ 611 static const uint8_t op8[] = {0xC6, 0x88}; 612 613 /* 16/32/64 bit instruction opcodes */ 614 static const uint8_t ops[] = {0xC7, 0x89}; 615 616 /* is the instruction has immediate value or src reg? */ 617 opx = (BPF_CLASS(op) == BPF_STX); 618 619 opsz = BPF_SIZE(op); 620 if (opsz == BPF_H) 621 emit_bytes(st, &prfx16, sizeof(prfx16)); 622 623 emit_rex(st, op, sreg, dreg); 624 625 if (opsz == BPF_B) 626 emit_bytes(st, &op8[opx], sizeof(op8[opx])); 627 else 628 emit_bytes(st, &ops[opx], sizeof(ops[opx])); 629 630 imsz = imm_size(ofs); 631 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32; 632 633 emit_modregrm(st, mods, sreg, dreg); 634 635 if (dreg == RSP || dreg == R12) 636 emit_sib(st, SIB_SCALE_1, dreg, dreg); 637 638 emit_imm(st, ofs, imsz); 639 640 if (opx == 0) { 641 imsz = RTE_MIN(bpf_size(opsz), sizeof(imm)); 642 emit_imm(st, imm, imsz); 643 } 644 } 645 646 static void 647 emit_st_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm, 648 int32_t ofs) 649 { 650 emit_st_common(st, op, 0, dreg, imm, ofs); 651 } 652 653 static void 654 emit_st_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, 655 int32_t ofs) 656 { 657 emit_st_common(st, op, sreg, dreg, 0, ofs); 658 } 659 660 /* 661 * emit lock add %<sreg>, <ofs>(%<dreg>) 662 */ 663 static void 664 emit_st_xadd(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 665 uint32_t dreg, int32_t ofs) 666 { 667 uint32_t imsz, mods; 668 669 const uint8_t lck = 0xF0; /* lock prefix */ 670 const uint8_t ops = 0x01; /* add opcode */ 671 672 imsz = imm_size(ofs); 673 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32; 674 675 emit_bytes(st, &lck, sizeof(lck)); 676 emit_rex(st, op, sreg, dreg); 677 emit_bytes(st, &ops, sizeof(ops)); 678 emit_modregrm(st, mods, sreg, dreg); 679 emit_imm(st, ofs, imsz); 680 } 681 682 /* 683 * emit: 684 * mov <imm64>, (%rax) 685 * call *%rax 686 */ 687 static void 688 emit_call(struct bpf_jit_state *st, uintptr_t trg) 689 { 690 const uint8_t ops = 0xFF; 691 const uint8_t mods = 2; 692 693 emit_ld_imm64(st, RAX, trg, trg >> 32); 694 emit_bytes(st, &ops, sizeof(ops)); 695 emit_modregrm(st, MOD_DIRECT, mods, RAX); 696 } 697 698 /* 699 * emit jmp <ofs> 700 * where 'ofs' is the target offset for the native code. 701 */ 702 static void 703 emit_abs_jmp(struct bpf_jit_state *st, int32_t ofs) 704 { 705 int32_t joff; 706 uint32_t imsz; 707 708 const uint8_t op8 = 0xEB; 709 const uint8_t op32 = 0xE9; 710 711 const int32_t sz8 = sizeof(op8) + sizeof(uint8_t); 712 const int32_t sz32 = sizeof(op32) + sizeof(uint32_t); 713 714 /* max possible jmp instruction size */ 715 const int32_t iszm = RTE_MAX(sz8, sz32); 716 717 joff = ofs - st->sz; 718 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm)); 719 720 if (imsz == 1) { 721 emit_bytes(st, &op8, sizeof(op8)); 722 joff -= sz8; 723 } else { 724 emit_bytes(st, &op32, sizeof(op32)); 725 joff -= sz32; 726 } 727 728 emit_imm(st, joff, imsz); 729 } 730 731 /* 732 * emit jmp <ofs> 733 * where 'ofs' is the target offset for the BPF bytecode. 734 */ 735 static void 736 emit_jmp(struct bpf_jit_state *st, int32_t ofs) 737 { 738 emit_abs_jmp(st, st->off[st->idx + ofs]); 739 } 740 741 /* 742 * emit one of: 743 * cmovz %<sreg>, <%dreg> 744 * cmovne %<sreg>, <%dreg> 745 * cmova %<sreg>, <%dreg> 746 * cmovb %<sreg>, <%dreg> 747 * cmovae %<sreg>, <%dreg> 748 * cmovbe %<sreg>, <%dreg> 749 * cmovg %<sreg>, <%dreg> 750 * cmovl %<sreg>, <%dreg> 751 * cmovge %<sreg>, <%dreg> 752 * cmovle %<sreg>, <%dreg> 753 */ 754 static void 755 emit_movcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 756 uint32_t dreg) 757 { 758 uint32_t bop; 759 760 static const uint8_t ops[][2] = { 761 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x44}, /* CMOVZ */ 762 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x45}, /* CMOVNE */ 763 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x47}, /* CMOVA */ 764 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x42}, /* CMOVB */ 765 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x43}, /* CMOVAE */ 766 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x46}, /* CMOVBE */ 767 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x4F}, /* CMOVG */ 768 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x4C}, /* CMOVL */ 769 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x4D}, /* CMOVGE */ 770 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x4E}, /* CMOVLE */ 771 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x45}, /* CMOVNE */ 772 }; 773 774 bop = GET_BPF_OP(op); 775 776 emit_rex(st, op, dreg, sreg); 777 emit_bytes(st, ops[bop], sizeof(ops[bop])); 778 emit_modregrm(st, MOD_DIRECT, dreg, sreg); 779 } 780 781 /* 782 * emit one of: 783 * je <ofs> 784 * jne <ofs> 785 * ja <ofs> 786 * jb <ofs> 787 * jae <ofs> 788 * jbe <ofs> 789 * jg <ofs> 790 * jl <ofs> 791 * jge <ofs> 792 * jle <ofs> 793 * where 'ofs' is the target offset for the native code. 794 */ 795 static void 796 emit_abs_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs) 797 { 798 uint32_t bop, imsz; 799 int32_t joff; 800 801 static const uint8_t op8[] = { 802 [GET_BPF_OP(BPF_JEQ)] = 0x74, /* JE */ 803 [GET_BPF_OP(EBPF_JNE)] = 0x75, /* JNE */ 804 [GET_BPF_OP(BPF_JGT)] = 0x77, /* JA */ 805 [GET_BPF_OP(EBPF_JLT)] = 0x72, /* JB */ 806 [GET_BPF_OP(BPF_JGE)] = 0x73, /* JAE */ 807 [GET_BPF_OP(EBPF_JLE)] = 0x76, /* JBE */ 808 [GET_BPF_OP(EBPF_JSGT)] = 0x7F, /* JG */ 809 [GET_BPF_OP(EBPF_JSLT)] = 0x7C, /* JL */ 810 [GET_BPF_OP(EBPF_JSGE)] = 0x7D, /*JGE */ 811 [GET_BPF_OP(EBPF_JSLE)] = 0x7E, /* JLE */ 812 [GET_BPF_OP(BPF_JSET)] = 0x75, /*JNE */ 813 }; 814 815 static const uint8_t op32[][2] = { 816 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x84}, /* JE */ 817 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x85}, /* JNE */ 818 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x87}, /* JA */ 819 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x82}, /* JB */ 820 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x83}, /* JAE */ 821 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x86}, /* JBE */ 822 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x8F}, /* JG */ 823 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x8C}, /* JL */ 824 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x8D}, /*JGE */ 825 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x8E}, /* JLE */ 826 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x85}, /*JNE */ 827 }; 828 829 const int32_t sz8 = sizeof(op8[0]) + sizeof(uint8_t); 830 const int32_t sz32 = sizeof(op32[0]) + sizeof(uint32_t); 831 832 /* max possible jcc instruction size */ 833 const int32_t iszm = RTE_MAX(sz8, sz32); 834 835 joff = ofs - st->sz; 836 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm)); 837 838 bop = GET_BPF_OP(op); 839 840 if (imsz == 1) { 841 emit_bytes(st, &op8[bop], sizeof(op8[bop])); 842 joff -= sz8; 843 } else { 844 emit_bytes(st, op32[bop], sizeof(op32[bop])); 845 joff -= sz32; 846 } 847 848 emit_imm(st, joff, imsz); 849 } 850 851 /* 852 * emit one of: 853 * je <ofs> 854 * jne <ofs> 855 * ja <ofs> 856 * jb <ofs> 857 * jae <ofs> 858 * jbe <ofs> 859 * jg <ofs> 860 * jl <ofs> 861 * jge <ofs> 862 * jle <ofs> 863 * where 'ofs' is the target offset for the BPF bytecode. 864 */ 865 static void 866 emit_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs) 867 { 868 emit_abs_jcc(st, op, st->off[st->idx + ofs]); 869 } 870 871 872 /* 873 * emit cmp <imm>, %<dreg> 874 */ 875 static void 876 emit_cmp_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm) 877 { 878 uint8_t ops; 879 uint32_t imsz; 880 881 const uint8_t op8 = 0x83; 882 const uint8_t op32 = 0x81; 883 const uint8_t mods = 7; 884 885 imsz = imm_size(imm); 886 ops = (imsz == 1) ? op8 : op32; 887 888 emit_rex(st, op, 0, dreg); 889 emit_bytes(st, &ops, sizeof(ops)); 890 emit_modregrm(st, MOD_DIRECT, mods, dreg); 891 emit_imm(st, imm, imsz); 892 } 893 894 /* 895 * emit test <imm>, %<dreg> 896 */ 897 static void 898 emit_tst_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm) 899 { 900 const uint8_t ops = 0xF7; 901 const uint8_t mods = 0; 902 903 emit_rex(st, op, 0, dreg); 904 emit_bytes(st, &ops, sizeof(ops)); 905 emit_modregrm(st, MOD_DIRECT, mods, dreg); 906 emit_imm(st, imm, imm_size(imm)); 907 } 908 909 static void 910 emit_jcc_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, 911 uint32_t imm, int32_t ofs) 912 { 913 if (BPF_OP(op) == BPF_JSET) 914 emit_tst_imm(st, EBPF_ALU64, dreg, imm); 915 else 916 emit_cmp_imm(st, EBPF_ALU64, dreg, imm); 917 918 emit_jcc(st, op, ofs); 919 } 920 921 /* 922 * emit test %<sreg>, %<dreg> 923 */ 924 static void 925 emit_tst_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 926 uint32_t dreg) 927 { 928 const uint8_t ops = 0x85; 929 930 emit_rex(st, op, sreg, dreg); 931 emit_bytes(st, &ops, sizeof(ops)); 932 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 933 } 934 935 /* 936 * emit cmp %<sreg>, %<dreg> 937 */ 938 static void 939 emit_cmp_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 940 uint32_t dreg) 941 { 942 const uint8_t ops = 0x39; 943 944 emit_rex(st, op, sreg, dreg); 945 emit_bytes(st, &ops, sizeof(ops)); 946 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 947 948 } 949 950 static void 951 emit_jcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 952 uint32_t dreg, int32_t ofs) 953 { 954 if (BPF_OP(op) == BPF_JSET) 955 emit_tst_reg(st, EBPF_ALU64, sreg, dreg); 956 else 957 emit_cmp_reg(st, EBPF_ALU64, sreg, dreg); 958 959 emit_jcc(st, op, ofs); 960 } 961 962 /* 963 * note that rax:rdx are implicitly used as source/destination registers, 964 * so some reg spillage is necessary. 965 * emit: 966 * mov %rax, %r11 967 * mov %rdx, %r10 968 * mov %<dreg>, %rax 969 * xor %rdx, %rdx 970 * for divisor as immediate value: 971 * mov <imm>, %r9 972 * div %<divisor_reg> 973 * mov %r10, %rdx 974 * mov %rax, %<dreg> 975 * mov %r11, %rax 976 * either: 977 * mov %rax, %<dreg> 978 * OR 979 * mov %rdx, %<dreg> 980 * mov %r11, %rax 981 * mov %r10, %rdx 982 */ 983 static void 984 emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, 985 uint32_t imm) 986 { 987 uint32_t sr; 988 989 const uint8_t ops = 0xF7; 990 const uint8_t mods = 6; 991 992 if (BPF_SRC(op) == BPF_X) { 993 994 /* check that src divisor is not zero */ 995 emit_tst_reg(st, BPF_CLASS(op), sreg, sreg); 996 997 /* exit with return value zero */ 998 emit_movcc_reg(st, BPF_CLASS(op) | BPF_JEQ | BPF_X, sreg, RAX); 999 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off); 1000 } 1001 1002 /* save rax & rdx */ 1003 if (dreg != RAX) 1004 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0); 1005 if (dreg != RDX) 1006 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1); 1007 1008 /* fill rax & rdx */ 1009 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX); 1010 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, 0); 1011 1012 if (BPF_SRC(op) == BPF_X) { 1013 sr = sreg; 1014 if (sr == RAX) 1015 sr = REG_TMP0; 1016 else if (sr == RDX) 1017 sr = REG_TMP1; 1018 } else { 1019 sr = REG_DIV_IMM; 1020 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, sr, imm); 1021 } 1022 1023 emit_rex(st, op, 0, sr); 1024 emit_bytes(st, &ops, sizeof(ops)); 1025 emit_modregrm(st, MOD_DIRECT, mods, sr); 1026 1027 if (BPF_OP(op) == BPF_DIV) 1028 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg); 1029 else 1030 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, dreg); 1031 1032 if (dreg != RAX) 1033 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX); 1034 if (dreg != RDX) 1035 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX); 1036 } 1037 1038 /* 1039 * helper function, used by emit_ld_mbuf(). 1040 * generates code for 'fast_path': 1041 * calculate load offset and check is it inside first packet segment. 1042 */ 1043 static void 1044 emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7], 1045 uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm, 1046 const int32_t ofs[LDMB_OFS_NUM]) 1047 { 1048 /* make R2 contain *off* value */ 1049 1050 if (sreg != rg[EBPF_REG_2]) { 1051 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, 1052 rg[EBPF_REG_2], imm); 1053 if (mode == BPF_IND) 1054 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, 1055 sreg, rg[EBPF_REG_2]); 1056 } else 1057 /* BPF_IND with sreg == R2 */ 1058 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, 1059 rg[EBPF_REG_2], imm); 1060 1061 /* R3 = mbuf->data_len */ 1062 emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H, 1063 rg[EBPF_REG_6], rg[EBPF_REG_3], 1064 offsetof(struct rte_mbuf, data_len)); 1065 1066 /* R3 = R3 - R2 */ 1067 emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X, 1068 rg[EBPF_REG_2], rg[EBPF_REG_3]); 1069 1070 /* JSLT R3, <sz> <slow_path> */ 1071 emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz); 1072 emit_abs_jcc(st, BPF_JMP | EBPF_JSLT | BPF_K, ofs[LDMB_SLP_OFS]); 1073 1074 /* R3 = mbuf->data_off */ 1075 emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H, 1076 rg[EBPF_REG_6], rg[EBPF_REG_3], 1077 offsetof(struct rte_mbuf, data_off)); 1078 1079 /* R0 = mbuf->buf_addr */ 1080 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW, 1081 rg[EBPF_REG_6], rg[EBPF_REG_0], 1082 offsetof(struct rte_mbuf, buf_addr)); 1083 1084 /* R0 = R0 + R3 */ 1085 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, 1086 rg[EBPF_REG_3], rg[EBPF_REG_0]); 1087 1088 /* R0 = R0 + R2 */ 1089 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, 1090 rg[EBPF_REG_2], rg[EBPF_REG_0]); 1091 1092 /* JMP <fin_part> */ 1093 emit_abs_jmp(st, ofs[LDMB_FIN_OFS]); 1094 } 1095 1096 /* 1097 * helper function, used by emit_ld_mbuf(). 1098 * generates code for 'slow_path': 1099 * call __rte_pktmbuf_read() and check return value. 1100 */ 1101 static void 1102 emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7], 1103 uint32_t sz) 1104 { 1105 /* make R3 contain *len* value (1/2/4) */ 1106 1107 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz); 1108 1109 /* make R4 contain (RBP - ldmb.stack_ofs) */ 1110 1111 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]); 1112 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4], 1113 st->ldmb.stack_ofs); 1114 1115 /* make R1 contain mbuf ptr */ 1116 1117 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, 1118 rg[EBPF_REG_6], rg[EBPF_REG_1]); 1119 1120 /* call rte_pktmbuf_read */ 1121 emit_call(st, (uintptr_t)__rte_pktmbuf_read); 1122 1123 /* check that return value (R0) is not zero */ 1124 emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]); 1125 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off); 1126 } 1127 1128 /* 1129 * helper function, used by emit_ld_mbuf(). 1130 * generates final part of code for BPF_ABS/BPF_IND load: 1131 * perform data load and endianness conversion. 1132 * expects dreg to contain valid data pointer. 1133 */ 1134 static void 1135 emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz, 1136 uint32_t sz) 1137 { 1138 emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0); 1139 if (sz != sizeof(uint8_t)) 1140 emit_be2le(st, dreg, sz * CHAR_BIT); 1141 } 1142 1143 /* 1144 * emit code for BPF_ABS/BPF_IND load. 1145 * generates the following construction: 1146 * fast_path: 1147 * off = ins->sreg + ins->imm 1148 * if (mbuf->data_len - off < ins->opsz) 1149 * goto slow_path; 1150 * ptr = mbuf->buf_addr + mbuf->data_off + off; 1151 * goto fin_part; 1152 * slow_path: 1153 * typeof(ins->opsz) buf; //allocate space on the stack 1154 * ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf); 1155 * if (ptr == NULL) 1156 * goto exit_label; 1157 * fin_part: 1158 * res = *(typeof(ins->opsz))ptr; 1159 * res = bswap(res); 1160 */ 1161 static void 1162 emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t imm) 1163 { 1164 uint32_t i, mode, opsz, sz; 1165 uint32_t rg[EBPF_REG_7]; 1166 int32_t ofs[LDMB_OFS_NUM]; 1167 1168 mode = BPF_MODE(op); 1169 opsz = BPF_SIZE(op); 1170 sz = bpf_size(opsz); 1171 1172 for (i = 0; i != RTE_DIM(rg); i++) 1173 rg[i] = ebpf2x86[i]; 1174 1175 /* fill with fake offsets */ 1176 for (i = 0; i != RTE_DIM(ofs); i++) 1177 ofs[i] = st->sz + INT8_MAX; 1178 1179 /* dry run first to calculate jump offsets */ 1180 1181 ofs[LDMB_FSP_OFS] = st->sz; 1182 emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs); 1183 ofs[LDMB_SLP_OFS] = st->sz; 1184 emit_ldmb_slow_path(st, rg, sz); 1185 ofs[LDMB_FIN_OFS] = st->sz; 1186 emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz); 1187 1188 RTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX); 1189 1190 /* reset dry-run code and do a proper run */ 1191 1192 st->sz = ofs[LDMB_FSP_OFS]; 1193 emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs); 1194 emit_ldmb_slow_path(st, rg, sz); 1195 emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz); 1196 } 1197 1198 static void 1199 emit_prolog(struct bpf_jit_state *st, int32_t stack_size) 1200 { 1201 uint32_t i; 1202 int32_t spil, ofs; 1203 1204 spil = 0; 1205 for (i = 0; i != RTE_DIM(save_regs); i++) 1206 spil += INUSE(st->reguse, save_regs[i]); 1207 1208 /* we can avoid touching the stack at all */ 1209 if (spil == 0) 1210 return; 1211 1212 1213 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP, 1214 spil * sizeof(uint64_t)); 1215 1216 ofs = 0; 1217 for (i = 0; i != RTE_DIM(save_regs); i++) { 1218 if (INUSE(st->reguse, save_regs[i]) != 0) { 1219 emit_st_reg(st, BPF_STX | BPF_MEM | EBPF_DW, 1220 save_regs[i], RSP, ofs); 1221 ofs += sizeof(uint64_t); 1222 } 1223 } 1224 1225 if (INUSE(st->reguse, RBP) != 0) { 1226 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RSP, RBP); 1227 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP, stack_size); 1228 } 1229 } 1230 1231 /* 1232 * emit ret 1233 */ 1234 static void 1235 emit_ret(struct bpf_jit_state *st) 1236 { 1237 const uint8_t ops = 0xC3; 1238 1239 emit_bytes(st, &ops, sizeof(ops)); 1240 } 1241 1242 static void 1243 emit_epilog(struct bpf_jit_state *st) 1244 { 1245 uint32_t i; 1246 int32_t spil, ofs; 1247 1248 /* if we allready have an epilog generate a jump to it */ 1249 if (st->exit.num++ != 0) { 1250 emit_abs_jmp(st, st->exit.off); 1251 return; 1252 } 1253 1254 /* store offset of epilog block */ 1255 st->exit.off = st->sz; 1256 1257 spil = 0; 1258 for (i = 0; i != RTE_DIM(save_regs); i++) 1259 spil += INUSE(st->reguse, save_regs[i]); 1260 1261 if (spil != 0) { 1262 1263 if (INUSE(st->reguse, RBP) != 0) 1264 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, 1265 RBP, RSP); 1266 1267 ofs = 0; 1268 for (i = 0; i != RTE_DIM(save_regs); i++) { 1269 if (INUSE(st->reguse, save_regs[i]) != 0) { 1270 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW, 1271 RSP, save_regs[i], ofs); 1272 ofs += sizeof(uint64_t); 1273 } 1274 } 1275 1276 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, RSP, 1277 spil * sizeof(uint64_t)); 1278 } 1279 1280 emit_ret(st); 1281 } 1282 1283 /* 1284 * walk through bpf code and translate them x86_64 one. 1285 */ 1286 static int 1287 emit(struct bpf_jit_state *st, const struct rte_bpf *bpf) 1288 { 1289 uint32_t i, dr, op, sr; 1290 const struct ebpf_insn *ins; 1291 1292 /* reset state fields */ 1293 st->sz = 0; 1294 st->exit.num = 0; 1295 st->ldmb.stack_ofs = bpf->stack_sz; 1296 1297 emit_prolog(st, bpf->stack_sz); 1298 1299 for (i = 0; i != bpf->prm.nb_ins; i++) { 1300 1301 st->idx = i; 1302 st->off[i] = st->sz; 1303 1304 ins = bpf->prm.ins + i; 1305 1306 dr = ebpf2x86[ins->dst_reg]; 1307 sr = ebpf2x86[ins->src_reg]; 1308 op = ins->code; 1309 1310 switch (op) { 1311 /* 32 bit ALU IMM operations */ 1312 case (BPF_ALU | BPF_ADD | BPF_K): 1313 case (BPF_ALU | BPF_SUB | BPF_K): 1314 case (BPF_ALU | BPF_AND | BPF_K): 1315 case (BPF_ALU | BPF_OR | BPF_K): 1316 case (BPF_ALU | BPF_XOR | BPF_K): 1317 emit_alu_imm(st, op, dr, ins->imm); 1318 break; 1319 case (BPF_ALU | BPF_LSH | BPF_K): 1320 case (BPF_ALU | BPF_RSH | BPF_K): 1321 emit_shift_imm(st, op, dr, ins->imm); 1322 break; 1323 case (BPF_ALU | EBPF_MOV | BPF_K): 1324 emit_mov_imm(st, op, dr, ins->imm); 1325 break; 1326 /* 32 bit ALU REG operations */ 1327 case (BPF_ALU | BPF_ADD | BPF_X): 1328 case (BPF_ALU | BPF_SUB | BPF_X): 1329 case (BPF_ALU | BPF_AND | BPF_X): 1330 case (BPF_ALU | BPF_OR | BPF_X): 1331 case (BPF_ALU | BPF_XOR | BPF_X): 1332 emit_alu_reg(st, op, sr, dr); 1333 break; 1334 case (BPF_ALU | BPF_LSH | BPF_X): 1335 case (BPF_ALU | BPF_RSH | BPF_X): 1336 emit_shift_reg(st, op, sr, dr); 1337 break; 1338 case (BPF_ALU | EBPF_MOV | BPF_X): 1339 emit_mov_reg(st, op, sr, dr); 1340 break; 1341 case (BPF_ALU | BPF_NEG): 1342 emit_neg(st, op, dr); 1343 break; 1344 case (BPF_ALU | EBPF_END | EBPF_TO_BE): 1345 emit_be2le(st, dr, ins->imm); 1346 break; 1347 case (BPF_ALU | EBPF_END | EBPF_TO_LE): 1348 emit_le2be(st, dr, ins->imm); 1349 break; 1350 /* 64 bit ALU IMM operations */ 1351 case (EBPF_ALU64 | BPF_ADD | BPF_K): 1352 case (EBPF_ALU64 | BPF_SUB | BPF_K): 1353 case (EBPF_ALU64 | BPF_AND | BPF_K): 1354 case (EBPF_ALU64 | BPF_OR | BPF_K): 1355 case (EBPF_ALU64 | BPF_XOR | BPF_K): 1356 emit_alu_imm(st, op, dr, ins->imm); 1357 break; 1358 case (EBPF_ALU64 | BPF_LSH | BPF_K): 1359 case (EBPF_ALU64 | BPF_RSH | BPF_K): 1360 case (EBPF_ALU64 | EBPF_ARSH | BPF_K): 1361 emit_shift_imm(st, op, dr, ins->imm); 1362 break; 1363 case (EBPF_ALU64 | EBPF_MOV | BPF_K): 1364 emit_mov_imm(st, op, dr, ins->imm); 1365 break; 1366 /* 64 bit ALU REG operations */ 1367 case (EBPF_ALU64 | BPF_ADD | BPF_X): 1368 case (EBPF_ALU64 | BPF_SUB | BPF_X): 1369 case (EBPF_ALU64 | BPF_AND | BPF_X): 1370 case (EBPF_ALU64 | BPF_OR | BPF_X): 1371 case (EBPF_ALU64 | BPF_XOR | BPF_X): 1372 emit_alu_reg(st, op, sr, dr); 1373 break; 1374 case (EBPF_ALU64 | BPF_LSH | BPF_X): 1375 case (EBPF_ALU64 | BPF_RSH | BPF_X): 1376 case (EBPF_ALU64 | EBPF_ARSH | BPF_X): 1377 emit_shift_reg(st, op, sr, dr); 1378 break; 1379 case (EBPF_ALU64 | EBPF_MOV | BPF_X): 1380 emit_mov_reg(st, op, sr, dr); 1381 break; 1382 case (EBPF_ALU64 | BPF_NEG): 1383 emit_neg(st, op, dr); 1384 break; 1385 /* multiply instructions */ 1386 case (BPF_ALU | BPF_MUL | BPF_K): 1387 case (BPF_ALU | BPF_MUL | BPF_X): 1388 case (EBPF_ALU64 | BPF_MUL | BPF_K): 1389 case (EBPF_ALU64 | BPF_MUL | BPF_X): 1390 emit_mul(st, op, sr, dr, ins->imm); 1391 break; 1392 /* divide instructions */ 1393 case (BPF_ALU | BPF_DIV | BPF_K): 1394 case (BPF_ALU | BPF_MOD | BPF_K): 1395 case (BPF_ALU | BPF_DIV | BPF_X): 1396 case (BPF_ALU | BPF_MOD | BPF_X): 1397 case (EBPF_ALU64 | BPF_DIV | BPF_K): 1398 case (EBPF_ALU64 | BPF_MOD | BPF_K): 1399 case (EBPF_ALU64 | BPF_DIV | BPF_X): 1400 case (EBPF_ALU64 | BPF_MOD | BPF_X): 1401 emit_div(st, op, sr, dr, ins->imm); 1402 break; 1403 /* load instructions */ 1404 case (BPF_LDX | BPF_MEM | BPF_B): 1405 case (BPF_LDX | BPF_MEM | BPF_H): 1406 case (BPF_LDX | BPF_MEM | BPF_W): 1407 case (BPF_LDX | BPF_MEM | EBPF_DW): 1408 emit_ld_reg(st, op, sr, dr, ins->off); 1409 break; 1410 /* load 64 bit immediate value */ 1411 case (BPF_LD | BPF_IMM | EBPF_DW): 1412 emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm); 1413 i++; 1414 break; 1415 /* load absolute/indirect instructions */ 1416 case (BPF_LD | BPF_ABS | BPF_B): 1417 case (BPF_LD | BPF_ABS | BPF_H): 1418 case (BPF_LD | BPF_ABS | BPF_W): 1419 case (BPF_LD | BPF_IND | BPF_B): 1420 case (BPF_LD | BPF_IND | BPF_H): 1421 case (BPF_LD | BPF_IND | BPF_W): 1422 emit_ld_mbuf(st, op, sr, ins->imm); 1423 break; 1424 /* store instructions */ 1425 case (BPF_STX | BPF_MEM | BPF_B): 1426 case (BPF_STX | BPF_MEM | BPF_H): 1427 case (BPF_STX | BPF_MEM | BPF_W): 1428 case (BPF_STX | BPF_MEM | EBPF_DW): 1429 emit_st_reg(st, op, sr, dr, ins->off); 1430 break; 1431 case (BPF_ST | BPF_MEM | BPF_B): 1432 case (BPF_ST | BPF_MEM | BPF_H): 1433 case (BPF_ST | BPF_MEM | BPF_W): 1434 case (BPF_ST | BPF_MEM | EBPF_DW): 1435 emit_st_imm(st, op, dr, ins->imm, ins->off); 1436 break; 1437 /* atomic add instructions */ 1438 case (BPF_STX | EBPF_XADD | BPF_W): 1439 case (BPF_STX | EBPF_XADD | EBPF_DW): 1440 emit_st_xadd(st, op, sr, dr, ins->off); 1441 break; 1442 /* jump instructions */ 1443 case (BPF_JMP | BPF_JA): 1444 emit_jmp(st, ins->off + 1); 1445 break; 1446 /* jump IMM instructions */ 1447 case (BPF_JMP | BPF_JEQ | BPF_K): 1448 case (BPF_JMP | EBPF_JNE | BPF_K): 1449 case (BPF_JMP | BPF_JGT | BPF_K): 1450 case (BPF_JMP | EBPF_JLT | BPF_K): 1451 case (BPF_JMP | BPF_JGE | BPF_K): 1452 case (BPF_JMP | EBPF_JLE | BPF_K): 1453 case (BPF_JMP | EBPF_JSGT | BPF_K): 1454 case (BPF_JMP | EBPF_JSLT | BPF_K): 1455 case (BPF_JMP | EBPF_JSGE | BPF_K): 1456 case (BPF_JMP | EBPF_JSLE | BPF_K): 1457 case (BPF_JMP | BPF_JSET | BPF_K): 1458 emit_jcc_imm(st, op, dr, ins->imm, ins->off + 1); 1459 break; 1460 /* jump REG instructions */ 1461 case (BPF_JMP | BPF_JEQ | BPF_X): 1462 case (BPF_JMP | EBPF_JNE | BPF_X): 1463 case (BPF_JMP | BPF_JGT | BPF_X): 1464 case (BPF_JMP | EBPF_JLT | BPF_X): 1465 case (BPF_JMP | BPF_JGE | BPF_X): 1466 case (BPF_JMP | EBPF_JLE | BPF_X): 1467 case (BPF_JMP | EBPF_JSGT | BPF_X): 1468 case (BPF_JMP | EBPF_JSLT | BPF_X): 1469 case (BPF_JMP | EBPF_JSGE | BPF_X): 1470 case (BPF_JMP | EBPF_JSLE | BPF_X): 1471 case (BPF_JMP | BPF_JSET | BPF_X): 1472 emit_jcc_reg(st, op, sr, dr, ins->off + 1); 1473 break; 1474 /* call instructions */ 1475 case (BPF_JMP | EBPF_CALL): 1476 emit_call(st, 1477 (uintptr_t)bpf->prm.xsym[ins->imm].func.val); 1478 break; 1479 /* return instruction */ 1480 case (BPF_JMP | EBPF_EXIT): 1481 emit_epilog(st); 1482 break; 1483 default: 1484 RTE_BPF_LOG(ERR, 1485 "%s(%p): invalid opcode %#x at pc: %u;\n", 1486 __func__, bpf, ins->code, i); 1487 return -EINVAL; 1488 } 1489 } 1490 1491 return 0; 1492 } 1493 1494 /* 1495 * produce a native ISA version of the given BPF code. 1496 */ 1497 int 1498 bpf_jit_x86(struct rte_bpf *bpf) 1499 { 1500 int32_t rc; 1501 uint32_t i; 1502 size_t sz; 1503 struct bpf_jit_state st; 1504 1505 /* init state */ 1506 memset(&st, 0, sizeof(st)); 1507 st.off = malloc(bpf->prm.nb_ins * sizeof(st.off[0])); 1508 if (st.off == NULL) 1509 return -ENOMEM; 1510 1511 /* fill with fake offsets */ 1512 st.exit.off = INT32_MAX; 1513 for (i = 0; i != bpf->prm.nb_ins; i++) 1514 st.off[i] = INT32_MAX; 1515 1516 /* 1517 * dry runs, used to calculate total code size and valid jump offsets. 1518 * stop when we get minimal possible size 1519 */ 1520 do { 1521 sz = st.sz; 1522 rc = emit(&st, bpf); 1523 } while (rc == 0 && sz != st.sz); 1524 1525 if (rc == 0) { 1526 1527 /* allocate memory needed */ 1528 st.ins = mmap(NULL, st.sz, PROT_READ | PROT_WRITE, 1529 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1530 if (st.ins == MAP_FAILED) 1531 rc = -ENOMEM; 1532 else 1533 /* generate code */ 1534 rc = emit(&st, bpf); 1535 } 1536 1537 if (rc == 0 && mprotect(st.ins, st.sz, PROT_READ | PROT_EXEC) != 0) 1538 rc = -ENOMEM; 1539 1540 if (rc != 0) 1541 munmap(st.ins, st.sz); 1542 else { 1543 bpf->jit.func = (void *)st.ins; 1544 bpf->jit.sz = st.sz; 1545 } 1546 1547 free(st.off); 1548 return rc; 1549 } 1550