1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 #include <errno.h> 6 #include <stdint.h> 7 8 #include <rte_common.h> 9 #include <rte_log.h> 10 #include <rte_debug.h> 11 12 #include "bpf_impl.h" 13 14 #define GET_BPF_OP(op) (BPF_OP(op) >> 4) 15 16 enum { 17 RAX = 0, /* scratch, return value */ 18 RCX = 1, /* scratch, 4th arg */ 19 RDX = 2, /* scratch, 3rd arg */ 20 RBX = 3, /* callee saved */ 21 RSP = 4, /* stack pointer */ 22 RBP = 5, /* frame pointer, callee saved */ 23 RSI = 6, /* scratch, 2nd arg */ 24 RDI = 7, /* scratch, 1st arg */ 25 R8 = 8, /* scratch, 5th arg */ 26 R9 = 9, /* scratch, 6th arg */ 27 R10 = 10, /* scratch */ 28 R11 = 11, /* scratch */ 29 R12 = 12, /* callee saved */ 30 R13 = 13, /* callee saved */ 31 R14 = 14, /* callee saved */ 32 R15 = 15, /* callee saved */ 33 }; 34 35 #define IS_EXT_REG(r) ((r) >= R8) 36 37 enum { 38 REX_PREFIX = 0x40, /* fixed value 0100 */ 39 REX_W = 0x8, /* 64bit operand size */ 40 REX_R = 0x4, /* extension of the ModRM.reg field */ 41 REX_X = 0x2, /* extension of the SIB.index field */ 42 REX_B = 0x1, /* extension of the ModRM.rm field */ 43 }; 44 45 enum { 46 MOD_INDIRECT = 0, 47 MOD_IDISP8 = 1, 48 MOD_IDISP32 = 2, 49 MOD_DIRECT = 3, 50 }; 51 52 enum { 53 SIB_SCALE_1 = 0, 54 SIB_SCALE_2 = 1, 55 SIB_SCALE_4 = 2, 56 SIB_SCALE_8 = 3, 57 }; 58 59 /* 60 * eBPF to x86_64 register mappings. 61 */ 62 static const uint32_t ebpf2x86[] = { 63 [EBPF_REG_0] = RAX, 64 [EBPF_REG_1] = RDI, 65 [EBPF_REG_2] = RSI, 66 [EBPF_REG_3] = RDX, 67 [EBPF_REG_4] = RCX, 68 [EBPF_REG_5] = R8, 69 [EBPF_REG_6] = RBX, 70 [EBPF_REG_7] = R13, 71 [EBPF_REG_8] = R14, 72 [EBPF_REG_9] = R15, 73 [EBPF_REG_10] = RBP, 74 }; 75 76 /* 77 * r10 and r11 are used as a scratch temporary registers. 78 */ 79 enum { 80 REG_DIV_IMM = R9, 81 REG_TMP0 = R11, 82 REG_TMP1 = R10, 83 }; 84 85 /* LD_ABS/LD_IMM offsets */ 86 enum { 87 LDMB_FSP_OFS, /* fast-path */ 88 LDMB_SLP_OFS, /* slow-path */ 89 LDMB_FIN_OFS, /* final part */ 90 LDMB_OFS_NUM 91 }; 92 93 /* 94 * callee saved registers list. 95 * keep RBP as the last one. 96 */ 97 static const uint32_t save_regs[] = {RBX, R12, R13, R14, R15, RBP}; 98 99 struct bpf_jit_state { 100 uint32_t idx; 101 size_t sz; 102 struct { 103 uint32_t num; 104 int32_t off; 105 } exit; 106 struct { 107 uint32_t stack_ofs; 108 } ldmb; 109 uint32_t reguse; 110 int32_t *off; 111 uint8_t *ins; 112 }; 113 114 #define INUSE(v, r) (((v) >> (r)) & 1) 115 #define USED(v, r) ((v) |= 1 << (r)) 116 117 union bpf_jit_imm { 118 uint32_t u32; 119 uint8_t u8[4]; 120 }; 121 122 /* 123 * In many cases for imm8 we can produce shorter code. 124 */ 125 static size_t 126 imm_size(int32_t v) 127 { 128 if (v == (int8_t)v) 129 return sizeof(int8_t); 130 return sizeof(int32_t); 131 } 132 133 static void 134 emit_bytes(struct bpf_jit_state *st, const uint8_t ins[], uint32_t sz) 135 { 136 uint32_t i; 137 138 if (st->ins != NULL) { 139 for (i = 0; i != sz; i++) 140 st->ins[st->sz + i] = ins[i]; 141 } 142 st->sz += sz; 143 } 144 145 static void 146 emit_imm(struct bpf_jit_state *st, const uint32_t imm, uint32_t sz) 147 { 148 union bpf_jit_imm v; 149 150 v.u32 = imm; 151 emit_bytes(st, v.u8, sz); 152 } 153 154 /* 155 * emit REX byte 156 */ 157 static void 158 emit_rex(struct bpf_jit_state *st, uint32_t op, uint32_t reg, uint32_t rm) 159 { 160 uint8_t rex; 161 162 /* mark operand registers as used*/ 163 USED(st->reguse, reg); 164 USED(st->reguse, rm); 165 166 rex = 0; 167 if (BPF_CLASS(op) == EBPF_ALU64 || 168 op == (BPF_ST | BPF_MEM | EBPF_DW) || 169 op == (BPF_STX | BPF_MEM | EBPF_DW) || 170 op == (BPF_STX | EBPF_XADD | EBPF_DW) || 171 op == (BPF_LD | BPF_IMM | EBPF_DW) || 172 (BPF_CLASS(op) == BPF_LDX && 173 BPF_MODE(op) == BPF_MEM && 174 BPF_SIZE(op) != BPF_W)) 175 rex |= REX_W; 176 177 if (IS_EXT_REG(reg)) 178 rex |= REX_R; 179 180 if (IS_EXT_REG(rm)) 181 rex |= REX_B; 182 183 /* store using SIL, DIL */ 184 if (op == (BPF_STX | BPF_MEM | BPF_B) && (reg == RDI || reg == RSI)) 185 rex |= REX_PREFIX; 186 187 if (rex != 0) { 188 rex |= REX_PREFIX; 189 emit_bytes(st, &rex, sizeof(rex)); 190 } 191 } 192 193 /* 194 * emit MODRegRM byte 195 */ 196 static void 197 emit_modregrm(struct bpf_jit_state *st, uint32_t mod, uint32_t reg, uint32_t rm) 198 { 199 uint8_t v; 200 201 v = mod << 6 | (reg & 7) << 3 | (rm & 7); 202 emit_bytes(st, &v, sizeof(v)); 203 } 204 205 /* 206 * emit SIB byte 207 */ 208 static void 209 emit_sib(struct bpf_jit_state *st, uint32_t scale, uint32_t idx, uint32_t base) 210 { 211 uint8_t v; 212 213 v = scale << 6 | (idx & 7) << 3 | (base & 7); 214 emit_bytes(st, &v, sizeof(v)); 215 } 216 217 /* 218 * emit OPCODE+REGIDX byte 219 */ 220 static void 221 emit_opcode(struct bpf_jit_state *st, uint8_t ops, uint32_t reg) 222 { 223 uint8_t v; 224 225 v = ops | (reg & 7); 226 emit_bytes(st, &v, sizeof(v)); 227 } 228 229 230 /* 231 * emit xchg %<sreg>, %<dreg> 232 */ 233 static void 234 emit_xchg_reg(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg) 235 { 236 const uint8_t ops = 0x87; 237 238 emit_rex(st, EBPF_ALU64, sreg, dreg); 239 emit_bytes(st, &ops, sizeof(ops)); 240 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 241 } 242 243 /* 244 * emit neg %<dreg> 245 */ 246 static void 247 emit_neg(struct bpf_jit_state *st, uint32_t op, uint32_t dreg) 248 { 249 const uint8_t ops = 0xF7; 250 const uint8_t mods = 3; 251 252 emit_rex(st, op, 0, dreg); 253 emit_bytes(st, &ops, sizeof(ops)); 254 emit_modregrm(st, MOD_DIRECT, mods, dreg); 255 } 256 257 /* 258 * emit mov %<sreg>, %<dreg> 259 */ 260 static void 261 emit_mov_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 262 uint32_t dreg) 263 { 264 const uint8_t ops = 0x89; 265 266 /* if operands are 32-bit, then it can be used to clear upper 32-bit */ 267 if (sreg != dreg || BPF_CLASS(op) == BPF_ALU) { 268 emit_rex(st, op, sreg, dreg); 269 emit_bytes(st, &ops, sizeof(ops)); 270 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 271 } 272 } 273 274 /* 275 * emit movzwl %<sreg>, %<dreg> 276 */ 277 static void 278 emit_movzwl(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg) 279 { 280 static const uint8_t ops[] = {0x0F, 0xB7}; 281 282 emit_rex(st, BPF_ALU, sreg, dreg); 283 emit_bytes(st, ops, sizeof(ops)); 284 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 285 } 286 287 /* 288 * emit ror <imm8>, %<dreg> 289 */ 290 static void 291 emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm) 292 { 293 const uint8_t prfx = 0x66; 294 const uint8_t ops = 0xC1; 295 const uint8_t mods = 1; 296 297 emit_bytes(st, &prfx, sizeof(prfx)); 298 emit_rex(st, BPF_ALU, 0, dreg); 299 emit_bytes(st, &ops, sizeof(ops)); 300 emit_modregrm(st, MOD_DIRECT, mods, dreg); 301 emit_imm(st, imm, imm_size(imm)); 302 } 303 304 /* 305 * emit bswap %<dreg> 306 */ 307 static void 308 emit_be2le_48(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm) 309 { 310 uint32_t rop; 311 312 const uint8_t ops = 0x0F; 313 const uint8_t mods = 1; 314 315 rop = (imm == 64) ? EBPF_ALU64 : BPF_ALU; 316 emit_rex(st, rop, 0, dreg); 317 emit_bytes(st, &ops, sizeof(ops)); 318 emit_modregrm(st, MOD_DIRECT, mods, dreg); 319 } 320 321 static void 322 emit_be2le(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm) 323 { 324 if (imm == 16) { 325 emit_ror_imm(st, dreg, 8); 326 emit_movzwl(st, dreg, dreg); 327 } else 328 emit_be2le_48(st, dreg, imm); 329 } 330 331 /* 332 * In general it is NOP for x86. 333 * Just clear the upper bits. 334 */ 335 static void 336 emit_le2be(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm) 337 { 338 if (imm == 16) 339 emit_movzwl(st, dreg, dreg); 340 else if (imm == 32) 341 emit_mov_reg(st, BPF_ALU | EBPF_MOV | BPF_X, dreg, dreg); 342 } 343 344 /* 345 * emit one of: 346 * add <imm>, %<dreg> 347 * and <imm>, %<dreg> 348 * or <imm>, %<dreg> 349 * sub <imm>, %<dreg> 350 * xor <imm>, %<dreg> 351 */ 352 static void 353 emit_alu_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm) 354 { 355 uint8_t mod, opcode; 356 uint32_t bop, imsz; 357 358 const uint8_t op8 = 0x83; 359 const uint8_t op32 = 0x81; 360 static const uint8_t mods[] = { 361 [GET_BPF_OP(BPF_ADD)] = 0, 362 [GET_BPF_OP(BPF_AND)] = 4, 363 [GET_BPF_OP(BPF_OR)] = 1, 364 [GET_BPF_OP(BPF_SUB)] = 5, 365 [GET_BPF_OP(BPF_XOR)] = 6, 366 }; 367 368 bop = GET_BPF_OP(op); 369 mod = mods[bop]; 370 371 imsz = imm_size(imm); 372 opcode = (imsz == 1) ? op8 : op32; 373 374 emit_rex(st, op, 0, dreg); 375 emit_bytes(st, &opcode, sizeof(opcode)); 376 emit_modregrm(st, MOD_DIRECT, mod, dreg); 377 emit_imm(st, imm, imsz); 378 } 379 380 /* 381 * emit one of: 382 * add %<sreg>, %<dreg> 383 * and %<sreg>, %<dreg> 384 * or %<sreg>, %<dreg> 385 * sub %<sreg>, %<dreg> 386 * xor %<sreg>, %<dreg> 387 */ 388 static void 389 emit_alu_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 390 uint32_t dreg) 391 { 392 uint32_t bop; 393 394 static const uint8_t ops[] = { 395 [GET_BPF_OP(BPF_ADD)] = 0x01, 396 [GET_BPF_OP(BPF_AND)] = 0x21, 397 [GET_BPF_OP(BPF_OR)] = 0x09, 398 [GET_BPF_OP(BPF_SUB)] = 0x29, 399 [GET_BPF_OP(BPF_XOR)] = 0x31, 400 }; 401 402 bop = GET_BPF_OP(op); 403 404 emit_rex(st, op, sreg, dreg); 405 emit_bytes(st, &ops[bop], sizeof(ops[bop])); 406 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 407 } 408 409 static void 410 emit_shift(struct bpf_jit_state *st, uint32_t op, uint32_t dreg) 411 { 412 uint8_t mod; 413 uint32_t bop, opx; 414 415 static const uint8_t ops[] = {0xC1, 0xD3}; 416 static const uint8_t mods[] = { 417 [GET_BPF_OP(BPF_LSH)] = 4, 418 [GET_BPF_OP(BPF_RSH)] = 5, 419 [GET_BPF_OP(EBPF_ARSH)] = 7, 420 }; 421 422 bop = GET_BPF_OP(op); 423 mod = mods[bop]; 424 opx = (BPF_SRC(op) == BPF_X); 425 426 emit_rex(st, op, 0, dreg); 427 emit_bytes(st, &ops[opx], sizeof(ops[opx])); 428 emit_modregrm(st, MOD_DIRECT, mod, dreg); 429 } 430 431 /* 432 * emit one of: 433 * shl <imm>, %<dreg> 434 * shr <imm>, %<dreg> 435 * sar <imm>, %<dreg> 436 */ 437 static void 438 emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, 439 uint32_t imm) 440 { 441 emit_shift(st, op, dreg); 442 emit_imm(st, imm, imm_size(imm)); 443 } 444 445 /* 446 * emit one of: 447 * shl %<dreg> 448 * shr %<dreg> 449 * sar %<dreg> 450 * note that rcx is implicitly used as a source register, so few extra 451 * instructions for register spillage might be necessary. 452 */ 453 static void 454 emit_shift_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 455 uint32_t dreg) 456 { 457 if (sreg != RCX) 458 emit_xchg_reg(st, RCX, sreg); 459 460 emit_shift(st, op, (dreg == RCX) ? sreg : dreg); 461 462 if (sreg != RCX) 463 emit_xchg_reg(st, RCX, sreg); 464 } 465 466 /* 467 * emit mov <imm>, %<dreg> 468 */ 469 static void 470 emit_mov_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm) 471 { 472 const uint8_t ops = 0xC7; 473 474 if (imm == 0) { 475 /* replace 'mov 0, %<dst>' with 'xor %<dst>, %<dst>' */ 476 op = BPF_CLASS(op) | BPF_XOR | BPF_X; 477 emit_alu_reg(st, op, dreg, dreg); 478 return; 479 } 480 481 emit_rex(st, op, 0, dreg); 482 emit_bytes(st, &ops, sizeof(ops)); 483 emit_modregrm(st, MOD_DIRECT, 0, dreg); 484 emit_imm(st, imm, sizeof(imm)); 485 } 486 487 /* 488 * emit mov <imm64>, %<dreg> 489 */ 490 static void 491 emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0, 492 uint32_t imm1) 493 { 494 uint32_t op; 495 496 const uint8_t ops = 0xB8; 497 498 op = (imm1 == 0) ? BPF_ALU : EBPF_ALU64; 499 500 emit_rex(st, op, 0, dreg); 501 emit_opcode(st, ops, dreg); 502 503 emit_imm(st, imm0, sizeof(imm0)); 504 if (imm1 != 0) 505 emit_imm(st, imm1, sizeof(imm1)); 506 } 507 508 /* 509 * note that rax:rdx are implicitly used as source/destination registers, 510 * so some reg spillage is necessary. 511 * emit: 512 * mov %rax, %r11 513 * mov %rdx, %r10 514 * mov %<dreg>, %rax 515 * either: 516 * mov %<sreg>, %rdx 517 * OR 518 * mov <imm>, %rdx 519 * mul %rdx 520 * mov %r10, %rdx 521 * mov %rax, %<dreg> 522 * mov %r11, %rax 523 */ 524 static void 525 emit_mul(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, 526 uint32_t imm) 527 { 528 const uint8_t ops = 0xF7; 529 const uint8_t mods = 4; 530 531 /* save rax & rdx */ 532 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0); 533 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1); 534 535 /* rax = dreg */ 536 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX); 537 538 if (BPF_SRC(op) == BPF_X) 539 /* rdx = sreg */ 540 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, 541 sreg == RAX ? REG_TMP0 : sreg, RDX); 542 else 543 /* rdx = imm */ 544 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, imm); 545 546 emit_rex(st, op, RAX, RDX); 547 emit_bytes(st, &ops, sizeof(ops)); 548 emit_modregrm(st, MOD_DIRECT, mods, RDX); 549 550 if (dreg != RDX) 551 /* restore rdx */ 552 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX); 553 554 if (dreg != RAX) { 555 /* dreg = rax */ 556 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg); 557 /* restore rax */ 558 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX); 559 } 560 } 561 562 /* 563 * emit mov <ofs>(%<sreg>), %<dreg> 564 * note that for non 64-bit ops, higher bits have to be cleared. 565 */ 566 static void 567 emit_ld_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, 568 int32_t ofs) 569 { 570 uint32_t mods, opsz; 571 const uint8_t op32 = 0x8B; 572 const uint8_t op16[] = {0x0F, 0xB7}; 573 const uint8_t op8[] = {0x0F, 0xB6}; 574 575 emit_rex(st, op, dreg, sreg); 576 577 opsz = BPF_SIZE(op); 578 if (opsz == BPF_B) 579 emit_bytes(st, op8, sizeof(op8)); 580 else if (opsz == BPF_H) 581 emit_bytes(st, op16, sizeof(op16)); 582 else 583 emit_bytes(st, &op32, sizeof(op32)); 584 585 mods = (imm_size(ofs) == 1) ? MOD_IDISP8 : MOD_IDISP32; 586 587 emit_modregrm(st, mods, dreg, sreg); 588 if (sreg == RSP || sreg == R12) 589 emit_sib(st, SIB_SCALE_1, sreg, sreg); 590 emit_imm(st, ofs, imm_size(ofs)); 591 } 592 593 /* 594 * emit one of: 595 * mov %<sreg>, <ofs>(%<dreg>) 596 * mov <imm>, <ofs>(%<dreg>) 597 */ 598 static void 599 emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 600 uint32_t dreg, uint32_t imm, int32_t ofs) 601 { 602 uint32_t mods, imsz, opsz, opx; 603 const uint8_t prfx16 = 0x66; 604 605 /* 8 bit instruction opcodes */ 606 static const uint8_t op8[] = {0xC6, 0x88}; 607 608 /* 16/32/64 bit instruction opcodes */ 609 static const uint8_t ops[] = {0xC7, 0x89}; 610 611 /* is the instruction has immediate value or src reg? */ 612 opx = (BPF_CLASS(op) == BPF_STX); 613 614 opsz = BPF_SIZE(op); 615 if (opsz == BPF_H) 616 emit_bytes(st, &prfx16, sizeof(prfx16)); 617 618 emit_rex(st, op, sreg, dreg); 619 620 if (opsz == BPF_B) 621 emit_bytes(st, &op8[opx], sizeof(op8[opx])); 622 else 623 emit_bytes(st, &ops[opx], sizeof(ops[opx])); 624 625 imsz = imm_size(ofs); 626 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32; 627 628 emit_modregrm(st, mods, sreg, dreg); 629 630 if (dreg == RSP || dreg == R12) 631 emit_sib(st, SIB_SCALE_1, dreg, dreg); 632 633 emit_imm(st, ofs, imsz); 634 635 if (opx == 0) { 636 imsz = RTE_MIN(bpf_size(opsz), sizeof(imm)); 637 emit_imm(st, imm, imsz); 638 } 639 } 640 641 static void 642 emit_st_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm, 643 int32_t ofs) 644 { 645 emit_st_common(st, op, 0, dreg, imm, ofs); 646 } 647 648 static void 649 emit_st_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, 650 int32_t ofs) 651 { 652 emit_st_common(st, op, sreg, dreg, 0, ofs); 653 } 654 655 /* 656 * emit lock add %<sreg>, <ofs>(%<dreg>) 657 */ 658 static void 659 emit_st_xadd(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 660 uint32_t dreg, int32_t ofs) 661 { 662 uint32_t imsz, mods; 663 664 const uint8_t lck = 0xF0; /* lock prefix */ 665 const uint8_t ops = 0x01; /* add opcode */ 666 667 imsz = imm_size(ofs); 668 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32; 669 670 emit_bytes(st, &lck, sizeof(lck)); 671 emit_rex(st, op, sreg, dreg); 672 emit_bytes(st, &ops, sizeof(ops)); 673 emit_modregrm(st, mods, sreg, dreg); 674 emit_imm(st, ofs, imsz); 675 } 676 677 /* 678 * emit: 679 * mov <imm64>, (%rax) 680 * call *%rax 681 */ 682 static void 683 emit_call(struct bpf_jit_state *st, uintptr_t trg) 684 { 685 const uint8_t ops = 0xFF; 686 const uint8_t mods = 2; 687 688 emit_ld_imm64(st, RAX, trg, trg >> 32); 689 emit_bytes(st, &ops, sizeof(ops)); 690 emit_modregrm(st, MOD_DIRECT, mods, RAX); 691 } 692 693 /* 694 * emit jmp <ofs> 695 * where 'ofs' is the target offset for the native code. 696 */ 697 static void 698 emit_abs_jmp(struct bpf_jit_state *st, int32_t ofs) 699 { 700 int32_t joff; 701 uint32_t imsz; 702 703 const uint8_t op8 = 0xEB; 704 const uint8_t op32 = 0xE9; 705 706 const int32_t sz8 = sizeof(op8) + sizeof(uint8_t); 707 const int32_t sz32 = sizeof(op32) + sizeof(uint32_t); 708 709 /* max possible jmp instruction size */ 710 const int32_t iszm = RTE_MAX(sz8, sz32); 711 712 joff = ofs - st->sz; 713 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm)); 714 715 if (imsz == 1) { 716 emit_bytes(st, &op8, sizeof(op8)); 717 joff -= sz8; 718 } else { 719 emit_bytes(st, &op32, sizeof(op32)); 720 joff -= sz32; 721 } 722 723 emit_imm(st, joff, imsz); 724 } 725 726 /* 727 * emit jmp <ofs> 728 * where 'ofs' is the target offset for the BPF bytecode. 729 */ 730 static void 731 emit_jmp(struct bpf_jit_state *st, int32_t ofs) 732 { 733 emit_abs_jmp(st, st->off[st->idx + ofs]); 734 } 735 736 /* 737 * emit one of: 738 * cmovz %<sreg>, <%dreg> 739 * cmovne %<sreg>, <%dreg> 740 * cmova %<sreg>, <%dreg> 741 * cmovb %<sreg>, <%dreg> 742 * cmovae %<sreg>, <%dreg> 743 * cmovbe %<sreg>, <%dreg> 744 * cmovg %<sreg>, <%dreg> 745 * cmovl %<sreg>, <%dreg> 746 * cmovge %<sreg>, <%dreg> 747 * cmovle %<sreg>, <%dreg> 748 */ 749 static void 750 emit_movcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 751 uint32_t dreg) 752 { 753 uint32_t bop; 754 755 static const uint8_t ops[][2] = { 756 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x44}, /* CMOVZ */ 757 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x45}, /* CMOVNE */ 758 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x47}, /* CMOVA */ 759 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x42}, /* CMOVB */ 760 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x43}, /* CMOVAE */ 761 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x46}, /* CMOVBE */ 762 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x4F}, /* CMOVG */ 763 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x4C}, /* CMOVL */ 764 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x4D}, /* CMOVGE */ 765 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x4E}, /* CMOVLE */ 766 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x45}, /* CMOVNE */ 767 }; 768 769 bop = GET_BPF_OP(op); 770 771 emit_rex(st, op, dreg, sreg); 772 emit_bytes(st, ops[bop], sizeof(ops[bop])); 773 emit_modregrm(st, MOD_DIRECT, dreg, sreg); 774 } 775 776 /* 777 * emit one of: 778 * je <ofs> 779 * jne <ofs> 780 * ja <ofs> 781 * jb <ofs> 782 * jae <ofs> 783 * jbe <ofs> 784 * jg <ofs> 785 * jl <ofs> 786 * jge <ofs> 787 * jle <ofs> 788 * where 'ofs' is the target offset for the native code. 789 */ 790 static void 791 emit_abs_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs) 792 { 793 uint32_t bop, imsz; 794 int32_t joff; 795 796 static const uint8_t op8[] = { 797 [GET_BPF_OP(BPF_JEQ)] = 0x74, /* JE */ 798 [GET_BPF_OP(EBPF_JNE)] = 0x75, /* JNE */ 799 [GET_BPF_OP(BPF_JGT)] = 0x77, /* JA */ 800 [GET_BPF_OP(EBPF_JLT)] = 0x72, /* JB */ 801 [GET_BPF_OP(BPF_JGE)] = 0x73, /* JAE */ 802 [GET_BPF_OP(EBPF_JLE)] = 0x76, /* JBE */ 803 [GET_BPF_OP(EBPF_JSGT)] = 0x7F, /* JG */ 804 [GET_BPF_OP(EBPF_JSLT)] = 0x7C, /* JL */ 805 [GET_BPF_OP(EBPF_JSGE)] = 0x7D, /*JGE */ 806 [GET_BPF_OP(EBPF_JSLE)] = 0x7E, /* JLE */ 807 [GET_BPF_OP(BPF_JSET)] = 0x75, /*JNE */ 808 }; 809 810 static const uint8_t op32[][2] = { 811 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x84}, /* JE */ 812 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x85}, /* JNE */ 813 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x87}, /* JA */ 814 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x82}, /* JB */ 815 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x83}, /* JAE */ 816 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x86}, /* JBE */ 817 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x8F}, /* JG */ 818 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x8C}, /* JL */ 819 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x8D}, /*JGE */ 820 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x8E}, /* JLE */ 821 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x85}, /*JNE */ 822 }; 823 824 const int32_t sz8 = sizeof(op8[0]) + sizeof(uint8_t); 825 const int32_t sz32 = sizeof(op32[0]) + sizeof(uint32_t); 826 827 /* max possible jcc instruction size */ 828 const int32_t iszm = RTE_MAX(sz8, sz32); 829 830 joff = ofs - st->sz; 831 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm)); 832 833 bop = GET_BPF_OP(op); 834 835 if (imsz == 1) { 836 emit_bytes(st, &op8[bop], sizeof(op8[bop])); 837 joff -= sz8; 838 } else { 839 emit_bytes(st, op32[bop], sizeof(op32[bop])); 840 joff -= sz32; 841 } 842 843 emit_imm(st, joff, imsz); 844 } 845 846 /* 847 * emit one of: 848 * je <ofs> 849 * jne <ofs> 850 * ja <ofs> 851 * jb <ofs> 852 * jae <ofs> 853 * jbe <ofs> 854 * jg <ofs> 855 * jl <ofs> 856 * jge <ofs> 857 * jle <ofs> 858 * where 'ofs' is the target offset for the BPF bytecode. 859 */ 860 static void 861 emit_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs) 862 { 863 emit_abs_jcc(st, op, st->off[st->idx + ofs]); 864 } 865 866 867 /* 868 * emit cmp <imm>, %<dreg> 869 */ 870 static void 871 emit_cmp_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm) 872 { 873 uint8_t ops; 874 uint32_t imsz; 875 876 const uint8_t op8 = 0x83; 877 const uint8_t op32 = 0x81; 878 const uint8_t mods = 7; 879 880 imsz = imm_size(imm); 881 ops = (imsz == 1) ? op8 : op32; 882 883 emit_rex(st, op, 0, dreg); 884 emit_bytes(st, &ops, sizeof(ops)); 885 emit_modregrm(st, MOD_DIRECT, mods, dreg); 886 emit_imm(st, imm, imsz); 887 } 888 889 /* 890 * emit test <imm>, %<dreg> 891 */ 892 static void 893 emit_tst_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm) 894 { 895 const uint8_t ops = 0xF7; 896 const uint8_t mods = 0; 897 898 emit_rex(st, op, 0, dreg); 899 emit_bytes(st, &ops, sizeof(ops)); 900 emit_modregrm(st, MOD_DIRECT, mods, dreg); 901 emit_imm(st, imm, imm_size(imm)); 902 } 903 904 static void 905 emit_jcc_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, 906 uint32_t imm, int32_t ofs) 907 { 908 if (BPF_OP(op) == BPF_JSET) 909 emit_tst_imm(st, EBPF_ALU64, dreg, imm); 910 else 911 emit_cmp_imm(st, EBPF_ALU64, dreg, imm); 912 913 emit_jcc(st, op, ofs); 914 } 915 916 /* 917 * emit test %<sreg>, %<dreg> 918 */ 919 static void 920 emit_tst_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 921 uint32_t dreg) 922 { 923 const uint8_t ops = 0x85; 924 925 emit_rex(st, op, sreg, dreg); 926 emit_bytes(st, &ops, sizeof(ops)); 927 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 928 } 929 930 /* 931 * emit cmp %<sreg>, %<dreg> 932 */ 933 static void 934 emit_cmp_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 935 uint32_t dreg) 936 { 937 const uint8_t ops = 0x39; 938 939 emit_rex(st, op, sreg, dreg); 940 emit_bytes(st, &ops, sizeof(ops)); 941 emit_modregrm(st, MOD_DIRECT, sreg, dreg); 942 943 } 944 945 static void 946 emit_jcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, 947 uint32_t dreg, int32_t ofs) 948 { 949 if (BPF_OP(op) == BPF_JSET) 950 emit_tst_reg(st, EBPF_ALU64, sreg, dreg); 951 else 952 emit_cmp_reg(st, EBPF_ALU64, sreg, dreg); 953 954 emit_jcc(st, op, ofs); 955 } 956 957 /* 958 * note that rax:rdx are implicitly used as source/destination registers, 959 * so some reg spillage is necessary. 960 * emit: 961 * mov %rax, %r11 962 * mov %rdx, %r10 963 * mov %<dreg>, %rax 964 * xor %rdx, %rdx 965 * for divisor as immediate value: 966 * mov <imm>, %r9 967 * div %<divisor_reg> 968 * mov %r10, %rdx 969 * mov %rax, %<dreg> 970 * mov %r11, %rax 971 * either: 972 * mov %rax, %<dreg> 973 * OR 974 * mov %rdx, %<dreg> 975 * mov %r11, %rax 976 * mov %r10, %rdx 977 */ 978 static void 979 emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, 980 uint32_t imm) 981 { 982 uint32_t sr; 983 984 const uint8_t ops = 0xF7; 985 const uint8_t mods = 6; 986 987 if (BPF_SRC(op) == BPF_X) { 988 989 /* check that src divisor is not zero */ 990 emit_tst_reg(st, BPF_CLASS(op), sreg, sreg); 991 992 /* exit with return value zero */ 993 emit_movcc_reg(st, BPF_CLASS(op) | BPF_JEQ | BPF_X, sreg, RAX); 994 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off); 995 } 996 997 /* save rax & rdx */ 998 if (dreg != RAX) 999 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0); 1000 if (dreg != RDX) 1001 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1); 1002 1003 /* fill rax & rdx */ 1004 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX); 1005 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, 0); 1006 1007 if (BPF_SRC(op) == BPF_X) { 1008 sr = sreg; 1009 if (sr == RAX) 1010 sr = REG_TMP0; 1011 else if (sr == RDX) 1012 sr = REG_TMP1; 1013 } else { 1014 sr = REG_DIV_IMM; 1015 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, sr, imm); 1016 } 1017 1018 emit_rex(st, op, 0, sr); 1019 emit_bytes(st, &ops, sizeof(ops)); 1020 emit_modregrm(st, MOD_DIRECT, mods, sr); 1021 1022 if (BPF_OP(op) == BPF_DIV) 1023 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg); 1024 else 1025 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, dreg); 1026 1027 if (dreg != RAX) 1028 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX); 1029 if (dreg != RDX) 1030 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX); 1031 } 1032 1033 /* 1034 * helper function, used by emit_ld_mbuf(). 1035 * generates code for 'fast_path': 1036 * calculate load offset and check is it inside first packet segment. 1037 */ 1038 static void 1039 emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7], 1040 uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm, 1041 const int32_t ofs[LDMB_OFS_NUM]) 1042 { 1043 /* make R2 contain *off* value */ 1044 1045 if (sreg != rg[EBPF_REG_2]) { 1046 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, 1047 rg[EBPF_REG_2], imm); 1048 if (mode == BPF_IND) 1049 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, 1050 sreg, rg[EBPF_REG_2]); 1051 } else 1052 /* BPF_IND with sreg == R2 */ 1053 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, 1054 rg[EBPF_REG_2], imm); 1055 1056 /* R3 = mbuf->data_len */ 1057 emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H, 1058 rg[EBPF_REG_6], rg[EBPF_REG_3], 1059 offsetof(struct rte_mbuf, data_len)); 1060 1061 /* R3 = R3 - R2 */ 1062 emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X, 1063 rg[EBPF_REG_2], rg[EBPF_REG_3]); 1064 1065 /* JSLT R3, <sz> <slow_path> */ 1066 emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz); 1067 emit_abs_jcc(st, BPF_JMP | EBPF_JSLT | BPF_K, ofs[LDMB_SLP_OFS]); 1068 1069 /* R3 = mbuf->data_off */ 1070 emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H, 1071 rg[EBPF_REG_6], rg[EBPF_REG_3], 1072 offsetof(struct rte_mbuf, data_off)); 1073 1074 /* R0 = mbuf->buf_addr */ 1075 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW, 1076 rg[EBPF_REG_6], rg[EBPF_REG_0], 1077 offsetof(struct rte_mbuf, buf_addr)); 1078 1079 /* R0 = R0 + R3 */ 1080 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, 1081 rg[EBPF_REG_3], rg[EBPF_REG_0]); 1082 1083 /* R0 = R0 + R2 */ 1084 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, 1085 rg[EBPF_REG_2], rg[EBPF_REG_0]); 1086 1087 /* JMP <fin_part> */ 1088 emit_abs_jmp(st, ofs[LDMB_FIN_OFS]); 1089 } 1090 1091 /* 1092 * helper function, used by emit_ld_mbuf(). 1093 * generates code for 'slow_path': 1094 * call __rte_pktmbuf_read() and check return value. 1095 */ 1096 static void 1097 emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7], 1098 uint32_t sz) 1099 { 1100 /* make R3 contain *len* value (1/2/4) */ 1101 1102 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz); 1103 1104 /* make R4 contain (RBP - ldmb.stack_ofs) */ 1105 1106 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]); 1107 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4], 1108 st->ldmb.stack_ofs); 1109 1110 /* make R1 contain mbuf ptr */ 1111 1112 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, 1113 rg[EBPF_REG_6], rg[EBPF_REG_1]); 1114 1115 /* call rte_pktmbuf_read */ 1116 emit_call(st, (uintptr_t)__rte_pktmbuf_read); 1117 1118 /* check that return value (R0) is not zero */ 1119 emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]); 1120 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off); 1121 } 1122 1123 /* 1124 * helper function, used by emit_ld_mbuf(). 1125 * generates final part of code for BPF_ABS/BPF_IND load: 1126 * perform data load and endianness conversion. 1127 * expects dreg to contain valid data pointer. 1128 */ 1129 static void 1130 emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz, 1131 uint32_t sz) 1132 { 1133 emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0); 1134 if (sz != sizeof(uint8_t)) 1135 emit_be2le(st, dreg, sz * CHAR_BIT); 1136 } 1137 1138 /* 1139 * emit code for BPF_ABS/BPF_IND load. 1140 * generates the following construction: 1141 * fast_path: 1142 * off = ins->sreg + ins->imm 1143 * if (mbuf->data_len - off < ins->opsz) 1144 * goto slow_path; 1145 * ptr = mbuf->buf_addr + mbuf->data_off + off; 1146 * goto fin_part; 1147 * slow_path: 1148 * typeof(ins->opsz) buf; //allocate space on the stack 1149 * ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf); 1150 * if (ptr == NULL) 1151 * goto exit_label; 1152 * fin_part: 1153 * res = *(typeof(ins->opsz))ptr; 1154 * res = bswap(res); 1155 */ 1156 static void 1157 emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t imm) 1158 { 1159 uint32_t i, mode, opsz, sz; 1160 uint32_t rg[EBPF_REG_7]; 1161 int32_t ofs[LDMB_OFS_NUM]; 1162 1163 mode = BPF_MODE(op); 1164 opsz = BPF_SIZE(op); 1165 sz = bpf_size(opsz); 1166 1167 for (i = 0; i != RTE_DIM(rg); i++) 1168 rg[i] = ebpf2x86[i]; 1169 1170 /* fill with fake offsets */ 1171 for (i = 0; i != RTE_DIM(ofs); i++) 1172 ofs[i] = st->sz + INT8_MAX; 1173 1174 /* dry run first to calculate jump offsets */ 1175 1176 ofs[LDMB_FSP_OFS] = st->sz; 1177 emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs); 1178 ofs[LDMB_SLP_OFS] = st->sz; 1179 emit_ldmb_slow_path(st, rg, sz); 1180 ofs[LDMB_FIN_OFS] = st->sz; 1181 emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz); 1182 1183 RTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX); 1184 1185 /* reset dry-run code and do a proper run */ 1186 1187 st->sz = ofs[LDMB_FSP_OFS]; 1188 emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs); 1189 emit_ldmb_slow_path(st, rg, sz); 1190 emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz); 1191 } 1192 1193 static void 1194 emit_prolog(struct bpf_jit_state *st, int32_t stack_size) 1195 { 1196 uint32_t i; 1197 int32_t spil, ofs; 1198 1199 spil = 0; 1200 for (i = 0; i != RTE_DIM(save_regs); i++) 1201 spil += INUSE(st->reguse, save_regs[i]); 1202 1203 /* we can avoid touching the stack at all */ 1204 if (spil == 0) 1205 return; 1206 1207 1208 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP, 1209 spil * sizeof(uint64_t)); 1210 1211 ofs = 0; 1212 for (i = 0; i != RTE_DIM(save_regs); i++) { 1213 if (INUSE(st->reguse, save_regs[i]) != 0) { 1214 emit_st_reg(st, BPF_STX | BPF_MEM | EBPF_DW, 1215 save_regs[i], RSP, ofs); 1216 ofs += sizeof(uint64_t); 1217 } 1218 } 1219 1220 if (INUSE(st->reguse, RBP) != 0) { 1221 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RSP, RBP); 1222 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP, stack_size); 1223 } 1224 } 1225 1226 /* 1227 * emit ret 1228 */ 1229 static void 1230 emit_ret(struct bpf_jit_state *st) 1231 { 1232 const uint8_t ops = 0xC3; 1233 1234 emit_bytes(st, &ops, sizeof(ops)); 1235 } 1236 1237 static void 1238 emit_epilog(struct bpf_jit_state *st) 1239 { 1240 uint32_t i; 1241 int32_t spil, ofs; 1242 1243 /* if we already have an epilog generate a jump to it */ 1244 if (st->exit.num++ != 0) { 1245 emit_abs_jmp(st, st->exit.off); 1246 return; 1247 } 1248 1249 /* store offset of epilog block */ 1250 st->exit.off = st->sz; 1251 1252 spil = 0; 1253 for (i = 0; i != RTE_DIM(save_regs); i++) 1254 spil += INUSE(st->reguse, save_regs[i]); 1255 1256 if (spil != 0) { 1257 1258 if (INUSE(st->reguse, RBP) != 0) 1259 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, 1260 RBP, RSP); 1261 1262 ofs = 0; 1263 for (i = 0; i != RTE_DIM(save_regs); i++) { 1264 if (INUSE(st->reguse, save_regs[i]) != 0) { 1265 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW, 1266 RSP, save_regs[i], ofs); 1267 ofs += sizeof(uint64_t); 1268 } 1269 } 1270 1271 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, RSP, 1272 spil * sizeof(uint64_t)); 1273 } 1274 1275 emit_ret(st); 1276 } 1277 1278 /* 1279 * walk through bpf code and translate them x86_64 one. 1280 */ 1281 static int 1282 emit(struct bpf_jit_state *st, const struct rte_bpf *bpf) 1283 { 1284 uint32_t i, dr, op, sr; 1285 const struct ebpf_insn *ins; 1286 1287 /* reset state fields */ 1288 st->sz = 0; 1289 st->exit.num = 0; 1290 st->ldmb.stack_ofs = bpf->stack_sz; 1291 1292 emit_prolog(st, bpf->stack_sz); 1293 1294 for (i = 0; i != bpf->prm.nb_ins; i++) { 1295 1296 st->idx = i; 1297 st->off[i] = st->sz; 1298 1299 ins = bpf->prm.ins + i; 1300 1301 dr = ebpf2x86[ins->dst_reg]; 1302 sr = ebpf2x86[ins->src_reg]; 1303 op = ins->code; 1304 1305 switch (op) { 1306 /* 32 bit ALU IMM operations */ 1307 case (BPF_ALU | BPF_ADD | BPF_K): 1308 case (BPF_ALU | BPF_SUB | BPF_K): 1309 case (BPF_ALU | BPF_AND | BPF_K): 1310 case (BPF_ALU | BPF_OR | BPF_K): 1311 case (BPF_ALU | BPF_XOR | BPF_K): 1312 emit_alu_imm(st, op, dr, ins->imm); 1313 break; 1314 case (BPF_ALU | BPF_LSH | BPF_K): 1315 case (BPF_ALU | BPF_RSH | BPF_K): 1316 emit_shift_imm(st, op, dr, ins->imm); 1317 break; 1318 case (BPF_ALU | EBPF_MOV | BPF_K): 1319 emit_mov_imm(st, op, dr, ins->imm); 1320 break; 1321 /* 32 bit ALU REG operations */ 1322 case (BPF_ALU | BPF_ADD | BPF_X): 1323 case (BPF_ALU | BPF_SUB | BPF_X): 1324 case (BPF_ALU | BPF_AND | BPF_X): 1325 case (BPF_ALU | BPF_OR | BPF_X): 1326 case (BPF_ALU | BPF_XOR | BPF_X): 1327 emit_alu_reg(st, op, sr, dr); 1328 break; 1329 case (BPF_ALU | BPF_LSH | BPF_X): 1330 case (BPF_ALU | BPF_RSH | BPF_X): 1331 emit_shift_reg(st, op, sr, dr); 1332 break; 1333 case (BPF_ALU | EBPF_MOV | BPF_X): 1334 emit_mov_reg(st, op, sr, dr); 1335 break; 1336 case (BPF_ALU | BPF_NEG): 1337 emit_neg(st, op, dr); 1338 break; 1339 case (BPF_ALU | EBPF_END | EBPF_TO_BE): 1340 emit_be2le(st, dr, ins->imm); 1341 break; 1342 case (BPF_ALU | EBPF_END | EBPF_TO_LE): 1343 emit_le2be(st, dr, ins->imm); 1344 break; 1345 /* 64 bit ALU IMM operations */ 1346 case (EBPF_ALU64 | BPF_ADD | BPF_K): 1347 case (EBPF_ALU64 | BPF_SUB | BPF_K): 1348 case (EBPF_ALU64 | BPF_AND | BPF_K): 1349 case (EBPF_ALU64 | BPF_OR | BPF_K): 1350 case (EBPF_ALU64 | BPF_XOR | BPF_K): 1351 emit_alu_imm(st, op, dr, ins->imm); 1352 break; 1353 case (EBPF_ALU64 | BPF_LSH | BPF_K): 1354 case (EBPF_ALU64 | BPF_RSH | BPF_K): 1355 case (EBPF_ALU64 | EBPF_ARSH | BPF_K): 1356 emit_shift_imm(st, op, dr, ins->imm); 1357 break; 1358 case (EBPF_ALU64 | EBPF_MOV | BPF_K): 1359 emit_mov_imm(st, op, dr, ins->imm); 1360 break; 1361 /* 64 bit ALU REG operations */ 1362 case (EBPF_ALU64 | BPF_ADD | BPF_X): 1363 case (EBPF_ALU64 | BPF_SUB | BPF_X): 1364 case (EBPF_ALU64 | BPF_AND | BPF_X): 1365 case (EBPF_ALU64 | BPF_OR | BPF_X): 1366 case (EBPF_ALU64 | BPF_XOR | BPF_X): 1367 emit_alu_reg(st, op, sr, dr); 1368 break; 1369 case (EBPF_ALU64 | BPF_LSH | BPF_X): 1370 case (EBPF_ALU64 | BPF_RSH | BPF_X): 1371 case (EBPF_ALU64 | EBPF_ARSH | BPF_X): 1372 emit_shift_reg(st, op, sr, dr); 1373 break; 1374 case (EBPF_ALU64 | EBPF_MOV | BPF_X): 1375 emit_mov_reg(st, op, sr, dr); 1376 break; 1377 case (EBPF_ALU64 | BPF_NEG): 1378 emit_neg(st, op, dr); 1379 break; 1380 /* multiply instructions */ 1381 case (BPF_ALU | BPF_MUL | BPF_K): 1382 case (BPF_ALU | BPF_MUL | BPF_X): 1383 case (EBPF_ALU64 | BPF_MUL | BPF_K): 1384 case (EBPF_ALU64 | BPF_MUL | BPF_X): 1385 emit_mul(st, op, sr, dr, ins->imm); 1386 break; 1387 /* divide instructions */ 1388 case (BPF_ALU | BPF_DIV | BPF_K): 1389 case (BPF_ALU | BPF_MOD | BPF_K): 1390 case (BPF_ALU | BPF_DIV | BPF_X): 1391 case (BPF_ALU | BPF_MOD | BPF_X): 1392 case (EBPF_ALU64 | BPF_DIV | BPF_K): 1393 case (EBPF_ALU64 | BPF_MOD | BPF_K): 1394 case (EBPF_ALU64 | BPF_DIV | BPF_X): 1395 case (EBPF_ALU64 | BPF_MOD | BPF_X): 1396 emit_div(st, op, sr, dr, ins->imm); 1397 break; 1398 /* load instructions */ 1399 case (BPF_LDX | BPF_MEM | BPF_B): 1400 case (BPF_LDX | BPF_MEM | BPF_H): 1401 case (BPF_LDX | BPF_MEM | BPF_W): 1402 case (BPF_LDX | BPF_MEM | EBPF_DW): 1403 emit_ld_reg(st, op, sr, dr, ins->off); 1404 break; 1405 /* load 64 bit immediate value */ 1406 case (BPF_LD | BPF_IMM | EBPF_DW): 1407 emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm); 1408 i++; 1409 break; 1410 /* load absolute/indirect instructions */ 1411 case (BPF_LD | BPF_ABS | BPF_B): 1412 case (BPF_LD | BPF_ABS | BPF_H): 1413 case (BPF_LD | BPF_ABS | BPF_W): 1414 case (BPF_LD | BPF_IND | BPF_B): 1415 case (BPF_LD | BPF_IND | BPF_H): 1416 case (BPF_LD | BPF_IND | BPF_W): 1417 emit_ld_mbuf(st, op, sr, ins->imm); 1418 break; 1419 /* store instructions */ 1420 case (BPF_STX | BPF_MEM | BPF_B): 1421 case (BPF_STX | BPF_MEM | BPF_H): 1422 case (BPF_STX | BPF_MEM | BPF_W): 1423 case (BPF_STX | BPF_MEM | EBPF_DW): 1424 emit_st_reg(st, op, sr, dr, ins->off); 1425 break; 1426 case (BPF_ST | BPF_MEM | BPF_B): 1427 case (BPF_ST | BPF_MEM | BPF_H): 1428 case (BPF_ST | BPF_MEM | BPF_W): 1429 case (BPF_ST | BPF_MEM | EBPF_DW): 1430 emit_st_imm(st, op, dr, ins->imm, ins->off); 1431 break; 1432 /* atomic add instructions */ 1433 case (BPF_STX | EBPF_XADD | BPF_W): 1434 case (BPF_STX | EBPF_XADD | EBPF_DW): 1435 emit_st_xadd(st, op, sr, dr, ins->off); 1436 break; 1437 /* jump instructions */ 1438 case (BPF_JMP | BPF_JA): 1439 emit_jmp(st, ins->off + 1); 1440 break; 1441 /* jump IMM instructions */ 1442 case (BPF_JMP | BPF_JEQ | BPF_K): 1443 case (BPF_JMP | EBPF_JNE | BPF_K): 1444 case (BPF_JMP | BPF_JGT | BPF_K): 1445 case (BPF_JMP | EBPF_JLT | BPF_K): 1446 case (BPF_JMP | BPF_JGE | BPF_K): 1447 case (BPF_JMP | EBPF_JLE | BPF_K): 1448 case (BPF_JMP | EBPF_JSGT | BPF_K): 1449 case (BPF_JMP | EBPF_JSLT | BPF_K): 1450 case (BPF_JMP | EBPF_JSGE | BPF_K): 1451 case (BPF_JMP | EBPF_JSLE | BPF_K): 1452 case (BPF_JMP | BPF_JSET | BPF_K): 1453 emit_jcc_imm(st, op, dr, ins->imm, ins->off + 1); 1454 break; 1455 /* jump REG instructions */ 1456 case (BPF_JMP | BPF_JEQ | BPF_X): 1457 case (BPF_JMP | EBPF_JNE | BPF_X): 1458 case (BPF_JMP | BPF_JGT | BPF_X): 1459 case (BPF_JMP | EBPF_JLT | BPF_X): 1460 case (BPF_JMP | BPF_JGE | BPF_X): 1461 case (BPF_JMP | EBPF_JLE | BPF_X): 1462 case (BPF_JMP | EBPF_JSGT | BPF_X): 1463 case (BPF_JMP | EBPF_JSLT | BPF_X): 1464 case (BPF_JMP | EBPF_JSGE | BPF_X): 1465 case (BPF_JMP | EBPF_JSLE | BPF_X): 1466 case (BPF_JMP | BPF_JSET | BPF_X): 1467 emit_jcc_reg(st, op, sr, dr, ins->off + 1); 1468 break; 1469 /* call instructions */ 1470 case (BPF_JMP | EBPF_CALL): 1471 emit_call(st, 1472 (uintptr_t)bpf->prm.xsym[ins->imm].func.val); 1473 break; 1474 /* return instruction */ 1475 case (BPF_JMP | EBPF_EXIT): 1476 emit_epilog(st); 1477 break; 1478 default: 1479 RTE_BPF_LOG(ERR, 1480 "%s(%p): invalid opcode %#x at pc: %u;\n", 1481 __func__, bpf, ins->code, i); 1482 return -EINVAL; 1483 } 1484 } 1485 1486 return 0; 1487 } 1488 1489 /* 1490 * produce a native ISA version of the given BPF code. 1491 */ 1492 int 1493 bpf_jit_x86(struct rte_bpf *bpf) 1494 { 1495 int32_t rc; 1496 uint32_t i; 1497 size_t sz; 1498 struct bpf_jit_state st; 1499 1500 /* init state */ 1501 memset(&st, 0, sizeof(st)); 1502 st.off = malloc(bpf->prm.nb_ins * sizeof(st.off[0])); 1503 if (st.off == NULL) 1504 return -ENOMEM; 1505 1506 /* fill with fake offsets */ 1507 st.exit.off = INT32_MAX; 1508 for (i = 0; i != bpf->prm.nb_ins; i++) 1509 st.off[i] = INT32_MAX; 1510 1511 /* 1512 * dry runs, used to calculate total code size and valid jump offsets. 1513 * stop when we get minimal possible size 1514 */ 1515 do { 1516 sz = st.sz; 1517 rc = emit(&st, bpf); 1518 } while (rc == 0 && sz != st.sz); 1519 1520 if (rc == 0) { 1521 1522 /* allocate memory needed */ 1523 st.ins = mmap(NULL, st.sz, PROT_READ | PROT_WRITE, 1524 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1525 if (st.ins == MAP_FAILED) 1526 rc = -ENOMEM; 1527 else 1528 /* generate code */ 1529 rc = emit(&st, bpf); 1530 } 1531 1532 if (rc == 0 && mprotect(st.ins, st.sz, PROT_READ | PROT_EXEC) != 0) 1533 rc = -ENOMEM; 1534 1535 if (rc != 0) 1536 munmap(st.ins, st.sz); 1537 else { 1538 bpf->jit.func = (void *)st.ins; 1539 bpf->jit.sz = st.sz; 1540 } 1541 1542 free(st.off); 1543 return rc; 1544 } 1545