1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2019 Marvell International Ltd. 3 */ 4 5 #include <errno.h> 6 #include <stdbool.h> 7 #include <stdlib.h> 8 9 #include <rte_common.h> 10 #include <rte_byteorder.h> 11 12 #include "bpf_impl.h" 13 14 #define A64_REG_MASK(r) ((r) & 0x1f) 15 #define A64_INVALID_OP_CODE (0xffffffff) 16 17 #define TMP_REG_1 (EBPF_REG_10 + 1) 18 #define TMP_REG_2 (EBPF_REG_10 + 2) 19 #define TMP_REG_3 (EBPF_REG_10 + 3) 20 21 #define EBPF_FP (EBPF_REG_10) 22 #define EBPF_OP_GET(op) (BPF_OP(op) >> 4) 23 24 #define A64_R(x) x 25 #define A64_FP 29 26 #define A64_LR 30 27 #define A64_SP 31 28 #define A64_ZR 31 29 30 #define check_imm(n, val) (((val) >= 0) ? !!((val) >> (n)) : !!((~val) >> (n))) 31 #define mask_imm(n, val) ((val) & ((1 << (n)) - 1)) 32 33 struct ebpf_a64_map { 34 uint32_t off; /* eBPF to arm64 insn offset mapping for jump */ 35 uint8_t off_to_b; /* Offset to branch instruction delta */ 36 }; 37 38 struct a64_jit_ctx { 39 size_t stack_sz; /* Stack size */ 40 uint32_t *ins; /* ARM64 instructions. NULL if first pass */ 41 struct ebpf_a64_map *map; /* eBPF to arm64 insn mapping for jump */ 42 uint32_t idx; /* Current instruction index */ 43 uint32_t program_start; /* Program index, Just after prologue */ 44 uint32_t program_sz; /* Program size. Found in first pass */ 45 uint8_t foundcall; /* Found EBPF_CALL class code in eBPF pgm */ 46 }; 47 48 static int 49 check_immr_imms(bool is64, uint8_t immr, uint8_t imms) 50 { 51 const unsigned int width = is64 ? 64 : 32; 52 53 if (immr >= width || imms >= width) 54 return 1; 55 56 return 0; 57 } 58 59 static int 60 check_mov_hw(bool is64, const uint8_t val) 61 { 62 if (val == 16 || val == 0) 63 return 0; 64 else if (is64 && val != 64 && val != 48 && val != 32) 65 return 1; 66 67 return 0; 68 } 69 70 static int 71 check_ls_sz(uint8_t sz) 72 { 73 if (sz == BPF_B || sz == BPF_H || sz == BPF_W || sz == EBPF_DW) 74 return 0; 75 76 return 1; 77 } 78 79 static int 80 check_reg(uint8_t r) 81 { 82 return (r > 31) ? 1 : 0; 83 } 84 85 static int 86 is_first_pass(struct a64_jit_ctx *ctx) 87 { 88 return (ctx->ins == NULL); 89 } 90 91 static int 92 check_invalid_args(struct a64_jit_ctx *ctx, uint32_t limit) 93 { 94 uint32_t idx; 95 96 if (is_first_pass(ctx)) 97 return 0; 98 99 for (idx = 0; idx < limit; idx++) { 100 if (rte_le_to_cpu_32(ctx->ins[idx]) == A64_INVALID_OP_CODE) { 101 RTE_BPF_LOG_LINE(ERR, 102 "%s: invalid opcode at %u;", __func__, idx); 103 return -EINVAL; 104 } 105 } 106 return 0; 107 } 108 109 static int 110 jump_offset_init(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) 111 { 112 uint32_t i; 113 114 ctx->map = malloc(bpf->prm.nb_ins * sizeof(ctx->map[0])); 115 if (ctx->map == NULL) 116 return -ENOMEM; 117 118 /* Fill with fake offsets */ 119 for (i = 0; i != bpf->prm.nb_ins; i++) { 120 ctx->map[i].off = INT32_MAX; 121 ctx->map[i].off_to_b = 0; 122 } 123 return 0; 124 } 125 126 static void 127 jump_offset_fini(struct a64_jit_ctx *ctx) 128 { 129 free(ctx->map); 130 } 131 132 static void 133 jump_offset_update(struct a64_jit_ctx *ctx, uint32_t ebpf_idx) 134 { 135 if (is_first_pass(ctx)) 136 ctx->map[ebpf_idx].off = ctx->idx; 137 } 138 139 static void 140 jump_offset_to_branch_update(struct a64_jit_ctx *ctx, uint32_t ebpf_idx) 141 { 142 if (is_first_pass(ctx)) 143 ctx->map[ebpf_idx].off_to_b = ctx->idx - ctx->map[ebpf_idx].off; 144 145 } 146 147 static int32_t 148 jump_offset_get(struct a64_jit_ctx *ctx, uint32_t from, int16_t offset) 149 { 150 int32_t a64_from, a64_to; 151 152 a64_from = ctx->map[from].off + ctx->map[from].off_to_b; 153 a64_to = ctx->map[from + offset + 1].off; 154 155 if (a64_to == INT32_MAX) 156 return a64_to; 157 158 return a64_to - a64_from; 159 } 160 161 enum a64_cond_e { 162 A64_EQ = 0x0, /* == */ 163 A64_NE = 0x1, /* != */ 164 A64_CS = 0x2, /* Unsigned >= */ 165 A64_CC = 0x3, /* Unsigned < */ 166 A64_MI = 0x4, /* < 0 */ 167 A64_PL = 0x5, /* >= 0 */ 168 A64_VS = 0x6, /* Overflow */ 169 A64_VC = 0x7, /* No overflow */ 170 A64_HI = 0x8, /* Unsigned > */ 171 A64_LS = 0x9, /* Unsigned <= */ 172 A64_GE = 0xa, /* Signed >= */ 173 A64_LT = 0xb, /* Signed < */ 174 A64_GT = 0xc, /* Signed > */ 175 A64_LE = 0xd, /* Signed <= */ 176 A64_AL = 0xe, /* Always */ 177 }; 178 179 static int 180 check_cond(uint8_t cond) 181 { 182 return (cond >= A64_AL) ? 1 : 0; 183 } 184 185 static uint8_t 186 ebpf_to_a64_cond(uint8_t op) 187 { 188 switch (BPF_OP(op)) { 189 case BPF_JEQ: 190 return A64_EQ; 191 case BPF_JGT: 192 return A64_HI; 193 case EBPF_JLT: 194 return A64_CC; 195 case BPF_JGE: 196 return A64_CS; 197 case EBPF_JLE: 198 return A64_LS; 199 case BPF_JSET: 200 case EBPF_JNE: 201 return A64_NE; 202 case EBPF_JSGT: 203 return A64_GT; 204 case EBPF_JSLT: 205 return A64_LT; 206 case EBPF_JSGE: 207 return A64_GE; 208 case EBPF_JSLE: 209 return A64_LE; 210 default: 211 return UINT8_MAX; 212 } 213 } 214 215 /* Emit an instruction */ 216 static inline void 217 emit_insn(struct a64_jit_ctx *ctx, uint32_t insn, int error) 218 { 219 if (error) 220 insn = A64_INVALID_OP_CODE; 221 222 if (ctx->ins) 223 ctx->ins[ctx->idx] = rte_cpu_to_le_32(insn); 224 225 ctx->idx++; 226 } 227 228 static void 229 emit_ret(struct a64_jit_ctx *ctx) 230 { 231 emit_insn(ctx, 0xd65f03c0, 0); 232 } 233 234 static void 235 emit_add_sub_imm(struct a64_jit_ctx *ctx, bool is64, bool sub, uint8_t rd, 236 uint8_t rn, int16_t imm12) 237 { 238 uint32_t insn, imm; 239 240 imm = mask_imm(12, imm12); 241 insn = (!!is64) << 31; 242 insn |= (!!sub) << 30; 243 insn |= 0x11000000; 244 insn |= rd; 245 insn |= rn << 5; 246 insn |= imm << 10; 247 248 emit_insn(ctx, insn, 249 check_reg(rd) || check_reg(rn) || check_imm(12, imm12)); 250 } 251 252 static void 253 emit_add_imm_64(struct a64_jit_ctx *ctx, uint8_t rd, uint8_t rn, uint16_t imm12) 254 { 255 emit_add_sub_imm(ctx, 1, 0, rd, rn, imm12); 256 } 257 258 static void 259 emit_sub_imm_64(struct a64_jit_ctx *ctx, uint8_t rd, uint8_t rn, uint16_t imm12) 260 { 261 emit_add_sub_imm(ctx, 1, 1, rd, rn, imm12); 262 } 263 264 static void 265 emit_mov(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn) 266 { 267 emit_add_sub_imm(ctx, is64, 0, rd, rn, 0); 268 } 269 270 static void 271 emit_mov_64(struct a64_jit_ctx *ctx, uint8_t rd, uint8_t rn) 272 { 273 emit_mov(ctx, 1, rd, rn); 274 } 275 276 static void 277 emit_ls_pair_64(struct a64_jit_ctx *ctx, uint8_t rt, uint8_t rt2, uint8_t rn, 278 bool push, bool load, bool pre_index) 279 { 280 uint32_t insn; 281 282 insn = (!!load) << 22; 283 insn |= (!!pre_index) << 24; 284 insn |= 0xa8800000; 285 insn |= rt; 286 insn |= rn << 5; 287 insn |= rt2 << 10; 288 if (push) 289 insn |= 0x7e << 15; /* 0x7e means -2 with imm7 */ 290 else 291 insn |= 0x2 << 15; 292 293 emit_insn(ctx, insn, check_reg(rn) || check_reg(rt) || check_reg(rt2)); 294 295 } 296 297 /* Emit stp rt, rt2, [sp, #-16]! */ 298 static void 299 emit_stack_push(struct a64_jit_ctx *ctx, uint8_t rt, uint8_t rt2) 300 { 301 emit_ls_pair_64(ctx, rt, rt2, A64_SP, 1, 0, 1); 302 } 303 304 /* Emit ldp rt, rt2, [sp, #16] */ 305 static void 306 emit_stack_pop(struct a64_jit_ctx *ctx, uint8_t rt, uint8_t rt2) 307 { 308 emit_ls_pair_64(ctx, rt, rt2, A64_SP, 0, 1, 0); 309 } 310 311 #define A64_MOVN 0 312 #define A64_MOVZ 2 313 #define A64_MOVK 3 314 static void 315 mov_imm(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t type, 316 uint16_t imm16, uint8_t shift) 317 { 318 uint32_t insn; 319 320 insn = (!!is64) << 31; 321 insn |= type << 29; 322 insn |= 0x25 << 23; 323 insn |= (shift/16) << 21; 324 insn |= imm16 << 5; 325 insn |= rd; 326 327 emit_insn(ctx, insn, check_reg(rd) || check_mov_hw(is64, shift)); 328 } 329 330 static void 331 emit_mov_imm32(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint32_t val) 332 { 333 uint16_t upper = val >> 16; 334 uint16_t lower = val & 0xffff; 335 336 /* Positive number */ 337 if ((val & 1UL << 31) == 0) { 338 mov_imm(ctx, is64, rd, A64_MOVZ, lower, 0); 339 if (upper) 340 mov_imm(ctx, is64, rd, A64_MOVK, upper, 16); 341 } else { /* Negative number */ 342 if (upper == 0xffff) { 343 mov_imm(ctx, is64, rd, A64_MOVN, ~lower, 0); 344 } else { 345 mov_imm(ctx, is64, rd, A64_MOVN, ~upper, 16); 346 if (lower != 0xffff) 347 mov_imm(ctx, is64, rd, A64_MOVK, lower, 0); 348 } 349 } 350 } 351 352 static int 353 u16_blocks_weight(const uint64_t val, bool one) 354 { 355 return (((val >> 0) & 0xffff) == (one ? 0xffff : 0x0000)) + 356 (((val >> 16) & 0xffff) == (one ? 0xffff : 0x0000)) + 357 (((val >> 32) & 0xffff) == (one ? 0xffff : 0x0000)) + 358 (((val >> 48) & 0xffff) == (one ? 0xffff : 0x0000)); 359 } 360 361 static void 362 emit_mov_imm(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint64_t val) 363 { 364 uint64_t nval = ~val; 365 int movn, sr; 366 367 if (is64 == 0) 368 return emit_mov_imm32(ctx, 0, rd, (uint32_t)(val & 0xffffffff)); 369 370 /* Find MOVN or MOVZ first */ 371 movn = u16_blocks_weight(val, true) > u16_blocks_weight(val, false); 372 /* Find shift right value */ 373 sr = movn ? rte_fls_u64(nval) - 1 : rte_fls_u64(val) - 1; 374 sr = RTE_ALIGN_FLOOR(sr, 16); 375 sr = RTE_MAX(sr, 0); 376 377 if (movn) 378 mov_imm(ctx, 1, rd, A64_MOVN, (nval >> sr) & 0xffff, sr); 379 else 380 mov_imm(ctx, 1, rd, A64_MOVZ, (val >> sr) & 0xffff, sr); 381 382 sr -= 16; 383 while (sr >= 0) { 384 if (((val >> sr) & 0xffff) != (movn ? 0xffff : 0x0000)) 385 mov_imm(ctx, 1, rd, A64_MOVK, (val >> sr) & 0xffff, sr); 386 sr -= 16; 387 } 388 } 389 390 static void 391 emit_ls(struct a64_jit_ctx *ctx, uint8_t sz, uint8_t rt, uint8_t rn, uint8_t rm, 392 bool load) 393 { 394 uint32_t insn; 395 396 insn = 0x1c1 << 21; 397 if (load) 398 insn |= 1 << 22; 399 if (sz == BPF_B) 400 insn |= 0 << 30; 401 else if (sz == BPF_H) 402 insn |= 1 << 30; 403 else if (sz == BPF_W) 404 insn |= 2 << 30; 405 else if (sz == EBPF_DW) 406 insn |= 3 << 30; 407 408 insn |= rm << 16; 409 insn |= 0x1a << 10; /* LSL and S = 0 */ 410 insn |= rn << 5; 411 insn |= rt; 412 413 emit_insn(ctx, insn, check_reg(rt) || check_reg(rn) || check_reg(rm) || 414 check_ls_sz(sz)); 415 } 416 417 static void 418 emit_str(struct a64_jit_ctx *ctx, uint8_t sz, uint8_t rt, uint8_t rn, 419 uint8_t rm) 420 { 421 emit_ls(ctx, sz, rt, rn, rm, 0); 422 } 423 424 static void 425 emit_ldr(struct a64_jit_ctx *ctx, uint8_t sz, uint8_t rt, uint8_t rn, 426 uint8_t rm) 427 { 428 emit_ls(ctx, sz, rt, rn, rm, 1); 429 } 430 431 #define A64_ADD 0x58 432 #define A64_SUB 0x258 433 static void 434 emit_add_sub(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn, 435 uint8_t rm, uint16_t op) 436 { 437 uint32_t insn; 438 439 insn = (!!is64) << 31; 440 insn |= op << 21; /* shift == 0 */ 441 insn |= rm << 16; 442 insn |= rn << 5; 443 insn |= rd; 444 445 emit_insn(ctx, insn, check_reg(rd) || check_reg(rm)); 446 } 447 448 static void 449 emit_add(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm) 450 { 451 emit_add_sub(ctx, is64, rd, rd, rm, A64_ADD); 452 } 453 454 static void 455 emit_sub(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm) 456 { 457 emit_add_sub(ctx, is64, rd, rd, rm, A64_SUB); 458 } 459 460 static void 461 emit_neg(struct a64_jit_ctx *ctx, bool is64, uint8_t rd) 462 { 463 emit_add_sub(ctx, is64, rd, A64_ZR, rd, A64_SUB); 464 } 465 466 static void 467 emit_mul(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm) 468 { 469 uint32_t insn; 470 471 insn = (!!is64) << 31; 472 insn |= 0xd8 << 21; 473 insn |= rm << 16; 474 insn |= A64_ZR << 10; 475 insn |= rd << 5; 476 insn |= rd; 477 478 emit_insn(ctx, insn, check_reg(rd) || check_reg(rm)); 479 } 480 481 #define A64_UDIV 0x2 482 #define A64_LSLV 0x8 483 #define A64_LSRV 0x9 484 #define A64_ASRV 0xA 485 static void 486 emit_data_process_two_src(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, 487 uint8_t rn, uint8_t rm, uint16_t op) 488 489 { 490 uint32_t insn; 491 492 insn = (!!is64) << 31; 493 insn |= 0xd6 << 21; 494 insn |= rm << 16; 495 insn |= op << 10; 496 insn |= rn << 5; 497 insn |= rd; 498 499 emit_insn(ctx, insn, check_reg(rd) || check_reg(rm)); 500 } 501 502 static void 503 emit_div(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm) 504 { 505 emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_UDIV); 506 } 507 508 static void 509 emit_lslv(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm) 510 { 511 emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_LSLV); 512 } 513 514 static void 515 emit_lsrv(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm) 516 { 517 emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_LSRV); 518 } 519 520 static void 521 emit_asrv(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm) 522 { 523 emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_ASRV); 524 } 525 526 #define A64_UBFM 0x2 527 #define A64_SBFM 0x0 528 static void 529 emit_bitfield(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn, 530 uint8_t immr, uint8_t imms, uint16_t op) 531 532 { 533 uint32_t insn; 534 535 insn = (!!is64) << 31; 536 if (insn) 537 insn |= 1 << 22; /* Set N bit when is64 is set */ 538 insn |= op << 29; 539 insn |= 0x26 << 23; 540 insn |= immr << 16; 541 insn |= imms << 10; 542 insn |= rn << 5; 543 insn |= rd; 544 545 emit_insn(ctx, insn, check_reg(rd) || check_reg(rn) || 546 check_immr_imms(is64, immr, imms)); 547 } 548 static void 549 emit_lsl(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t imm) 550 { 551 const unsigned int width = is64 ? 64 : 32; 552 uint8_t imms, immr; 553 554 immr = (width - imm) & (width - 1); 555 imms = width - 1 - imm; 556 557 emit_bitfield(ctx, is64, rd, rd, immr, imms, A64_UBFM); 558 } 559 560 static void 561 emit_lsr(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t imm) 562 { 563 emit_bitfield(ctx, is64, rd, rd, imm, is64 ? 63 : 31, A64_UBFM); 564 } 565 566 static void 567 emit_asr(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t imm) 568 { 569 emit_bitfield(ctx, is64, rd, rd, imm, is64 ? 63 : 31, A64_SBFM); 570 } 571 572 #define A64_AND 0 573 #define A64_OR 1 574 #define A64_XOR 2 575 static void 576 emit_logical(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, 577 uint8_t rm, uint16_t op) 578 { 579 uint32_t insn; 580 581 insn = (!!is64) << 31; 582 insn |= op << 29; 583 insn |= 0x50 << 21; 584 insn |= rm << 16; 585 insn |= rd << 5; 586 insn |= rd; 587 588 emit_insn(ctx, insn, check_reg(rd) || check_reg(rm)); 589 } 590 591 static void 592 emit_or(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm) 593 { 594 emit_logical(ctx, is64, rd, rm, A64_OR); 595 } 596 597 static void 598 emit_and(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm) 599 { 600 emit_logical(ctx, is64, rd, rm, A64_AND); 601 } 602 603 static void 604 emit_xor(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm) 605 { 606 emit_logical(ctx, is64, rd, rm, A64_XOR); 607 } 608 609 static void 610 emit_msub(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn, 611 uint8_t rm, uint8_t ra) 612 { 613 uint32_t insn; 614 615 insn = (!!is64) << 31; 616 insn |= 0xd8 << 21; 617 insn |= rm << 16; 618 insn |= 0x1 << 15; 619 insn |= ra << 10; 620 insn |= rn << 5; 621 insn |= rd; 622 623 emit_insn(ctx, insn, check_reg(rd) || check_reg(rn) || check_reg(rm) || 624 check_reg(ra)); 625 } 626 627 static void 628 emit_mod(struct a64_jit_ctx *ctx, bool is64, uint8_t tmp, uint8_t rd, 629 uint8_t rm) 630 { 631 emit_data_process_two_src(ctx, is64, tmp, rd, rm, A64_UDIV); 632 emit_msub(ctx, is64, rd, tmp, rm, rd); 633 } 634 635 static void 636 emit_blr(struct a64_jit_ctx *ctx, uint8_t rn) 637 { 638 uint32_t insn; 639 640 insn = 0xd63f0000; 641 insn |= rn << 5; 642 643 emit_insn(ctx, insn, check_reg(rn)); 644 } 645 646 static void 647 emit_zero_extend(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm) 648 { 649 switch (imm) { 650 case 16: 651 /* Zero-extend 16 bits into 64 bits */ 652 emit_bitfield(ctx, 1, rd, rd, 0, 15, A64_UBFM); 653 break; 654 case 32: 655 /* Zero-extend 32 bits into 64 bits */ 656 emit_bitfield(ctx, 1, rd, rd, 0, 31, A64_UBFM); 657 break; 658 case 64: 659 break; 660 default: 661 /* Generate error */ 662 emit_insn(ctx, 0, 1); 663 } 664 } 665 666 static void 667 emit_rev(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm) 668 { 669 uint32_t insn; 670 671 insn = 0xdac00000; 672 insn |= rd << 5; 673 insn |= rd; 674 675 switch (imm) { 676 case 16: 677 insn |= 1 << 10; 678 emit_insn(ctx, insn, check_reg(rd)); 679 emit_zero_extend(ctx, rd, 16); 680 break; 681 case 32: 682 insn |= 2 << 10; 683 emit_insn(ctx, insn, check_reg(rd)); 684 /* Upper 32 bits already cleared */ 685 break; 686 case 64: 687 insn |= 3 << 10; 688 emit_insn(ctx, insn, check_reg(rd)); 689 break; 690 default: 691 /* Generate error */ 692 emit_insn(ctx, insn, 1); 693 } 694 } 695 696 static int 697 is_be(void) 698 { 699 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 700 return 1; 701 #else 702 return 0; 703 #endif 704 } 705 706 static void 707 emit_be(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm) 708 { 709 if (is_be()) 710 emit_zero_extend(ctx, rd, imm); 711 else 712 emit_rev(ctx, rd, imm); 713 } 714 715 static void 716 emit_le(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm) 717 { 718 if (is_be()) 719 emit_rev(ctx, rd, imm); 720 else 721 emit_zero_extend(ctx, rd, imm); 722 } 723 724 static uint8_t 725 ebpf_to_a64_reg(struct a64_jit_ctx *ctx, uint8_t reg) 726 { 727 const uint32_t ebpf2a64_has_call[] = { 728 /* Map A64 R7 register as EBPF return register */ 729 [EBPF_REG_0] = A64_R(7), 730 /* Map A64 arguments register as EBPF arguments register */ 731 [EBPF_REG_1] = A64_R(0), 732 [EBPF_REG_2] = A64_R(1), 733 [EBPF_REG_3] = A64_R(2), 734 [EBPF_REG_4] = A64_R(3), 735 [EBPF_REG_5] = A64_R(4), 736 /* Map A64 callee save register as EBPF callee save register */ 737 [EBPF_REG_6] = A64_R(19), 738 [EBPF_REG_7] = A64_R(20), 739 [EBPF_REG_8] = A64_R(21), 740 [EBPF_REG_9] = A64_R(22), 741 [EBPF_FP] = A64_R(25), 742 /* Map A64 scratch registers as temporary storage */ 743 [TMP_REG_1] = A64_R(9), 744 [TMP_REG_2] = A64_R(10), 745 [TMP_REG_3] = A64_R(11), 746 }; 747 748 const uint32_t ebpf2a64_no_call[] = { 749 /* Map A64 R7 register as EBPF return register */ 750 [EBPF_REG_0] = A64_R(7), 751 /* Map A64 arguments register as EBPF arguments register */ 752 [EBPF_REG_1] = A64_R(0), 753 [EBPF_REG_2] = A64_R(1), 754 [EBPF_REG_3] = A64_R(2), 755 [EBPF_REG_4] = A64_R(3), 756 [EBPF_REG_5] = A64_R(4), 757 /* 758 * EBPF program does not have EBPF_CALL op code, 759 * Map A64 scratch registers as EBPF callee save registers. 760 */ 761 [EBPF_REG_6] = A64_R(9), 762 [EBPF_REG_7] = A64_R(10), 763 [EBPF_REG_8] = A64_R(11), 764 [EBPF_REG_9] = A64_R(12), 765 /* Map A64 FP register as EBPF FP register */ 766 [EBPF_FP] = A64_FP, 767 /* Map remaining A64 scratch registers as temporary storage */ 768 [TMP_REG_1] = A64_R(13), 769 [TMP_REG_2] = A64_R(14), 770 [TMP_REG_3] = A64_R(15), 771 }; 772 773 if (ctx->foundcall) 774 return ebpf2a64_has_call[reg]; 775 else 776 return ebpf2a64_no_call[reg]; 777 } 778 779 /* 780 * Procedure call standard for the arm64 781 * ------------------------------------- 782 * R0..R7 - Parameter/result registers 783 * R8 - Indirect result location register 784 * R9..R15 - Scratch registers 785 * R15 - Platform Register 786 * R16 - First intra-procedure-call scratch register 787 * R17 - Second intra-procedure-call temporary register 788 * R19-R28 - Callee saved registers 789 * R29 - Frame pointer 790 * R30 - Link register 791 * R31 - Stack pointer 792 */ 793 static void 794 emit_prologue_has_call(struct a64_jit_ctx *ctx) 795 { 796 uint8_t r6, r7, r8, r9, fp; 797 798 r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6); 799 r7 = ebpf_to_a64_reg(ctx, EBPF_REG_7); 800 r8 = ebpf_to_a64_reg(ctx, EBPF_REG_8); 801 r9 = ebpf_to_a64_reg(ctx, EBPF_REG_9); 802 fp = ebpf_to_a64_reg(ctx, EBPF_FP); 803 804 /* 805 * eBPF prog stack layout 806 * 807 * high 808 * eBPF prologue 0:+-----+ <= original A64_SP 809 * |FP/LR| 810 * -16:+-----+ <= current A64_FP 811 * Callee saved registers | ... | 812 * EBPF_FP => -64:+-----+ 813 * | | 814 * eBPF prog stack | ... | 815 * | | 816 * (EBPF_FP - bpf->stack_sz)=> +-----+ 817 * Pad for A64_SP 16B alignment| PAD | 818 * (EBPF_FP - ctx->stack_sz)=> +-----+ <= current A64_SP 819 * | | 820 * | ... | Function call stack 821 * | | 822 * +-----+ 823 * low 824 */ 825 emit_stack_push(ctx, A64_FP, A64_LR); 826 emit_mov_64(ctx, A64_FP, A64_SP); 827 emit_stack_push(ctx, r6, r7); 828 emit_stack_push(ctx, r8, r9); 829 /* 830 * There is no requirement to save A64_R(28) in stack. Doing it here, 831 * because, A64_SP needs be to 16B aligned and STR vs STP 832 * takes same number of cycles(typically). 833 */ 834 emit_stack_push(ctx, fp, A64_R(28)); 835 emit_mov_64(ctx, fp, A64_SP); 836 if (ctx->stack_sz) 837 emit_sub_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz); 838 } 839 840 static void 841 emit_epilogue_has_call(struct a64_jit_ctx *ctx) 842 { 843 uint8_t r6, r7, r8, r9, fp, r0; 844 845 r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6); 846 r7 = ebpf_to_a64_reg(ctx, EBPF_REG_7); 847 r8 = ebpf_to_a64_reg(ctx, EBPF_REG_8); 848 r9 = ebpf_to_a64_reg(ctx, EBPF_REG_9); 849 fp = ebpf_to_a64_reg(ctx, EBPF_FP); 850 r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); 851 852 if (ctx->stack_sz) 853 emit_add_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz); 854 emit_stack_pop(ctx, fp, A64_R(28)); 855 emit_stack_pop(ctx, r8, r9); 856 emit_stack_pop(ctx, r6, r7); 857 emit_stack_pop(ctx, A64_FP, A64_LR); 858 emit_mov_64(ctx, A64_R(0), r0); 859 emit_ret(ctx); 860 } 861 862 static void 863 emit_prologue_no_call(struct a64_jit_ctx *ctx) 864 { 865 /* 866 * eBPF prog stack layout without EBPF_CALL opcode 867 * 868 * high 869 * eBPF prologue(EBPF_FP) 0:+-----+ <= original A64_SP/current A64_FP 870 * | | 871 * | ... | 872 * eBPF prog stack | | 873 * | | 874 * (EBPF_FP - bpf->stack_sz)=> +-----+ 875 * Pad for A64_SP 16B alignment| PAD | 876 * (EBPF_FP - ctx->stack_sz)=> +-----+ <= current A64_SP 877 * | | 878 * | ... | Function call stack 879 * | | 880 * +-----+ 881 * low 882 */ 883 if (ctx->stack_sz) { 884 emit_mov_64(ctx, A64_FP, A64_SP); 885 emit_sub_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz); 886 } 887 } 888 889 static void 890 emit_epilogue_no_call(struct a64_jit_ctx *ctx) 891 { 892 if (ctx->stack_sz) 893 emit_add_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz); 894 emit_mov_64(ctx, A64_R(0), ebpf_to_a64_reg(ctx, EBPF_REG_0)); 895 emit_ret(ctx); 896 } 897 898 static void 899 emit_prologue(struct a64_jit_ctx *ctx) 900 { 901 if (ctx->foundcall) 902 emit_prologue_has_call(ctx); 903 else 904 emit_prologue_no_call(ctx); 905 906 ctx->program_start = ctx->idx; 907 } 908 909 static void 910 emit_epilogue(struct a64_jit_ctx *ctx) 911 { 912 ctx->program_sz = ctx->idx - ctx->program_start; 913 914 if (ctx->foundcall) 915 emit_epilogue_has_call(ctx); 916 else 917 emit_epilogue_no_call(ctx); 918 } 919 920 static void 921 emit_call(struct a64_jit_ctx *ctx, uint8_t tmp, void *func) 922 { 923 uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); 924 925 emit_mov_imm(ctx, 1, tmp, (uint64_t)func); 926 emit_blr(ctx, tmp); 927 emit_mov_64(ctx, r0, A64_R(0)); 928 } 929 930 static void 931 emit_cbnz(struct a64_jit_ctx *ctx, bool is64, uint8_t rt, int32_t imm19) 932 { 933 uint32_t insn, imm; 934 935 imm = mask_imm(19, imm19); 936 insn = (!!is64) << 31; 937 insn |= 0x35 << 24; 938 insn |= imm << 5; 939 insn |= rt; 940 941 emit_insn(ctx, insn, check_reg(rt) || check_imm(19, imm19)); 942 } 943 944 static void 945 emit_b(struct a64_jit_ctx *ctx, int32_t imm26) 946 { 947 uint32_t insn, imm; 948 949 imm = mask_imm(26, imm26); 950 insn = 0x5 << 26; 951 insn |= imm; 952 953 emit_insn(ctx, insn, check_imm(26, imm26)); 954 } 955 956 static void 957 emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src) 958 { 959 uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); 960 uint16_t jump_to_epilogue; 961 962 emit_cbnz(ctx, is64, src, 3); 963 emit_mov_imm(ctx, is64, r0, 0); 964 jump_to_epilogue = (ctx->program_start + ctx->program_sz) - ctx->idx; 965 emit_b(ctx, jump_to_epilogue); 966 } 967 968 static void 969 emit_stadd(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rn) 970 { 971 uint32_t insn; 972 973 insn = 0xb820001f; 974 insn |= (!!is64) << 30; 975 insn |= rs << 16; 976 insn |= rn << 5; 977 978 emit_insn(ctx, insn, check_reg(rs) || check_reg(rn)); 979 } 980 981 static void 982 emit_ldxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rt, uint8_t rn) 983 { 984 uint32_t insn; 985 986 insn = 0x885f7c00; 987 insn |= (!!is64) << 30; 988 insn |= rn << 5; 989 insn |= rt; 990 991 emit_insn(ctx, insn, check_reg(rt) || check_reg(rn)); 992 } 993 994 static void 995 emit_stxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rt, 996 uint8_t rn) 997 { 998 uint32_t insn; 999 1000 insn = 0x88007c00; 1001 insn |= (!!is64) << 30; 1002 insn |= rs << 16; 1003 insn |= rn << 5; 1004 insn |= rt; 1005 1006 emit_insn(ctx, insn, check_reg(rs) || check_reg(rt) || check_reg(rn)); 1007 } 1008 1009 static int 1010 has_atomics(void) 1011 { 1012 int rc = 0; 1013 1014 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) 1015 rc = 1; 1016 #endif 1017 return rc; 1018 } 1019 1020 static void 1021 emit_xadd(struct a64_jit_ctx *ctx, uint8_t op, uint8_t tmp1, uint8_t tmp2, 1022 uint8_t tmp3, uint8_t dst, int16_t off, uint8_t src) 1023 { 1024 bool is64 = (BPF_SIZE(op) == EBPF_DW); 1025 uint8_t rn; 1026 1027 if (off) { 1028 emit_mov_imm(ctx, 1, tmp1, off); 1029 emit_add(ctx, 1, tmp1, dst); 1030 rn = tmp1; 1031 } else { 1032 rn = dst; 1033 } 1034 1035 if (has_atomics()) { 1036 emit_stadd(ctx, is64, src, rn); 1037 } else { 1038 emit_ldxr(ctx, is64, tmp2, rn); 1039 emit_add(ctx, is64, tmp2, src); 1040 emit_stxr(ctx, is64, tmp3, tmp2, rn); 1041 emit_cbnz(ctx, is64, tmp3, -3); 1042 } 1043 } 1044 1045 #define A64_CMP 0x6b00000f 1046 #define A64_TST 0x6a00000f 1047 static void 1048 emit_cmp_tst(struct a64_jit_ctx *ctx, bool is64, uint8_t rn, uint8_t rm, 1049 uint32_t opc) 1050 { 1051 uint32_t insn; 1052 1053 insn = opc; 1054 insn |= (!!is64) << 31; 1055 insn |= rm << 16; 1056 insn |= rn << 5; 1057 1058 emit_insn(ctx, insn, check_reg(rn) || check_reg(rm)); 1059 } 1060 1061 static void 1062 emit_cmp(struct a64_jit_ctx *ctx, bool is64, uint8_t rn, uint8_t rm) 1063 { 1064 emit_cmp_tst(ctx, is64, rn, rm, A64_CMP); 1065 } 1066 1067 static void 1068 emit_tst(struct a64_jit_ctx *ctx, bool is64, uint8_t rn, uint8_t rm) 1069 { 1070 emit_cmp_tst(ctx, is64, rn, rm, A64_TST); 1071 } 1072 1073 static void 1074 emit_b_cond(struct a64_jit_ctx *ctx, uint8_t cond, int32_t imm19) 1075 { 1076 uint32_t insn, imm; 1077 1078 imm = mask_imm(19, imm19); 1079 insn = 0x15 << 26; 1080 insn |= imm << 5; 1081 insn |= cond; 1082 1083 emit_insn(ctx, insn, check_cond(cond) || check_imm(19, imm19)); 1084 } 1085 1086 static void 1087 emit_branch(struct a64_jit_ctx *ctx, uint8_t op, uint32_t i, int16_t off) 1088 { 1089 jump_offset_to_branch_update(ctx, i); 1090 emit_b_cond(ctx, ebpf_to_a64_cond(op), jump_offset_get(ctx, i, off)); 1091 } 1092 1093 static void 1094 check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) 1095 { 1096 const struct ebpf_insn *ins; 1097 uint8_t op; 1098 uint32_t i; 1099 1100 for (i = 0; i != bpf->prm.nb_ins; i++) { 1101 ins = bpf->prm.ins + i; 1102 op = ins->code; 1103 1104 switch (op) { 1105 /* Call imm */ 1106 case (BPF_JMP | EBPF_CALL): 1107 ctx->foundcall = 1; 1108 return; 1109 } 1110 } 1111 } 1112 1113 /* 1114 * Walk through eBPF code and translate them to arm64 one. 1115 */ 1116 static int 1117 emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) 1118 { 1119 uint8_t op, dst, src, tmp1, tmp2, tmp3; 1120 const struct ebpf_insn *ins; 1121 uint64_t u64; 1122 int16_t off; 1123 int32_t imm; 1124 uint32_t i; 1125 bool is64; 1126 int rc; 1127 1128 /* Reset context fields */ 1129 ctx->idx = 0; 1130 /* arm64 SP must be aligned to 16 */ 1131 ctx->stack_sz = RTE_ALIGN_MUL_CEIL(bpf->stack_sz, 16); 1132 tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1); 1133 tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2); 1134 tmp3 = ebpf_to_a64_reg(ctx, TMP_REG_3); 1135 1136 emit_prologue(ctx); 1137 1138 for (i = 0; i != bpf->prm.nb_ins; i++) { 1139 1140 jump_offset_update(ctx, i); 1141 ins = bpf->prm.ins + i; 1142 op = ins->code; 1143 off = ins->off; 1144 imm = ins->imm; 1145 1146 dst = ebpf_to_a64_reg(ctx, ins->dst_reg); 1147 src = ebpf_to_a64_reg(ctx, ins->src_reg); 1148 is64 = (BPF_CLASS(op) == EBPF_ALU64); 1149 1150 switch (op) { 1151 /* dst = src */ 1152 case (BPF_ALU | EBPF_MOV | BPF_X): 1153 case (EBPF_ALU64 | EBPF_MOV | BPF_X): 1154 emit_mov(ctx, is64, dst, src); 1155 break; 1156 /* dst = imm */ 1157 case (BPF_ALU | EBPF_MOV | BPF_K): 1158 case (EBPF_ALU64 | EBPF_MOV | BPF_K): 1159 emit_mov_imm(ctx, is64, dst, imm); 1160 break; 1161 /* dst += src */ 1162 case (BPF_ALU | BPF_ADD | BPF_X): 1163 case (EBPF_ALU64 | BPF_ADD | BPF_X): 1164 emit_add(ctx, is64, dst, src); 1165 break; 1166 /* dst += imm */ 1167 case (BPF_ALU | BPF_ADD | BPF_K): 1168 case (EBPF_ALU64 | BPF_ADD | BPF_K): 1169 emit_mov_imm(ctx, is64, tmp1, imm); 1170 emit_add(ctx, is64, dst, tmp1); 1171 break; 1172 /* dst -= src */ 1173 case (BPF_ALU | BPF_SUB | BPF_X): 1174 case (EBPF_ALU64 | BPF_SUB | BPF_X): 1175 emit_sub(ctx, is64, dst, src); 1176 break; 1177 /* dst -= imm */ 1178 case (BPF_ALU | BPF_SUB | BPF_K): 1179 case (EBPF_ALU64 | BPF_SUB | BPF_K): 1180 emit_mov_imm(ctx, is64, tmp1, imm); 1181 emit_sub(ctx, is64, dst, tmp1); 1182 break; 1183 /* dst *= src */ 1184 case (BPF_ALU | BPF_MUL | BPF_X): 1185 case (EBPF_ALU64 | BPF_MUL | BPF_X): 1186 emit_mul(ctx, is64, dst, src); 1187 break; 1188 /* dst *= imm */ 1189 case (BPF_ALU | BPF_MUL | BPF_K): 1190 case (EBPF_ALU64 | BPF_MUL | BPF_K): 1191 emit_mov_imm(ctx, is64, tmp1, imm); 1192 emit_mul(ctx, is64, dst, tmp1); 1193 break; 1194 /* dst /= src */ 1195 case (BPF_ALU | BPF_DIV | BPF_X): 1196 case (EBPF_ALU64 | BPF_DIV | BPF_X): 1197 emit_return_zero_if_src_zero(ctx, is64, src); 1198 emit_div(ctx, is64, dst, src); 1199 break; 1200 /* dst /= imm */ 1201 case (BPF_ALU | BPF_DIV | BPF_K): 1202 case (EBPF_ALU64 | BPF_DIV | BPF_K): 1203 emit_mov_imm(ctx, is64, tmp1, imm); 1204 emit_div(ctx, is64, dst, tmp1); 1205 break; 1206 /* dst %= src */ 1207 case (BPF_ALU | BPF_MOD | BPF_X): 1208 case (EBPF_ALU64 | BPF_MOD | BPF_X): 1209 emit_return_zero_if_src_zero(ctx, is64, src); 1210 emit_mod(ctx, is64, tmp1, dst, src); 1211 break; 1212 /* dst %= imm */ 1213 case (BPF_ALU | BPF_MOD | BPF_K): 1214 case (EBPF_ALU64 | BPF_MOD | BPF_K): 1215 emit_mov_imm(ctx, is64, tmp1, imm); 1216 emit_mod(ctx, is64, tmp2, dst, tmp1); 1217 break; 1218 /* dst |= src */ 1219 case (BPF_ALU | BPF_OR | BPF_X): 1220 case (EBPF_ALU64 | BPF_OR | BPF_X): 1221 emit_or(ctx, is64, dst, src); 1222 break; 1223 /* dst |= imm */ 1224 case (BPF_ALU | BPF_OR | BPF_K): 1225 case (EBPF_ALU64 | BPF_OR | BPF_K): 1226 emit_mov_imm(ctx, is64, tmp1, imm); 1227 emit_or(ctx, is64, dst, tmp1); 1228 break; 1229 /* dst &= src */ 1230 case (BPF_ALU | BPF_AND | BPF_X): 1231 case (EBPF_ALU64 | BPF_AND | BPF_X): 1232 emit_and(ctx, is64, dst, src); 1233 break; 1234 /* dst &= imm */ 1235 case (BPF_ALU | BPF_AND | BPF_K): 1236 case (EBPF_ALU64 | BPF_AND | BPF_K): 1237 emit_mov_imm(ctx, is64, tmp1, imm); 1238 emit_and(ctx, is64, dst, tmp1); 1239 break; 1240 /* dst ^= src */ 1241 case (BPF_ALU | BPF_XOR | BPF_X): 1242 case (EBPF_ALU64 | BPF_XOR | BPF_X): 1243 emit_xor(ctx, is64, dst, src); 1244 break; 1245 /* dst ^= imm */ 1246 case (BPF_ALU | BPF_XOR | BPF_K): 1247 case (EBPF_ALU64 | BPF_XOR | BPF_K): 1248 emit_mov_imm(ctx, is64, tmp1, imm); 1249 emit_xor(ctx, is64, dst, tmp1); 1250 break; 1251 /* dst = -dst */ 1252 case (BPF_ALU | BPF_NEG): 1253 case (EBPF_ALU64 | BPF_NEG): 1254 emit_neg(ctx, is64, dst); 1255 break; 1256 /* dst <<= src */ 1257 case BPF_ALU | BPF_LSH | BPF_X: 1258 case EBPF_ALU64 | BPF_LSH | BPF_X: 1259 emit_lslv(ctx, is64, dst, src); 1260 break; 1261 /* dst <<= imm */ 1262 case BPF_ALU | BPF_LSH | BPF_K: 1263 case EBPF_ALU64 | BPF_LSH | BPF_K: 1264 emit_lsl(ctx, is64, dst, imm); 1265 break; 1266 /* dst >>= src */ 1267 case BPF_ALU | BPF_RSH | BPF_X: 1268 case EBPF_ALU64 | BPF_RSH | BPF_X: 1269 emit_lsrv(ctx, is64, dst, src); 1270 break; 1271 /* dst >>= imm */ 1272 case BPF_ALU | BPF_RSH | BPF_K: 1273 case EBPF_ALU64 | BPF_RSH | BPF_K: 1274 emit_lsr(ctx, is64, dst, imm); 1275 break; 1276 /* dst >>= src (arithmetic) */ 1277 case BPF_ALU | EBPF_ARSH | BPF_X: 1278 case EBPF_ALU64 | EBPF_ARSH | BPF_X: 1279 emit_asrv(ctx, is64, dst, src); 1280 break; 1281 /* dst >>= imm (arithmetic) */ 1282 case BPF_ALU | EBPF_ARSH | BPF_K: 1283 case EBPF_ALU64 | EBPF_ARSH | BPF_K: 1284 emit_asr(ctx, is64, dst, imm); 1285 break; 1286 /* dst = be##imm(dst) */ 1287 case (BPF_ALU | EBPF_END | EBPF_TO_BE): 1288 emit_be(ctx, dst, imm); 1289 break; 1290 /* dst = le##imm(dst) */ 1291 case (BPF_ALU | EBPF_END | EBPF_TO_LE): 1292 emit_le(ctx, dst, imm); 1293 break; 1294 /* dst = *(size *) (src + off) */ 1295 case (BPF_LDX | BPF_MEM | BPF_B): 1296 case (BPF_LDX | BPF_MEM | BPF_H): 1297 case (BPF_LDX | BPF_MEM | BPF_W): 1298 case (BPF_LDX | BPF_MEM | EBPF_DW): 1299 emit_mov_imm(ctx, 1, tmp1, off); 1300 emit_ldr(ctx, BPF_SIZE(op), dst, src, tmp1); 1301 break; 1302 /* dst = imm64 */ 1303 case (BPF_LD | BPF_IMM | EBPF_DW): 1304 u64 = ((uint64_t)ins[1].imm << 32) | (uint32_t)imm; 1305 emit_mov_imm(ctx, 1, dst, u64); 1306 i++; 1307 break; 1308 /* *(size *)(dst + off) = src */ 1309 case (BPF_STX | BPF_MEM | BPF_B): 1310 case (BPF_STX | BPF_MEM | BPF_H): 1311 case (BPF_STX | BPF_MEM | BPF_W): 1312 case (BPF_STX | BPF_MEM | EBPF_DW): 1313 emit_mov_imm(ctx, 1, tmp1, off); 1314 emit_str(ctx, BPF_SIZE(op), src, dst, tmp1); 1315 break; 1316 /* *(size *)(dst + off) = imm */ 1317 case (BPF_ST | BPF_MEM | BPF_B): 1318 case (BPF_ST | BPF_MEM | BPF_H): 1319 case (BPF_ST | BPF_MEM | BPF_W): 1320 case (BPF_ST | BPF_MEM | EBPF_DW): 1321 emit_mov_imm(ctx, 1, tmp1, imm); 1322 emit_mov_imm(ctx, 1, tmp2, off); 1323 emit_str(ctx, BPF_SIZE(op), tmp1, dst, tmp2); 1324 break; 1325 /* STX XADD: lock *(size *)(dst + off) += src */ 1326 case (BPF_STX | EBPF_XADD | BPF_W): 1327 case (BPF_STX | EBPF_XADD | EBPF_DW): 1328 emit_xadd(ctx, op, tmp1, tmp2, tmp3, dst, off, src); 1329 break; 1330 /* PC += off */ 1331 case (BPF_JMP | BPF_JA): 1332 emit_b(ctx, jump_offset_get(ctx, i, off)); 1333 break; 1334 /* PC += off if dst COND imm */ 1335 case (BPF_JMP | BPF_JEQ | BPF_K): 1336 case (BPF_JMP | EBPF_JNE | BPF_K): 1337 case (BPF_JMP | BPF_JGT | BPF_K): 1338 case (BPF_JMP | EBPF_JLT | BPF_K): 1339 case (BPF_JMP | BPF_JGE | BPF_K): 1340 case (BPF_JMP | EBPF_JLE | BPF_K): 1341 case (BPF_JMP | EBPF_JSGT | BPF_K): 1342 case (BPF_JMP | EBPF_JSLT | BPF_K): 1343 case (BPF_JMP | EBPF_JSGE | BPF_K): 1344 case (BPF_JMP | EBPF_JSLE | BPF_K): 1345 emit_mov_imm(ctx, 1, tmp1, imm); 1346 emit_cmp(ctx, 1, dst, tmp1); 1347 emit_branch(ctx, op, i, off); 1348 break; 1349 case (BPF_JMP | BPF_JSET | BPF_K): 1350 emit_mov_imm(ctx, 1, tmp1, imm); 1351 emit_tst(ctx, 1, dst, tmp1); 1352 emit_branch(ctx, op, i, off); 1353 break; 1354 /* PC += off if dst COND src */ 1355 case (BPF_JMP | BPF_JEQ | BPF_X): 1356 case (BPF_JMP | EBPF_JNE | BPF_X): 1357 case (BPF_JMP | BPF_JGT | BPF_X): 1358 case (BPF_JMP | EBPF_JLT | BPF_X): 1359 case (BPF_JMP | BPF_JGE | BPF_X): 1360 case (BPF_JMP | EBPF_JLE | BPF_X): 1361 case (BPF_JMP | EBPF_JSGT | BPF_X): 1362 case (BPF_JMP | EBPF_JSLT | BPF_X): 1363 case (BPF_JMP | EBPF_JSGE | BPF_X): 1364 case (BPF_JMP | EBPF_JSLE | BPF_X): 1365 emit_cmp(ctx, 1, dst, src); 1366 emit_branch(ctx, op, i, off); 1367 break; 1368 case (BPF_JMP | BPF_JSET | BPF_X): 1369 emit_tst(ctx, 1, dst, src); 1370 emit_branch(ctx, op, i, off); 1371 break; 1372 /* Call imm */ 1373 case (BPF_JMP | EBPF_CALL): 1374 emit_call(ctx, tmp1, bpf->prm.xsym[ins->imm].func.val); 1375 break; 1376 /* Return r0 */ 1377 case (BPF_JMP | EBPF_EXIT): 1378 emit_epilogue(ctx); 1379 break; 1380 default: 1381 RTE_BPF_LOG_LINE(ERR, 1382 "%s(%p): invalid opcode %#x at pc: %u;", 1383 __func__, bpf, ins->code, i); 1384 return -EINVAL; 1385 } 1386 } 1387 rc = check_invalid_args(ctx, ctx->idx); 1388 1389 return rc; 1390 } 1391 1392 /* 1393 * Produce a native ISA version of the given BPF code. 1394 */ 1395 int 1396 __rte_bpf_jit_arm64(struct rte_bpf *bpf) 1397 { 1398 struct a64_jit_ctx ctx; 1399 size_t size; 1400 int rc; 1401 1402 /* Init JIT context */ 1403 memset(&ctx, 0, sizeof(ctx)); 1404 1405 /* Initialize the memory for eBPF to a64 insn offset map for jump */ 1406 rc = jump_offset_init(&ctx, bpf); 1407 if (rc) 1408 goto error; 1409 1410 /* Find eBPF program has call class or not */ 1411 check_program_has_call(&ctx, bpf); 1412 1413 /* First pass to calculate total code size and valid jump offsets */ 1414 rc = emit(&ctx, bpf); 1415 if (rc) 1416 goto finish; 1417 1418 size = ctx.idx * sizeof(uint32_t); 1419 /* Allocate JIT program memory */ 1420 ctx.ins = mmap(NULL, size, PROT_READ | PROT_WRITE, 1421 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 1422 if (ctx.ins == MAP_FAILED) { 1423 rc = -ENOMEM; 1424 goto finish; 1425 } 1426 1427 /* Second pass to generate code */ 1428 rc = emit(&ctx, bpf); 1429 if (rc) 1430 goto munmap; 1431 1432 rc = mprotect(ctx.ins, size, PROT_READ | PROT_EXEC) != 0; 1433 if (rc) { 1434 rc = -errno; 1435 goto munmap; 1436 } 1437 1438 /* Flush the icache */ 1439 __builtin___clear_cache((char *)ctx.ins, (char *)(ctx.ins + ctx.idx)); 1440 1441 bpf->jit.func = (void *)ctx.ins; 1442 bpf->jit.sz = size; 1443 1444 goto finish; 1445 1446 munmap: 1447 munmap(ctx.ins, size); 1448 finish: 1449 jump_offset_fini(&ctx); 1450 error: 1451 return rc; 1452 } 1453