1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdint.h> 7 8 #include <rte_common.h> 9 #include <rte_log.h> 10 #include <rte_debug.h> 11 #include <rte_byteorder.h> 12 13 #include "bpf_impl.h" 14 15 #define BPF_JMP_UNC(ins) ((ins) += (ins)->off) 16 17 #define BPF_JMP_CND_REG(reg, ins, op, type) \ 18 ((ins) += \ 19 ((type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg]) ? \ 20 (ins)->off : 0) 21 22 #define BPF_JMP_CND_IMM(reg, ins, op, type) \ 23 ((ins) += \ 24 ((type)(reg)[(ins)->dst_reg] op (type)(ins)->imm) ? \ 25 (ins)->off : 0) 26 27 #define BPF_NEG_ALU(reg, ins, type) \ 28 ((reg)[(ins)->dst_reg] = (type)(-(reg)[(ins)->dst_reg])) 29 30 #define EBPF_MOV_ALU_REG(reg, ins, type) \ 31 ((reg)[(ins)->dst_reg] = (type)(reg)[(ins)->src_reg]) 32 33 #define BPF_OP_ALU_REG(reg, ins, op, type) \ 34 ((reg)[(ins)->dst_reg] = \ 35 (type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg]) 36 37 #define EBPF_MOV_ALU_IMM(reg, ins, type) \ 38 ((reg)[(ins)->dst_reg] = (type)(ins)->imm) 39 40 #define BPF_OP_ALU_IMM(reg, ins, op, type) \ 41 ((reg)[(ins)->dst_reg] = \ 42 (type)(reg)[(ins)->dst_reg] op (type)(ins)->imm) 43 44 #define BPF_DIV_ZERO_CHECK(bpf, reg, ins, type) do { \ 45 if ((type)(reg)[(ins)->src_reg] == 0) { \ 46 RTE_BPF_LOG(ERR, \ 47 "%s(%p): division by 0 at pc: %#zx;\n", \ 48 __func__, bpf, \ 49 (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \ 50 return 0; \ 51 } \ 52 } while (0) 53 54 #define BPF_LD_REG(reg, ins, type) \ 55 ((reg)[(ins)->dst_reg] = \ 56 *(type *)(uintptr_t)((reg)[(ins)->src_reg] + (ins)->off)) 57 58 #define BPF_ST_IMM(reg, ins, type) \ 59 (*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \ 60 (type)(ins)->imm) 61 62 #define BPF_ST_REG(reg, ins, type) \ 63 (*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \ 64 (type)(reg)[(ins)->src_reg]) 65 66 #define BPF_ST_XADD_REG(reg, ins, tp) \ 67 (rte_atomic##tp##_add((rte_atomic##tp##_t *) \ 68 (uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \ 69 reg[ins->src_reg])) 70 71 /* BPF_LD | BPF_ABS/BPF_IND */ 72 73 #define NOP(x) (x) 74 75 #define BPF_LD_ABS(bpf, reg, ins, type, op) do { \ 76 const type *p = bpf_ld_mbuf(bpf, reg, ins, (ins)->imm, sizeof(type)); \ 77 if (p == NULL) \ 78 return 0; \ 79 reg[EBPF_REG_0] = op(p[0]); \ 80 } while (0) 81 82 #define BPF_LD_IND(bpf, reg, ins, type, op) do { \ 83 uint32_t ofs = reg[ins->src_reg] + (ins)->imm; \ 84 const type *p = bpf_ld_mbuf(bpf, reg, ins, ofs, sizeof(type)); \ 85 if (p == NULL) \ 86 return 0; \ 87 reg[EBPF_REG_0] = op(p[0]); \ 88 } while (0) 89 90 91 static inline void 92 bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins) 93 { 94 uint64_t *v; 95 96 v = reg + ins->dst_reg; 97 switch (ins->imm) { 98 case 16: 99 *v = rte_cpu_to_be_16(*v); 100 break; 101 case 32: 102 *v = rte_cpu_to_be_32(*v); 103 break; 104 case 64: 105 *v = rte_cpu_to_be_64(*v); 106 break; 107 } 108 } 109 110 static inline void 111 bpf_alu_le(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins) 112 { 113 uint64_t *v; 114 115 v = reg + ins->dst_reg; 116 switch (ins->imm) { 117 case 16: 118 *v = rte_cpu_to_le_16(*v); 119 break; 120 case 32: 121 *v = rte_cpu_to_le_32(*v); 122 break; 123 case 64: 124 *v = rte_cpu_to_le_64(*v); 125 break; 126 } 127 } 128 129 static inline const void * 130 bpf_ld_mbuf(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM], 131 const struct ebpf_insn *ins, uint32_t off, uint32_t len) 132 { 133 const struct rte_mbuf *mb; 134 const void *p; 135 136 mb = (const struct rte_mbuf *)(uintptr_t)reg[EBPF_REG_6]; 137 p = rte_pktmbuf_read(mb, off, len, reg + EBPF_REG_0); 138 if (p == NULL) 139 RTE_BPF_LOG(DEBUG, "%s(bpf=%p, mbuf=%p, ofs=%u, len=%u): " 140 "load beyond packet boundary at pc: %#zx;\n", 141 __func__, bpf, mb, off, len, 142 (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); 143 return p; 144 } 145 146 static inline uint64_t 147 bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) 148 { 149 const struct ebpf_insn *ins; 150 151 for (ins = bpf->prm.ins; ; ins++) { 152 switch (ins->code) { 153 /* 32 bit ALU IMM operations */ 154 case (BPF_ALU | BPF_ADD | BPF_K): 155 BPF_OP_ALU_IMM(reg, ins, +, uint32_t); 156 break; 157 case (BPF_ALU | BPF_SUB | BPF_K): 158 BPF_OP_ALU_IMM(reg, ins, -, uint32_t); 159 break; 160 case (BPF_ALU | BPF_AND | BPF_K): 161 BPF_OP_ALU_IMM(reg, ins, &, uint32_t); 162 break; 163 case (BPF_ALU | BPF_OR | BPF_K): 164 BPF_OP_ALU_IMM(reg, ins, |, uint32_t); 165 break; 166 case (BPF_ALU | BPF_LSH | BPF_K): 167 BPF_OP_ALU_IMM(reg, ins, <<, uint32_t); 168 break; 169 case (BPF_ALU | BPF_RSH | BPF_K): 170 BPF_OP_ALU_IMM(reg, ins, >>, uint32_t); 171 break; 172 case (BPF_ALU | BPF_XOR | BPF_K): 173 BPF_OP_ALU_IMM(reg, ins, ^, uint32_t); 174 break; 175 case (BPF_ALU | BPF_MUL | BPF_K): 176 BPF_OP_ALU_IMM(reg, ins, *, uint32_t); 177 break; 178 case (BPF_ALU | BPF_DIV | BPF_K): 179 BPF_OP_ALU_IMM(reg, ins, /, uint32_t); 180 break; 181 case (BPF_ALU | BPF_MOD | BPF_K): 182 BPF_OP_ALU_IMM(reg, ins, %, uint32_t); 183 break; 184 case (BPF_ALU | EBPF_MOV | BPF_K): 185 EBPF_MOV_ALU_IMM(reg, ins, uint32_t); 186 break; 187 /* 32 bit ALU REG operations */ 188 case (BPF_ALU | BPF_ADD | BPF_X): 189 BPF_OP_ALU_REG(reg, ins, +, uint32_t); 190 break; 191 case (BPF_ALU | BPF_SUB | BPF_X): 192 BPF_OP_ALU_REG(reg, ins, -, uint32_t); 193 break; 194 case (BPF_ALU | BPF_AND | BPF_X): 195 BPF_OP_ALU_REG(reg, ins, &, uint32_t); 196 break; 197 case (BPF_ALU | BPF_OR | BPF_X): 198 BPF_OP_ALU_REG(reg, ins, |, uint32_t); 199 break; 200 case (BPF_ALU | BPF_LSH | BPF_X): 201 BPF_OP_ALU_REG(reg, ins, <<, uint32_t); 202 break; 203 case (BPF_ALU | BPF_RSH | BPF_X): 204 BPF_OP_ALU_REG(reg, ins, >>, uint32_t); 205 break; 206 case (BPF_ALU | BPF_XOR | BPF_X): 207 BPF_OP_ALU_REG(reg, ins, ^, uint32_t); 208 break; 209 case (BPF_ALU | BPF_MUL | BPF_X): 210 BPF_OP_ALU_REG(reg, ins, *, uint32_t); 211 break; 212 case (BPF_ALU | BPF_DIV | BPF_X): 213 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t); 214 BPF_OP_ALU_REG(reg, ins, /, uint32_t); 215 break; 216 case (BPF_ALU | BPF_MOD | BPF_X): 217 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t); 218 BPF_OP_ALU_REG(reg, ins, %, uint32_t); 219 break; 220 case (BPF_ALU | EBPF_MOV | BPF_X): 221 EBPF_MOV_ALU_REG(reg, ins, uint32_t); 222 break; 223 case (BPF_ALU | BPF_NEG): 224 BPF_NEG_ALU(reg, ins, uint32_t); 225 break; 226 case (BPF_ALU | EBPF_END | EBPF_TO_BE): 227 bpf_alu_be(reg, ins); 228 break; 229 case (BPF_ALU | EBPF_END | EBPF_TO_LE): 230 bpf_alu_le(reg, ins); 231 break; 232 /* 64 bit ALU IMM operations */ 233 case (EBPF_ALU64 | BPF_ADD | BPF_K): 234 BPF_OP_ALU_IMM(reg, ins, +, uint64_t); 235 break; 236 case (EBPF_ALU64 | BPF_SUB | BPF_K): 237 BPF_OP_ALU_IMM(reg, ins, -, uint64_t); 238 break; 239 case (EBPF_ALU64 | BPF_AND | BPF_K): 240 BPF_OP_ALU_IMM(reg, ins, &, uint64_t); 241 break; 242 case (EBPF_ALU64 | BPF_OR | BPF_K): 243 BPF_OP_ALU_IMM(reg, ins, |, uint64_t); 244 break; 245 case (EBPF_ALU64 | BPF_LSH | BPF_K): 246 BPF_OP_ALU_IMM(reg, ins, <<, uint64_t); 247 break; 248 case (EBPF_ALU64 | BPF_RSH | BPF_K): 249 BPF_OP_ALU_IMM(reg, ins, >>, uint64_t); 250 break; 251 case (EBPF_ALU64 | EBPF_ARSH | BPF_K): 252 BPF_OP_ALU_IMM(reg, ins, >>, int64_t); 253 break; 254 case (EBPF_ALU64 | BPF_XOR | BPF_K): 255 BPF_OP_ALU_IMM(reg, ins, ^, uint64_t); 256 break; 257 case (EBPF_ALU64 | BPF_MUL | BPF_K): 258 BPF_OP_ALU_IMM(reg, ins, *, uint64_t); 259 break; 260 case (EBPF_ALU64 | BPF_DIV | BPF_K): 261 BPF_OP_ALU_IMM(reg, ins, /, uint64_t); 262 break; 263 case (EBPF_ALU64 | BPF_MOD | BPF_K): 264 BPF_OP_ALU_IMM(reg, ins, %, uint64_t); 265 break; 266 case (EBPF_ALU64 | EBPF_MOV | BPF_K): 267 EBPF_MOV_ALU_IMM(reg, ins, uint64_t); 268 break; 269 /* 64 bit ALU REG operations */ 270 case (EBPF_ALU64 | BPF_ADD | BPF_X): 271 BPF_OP_ALU_REG(reg, ins, +, uint64_t); 272 break; 273 case (EBPF_ALU64 | BPF_SUB | BPF_X): 274 BPF_OP_ALU_REG(reg, ins, -, uint64_t); 275 break; 276 case (EBPF_ALU64 | BPF_AND | BPF_X): 277 BPF_OP_ALU_REG(reg, ins, &, uint64_t); 278 break; 279 case (EBPF_ALU64 | BPF_OR | BPF_X): 280 BPF_OP_ALU_REG(reg, ins, |, uint64_t); 281 break; 282 case (EBPF_ALU64 | BPF_LSH | BPF_X): 283 BPF_OP_ALU_REG(reg, ins, <<, uint64_t); 284 break; 285 case (EBPF_ALU64 | BPF_RSH | BPF_X): 286 BPF_OP_ALU_REG(reg, ins, >>, uint64_t); 287 break; 288 case (EBPF_ALU64 | EBPF_ARSH | BPF_X): 289 BPF_OP_ALU_REG(reg, ins, >>, int64_t); 290 break; 291 case (EBPF_ALU64 | BPF_XOR | BPF_X): 292 BPF_OP_ALU_REG(reg, ins, ^, uint64_t); 293 break; 294 case (EBPF_ALU64 | BPF_MUL | BPF_X): 295 BPF_OP_ALU_REG(reg, ins, *, uint64_t); 296 break; 297 case (EBPF_ALU64 | BPF_DIV | BPF_X): 298 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t); 299 BPF_OP_ALU_REG(reg, ins, /, uint64_t); 300 break; 301 case (EBPF_ALU64 | BPF_MOD | BPF_X): 302 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t); 303 BPF_OP_ALU_REG(reg, ins, %, uint64_t); 304 break; 305 case (EBPF_ALU64 | EBPF_MOV | BPF_X): 306 EBPF_MOV_ALU_REG(reg, ins, uint64_t); 307 break; 308 case (EBPF_ALU64 | BPF_NEG): 309 BPF_NEG_ALU(reg, ins, uint64_t); 310 break; 311 /* load instructions */ 312 case (BPF_LDX | BPF_MEM | BPF_B): 313 BPF_LD_REG(reg, ins, uint8_t); 314 break; 315 case (BPF_LDX | BPF_MEM | BPF_H): 316 BPF_LD_REG(reg, ins, uint16_t); 317 break; 318 case (BPF_LDX | BPF_MEM | BPF_W): 319 BPF_LD_REG(reg, ins, uint32_t); 320 break; 321 case (BPF_LDX | BPF_MEM | EBPF_DW): 322 BPF_LD_REG(reg, ins, uint64_t); 323 break; 324 /* load 64 bit immediate value */ 325 case (BPF_LD | BPF_IMM | EBPF_DW): 326 reg[ins->dst_reg] = (uint32_t)ins[0].imm | 327 (uint64_t)(uint32_t)ins[1].imm << 32; 328 ins++; 329 break; 330 /* load absolute instructions */ 331 case (BPF_LD | BPF_ABS | BPF_B): 332 BPF_LD_ABS(bpf, reg, ins, uint8_t, NOP); 333 break; 334 case (BPF_LD | BPF_ABS | BPF_H): 335 BPF_LD_ABS(bpf, reg, ins, uint16_t, rte_be_to_cpu_16); 336 break; 337 case (BPF_LD | BPF_ABS | BPF_W): 338 BPF_LD_ABS(bpf, reg, ins, uint32_t, rte_be_to_cpu_32); 339 break; 340 /* load indirect instructions */ 341 case (BPF_LD | BPF_IND | BPF_B): 342 BPF_LD_IND(bpf, reg, ins, uint8_t, NOP); 343 break; 344 case (BPF_LD | BPF_IND | BPF_H): 345 BPF_LD_IND(bpf, reg, ins, uint16_t, rte_be_to_cpu_16); 346 break; 347 case (BPF_LD | BPF_IND | BPF_W): 348 BPF_LD_IND(bpf, reg, ins, uint32_t, rte_be_to_cpu_32); 349 break; 350 /* store instructions */ 351 case (BPF_STX | BPF_MEM | BPF_B): 352 BPF_ST_REG(reg, ins, uint8_t); 353 break; 354 case (BPF_STX | BPF_MEM | BPF_H): 355 BPF_ST_REG(reg, ins, uint16_t); 356 break; 357 case (BPF_STX | BPF_MEM | BPF_W): 358 BPF_ST_REG(reg, ins, uint32_t); 359 break; 360 case (BPF_STX | BPF_MEM | EBPF_DW): 361 BPF_ST_REG(reg, ins, uint64_t); 362 break; 363 case (BPF_ST | BPF_MEM | BPF_B): 364 BPF_ST_IMM(reg, ins, uint8_t); 365 break; 366 case (BPF_ST | BPF_MEM | BPF_H): 367 BPF_ST_IMM(reg, ins, uint16_t); 368 break; 369 case (BPF_ST | BPF_MEM | BPF_W): 370 BPF_ST_IMM(reg, ins, uint32_t); 371 break; 372 case (BPF_ST | BPF_MEM | EBPF_DW): 373 BPF_ST_IMM(reg, ins, uint64_t); 374 break; 375 /* atomic add instructions */ 376 case (BPF_STX | EBPF_XADD | BPF_W): 377 BPF_ST_XADD_REG(reg, ins, 32); 378 break; 379 case (BPF_STX | EBPF_XADD | EBPF_DW): 380 BPF_ST_XADD_REG(reg, ins, 64); 381 break; 382 /* jump instructions */ 383 case (BPF_JMP | BPF_JA): 384 BPF_JMP_UNC(ins); 385 break; 386 /* jump IMM instructions */ 387 case (BPF_JMP | BPF_JEQ | BPF_K): 388 BPF_JMP_CND_IMM(reg, ins, ==, uint64_t); 389 break; 390 case (BPF_JMP | EBPF_JNE | BPF_K): 391 BPF_JMP_CND_IMM(reg, ins, !=, uint64_t); 392 break; 393 case (BPF_JMP | BPF_JGT | BPF_K): 394 BPF_JMP_CND_IMM(reg, ins, >, uint64_t); 395 break; 396 case (BPF_JMP | EBPF_JLT | BPF_K): 397 BPF_JMP_CND_IMM(reg, ins, <, uint64_t); 398 break; 399 case (BPF_JMP | BPF_JGE | BPF_K): 400 BPF_JMP_CND_IMM(reg, ins, >=, uint64_t); 401 break; 402 case (BPF_JMP | EBPF_JLE | BPF_K): 403 BPF_JMP_CND_IMM(reg, ins, <=, uint64_t); 404 break; 405 case (BPF_JMP | EBPF_JSGT | BPF_K): 406 BPF_JMP_CND_IMM(reg, ins, >, int64_t); 407 break; 408 case (BPF_JMP | EBPF_JSLT | BPF_K): 409 BPF_JMP_CND_IMM(reg, ins, <, int64_t); 410 break; 411 case (BPF_JMP | EBPF_JSGE | BPF_K): 412 BPF_JMP_CND_IMM(reg, ins, >=, int64_t); 413 break; 414 case (BPF_JMP | EBPF_JSLE | BPF_K): 415 BPF_JMP_CND_IMM(reg, ins, <=, int64_t); 416 break; 417 case (BPF_JMP | BPF_JSET | BPF_K): 418 BPF_JMP_CND_IMM(reg, ins, &, uint64_t); 419 break; 420 /* jump REG instructions */ 421 case (BPF_JMP | BPF_JEQ | BPF_X): 422 BPF_JMP_CND_REG(reg, ins, ==, uint64_t); 423 break; 424 case (BPF_JMP | EBPF_JNE | BPF_X): 425 BPF_JMP_CND_REG(reg, ins, !=, uint64_t); 426 break; 427 case (BPF_JMP | BPF_JGT | BPF_X): 428 BPF_JMP_CND_REG(reg, ins, >, uint64_t); 429 break; 430 case (BPF_JMP | EBPF_JLT | BPF_X): 431 BPF_JMP_CND_REG(reg, ins, <, uint64_t); 432 break; 433 case (BPF_JMP | BPF_JGE | BPF_X): 434 BPF_JMP_CND_REG(reg, ins, >=, uint64_t); 435 break; 436 case (BPF_JMP | EBPF_JLE | BPF_X): 437 BPF_JMP_CND_REG(reg, ins, <=, uint64_t); 438 break; 439 case (BPF_JMP | EBPF_JSGT | BPF_X): 440 BPF_JMP_CND_REG(reg, ins, >, int64_t); 441 break; 442 case (BPF_JMP | EBPF_JSLT | BPF_X): 443 BPF_JMP_CND_REG(reg, ins, <, int64_t); 444 break; 445 case (BPF_JMP | EBPF_JSGE | BPF_X): 446 BPF_JMP_CND_REG(reg, ins, >=, int64_t); 447 break; 448 case (BPF_JMP | EBPF_JSLE | BPF_X): 449 BPF_JMP_CND_REG(reg, ins, <=, int64_t); 450 break; 451 case (BPF_JMP | BPF_JSET | BPF_X): 452 BPF_JMP_CND_REG(reg, ins, &, uint64_t); 453 break; 454 /* call instructions */ 455 case (BPF_JMP | EBPF_CALL): 456 reg[EBPF_REG_0] = bpf->prm.xsym[ins->imm].func.val( 457 reg[EBPF_REG_1], reg[EBPF_REG_2], 458 reg[EBPF_REG_3], reg[EBPF_REG_4], 459 reg[EBPF_REG_5]); 460 break; 461 /* return instruction */ 462 case (BPF_JMP | EBPF_EXIT): 463 return reg[EBPF_REG_0]; 464 default: 465 RTE_BPF_LOG(ERR, 466 "%s(%p): invalid opcode %#x at pc: %#zx;\n", 467 __func__, bpf, ins->code, 468 (uintptr_t)ins - (uintptr_t)bpf->prm.ins); 469 return 0; 470 } 471 } 472 473 /* should never be reached */ 474 RTE_VERIFY(0); 475 return 0; 476 } 477 478 uint32_t 479 rte_bpf_exec_burst(const struct rte_bpf *bpf, void *ctx[], uint64_t rc[], 480 uint32_t num) 481 { 482 uint32_t i; 483 uint64_t reg[EBPF_REG_NUM]; 484 uint64_t stack[MAX_BPF_STACK_SIZE / sizeof(uint64_t)]; 485 486 for (i = 0; i != num; i++) { 487 488 reg[EBPF_REG_1] = (uintptr_t)ctx[i]; 489 reg[EBPF_REG_10] = (uintptr_t)(stack + RTE_DIM(stack)); 490 491 rc[i] = bpf_exec(bpf, reg); 492 } 493 494 return i; 495 } 496 497 uint64_t 498 rte_bpf_exec(const struct rte_bpf *bpf, void *ctx) 499 { 500 uint64_t rc; 501 502 rte_bpf_exec_burst(bpf, &ctx, &rc, 1); 503 return rc; 504 } 505