1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <string.h> 8 #include <errno.h> 9 #include <stdint.h> 10 #include <unistd.h> 11 #include <inttypes.h> 12 13 #include <sys/types.h> 14 #include <sys/stat.h> 15 #include <sys/queue.h> 16 #include <fcntl.h> 17 18 #include <libelf.h> 19 20 #include <rte_common.h> 21 #include <rte_log.h> 22 #include <rte_debug.h> 23 #include <rte_memory.h> 24 #include <rte_eal.h> 25 #include <rte_byteorder.h> 26 #include <rte_errno.h> 27 28 #include "bpf_impl.h" 29 30 /* To overcome compatibility issue */ 31 #ifndef EM_BPF 32 #define EM_BPF 247 33 #endif 34 35 static uint32_t 36 bpf_find_xsym(const char *sn, enum rte_bpf_xtype type, 37 const struct rte_bpf_xsym fp[], uint32_t fn) 38 { 39 uint32_t i; 40 41 if (sn == NULL || fp == NULL) 42 return UINT32_MAX; 43 44 for (i = 0; i != fn; i++) { 45 if (fp[i].type == type && strcmp(sn, fp[i].name) == 0) 46 break; 47 } 48 49 return (i != fn) ? i : UINT32_MAX; 50 } 51 52 /* 53 * update BPF code at offset *ofs* with a proper address(index) for external 54 * symbol *sn* 55 */ 56 static int 57 resolve_xsym(const char *sn, size_t ofs, struct ebpf_insn *ins, size_t ins_sz, 58 const struct rte_bpf_prm *prm) 59 { 60 uint32_t idx, fidx; 61 enum rte_bpf_xtype type; 62 63 if (ofs % sizeof(ins[0]) != 0 || ofs >= ins_sz) 64 return -EINVAL; 65 66 idx = ofs / sizeof(ins[0]); 67 if (ins[idx].code == (BPF_JMP | EBPF_CALL)) 68 type = RTE_BPF_XTYPE_FUNC; 69 else if (ins[idx].code == (BPF_LD | BPF_IMM | EBPF_DW) && 70 ofs < ins_sz - sizeof(ins[idx])) 71 type = RTE_BPF_XTYPE_VAR; 72 else 73 return -EINVAL; 74 75 fidx = bpf_find_xsym(sn, type, prm->xsym, prm->nb_xsym); 76 if (fidx == UINT32_MAX) 77 return -ENOENT; 78 79 /* for function we just need an index in our xsym table */ 80 if (type == RTE_BPF_XTYPE_FUNC) { 81 82 /* we don't support multiple functions per BPF module, 83 * so treat EBPF_PSEUDO_CALL to external function 84 * as an ordinary EBPF_CALL. 85 */ 86 if (ins[idx].src_reg == EBPF_PSEUDO_CALL) { 87 RTE_BPF_LOG_LINE(INFO, "%s(%u): " 88 "EBPF_PSEUDO_CALL to external function: %s", 89 __func__, idx, sn); 90 ins[idx].src_reg = EBPF_REG_0; 91 } 92 ins[idx].imm = fidx; 93 /* for variable we need to store its absolute address */ 94 } else { 95 ins[idx].imm = (uintptr_t)prm->xsym[fidx].var.val; 96 ins[idx + 1].imm = 97 (uint64_t)(uintptr_t)prm->xsym[fidx].var.val >> 32; 98 } 99 100 return 0; 101 } 102 103 static int 104 check_elf_header(const Elf64_Ehdr *eh) 105 { 106 const char *err; 107 108 err = NULL; 109 110 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 111 if (eh->e_ident[EI_DATA] != ELFDATA2LSB) 112 #else 113 if (eh->e_ident[EI_DATA] != ELFDATA2MSB) 114 #endif 115 err = "not native byte order"; 116 else if (eh->e_ident[EI_OSABI] != ELFOSABI_NONE) 117 err = "unexpected OS ABI"; 118 else if (eh->e_type != ET_REL) 119 err = "unexpected ELF type"; 120 else if (eh->e_machine != EM_NONE && eh->e_machine != EM_BPF) 121 err = "unexpected machine type"; 122 123 if (err != NULL) { 124 RTE_BPF_LOG_LINE(ERR, "%s(): %s", __func__, err); 125 return -EINVAL; 126 } 127 128 return 0; 129 } 130 131 /* 132 * helper function, find executable section by name. 133 */ 134 static int 135 find_elf_code(Elf *elf, const char *section, Elf_Data **psd, size_t *pidx) 136 { 137 Elf_Scn *sc; 138 const Elf64_Ehdr *eh; 139 const Elf64_Shdr *sh; 140 Elf_Data *sd; 141 const char *sn; 142 int32_t rc; 143 144 eh = elf64_getehdr(elf); 145 if (eh == NULL) { 146 rc = elf_errno(); 147 RTE_BPF_LOG_LINE(ERR, "%s(%p, %s) error code: %d(%s)", 148 __func__, elf, section, rc, elf_errmsg(rc)); 149 return -EINVAL; 150 } 151 152 if (check_elf_header(eh) != 0) 153 return -EINVAL; 154 155 /* find given section by name */ 156 for (sc = elf_nextscn(elf, NULL); sc != NULL; 157 sc = elf_nextscn(elf, sc)) { 158 sh = elf64_getshdr(sc); 159 sn = elf_strptr(elf, eh->e_shstrndx, sh->sh_name); 160 if (sn != NULL && strcmp(section, sn) == 0 && 161 sh->sh_type == SHT_PROGBITS && 162 sh->sh_flags == (SHF_ALLOC | SHF_EXECINSTR)) 163 break; 164 } 165 166 sd = elf_getdata(sc, NULL); 167 if (sd == NULL || sd->d_size == 0 || 168 sd->d_size % sizeof(struct ebpf_insn) != 0) { 169 rc = elf_errno(); 170 RTE_BPF_LOG_LINE(ERR, "%s(%p, %s) error code: %d(%s)", 171 __func__, elf, section, rc, elf_errmsg(rc)); 172 return -EINVAL; 173 } 174 175 *psd = sd; 176 *pidx = elf_ndxscn(sc); 177 return 0; 178 } 179 180 /* 181 * helper function to process data from relocation table. 182 */ 183 static int 184 process_reloc(Elf *elf, size_t sym_idx, Elf64_Rel *re, size_t re_sz, 185 struct ebpf_insn *ins, size_t ins_sz, const struct rte_bpf_prm *prm) 186 { 187 int32_t rc; 188 uint32_t i, n; 189 size_t ofs, sym; 190 const char *sn; 191 const Elf64_Ehdr *eh; 192 Elf_Scn *sc; 193 const Elf_Data *sd; 194 Elf64_Sym *sm; 195 196 eh = elf64_getehdr(elf); 197 198 /* get symtable by section index */ 199 sc = elf_getscn(elf, sym_idx); 200 sd = elf_getdata(sc, NULL); 201 if (sd == NULL) 202 return -EINVAL; 203 sm = sd->d_buf; 204 205 n = re_sz / sizeof(re[0]); 206 for (i = 0; i != n; i++) { 207 208 ofs = re[i].r_offset; 209 210 /* retrieve index in the symtable */ 211 sym = ELF64_R_SYM(re[i].r_info); 212 if (sym * sizeof(sm[0]) >= sd->d_size) 213 return -EINVAL; 214 215 sn = elf_strptr(elf, eh->e_shstrndx, sm[sym].st_name); 216 217 rc = resolve_xsym(sn, ofs, ins, ins_sz, prm); 218 if (rc != 0) { 219 RTE_BPF_LOG_LINE(ERR, 220 "resolve_xsym(%s, %zu) error code: %d", 221 sn, ofs, rc); 222 return rc; 223 } 224 } 225 226 return 0; 227 } 228 229 /* 230 * helper function, find relocation information (if any) 231 * and update bpf code. 232 */ 233 static int 234 elf_reloc_code(Elf *elf, Elf_Data *ed, size_t sidx, 235 const struct rte_bpf_prm *prm) 236 { 237 Elf64_Rel *re; 238 Elf_Scn *sc; 239 const Elf64_Shdr *sh; 240 const Elf_Data *sd; 241 int32_t rc; 242 243 rc = 0; 244 245 /* walk through all sections */ 246 for (sc = elf_nextscn(elf, NULL); sc != NULL && rc == 0; 247 sc = elf_nextscn(elf, sc)) { 248 249 sh = elf64_getshdr(sc); 250 251 /* relocation data for our code section */ 252 if (sh->sh_type == SHT_REL && sh->sh_info == sidx) { 253 sd = elf_getdata(sc, NULL); 254 if (sd == NULL || sd->d_size == 0 || 255 sd->d_size % sizeof(re[0]) != 0) 256 return -EINVAL; 257 rc = process_reloc(elf, sh->sh_link, 258 sd->d_buf, sd->d_size, ed->d_buf, ed->d_size, 259 prm); 260 } 261 } 262 263 return rc; 264 } 265 266 static struct rte_bpf * 267 bpf_load_elf(const struct rte_bpf_prm *prm, int32_t fd, const char *section) 268 { 269 Elf *elf; 270 Elf_Data *sd; 271 size_t sidx; 272 int32_t rc; 273 struct rte_bpf *bpf; 274 struct rte_bpf_prm np; 275 276 elf_version(EV_CURRENT); 277 elf = elf_begin(fd, ELF_C_READ, NULL); 278 279 rc = find_elf_code(elf, section, &sd, &sidx); 280 if (rc == 0) 281 rc = elf_reloc_code(elf, sd, sidx, prm); 282 283 if (rc == 0) { 284 np = prm[0]; 285 np.ins = sd->d_buf; 286 np.nb_ins = sd->d_size / sizeof(struct ebpf_insn); 287 bpf = rte_bpf_load(&np); 288 } else { 289 bpf = NULL; 290 rte_errno = -rc; 291 } 292 293 elf_end(elf); 294 return bpf; 295 } 296 297 struct rte_bpf * 298 rte_bpf_elf_load(const struct rte_bpf_prm *prm, const char *fname, 299 const char *sname) 300 { 301 int32_t fd, rc; 302 struct rte_bpf *bpf; 303 304 if (prm == NULL || fname == NULL || sname == NULL) { 305 rte_errno = EINVAL; 306 return NULL; 307 } 308 309 fd = open(fname, O_RDONLY); 310 if (fd < 0) { 311 rc = errno; 312 RTE_BPF_LOG_LINE(ERR, "%s(%s) error code: %d(%s)", 313 __func__, fname, rc, strerror(rc)); 314 rte_errno = EINVAL; 315 return NULL; 316 } 317 318 bpf = bpf_load_elf(prm, fd, sname); 319 close(fd); 320 321 if (bpf == NULL) { 322 RTE_BPF_LOG_LINE(ERR, 323 "%s(fname=\"%s\", sname=\"%s\") failed, " 324 "error code: %d", 325 __func__, fname, sname, rte_errno); 326 return NULL; 327 } 328 329 RTE_BPF_LOG_LINE(INFO, "%s(fname=\"%s\", sname=\"%s\") " 330 "successfully creates %p(jit={.func=%p,.sz=%zu});", 331 __func__, fname, sname, bpf, bpf->jit.func, bpf->jit.sz); 332 return bpf; 333 } 334