1 /* $NetBSD: bpf.c,v 1.238 2020/08/02 07:19:39 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.238 2020/08/02 07:19:39 maxv Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "opt_net_mpsafe.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/mbuf.h> 53 #include <sys/buf.h> 54 #include <sys/time.h> 55 #include <sys/proc.h> 56 #include <sys/ioctl.h> 57 #include <sys/conf.h> 58 #include <sys/vnode.h> 59 #include <sys/queue.h> 60 #include <sys/stat.h> 61 #include <sys/module.h> 62 #include <sys/atomic.h> 63 #include <sys/cpu.h> 64 65 #include <sys/file.h> 66 #include <sys/filedesc.h> 67 #include <sys/tty.h> 68 #include <sys/uio.h> 69 70 #include <sys/protosw.h> 71 #include <sys/socket.h> 72 #include <sys/errno.h> 73 #include <sys/kernel.h> 74 #include <sys/poll.h> 75 #include <sys/sysctl.h> 76 #include <sys/kauth.h> 77 #include <sys/syslog.h> 78 #include <sys/percpu.h> 79 #include <sys/pserialize.h> 80 #include <sys/lwp.h> 81 #include <sys/xcall.h> 82 83 #include <net/if.h> 84 #include <net/slip.h> 85 86 #include <net/bpf.h> 87 #include <net/bpfdesc.h> 88 #include <net/bpfjit.h> 89 90 #include <net/if_arc.h> 91 #include <net/if_ether.h> 92 93 #include <netinet/in.h> 94 #include <netinet/if_inarp.h> 95 96 97 #include <compat/sys/sockio.h> 98 99 #ifndef BPF_BUFSIZE 100 /* 101 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 102 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 103 */ 104 # define BPF_BUFSIZE 32768 105 #endif 106 107 #define PRINET 26 /* interruptible */ 108 109 /* 110 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 111 * XXX the default values should be computed dynamically based 112 * on available memory size and available mbuf clusters. 113 */ 114 static int bpf_bufsize = BPF_BUFSIZE; 115 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 116 static bool bpf_jit = false; 117 118 struct bpfjit_ops bpfjit_module_ops = { 119 .bj_generate_code = NULL, 120 .bj_free_code = NULL 121 }; 122 123 /* 124 * Global BPF statistics returned by net.bpf.stats sysctl. 125 */ 126 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */ 127 128 #define BPF_STATINC(id) \ 129 { \ 130 struct bpf_stat *__stats = \ 131 percpu_getref(bpf_gstats_percpu); \ 132 __stats->bs_##id++; \ 133 percpu_putref(bpf_gstats_percpu); \ 134 } 135 136 /* 137 * Locking notes: 138 * - bpf_mtx (adaptive mutex) protects: 139 * - Gobal lists: bpf_iflist and bpf_dlist 140 * - struct bpf_if 141 * - bpf_close 142 * - bpf_psz (pserialize) 143 * - struct bpf_d has two mutexes: 144 * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed 145 * on packet tapping 146 * - bd_mtx (adaptive mutex) protects member variables other than the buffers 147 * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx 148 * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is 149 * never freed because struct bpf_d is only freed in bpf_close and 150 * bpf_close never be called while executing bpf_read and bpf_write 151 * - A filter that is assigned to bpf_d can be replaced with another filter 152 * while tapping packets, so it needs to be done atomically 153 * - struct bpf_d is iterated on bpf_dlist with psz 154 * - struct bpf_if is iterated on bpf_iflist with psz or psref 155 */ 156 /* 157 * Use a mutex to avoid a race condition between gathering the stats/peers 158 * and opening/closing the device. 159 */ 160 static kmutex_t bpf_mtx; 161 162 static struct psref_class *bpf_psref_class __read_mostly; 163 static pserialize_t bpf_psz; 164 165 static inline void 166 bpf_if_acquire(struct bpf_if *bp, struct psref *psref) 167 { 168 169 psref_acquire(psref, &bp->bif_psref, bpf_psref_class); 170 } 171 172 static inline void 173 bpf_if_release(struct bpf_if *bp, struct psref *psref) 174 { 175 176 psref_release(psref, &bp->bif_psref, bpf_psref_class); 177 } 178 179 /* 180 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 181 * bpf_dtab holds the descriptors, indexed by minor device # 182 */ 183 static struct pslist_head bpf_iflist; 184 static struct pslist_head bpf_dlist; 185 186 /* Macros for bpf_d on bpf_dlist */ 187 #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \ 188 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) 189 #define BPF_DLIST_READER_FOREACH(__d) \ 190 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 191 bd_bpf_dlist_entry) 192 #define BPF_DLIST_WRITER_FOREACH(__d) \ 193 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 194 bd_bpf_dlist_entry) 195 #define BPF_DLIST_ENTRY_INIT(__d) \ 196 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) 197 #define BPF_DLIST_WRITER_REMOVE(__d) \ 198 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) 199 #define BPF_DLIST_ENTRY_DESTROY(__d) \ 200 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) 201 202 /* Macros for bpf_if on bpf_iflist */ 203 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ 204 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) 205 #define BPF_IFLIST_READER_FOREACH(__bp) \ 206 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 207 bif_iflist_entry) 208 #define BPF_IFLIST_WRITER_FOREACH(__bp) \ 209 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 210 bif_iflist_entry) 211 #define BPF_IFLIST_WRITER_REMOVE(__bp) \ 212 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) 213 #define BPF_IFLIST_ENTRY_INIT(__bp) \ 214 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) 215 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ 216 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) 217 218 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ 219 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ 220 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ 221 bd_bif_dlist_entry) 222 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ 223 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ 224 bd_bif_dlist_entry) 225 #define BPFIF_DLIST_WRITER_REMOVE(__d) \ 226 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) 227 #define BPFIF_DLIST_ENTRY_INIT(__d) \ 228 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) 229 #define BPFIF_DLIST_READER_EMPTY(__bp) \ 230 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 231 bd_bif_dlist_entry) == NULL) 232 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ 233 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 234 bd_bif_dlist_entry) == NULL) 235 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ 236 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) 237 238 static int bpf_allocbufs(struct bpf_d *); 239 static u_int bpf_xfilter(struct bpf_filter **, void *, u_int, u_int); 240 static void bpf_deliver(struct bpf_if *, 241 void *(*cpfn)(void *, const void *, size_t), 242 void *, u_int, u_int, const u_int); 243 static void bpf_freed(struct bpf_d *); 244 static void bpf_free_filter(struct bpf_filter *); 245 static void bpf_ifname(struct ifnet *, struct ifreq *); 246 static void *bpf_mcpy(void *, const void *, size_t); 247 static int bpf_movein(struct uio *, int, uint64_t, 248 struct mbuf **, struct sockaddr *, 249 struct bpf_filter **); 250 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 251 static void bpf_detachd(struct bpf_d *); 252 static int bpf_setif(struct bpf_d *, struct ifreq *); 253 static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long); 254 static void bpf_timed_out(void *); 255 static inline void 256 bpf_wakeup(struct bpf_d *); 257 static int bpf_hdrlen(struct bpf_d *); 258 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 259 void *(*)(void *, const void *, size_t), struct timespec *); 260 static void reset_d(struct bpf_d *); 261 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 262 static int bpf_setdlt(struct bpf_d *, u_int); 263 264 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 265 int); 266 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 267 int); 268 static int bpf_ioctl(struct file *, u_long, void *); 269 static int bpf_poll(struct file *, int); 270 static int bpf_stat(struct file *, struct stat *); 271 static int bpf_close(struct file *); 272 static int bpf_kqfilter(struct file *, struct knote *); 273 274 static const struct fileops bpf_fileops = { 275 .fo_name = "bpf", 276 .fo_read = bpf_read, 277 .fo_write = bpf_write, 278 .fo_ioctl = bpf_ioctl, 279 .fo_fcntl = fnullop_fcntl, 280 .fo_poll = bpf_poll, 281 .fo_stat = bpf_stat, 282 .fo_close = bpf_close, 283 .fo_kqfilter = bpf_kqfilter, 284 .fo_restart = fnullop_restart, 285 }; 286 287 dev_type_open(bpfopen); 288 289 const struct cdevsw bpf_cdevsw = { 290 .d_open = bpfopen, 291 .d_close = noclose, 292 .d_read = noread, 293 .d_write = nowrite, 294 .d_ioctl = noioctl, 295 .d_stop = nostop, 296 .d_tty = notty, 297 .d_poll = nopoll, 298 .d_mmap = nommap, 299 .d_kqfilter = nokqfilter, 300 .d_discard = nodiscard, 301 .d_flag = D_OTHER | D_MPSAFE 302 }; 303 304 bpfjit_func_t 305 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 306 { 307 struct bpfjit_ops *ops = &bpfjit_module_ops; 308 bpfjit_func_t (*generate_code)(const bpf_ctx_t *, 309 const struct bpf_insn *, size_t); 310 311 generate_code = atomic_load_acquire(&ops->bj_generate_code); 312 if (generate_code != NULL) { 313 return generate_code(bc, code, size); 314 } 315 return NULL; 316 } 317 318 void 319 bpf_jit_freecode(bpfjit_func_t jcode) 320 { 321 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 322 bpfjit_module_ops.bj_free_code(jcode); 323 } 324 325 static int 326 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 327 struct sockaddr *sockp, struct bpf_filter **wfilter) 328 { 329 struct mbuf *m, *m0, *n; 330 int error; 331 size_t len; 332 size_t hlen; 333 size_t align; 334 u_int slen; 335 336 /* 337 * Build a sockaddr based on the data link layer type. 338 * We do this at this level because the ethernet header 339 * is copied directly into the data field of the sockaddr. 340 * In the case of SLIP, there is no header and the packet 341 * is forwarded as is. 342 * Also, we are careful to leave room at the front of the mbuf 343 * for the link level header. 344 */ 345 switch (linktype) { 346 347 case DLT_SLIP: 348 sockp->sa_family = AF_INET; 349 hlen = 0; 350 align = 0; 351 break; 352 353 case DLT_PPP: 354 sockp->sa_family = AF_UNSPEC; 355 hlen = 0; 356 align = 0; 357 break; 358 359 case DLT_EN10MB: 360 sockp->sa_family = AF_UNSPEC; 361 /* XXX Would MAXLINKHDR be better? */ 362 /* 6(dst)+6(src)+2(type) */ 363 hlen = sizeof(struct ether_header); 364 align = 2; 365 break; 366 367 case DLT_ARCNET: 368 sockp->sa_family = AF_UNSPEC; 369 hlen = ARC_HDRLEN; 370 align = 5; 371 break; 372 373 case DLT_FDDI: 374 sockp->sa_family = AF_LINK; 375 /* XXX 4(FORMAC)+6(dst)+6(src) */ 376 hlen = 16; 377 align = 0; 378 break; 379 380 case DLT_ECONET: 381 sockp->sa_family = AF_UNSPEC; 382 hlen = 6; 383 align = 2; 384 break; 385 386 case DLT_NULL: 387 sockp->sa_family = AF_UNSPEC; 388 hlen = 0; 389 align = 0; 390 break; 391 392 default: 393 return (EIO); 394 } 395 396 len = uio->uio_resid; 397 /* 398 * If there aren't enough bytes for a link level header or the 399 * packet length exceeds the interface mtu, return an error. 400 */ 401 if (len - hlen > mtu) 402 return (EMSGSIZE); 403 404 m0 = m = m_gethdr(M_WAIT, MT_DATA); 405 m_reset_rcvif(m); 406 m->m_pkthdr.len = (int)(len - hlen); 407 if (len + align > MHLEN) { 408 m_clget(m, M_WAIT); 409 if ((m->m_flags & M_EXT) == 0) { 410 error = ENOBUFS; 411 goto bad; 412 } 413 } 414 415 /* Insure the data is properly aligned */ 416 if (align > 0) 417 m->m_data += align; 418 419 for (;;) { 420 len = M_TRAILINGSPACE(m); 421 if (len > uio->uio_resid) 422 len = uio->uio_resid; 423 error = uiomove(mtod(m, void *), len, uio); 424 if (error) 425 goto bad; 426 m->m_len = len; 427 428 if (uio->uio_resid == 0) 429 break; 430 431 n = m_get(M_WAIT, MT_DATA); 432 m_clget(n, M_WAIT); /* if fails, there is no problem */ 433 m->m_next = n; 434 m = n; 435 } 436 437 slen = bpf_xfilter(wfilter, mtod(m, u_char *), len, len); 438 if (slen == 0) { 439 error = EPERM; 440 goto bad; 441 } 442 443 if (hlen != 0) { 444 /* move link level header in the top of mbuf to sa_data */ 445 memcpy(sockp->sa_data, mtod(m0, void *), hlen); 446 m0->m_data += hlen; 447 m0->m_len -= hlen; 448 } 449 450 *mp = m0; 451 return (0); 452 453 bad: 454 m_freem(m0); 455 return (error); 456 } 457 458 /* 459 * Attach file to the bpf interface, i.e. make d listen on bp. 460 */ 461 static void 462 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 463 { 464 465 KASSERT(mutex_owned(&bpf_mtx)); 466 KASSERT(mutex_owned(d->bd_mtx)); 467 /* 468 * Point d at bp, and add d to the interface's list of listeners. 469 * Finally, point the driver's bpf cookie at the interface so 470 * it will divert packets to bpf. 471 */ 472 d->bd_bif = bp; 473 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); 474 475 *bp->bif_driverp = bp; 476 } 477 478 /* 479 * Detach a file from its interface. 480 */ 481 static void 482 bpf_detachd(struct bpf_d *d) 483 { 484 struct bpf_if *bp; 485 486 KASSERT(mutex_owned(&bpf_mtx)); 487 KASSERT(mutex_owned(d->bd_mtx)); 488 489 bp = d->bd_bif; 490 /* 491 * Check if this descriptor had requested promiscuous mode. 492 * If so, turn it off. 493 */ 494 if (d->bd_promisc) { 495 int error __diagused; 496 497 d->bd_promisc = 0; 498 /* 499 * Take device out of promiscuous mode. Since we were 500 * able to enter promiscuous mode, we should be able 501 * to turn it off. But we can get an error if 502 * the interface was configured down, so only panic 503 * if we don't get an unexpected error. 504 */ 505 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 506 error = ifpromisc(bp->bif_ifp, 0); 507 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 508 #ifdef DIAGNOSTIC 509 if (error) 510 printf("%s: ifpromisc failed: %d", __func__, error); 511 #endif 512 } 513 514 /* Remove d from the interface's descriptor list. */ 515 BPFIF_DLIST_WRITER_REMOVE(d); 516 517 pserialize_perform(bpf_psz); 518 519 if (BPFIF_DLIST_WRITER_EMPTY(bp)) { 520 /* 521 * Let the driver know that there are no more listeners. 522 */ 523 *d->bd_bif->bif_driverp = NULL; 524 } 525 d->bd_bif = NULL; 526 } 527 528 static void 529 bpf_init(void) 530 { 531 532 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 533 bpf_psz = pserialize_create(); 534 bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET); 535 536 PSLIST_INIT(&bpf_iflist); 537 PSLIST_INIT(&bpf_dlist); 538 539 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat)); 540 541 return; 542 } 543 544 /* 545 * bpfilterattach() is called at boot time. We don't need to do anything 546 * here, since any initialization will happen as part of module init code. 547 */ 548 /* ARGSUSED */ 549 void 550 bpfilterattach(int n) 551 { 552 553 } 554 555 /* 556 * Open ethernet device. Clones. 557 */ 558 /* ARGSUSED */ 559 int 560 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 561 { 562 struct bpf_d *d; 563 struct file *fp; 564 int error, fd; 565 566 /* falloc() will fill in the descriptor for us. */ 567 if ((error = fd_allocfile(&fp, &fd)) != 0) 568 return error; 569 570 d = kmem_zalloc(sizeof(*d), KM_SLEEP); 571 d->bd_bufsize = bpf_bufsize; 572 d->bd_direction = BPF_D_INOUT; 573 d->bd_feedback = 0; 574 d->bd_pid = l->l_proc->p_pid; 575 #ifdef _LP64 576 if (curproc->p_flag & PK_32) 577 d->bd_compat32 = 1; 578 #endif 579 getnanotime(&d->bd_btime); 580 d->bd_atime = d->bd_mtime = d->bd_btime; 581 callout_init(&d->bd_callout, CALLOUT_MPSAFE); 582 selinit(&d->bd_sel); 583 d->bd_jitcode = NULL; 584 d->bd_rfilter = NULL; 585 d->bd_wfilter = NULL; 586 d->bd_locked = 0; 587 BPF_DLIST_ENTRY_INIT(d); 588 BPFIF_DLIST_ENTRY_INIT(d); 589 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 590 d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 591 cv_init(&d->bd_cv, "bpf"); 592 593 mutex_enter(&bpf_mtx); 594 BPF_DLIST_WRITER_INSERT_HEAD(d); 595 mutex_exit(&bpf_mtx); 596 597 return fd_clone(fp, fd, flag, &bpf_fileops, d); 598 } 599 600 /* 601 * Close the descriptor by detaching it from its interface, 602 * deallocating its buffers, and marking it free. 603 */ 604 /* ARGSUSED */ 605 static int 606 bpf_close(struct file *fp) 607 { 608 struct bpf_d *d; 609 610 mutex_enter(&bpf_mtx); 611 612 if ((d = fp->f_bpf) == NULL) { 613 mutex_exit(&bpf_mtx); 614 return 0; 615 } 616 617 /* 618 * Refresh the PID associated with this bpf file. 619 */ 620 d->bd_pid = curproc->p_pid; 621 622 mutex_enter(d->bd_mtx); 623 if (d->bd_state == BPF_WAITING) 624 callout_halt(&d->bd_callout, d->bd_mtx); 625 d->bd_state = BPF_IDLE; 626 if (d->bd_bif) 627 bpf_detachd(d); 628 mutex_exit(d->bd_mtx); 629 630 BPF_DLIST_WRITER_REMOVE(d); 631 632 pserialize_perform(bpf_psz); 633 mutex_exit(&bpf_mtx); 634 635 BPFIF_DLIST_ENTRY_DESTROY(d); 636 BPF_DLIST_ENTRY_DESTROY(d); 637 fp->f_bpf = NULL; 638 bpf_freed(d); 639 callout_destroy(&d->bd_callout); 640 seldestroy(&d->bd_sel); 641 mutex_obj_free(d->bd_mtx); 642 mutex_obj_free(d->bd_buf_mtx); 643 cv_destroy(&d->bd_cv); 644 645 kmem_free(d, sizeof(*d)); 646 647 return (0); 648 } 649 650 /* 651 * Rotate the packet buffers in descriptor d. Move the store buffer 652 * into the hold slot, and the free buffer into the store slot. 653 * Zero the length of the new store buffer. 654 */ 655 #define ROTATE_BUFFERS(d) \ 656 (d)->bd_hbuf = (d)->bd_sbuf; \ 657 (d)->bd_hlen = (d)->bd_slen; \ 658 (d)->bd_sbuf = (d)->bd_fbuf; \ 659 (d)->bd_slen = 0; \ 660 (d)->bd_fbuf = NULL; 661 /* 662 * bpfread - read next chunk of packets from buffers 663 */ 664 static int 665 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 666 kauth_cred_t cred, int flags) 667 { 668 struct bpf_d *d = fp->f_bpf; 669 int timed_out; 670 int error; 671 672 getnanotime(&d->bd_atime); 673 /* 674 * Restrict application to use a buffer the same size as 675 * the kernel buffers. 676 */ 677 if (uio->uio_resid != d->bd_bufsize) 678 return (EINVAL); 679 680 mutex_enter(d->bd_mtx); 681 if (d->bd_state == BPF_WAITING) 682 callout_halt(&d->bd_callout, d->bd_mtx); 683 timed_out = (d->bd_state == BPF_TIMED_OUT); 684 d->bd_state = BPF_IDLE; 685 mutex_exit(d->bd_mtx); 686 /* 687 * If the hold buffer is empty, then do a timed sleep, which 688 * ends when the timeout expires or when enough packets 689 * have arrived to fill the store buffer. 690 */ 691 mutex_enter(d->bd_buf_mtx); 692 while (d->bd_hbuf == NULL) { 693 if (fp->f_flag & FNONBLOCK) { 694 if (d->bd_slen == 0) { 695 error = EWOULDBLOCK; 696 goto out; 697 } 698 ROTATE_BUFFERS(d); 699 break; 700 } 701 702 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 703 /* 704 * A packet(s) either arrived since the previous 705 * read or arrived while we were asleep. 706 * Rotate the buffers and return what's here. 707 */ 708 ROTATE_BUFFERS(d); 709 break; 710 } 711 712 error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout); 713 714 if (error == EINTR || error == ERESTART) 715 goto out; 716 717 if (error == EWOULDBLOCK) { 718 /* 719 * On a timeout, return what's in the buffer, 720 * which may be nothing. If there is something 721 * in the store buffer, we can rotate the buffers. 722 */ 723 if (d->bd_hbuf) 724 /* 725 * We filled up the buffer in between 726 * getting the timeout and arriving 727 * here, so we don't need to rotate. 728 */ 729 break; 730 731 if (d->bd_slen == 0) { 732 error = 0; 733 goto out; 734 } 735 ROTATE_BUFFERS(d); 736 break; 737 } 738 if (error != 0) 739 goto out; 740 } 741 /* 742 * At this point, we know we have something in the hold slot. 743 */ 744 mutex_exit(d->bd_buf_mtx); 745 746 /* 747 * Move data from hold buffer into user space. 748 * We know the entire buffer is transferred since 749 * we checked above that the read buffer is bpf_bufsize bytes. 750 */ 751 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 752 753 mutex_enter(d->bd_buf_mtx); 754 d->bd_fbuf = d->bd_hbuf; 755 d->bd_hbuf = NULL; 756 d->bd_hlen = 0; 757 out: 758 mutex_exit(d->bd_buf_mtx); 759 return (error); 760 } 761 762 763 /* 764 * If there are processes sleeping on this descriptor, wake them up. 765 */ 766 static inline void 767 bpf_wakeup(struct bpf_d *d) 768 { 769 770 mutex_enter(d->bd_buf_mtx); 771 cv_broadcast(&d->bd_cv); 772 mutex_exit(d->bd_buf_mtx); 773 774 if (d->bd_async) 775 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 776 selnotify(&d->bd_sel, 0, 0); 777 } 778 779 static void 780 bpf_timed_out(void *arg) 781 { 782 struct bpf_d *d = arg; 783 784 mutex_enter(d->bd_mtx); 785 if (d->bd_state == BPF_WAITING) { 786 d->bd_state = BPF_TIMED_OUT; 787 if (d->bd_slen != 0) 788 bpf_wakeup(d); 789 } 790 mutex_exit(d->bd_mtx); 791 } 792 793 794 static int 795 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 796 kauth_cred_t cred, int flags) 797 { 798 struct bpf_d *d = fp->f_bpf; 799 struct bpf_if *bp; 800 struct ifnet *ifp; 801 struct mbuf *m, *mc; 802 int error; 803 static struct sockaddr_storage dst; 804 struct psref psref; 805 int bound; 806 807 m = NULL; /* XXX gcc */ 808 809 bound = curlwp_bind(); 810 mutex_enter(d->bd_mtx); 811 bp = d->bd_bif; 812 if (bp == NULL) { 813 mutex_exit(d->bd_mtx); 814 error = ENXIO; 815 goto out_bindx; 816 } 817 bpf_if_acquire(bp, &psref); 818 mutex_exit(d->bd_mtx); 819 820 getnanotime(&d->bd_mtime); 821 822 ifp = bp->bif_ifp; 823 if (if_is_deactivated(ifp)) { 824 error = ENXIO; 825 goto out; 826 } 827 828 if (uio->uio_resid == 0) { 829 error = 0; 830 goto out; 831 } 832 833 error = bpf_movein(uio, (int)bp->bif_dlt, ifp->if_mtu, &m, 834 (struct sockaddr *) &dst, &d->bd_wfilter); 835 if (error) 836 goto out; 837 838 if (m->m_pkthdr.len > ifp->if_mtu) { 839 m_freem(m); 840 error = EMSGSIZE; 841 goto out; 842 } 843 844 if (d->bd_hdrcmplt) 845 dst.ss_family = pseudo_AF_HDRCMPLT; 846 847 if (d->bd_feedback) { 848 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 849 if (mc != NULL) 850 m_set_rcvif(mc, ifp); 851 /* Set M_PROMISC for outgoing packets to be discarded. */ 852 if (1 /*d->bd_direction == BPF_D_INOUT*/) 853 m->m_flags |= M_PROMISC; 854 } else 855 mc = NULL; 856 857 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); 858 859 if (mc != NULL) { 860 if (error == 0) { 861 int s = splsoftnet(); 862 KERNEL_LOCK_UNLESS_IFP_MPSAFE(ifp); 863 ifp->_if_input(ifp, mc); 864 KERNEL_UNLOCK_UNLESS_IFP_MPSAFE(ifp); 865 splx(s); 866 } else 867 m_freem(mc); 868 } 869 /* 870 * The driver frees the mbuf. 871 */ 872 out: 873 bpf_if_release(bp, &psref); 874 out_bindx: 875 curlwp_bindx(bound); 876 return error; 877 } 878 879 /* 880 * Reset a descriptor by flushing its packet buffer and clearing the 881 * receive and drop counts. 882 */ 883 static void 884 reset_d(struct bpf_d *d) 885 { 886 887 KASSERT(mutex_owned(d->bd_mtx)); 888 889 mutex_enter(d->bd_buf_mtx); 890 if (d->bd_hbuf) { 891 /* Free the hold buffer. */ 892 d->bd_fbuf = d->bd_hbuf; 893 d->bd_hbuf = NULL; 894 } 895 d->bd_slen = 0; 896 d->bd_hlen = 0; 897 d->bd_rcount = 0; 898 d->bd_dcount = 0; 899 d->bd_ccount = 0; 900 mutex_exit(d->bd_buf_mtx); 901 } 902 903 /* 904 * FIONREAD Check for read packet available. 905 * BIOCGBLEN Get buffer len [for read()]. 906 * BIOCSETF Set ethernet read filter. 907 * BIOCFLUSH Flush read packet buffer. 908 * BIOCPROMISC Put interface into promiscuous mode. 909 * BIOCGDLT Get link layer type. 910 * BIOCGETIF Get interface name. 911 * BIOCSETIF Set interface. 912 * BIOCSRTIMEOUT Set read timeout. 913 * BIOCGRTIMEOUT Get read timeout. 914 * BIOCGSTATS Get packet stats. 915 * BIOCIMMEDIATE Set immediate mode. 916 * BIOCVERSION Get filter language version. 917 * BIOCGHDRCMPLT Get "header already complete" flag. 918 * BIOCSHDRCMPLT Set "header already complete" flag. 919 * BIOCSFEEDBACK Set packet feedback mode. 920 * BIOCGFEEDBACK Get packet feedback mode. 921 * BIOCGDIRECTION Get packet direction flag 922 * BIOCSDIRECTION Set packet direction flag 923 */ 924 /* ARGSUSED */ 925 static int 926 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 927 { 928 struct bpf_d *d = fp->f_bpf; 929 int error = 0; 930 931 /* 932 * Refresh the PID associated with this bpf file. 933 */ 934 d->bd_pid = curproc->p_pid; 935 #ifdef _LP64 936 if (curproc->p_flag & PK_32) 937 d->bd_compat32 = 1; 938 else 939 d->bd_compat32 = 0; 940 #endif 941 942 mutex_enter(d->bd_mtx); 943 if (d->bd_state == BPF_WAITING) 944 callout_halt(&d->bd_callout, d->bd_mtx); 945 d->bd_state = BPF_IDLE; 946 mutex_exit(d->bd_mtx); 947 948 if (d->bd_locked) { 949 switch (cmd) { 950 case BIOCGBLEN: /* FALLTHROUGH */ 951 case BIOCFLUSH: /* FALLTHROUGH */ 952 case BIOCGDLT: /* FALLTHROUGH */ 953 case BIOCGDLTLIST: /* FALLTHROUGH */ 954 case BIOCGETIF: /* FALLTHROUGH */ 955 case BIOCGRTIMEOUT: /* FALLTHROUGH */ 956 case BIOCGSTATS: /* FALLTHROUGH */ 957 case BIOCVERSION: /* FALLTHROUGH */ 958 case BIOCGHDRCMPLT: /* FALLTHROUGH */ 959 case FIONREAD: /* FALLTHROUGH */ 960 case BIOCLOCK: /* FALLTHROUGH */ 961 case BIOCSRTIMEOUT: /* FALLTHROUGH */ 962 case BIOCIMMEDIATE: /* FALLTHROUGH */ 963 case TIOCGPGRP: 964 break; 965 default: 966 return EPERM; 967 } 968 } 969 970 switch (cmd) { 971 972 default: 973 error = EINVAL; 974 break; 975 976 /* 977 * Check for read packet available. 978 */ 979 case FIONREAD: 980 { 981 int n; 982 983 mutex_enter(d->bd_buf_mtx); 984 n = d->bd_slen; 985 if (d->bd_hbuf) 986 n += d->bd_hlen; 987 mutex_exit(d->bd_buf_mtx); 988 989 *(int *)addr = n; 990 break; 991 } 992 993 /* 994 * Get buffer len [for read()]. 995 */ 996 case BIOCGBLEN: 997 *(u_int *)addr = d->bd_bufsize; 998 break; 999 1000 /* 1001 * Set buffer length. 1002 */ 1003 case BIOCSBLEN: 1004 /* 1005 * Forbid to change the buffer length if buffers are already 1006 * allocated. 1007 */ 1008 mutex_enter(d->bd_mtx); 1009 mutex_enter(d->bd_buf_mtx); 1010 if (d->bd_bif != NULL || d->bd_sbuf != NULL) 1011 error = EINVAL; 1012 else { 1013 u_int size = *(u_int *)addr; 1014 1015 if (size > bpf_maxbufsize) 1016 *(u_int *)addr = size = bpf_maxbufsize; 1017 else if (size < BPF_MINBUFSIZE) 1018 *(u_int *)addr = size = BPF_MINBUFSIZE; 1019 d->bd_bufsize = size; 1020 } 1021 mutex_exit(d->bd_buf_mtx); 1022 mutex_exit(d->bd_mtx); 1023 break; 1024 1025 /* 1026 * Set link layer read filter. 1027 */ 1028 case BIOCSETF: /* FALLTHROUGH */ 1029 case BIOCSETWF: 1030 error = bpf_setf(d, addr, cmd); 1031 break; 1032 1033 case BIOCLOCK: 1034 d->bd_locked = 1; 1035 break; 1036 1037 /* 1038 * Flush read packet buffer. 1039 */ 1040 case BIOCFLUSH: 1041 mutex_enter(d->bd_mtx); 1042 reset_d(d); 1043 mutex_exit(d->bd_mtx); 1044 break; 1045 1046 /* 1047 * Put interface into promiscuous mode. 1048 */ 1049 case BIOCPROMISC: 1050 mutex_enter(d->bd_mtx); 1051 if (d->bd_bif == NULL) { 1052 mutex_exit(d->bd_mtx); 1053 /* 1054 * No interface attached yet. 1055 */ 1056 error = EINVAL; 1057 break; 1058 } 1059 if (d->bd_promisc == 0) { 1060 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 1061 error = ifpromisc(d->bd_bif->bif_ifp, 1); 1062 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 1063 if (error == 0) 1064 d->bd_promisc = 1; 1065 } 1066 mutex_exit(d->bd_mtx); 1067 break; 1068 1069 /* 1070 * Get device parameters. 1071 */ 1072 case BIOCGDLT: 1073 mutex_enter(d->bd_mtx); 1074 if (d->bd_bif == NULL) 1075 error = EINVAL; 1076 else 1077 *(u_int *)addr = d->bd_bif->bif_dlt; 1078 mutex_exit(d->bd_mtx); 1079 break; 1080 1081 /* 1082 * Get a list of supported device parameters. 1083 */ 1084 case BIOCGDLTLIST: 1085 mutex_enter(d->bd_mtx); 1086 if (d->bd_bif == NULL) 1087 error = EINVAL; 1088 else 1089 error = bpf_getdltlist(d, addr); 1090 mutex_exit(d->bd_mtx); 1091 break; 1092 1093 /* 1094 * Set device parameters. 1095 */ 1096 case BIOCSDLT: 1097 mutex_enter(&bpf_mtx); 1098 mutex_enter(d->bd_mtx); 1099 if (d->bd_bif == NULL) 1100 error = EINVAL; 1101 else 1102 error = bpf_setdlt(d, *(u_int *)addr); 1103 mutex_exit(d->bd_mtx); 1104 mutex_exit(&bpf_mtx); 1105 break; 1106 1107 /* 1108 * Set interface name. 1109 */ 1110 #ifdef OBIOCGETIF 1111 case OBIOCGETIF: 1112 #endif 1113 case BIOCGETIF: 1114 mutex_enter(d->bd_mtx); 1115 if (d->bd_bif == NULL) 1116 error = EINVAL; 1117 else 1118 bpf_ifname(d->bd_bif->bif_ifp, addr); 1119 mutex_exit(d->bd_mtx); 1120 break; 1121 1122 /* 1123 * Set interface. 1124 */ 1125 #ifdef OBIOCSETIF 1126 case OBIOCSETIF: 1127 #endif 1128 case BIOCSETIF: 1129 mutex_enter(&bpf_mtx); 1130 error = bpf_setif(d, addr); 1131 mutex_exit(&bpf_mtx); 1132 break; 1133 1134 /* 1135 * Set read timeout. 1136 */ 1137 case BIOCSRTIMEOUT: 1138 { 1139 struct timeval *tv = addr; 1140 1141 /* Compute number of ticks. */ 1142 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1143 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1144 d->bd_rtout = 1; 1145 break; 1146 } 1147 1148 #ifdef BIOCGORTIMEOUT 1149 /* 1150 * Get read timeout. 1151 */ 1152 case BIOCGORTIMEOUT: 1153 { 1154 struct timeval50 *tv = addr; 1155 1156 tv->tv_sec = d->bd_rtout / hz; 1157 tv->tv_usec = (d->bd_rtout % hz) * tick; 1158 break; 1159 } 1160 #endif 1161 1162 #ifdef BIOCSORTIMEOUT 1163 /* 1164 * Set read timeout. 1165 */ 1166 case BIOCSORTIMEOUT: 1167 { 1168 struct timeval50 *tv = addr; 1169 1170 /* Compute number of ticks. */ 1171 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1172 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1173 d->bd_rtout = 1; 1174 break; 1175 } 1176 #endif 1177 1178 /* 1179 * Get read timeout. 1180 */ 1181 case BIOCGRTIMEOUT: 1182 { 1183 struct timeval *tv = addr; 1184 1185 tv->tv_sec = d->bd_rtout / hz; 1186 tv->tv_usec = (d->bd_rtout % hz) * tick; 1187 break; 1188 } 1189 /* 1190 * Get packet stats. 1191 */ 1192 case BIOCGSTATS: 1193 { 1194 struct bpf_stat *bs = addr; 1195 1196 bs->bs_recv = d->bd_rcount; 1197 bs->bs_drop = d->bd_dcount; 1198 bs->bs_capt = d->bd_ccount; 1199 break; 1200 } 1201 1202 case BIOCGSTATSOLD: 1203 { 1204 struct bpf_stat_old *bs = addr; 1205 1206 bs->bs_recv = d->bd_rcount; 1207 bs->bs_drop = d->bd_dcount; 1208 break; 1209 } 1210 1211 /* 1212 * Set immediate mode. 1213 */ 1214 case BIOCIMMEDIATE: 1215 d->bd_immediate = *(u_int *)addr; 1216 break; 1217 1218 case BIOCVERSION: 1219 { 1220 struct bpf_version *bv = addr; 1221 1222 bv->bv_major = BPF_MAJOR_VERSION; 1223 bv->bv_minor = BPF_MINOR_VERSION; 1224 break; 1225 } 1226 1227 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1228 *(u_int *)addr = d->bd_hdrcmplt; 1229 break; 1230 1231 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1232 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1233 break; 1234 1235 /* 1236 * Get packet direction flag 1237 */ 1238 case BIOCGDIRECTION: 1239 *(u_int *)addr = d->bd_direction; 1240 break; 1241 1242 /* 1243 * Set packet direction flag 1244 */ 1245 case BIOCSDIRECTION: 1246 { 1247 u_int direction; 1248 1249 direction = *(u_int *)addr; 1250 switch (direction) { 1251 case BPF_D_IN: 1252 case BPF_D_INOUT: 1253 case BPF_D_OUT: 1254 d->bd_direction = direction; 1255 break; 1256 default: 1257 error = EINVAL; 1258 } 1259 } 1260 break; 1261 1262 /* 1263 * Set "feed packets from bpf back to input" mode 1264 */ 1265 case BIOCSFEEDBACK: 1266 d->bd_feedback = *(u_int *)addr; 1267 break; 1268 1269 /* 1270 * Get "feed packets from bpf back to input" mode 1271 */ 1272 case BIOCGFEEDBACK: 1273 *(u_int *)addr = d->bd_feedback; 1274 break; 1275 1276 case FIONBIO: /* Non-blocking I/O */ 1277 /* 1278 * No need to do anything special as we use IO_NDELAY in 1279 * bpfread() as an indication of whether or not to block 1280 * the read. 1281 */ 1282 break; 1283 1284 case FIOASYNC: /* Send signal on receive packets */ 1285 mutex_enter(d->bd_mtx); 1286 d->bd_async = *(int *)addr; 1287 mutex_exit(d->bd_mtx); 1288 break; 1289 1290 case TIOCSPGRP: /* Process or group to send signals to */ 1291 case FIOSETOWN: 1292 error = fsetown(&d->bd_pgid, cmd, addr); 1293 break; 1294 1295 case TIOCGPGRP: 1296 case FIOGETOWN: 1297 error = fgetown(d->bd_pgid, cmd, addr); 1298 break; 1299 } 1300 return (error); 1301 } 1302 1303 /* 1304 * Set d's packet filter program to fp. If this file already has a filter, 1305 * free it and replace it. Returns EINVAL for bogus requests. 1306 */ 1307 static int 1308 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) 1309 { 1310 struct bpf_insn *fcode; 1311 bpfjit_func_t jcode; 1312 size_t flen, size = 0; 1313 struct bpf_filter *oldf, *newf, **storef; 1314 1315 jcode = NULL; 1316 flen = fp->bf_len; 1317 1318 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1319 return EINVAL; 1320 } 1321 1322 if (flen) { 1323 /* 1324 * Allocate the buffer, copy the byte-code from 1325 * userspace and validate it. 1326 */ 1327 size = flen * sizeof(*fp->bf_insns); 1328 fcode = kmem_alloc(size, KM_SLEEP); 1329 if (copyin(fp->bf_insns, fcode, size) != 0 || 1330 !bpf_validate(fcode, (int)flen)) { 1331 kmem_free(fcode, size); 1332 return EINVAL; 1333 } 1334 if (bpf_jit) 1335 jcode = bpf_jit_generate(NULL, fcode, flen); 1336 } else { 1337 fcode = NULL; 1338 } 1339 1340 newf = kmem_alloc(sizeof(*newf), KM_SLEEP); 1341 newf->bf_insn = fcode; 1342 newf->bf_size = size; 1343 newf->bf_jitcode = jcode; 1344 if (cmd == BIOCSETF) 1345 d->bd_jitcode = jcode; /* XXX just for kvm(3) users */ 1346 1347 /* Need to hold bpf_mtx for pserialize_perform */ 1348 mutex_enter(&bpf_mtx); 1349 mutex_enter(d->bd_mtx); 1350 if (cmd == BIOCSETWF) { 1351 oldf = d->bd_wfilter; 1352 storef = &d->bd_wfilter; 1353 } else { 1354 oldf = d->bd_rfilter; 1355 storef = &d->bd_rfilter; 1356 } 1357 atomic_store_release(storef, newf); 1358 reset_d(d); 1359 pserialize_perform(bpf_psz); 1360 mutex_exit(d->bd_mtx); 1361 mutex_exit(&bpf_mtx); 1362 1363 if (oldf != NULL) 1364 bpf_free_filter(oldf); 1365 1366 return 0; 1367 } 1368 1369 /* 1370 * Detach a file from its current interface (if attached at all) and attach 1371 * to the interface indicated by the name stored in ifr. 1372 * Return an errno or 0. 1373 */ 1374 static int 1375 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1376 { 1377 struct bpf_if *bp; 1378 char *cp; 1379 int unit_seen, i, error; 1380 1381 KASSERT(mutex_owned(&bpf_mtx)); 1382 /* 1383 * Make sure the provided name has a unit number, and default 1384 * it to '0' if not specified. 1385 * XXX This is ugly ... do this differently? 1386 */ 1387 unit_seen = 0; 1388 cp = ifr->ifr_name; 1389 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1390 while (*cp++) 1391 if (*cp >= '0' && *cp <= '9') 1392 unit_seen = 1; 1393 if (!unit_seen) { 1394 /* Make sure to leave room for the '\0'. */ 1395 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1396 if ((ifr->ifr_name[i] >= 'a' && 1397 ifr->ifr_name[i] <= 'z') || 1398 (ifr->ifr_name[i] >= 'A' && 1399 ifr->ifr_name[i] <= 'Z')) 1400 continue; 1401 ifr->ifr_name[i] = '0'; 1402 } 1403 } 1404 1405 /* 1406 * Look through attached interfaces for the named one. 1407 */ 1408 BPF_IFLIST_WRITER_FOREACH(bp) { 1409 struct ifnet *ifp = bp->bif_ifp; 1410 1411 if (ifp == NULL || 1412 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1413 continue; 1414 /* skip additional entry */ 1415 if (bp->bif_driverp != &ifp->if_bpf) 1416 continue; 1417 /* 1418 * We found the requested interface. 1419 * Allocate the packet buffers if we need to. 1420 * If we're already attached to requested interface, 1421 * just flush the buffer. 1422 */ 1423 /* 1424 * bpf_allocbufs is called only here. bpf_mtx ensures that 1425 * no race condition happen on d->bd_sbuf. 1426 */ 1427 if (d->bd_sbuf == NULL) { 1428 error = bpf_allocbufs(d); 1429 if (error != 0) 1430 return (error); 1431 } 1432 mutex_enter(d->bd_mtx); 1433 if (bp != d->bd_bif) { 1434 if (d->bd_bif) { 1435 /* 1436 * Detach if attached to something else. 1437 */ 1438 bpf_detachd(d); 1439 BPFIF_DLIST_ENTRY_INIT(d); 1440 } 1441 1442 bpf_attachd(d, bp); 1443 } 1444 reset_d(d); 1445 mutex_exit(d->bd_mtx); 1446 return (0); 1447 } 1448 /* Not found. */ 1449 return (ENXIO); 1450 } 1451 1452 /* 1453 * Copy the interface name to the ifreq. 1454 */ 1455 static void 1456 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1457 { 1458 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1459 } 1460 1461 static int 1462 bpf_stat(struct file *fp, struct stat *st) 1463 { 1464 struct bpf_d *d = fp->f_bpf; 1465 1466 (void)memset(st, 0, sizeof(*st)); 1467 mutex_enter(d->bd_mtx); 1468 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1469 st->st_atimespec = d->bd_atime; 1470 st->st_mtimespec = d->bd_mtime; 1471 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1472 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1473 st->st_gid = kauth_cred_getegid(fp->f_cred); 1474 st->st_mode = S_IFCHR; 1475 mutex_exit(d->bd_mtx); 1476 return 0; 1477 } 1478 1479 /* 1480 * Support for poll() system call 1481 * 1482 * Return true iff the specific operation will not block indefinitely - with 1483 * the assumption that it is safe to positively acknowledge a request for the 1484 * ability to write to the BPF device. 1485 * Otherwise, return false but make a note that a selnotify() must be done. 1486 */ 1487 static int 1488 bpf_poll(struct file *fp, int events) 1489 { 1490 struct bpf_d *d = fp->f_bpf; 1491 int revents; 1492 1493 /* 1494 * Refresh the PID associated with this bpf file. 1495 */ 1496 mutex_enter(&bpf_mtx); 1497 d->bd_pid = curproc->p_pid; 1498 1499 revents = events & (POLLOUT | POLLWRNORM); 1500 if (events & (POLLIN | POLLRDNORM)) { 1501 /* 1502 * An imitation of the FIONREAD ioctl code. 1503 */ 1504 mutex_enter(d->bd_mtx); 1505 if (d->bd_hlen != 0 || 1506 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1507 d->bd_slen != 0)) { 1508 revents |= events & (POLLIN | POLLRDNORM); 1509 } else { 1510 selrecord(curlwp, &d->bd_sel); 1511 /* Start the read timeout if necessary */ 1512 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1513 callout_reset(&d->bd_callout, d->bd_rtout, 1514 bpf_timed_out, d); 1515 d->bd_state = BPF_WAITING; 1516 } 1517 } 1518 mutex_exit(d->bd_mtx); 1519 } 1520 1521 mutex_exit(&bpf_mtx); 1522 return (revents); 1523 } 1524 1525 static void 1526 filt_bpfrdetach(struct knote *kn) 1527 { 1528 struct bpf_d *d = kn->kn_hook; 1529 1530 mutex_enter(d->bd_buf_mtx); 1531 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1532 mutex_exit(d->bd_buf_mtx); 1533 } 1534 1535 static int 1536 filt_bpfread(struct knote *kn, long hint) 1537 { 1538 struct bpf_d *d = kn->kn_hook; 1539 int rv; 1540 1541 mutex_enter(d->bd_buf_mtx); 1542 kn->kn_data = d->bd_hlen; 1543 if (d->bd_immediate) 1544 kn->kn_data += d->bd_slen; 1545 rv = (kn->kn_data > 0); 1546 mutex_exit(d->bd_buf_mtx); 1547 return rv; 1548 } 1549 1550 static const struct filterops bpfread_filtops = { 1551 .f_isfd = 1, 1552 .f_attach = NULL, 1553 .f_detach = filt_bpfrdetach, 1554 .f_event = filt_bpfread, 1555 }; 1556 1557 static int 1558 bpf_kqfilter(struct file *fp, struct knote *kn) 1559 { 1560 struct bpf_d *d = fp->f_bpf; 1561 struct klist *klist; 1562 1563 mutex_enter(d->bd_buf_mtx); 1564 switch (kn->kn_filter) { 1565 case EVFILT_READ: 1566 klist = &d->bd_sel.sel_klist; 1567 kn->kn_fop = &bpfread_filtops; 1568 break; 1569 1570 default: 1571 mutex_exit(d->bd_buf_mtx); 1572 return (EINVAL); 1573 } 1574 1575 kn->kn_hook = d; 1576 1577 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1578 mutex_exit(d->bd_buf_mtx); 1579 1580 return (0); 1581 } 1582 1583 /* 1584 * Copy data from an mbuf chain into a buffer. This code is derived 1585 * from m_copydata in sys/uipc_mbuf.c. 1586 */ 1587 static void * 1588 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1589 { 1590 const struct mbuf *m; 1591 u_int count; 1592 u_char *dst; 1593 1594 m = src_arg; 1595 dst = dst_arg; 1596 while (len > 0) { 1597 if (m == NULL) 1598 panic("bpf_mcpy"); 1599 count = uimin(m->m_len, len); 1600 memcpy(dst, mtod(m, const void *), count); 1601 m = m->m_next; 1602 dst += count; 1603 len -= count; 1604 } 1605 return dst_arg; 1606 } 1607 1608 static inline u_int 1609 bpf_xfilter(struct bpf_filter **filter, void *pkt, u_int pktlen, u_int buflen) 1610 { 1611 struct bpf_filter *filt; 1612 uint32_t mem[BPF_MEMWORDS]; 1613 bpf_args_t args = { 1614 .pkt = (const uint8_t *)pkt, 1615 .wirelen = pktlen, 1616 .buflen = buflen, 1617 .mem = mem, 1618 .arg = NULL 1619 }; 1620 u_int slen; 1621 1622 filt = atomic_load_consume(filter); 1623 if (filt == NULL) /* No filter means accept all. */ 1624 return (u_int)-1; 1625 1626 if (filt->bf_jitcode != NULL) 1627 slen = filt->bf_jitcode(NULL, &args); 1628 else 1629 slen = bpf_filter_ext(NULL, filt->bf_insn, &args); 1630 return slen; 1631 } 1632 1633 /* 1634 * Dispatch a packet to all the listeners on interface bp. 1635 * 1636 * pkt pointer to the packet, either a data buffer or an mbuf chain 1637 * buflen buffer length, if pkt is a data buffer 1638 * cpfn a function that can copy pkt into the listener's buffer 1639 * pktlen length of the packet 1640 * direction BPF_D_IN or BPF_D_OUT 1641 */ 1642 static inline void 1643 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1644 void *pkt, u_int pktlen, u_int buflen, const u_int direction) 1645 { 1646 bool gottime = false; 1647 struct timespec ts; 1648 struct bpf_d *d; 1649 int s; 1650 u_int slen; 1651 1652 KASSERT(!cpu_intr_p()); 1653 1654 /* 1655 * Note that the IPL does not have to be raised at this point. 1656 * The only problem that could arise here is that if two different 1657 * interfaces shared any data. This is not the case. 1658 */ 1659 s = pserialize_read_enter(); 1660 BPFIF_DLIST_READER_FOREACH(d, bp) { 1661 if (direction == BPF_D_IN) { 1662 if (d->bd_direction == BPF_D_OUT) 1663 continue; 1664 } else { /* BPF_D_OUT */ 1665 if (d->bd_direction == BPF_D_IN) 1666 continue; 1667 } 1668 1669 atomic_inc_ulong(&d->bd_rcount); 1670 BPF_STATINC(recv); 1671 1672 slen = bpf_xfilter(&d->bd_rfilter, pkt, pktlen, buflen); 1673 if (slen == 0) 1674 continue; 1675 1676 if (!gottime) { 1677 gottime = true; 1678 nanotime(&ts); 1679 } 1680 /* Assume catchpacket doesn't sleep */ 1681 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1682 } 1683 pserialize_read_exit(s); 1684 } 1685 1686 /* 1687 * Incoming linkage from device drivers, when the head of the packet is in 1688 * a buffer, and the tail is in an mbuf chain. 1689 */ 1690 static void 1691 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m, 1692 u_int direction) 1693 { 1694 u_int pktlen; 1695 struct mbuf mb; 1696 1697 /* Skip outgoing duplicate packets. */ 1698 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1699 m->m_flags &= ~M_PROMISC; 1700 return; 1701 } 1702 1703 pktlen = m_length(m) + dlen; 1704 1705 /* 1706 * Craft on-stack mbuf suitable for passing to bpf_filter. 1707 * Note that we cut corners here; we only setup what's 1708 * absolutely needed--this mbuf should never go anywhere else. 1709 */ 1710 (void)memset(&mb, 0, sizeof(mb)); 1711 mb.m_type = MT_DATA; 1712 mb.m_next = m; 1713 mb.m_data = data; 1714 mb.m_len = dlen; 1715 1716 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, direction); 1717 } 1718 1719 /* 1720 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1721 */ 1722 static void 1723 _bpf_mtap(struct bpf_if *bp, struct mbuf *m, u_int direction) 1724 { 1725 void *(*cpfn)(void *, const void *, size_t); 1726 u_int pktlen, buflen; 1727 void *marg; 1728 1729 /* Skip outgoing duplicate packets. */ 1730 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1731 m->m_flags &= ~M_PROMISC; 1732 return; 1733 } 1734 1735 pktlen = m_length(m); 1736 1737 /* Skip zero-sized packets. */ 1738 if (__predict_false(pktlen == 0)) { 1739 return; 1740 } 1741 1742 if (pktlen == m->m_len) { 1743 cpfn = (void *)memcpy; 1744 marg = mtod(m, void *); 1745 buflen = pktlen; 1746 KASSERT(buflen != 0); 1747 } else { 1748 cpfn = bpf_mcpy; 1749 marg = m; 1750 buflen = 0; 1751 } 1752 1753 bpf_deliver(bp, cpfn, marg, pktlen, buflen, direction); 1754 } 1755 1756 /* 1757 * We need to prepend the address family as 1758 * a four byte field. Cons up a dummy header 1759 * to pacify bpf. This is safe because bpf 1760 * will only read from the mbuf (i.e., it won't 1761 * try to free it or keep a pointer a to it). 1762 */ 1763 static void 1764 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m, u_int direction) 1765 { 1766 struct mbuf m0; 1767 1768 m0.m_type = MT_DATA; 1769 m0.m_flags = 0; 1770 m0.m_next = m; 1771 m0.m_nextpkt = NULL; 1772 m0.m_owner = NULL; 1773 m0.m_len = 4; 1774 m0.m_data = (char *)⁡ 1775 1776 _bpf_mtap(bp, &m0, direction); 1777 } 1778 1779 /* 1780 * Put the SLIP pseudo-"link header" in place. 1781 * Note this M_PREPEND() should never fail, 1782 * swince we know we always have enough space 1783 * in the input buffer. 1784 */ 1785 static void 1786 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1787 { 1788 u_char *hp; 1789 1790 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1791 if (*m == NULL) 1792 return; 1793 1794 hp = mtod(*m, u_char *); 1795 hp[SLX_DIR] = SLIPDIR_IN; 1796 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1797 1798 _bpf_mtap(bp, *m, BPF_D_IN); 1799 1800 m_adj(*m, SLIP_HDRLEN); 1801 } 1802 1803 /* 1804 * Put the SLIP pseudo-"link header" in 1805 * place. The compressed header is now 1806 * at the beginning of the mbuf. 1807 */ 1808 static void 1809 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1810 { 1811 struct mbuf m0; 1812 u_char *hp; 1813 1814 m0.m_type = MT_DATA; 1815 m0.m_flags = 0; 1816 m0.m_next = m; 1817 m0.m_nextpkt = NULL; 1818 m0.m_owner = NULL; 1819 m0.m_data = m0.m_dat; 1820 m0.m_len = SLIP_HDRLEN; 1821 1822 hp = mtod(&m0, u_char *); 1823 1824 hp[SLX_DIR] = SLIPDIR_OUT; 1825 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1826 1827 _bpf_mtap(bp, &m0, BPF_D_OUT); 1828 m_freem(m); 1829 } 1830 1831 static struct mbuf * 1832 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) 1833 { 1834 struct mbuf *dup; 1835 1836 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); 1837 if (dup == NULL) 1838 return NULL; 1839 1840 if (bp->bif_mbuf_tail != NULL) { 1841 bp->bif_mbuf_tail->m_nextpkt = dup; 1842 } else { 1843 bp->bif_mbuf_head = dup; 1844 } 1845 bp->bif_mbuf_tail = dup; 1846 #ifdef BPF_MTAP_SOFTINT_DEBUG 1847 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", 1848 __func__, dup, bp->bif_ifp->if_xname); 1849 #endif 1850 1851 return dup; 1852 } 1853 1854 static struct mbuf * 1855 bpf_mbuf_dequeue(struct bpf_if *bp) 1856 { 1857 struct mbuf *m; 1858 int s; 1859 1860 /* XXX NOMPSAFE: assumed running on one CPU */ 1861 s = splnet(); 1862 m = bp->bif_mbuf_head; 1863 if (m != NULL) { 1864 bp->bif_mbuf_head = m->m_nextpkt; 1865 m->m_nextpkt = NULL; 1866 1867 if (bp->bif_mbuf_head == NULL) 1868 bp->bif_mbuf_tail = NULL; 1869 #ifdef BPF_MTAP_SOFTINT_DEBUG 1870 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", 1871 __func__, m, bp->bif_ifp->if_xname); 1872 #endif 1873 } 1874 splx(s); 1875 1876 return m; 1877 } 1878 1879 static void 1880 bpf_mtap_si(void *arg) 1881 { 1882 struct bpf_if *bp = arg; 1883 struct mbuf *m; 1884 1885 while ((m = bpf_mbuf_dequeue(bp)) != NULL) { 1886 #ifdef BPF_MTAP_SOFTINT_DEBUG 1887 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", 1888 __func__, m, bp->bif_ifp->if_xname); 1889 #endif 1890 bpf_ops->bpf_mtap(bp, m, BPF_D_IN); 1891 m_freem(m); 1892 } 1893 } 1894 1895 static void 1896 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) 1897 { 1898 struct bpf_if *bp = ifp->if_bpf; 1899 struct mbuf *dup; 1900 1901 KASSERT(cpu_intr_p()); 1902 1903 /* To avoid extra invocations of the softint */ 1904 if (BPFIF_DLIST_READER_EMPTY(bp)) 1905 return; 1906 KASSERT(bp->bif_si != NULL); 1907 1908 dup = bpf_mbuf_enqueue(bp, m); 1909 if (dup != NULL) 1910 softint_schedule(bp->bif_si); 1911 } 1912 1913 static int 1914 bpf_hdrlen(struct bpf_d *d) 1915 { 1916 int hdrlen = d->bd_bif->bif_hdrlen; 1917 /* 1918 * Compute the length of the bpf header. This is not necessarily 1919 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1920 * that the network layer header begins on a longword boundary (for 1921 * performance reasons and to alleviate alignment restrictions). 1922 */ 1923 #ifdef _LP64 1924 if (d->bd_compat32) 1925 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1926 else 1927 #endif 1928 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1929 } 1930 1931 /* 1932 * Move the packet data from interface memory (pkt) into the 1933 * store buffer. Call the wakeup functions if it's time to wakeup 1934 * a listener (buffer full), "cpfn" is the routine called to do the 1935 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1936 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1937 * pkt is really an mbuf. 1938 */ 1939 static void 1940 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1941 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1942 { 1943 char *h; 1944 int totlen, curlen, caplen; 1945 int hdrlen = bpf_hdrlen(d); 1946 int do_wakeup = 0; 1947 1948 atomic_inc_ulong(&d->bd_ccount); 1949 BPF_STATINC(capt); 1950 /* 1951 * Figure out how many bytes to move. If the packet is 1952 * greater or equal to the snapshot length, transfer that 1953 * much. Otherwise, transfer the whole packet (unless 1954 * we hit the buffer size limit). 1955 */ 1956 totlen = hdrlen + uimin(snaplen, pktlen); 1957 if (totlen > d->bd_bufsize) 1958 totlen = d->bd_bufsize; 1959 /* 1960 * If we adjusted totlen to fit the bufsize, it could be that 1961 * totlen is smaller than hdrlen because of the link layer header. 1962 */ 1963 caplen = totlen - hdrlen; 1964 if (caplen < 0) 1965 caplen = 0; 1966 1967 mutex_enter(d->bd_buf_mtx); 1968 /* 1969 * Round up the end of the previous packet to the next longword. 1970 */ 1971 #ifdef _LP64 1972 if (d->bd_compat32) 1973 curlen = BPF_WORDALIGN32(d->bd_slen); 1974 else 1975 #endif 1976 curlen = BPF_WORDALIGN(d->bd_slen); 1977 if (curlen + totlen > d->bd_bufsize) { 1978 /* 1979 * This packet will overflow the storage buffer. 1980 * Rotate the buffers if we can, then wakeup any 1981 * pending reads. 1982 */ 1983 if (d->bd_fbuf == NULL) { 1984 mutex_exit(d->bd_buf_mtx); 1985 /* 1986 * We haven't completed the previous read yet, 1987 * so drop the packet. 1988 */ 1989 atomic_inc_ulong(&d->bd_dcount); 1990 BPF_STATINC(drop); 1991 return; 1992 } 1993 ROTATE_BUFFERS(d); 1994 do_wakeup = 1; 1995 curlen = 0; 1996 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1997 /* 1998 * Immediate mode is set, or the read timeout has 1999 * already expired during a select call. A packet 2000 * arrived, so the reader should be woken up. 2001 */ 2002 do_wakeup = 1; 2003 } 2004 2005 /* 2006 * Append the bpf header. 2007 */ 2008 h = (char *)d->bd_sbuf + curlen; 2009 #ifdef _LP64 2010 if (d->bd_compat32) { 2011 struct bpf_hdr32 *hp32; 2012 2013 hp32 = (struct bpf_hdr32 *)h; 2014 hp32->bh_tstamp.tv_sec = ts->tv_sec; 2015 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 2016 hp32->bh_datalen = pktlen; 2017 hp32->bh_hdrlen = hdrlen; 2018 hp32->bh_caplen = caplen; 2019 } else 2020 #endif 2021 { 2022 struct bpf_hdr *hp; 2023 2024 hp = (struct bpf_hdr *)h; 2025 hp->bh_tstamp.tv_sec = ts->tv_sec; 2026 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 2027 hp->bh_datalen = pktlen; 2028 hp->bh_hdrlen = hdrlen; 2029 hp->bh_caplen = caplen; 2030 } 2031 2032 /* 2033 * Copy the packet data into the store buffer and update its length. 2034 */ 2035 (*cpfn)(h + hdrlen, pkt, caplen); 2036 d->bd_slen = curlen + totlen; 2037 mutex_exit(d->bd_buf_mtx); 2038 2039 /* 2040 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 2041 * will cause filt_bpfread() to be called with it adjusted. 2042 */ 2043 if (do_wakeup) 2044 bpf_wakeup(d); 2045 } 2046 2047 /* 2048 * Initialize all nonzero fields of a descriptor. 2049 */ 2050 static int 2051 bpf_allocbufs(struct bpf_d *d) 2052 { 2053 2054 d->bd_fbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); 2055 if (!d->bd_fbuf) 2056 return (ENOBUFS); 2057 d->bd_sbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); 2058 if (!d->bd_sbuf) { 2059 kmem_free(d->bd_fbuf, d->bd_bufsize); 2060 return (ENOBUFS); 2061 } 2062 d->bd_slen = 0; 2063 d->bd_hlen = 0; 2064 return (0); 2065 } 2066 2067 static void 2068 bpf_free_filter(struct bpf_filter *filter) 2069 { 2070 2071 KASSERT(filter != NULL); 2072 KASSERT(filter->bf_insn != NULL); 2073 2074 kmem_free(filter->bf_insn, filter->bf_size); 2075 if (filter->bf_jitcode != NULL) 2076 bpf_jit_freecode(filter->bf_jitcode); 2077 kmem_free(filter, sizeof(*filter)); 2078 } 2079 2080 /* 2081 * Free buffers currently in use by a descriptor. 2082 * Called on close. 2083 */ 2084 static void 2085 bpf_freed(struct bpf_d *d) 2086 { 2087 /* 2088 * We don't need to lock out interrupts since this descriptor has 2089 * been detached from its interface and it yet hasn't been marked 2090 * free. 2091 */ 2092 if (d->bd_sbuf != NULL) { 2093 kmem_free(d->bd_sbuf, d->bd_bufsize); 2094 if (d->bd_hbuf != NULL) 2095 kmem_free(d->bd_hbuf, d->bd_bufsize); 2096 if (d->bd_fbuf != NULL) 2097 kmem_free(d->bd_fbuf, d->bd_bufsize); 2098 } 2099 if (d->bd_rfilter != NULL) { 2100 bpf_free_filter(d->bd_rfilter); 2101 d->bd_rfilter = NULL; 2102 } 2103 if (d->bd_wfilter != NULL) { 2104 bpf_free_filter(d->bd_wfilter); 2105 d->bd_wfilter = NULL; 2106 } 2107 d->bd_jitcode = NULL; 2108 } 2109 2110 /* 2111 * Attach an interface to bpf. dlt is the link layer type; 2112 * hdrlen is the fixed size of the link header for the specified dlt 2113 * (variable length headers not yet supported). 2114 */ 2115 static void 2116 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2117 { 2118 struct bpf_if *bp; 2119 bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP); 2120 if (bp == NULL) 2121 panic("%s: out of memory", __func__); 2122 2123 mutex_enter(&bpf_mtx); 2124 bp->bif_driverp = driverp; 2125 bp->bif_ifp = ifp; 2126 bp->bif_dlt = dlt; 2127 bp->bif_si = NULL; 2128 BPF_IFLIST_ENTRY_INIT(bp); 2129 PSLIST_INIT(&bp->bif_dlist_head); 2130 psref_target_init(&bp->bif_psref, bpf_psref_class); 2131 2132 BPF_IFLIST_WRITER_INSERT_HEAD(bp); 2133 2134 *bp->bif_driverp = NULL; 2135 2136 bp->bif_hdrlen = hdrlen; 2137 mutex_exit(&bpf_mtx); 2138 #if 0 2139 printf("bpf: %s attached\n", ifp->if_xname); 2140 #endif 2141 } 2142 2143 static void 2144 _bpf_mtap_softint_init(struct ifnet *ifp) 2145 { 2146 struct bpf_if *bp; 2147 2148 mutex_enter(&bpf_mtx); 2149 BPF_IFLIST_WRITER_FOREACH(bp) { 2150 if (bp->bif_ifp != ifp) 2151 continue; 2152 2153 bp->bif_mbuf_head = NULL; 2154 bp->bif_mbuf_tail = NULL; 2155 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); 2156 if (bp->bif_si == NULL) 2157 panic("%s: softint_establish() failed", __func__); 2158 break; 2159 } 2160 mutex_exit(&bpf_mtx); 2161 2162 if (bp == NULL) 2163 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); 2164 } 2165 2166 /* 2167 * Remove an interface from bpf. 2168 */ 2169 static void 2170 _bpfdetach(struct ifnet *ifp) 2171 { 2172 struct bpf_if *bp; 2173 struct bpf_d *d; 2174 int s; 2175 2176 mutex_enter(&bpf_mtx); 2177 /* Nuke the vnodes for any open instances */ 2178 again_d: 2179 BPF_DLIST_WRITER_FOREACH(d) { 2180 mutex_enter(d->bd_mtx); 2181 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 2182 /* 2183 * Detach the descriptor from an interface now. 2184 * It will be free'ed later by close routine. 2185 */ 2186 d->bd_promisc = 0; /* we can't touch device. */ 2187 bpf_detachd(d); 2188 mutex_exit(d->bd_mtx); 2189 goto again_d; 2190 } 2191 mutex_exit(d->bd_mtx); 2192 } 2193 2194 again: 2195 BPF_IFLIST_WRITER_FOREACH(bp) { 2196 if (bp->bif_ifp == ifp) { 2197 BPF_IFLIST_WRITER_REMOVE(bp); 2198 2199 pserialize_perform(bpf_psz); 2200 psref_target_destroy(&bp->bif_psref, bpf_psref_class); 2201 2202 BPF_IFLIST_ENTRY_DESTROY(bp); 2203 if (bp->bif_si != NULL) { 2204 /* XXX NOMPSAFE: assumed running on one CPU */ 2205 s = splnet(); 2206 while (bp->bif_mbuf_head != NULL) { 2207 struct mbuf *m = bp->bif_mbuf_head; 2208 bp->bif_mbuf_head = m->m_nextpkt; 2209 m_freem(m); 2210 } 2211 splx(s); 2212 softint_disestablish(bp->bif_si); 2213 } 2214 kmem_free(bp, sizeof(*bp)); 2215 goto again; 2216 } 2217 } 2218 mutex_exit(&bpf_mtx); 2219 } 2220 2221 /* 2222 * Change the data link type of a interface. 2223 */ 2224 static void 2225 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2226 { 2227 struct bpf_if *bp; 2228 2229 mutex_enter(&bpf_mtx); 2230 BPF_IFLIST_WRITER_FOREACH(bp) { 2231 if (bp->bif_driverp == &ifp->if_bpf) 2232 break; 2233 } 2234 if (bp == NULL) 2235 panic("bpf_change_type"); 2236 2237 bp->bif_dlt = dlt; 2238 2239 bp->bif_hdrlen = hdrlen; 2240 mutex_exit(&bpf_mtx); 2241 } 2242 2243 /* 2244 * Get a list of available data link type of the interface. 2245 */ 2246 static int 2247 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2248 { 2249 int n, error; 2250 struct ifnet *ifp; 2251 struct bpf_if *bp; 2252 int s, bound; 2253 2254 KASSERT(mutex_owned(d->bd_mtx)); 2255 2256 ifp = d->bd_bif->bif_ifp; 2257 n = 0; 2258 error = 0; 2259 2260 bound = curlwp_bind(); 2261 s = pserialize_read_enter(); 2262 BPF_IFLIST_READER_FOREACH(bp) { 2263 if (bp->bif_ifp != ifp) 2264 continue; 2265 if (bfl->bfl_list != NULL) { 2266 struct psref psref; 2267 2268 if (n >= bfl->bfl_len) { 2269 pserialize_read_exit(s); 2270 return ENOMEM; 2271 } 2272 2273 bpf_if_acquire(bp, &psref); 2274 pserialize_read_exit(s); 2275 2276 error = copyout(&bp->bif_dlt, 2277 bfl->bfl_list + n, sizeof(u_int)); 2278 2279 s = pserialize_read_enter(); 2280 bpf_if_release(bp, &psref); 2281 } 2282 n++; 2283 } 2284 pserialize_read_exit(s); 2285 curlwp_bindx(bound); 2286 2287 bfl->bfl_len = n; 2288 return error; 2289 } 2290 2291 /* 2292 * Set the data link type of a BPF instance. 2293 */ 2294 static int 2295 bpf_setdlt(struct bpf_d *d, u_int dlt) 2296 { 2297 int error, opromisc; 2298 struct ifnet *ifp; 2299 struct bpf_if *bp; 2300 2301 KASSERT(mutex_owned(&bpf_mtx)); 2302 KASSERT(mutex_owned(d->bd_mtx)); 2303 2304 if (d->bd_bif->bif_dlt == dlt) 2305 return 0; 2306 ifp = d->bd_bif->bif_ifp; 2307 BPF_IFLIST_WRITER_FOREACH(bp) { 2308 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2309 break; 2310 } 2311 if (bp == NULL) 2312 return EINVAL; 2313 opromisc = d->bd_promisc; 2314 bpf_detachd(d); 2315 BPFIF_DLIST_ENTRY_INIT(d); 2316 bpf_attachd(d, bp); 2317 reset_d(d); 2318 if (opromisc) { 2319 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 2320 error = ifpromisc(bp->bif_ifp, 1); 2321 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 2322 if (error) 2323 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 2324 bp->bif_ifp->if_xname, error); 2325 else 2326 d->bd_promisc = 1; 2327 } 2328 return 0; 2329 } 2330 2331 static int 2332 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 2333 { 2334 int newsize, error; 2335 struct sysctlnode node; 2336 2337 node = *rnode; 2338 node.sysctl_data = &newsize; 2339 newsize = bpf_maxbufsize; 2340 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2341 if (error || newp == NULL) 2342 return (error); 2343 2344 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 2345 return (EINVAL); 2346 2347 bpf_maxbufsize = newsize; 2348 2349 return (0); 2350 } 2351 2352 #if defined(MODULAR) || defined(BPFJIT) 2353 static int 2354 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 2355 { 2356 bool newval; 2357 int error; 2358 struct sysctlnode node; 2359 2360 node = *rnode; 2361 node.sysctl_data = &newval; 2362 newval = bpf_jit; 2363 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2364 if (error != 0 || newp == NULL) 2365 return error; 2366 2367 bpf_jit = newval; 2368 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 2369 printf("JIT compilation is postponed " 2370 "until after bpfjit module is loaded\n"); 2371 } 2372 2373 return 0; 2374 } 2375 #endif 2376 2377 static int 2378 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 2379 { 2380 int error, elem_count; 2381 struct bpf_d *dp; 2382 struct bpf_d_ext dpe; 2383 size_t len, needed, elem_size, out_size; 2384 char *sp; 2385 2386 if (namelen == 1 && name[0] == CTL_QUERY) 2387 return (sysctl_query(SYSCTLFN_CALL(rnode))); 2388 2389 if (namelen != 2) 2390 return (EINVAL); 2391 2392 /* BPF peers is privileged information. */ 2393 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 2394 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 2395 if (error) 2396 return (EPERM); 2397 2398 len = (oldp != NULL) ? *oldlenp : 0; 2399 sp = oldp; 2400 elem_size = name[0]; 2401 elem_count = name[1]; 2402 out_size = MIN(sizeof(dpe), elem_size); 2403 needed = 0; 2404 2405 if (elem_size < 1 || elem_count < 0) 2406 return (EINVAL); 2407 2408 mutex_enter(&bpf_mtx); 2409 BPF_DLIST_WRITER_FOREACH(dp) { 2410 if (len >= elem_size && elem_count > 0) { 2411 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 2412 BPF_EXT(bufsize); 2413 BPF_EXT(promisc); 2414 BPF_EXT(state); 2415 BPF_EXT(immediate); 2416 BPF_EXT(hdrcmplt); 2417 BPF_EXT(direction); 2418 BPF_EXT(pid); 2419 BPF_EXT(rcount); 2420 BPF_EXT(dcount); 2421 BPF_EXT(ccount); 2422 #undef BPF_EXT 2423 mutex_enter(dp->bd_mtx); 2424 if (dp->bd_bif) 2425 (void)strlcpy(dpe.bde_ifname, 2426 dp->bd_bif->bif_ifp->if_xname, 2427 IFNAMSIZ - 1); 2428 else 2429 dpe.bde_ifname[0] = '\0'; 2430 dpe.bde_locked = dp->bd_locked; 2431 mutex_exit(dp->bd_mtx); 2432 2433 error = copyout(&dpe, sp, out_size); 2434 if (error) 2435 break; 2436 sp += elem_size; 2437 len -= elem_size; 2438 } 2439 needed += elem_size; 2440 if (elem_count > 0 && elem_count != INT_MAX) 2441 elem_count--; 2442 } 2443 mutex_exit(&bpf_mtx); 2444 2445 *oldlenp = needed; 2446 2447 return (error); 2448 } 2449 2450 static void 2451 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused) 2452 { 2453 struct bpf_stat *const stats = p; 2454 struct bpf_stat *sum = arg; 2455 2456 int s = splnet(); 2457 2458 sum->bs_recv += stats->bs_recv; 2459 sum->bs_drop += stats->bs_drop; 2460 sum->bs_capt += stats->bs_capt; 2461 2462 splx(s); 2463 } 2464 2465 static int 2466 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS) 2467 { 2468 struct sysctlnode node; 2469 int error; 2470 struct bpf_stat sum; 2471 2472 memset(&sum, 0, sizeof(sum)); 2473 node = *rnode; 2474 2475 percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET), 2476 bpf_stats, &sum); 2477 2478 node.sysctl_data = ∑ 2479 node.sysctl_size = sizeof(sum); 2480 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2481 if (error != 0 || newp == NULL) 2482 return error; 2483 2484 return 0; 2485 } 2486 2487 SYSCTL_SETUP(sysctl_net_bpf_setup, "bpf sysctls") 2488 { 2489 const struct sysctlnode *node; 2490 2491 node = NULL; 2492 sysctl_createv(clog, 0, NULL, &node, 2493 CTLFLAG_PERMANENT, 2494 CTLTYPE_NODE, "bpf", 2495 SYSCTL_DESCR("BPF options"), 2496 NULL, 0, NULL, 0, 2497 CTL_NET, CTL_CREATE, CTL_EOL); 2498 if (node != NULL) { 2499 #if defined(MODULAR) || defined(BPFJIT) 2500 sysctl_createv(clog, 0, NULL, NULL, 2501 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2502 CTLTYPE_BOOL, "jit", 2503 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2504 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2505 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2506 #endif 2507 sysctl_createv(clog, 0, NULL, NULL, 2508 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2509 CTLTYPE_INT, "maxbufsize", 2510 SYSCTL_DESCR("Maximum size for data capture buffer"), 2511 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2512 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2513 sysctl_createv(clog, 0, NULL, NULL, 2514 CTLFLAG_PERMANENT, 2515 CTLTYPE_STRUCT, "stats", 2516 SYSCTL_DESCR("BPF stats"), 2517 bpf_sysctl_gstats_handler, 0, NULL, 0, 2518 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2519 sysctl_createv(clog, 0, NULL, NULL, 2520 CTLFLAG_PERMANENT, 2521 CTLTYPE_STRUCT, "peers", 2522 SYSCTL_DESCR("BPF peers"), 2523 sysctl_net_bpf_peers, 0, NULL, 0, 2524 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2525 } 2526 2527 } 2528 2529 struct bpf_ops bpf_ops_kernel = { 2530 .bpf_attach = _bpfattach, 2531 .bpf_detach = _bpfdetach, 2532 .bpf_change_type = _bpf_change_type, 2533 2534 .bpf_mtap = _bpf_mtap, 2535 .bpf_mtap2 = _bpf_mtap2, 2536 .bpf_mtap_af = _bpf_mtap_af, 2537 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2538 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2539 2540 .bpf_mtap_softint = _bpf_mtap_softint, 2541 .bpf_mtap_softint_init = _bpf_mtap_softint_init, 2542 }; 2543 2544 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); 2545 2546 static int 2547 bpf_modcmd(modcmd_t cmd, void *arg) 2548 { 2549 #ifdef _MODULE 2550 devmajor_t bmajor, cmajor; 2551 #endif 2552 int error = 0; 2553 2554 switch (cmd) { 2555 case MODULE_CMD_INIT: 2556 bpf_init(); 2557 #ifdef _MODULE 2558 bmajor = cmajor = NODEVMAJOR; 2559 error = devsw_attach("bpf", NULL, &bmajor, 2560 &bpf_cdevsw, &cmajor); 2561 if (error) 2562 break; 2563 #endif 2564 2565 bpf_ops_handover_enter(&bpf_ops_kernel); 2566 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2567 bpf_ops_handover_exit(); 2568 break; 2569 2570 case MODULE_CMD_FINI: 2571 /* 2572 * While there is no reference counting for bpf callers, 2573 * unload could at least in theory be done similarly to 2574 * system call disestablishment. This should even be 2575 * a little simpler: 2576 * 2577 * 1) replace op vector with stubs 2578 * 2) post update to all cpus with xc 2579 * 3) check that nobody is in bpf anymore 2580 * (it's doubtful we'd want something like l_sysent, 2581 * but we could do something like *signed* percpu 2582 * counters. if the sum is 0, we're good). 2583 * 4) if fail, unroll changes 2584 * 2585 * NOTE: change won't be atomic to the outside. some 2586 * packets may be not captured even if unload is 2587 * not succesful. I think packet capture not working 2588 * is a perfectly logical consequence of trying to 2589 * disable packet capture. 2590 */ 2591 error = EOPNOTSUPP; 2592 break; 2593 2594 default: 2595 error = ENOTTY; 2596 break; 2597 } 2598 2599 return error; 2600 } 2601