1 /* $NetBSD: bpf.c,v 1.189 2014/09/13 17:18:45 rmind Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.189 2014/09/13 17:18:45 rmind Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "strip.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/mbuf.h> 53 #include <sys/buf.h> 54 #include <sys/time.h> 55 #include <sys/proc.h> 56 #include <sys/ioctl.h> 57 #include <sys/conf.h> 58 #include <sys/vnode.h> 59 #include <sys/queue.h> 60 #include <sys/stat.h> 61 #include <sys/module.h> 62 #include <sys/once.h> 63 #include <sys/atomic.h> 64 65 #include <sys/file.h> 66 #include <sys/filedesc.h> 67 #include <sys/tty.h> 68 #include <sys/uio.h> 69 70 #include <sys/protosw.h> 71 #include <sys/socket.h> 72 #include <sys/errno.h> 73 #include <sys/kernel.h> 74 #include <sys/poll.h> 75 #include <sys/sysctl.h> 76 #include <sys/kauth.h> 77 78 #include <net/if.h> 79 #include <net/slip.h> 80 81 #include <net/bpf.h> 82 #include <net/bpfdesc.h> 83 #include <net/bpfjit.h> 84 85 #include <net/if_arc.h> 86 #include <net/if_ether.h> 87 88 #include <netinet/in.h> 89 #include <netinet/if_inarp.h> 90 91 92 #include <compat/sys/sockio.h> 93 94 #ifndef BPF_BUFSIZE 95 /* 96 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 97 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 98 */ 99 # define BPF_BUFSIZE 32768 100 #endif 101 102 #define PRINET 26 /* interruptible */ 103 104 /* 105 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 106 * XXX the default values should be computed dynamically based 107 * on available memory size and available mbuf clusters. 108 */ 109 int bpf_bufsize = BPF_BUFSIZE; 110 int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 111 bool bpf_jit = false; 112 113 struct bpfjit_ops bpfjit_module_ops = { 114 .bj_generate_code = NULL, 115 .bj_free_code = NULL 116 }; 117 118 /* 119 * Global BPF statistics returned by net.bpf.stats sysctl. 120 */ 121 struct bpf_stat bpf_gstats; 122 123 /* 124 * Use a mutex to avoid a race condition between gathering the stats/peers 125 * and opening/closing the device. 126 */ 127 static kmutex_t bpf_mtx; 128 129 /* 130 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 131 * bpf_dtab holds the descriptors, indexed by minor device # 132 */ 133 struct bpf_if *bpf_iflist; 134 LIST_HEAD(, bpf_d) bpf_list; 135 136 static int bpf_allocbufs(struct bpf_d *); 137 static void bpf_deliver(struct bpf_if *, 138 void *(*cpfn)(void *, const void *, size_t), 139 void *, u_int, u_int, const bool); 140 static void bpf_freed(struct bpf_d *); 141 static void bpf_ifname(struct ifnet *, struct ifreq *); 142 static void *bpf_mcpy(void *, const void *, size_t); 143 static int bpf_movein(struct uio *, int, uint64_t, 144 struct mbuf **, struct sockaddr *); 145 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 146 static void bpf_detachd(struct bpf_d *); 147 static int bpf_setif(struct bpf_d *, struct ifreq *); 148 static void bpf_timed_out(void *); 149 static inline void 150 bpf_wakeup(struct bpf_d *); 151 static int bpf_hdrlen(struct bpf_d *); 152 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 153 void *(*)(void *, const void *, size_t), struct timespec *); 154 static void reset_d(struct bpf_d *); 155 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 156 static int bpf_setdlt(struct bpf_d *, u_int); 157 158 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 159 int); 160 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 161 int); 162 static int bpf_ioctl(struct file *, u_long, void *); 163 static int bpf_poll(struct file *, int); 164 static int bpf_stat(struct file *, struct stat *); 165 static int bpf_close(struct file *); 166 static int bpf_kqfilter(struct file *, struct knote *); 167 static void bpf_softintr(void *); 168 169 static const struct fileops bpf_fileops = { 170 .fo_read = bpf_read, 171 .fo_write = bpf_write, 172 .fo_ioctl = bpf_ioctl, 173 .fo_fcntl = fnullop_fcntl, 174 .fo_poll = bpf_poll, 175 .fo_stat = bpf_stat, 176 .fo_close = bpf_close, 177 .fo_kqfilter = bpf_kqfilter, 178 .fo_restart = fnullop_restart, 179 }; 180 181 dev_type_open(bpfopen); 182 183 const struct cdevsw bpf_cdevsw = { 184 .d_open = bpfopen, 185 .d_close = noclose, 186 .d_read = noread, 187 .d_write = nowrite, 188 .d_ioctl = noioctl, 189 .d_stop = nostop, 190 .d_tty = notty, 191 .d_poll = nopoll, 192 .d_mmap = nommap, 193 .d_kqfilter = nokqfilter, 194 .d_discard = nodiscard, 195 .d_flag = D_OTHER 196 }; 197 198 bpfjit_func_t 199 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 200 { 201 202 membar_consumer(); 203 if (bpfjit_module_ops.bj_generate_code != NULL) { 204 return bpfjit_module_ops.bj_generate_code(bc, code, size); 205 } 206 return NULL; 207 } 208 209 void 210 bpf_jit_freecode(bpfjit_func_t jcode) 211 { 212 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 213 bpfjit_module_ops.bj_free_code(jcode); 214 } 215 216 static int 217 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 218 struct sockaddr *sockp) 219 { 220 struct mbuf *m; 221 int error; 222 size_t len; 223 size_t hlen; 224 size_t align; 225 226 /* 227 * Build a sockaddr based on the data link layer type. 228 * We do this at this level because the ethernet header 229 * is copied directly into the data field of the sockaddr. 230 * In the case of SLIP, there is no header and the packet 231 * is forwarded as is. 232 * Also, we are careful to leave room at the front of the mbuf 233 * for the link level header. 234 */ 235 switch (linktype) { 236 237 case DLT_SLIP: 238 sockp->sa_family = AF_INET; 239 hlen = 0; 240 align = 0; 241 break; 242 243 case DLT_PPP: 244 sockp->sa_family = AF_UNSPEC; 245 hlen = 0; 246 align = 0; 247 break; 248 249 case DLT_EN10MB: 250 sockp->sa_family = AF_UNSPEC; 251 /* XXX Would MAXLINKHDR be better? */ 252 /* 6(dst)+6(src)+2(type) */ 253 hlen = sizeof(struct ether_header); 254 align = 2; 255 break; 256 257 case DLT_ARCNET: 258 sockp->sa_family = AF_UNSPEC; 259 hlen = ARC_HDRLEN; 260 align = 5; 261 break; 262 263 case DLT_FDDI: 264 sockp->sa_family = AF_LINK; 265 /* XXX 4(FORMAC)+6(dst)+6(src) */ 266 hlen = 16; 267 align = 0; 268 break; 269 270 case DLT_ECONET: 271 sockp->sa_family = AF_UNSPEC; 272 hlen = 6; 273 align = 2; 274 break; 275 276 case DLT_NULL: 277 sockp->sa_family = AF_UNSPEC; 278 hlen = 0; 279 align = 0; 280 break; 281 282 default: 283 return (EIO); 284 } 285 286 len = uio->uio_resid; 287 /* 288 * If there aren't enough bytes for a link level header or the 289 * packet length exceeds the interface mtu, return an error. 290 */ 291 if (len - hlen > mtu) 292 return (EMSGSIZE); 293 294 /* 295 * XXX Avoid complicated buffer chaining --- 296 * bail if it won't fit in a single mbuf. 297 * (Take into account possible alignment bytes) 298 */ 299 if (len + align > MCLBYTES) 300 return (EIO); 301 302 m = m_gethdr(M_WAIT, MT_DATA); 303 m->m_pkthdr.rcvif = NULL; 304 m->m_pkthdr.len = (int)(len - hlen); 305 if (len + align > MHLEN) { 306 m_clget(m, M_WAIT); 307 if ((m->m_flags & M_EXT) == 0) { 308 error = ENOBUFS; 309 goto bad; 310 } 311 } 312 313 /* Insure the data is properly aligned */ 314 if (align > 0) { 315 m->m_data += align; 316 m->m_len -= (int)align; 317 } 318 319 error = uiomove(mtod(m, void *), len, uio); 320 if (error) 321 goto bad; 322 if (hlen != 0) { 323 memcpy(sockp->sa_data, mtod(m, void *), hlen); 324 m->m_data += hlen; /* XXX */ 325 len -= hlen; 326 } 327 m->m_len = (int)len; 328 *mp = m; 329 return (0); 330 331 bad: 332 m_freem(m); 333 return (error); 334 } 335 336 /* 337 * Attach file to the bpf interface, i.e. make d listen on bp. 338 * Must be called at splnet. 339 */ 340 static void 341 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 342 { 343 /* 344 * Point d at bp, and add d to the interface's list of listeners. 345 * Finally, point the driver's bpf cookie at the interface so 346 * it will divert packets to bpf. 347 */ 348 d->bd_bif = bp; 349 d->bd_next = bp->bif_dlist; 350 bp->bif_dlist = d; 351 352 *bp->bif_driverp = bp; 353 } 354 355 /* 356 * Detach a file from its interface. 357 */ 358 static void 359 bpf_detachd(struct bpf_d *d) 360 { 361 struct bpf_d **p; 362 struct bpf_if *bp; 363 364 bp = d->bd_bif; 365 /* 366 * Check if this descriptor had requested promiscuous mode. 367 * If so, turn it off. 368 */ 369 if (d->bd_promisc) { 370 int error __diagused; 371 372 d->bd_promisc = 0; 373 /* 374 * Take device out of promiscuous mode. Since we were 375 * able to enter promiscuous mode, we should be able 376 * to turn it off. But we can get an error if 377 * the interface was configured down, so only panic 378 * if we don't get an unexpected error. 379 */ 380 error = ifpromisc(bp->bif_ifp, 0); 381 #ifdef DIAGNOSTIC 382 if (error) 383 printf("%s: ifpromisc failed: %d", __func__, error); 384 #endif 385 } 386 /* Remove d from the interface's descriptor list. */ 387 p = &bp->bif_dlist; 388 while (*p != d) { 389 p = &(*p)->bd_next; 390 if (*p == NULL) 391 panic("%s: descriptor not in list", __func__); 392 } 393 *p = (*p)->bd_next; 394 if (bp->bif_dlist == NULL) 395 /* 396 * Let the driver know that there are no more listeners. 397 */ 398 *d->bd_bif->bif_driverp = NULL; 399 d->bd_bif = NULL; 400 } 401 402 static int 403 doinit(void) 404 { 405 406 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 407 408 LIST_INIT(&bpf_list); 409 410 bpf_gstats.bs_recv = 0; 411 bpf_gstats.bs_drop = 0; 412 bpf_gstats.bs_capt = 0; 413 414 return 0; 415 } 416 417 /* 418 * bpfilterattach() is called at boot time. 419 */ 420 /* ARGSUSED */ 421 void 422 bpfilterattach(int n) 423 { 424 static ONCE_DECL(control); 425 426 RUN_ONCE(&control, doinit); 427 } 428 429 /* 430 * Open ethernet device. Clones. 431 */ 432 /* ARGSUSED */ 433 int 434 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 435 { 436 struct bpf_d *d; 437 struct file *fp; 438 int error, fd; 439 440 /* falloc() will use the descriptor for us. */ 441 if ((error = fd_allocfile(&fp, &fd)) != 0) 442 return error; 443 444 d = malloc(sizeof(*d), M_DEVBUF, M_WAITOK|M_ZERO); 445 d->bd_bufsize = bpf_bufsize; 446 d->bd_seesent = 1; 447 d->bd_feedback = 0; 448 d->bd_pid = l->l_proc->p_pid; 449 #ifdef _LP64 450 if (curproc->p_flag & PK_32) 451 d->bd_compat32 = 1; 452 #endif 453 getnanotime(&d->bd_btime); 454 d->bd_atime = d->bd_mtime = d->bd_btime; 455 callout_init(&d->bd_callout, 0); 456 selinit(&d->bd_sel); 457 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d); 458 d->bd_jitcode = NULL; 459 460 mutex_enter(&bpf_mtx); 461 LIST_INSERT_HEAD(&bpf_list, d, bd_list); 462 mutex_exit(&bpf_mtx); 463 464 return fd_clone(fp, fd, flag, &bpf_fileops, d); 465 } 466 467 /* 468 * Close the descriptor by detaching it from its interface, 469 * deallocating its buffers, and marking it free. 470 */ 471 /* ARGSUSED */ 472 static int 473 bpf_close(struct file *fp) 474 { 475 struct bpf_d *d = fp->f_bpf; 476 int s; 477 478 KERNEL_LOCK(1, NULL); 479 480 /* 481 * Refresh the PID associated with this bpf file. 482 */ 483 d->bd_pid = curproc->p_pid; 484 485 s = splnet(); 486 if (d->bd_state == BPF_WAITING) 487 callout_stop(&d->bd_callout); 488 d->bd_state = BPF_IDLE; 489 if (d->bd_bif) 490 bpf_detachd(d); 491 splx(s); 492 bpf_freed(d); 493 mutex_enter(&bpf_mtx); 494 LIST_REMOVE(d, bd_list); 495 mutex_exit(&bpf_mtx); 496 callout_destroy(&d->bd_callout); 497 seldestroy(&d->bd_sel); 498 softint_disestablish(d->bd_sih); 499 free(d, M_DEVBUF); 500 fp->f_bpf = NULL; 501 502 KERNEL_UNLOCK_ONE(NULL); 503 504 return (0); 505 } 506 507 /* 508 * Rotate the packet buffers in descriptor d. Move the store buffer 509 * into the hold slot, and the free buffer into the store slot. 510 * Zero the length of the new store buffer. 511 */ 512 #define ROTATE_BUFFERS(d) \ 513 (d)->bd_hbuf = (d)->bd_sbuf; \ 514 (d)->bd_hlen = (d)->bd_slen; \ 515 (d)->bd_sbuf = (d)->bd_fbuf; \ 516 (d)->bd_slen = 0; \ 517 (d)->bd_fbuf = NULL; 518 /* 519 * bpfread - read next chunk of packets from buffers 520 */ 521 static int 522 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 523 kauth_cred_t cred, int flags) 524 { 525 struct bpf_d *d = fp->f_bpf; 526 int timed_out; 527 int error; 528 int s; 529 530 getnanotime(&d->bd_atime); 531 /* 532 * Restrict application to use a buffer the same size as 533 * the kernel buffers. 534 */ 535 if (uio->uio_resid != d->bd_bufsize) 536 return (EINVAL); 537 538 KERNEL_LOCK(1, NULL); 539 s = splnet(); 540 if (d->bd_state == BPF_WAITING) 541 callout_stop(&d->bd_callout); 542 timed_out = (d->bd_state == BPF_TIMED_OUT); 543 d->bd_state = BPF_IDLE; 544 /* 545 * If the hold buffer is empty, then do a timed sleep, which 546 * ends when the timeout expires or when enough packets 547 * have arrived to fill the store buffer. 548 */ 549 while (d->bd_hbuf == NULL) { 550 if (fp->f_flag & FNONBLOCK) { 551 if (d->bd_slen == 0) { 552 splx(s); 553 KERNEL_UNLOCK_ONE(NULL); 554 return (EWOULDBLOCK); 555 } 556 ROTATE_BUFFERS(d); 557 break; 558 } 559 560 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 561 /* 562 * A packet(s) either arrived since the previous 563 * read or arrived while we were asleep. 564 * Rotate the buffers and return what's here. 565 */ 566 ROTATE_BUFFERS(d); 567 break; 568 } 569 error = tsleep(d, PRINET|PCATCH, "bpf", 570 d->bd_rtout); 571 if (error == EINTR || error == ERESTART) { 572 splx(s); 573 KERNEL_UNLOCK_ONE(NULL); 574 return (error); 575 } 576 if (error == EWOULDBLOCK) { 577 /* 578 * On a timeout, return what's in the buffer, 579 * which may be nothing. If there is something 580 * in the store buffer, we can rotate the buffers. 581 */ 582 if (d->bd_hbuf) 583 /* 584 * We filled up the buffer in between 585 * getting the timeout and arriving 586 * here, so we don't need to rotate. 587 */ 588 break; 589 590 if (d->bd_slen == 0) { 591 splx(s); 592 KERNEL_UNLOCK_ONE(NULL); 593 return (0); 594 } 595 ROTATE_BUFFERS(d); 596 break; 597 } 598 if (error != 0) 599 goto done; 600 } 601 /* 602 * At this point, we know we have something in the hold slot. 603 */ 604 splx(s); 605 606 /* 607 * Move data from hold buffer into user space. 608 * We know the entire buffer is transferred since 609 * we checked above that the read buffer is bpf_bufsize bytes. 610 */ 611 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 612 613 s = splnet(); 614 d->bd_fbuf = d->bd_hbuf; 615 d->bd_hbuf = NULL; 616 d->bd_hlen = 0; 617 done: 618 splx(s); 619 KERNEL_UNLOCK_ONE(NULL); 620 return (error); 621 } 622 623 624 /* 625 * If there are processes sleeping on this descriptor, wake them up. 626 */ 627 static inline void 628 bpf_wakeup(struct bpf_d *d) 629 { 630 wakeup(d); 631 if (d->bd_async) 632 softint_schedule(d->bd_sih); 633 selnotify(&d->bd_sel, 0, 0); 634 } 635 636 static void 637 bpf_softintr(void *cookie) 638 { 639 struct bpf_d *d; 640 641 d = cookie; 642 if (d->bd_async) 643 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 644 } 645 646 static void 647 bpf_timed_out(void *arg) 648 { 649 struct bpf_d *d = arg; 650 int s; 651 652 s = splnet(); 653 if (d->bd_state == BPF_WAITING) { 654 d->bd_state = BPF_TIMED_OUT; 655 if (d->bd_slen != 0) 656 bpf_wakeup(d); 657 } 658 splx(s); 659 } 660 661 662 static int 663 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 664 kauth_cred_t cred, int flags) 665 { 666 struct bpf_d *d = fp->f_bpf; 667 struct ifnet *ifp; 668 struct mbuf *m, *mc; 669 int error, s; 670 static struct sockaddr_storage dst; 671 672 m = NULL; /* XXX gcc */ 673 674 KERNEL_LOCK(1, NULL); 675 676 if (d->bd_bif == NULL) { 677 KERNEL_UNLOCK_ONE(NULL); 678 return (ENXIO); 679 } 680 getnanotime(&d->bd_mtime); 681 682 ifp = d->bd_bif->bif_ifp; 683 684 if (uio->uio_resid == 0) { 685 KERNEL_UNLOCK_ONE(NULL); 686 return (0); 687 } 688 689 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp->if_mtu, &m, 690 (struct sockaddr *) &dst); 691 if (error) { 692 KERNEL_UNLOCK_ONE(NULL); 693 return (error); 694 } 695 696 if (m->m_pkthdr.len > ifp->if_mtu) { 697 KERNEL_UNLOCK_ONE(NULL); 698 m_freem(m); 699 return (EMSGSIZE); 700 } 701 702 if (d->bd_hdrcmplt) 703 dst.ss_family = pseudo_AF_HDRCMPLT; 704 705 if (d->bd_feedback) { 706 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 707 if (mc != NULL) 708 mc->m_pkthdr.rcvif = ifp; 709 /* Set M_PROMISC for outgoing packets to be discarded. */ 710 if (1 /*d->bd_direction == BPF_D_INOUT*/) 711 m->m_flags |= M_PROMISC; 712 } else 713 mc = NULL; 714 715 s = splsoftnet(); 716 error = (*ifp->if_output)(ifp, m, (struct sockaddr *) &dst, NULL); 717 718 if (mc != NULL) { 719 if (error == 0) 720 (*ifp->if_input)(ifp, mc); 721 m_freem(mc); 722 } 723 splx(s); 724 KERNEL_UNLOCK_ONE(NULL); 725 /* 726 * The driver frees the mbuf. 727 */ 728 return (error); 729 } 730 731 /* 732 * Reset a descriptor by flushing its packet buffer and clearing the 733 * receive and drop counts. Should be called at splnet. 734 */ 735 static void 736 reset_d(struct bpf_d *d) 737 { 738 if (d->bd_hbuf) { 739 /* Free the hold buffer. */ 740 d->bd_fbuf = d->bd_hbuf; 741 d->bd_hbuf = NULL; 742 } 743 d->bd_slen = 0; 744 d->bd_hlen = 0; 745 d->bd_rcount = 0; 746 d->bd_dcount = 0; 747 d->bd_ccount = 0; 748 } 749 750 /* 751 * FIONREAD Check for read packet available. 752 * BIOCGBLEN Get buffer len [for read()]. 753 * BIOCSETF Set ethernet read filter. 754 * BIOCFLUSH Flush read packet buffer. 755 * BIOCPROMISC Put interface into promiscuous mode. 756 * BIOCGDLT Get link layer type. 757 * BIOCGETIF Get interface name. 758 * BIOCSETIF Set interface. 759 * BIOCSRTIMEOUT Set read timeout. 760 * BIOCGRTIMEOUT Get read timeout. 761 * BIOCGSTATS Get packet stats. 762 * BIOCIMMEDIATE Set immediate mode. 763 * BIOCVERSION Get filter language version. 764 * BIOCGHDRCMPLT Get "header already complete" flag. 765 * BIOCSHDRCMPLT Set "header already complete" flag. 766 * BIOCSFEEDBACK Set packet feedback mode. 767 * BIOCGFEEDBACK Get packet feedback mode. 768 * BIOCGSEESENT Get "see sent packets" mode. 769 * BIOCSSEESENT Set "see sent packets" mode. 770 */ 771 /* ARGSUSED */ 772 static int 773 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 774 { 775 struct bpf_d *d = fp->f_bpf; 776 int s, error = 0; 777 778 /* 779 * Refresh the PID associated with this bpf file. 780 */ 781 KERNEL_LOCK(1, NULL); 782 d->bd_pid = curproc->p_pid; 783 #ifdef _LP64 784 if (curproc->p_flag & PK_32) 785 d->bd_compat32 = 1; 786 else 787 d->bd_compat32 = 0; 788 #endif 789 790 s = splnet(); 791 if (d->bd_state == BPF_WAITING) 792 callout_stop(&d->bd_callout); 793 d->bd_state = BPF_IDLE; 794 splx(s); 795 796 switch (cmd) { 797 798 default: 799 error = EINVAL; 800 break; 801 802 /* 803 * Check for read packet available. 804 */ 805 case FIONREAD: 806 { 807 int n; 808 809 s = splnet(); 810 n = d->bd_slen; 811 if (d->bd_hbuf) 812 n += d->bd_hlen; 813 splx(s); 814 815 *(int *)addr = n; 816 break; 817 } 818 819 /* 820 * Get buffer len [for read()]. 821 */ 822 case BIOCGBLEN: 823 *(u_int *)addr = d->bd_bufsize; 824 break; 825 826 /* 827 * Set buffer length. 828 */ 829 case BIOCSBLEN: 830 if (d->bd_bif != NULL) 831 error = EINVAL; 832 else { 833 u_int size = *(u_int *)addr; 834 835 if (size > bpf_maxbufsize) 836 *(u_int *)addr = size = bpf_maxbufsize; 837 else if (size < BPF_MINBUFSIZE) 838 *(u_int *)addr = size = BPF_MINBUFSIZE; 839 d->bd_bufsize = size; 840 } 841 break; 842 843 /* 844 * Set link layer read filter. 845 */ 846 case BIOCSETF: 847 error = bpf_setf(d, addr); 848 break; 849 850 /* 851 * Flush read packet buffer. 852 */ 853 case BIOCFLUSH: 854 s = splnet(); 855 reset_d(d); 856 splx(s); 857 break; 858 859 /* 860 * Put interface into promiscuous mode. 861 */ 862 case BIOCPROMISC: 863 if (d->bd_bif == NULL) { 864 /* 865 * No interface attached yet. 866 */ 867 error = EINVAL; 868 break; 869 } 870 s = splnet(); 871 if (d->bd_promisc == 0) { 872 error = ifpromisc(d->bd_bif->bif_ifp, 1); 873 if (error == 0) 874 d->bd_promisc = 1; 875 } 876 splx(s); 877 break; 878 879 /* 880 * Get device parameters. 881 */ 882 case BIOCGDLT: 883 if (d->bd_bif == NULL) 884 error = EINVAL; 885 else 886 *(u_int *)addr = d->bd_bif->bif_dlt; 887 break; 888 889 /* 890 * Get a list of supported device parameters. 891 */ 892 case BIOCGDLTLIST: 893 if (d->bd_bif == NULL) 894 error = EINVAL; 895 else 896 error = bpf_getdltlist(d, addr); 897 break; 898 899 /* 900 * Set device parameters. 901 */ 902 case BIOCSDLT: 903 if (d->bd_bif == NULL) 904 error = EINVAL; 905 else 906 error = bpf_setdlt(d, *(u_int *)addr); 907 break; 908 909 /* 910 * Set interface name. 911 */ 912 #ifdef OBIOCGETIF 913 case OBIOCGETIF: 914 #endif 915 case BIOCGETIF: 916 if (d->bd_bif == NULL) 917 error = EINVAL; 918 else 919 bpf_ifname(d->bd_bif->bif_ifp, addr); 920 break; 921 922 /* 923 * Set interface. 924 */ 925 #ifdef OBIOCSETIF 926 case OBIOCSETIF: 927 #endif 928 case BIOCSETIF: 929 error = bpf_setif(d, addr); 930 break; 931 932 /* 933 * Set read timeout. 934 */ 935 case BIOCSRTIMEOUT: 936 { 937 struct timeval *tv = addr; 938 939 /* Compute number of ticks. */ 940 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 941 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 942 d->bd_rtout = 1; 943 break; 944 } 945 946 #ifdef BIOCGORTIMEOUT 947 /* 948 * Get read timeout. 949 */ 950 case BIOCGORTIMEOUT: 951 { 952 struct timeval50 *tv = addr; 953 954 tv->tv_sec = d->bd_rtout / hz; 955 tv->tv_usec = (d->bd_rtout % hz) * tick; 956 break; 957 } 958 #endif 959 960 #ifdef BIOCSORTIMEOUT 961 /* 962 * Set read timeout. 963 */ 964 case BIOCSORTIMEOUT: 965 { 966 struct timeval50 *tv = addr; 967 968 /* Compute number of ticks. */ 969 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 970 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 971 d->bd_rtout = 1; 972 break; 973 } 974 #endif 975 976 /* 977 * Get read timeout. 978 */ 979 case BIOCGRTIMEOUT: 980 { 981 struct timeval *tv = addr; 982 983 tv->tv_sec = d->bd_rtout / hz; 984 tv->tv_usec = (d->bd_rtout % hz) * tick; 985 break; 986 } 987 /* 988 * Get packet stats. 989 */ 990 case BIOCGSTATS: 991 { 992 struct bpf_stat *bs = addr; 993 994 bs->bs_recv = d->bd_rcount; 995 bs->bs_drop = d->bd_dcount; 996 bs->bs_capt = d->bd_ccount; 997 break; 998 } 999 1000 case BIOCGSTATSOLD: 1001 { 1002 struct bpf_stat_old *bs = addr; 1003 1004 bs->bs_recv = d->bd_rcount; 1005 bs->bs_drop = d->bd_dcount; 1006 break; 1007 } 1008 1009 /* 1010 * Set immediate mode. 1011 */ 1012 case BIOCIMMEDIATE: 1013 d->bd_immediate = *(u_int *)addr; 1014 break; 1015 1016 case BIOCVERSION: 1017 { 1018 struct bpf_version *bv = addr; 1019 1020 bv->bv_major = BPF_MAJOR_VERSION; 1021 bv->bv_minor = BPF_MINOR_VERSION; 1022 break; 1023 } 1024 1025 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1026 *(u_int *)addr = d->bd_hdrcmplt; 1027 break; 1028 1029 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1030 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1031 break; 1032 1033 /* 1034 * Get "see sent packets" flag 1035 */ 1036 case BIOCGSEESENT: 1037 *(u_int *)addr = d->bd_seesent; 1038 break; 1039 1040 /* 1041 * Set "see sent" packets flag 1042 */ 1043 case BIOCSSEESENT: 1044 d->bd_seesent = *(u_int *)addr; 1045 break; 1046 1047 /* 1048 * Set "feed packets from bpf back to input" mode 1049 */ 1050 case BIOCSFEEDBACK: 1051 d->bd_feedback = *(u_int *)addr; 1052 break; 1053 1054 /* 1055 * Get "feed packets from bpf back to input" mode 1056 */ 1057 case BIOCGFEEDBACK: 1058 *(u_int *)addr = d->bd_feedback; 1059 break; 1060 1061 case FIONBIO: /* Non-blocking I/O */ 1062 /* 1063 * No need to do anything special as we use IO_NDELAY in 1064 * bpfread() as an indication of whether or not to block 1065 * the read. 1066 */ 1067 break; 1068 1069 case FIOASYNC: /* Send signal on receive packets */ 1070 d->bd_async = *(int *)addr; 1071 break; 1072 1073 case TIOCSPGRP: /* Process or group to send signals to */ 1074 case FIOSETOWN: 1075 error = fsetown(&d->bd_pgid, cmd, addr); 1076 break; 1077 1078 case TIOCGPGRP: 1079 case FIOGETOWN: 1080 error = fgetown(d->bd_pgid, cmd, addr); 1081 break; 1082 } 1083 KERNEL_UNLOCK_ONE(NULL); 1084 return (error); 1085 } 1086 1087 /* 1088 * Set d's packet filter program to fp. If this file already has a filter, 1089 * free it and replace it. Returns EINVAL for bogus requests. 1090 */ 1091 int 1092 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1093 { 1094 struct bpf_insn *fcode, *old; 1095 bpfjit_func_t jcode, oldj; 1096 size_t flen, size; 1097 int s; 1098 1099 jcode = NULL; 1100 flen = fp->bf_len; 1101 1102 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1103 return EINVAL; 1104 } 1105 1106 if (flen) { 1107 /* 1108 * Allocate the buffer, copy the byte-code from 1109 * userspace and validate it. 1110 */ 1111 size = flen * sizeof(*fp->bf_insns); 1112 fcode = malloc(size, M_DEVBUF, M_WAITOK); 1113 if (copyin(fp->bf_insns, fcode, size) != 0 || 1114 !bpf_validate(fcode, (int)flen)) { 1115 free(fcode, M_DEVBUF); 1116 return EINVAL; 1117 } 1118 membar_consumer(); 1119 if (bpf_jit) 1120 jcode = bpf_jit_generate(NULL, fcode, flen); 1121 } else { 1122 fcode = NULL; 1123 } 1124 1125 s = splnet(); 1126 old = d->bd_filter; 1127 d->bd_filter = fcode; 1128 oldj = d->bd_jitcode; 1129 d->bd_jitcode = jcode; 1130 reset_d(d); 1131 splx(s); 1132 1133 if (old) { 1134 free(old, M_DEVBUF); 1135 } 1136 if (oldj) { 1137 bpf_jit_freecode(oldj); 1138 } 1139 1140 return 0; 1141 } 1142 1143 /* 1144 * Detach a file from its current interface (if attached at all) and attach 1145 * to the interface indicated by the name stored in ifr. 1146 * Return an errno or 0. 1147 */ 1148 static int 1149 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1150 { 1151 struct bpf_if *bp; 1152 char *cp; 1153 int unit_seen, i, s, error; 1154 1155 /* 1156 * Make sure the provided name has a unit number, and default 1157 * it to '0' if not specified. 1158 * XXX This is ugly ... do this differently? 1159 */ 1160 unit_seen = 0; 1161 cp = ifr->ifr_name; 1162 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1163 while (*cp++) 1164 if (*cp >= '0' && *cp <= '9') 1165 unit_seen = 1; 1166 if (!unit_seen) { 1167 /* Make sure to leave room for the '\0'. */ 1168 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1169 if ((ifr->ifr_name[i] >= 'a' && 1170 ifr->ifr_name[i] <= 'z') || 1171 (ifr->ifr_name[i] >= 'A' && 1172 ifr->ifr_name[i] <= 'Z')) 1173 continue; 1174 ifr->ifr_name[i] = '0'; 1175 } 1176 } 1177 1178 /* 1179 * Look through attached interfaces for the named one. 1180 */ 1181 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1182 struct ifnet *ifp = bp->bif_ifp; 1183 1184 if (ifp == NULL || 1185 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1186 continue; 1187 /* skip additional entry */ 1188 if (bp->bif_driverp != &ifp->if_bpf) 1189 continue; 1190 /* 1191 * We found the requested interface. 1192 * Allocate the packet buffers if we need to. 1193 * If we're already attached to requested interface, 1194 * just flush the buffer. 1195 */ 1196 if (d->bd_sbuf == NULL) { 1197 error = bpf_allocbufs(d); 1198 if (error != 0) 1199 return (error); 1200 } 1201 s = splnet(); 1202 if (bp != d->bd_bif) { 1203 if (d->bd_bif) 1204 /* 1205 * Detach if attached to something else. 1206 */ 1207 bpf_detachd(d); 1208 1209 bpf_attachd(d, bp); 1210 } 1211 reset_d(d); 1212 splx(s); 1213 return (0); 1214 } 1215 /* Not found. */ 1216 return (ENXIO); 1217 } 1218 1219 /* 1220 * Copy the interface name to the ifreq. 1221 */ 1222 static void 1223 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1224 { 1225 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1226 } 1227 1228 static int 1229 bpf_stat(struct file *fp, struct stat *st) 1230 { 1231 struct bpf_d *d = fp->f_bpf; 1232 1233 (void)memset(st, 0, sizeof(*st)); 1234 KERNEL_LOCK(1, NULL); 1235 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1236 st->st_atimespec = d->bd_atime; 1237 st->st_mtimespec = d->bd_mtime; 1238 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1239 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1240 st->st_gid = kauth_cred_getegid(fp->f_cred); 1241 st->st_mode = S_IFCHR; 1242 KERNEL_UNLOCK_ONE(NULL); 1243 return 0; 1244 } 1245 1246 /* 1247 * Support for poll() system call 1248 * 1249 * Return true iff the specific operation will not block indefinitely - with 1250 * the assumption that it is safe to positively acknowledge a request for the 1251 * ability to write to the BPF device. 1252 * Otherwise, return false but make a note that a selnotify() must be done. 1253 */ 1254 static int 1255 bpf_poll(struct file *fp, int events) 1256 { 1257 struct bpf_d *d = fp->f_bpf; 1258 int s = splnet(); 1259 int revents; 1260 1261 /* 1262 * Refresh the PID associated with this bpf file. 1263 */ 1264 KERNEL_LOCK(1, NULL); 1265 d->bd_pid = curproc->p_pid; 1266 1267 revents = events & (POLLOUT | POLLWRNORM); 1268 if (events & (POLLIN | POLLRDNORM)) { 1269 /* 1270 * An imitation of the FIONREAD ioctl code. 1271 */ 1272 if (d->bd_hlen != 0 || 1273 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1274 d->bd_slen != 0)) { 1275 revents |= events & (POLLIN | POLLRDNORM); 1276 } else { 1277 selrecord(curlwp, &d->bd_sel); 1278 /* Start the read timeout if necessary */ 1279 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1280 callout_reset(&d->bd_callout, d->bd_rtout, 1281 bpf_timed_out, d); 1282 d->bd_state = BPF_WAITING; 1283 } 1284 } 1285 } 1286 1287 KERNEL_UNLOCK_ONE(NULL); 1288 splx(s); 1289 return (revents); 1290 } 1291 1292 static void 1293 filt_bpfrdetach(struct knote *kn) 1294 { 1295 struct bpf_d *d = kn->kn_hook; 1296 int s; 1297 1298 KERNEL_LOCK(1, NULL); 1299 s = splnet(); 1300 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1301 splx(s); 1302 KERNEL_UNLOCK_ONE(NULL); 1303 } 1304 1305 static int 1306 filt_bpfread(struct knote *kn, long hint) 1307 { 1308 struct bpf_d *d = kn->kn_hook; 1309 int rv; 1310 1311 KERNEL_LOCK(1, NULL); 1312 kn->kn_data = d->bd_hlen; 1313 if (d->bd_immediate) 1314 kn->kn_data += d->bd_slen; 1315 rv = (kn->kn_data > 0); 1316 KERNEL_UNLOCK_ONE(NULL); 1317 return rv; 1318 } 1319 1320 static const struct filterops bpfread_filtops = 1321 { 1, NULL, filt_bpfrdetach, filt_bpfread }; 1322 1323 static int 1324 bpf_kqfilter(struct file *fp, struct knote *kn) 1325 { 1326 struct bpf_d *d = fp->f_bpf; 1327 struct klist *klist; 1328 int s; 1329 1330 KERNEL_LOCK(1, NULL); 1331 1332 switch (kn->kn_filter) { 1333 case EVFILT_READ: 1334 klist = &d->bd_sel.sel_klist; 1335 kn->kn_fop = &bpfread_filtops; 1336 break; 1337 1338 default: 1339 KERNEL_UNLOCK_ONE(NULL); 1340 return (EINVAL); 1341 } 1342 1343 kn->kn_hook = d; 1344 1345 s = splnet(); 1346 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1347 splx(s); 1348 KERNEL_UNLOCK_ONE(NULL); 1349 1350 return (0); 1351 } 1352 1353 /* 1354 * Copy data from an mbuf chain into a buffer. This code is derived 1355 * from m_copydata in sys/uipc_mbuf.c. 1356 */ 1357 static void * 1358 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1359 { 1360 const struct mbuf *m; 1361 u_int count; 1362 u_char *dst; 1363 1364 m = src_arg; 1365 dst = dst_arg; 1366 while (len > 0) { 1367 if (m == NULL) 1368 panic("bpf_mcpy"); 1369 count = min(m->m_len, len); 1370 memcpy(dst, mtod(m, const void *), count); 1371 m = m->m_next; 1372 dst += count; 1373 len -= count; 1374 } 1375 return dst_arg; 1376 } 1377 1378 /* 1379 * Dispatch a packet to all the listeners on interface bp. 1380 * 1381 * pkt pointer to the packet, either a data buffer or an mbuf chain 1382 * buflen buffer length, if pkt is a data buffer 1383 * cpfn a function that can copy pkt into the listener's buffer 1384 * pktlen length of the packet 1385 * rcv true if packet came in 1386 */ 1387 static inline void 1388 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1389 void *pkt, u_int pktlen, u_int buflen, const bool rcv) 1390 { 1391 const bpf_ctx_t *bc = NULL; 1392 uint32_t mem[BPF_MEMWORDS]; 1393 bpf_args_t args = { 1394 .pkt = (const uint8_t *)pkt, 1395 .wirelen = pktlen, 1396 .buflen = buflen, 1397 .mem = mem, 1398 .arg = NULL 1399 }; 1400 bool gottime = false; 1401 struct timespec ts; 1402 1403 /* 1404 * Note that the IPL does not have to be raised at this point. 1405 * The only problem that could arise here is that if two different 1406 * interfaces shared any data. This is not the case. 1407 */ 1408 for (struct bpf_d *d = bp->bif_dlist; d != NULL; d = d->bd_next) { 1409 u_int slen; 1410 1411 if (!d->bd_seesent && !rcv) { 1412 continue; 1413 } 1414 d->bd_rcount++; 1415 bpf_gstats.bs_recv++; 1416 1417 if (d->bd_jitcode) 1418 slen = d->bd_jitcode(bc, &args); 1419 else 1420 slen = bpf_filter_ext(bc, d->bd_filter, &args); 1421 1422 if (!slen) { 1423 continue; 1424 } 1425 if (!gottime) { 1426 gottime = true; 1427 nanotime(&ts); 1428 } 1429 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1430 } 1431 } 1432 1433 /* 1434 * Incoming linkage from device drivers. Process the packet pkt, of length 1435 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1436 * by each process' filter, and if accepted, stashed into the corresponding 1437 * buffer. 1438 */ 1439 static void 1440 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1441 { 1442 1443 bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true); 1444 } 1445 1446 /* 1447 * Incoming linkage from device drivers, when the head of the packet is in 1448 * a buffer, and the tail is in an mbuf chain. 1449 */ 1450 static void 1451 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1452 { 1453 u_int pktlen; 1454 struct mbuf mb; 1455 1456 /* Skip outgoing duplicate packets. */ 1457 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1458 m->m_flags &= ~M_PROMISC; 1459 return; 1460 } 1461 1462 pktlen = m_length(m) + dlen; 1463 1464 /* 1465 * Craft on-stack mbuf suitable for passing to bpf_filter. 1466 * Note that we cut corners here; we only setup what's 1467 * absolutely needed--this mbuf should never go anywhere else. 1468 */ 1469 (void)memset(&mb, 0, sizeof(mb)); 1470 mb.m_next = m; 1471 mb.m_data = data; 1472 mb.m_len = dlen; 1473 1474 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif != NULL); 1475 } 1476 1477 /* 1478 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1479 */ 1480 static void 1481 _bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1482 { 1483 void *(*cpfn)(void *, const void *, size_t); 1484 u_int pktlen, buflen; 1485 void *marg; 1486 1487 /* Skip outgoing duplicate packets. */ 1488 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1489 m->m_flags &= ~M_PROMISC; 1490 return; 1491 } 1492 1493 pktlen = m_length(m); 1494 1495 if (pktlen == m->m_len) { 1496 cpfn = (void *)memcpy; 1497 marg = mtod(m, void *); 1498 buflen = pktlen; 1499 } else { 1500 cpfn = bpf_mcpy; 1501 marg = m; 1502 buflen = 0; 1503 } 1504 1505 bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif != NULL); 1506 } 1507 1508 /* 1509 * We need to prepend the address family as 1510 * a four byte field. Cons up a dummy header 1511 * to pacify bpf. This is safe because bpf 1512 * will only read from the mbuf (i.e., it won't 1513 * try to free it or keep a pointer a to it). 1514 */ 1515 static void 1516 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m) 1517 { 1518 struct mbuf m0; 1519 1520 m0.m_flags = 0; 1521 m0.m_next = m; 1522 m0.m_len = 4; 1523 m0.m_data = (char *)⁡ 1524 1525 _bpf_mtap(bp, &m0); 1526 } 1527 1528 /* 1529 * Put the SLIP pseudo-"link header" in place. 1530 * Note this M_PREPEND() should never fail, 1531 * swince we know we always have enough space 1532 * in the input buffer. 1533 */ 1534 static void 1535 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1536 { 1537 int s; 1538 u_char *hp; 1539 1540 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1541 if (*m == NULL) 1542 return; 1543 1544 hp = mtod(*m, u_char *); 1545 hp[SLX_DIR] = SLIPDIR_IN; 1546 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1547 1548 s = splnet(); 1549 _bpf_mtap(bp, *m); 1550 splx(s); 1551 1552 m_adj(*m, SLIP_HDRLEN); 1553 } 1554 1555 /* 1556 * Put the SLIP pseudo-"link header" in 1557 * place. The compressed header is now 1558 * at the beginning of the mbuf. 1559 */ 1560 static void 1561 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1562 { 1563 struct mbuf m0; 1564 u_char *hp; 1565 int s; 1566 1567 m0.m_flags = 0; 1568 m0.m_next = m; 1569 m0.m_data = m0.m_dat; 1570 m0.m_len = SLIP_HDRLEN; 1571 1572 hp = mtod(&m0, u_char *); 1573 1574 hp[SLX_DIR] = SLIPDIR_OUT; 1575 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1576 1577 s = splnet(); 1578 _bpf_mtap(bp, &m0); 1579 splx(s); 1580 m_freem(m); 1581 } 1582 1583 static int 1584 bpf_hdrlen(struct bpf_d *d) 1585 { 1586 int hdrlen = d->bd_bif->bif_hdrlen; 1587 /* 1588 * Compute the length of the bpf header. This is not necessarily 1589 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1590 * that the network layer header begins on a longword boundary (for 1591 * performance reasons and to alleviate alignment restrictions). 1592 */ 1593 #ifdef _LP64 1594 if (d->bd_compat32) 1595 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1596 else 1597 #endif 1598 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1599 } 1600 1601 /* 1602 * Move the packet data from interface memory (pkt) into the 1603 * store buffer. Call the wakeup functions if it's time to wakeup 1604 * a listener (buffer full), "cpfn" is the routine called to do the 1605 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1606 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1607 * pkt is really an mbuf. 1608 */ 1609 static void 1610 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1611 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1612 { 1613 char *h; 1614 int totlen, curlen, caplen; 1615 int hdrlen = bpf_hdrlen(d); 1616 int do_wakeup = 0; 1617 1618 ++d->bd_ccount; 1619 ++bpf_gstats.bs_capt; 1620 /* 1621 * Figure out how many bytes to move. If the packet is 1622 * greater or equal to the snapshot length, transfer that 1623 * much. Otherwise, transfer the whole packet (unless 1624 * we hit the buffer size limit). 1625 */ 1626 totlen = hdrlen + min(snaplen, pktlen); 1627 if (totlen > d->bd_bufsize) 1628 totlen = d->bd_bufsize; 1629 /* 1630 * If we adjusted totlen to fit the bufsize, it could be that 1631 * totlen is smaller than hdrlen because of the link layer header. 1632 */ 1633 caplen = totlen - hdrlen; 1634 if (caplen < 0) 1635 caplen = 0; 1636 1637 /* 1638 * Round up the end of the previous packet to the next longword. 1639 */ 1640 #ifdef _LP64 1641 if (d->bd_compat32) 1642 curlen = BPF_WORDALIGN32(d->bd_slen); 1643 else 1644 #endif 1645 curlen = BPF_WORDALIGN(d->bd_slen); 1646 if (curlen + totlen > d->bd_bufsize) { 1647 /* 1648 * This packet will overflow the storage buffer. 1649 * Rotate the buffers if we can, then wakeup any 1650 * pending reads. 1651 */ 1652 if (d->bd_fbuf == NULL) { 1653 /* 1654 * We haven't completed the previous read yet, 1655 * so drop the packet. 1656 */ 1657 ++d->bd_dcount; 1658 ++bpf_gstats.bs_drop; 1659 return; 1660 } 1661 ROTATE_BUFFERS(d); 1662 do_wakeup = 1; 1663 curlen = 0; 1664 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1665 /* 1666 * Immediate mode is set, or the read timeout has 1667 * already expired during a select call. A packet 1668 * arrived, so the reader should be woken up. 1669 */ 1670 do_wakeup = 1; 1671 } 1672 1673 /* 1674 * Append the bpf header. 1675 */ 1676 h = (char *)d->bd_sbuf + curlen; 1677 #ifdef _LP64 1678 if (d->bd_compat32) { 1679 struct bpf_hdr32 *hp32; 1680 1681 hp32 = (struct bpf_hdr32 *)h; 1682 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1683 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1684 hp32->bh_datalen = pktlen; 1685 hp32->bh_hdrlen = hdrlen; 1686 hp32->bh_caplen = caplen; 1687 } else 1688 #endif 1689 { 1690 struct bpf_hdr *hp; 1691 1692 hp = (struct bpf_hdr *)h; 1693 hp->bh_tstamp.tv_sec = ts->tv_sec; 1694 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1695 hp->bh_datalen = pktlen; 1696 hp->bh_hdrlen = hdrlen; 1697 hp->bh_caplen = caplen; 1698 } 1699 1700 /* 1701 * Copy the packet data into the store buffer and update its length. 1702 */ 1703 (*cpfn)(h + hdrlen, pkt, caplen); 1704 d->bd_slen = curlen + totlen; 1705 1706 /* 1707 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1708 * will cause filt_bpfread() to be called with it adjusted. 1709 */ 1710 if (do_wakeup) 1711 bpf_wakeup(d); 1712 } 1713 1714 /* 1715 * Initialize all nonzero fields of a descriptor. 1716 */ 1717 static int 1718 bpf_allocbufs(struct bpf_d *d) 1719 { 1720 1721 d->bd_fbuf = malloc(d->bd_bufsize, M_DEVBUF, M_WAITOK | M_CANFAIL); 1722 if (!d->bd_fbuf) 1723 return (ENOBUFS); 1724 d->bd_sbuf = malloc(d->bd_bufsize, M_DEVBUF, M_WAITOK | M_CANFAIL); 1725 if (!d->bd_sbuf) { 1726 free(d->bd_fbuf, M_DEVBUF); 1727 return (ENOBUFS); 1728 } 1729 d->bd_slen = 0; 1730 d->bd_hlen = 0; 1731 return (0); 1732 } 1733 1734 /* 1735 * Free buffers currently in use by a descriptor. 1736 * Called on close. 1737 */ 1738 static void 1739 bpf_freed(struct bpf_d *d) 1740 { 1741 /* 1742 * We don't need to lock out interrupts since this descriptor has 1743 * been detached from its interface and it yet hasn't been marked 1744 * free. 1745 */ 1746 if (d->bd_sbuf != NULL) { 1747 free(d->bd_sbuf, M_DEVBUF); 1748 if (d->bd_hbuf != NULL) 1749 free(d->bd_hbuf, M_DEVBUF); 1750 if (d->bd_fbuf != NULL) 1751 free(d->bd_fbuf, M_DEVBUF); 1752 } 1753 if (d->bd_filter) 1754 free(d->bd_filter, M_DEVBUF); 1755 1756 if (d->bd_jitcode != NULL) { 1757 bpf_jit_freecode(d->bd_jitcode); 1758 } 1759 } 1760 1761 /* 1762 * Attach an interface to bpf. dlt is the link layer type; 1763 * hdrlen is the fixed size of the link header for the specified dlt 1764 * (variable length headers not yet supported). 1765 */ 1766 static void 1767 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 1768 { 1769 struct bpf_if *bp; 1770 bp = malloc(sizeof(*bp), M_DEVBUF, M_DONTWAIT); 1771 if (bp == NULL) 1772 panic("bpfattach"); 1773 1774 bp->bif_dlist = NULL; 1775 bp->bif_driverp = driverp; 1776 bp->bif_ifp = ifp; 1777 bp->bif_dlt = dlt; 1778 1779 bp->bif_next = bpf_iflist; 1780 bpf_iflist = bp; 1781 1782 *bp->bif_driverp = NULL; 1783 1784 bp->bif_hdrlen = hdrlen; 1785 #if 0 1786 printf("bpf: %s attached\n", ifp->if_xname); 1787 #endif 1788 } 1789 1790 /* 1791 * Remove an interface from bpf. 1792 */ 1793 static void 1794 _bpfdetach(struct ifnet *ifp) 1795 { 1796 struct bpf_if *bp, **pbp; 1797 struct bpf_d *d; 1798 int s; 1799 1800 /* Nuke the vnodes for any open instances */ 1801 LIST_FOREACH(d, &bpf_list, bd_list) { 1802 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 1803 /* 1804 * Detach the descriptor from an interface now. 1805 * It will be free'ed later by close routine. 1806 */ 1807 s = splnet(); 1808 d->bd_promisc = 0; /* we can't touch device. */ 1809 bpf_detachd(d); 1810 splx(s); 1811 } 1812 } 1813 1814 again: 1815 for (bp = bpf_iflist, pbp = &bpf_iflist; 1816 bp != NULL; pbp = &bp->bif_next, bp = bp->bif_next) { 1817 if (bp->bif_ifp == ifp) { 1818 *pbp = bp->bif_next; 1819 free(bp, M_DEVBUF); 1820 goto again; 1821 } 1822 } 1823 } 1824 1825 /* 1826 * Change the data link type of a interface. 1827 */ 1828 static void 1829 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 1830 { 1831 struct bpf_if *bp; 1832 1833 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1834 if (bp->bif_driverp == &ifp->if_bpf) 1835 break; 1836 } 1837 if (bp == NULL) 1838 panic("bpf_change_type"); 1839 1840 bp->bif_dlt = dlt; 1841 1842 bp->bif_hdrlen = hdrlen; 1843 } 1844 1845 /* 1846 * Get a list of available data link type of the interface. 1847 */ 1848 static int 1849 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 1850 { 1851 int n, error; 1852 struct ifnet *ifp; 1853 struct bpf_if *bp; 1854 1855 ifp = d->bd_bif->bif_ifp; 1856 n = 0; 1857 error = 0; 1858 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1859 if (bp->bif_ifp != ifp) 1860 continue; 1861 if (bfl->bfl_list != NULL) { 1862 if (n >= bfl->bfl_len) 1863 return ENOMEM; 1864 error = copyout(&bp->bif_dlt, 1865 bfl->bfl_list + n, sizeof(u_int)); 1866 } 1867 n++; 1868 } 1869 bfl->bfl_len = n; 1870 return error; 1871 } 1872 1873 /* 1874 * Set the data link type of a BPF instance. 1875 */ 1876 static int 1877 bpf_setdlt(struct bpf_d *d, u_int dlt) 1878 { 1879 int s, error, opromisc; 1880 struct ifnet *ifp; 1881 struct bpf_if *bp; 1882 1883 if (d->bd_bif->bif_dlt == dlt) 1884 return 0; 1885 ifp = d->bd_bif->bif_ifp; 1886 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1887 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 1888 break; 1889 } 1890 if (bp == NULL) 1891 return EINVAL; 1892 s = splnet(); 1893 opromisc = d->bd_promisc; 1894 bpf_detachd(d); 1895 bpf_attachd(d, bp); 1896 reset_d(d); 1897 if (opromisc) { 1898 error = ifpromisc(bp->bif_ifp, 1); 1899 if (error) 1900 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 1901 bp->bif_ifp->if_xname, error); 1902 else 1903 d->bd_promisc = 1; 1904 } 1905 splx(s); 1906 return 0; 1907 } 1908 1909 static int 1910 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 1911 { 1912 int newsize, error; 1913 struct sysctlnode node; 1914 1915 node = *rnode; 1916 node.sysctl_data = &newsize; 1917 newsize = bpf_maxbufsize; 1918 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1919 if (error || newp == NULL) 1920 return (error); 1921 1922 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 1923 return (EINVAL); 1924 1925 bpf_maxbufsize = newsize; 1926 1927 return (0); 1928 } 1929 1930 #if defined(MODULAR) || defined(BPFJIT) 1931 static int 1932 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 1933 { 1934 bool newval; 1935 int error; 1936 struct sysctlnode node; 1937 1938 node = *rnode; 1939 node.sysctl_data = &newval; 1940 newval = bpf_jit; 1941 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1942 if (error != 0 || newp == NULL) 1943 return error; 1944 1945 bpf_jit = newval; 1946 1947 /* 1948 * Do a full sync to publish new bpf_jit value and 1949 * update bpfjit_module_ops.bj_generate_code variable. 1950 */ 1951 membar_sync(); 1952 1953 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 1954 printf("JIT compilation is postponed " 1955 "until after bpfjit module is loaded\n"); 1956 } 1957 1958 return 0; 1959 } 1960 #endif 1961 1962 static int 1963 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 1964 { 1965 int error, elem_count; 1966 struct bpf_d *dp; 1967 struct bpf_d_ext dpe; 1968 size_t len, needed, elem_size, out_size; 1969 char *sp; 1970 1971 if (namelen == 1 && name[0] == CTL_QUERY) 1972 return (sysctl_query(SYSCTLFN_CALL(rnode))); 1973 1974 if (namelen != 2) 1975 return (EINVAL); 1976 1977 /* BPF peers is privileged information. */ 1978 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 1979 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 1980 if (error) 1981 return (EPERM); 1982 1983 len = (oldp != NULL) ? *oldlenp : 0; 1984 sp = oldp; 1985 elem_size = name[0]; 1986 elem_count = name[1]; 1987 out_size = MIN(sizeof(dpe), elem_size); 1988 needed = 0; 1989 1990 if (elem_size < 1 || elem_count < 0) 1991 return (EINVAL); 1992 1993 mutex_enter(&bpf_mtx); 1994 LIST_FOREACH(dp, &bpf_list, bd_list) { 1995 if (len >= elem_size && elem_count > 0) { 1996 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 1997 BPF_EXT(bufsize); 1998 BPF_EXT(promisc); 1999 BPF_EXT(state); 2000 BPF_EXT(immediate); 2001 BPF_EXT(hdrcmplt); 2002 BPF_EXT(seesent); 2003 BPF_EXT(pid); 2004 BPF_EXT(rcount); 2005 BPF_EXT(dcount); 2006 BPF_EXT(ccount); 2007 #undef BPF_EXT 2008 if (dp->bd_bif) 2009 (void)strlcpy(dpe.bde_ifname, 2010 dp->bd_bif->bif_ifp->if_xname, 2011 IFNAMSIZ - 1); 2012 else 2013 dpe.bde_ifname[0] = '\0'; 2014 2015 error = copyout(&dpe, sp, out_size); 2016 if (error) 2017 break; 2018 sp += elem_size; 2019 len -= elem_size; 2020 } 2021 needed += elem_size; 2022 if (elem_count > 0 && elem_count != INT_MAX) 2023 elem_count--; 2024 } 2025 mutex_exit(&bpf_mtx); 2026 2027 *oldlenp = needed; 2028 2029 return (error); 2030 } 2031 2032 static struct sysctllog *bpf_sysctllog; 2033 static void 2034 sysctl_net_bpf_setup(void) 2035 { 2036 const struct sysctlnode *node; 2037 2038 node = NULL; 2039 sysctl_createv(&bpf_sysctllog, 0, NULL, &node, 2040 CTLFLAG_PERMANENT, 2041 CTLTYPE_NODE, "bpf", 2042 SYSCTL_DESCR("BPF options"), 2043 NULL, 0, NULL, 0, 2044 CTL_NET, CTL_CREATE, CTL_EOL); 2045 if (node != NULL) { 2046 #if defined(MODULAR) || defined(BPFJIT) 2047 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2048 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2049 CTLTYPE_BOOL, "jit", 2050 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2051 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2052 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2053 #endif 2054 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2055 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2056 CTLTYPE_INT, "maxbufsize", 2057 SYSCTL_DESCR("Maximum size for data capture buffer"), 2058 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2059 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2060 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2061 CTLFLAG_PERMANENT, 2062 CTLTYPE_STRUCT, "stats", 2063 SYSCTL_DESCR("BPF stats"), 2064 NULL, 0, &bpf_gstats, sizeof(bpf_gstats), 2065 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2066 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2067 CTLFLAG_PERMANENT, 2068 CTLTYPE_STRUCT, "peers", 2069 SYSCTL_DESCR("BPF peers"), 2070 sysctl_net_bpf_peers, 0, NULL, 0, 2071 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2072 } 2073 2074 } 2075 2076 struct bpf_ops bpf_ops_kernel = { 2077 .bpf_attach = _bpfattach, 2078 .bpf_detach = _bpfdetach, 2079 .bpf_change_type = _bpf_change_type, 2080 2081 .bpf_tap = _bpf_tap, 2082 .bpf_mtap = _bpf_mtap, 2083 .bpf_mtap2 = _bpf_mtap2, 2084 .bpf_mtap_af = _bpf_mtap_af, 2085 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2086 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2087 }; 2088 2089 MODULE(MODULE_CLASS_DRIVER, bpf, NULL); 2090 2091 static int 2092 bpf_modcmd(modcmd_t cmd, void *arg) 2093 { 2094 devmajor_t bmajor, cmajor; 2095 int error; 2096 2097 bmajor = cmajor = NODEVMAJOR; 2098 2099 switch (cmd) { 2100 case MODULE_CMD_INIT: 2101 bpfilterattach(0); 2102 error = devsw_attach("bpf", NULL, &bmajor, 2103 &bpf_cdevsw, &cmajor); 2104 if (error == EEXIST) 2105 error = 0; /* maybe built-in ... improve eventually */ 2106 if (error) 2107 break; 2108 2109 bpf_ops_handover_enter(&bpf_ops_kernel); 2110 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2111 bpf_ops_handover_exit(); 2112 sysctl_net_bpf_setup(); 2113 break; 2114 2115 case MODULE_CMD_FINI: 2116 /* 2117 * While there is no reference counting for bpf callers, 2118 * unload could at least in theory be done similarly to 2119 * system call disestablishment. This should even be 2120 * a little simpler: 2121 * 2122 * 1) replace op vector with stubs 2123 * 2) post update to all cpus with xc 2124 * 3) check that nobody is in bpf anymore 2125 * (it's doubtful we'd want something like l_sysent, 2126 * but we could do something like *signed* percpu 2127 * counters. if the sum is 0, we're good). 2128 * 4) if fail, unroll changes 2129 * 2130 * NOTE: change won't be atomic to the outside. some 2131 * packets may be not captured even if unload is 2132 * not succesful. I think packet capture not working 2133 * is a perfectly logical consequence of trying to 2134 * disable packet capture. 2135 */ 2136 error = EOPNOTSUPP; 2137 /* insert sysctl teardown */ 2138 break; 2139 2140 default: 2141 error = ENOTTY; 2142 break; 2143 } 2144 2145 return error; 2146 } 2147