1 /* $NetBSD: bpf.c,v 1.173 2012/10/27 22:36:14 alnsn Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.173 2012/10/27 22:36:14 alnsn Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "strip.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/mbuf.h> 53 #include <sys/buf.h> 54 #include <sys/time.h> 55 #include <sys/proc.h> 56 #include <sys/ioctl.h> 57 #include <sys/conf.h> 58 #include <sys/vnode.h> 59 #include <sys/queue.h> 60 #include <sys/stat.h> 61 #include <sys/module.h> 62 #include <sys/once.h> 63 #include <sys/atomic.h> 64 65 #include <sys/file.h> 66 #include <sys/filedesc.h> 67 #include <sys/tty.h> 68 #include <sys/uio.h> 69 70 #include <sys/protosw.h> 71 #include <sys/socket.h> 72 #include <sys/errno.h> 73 #include <sys/kernel.h> 74 #include <sys/poll.h> 75 #include <sys/sysctl.h> 76 #include <sys/kauth.h> 77 78 #include <net/if.h> 79 #include <net/slip.h> 80 81 #include <net/bpf.h> 82 #include <net/bpfdesc.h> 83 #include <net/bpfjit.h> 84 85 #include <net/if_arc.h> 86 #include <net/if_ether.h> 87 88 #include <netinet/in.h> 89 #include <netinet/if_inarp.h> 90 91 92 #include <compat/sys/sockio.h> 93 94 #ifndef BPF_BUFSIZE 95 /* 96 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 97 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 98 */ 99 # define BPF_BUFSIZE 32768 100 #endif 101 102 #define PRINET 26 /* interruptible */ 103 104 /* 105 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 106 * XXX the default values should be computed dynamically based 107 * on available memory size and available mbuf clusters. 108 */ 109 int bpf_bufsize = BPF_BUFSIZE; 110 int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 111 bool bpf_jit = false; 112 113 struct bpfjit_ops bpfjit_module_ops = { 114 .bj_generate_code = NULL, 115 .bj_free_code = NULL 116 }; 117 118 /* 119 * Global BPF statistics returned by net.bpf.stats sysctl. 120 */ 121 struct bpf_stat bpf_gstats; 122 123 /* 124 * Use a mutex to avoid a race condition between gathering the stats/peers 125 * and opening/closing the device. 126 */ 127 static kmutex_t bpf_mtx; 128 129 /* 130 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 131 * bpf_dtab holds the descriptors, indexed by minor device # 132 */ 133 struct bpf_if *bpf_iflist; 134 LIST_HEAD(, bpf_d) bpf_list; 135 136 static int bpf_allocbufs(struct bpf_d *); 137 static void bpf_deliver(struct bpf_if *, 138 void *(*cpfn)(void *, const void *, size_t), 139 void *, u_int, u_int, const bool); 140 static void bpf_freed(struct bpf_d *); 141 static void bpf_ifname(struct ifnet *, struct ifreq *); 142 static void *bpf_mcpy(void *, const void *, size_t); 143 static int bpf_movein(struct uio *, int, uint64_t, 144 struct mbuf **, struct sockaddr *); 145 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 146 static void bpf_detachd(struct bpf_d *); 147 static int bpf_setif(struct bpf_d *, struct ifreq *); 148 static void bpf_timed_out(void *); 149 static inline void 150 bpf_wakeup(struct bpf_d *); 151 static int bpf_hdrlen(struct bpf_d *); 152 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 153 void *(*)(void *, const void *, size_t), struct timespec *); 154 static void reset_d(struct bpf_d *); 155 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 156 static int bpf_setdlt(struct bpf_d *, u_int); 157 158 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 159 int); 160 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 161 int); 162 static int bpf_ioctl(struct file *, u_long, void *); 163 static int bpf_poll(struct file *, int); 164 static int bpf_stat(struct file *, struct stat *); 165 static int bpf_close(struct file *); 166 static int bpf_kqfilter(struct file *, struct knote *); 167 static void bpf_softintr(void *); 168 169 static const struct fileops bpf_fileops = { 170 .fo_read = bpf_read, 171 .fo_write = bpf_write, 172 .fo_ioctl = bpf_ioctl, 173 .fo_fcntl = fnullop_fcntl, 174 .fo_poll = bpf_poll, 175 .fo_stat = bpf_stat, 176 .fo_close = bpf_close, 177 .fo_kqfilter = bpf_kqfilter, 178 .fo_restart = fnullop_restart, 179 }; 180 181 dev_type_open(bpfopen); 182 183 const struct cdevsw bpf_cdevsw = { 184 bpfopen, noclose, noread, nowrite, noioctl, 185 nostop, notty, nopoll, nommap, nokqfilter, D_OTHER 186 }; 187 188 static int 189 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 190 struct sockaddr *sockp) 191 { 192 struct mbuf *m; 193 int error; 194 size_t len; 195 size_t hlen; 196 size_t align; 197 198 /* 199 * Build a sockaddr based on the data link layer type. 200 * We do this at this level because the ethernet header 201 * is copied directly into the data field of the sockaddr. 202 * In the case of SLIP, there is no header and the packet 203 * is forwarded as is. 204 * Also, we are careful to leave room at the front of the mbuf 205 * for the link level header. 206 */ 207 switch (linktype) { 208 209 case DLT_SLIP: 210 sockp->sa_family = AF_INET; 211 hlen = 0; 212 align = 0; 213 break; 214 215 case DLT_PPP: 216 sockp->sa_family = AF_UNSPEC; 217 hlen = 0; 218 align = 0; 219 break; 220 221 case DLT_EN10MB: 222 sockp->sa_family = AF_UNSPEC; 223 /* XXX Would MAXLINKHDR be better? */ 224 /* 6(dst)+6(src)+2(type) */ 225 hlen = sizeof(struct ether_header); 226 align = 2; 227 break; 228 229 case DLT_ARCNET: 230 sockp->sa_family = AF_UNSPEC; 231 hlen = ARC_HDRLEN; 232 align = 5; 233 break; 234 235 case DLT_FDDI: 236 sockp->sa_family = AF_LINK; 237 /* XXX 4(FORMAC)+6(dst)+6(src) */ 238 hlen = 16; 239 align = 0; 240 break; 241 242 case DLT_ECONET: 243 sockp->sa_family = AF_UNSPEC; 244 hlen = 6; 245 align = 2; 246 break; 247 248 case DLT_NULL: 249 sockp->sa_family = AF_UNSPEC; 250 hlen = 0; 251 align = 0; 252 break; 253 254 default: 255 return (EIO); 256 } 257 258 len = uio->uio_resid; 259 /* 260 * If there aren't enough bytes for a link level header or the 261 * packet length exceeds the interface mtu, return an error. 262 */ 263 if (len - hlen > mtu) 264 return (EMSGSIZE); 265 266 /* 267 * XXX Avoid complicated buffer chaining --- 268 * bail if it won't fit in a single mbuf. 269 * (Take into account possible alignment bytes) 270 */ 271 if (len + align > MCLBYTES) 272 return (EIO); 273 274 m = m_gethdr(M_WAIT, MT_DATA); 275 m->m_pkthdr.rcvif = 0; 276 m->m_pkthdr.len = (int)(len - hlen); 277 if (len + align > MHLEN) { 278 m_clget(m, M_WAIT); 279 if ((m->m_flags & M_EXT) == 0) { 280 error = ENOBUFS; 281 goto bad; 282 } 283 } 284 285 /* Insure the data is properly aligned */ 286 if (align > 0) { 287 m->m_data += align; 288 m->m_len -= (int)align; 289 } 290 291 error = uiomove(mtod(m, void *), len, uio); 292 if (error) 293 goto bad; 294 if (hlen != 0) { 295 memcpy(sockp->sa_data, mtod(m, void *), hlen); 296 m->m_data += hlen; /* XXX */ 297 len -= hlen; 298 } 299 m->m_len = (int)len; 300 *mp = m; 301 return (0); 302 303 bad: 304 m_freem(m); 305 return (error); 306 } 307 308 /* 309 * Attach file to the bpf interface, i.e. make d listen on bp. 310 * Must be called at splnet. 311 */ 312 static void 313 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 314 { 315 /* 316 * Point d at bp, and add d to the interface's list of listeners. 317 * Finally, point the driver's bpf cookie at the interface so 318 * it will divert packets to bpf. 319 */ 320 d->bd_bif = bp; 321 d->bd_next = bp->bif_dlist; 322 bp->bif_dlist = d; 323 324 *bp->bif_driverp = bp; 325 } 326 327 /* 328 * Detach a file from its interface. 329 */ 330 static void 331 bpf_detachd(struct bpf_d *d) 332 { 333 struct bpf_d **p; 334 struct bpf_if *bp; 335 336 bp = d->bd_bif; 337 /* 338 * Check if this descriptor had requested promiscuous mode. 339 * If so, turn it off. 340 */ 341 if (d->bd_promisc) { 342 int error; 343 344 d->bd_promisc = 0; 345 /* 346 * Take device out of promiscuous mode. Since we were 347 * able to enter promiscuous mode, we should be able 348 * to turn it off. But we can get an error if 349 * the interface was configured down, so only panic 350 * if we don't get an unexpected error. 351 */ 352 error = ifpromisc(bp->bif_ifp, 0); 353 if (error && error != EINVAL) 354 panic("%s: ifpromisc failed: %d", __func__, error); 355 } 356 /* Remove d from the interface's descriptor list. */ 357 p = &bp->bif_dlist; 358 while (*p != d) { 359 p = &(*p)->bd_next; 360 if (*p == 0) 361 panic("%s: descriptor not in list", __func__); 362 } 363 *p = (*p)->bd_next; 364 if (bp->bif_dlist == 0) 365 /* 366 * Let the driver know that there are no more listeners. 367 */ 368 *d->bd_bif->bif_driverp = 0; 369 d->bd_bif = 0; 370 } 371 372 static int 373 doinit(void) 374 { 375 376 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 377 378 LIST_INIT(&bpf_list); 379 380 bpf_gstats.bs_recv = 0; 381 bpf_gstats.bs_drop = 0; 382 bpf_gstats.bs_capt = 0; 383 384 return 0; 385 } 386 387 /* 388 * bpfilterattach() is called at boot time. 389 */ 390 /* ARGSUSED */ 391 void 392 bpfilterattach(int n) 393 { 394 static ONCE_DECL(control); 395 396 RUN_ONCE(&control, doinit); 397 } 398 399 /* 400 * Open ethernet device. Clones. 401 */ 402 /* ARGSUSED */ 403 int 404 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 405 { 406 struct bpf_d *d; 407 struct file *fp; 408 int error, fd; 409 410 /* falloc() will use the descriptor for us. */ 411 if ((error = fd_allocfile(&fp, &fd)) != 0) 412 return error; 413 414 d = malloc(sizeof(*d), M_DEVBUF, M_WAITOK|M_ZERO); 415 d->bd_bufsize = bpf_bufsize; 416 d->bd_seesent = 1; 417 d->bd_feedback = 0; 418 d->bd_pid = l->l_proc->p_pid; 419 #ifdef _LP64 420 if (curproc->p_flag & PK_32) 421 d->bd_compat32 = 1; 422 #endif 423 getnanotime(&d->bd_btime); 424 d->bd_atime = d->bd_mtime = d->bd_btime; 425 callout_init(&d->bd_callout, 0); 426 selinit(&d->bd_sel); 427 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d); 428 d->bd_jitcode = NULL; 429 430 mutex_enter(&bpf_mtx); 431 LIST_INSERT_HEAD(&bpf_list, d, bd_list); 432 mutex_exit(&bpf_mtx); 433 434 return fd_clone(fp, fd, flag, &bpf_fileops, d); 435 } 436 437 /* 438 * Close the descriptor by detaching it from its interface, 439 * deallocating its buffers, and marking it free. 440 */ 441 /* ARGSUSED */ 442 static int 443 bpf_close(struct file *fp) 444 { 445 struct bpf_d *d = fp->f_data; 446 int s; 447 448 KERNEL_LOCK(1, NULL); 449 450 /* 451 * Refresh the PID associated with this bpf file. 452 */ 453 d->bd_pid = curproc->p_pid; 454 455 s = splnet(); 456 if (d->bd_state == BPF_WAITING) 457 callout_stop(&d->bd_callout); 458 d->bd_state = BPF_IDLE; 459 if (d->bd_bif) 460 bpf_detachd(d); 461 splx(s); 462 bpf_freed(d); 463 mutex_enter(&bpf_mtx); 464 LIST_REMOVE(d, bd_list); 465 mutex_exit(&bpf_mtx); 466 callout_destroy(&d->bd_callout); 467 seldestroy(&d->bd_sel); 468 softint_disestablish(d->bd_sih); 469 free(d, M_DEVBUF); 470 fp->f_data = NULL; 471 472 KERNEL_UNLOCK_ONE(NULL); 473 474 return (0); 475 } 476 477 /* 478 * Rotate the packet buffers in descriptor d. Move the store buffer 479 * into the hold slot, and the free buffer into the store slot. 480 * Zero the length of the new store buffer. 481 */ 482 #define ROTATE_BUFFERS(d) \ 483 (d)->bd_hbuf = (d)->bd_sbuf; \ 484 (d)->bd_hlen = (d)->bd_slen; \ 485 (d)->bd_sbuf = (d)->bd_fbuf; \ 486 (d)->bd_slen = 0; \ 487 (d)->bd_fbuf = 0; 488 /* 489 * bpfread - read next chunk of packets from buffers 490 */ 491 static int 492 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 493 kauth_cred_t cred, int flags) 494 { 495 struct bpf_d *d = fp->f_data; 496 int timed_out; 497 int error; 498 int s; 499 500 getnanotime(&d->bd_atime); 501 /* 502 * Restrict application to use a buffer the same size as 503 * the kernel buffers. 504 */ 505 if (uio->uio_resid != d->bd_bufsize) 506 return (EINVAL); 507 508 KERNEL_LOCK(1, NULL); 509 s = splnet(); 510 if (d->bd_state == BPF_WAITING) 511 callout_stop(&d->bd_callout); 512 timed_out = (d->bd_state == BPF_TIMED_OUT); 513 d->bd_state = BPF_IDLE; 514 /* 515 * If the hold buffer is empty, then do a timed sleep, which 516 * ends when the timeout expires or when enough packets 517 * have arrived to fill the store buffer. 518 */ 519 while (d->bd_hbuf == 0) { 520 if (fp->f_flag & FNONBLOCK) { 521 if (d->bd_slen == 0) { 522 splx(s); 523 KERNEL_UNLOCK_ONE(NULL); 524 return (EWOULDBLOCK); 525 } 526 ROTATE_BUFFERS(d); 527 break; 528 } 529 530 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 531 /* 532 * A packet(s) either arrived since the previous 533 * read or arrived while we were asleep. 534 * Rotate the buffers and return what's here. 535 */ 536 ROTATE_BUFFERS(d); 537 break; 538 } 539 error = tsleep(d, PRINET|PCATCH, "bpf", 540 d->bd_rtout); 541 if (error == EINTR || error == ERESTART) { 542 splx(s); 543 KERNEL_UNLOCK_ONE(NULL); 544 return (error); 545 } 546 if (error == EWOULDBLOCK) { 547 /* 548 * On a timeout, return what's in the buffer, 549 * which may be nothing. If there is something 550 * in the store buffer, we can rotate the buffers. 551 */ 552 if (d->bd_hbuf) 553 /* 554 * We filled up the buffer in between 555 * getting the timeout and arriving 556 * here, so we don't need to rotate. 557 */ 558 break; 559 560 if (d->bd_slen == 0) { 561 splx(s); 562 KERNEL_UNLOCK_ONE(NULL); 563 return (0); 564 } 565 ROTATE_BUFFERS(d); 566 break; 567 } 568 if (error != 0) 569 goto done; 570 } 571 /* 572 * At this point, we know we have something in the hold slot. 573 */ 574 splx(s); 575 576 /* 577 * Move data from hold buffer into user space. 578 * We know the entire buffer is transferred since 579 * we checked above that the read buffer is bpf_bufsize bytes. 580 */ 581 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 582 583 s = splnet(); 584 d->bd_fbuf = d->bd_hbuf; 585 d->bd_hbuf = 0; 586 d->bd_hlen = 0; 587 done: 588 splx(s); 589 KERNEL_UNLOCK_ONE(NULL); 590 return (error); 591 } 592 593 594 /* 595 * If there are processes sleeping on this descriptor, wake them up. 596 */ 597 static inline void 598 bpf_wakeup(struct bpf_d *d) 599 { 600 wakeup(d); 601 if (d->bd_async) 602 softint_schedule(d->bd_sih); 603 selnotify(&d->bd_sel, 0, 0); 604 } 605 606 static void 607 bpf_softintr(void *cookie) 608 { 609 struct bpf_d *d; 610 611 d = cookie; 612 if (d->bd_async) 613 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 614 } 615 616 static void 617 bpf_timed_out(void *arg) 618 { 619 struct bpf_d *d = arg; 620 int s; 621 622 s = splnet(); 623 if (d->bd_state == BPF_WAITING) { 624 d->bd_state = BPF_TIMED_OUT; 625 if (d->bd_slen != 0) 626 bpf_wakeup(d); 627 } 628 splx(s); 629 } 630 631 632 static int 633 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 634 kauth_cred_t cred, int flags) 635 { 636 struct bpf_d *d = fp->f_data; 637 struct ifnet *ifp; 638 struct mbuf *m, *mc; 639 int error, s; 640 static struct sockaddr_storage dst; 641 642 m = NULL; /* XXX gcc */ 643 644 KERNEL_LOCK(1, NULL); 645 646 if (d->bd_bif == 0) { 647 KERNEL_UNLOCK_ONE(NULL); 648 return (ENXIO); 649 } 650 getnanotime(&d->bd_mtime); 651 652 ifp = d->bd_bif->bif_ifp; 653 654 if (uio->uio_resid == 0) { 655 KERNEL_UNLOCK_ONE(NULL); 656 return (0); 657 } 658 659 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp->if_mtu, &m, 660 (struct sockaddr *) &dst); 661 if (error) { 662 KERNEL_UNLOCK_ONE(NULL); 663 return (error); 664 } 665 666 if (m->m_pkthdr.len > ifp->if_mtu) { 667 KERNEL_UNLOCK_ONE(NULL); 668 m_freem(m); 669 return (EMSGSIZE); 670 } 671 672 if (d->bd_hdrcmplt) 673 dst.ss_family = pseudo_AF_HDRCMPLT; 674 675 if (d->bd_feedback) { 676 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 677 if (mc != NULL) 678 mc->m_pkthdr.rcvif = ifp; 679 /* Set M_PROMISC for outgoing packets to be discarded. */ 680 if (1 /*d->bd_direction == BPF_D_INOUT*/) 681 m->m_flags |= M_PROMISC; 682 } else 683 mc = NULL; 684 685 s = splsoftnet(); 686 error = (*ifp->if_output)(ifp, m, (struct sockaddr *) &dst, NULL); 687 688 if (mc != NULL) { 689 if (error == 0) 690 (*ifp->if_input)(ifp, mc); 691 m_freem(mc); 692 } 693 splx(s); 694 KERNEL_UNLOCK_ONE(NULL); 695 /* 696 * The driver frees the mbuf. 697 */ 698 return (error); 699 } 700 701 /* 702 * Reset a descriptor by flushing its packet buffer and clearing the 703 * receive and drop counts. Should be called at splnet. 704 */ 705 static void 706 reset_d(struct bpf_d *d) 707 { 708 if (d->bd_hbuf) { 709 /* Free the hold buffer. */ 710 d->bd_fbuf = d->bd_hbuf; 711 d->bd_hbuf = 0; 712 } 713 d->bd_slen = 0; 714 d->bd_hlen = 0; 715 d->bd_rcount = 0; 716 d->bd_dcount = 0; 717 d->bd_ccount = 0; 718 } 719 720 /* 721 * FIONREAD Check for read packet available. 722 * BIOCGBLEN Get buffer len [for read()]. 723 * BIOCSETF Set ethernet read filter. 724 * BIOCFLUSH Flush read packet buffer. 725 * BIOCPROMISC Put interface into promiscuous mode. 726 * BIOCGDLT Get link layer type. 727 * BIOCGETIF Get interface name. 728 * BIOCSETIF Set interface. 729 * BIOCSRTIMEOUT Set read timeout. 730 * BIOCGRTIMEOUT Get read timeout. 731 * BIOCGSTATS Get packet stats. 732 * BIOCIMMEDIATE Set immediate mode. 733 * BIOCVERSION Get filter language version. 734 * BIOCGHDRCMPLT Get "header already complete" flag. 735 * BIOCSHDRCMPLT Set "header already complete" flag. 736 * BIOCSFEEDBACK Set packet feedback mode. 737 * BIOCGFEEDBACK Get packet feedback mode. 738 * BIOCGSEESENT Get "see sent packets" mode. 739 * BIOCSSEESENT Set "see sent packets" mode. 740 */ 741 /* ARGSUSED */ 742 static int 743 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 744 { 745 struct bpf_d *d = fp->f_data; 746 int s, error = 0; 747 748 /* 749 * Refresh the PID associated with this bpf file. 750 */ 751 KERNEL_LOCK(1, NULL); 752 d->bd_pid = curproc->p_pid; 753 #ifdef _LP64 754 if (curproc->p_flag & PK_32) 755 d->bd_compat32 = 1; 756 else 757 d->bd_compat32 = 0; 758 #endif 759 760 s = splnet(); 761 if (d->bd_state == BPF_WAITING) 762 callout_stop(&d->bd_callout); 763 d->bd_state = BPF_IDLE; 764 splx(s); 765 766 switch (cmd) { 767 768 default: 769 error = EINVAL; 770 break; 771 772 /* 773 * Check for read packet available. 774 */ 775 case FIONREAD: 776 { 777 int n; 778 779 s = splnet(); 780 n = d->bd_slen; 781 if (d->bd_hbuf) 782 n += d->bd_hlen; 783 splx(s); 784 785 *(int *)addr = n; 786 break; 787 } 788 789 /* 790 * Get buffer len [for read()]. 791 */ 792 case BIOCGBLEN: 793 *(u_int *)addr = d->bd_bufsize; 794 break; 795 796 /* 797 * Set buffer length. 798 */ 799 case BIOCSBLEN: 800 if (d->bd_bif != 0) 801 error = EINVAL; 802 else { 803 u_int size = *(u_int *)addr; 804 805 if (size > bpf_maxbufsize) 806 *(u_int *)addr = size = bpf_maxbufsize; 807 else if (size < BPF_MINBUFSIZE) 808 *(u_int *)addr = size = BPF_MINBUFSIZE; 809 d->bd_bufsize = size; 810 } 811 break; 812 813 /* 814 * Set link layer read filter. 815 */ 816 case BIOCSETF: 817 error = bpf_setf(d, addr); 818 break; 819 820 /* 821 * Flush read packet buffer. 822 */ 823 case BIOCFLUSH: 824 s = splnet(); 825 reset_d(d); 826 splx(s); 827 break; 828 829 /* 830 * Put interface into promiscuous mode. 831 */ 832 case BIOCPROMISC: 833 if (d->bd_bif == 0) { 834 /* 835 * No interface attached yet. 836 */ 837 error = EINVAL; 838 break; 839 } 840 s = splnet(); 841 if (d->bd_promisc == 0) { 842 error = ifpromisc(d->bd_bif->bif_ifp, 1); 843 if (error == 0) 844 d->bd_promisc = 1; 845 } 846 splx(s); 847 break; 848 849 /* 850 * Get device parameters. 851 */ 852 case BIOCGDLT: 853 if (d->bd_bif == 0) 854 error = EINVAL; 855 else 856 *(u_int *)addr = d->bd_bif->bif_dlt; 857 break; 858 859 /* 860 * Get a list of supported device parameters. 861 */ 862 case BIOCGDLTLIST: 863 if (d->bd_bif == 0) 864 error = EINVAL; 865 else 866 error = bpf_getdltlist(d, addr); 867 break; 868 869 /* 870 * Set device parameters. 871 */ 872 case BIOCSDLT: 873 if (d->bd_bif == 0) 874 error = EINVAL; 875 else 876 error = bpf_setdlt(d, *(u_int *)addr); 877 break; 878 879 /* 880 * Set interface name. 881 */ 882 #ifdef OBIOCGETIF 883 case OBIOCGETIF: 884 #endif 885 case BIOCGETIF: 886 if (d->bd_bif == 0) 887 error = EINVAL; 888 else 889 bpf_ifname(d->bd_bif->bif_ifp, addr); 890 break; 891 892 /* 893 * Set interface. 894 */ 895 #ifdef OBIOCSETIF 896 case OBIOCSETIF: 897 #endif 898 case BIOCSETIF: 899 error = bpf_setif(d, addr); 900 break; 901 902 /* 903 * Set read timeout. 904 */ 905 case BIOCSRTIMEOUT: 906 { 907 struct timeval *tv = addr; 908 909 /* Compute number of ticks. */ 910 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 911 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 912 d->bd_rtout = 1; 913 break; 914 } 915 916 #ifdef BIOCGORTIMEOUT 917 /* 918 * Get read timeout. 919 */ 920 case BIOCGORTIMEOUT: 921 { 922 struct timeval50 *tv = addr; 923 924 tv->tv_sec = d->bd_rtout / hz; 925 tv->tv_usec = (d->bd_rtout % hz) * tick; 926 break; 927 } 928 #endif 929 930 #ifdef BIOCSORTIMEOUT 931 /* 932 * Set read timeout. 933 */ 934 case BIOCSORTIMEOUT: 935 { 936 struct timeval50 *tv = addr; 937 938 /* Compute number of ticks. */ 939 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 940 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 941 d->bd_rtout = 1; 942 break; 943 } 944 #endif 945 946 /* 947 * Get read timeout. 948 */ 949 case BIOCGRTIMEOUT: 950 { 951 struct timeval *tv = addr; 952 953 tv->tv_sec = d->bd_rtout / hz; 954 tv->tv_usec = (d->bd_rtout % hz) * tick; 955 break; 956 } 957 /* 958 * Get packet stats. 959 */ 960 case BIOCGSTATS: 961 { 962 struct bpf_stat *bs = addr; 963 964 bs->bs_recv = d->bd_rcount; 965 bs->bs_drop = d->bd_dcount; 966 bs->bs_capt = d->bd_ccount; 967 break; 968 } 969 970 case BIOCGSTATSOLD: 971 { 972 struct bpf_stat_old *bs = addr; 973 974 bs->bs_recv = d->bd_rcount; 975 bs->bs_drop = d->bd_dcount; 976 break; 977 } 978 979 /* 980 * Set immediate mode. 981 */ 982 case BIOCIMMEDIATE: 983 d->bd_immediate = *(u_int *)addr; 984 break; 985 986 case BIOCVERSION: 987 { 988 struct bpf_version *bv = addr; 989 990 bv->bv_major = BPF_MAJOR_VERSION; 991 bv->bv_minor = BPF_MINOR_VERSION; 992 break; 993 } 994 995 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 996 *(u_int *)addr = d->bd_hdrcmplt; 997 break; 998 999 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1000 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1001 break; 1002 1003 /* 1004 * Get "see sent packets" flag 1005 */ 1006 case BIOCGSEESENT: 1007 *(u_int *)addr = d->bd_seesent; 1008 break; 1009 1010 /* 1011 * Set "see sent" packets flag 1012 */ 1013 case BIOCSSEESENT: 1014 d->bd_seesent = *(u_int *)addr; 1015 break; 1016 1017 /* 1018 * Set "feed packets from bpf back to input" mode 1019 */ 1020 case BIOCSFEEDBACK: 1021 d->bd_feedback = *(u_int *)addr; 1022 break; 1023 1024 /* 1025 * Get "feed packets from bpf back to input" mode 1026 */ 1027 case BIOCGFEEDBACK: 1028 *(u_int *)addr = d->bd_feedback; 1029 break; 1030 1031 case FIONBIO: /* Non-blocking I/O */ 1032 /* 1033 * No need to do anything special as we use IO_NDELAY in 1034 * bpfread() as an indication of whether or not to block 1035 * the read. 1036 */ 1037 break; 1038 1039 case FIOASYNC: /* Send signal on receive packets */ 1040 d->bd_async = *(int *)addr; 1041 break; 1042 1043 case TIOCSPGRP: /* Process or group to send signals to */ 1044 case FIOSETOWN: 1045 error = fsetown(&d->bd_pgid, cmd, addr); 1046 break; 1047 1048 case TIOCGPGRP: 1049 case FIOGETOWN: 1050 error = fgetown(d->bd_pgid, cmd, addr); 1051 break; 1052 } 1053 KERNEL_UNLOCK_ONE(NULL); 1054 return (error); 1055 } 1056 1057 /* 1058 * Set d's packet filter program to fp. If this file already has a filter, 1059 * free it and replace it. Returns EINVAL for bogus requests. 1060 */ 1061 int 1062 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1063 { 1064 struct bpf_insn *fcode, *old; 1065 bpfjit_function_t jcode, oldj; 1066 size_t flen, size; 1067 int s; 1068 1069 jcode = NULL; 1070 flen = fp->bf_len; 1071 1072 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1073 return EINVAL; 1074 } 1075 1076 if (flen) { 1077 /* 1078 * Allocate the buffer, copy the byte-code from 1079 * userspace and validate it. 1080 */ 1081 size = flen * sizeof(*fp->bf_insns); 1082 fcode = malloc(size, M_DEVBUF, M_WAITOK); 1083 if (copyin(fp->bf_insns, fcode, size) != 0 || 1084 !bpf_validate(fcode, (int)flen)) { 1085 free(fcode, M_DEVBUF); 1086 return EINVAL; 1087 } 1088 membar_consumer(); 1089 if (bpf_jit && bpfjit_module_ops.bj_generate_code != NULL) { 1090 jcode = bpfjit_module_ops.bj_generate_code(fcode, flen); 1091 } 1092 } else { 1093 fcode = NULL; 1094 } 1095 1096 s = splnet(); 1097 old = d->bd_filter; 1098 d->bd_filter = fcode; 1099 oldj = d->bd_jitcode; 1100 d->bd_jitcode = jcode; 1101 reset_d(d); 1102 splx(s); 1103 1104 if (old) { 1105 free(old, M_DEVBUF); 1106 } 1107 1108 if (oldj != NULL) { 1109 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 1110 bpfjit_module_ops.bj_free_code(oldj); 1111 } 1112 1113 return 0; 1114 } 1115 1116 /* 1117 * Detach a file from its current interface (if attached at all) and attach 1118 * to the interface indicated by the name stored in ifr. 1119 * Return an errno or 0. 1120 */ 1121 static int 1122 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1123 { 1124 struct bpf_if *bp; 1125 char *cp; 1126 int unit_seen, i, s, error; 1127 1128 /* 1129 * Make sure the provided name has a unit number, and default 1130 * it to '0' if not specified. 1131 * XXX This is ugly ... do this differently? 1132 */ 1133 unit_seen = 0; 1134 cp = ifr->ifr_name; 1135 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1136 while (*cp++) 1137 if (*cp >= '0' && *cp <= '9') 1138 unit_seen = 1; 1139 if (!unit_seen) { 1140 /* Make sure to leave room for the '\0'. */ 1141 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1142 if ((ifr->ifr_name[i] >= 'a' && 1143 ifr->ifr_name[i] <= 'z') || 1144 (ifr->ifr_name[i] >= 'A' && 1145 ifr->ifr_name[i] <= 'Z')) 1146 continue; 1147 ifr->ifr_name[i] = '0'; 1148 } 1149 } 1150 1151 /* 1152 * Look through attached interfaces for the named one. 1153 */ 1154 for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) { 1155 struct ifnet *ifp = bp->bif_ifp; 1156 1157 if (ifp == 0 || 1158 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1159 continue; 1160 /* skip additional entry */ 1161 if (bp->bif_driverp != &ifp->if_bpf) 1162 continue; 1163 /* 1164 * We found the requested interface. 1165 * Allocate the packet buffers if we need to. 1166 * If we're already attached to requested interface, 1167 * just flush the buffer. 1168 */ 1169 if (d->bd_sbuf == 0) { 1170 error = bpf_allocbufs(d); 1171 if (error != 0) 1172 return (error); 1173 } 1174 s = splnet(); 1175 if (bp != d->bd_bif) { 1176 if (d->bd_bif) 1177 /* 1178 * Detach if attached to something else. 1179 */ 1180 bpf_detachd(d); 1181 1182 bpf_attachd(d, bp); 1183 } 1184 reset_d(d); 1185 splx(s); 1186 return (0); 1187 } 1188 /* Not found. */ 1189 return (ENXIO); 1190 } 1191 1192 /* 1193 * Copy the interface name to the ifreq. 1194 */ 1195 static void 1196 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1197 { 1198 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1199 } 1200 1201 static int 1202 bpf_stat(struct file *fp, struct stat *st) 1203 { 1204 struct bpf_d *d = fp->f_data; 1205 1206 (void)memset(st, 0, sizeof(*st)); 1207 KERNEL_LOCK(1, NULL); 1208 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1209 st->st_atimespec = d->bd_atime; 1210 st->st_mtimespec = d->bd_mtime; 1211 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1212 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1213 st->st_gid = kauth_cred_getegid(fp->f_cred); 1214 st->st_mode = S_IFCHR; 1215 KERNEL_UNLOCK_ONE(NULL); 1216 return 0; 1217 } 1218 1219 /* 1220 * Support for poll() system call 1221 * 1222 * Return true iff the specific operation will not block indefinitely - with 1223 * the assumption that it is safe to positively acknowledge a request for the 1224 * ability to write to the BPF device. 1225 * Otherwise, return false but make a note that a selnotify() must be done. 1226 */ 1227 static int 1228 bpf_poll(struct file *fp, int events) 1229 { 1230 struct bpf_d *d = fp->f_data; 1231 int s = splnet(); 1232 int revents; 1233 1234 /* 1235 * Refresh the PID associated with this bpf file. 1236 */ 1237 KERNEL_LOCK(1, NULL); 1238 d->bd_pid = curproc->p_pid; 1239 1240 revents = events & (POLLOUT | POLLWRNORM); 1241 if (events & (POLLIN | POLLRDNORM)) { 1242 /* 1243 * An imitation of the FIONREAD ioctl code. 1244 */ 1245 if (d->bd_hlen != 0 || 1246 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1247 d->bd_slen != 0)) { 1248 revents |= events & (POLLIN | POLLRDNORM); 1249 } else { 1250 selrecord(curlwp, &d->bd_sel); 1251 /* Start the read timeout if necessary */ 1252 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1253 callout_reset(&d->bd_callout, d->bd_rtout, 1254 bpf_timed_out, d); 1255 d->bd_state = BPF_WAITING; 1256 } 1257 } 1258 } 1259 1260 KERNEL_UNLOCK_ONE(NULL); 1261 splx(s); 1262 return (revents); 1263 } 1264 1265 static void 1266 filt_bpfrdetach(struct knote *kn) 1267 { 1268 struct bpf_d *d = kn->kn_hook; 1269 int s; 1270 1271 KERNEL_LOCK(1, NULL); 1272 s = splnet(); 1273 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1274 splx(s); 1275 KERNEL_UNLOCK_ONE(NULL); 1276 } 1277 1278 static int 1279 filt_bpfread(struct knote *kn, long hint) 1280 { 1281 struct bpf_d *d = kn->kn_hook; 1282 int rv; 1283 1284 KERNEL_LOCK(1, NULL); 1285 kn->kn_data = d->bd_hlen; 1286 if (d->bd_immediate) 1287 kn->kn_data += d->bd_slen; 1288 rv = (kn->kn_data > 0); 1289 KERNEL_UNLOCK_ONE(NULL); 1290 return rv; 1291 } 1292 1293 static const struct filterops bpfread_filtops = 1294 { 1, NULL, filt_bpfrdetach, filt_bpfread }; 1295 1296 static int 1297 bpf_kqfilter(struct file *fp, struct knote *kn) 1298 { 1299 struct bpf_d *d = fp->f_data; 1300 struct klist *klist; 1301 int s; 1302 1303 KERNEL_LOCK(1, NULL); 1304 1305 switch (kn->kn_filter) { 1306 case EVFILT_READ: 1307 klist = &d->bd_sel.sel_klist; 1308 kn->kn_fop = &bpfread_filtops; 1309 break; 1310 1311 default: 1312 KERNEL_UNLOCK_ONE(NULL); 1313 return (EINVAL); 1314 } 1315 1316 kn->kn_hook = d; 1317 1318 s = splnet(); 1319 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1320 splx(s); 1321 KERNEL_UNLOCK_ONE(NULL); 1322 1323 return (0); 1324 } 1325 1326 /* 1327 * Copy data from an mbuf chain into a buffer. This code is derived 1328 * from m_copydata in sys/uipc_mbuf.c. 1329 */ 1330 static void * 1331 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1332 { 1333 const struct mbuf *m; 1334 u_int count; 1335 u_char *dst; 1336 1337 m = src_arg; 1338 dst = dst_arg; 1339 while (len > 0) { 1340 if (m == NULL) 1341 panic("bpf_mcpy"); 1342 count = min(m->m_len, len); 1343 memcpy(dst, mtod(m, const void *), count); 1344 m = m->m_next; 1345 dst += count; 1346 len -= count; 1347 } 1348 return dst_arg; 1349 } 1350 1351 /* 1352 * Dispatch a packet to all the listeners on interface bp. 1353 * 1354 * pkt pointer to the packet, either a data buffer or an mbuf chain 1355 * buflen buffer length, if pkt is a data buffer 1356 * cpfn a function that can copy pkt into the listener's buffer 1357 * pktlen length of the packet 1358 * rcv true if packet came in 1359 */ 1360 static inline void 1361 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1362 void *pkt, u_int pktlen, u_int buflen, const bool rcv) 1363 { 1364 struct bpf_d *d; 1365 struct timespec ts; 1366 bool gottime = false; 1367 1368 /* 1369 * Note that the IPL does not have to be raised at this point. 1370 * The only problem that could arise here is that if two different 1371 * interfaces shared any data. This is not the case. 1372 */ 1373 for (d = bp->bif_dlist; d != NULL; d = d->bd_next) { 1374 u_int slen; 1375 1376 if (!d->bd_seesent && !rcv) { 1377 continue; 1378 } 1379 d->bd_rcount++; 1380 bpf_gstats.bs_recv++; 1381 1382 if (d->bd_jitcode != NULL) 1383 slen = d->bd_jitcode(pkt, pktlen, buflen); 1384 else 1385 slen = bpf_filter(d->bd_filter, pkt, pktlen, buflen); 1386 1387 if (!slen) { 1388 continue; 1389 } 1390 if (!gottime) { 1391 gottime = true; 1392 nanotime(&ts); 1393 } 1394 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1395 } 1396 } 1397 1398 /* 1399 * Incoming linkage from device drivers. Process the packet pkt, of length 1400 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1401 * by each process' filter, and if accepted, stashed into the corresponding 1402 * buffer. 1403 */ 1404 static void 1405 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1406 { 1407 1408 bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true); 1409 } 1410 1411 /* 1412 * Incoming linkage from device drivers, when the head of the packet is in 1413 * a buffer, and the tail is in an mbuf chain. 1414 */ 1415 static void 1416 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1417 { 1418 u_int pktlen; 1419 struct mbuf mb; 1420 1421 /* Skip outgoing duplicate packets. */ 1422 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1423 m->m_flags &= ~M_PROMISC; 1424 return; 1425 } 1426 1427 pktlen = m_length(m) + dlen; 1428 1429 /* 1430 * Craft on-stack mbuf suitable for passing to bpf_filter. 1431 * Note that we cut corners here; we only setup what's 1432 * absolutely needed--this mbuf should never go anywhere else. 1433 */ 1434 (void)memset(&mb, 0, sizeof(mb)); 1435 mb.m_next = m; 1436 mb.m_data = data; 1437 mb.m_len = dlen; 1438 1439 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif != NULL); 1440 } 1441 1442 /* 1443 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1444 */ 1445 static void 1446 _bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1447 { 1448 void *(*cpfn)(void *, const void *, size_t); 1449 u_int pktlen, buflen; 1450 void *marg; 1451 1452 /* Skip outgoing duplicate packets. */ 1453 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1454 m->m_flags &= ~M_PROMISC; 1455 return; 1456 } 1457 1458 pktlen = m_length(m); 1459 1460 if (pktlen == m->m_len) { 1461 cpfn = (void *)memcpy; 1462 marg = mtod(m, void *); 1463 buflen = pktlen; 1464 } else { 1465 cpfn = bpf_mcpy; 1466 marg = m; 1467 buflen = 0; 1468 } 1469 1470 bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif != NULL); 1471 } 1472 1473 /* 1474 * We need to prepend the address family as 1475 * a four byte field. Cons up a dummy header 1476 * to pacify bpf. This is safe because bpf 1477 * will only read from the mbuf (i.e., it won't 1478 * try to free it or keep a pointer a to it). 1479 */ 1480 static void 1481 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m) 1482 { 1483 struct mbuf m0; 1484 1485 m0.m_flags = 0; 1486 m0.m_next = m; 1487 m0.m_len = 4; 1488 m0.m_data = (char *)⁡ 1489 1490 _bpf_mtap(bp, &m0); 1491 } 1492 1493 /* 1494 * Put the SLIP pseudo-"link header" in place. 1495 * Note this M_PREPEND() should never fail, 1496 * swince we know we always have enough space 1497 * in the input buffer. 1498 */ 1499 static void 1500 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1501 { 1502 int s; 1503 u_char *hp; 1504 1505 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1506 if (*m == NULL) 1507 return; 1508 1509 hp = mtod(*m, u_char *); 1510 hp[SLX_DIR] = SLIPDIR_IN; 1511 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1512 1513 s = splnet(); 1514 _bpf_mtap(bp, *m); 1515 splx(s); 1516 1517 m_adj(*m, SLIP_HDRLEN); 1518 } 1519 1520 /* 1521 * Put the SLIP pseudo-"link header" in 1522 * place. The compressed header is now 1523 * at the beginning of the mbuf. 1524 */ 1525 static void 1526 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1527 { 1528 struct mbuf m0; 1529 u_char *hp; 1530 int s; 1531 1532 m0.m_flags = 0; 1533 m0.m_next = m; 1534 m0.m_data = m0.m_dat; 1535 m0.m_len = SLIP_HDRLEN; 1536 1537 hp = mtod(&m0, u_char *); 1538 1539 hp[SLX_DIR] = SLIPDIR_OUT; 1540 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1541 1542 s = splnet(); 1543 _bpf_mtap(bp, &m0); 1544 splx(s); 1545 m_freem(m); 1546 } 1547 1548 static int 1549 bpf_hdrlen(struct bpf_d *d) 1550 { 1551 int hdrlen = d->bd_bif->bif_hdrlen; 1552 /* 1553 * Compute the length of the bpf header. This is not necessarily 1554 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1555 * that the network layer header begins on a longword boundary (for 1556 * performance reasons and to alleviate alignment restrictions). 1557 */ 1558 #ifdef _LP64 1559 if (d->bd_compat32) 1560 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1561 else 1562 #endif 1563 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1564 } 1565 1566 /* 1567 * Move the packet data from interface memory (pkt) into the 1568 * store buffer. Call the wakeup functions if it's time to wakeup 1569 * a listener (buffer full), "cpfn" is the routine called to do the 1570 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1571 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1572 * pkt is really an mbuf. 1573 */ 1574 static void 1575 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1576 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1577 { 1578 struct bpf_hdr *hp; 1579 #ifdef _LP64 1580 struct bpf_hdr32 *hp32; 1581 #endif 1582 int totlen, curlen; 1583 int hdrlen = bpf_hdrlen(d); 1584 int do_wakeup = 0; 1585 1586 ++d->bd_ccount; 1587 ++bpf_gstats.bs_capt; 1588 /* 1589 * Figure out how many bytes to move. If the packet is 1590 * greater or equal to the snapshot length, transfer that 1591 * much. Otherwise, transfer the whole packet (unless 1592 * we hit the buffer size limit). 1593 */ 1594 totlen = hdrlen + min(snaplen, pktlen); 1595 if (totlen > d->bd_bufsize) 1596 totlen = d->bd_bufsize; 1597 1598 /* 1599 * Round up the end of the previous packet to the next longword. 1600 */ 1601 #ifdef _LP64 1602 if (d->bd_compat32) 1603 curlen = BPF_WORDALIGN32(d->bd_slen); 1604 else 1605 #endif 1606 curlen = BPF_WORDALIGN(d->bd_slen); 1607 if (curlen + totlen > d->bd_bufsize) { 1608 /* 1609 * This packet will overflow the storage buffer. 1610 * Rotate the buffers if we can, then wakeup any 1611 * pending reads. 1612 */ 1613 if (d->bd_fbuf == 0) { 1614 /* 1615 * We haven't completed the previous read yet, 1616 * so drop the packet. 1617 */ 1618 ++d->bd_dcount; 1619 ++bpf_gstats.bs_drop; 1620 return; 1621 } 1622 ROTATE_BUFFERS(d); 1623 do_wakeup = 1; 1624 curlen = 0; 1625 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1626 /* 1627 * Immediate mode is set, or the read timeout has 1628 * already expired during a select call. A packet 1629 * arrived, so the reader should be woken up. 1630 */ 1631 do_wakeup = 1; 1632 } 1633 1634 /* 1635 * Append the bpf header. 1636 */ 1637 #ifdef _LP64 1638 if (d->bd_compat32) { 1639 hp32 = (struct bpf_hdr32 *)((char *)d->bd_sbuf + curlen); 1640 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1641 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1642 hp32->bh_datalen = pktlen; 1643 hp32->bh_hdrlen = hdrlen; 1644 /* 1645 * Copy the packet data into the store buffer and update its length. 1646 */ 1647 (*cpfn)((u_char *)hp32 + hdrlen, pkt, 1648 (hp32->bh_caplen = totlen - hdrlen)); 1649 } else 1650 #endif 1651 { 1652 hp = (struct bpf_hdr *)((char *)d->bd_sbuf + curlen); 1653 hp->bh_tstamp.tv_sec = ts->tv_sec; 1654 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1655 hp->bh_datalen = pktlen; 1656 hp->bh_hdrlen = hdrlen; 1657 /* 1658 * Copy the packet data into the store buffer and update 1659 * its length. 1660 */ 1661 (*cpfn)((u_char *)hp + hdrlen, pkt, 1662 (hp->bh_caplen = totlen - hdrlen)); 1663 } 1664 d->bd_slen = curlen + totlen; 1665 1666 /* 1667 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1668 * will cause filt_bpfread() to be called with it adjusted. 1669 */ 1670 if (do_wakeup) 1671 bpf_wakeup(d); 1672 } 1673 1674 /* 1675 * Initialize all nonzero fields of a descriptor. 1676 */ 1677 static int 1678 bpf_allocbufs(struct bpf_d *d) 1679 { 1680 1681 d->bd_fbuf = malloc(d->bd_bufsize, M_DEVBUF, M_WAITOK | M_CANFAIL); 1682 if (!d->bd_fbuf) 1683 return (ENOBUFS); 1684 d->bd_sbuf = malloc(d->bd_bufsize, M_DEVBUF, M_WAITOK | M_CANFAIL); 1685 if (!d->bd_sbuf) { 1686 free(d->bd_fbuf, M_DEVBUF); 1687 return (ENOBUFS); 1688 } 1689 d->bd_slen = 0; 1690 d->bd_hlen = 0; 1691 return (0); 1692 } 1693 1694 /* 1695 * Free buffers currently in use by a descriptor. 1696 * Called on close. 1697 */ 1698 static void 1699 bpf_freed(struct bpf_d *d) 1700 { 1701 /* 1702 * We don't need to lock out interrupts since this descriptor has 1703 * been detached from its interface and it yet hasn't been marked 1704 * free. 1705 */ 1706 if (d->bd_sbuf != NULL) { 1707 free(d->bd_sbuf, M_DEVBUF); 1708 if (d->bd_hbuf != NULL) 1709 free(d->bd_hbuf, M_DEVBUF); 1710 if (d->bd_fbuf != NULL) 1711 free(d->bd_fbuf, M_DEVBUF); 1712 } 1713 if (d->bd_filter) 1714 free(d->bd_filter, M_DEVBUF); 1715 1716 if (d->bd_jitcode != NULL) { 1717 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 1718 bpfjit_module_ops.bj_free_code(d->bd_jitcode); 1719 } 1720 } 1721 1722 /* 1723 * Attach an interface to bpf. dlt is the link layer type; 1724 * hdrlen is the fixed size of the link header for the specified dlt 1725 * (variable length headers not yet supported). 1726 */ 1727 static void 1728 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 1729 { 1730 struct bpf_if *bp; 1731 bp = malloc(sizeof(*bp), M_DEVBUF, M_DONTWAIT); 1732 if (bp == 0) 1733 panic("bpfattach"); 1734 1735 bp->bif_dlist = 0; 1736 bp->bif_driverp = driverp; 1737 bp->bif_ifp = ifp; 1738 bp->bif_dlt = dlt; 1739 1740 bp->bif_next = bpf_iflist; 1741 bpf_iflist = bp; 1742 1743 *bp->bif_driverp = 0; 1744 1745 bp->bif_hdrlen = hdrlen; 1746 #if 0 1747 printf("bpf: %s attached\n", ifp->if_xname); 1748 #endif 1749 } 1750 1751 /* 1752 * Remove an interface from bpf. 1753 */ 1754 static void 1755 _bpfdetach(struct ifnet *ifp) 1756 { 1757 struct bpf_if *bp, **pbp; 1758 struct bpf_d *d; 1759 int s; 1760 1761 /* Nuke the vnodes for any open instances */ 1762 LIST_FOREACH(d, &bpf_list, bd_list) { 1763 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 1764 /* 1765 * Detach the descriptor from an interface now. 1766 * It will be free'ed later by close routine. 1767 */ 1768 s = splnet(); 1769 d->bd_promisc = 0; /* we can't touch device. */ 1770 bpf_detachd(d); 1771 splx(s); 1772 } 1773 } 1774 1775 again: 1776 for (bp = bpf_iflist, pbp = &bpf_iflist; 1777 bp != NULL; pbp = &bp->bif_next, bp = bp->bif_next) { 1778 if (bp->bif_ifp == ifp) { 1779 *pbp = bp->bif_next; 1780 free(bp, M_DEVBUF); 1781 goto again; 1782 } 1783 } 1784 } 1785 1786 /* 1787 * Change the data link type of a interface. 1788 */ 1789 static void 1790 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 1791 { 1792 struct bpf_if *bp; 1793 1794 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1795 if (bp->bif_driverp == &ifp->if_bpf) 1796 break; 1797 } 1798 if (bp == NULL) 1799 panic("bpf_change_type"); 1800 1801 bp->bif_dlt = dlt; 1802 1803 bp->bif_hdrlen = hdrlen; 1804 } 1805 1806 /* 1807 * Get a list of available data link type of the interface. 1808 */ 1809 static int 1810 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 1811 { 1812 int n, error; 1813 struct ifnet *ifp; 1814 struct bpf_if *bp; 1815 1816 ifp = d->bd_bif->bif_ifp; 1817 n = 0; 1818 error = 0; 1819 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1820 if (bp->bif_ifp != ifp) 1821 continue; 1822 if (bfl->bfl_list != NULL) { 1823 if (n >= bfl->bfl_len) 1824 return ENOMEM; 1825 error = copyout(&bp->bif_dlt, 1826 bfl->bfl_list + n, sizeof(u_int)); 1827 } 1828 n++; 1829 } 1830 bfl->bfl_len = n; 1831 return error; 1832 } 1833 1834 /* 1835 * Set the data link type of a BPF instance. 1836 */ 1837 static int 1838 bpf_setdlt(struct bpf_d *d, u_int dlt) 1839 { 1840 int s, error, opromisc; 1841 struct ifnet *ifp; 1842 struct bpf_if *bp; 1843 1844 if (d->bd_bif->bif_dlt == dlt) 1845 return 0; 1846 ifp = d->bd_bif->bif_ifp; 1847 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1848 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 1849 break; 1850 } 1851 if (bp == NULL) 1852 return EINVAL; 1853 s = splnet(); 1854 opromisc = d->bd_promisc; 1855 bpf_detachd(d); 1856 bpf_attachd(d, bp); 1857 reset_d(d); 1858 if (opromisc) { 1859 error = ifpromisc(bp->bif_ifp, 1); 1860 if (error) 1861 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 1862 bp->bif_ifp->if_xname, error); 1863 else 1864 d->bd_promisc = 1; 1865 } 1866 splx(s); 1867 return 0; 1868 } 1869 1870 static int 1871 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 1872 { 1873 int newsize, error; 1874 struct sysctlnode node; 1875 1876 node = *rnode; 1877 node.sysctl_data = &newsize; 1878 newsize = bpf_maxbufsize; 1879 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1880 if (error || newp == NULL) 1881 return (error); 1882 1883 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 1884 return (EINVAL); 1885 1886 bpf_maxbufsize = newsize; 1887 1888 return (0); 1889 } 1890 1891 static int 1892 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 1893 { 1894 bool newval; 1895 int error; 1896 struct sysctlnode node; 1897 1898 node = *rnode; 1899 node.sysctl_data = &newval; 1900 newval = bpf_jit; 1901 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1902 if (error != 0 || newp == NULL) 1903 return error; 1904 1905 bpf_jit = newval; 1906 1907 /* 1908 * Do a full sync to publish new bpf_jit value and 1909 * update bpfjit_module_ops.bj_generate_code variable. 1910 */ 1911 membar_sync(); 1912 1913 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 1914 printf("WARNING jit activation is postponed " 1915 "until after bpfjit module is loaded\n"); 1916 } 1917 1918 return 0; 1919 } 1920 1921 static int 1922 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 1923 { 1924 int error, elem_count; 1925 struct bpf_d *dp; 1926 struct bpf_d_ext dpe; 1927 size_t len, needed, elem_size, out_size; 1928 char *sp; 1929 1930 if (namelen == 1 && name[0] == CTL_QUERY) 1931 return (sysctl_query(SYSCTLFN_CALL(rnode))); 1932 1933 if (namelen != 2) 1934 return (EINVAL); 1935 1936 /* BPF peers is privileged information. */ 1937 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 1938 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 1939 if (error) 1940 return (EPERM); 1941 1942 len = (oldp != NULL) ? *oldlenp : 0; 1943 sp = oldp; 1944 elem_size = name[0]; 1945 elem_count = name[1]; 1946 out_size = MIN(sizeof(dpe), elem_size); 1947 needed = 0; 1948 1949 if (elem_size < 1 || elem_count < 0) 1950 return (EINVAL); 1951 1952 mutex_enter(&bpf_mtx); 1953 LIST_FOREACH(dp, &bpf_list, bd_list) { 1954 if (len >= elem_size && elem_count > 0) { 1955 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 1956 BPF_EXT(bufsize); 1957 BPF_EXT(promisc); 1958 BPF_EXT(state); 1959 BPF_EXT(immediate); 1960 BPF_EXT(hdrcmplt); 1961 BPF_EXT(seesent); 1962 BPF_EXT(pid); 1963 BPF_EXT(rcount); 1964 BPF_EXT(dcount); 1965 BPF_EXT(ccount); 1966 #undef BPF_EXT 1967 if (dp->bd_bif) 1968 (void)strlcpy(dpe.bde_ifname, 1969 dp->bd_bif->bif_ifp->if_xname, 1970 IFNAMSIZ - 1); 1971 else 1972 dpe.bde_ifname[0] = '\0'; 1973 1974 error = copyout(&dpe, sp, out_size); 1975 if (error) 1976 break; 1977 sp += elem_size; 1978 len -= elem_size; 1979 } 1980 needed += elem_size; 1981 if (elem_count > 0 && elem_count != INT_MAX) 1982 elem_count--; 1983 } 1984 mutex_exit(&bpf_mtx); 1985 1986 *oldlenp = needed; 1987 1988 return (error); 1989 } 1990 1991 static struct sysctllog *bpf_sysctllog; 1992 static void 1993 sysctl_net_bpf_setup(void) 1994 { 1995 const struct sysctlnode *node; 1996 1997 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 1998 CTLFLAG_PERMANENT, 1999 CTLTYPE_NODE, "net", NULL, 2000 NULL, 0, NULL, 0, 2001 CTL_NET, CTL_EOL); 2002 2003 node = NULL; 2004 sysctl_createv(&bpf_sysctllog, 0, NULL, &node, 2005 CTLFLAG_PERMANENT, 2006 CTLTYPE_NODE, "bpf", 2007 SYSCTL_DESCR("BPF options"), 2008 NULL, 0, NULL, 0, 2009 CTL_NET, CTL_CREATE, CTL_EOL); 2010 if (node != NULL) { 2011 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2012 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2013 CTLTYPE_BOOL, "jit", 2014 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2015 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2016 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2017 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2018 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2019 CTLTYPE_INT, "maxbufsize", 2020 SYSCTL_DESCR("Maximum size for data capture buffer"), 2021 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2022 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2023 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2024 CTLFLAG_PERMANENT, 2025 CTLTYPE_STRUCT, "stats", 2026 SYSCTL_DESCR("BPF stats"), 2027 NULL, 0, &bpf_gstats, sizeof(bpf_gstats), 2028 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2029 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2030 CTLFLAG_PERMANENT, 2031 CTLTYPE_STRUCT, "peers", 2032 SYSCTL_DESCR("BPF peers"), 2033 sysctl_net_bpf_peers, 0, NULL, 0, 2034 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2035 } 2036 2037 } 2038 2039 struct bpf_ops bpf_ops_kernel = { 2040 .bpf_attach = _bpfattach, 2041 .bpf_detach = _bpfdetach, 2042 .bpf_change_type = _bpf_change_type, 2043 2044 .bpf_tap = _bpf_tap, 2045 .bpf_mtap = _bpf_mtap, 2046 .bpf_mtap2 = _bpf_mtap2, 2047 .bpf_mtap_af = _bpf_mtap_af, 2048 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2049 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2050 }; 2051 2052 MODULE(MODULE_CLASS_DRIVER, bpf, NULL); 2053 2054 static int 2055 bpf_modcmd(modcmd_t cmd, void *arg) 2056 { 2057 devmajor_t bmajor, cmajor; 2058 int error; 2059 2060 bmajor = cmajor = NODEVMAJOR; 2061 2062 switch (cmd) { 2063 case MODULE_CMD_INIT: 2064 bpfilterattach(0); 2065 error = devsw_attach("bpf", NULL, &bmajor, 2066 &bpf_cdevsw, &cmajor); 2067 if (error == EEXIST) 2068 error = 0; /* maybe built-in ... improve eventually */ 2069 if (error) 2070 break; 2071 2072 bpf_ops_handover_enter(&bpf_ops_kernel); 2073 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2074 bpf_ops_handover_exit(); 2075 sysctl_net_bpf_setup(); 2076 break; 2077 2078 case MODULE_CMD_FINI: 2079 /* 2080 * While there is no reference counting for bpf callers, 2081 * unload could at least in theory be done similarly to 2082 * system call disestablishment. This should even be 2083 * a little simpler: 2084 * 2085 * 1) replace op vector with stubs 2086 * 2) post update to all cpus with xc 2087 * 3) check that nobody is in bpf anymore 2088 * (it's doubtful we'd want something like l_sysent, 2089 * but we could do something like *signed* percpu 2090 * counters. if the sum is 0, we're good). 2091 * 4) if fail, unroll changes 2092 * 2093 * NOTE: change won't be atomic to the outside. some 2094 * packets may be not captured even if unload is 2095 * not succesful. I think packet capture not working 2096 * is a perfectly logical consequence of trying to 2097 * disable packet capture. 2098 */ 2099 error = EOPNOTSUPP; 2100 /* insert sysctl teardown */ 2101 break; 2102 2103 default: 2104 error = ENOTTY; 2105 break; 2106 } 2107 2108 return error; 2109 } 2110