1 /* $NetBSD: bpf.c,v 1.193 2015/12/16 23:14:42 christos Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.193 2015/12/16 23:14:42 christos Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "strip.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/mbuf.h> 53 #include <sys/buf.h> 54 #include <sys/time.h> 55 #include <sys/proc.h> 56 #include <sys/ioctl.h> 57 #include <sys/conf.h> 58 #include <sys/vnode.h> 59 #include <sys/queue.h> 60 #include <sys/stat.h> 61 #include <sys/module.h> 62 #include <sys/once.h> 63 #include <sys/atomic.h> 64 65 #include <sys/file.h> 66 #include <sys/filedesc.h> 67 #include <sys/tty.h> 68 #include <sys/uio.h> 69 70 #include <sys/protosw.h> 71 #include <sys/socket.h> 72 #include <sys/errno.h> 73 #include <sys/kernel.h> 74 #include <sys/poll.h> 75 #include <sys/sysctl.h> 76 #include <sys/kauth.h> 77 78 #include <net/if.h> 79 #include <net/slip.h> 80 81 #include <net/bpf.h> 82 #include <net/bpfdesc.h> 83 #include <net/bpfjit.h> 84 85 #include <net/if_arc.h> 86 #include <net/if_ether.h> 87 88 #include <netinet/in.h> 89 #include <netinet/if_inarp.h> 90 91 92 #include <compat/sys/sockio.h> 93 94 #ifndef BPF_BUFSIZE 95 /* 96 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 97 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 98 */ 99 # define BPF_BUFSIZE 32768 100 #endif 101 102 #define PRINET 26 /* interruptible */ 103 104 /* 105 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 106 * XXX the default values should be computed dynamically based 107 * on available memory size and available mbuf clusters. 108 */ 109 int bpf_bufsize = BPF_BUFSIZE; 110 int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 111 bool bpf_jit = false; 112 113 struct bpfjit_ops bpfjit_module_ops = { 114 .bj_generate_code = NULL, 115 .bj_free_code = NULL 116 }; 117 118 /* 119 * Global BPF statistics returned by net.bpf.stats sysctl. 120 */ 121 struct bpf_stat bpf_gstats; 122 123 /* 124 * Use a mutex to avoid a race condition between gathering the stats/peers 125 * and opening/closing the device. 126 */ 127 static kmutex_t bpf_mtx; 128 129 /* 130 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 131 * bpf_dtab holds the descriptors, indexed by minor device # 132 */ 133 struct bpf_if *bpf_iflist; 134 LIST_HEAD(, bpf_d) bpf_list; 135 136 static int bpf_allocbufs(struct bpf_d *); 137 static void bpf_deliver(struct bpf_if *, 138 void *(*cpfn)(void *, const void *, size_t), 139 void *, u_int, u_int, const bool); 140 static void bpf_freed(struct bpf_d *); 141 static void bpf_ifname(struct ifnet *, struct ifreq *); 142 static void *bpf_mcpy(void *, const void *, size_t); 143 static int bpf_movein(struct uio *, int, uint64_t, 144 struct mbuf **, struct sockaddr *); 145 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 146 static void bpf_detachd(struct bpf_d *); 147 static int bpf_setif(struct bpf_d *, struct ifreq *); 148 static void bpf_timed_out(void *); 149 static inline void 150 bpf_wakeup(struct bpf_d *); 151 static int bpf_hdrlen(struct bpf_d *); 152 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 153 void *(*)(void *, const void *, size_t), struct timespec *); 154 static void reset_d(struct bpf_d *); 155 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 156 static int bpf_setdlt(struct bpf_d *, u_int); 157 158 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 159 int); 160 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 161 int); 162 static int bpf_ioctl(struct file *, u_long, void *); 163 static int bpf_poll(struct file *, int); 164 static int bpf_stat(struct file *, struct stat *); 165 static int bpf_close(struct file *); 166 static int bpf_kqfilter(struct file *, struct knote *); 167 static void bpf_softintr(void *); 168 169 static const struct fileops bpf_fileops = { 170 .fo_read = bpf_read, 171 .fo_write = bpf_write, 172 .fo_ioctl = bpf_ioctl, 173 .fo_fcntl = fnullop_fcntl, 174 .fo_poll = bpf_poll, 175 .fo_stat = bpf_stat, 176 .fo_close = bpf_close, 177 .fo_kqfilter = bpf_kqfilter, 178 .fo_restart = fnullop_restart, 179 }; 180 181 dev_type_open(bpfopen); 182 183 const struct cdevsw bpf_cdevsw = { 184 .d_open = bpfopen, 185 .d_close = noclose, 186 .d_read = noread, 187 .d_write = nowrite, 188 .d_ioctl = noioctl, 189 .d_stop = nostop, 190 .d_tty = notty, 191 .d_poll = nopoll, 192 .d_mmap = nommap, 193 .d_kqfilter = nokqfilter, 194 .d_discard = nodiscard, 195 .d_flag = D_OTHER 196 }; 197 198 bpfjit_func_t 199 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 200 { 201 202 membar_consumer(); 203 if (bpfjit_module_ops.bj_generate_code != NULL) { 204 return bpfjit_module_ops.bj_generate_code(bc, code, size); 205 } 206 return NULL; 207 } 208 209 void 210 bpf_jit_freecode(bpfjit_func_t jcode) 211 { 212 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 213 bpfjit_module_ops.bj_free_code(jcode); 214 } 215 216 static int 217 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 218 struct sockaddr *sockp) 219 { 220 struct mbuf *m; 221 int error; 222 size_t len; 223 size_t hlen; 224 size_t align; 225 226 /* 227 * Build a sockaddr based on the data link layer type. 228 * We do this at this level because the ethernet header 229 * is copied directly into the data field of the sockaddr. 230 * In the case of SLIP, there is no header and the packet 231 * is forwarded as is. 232 * Also, we are careful to leave room at the front of the mbuf 233 * for the link level header. 234 */ 235 switch (linktype) { 236 237 case DLT_SLIP: 238 sockp->sa_family = AF_INET; 239 hlen = 0; 240 align = 0; 241 break; 242 243 case DLT_PPP: 244 sockp->sa_family = AF_UNSPEC; 245 hlen = 0; 246 align = 0; 247 break; 248 249 case DLT_EN10MB: 250 sockp->sa_family = AF_UNSPEC; 251 /* XXX Would MAXLINKHDR be better? */ 252 /* 6(dst)+6(src)+2(type) */ 253 hlen = sizeof(struct ether_header); 254 align = 2; 255 break; 256 257 case DLT_ARCNET: 258 sockp->sa_family = AF_UNSPEC; 259 hlen = ARC_HDRLEN; 260 align = 5; 261 break; 262 263 case DLT_FDDI: 264 sockp->sa_family = AF_LINK; 265 /* XXX 4(FORMAC)+6(dst)+6(src) */ 266 hlen = 16; 267 align = 0; 268 break; 269 270 case DLT_ECONET: 271 sockp->sa_family = AF_UNSPEC; 272 hlen = 6; 273 align = 2; 274 break; 275 276 case DLT_NULL: 277 sockp->sa_family = AF_UNSPEC; 278 hlen = 0; 279 align = 0; 280 break; 281 282 default: 283 return (EIO); 284 } 285 286 len = uio->uio_resid; 287 /* 288 * If there aren't enough bytes for a link level header or the 289 * packet length exceeds the interface mtu, return an error. 290 */ 291 if (len - hlen > mtu) 292 return (EMSGSIZE); 293 294 /* 295 * XXX Avoid complicated buffer chaining --- 296 * bail if it won't fit in a single mbuf. 297 * (Take into account possible alignment bytes) 298 */ 299 if (len + align > MCLBYTES) 300 return (EIO); 301 302 m = m_gethdr(M_WAIT, MT_DATA); 303 m->m_pkthdr.rcvif = NULL; 304 m->m_pkthdr.len = (int)(len - hlen); 305 if (len + align > MHLEN) { 306 m_clget(m, M_WAIT); 307 if ((m->m_flags & M_EXT) == 0) { 308 error = ENOBUFS; 309 goto bad; 310 } 311 } 312 313 /* Insure the data is properly aligned */ 314 if (align > 0) { 315 m->m_data += align; 316 m->m_len -= (int)align; 317 } 318 319 error = uiomove(mtod(m, void *), len, uio); 320 if (error) 321 goto bad; 322 if (hlen != 0) { 323 memcpy(sockp->sa_data, mtod(m, void *), hlen); 324 m->m_data += hlen; /* XXX */ 325 len -= hlen; 326 } 327 m->m_len = (int)len; 328 *mp = m; 329 return (0); 330 331 bad: 332 m_freem(m); 333 return (error); 334 } 335 336 /* 337 * Attach file to the bpf interface, i.e. make d listen on bp. 338 * Must be called at splnet. 339 */ 340 static void 341 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 342 { 343 KASSERT(mutex_owned(&bpf_mtx)); 344 /* 345 * Point d at bp, and add d to the interface's list of listeners. 346 * Finally, point the driver's bpf cookie at the interface so 347 * it will divert packets to bpf. 348 */ 349 d->bd_bif = bp; 350 d->bd_next = bp->bif_dlist; 351 bp->bif_dlist = d; 352 353 *bp->bif_driverp = bp; 354 } 355 356 /* 357 * Detach a file from its interface. 358 */ 359 static void 360 bpf_detachd(struct bpf_d *d) 361 { 362 struct bpf_d **p; 363 struct bpf_if *bp; 364 365 KASSERT(mutex_owned(&bpf_mtx)); 366 367 bp = d->bd_bif; 368 /* 369 * Check if this descriptor had requested promiscuous mode. 370 * If so, turn it off. 371 */ 372 if (d->bd_promisc) { 373 int error __diagused; 374 375 d->bd_promisc = 0; 376 /* 377 * Take device out of promiscuous mode. Since we were 378 * able to enter promiscuous mode, we should be able 379 * to turn it off. But we can get an error if 380 * the interface was configured down, so only panic 381 * if we don't get an unexpected error. 382 */ 383 error = ifpromisc(bp->bif_ifp, 0); 384 #ifdef DIAGNOSTIC 385 if (error) 386 printf("%s: ifpromisc failed: %d", __func__, error); 387 #endif 388 } 389 /* Remove d from the interface's descriptor list. */ 390 p = &bp->bif_dlist; 391 while (*p != d) { 392 p = &(*p)->bd_next; 393 if (*p == NULL) 394 panic("%s: descriptor not in list", __func__); 395 } 396 *p = (*p)->bd_next; 397 if (bp->bif_dlist == NULL) 398 /* 399 * Let the driver know that there are no more listeners. 400 */ 401 *d->bd_bif->bif_driverp = NULL; 402 d->bd_bif = NULL; 403 } 404 405 static int 406 doinit(void) 407 { 408 409 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 410 411 LIST_INIT(&bpf_list); 412 413 bpf_gstats.bs_recv = 0; 414 bpf_gstats.bs_drop = 0; 415 bpf_gstats.bs_capt = 0; 416 417 return 0; 418 } 419 420 /* 421 * bpfilterattach() is called at boot time. 422 */ 423 /* ARGSUSED */ 424 void 425 bpfilterattach(int n) 426 { 427 static ONCE_DECL(control); 428 429 RUN_ONCE(&control, doinit); 430 } 431 432 /* 433 * Open ethernet device. Clones. 434 */ 435 /* ARGSUSED */ 436 int 437 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 438 { 439 struct bpf_d *d; 440 struct file *fp; 441 int error, fd; 442 443 /* falloc() will fill in the descriptor for us. */ 444 if ((error = fd_allocfile(&fp, &fd)) != 0) 445 return error; 446 447 d = malloc(sizeof(*d), M_DEVBUF, M_WAITOK|M_ZERO); 448 d->bd_bufsize = bpf_bufsize; 449 d->bd_seesent = 1; 450 d->bd_feedback = 0; 451 d->bd_pid = l->l_proc->p_pid; 452 #ifdef _LP64 453 if (curproc->p_flag & PK_32) 454 d->bd_compat32 = 1; 455 #endif 456 getnanotime(&d->bd_btime); 457 d->bd_atime = d->bd_mtime = d->bd_btime; 458 callout_init(&d->bd_callout, 0); 459 selinit(&d->bd_sel); 460 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d); 461 d->bd_jitcode = NULL; 462 463 mutex_enter(&bpf_mtx); 464 LIST_INSERT_HEAD(&bpf_list, d, bd_list); 465 mutex_exit(&bpf_mtx); 466 467 return fd_clone(fp, fd, flag, &bpf_fileops, d); 468 } 469 470 /* 471 * Close the descriptor by detaching it from its interface, 472 * deallocating its buffers, and marking it free. 473 */ 474 /* ARGSUSED */ 475 static int 476 bpf_close(struct file *fp) 477 { 478 struct bpf_d *d = fp->f_bpf; 479 int s; 480 481 KERNEL_LOCK(1, NULL); 482 mutex_enter(&bpf_mtx); 483 484 /* 485 * Refresh the PID associated with this bpf file. 486 */ 487 d->bd_pid = curproc->p_pid; 488 489 s = splnet(); 490 if (d->bd_state == BPF_WAITING) 491 callout_stop(&d->bd_callout); 492 d->bd_state = BPF_IDLE; 493 if (d->bd_bif) 494 bpf_detachd(d); 495 splx(s); 496 bpf_freed(d); 497 LIST_REMOVE(d, bd_list); 498 callout_destroy(&d->bd_callout); 499 seldestroy(&d->bd_sel); 500 softint_disestablish(d->bd_sih); 501 free(d, M_DEVBUF); 502 fp->f_bpf = NULL; 503 504 mutex_exit(&bpf_mtx); 505 KERNEL_UNLOCK_ONE(NULL); 506 507 return (0); 508 } 509 510 /* 511 * Rotate the packet buffers in descriptor d. Move the store buffer 512 * into the hold slot, and the free buffer into the store slot. 513 * Zero the length of the new store buffer. 514 */ 515 #define ROTATE_BUFFERS(d) \ 516 (d)->bd_hbuf = (d)->bd_sbuf; \ 517 (d)->bd_hlen = (d)->bd_slen; \ 518 (d)->bd_sbuf = (d)->bd_fbuf; \ 519 (d)->bd_slen = 0; \ 520 (d)->bd_fbuf = NULL; 521 /* 522 * bpfread - read next chunk of packets from buffers 523 */ 524 static int 525 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 526 kauth_cred_t cred, int flags) 527 { 528 struct bpf_d *d = fp->f_bpf; 529 int timed_out; 530 int error; 531 int s; 532 533 getnanotime(&d->bd_atime); 534 /* 535 * Restrict application to use a buffer the same size as 536 * the kernel buffers. 537 */ 538 if (uio->uio_resid != d->bd_bufsize) 539 return (EINVAL); 540 541 KERNEL_LOCK(1, NULL); 542 s = splnet(); 543 if (d->bd_state == BPF_WAITING) 544 callout_stop(&d->bd_callout); 545 timed_out = (d->bd_state == BPF_TIMED_OUT); 546 d->bd_state = BPF_IDLE; 547 /* 548 * If the hold buffer is empty, then do a timed sleep, which 549 * ends when the timeout expires or when enough packets 550 * have arrived to fill the store buffer. 551 */ 552 while (d->bd_hbuf == NULL) { 553 if (fp->f_flag & FNONBLOCK) { 554 if (d->bd_slen == 0) { 555 splx(s); 556 KERNEL_UNLOCK_ONE(NULL); 557 return (EWOULDBLOCK); 558 } 559 ROTATE_BUFFERS(d); 560 break; 561 } 562 563 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 564 /* 565 * A packet(s) either arrived since the previous 566 * read or arrived while we were asleep. 567 * Rotate the buffers and return what's here. 568 */ 569 ROTATE_BUFFERS(d); 570 break; 571 } 572 error = tsleep(d, PRINET|PCATCH, "bpf", 573 d->bd_rtout); 574 if (error == EINTR || error == ERESTART) { 575 splx(s); 576 KERNEL_UNLOCK_ONE(NULL); 577 return (error); 578 } 579 if (error == EWOULDBLOCK) { 580 /* 581 * On a timeout, return what's in the buffer, 582 * which may be nothing. If there is something 583 * in the store buffer, we can rotate the buffers. 584 */ 585 if (d->bd_hbuf) 586 /* 587 * We filled up the buffer in between 588 * getting the timeout and arriving 589 * here, so we don't need to rotate. 590 */ 591 break; 592 593 if (d->bd_slen == 0) { 594 splx(s); 595 KERNEL_UNLOCK_ONE(NULL); 596 return (0); 597 } 598 ROTATE_BUFFERS(d); 599 break; 600 } 601 if (error != 0) 602 goto done; 603 } 604 /* 605 * At this point, we know we have something in the hold slot. 606 */ 607 splx(s); 608 609 /* 610 * Move data from hold buffer into user space. 611 * We know the entire buffer is transferred since 612 * we checked above that the read buffer is bpf_bufsize bytes. 613 */ 614 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 615 616 s = splnet(); 617 d->bd_fbuf = d->bd_hbuf; 618 d->bd_hbuf = NULL; 619 d->bd_hlen = 0; 620 done: 621 splx(s); 622 KERNEL_UNLOCK_ONE(NULL); 623 return (error); 624 } 625 626 627 /* 628 * If there are processes sleeping on this descriptor, wake them up. 629 */ 630 static inline void 631 bpf_wakeup(struct bpf_d *d) 632 { 633 wakeup(d); 634 if (d->bd_async) 635 softint_schedule(d->bd_sih); 636 selnotify(&d->bd_sel, 0, 0); 637 } 638 639 static void 640 bpf_softintr(void *cookie) 641 { 642 struct bpf_d *d; 643 644 d = cookie; 645 if (d->bd_async) 646 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 647 } 648 649 static void 650 bpf_timed_out(void *arg) 651 { 652 struct bpf_d *d = arg; 653 int s; 654 655 s = splnet(); 656 if (d->bd_state == BPF_WAITING) { 657 d->bd_state = BPF_TIMED_OUT; 658 if (d->bd_slen != 0) 659 bpf_wakeup(d); 660 } 661 splx(s); 662 } 663 664 665 static int 666 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 667 kauth_cred_t cred, int flags) 668 { 669 struct bpf_d *d = fp->f_bpf; 670 struct ifnet *ifp; 671 struct mbuf *m, *mc; 672 int error, s; 673 static struct sockaddr_storage dst; 674 675 m = NULL; /* XXX gcc */ 676 677 KERNEL_LOCK(1, NULL); 678 679 if (d->bd_bif == NULL) { 680 KERNEL_UNLOCK_ONE(NULL); 681 return (ENXIO); 682 } 683 getnanotime(&d->bd_mtime); 684 685 ifp = d->bd_bif->bif_ifp; 686 687 if (uio->uio_resid == 0) { 688 KERNEL_UNLOCK_ONE(NULL); 689 return (0); 690 } 691 692 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp->if_mtu, &m, 693 (struct sockaddr *) &dst); 694 if (error) { 695 KERNEL_UNLOCK_ONE(NULL); 696 return (error); 697 } 698 699 if (m->m_pkthdr.len > ifp->if_mtu) { 700 KERNEL_UNLOCK_ONE(NULL); 701 m_freem(m); 702 return (EMSGSIZE); 703 } 704 705 if (d->bd_hdrcmplt) 706 dst.ss_family = pseudo_AF_HDRCMPLT; 707 708 if (d->bd_feedback) { 709 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 710 if (mc != NULL) 711 mc->m_pkthdr.rcvif = ifp; 712 /* Set M_PROMISC for outgoing packets to be discarded. */ 713 if (1 /*d->bd_direction == BPF_D_INOUT*/) 714 m->m_flags |= M_PROMISC; 715 } else 716 mc = NULL; 717 718 s = splsoftnet(); 719 error = (*ifp->if_output)(ifp, m, (struct sockaddr *) &dst, NULL); 720 721 if (mc != NULL) { 722 if (error == 0) 723 (*ifp->if_input)(ifp, mc); 724 else 725 m_freem(mc); 726 } 727 splx(s); 728 KERNEL_UNLOCK_ONE(NULL); 729 /* 730 * The driver frees the mbuf. 731 */ 732 return (error); 733 } 734 735 /* 736 * Reset a descriptor by flushing its packet buffer and clearing the 737 * receive and drop counts. Should be called at splnet. 738 */ 739 static void 740 reset_d(struct bpf_d *d) 741 { 742 if (d->bd_hbuf) { 743 /* Free the hold buffer. */ 744 d->bd_fbuf = d->bd_hbuf; 745 d->bd_hbuf = NULL; 746 } 747 d->bd_slen = 0; 748 d->bd_hlen = 0; 749 d->bd_rcount = 0; 750 d->bd_dcount = 0; 751 d->bd_ccount = 0; 752 } 753 754 /* 755 * FIONREAD Check for read packet available. 756 * BIOCGBLEN Get buffer len [for read()]. 757 * BIOCSETF Set ethernet read filter. 758 * BIOCFLUSH Flush read packet buffer. 759 * BIOCPROMISC Put interface into promiscuous mode. 760 * BIOCGDLT Get link layer type. 761 * BIOCGETIF Get interface name. 762 * BIOCSETIF Set interface. 763 * BIOCSRTIMEOUT Set read timeout. 764 * BIOCGRTIMEOUT Get read timeout. 765 * BIOCGSTATS Get packet stats. 766 * BIOCIMMEDIATE Set immediate mode. 767 * BIOCVERSION Get filter language version. 768 * BIOCGHDRCMPLT Get "header already complete" flag. 769 * BIOCSHDRCMPLT Set "header already complete" flag. 770 * BIOCSFEEDBACK Set packet feedback mode. 771 * BIOCGFEEDBACK Get packet feedback mode. 772 * BIOCGSEESENT Get "see sent packets" mode. 773 * BIOCSSEESENT Set "see sent packets" mode. 774 */ 775 /* ARGSUSED */ 776 static int 777 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 778 { 779 struct bpf_d *d = fp->f_bpf; 780 int s, error = 0; 781 782 /* 783 * Refresh the PID associated with this bpf file. 784 */ 785 KERNEL_LOCK(1, NULL); 786 d->bd_pid = curproc->p_pid; 787 #ifdef _LP64 788 if (curproc->p_flag & PK_32) 789 d->bd_compat32 = 1; 790 else 791 d->bd_compat32 = 0; 792 #endif 793 794 s = splnet(); 795 if (d->bd_state == BPF_WAITING) 796 callout_stop(&d->bd_callout); 797 d->bd_state = BPF_IDLE; 798 splx(s); 799 800 switch (cmd) { 801 802 default: 803 error = EINVAL; 804 break; 805 806 /* 807 * Check for read packet available. 808 */ 809 case FIONREAD: 810 { 811 int n; 812 813 s = splnet(); 814 n = d->bd_slen; 815 if (d->bd_hbuf) 816 n += d->bd_hlen; 817 splx(s); 818 819 *(int *)addr = n; 820 break; 821 } 822 823 /* 824 * Get buffer len [for read()]. 825 */ 826 case BIOCGBLEN: 827 *(u_int *)addr = d->bd_bufsize; 828 break; 829 830 /* 831 * Set buffer length. 832 */ 833 case BIOCSBLEN: 834 if (d->bd_bif != NULL) 835 error = EINVAL; 836 else { 837 u_int size = *(u_int *)addr; 838 839 if (size > bpf_maxbufsize) 840 *(u_int *)addr = size = bpf_maxbufsize; 841 else if (size < BPF_MINBUFSIZE) 842 *(u_int *)addr = size = BPF_MINBUFSIZE; 843 d->bd_bufsize = size; 844 } 845 break; 846 847 /* 848 * Set link layer read filter. 849 */ 850 case BIOCSETF: 851 error = bpf_setf(d, addr); 852 break; 853 854 /* 855 * Flush read packet buffer. 856 */ 857 case BIOCFLUSH: 858 s = splnet(); 859 reset_d(d); 860 splx(s); 861 break; 862 863 /* 864 * Put interface into promiscuous mode. 865 */ 866 case BIOCPROMISC: 867 if (d->bd_bif == NULL) { 868 /* 869 * No interface attached yet. 870 */ 871 error = EINVAL; 872 break; 873 } 874 s = splnet(); 875 if (d->bd_promisc == 0) { 876 error = ifpromisc(d->bd_bif->bif_ifp, 1); 877 if (error == 0) 878 d->bd_promisc = 1; 879 } 880 splx(s); 881 break; 882 883 /* 884 * Get device parameters. 885 */ 886 case BIOCGDLT: 887 if (d->bd_bif == NULL) 888 error = EINVAL; 889 else 890 *(u_int *)addr = d->bd_bif->bif_dlt; 891 break; 892 893 /* 894 * Get a list of supported device parameters. 895 */ 896 case BIOCGDLTLIST: 897 if (d->bd_bif == NULL) 898 error = EINVAL; 899 else 900 error = bpf_getdltlist(d, addr); 901 break; 902 903 /* 904 * Set device parameters. 905 */ 906 case BIOCSDLT: 907 mutex_enter(&bpf_mtx); 908 if (d->bd_bif == NULL) 909 error = EINVAL; 910 else 911 error = bpf_setdlt(d, *(u_int *)addr); 912 mutex_exit(&bpf_mtx); 913 break; 914 915 /* 916 * Set interface name. 917 */ 918 #ifdef OBIOCGETIF 919 case OBIOCGETIF: 920 #endif 921 case BIOCGETIF: 922 if (d->bd_bif == NULL) 923 error = EINVAL; 924 else 925 bpf_ifname(d->bd_bif->bif_ifp, addr); 926 break; 927 928 /* 929 * Set interface. 930 */ 931 #ifdef OBIOCSETIF 932 case OBIOCSETIF: 933 #endif 934 case BIOCSETIF: 935 mutex_enter(&bpf_mtx); 936 error = bpf_setif(d, addr); 937 mutex_exit(&bpf_mtx); 938 break; 939 940 /* 941 * Set read timeout. 942 */ 943 case BIOCSRTIMEOUT: 944 { 945 struct timeval *tv = addr; 946 947 /* Compute number of ticks. */ 948 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 949 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 950 d->bd_rtout = 1; 951 break; 952 } 953 954 #ifdef BIOCGORTIMEOUT 955 /* 956 * Get read timeout. 957 */ 958 case BIOCGORTIMEOUT: 959 { 960 struct timeval50 *tv = addr; 961 962 tv->tv_sec = d->bd_rtout / hz; 963 tv->tv_usec = (d->bd_rtout % hz) * tick; 964 break; 965 } 966 #endif 967 968 #ifdef BIOCSORTIMEOUT 969 /* 970 * Set read timeout. 971 */ 972 case BIOCSORTIMEOUT: 973 { 974 struct timeval50 *tv = addr; 975 976 /* Compute number of ticks. */ 977 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 978 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 979 d->bd_rtout = 1; 980 break; 981 } 982 #endif 983 984 /* 985 * Get read timeout. 986 */ 987 case BIOCGRTIMEOUT: 988 { 989 struct timeval *tv = addr; 990 991 tv->tv_sec = d->bd_rtout / hz; 992 tv->tv_usec = (d->bd_rtout % hz) * tick; 993 break; 994 } 995 /* 996 * Get packet stats. 997 */ 998 case BIOCGSTATS: 999 { 1000 struct bpf_stat *bs = addr; 1001 1002 bs->bs_recv = d->bd_rcount; 1003 bs->bs_drop = d->bd_dcount; 1004 bs->bs_capt = d->bd_ccount; 1005 break; 1006 } 1007 1008 case BIOCGSTATSOLD: 1009 { 1010 struct bpf_stat_old *bs = addr; 1011 1012 bs->bs_recv = d->bd_rcount; 1013 bs->bs_drop = d->bd_dcount; 1014 break; 1015 } 1016 1017 /* 1018 * Set immediate mode. 1019 */ 1020 case BIOCIMMEDIATE: 1021 d->bd_immediate = *(u_int *)addr; 1022 break; 1023 1024 case BIOCVERSION: 1025 { 1026 struct bpf_version *bv = addr; 1027 1028 bv->bv_major = BPF_MAJOR_VERSION; 1029 bv->bv_minor = BPF_MINOR_VERSION; 1030 break; 1031 } 1032 1033 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1034 *(u_int *)addr = d->bd_hdrcmplt; 1035 break; 1036 1037 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1038 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1039 break; 1040 1041 /* 1042 * Get "see sent packets" flag 1043 */ 1044 case BIOCGSEESENT: 1045 *(u_int *)addr = d->bd_seesent; 1046 break; 1047 1048 /* 1049 * Set "see sent" packets flag 1050 */ 1051 case BIOCSSEESENT: 1052 d->bd_seesent = *(u_int *)addr; 1053 break; 1054 1055 /* 1056 * Set "feed packets from bpf back to input" mode 1057 */ 1058 case BIOCSFEEDBACK: 1059 d->bd_feedback = *(u_int *)addr; 1060 break; 1061 1062 /* 1063 * Get "feed packets from bpf back to input" mode 1064 */ 1065 case BIOCGFEEDBACK: 1066 *(u_int *)addr = d->bd_feedback; 1067 break; 1068 1069 case FIONBIO: /* Non-blocking I/O */ 1070 /* 1071 * No need to do anything special as we use IO_NDELAY in 1072 * bpfread() as an indication of whether or not to block 1073 * the read. 1074 */ 1075 break; 1076 1077 case FIOASYNC: /* Send signal on receive packets */ 1078 d->bd_async = *(int *)addr; 1079 break; 1080 1081 case TIOCSPGRP: /* Process or group to send signals to */ 1082 case FIOSETOWN: 1083 error = fsetown(&d->bd_pgid, cmd, addr); 1084 break; 1085 1086 case TIOCGPGRP: 1087 case FIOGETOWN: 1088 error = fgetown(d->bd_pgid, cmd, addr); 1089 break; 1090 } 1091 KERNEL_UNLOCK_ONE(NULL); 1092 return (error); 1093 } 1094 1095 /* 1096 * Set d's packet filter program to fp. If this file already has a filter, 1097 * free it and replace it. Returns EINVAL for bogus requests. 1098 */ 1099 int 1100 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1101 { 1102 struct bpf_insn *fcode, *old; 1103 bpfjit_func_t jcode, oldj; 1104 size_t flen, size; 1105 int s; 1106 1107 jcode = NULL; 1108 flen = fp->bf_len; 1109 1110 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1111 return EINVAL; 1112 } 1113 1114 if (flen) { 1115 /* 1116 * Allocate the buffer, copy the byte-code from 1117 * userspace and validate it. 1118 */ 1119 size = flen * sizeof(*fp->bf_insns); 1120 fcode = malloc(size, M_DEVBUF, M_WAITOK); 1121 if (copyin(fp->bf_insns, fcode, size) != 0 || 1122 !bpf_validate(fcode, (int)flen)) { 1123 free(fcode, M_DEVBUF); 1124 return EINVAL; 1125 } 1126 membar_consumer(); 1127 if (bpf_jit) 1128 jcode = bpf_jit_generate(NULL, fcode, flen); 1129 } else { 1130 fcode = NULL; 1131 } 1132 1133 s = splnet(); 1134 old = d->bd_filter; 1135 d->bd_filter = fcode; 1136 oldj = d->bd_jitcode; 1137 d->bd_jitcode = jcode; 1138 reset_d(d); 1139 splx(s); 1140 1141 if (old) { 1142 free(old, M_DEVBUF); 1143 } 1144 if (oldj) { 1145 bpf_jit_freecode(oldj); 1146 } 1147 1148 return 0; 1149 } 1150 1151 /* 1152 * Detach a file from its current interface (if attached at all) and attach 1153 * to the interface indicated by the name stored in ifr. 1154 * Return an errno or 0. 1155 */ 1156 static int 1157 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1158 { 1159 struct bpf_if *bp; 1160 char *cp; 1161 int unit_seen, i, s, error; 1162 1163 KASSERT(mutex_owned(&bpf_mtx)); 1164 /* 1165 * Make sure the provided name has a unit number, and default 1166 * it to '0' if not specified. 1167 * XXX This is ugly ... do this differently? 1168 */ 1169 unit_seen = 0; 1170 cp = ifr->ifr_name; 1171 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1172 while (*cp++) 1173 if (*cp >= '0' && *cp <= '9') 1174 unit_seen = 1; 1175 if (!unit_seen) { 1176 /* Make sure to leave room for the '\0'. */ 1177 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1178 if ((ifr->ifr_name[i] >= 'a' && 1179 ifr->ifr_name[i] <= 'z') || 1180 (ifr->ifr_name[i] >= 'A' && 1181 ifr->ifr_name[i] <= 'Z')) 1182 continue; 1183 ifr->ifr_name[i] = '0'; 1184 } 1185 } 1186 1187 /* 1188 * Look through attached interfaces for the named one. 1189 */ 1190 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1191 struct ifnet *ifp = bp->bif_ifp; 1192 1193 if (ifp == NULL || 1194 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1195 continue; 1196 /* skip additional entry */ 1197 if (bp->bif_driverp != &ifp->if_bpf) 1198 continue; 1199 /* 1200 * We found the requested interface. 1201 * Allocate the packet buffers if we need to. 1202 * If we're already attached to requested interface, 1203 * just flush the buffer. 1204 */ 1205 if (d->bd_sbuf == NULL) { 1206 error = bpf_allocbufs(d); 1207 if (error != 0) 1208 return (error); 1209 } 1210 s = splnet(); 1211 if (bp != d->bd_bif) { 1212 if (d->bd_bif) 1213 /* 1214 * Detach if attached to something else. 1215 */ 1216 bpf_detachd(d); 1217 1218 bpf_attachd(d, bp); 1219 } 1220 reset_d(d); 1221 splx(s); 1222 return (0); 1223 } 1224 /* Not found. */ 1225 return (ENXIO); 1226 } 1227 1228 /* 1229 * Copy the interface name to the ifreq. 1230 */ 1231 static void 1232 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1233 { 1234 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1235 } 1236 1237 static int 1238 bpf_stat(struct file *fp, struct stat *st) 1239 { 1240 struct bpf_d *d = fp->f_bpf; 1241 1242 (void)memset(st, 0, sizeof(*st)); 1243 KERNEL_LOCK(1, NULL); 1244 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1245 st->st_atimespec = d->bd_atime; 1246 st->st_mtimespec = d->bd_mtime; 1247 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1248 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1249 st->st_gid = kauth_cred_getegid(fp->f_cred); 1250 st->st_mode = S_IFCHR; 1251 KERNEL_UNLOCK_ONE(NULL); 1252 return 0; 1253 } 1254 1255 /* 1256 * Support for poll() system call 1257 * 1258 * Return true iff the specific operation will not block indefinitely - with 1259 * the assumption that it is safe to positively acknowledge a request for the 1260 * ability to write to the BPF device. 1261 * Otherwise, return false but make a note that a selnotify() must be done. 1262 */ 1263 static int 1264 bpf_poll(struct file *fp, int events) 1265 { 1266 struct bpf_d *d = fp->f_bpf; 1267 int s = splnet(); 1268 int revents; 1269 1270 /* 1271 * Refresh the PID associated with this bpf file. 1272 */ 1273 KERNEL_LOCK(1, NULL); 1274 d->bd_pid = curproc->p_pid; 1275 1276 revents = events & (POLLOUT | POLLWRNORM); 1277 if (events & (POLLIN | POLLRDNORM)) { 1278 /* 1279 * An imitation of the FIONREAD ioctl code. 1280 */ 1281 if (d->bd_hlen != 0 || 1282 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1283 d->bd_slen != 0)) { 1284 revents |= events & (POLLIN | POLLRDNORM); 1285 } else { 1286 selrecord(curlwp, &d->bd_sel); 1287 /* Start the read timeout if necessary */ 1288 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1289 callout_reset(&d->bd_callout, d->bd_rtout, 1290 bpf_timed_out, d); 1291 d->bd_state = BPF_WAITING; 1292 } 1293 } 1294 } 1295 1296 KERNEL_UNLOCK_ONE(NULL); 1297 splx(s); 1298 return (revents); 1299 } 1300 1301 static void 1302 filt_bpfrdetach(struct knote *kn) 1303 { 1304 struct bpf_d *d = kn->kn_hook; 1305 int s; 1306 1307 KERNEL_LOCK(1, NULL); 1308 s = splnet(); 1309 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1310 splx(s); 1311 KERNEL_UNLOCK_ONE(NULL); 1312 } 1313 1314 static int 1315 filt_bpfread(struct knote *kn, long hint) 1316 { 1317 struct bpf_d *d = kn->kn_hook; 1318 int rv; 1319 1320 KERNEL_LOCK(1, NULL); 1321 kn->kn_data = d->bd_hlen; 1322 if (d->bd_immediate) 1323 kn->kn_data += d->bd_slen; 1324 rv = (kn->kn_data > 0); 1325 KERNEL_UNLOCK_ONE(NULL); 1326 return rv; 1327 } 1328 1329 static const struct filterops bpfread_filtops = 1330 { 1, NULL, filt_bpfrdetach, filt_bpfread }; 1331 1332 static int 1333 bpf_kqfilter(struct file *fp, struct knote *kn) 1334 { 1335 struct bpf_d *d = fp->f_bpf; 1336 struct klist *klist; 1337 int s; 1338 1339 KERNEL_LOCK(1, NULL); 1340 1341 switch (kn->kn_filter) { 1342 case EVFILT_READ: 1343 klist = &d->bd_sel.sel_klist; 1344 kn->kn_fop = &bpfread_filtops; 1345 break; 1346 1347 default: 1348 KERNEL_UNLOCK_ONE(NULL); 1349 return (EINVAL); 1350 } 1351 1352 kn->kn_hook = d; 1353 1354 s = splnet(); 1355 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1356 splx(s); 1357 KERNEL_UNLOCK_ONE(NULL); 1358 1359 return (0); 1360 } 1361 1362 /* 1363 * Copy data from an mbuf chain into a buffer. This code is derived 1364 * from m_copydata in sys/uipc_mbuf.c. 1365 */ 1366 static void * 1367 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1368 { 1369 const struct mbuf *m; 1370 u_int count; 1371 u_char *dst; 1372 1373 m = src_arg; 1374 dst = dst_arg; 1375 while (len > 0) { 1376 if (m == NULL) 1377 panic("bpf_mcpy"); 1378 count = min(m->m_len, len); 1379 memcpy(dst, mtod(m, const void *), count); 1380 m = m->m_next; 1381 dst += count; 1382 len -= count; 1383 } 1384 return dst_arg; 1385 } 1386 1387 /* 1388 * Dispatch a packet to all the listeners on interface bp. 1389 * 1390 * pkt pointer to the packet, either a data buffer or an mbuf chain 1391 * buflen buffer length, if pkt is a data buffer 1392 * cpfn a function that can copy pkt into the listener's buffer 1393 * pktlen length of the packet 1394 * rcv true if packet came in 1395 */ 1396 static inline void 1397 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1398 void *pkt, u_int pktlen, u_int buflen, const bool rcv) 1399 { 1400 uint32_t mem[BPF_MEMWORDS]; 1401 bpf_args_t args = { 1402 .pkt = (const uint8_t *)pkt, 1403 .wirelen = pktlen, 1404 .buflen = buflen, 1405 .mem = mem, 1406 .arg = NULL 1407 }; 1408 bool gottime = false; 1409 struct timespec ts; 1410 1411 /* 1412 * Note that the IPL does not have to be raised at this point. 1413 * The only problem that could arise here is that if two different 1414 * interfaces shared any data. This is not the case. 1415 */ 1416 for (struct bpf_d *d = bp->bif_dlist; d != NULL; d = d->bd_next) { 1417 u_int slen; 1418 1419 if (!d->bd_seesent && !rcv) { 1420 continue; 1421 } 1422 d->bd_rcount++; 1423 bpf_gstats.bs_recv++; 1424 1425 if (d->bd_jitcode) 1426 slen = d->bd_jitcode(NULL, &args); 1427 else 1428 slen = bpf_filter_ext(NULL, d->bd_filter, &args); 1429 1430 if (!slen) { 1431 continue; 1432 } 1433 if (!gottime) { 1434 gottime = true; 1435 nanotime(&ts); 1436 } 1437 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1438 } 1439 } 1440 1441 /* 1442 * Incoming linkage from device drivers. Process the packet pkt, of length 1443 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1444 * by each process' filter, and if accepted, stashed into the corresponding 1445 * buffer. 1446 */ 1447 static void 1448 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1449 { 1450 1451 bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true); 1452 } 1453 1454 /* 1455 * Incoming linkage from device drivers, when the head of the packet is in 1456 * a buffer, and the tail is in an mbuf chain. 1457 */ 1458 static void 1459 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1460 { 1461 u_int pktlen; 1462 struct mbuf mb; 1463 1464 /* Skip outgoing duplicate packets. */ 1465 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1466 m->m_flags &= ~M_PROMISC; 1467 return; 1468 } 1469 1470 pktlen = m_length(m) + dlen; 1471 1472 /* 1473 * Craft on-stack mbuf suitable for passing to bpf_filter. 1474 * Note that we cut corners here; we only setup what's 1475 * absolutely needed--this mbuf should never go anywhere else. 1476 */ 1477 (void)memset(&mb, 0, sizeof(mb)); 1478 mb.m_next = m; 1479 mb.m_data = data; 1480 mb.m_len = dlen; 1481 1482 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif != NULL); 1483 } 1484 1485 /* 1486 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1487 */ 1488 static void 1489 _bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1490 { 1491 void *(*cpfn)(void *, const void *, size_t); 1492 u_int pktlen, buflen; 1493 void *marg; 1494 1495 /* Skip outgoing duplicate packets. */ 1496 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1497 m->m_flags &= ~M_PROMISC; 1498 return; 1499 } 1500 1501 pktlen = m_length(m); 1502 1503 if (pktlen == m->m_len) { 1504 cpfn = (void *)memcpy; 1505 marg = mtod(m, void *); 1506 buflen = pktlen; 1507 } else { 1508 cpfn = bpf_mcpy; 1509 marg = m; 1510 buflen = 0; 1511 } 1512 1513 bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif != NULL); 1514 } 1515 1516 /* 1517 * We need to prepend the address family as 1518 * a four byte field. Cons up a dummy header 1519 * to pacify bpf. This is safe because bpf 1520 * will only read from the mbuf (i.e., it won't 1521 * try to free it or keep a pointer a to it). 1522 */ 1523 static void 1524 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m) 1525 { 1526 struct mbuf m0; 1527 1528 m0.m_flags = 0; 1529 m0.m_next = m; 1530 m0.m_len = 4; 1531 m0.m_data = (char *)⁡ 1532 1533 _bpf_mtap(bp, &m0); 1534 } 1535 1536 /* 1537 * Put the SLIP pseudo-"link header" in place. 1538 * Note this M_PREPEND() should never fail, 1539 * swince we know we always have enough space 1540 * in the input buffer. 1541 */ 1542 static void 1543 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1544 { 1545 int s; 1546 u_char *hp; 1547 1548 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1549 if (*m == NULL) 1550 return; 1551 1552 hp = mtod(*m, u_char *); 1553 hp[SLX_DIR] = SLIPDIR_IN; 1554 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1555 1556 s = splnet(); 1557 _bpf_mtap(bp, *m); 1558 splx(s); 1559 1560 m_adj(*m, SLIP_HDRLEN); 1561 } 1562 1563 /* 1564 * Put the SLIP pseudo-"link header" in 1565 * place. The compressed header is now 1566 * at the beginning of the mbuf. 1567 */ 1568 static void 1569 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1570 { 1571 struct mbuf m0; 1572 u_char *hp; 1573 int s; 1574 1575 m0.m_flags = 0; 1576 m0.m_next = m; 1577 m0.m_data = m0.m_dat; 1578 m0.m_len = SLIP_HDRLEN; 1579 1580 hp = mtod(&m0, u_char *); 1581 1582 hp[SLX_DIR] = SLIPDIR_OUT; 1583 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1584 1585 s = splnet(); 1586 _bpf_mtap(bp, &m0); 1587 splx(s); 1588 m_freem(m); 1589 } 1590 1591 static int 1592 bpf_hdrlen(struct bpf_d *d) 1593 { 1594 int hdrlen = d->bd_bif->bif_hdrlen; 1595 /* 1596 * Compute the length of the bpf header. This is not necessarily 1597 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1598 * that the network layer header begins on a longword boundary (for 1599 * performance reasons and to alleviate alignment restrictions). 1600 */ 1601 #ifdef _LP64 1602 if (d->bd_compat32) 1603 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1604 else 1605 #endif 1606 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1607 } 1608 1609 /* 1610 * Move the packet data from interface memory (pkt) into the 1611 * store buffer. Call the wakeup functions if it's time to wakeup 1612 * a listener (buffer full), "cpfn" is the routine called to do the 1613 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1614 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1615 * pkt is really an mbuf. 1616 */ 1617 static void 1618 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1619 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1620 { 1621 char *h; 1622 int totlen, curlen, caplen; 1623 int hdrlen = bpf_hdrlen(d); 1624 int do_wakeup = 0; 1625 1626 ++d->bd_ccount; 1627 ++bpf_gstats.bs_capt; 1628 /* 1629 * Figure out how many bytes to move. If the packet is 1630 * greater or equal to the snapshot length, transfer that 1631 * much. Otherwise, transfer the whole packet (unless 1632 * we hit the buffer size limit). 1633 */ 1634 totlen = hdrlen + min(snaplen, pktlen); 1635 if (totlen > d->bd_bufsize) 1636 totlen = d->bd_bufsize; 1637 /* 1638 * If we adjusted totlen to fit the bufsize, it could be that 1639 * totlen is smaller than hdrlen because of the link layer header. 1640 */ 1641 caplen = totlen - hdrlen; 1642 if (caplen < 0) 1643 caplen = 0; 1644 1645 /* 1646 * Round up the end of the previous packet to the next longword. 1647 */ 1648 #ifdef _LP64 1649 if (d->bd_compat32) 1650 curlen = BPF_WORDALIGN32(d->bd_slen); 1651 else 1652 #endif 1653 curlen = BPF_WORDALIGN(d->bd_slen); 1654 if (curlen + totlen > d->bd_bufsize) { 1655 /* 1656 * This packet will overflow the storage buffer. 1657 * Rotate the buffers if we can, then wakeup any 1658 * pending reads. 1659 */ 1660 if (d->bd_fbuf == NULL) { 1661 /* 1662 * We haven't completed the previous read yet, 1663 * so drop the packet. 1664 */ 1665 ++d->bd_dcount; 1666 ++bpf_gstats.bs_drop; 1667 return; 1668 } 1669 ROTATE_BUFFERS(d); 1670 do_wakeup = 1; 1671 curlen = 0; 1672 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1673 /* 1674 * Immediate mode is set, or the read timeout has 1675 * already expired during a select call. A packet 1676 * arrived, so the reader should be woken up. 1677 */ 1678 do_wakeup = 1; 1679 } 1680 1681 /* 1682 * Append the bpf header. 1683 */ 1684 h = (char *)d->bd_sbuf + curlen; 1685 #ifdef _LP64 1686 if (d->bd_compat32) { 1687 struct bpf_hdr32 *hp32; 1688 1689 hp32 = (struct bpf_hdr32 *)h; 1690 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1691 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1692 hp32->bh_datalen = pktlen; 1693 hp32->bh_hdrlen = hdrlen; 1694 hp32->bh_caplen = caplen; 1695 } else 1696 #endif 1697 { 1698 struct bpf_hdr *hp; 1699 1700 hp = (struct bpf_hdr *)h; 1701 hp->bh_tstamp.tv_sec = ts->tv_sec; 1702 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1703 hp->bh_datalen = pktlen; 1704 hp->bh_hdrlen = hdrlen; 1705 hp->bh_caplen = caplen; 1706 } 1707 1708 /* 1709 * Copy the packet data into the store buffer and update its length. 1710 */ 1711 (*cpfn)(h + hdrlen, pkt, caplen); 1712 d->bd_slen = curlen + totlen; 1713 1714 /* 1715 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1716 * will cause filt_bpfread() to be called with it adjusted. 1717 */ 1718 if (do_wakeup) 1719 bpf_wakeup(d); 1720 } 1721 1722 /* 1723 * Initialize all nonzero fields of a descriptor. 1724 */ 1725 static int 1726 bpf_allocbufs(struct bpf_d *d) 1727 { 1728 1729 d->bd_fbuf = malloc(d->bd_bufsize, M_DEVBUF, M_NOWAIT); 1730 if (!d->bd_fbuf) 1731 return (ENOBUFS); 1732 d->bd_sbuf = malloc(d->bd_bufsize, M_DEVBUF, M_NOWAIT); 1733 if (!d->bd_sbuf) { 1734 free(d->bd_fbuf, M_DEVBUF); 1735 return (ENOBUFS); 1736 } 1737 d->bd_slen = 0; 1738 d->bd_hlen = 0; 1739 return (0); 1740 } 1741 1742 /* 1743 * Free buffers currently in use by a descriptor. 1744 * Called on close. 1745 */ 1746 static void 1747 bpf_freed(struct bpf_d *d) 1748 { 1749 /* 1750 * We don't need to lock out interrupts since this descriptor has 1751 * been detached from its interface and it yet hasn't been marked 1752 * free. 1753 */ 1754 if (d->bd_sbuf != NULL) { 1755 free(d->bd_sbuf, M_DEVBUF); 1756 if (d->bd_hbuf != NULL) 1757 free(d->bd_hbuf, M_DEVBUF); 1758 if (d->bd_fbuf != NULL) 1759 free(d->bd_fbuf, M_DEVBUF); 1760 } 1761 if (d->bd_filter) 1762 free(d->bd_filter, M_DEVBUF); 1763 1764 if (d->bd_jitcode != NULL) { 1765 bpf_jit_freecode(d->bd_jitcode); 1766 } 1767 } 1768 1769 /* 1770 * Attach an interface to bpf. dlt is the link layer type; 1771 * hdrlen is the fixed size of the link header for the specified dlt 1772 * (variable length headers not yet supported). 1773 */ 1774 static void 1775 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 1776 { 1777 struct bpf_if *bp; 1778 bp = malloc(sizeof(*bp), M_DEVBUF, M_DONTWAIT); 1779 if (bp == NULL) 1780 panic("bpfattach"); 1781 1782 mutex_enter(&bpf_mtx); 1783 bp->bif_dlist = NULL; 1784 bp->bif_driverp = driverp; 1785 bp->bif_ifp = ifp; 1786 bp->bif_dlt = dlt; 1787 1788 bp->bif_next = bpf_iflist; 1789 bpf_iflist = bp; 1790 1791 *bp->bif_driverp = NULL; 1792 1793 bp->bif_hdrlen = hdrlen; 1794 mutex_exit(&bpf_mtx); 1795 #if 0 1796 printf("bpf: %s attached\n", ifp->if_xname); 1797 #endif 1798 } 1799 1800 /* 1801 * Remove an interface from bpf. 1802 */ 1803 static void 1804 _bpfdetach(struct ifnet *ifp) 1805 { 1806 struct bpf_if *bp, **pbp; 1807 struct bpf_d *d; 1808 int s; 1809 1810 mutex_enter(&bpf_mtx); 1811 /* Nuke the vnodes for any open instances */ 1812 LIST_FOREACH(d, &bpf_list, bd_list) { 1813 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 1814 /* 1815 * Detach the descriptor from an interface now. 1816 * It will be free'ed later by close routine. 1817 */ 1818 s = splnet(); 1819 d->bd_promisc = 0; /* we can't touch device. */ 1820 bpf_detachd(d); 1821 splx(s); 1822 } 1823 } 1824 1825 again: 1826 for (bp = bpf_iflist, pbp = &bpf_iflist; 1827 bp != NULL; pbp = &bp->bif_next, bp = bp->bif_next) { 1828 if (bp->bif_ifp == ifp) { 1829 *pbp = bp->bif_next; 1830 free(bp, M_DEVBUF); 1831 goto again; 1832 } 1833 } 1834 mutex_exit(&bpf_mtx); 1835 } 1836 1837 /* 1838 * Change the data link type of a interface. 1839 */ 1840 static void 1841 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 1842 { 1843 struct bpf_if *bp; 1844 1845 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1846 if (bp->bif_driverp == &ifp->if_bpf) 1847 break; 1848 } 1849 if (bp == NULL) 1850 panic("bpf_change_type"); 1851 1852 bp->bif_dlt = dlt; 1853 1854 bp->bif_hdrlen = hdrlen; 1855 } 1856 1857 /* 1858 * Get a list of available data link type of the interface. 1859 */ 1860 static int 1861 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 1862 { 1863 int n, error; 1864 struct ifnet *ifp; 1865 struct bpf_if *bp; 1866 1867 ifp = d->bd_bif->bif_ifp; 1868 n = 0; 1869 error = 0; 1870 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1871 if (bp->bif_ifp != ifp) 1872 continue; 1873 if (bfl->bfl_list != NULL) { 1874 if (n >= bfl->bfl_len) 1875 return ENOMEM; 1876 error = copyout(&bp->bif_dlt, 1877 bfl->bfl_list + n, sizeof(u_int)); 1878 } 1879 n++; 1880 } 1881 bfl->bfl_len = n; 1882 return error; 1883 } 1884 1885 /* 1886 * Set the data link type of a BPF instance. 1887 */ 1888 static int 1889 bpf_setdlt(struct bpf_d *d, u_int dlt) 1890 { 1891 int s, error, opromisc; 1892 struct ifnet *ifp; 1893 struct bpf_if *bp; 1894 1895 KASSERT(mutex_owned(&bpf_mtx)); 1896 1897 if (d->bd_bif->bif_dlt == dlt) 1898 return 0; 1899 ifp = d->bd_bif->bif_ifp; 1900 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1901 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 1902 break; 1903 } 1904 if (bp == NULL) 1905 return EINVAL; 1906 s = splnet(); 1907 opromisc = d->bd_promisc; 1908 bpf_detachd(d); 1909 bpf_attachd(d, bp); 1910 reset_d(d); 1911 if (opromisc) { 1912 error = ifpromisc(bp->bif_ifp, 1); 1913 if (error) 1914 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 1915 bp->bif_ifp->if_xname, error); 1916 else 1917 d->bd_promisc = 1; 1918 } 1919 splx(s); 1920 return 0; 1921 } 1922 1923 static int 1924 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 1925 { 1926 int newsize, error; 1927 struct sysctlnode node; 1928 1929 node = *rnode; 1930 node.sysctl_data = &newsize; 1931 newsize = bpf_maxbufsize; 1932 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1933 if (error || newp == NULL) 1934 return (error); 1935 1936 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 1937 return (EINVAL); 1938 1939 bpf_maxbufsize = newsize; 1940 1941 return (0); 1942 } 1943 1944 #if defined(MODULAR) || defined(BPFJIT) 1945 static int 1946 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 1947 { 1948 bool newval; 1949 int error; 1950 struct sysctlnode node; 1951 1952 node = *rnode; 1953 node.sysctl_data = &newval; 1954 newval = bpf_jit; 1955 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1956 if (error != 0 || newp == NULL) 1957 return error; 1958 1959 bpf_jit = newval; 1960 1961 /* 1962 * Do a full sync to publish new bpf_jit value and 1963 * update bpfjit_module_ops.bj_generate_code variable. 1964 */ 1965 membar_sync(); 1966 1967 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 1968 printf("JIT compilation is postponed " 1969 "until after bpfjit module is loaded\n"); 1970 } 1971 1972 return 0; 1973 } 1974 #endif 1975 1976 static int 1977 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 1978 { 1979 int error, elem_count; 1980 struct bpf_d *dp; 1981 struct bpf_d_ext dpe; 1982 size_t len, needed, elem_size, out_size; 1983 char *sp; 1984 1985 if (namelen == 1 && name[0] == CTL_QUERY) 1986 return (sysctl_query(SYSCTLFN_CALL(rnode))); 1987 1988 if (namelen != 2) 1989 return (EINVAL); 1990 1991 /* BPF peers is privileged information. */ 1992 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 1993 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 1994 if (error) 1995 return (EPERM); 1996 1997 len = (oldp != NULL) ? *oldlenp : 0; 1998 sp = oldp; 1999 elem_size = name[0]; 2000 elem_count = name[1]; 2001 out_size = MIN(sizeof(dpe), elem_size); 2002 needed = 0; 2003 2004 if (elem_size < 1 || elem_count < 0) 2005 return (EINVAL); 2006 2007 mutex_enter(&bpf_mtx); 2008 LIST_FOREACH(dp, &bpf_list, bd_list) { 2009 if (len >= elem_size && elem_count > 0) { 2010 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 2011 BPF_EXT(bufsize); 2012 BPF_EXT(promisc); 2013 BPF_EXT(state); 2014 BPF_EXT(immediate); 2015 BPF_EXT(hdrcmplt); 2016 BPF_EXT(seesent); 2017 BPF_EXT(pid); 2018 BPF_EXT(rcount); 2019 BPF_EXT(dcount); 2020 BPF_EXT(ccount); 2021 #undef BPF_EXT 2022 if (dp->bd_bif) 2023 (void)strlcpy(dpe.bde_ifname, 2024 dp->bd_bif->bif_ifp->if_xname, 2025 IFNAMSIZ - 1); 2026 else 2027 dpe.bde_ifname[0] = '\0'; 2028 2029 error = copyout(&dpe, sp, out_size); 2030 if (error) 2031 break; 2032 sp += elem_size; 2033 len -= elem_size; 2034 } 2035 needed += elem_size; 2036 if (elem_count > 0 && elem_count != INT_MAX) 2037 elem_count--; 2038 } 2039 mutex_exit(&bpf_mtx); 2040 2041 *oldlenp = needed; 2042 2043 return (error); 2044 } 2045 2046 static struct sysctllog *bpf_sysctllog; 2047 static void 2048 sysctl_net_bpf_setup(void) 2049 { 2050 const struct sysctlnode *node; 2051 2052 node = NULL; 2053 sysctl_createv(&bpf_sysctllog, 0, NULL, &node, 2054 CTLFLAG_PERMANENT, 2055 CTLTYPE_NODE, "bpf", 2056 SYSCTL_DESCR("BPF options"), 2057 NULL, 0, NULL, 0, 2058 CTL_NET, CTL_CREATE, CTL_EOL); 2059 if (node != NULL) { 2060 #if defined(MODULAR) || defined(BPFJIT) 2061 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2062 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2063 CTLTYPE_BOOL, "jit", 2064 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2065 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2066 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2067 #endif 2068 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2069 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2070 CTLTYPE_INT, "maxbufsize", 2071 SYSCTL_DESCR("Maximum size for data capture buffer"), 2072 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2073 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2074 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2075 CTLFLAG_PERMANENT, 2076 CTLTYPE_STRUCT, "stats", 2077 SYSCTL_DESCR("BPF stats"), 2078 NULL, 0, &bpf_gstats, sizeof(bpf_gstats), 2079 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2080 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2081 CTLFLAG_PERMANENT, 2082 CTLTYPE_STRUCT, "peers", 2083 SYSCTL_DESCR("BPF peers"), 2084 sysctl_net_bpf_peers, 0, NULL, 0, 2085 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2086 } 2087 2088 } 2089 2090 struct bpf_ops bpf_ops_kernel = { 2091 .bpf_attach = _bpfattach, 2092 .bpf_detach = _bpfdetach, 2093 .bpf_change_type = _bpf_change_type, 2094 2095 .bpf_tap = _bpf_tap, 2096 .bpf_mtap = _bpf_mtap, 2097 .bpf_mtap2 = _bpf_mtap2, 2098 .bpf_mtap_af = _bpf_mtap_af, 2099 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2100 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2101 }; 2102 2103 MODULE(MODULE_CLASS_DRIVER, bpf, NULL); 2104 2105 static int 2106 bpf_modcmd(modcmd_t cmd, void *arg) 2107 { 2108 devmajor_t bmajor, cmajor; 2109 int error; 2110 2111 bmajor = cmajor = NODEVMAJOR; 2112 2113 switch (cmd) { 2114 case MODULE_CMD_INIT: 2115 bpfilterattach(0); 2116 error = devsw_attach("bpf", NULL, &bmajor, 2117 &bpf_cdevsw, &cmajor); 2118 if (error == EEXIST) 2119 error = 0; /* maybe built-in ... improve eventually */ 2120 if (error) 2121 break; 2122 2123 bpf_ops_handover_enter(&bpf_ops_kernel); 2124 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2125 bpf_ops_handover_exit(); 2126 sysctl_net_bpf_setup(); 2127 break; 2128 2129 case MODULE_CMD_FINI: 2130 /* 2131 * While there is no reference counting for bpf callers, 2132 * unload could at least in theory be done similarly to 2133 * system call disestablishment. This should even be 2134 * a little simpler: 2135 * 2136 * 1) replace op vector with stubs 2137 * 2) post update to all cpus with xc 2138 * 3) check that nobody is in bpf anymore 2139 * (it's doubtful we'd want something like l_sysent, 2140 * but we could do something like *signed* percpu 2141 * counters. if the sum is 0, we're good). 2142 * 4) if fail, unroll changes 2143 * 2144 * NOTE: change won't be atomic to the outside. some 2145 * packets may be not captured even if unload is 2146 * not succesful. I think packet capture not working 2147 * is a perfectly logical consequence of trying to 2148 * disable packet capture. 2149 */ 2150 error = EOPNOTSUPP; 2151 /* insert sysctl teardown */ 2152 break; 2153 2154 default: 2155 error = ENOTTY; 2156 break; 2157 } 2158 2159 return error; 2160 } 2161