1 /* $NetBSD: bpf.c,v 1.179 2013/11/16 01:13:52 rmind Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.179 2013/11/16 01:13:52 rmind Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "strip.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/mbuf.h> 53 #include <sys/buf.h> 54 #include <sys/time.h> 55 #include <sys/proc.h> 56 #include <sys/ioctl.h> 57 #include <sys/conf.h> 58 #include <sys/vnode.h> 59 #include <sys/queue.h> 60 #include <sys/stat.h> 61 #include <sys/module.h> 62 #include <sys/once.h> 63 #include <sys/atomic.h> 64 65 #include <sys/file.h> 66 #include <sys/filedesc.h> 67 #include <sys/tty.h> 68 #include <sys/uio.h> 69 70 #include <sys/protosw.h> 71 #include <sys/socket.h> 72 #include <sys/errno.h> 73 #include <sys/kernel.h> 74 #include <sys/poll.h> 75 #include <sys/sysctl.h> 76 #include <sys/kauth.h> 77 78 #include <net/if.h> 79 #include <net/slip.h> 80 81 #include <net/bpf.h> 82 #include <net/bpfdesc.h> 83 #include <net/bpfjit.h> 84 85 #include <net/if_arc.h> 86 #include <net/if_ether.h> 87 88 #include <netinet/in.h> 89 #include <netinet/if_inarp.h> 90 91 92 #include <compat/sys/sockio.h> 93 94 #ifndef BPF_BUFSIZE 95 /* 96 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 97 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 98 */ 99 # define BPF_BUFSIZE 32768 100 #endif 101 102 #define PRINET 26 /* interruptible */ 103 104 /* 105 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 106 * XXX the default values should be computed dynamically based 107 * on available memory size and available mbuf clusters. 108 */ 109 int bpf_bufsize = BPF_BUFSIZE; 110 int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 111 bool bpf_jit = false; 112 113 struct bpfjit_ops bpfjit_module_ops = { 114 .bj_generate_code = NULL, 115 .bj_free_code = NULL 116 }; 117 118 /* 119 * Global BPF statistics returned by net.bpf.stats sysctl. 120 */ 121 struct bpf_stat bpf_gstats; 122 123 /* 124 * Use a mutex to avoid a race condition between gathering the stats/peers 125 * and opening/closing the device. 126 */ 127 static kmutex_t bpf_mtx; 128 129 /* 130 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 131 * bpf_dtab holds the descriptors, indexed by minor device # 132 */ 133 struct bpf_if *bpf_iflist; 134 LIST_HEAD(, bpf_d) bpf_list; 135 136 static int bpf_allocbufs(struct bpf_d *); 137 static void bpf_deliver(struct bpf_if *, 138 void *(*cpfn)(void *, const void *, size_t), 139 void *, u_int, u_int, const bool); 140 static void bpf_freed(struct bpf_d *); 141 static void bpf_ifname(struct ifnet *, struct ifreq *); 142 static void *bpf_mcpy(void *, const void *, size_t); 143 static int bpf_movein(struct uio *, int, uint64_t, 144 struct mbuf **, struct sockaddr *); 145 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 146 static void bpf_detachd(struct bpf_d *); 147 static int bpf_setif(struct bpf_d *, struct ifreq *); 148 static void bpf_timed_out(void *); 149 static inline void 150 bpf_wakeup(struct bpf_d *); 151 static int bpf_hdrlen(struct bpf_d *); 152 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 153 void *(*)(void *, const void *, size_t), struct timespec *); 154 static void reset_d(struct bpf_d *); 155 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 156 static int bpf_setdlt(struct bpf_d *, u_int); 157 158 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 159 int); 160 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 161 int); 162 static int bpf_ioctl(struct file *, u_long, void *); 163 static int bpf_poll(struct file *, int); 164 static int bpf_stat(struct file *, struct stat *); 165 static int bpf_close(struct file *); 166 static int bpf_kqfilter(struct file *, struct knote *); 167 static void bpf_softintr(void *); 168 169 static const struct fileops bpf_fileops = { 170 .fo_read = bpf_read, 171 .fo_write = bpf_write, 172 .fo_ioctl = bpf_ioctl, 173 .fo_fcntl = fnullop_fcntl, 174 .fo_poll = bpf_poll, 175 .fo_stat = bpf_stat, 176 .fo_close = bpf_close, 177 .fo_kqfilter = bpf_kqfilter, 178 .fo_restart = fnullop_restart, 179 }; 180 181 dev_type_open(bpfopen); 182 183 const struct cdevsw bpf_cdevsw = { 184 bpfopen, noclose, noread, nowrite, noioctl, 185 nostop, notty, nopoll, nommap, nokqfilter, D_OTHER 186 }; 187 188 bpfjit_func_t 189 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 190 { 191 membar_consumer(); 192 if (bpfjit_module_ops.bj_generate_code != NULL) { 193 return bpfjit_module_ops.bj_generate_code(bc, code, size); 194 } 195 return NULL; 196 } 197 198 void 199 bpf_jit_freecode(bpfjit_func_t jcode) 200 { 201 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 202 bpfjit_module_ops.bj_free_code(jcode); 203 } 204 205 static int 206 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 207 struct sockaddr *sockp) 208 { 209 struct mbuf *m; 210 int error; 211 size_t len; 212 size_t hlen; 213 size_t align; 214 215 /* 216 * Build a sockaddr based on the data link layer type. 217 * We do this at this level because the ethernet header 218 * is copied directly into the data field of the sockaddr. 219 * In the case of SLIP, there is no header and the packet 220 * is forwarded as is. 221 * Also, we are careful to leave room at the front of the mbuf 222 * for the link level header. 223 */ 224 switch (linktype) { 225 226 case DLT_SLIP: 227 sockp->sa_family = AF_INET; 228 hlen = 0; 229 align = 0; 230 break; 231 232 case DLT_PPP: 233 sockp->sa_family = AF_UNSPEC; 234 hlen = 0; 235 align = 0; 236 break; 237 238 case DLT_EN10MB: 239 sockp->sa_family = AF_UNSPEC; 240 /* XXX Would MAXLINKHDR be better? */ 241 /* 6(dst)+6(src)+2(type) */ 242 hlen = sizeof(struct ether_header); 243 align = 2; 244 break; 245 246 case DLT_ARCNET: 247 sockp->sa_family = AF_UNSPEC; 248 hlen = ARC_HDRLEN; 249 align = 5; 250 break; 251 252 case DLT_FDDI: 253 sockp->sa_family = AF_LINK; 254 /* XXX 4(FORMAC)+6(dst)+6(src) */ 255 hlen = 16; 256 align = 0; 257 break; 258 259 case DLT_ECONET: 260 sockp->sa_family = AF_UNSPEC; 261 hlen = 6; 262 align = 2; 263 break; 264 265 case DLT_NULL: 266 sockp->sa_family = AF_UNSPEC; 267 hlen = 0; 268 align = 0; 269 break; 270 271 default: 272 return (EIO); 273 } 274 275 len = uio->uio_resid; 276 /* 277 * If there aren't enough bytes for a link level header or the 278 * packet length exceeds the interface mtu, return an error. 279 */ 280 if (len - hlen > mtu) 281 return (EMSGSIZE); 282 283 /* 284 * XXX Avoid complicated buffer chaining --- 285 * bail if it won't fit in a single mbuf. 286 * (Take into account possible alignment bytes) 287 */ 288 if (len + align > MCLBYTES) 289 return (EIO); 290 291 m = m_gethdr(M_WAIT, MT_DATA); 292 m->m_pkthdr.rcvif = 0; 293 m->m_pkthdr.len = (int)(len - hlen); 294 if (len + align > MHLEN) { 295 m_clget(m, M_WAIT); 296 if ((m->m_flags & M_EXT) == 0) { 297 error = ENOBUFS; 298 goto bad; 299 } 300 } 301 302 /* Insure the data is properly aligned */ 303 if (align > 0) { 304 m->m_data += align; 305 m->m_len -= (int)align; 306 } 307 308 error = uiomove(mtod(m, void *), len, uio); 309 if (error) 310 goto bad; 311 if (hlen != 0) { 312 memcpy(sockp->sa_data, mtod(m, void *), hlen); 313 m->m_data += hlen; /* XXX */ 314 len -= hlen; 315 } 316 m->m_len = (int)len; 317 *mp = m; 318 return (0); 319 320 bad: 321 m_freem(m); 322 return (error); 323 } 324 325 /* 326 * Attach file to the bpf interface, i.e. make d listen on bp. 327 * Must be called at splnet. 328 */ 329 static void 330 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 331 { 332 /* 333 * Point d at bp, and add d to the interface's list of listeners. 334 * Finally, point the driver's bpf cookie at the interface so 335 * it will divert packets to bpf. 336 */ 337 d->bd_bif = bp; 338 d->bd_next = bp->bif_dlist; 339 bp->bif_dlist = d; 340 341 *bp->bif_driverp = bp; 342 } 343 344 /* 345 * Detach a file from its interface. 346 */ 347 static void 348 bpf_detachd(struct bpf_d *d) 349 { 350 struct bpf_d **p; 351 struct bpf_if *bp; 352 353 bp = d->bd_bif; 354 /* 355 * Check if this descriptor had requested promiscuous mode. 356 * If so, turn it off. 357 */ 358 if (d->bd_promisc) { 359 int error; 360 361 d->bd_promisc = 0; 362 /* 363 * Take device out of promiscuous mode. Since we were 364 * able to enter promiscuous mode, we should be able 365 * to turn it off. But we can get an error if 366 * the interface was configured down, so only panic 367 * if we don't get an unexpected error. 368 */ 369 error = ifpromisc(bp->bif_ifp, 0); 370 if (error && error != EINVAL) 371 panic("%s: ifpromisc failed: %d", __func__, error); 372 } 373 /* Remove d from the interface's descriptor list. */ 374 p = &bp->bif_dlist; 375 while (*p != d) { 376 p = &(*p)->bd_next; 377 if (*p == 0) 378 panic("%s: descriptor not in list", __func__); 379 } 380 *p = (*p)->bd_next; 381 if (bp->bif_dlist == 0) 382 /* 383 * Let the driver know that there are no more listeners. 384 */ 385 *d->bd_bif->bif_driverp = 0; 386 d->bd_bif = 0; 387 } 388 389 static int 390 doinit(void) 391 { 392 393 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 394 395 LIST_INIT(&bpf_list); 396 397 bpf_gstats.bs_recv = 0; 398 bpf_gstats.bs_drop = 0; 399 bpf_gstats.bs_capt = 0; 400 401 return 0; 402 } 403 404 /* 405 * bpfilterattach() is called at boot time. 406 */ 407 /* ARGSUSED */ 408 void 409 bpfilterattach(int n) 410 { 411 static ONCE_DECL(control); 412 413 RUN_ONCE(&control, doinit); 414 } 415 416 /* 417 * Open ethernet device. Clones. 418 */ 419 /* ARGSUSED */ 420 int 421 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 422 { 423 struct bpf_d *d; 424 struct file *fp; 425 int error, fd; 426 427 /* falloc() will use the descriptor for us. */ 428 if ((error = fd_allocfile(&fp, &fd)) != 0) 429 return error; 430 431 d = malloc(sizeof(*d), M_DEVBUF, M_WAITOK|M_ZERO); 432 d->bd_bufsize = bpf_bufsize; 433 d->bd_seesent = 1; 434 d->bd_feedback = 0; 435 d->bd_pid = l->l_proc->p_pid; 436 #ifdef _LP64 437 if (curproc->p_flag & PK_32) 438 d->bd_compat32 = 1; 439 #endif 440 getnanotime(&d->bd_btime); 441 d->bd_atime = d->bd_mtime = d->bd_btime; 442 callout_init(&d->bd_callout, 0); 443 selinit(&d->bd_sel); 444 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d); 445 d->bd_jitcode = NULL; 446 447 mutex_enter(&bpf_mtx); 448 LIST_INSERT_HEAD(&bpf_list, d, bd_list); 449 mutex_exit(&bpf_mtx); 450 451 return fd_clone(fp, fd, flag, &bpf_fileops, d); 452 } 453 454 /* 455 * Close the descriptor by detaching it from its interface, 456 * deallocating its buffers, and marking it free. 457 */ 458 /* ARGSUSED */ 459 static int 460 bpf_close(struct file *fp) 461 { 462 struct bpf_d *d = fp->f_data; 463 int s; 464 465 KERNEL_LOCK(1, NULL); 466 467 /* 468 * Refresh the PID associated with this bpf file. 469 */ 470 d->bd_pid = curproc->p_pid; 471 472 s = splnet(); 473 if (d->bd_state == BPF_WAITING) 474 callout_stop(&d->bd_callout); 475 d->bd_state = BPF_IDLE; 476 if (d->bd_bif) 477 bpf_detachd(d); 478 splx(s); 479 bpf_freed(d); 480 mutex_enter(&bpf_mtx); 481 LIST_REMOVE(d, bd_list); 482 mutex_exit(&bpf_mtx); 483 callout_destroy(&d->bd_callout); 484 seldestroy(&d->bd_sel); 485 softint_disestablish(d->bd_sih); 486 free(d, M_DEVBUF); 487 fp->f_data = NULL; 488 489 KERNEL_UNLOCK_ONE(NULL); 490 491 return (0); 492 } 493 494 /* 495 * Rotate the packet buffers in descriptor d. Move the store buffer 496 * into the hold slot, and the free buffer into the store slot. 497 * Zero the length of the new store buffer. 498 */ 499 #define ROTATE_BUFFERS(d) \ 500 (d)->bd_hbuf = (d)->bd_sbuf; \ 501 (d)->bd_hlen = (d)->bd_slen; \ 502 (d)->bd_sbuf = (d)->bd_fbuf; \ 503 (d)->bd_slen = 0; \ 504 (d)->bd_fbuf = 0; 505 /* 506 * bpfread - read next chunk of packets from buffers 507 */ 508 static int 509 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 510 kauth_cred_t cred, int flags) 511 { 512 struct bpf_d *d = fp->f_data; 513 int timed_out; 514 int error; 515 int s; 516 517 getnanotime(&d->bd_atime); 518 /* 519 * Restrict application to use a buffer the same size as 520 * the kernel buffers. 521 */ 522 if (uio->uio_resid != d->bd_bufsize) 523 return (EINVAL); 524 525 KERNEL_LOCK(1, NULL); 526 s = splnet(); 527 if (d->bd_state == BPF_WAITING) 528 callout_stop(&d->bd_callout); 529 timed_out = (d->bd_state == BPF_TIMED_OUT); 530 d->bd_state = BPF_IDLE; 531 /* 532 * If the hold buffer is empty, then do a timed sleep, which 533 * ends when the timeout expires or when enough packets 534 * have arrived to fill the store buffer. 535 */ 536 while (d->bd_hbuf == 0) { 537 if (fp->f_flag & FNONBLOCK) { 538 if (d->bd_slen == 0) { 539 splx(s); 540 KERNEL_UNLOCK_ONE(NULL); 541 return (EWOULDBLOCK); 542 } 543 ROTATE_BUFFERS(d); 544 break; 545 } 546 547 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 548 /* 549 * A packet(s) either arrived since the previous 550 * read or arrived while we were asleep. 551 * Rotate the buffers and return what's here. 552 */ 553 ROTATE_BUFFERS(d); 554 break; 555 } 556 error = tsleep(d, PRINET|PCATCH, "bpf", 557 d->bd_rtout); 558 if (error == EINTR || error == ERESTART) { 559 splx(s); 560 KERNEL_UNLOCK_ONE(NULL); 561 return (error); 562 } 563 if (error == EWOULDBLOCK) { 564 /* 565 * On a timeout, return what's in the buffer, 566 * which may be nothing. If there is something 567 * in the store buffer, we can rotate the buffers. 568 */ 569 if (d->bd_hbuf) 570 /* 571 * We filled up the buffer in between 572 * getting the timeout and arriving 573 * here, so we don't need to rotate. 574 */ 575 break; 576 577 if (d->bd_slen == 0) { 578 splx(s); 579 KERNEL_UNLOCK_ONE(NULL); 580 return (0); 581 } 582 ROTATE_BUFFERS(d); 583 break; 584 } 585 if (error != 0) 586 goto done; 587 } 588 /* 589 * At this point, we know we have something in the hold slot. 590 */ 591 splx(s); 592 593 /* 594 * Move data from hold buffer into user space. 595 * We know the entire buffer is transferred since 596 * we checked above that the read buffer is bpf_bufsize bytes. 597 */ 598 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 599 600 s = splnet(); 601 d->bd_fbuf = d->bd_hbuf; 602 d->bd_hbuf = 0; 603 d->bd_hlen = 0; 604 done: 605 splx(s); 606 KERNEL_UNLOCK_ONE(NULL); 607 return (error); 608 } 609 610 611 /* 612 * If there are processes sleeping on this descriptor, wake them up. 613 */ 614 static inline void 615 bpf_wakeup(struct bpf_d *d) 616 { 617 wakeup(d); 618 if (d->bd_async) 619 softint_schedule(d->bd_sih); 620 selnotify(&d->bd_sel, 0, 0); 621 } 622 623 static void 624 bpf_softintr(void *cookie) 625 { 626 struct bpf_d *d; 627 628 d = cookie; 629 if (d->bd_async) 630 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 631 } 632 633 static void 634 bpf_timed_out(void *arg) 635 { 636 struct bpf_d *d = arg; 637 int s; 638 639 s = splnet(); 640 if (d->bd_state == BPF_WAITING) { 641 d->bd_state = BPF_TIMED_OUT; 642 if (d->bd_slen != 0) 643 bpf_wakeup(d); 644 } 645 splx(s); 646 } 647 648 649 static int 650 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 651 kauth_cred_t cred, int flags) 652 { 653 struct bpf_d *d = fp->f_data; 654 struct ifnet *ifp; 655 struct mbuf *m, *mc; 656 int error, s; 657 static struct sockaddr_storage dst; 658 659 m = NULL; /* XXX gcc */ 660 661 KERNEL_LOCK(1, NULL); 662 663 if (d->bd_bif == 0) { 664 KERNEL_UNLOCK_ONE(NULL); 665 return (ENXIO); 666 } 667 getnanotime(&d->bd_mtime); 668 669 ifp = d->bd_bif->bif_ifp; 670 671 if (uio->uio_resid == 0) { 672 KERNEL_UNLOCK_ONE(NULL); 673 return (0); 674 } 675 676 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp->if_mtu, &m, 677 (struct sockaddr *) &dst); 678 if (error) { 679 KERNEL_UNLOCK_ONE(NULL); 680 return (error); 681 } 682 683 if (m->m_pkthdr.len > ifp->if_mtu) { 684 KERNEL_UNLOCK_ONE(NULL); 685 m_freem(m); 686 return (EMSGSIZE); 687 } 688 689 if (d->bd_hdrcmplt) 690 dst.ss_family = pseudo_AF_HDRCMPLT; 691 692 if (d->bd_feedback) { 693 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 694 if (mc != NULL) 695 mc->m_pkthdr.rcvif = ifp; 696 /* Set M_PROMISC for outgoing packets to be discarded. */ 697 if (1 /*d->bd_direction == BPF_D_INOUT*/) 698 m->m_flags |= M_PROMISC; 699 } else 700 mc = NULL; 701 702 s = splsoftnet(); 703 error = (*ifp->if_output)(ifp, m, (struct sockaddr *) &dst, NULL); 704 705 if (mc != NULL) { 706 if (error == 0) 707 (*ifp->if_input)(ifp, mc); 708 m_freem(mc); 709 } 710 splx(s); 711 KERNEL_UNLOCK_ONE(NULL); 712 /* 713 * The driver frees the mbuf. 714 */ 715 return (error); 716 } 717 718 /* 719 * Reset a descriptor by flushing its packet buffer and clearing the 720 * receive and drop counts. Should be called at splnet. 721 */ 722 static void 723 reset_d(struct bpf_d *d) 724 { 725 if (d->bd_hbuf) { 726 /* Free the hold buffer. */ 727 d->bd_fbuf = d->bd_hbuf; 728 d->bd_hbuf = 0; 729 } 730 d->bd_slen = 0; 731 d->bd_hlen = 0; 732 d->bd_rcount = 0; 733 d->bd_dcount = 0; 734 d->bd_ccount = 0; 735 } 736 737 /* 738 * FIONREAD Check for read packet available. 739 * BIOCGBLEN Get buffer len [for read()]. 740 * BIOCSETF Set ethernet read filter. 741 * BIOCFLUSH Flush read packet buffer. 742 * BIOCPROMISC Put interface into promiscuous mode. 743 * BIOCGDLT Get link layer type. 744 * BIOCGETIF Get interface name. 745 * BIOCSETIF Set interface. 746 * BIOCSRTIMEOUT Set read timeout. 747 * BIOCGRTIMEOUT Get read timeout. 748 * BIOCGSTATS Get packet stats. 749 * BIOCIMMEDIATE Set immediate mode. 750 * BIOCVERSION Get filter language version. 751 * BIOCGHDRCMPLT Get "header already complete" flag. 752 * BIOCSHDRCMPLT Set "header already complete" flag. 753 * BIOCSFEEDBACK Set packet feedback mode. 754 * BIOCGFEEDBACK Get packet feedback mode. 755 * BIOCGSEESENT Get "see sent packets" mode. 756 * BIOCSSEESENT Set "see sent packets" mode. 757 */ 758 /* ARGSUSED */ 759 static int 760 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 761 { 762 struct bpf_d *d = fp->f_data; 763 int s, error = 0; 764 765 /* 766 * Refresh the PID associated with this bpf file. 767 */ 768 KERNEL_LOCK(1, NULL); 769 d->bd_pid = curproc->p_pid; 770 #ifdef _LP64 771 if (curproc->p_flag & PK_32) 772 d->bd_compat32 = 1; 773 else 774 d->bd_compat32 = 0; 775 #endif 776 777 s = splnet(); 778 if (d->bd_state == BPF_WAITING) 779 callout_stop(&d->bd_callout); 780 d->bd_state = BPF_IDLE; 781 splx(s); 782 783 switch (cmd) { 784 785 default: 786 error = EINVAL; 787 break; 788 789 /* 790 * Check for read packet available. 791 */ 792 case FIONREAD: 793 { 794 int n; 795 796 s = splnet(); 797 n = d->bd_slen; 798 if (d->bd_hbuf) 799 n += d->bd_hlen; 800 splx(s); 801 802 *(int *)addr = n; 803 break; 804 } 805 806 /* 807 * Get buffer len [for read()]. 808 */ 809 case BIOCGBLEN: 810 *(u_int *)addr = d->bd_bufsize; 811 break; 812 813 /* 814 * Set buffer length. 815 */ 816 case BIOCSBLEN: 817 if (d->bd_bif != 0) 818 error = EINVAL; 819 else { 820 u_int size = *(u_int *)addr; 821 822 if (size > bpf_maxbufsize) 823 *(u_int *)addr = size = bpf_maxbufsize; 824 else if (size < BPF_MINBUFSIZE) 825 *(u_int *)addr = size = BPF_MINBUFSIZE; 826 d->bd_bufsize = size; 827 } 828 break; 829 830 /* 831 * Set link layer read filter. 832 */ 833 case BIOCSETF: 834 error = bpf_setf(d, addr); 835 break; 836 837 /* 838 * Flush read packet buffer. 839 */ 840 case BIOCFLUSH: 841 s = splnet(); 842 reset_d(d); 843 splx(s); 844 break; 845 846 /* 847 * Put interface into promiscuous mode. 848 */ 849 case BIOCPROMISC: 850 if (d->bd_bif == 0) { 851 /* 852 * No interface attached yet. 853 */ 854 error = EINVAL; 855 break; 856 } 857 s = splnet(); 858 if (d->bd_promisc == 0) { 859 error = ifpromisc(d->bd_bif->bif_ifp, 1); 860 if (error == 0) 861 d->bd_promisc = 1; 862 } 863 splx(s); 864 break; 865 866 /* 867 * Get device parameters. 868 */ 869 case BIOCGDLT: 870 if (d->bd_bif == 0) 871 error = EINVAL; 872 else 873 *(u_int *)addr = d->bd_bif->bif_dlt; 874 break; 875 876 /* 877 * Get a list of supported device parameters. 878 */ 879 case BIOCGDLTLIST: 880 if (d->bd_bif == 0) 881 error = EINVAL; 882 else 883 error = bpf_getdltlist(d, addr); 884 break; 885 886 /* 887 * Set device parameters. 888 */ 889 case BIOCSDLT: 890 if (d->bd_bif == 0) 891 error = EINVAL; 892 else 893 error = bpf_setdlt(d, *(u_int *)addr); 894 break; 895 896 /* 897 * Set interface name. 898 */ 899 #ifdef OBIOCGETIF 900 case OBIOCGETIF: 901 #endif 902 case BIOCGETIF: 903 if (d->bd_bif == 0) 904 error = EINVAL; 905 else 906 bpf_ifname(d->bd_bif->bif_ifp, addr); 907 break; 908 909 /* 910 * Set interface. 911 */ 912 #ifdef OBIOCSETIF 913 case OBIOCSETIF: 914 #endif 915 case BIOCSETIF: 916 error = bpf_setif(d, addr); 917 break; 918 919 /* 920 * Set read timeout. 921 */ 922 case BIOCSRTIMEOUT: 923 { 924 struct timeval *tv = addr; 925 926 /* Compute number of ticks. */ 927 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 928 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 929 d->bd_rtout = 1; 930 break; 931 } 932 933 #ifdef BIOCGORTIMEOUT 934 /* 935 * Get read timeout. 936 */ 937 case BIOCGORTIMEOUT: 938 { 939 struct timeval50 *tv = addr; 940 941 tv->tv_sec = d->bd_rtout / hz; 942 tv->tv_usec = (d->bd_rtout % hz) * tick; 943 break; 944 } 945 #endif 946 947 #ifdef BIOCSORTIMEOUT 948 /* 949 * Set read timeout. 950 */ 951 case BIOCSORTIMEOUT: 952 { 953 struct timeval50 *tv = addr; 954 955 /* Compute number of ticks. */ 956 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 957 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 958 d->bd_rtout = 1; 959 break; 960 } 961 #endif 962 963 /* 964 * Get read timeout. 965 */ 966 case BIOCGRTIMEOUT: 967 { 968 struct timeval *tv = addr; 969 970 tv->tv_sec = d->bd_rtout / hz; 971 tv->tv_usec = (d->bd_rtout % hz) * tick; 972 break; 973 } 974 /* 975 * Get packet stats. 976 */ 977 case BIOCGSTATS: 978 { 979 struct bpf_stat *bs = addr; 980 981 bs->bs_recv = d->bd_rcount; 982 bs->bs_drop = d->bd_dcount; 983 bs->bs_capt = d->bd_ccount; 984 break; 985 } 986 987 case BIOCGSTATSOLD: 988 { 989 struct bpf_stat_old *bs = addr; 990 991 bs->bs_recv = d->bd_rcount; 992 bs->bs_drop = d->bd_dcount; 993 break; 994 } 995 996 /* 997 * Set immediate mode. 998 */ 999 case BIOCIMMEDIATE: 1000 d->bd_immediate = *(u_int *)addr; 1001 break; 1002 1003 case BIOCVERSION: 1004 { 1005 struct bpf_version *bv = addr; 1006 1007 bv->bv_major = BPF_MAJOR_VERSION; 1008 bv->bv_minor = BPF_MINOR_VERSION; 1009 break; 1010 } 1011 1012 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1013 *(u_int *)addr = d->bd_hdrcmplt; 1014 break; 1015 1016 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1017 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1018 break; 1019 1020 /* 1021 * Get "see sent packets" flag 1022 */ 1023 case BIOCGSEESENT: 1024 *(u_int *)addr = d->bd_seesent; 1025 break; 1026 1027 /* 1028 * Set "see sent" packets flag 1029 */ 1030 case BIOCSSEESENT: 1031 d->bd_seesent = *(u_int *)addr; 1032 break; 1033 1034 /* 1035 * Set "feed packets from bpf back to input" mode 1036 */ 1037 case BIOCSFEEDBACK: 1038 d->bd_feedback = *(u_int *)addr; 1039 break; 1040 1041 /* 1042 * Get "feed packets from bpf back to input" mode 1043 */ 1044 case BIOCGFEEDBACK: 1045 *(u_int *)addr = d->bd_feedback; 1046 break; 1047 1048 case FIONBIO: /* Non-blocking I/O */ 1049 /* 1050 * No need to do anything special as we use IO_NDELAY in 1051 * bpfread() as an indication of whether or not to block 1052 * the read. 1053 */ 1054 break; 1055 1056 case FIOASYNC: /* Send signal on receive packets */ 1057 d->bd_async = *(int *)addr; 1058 break; 1059 1060 case TIOCSPGRP: /* Process or group to send signals to */ 1061 case FIOSETOWN: 1062 error = fsetown(&d->bd_pgid, cmd, addr); 1063 break; 1064 1065 case TIOCGPGRP: 1066 case FIOGETOWN: 1067 error = fgetown(d->bd_pgid, cmd, addr); 1068 break; 1069 } 1070 KERNEL_UNLOCK_ONE(NULL); 1071 return (error); 1072 } 1073 1074 /* 1075 * Set d's packet filter program to fp. If this file already has a filter, 1076 * free it and replace it. Returns EINVAL for bogus requests. 1077 */ 1078 int 1079 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1080 { 1081 struct bpf_insn *fcode, *old; 1082 bpfjit_func_t jcode, oldj; 1083 size_t flen, size; 1084 int s; 1085 1086 jcode = NULL; 1087 flen = fp->bf_len; 1088 1089 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1090 return EINVAL; 1091 } 1092 1093 if (flen) { 1094 /* 1095 * Allocate the buffer, copy the byte-code from 1096 * userspace and validate it. 1097 */ 1098 size = flen * sizeof(*fp->bf_insns); 1099 fcode = malloc(size, M_DEVBUF, M_WAITOK); 1100 if (copyin(fp->bf_insns, fcode, size) != 0 || 1101 !bpf_validate(fcode, (int)flen)) { 1102 free(fcode, M_DEVBUF); 1103 return EINVAL; 1104 } 1105 membar_consumer(); 1106 if (bpf_jit) { 1107 bpf_ctx_t *bc = bpf_default_ctx(); 1108 jcode = bpf_jit_generate(bc, fcode, flen); 1109 } 1110 } else { 1111 fcode = NULL; 1112 } 1113 1114 s = splnet(); 1115 old = d->bd_filter; 1116 d->bd_filter = fcode; 1117 oldj = d->bd_jitcode; 1118 d->bd_jitcode = jcode; 1119 reset_d(d); 1120 splx(s); 1121 1122 if (old) { 1123 free(old, M_DEVBUF); 1124 } 1125 if (oldj) { 1126 bpf_jit_freecode(oldj); 1127 } 1128 1129 return 0; 1130 } 1131 1132 /* 1133 * Detach a file from its current interface (if attached at all) and attach 1134 * to the interface indicated by the name stored in ifr. 1135 * Return an errno or 0. 1136 */ 1137 static int 1138 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1139 { 1140 struct bpf_if *bp; 1141 char *cp; 1142 int unit_seen, i, s, error; 1143 1144 /* 1145 * Make sure the provided name has a unit number, and default 1146 * it to '0' if not specified. 1147 * XXX This is ugly ... do this differently? 1148 */ 1149 unit_seen = 0; 1150 cp = ifr->ifr_name; 1151 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1152 while (*cp++) 1153 if (*cp >= '0' && *cp <= '9') 1154 unit_seen = 1; 1155 if (!unit_seen) { 1156 /* Make sure to leave room for the '\0'. */ 1157 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1158 if ((ifr->ifr_name[i] >= 'a' && 1159 ifr->ifr_name[i] <= 'z') || 1160 (ifr->ifr_name[i] >= 'A' && 1161 ifr->ifr_name[i] <= 'Z')) 1162 continue; 1163 ifr->ifr_name[i] = '0'; 1164 } 1165 } 1166 1167 /* 1168 * Look through attached interfaces for the named one. 1169 */ 1170 for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) { 1171 struct ifnet *ifp = bp->bif_ifp; 1172 1173 if (ifp == 0 || 1174 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1175 continue; 1176 /* skip additional entry */ 1177 if (bp->bif_driverp != &ifp->if_bpf) 1178 continue; 1179 /* 1180 * We found the requested interface. 1181 * Allocate the packet buffers if we need to. 1182 * If we're already attached to requested interface, 1183 * just flush the buffer. 1184 */ 1185 if (d->bd_sbuf == 0) { 1186 error = bpf_allocbufs(d); 1187 if (error != 0) 1188 return (error); 1189 } 1190 s = splnet(); 1191 if (bp != d->bd_bif) { 1192 if (d->bd_bif) 1193 /* 1194 * Detach if attached to something else. 1195 */ 1196 bpf_detachd(d); 1197 1198 bpf_attachd(d, bp); 1199 } 1200 reset_d(d); 1201 splx(s); 1202 return (0); 1203 } 1204 /* Not found. */ 1205 return (ENXIO); 1206 } 1207 1208 /* 1209 * Copy the interface name to the ifreq. 1210 */ 1211 static void 1212 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1213 { 1214 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1215 } 1216 1217 static int 1218 bpf_stat(struct file *fp, struct stat *st) 1219 { 1220 struct bpf_d *d = fp->f_data; 1221 1222 (void)memset(st, 0, sizeof(*st)); 1223 KERNEL_LOCK(1, NULL); 1224 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1225 st->st_atimespec = d->bd_atime; 1226 st->st_mtimespec = d->bd_mtime; 1227 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1228 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1229 st->st_gid = kauth_cred_getegid(fp->f_cred); 1230 st->st_mode = S_IFCHR; 1231 KERNEL_UNLOCK_ONE(NULL); 1232 return 0; 1233 } 1234 1235 /* 1236 * Support for poll() system call 1237 * 1238 * Return true iff the specific operation will not block indefinitely - with 1239 * the assumption that it is safe to positively acknowledge a request for the 1240 * ability to write to the BPF device. 1241 * Otherwise, return false but make a note that a selnotify() must be done. 1242 */ 1243 static int 1244 bpf_poll(struct file *fp, int events) 1245 { 1246 struct bpf_d *d = fp->f_data; 1247 int s = splnet(); 1248 int revents; 1249 1250 /* 1251 * Refresh the PID associated with this bpf file. 1252 */ 1253 KERNEL_LOCK(1, NULL); 1254 d->bd_pid = curproc->p_pid; 1255 1256 revents = events & (POLLOUT | POLLWRNORM); 1257 if (events & (POLLIN | POLLRDNORM)) { 1258 /* 1259 * An imitation of the FIONREAD ioctl code. 1260 */ 1261 if (d->bd_hlen != 0 || 1262 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1263 d->bd_slen != 0)) { 1264 revents |= events & (POLLIN | POLLRDNORM); 1265 } else { 1266 selrecord(curlwp, &d->bd_sel); 1267 /* Start the read timeout if necessary */ 1268 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1269 callout_reset(&d->bd_callout, d->bd_rtout, 1270 bpf_timed_out, d); 1271 d->bd_state = BPF_WAITING; 1272 } 1273 } 1274 } 1275 1276 KERNEL_UNLOCK_ONE(NULL); 1277 splx(s); 1278 return (revents); 1279 } 1280 1281 static void 1282 filt_bpfrdetach(struct knote *kn) 1283 { 1284 struct bpf_d *d = kn->kn_hook; 1285 int s; 1286 1287 KERNEL_LOCK(1, NULL); 1288 s = splnet(); 1289 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1290 splx(s); 1291 KERNEL_UNLOCK_ONE(NULL); 1292 } 1293 1294 static int 1295 filt_bpfread(struct knote *kn, long hint) 1296 { 1297 struct bpf_d *d = kn->kn_hook; 1298 int rv; 1299 1300 KERNEL_LOCK(1, NULL); 1301 kn->kn_data = d->bd_hlen; 1302 if (d->bd_immediate) 1303 kn->kn_data += d->bd_slen; 1304 rv = (kn->kn_data > 0); 1305 KERNEL_UNLOCK_ONE(NULL); 1306 return rv; 1307 } 1308 1309 static const struct filterops bpfread_filtops = 1310 { 1, NULL, filt_bpfrdetach, filt_bpfread }; 1311 1312 static int 1313 bpf_kqfilter(struct file *fp, struct knote *kn) 1314 { 1315 struct bpf_d *d = fp->f_data; 1316 struct klist *klist; 1317 int s; 1318 1319 KERNEL_LOCK(1, NULL); 1320 1321 switch (kn->kn_filter) { 1322 case EVFILT_READ: 1323 klist = &d->bd_sel.sel_klist; 1324 kn->kn_fop = &bpfread_filtops; 1325 break; 1326 1327 default: 1328 KERNEL_UNLOCK_ONE(NULL); 1329 return (EINVAL); 1330 } 1331 1332 kn->kn_hook = d; 1333 1334 s = splnet(); 1335 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1336 splx(s); 1337 KERNEL_UNLOCK_ONE(NULL); 1338 1339 return (0); 1340 } 1341 1342 /* 1343 * Copy data from an mbuf chain into a buffer. This code is derived 1344 * from m_copydata in sys/uipc_mbuf.c. 1345 */ 1346 static void * 1347 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1348 { 1349 const struct mbuf *m; 1350 u_int count; 1351 u_char *dst; 1352 1353 m = src_arg; 1354 dst = dst_arg; 1355 while (len > 0) { 1356 if (m == NULL) 1357 panic("bpf_mcpy"); 1358 count = min(m->m_len, len); 1359 memcpy(dst, mtod(m, const void *), count); 1360 m = m->m_next; 1361 dst += count; 1362 len -= count; 1363 } 1364 return dst_arg; 1365 } 1366 1367 /* 1368 * Dispatch a packet to all the listeners on interface bp. 1369 * 1370 * pkt pointer to the packet, either a data buffer or an mbuf chain 1371 * buflen buffer length, if pkt is a data buffer 1372 * cpfn a function that can copy pkt into the listener's buffer 1373 * pktlen length of the packet 1374 * rcv true if packet came in 1375 */ 1376 static inline void 1377 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1378 void *pkt, u_int pktlen, u_int buflen, const bool rcv) 1379 { 1380 bpf_ctx_t *bc = bpf_default_ctx(); 1381 bpf_args_t args = { 1382 .pkt = pkt, 1383 .wirelen = pktlen, 1384 .buflen = buflen, 1385 .arg = NULL 1386 }; 1387 struct bpf_d *d; 1388 struct timespec ts; 1389 bool gottime = false; 1390 1391 /* 1392 * Note that the IPL does not have to be raised at this point. 1393 * The only problem that could arise here is that if two different 1394 * interfaces shared any data. This is not the case. 1395 */ 1396 for (d = bp->bif_dlist; d != NULL; d = d->bd_next) { 1397 u_int slen; 1398 1399 if (!d->bd_seesent && !rcv) { 1400 continue; 1401 } 1402 d->bd_rcount++; 1403 bpf_gstats.bs_recv++; 1404 1405 if (d->bd_jitcode) 1406 slen = d->bd_jitcode(pkt, pktlen, buflen); 1407 else 1408 slen = bpf_filter_ext(bc, d->bd_filter, &args); 1409 1410 if (!slen) { 1411 continue; 1412 } 1413 if (!gottime) { 1414 gottime = true; 1415 nanotime(&ts); 1416 } 1417 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1418 } 1419 } 1420 1421 /* 1422 * Incoming linkage from device drivers. Process the packet pkt, of length 1423 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1424 * by each process' filter, and if accepted, stashed into the corresponding 1425 * buffer. 1426 */ 1427 static void 1428 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1429 { 1430 1431 bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true); 1432 } 1433 1434 /* 1435 * Incoming linkage from device drivers, when the head of the packet is in 1436 * a buffer, and the tail is in an mbuf chain. 1437 */ 1438 static void 1439 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1440 { 1441 u_int pktlen; 1442 struct mbuf mb; 1443 1444 /* Skip outgoing duplicate packets. */ 1445 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1446 m->m_flags &= ~M_PROMISC; 1447 return; 1448 } 1449 1450 pktlen = m_length(m) + dlen; 1451 1452 /* 1453 * Craft on-stack mbuf suitable for passing to bpf_filter. 1454 * Note that we cut corners here; we only setup what's 1455 * absolutely needed--this mbuf should never go anywhere else. 1456 */ 1457 (void)memset(&mb, 0, sizeof(mb)); 1458 mb.m_next = m; 1459 mb.m_data = data; 1460 mb.m_len = dlen; 1461 1462 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif != NULL); 1463 } 1464 1465 /* 1466 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1467 */ 1468 static void 1469 _bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1470 { 1471 void *(*cpfn)(void *, const void *, size_t); 1472 u_int pktlen, buflen; 1473 void *marg; 1474 1475 /* Skip outgoing duplicate packets. */ 1476 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1477 m->m_flags &= ~M_PROMISC; 1478 return; 1479 } 1480 1481 pktlen = m_length(m); 1482 1483 if (pktlen == m->m_len) { 1484 cpfn = (void *)memcpy; 1485 marg = mtod(m, void *); 1486 buflen = pktlen; 1487 } else { 1488 cpfn = bpf_mcpy; 1489 marg = m; 1490 buflen = 0; 1491 } 1492 1493 bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif != NULL); 1494 } 1495 1496 /* 1497 * We need to prepend the address family as 1498 * a four byte field. Cons up a dummy header 1499 * to pacify bpf. This is safe because bpf 1500 * will only read from the mbuf (i.e., it won't 1501 * try to free it or keep a pointer a to it). 1502 */ 1503 static void 1504 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m) 1505 { 1506 struct mbuf m0; 1507 1508 m0.m_flags = 0; 1509 m0.m_next = m; 1510 m0.m_len = 4; 1511 m0.m_data = (char *)⁡ 1512 1513 _bpf_mtap(bp, &m0); 1514 } 1515 1516 /* 1517 * Put the SLIP pseudo-"link header" in place. 1518 * Note this M_PREPEND() should never fail, 1519 * swince we know we always have enough space 1520 * in the input buffer. 1521 */ 1522 static void 1523 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1524 { 1525 int s; 1526 u_char *hp; 1527 1528 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1529 if (*m == NULL) 1530 return; 1531 1532 hp = mtod(*m, u_char *); 1533 hp[SLX_DIR] = SLIPDIR_IN; 1534 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1535 1536 s = splnet(); 1537 _bpf_mtap(bp, *m); 1538 splx(s); 1539 1540 m_adj(*m, SLIP_HDRLEN); 1541 } 1542 1543 /* 1544 * Put the SLIP pseudo-"link header" in 1545 * place. The compressed header is now 1546 * at the beginning of the mbuf. 1547 */ 1548 static void 1549 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1550 { 1551 struct mbuf m0; 1552 u_char *hp; 1553 int s; 1554 1555 m0.m_flags = 0; 1556 m0.m_next = m; 1557 m0.m_data = m0.m_dat; 1558 m0.m_len = SLIP_HDRLEN; 1559 1560 hp = mtod(&m0, u_char *); 1561 1562 hp[SLX_DIR] = SLIPDIR_OUT; 1563 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1564 1565 s = splnet(); 1566 _bpf_mtap(bp, &m0); 1567 splx(s); 1568 m_freem(m); 1569 } 1570 1571 static int 1572 bpf_hdrlen(struct bpf_d *d) 1573 { 1574 int hdrlen = d->bd_bif->bif_hdrlen; 1575 /* 1576 * Compute the length of the bpf header. This is not necessarily 1577 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1578 * that the network layer header begins on a longword boundary (for 1579 * performance reasons and to alleviate alignment restrictions). 1580 */ 1581 #ifdef _LP64 1582 if (d->bd_compat32) 1583 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1584 else 1585 #endif 1586 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1587 } 1588 1589 /* 1590 * Move the packet data from interface memory (pkt) into the 1591 * store buffer. Call the wakeup functions if it's time to wakeup 1592 * a listener (buffer full), "cpfn" is the routine called to do the 1593 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1594 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1595 * pkt is really an mbuf. 1596 */ 1597 static void 1598 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1599 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1600 { 1601 char *h; 1602 int totlen, curlen, caplen; 1603 int hdrlen = bpf_hdrlen(d); 1604 int do_wakeup = 0; 1605 1606 ++d->bd_ccount; 1607 ++bpf_gstats.bs_capt; 1608 /* 1609 * Figure out how many bytes to move. If the packet is 1610 * greater or equal to the snapshot length, transfer that 1611 * much. Otherwise, transfer the whole packet (unless 1612 * we hit the buffer size limit). 1613 */ 1614 totlen = hdrlen + min(snaplen, pktlen); 1615 if (totlen > d->bd_bufsize) 1616 totlen = d->bd_bufsize; 1617 /* 1618 * If we adjusted totlen to fit the bufsize, it could be that 1619 * totlen is smaller than hdrlen because of the link layer header. 1620 */ 1621 caplen = totlen - hdrlen; 1622 if (caplen < 0) 1623 caplen = 0; 1624 1625 /* 1626 * Round up the end of the previous packet to the next longword. 1627 */ 1628 #ifdef _LP64 1629 if (d->bd_compat32) 1630 curlen = BPF_WORDALIGN32(d->bd_slen); 1631 else 1632 #endif 1633 curlen = BPF_WORDALIGN(d->bd_slen); 1634 if (curlen + totlen > d->bd_bufsize) { 1635 /* 1636 * This packet will overflow the storage buffer. 1637 * Rotate the buffers if we can, then wakeup any 1638 * pending reads. 1639 */ 1640 if (d->bd_fbuf == 0) { 1641 /* 1642 * We haven't completed the previous read yet, 1643 * so drop the packet. 1644 */ 1645 ++d->bd_dcount; 1646 ++bpf_gstats.bs_drop; 1647 return; 1648 } 1649 ROTATE_BUFFERS(d); 1650 do_wakeup = 1; 1651 curlen = 0; 1652 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1653 /* 1654 * Immediate mode is set, or the read timeout has 1655 * already expired during a select call. A packet 1656 * arrived, so the reader should be woken up. 1657 */ 1658 do_wakeup = 1; 1659 } 1660 1661 /* 1662 * Append the bpf header. 1663 */ 1664 h = (char *)d->bd_sbuf + curlen; 1665 #ifdef _LP64 1666 if (d->bd_compat32) { 1667 struct bpf_hdr32 *hp32; 1668 1669 hp32 = (struct bpf_hdr32 *)h; 1670 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1671 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1672 hp32->bh_datalen = pktlen; 1673 hp32->bh_hdrlen = hdrlen; 1674 hp32->bh_caplen = caplen; 1675 } else 1676 #endif 1677 { 1678 struct bpf_hdr *hp; 1679 1680 hp = (struct bpf_hdr *)h; 1681 hp->bh_tstamp.tv_sec = ts->tv_sec; 1682 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1683 hp->bh_datalen = pktlen; 1684 hp->bh_hdrlen = hdrlen; 1685 hp->bh_caplen = caplen; 1686 } 1687 1688 /* 1689 * Copy the packet data into the store buffer and update its length. 1690 */ 1691 (*cpfn)(h + hdrlen, pkt, caplen); 1692 d->bd_slen = curlen + totlen; 1693 1694 /* 1695 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1696 * will cause filt_bpfread() to be called with it adjusted. 1697 */ 1698 if (do_wakeup) 1699 bpf_wakeup(d); 1700 } 1701 1702 /* 1703 * Initialize all nonzero fields of a descriptor. 1704 */ 1705 static int 1706 bpf_allocbufs(struct bpf_d *d) 1707 { 1708 1709 d->bd_fbuf = malloc(d->bd_bufsize, M_DEVBUF, M_WAITOK | M_CANFAIL); 1710 if (!d->bd_fbuf) 1711 return (ENOBUFS); 1712 d->bd_sbuf = malloc(d->bd_bufsize, M_DEVBUF, M_WAITOK | M_CANFAIL); 1713 if (!d->bd_sbuf) { 1714 free(d->bd_fbuf, M_DEVBUF); 1715 return (ENOBUFS); 1716 } 1717 d->bd_slen = 0; 1718 d->bd_hlen = 0; 1719 return (0); 1720 } 1721 1722 /* 1723 * Free buffers currently in use by a descriptor. 1724 * Called on close. 1725 */ 1726 static void 1727 bpf_freed(struct bpf_d *d) 1728 { 1729 /* 1730 * We don't need to lock out interrupts since this descriptor has 1731 * been detached from its interface and it yet hasn't been marked 1732 * free. 1733 */ 1734 if (d->bd_sbuf != NULL) { 1735 free(d->bd_sbuf, M_DEVBUF); 1736 if (d->bd_hbuf != NULL) 1737 free(d->bd_hbuf, M_DEVBUF); 1738 if (d->bd_fbuf != NULL) 1739 free(d->bd_fbuf, M_DEVBUF); 1740 } 1741 if (d->bd_filter) 1742 free(d->bd_filter, M_DEVBUF); 1743 1744 if (d->bd_jitcode != NULL) { 1745 bpf_jit_freecode(d->bd_jitcode); 1746 } 1747 } 1748 1749 /* 1750 * Attach an interface to bpf. dlt is the link layer type; 1751 * hdrlen is the fixed size of the link header for the specified dlt 1752 * (variable length headers not yet supported). 1753 */ 1754 static void 1755 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 1756 { 1757 struct bpf_if *bp; 1758 bp = malloc(sizeof(*bp), M_DEVBUF, M_DONTWAIT); 1759 if (bp == 0) 1760 panic("bpfattach"); 1761 1762 bp->bif_dlist = 0; 1763 bp->bif_driverp = driverp; 1764 bp->bif_ifp = ifp; 1765 bp->bif_dlt = dlt; 1766 1767 bp->bif_next = bpf_iflist; 1768 bpf_iflist = bp; 1769 1770 *bp->bif_driverp = 0; 1771 1772 bp->bif_hdrlen = hdrlen; 1773 #if 0 1774 printf("bpf: %s attached\n", ifp->if_xname); 1775 #endif 1776 } 1777 1778 /* 1779 * Remove an interface from bpf. 1780 */ 1781 static void 1782 _bpfdetach(struct ifnet *ifp) 1783 { 1784 struct bpf_if *bp, **pbp; 1785 struct bpf_d *d; 1786 int s; 1787 1788 /* Nuke the vnodes for any open instances */ 1789 LIST_FOREACH(d, &bpf_list, bd_list) { 1790 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 1791 /* 1792 * Detach the descriptor from an interface now. 1793 * It will be free'ed later by close routine. 1794 */ 1795 s = splnet(); 1796 d->bd_promisc = 0; /* we can't touch device. */ 1797 bpf_detachd(d); 1798 splx(s); 1799 } 1800 } 1801 1802 again: 1803 for (bp = bpf_iflist, pbp = &bpf_iflist; 1804 bp != NULL; pbp = &bp->bif_next, bp = bp->bif_next) { 1805 if (bp->bif_ifp == ifp) { 1806 *pbp = bp->bif_next; 1807 free(bp, M_DEVBUF); 1808 goto again; 1809 } 1810 } 1811 } 1812 1813 /* 1814 * Change the data link type of a interface. 1815 */ 1816 static void 1817 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 1818 { 1819 struct bpf_if *bp; 1820 1821 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1822 if (bp->bif_driverp == &ifp->if_bpf) 1823 break; 1824 } 1825 if (bp == NULL) 1826 panic("bpf_change_type"); 1827 1828 bp->bif_dlt = dlt; 1829 1830 bp->bif_hdrlen = hdrlen; 1831 } 1832 1833 /* 1834 * Get a list of available data link type of the interface. 1835 */ 1836 static int 1837 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 1838 { 1839 int n, error; 1840 struct ifnet *ifp; 1841 struct bpf_if *bp; 1842 1843 ifp = d->bd_bif->bif_ifp; 1844 n = 0; 1845 error = 0; 1846 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1847 if (bp->bif_ifp != ifp) 1848 continue; 1849 if (bfl->bfl_list != NULL) { 1850 if (n >= bfl->bfl_len) 1851 return ENOMEM; 1852 error = copyout(&bp->bif_dlt, 1853 bfl->bfl_list + n, sizeof(u_int)); 1854 } 1855 n++; 1856 } 1857 bfl->bfl_len = n; 1858 return error; 1859 } 1860 1861 /* 1862 * Set the data link type of a BPF instance. 1863 */ 1864 static int 1865 bpf_setdlt(struct bpf_d *d, u_int dlt) 1866 { 1867 int s, error, opromisc; 1868 struct ifnet *ifp; 1869 struct bpf_if *bp; 1870 1871 if (d->bd_bif->bif_dlt == dlt) 1872 return 0; 1873 ifp = d->bd_bif->bif_ifp; 1874 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1875 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 1876 break; 1877 } 1878 if (bp == NULL) 1879 return EINVAL; 1880 s = splnet(); 1881 opromisc = d->bd_promisc; 1882 bpf_detachd(d); 1883 bpf_attachd(d, bp); 1884 reset_d(d); 1885 if (opromisc) { 1886 error = ifpromisc(bp->bif_ifp, 1); 1887 if (error) 1888 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 1889 bp->bif_ifp->if_xname, error); 1890 else 1891 d->bd_promisc = 1; 1892 } 1893 splx(s); 1894 return 0; 1895 } 1896 1897 static int 1898 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 1899 { 1900 int newsize, error; 1901 struct sysctlnode node; 1902 1903 node = *rnode; 1904 node.sysctl_data = &newsize; 1905 newsize = bpf_maxbufsize; 1906 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1907 if (error || newp == NULL) 1908 return (error); 1909 1910 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 1911 return (EINVAL); 1912 1913 bpf_maxbufsize = newsize; 1914 1915 return (0); 1916 } 1917 1918 static int 1919 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 1920 { 1921 bool newval; 1922 int error; 1923 struct sysctlnode node; 1924 1925 node = *rnode; 1926 node.sysctl_data = &newval; 1927 newval = bpf_jit; 1928 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1929 if (error != 0 || newp == NULL) 1930 return error; 1931 1932 bpf_jit = newval; 1933 1934 /* 1935 * Do a full sync to publish new bpf_jit value and 1936 * update bpfjit_module_ops.bj_generate_code variable. 1937 */ 1938 membar_sync(); 1939 1940 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 1941 printf("WARNING jit activation is postponed " 1942 "until after bpfjit module is loaded\n"); 1943 } 1944 1945 return 0; 1946 } 1947 1948 static int 1949 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 1950 { 1951 int error, elem_count; 1952 struct bpf_d *dp; 1953 struct bpf_d_ext dpe; 1954 size_t len, needed, elem_size, out_size; 1955 char *sp; 1956 1957 if (namelen == 1 && name[0] == CTL_QUERY) 1958 return (sysctl_query(SYSCTLFN_CALL(rnode))); 1959 1960 if (namelen != 2) 1961 return (EINVAL); 1962 1963 /* BPF peers is privileged information. */ 1964 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 1965 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 1966 if (error) 1967 return (EPERM); 1968 1969 len = (oldp != NULL) ? *oldlenp : 0; 1970 sp = oldp; 1971 elem_size = name[0]; 1972 elem_count = name[1]; 1973 out_size = MIN(sizeof(dpe), elem_size); 1974 needed = 0; 1975 1976 if (elem_size < 1 || elem_count < 0) 1977 return (EINVAL); 1978 1979 mutex_enter(&bpf_mtx); 1980 LIST_FOREACH(dp, &bpf_list, bd_list) { 1981 if (len >= elem_size && elem_count > 0) { 1982 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 1983 BPF_EXT(bufsize); 1984 BPF_EXT(promisc); 1985 BPF_EXT(state); 1986 BPF_EXT(immediate); 1987 BPF_EXT(hdrcmplt); 1988 BPF_EXT(seesent); 1989 BPF_EXT(pid); 1990 BPF_EXT(rcount); 1991 BPF_EXT(dcount); 1992 BPF_EXT(ccount); 1993 #undef BPF_EXT 1994 if (dp->bd_bif) 1995 (void)strlcpy(dpe.bde_ifname, 1996 dp->bd_bif->bif_ifp->if_xname, 1997 IFNAMSIZ - 1); 1998 else 1999 dpe.bde_ifname[0] = '\0'; 2000 2001 error = copyout(&dpe, sp, out_size); 2002 if (error) 2003 break; 2004 sp += elem_size; 2005 len -= elem_size; 2006 } 2007 needed += elem_size; 2008 if (elem_count > 0 && elem_count != INT_MAX) 2009 elem_count--; 2010 } 2011 mutex_exit(&bpf_mtx); 2012 2013 *oldlenp = needed; 2014 2015 return (error); 2016 } 2017 2018 static struct sysctllog *bpf_sysctllog; 2019 static void 2020 sysctl_net_bpf_setup(void) 2021 { 2022 const struct sysctlnode *node; 2023 2024 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2025 CTLFLAG_PERMANENT, 2026 CTLTYPE_NODE, "net", NULL, 2027 NULL, 0, NULL, 0, 2028 CTL_NET, CTL_EOL); 2029 2030 node = NULL; 2031 sysctl_createv(&bpf_sysctllog, 0, NULL, &node, 2032 CTLFLAG_PERMANENT, 2033 CTLTYPE_NODE, "bpf", 2034 SYSCTL_DESCR("BPF options"), 2035 NULL, 0, NULL, 0, 2036 CTL_NET, CTL_CREATE, CTL_EOL); 2037 if (node != NULL) { 2038 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2039 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2040 CTLTYPE_BOOL, "jit", 2041 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2042 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2043 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2044 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2045 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2046 CTLTYPE_INT, "maxbufsize", 2047 SYSCTL_DESCR("Maximum size for data capture buffer"), 2048 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2049 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2050 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2051 CTLFLAG_PERMANENT, 2052 CTLTYPE_STRUCT, "stats", 2053 SYSCTL_DESCR("BPF stats"), 2054 NULL, 0, &bpf_gstats, sizeof(bpf_gstats), 2055 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2056 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2057 CTLFLAG_PERMANENT, 2058 CTLTYPE_STRUCT, "peers", 2059 SYSCTL_DESCR("BPF peers"), 2060 sysctl_net_bpf_peers, 0, NULL, 0, 2061 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2062 } 2063 2064 } 2065 2066 struct bpf_ops bpf_ops_kernel = { 2067 .bpf_attach = _bpfattach, 2068 .bpf_detach = _bpfdetach, 2069 .bpf_change_type = _bpf_change_type, 2070 2071 .bpf_tap = _bpf_tap, 2072 .bpf_mtap = _bpf_mtap, 2073 .bpf_mtap2 = _bpf_mtap2, 2074 .bpf_mtap_af = _bpf_mtap_af, 2075 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2076 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2077 }; 2078 2079 MODULE(MODULE_CLASS_DRIVER, bpf, NULL); 2080 2081 static int 2082 bpf_modcmd(modcmd_t cmd, void *arg) 2083 { 2084 devmajor_t bmajor, cmajor; 2085 int error; 2086 2087 bmajor = cmajor = NODEVMAJOR; 2088 2089 switch (cmd) { 2090 case MODULE_CMD_INIT: 2091 bpfilterattach(0); 2092 error = devsw_attach("bpf", NULL, &bmajor, 2093 &bpf_cdevsw, &cmajor); 2094 if (error == EEXIST) 2095 error = 0; /* maybe built-in ... improve eventually */ 2096 if (error) 2097 break; 2098 2099 bpf_ops_handover_enter(&bpf_ops_kernel); 2100 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2101 bpf_ops_handover_exit(); 2102 sysctl_net_bpf_setup(); 2103 break; 2104 2105 case MODULE_CMD_FINI: 2106 /* 2107 * While there is no reference counting for bpf callers, 2108 * unload could at least in theory be done similarly to 2109 * system call disestablishment. This should even be 2110 * a little simpler: 2111 * 2112 * 1) replace op vector with stubs 2113 * 2) post update to all cpus with xc 2114 * 3) check that nobody is in bpf anymore 2115 * (it's doubtful we'd want something like l_sysent, 2116 * but we could do something like *signed* percpu 2117 * counters. if the sum is 0, we're good). 2118 * 4) if fail, unroll changes 2119 * 2120 * NOTE: change won't be atomic to the outside. some 2121 * packets may be not captured even if unload is 2122 * not succesful. I think packet capture not working 2123 * is a perfectly logical consequence of trying to 2124 * disable packet capture. 2125 */ 2126 error = EOPNOTSUPP; 2127 /* insert sysctl teardown */ 2128 break; 2129 2130 default: 2131 error = ENOTTY; 2132 break; 2133 } 2134 2135 return error; 2136 } 2137