1 /* $NetBSD: bpf.c,v 1.180 2013/12/05 15:55:35 christos Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.180 2013/12/05 15:55:35 christos Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "strip.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/mbuf.h> 53 #include <sys/buf.h> 54 #include <sys/time.h> 55 #include <sys/proc.h> 56 #include <sys/ioctl.h> 57 #include <sys/conf.h> 58 #include <sys/vnode.h> 59 #include <sys/queue.h> 60 #include <sys/stat.h> 61 #include <sys/module.h> 62 #include <sys/once.h> 63 #include <sys/atomic.h> 64 65 #include <sys/file.h> 66 #include <sys/filedesc.h> 67 #include <sys/tty.h> 68 #include <sys/uio.h> 69 70 #include <sys/protosw.h> 71 #include <sys/socket.h> 72 #include <sys/errno.h> 73 #include <sys/kernel.h> 74 #include <sys/poll.h> 75 #include <sys/sysctl.h> 76 #include <sys/kauth.h> 77 78 #include <net/if.h> 79 #include <net/slip.h> 80 81 #include <net/bpf.h> 82 #include <net/bpfdesc.h> 83 #include <net/bpfjit.h> 84 85 #include <net/if_arc.h> 86 #include <net/if_ether.h> 87 88 #include <netinet/in.h> 89 #include <netinet/if_inarp.h> 90 91 92 #include <compat/sys/sockio.h> 93 94 #ifndef BPF_BUFSIZE 95 /* 96 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 97 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 98 */ 99 # define BPF_BUFSIZE 32768 100 #endif 101 102 #define PRINET 26 /* interruptible */ 103 104 /* 105 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 106 * XXX the default values should be computed dynamically based 107 * on available memory size and available mbuf clusters. 108 */ 109 int bpf_bufsize = BPF_BUFSIZE; 110 int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 111 bool bpf_jit = false; 112 113 struct bpfjit_ops bpfjit_module_ops = { 114 .bj_generate_code = NULL, 115 .bj_free_code = NULL 116 }; 117 118 /* 119 * Global BPF statistics returned by net.bpf.stats sysctl. 120 */ 121 struct bpf_stat bpf_gstats; 122 123 /* 124 * Use a mutex to avoid a race condition between gathering the stats/peers 125 * and opening/closing the device. 126 */ 127 static kmutex_t bpf_mtx; 128 129 /* 130 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 131 * bpf_dtab holds the descriptors, indexed by minor device # 132 */ 133 struct bpf_if *bpf_iflist; 134 LIST_HEAD(, bpf_d) bpf_list; 135 136 static int bpf_allocbufs(struct bpf_d *); 137 static void bpf_deliver(struct bpf_if *, 138 void *(*cpfn)(void *, const void *, size_t), 139 void *, u_int, u_int, const bool); 140 static void bpf_freed(struct bpf_d *); 141 static void bpf_ifname(struct ifnet *, struct ifreq *); 142 static void *bpf_mcpy(void *, const void *, size_t); 143 static int bpf_movein(struct uio *, int, uint64_t, 144 struct mbuf **, struct sockaddr *); 145 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 146 static void bpf_detachd(struct bpf_d *); 147 static int bpf_setif(struct bpf_d *, struct ifreq *); 148 static void bpf_timed_out(void *); 149 static inline void 150 bpf_wakeup(struct bpf_d *); 151 static int bpf_hdrlen(struct bpf_d *); 152 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 153 void *(*)(void *, const void *, size_t), struct timespec *); 154 static void reset_d(struct bpf_d *); 155 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 156 static int bpf_setdlt(struct bpf_d *, u_int); 157 158 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 159 int); 160 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 161 int); 162 static int bpf_ioctl(struct file *, u_long, void *); 163 static int bpf_poll(struct file *, int); 164 static int bpf_stat(struct file *, struct stat *); 165 static int bpf_close(struct file *); 166 static int bpf_kqfilter(struct file *, struct knote *); 167 static void bpf_softintr(void *); 168 169 static const struct fileops bpf_fileops = { 170 .fo_read = bpf_read, 171 .fo_write = bpf_write, 172 .fo_ioctl = bpf_ioctl, 173 .fo_fcntl = fnullop_fcntl, 174 .fo_poll = bpf_poll, 175 .fo_stat = bpf_stat, 176 .fo_close = bpf_close, 177 .fo_kqfilter = bpf_kqfilter, 178 .fo_restart = fnullop_restart, 179 }; 180 181 dev_type_open(bpfopen); 182 183 const struct cdevsw bpf_cdevsw = { 184 bpfopen, noclose, noread, nowrite, noioctl, 185 nostop, notty, nopoll, nommap, nokqfilter, D_OTHER 186 }; 187 188 bpfjit_func_t 189 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 190 { 191 membar_consumer(); 192 if (bpfjit_module_ops.bj_generate_code != NULL) { 193 return bpfjit_module_ops.bj_generate_code(bc, code, size); 194 } 195 return NULL; 196 } 197 198 void 199 bpf_jit_freecode(bpfjit_func_t jcode) 200 { 201 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 202 bpfjit_module_ops.bj_free_code(jcode); 203 } 204 205 static int 206 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 207 struct sockaddr *sockp) 208 { 209 struct mbuf *m; 210 int error; 211 size_t len; 212 size_t hlen; 213 size_t align; 214 215 /* 216 * Build a sockaddr based on the data link layer type. 217 * We do this at this level because the ethernet header 218 * is copied directly into the data field of the sockaddr. 219 * In the case of SLIP, there is no header and the packet 220 * is forwarded as is. 221 * Also, we are careful to leave room at the front of the mbuf 222 * for the link level header. 223 */ 224 switch (linktype) { 225 226 case DLT_SLIP: 227 sockp->sa_family = AF_INET; 228 hlen = 0; 229 align = 0; 230 break; 231 232 case DLT_PPP: 233 sockp->sa_family = AF_UNSPEC; 234 hlen = 0; 235 align = 0; 236 break; 237 238 case DLT_EN10MB: 239 sockp->sa_family = AF_UNSPEC; 240 /* XXX Would MAXLINKHDR be better? */ 241 /* 6(dst)+6(src)+2(type) */ 242 hlen = sizeof(struct ether_header); 243 align = 2; 244 break; 245 246 case DLT_ARCNET: 247 sockp->sa_family = AF_UNSPEC; 248 hlen = ARC_HDRLEN; 249 align = 5; 250 break; 251 252 case DLT_FDDI: 253 sockp->sa_family = AF_LINK; 254 /* XXX 4(FORMAC)+6(dst)+6(src) */ 255 hlen = 16; 256 align = 0; 257 break; 258 259 case DLT_ECONET: 260 sockp->sa_family = AF_UNSPEC; 261 hlen = 6; 262 align = 2; 263 break; 264 265 case DLT_NULL: 266 sockp->sa_family = AF_UNSPEC; 267 hlen = 0; 268 align = 0; 269 break; 270 271 default: 272 return (EIO); 273 } 274 275 len = uio->uio_resid; 276 /* 277 * If there aren't enough bytes for a link level header or the 278 * packet length exceeds the interface mtu, return an error. 279 */ 280 if (len - hlen > mtu) 281 return (EMSGSIZE); 282 283 /* 284 * XXX Avoid complicated buffer chaining --- 285 * bail if it won't fit in a single mbuf. 286 * (Take into account possible alignment bytes) 287 */ 288 if (len + align > MCLBYTES) 289 return (EIO); 290 291 m = m_gethdr(M_WAIT, MT_DATA); 292 m->m_pkthdr.rcvif = 0; 293 m->m_pkthdr.len = (int)(len - hlen); 294 if (len + align > MHLEN) { 295 m_clget(m, M_WAIT); 296 if ((m->m_flags & M_EXT) == 0) { 297 error = ENOBUFS; 298 goto bad; 299 } 300 } 301 302 /* Insure the data is properly aligned */ 303 if (align > 0) { 304 m->m_data += align; 305 m->m_len -= (int)align; 306 } 307 308 error = uiomove(mtod(m, void *), len, uio); 309 if (error) 310 goto bad; 311 if (hlen != 0) { 312 memcpy(sockp->sa_data, mtod(m, void *), hlen); 313 m->m_data += hlen; /* XXX */ 314 len -= hlen; 315 } 316 m->m_len = (int)len; 317 *mp = m; 318 return (0); 319 320 bad: 321 m_freem(m); 322 return (error); 323 } 324 325 /* 326 * Attach file to the bpf interface, i.e. make d listen on bp. 327 * Must be called at splnet. 328 */ 329 static void 330 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 331 { 332 /* 333 * Point d at bp, and add d to the interface's list of listeners. 334 * Finally, point the driver's bpf cookie at the interface so 335 * it will divert packets to bpf. 336 */ 337 d->bd_bif = bp; 338 d->bd_next = bp->bif_dlist; 339 bp->bif_dlist = d; 340 341 *bp->bif_driverp = bp; 342 } 343 344 /* 345 * Detach a file from its interface. 346 */ 347 static void 348 bpf_detachd(struct bpf_d *d) 349 { 350 struct bpf_d **p; 351 struct bpf_if *bp; 352 353 bp = d->bd_bif; 354 /* 355 * Check if this descriptor had requested promiscuous mode. 356 * If so, turn it off. 357 */ 358 if (d->bd_promisc) { 359 int error __diagused; 360 361 d->bd_promisc = 0; 362 /* 363 * Take device out of promiscuous mode. Since we were 364 * able to enter promiscuous mode, we should be able 365 * to turn it off. But we can get an error if 366 * the interface was configured down, so only panic 367 * if we don't get an unexpected error. 368 */ 369 error = ifpromisc(bp->bif_ifp, 0); 370 #ifdef DIAGNOSTIC 371 if (error) 372 printf("%s: ifpromisc failed: %d", __func__, error); 373 #endif 374 } 375 /* Remove d from the interface's descriptor list. */ 376 p = &bp->bif_dlist; 377 while (*p != d) { 378 p = &(*p)->bd_next; 379 if (*p == 0) 380 panic("%s: descriptor not in list", __func__); 381 } 382 *p = (*p)->bd_next; 383 if (bp->bif_dlist == 0) 384 /* 385 * Let the driver know that there are no more listeners. 386 */ 387 *d->bd_bif->bif_driverp = 0; 388 d->bd_bif = 0; 389 } 390 391 static int 392 doinit(void) 393 { 394 395 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 396 397 LIST_INIT(&bpf_list); 398 399 bpf_gstats.bs_recv = 0; 400 bpf_gstats.bs_drop = 0; 401 bpf_gstats.bs_capt = 0; 402 403 return 0; 404 } 405 406 /* 407 * bpfilterattach() is called at boot time. 408 */ 409 /* ARGSUSED */ 410 void 411 bpfilterattach(int n) 412 { 413 static ONCE_DECL(control); 414 415 RUN_ONCE(&control, doinit); 416 } 417 418 /* 419 * Open ethernet device. Clones. 420 */ 421 /* ARGSUSED */ 422 int 423 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 424 { 425 struct bpf_d *d; 426 struct file *fp; 427 int error, fd; 428 429 /* falloc() will use the descriptor for us. */ 430 if ((error = fd_allocfile(&fp, &fd)) != 0) 431 return error; 432 433 d = malloc(sizeof(*d), M_DEVBUF, M_WAITOK|M_ZERO); 434 d->bd_bufsize = bpf_bufsize; 435 d->bd_seesent = 1; 436 d->bd_feedback = 0; 437 d->bd_pid = l->l_proc->p_pid; 438 #ifdef _LP64 439 if (curproc->p_flag & PK_32) 440 d->bd_compat32 = 1; 441 #endif 442 getnanotime(&d->bd_btime); 443 d->bd_atime = d->bd_mtime = d->bd_btime; 444 callout_init(&d->bd_callout, 0); 445 selinit(&d->bd_sel); 446 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d); 447 d->bd_jitcode = NULL; 448 449 mutex_enter(&bpf_mtx); 450 LIST_INSERT_HEAD(&bpf_list, d, bd_list); 451 mutex_exit(&bpf_mtx); 452 453 return fd_clone(fp, fd, flag, &bpf_fileops, d); 454 } 455 456 /* 457 * Close the descriptor by detaching it from its interface, 458 * deallocating its buffers, and marking it free. 459 */ 460 /* ARGSUSED */ 461 static int 462 bpf_close(struct file *fp) 463 { 464 struct bpf_d *d = fp->f_data; 465 int s; 466 467 KERNEL_LOCK(1, NULL); 468 469 /* 470 * Refresh the PID associated with this bpf file. 471 */ 472 d->bd_pid = curproc->p_pid; 473 474 s = splnet(); 475 if (d->bd_state == BPF_WAITING) 476 callout_stop(&d->bd_callout); 477 d->bd_state = BPF_IDLE; 478 if (d->bd_bif) 479 bpf_detachd(d); 480 splx(s); 481 bpf_freed(d); 482 mutex_enter(&bpf_mtx); 483 LIST_REMOVE(d, bd_list); 484 mutex_exit(&bpf_mtx); 485 callout_destroy(&d->bd_callout); 486 seldestroy(&d->bd_sel); 487 softint_disestablish(d->bd_sih); 488 free(d, M_DEVBUF); 489 fp->f_data = NULL; 490 491 KERNEL_UNLOCK_ONE(NULL); 492 493 return (0); 494 } 495 496 /* 497 * Rotate the packet buffers in descriptor d. Move the store buffer 498 * into the hold slot, and the free buffer into the store slot. 499 * Zero the length of the new store buffer. 500 */ 501 #define ROTATE_BUFFERS(d) \ 502 (d)->bd_hbuf = (d)->bd_sbuf; \ 503 (d)->bd_hlen = (d)->bd_slen; \ 504 (d)->bd_sbuf = (d)->bd_fbuf; \ 505 (d)->bd_slen = 0; \ 506 (d)->bd_fbuf = 0; 507 /* 508 * bpfread - read next chunk of packets from buffers 509 */ 510 static int 511 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 512 kauth_cred_t cred, int flags) 513 { 514 struct bpf_d *d = fp->f_data; 515 int timed_out; 516 int error; 517 int s; 518 519 getnanotime(&d->bd_atime); 520 /* 521 * Restrict application to use a buffer the same size as 522 * the kernel buffers. 523 */ 524 if (uio->uio_resid != d->bd_bufsize) 525 return (EINVAL); 526 527 KERNEL_LOCK(1, NULL); 528 s = splnet(); 529 if (d->bd_state == BPF_WAITING) 530 callout_stop(&d->bd_callout); 531 timed_out = (d->bd_state == BPF_TIMED_OUT); 532 d->bd_state = BPF_IDLE; 533 /* 534 * If the hold buffer is empty, then do a timed sleep, which 535 * ends when the timeout expires or when enough packets 536 * have arrived to fill the store buffer. 537 */ 538 while (d->bd_hbuf == 0) { 539 if (fp->f_flag & FNONBLOCK) { 540 if (d->bd_slen == 0) { 541 splx(s); 542 KERNEL_UNLOCK_ONE(NULL); 543 return (EWOULDBLOCK); 544 } 545 ROTATE_BUFFERS(d); 546 break; 547 } 548 549 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 550 /* 551 * A packet(s) either arrived since the previous 552 * read or arrived while we were asleep. 553 * Rotate the buffers and return what's here. 554 */ 555 ROTATE_BUFFERS(d); 556 break; 557 } 558 error = tsleep(d, PRINET|PCATCH, "bpf", 559 d->bd_rtout); 560 if (error == EINTR || error == ERESTART) { 561 splx(s); 562 KERNEL_UNLOCK_ONE(NULL); 563 return (error); 564 } 565 if (error == EWOULDBLOCK) { 566 /* 567 * On a timeout, return what's in the buffer, 568 * which may be nothing. If there is something 569 * in the store buffer, we can rotate the buffers. 570 */ 571 if (d->bd_hbuf) 572 /* 573 * We filled up the buffer in between 574 * getting the timeout and arriving 575 * here, so we don't need to rotate. 576 */ 577 break; 578 579 if (d->bd_slen == 0) { 580 splx(s); 581 KERNEL_UNLOCK_ONE(NULL); 582 return (0); 583 } 584 ROTATE_BUFFERS(d); 585 break; 586 } 587 if (error != 0) 588 goto done; 589 } 590 /* 591 * At this point, we know we have something in the hold slot. 592 */ 593 splx(s); 594 595 /* 596 * Move data from hold buffer into user space. 597 * We know the entire buffer is transferred since 598 * we checked above that the read buffer is bpf_bufsize bytes. 599 */ 600 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 601 602 s = splnet(); 603 d->bd_fbuf = d->bd_hbuf; 604 d->bd_hbuf = 0; 605 d->bd_hlen = 0; 606 done: 607 splx(s); 608 KERNEL_UNLOCK_ONE(NULL); 609 return (error); 610 } 611 612 613 /* 614 * If there are processes sleeping on this descriptor, wake them up. 615 */ 616 static inline void 617 bpf_wakeup(struct bpf_d *d) 618 { 619 wakeup(d); 620 if (d->bd_async) 621 softint_schedule(d->bd_sih); 622 selnotify(&d->bd_sel, 0, 0); 623 } 624 625 static void 626 bpf_softintr(void *cookie) 627 { 628 struct bpf_d *d; 629 630 d = cookie; 631 if (d->bd_async) 632 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 633 } 634 635 static void 636 bpf_timed_out(void *arg) 637 { 638 struct bpf_d *d = arg; 639 int s; 640 641 s = splnet(); 642 if (d->bd_state == BPF_WAITING) { 643 d->bd_state = BPF_TIMED_OUT; 644 if (d->bd_slen != 0) 645 bpf_wakeup(d); 646 } 647 splx(s); 648 } 649 650 651 static int 652 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 653 kauth_cred_t cred, int flags) 654 { 655 struct bpf_d *d = fp->f_data; 656 struct ifnet *ifp; 657 struct mbuf *m, *mc; 658 int error, s; 659 static struct sockaddr_storage dst; 660 661 m = NULL; /* XXX gcc */ 662 663 KERNEL_LOCK(1, NULL); 664 665 if (d->bd_bif == 0) { 666 KERNEL_UNLOCK_ONE(NULL); 667 return (ENXIO); 668 } 669 getnanotime(&d->bd_mtime); 670 671 ifp = d->bd_bif->bif_ifp; 672 673 if (uio->uio_resid == 0) { 674 KERNEL_UNLOCK_ONE(NULL); 675 return (0); 676 } 677 678 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp->if_mtu, &m, 679 (struct sockaddr *) &dst); 680 if (error) { 681 KERNEL_UNLOCK_ONE(NULL); 682 return (error); 683 } 684 685 if (m->m_pkthdr.len > ifp->if_mtu) { 686 KERNEL_UNLOCK_ONE(NULL); 687 m_freem(m); 688 return (EMSGSIZE); 689 } 690 691 if (d->bd_hdrcmplt) 692 dst.ss_family = pseudo_AF_HDRCMPLT; 693 694 if (d->bd_feedback) { 695 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 696 if (mc != NULL) 697 mc->m_pkthdr.rcvif = ifp; 698 /* Set M_PROMISC for outgoing packets to be discarded. */ 699 if (1 /*d->bd_direction == BPF_D_INOUT*/) 700 m->m_flags |= M_PROMISC; 701 } else 702 mc = NULL; 703 704 s = splsoftnet(); 705 error = (*ifp->if_output)(ifp, m, (struct sockaddr *) &dst, NULL); 706 707 if (mc != NULL) { 708 if (error == 0) 709 (*ifp->if_input)(ifp, mc); 710 m_freem(mc); 711 } 712 splx(s); 713 KERNEL_UNLOCK_ONE(NULL); 714 /* 715 * The driver frees the mbuf. 716 */ 717 return (error); 718 } 719 720 /* 721 * Reset a descriptor by flushing its packet buffer and clearing the 722 * receive and drop counts. Should be called at splnet. 723 */ 724 static void 725 reset_d(struct bpf_d *d) 726 { 727 if (d->bd_hbuf) { 728 /* Free the hold buffer. */ 729 d->bd_fbuf = d->bd_hbuf; 730 d->bd_hbuf = 0; 731 } 732 d->bd_slen = 0; 733 d->bd_hlen = 0; 734 d->bd_rcount = 0; 735 d->bd_dcount = 0; 736 d->bd_ccount = 0; 737 } 738 739 /* 740 * FIONREAD Check for read packet available. 741 * BIOCGBLEN Get buffer len [for read()]. 742 * BIOCSETF Set ethernet read filter. 743 * BIOCFLUSH Flush read packet buffer. 744 * BIOCPROMISC Put interface into promiscuous mode. 745 * BIOCGDLT Get link layer type. 746 * BIOCGETIF Get interface name. 747 * BIOCSETIF Set interface. 748 * BIOCSRTIMEOUT Set read timeout. 749 * BIOCGRTIMEOUT Get read timeout. 750 * BIOCGSTATS Get packet stats. 751 * BIOCIMMEDIATE Set immediate mode. 752 * BIOCVERSION Get filter language version. 753 * BIOCGHDRCMPLT Get "header already complete" flag. 754 * BIOCSHDRCMPLT Set "header already complete" flag. 755 * BIOCSFEEDBACK Set packet feedback mode. 756 * BIOCGFEEDBACK Get packet feedback mode. 757 * BIOCGSEESENT Get "see sent packets" mode. 758 * BIOCSSEESENT Set "see sent packets" mode. 759 */ 760 /* ARGSUSED */ 761 static int 762 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 763 { 764 struct bpf_d *d = fp->f_data; 765 int s, error = 0; 766 767 /* 768 * Refresh the PID associated with this bpf file. 769 */ 770 KERNEL_LOCK(1, NULL); 771 d->bd_pid = curproc->p_pid; 772 #ifdef _LP64 773 if (curproc->p_flag & PK_32) 774 d->bd_compat32 = 1; 775 else 776 d->bd_compat32 = 0; 777 #endif 778 779 s = splnet(); 780 if (d->bd_state == BPF_WAITING) 781 callout_stop(&d->bd_callout); 782 d->bd_state = BPF_IDLE; 783 splx(s); 784 785 switch (cmd) { 786 787 default: 788 error = EINVAL; 789 break; 790 791 /* 792 * Check for read packet available. 793 */ 794 case FIONREAD: 795 { 796 int n; 797 798 s = splnet(); 799 n = d->bd_slen; 800 if (d->bd_hbuf) 801 n += d->bd_hlen; 802 splx(s); 803 804 *(int *)addr = n; 805 break; 806 } 807 808 /* 809 * Get buffer len [for read()]. 810 */ 811 case BIOCGBLEN: 812 *(u_int *)addr = d->bd_bufsize; 813 break; 814 815 /* 816 * Set buffer length. 817 */ 818 case BIOCSBLEN: 819 if (d->bd_bif != 0) 820 error = EINVAL; 821 else { 822 u_int size = *(u_int *)addr; 823 824 if (size > bpf_maxbufsize) 825 *(u_int *)addr = size = bpf_maxbufsize; 826 else if (size < BPF_MINBUFSIZE) 827 *(u_int *)addr = size = BPF_MINBUFSIZE; 828 d->bd_bufsize = size; 829 } 830 break; 831 832 /* 833 * Set link layer read filter. 834 */ 835 case BIOCSETF: 836 error = bpf_setf(d, addr); 837 break; 838 839 /* 840 * Flush read packet buffer. 841 */ 842 case BIOCFLUSH: 843 s = splnet(); 844 reset_d(d); 845 splx(s); 846 break; 847 848 /* 849 * Put interface into promiscuous mode. 850 */ 851 case BIOCPROMISC: 852 if (d->bd_bif == 0) { 853 /* 854 * No interface attached yet. 855 */ 856 error = EINVAL; 857 break; 858 } 859 s = splnet(); 860 if (d->bd_promisc == 0) { 861 error = ifpromisc(d->bd_bif->bif_ifp, 1); 862 if (error == 0) 863 d->bd_promisc = 1; 864 } 865 splx(s); 866 break; 867 868 /* 869 * Get device parameters. 870 */ 871 case BIOCGDLT: 872 if (d->bd_bif == 0) 873 error = EINVAL; 874 else 875 *(u_int *)addr = d->bd_bif->bif_dlt; 876 break; 877 878 /* 879 * Get a list of supported device parameters. 880 */ 881 case BIOCGDLTLIST: 882 if (d->bd_bif == 0) 883 error = EINVAL; 884 else 885 error = bpf_getdltlist(d, addr); 886 break; 887 888 /* 889 * Set device parameters. 890 */ 891 case BIOCSDLT: 892 if (d->bd_bif == 0) 893 error = EINVAL; 894 else 895 error = bpf_setdlt(d, *(u_int *)addr); 896 break; 897 898 /* 899 * Set interface name. 900 */ 901 #ifdef OBIOCGETIF 902 case OBIOCGETIF: 903 #endif 904 case BIOCGETIF: 905 if (d->bd_bif == 0) 906 error = EINVAL; 907 else 908 bpf_ifname(d->bd_bif->bif_ifp, addr); 909 break; 910 911 /* 912 * Set interface. 913 */ 914 #ifdef OBIOCSETIF 915 case OBIOCSETIF: 916 #endif 917 case BIOCSETIF: 918 error = bpf_setif(d, addr); 919 break; 920 921 /* 922 * Set read timeout. 923 */ 924 case BIOCSRTIMEOUT: 925 { 926 struct timeval *tv = addr; 927 928 /* Compute number of ticks. */ 929 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 930 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 931 d->bd_rtout = 1; 932 break; 933 } 934 935 #ifdef BIOCGORTIMEOUT 936 /* 937 * Get read timeout. 938 */ 939 case BIOCGORTIMEOUT: 940 { 941 struct timeval50 *tv = addr; 942 943 tv->tv_sec = d->bd_rtout / hz; 944 tv->tv_usec = (d->bd_rtout % hz) * tick; 945 break; 946 } 947 #endif 948 949 #ifdef BIOCSORTIMEOUT 950 /* 951 * Set read timeout. 952 */ 953 case BIOCSORTIMEOUT: 954 { 955 struct timeval50 *tv = addr; 956 957 /* Compute number of ticks. */ 958 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 959 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 960 d->bd_rtout = 1; 961 break; 962 } 963 #endif 964 965 /* 966 * Get read timeout. 967 */ 968 case BIOCGRTIMEOUT: 969 { 970 struct timeval *tv = addr; 971 972 tv->tv_sec = d->bd_rtout / hz; 973 tv->tv_usec = (d->bd_rtout % hz) * tick; 974 break; 975 } 976 /* 977 * Get packet stats. 978 */ 979 case BIOCGSTATS: 980 { 981 struct bpf_stat *bs = addr; 982 983 bs->bs_recv = d->bd_rcount; 984 bs->bs_drop = d->bd_dcount; 985 bs->bs_capt = d->bd_ccount; 986 break; 987 } 988 989 case BIOCGSTATSOLD: 990 { 991 struct bpf_stat_old *bs = addr; 992 993 bs->bs_recv = d->bd_rcount; 994 bs->bs_drop = d->bd_dcount; 995 break; 996 } 997 998 /* 999 * Set immediate mode. 1000 */ 1001 case BIOCIMMEDIATE: 1002 d->bd_immediate = *(u_int *)addr; 1003 break; 1004 1005 case BIOCVERSION: 1006 { 1007 struct bpf_version *bv = addr; 1008 1009 bv->bv_major = BPF_MAJOR_VERSION; 1010 bv->bv_minor = BPF_MINOR_VERSION; 1011 break; 1012 } 1013 1014 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1015 *(u_int *)addr = d->bd_hdrcmplt; 1016 break; 1017 1018 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1019 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1020 break; 1021 1022 /* 1023 * Get "see sent packets" flag 1024 */ 1025 case BIOCGSEESENT: 1026 *(u_int *)addr = d->bd_seesent; 1027 break; 1028 1029 /* 1030 * Set "see sent" packets flag 1031 */ 1032 case BIOCSSEESENT: 1033 d->bd_seesent = *(u_int *)addr; 1034 break; 1035 1036 /* 1037 * Set "feed packets from bpf back to input" mode 1038 */ 1039 case BIOCSFEEDBACK: 1040 d->bd_feedback = *(u_int *)addr; 1041 break; 1042 1043 /* 1044 * Get "feed packets from bpf back to input" mode 1045 */ 1046 case BIOCGFEEDBACK: 1047 *(u_int *)addr = d->bd_feedback; 1048 break; 1049 1050 case FIONBIO: /* Non-blocking I/O */ 1051 /* 1052 * No need to do anything special as we use IO_NDELAY in 1053 * bpfread() as an indication of whether or not to block 1054 * the read. 1055 */ 1056 break; 1057 1058 case FIOASYNC: /* Send signal on receive packets */ 1059 d->bd_async = *(int *)addr; 1060 break; 1061 1062 case TIOCSPGRP: /* Process or group to send signals to */ 1063 case FIOSETOWN: 1064 error = fsetown(&d->bd_pgid, cmd, addr); 1065 break; 1066 1067 case TIOCGPGRP: 1068 case FIOGETOWN: 1069 error = fgetown(d->bd_pgid, cmd, addr); 1070 break; 1071 } 1072 KERNEL_UNLOCK_ONE(NULL); 1073 return (error); 1074 } 1075 1076 /* 1077 * Set d's packet filter program to fp. If this file already has a filter, 1078 * free it and replace it. Returns EINVAL for bogus requests. 1079 */ 1080 int 1081 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1082 { 1083 struct bpf_insn *fcode, *old; 1084 bpfjit_func_t jcode, oldj; 1085 size_t flen, size; 1086 int s; 1087 1088 jcode = NULL; 1089 flen = fp->bf_len; 1090 1091 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1092 return EINVAL; 1093 } 1094 1095 if (flen) { 1096 /* 1097 * Allocate the buffer, copy the byte-code from 1098 * userspace and validate it. 1099 */ 1100 size = flen * sizeof(*fp->bf_insns); 1101 fcode = malloc(size, M_DEVBUF, M_WAITOK); 1102 if (copyin(fp->bf_insns, fcode, size) != 0 || 1103 !bpf_validate(fcode, (int)flen)) { 1104 free(fcode, M_DEVBUF); 1105 return EINVAL; 1106 } 1107 membar_consumer(); 1108 if (bpf_jit) { 1109 bpf_ctx_t *bc = bpf_default_ctx(); 1110 jcode = bpf_jit_generate(bc, fcode, flen); 1111 } 1112 } else { 1113 fcode = NULL; 1114 } 1115 1116 s = splnet(); 1117 old = d->bd_filter; 1118 d->bd_filter = fcode; 1119 oldj = d->bd_jitcode; 1120 d->bd_jitcode = jcode; 1121 reset_d(d); 1122 splx(s); 1123 1124 if (old) { 1125 free(old, M_DEVBUF); 1126 } 1127 if (oldj) { 1128 bpf_jit_freecode(oldj); 1129 } 1130 1131 return 0; 1132 } 1133 1134 /* 1135 * Detach a file from its current interface (if attached at all) and attach 1136 * to the interface indicated by the name stored in ifr. 1137 * Return an errno or 0. 1138 */ 1139 static int 1140 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1141 { 1142 struct bpf_if *bp; 1143 char *cp; 1144 int unit_seen, i, s, error; 1145 1146 /* 1147 * Make sure the provided name has a unit number, and default 1148 * it to '0' if not specified. 1149 * XXX This is ugly ... do this differently? 1150 */ 1151 unit_seen = 0; 1152 cp = ifr->ifr_name; 1153 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1154 while (*cp++) 1155 if (*cp >= '0' && *cp <= '9') 1156 unit_seen = 1; 1157 if (!unit_seen) { 1158 /* Make sure to leave room for the '\0'. */ 1159 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1160 if ((ifr->ifr_name[i] >= 'a' && 1161 ifr->ifr_name[i] <= 'z') || 1162 (ifr->ifr_name[i] >= 'A' && 1163 ifr->ifr_name[i] <= 'Z')) 1164 continue; 1165 ifr->ifr_name[i] = '0'; 1166 } 1167 } 1168 1169 /* 1170 * Look through attached interfaces for the named one. 1171 */ 1172 for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) { 1173 struct ifnet *ifp = bp->bif_ifp; 1174 1175 if (ifp == 0 || 1176 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1177 continue; 1178 /* skip additional entry */ 1179 if (bp->bif_driverp != &ifp->if_bpf) 1180 continue; 1181 /* 1182 * We found the requested interface. 1183 * Allocate the packet buffers if we need to. 1184 * If we're already attached to requested interface, 1185 * just flush the buffer. 1186 */ 1187 if (d->bd_sbuf == 0) { 1188 error = bpf_allocbufs(d); 1189 if (error != 0) 1190 return (error); 1191 } 1192 s = splnet(); 1193 if (bp != d->bd_bif) { 1194 if (d->bd_bif) 1195 /* 1196 * Detach if attached to something else. 1197 */ 1198 bpf_detachd(d); 1199 1200 bpf_attachd(d, bp); 1201 } 1202 reset_d(d); 1203 splx(s); 1204 return (0); 1205 } 1206 /* Not found. */ 1207 return (ENXIO); 1208 } 1209 1210 /* 1211 * Copy the interface name to the ifreq. 1212 */ 1213 static void 1214 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1215 { 1216 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1217 } 1218 1219 static int 1220 bpf_stat(struct file *fp, struct stat *st) 1221 { 1222 struct bpf_d *d = fp->f_data; 1223 1224 (void)memset(st, 0, sizeof(*st)); 1225 KERNEL_LOCK(1, NULL); 1226 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1227 st->st_atimespec = d->bd_atime; 1228 st->st_mtimespec = d->bd_mtime; 1229 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1230 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1231 st->st_gid = kauth_cred_getegid(fp->f_cred); 1232 st->st_mode = S_IFCHR; 1233 KERNEL_UNLOCK_ONE(NULL); 1234 return 0; 1235 } 1236 1237 /* 1238 * Support for poll() system call 1239 * 1240 * Return true iff the specific operation will not block indefinitely - with 1241 * the assumption that it is safe to positively acknowledge a request for the 1242 * ability to write to the BPF device. 1243 * Otherwise, return false but make a note that a selnotify() must be done. 1244 */ 1245 static int 1246 bpf_poll(struct file *fp, int events) 1247 { 1248 struct bpf_d *d = fp->f_data; 1249 int s = splnet(); 1250 int revents; 1251 1252 /* 1253 * Refresh the PID associated with this bpf file. 1254 */ 1255 KERNEL_LOCK(1, NULL); 1256 d->bd_pid = curproc->p_pid; 1257 1258 revents = events & (POLLOUT | POLLWRNORM); 1259 if (events & (POLLIN | POLLRDNORM)) { 1260 /* 1261 * An imitation of the FIONREAD ioctl code. 1262 */ 1263 if (d->bd_hlen != 0 || 1264 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1265 d->bd_slen != 0)) { 1266 revents |= events & (POLLIN | POLLRDNORM); 1267 } else { 1268 selrecord(curlwp, &d->bd_sel); 1269 /* Start the read timeout if necessary */ 1270 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1271 callout_reset(&d->bd_callout, d->bd_rtout, 1272 bpf_timed_out, d); 1273 d->bd_state = BPF_WAITING; 1274 } 1275 } 1276 } 1277 1278 KERNEL_UNLOCK_ONE(NULL); 1279 splx(s); 1280 return (revents); 1281 } 1282 1283 static void 1284 filt_bpfrdetach(struct knote *kn) 1285 { 1286 struct bpf_d *d = kn->kn_hook; 1287 int s; 1288 1289 KERNEL_LOCK(1, NULL); 1290 s = splnet(); 1291 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1292 splx(s); 1293 KERNEL_UNLOCK_ONE(NULL); 1294 } 1295 1296 static int 1297 filt_bpfread(struct knote *kn, long hint) 1298 { 1299 struct bpf_d *d = kn->kn_hook; 1300 int rv; 1301 1302 KERNEL_LOCK(1, NULL); 1303 kn->kn_data = d->bd_hlen; 1304 if (d->bd_immediate) 1305 kn->kn_data += d->bd_slen; 1306 rv = (kn->kn_data > 0); 1307 KERNEL_UNLOCK_ONE(NULL); 1308 return rv; 1309 } 1310 1311 static const struct filterops bpfread_filtops = 1312 { 1, NULL, filt_bpfrdetach, filt_bpfread }; 1313 1314 static int 1315 bpf_kqfilter(struct file *fp, struct knote *kn) 1316 { 1317 struct bpf_d *d = fp->f_data; 1318 struct klist *klist; 1319 int s; 1320 1321 KERNEL_LOCK(1, NULL); 1322 1323 switch (kn->kn_filter) { 1324 case EVFILT_READ: 1325 klist = &d->bd_sel.sel_klist; 1326 kn->kn_fop = &bpfread_filtops; 1327 break; 1328 1329 default: 1330 KERNEL_UNLOCK_ONE(NULL); 1331 return (EINVAL); 1332 } 1333 1334 kn->kn_hook = d; 1335 1336 s = splnet(); 1337 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1338 splx(s); 1339 KERNEL_UNLOCK_ONE(NULL); 1340 1341 return (0); 1342 } 1343 1344 /* 1345 * Copy data from an mbuf chain into a buffer. This code is derived 1346 * from m_copydata in sys/uipc_mbuf.c. 1347 */ 1348 static void * 1349 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1350 { 1351 const struct mbuf *m; 1352 u_int count; 1353 u_char *dst; 1354 1355 m = src_arg; 1356 dst = dst_arg; 1357 while (len > 0) { 1358 if (m == NULL) 1359 panic("bpf_mcpy"); 1360 count = min(m->m_len, len); 1361 memcpy(dst, mtod(m, const void *), count); 1362 m = m->m_next; 1363 dst += count; 1364 len -= count; 1365 } 1366 return dst_arg; 1367 } 1368 1369 /* 1370 * Dispatch a packet to all the listeners on interface bp. 1371 * 1372 * pkt pointer to the packet, either a data buffer or an mbuf chain 1373 * buflen buffer length, if pkt is a data buffer 1374 * cpfn a function that can copy pkt into the listener's buffer 1375 * pktlen length of the packet 1376 * rcv true if packet came in 1377 */ 1378 static inline void 1379 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1380 void *pkt, u_int pktlen, u_int buflen, const bool rcv) 1381 { 1382 bpf_ctx_t *bc = bpf_default_ctx(); 1383 bpf_args_t args = { 1384 .pkt = pkt, 1385 .wirelen = pktlen, 1386 .buflen = buflen, 1387 .arg = NULL 1388 }; 1389 struct bpf_d *d; 1390 struct timespec ts; 1391 bool gottime = false; 1392 1393 /* 1394 * Note that the IPL does not have to be raised at this point. 1395 * The only problem that could arise here is that if two different 1396 * interfaces shared any data. This is not the case. 1397 */ 1398 for (d = bp->bif_dlist; d != NULL; d = d->bd_next) { 1399 u_int slen; 1400 1401 if (!d->bd_seesent && !rcv) { 1402 continue; 1403 } 1404 d->bd_rcount++; 1405 bpf_gstats.bs_recv++; 1406 1407 if (d->bd_jitcode) 1408 slen = d->bd_jitcode(pkt, pktlen, buflen); 1409 else 1410 slen = bpf_filter_ext(bc, d->bd_filter, &args); 1411 1412 if (!slen) { 1413 continue; 1414 } 1415 if (!gottime) { 1416 gottime = true; 1417 nanotime(&ts); 1418 } 1419 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1420 } 1421 } 1422 1423 /* 1424 * Incoming linkage from device drivers. Process the packet pkt, of length 1425 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1426 * by each process' filter, and if accepted, stashed into the corresponding 1427 * buffer. 1428 */ 1429 static void 1430 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1431 { 1432 1433 bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true); 1434 } 1435 1436 /* 1437 * Incoming linkage from device drivers, when the head of the packet is in 1438 * a buffer, and the tail is in an mbuf chain. 1439 */ 1440 static void 1441 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1442 { 1443 u_int pktlen; 1444 struct mbuf mb; 1445 1446 /* Skip outgoing duplicate packets. */ 1447 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1448 m->m_flags &= ~M_PROMISC; 1449 return; 1450 } 1451 1452 pktlen = m_length(m) + dlen; 1453 1454 /* 1455 * Craft on-stack mbuf suitable for passing to bpf_filter. 1456 * Note that we cut corners here; we only setup what's 1457 * absolutely needed--this mbuf should never go anywhere else. 1458 */ 1459 (void)memset(&mb, 0, sizeof(mb)); 1460 mb.m_next = m; 1461 mb.m_data = data; 1462 mb.m_len = dlen; 1463 1464 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif != NULL); 1465 } 1466 1467 /* 1468 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1469 */ 1470 static void 1471 _bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1472 { 1473 void *(*cpfn)(void *, const void *, size_t); 1474 u_int pktlen, buflen; 1475 void *marg; 1476 1477 /* Skip outgoing duplicate packets. */ 1478 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif == NULL) { 1479 m->m_flags &= ~M_PROMISC; 1480 return; 1481 } 1482 1483 pktlen = m_length(m); 1484 1485 if (pktlen == m->m_len) { 1486 cpfn = (void *)memcpy; 1487 marg = mtod(m, void *); 1488 buflen = pktlen; 1489 } else { 1490 cpfn = bpf_mcpy; 1491 marg = m; 1492 buflen = 0; 1493 } 1494 1495 bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif != NULL); 1496 } 1497 1498 /* 1499 * We need to prepend the address family as 1500 * a four byte field. Cons up a dummy header 1501 * to pacify bpf. This is safe because bpf 1502 * will only read from the mbuf (i.e., it won't 1503 * try to free it or keep a pointer a to it). 1504 */ 1505 static void 1506 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m) 1507 { 1508 struct mbuf m0; 1509 1510 m0.m_flags = 0; 1511 m0.m_next = m; 1512 m0.m_len = 4; 1513 m0.m_data = (char *)⁡ 1514 1515 _bpf_mtap(bp, &m0); 1516 } 1517 1518 /* 1519 * Put the SLIP pseudo-"link header" in place. 1520 * Note this M_PREPEND() should never fail, 1521 * swince we know we always have enough space 1522 * in the input buffer. 1523 */ 1524 static void 1525 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1526 { 1527 int s; 1528 u_char *hp; 1529 1530 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1531 if (*m == NULL) 1532 return; 1533 1534 hp = mtod(*m, u_char *); 1535 hp[SLX_DIR] = SLIPDIR_IN; 1536 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1537 1538 s = splnet(); 1539 _bpf_mtap(bp, *m); 1540 splx(s); 1541 1542 m_adj(*m, SLIP_HDRLEN); 1543 } 1544 1545 /* 1546 * Put the SLIP pseudo-"link header" in 1547 * place. The compressed header is now 1548 * at the beginning of the mbuf. 1549 */ 1550 static void 1551 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1552 { 1553 struct mbuf m0; 1554 u_char *hp; 1555 int s; 1556 1557 m0.m_flags = 0; 1558 m0.m_next = m; 1559 m0.m_data = m0.m_dat; 1560 m0.m_len = SLIP_HDRLEN; 1561 1562 hp = mtod(&m0, u_char *); 1563 1564 hp[SLX_DIR] = SLIPDIR_OUT; 1565 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1566 1567 s = splnet(); 1568 _bpf_mtap(bp, &m0); 1569 splx(s); 1570 m_freem(m); 1571 } 1572 1573 static int 1574 bpf_hdrlen(struct bpf_d *d) 1575 { 1576 int hdrlen = d->bd_bif->bif_hdrlen; 1577 /* 1578 * Compute the length of the bpf header. This is not necessarily 1579 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1580 * that the network layer header begins on a longword boundary (for 1581 * performance reasons and to alleviate alignment restrictions). 1582 */ 1583 #ifdef _LP64 1584 if (d->bd_compat32) 1585 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1586 else 1587 #endif 1588 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1589 } 1590 1591 /* 1592 * Move the packet data from interface memory (pkt) into the 1593 * store buffer. Call the wakeup functions if it's time to wakeup 1594 * a listener (buffer full), "cpfn" is the routine called to do the 1595 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1596 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1597 * pkt is really an mbuf. 1598 */ 1599 static void 1600 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1601 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1602 { 1603 char *h; 1604 int totlen, curlen, caplen; 1605 int hdrlen = bpf_hdrlen(d); 1606 int do_wakeup = 0; 1607 1608 ++d->bd_ccount; 1609 ++bpf_gstats.bs_capt; 1610 /* 1611 * Figure out how many bytes to move. If the packet is 1612 * greater or equal to the snapshot length, transfer that 1613 * much. Otherwise, transfer the whole packet (unless 1614 * we hit the buffer size limit). 1615 */ 1616 totlen = hdrlen + min(snaplen, pktlen); 1617 if (totlen > d->bd_bufsize) 1618 totlen = d->bd_bufsize; 1619 /* 1620 * If we adjusted totlen to fit the bufsize, it could be that 1621 * totlen is smaller than hdrlen because of the link layer header. 1622 */ 1623 caplen = totlen - hdrlen; 1624 if (caplen < 0) 1625 caplen = 0; 1626 1627 /* 1628 * Round up the end of the previous packet to the next longword. 1629 */ 1630 #ifdef _LP64 1631 if (d->bd_compat32) 1632 curlen = BPF_WORDALIGN32(d->bd_slen); 1633 else 1634 #endif 1635 curlen = BPF_WORDALIGN(d->bd_slen); 1636 if (curlen + totlen > d->bd_bufsize) { 1637 /* 1638 * This packet will overflow the storage buffer. 1639 * Rotate the buffers if we can, then wakeup any 1640 * pending reads. 1641 */ 1642 if (d->bd_fbuf == 0) { 1643 /* 1644 * We haven't completed the previous read yet, 1645 * so drop the packet. 1646 */ 1647 ++d->bd_dcount; 1648 ++bpf_gstats.bs_drop; 1649 return; 1650 } 1651 ROTATE_BUFFERS(d); 1652 do_wakeup = 1; 1653 curlen = 0; 1654 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1655 /* 1656 * Immediate mode is set, or the read timeout has 1657 * already expired during a select call. A packet 1658 * arrived, so the reader should be woken up. 1659 */ 1660 do_wakeup = 1; 1661 } 1662 1663 /* 1664 * Append the bpf header. 1665 */ 1666 h = (char *)d->bd_sbuf + curlen; 1667 #ifdef _LP64 1668 if (d->bd_compat32) { 1669 struct bpf_hdr32 *hp32; 1670 1671 hp32 = (struct bpf_hdr32 *)h; 1672 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1673 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1674 hp32->bh_datalen = pktlen; 1675 hp32->bh_hdrlen = hdrlen; 1676 hp32->bh_caplen = caplen; 1677 } else 1678 #endif 1679 { 1680 struct bpf_hdr *hp; 1681 1682 hp = (struct bpf_hdr *)h; 1683 hp->bh_tstamp.tv_sec = ts->tv_sec; 1684 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1685 hp->bh_datalen = pktlen; 1686 hp->bh_hdrlen = hdrlen; 1687 hp->bh_caplen = caplen; 1688 } 1689 1690 /* 1691 * Copy the packet data into the store buffer and update its length. 1692 */ 1693 (*cpfn)(h + hdrlen, pkt, caplen); 1694 d->bd_slen = curlen + totlen; 1695 1696 /* 1697 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1698 * will cause filt_bpfread() to be called with it adjusted. 1699 */ 1700 if (do_wakeup) 1701 bpf_wakeup(d); 1702 } 1703 1704 /* 1705 * Initialize all nonzero fields of a descriptor. 1706 */ 1707 static int 1708 bpf_allocbufs(struct bpf_d *d) 1709 { 1710 1711 d->bd_fbuf = malloc(d->bd_bufsize, M_DEVBUF, M_WAITOK | M_CANFAIL); 1712 if (!d->bd_fbuf) 1713 return (ENOBUFS); 1714 d->bd_sbuf = malloc(d->bd_bufsize, M_DEVBUF, M_WAITOK | M_CANFAIL); 1715 if (!d->bd_sbuf) { 1716 free(d->bd_fbuf, M_DEVBUF); 1717 return (ENOBUFS); 1718 } 1719 d->bd_slen = 0; 1720 d->bd_hlen = 0; 1721 return (0); 1722 } 1723 1724 /* 1725 * Free buffers currently in use by a descriptor. 1726 * Called on close. 1727 */ 1728 static void 1729 bpf_freed(struct bpf_d *d) 1730 { 1731 /* 1732 * We don't need to lock out interrupts since this descriptor has 1733 * been detached from its interface and it yet hasn't been marked 1734 * free. 1735 */ 1736 if (d->bd_sbuf != NULL) { 1737 free(d->bd_sbuf, M_DEVBUF); 1738 if (d->bd_hbuf != NULL) 1739 free(d->bd_hbuf, M_DEVBUF); 1740 if (d->bd_fbuf != NULL) 1741 free(d->bd_fbuf, M_DEVBUF); 1742 } 1743 if (d->bd_filter) 1744 free(d->bd_filter, M_DEVBUF); 1745 1746 if (d->bd_jitcode != NULL) { 1747 bpf_jit_freecode(d->bd_jitcode); 1748 } 1749 } 1750 1751 /* 1752 * Attach an interface to bpf. dlt is the link layer type; 1753 * hdrlen is the fixed size of the link header for the specified dlt 1754 * (variable length headers not yet supported). 1755 */ 1756 static void 1757 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 1758 { 1759 struct bpf_if *bp; 1760 bp = malloc(sizeof(*bp), M_DEVBUF, M_DONTWAIT); 1761 if (bp == 0) 1762 panic("bpfattach"); 1763 1764 bp->bif_dlist = 0; 1765 bp->bif_driverp = driverp; 1766 bp->bif_ifp = ifp; 1767 bp->bif_dlt = dlt; 1768 1769 bp->bif_next = bpf_iflist; 1770 bpf_iflist = bp; 1771 1772 *bp->bif_driverp = 0; 1773 1774 bp->bif_hdrlen = hdrlen; 1775 #if 0 1776 printf("bpf: %s attached\n", ifp->if_xname); 1777 #endif 1778 } 1779 1780 /* 1781 * Remove an interface from bpf. 1782 */ 1783 static void 1784 _bpfdetach(struct ifnet *ifp) 1785 { 1786 struct bpf_if *bp, **pbp; 1787 struct bpf_d *d; 1788 int s; 1789 1790 /* Nuke the vnodes for any open instances */ 1791 LIST_FOREACH(d, &bpf_list, bd_list) { 1792 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 1793 /* 1794 * Detach the descriptor from an interface now. 1795 * It will be free'ed later by close routine. 1796 */ 1797 s = splnet(); 1798 d->bd_promisc = 0; /* we can't touch device. */ 1799 bpf_detachd(d); 1800 splx(s); 1801 } 1802 } 1803 1804 again: 1805 for (bp = bpf_iflist, pbp = &bpf_iflist; 1806 bp != NULL; pbp = &bp->bif_next, bp = bp->bif_next) { 1807 if (bp->bif_ifp == ifp) { 1808 *pbp = bp->bif_next; 1809 free(bp, M_DEVBUF); 1810 goto again; 1811 } 1812 } 1813 } 1814 1815 /* 1816 * Change the data link type of a interface. 1817 */ 1818 static void 1819 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 1820 { 1821 struct bpf_if *bp; 1822 1823 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1824 if (bp->bif_driverp == &ifp->if_bpf) 1825 break; 1826 } 1827 if (bp == NULL) 1828 panic("bpf_change_type"); 1829 1830 bp->bif_dlt = dlt; 1831 1832 bp->bif_hdrlen = hdrlen; 1833 } 1834 1835 /* 1836 * Get a list of available data link type of the interface. 1837 */ 1838 static int 1839 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 1840 { 1841 int n, error; 1842 struct ifnet *ifp; 1843 struct bpf_if *bp; 1844 1845 ifp = d->bd_bif->bif_ifp; 1846 n = 0; 1847 error = 0; 1848 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1849 if (bp->bif_ifp != ifp) 1850 continue; 1851 if (bfl->bfl_list != NULL) { 1852 if (n >= bfl->bfl_len) 1853 return ENOMEM; 1854 error = copyout(&bp->bif_dlt, 1855 bfl->bfl_list + n, sizeof(u_int)); 1856 } 1857 n++; 1858 } 1859 bfl->bfl_len = n; 1860 return error; 1861 } 1862 1863 /* 1864 * Set the data link type of a BPF instance. 1865 */ 1866 static int 1867 bpf_setdlt(struct bpf_d *d, u_int dlt) 1868 { 1869 int s, error, opromisc; 1870 struct ifnet *ifp; 1871 struct bpf_if *bp; 1872 1873 if (d->bd_bif->bif_dlt == dlt) 1874 return 0; 1875 ifp = d->bd_bif->bif_ifp; 1876 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) { 1877 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 1878 break; 1879 } 1880 if (bp == NULL) 1881 return EINVAL; 1882 s = splnet(); 1883 opromisc = d->bd_promisc; 1884 bpf_detachd(d); 1885 bpf_attachd(d, bp); 1886 reset_d(d); 1887 if (opromisc) { 1888 error = ifpromisc(bp->bif_ifp, 1); 1889 if (error) 1890 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 1891 bp->bif_ifp->if_xname, error); 1892 else 1893 d->bd_promisc = 1; 1894 } 1895 splx(s); 1896 return 0; 1897 } 1898 1899 static int 1900 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 1901 { 1902 int newsize, error; 1903 struct sysctlnode node; 1904 1905 node = *rnode; 1906 node.sysctl_data = &newsize; 1907 newsize = bpf_maxbufsize; 1908 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1909 if (error || newp == NULL) 1910 return (error); 1911 1912 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 1913 return (EINVAL); 1914 1915 bpf_maxbufsize = newsize; 1916 1917 return (0); 1918 } 1919 1920 static int 1921 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 1922 { 1923 bool newval; 1924 int error; 1925 struct sysctlnode node; 1926 1927 node = *rnode; 1928 node.sysctl_data = &newval; 1929 newval = bpf_jit; 1930 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1931 if (error != 0 || newp == NULL) 1932 return error; 1933 1934 bpf_jit = newval; 1935 1936 /* 1937 * Do a full sync to publish new bpf_jit value and 1938 * update bpfjit_module_ops.bj_generate_code variable. 1939 */ 1940 membar_sync(); 1941 1942 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 1943 printf("WARNING jit activation is postponed " 1944 "until after bpfjit module is loaded\n"); 1945 } 1946 1947 return 0; 1948 } 1949 1950 static int 1951 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 1952 { 1953 int error, elem_count; 1954 struct bpf_d *dp; 1955 struct bpf_d_ext dpe; 1956 size_t len, needed, elem_size, out_size; 1957 char *sp; 1958 1959 if (namelen == 1 && name[0] == CTL_QUERY) 1960 return (sysctl_query(SYSCTLFN_CALL(rnode))); 1961 1962 if (namelen != 2) 1963 return (EINVAL); 1964 1965 /* BPF peers is privileged information. */ 1966 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 1967 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 1968 if (error) 1969 return (EPERM); 1970 1971 len = (oldp != NULL) ? *oldlenp : 0; 1972 sp = oldp; 1973 elem_size = name[0]; 1974 elem_count = name[1]; 1975 out_size = MIN(sizeof(dpe), elem_size); 1976 needed = 0; 1977 1978 if (elem_size < 1 || elem_count < 0) 1979 return (EINVAL); 1980 1981 mutex_enter(&bpf_mtx); 1982 LIST_FOREACH(dp, &bpf_list, bd_list) { 1983 if (len >= elem_size && elem_count > 0) { 1984 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 1985 BPF_EXT(bufsize); 1986 BPF_EXT(promisc); 1987 BPF_EXT(state); 1988 BPF_EXT(immediate); 1989 BPF_EXT(hdrcmplt); 1990 BPF_EXT(seesent); 1991 BPF_EXT(pid); 1992 BPF_EXT(rcount); 1993 BPF_EXT(dcount); 1994 BPF_EXT(ccount); 1995 #undef BPF_EXT 1996 if (dp->bd_bif) 1997 (void)strlcpy(dpe.bde_ifname, 1998 dp->bd_bif->bif_ifp->if_xname, 1999 IFNAMSIZ - 1); 2000 else 2001 dpe.bde_ifname[0] = '\0'; 2002 2003 error = copyout(&dpe, sp, out_size); 2004 if (error) 2005 break; 2006 sp += elem_size; 2007 len -= elem_size; 2008 } 2009 needed += elem_size; 2010 if (elem_count > 0 && elem_count != INT_MAX) 2011 elem_count--; 2012 } 2013 mutex_exit(&bpf_mtx); 2014 2015 *oldlenp = needed; 2016 2017 return (error); 2018 } 2019 2020 static struct sysctllog *bpf_sysctllog; 2021 static void 2022 sysctl_net_bpf_setup(void) 2023 { 2024 const struct sysctlnode *node; 2025 2026 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2027 CTLFLAG_PERMANENT, 2028 CTLTYPE_NODE, "net", NULL, 2029 NULL, 0, NULL, 0, 2030 CTL_NET, CTL_EOL); 2031 2032 node = NULL; 2033 sysctl_createv(&bpf_sysctllog, 0, NULL, &node, 2034 CTLFLAG_PERMANENT, 2035 CTLTYPE_NODE, "bpf", 2036 SYSCTL_DESCR("BPF options"), 2037 NULL, 0, NULL, 0, 2038 CTL_NET, CTL_CREATE, CTL_EOL); 2039 if (node != NULL) { 2040 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2041 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2042 CTLTYPE_BOOL, "jit", 2043 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2044 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2045 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2046 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2047 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2048 CTLTYPE_INT, "maxbufsize", 2049 SYSCTL_DESCR("Maximum size for data capture buffer"), 2050 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2051 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2052 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2053 CTLFLAG_PERMANENT, 2054 CTLTYPE_STRUCT, "stats", 2055 SYSCTL_DESCR("BPF stats"), 2056 NULL, 0, &bpf_gstats, sizeof(bpf_gstats), 2057 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2058 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2059 CTLFLAG_PERMANENT, 2060 CTLTYPE_STRUCT, "peers", 2061 SYSCTL_DESCR("BPF peers"), 2062 sysctl_net_bpf_peers, 0, NULL, 0, 2063 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2064 } 2065 2066 } 2067 2068 struct bpf_ops bpf_ops_kernel = { 2069 .bpf_attach = _bpfattach, 2070 .bpf_detach = _bpfdetach, 2071 .bpf_change_type = _bpf_change_type, 2072 2073 .bpf_tap = _bpf_tap, 2074 .bpf_mtap = _bpf_mtap, 2075 .bpf_mtap2 = _bpf_mtap2, 2076 .bpf_mtap_af = _bpf_mtap_af, 2077 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2078 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2079 }; 2080 2081 MODULE(MODULE_CLASS_DRIVER, bpf, NULL); 2082 2083 static int 2084 bpf_modcmd(modcmd_t cmd, void *arg) 2085 { 2086 devmajor_t bmajor, cmajor; 2087 int error; 2088 2089 bmajor = cmajor = NODEVMAJOR; 2090 2091 switch (cmd) { 2092 case MODULE_CMD_INIT: 2093 bpfilterattach(0); 2094 error = devsw_attach("bpf", NULL, &bmajor, 2095 &bpf_cdevsw, &cmajor); 2096 if (error == EEXIST) 2097 error = 0; /* maybe built-in ... improve eventually */ 2098 if (error) 2099 break; 2100 2101 bpf_ops_handover_enter(&bpf_ops_kernel); 2102 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2103 bpf_ops_handover_exit(); 2104 sysctl_net_bpf_setup(); 2105 break; 2106 2107 case MODULE_CMD_FINI: 2108 /* 2109 * While there is no reference counting for bpf callers, 2110 * unload could at least in theory be done similarly to 2111 * system call disestablishment. This should even be 2112 * a little simpler: 2113 * 2114 * 1) replace op vector with stubs 2115 * 2) post update to all cpus with xc 2116 * 3) check that nobody is in bpf anymore 2117 * (it's doubtful we'd want something like l_sysent, 2118 * but we could do something like *signed* percpu 2119 * counters. if the sum is 0, we're good). 2120 * 4) if fail, unroll changes 2121 * 2122 * NOTE: change won't be atomic to the outside. some 2123 * packets may be not captured even if unload is 2124 * not succesful. I think packet capture not working 2125 * is a perfectly logical consequence of trying to 2126 * disable packet capture. 2127 */ 2128 error = EOPNOTSUPP; 2129 /* insert sysctl teardown */ 2130 break; 2131 2132 default: 2133 error = ENOTTY; 2134 break; 2135 } 2136 2137 return error; 2138 } 2139