1 /* $NetBSD: bpf.c,v 1.212 2017/02/01 08:18:33 ozaki-r Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.212 2017/02/01 08:18:33 ozaki-r Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "strip.h" 48 #include "opt_net_mpsafe.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/systm.h> 53 #include <sys/mbuf.h> 54 #include <sys/buf.h> 55 #include <sys/time.h> 56 #include <sys/proc.h> 57 #include <sys/ioctl.h> 58 #include <sys/conf.h> 59 #include <sys/vnode.h> 60 #include <sys/queue.h> 61 #include <sys/stat.h> 62 #include <sys/module.h> 63 #include <sys/atomic.h> 64 #include <sys/cpu.h> 65 66 #include <sys/file.h> 67 #include <sys/filedesc.h> 68 #include <sys/tty.h> 69 #include <sys/uio.h> 70 71 #include <sys/protosw.h> 72 #include <sys/socket.h> 73 #include <sys/errno.h> 74 #include <sys/kernel.h> 75 #include <sys/poll.h> 76 #include <sys/sysctl.h> 77 #include <sys/kauth.h> 78 #include <sys/syslog.h> 79 #include <sys/percpu.h> 80 81 #include <net/if.h> 82 #include <net/slip.h> 83 84 #include <net/bpf.h> 85 #include <net/bpfdesc.h> 86 #include <net/bpfjit.h> 87 88 #include <net/if_arc.h> 89 #include <net/if_ether.h> 90 91 #include <netinet/in.h> 92 #include <netinet/if_inarp.h> 93 94 95 #include <compat/sys/sockio.h> 96 97 #ifndef BPF_BUFSIZE 98 /* 99 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 100 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 101 */ 102 # define BPF_BUFSIZE 32768 103 #endif 104 105 #define PRINET 26 /* interruptible */ 106 107 /* 108 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 109 * XXX the default values should be computed dynamically based 110 * on available memory size and available mbuf clusters. 111 */ 112 static int bpf_bufsize = BPF_BUFSIZE; 113 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 114 static bool bpf_jit = false; 115 116 struct bpfjit_ops bpfjit_module_ops = { 117 .bj_generate_code = NULL, 118 .bj_free_code = NULL 119 }; 120 121 /* 122 * Global BPF statistics returned by net.bpf.stats sysctl. 123 */ 124 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */ 125 126 #define BPF_STATINC(id) \ 127 { \ 128 struct bpf_stat *__stats = \ 129 percpu_getref(bpf_gstats_percpu); \ 130 __stats->bs_##id++; \ 131 percpu_putref(bpf_gstats_percpu); \ 132 } 133 134 /* 135 * Use a mutex to avoid a race condition between gathering the stats/peers 136 * and opening/closing the device. 137 */ 138 static kmutex_t bpf_mtx; 139 140 /* 141 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 142 * bpf_dtab holds the descriptors, indexed by minor device # 143 */ 144 static struct pslist_head bpf_iflist; 145 static struct pslist_head bpf_dlist; 146 147 /* Macros for bpf_d on bpf_dlist */ 148 #define BPF_DLIST_WRITER_INSEART_HEAD(__d) \ 149 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) 150 #define BPF_DLIST_READER_FOREACH(__d) \ 151 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 152 bd_bpf_dlist_entry) 153 #define BPF_DLIST_WRITER_FOREACH(__d) \ 154 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 155 bd_bpf_dlist_entry) 156 #define BPF_DLIST_ENTRY_INIT(__d) \ 157 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) 158 #define BPF_DLIST_WRITER_REMOVE(__d) \ 159 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) 160 #define BPF_DLIST_ENTRY_DESTROY(__d) \ 161 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) 162 163 /* Macros for bpf_if on bpf_iflist */ 164 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ 165 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) 166 #define BPF_IFLIST_READER_FOREACH(__bp) \ 167 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 168 bif_iflist_entry) 169 #define BPF_IFLIST_WRITER_FOREACH(__bp) \ 170 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 171 bif_iflist_entry) 172 #define BPF_IFLIST_WRITER_REMOVE(__bp) \ 173 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) 174 #define BPF_IFLIST_ENTRY_INIT(__bp) \ 175 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) 176 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ 177 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) 178 179 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ 180 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ 181 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ 182 bd_bif_dlist_entry) 183 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ 184 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ 185 bd_bif_dlist_entry) 186 #define BPFIF_DLIST_WRITER_REMOVE(__d) \ 187 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) 188 #define BPFIF_DLIST_ENTRY_INIT(__d) \ 189 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) 190 #define BPFIF_DLIST_READER_EMPTY(__bp) \ 191 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 192 bd_bif_dlist_entry) == NULL) 193 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ 194 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 195 bd_bif_dlist_entry) == NULL) 196 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ 197 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) 198 199 static int bpf_allocbufs(struct bpf_d *); 200 static void bpf_deliver(struct bpf_if *, 201 void *(*cpfn)(void *, const void *, size_t), 202 void *, u_int, u_int, const bool); 203 static void bpf_freed(struct bpf_d *); 204 static void bpf_ifname(struct ifnet *, struct ifreq *); 205 static void *bpf_mcpy(void *, const void *, size_t); 206 static int bpf_movein(struct uio *, int, uint64_t, 207 struct mbuf **, struct sockaddr *); 208 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 209 static void bpf_detachd(struct bpf_d *); 210 static int bpf_setif(struct bpf_d *, struct ifreq *); 211 static int bpf_setf(struct bpf_d *, struct bpf_program *); 212 static void bpf_timed_out(void *); 213 static inline void 214 bpf_wakeup(struct bpf_d *); 215 static int bpf_hdrlen(struct bpf_d *); 216 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 217 void *(*)(void *, const void *, size_t), struct timespec *); 218 static void reset_d(struct bpf_d *); 219 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 220 static int bpf_setdlt(struct bpf_d *, u_int); 221 222 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 223 int); 224 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 225 int); 226 static int bpf_ioctl(struct file *, u_long, void *); 227 static int bpf_poll(struct file *, int); 228 static int bpf_stat(struct file *, struct stat *); 229 static int bpf_close(struct file *); 230 static int bpf_kqfilter(struct file *, struct knote *); 231 static void bpf_softintr(void *); 232 233 static const struct fileops bpf_fileops = { 234 .fo_read = bpf_read, 235 .fo_write = bpf_write, 236 .fo_ioctl = bpf_ioctl, 237 .fo_fcntl = fnullop_fcntl, 238 .fo_poll = bpf_poll, 239 .fo_stat = bpf_stat, 240 .fo_close = bpf_close, 241 .fo_kqfilter = bpf_kqfilter, 242 .fo_restart = fnullop_restart, 243 }; 244 245 dev_type_open(bpfopen); 246 247 const struct cdevsw bpf_cdevsw = { 248 .d_open = bpfopen, 249 .d_close = noclose, 250 .d_read = noread, 251 .d_write = nowrite, 252 .d_ioctl = noioctl, 253 .d_stop = nostop, 254 .d_tty = notty, 255 .d_poll = nopoll, 256 .d_mmap = nommap, 257 .d_kqfilter = nokqfilter, 258 .d_discard = nodiscard, 259 .d_flag = D_OTHER 260 }; 261 262 bpfjit_func_t 263 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 264 { 265 266 membar_consumer(); 267 if (bpfjit_module_ops.bj_generate_code != NULL) { 268 return bpfjit_module_ops.bj_generate_code(bc, code, size); 269 } 270 return NULL; 271 } 272 273 void 274 bpf_jit_freecode(bpfjit_func_t jcode) 275 { 276 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 277 bpfjit_module_ops.bj_free_code(jcode); 278 } 279 280 static int 281 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 282 struct sockaddr *sockp) 283 { 284 struct mbuf *m; 285 int error; 286 size_t len; 287 size_t hlen; 288 size_t align; 289 290 /* 291 * Build a sockaddr based on the data link layer type. 292 * We do this at this level because the ethernet header 293 * is copied directly into the data field of the sockaddr. 294 * In the case of SLIP, there is no header and the packet 295 * is forwarded as is. 296 * Also, we are careful to leave room at the front of the mbuf 297 * for the link level header. 298 */ 299 switch (linktype) { 300 301 case DLT_SLIP: 302 sockp->sa_family = AF_INET; 303 hlen = 0; 304 align = 0; 305 break; 306 307 case DLT_PPP: 308 sockp->sa_family = AF_UNSPEC; 309 hlen = 0; 310 align = 0; 311 break; 312 313 case DLT_EN10MB: 314 sockp->sa_family = AF_UNSPEC; 315 /* XXX Would MAXLINKHDR be better? */ 316 /* 6(dst)+6(src)+2(type) */ 317 hlen = sizeof(struct ether_header); 318 align = 2; 319 break; 320 321 case DLT_ARCNET: 322 sockp->sa_family = AF_UNSPEC; 323 hlen = ARC_HDRLEN; 324 align = 5; 325 break; 326 327 case DLT_FDDI: 328 sockp->sa_family = AF_LINK; 329 /* XXX 4(FORMAC)+6(dst)+6(src) */ 330 hlen = 16; 331 align = 0; 332 break; 333 334 case DLT_ECONET: 335 sockp->sa_family = AF_UNSPEC; 336 hlen = 6; 337 align = 2; 338 break; 339 340 case DLT_NULL: 341 sockp->sa_family = AF_UNSPEC; 342 hlen = 0; 343 align = 0; 344 break; 345 346 default: 347 return (EIO); 348 } 349 350 len = uio->uio_resid; 351 /* 352 * If there aren't enough bytes for a link level header or the 353 * packet length exceeds the interface mtu, return an error. 354 */ 355 if (len - hlen > mtu) 356 return (EMSGSIZE); 357 358 /* 359 * XXX Avoid complicated buffer chaining --- 360 * bail if it won't fit in a single mbuf. 361 * (Take into account possible alignment bytes) 362 */ 363 if (len + align > MCLBYTES) 364 return (EIO); 365 366 m = m_gethdr(M_WAIT, MT_DATA); 367 m_reset_rcvif(m); 368 m->m_pkthdr.len = (int)(len - hlen); 369 if (len + align > MHLEN) { 370 m_clget(m, M_WAIT); 371 if ((m->m_flags & M_EXT) == 0) { 372 error = ENOBUFS; 373 goto bad; 374 } 375 } 376 377 /* Insure the data is properly aligned */ 378 if (align > 0) { 379 m->m_data += align; 380 m->m_len -= (int)align; 381 } 382 383 error = uiomove(mtod(m, void *), len, uio); 384 if (error) 385 goto bad; 386 if (hlen != 0) { 387 memcpy(sockp->sa_data, mtod(m, void *), hlen); 388 m->m_data += hlen; /* XXX */ 389 len -= hlen; 390 } 391 m->m_len = (int)len; 392 *mp = m; 393 return (0); 394 395 bad: 396 m_freem(m); 397 return (error); 398 } 399 400 /* 401 * Attach file to the bpf interface, i.e. make d listen on bp. 402 * Must be called at splnet. 403 */ 404 static void 405 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 406 { 407 KASSERT(mutex_owned(&bpf_mtx)); 408 /* 409 * Point d at bp, and add d to the interface's list of listeners. 410 * Finally, point the driver's bpf cookie at the interface so 411 * it will divert packets to bpf. 412 */ 413 d->bd_bif = bp; 414 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); 415 416 *bp->bif_driverp = bp; 417 } 418 419 /* 420 * Detach a file from its interface. 421 */ 422 static void 423 bpf_detachd(struct bpf_d *d) 424 { 425 struct bpf_if *bp; 426 427 KASSERT(mutex_owned(&bpf_mtx)); 428 429 bp = d->bd_bif; 430 /* 431 * Check if this descriptor had requested promiscuous mode. 432 * If so, turn it off. 433 */ 434 if (d->bd_promisc) { 435 int error __diagused; 436 437 d->bd_promisc = 0; 438 /* 439 * Take device out of promiscuous mode. Since we were 440 * able to enter promiscuous mode, we should be able 441 * to turn it off. But we can get an error if 442 * the interface was configured down, so only panic 443 * if we don't get an unexpected error. 444 */ 445 error = ifpromisc(bp->bif_ifp, 0); 446 #ifdef DIAGNOSTIC 447 if (error) 448 printf("%s: ifpromisc failed: %d", __func__, error); 449 #endif 450 } 451 452 /* Remove d from the interface's descriptor list. */ 453 BPFIF_DLIST_WRITER_REMOVE(d); 454 455 /* TODO pserialize_perform(); */ 456 /* TODO psref_target_destroy(); */ 457 BPFIF_DLIST_ENTRY_DESTROY(d); 458 459 /* XXX NOMPSAFE? */ 460 if (BPFIF_DLIST_WRITER_EMPTY(bp)) { 461 /* 462 * Let the driver know that there are no more listeners. 463 */ 464 *d->bd_bif->bif_driverp = NULL; 465 } 466 d->bd_bif = NULL; 467 } 468 469 static void 470 bpf_init(void) 471 { 472 473 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 474 475 PSLIST_INIT(&bpf_iflist); 476 PSLIST_INIT(&bpf_dlist); 477 478 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat)); 479 480 return; 481 } 482 483 /* 484 * bpfilterattach() is called at boot time. We don't need to do anything 485 * here, since any initialization will happen as part of module init code. 486 */ 487 /* ARGSUSED */ 488 void 489 bpfilterattach(int n) 490 { 491 492 } 493 494 /* 495 * Open ethernet device. Clones. 496 */ 497 /* ARGSUSED */ 498 int 499 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 500 { 501 struct bpf_d *d; 502 struct file *fp; 503 int error, fd; 504 505 /* falloc() will fill in the descriptor for us. */ 506 if ((error = fd_allocfile(&fp, &fd)) != 0) 507 return error; 508 509 d = kmem_zalloc(sizeof(*d), KM_SLEEP); 510 d->bd_bufsize = bpf_bufsize; 511 d->bd_seesent = 1; 512 d->bd_feedback = 0; 513 d->bd_pid = l->l_proc->p_pid; 514 #ifdef _LP64 515 if (curproc->p_flag & PK_32) 516 d->bd_compat32 = 1; 517 #endif 518 getnanotime(&d->bd_btime); 519 d->bd_atime = d->bd_mtime = d->bd_btime; 520 callout_init(&d->bd_callout, 0); 521 selinit(&d->bd_sel); 522 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d); 523 d->bd_jitcode = NULL; 524 BPF_DLIST_ENTRY_INIT(d); 525 BPFIF_DLIST_ENTRY_INIT(d); 526 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 527 cv_init(&d->bd_cv, "bpf"); 528 529 mutex_enter(&bpf_mtx); 530 BPF_DLIST_WRITER_INSEART_HEAD(d); 531 mutex_exit(&bpf_mtx); 532 533 return fd_clone(fp, fd, flag, &bpf_fileops, d); 534 } 535 536 /* 537 * Close the descriptor by detaching it from its interface, 538 * deallocating its buffers, and marking it free. 539 */ 540 /* ARGSUSED */ 541 static int 542 bpf_close(struct file *fp) 543 { 544 struct bpf_d *d; 545 int s; 546 547 KERNEL_LOCK(1, NULL); 548 mutex_enter(&bpf_mtx); 549 550 if ((d = fp->f_bpf) == NULL) { 551 mutex_exit(&bpf_mtx); 552 KERNEL_UNLOCK_ONE(NULL); 553 return 0; 554 } 555 556 /* 557 * Refresh the PID associated with this bpf file. 558 */ 559 d->bd_pid = curproc->p_pid; 560 561 s = splnet(); 562 if (d->bd_state == BPF_WAITING) 563 callout_stop(&d->bd_callout); 564 d->bd_state = BPF_IDLE; 565 if (d->bd_bif) 566 bpf_detachd(d); 567 splx(s); 568 bpf_freed(d); 569 BPF_DLIST_WRITER_REMOVE(d); 570 fp->f_bpf = NULL; 571 572 mutex_exit(&bpf_mtx); 573 KERNEL_UNLOCK_ONE(NULL); 574 575 /* TODO pserialize_perform(); */ 576 /* TODO psref_target_destroy(); */ 577 BPF_DLIST_ENTRY_DESTROY(d); 578 579 callout_destroy(&d->bd_callout); 580 seldestroy(&d->bd_sel); 581 softint_disestablish(d->bd_sih); 582 mutex_obj_free(d->bd_mtx); 583 cv_destroy(&d->bd_cv); 584 585 kmem_free(d, sizeof(*d)); 586 587 return (0); 588 } 589 590 /* 591 * Rotate the packet buffers in descriptor d. Move the store buffer 592 * into the hold slot, and the free buffer into the store slot. 593 * Zero the length of the new store buffer. 594 */ 595 #define ROTATE_BUFFERS(d) \ 596 (d)->bd_hbuf = (d)->bd_sbuf; \ 597 (d)->bd_hlen = (d)->bd_slen; \ 598 (d)->bd_sbuf = (d)->bd_fbuf; \ 599 (d)->bd_slen = 0; \ 600 (d)->bd_fbuf = NULL; 601 /* 602 * bpfread - read next chunk of packets from buffers 603 */ 604 static int 605 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 606 kauth_cred_t cred, int flags) 607 { 608 struct bpf_d *d = fp->f_bpf; 609 int timed_out; 610 int error; 611 int s; 612 613 getnanotime(&d->bd_atime); 614 /* 615 * Restrict application to use a buffer the same size as 616 * the kernel buffers. 617 */ 618 if (uio->uio_resid != d->bd_bufsize) 619 return (EINVAL); 620 621 KERNEL_LOCK(1, NULL); 622 s = splnet(); 623 if (d->bd_state == BPF_WAITING) 624 callout_stop(&d->bd_callout); 625 timed_out = (d->bd_state == BPF_TIMED_OUT); 626 d->bd_state = BPF_IDLE; 627 /* 628 * If the hold buffer is empty, then do a timed sleep, which 629 * ends when the timeout expires or when enough packets 630 * have arrived to fill the store buffer. 631 */ 632 while (d->bd_hbuf == NULL) { 633 if (fp->f_flag & FNONBLOCK) { 634 if (d->bd_slen == 0) { 635 error = EWOULDBLOCK; 636 goto out; 637 } 638 ROTATE_BUFFERS(d); 639 break; 640 } 641 642 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 643 /* 644 * A packet(s) either arrived since the previous 645 * read or arrived while we were asleep. 646 * Rotate the buffers and return what's here. 647 */ 648 ROTATE_BUFFERS(d); 649 break; 650 } 651 652 mutex_enter(d->bd_mtx); 653 error = cv_timedwait_sig(&d->bd_cv, d->bd_mtx, d->bd_rtout); 654 mutex_exit(d->bd_mtx); 655 656 if (error == EINTR || error == ERESTART) 657 goto out; 658 659 if (error == EWOULDBLOCK) { 660 /* 661 * On a timeout, return what's in the buffer, 662 * which may be nothing. If there is something 663 * in the store buffer, we can rotate the buffers. 664 */ 665 if (d->bd_hbuf) 666 /* 667 * We filled up the buffer in between 668 * getting the timeout and arriving 669 * here, so we don't need to rotate. 670 */ 671 break; 672 673 if (d->bd_slen == 0) { 674 error = 0; 675 goto out; 676 } 677 ROTATE_BUFFERS(d); 678 break; 679 } 680 if (error != 0) 681 goto out; 682 } 683 /* 684 * At this point, we know we have something in the hold slot. 685 */ 686 splx(s); 687 688 /* 689 * Move data from hold buffer into user space. 690 * We know the entire buffer is transferred since 691 * we checked above that the read buffer is bpf_bufsize bytes. 692 */ 693 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 694 695 s = splnet(); 696 d->bd_fbuf = d->bd_hbuf; 697 d->bd_hbuf = NULL; 698 d->bd_hlen = 0; 699 out: 700 splx(s); 701 KERNEL_UNLOCK_ONE(NULL); 702 return (error); 703 } 704 705 706 /* 707 * If there are processes sleeping on this descriptor, wake them up. 708 */ 709 static inline void 710 bpf_wakeup(struct bpf_d *d) 711 { 712 713 mutex_enter(d->bd_mtx); 714 cv_broadcast(&d->bd_cv); 715 mutex_exit(d->bd_mtx); 716 717 if (d->bd_async) 718 softint_schedule(d->bd_sih); 719 selnotify(&d->bd_sel, 0, 0); 720 } 721 722 static void 723 bpf_softintr(void *cookie) 724 { 725 struct bpf_d *d; 726 727 d = cookie; 728 if (d->bd_async) 729 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 730 } 731 732 static void 733 bpf_timed_out(void *arg) 734 { 735 struct bpf_d *d = arg; 736 int s; 737 738 s = splnet(); 739 if (d->bd_state == BPF_WAITING) { 740 d->bd_state = BPF_TIMED_OUT; 741 if (d->bd_slen != 0) 742 bpf_wakeup(d); 743 } 744 splx(s); 745 } 746 747 748 static int 749 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 750 kauth_cred_t cred, int flags) 751 { 752 struct bpf_d *d = fp->f_bpf; 753 struct ifnet *ifp; 754 struct mbuf *m, *mc; 755 int error, s; 756 static struct sockaddr_storage dst; 757 758 m = NULL; /* XXX gcc */ 759 760 KERNEL_LOCK(1, NULL); 761 762 if (d->bd_bif == NULL) { 763 KERNEL_UNLOCK_ONE(NULL); 764 return (ENXIO); 765 } 766 getnanotime(&d->bd_mtime); 767 768 ifp = d->bd_bif->bif_ifp; 769 770 if (uio->uio_resid == 0) { 771 KERNEL_UNLOCK_ONE(NULL); 772 return (0); 773 } 774 775 error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, ifp->if_mtu, &m, 776 (struct sockaddr *) &dst); 777 if (error) { 778 KERNEL_UNLOCK_ONE(NULL); 779 return (error); 780 } 781 782 if (m->m_pkthdr.len > ifp->if_mtu) { 783 KERNEL_UNLOCK_ONE(NULL); 784 m_freem(m); 785 return (EMSGSIZE); 786 } 787 788 if (d->bd_hdrcmplt) 789 dst.ss_family = pseudo_AF_HDRCMPLT; 790 791 if (d->bd_feedback) { 792 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 793 if (mc != NULL) 794 m_set_rcvif(mc, ifp); 795 /* Set M_PROMISC for outgoing packets to be discarded. */ 796 if (1 /*d->bd_direction == BPF_D_INOUT*/) 797 m->m_flags |= M_PROMISC; 798 } else 799 mc = NULL; 800 801 s = splsoftnet(); 802 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); 803 804 if (mc != NULL) { 805 if (error == 0) 806 ifp->_if_input(ifp, mc); 807 else 808 m_freem(mc); 809 } 810 splx(s); 811 KERNEL_UNLOCK_ONE(NULL); 812 /* 813 * The driver frees the mbuf. 814 */ 815 return (error); 816 } 817 818 /* 819 * Reset a descriptor by flushing its packet buffer and clearing the 820 * receive and drop counts. Should be called at splnet. 821 */ 822 static void 823 reset_d(struct bpf_d *d) 824 { 825 if (d->bd_hbuf) { 826 /* Free the hold buffer. */ 827 d->bd_fbuf = d->bd_hbuf; 828 d->bd_hbuf = NULL; 829 } 830 d->bd_slen = 0; 831 d->bd_hlen = 0; 832 d->bd_rcount = 0; 833 d->bd_dcount = 0; 834 d->bd_ccount = 0; 835 } 836 837 /* 838 * FIONREAD Check for read packet available. 839 * BIOCGBLEN Get buffer len [for read()]. 840 * BIOCSETF Set ethernet read filter. 841 * BIOCFLUSH Flush read packet buffer. 842 * BIOCPROMISC Put interface into promiscuous mode. 843 * BIOCGDLT Get link layer type. 844 * BIOCGETIF Get interface name. 845 * BIOCSETIF Set interface. 846 * BIOCSRTIMEOUT Set read timeout. 847 * BIOCGRTIMEOUT Get read timeout. 848 * BIOCGSTATS Get packet stats. 849 * BIOCIMMEDIATE Set immediate mode. 850 * BIOCVERSION Get filter language version. 851 * BIOCGHDRCMPLT Get "header already complete" flag. 852 * BIOCSHDRCMPLT Set "header already complete" flag. 853 * BIOCSFEEDBACK Set packet feedback mode. 854 * BIOCGFEEDBACK Get packet feedback mode. 855 * BIOCGSEESENT Get "see sent packets" mode. 856 * BIOCSSEESENT Set "see sent packets" mode. 857 */ 858 /* ARGSUSED */ 859 static int 860 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 861 { 862 struct bpf_d *d = fp->f_bpf; 863 int s, error = 0; 864 865 /* 866 * Refresh the PID associated with this bpf file. 867 */ 868 KERNEL_LOCK(1, NULL); 869 d->bd_pid = curproc->p_pid; 870 #ifdef _LP64 871 if (curproc->p_flag & PK_32) 872 d->bd_compat32 = 1; 873 else 874 d->bd_compat32 = 0; 875 #endif 876 877 s = splnet(); 878 if (d->bd_state == BPF_WAITING) 879 callout_stop(&d->bd_callout); 880 d->bd_state = BPF_IDLE; 881 splx(s); 882 883 switch (cmd) { 884 885 default: 886 error = EINVAL; 887 break; 888 889 /* 890 * Check for read packet available. 891 */ 892 case FIONREAD: 893 { 894 int n; 895 896 s = splnet(); 897 n = d->bd_slen; 898 if (d->bd_hbuf) 899 n += d->bd_hlen; 900 splx(s); 901 902 *(int *)addr = n; 903 break; 904 } 905 906 /* 907 * Get buffer len [for read()]. 908 */ 909 case BIOCGBLEN: 910 *(u_int *)addr = d->bd_bufsize; 911 break; 912 913 /* 914 * Set buffer length. 915 */ 916 case BIOCSBLEN: 917 /* 918 * Forbid to change the buffer length if buffers are already 919 * allocated. 920 */ 921 if (d->bd_bif != NULL || d->bd_sbuf != NULL) 922 error = EINVAL; 923 else { 924 u_int size = *(u_int *)addr; 925 926 if (size > bpf_maxbufsize) 927 *(u_int *)addr = size = bpf_maxbufsize; 928 else if (size < BPF_MINBUFSIZE) 929 *(u_int *)addr = size = BPF_MINBUFSIZE; 930 d->bd_bufsize = size; 931 } 932 break; 933 934 /* 935 * Set link layer read filter. 936 */ 937 case BIOCSETF: 938 error = bpf_setf(d, addr); 939 break; 940 941 /* 942 * Flush read packet buffer. 943 */ 944 case BIOCFLUSH: 945 s = splnet(); 946 reset_d(d); 947 splx(s); 948 break; 949 950 /* 951 * Put interface into promiscuous mode. 952 */ 953 case BIOCPROMISC: 954 if (d->bd_bif == NULL) { 955 /* 956 * No interface attached yet. 957 */ 958 error = EINVAL; 959 break; 960 } 961 s = splnet(); 962 if (d->bd_promisc == 0) { 963 error = ifpromisc(d->bd_bif->bif_ifp, 1); 964 if (error == 0) 965 d->bd_promisc = 1; 966 } 967 splx(s); 968 break; 969 970 /* 971 * Get device parameters. 972 */ 973 case BIOCGDLT: 974 if (d->bd_bif == NULL) 975 error = EINVAL; 976 else 977 *(u_int *)addr = d->bd_bif->bif_dlt; 978 break; 979 980 /* 981 * Get a list of supported device parameters. 982 */ 983 case BIOCGDLTLIST: 984 if (d->bd_bif == NULL) 985 error = EINVAL; 986 else 987 error = bpf_getdltlist(d, addr); 988 break; 989 990 /* 991 * Set device parameters. 992 */ 993 case BIOCSDLT: 994 mutex_enter(&bpf_mtx); 995 if (d->bd_bif == NULL) 996 error = EINVAL; 997 else 998 error = bpf_setdlt(d, *(u_int *)addr); 999 mutex_exit(&bpf_mtx); 1000 break; 1001 1002 /* 1003 * Set interface name. 1004 */ 1005 #ifdef OBIOCGETIF 1006 case OBIOCGETIF: 1007 #endif 1008 case BIOCGETIF: 1009 if (d->bd_bif == NULL) 1010 error = EINVAL; 1011 else 1012 bpf_ifname(d->bd_bif->bif_ifp, addr); 1013 break; 1014 1015 /* 1016 * Set interface. 1017 */ 1018 #ifdef OBIOCSETIF 1019 case OBIOCSETIF: 1020 #endif 1021 case BIOCSETIF: 1022 mutex_enter(&bpf_mtx); 1023 error = bpf_setif(d, addr); 1024 mutex_exit(&bpf_mtx); 1025 break; 1026 1027 /* 1028 * Set read timeout. 1029 */ 1030 case BIOCSRTIMEOUT: 1031 { 1032 struct timeval *tv = addr; 1033 1034 /* Compute number of ticks. */ 1035 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1036 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1037 d->bd_rtout = 1; 1038 break; 1039 } 1040 1041 #ifdef BIOCGORTIMEOUT 1042 /* 1043 * Get read timeout. 1044 */ 1045 case BIOCGORTIMEOUT: 1046 { 1047 struct timeval50 *tv = addr; 1048 1049 tv->tv_sec = d->bd_rtout / hz; 1050 tv->tv_usec = (d->bd_rtout % hz) * tick; 1051 break; 1052 } 1053 #endif 1054 1055 #ifdef BIOCSORTIMEOUT 1056 /* 1057 * Set read timeout. 1058 */ 1059 case BIOCSORTIMEOUT: 1060 { 1061 struct timeval50 *tv = addr; 1062 1063 /* Compute number of ticks. */ 1064 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1065 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1066 d->bd_rtout = 1; 1067 break; 1068 } 1069 #endif 1070 1071 /* 1072 * Get read timeout. 1073 */ 1074 case BIOCGRTIMEOUT: 1075 { 1076 struct timeval *tv = addr; 1077 1078 tv->tv_sec = d->bd_rtout / hz; 1079 tv->tv_usec = (d->bd_rtout % hz) * tick; 1080 break; 1081 } 1082 /* 1083 * Get packet stats. 1084 */ 1085 case BIOCGSTATS: 1086 { 1087 struct bpf_stat *bs = addr; 1088 1089 bs->bs_recv = d->bd_rcount; 1090 bs->bs_drop = d->bd_dcount; 1091 bs->bs_capt = d->bd_ccount; 1092 break; 1093 } 1094 1095 case BIOCGSTATSOLD: 1096 { 1097 struct bpf_stat_old *bs = addr; 1098 1099 bs->bs_recv = d->bd_rcount; 1100 bs->bs_drop = d->bd_dcount; 1101 break; 1102 } 1103 1104 /* 1105 * Set immediate mode. 1106 */ 1107 case BIOCIMMEDIATE: 1108 d->bd_immediate = *(u_int *)addr; 1109 break; 1110 1111 case BIOCVERSION: 1112 { 1113 struct bpf_version *bv = addr; 1114 1115 bv->bv_major = BPF_MAJOR_VERSION; 1116 bv->bv_minor = BPF_MINOR_VERSION; 1117 break; 1118 } 1119 1120 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1121 *(u_int *)addr = d->bd_hdrcmplt; 1122 break; 1123 1124 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1125 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1126 break; 1127 1128 /* 1129 * Get "see sent packets" flag 1130 */ 1131 case BIOCGSEESENT: 1132 *(u_int *)addr = d->bd_seesent; 1133 break; 1134 1135 /* 1136 * Set "see sent" packets flag 1137 */ 1138 case BIOCSSEESENT: 1139 d->bd_seesent = *(u_int *)addr; 1140 break; 1141 1142 /* 1143 * Set "feed packets from bpf back to input" mode 1144 */ 1145 case BIOCSFEEDBACK: 1146 d->bd_feedback = *(u_int *)addr; 1147 break; 1148 1149 /* 1150 * Get "feed packets from bpf back to input" mode 1151 */ 1152 case BIOCGFEEDBACK: 1153 *(u_int *)addr = d->bd_feedback; 1154 break; 1155 1156 case FIONBIO: /* Non-blocking I/O */ 1157 /* 1158 * No need to do anything special as we use IO_NDELAY in 1159 * bpfread() as an indication of whether or not to block 1160 * the read. 1161 */ 1162 break; 1163 1164 case FIOASYNC: /* Send signal on receive packets */ 1165 d->bd_async = *(int *)addr; 1166 break; 1167 1168 case TIOCSPGRP: /* Process or group to send signals to */ 1169 case FIOSETOWN: 1170 error = fsetown(&d->bd_pgid, cmd, addr); 1171 break; 1172 1173 case TIOCGPGRP: 1174 case FIOGETOWN: 1175 error = fgetown(d->bd_pgid, cmd, addr); 1176 break; 1177 } 1178 KERNEL_UNLOCK_ONE(NULL); 1179 return (error); 1180 } 1181 1182 /* 1183 * Set d's packet filter program to fp. If this file already has a filter, 1184 * free it and replace it. Returns EINVAL for bogus requests. 1185 */ 1186 static int 1187 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1188 { 1189 struct bpf_insn *fcode, *old; 1190 bpfjit_func_t jcode, oldj; 1191 size_t flen, size = 0, old_size; 1192 int s; 1193 1194 jcode = NULL; 1195 flen = fp->bf_len; 1196 1197 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1198 return EINVAL; 1199 } 1200 1201 if (flen) { 1202 /* 1203 * Allocate the buffer, copy the byte-code from 1204 * userspace and validate it. 1205 */ 1206 size = flen * sizeof(*fp->bf_insns); 1207 fcode = kmem_alloc(size, KM_SLEEP); 1208 if (copyin(fp->bf_insns, fcode, size) != 0 || 1209 !bpf_validate(fcode, (int)flen)) { 1210 kmem_free(fcode, size); 1211 return EINVAL; 1212 } 1213 membar_consumer(); 1214 if (bpf_jit) 1215 jcode = bpf_jit_generate(NULL, fcode, flen); 1216 } else { 1217 fcode = NULL; 1218 } 1219 1220 old_size = d->bd_filter_size; 1221 1222 s = splnet(); 1223 old = d->bd_filter; 1224 d->bd_filter = fcode; 1225 d->bd_filter_size = size; 1226 oldj = d->bd_jitcode; 1227 d->bd_jitcode = jcode; 1228 reset_d(d); 1229 splx(s); 1230 1231 if (old) { 1232 kmem_free(old, old_size); 1233 } 1234 if (oldj) { 1235 bpf_jit_freecode(oldj); 1236 } 1237 1238 return 0; 1239 } 1240 1241 /* 1242 * Detach a file from its current interface (if attached at all) and attach 1243 * to the interface indicated by the name stored in ifr. 1244 * Return an errno or 0. 1245 */ 1246 static int 1247 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1248 { 1249 struct bpf_if *bp; 1250 char *cp; 1251 int unit_seen, i, s, error; 1252 1253 KASSERT(mutex_owned(&bpf_mtx)); 1254 /* 1255 * Make sure the provided name has a unit number, and default 1256 * it to '0' if not specified. 1257 * XXX This is ugly ... do this differently? 1258 */ 1259 unit_seen = 0; 1260 cp = ifr->ifr_name; 1261 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1262 while (*cp++) 1263 if (*cp >= '0' && *cp <= '9') 1264 unit_seen = 1; 1265 if (!unit_seen) { 1266 /* Make sure to leave room for the '\0'. */ 1267 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1268 if ((ifr->ifr_name[i] >= 'a' && 1269 ifr->ifr_name[i] <= 'z') || 1270 (ifr->ifr_name[i] >= 'A' && 1271 ifr->ifr_name[i] <= 'Z')) 1272 continue; 1273 ifr->ifr_name[i] = '0'; 1274 } 1275 } 1276 1277 /* 1278 * Look through attached interfaces for the named one. 1279 */ 1280 BPF_IFLIST_WRITER_FOREACH(bp) { 1281 struct ifnet *ifp = bp->bif_ifp; 1282 1283 if (ifp == NULL || 1284 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1285 continue; 1286 /* skip additional entry */ 1287 if (bp->bif_driverp != &ifp->if_bpf) 1288 continue; 1289 /* 1290 * We found the requested interface. 1291 * Allocate the packet buffers if we need to. 1292 * If we're already attached to requested interface, 1293 * just flush the buffer. 1294 */ 1295 if (d->bd_sbuf == NULL) { 1296 error = bpf_allocbufs(d); 1297 if (error != 0) 1298 return (error); 1299 } 1300 s = splnet(); 1301 if (bp != d->bd_bif) { 1302 if (d->bd_bif) 1303 /* 1304 * Detach if attached to something else. 1305 */ 1306 bpf_detachd(d); 1307 1308 bpf_attachd(d, bp); 1309 } 1310 reset_d(d); 1311 splx(s); 1312 return (0); 1313 } 1314 /* Not found. */ 1315 return (ENXIO); 1316 } 1317 1318 /* 1319 * Copy the interface name to the ifreq. 1320 */ 1321 static void 1322 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1323 { 1324 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1325 } 1326 1327 static int 1328 bpf_stat(struct file *fp, struct stat *st) 1329 { 1330 struct bpf_d *d = fp->f_bpf; 1331 1332 (void)memset(st, 0, sizeof(*st)); 1333 KERNEL_LOCK(1, NULL); 1334 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1335 st->st_atimespec = d->bd_atime; 1336 st->st_mtimespec = d->bd_mtime; 1337 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1338 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1339 st->st_gid = kauth_cred_getegid(fp->f_cred); 1340 st->st_mode = S_IFCHR; 1341 KERNEL_UNLOCK_ONE(NULL); 1342 return 0; 1343 } 1344 1345 /* 1346 * Support for poll() system call 1347 * 1348 * Return true iff the specific operation will not block indefinitely - with 1349 * the assumption that it is safe to positively acknowledge a request for the 1350 * ability to write to the BPF device. 1351 * Otherwise, return false but make a note that a selnotify() must be done. 1352 */ 1353 static int 1354 bpf_poll(struct file *fp, int events) 1355 { 1356 struct bpf_d *d = fp->f_bpf; 1357 int s = splnet(); 1358 int revents; 1359 1360 /* 1361 * Refresh the PID associated with this bpf file. 1362 */ 1363 KERNEL_LOCK(1, NULL); 1364 d->bd_pid = curproc->p_pid; 1365 1366 revents = events & (POLLOUT | POLLWRNORM); 1367 if (events & (POLLIN | POLLRDNORM)) { 1368 /* 1369 * An imitation of the FIONREAD ioctl code. 1370 */ 1371 if (d->bd_hlen != 0 || 1372 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1373 d->bd_slen != 0)) { 1374 revents |= events & (POLLIN | POLLRDNORM); 1375 } else { 1376 selrecord(curlwp, &d->bd_sel); 1377 /* Start the read timeout if necessary */ 1378 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1379 callout_reset(&d->bd_callout, d->bd_rtout, 1380 bpf_timed_out, d); 1381 d->bd_state = BPF_WAITING; 1382 } 1383 } 1384 } 1385 1386 KERNEL_UNLOCK_ONE(NULL); 1387 splx(s); 1388 return (revents); 1389 } 1390 1391 static void 1392 filt_bpfrdetach(struct knote *kn) 1393 { 1394 struct bpf_d *d = kn->kn_hook; 1395 int s; 1396 1397 KERNEL_LOCK(1, NULL); 1398 s = splnet(); 1399 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1400 splx(s); 1401 KERNEL_UNLOCK_ONE(NULL); 1402 } 1403 1404 static int 1405 filt_bpfread(struct knote *kn, long hint) 1406 { 1407 struct bpf_d *d = kn->kn_hook; 1408 int rv; 1409 1410 KERNEL_LOCK(1, NULL); 1411 kn->kn_data = d->bd_hlen; 1412 if (d->bd_immediate) 1413 kn->kn_data += d->bd_slen; 1414 rv = (kn->kn_data > 0); 1415 KERNEL_UNLOCK_ONE(NULL); 1416 return rv; 1417 } 1418 1419 static const struct filterops bpfread_filtops = 1420 { 1, NULL, filt_bpfrdetach, filt_bpfread }; 1421 1422 static int 1423 bpf_kqfilter(struct file *fp, struct knote *kn) 1424 { 1425 struct bpf_d *d = fp->f_bpf; 1426 struct klist *klist; 1427 int s; 1428 1429 KERNEL_LOCK(1, NULL); 1430 1431 switch (kn->kn_filter) { 1432 case EVFILT_READ: 1433 klist = &d->bd_sel.sel_klist; 1434 kn->kn_fop = &bpfread_filtops; 1435 break; 1436 1437 default: 1438 KERNEL_UNLOCK_ONE(NULL); 1439 return (EINVAL); 1440 } 1441 1442 kn->kn_hook = d; 1443 1444 s = splnet(); 1445 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1446 splx(s); 1447 KERNEL_UNLOCK_ONE(NULL); 1448 1449 return (0); 1450 } 1451 1452 /* 1453 * Copy data from an mbuf chain into a buffer. This code is derived 1454 * from m_copydata in sys/uipc_mbuf.c. 1455 */ 1456 static void * 1457 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1458 { 1459 const struct mbuf *m; 1460 u_int count; 1461 u_char *dst; 1462 1463 m = src_arg; 1464 dst = dst_arg; 1465 while (len > 0) { 1466 if (m == NULL) 1467 panic("bpf_mcpy"); 1468 count = min(m->m_len, len); 1469 memcpy(dst, mtod(m, const void *), count); 1470 m = m->m_next; 1471 dst += count; 1472 len -= count; 1473 } 1474 return dst_arg; 1475 } 1476 1477 /* 1478 * Dispatch a packet to all the listeners on interface bp. 1479 * 1480 * pkt pointer to the packet, either a data buffer or an mbuf chain 1481 * buflen buffer length, if pkt is a data buffer 1482 * cpfn a function that can copy pkt into the listener's buffer 1483 * pktlen length of the packet 1484 * rcv true if packet came in 1485 */ 1486 static inline void 1487 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1488 void *pkt, u_int pktlen, u_int buflen, const bool rcv) 1489 { 1490 uint32_t mem[BPF_MEMWORDS]; 1491 bpf_args_t args = { 1492 .pkt = (const uint8_t *)pkt, 1493 .wirelen = pktlen, 1494 .buflen = buflen, 1495 .mem = mem, 1496 .arg = NULL 1497 }; 1498 bool gottime = false; 1499 struct timespec ts; 1500 struct bpf_d *d; 1501 1502 /* 1503 * Note that the IPL does not have to be raised at this point. 1504 * The only problem that could arise here is that if two different 1505 * interfaces shared any data. This is not the case. 1506 */ 1507 BPFIF_DLIST_READER_FOREACH(d, bp) { 1508 u_int slen; 1509 1510 if (!d->bd_seesent && !rcv) { 1511 continue; 1512 } 1513 d->bd_rcount++; 1514 BPF_STATINC(recv); 1515 1516 if (d->bd_jitcode) 1517 slen = d->bd_jitcode(NULL, &args); 1518 else 1519 slen = bpf_filter_ext(NULL, d->bd_filter, &args); 1520 1521 if (!slen) { 1522 continue; 1523 } 1524 if (!gottime) { 1525 gottime = true; 1526 nanotime(&ts); 1527 } 1528 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1529 } 1530 } 1531 1532 /* 1533 * Incoming linkage from device drivers. Process the packet pkt, of length 1534 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1535 * by each process' filter, and if accepted, stashed into the corresponding 1536 * buffer. 1537 */ 1538 static void 1539 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1540 { 1541 1542 bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true); 1543 } 1544 1545 /* 1546 * Incoming linkage from device drivers, when the head of the packet is in 1547 * a buffer, and the tail is in an mbuf chain. 1548 */ 1549 static void 1550 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1551 { 1552 u_int pktlen; 1553 struct mbuf mb; 1554 1555 /* Skip outgoing duplicate packets. */ 1556 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1557 m->m_flags &= ~M_PROMISC; 1558 return; 1559 } 1560 1561 pktlen = m_length(m) + dlen; 1562 1563 /* 1564 * Craft on-stack mbuf suitable for passing to bpf_filter. 1565 * Note that we cut corners here; we only setup what's 1566 * absolutely needed--this mbuf should never go anywhere else. 1567 */ 1568 (void)memset(&mb, 0, sizeof(mb)); 1569 mb.m_next = m; 1570 mb.m_data = data; 1571 mb.m_len = dlen; 1572 1573 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif_index != 0); 1574 } 1575 1576 /* 1577 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1578 */ 1579 static void 1580 _bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1581 { 1582 void *(*cpfn)(void *, const void *, size_t); 1583 u_int pktlen, buflen; 1584 void *marg; 1585 1586 /* Skip outgoing duplicate packets. */ 1587 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1588 m->m_flags &= ~M_PROMISC; 1589 return; 1590 } 1591 1592 pktlen = m_length(m); 1593 1594 if (pktlen == m->m_len) { 1595 cpfn = (void *)memcpy; 1596 marg = mtod(m, void *); 1597 buflen = pktlen; 1598 } else { 1599 cpfn = bpf_mcpy; 1600 marg = m; 1601 buflen = 0; 1602 } 1603 1604 bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif_index != 0); 1605 } 1606 1607 /* 1608 * We need to prepend the address family as 1609 * a four byte field. Cons up a dummy header 1610 * to pacify bpf. This is safe because bpf 1611 * will only read from the mbuf (i.e., it won't 1612 * try to free it or keep a pointer a to it). 1613 */ 1614 static void 1615 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m) 1616 { 1617 struct mbuf m0; 1618 1619 m0.m_flags = 0; 1620 m0.m_next = m; 1621 m0.m_len = 4; 1622 m0.m_data = (char *)⁡ 1623 1624 _bpf_mtap(bp, &m0); 1625 } 1626 1627 /* 1628 * Put the SLIP pseudo-"link header" in place. 1629 * Note this M_PREPEND() should never fail, 1630 * swince we know we always have enough space 1631 * in the input buffer. 1632 */ 1633 static void 1634 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1635 { 1636 int s; 1637 u_char *hp; 1638 1639 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1640 if (*m == NULL) 1641 return; 1642 1643 hp = mtod(*m, u_char *); 1644 hp[SLX_DIR] = SLIPDIR_IN; 1645 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1646 1647 s = splnet(); 1648 _bpf_mtap(bp, *m); 1649 splx(s); 1650 1651 m_adj(*m, SLIP_HDRLEN); 1652 } 1653 1654 /* 1655 * Put the SLIP pseudo-"link header" in 1656 * place. The compressed header is now 1657 * at the beginning of the mbuf. 1658 */ 1659 static void 1660 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1661 { 1662 struct mbuf m0; 1663 u_char *hp; 1664 int s; 1665 1666 m0.m_flags = 0; 1667 m0.m_next = m; 1668 m0.m_data = m0.m_dat; 1669 m0.m_len = SLIP_HDRLEN; 1670 1671 hp = mtod(&m0, u_char *); 1672 1673 hp[SLX_DIR] = SLIPDIR_OUT; 1674 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1675 1676 s = splnet(); 1677 _bpf_mtap(bp, &m0); 1678 splx(s); 1679 m_freem(m); 1680 } 1681 1682 static struct mbuf * 1683 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) 1684 { 1685 struct mbuf *dup; 1686 1687 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); 1688 if (dup == NULL) 1689 return NULL; 1690 1691 if (bp->bif_mbuf_tail != NULL) { 1692 bp->bif_mbuf_tail->m_nextpkt = dup; 1693 } else { 1694 bp->bif_mbuf_head = dup; 1695 } 1696 bp->bif_mbuf_tail = dup; 1697 #ifdef BPF_MTAP_SOFTINT_DEBUG 1698 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", 1699 __func__, dup, bp->bif_ifp->if_xname); 1700 #endif 1701 1702 return dup; 1703 } 1704 1705 static struct mbuf * 1706 bpf_mbuf_dequeue(struct bpf_if *bp) 1707 { 1708 struct mbuf *m; 1709 int s; 1710 1711 s = splnet(); 1712 m = bp->bif_mbuf_head; 1713 if (m != NULL) { 1714 bp->bif_mbuf_head = m->m_nextpkt; 1715 m->m_nextpkt = NULL; 1716 1717 if (bp->bif_mbuf_head == NULL) 1718 bp->bif_mbuf_tail = NULL; 1719 #ifdef BPF_MTAP_SOFTINT_DEBUG 1720 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", 1721 __func__, m, bp->bif_ifp->if_xname); 1722 #endif 1723 } 1724 splx(s); 1725 1726 return m; 1727 } 1728 1729 static void 1730 bpf_mtap_si(void *arg) 1731 { 1732 struct bpf_if *bp = arg; 1733 struct mbuf *m; 1734 1735 while ((m = bpf_mbuf_dequeue(bp)) != NULL) { 1736 #ifdef BPF_MTAP_SOFTINT_DEBUG 1737 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", 1738 __func__, m, bp->bif_ifp->if_xname); 1739 #endif 1740 #ifndef NET_MPSAFE 1741 KERNEL_LOCK(1, NULL); 1742 #endif 1743 bpf_ops->bpf_mtap(bp, m); 1744 #ifndef NET_MPSAFE 1745 KERNEL_UNLOCK_ONE(NULL); 1746 #endif 1747 m_freem(m); 1748 } 1749 } 1750 1751 static void 1752 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) 1753 { 1754 struct bpf_if *bp = ifp->if_bpf; 1755 struct mbuf *dup; 1756 1757 KASSERT(cpu_intr_p()); 1758 1759 /* To avoid extra invocations of the softint */ 1760 if (BPFIF_DLIST_READER_EMPTY(bp)) 1761 return; 1762 KASSERT(bp->bif_si != NULL); 1763 1764 dup = bpf_mbuf_enqueue(bp, m); 1765 if (dup != NULL) 1766 softint_schedule(bp->bif_si); 1767 } 1768 1769 static int 1770 bpf_hdrlen(struct bpf_d *d) 1771 { 1772 int hdrlen = d->bd_bif->bif_hdrlen; 1773 /* 1774 * Compute the length of the bpf header. This is not necessarily 1775 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1776 * that the network layer header begins on a longword boundary (for 1777 * performance reasons and to alleviate alignment restrictions). 1778 */ 1779 #ifdef _LP64 1780 if (d->bd_compat32) 1781 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1782 else 1783 #endif 1784 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1785 } 1786 1787 /* 1788 * Move the packet data from interface memory (pkt) into the 1789 * store buffer. Call the wakeup functions if it's time to wakeup 1790 * a listener (buffer full), "cpfn" is the routine called to do the 1791 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1792 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1793 * pkt is really an mbuf. 1794 */ 1795 static void 1796 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1797 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1798 { 1799 char *h; 1800 int totlen, curlen, caplen; 1801 int hdrlen = bpf_hdrlen(d); 1802 int do_wakeup = 0; 1803 1804 ++d->bd_ccount; 1805 BPF_STATINC(capt); 1806 /* 1807 * Figure out how many bytes to move. If the packet is 1808 * greater or equal to the snapshot length, transfer that 1809 * much. Otherwise, transfer the whole packet (unless 1810 * we hit the buffer size limit). 1811 */ 1812 totlen = hdrlen + min(snaplen, pktlen); 1813 if (totlen > d->bd_bufsize) 1814 totlen = d->bd_bufsize; 1815 /* 1816 * If we adjusted totlen to fit the bufsize, it could be that 1817 * totlen is smaller than hdrlen because of the link layer header. 1818 */ 1819 caplen = totlen - hdrlen; 1820 if (caplen < 0) 1821 caplen = 0; 1822 1823 /* 1824 * Round up the end of the previous packet to the next longword. 1825 */ 1826 #ifdef _LP64 1827 if (d->bd_compat32) 1828 curlen = BPF_WORDALIGN32(d->bd_slen); 1829 else 1830 #endif 1831 curlen = BPF_WORDALIGN(d->bd_slen); 1832 if (curlen + totlen > d->bd_bufsize) { 1833 /* 1834 * This packet will overflow the storage buffer. 1835 * Rotate the buffers if we can, then wakeup any 1836 * pending reads. 1837 */ 1838 if (d->bd_fbuf == NULL) { 1839 /* 1840 * We haven't completed the previous read yet, 1841 * so drop the packet. 1842 */ 1843 ++d->bd_dcount; 1844 BPF_STATINC(drop); 1845 return; 1846 } 1847 ROTATE_BUFFERS(d); 1848 do_wakeup = 1; 1849 curlen = 0; 1850 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1851 /* 1852 * Immediate mode is set, or the read timeout has 1853 * already expired during a select call. A packet 1854 * arrived, so the reader should be woken up. 1855 */ 1856 do_wakeup = 1; 1857 } 1858 1859 /* 1860 * Append the bpf header. 1861 */ 1862 h = (char *)d->bd_sbuf + curlen; 1863 #ifdef _LP64 1864 if (d->bd_compat32) { 1865 struct bpf_hdr32 *hp32; 1866 1867 hp32 = (struct bpf_hdr32 *)h; 1868 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1869 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1870 hp32->bh_datalen = pktlen; 1871 hp32->bh_hdrlen = hdrlen; 1872 hp32->bh_caplen = caplen; 1873 } else 1874 #endif 1875 { 1876 struct bpf_hdr *hp; 1877 1878 hp = (struct bpf_hdr *)h; 1879 hp->bh_tstamp.tv_sec = ts->tv_sec; 1880 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1881 hp->bh_datalen = pktlen; 1882 hp->bh_hdrlen = hdrlen; 1883 hp->bh_caplen = caplen; 1884 } 1885 1886 /* 1887 * Copy the packet data into the store buffer and update its length. 1888 */ 1889 (*cpfn)(h + hdrlen, pkt, caplen); 1890 d->bd_slen = curlen + totlen; 1891 1892 /* 1893 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1894 * will cause filt_bpfread() to be called with it adjusted. 1895 */ 1896 if (do_wakeup) 1897 bpf_wakeup(d); 1898 } 1899 1900 /* 1901 * Initialize all nonzero fields of a descriptor. 1902 */ 1903 static int 1904 bpf_allocbufs(struct bpf_d *d) 1905 { 1906 1907 d->bd_fbuf = kmem_alloc(d->bd_bufsize, KM_NOSLEEP); 1908 if (!d->bd_fbuf) 1909 return (ENOBUFS); 1910 d->bd_sbuf = kmem_alloc(d->bd_bufsize, KM_NOSLEEP); 1911 if (!d->bd_sbuf) { 1912 kmem_free(d->bd_fbuf, d->bd_bufsize); 1913 return (ENOBUFS); 1914 } 1915 d->bd_slen = 0; 1916 d->bd_hlen = 0; 1917 return (0); 1918 } 1919 1920 /* 1921 * Free buffers currently in use by a descriptor. 1922 * Called on close. 1923 */ 1924 static void 1925 bpf_freed(struct bpf_d *d) 1926 { 1927 /* 1928 * We don't need to lock out interrupts since this descriptor has 1929 * been detached from its interface and it yet hasn't been marked 1930 * free. 1931 */ 1932 if (d->bd_sbuf != NULL) { 1933 kmem_free(d->bd_sbuf, d->bd_bufsize); 1934 if (d->bd_hbuf != NULL) 1935 kmem_free(d->bd_hbuf, d->bd_bufsize); 1936 if (d->bd_fbuf != NULL) 1937 kmem_free(d->bd_fbuf, d->bd_bufsize); 1938 } 1939 if (d->bd_filter) 1940 kmem_free(d->bd_filter, d->bd_filter_size); 1941 1942 if (d->bd_jitcode != NULL) { 1943 bpf_jit_freecode(d->bd_jitcode); 1944 } 1945 } 1946 1947 /* 1948 * Attach an interface to bpf. dlt is the link layer type; 1949 * hdrlen is the fixed size of the link header for the specified dlt 1950 * (variable length headers not yet supported). 1951 */ 1952 static void 1953 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 1954 { 1955 struct bpf_if *bp; 1956 bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP); 1957 if (bp == NULL) 1958 panic("bpfattach"); 1959 1960 mutex_enter(&bpf_mtx); 1961 bp->bif_driverp = driverp; 1962 bp->bif_ifp = ifp; 1963 bp->bif_dlt = dlt; 1964 bp->bif_si = NULL; 1965 BPF_IFLIST_ENTRY_INIT(bp); 1966 PSLIST_INIT(&bp->bif_dlist_head); 1967 1968 BPF_IFLIST_WRITER_INSERT_HEAD(bp); 1969 1970 *bp->bif_driverp = NULL; 1971 1972 bp->bif_hdrlen = hdrlen; 1973 mutex_exit(&bpf_mtx); 1974 #if 0 1975 printf("bpf: %s attached\n", ifp->if_xname); 1976 #endif 1977 } 1978 1979 static void 1980 _bpf_mtap_softint_init(struct ifnet *ifp) 1981 { 1982 struct bpf_if *bp; 1983 1984 mutex_enter(&bpf_mtx); 1985 BPF_IFLIST_WRITER_FOREACH(bp) { 1986 if (bp->bif_ifp != ifp) 1987 continue; 1988 1989 bp->bif_mbuf_head = NULL; 1990 bp->bif_mbuf_tail = NULL; 1991 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); 1992 if (bp->bif_si == NULL) 1993 panic("%s: softint_establish() failed", __func__); 1994 break; 1995 } 1996 mutex_exit(&bpf_mtx); 1997 1998 if (bp == NULL) 1999 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); 2000 } 2001 2002 /* 2003 * Remove an interface from bpf. 2004 */ 2005 static void 2006 _bpfdetach(struct ifnet *ifp) 2007 { 2008 struct bpf_if *bp; 2009 struct bpf_d *d; 2010 int s; 2011 2012 mutex_enter(&bpf_mtx); 2013 /* Nuke the vnodes for any open instances */ 2014 again_d: 2015 BPF_DLIST_WRITER_FOREACH(d) { 2016 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 2017 /* 2018 * Detach the descriptor from an interface now. 2019 * It will be free'ed later by close routine. 2020 */ 2021 s = splnet(); 2022 d->bd_promisc = 0; /* we can't touch device. */ 2023 bpf_detachd(d); 2024 splx(s); 2025 goto again_d; 2026 } 2027 } 2028 2029 again: 2030 BPF_IFLIST_WRITER_FOREACH(bp) { 2031 if (bp->bif_ifp == ifp) { 2032 BPF_IFLIST_WRITER_REMOVE(bp); 2033 /* TODO pserialize_perform(); */ 2034 /* TODO psref_target_destroy(); */ 2035 BPF_IFLIST_ENTRY_DESTROY(bp); 2036 if (bp->bif_si != NULL) { 2037 s = splnet(); 2038 while (bp->bif_mbuf_head != NULL) { 2039 struct mbuf *m = bp->bif_mbuf_head; 2040 bp->bif_mbuf_head = m->m_nextpkt; 2041 m_freem(m); 2042 } 2043 splx(s); 2044 softint_disestablish(bp->bif_si); 2045 } 2046 kmem_free(bp, sizeof(*bp)); 2047 goto again; 2048 } 2049 } 2050 mutex_exit(&bpf_mtx); 2051 } 2052 2053 /* 2054 * Change the data link type of a interface. 2055 */ 2056 static void 2057 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2058 { 2059 struct bpf_if *bp; 2060 2061 BPF_IFLIST_READER_FOREACH(bp) { 2062 if (bp->bif_driverp == &ifp->if_bpf) 2063 break; 2064 } 2065 if (bp == NULL) 2066 panic("bpf_change_type"); 2067 2068 bp->bif_dlt = dlt; 2069 2070 bp->bif_hdrlen = hdrlen; 2071 } 2072 2073 /* 2074 * Get a list of available data link type of the interface. 2075 */ 2076 static int 2077 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2078 { 2079 int n, error; 2080 struct ifnet *ifp; 2081 struct bpf_if *bp; 2082 2083 ifp = d->bd_bif->bif_ifp; 2084 n = 0; 2085 error = 0; 2086 BPF_IFLIST_READER_FOREACH(bp) { 2087 if (bp->bif_ifp != ifp) 2088 continue; 2089 if (bfl->bfl_list != NULL) { 2090 if (n >= bfl->bfl_len) 2091 return ENOMEM; 2092 error = copyout(&bp->bif_dlt, 2093 bfl->bfl_list + n, sizeof(u_int)); 2094 } 2095 n++; 2096 } 2097 bfl->bfl_len = n; 2098 return error; 2099 } 2100 2101 /* 2102 * Set the data link type of a BPF instance. 2103 */ 2104 static int 2105 bpf_setdlt(struct bpf_d *d, u_int dlt) 2106 { 2107 int s, error, opromisc; 2108 struct ifnet *ifp; 2109 struct bpf_if *bp; 2110 2111 KASSERT(mutex_owned(&bpf_mtx)); 2112 2113 if (d->bd_bif->bif_dlt == dlt) 2114 return 0; 2115 ifp = d->bd_bif->bif_ifp; 2116 BPF_IFLIST_WRITER_FOREACH(bp) { 2117 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2118 break; 2119 } 2120 if (bp == NULL) 2121 return EINVAL; 2122 s = splnet(); 2123 opromisc = d->bd_promisc; 2124 bpf_detachd(d); 2125 bpf_attachd(d, bp); 2126 reset_d(d); 2127 if (opromisc) { 2128 error = ifpromisc(bp->bif_ifp, 1); 2129 if (error) 2130 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 2131 bp->bif_ifp->if_xname, error); 2132 else 2133 d->bd_promisc = 1; 2134 } 2135 splx(s); 2136 return 0; 2137 } 2138 2139 static int 2140 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 2141 { 2142 int newsize, error; 2143 struct sysctlnode node; 2144 2145 node = *rnode; 2146 node.sysctl_data = &newsize; 2147 newsize = bpf_maxbufsize; 2148 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2149 if (error || newp == NULL) 2150 return (error); 2151 2152 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 2153 return (EINVAL); 2154 2155 bpf_maxbufsize = newsize; 2156 2157 return (0); 2158 } 2159 2160 #if defined(MODULAR) || defined(BPFJIT) 2161 static int 2162 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 2163 { 2164 bool newval; 2165 int error; 2166 struct sysctlnode node; 2167 2168 node = *rnode; 2169 node.sysctl_data = &newval; 2170 newval = bpf_jit; 2171 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2172 if (error != 0 || newp == NULL) 2173 return error; 2174 2175 bpf_jit = newval; 2176 2177 /* 2178 * Do a full sync to publish new bpf_jit value and 2179 * update bpfjit_module_ops.bj_generate_code variable. 2180 */ 2181 membar_sync(); 2182 2183 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 2184 printf("JIT compilation is postponed " 2185 "until after bpfjit module is loaded\n"); 2186 } 2187 2188 return 0; 2189 } 2190 #endif 2191 2192 static int 2193 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 2194 { 2195 int error, elem_count; 2196 struct bpf_d *dp; 2197 struct bpf_d_ext dpe; 2198 size_t len, needed, elem_size, out_size; 2199 char *sp; 2200 2201 if (namelen == 1 && name[0] == CTL_QUERY) 2202 return (sysctl_query(SYSCTLFN_CALL(rnode))); 2203 2204 if (namelen != 2) 2205 return (EINVAL); 2206 2207 /* BPF peers is privileged information. */ 2208 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 2209 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 2210 if (error) 2211 return (EPERM); 2212 2213 len = (oldp != NULL) ? *oldlenp : 0; 2214 sp = oldp; 2215 elem_size = name[0]; 2216 elem_count = name[1]; 2217 out_size = MIN(sizeof(dpe), elem_size); 2218 needed = 0; 2219 2220 if (elem_size < 1 || elem_count < 0) 2221 return (EINVAL); 2222 2223 mutex_enter(&bpf_mtx); 2224 BPF_DLIST_WRITER_FOREACH(dp) { 2225 if (len >= elem_size && elem_count > 0) { 2226 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 2227 BPF_EXT(bufsize); 2228 BPF_EXT(promisc); 2229 BPF_EXT(state); 2230 BPF_EXT(immediate); 2231 BPF_EXT(hdrcmplt); 2232 BPF_EXT(seesent); 2233 BPF_EXT(pid); 2234 BPF_EXT(rcount); 2235 BPF_EXT(dcount); 2236 BPF_EXT(ccount); 2237 #undef BPF_EXT 2238 if (dp->bd_bif) 2239 (void)strlcpy(dpe.bde_ifname, 2240 dp->bd_bif->bif_ifp->if_xname, 2241 IFNAMSIZ - 1); 2242 else 2243 dpe.bde_ifname[0] = '\0'; 2244 2245 error = copyout(&dpe, sp, out_size); 2246 if (error) 2247 break; 2248 sp += elem_size; 2249 len -= elem_size; 2250 } 2251 needed += elem_size; 2252 if (elem_count > 0 && elem_count != INT_MAX) 2253 elem_count--; 2254 } 2255 mutex_exit(&bpf_mtx); 2256 2257 *oldlenp = needed; 2258 2259 return (error); 2260 } 2261 2262 static void 2263 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused) 2264 { 2265 struct bpf_stat *const stats = p; 2266 struct bpf_stat *sum = arg; 2267 2268 sum->bs_recv += stats->bs_recv; 2269 sum->bs_drop += stats->bs_drop; 2270 sum->bs_capt += stats->bs_capt; 2271 } 2272 2273 static int 2274 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS) 2275 { 2276 struct sysctlnode node; 2277 int error; 2278 struct bpf_stat sum; 2279 2280 memset(&sum, 0, sizeof(sum)); 2281 node = *rnode; 2282 2283 percpu_foreach(bpf_gstats_percpu, bpf_stats, &sum); 2284 2285 node.sysctl_data = ∑ 2286 node.sysctl_size = sizeof(sum); 2287 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2288 if (error != 0 || newp == NULL) 2289 return error; 2290 2291 return 0; 2292 } 2293 2294 static struct sysctllog *bpf_sysctllog; 2295 static void 2296 sysctl_net_bpf_setup(void) 2297 { 2298 const struct sysctlnode *node; 2299 2300 node = NULL; 2301 sysctl_createv(&bpf_sysctllog, 0, NULL, &node, 2302 CTLFLAG_PERMANENT, 2303 CTLTYPE_NODE, "bpf", 2304 SYSCTL_DESCR("BPF options"), 2305 NULL, 0, NULL, 0, 2306 CTL_NET, CTL_CREATE, CTL_EOL); 2307 if (node != NULL) { 2308 #if defined(MODULAR) || defined(BPFJIT) 2309 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2310 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2311 CTLTYPE_BOOL, "jit", 2312 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2313 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2314 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2315 #endif 2316 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2317 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2318 CTLTYPE_INT, "maxbufsize", 2319 SYSCTL_DESCR("Maximum size for data capture buffer"), 2320 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2321 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2322 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2323 CTLFLAG_PERMANENT, 2324 CTLTYPE_STRUCT, "stats", 2325 SYSCTL_DESCR("BPF stats"), 2326 bpf_sysctl_gstats_handler, 0, NULL, 0, 2327 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2328 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2329 CTLFLAG_PERMANENT, 2330 CTLTYPE_STRUCT, "peers", 2331 SYSCTL_DESCR("BPF peers"), 2332 sysctl_net_bpf_peers, 0, NULL, 0, 2333 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2334 } 2335 2336 } 2337 2338 struct bpf_ops bpf_ops_kernel = { 2339 .bpf_attach = _bpfattach, 2340 .bpf_detach = _bpfdetach, 2341 .bpf_change_type = _bpf_change_type, 2342 2343 .bpf_tap = _bpf_tap, 2344 .bpf_mtap = _bpf_mtap, 2345 .bpf_mtap2 = _bpf_mtap2, 2346 .bpf_mtap_af = _bpf_mtap_af, 2347 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2348 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2349 2350 .bpf_mtap_softint = _bpf_mtap_softint, 2351 .bpf_mtap_softint_init = _bpf_mtap_softint_init, 2352 }; 2353 2354 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); 2355 2356 static int 2357 bpf_modcmd(modcmd_t cmd, void *arg) 2358 { 2359 #ifdef _MODULE 2360 devmajor_t bmajor, cmajor; 2361 #endif 2362 int error = 0; 2363 2364 switch (cmd) { 2365 case MODULE_CMD_INIT: 2366 bpf_init(); 2367 #ifdef _MODULE 2368 bmajor = cmajor = NODEVMAJOR; 2369 error = devsw_attach("bpf", NULL, &bmajor, 2370 &bpf_cdevsw, &cmajor); 2371 if (error) 2372 break; 2373 #endif 2374 2375 bpf_ops_handover_enter(&bpf_ops_kernel); 2376 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2377 bpf_ops_handover_exit(); 2378 sysctl_net_bpf_setup(); 2379 break; 2380 2381 case MODULE_CMD_FINI: 2382 /* 2383 * While there is no reference counting for bpf callers, 2384 * unload could at least in theory be done similarly to 2385 * system call disestablishment. This should even be 2386 * a little simpler: 2387 * 2388 * 1) replace op vector with stubs 2389 * 2) post update to all cpus with xc 2390 * 3) check that nobody is in bpf anymore 2391 * (it's doubtful we'd want something like l_sysent, 2392 * but we could do something like *signed* percpu 2393 * counters. if the sum is 0, we're good). 2394 * 4) if fail, unroll changes 2395 * 2396 * NOTE: change won't be atomic to the outside. some 2397 * packets may be not captured even if unload is 2398 * not succesful. I think packet capture not working 2399 * is a perfectly logical consequence of trying to 2400 * disable packet capture. 2401 */ 2402 error = EOPNOTSUPP; 2403 /* insert sysctl teardown */ 2404 break; 2405 2406 default: 2407 error = ENOTTY; 2408 break; 2409 } 2410 2411 return error; 2412 } 2413