1 /* $NetBSD: bpf.c,v 1.222 2017/12/15 07:29:11 ozaki-r Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.222 2017/12/15 07:29:11 ozaki-r Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "strip.h" 48 #include "opt_net_mpsafe.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/systm.h> 53 #include <sys/mbuf.h> 54 #include <sys/buf.h> 55 #include <sys/time.h> 56 #include <sys/proc.h> 57 #include <sys/ioctl.h> 58 #include <sys/conf.h> 59 #include <sys/vnode.h> 60 #include <sys/queue.h> 61 #include <sys/stat.h> 62 #include <sys/module.h> 63 #include <sys/atomic.h> 64 #include <sys/cpu.h> 65 66 #include <sys/file.h> 67 #include <sys/filedesc.h> 68 #include <sys/tty.h> 69 #include <sys/uio.h> 70 71 #include <sys/protosw.h> 72 #include <sys/socket.h> 73 #include <sys/errno.h> 74 #include <sys/kernel.h> 75 #include <sys/poll.h> 76 #include <sys/sysctl.h> 77 #include <sys/kauth.h> 78 #include <sys/syslog.h> 79 #include <sys/percpu.h> 80 #include <sys/pserialize.h> 81 #include <sys/lwp.h> 82 83 #include <net/if.h> 84 #include <net/slip.h> 85 86 #include <net/bpf.h> 87 #include <net/bpfdesc.h> 88 #include <net/bpfjit.h> 89 90 #include <net/if_arc.h> 91 #include <net/if_ether.h> 92 93 #include <netinet/in.h> 94 #include <netinet/if_inarp.h> 95 96 97 #include <compat/sys/sockio.h> 98 99 #ifndef BPF_BUFSIZE 100 /* 101 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 102 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 103 */ 104 # define BPF_BUFSIZE 32768 105 #endif 106 107 #define PRINET 26 /* interruptible */ 108 109 /* 110 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 111 * XXX the default values should be computed dynamically based 112 * on available memory size and available mbuf clusters. 113 */ 114 static int bpf_bufsize = BPF_BUFSIZE; 115 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 116 static bool bpf_jit = false; 117 118 struct bpfjit_ops bpfjit_module_ops = { 119 .bj_generate_code = NULL, 120 .bj_free_code = NULL 121 }; 122 123 /* 124 * Global BPF statistics returned by net.bpf.stats sysctl. 125 */ 126 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */ 127 128 #define BPF_STATINC(id) \ 129 { \ 130 struct bpf_stat *__stats = \ 131 percpu_getref(bpf_gstats_percpu); \ 132 __stats->bs_##id++; \ 133 percpu_putref(bpf_gstats_percpu); \ 134 } 135 136 /* 137 * Locking notes: 138 * - bpf_mtx (adaptive mutex) protects: 139 * - Gobal lists: bpf_iflist and bpf_dlist 140 * - struct bpf_if 141 * - bpf_close 142 * - bpf_psz (pserialize) 143 * - struct bpf_d has two mutexes: 144 * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed 145 * on packet tapping 146 * - bd_mtx (adaptive mutex) protects member variables other than the buffers 147 * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx 148 * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is 149 * never freed because struct bpf_d is only freed in bpf_close and 150 * bpf_close never be called while executing bpf_read and bpf_write 151 * - A filter that is assigned to bpf_d can be replaced with another filter 152 * while tapping packets, so it needs to be done atomically 153 * - struct bpf_d is iterated on bpf_dlist with psz 154 * - struct bpf_if is iterated on bpf_iflist with psz or psref 155 */ 156 /* 157 * Use a mutex to avoid a race condition between gathering the stats/peers 158 * and opening/closing the device. 159 */ 160 static kmutex_t bpf_mtx; 161 162 static struct psref_class *bpf_psref_class __read_mostly; 163 static pserialize_t bpf_psz; 164 165 static inline void 166 bpf_if_acquire(struct bpf_if *bp, struct psref *psref) 167 { 168 169 psref_acquire(psref, &bp->bif_psref, bpf_psref_class); 170 } 171 172 static inline void 173 bpf_if_release(struct bpf_if *bp, struct psref *psref) 174 { 175 176 psref_release(psref, &bp->bif_psref, bpf_psref_class); 177 } 178 179 /* 180 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 181 * bpf_dtab holds the descriptors, indexed by minor device # 182 */ 183 static struct pslist_head bpf_iflist; 184 static struct pslist_head bpf_dlist; 185 186 /* Macros for bpf_d on bpf_dlist */ 187 #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \ 188 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) 189 #define BPF_DLIST_READER_FOREACH(__d) \ 190 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 191 bd_bpf_dlist_entry) 192 #define BPF_DLIST_WRITER_FOREACH(__d) \ 193 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 194 bd_bpf_dlist_entry) 195 #define BPF_DLIST_ENTRY_INIT(__d) \ 196 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) 197 #define BPF_DLIST_WRITER_REMOVE(__d) \ 198 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) 199 #define BPF_DLIST_ENTRY_DESTROY(__d) \ 200 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) 201 202 /* Macros for bpf_if on bpf_iflist */ 203 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ 204 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) 205 #define BPF_IFLIST_READER_FOREACH(__bp) \ 206 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 207 bif_iflist_entry) 208 #define BPF_IFLIST_WRITER_FOREACH(__bp) \ 209 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 210 bif_iflist_entry) 211 #define BPF_IFLIST_WRITER_REMOVE(__bp) \ 212 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) 213 #define BPF_IFLIST_ENTRY_INIT(__bp) \ 214 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) 215 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ 216 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) 217 218 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ 219 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ 220 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ 221 bd_bif_dlist_entry) 222 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ 223 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ 224 bd_bif_dlist_entry) 225 #define BPFIF_DLIST_WRITER_REMOVE(__d) \ 226 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) 227 #define BPFIF_DLIST_ENTRY_INIT(__d) \ 228 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) 229 #define BPFIF_DLIST_READER_EMPTY(__bp) \ 230 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 231 bd_bif_dlist_entry) == NULL) 232 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ 233 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 234 bd_bif_dlist_entry) == NULL) 235 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ 236 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) 237 238 static int bpf_allocbufs(struct bpf_d *); 239 static void bpf_deliver(struct bpf_if *, 240 void *(*cpfn)(void *, const void *, size_t), 241 void *, u_int, u_int, const bool); 242 static void bpf_freed(struct bpf_d *); 243 static void bpf_free_filter(struct bpf_filter *); 244 static void bpf_ifname(struct ifnet *, struct ifreq *); 245 static void *bpf_mcpy(void *, const void *, size_t); 246 static int bpf_movein(struct uio *, int, uint64_t, 247 struct mbuf **, struct sockaddr *); 248 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 249 static void bpf_detachd(struct bpf_d *); 250 static int bpf_setif(struct bpf_d *, struct ifreq *); 251 static int bpf_setf(struct bpf_d *, struct bpf_program *); 252 static void bpf_timed_out(void *); 253 static inline void 254 bpf_wakeup(struct bpf_d *); 255 static int bpf_hdrlen(struct bpf_d *); 256 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 257 void *(*)(void *, const void *, size_t), struct timespec *); 258 static void reset_d(struct bpf_d *); 259 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 260 static int bpf_setdlt(struct bpf_d *, u_int); 261 262 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 263 int); 264 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 265 int); 266 static int bpf_ioctl(struct file *, u_long, void *); 267 static int bpf_poll(struct file *, int); 268 static int bpf_stat(struct file *, struct stat *); 269 static int bpf_close(struct file *); 270 static int bpf_kqfilter(struct file *, struct knote *); 271 static void bpf_softintr(void *); 272 273 static const struct fileops bpf_fileops = { 274 .fo_name = "bpf", 275 .fo_read = bpf_read, 276 .fo_write = bpf_write, 277 .fo_ioctl = bpf_ioctl, 278 .fo_fcntl = fnullop_fcntl, 279 .fo_poll = bpf_poll, 280 .fo_stat = bpf_stat, 281 .fo_close = bpf_close, 282 .fo_kqfilter = bpf_kqfilter, 283 .fo_restart = fnullop_restart, 284 }; 285 286 dev_type_open(bpfopen); 287 288 const struct cdevsw bpf_cdevsw = { 289 .d_open = bpfopen, 290 .d_close = noclose, 291 .d_read = noread, 292 .d_write = nowrite, 293 .d_ioctl = noioctl, 294 .d_stop = nostop, 295 .d_tty = notty, 296 .d_poll = nopoll, 297 .d_mmap = nommap, 298 .d_kqfilter = nokqfilter, 299 .d_discard = nodiscard, 300 .d_flag = D_OTHER | D_MPSAFE 301 }; 302 303 bpfjit_func_t 304 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 305 { 306 307 membar_consumer(); 308 if (bpfjit_module_ops.bj_generate_code != NULL) { 309 return bpfjit_module_ops.bj_generate_code(bc, code, size); 310 } 311 return NULL; 312 } 313 314 void 315 bpf_jit_freecode(bpfjit_func_t jcode) 316 { 317 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 318 bpfjit_module_ops.bj_free_code(jcode); 319 } 320 321 static int 322 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 323 struct sockaddr *sockp) 324 { 325 struct mbuf *m; 326 int error; 327 size_t len; 328 size_t hlen; 329 size_t align; 330 331 /* 332 * Build a sockaddr based on the data link layer type. 333 * We do this at this level because the ethernet header 334 * is copied directly into the data field of the sockaddr. 335 * In the case of SLIP, there is no header and the packet 336 * is forwarded as is. 337 * Also, we are careful to leave room at the front of the mbuf 338 * for the link level header. 339 */ 340 switch (linktype) { 341 342 case DLT_SLIP: 343 sockp->sa_family = AF_INET; 344 hlen = 0; 345 align = 0; 346 break; 347 348 case DLT_PPP: 349 sockp->sa_family = AF_UNSPEC; 350 hlen = 0; 351 align = 0; 352 break; 353 354 case DLT_EN10MB: 355 sockp->sa_family = AF_UNSPEC; 356 /* XXX Would MAXLINKHDR be better? */ 357 /* 6(dst)+6(src)+2(type) */ 358 hlen = sizeof(struct ether_header); 359 align = 2; 360 break; 361 362 case DLT_ARCNET: 363 sockp->sa_family = AF_UNSPEC; 364 hlen = ARC_HDRLEN; 365 align = 5; 366 break; 367 368 case DLT_FDDI: 369 sockp->sa_family = AF_LINK; 370 /* XXX 4(FORMAC)+6(dst)+6(src) */ 371 hlen = 16; 372 align = 0; 373 break; 374 375 case DLT_ECONET: 376 sockp->sa_family = AF_UNSPEC; 377 hlen = 6; 378 align = 2; 379 break; 380 381 case DLT_NULL: 382 sockp->sa_family = AF_UNSPEC; 383 hlen = 0; 384 align = 0; 385 break; 386 387 default: 388 return (EIO); 389 } 390 391 len = uio->uio_resid; 392 /* 393 * If there aren't enough bytes for a link level header or the 394 * packet length exceeds the interface mtu, return an error. 395 */ 396 if (len - hlen > mtu) 397 return (EMSGSIZE); 398 399 /* 400 * XXX Avoid complicated buffer chaining --- 401 * bail if it won't fit in a single mbuf. 402 * (Take into account possible alignment bytes) 403 */ 404 if (len + align > MCLBYTES) 405 return (EIO); 406 407 m = m_gethdr(M_WAIT, MT_DATA); 408 m_reset_rcvif(m); 409 m->m_pkthdr.len = (int)(len - hlen); 410 if (len + align > MHLEN) { 411 m_clget(m, M_WAIT); 412 if ((m->m_flags & M_EXT) == 0) { 413 error = ENOBUFS; 414 goto bad; 415 } 416 } 417 418 /* Insure the data is properly aligned */ 419 if (align > 0) { 420 m->m_data += align; 421 m->m_len -= (int)align; 422 } 423 424 error = uiomove(mtod(m, void *), len, uio); 425 if (error) 426 goto bad; 427 if (hlen != 0) { 428 memcpy(sockp->sa_data, mtod(m, void *), hlen); 429 m->m_data += hlen; /* XXX */ 430 len -= hlen; 431 } 432 m->m_len = (int)len; 433 *mp = m; 434 return (0); 435 436 bad: 437 m_freem(m); 438 return (error); 439 } 440 441 /* 442 * Attach file to the bpf interface, i.e. make d listen on bp. 443 */ 444 static void 445 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 446 { 447 448 KASSERT(mutex_owned(&bpf_mtx)); 449 KASSERT(mutex_owned(d->bd_mtx)); 450 /* 451 * Point d at bp, and add d to the interface's list of listeners. 452 * Finally, point the driver's bpf cookie at the interface so 453 * it will divert packets to bpf. 454 */ 455 d->bd_bif = bp; 456 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); 457 458 *bp->bif_driverp = bp; 459 } 460 461 /* 462 * Detach a file from its interface. 463 */ 464 static void 465 bpf_detachd(struct bpf_d *d) 466 { 467 struct bpf_if *bp; 468 469 KASSERT(mutex_owned(&bpf_mtx)); 470 KASSERT(mutex_owned(d->bd_mtx)); 471 472 bp = d->bd_bif; 473 /* 474 * Check if this descriptor had requested promiscuous mode. 475 * If so, turn it off. 476 */ 477 if (d->bd_promisc) { 478 int error __diagused; 479 480 d->bd_promisc = 0; 481 /* 482 * Take device out of promiscuous mode. Since we were 483 * able to enter promiscuous mode, we should be able 484 * to turn it off. But we can get an error if 485 * the interface was configured down, so only panic 486 * if we don't get an unexpected error. 487 */ 488 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 489 error = ifpromisc(bp->bif_ifp, 0); 490 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 491 #ifdef DIAGNOSTIC 492 if (error) 493 printf("%s: ifpromisc failed: %d", __func__, error); 494 #endif 495 } 496 497 /* Remove d from the interface's descriptor list. */ 498 BPFIF_DLIST_WRITER_REMOVE(d); 499 500 pserialize_perform(bpf_psz); 501 502 if (BPFIF_DLIST_WRITER_EMPTY(bp)) { 503 /* 504 * Let the driver know that there are no more listeners. 505 */ 506 *d->bd_bif->bif_driverp = NULL; 507 } 508 d->bd_bif = NULL; 509 } 510 511 static void 512 bpf_init(void) 513 { 514 515 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 516 bpf_psz = pserialize_create(); 517 bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET); 518 519 PSLIST_INIT(&bpf_iflist); 520 PSLIST_INIT(&bpf_dlist); 521 522 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat)); 523 524 return; 525 } 526 527 /* 528 * bpfilterattach() is called at boot time. We don't need to do anything 529 * here, since any initialization will happen as part of module init code. 530 */ 531 /* ARGSUSED */ 532 void 533 bpfilterattach(int n) 534 { 535 536 } 537 538 /* 539 * Open ethernet device. Clones. 540 */ 541 /* ARGSUSED */ 542 int 543 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 544 { 545 struct bpf_d *d; 546 struct file *fp; 547 int error, fd; 548 549 /* falloc() will fill in the descriptor for us. */ 550 if ((error = fd_allocfile(&fp, &fd)) != 0) 551 return error; 552 553 d = kmem_zalloc(sizeof(*d), KM_SLEEP); 554 d->bd_bufsize = bpf_bufsize; 555 d->bd_seesent = 1; 556 d->bd_feedback = 0; 557 d->bd_pid = l->l_proc->p_pid; 558 #ifdef _LP64 559 if (curproc->p_flag & PK_32) 560 d->bd_compat32 = 1; 561 #endif 562 getnanotime(&d->bd_btime); 563 d->bd_atime = d->bd_mtime = d->bd_btime; 564 callout_init(&d->bd_callout, CALLOUT_MPSAFE); 565 selinit(&d->bd_sel); 566 d->bd_sih = softint_establish(SOFTINT_CLOCK|SOFTINT_MPSAFE, 567 bpf_softintr, d); 568 d->bd_jitcode = NULL; 569 d->bd_filter = NULL; 570 BPF_DLIST_ENTRY_INIT(d); 571 BPFIF_DLIST_ENTRY_INIT(d); 572 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 573 d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 574 cv_init(&d->bd_cv, "bpf"); 575 576 mutex_enter(&bpf_mtx); 577 BPF_DLIST_WRITER_INSERT_HEAD(d); 578 mutex_exit(&bpf_mtx); 579 580 return fd_clone(fp, fd, flag, &bpf_fileops, d); 581 } 582 583 /* 584 * Close the descriptor by detaching it from its interface, 585 * deallocating its buffers, and marking it free. 586 */ 587 /* ARGSUSED */ 588 static int 589 bpf_close(struct file *fp) 590 { 591 struct bpf_d *d; 592 593 mutex_enter(&bpf_mtx); 594 595 if ((d = fp->f_bpf) == NULL) { 596 mutex_exit(&bpf_mtx); 597 return 0; 598 } 599 600 /* 601 * Refresh the PID associated with this bpf file. 602 */ 603 d->bd_pid = curproc->p_pid; 604 605 mutex_enter(d->bd_mtx); 606 if (d->bd_state == BPF_WAITING) 607 callout_halt(&d->bd_callout, d->bd_mtx); 608 d->bd_state = BPF_IDLE; 609 if (d->bd_bif) 610 bpf_detachd(d); 611 mutex_exit(d->bd_mtx); 612 613 BPF_DLIST_WRITER_REMOVE(d); 614 615 pserialize_perform(bpf_psz); 616 mutex_exit(&bpf_mtx); 617 618 BPFIF_DLIST_ENTRY_DESTROY(d); 619 BPF_DLIST_ENTRY_DESTROY(d); 620 fp->f_bpf = NULL; 621 bpf_freed(d); 622 callout_destroy(&d->bd_callout); 623 seldestroy(&d->bd_sel); 624 softint_disestablish(d->bd_sih); 625 mutex_obj_free(d->bd_mtx); 626 mutex_obj_free(d->bd_buf_mtx); 627 cv_destroy(&d->bd_cv); 628 629 kmem_free(d, sizeof(*d)); 630 631 return (0); 632 } 633 634 /* 635 * Rotate the packet buffers in descriptor d. Move the store buffer 636 * into the hold slot, and the free buffer into the store slot. 637 * Zero the length of the new store buffer. 638 */ 639 #define ROTATE_BUFFERS(d) \ 640 (d)->bd_hbuf = (d)->bd_sbuf; \ 641 (d)->bd_hlen = (d)->bd_slen; \ 642 (d)->bd_sbuf = (d)->bd_fbuf; \ 643 (d)->bd_slen = 0; \ 644 (d)->bd_fbuf = NULL; 645 /* 646 * bpfread - read next chunk of packets from buffers 647 */ 648 static int 649 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 650 kauth_cred_t cred, int flags) 651 { 652 struct bpf_d *d = fp->f_bpf; 653 int timed_out; 654 int error; 655 656 getnanotime(&d->bd_atime); 657 /* 658 * Restrict application to use a buffer the same size as 659 * the kernel buffers. 660 */ 661 if (uio->uio_resid != d->bd_bufsize) 662 return (EINVAL); 663 664 mutex_enter(d->bd_mtx); 665 if (d->bd_state == BPF_WAITING) 666 callout_halt(&d->bd_callout, d->bd_mtx); 667 timed_out = (d->bd_state == BPF_TIMED_OUT); 668 d->bd_state = BPF_IDLE; 669 mutex_exit(d->bd_mtx); 670 /* 671 * If the hold buffer is empty, then do a timed sleep, which 672 * ends when the timeout expires or when enough packets 673 * have arrived to fill the store buffer. 674 */ 675 mutex_enter(d->bd_buf_mtx); 676 while (d->bd_hbuf == NULL) { 677 if (fp->f_flag & FNONBLOCK) { 678 if (d->bd_slen == 0) { 679 error = EWOULDBLOCK; 680 goto out; 681 } 682 ROTATE_BUFFERS(d); 683 break; 684 } 685 686 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 687 /* 688 * A packet(s) either arrived since the previous 689 * read or arrived while we were asleep. 690 * Rotate the buffers and return what's here. 691 */ 692 ROTATE_BUFFERS(d); 693 break; 694 } 695 696 error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout); 697 698 if (error == EINTR || error == ERESTART) 699 goto out; 700 701 if (error == EWOULDBLOCK) { 702 /* 703 * On a timeout, return what's in the buffer, 704 * which may be nothing. If there is something 705 * in the store buffer, we can rotate the buffers. 706 */ 707 if (d->bd_hbuf) 708 /* 709 * We filled up the buffer in between 710 * getting the timeout and arriving 711 * here, so we don't need to rotate. 712 */ 713 break; 714 715 if (d->bd_slen == 0) { 716 error = 0; 717 goto out; 718 } 719 ROTATE_BUFFERS(d); 720 break; 721 } 722 if (error != 0) 723 goto out; 724 } 725 /* 726 * At this point, we know we have something in the hold slot. 727 */ 728 mutex_exit(d->bd_buf_mtx); 729 730 /* 731 * Move data from hold buffer into user space. 732 * We know the entire buffer is transferred since 733 * we checked above that the read buffer is bpf_bufsize bytes. 734 */ 735 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 736 737 mutex_enter(d->bd_buf_mtx); 738 d->bd_fbuf = d->bd_hbuf; 739 d->bd_hbuf = NULL; 740 d->bd_hlen = 0; 741 out: 742 mutex_exit(d->bd_buf_mtx); 743 return (error); 744 } 745 746 747 /* 748 * If there are processes sleeping on this descriptor, wake them up. 749 */ 750 static inline void 751 bpf_wakeup(struct bpf_d *d) 752 { 753 754 mutex_enter(d->bd_buf_mtx); 755 cv_broadcast(&d->bd_cv); 756 mutex_exit(d->bd_buf_mtx); 757 758 if (d->bd_async) 759 softint_schedule(d->bd_sih); 760 selnotify(&d->bd_sel, 0, 0); 761 } 762 763 static void 764 bpf_softintr(void *cookie) 765 { 766 struct bpf_d *d; 767 768 d = cookie; 769 mutex_enter(d->bd_mtx); 770 if (d->bd_async) 771 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 772 mutex_exit(d->bd_mtx); 773 } 774 775 static void 776 bpf_timed_out(void *arg) 777 { 778 struct bpf_d *d = arg; 779 780 mutex_enter(d->bd_mtx); 781 if (d->bd_state == BPF_WAITING) { 782 d->bd_state = BPF_TIMED_OUT; 783 if (d->bd_slen != 0) 784 bpf_wakeup(d); 785 } 786 mutex_exit(d->bd_mtx); 787 } 788 789 790 static int 791 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 792 kauth_cred_t cred, int flags) 793 { 794 struct bpf_d *d = fp->f_bpf; 795 struct bpf_if *bp; 796 struct ifnet *ifp; 797 struct mbuf *m, *mc; 798 int error; 799 static struct sockaddr_storage dst; 800 struct psref psref; 801 int bound; 802 803 m = NULL; /* XXX gcc */ 804 805 bound = curlwp_bind(); 806 mutex_enter(d->bd_mtx); 807 bp = d->bd_bif; 808 if (bp == NULL) { 809 mutex_exit(d->bd_mtx); 810 error = ENXIO; 811 goto out_bindx; 812 } 813 bpf_if_acquire(bp, &psref); 814 mutex_exit(d->bd_mtx); 815 816 getnanotime(&d->bd_mtime); 817 818 ifp = bp->bif_ifp; 819 if (if_is_deactivated(ifp)) { 820 error = ENXIO; 821 goto out; 822 } 823 824 if (uio->uio_resid == 0) { 825 error = 0; 826 goto out; 827 } 828 829 error = bpf_movein(uio, (int)bp->bif_dlt, ifp->if_mtu, &m, 830 (struct sockaddr *) &dst); 831 if (error) 832 goto out; 833 834 if (m->m_pkthdr.len > ifp->if_mtu) { 835 m_freem(m); 836 error = EMSGSIZE; 837 goto out; 838 } 839 840 if (d->bd_hdrcmplt) 841 dst.ss_family = pseudo_AF_HDRCMPLT; 842 843 if (d->bd_feedback) { 844 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 845 if (mc != NULL) 846 m_set_rcvif(mc, ifp); 847 /* Set M_PROMISC for outgoing packets to be discarded. */ 848 if (1 /*d->bd_direction == BPF_D_INOUT*/) 849 m->m_flags |= M_PROMISC; 850 } else 851 mc = NULL; 852 853 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); 854 855 if (mc != NULL) { 856 if (error == 0) 857 ifp->_if_input(ifp, mc); 858 else 859 m_freem(mc); 860 } 861 /* 862 * The driver frees the mbuf. 863 */ 864 out: 865 bpf_if_release(bp, &psref); 866 out_bindx: 867 curlwp_bindx(bound); 868 return error; 869 } 870 871 /* 872 * Reset a descriptor by flushing its packet buffer and clearing the 873 * receive and drop counts. 874 */ 875 static void 876 reset_d(struct bpf_d *d) 877 { 878 879 KASSERT(mutex_owned(d->bd_mtx)); 880 881 mutex_enter(d->bd_buf_mtx); 882 if (d->bd_hbuf) { 883 /* Free the hold buffer. */ 884 d->bd_fbuf = d->bd_hbuf; 885 d->bd_hbuf = NULL; 886 } 887 d->bd_slen = 0; 888 d->bd_hlen = 0; 889 d->bd_rcount = 0; 890 d->bd_dcount = 0; 891 d->bd_ccount = 0; 892 mutex_exit(d->bd_buf_mtx); 893 } 894 895 /* 896 * FIONREAD Check for read packet available. 897 * BIOCGBLEN Get buffer len [for read()]. 898 * BIOCSETF Set ethernet read filter. 899 * BIOCFLUSH Flush read packet buffer. 900 * BIOCPROMISC Put interface into promiscuous mode. 901 * BIOCGDLT Get link layer type. 902 * BIOCGETIF Get interface name. 903 * BIOCSETIF Set interface. 904 * BIOCSRTIMEOUT Set read timeout. 905 * BIOCGRTIMEOUT Get read timeout. 906 * BIOCGSTATS Get packet stats. 907 * BIOCIMMEDIATE Set immediate mode. 908 * BIOCVERSION Get filter language version. 909 * BIOCGHDRCMPLT Get "header already complete" flag. 910 * BIOCSHDRCMPLT Set "header already complete" flag. 911 * BIOCSFEEDBACK Set packet feedback mode. 912 * BIOCGFEEDBACK Get packet feedback mode. 913 * BIOCGSEESENT Get "see sent packets" mode. 914 * BIOCSSEESENT Set "see sent packets" mode. 915 */ 916 /* ARGSUSED */ 917 static int 918 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 919 { 920 struct bpf_d *d = fp->f_bpf; 921 int error = 0; 922 923 /* 924 * Refresh the PID associated with this bpf file. 925 */ 926 d->bd_pid = curproc->p_pid; 927 #ifdef _LP64 928 if (curproc->p_flag & PK_32) 929 d->bd_compat32 = 1; 930 else 931 d->bd_compat32 = 0; 932 #endif 933 934 mutex_enter(d->bd_mtx); 935 if (d->bd_state == BPF_WAITING) 936 callout_halt(&d->bd_callout, d->bd_mtx); 937 d->bd_state = BPF_IDLE; 938 mutex_exit(d->bd_mtx); 939 940 switch (cmd) { 941 942 default: 943 error = EINVAL; 944 break; 945 946 /* 947 * Check for read packet available. 948 */ 949 case FIONREAD: 950 { 951 int n; 952 953 mutex_enter(d->bd_buf_mtx); 954 n = d->bd_slen; 955 if (d->bd_hbuf) 956 n += d->bd_hlen; 957 mutex_exit(d->bd_buf_mtx); 958 959 *(int *)addr = n; 960 break; 961 } 962 963 /* 964 * Get buffer len [for read()]. 965 */ 966 case BIOCGBLEN: 967 *(u_int *)addr = d->bd_bufsize; 968 break; 969 970 /* 971 * Set buffer length. 972 */ 973 case BIOCSBLEN: 974 /* 975 * Forbid to change the buffer length if buffers are already 976 * allocated. 977 */ 978 mutex_enter(d->bd_mtx); 979 mutex_enter(d->bd_buf_mtx); 980 if (d->bd_bif != NULL || d->bd_sbuf != NULL) 981 error = EINVAL; 982 else { 983 u_int size = *(u_int *)addr; 984 985 if (size > bpf_maxbufsize) 986 *(u_int *)addr = size = bpf_maxbufsize; 987 else if (size < BPF_MINBUFSIZE) 988 *(u_int *)addr = size = BPF_MINBUFSIZE; 989 d->bd_bufsize = size; 990 } 991 mutex_exit(d->bd_buf_mtx); 992 mutex_exit(d->bd_mtx); 993 break; 994 995 /* 996 * Set link layer read filter. 997 */ 998 case BIOCSETF: 999 error = bpf_setf(d, addr); 1000 break; 1001 1002 /* 1003 * Flush read packet buffer. 1004 */ 1005 case BIOCFLUSH: 1006 mutex_enter(d->bd_mtx); 1007 reset_d(d); 1008 mutex_exit(d->bd_mtx); 1009 break; 1010 1011 /* 1012 * Put interface into promiscuous mode. 1013 */ 1014 case BIOCPROMISC: 1015 mutex_enter(d->bd_mtx); 1016 if (d->bd_bif == NULL) { 1017 mutex_exit(d->bd_mtx); 1018 /* 1019 * No interface attached yet. 1020 */ 1021 error = EINVAL; 1022 break; 1023 } 1024 if (d->bd_promisc == 0) { 1025 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 1026 error = ifpromisc(d->bd_bif->bif_ifp, 1); 1027 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 1028 if (error == 0) 1029 d->bd_promisc = 1; 1030 } 1031 mutex_exit(d->bd_mtx); 1032 break; 1033 1034 /* 1035 * Get device parameters. 1036 */ 1037 case BIOCGDLT: 1038 mutex_enter(d->bd_mtx); 1039 if (d->bd_bif == NULL) 1040 error = EINVAL; 1041 else 1042 *(u_int *)addr = d->bd_bif->bif_dlt; 1043 mutex_exit(d->bd_mtx); 1044 break; 1045 1046 /* 1047 * Get a list of supported device parameters. 1048 */ 1049 case BIOCGDLTLIST: 1050 mutex_enter(d->bd_mtx); 1051 if (d->bd_bif == NULL) 1052 error = EINVAL; 1053 else 1054 error = bpf_getdltlist(d, addr); 1055 mutex_exit(d->bd_mtx); 1056 break; 1057 1058 /* 1059 * Set device parameters. 1060 */ 1061 case BIOCSDLT: 1062 mutex_enter(&bpf_mtx); 1063 mutex_enter(d->bd_mtx); 1064 if (d->bd_bif == NULL) 1065 error = EINVAL; 1066 else 1067 error = bpf_setdlt(d, *(u_int *)addr); 1068 mutex_exit(d->bd_mtx); 1069 mutex_exit(&bpf_mtx); 1070 break; 1071 1072 /* 1073 * Set interface name. 1074 */ 1075 #ifdef OBIOCGETIF 1076 case OBIOCGETIF: 1077 #endif 1078 case BIOCGETIF: 1079 mutex_enter(d->bd_mtx); 1080 if (d->bd_bif == NULL) 1081 error = EINVAL; 1082 else 1083 bpf_ifname(d->bd_bif->bif_ifp, addr); 1084 mutex_exit(d->bd_mtx); 1085 break; 1086 1087 /* 1088 * Set interface. 1089 */ 1090 #ifdef OBIOCSETIF 1091 case OBIOCSETIF: 1092 #endif 1093 case BIOCSETIF: 1094 mutex_enter(&bpf_mtx); 1095 error = bpf_setif(d, addr); 1096 mutex_exit(&bpf_mtx); 1097 break; 1098 1099 /* 1100 * Set read timeout. 1101 */ 1102 case BIOCSRTIMEOUT: 1103 { 1104 struct timeval *tv = addr; 1105 1106 /* Compute number of ticks. */ 1107 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1108 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1109 d->bd_rtout = 1; 1110 break; 1111 } 1112 1113 #ifdef BIOCGORTIMEOUT 1114 /* 1115 * Get read timeout. 1116 */ 1117 case BIOCGORTIMEOUT: 1118 { 1119 struct timeval50 *tv = addr; 1120 1121 tv->tv_sec = d->bd_rtout / hz; 1122 tv->tv_usec = (d->bd_rtout % hz) * tick; 1123 break; 1124 } 1125 #endif 1126 1127 #ifdef BIOCSORTIMEOUT 1128 /* 1129 * Set read timeout. 1130 */ 1131 case BIOCSORTIMEOUT: 1132 { 1133 struct timeval50 *tv = addr; 1134 1135 /* Compute number of ticks. */ 1136 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1137 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1138 d->bd_rtout = 1; 1139 break; 1140 } 1141 #endif 1142 1143 /* 1144 * Get read timeout. 1145 */ 1146 case BIOCGRTIMEOUT: 1147 { 1148 struct timeval *tv = addr; 1149 1150 tv->tv_sec = d->bd_rtout / hz; 1151 tv->tv_usec = (d->bd_rtout % hz) * tick; 1152 break; 1153 } 1154 /* 1155 * Get packet stats. 1156 */ 1157 case BIOCGSTATS: 1158 { 1159 struct bpf_stat *bs = addr; 1160 1161 bs->bs_recv = d->bd_rcount; 1162 bs->bs_drop = d->bd_dcount; 1163 bs->bs_capt = d->bd_ccount; 1164 break; 1165 } 1166 1167 case BIOCGSTATSOLD: 1168 { 1169 struct bpf_stat_old *bs = addr; 1170 1171 bs->bs_recv = d->bd_rcount; 1172 bs->bs_drop = d->bd_dcount; 1173 break; 1174 } 1175 1176 /* 1177 * Set immediate mode. 1178 */ 1179 case BIOCIMMEDIATE: 1180 d->bd_immediate = *(u_int *)addr; 1181 break; 1182 1183 case BIOCVERSION: 1184 { 1185 struct bpf_version *bv = addr; 1186 1187 bv->bv_major = BPF_MAJOR_VERSION; 1188 bv->bv_minor = BPF_MINOR_VERSION; 1189 break; 1190 } 1191 1192 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1193 *(u_int *)addr = d->bd_hdrcmplt; 1194 break; 1195 1196 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1197 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1198 break; 1199 1200 /* 1201 * Get "see sent packets" flag 1202 */ 1203 case BIOCGSEESENT: 1204 *(u_int *)addr = d->bd_seesent; 1205 break; 1206 1207 /* 1208 * Set "see sent" packets flag 1209 */ 1210 case BIOCSSEESENT: 1211 d->bd_seesent = *(u_int *)addr; 1212 break; 1213 1214 /* 1215 * Set "feed packets from bpf back to input" mode 1216 */ 1217 case BIOCSFEEDBACK: 1218 d->bd_feedback = *(u_int *)addr; 1219 break; 1220 1221 /* 1222 * Get "feed packets from bpf back to input" mode 1223 */ 1224 case BIOCGFEEDBACK: 1225 *(u_int *)addr = d->bd_feedback; 1226 break; 1227 1228 case FIONBIO: /* Non-blocking I/O */ 1229 /* 1230 * No need to do anything special as we use IO_NDELAY in 1231 * bpfread() as an indication of whether or not to block 1232 * the read. 1233 */ 1234 break; 1235 1236 case FIOASYNC: /* Send signal on receive packets */ 1237 mutex_enter(d->bd_mtx); 1238 d->bd_async = *(int *)addr; 1239 mutex_exit(d->bd_mtx); 1240 break; 1241 1242 case TIOCSPGRP: /* Process or group to send signals to */ 1243 case FIOSETOWN: 1244 error = fsetown(&d->bd_pgid, cmd, addr); 1245 break; 1246 1247 case TIOCGPGRP: 1248 case FIOGETOWN: 1249 error = fgetown(d->bd_pgid, cmd, addr); 1250 break; 1251 } 1252 return (error); 1253 } 1254 1255 /* 1256 * Set d's packet filter program to fp. If this file already has a filter, 1257 * free it and replace it. Returns EINVAL for bogus requests. 1258 */ 1259 static int 1260 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1261 { 1262 struct bpf_insn *fcode; 1263 bpfjit_func_t jcode; 1264 size_t flen, size = 0; 1265 struct bpf_filter *oldf, *newf; 1266 1267 jcode = NULL; 1268 flen = fp->bf_len; 1269 1270 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1271 return EINVAL; 1272 } 1273 1274 if (flen) { 1275 /* 1276 * Allocate the buffer, copy the byte-code from 1277 * userspace and validate it. 1278 */ 1279 size = flen * sizeof(*fp->bf_insns); 1280 fcode = kmem_alloc(size, KM_SLEEP); 1281 if (copyin(fp->bf_insns, fcode, size) != 0 || 1282 !bpf_validate(fcode, (int)flen)) { 1283 kmem_free(fcode, size); 1284 return EINVAL; 1285 } 1286 membar_consumer(); 1287 if (bpf_jit) 1288 jcode = bpf_jit_generate(NULL, fcode, flen); 1289 } else { 1290 fcode = NULL; 1291 } 1292 1293 newf = kmem_alloc(sizeof(*newf), KM_SLEEP); 1294 newf->bf_insn = fcode; 1295 newf->bf_size = size; 1296 newf->bf_jitcode = jcode; 1297 d->bd_jitcode = jcode; /* XXX just for kvm(3) users */ 1298 1299 /* Need to hold bpf_mtx for pserialize_perform */ 1300 mutex_enter(&bpf_mtx); 1301 mutex_enter(d->bd_mtx); 1302 oldf = d->bd_filter; 1303 d->bd_filter = newf; 1304 membar_producer(); 1305 reset_d(d); 1306 pserialize_perform(bpf_psz); 1307 mutex_exit(d->bd_mtx); 1308 mutex_exit(&bpf_mtx); 1309 1310 if (oldf != NULL) 1311 bpf_free_filter(oldf); 1312 1313 return 0; 1314 } 1315 1316 /* 1317 * Detach a file from its current interface (if attached at all) and attach 1318 * to the interface indicated by the name stored in ifr. 1319 * Return an errno or 0. 1320 */ 1321 static int 1322 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1323 { 1324 struct bpf_if *bp; 1325 char *cp; 1326 int unit_seen, i, error; 1327 1328 KASSERT(mutex_owned(&bpf_mtx)); 1329 /* 1330 * Make sure the provided name has a unit number, and default 1331 * it to '0' if not specified. 1332 * XXX This is ugly ... do this differently? 1333 */ 1334 unit_seen = 0; 1335 cp = ifr->ifr_name; 1336 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1337 while (*cp++) 1338 if (*cp >= '0' && *cp <= '9') 1339 unit_seen = 1; 1340 if (!unit_seen) { 1341 /* Make sure to leave room for the '\0'. */ 1342 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1343 if ((ifr->ifr_name[i] >= 'a' && 1344 ifr->ifr_name[i] <= 'z') || 1345 (ifr->ifr_name[i] >= 'A' && 1346 ifr->ifr_name[i] <= 'Z')) 1347 continue; 1348 ifr->ifr_name[i] = '0'; 1349 } 1350 } 1351 1352 /* 1353 * Look through attached interfaces for the named one. 1354 */ 1355 BPF_IFLIST_WRITER_FOREACH(bp) { 1356 struct ifnet *ifp = bp->bif_ifp; 1357 1358 if (ifp == NULL || 1359 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1360 continue; 1361 /* skip additional entry */ 1362 if (bp->bif_driverp != &ifp->if_bpf) 1363 continue; 1364 /* 1365 * We found the requested interface. 1366 * Allocate the packet buffers if we need to. 1367 * If we're already attached to requested interface, 1368 * just flush the buffer. 1369 */ 1370 /* 1371 * bpf_allocbufs is called only here. bpf_mtx ensures that 1372 * no race condition happen on d->bd_sbuf. 1373 */ 1374 if (d->bd_sbuf == NULL) { 1375 error = bpf_allocbufs(d); 1376 if (error != 0) 1377 return (error); 1378 } 1379 mutex_enter(d->bd_mtx); 1380 if (bp != d->bd_bif) { 1381 if (d->bd_bif) { 1382 /* 1383 * Detach if attached to something else. 1384 */ 1385 bpf_detachd(d); 1386 BPFIF_DLIST_ENTRY_INIT(d); 1387 } 1388 1389 bpf_attachd(d, bp); 1390 } 1391 reset_d(d); 1392 mutex_exit(d->bd_mtx); 1393 return (0); 1394 } 1395 /* Not found. */ 1396 return (ENXIO); 1397 } 1398 1399 /* 1400 * Copy the interface name to the ifreq. 1401 */ 1402 static void 1403 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1404 { 1405 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1406 } 1407 1408 static int 1409 bpf_stat(struct file *fp, struct stat *st) 1410 { 1411 struct bpf_d *d = fp->f_bpf; 1412 1413 (void)memset(st, 0, sizeof(*st)); 1414 mutex_enter(d->bd_mtx); 1415 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1416 st->st_atimespec = d->bd_atime; 1417 st->st_mtimespec = d->bd_mtime; 1418 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1419 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1420 st->st_gid = kauth_cred_getegid(fp->f_cred); 1421 st->st_mode = S_IFCHR; 1422 mutex_exit(d->bd_mtx); 1423 return 0; 1424 } 1425 1426 /* 1427 * Support for poll() system call 1428 * 1429 * Return true iff the specific operation will not block indefinitely - with 1430 * the assumption that it is safe to positively acknowledge a request for the 1431 * ability to write to the BPF device. 1432 * Otherwise, return false but make a note that a selnotify() must be done. 1433 */ 1434 static int 1435 bpf_poll(struct file *fp, int events) 1436 { 1437 struct bpf_d *d = fp->f_bpf; 1438 int revents; 1439 1440 /* 1441 * Refresh the PID associated with this bpf file. 1442 */ 1443 mutex_enter(&bpf_mtx); 1444 d->bd_pid = curproc->p_pid; 1445 1446 revents = events & (POLLOUT | POLLWRNORM); 1447 if (events & (POLLIN | POLLRDNORM)) { 1448 /* 1449 * An imitation of the FIONREAD ioctl code. 1450 */ 1451 mutex_enter(d->bd_mtx); 1452 if (d->bd_hlen != 0 || 1453 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1454 d->bd_slen != 0)) { 1455 revents |= events & (POLLIN | POLLRDNORM); 1456 } else { 1457 selrecord(curlwp, &d->bd_sel); 1458 /* Start the read timeout if necessary */ 1459 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1460 callout_reset(&d->bd_callout, d->bd_rtout, 1461 bpf_timed_out, d); 1462 d->bd_state = BPF_WAITING; 1463 } 1464 } 1465 mutex_exit(d->bd_mtx); 1466 } 1467 1468 mutex_exit(&bpf_mtx); 1469 return (revents); 1470 } 1471 1472 static void 1473 filt_bpfrdetach(struct knote *kn) 1474 { 1475 struct bpf_d *d = kn->kn_hook; 1476 1477 mutex_enter(d->bd_buf_mtx); 1478 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1479 mutex_exit(d->bd_buf_mtx); 1480 } 1481 1482 static int 1483 filt_bpfread(struct knote *kn, long hint) 1484 { 1485 struct bpf_d *d = kn->kn_hook; 1486 int rv; 1487 1488 mutex_enter(d->bd_buf_mtx); 1489 kn->kn_data = d->bd_hlen; 1490 if (d->bd_immediate) 1491 kn->kn_data += d->bd_slen; 1492 rv = (kn->kn_data > 0); 1493 mutex_exit(d->bd_buf_mtx); 1494 return rv; 1495 } 1496 1497 static const struct filterops bpfread_filtops = { 1498 .f_isfd = 1, 1499 .f_attach = NULL, 1500 .f_detach = filt_bpfrdetach, 1501 .f_event = filt_bpfread, 1502 }; 1503 1504 static int 1505 bpf_kqfilter(struct file *fp, struct knote *kn) 1506 { 1507 struct bpf_d *d = fp->f_bpf; 1508 struct klist *klist; 1509 1510 mutex_enter(d->bd_buf_mtx); 1511 switch (kn->kn_filter) { 1512 case EVFILT_READ: 1513 klist = &d->bd_sel.sel_klist; 1514 kn->kn_fop = &bpfread_filtops; 1515 break; 1516 1517 default: 1518 mutex_exit(d->bd_buf_mtx); 1519 return (EINVAL); 1520 } 1521 1522 kn->kn_hook = d; 1523 1524 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1525 mutex_exit(d->bd_buf_mtx); 1526 1527 return (0); 1528 } 1529 1530 /* 1531 * Copy data from an mbuf chain into a buffer. This code is derived 1532 * from m_copydata in sys/uipc_mbuf.c. 1533 */ 1534 static void * 1535 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1536 { 1537 const struct mbuf *m; 1538 u_int count; 1539 u_char *dst; 1540 1541 m = src_arg; 1542 dst = dst_arg; 1543 while (len > 0) { 1544 if (m == NULL) 1545 panic("bpf_mcpy"); 1546 count = min(m->m_len, len); 1547 memcpy(dst, mtod(m, const void *), count); 1548 m = m->m_next; 1549 dst += count; 1550 len -= count; 1551 } 1552 return dst_arg; 1553 } 1554 1555 /* 1556 * Dispatch a packet to all the listeners on interface bp. 1557 * 1558 * pkt pointer to the packet, either a data buffer or an mbuf chain 1559 * buflen buffer length, if pkt is a data buffer 1560 * cpfn a function that can copy pkt into the listener's buffer 1561 * pktlen length of the packet 1562 * rcv true if packet came in 1563 */ 1564 static inline void 1565 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1566 void *pkt, u_int pktlen, u_int buflen, const bool rcv) 1567 { 1568 uint32_t mem[BPF_MEMWORDS]; 1569 bpf_args_t args = { 1570 .pkt = (const uint8_t *)pkt, 1571 .wirelen = pktlen, 1572 .buflen = buflen, 1573 .mem = mem, 1574 .arg = NULL 1575 }; 1576 bool gottime = false; 1577 struct timespec ts; 1578 struct bpf_d *d; 1579 int s; 1580 1581 KASSERT(!cpu_intr_p()); 1582 1583 /* 1584 * Note that the IPL does not have to be raised at this point. 1585 * The only problem that could arise here is that if two different 1586 * interfaces shared any data. This is not the case. 1587 */ 1588 s = pserialize_read_enter(); 1589 BPFIF_DLIST_READER_FOREACH(d, bp) { 1590 u_int slen = 0; 1591 struct bpf_filter *filter; 1592 1593 if (!d->bd_seesent && !rcv) { 1594 continue; 1595 } 1596 atomic_inc_ulong(&d->bd_rcount); 1597 BPF_STATINC(recv); 1598 1599 filter = d->bd_filter; 1600 membar_datadep_consumer(); 1601 if (filter != NULL) { 1602 if (filter->bf_jitcode != NULL) 1603 slen = filter->bf_jitcode(NULL, &args); 1604 else 1605 slen = bpf_filter_ext(NULL, filter->bf_insn, 1606 &args); 1607 } 1608 1609 if (!slen) { 1610 continue; 1611 } 1612 if (!gottime) { 1613 gottime = true; 1614 nanotime(&ts); 1615 } 1616 /* Assume catchpacket doesn't sleep */ 1617 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1618 } 1619 pserialize_read_exit(s); 1620 } 1621 1622 /* 1623 * Incoming linkage from device drivers. Process the packet pkt, of length 1624 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1625 * by each process' filter, and if accepted, stashed into the corresponding 1626 * buffer. 1627 */ 1628 static void 1629 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1630 { 1631 1632 bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true); 1633 } 1634 1635 /* 1636 * Incoming linkage from device drivers, when the head of the packet is in 1637 * a buffer, and the tail is in an mbuf chain. 1638 */ 1639 static void 1640 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1641 { 1642 u_int pktlen; 1643 struct mbuf mb; 1644 1645 /* Skip outgoing duplicate packets. */ 1646 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1647 m->m_flags &= ~M_PROMISC; 1648 return; 1649 } 1650 1651 pktlen = m_length(m) + dlen; 1652 1653 /* 1654 * Craft on-stack mbuf suitable for passing to bpf_filter. 1655 * Note that we cut corners here; we only setup what's 1656 * absolutely needed--this mbuf should never go anywhere else. 1657 */ 1658 (void)memset(&mb, 0, sizeof(mb)); 1659 mb.m_next = m; 1660 mb.m_data = data; 1661 mb.m_len = dlen; 1662 1663 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif_index != 0); 1664 } 1665 1666 /* 1667 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1668 */ 1669 static void 1670 _bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1671 { 1672 void *(*cpfn)(void *, const void *, size_t); 1673 u_int pktlen, buflen; 1674 void *marg; 1675 1676 /* Skip outgoing duplicate packets. */ 1677 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1678 m->m_flags &= ~M_PROMISC; 1679 return; 1680 } 1681 1682 pktlen = m_length(m); 1683 1684 if (pktlen == m->m_len) { 1685 cpfn = (void *)memcpy; 1686 marg = mtod(m, void *); 1687 buflen = pktlen; 1688 } else { 1689 cpfn = bpf_mcpy; 1690 marg = m; 1691 buflen = 0; 1692 } 1693 1694 bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif_index != 0); 1695 } 1696 1697 /* 1698 * We need to prepend the address family as 1699 * a four byte field. Cons up a dummy header 1700 * to pacify bpf. This is safe because bpf 1701 * will only read from the mbuf (i.e., it won't 1702 * try to free it or keep a pointer a to it). 1703 */ 1704 static void 1705 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m) 1706 { 1707 struct mbuf m0; 1708 1709 m0.m_flags = 0; 1710 m0.m_next = m; 1711 m0.m_len = 4; 1712 m0.m_data = (char *)⁡ 1713 1714 _bpf_mtap(bp, &m0); 1715 } 1716 1717 /* 1718 * Put the SLIP pseudo-"link header" in place. 1719 * Note this M_PREPEND() should never fail, 1720 * swince we know we always have enough space 1721 * in the input buffer. 1722 */ 1723 static void 1724 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1725 { 1726 u_char *hp; 1727 1728 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1729 if (*m == NULL) 1730 return; 1731 1732 hp = mtod(*m, u_char *); 1733 hp[SLX_DIR] = SLIPDIR_IN; 1734 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1735 1736 _bpf_mtap(bp, *m); 1737 1738 m_adj(*m, SLIP_HDRLEN); 1739 } 1740 1741 /* 1742 * Put the SLIP pseudo-"link header" in 1743 * place. The compressed header is now 1744 * at the beginning of the mbuf. 1745 */ 1746 static void 1747 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1748 { 1749 struct mbuf m0; 1750 u_char *hp; 1751 1752 m0.m_flags = 0; 1753 m0.m_next = m; 1754 m0.m_data = m0.m_dat; 1755 m0.m_len = SLIP_HDRLEN; 1756 1757 hp = mtod(&m0, u_char *); 1758 1759 hp[SLX_DIR] = SLIPDIR_OUT; 1760 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1761 1762 _bpf_mtap(bp, &m0); 1763 m_freem(m); 1764 } 1765 1766 static struct mbuf * 1767 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) 1768 { 1769 struct mbuf *dup; 1770 1771 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); 1772 if (dup == NULL) 1773 return NULL; 1774 1775 if (bp->bif_mbuf_tail != NULL) { 1776 bp->bif_mbuf_tail->m_nextpkt = dup; 1777 } else { 1778 bp->bif_mbuf_head = dup; 1779 } 1780 bp->bif_mbuf_tail = dup; 1781 #ifdef BPF_MTAP_SOFTINT_DEBUG 1782 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", 1783 __func__, dup, bp->bif_ifp->if_xname); 1784 #endif 1785 1786 return dup; 1787 } 1788 1789 static struct mbuf * 1790 bpf_mbuf_dequeue(struct bpf_if *bp) 1791 { 1792 struct mbuf *m; 1793 int s; 1794 1795 /* XXX NOMPSAFE: assumed running on one CPU */ 1796 s = splnet(); 1797 m = bp->bif_mbuf_head; 1798 if (m != NULL) { 1799 bp->bif_mbuf_head = m->m_nextpkt; 1800 m->m_nextpkt = NULL; 1801 1802 if (bp->bif_mbuf_head == NULL) 1803 bp->bif_mbuf_tail = NULL; 1804 #ifdef BPF_MTAP_SOFTINT_DEBUG 1805 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", 1806 __func__, m, bp->bif_ifp->if_xname); 1807 #endif 1808 } 1809 splx(s); 1810 1811 return m; 1812 } 1813 1814 static void 1815 bpf_mtap_si(void *arg) 1816 { 1817 struct bpf_if *bp = arg; 1818 struct mbuf *m; 1819 1820 while ((m = bpf_mbuf_dequeue(bp)) != NULL) { 1821 #ifdef BPF_MTAP_SOFTINT_DEBUG 1822 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", 1823 __func__, m, bp->bif_ifp->if_xname); 1824 #endif 1825 bpf_ops->bpf_mtap(bp, m); 1826 m_freem(m); 1827 } 1828 } 1829 1830 static void 1831 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) 1832 { 1833 struct bpf_if *bp = ifp->if_bpf; 1834 struct mbuf *dup; 1835 1836 KASSERT(cpu_intr_p()); 1837 1838 /* To avoid extra invocations of the softint */ 1839 if (BPFIF_DLIST_READER_EMPTY(bp)) 1840 return; 1841 KASSERT(bp->bif_si != NULL); 1842 1843 dup = bpf_mbuf_enqueue(bp, m); 1844 if (dup != NULL) 1845 softint_schedule(bp->bif_si); 1846 } 1847 1848 static int 1849 bpf_hdrlen(struct bpf_d *d) 1850 { 1851 int hdrlen = d->bd_bif->bif_hdrlen; 1852 /* 1853 * Compute the length of the bpf header. This is not necessarily 1854 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1855 * that the network layer header begins on a longword boundary (for 1856 * performance reasons and to alleviate alignment restrictions). 1857 */ 1858 #ifdef _LP64 1859 if (d->bd_compat32) 1860 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1861 else 1862 #endif 1863 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1864 } 1865 1866 /* 1867 * Move the packet data from interface memory (pkt) into the 1868 * store buffer. Call the wakeup functions if it's time to wakeup 1869 * a listener (buffer full), "cpfn" is the routine called to do the 1870 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1871 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1872 * pkt is really an mbuf. 1873 */ 1874 static void 1875 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1876 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1877 { 1878 char *h; 1879 int totlen, curlen, caplen; 1880 int hdrlen = bpf_hdrlen(d); 1881 int do_wakeup = 0; 1882 1883 atomic_inc_ulong(&d->bd_ccount); 1884 BPF_STATINC(capt); 1885 /* 1886 * Figure out how many bytes to move. If the packet is 1887 * greater or equal to the snapshot length, transfer that 1888 * much. Otherwise, transfer the whole packet (unless 1889 * we hit the buffer size limit). 1890 */ 1891 totlen = hdrlen + min(snaplen, pktlen); 1892 if (totlen > d->bd_bufsize) 1893 totlen = d->bd_bufsize; 1894 /* 1895 * If we adjusted totlen to fit the bufsize, it could be that 1896 * totlen is smaller than hdrlen because of the link layer header. 1897 */ 1898 caplen = totlen - hdrlen; 1899 if (caplen < 0) 1900 caplen = 0; 1901 1902 mutex_enter(d->bd_buf_mtx); 1903 /* 1904 * Round up the end of the previous packet to the next longword. 1905 */ 1906 #ifdef _LP64 1907 if (d->bd_compat32) 1908 curlen = BPF_WORDALIGN32(d->bd_slen); 1909 else 1910 #endif 1911 curlen = BPF_WORDALIGN(d->bd_slen); 1912 if (curlen + totlen > d->bd_bufsize) { 1913 /* 1914 * This packet will overflow the storage buffer. 1915 * Rotate the buffers if we can, then wakeup any 1916 * pending reads. 1917 */ 1918 if (d->bd_fbuf == NULL) { 1919 mutex_exit(d->bd_buf_mtx); 1920 /* 1921 * We haven't completed the previous read yet, 1922 * so drop the packet. 1923 */ 1924 atomic_inc_ulong(&d->bd_dcount); 1925 BPF_STATINC(drop); 1926 return; 1927 } 1928 ROTATE_BUFFERS(d); 1929 do_wakeup = 1; 1930 curlen = 0; 1931 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1932 /* 1933 * Immediate mode is set, or the read timeout has 1934 * already expired during a select call. A packet 1935 * arrived, so the reader should be woken up. 1936 */ 1937 do_wakeup = 1; 1938 } 1939 1940 /* 1941 * Append the bpf header. 1942 */ 1943 h = (char *)d->bd_sbuf + curlen; 1944 #ifdef _LP64 1945 if (d->bd_compat32) { 1946 struct bpf_hdr32 *hp32; 1947 1948 hp32 = (struct bpf_hdr32 *)h; 1949 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1950 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1951 hp32->bh_datalen = pktlen; 1952 hp32->bh_hdrlen = hdrlen; 1953 hp32->bh_caplen = caplen; 1954 } else 1955 #endif 1956 { 1957 struct bpf_hdr *hp; 1958 1959 hp = (struct bpf_hdr *)h; 1960 hp->bh_tstamp.tv_sec = ts->tv_sec; 1961 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1962 hp->bh_datalen = pktlen; 1963 hp->bh_hdrlen = hdrlen; 1964 hp->bh_caplen = caplen; 1965 } 1966 1967 /* 1968 * Copy the packet data into the store buffer and update its length. 1969 */ 1970 (*cpfn)(h + hdrlen, pkt, caplen); 1971 d->bd_slen = curlen + totlen; 1972 mutex_exit(d->bd_buf_mtx); 1973 1974 /* 1975 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1976 * will cause filt_bpfread() to be called with it adjusted. 1977 */ 1978 if (do_wakeup) 1979 bpf_wakeup(d); 1980 } 1981 1982 /* 1983 * Initialize all nonzero fields of a descriptor. 1984 */ 1985 static int 1986 bpf_allocbufs(struct bpf_d *d) 1987 { 1988 1989 d->bd_fbuf = kmem_alloc(d->bd_bufsize, KM_NOSLEEP); 1990 if (!d->bd_fbuf) 1991 return (ENOBUFS); 1992 d->bd_sbuf = kmem_alloc(d->bd_bufsize, KM_NOSLEEP); 1993 if (!d->bd_sbuf) { 1994 kmem_free(d->bd_fbuf, d->bd_bufsize); 1995 return (ENOBUFS); 1996 } 1997 d->bd_slen = 0; 1998 d->bd_hlen = 0; 1999 return (0); 2000 } 2001 2002 static void 2003 bpf_free_filter(struct bpf_filter *filter) 2004 { 2005 2006 KASSERT(filter != NULL); 2007 KASSERT(filter->bf_insn != NULL); 2008 2009 kmem_free(filter->bf_insn, filter->bf_size); 2010 if (filter->bf_jitcode != NULL) 2011 bpf_jit_freecode(filter->bf_jitcode); 2012 kmem_free(filter, sizeof(*filter)); 2013 } 2014 2015 /* 2016 * Free buffers currently in use by a descriptor. 2017 * Called on close. 2018 */ 2019 static void 2020 bpf_freed(struct bpf_d *d) 2021 { 2022 /* 2023 * We don't need to lock out interrupts since this descriptor has 2024 * been detached from its interface and it yet hasn't been marked 2025 * free. 2026 */ 2027 if (d->bd_sbuf != NULL) { 2028 kmem_free(d->bd_sbuf, d->bd_bufsize); 2029 if (d->bd_hbuf != NULL) 2030 kmem_free(d->bd_hbuf, d->bd_bufsize); 2031 if (d->bd_fbuf != NULL) 2032 kmem_free(d->bd_fbuf, d->bd_bufsize); 2033 } 2034 if (d->bd_filter != NULL) { 2035 bpf_free_filter(d->bd_filter); 2036 d->bd_filter = NULL; 2037 } 2038 d->bd_jitcode = NULL; 2039 } 2040 2041 /* 2042 * Attach an interface to bpf. dlt is the link layer type; 2043 * hdrlen is the fixed size of the link header for the specified dlt 2044 * (variable length headers not yet supported). 2045 */ 2046 static void 2047 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2048 { 2049 struct bpf_if *bp; 2050 bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP); 2051 if (bp == NULL) 2052 panic("bpfattach"); 2053 2054 mutex_enter(&bpf_mtx); 2055 bp->bif_driverp = driverp; 2056 bp->bif_ifp = ifp; 2057 bp->bif_dlt = dlt; 2058 bp->bif_si = NULL; 2059 BPF_IFLIST_ENTRY_INIT(bp); 2060 PSLIST_INIT(&bp->bif_dlist_head); 2061 psref_target_init(&bp->bif_psref, bpf_psref_class); 2062 2063 BPF_IFLIST_WRITER_INSERT_HEAD(bp); 2064 2065 *bp->bif_driverp = NULL; 2066 2067 bp->bif_hdrlen = hdrlen; 2068 mutex_exit(&bpf_mtx); 2069 #if 0 2070 printf("bpf: %s attached\n", ifp->if_xname); 2071 #endif 2072 } 2073 2074 static void 2075 _bpf_mtap_softint_init(struct ifnet *ifp) 2076 { 2077 struct bpf_if *bp; 2078 2079 mutex_enter(&bpf_mtx); 2080 BPF_IFLIST_WRITER_FOREACH(bp) { 2081 if (bp->bif_ifp != ifp) 2082 continue; 2083 2084 bp->bif_mbuf_head = NULL; 2085 bp->bif_mbuf_tail = NULL; 2086 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); 2087 if (bp->bif_si == NULL) 2088 panic("%s: softint_establish() failed", __func__); 2089 break; 2090 } 2091 mutex_exit(&bpf_mtx); 2092 2093 if (bp == NULL) 2094 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); 2095 } 2096 2097 /* 2098 * Remove an interface from bpf. 2099 */ 2100 static void 2101 _bpfdetach(struct ifnet *ifp) 2102 { 2103 struct bpf_if *bp; 2104 struct bpf_d *d; 2105 int s; 2106 2107 mutex_enter(&bpf_mtx); 2108 /* Nuke the vnodes for any open instances */ 2109 again_d: 2110 BPF_DLIST_WRITER_FOREACH(d) { 2111 mutex_enter(d->bd_mtx); 2112 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 2113 /* 2114 * Detach the descriptor from an interface now. 2115 * It will be free'ed later by close routine. 2116 */ 2117 d->bd_promisc = 0; /* we can't touch device. */ 2118 bpf_detachd(d); 2119 mutex_exit(d->bd_mtx); 2120 goto again_d; 2121 } 2122 mutex_exit(d->bd_mtx); 2123 } 2124 2125 again: 2126 BPF_IFLIST_WRITER_FOREACH(bp) { 2127 if (bp->bif_ifp == ifp) { 2128 BPF_IFLIST_WRITER_REMOVE(bp); 2129 2130 pserialize_perform(bpf_psz); 2131 psref_target_destroy(&bp->bif_psref, bpf_psref_class); 2132 2133 BPF_IFLIST_ENTRY_DESTROY(bp); 2134 if (bp->bif_si != NULL) { 2135 /* XXX NOMPSAFE: assumed running on one CPU */ 2136 s = splnet(); 2137 while (bp->bif_mbuf_head != NULL) { 2138 struct mbuf *m = bp->bif_mbuf_head; 2139 bp->bif_mbuf_head = m->m_nextpkt; 2140 m_freem(m); 2141 } 2142 splx(s); 2143 softint_disestablish(bp->bif_si); 2144 } 2145 kmem_free(bp, sizeof(*bp)); 2146 goto again; 2147 } 2148 } 2149 mutex_exit(&bpf_mtx); 2150 } 2151 2152 /* 2153 * Change the data link type of a interface. 2154 */ 2155 static void 2156 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2157 { 2158 struct bpf_if *bp; 2159 2160 mutex_enter(&bpf_mtx); 2161 BPF_IFLIST_WRITER_FOREACH(bp) { 2162 if (bp->bif_driverp == &ifp->if_bpf) 2163 break; 2164 } 2165 if (bp == NULL) 2166 panic("bpf_change_type"); 2167 2168 bp->bif_dlt = dlt; 2169 2170 bp->bif_hdrlen = hdrlen; 2171 mutex_exit(&bpf_mtx); 2172 } 2173 2174 /* 2175 * Get a list of available data link type of the interface. 2176 */ 2177 static int 2178 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2179 { 2180 int n, error; 2181 struct ifnet *ifp; 2182 struct bpf_if *bp; 2183 int s, bound; 2184 2185 KASSERT(mutex_owned(d->bd_mtx)); 2186 2187 ifp = d->bd_bif->bif_ifp; 2188 n = 0; 2189 error = 0; 2190 2191 bound = curlwp_bind(); 2192 s = pserialize_read_enter(); 2193 BPF_IFLIST_READER_FOREACH(bp) { 2194 if (bp->bif_ifp != ifp) 2195 continue; 2196 if (bfl->bfl_list != NULL) { 2197 struct psref psref; 2198 2199 if (n >= bfl->bfl_len) { 2200 pserialize_read_exit(s); 2201 return ENOMEM; 2202 } 2203 2204 bpf_if_acquire(bp, &psref); 2205 pserialize_read_exit(s); 2206 2207 error = copyout(&bp->bif_dlt, 2208 bfl->bfl_list + n, sizeof(u_int)); 2209 2210 s = pserialize_read_enter(); 2211 bpf_if_release(bp, &psref); 2212 } 2213 n++; 2214 } 2215 pserialize_read_exit(s); 2216 curlwp_bindx(bound); 2217 2218 bfl->bfl_len = n; 2219 return error; 2220 } 2221 2222 /* 2223 * Set the data link type of a BPF instance. 2224 */ 2225 static int 2226 bpf_setdlt(struct bpf_d *d, u_int dlt) 2227 { 2228 int error, opromisc; 2229 struct ifnet *ifp; 2230 struct bpf_if *bp; 2231 2232 KASSERT(mutex_owned(&bpf_mtx)); 2233 KASSERT(mutex_owned(d->bd_mtx)); 2234 2235 if (d->bd_bif->bif_dlt == dlt) 2236 return 0; 2237 ifp = d->bd_bif->bif_ifp; 2238 BPF_IFLIST_WRITER_FOREACH(bp) { 2239 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2240 break; 2241 } 2242 if (bp == NULL) 2243 return EINVAL; 2244 opromisc = d->bd_promisc; 2245 bpf_detachd(d); 2246 BPFIF_DLIST_ENTRY_INIT(d); 2247 bpf_attachd(d, bp); 2248 reset_d(d); 2249 if (opromisc) { 2250 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 2251 error = ifpromisc(bp->bif_ifp, 1); 2252 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 2253 if (error) 2254 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 2255 bp->bif_ifp->if_xname, error); 2256 else 2257 d->bd_promisc = 1; 2258 } 2259 return 0; 2260 } 2261 2262 static int 2263 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 2264 { 2265 int newsize, error; 2266 struct sysctlnode node; 2267 2268 node = *rnode; 2269 node.sysctl_data = &newsize; 2270 newsize = bpf_maxbufsize; 2271 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2272 if (error || newp == NULL) 2273 return (error); 2274 2275 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 2276 return (EINVAL); 2277 2278 bpf_maxbufsize = newsize; 2279 2280 return (0); 2281 } 2282 2283 #if defined(MODULAR) || defined(BPFJIT) 2284 static int 2285 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 2286 { 2287 bool newval; 2288 int error; 2289 struct sysctlnode node; 2290 2291 node = *rnode; 2292 node.sysctl_data = &newval; 2293 newval = bpf_jit; 2294 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2295 if (error != 0 || newp == NULL) 2296 return error; 2297 2298 bpf_jit = newval; 2299 2300 /* 2301 * Do a full sync to publish new bpf_jit value and 2302 * update bpfjit_module_ops.bj_generate_code variable. 2303 */ 2304 membar_sync(); 2305 2306 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 2307 printf("JIT compilation is postponed " 2308 "until after bpfjit module is loaded\n"); 2309 } 2310 2311 return 0; 2312 } 2313 #endif 2314 2315 static int 2316 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 2317 { 2318 int error, elem_count; 2319 struct bpf_d *dp; 2320 struct bpf_d_ext dpe; 2321 size_t len, needed, elem_size, out_size; 2322 char *sp; 2323 2324 if (namelen == 1 && name[0] == CTL_QUERY) 2325 return (sysctl_query(SYSCTLFN_CALL(rnode))); 2326 2327 if (namelen != 2) 2328 return (EINVAL); 2329 2330 /* BPF peers is privileged information. */ 2331 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 2332 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 2333 if (error) 2334 return (EPERM); 2335 2336 len = (oldp != NULL) ? *oldlenp : 0; 2337 sp = oldp; 2338 elem_size = name[0]; 2339 elem_count = name[1]; 2340 out_size = MIN(sizeof(dpe), elem_size); 2341 needed = 0; 2342 2343 if (elem_size < 1 || elem_count < 0) 2344 return (EINVAL); 2345 2346 mutex_enter(&bpf_mtx); 2347 BPF_DLIST_WRITER_FOREACH(dp) { 2348 if (len >= elem_size && elem_count > 0) { 2349 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 2350 BPF_EXT(bufsize); 2351 BPF_EXT(promisc); 2352 BPF_EXT(state); 2353 BPF_EXT(immediate); 2354 BPF_EXT(hdrcmplt); 2355 BPF_EXT(seesent); 2356 BPF_EXT(pid); 2357 BPF_EXT(rcount); 2358 BPF_EXT(dcount); 2359 BPF_EXT(ccount); 2360 #undef BPF_EXT 2361 mutex_enter(dp->bd_mtx); 2362 if (dp->bd_bif) 2363 (void)strlcpy(dpe.bde_ifname, 2364 dp->bd_bif->bif_ifp->if_xname, 2365 IFNAMSIZ - 1); 2366 else 2367 dpe.bde_ifname[0] = '\0'; 2368 mutex_exit(dp->bd_mtx); 2369 2370 error = copyout(&dpe, sp, out_size); 2371 if (error) 2372 break; 2373 sp += elem_size; 2374 len -= elem_size; 2375 } 2376 needed += elem_size; 2377 if (elem_count > 0 && elem_count != INT_MAX) 2378 elem_count--; 2379 } 2380 mutex_exit(&bpf_mtx); 2381 2382 *oldlenp = needed; 2383 2384 return (error); 2385 } 2386 2387 static void 2388 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused) 2389 { 2390 struct bpf_stat *const stats = p; 2391 struct bpf_stat *sum = arg; 2392 2393 sum->bs_recv += stats->bs_recv; 2394 sum->bs_drop += stats->bs_drop; 2395 sum->bs_capt += stats->bs_capt; 2396 } 2397 2398 static int 2399 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS) 2400 { 2401 struct sysctlnode node; 2402 int error; 2403 struct bpf_stat sum; 2404 2405 memset(&sum, 0, sizeof(sum)); 2406 node = *rnode; 2407 2408 percpu_foreach(bpf_gstats_percpu, bpf_stats, &sum); 2409 2410 node.sysctl_data = ∑ 2411 node.sysctl_size = sizeof(sum); 2412 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2413 if (error != 0 || newp == NULL) 2414 return error; 2415 2416 return 0; 2417 } 2418 2419 static struct sysctllog *bpf_sysctllog; 2420 static void 2421 sysctl_net_bpf_setup(void) 2422 { 2423 const struct sysctlnode *node; 2424 2425 node = NULL; 2426 sysctl_createv(&bpf_sysctllog, 0, NULL, &node, 2427 CTLFLAG_PERMANENT, 2428 CTLTYPE_NODE, "bpf", 2429 SYSCTL_DESCR("BPF options"), 2430 NULL, 0, NULL, 0, 2431 CTL_NET, CTL_CREATE, CTL_EOL); 2432 if (node != NULL) { 2433 #if defined(MODULAR) || defined(BPFJIT) 2434 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2435 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2436 CTLTYPE_BOOL, "jit", 2437 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2438 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2439 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2440 #endif 2441 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2442 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2443 CTLTYPE_INT, "maxbufsize", 2444 SYSCTL_DESCR("Maximum size for data capture buffer"), 2445 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2446 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2447 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2448 CTLFLAG_PERMANENT, 2449 CTLTYPE_STRUCT, "stats", 2450 SYSCTL_DESCR("BPF stats"), 2451 bpf_sysctl_gstats_handler, 0, NULL, 0, 2452 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2453 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2454 CTLFLAG_PERMANENT, 2455 CTLTYPE_STRUCT, "peers", 2456 SYSCTL_DESCR("BPF peers"), 2457 sysctl_net_bpf_peers, 0, NULL, 0, 2458 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2459 } 2460 2461 } 2462 2463 struct bpf_ops bpf_ops_kernel = { 2464 .bpf_attach = _bpfattach, 2465 .bpf_detach = _bpfdetach, 2466 .bpf_change_type = _bpf_change_type, 2467 2468 .bpf_tap = _bpf_tap, 2469 .bpf_mtap = _bpf_mtap, 2470 .bpf_mtap2 = _bpf_mtap2, 2471 .bpf_mtap_af = _bpf_mtap_af, 2472 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2473 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2474 2475 .bpf_mtap_softint = _bpf_mtap_softint, 2476 .bpf_mtap_softint_init = _bpf_mtap_softint_init, 2477 }; 2478 2479 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); 2480 2481 static int 2482 bpf_modcmd(modcmd_t cmd, void *arg) 2483 { 2484 #ifdef _MODULE 2485 devmajor_t bmajor, cmajor; 2486 #endif 2487 int error = 0; 2488 2489 switch (cmd) { 2490 case MODULE_CMD_INIT: 2491 bpf_init(); 2492 #ifdef _MODULE 2493 bmajor = cmajor = NODEVMAJOR; 2494 error = devsw_attach("bpf", NULL, &bmajor, 2495 &bpf_cdevsw, &cmajor); 2496 if (error) 2497 break; 2498 #endif 2499 2500 bpf_ops_handover_enter(&bpf_ops_kernel); 2501 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2502 bpf_ops_handover_exit(); 2503 sysctl_net_bpf_setup(); 2504 break; 2505 2506 case MODULE_CMD_FINI: 2507 /* 2508 * While there is no reference counting for bpf callers, 2509 * unload could at least in theory be done similarly to 2510 * system call disestablishment. This should even be 2511 * a little simpler: 2512 * 2513 * 1) replace op vector with stubs 2514 * 2) post update to all cpus with xc 2515 * 3) check that nobody is in bpf anymore 2516 * (it's doubtful we'd want something like l_sysent, 2517 * but we could do something like *signed* percpu 2518 * counters. if the sum is 0, we're good). 2519 * 4) if fail, unroll changes 2520 * 2521 * NOTE: change won't be atomic to the outside. some 2522 * packets may be not captured even if unload is 2523 * not succesful. I think packet capture not working 2524 * is a perfectly logical consequence of trying to 2525 * disable packet capture. 2526 */ 2527 error = EOPNOTSUPP; 2528 /* insert sysctl teardown */ 2529 break; 2530 2531 default: 2532 error = ENOTTY; 2533 break; 2534 } 2535 2536 return error; 2537 } 2538