1 /* $NetBSD: bpf.c,v 1.247 2022/09/03 10:03:20 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.247 2022/09/03 10:03:20 riastradh Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "opt_net_mpsafe.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/mbuf.h> 53 #include <sys/buf.h> 54 #include <sys/time.h> 55 #include <sys/proc.h> 56 #include <sys/ioctl.h> 57 #include <sys/conf.h> 58 #include <sys/vnode.h> 59 #include <sys/queue.h> 60 #include <sys/stat.h> 61 #include <sys/module.h> 62 #include <sys/atomic.h> 63 #include <sys/cpu.h> 64 65 #include <sys/file.h> 66 #include <sys/filedesc.h> 67 #include <sys/tty.h> 68 #include <sys/uio.h> 69 70 #include <sys/protosw.h> 71 #include <sys/socket.h> 72 #include <sys/errno.h> 73 #include <sys/kernel.h> 74 #include <sys/poll.h> 75 #include <sys/sysctl.h> 76 #include <sys/kauth.h> 77 #include <sys/syslog.h> 78 #include <sys/percpu.h> 79 #include <sys/pserialize.h> 80 #include <sys/lwp.h> 81 #include <sys/xcall.h> 82 83 #include <net/if.h> 84 #include <net/slip.h> 85 86 #include <net/bpf.h> 87 #include <net/bpfdesc.h> 88 #include <net/bpfjit.h> 89 90 #include <net/if_arc.h> 91 #include <net/if_ether.h> 92 93 #include <netinet/in.h> 94 #include <netinet/if_inarp.h> 95 96 97 #include <compat/sys/sockio.h> 98 99 #ifndef BPF_BUFSIZE 100 /* 101 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 102 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 103 */ 104 # define BPF_BUFSIZE 32768 105 #endif 106 107 #define PRINET 26 /* interruptible */ 108 109 /* 110 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 111 * XXX the default values should be computed dynamically based 112 * on available memory size and available mbuf clusters. 113 */ 114 static int bpf_bufsize = BPF_BUFSIZE; 115 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 116 static bool bpf_jit = false; 117 118 struct bpfjit_ops bpfjit_module_ops = { 119 .bj_generate_code = NULL, 120 .bj_free_code = NULL 121 }; 122 123 /* 124 * Global BPF statistics returned by net.bpf.stats sysctl. 125 */ 126 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */ 127 128 #define BPF_STATINC(id) \ 129 { \ 130 struct bpf_stat *__stats = \ 131 percpu_getref(bpf_gstats_percpu); \ 132 __stats->bs_##id++; \ 133 percpu_putref(bpf_gstats_percpu); \ 134 } 135 136 /* 137 * Locking notes: 138 * - bpf_mtx (adaptive mutex) protects: 139 * - Gobal lists: bpf_iflist and bpf_dlist 140 * - struct bpf_if 141 * - bpf_close 142 * - bpf_psz (pserialize) 143 * - struct bpf_d has two mutexes: 144 * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed 145 * on packet tapping 146 * - bd_mtx (adaptive mutex) protects member variables other than the buffers 147 * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx 148 * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is 149 * never freed because struct bpf_d is only freed in bpf_close and 150 * bpf_close never be called while executing bpf_read and bpf_write 151 * - A filter that is assigned to bpf_d can be replaced with another filter 152 * while tapping packets, so it needs to be done atomically 153 * - struct bpf_d is iterated on bpf_dlist with psz 154 * - struct bpf_if is iterated on bpf_iflist with psz or psref 155 */ 156 /* 157 * Use a mutex to avoid a race condition between gathering the stats/peers 158 * and opening/closing the device. 159 */ 160 static kmutex_t bpf_mtx; 161 162 static struct psref_class *bpf_psref_class __read_mostly; 163 static pserialize_t bpf_psz; 164 165 static inline void 166 bpf_if_acquire(struct bpf_if *bp, struct psref *psref) 167 { 168 169 psref_acquire(psref, &bp->bif_psref, bpf_psref_class); 170 } 171 172 static inline void 173 bpf_if_release(struct bpf_if *bp, struct psref *psref) 174 { 175 176 psref_release(psref, &bp->bif_psref, bpf_psref_class); 177 } 178 179 /* 180 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 181 * bpf_dtab holds the descriptors, indexed by minor device # 182 */ 183 static struct pslist_head bpf_iflist; 184 static struct pslist_head bpf_dlist; 185 186 /* Macros for bpf_d on bpf_dlist */ 187 #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \ 188 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) 189 #define BPF_DLIST_READER_FOREACH(__d) \ 190 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 191 bd_bpf_dlist_entry) 192 #define BPF_DLIST_WRITER_FOREACH(__d) \ 193 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 194 bd_bpf_dlist_entry) 195 #define BPF_DLIST_ENTRY_INIT(__d) \ 196 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) 197 #define BPF_DLIST_WRITER_REMOVE(__d) \ 198 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) 199 #define BPF_DLIST_ENTRY_DESTROY(__d) \ 200 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) 201 202 /* Macros for bpf_if on bpf_iflist */ 203 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ 204 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) 205 #define BPF_IFLIST_READER_FOREACH(__bp) \ 206 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 207 bif_iflist_entry) 208 #define BPF_IFLIST_WRITER_FOREACH(__bp) \ 209 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 210 bif_iflist_entry) 211 #define BPF_IFLIST_WRITER_REMOVE(__bp) \ 212 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) 213 #define BPF_IFLIST_ENTRY_INIT(__bp) \ 214 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) 215 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ 216 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) 217 218 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ 219 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ 220 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ 221 bd_bif_dlist_entry) 222 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ 223 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ 224 bd_bif_dlist_entry) 225 #define BPFIF_DLIST_WRITER_REMOVE(__d) \ 226 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) 227 #define BPFIF_DLIST_ENTRY_INIT(__d) \ 228 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) 229 #define BPFIF_DLIST_READER_EMPTY(__bp) \ 230 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 231 bd_bif_dlist_entry) == NULL) 232 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ 233 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 234 bd_bif_dlist_entry) == NULL) 235 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ 236 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) 237 238 static int bpf_allocbufs(struct bpf_d *); 239 static u_int bpf_xfilter(struct bpf_filter **, void *, u_int, u_int); 240 static void bpf_deliver(struct bpf_if *, 241 void *(*cpfn)(void *, const void *, size_t), 242 void *, u_int, u_int, const u_int); 243 static void bpf_freed(struct bpf_d *); 244 static void bpf_free_filter(struct bpf_filter *); 245 static void bpf_ifname(struct ifnet *, struct ifreq *); 246 static void *bpf_mcpy(void *, const void *, size_t); 247 static int bpf_movein(struct uio *, int, uint64_t, 248 struct mbuf **, struct sockaddr *, 249 struct bpf_filter **); 250 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 251 static void bpf_detachd(struct bpf_d *); 252 static int bpf_setif(struct bpf_d *, struct ifreq *); 253 static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long); 254 static void bpf_timed_out(void *); 255 static inline void 256 bpf_wakeup(struct bpf_d *); 257 static int bpf_hdrlen(struct bpf_d *); 258 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 259 void *(*)(void *, const void *, size_t), struct timespec *); 260 static void reset_d(struct bpf_d *); 261 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 262 static int bpf_setdlt(struct bpf_d *, u_int); 263 264 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 265 int); 266 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 267 int); 268 static int bpf_ioctl(struct file *, u_long, void *); 269 static int bpf_poll(struct file *, int); 270 static int bpf_stat(struct file *, struct stat *); 271 static int bpf_close(struct file *); 272 static int bpf_kqfilter(struct file *, struct knote *); 273 274 static const struct fileops bpf_fileops = { 275 .fo_name = "bpf", 276 .fo_read = bpf_read, 277 .fo_write = bpf_write, 278 .fo_ioctl = bpf_ioctl, 279 .fo_fcntl = fnullop_fcntl, 280 .fo_poll = bpf_poll, 281 .fo_stat = bpf_stat, 282 .fo_close = bpf_close, 283 .fo_kqfilter = bpf_kqfilter, 284 .fo_restart = fnullop_restart, 285 }; 286 287 dev_type_open(bpfopen); 288 289 const struct cdevsw bpf_cdevsw = { 290 .d_open = bpfopen, 291 .d_close = noclose, 292 .d_read = noread, 293 .d_write = nowrite, 294 .d_ioctl = noioctl, 295 .d_stop = nostop, 296 .d_tty = notty, 297 .d_poll = nopoll, 298 .d_mmap = nommap, 299 .d_kqfilter = nokqfilter, 300 .d_discard = nodiscard, 301 .d_flag = D_OTHER | D_MPSAFE 302 }; 303 304 bpfjit_func_t 305 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 306 { 307 struct bpfjit_ops *ops = &bpfjit_module_ops; 308 bpfjit_func_t (*generate_code)(const bpf_ctx_t *, 309 const struct bpf_insn *, size_t); 310 311 generate_code = atomic_load_acquire(&ops->bj_generate_code); 312 if (generate_code != NULL) { 313 return generate_code(bc, code, size); 314 } 315 return NULL; 316 } 317 318 void 319 bpf_jit_freecode(bpfjit_func_t jcode) 320 { 321 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 322 bpfjit_module_ops.bj_free_code(jcode); 323 } 324 325 static int 326 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 327 struct sockaddr *sockp, struct bpf_filter **wfilter) 328 { 329 struct mbuf *m, *m0, *n; 330 int error; 331 size_t len; 332 size_t hlen; 333 size_t align; 334 u_int slen; 335 336 /* 337 * Build a sockaddr based on the data link layer type. 338 * We do this at this level because the ethernet header 339 * is copied directly into the data field of the sockaddr. 340 * In the case of SLIP, there is no header and the packet 341 * is forwarded as is. 342 * Also, we are careful to leave room at the front of the mbuf 343 * for the link level header. 344 */ 345 switch (linktype) { 346 347 case DLT_SLIP: 348 sockp->sa_family = AF_INET; 349 hlen = 0; 350 align = 0; 351 break; 352 353 case DLT_PPP: 354 sockp->sa_family = AF_UNSPEC; 355 hlen = 0; 356 align = 0; 357 break; 358 359 case DLT_EN10MB: 360 sockp->sa_family = AF_UNSPEC; 361 /* XXX Would MAXLINKHDR be better? */ 362 /* 6(dst)+6(src)+2(type) */ 363 hlen = sizeof(struct ether_header); 364 align = 2; 365 break; 366 367 case DLT_ARCNET: 368 sockp->sa_family = AF_UNSPEC; 369 hlen = ARC_HDRLEN; 370 align = 5; 371 break; 372 373 case DLT_FDDI: 374 sockp->sa_family = AF_LINK; 375 /* XXX 4(FORMAC)+6(dst)+6(src) */ 376 hlen = 16; 377 align = 0; 378 break; 379 380 case DLT_ECONET: 381 sockp->sa_family = AF_UNSPEC; 382 hlen = 6; 383 align = 2; 384 break; 385 386 case DLT_NULL: 387 sockp->sa_family = AF_UNSPEC; 388 hlen = 0; 389 align = 0; 390 break; 391 392 default: 393 return (EIO); 394 } 395 396 len = uio->uio_resid; 397 /* 398 * If there aren't enough bytes for a link level header or the 399 * packet length exceeds the interface mtu, return an error. 400 */ 401 if (len - hlen > mtu) 402 return (EMSGSIZE); 403 404 m0 = m = m_gethdr(M_WAIT, MT_DATA); 405 m_reset_rcvif(m); 406 m->m_pkthdr.len = (int)(len - hlen); 407 if (len + align > MHLEN) { 408 m_clget(m, M_WAIT); 409 if ((m->m_flags & M_EXT) == 0) { 410 error = ENOBUFS; 411 goto bad; 412 } 413 } 414 415 /* Insure the data is properly aligned */ 416 if (align > 0) 417 m->m_data += align; 418 419 for (;;) { 420 len = M_TRAILINGSPACE(m); 421 if (len > uio->uio_resid) 422 len = uio->uio_resid; 423 error = uiomove(mtod(m, void *), len, uio); 424 if (error) 425 goto bad; 426 m->m_len = len; 427 428 if (uio->uio_resid == 0) 429 break; 430 431 n = m_get(M_WAIT, MT_DATA); 432 m_clget(n, M_WAIT); /* if fails, there is no problem */ 433 m->m_next = n; 434 m = n; 435 } 436 437 slen = bpf_xfilter(wfilter, mtod(m, u_char *), len, len); 438 if (slen == 0) { 439 error = EPERM; 440 goto bad; 441 } 442 443 if (hlen != 0) { 444 /* move link level header in the top of mbuf to sa_data */ 445 memcpy(sockp->sa_data, mtod(m0, void *), hlen); 446 m0->m_data += hlen; 447 m0->m_len -= hlen; 448 } 449 450 *mp = m0; 451 return (0); 452 453 bad: 454 m_freem(m0); 455 return (error); 456 } 457 458 /* 459 * Attach file to the bpf interface, i.e. make d listen on bp. 460 */ 461 static void 462 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 463 { 464 struct bpf_event_tracker *t; 465 466 KASSERT(mutex_owned(&bpf_mtx)); 467 KASSERT(mutex_owned(d->bd_mtx)); 468 /* 469 * Point d at bp, and add d to the interface's list of listeners. 470 * Finally, point the driver's bpf cookie at the interface so 471 * it will divert packets to bpf. 472 */ 473 d->bd_bif = bp; 474 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); 475 476 *bp->bif_driverp = bp; 477 478 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { 479 t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt, 480 BPF_TRACK_EVENT_ATTACH); 481 } 482 } 483 484 /* 485 * Detach a file from its interface. 486 */ 487 static void 488 bpf_detachd(struct bpf_d *d) 489 { 490 struct bpf_if *bp; 491 struct bpf_event_tracker *t; 492 493 KASSERT(mutex_owned(&bpf_mtx)); 494 KASSERT(mutex_owned(d->bd_mtx)); 495 496 bp = d->bd_bif; 497 /* 498 * Check if this descriptor had requested promiscuous mode. 499 * If so, turn it off. 500 */ 501 if (d->bd_promisc) { 502 int error __diagused; 503 504 d->bd_promisc = 0; 505 /* 506 * Take device out of promiscuous mode. Since we were 507 * able to enter promiscuous mode, we should be able 508 * to turn it off. But we can get an error if 509 * the interface was configured down, so only panic 510 * if we don't get an unexpected error. 511 */ 512 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 513 error = ifpromisc(bp->bif_ifp, 0); 514 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 515 #ifdef DIAGNOSTIC 516 if (error) 517 printf("%s: ifpromisc failed: %d", __func__, error); 518 #endif 519 } 520 521 /* Remove d from the interface's descriptor list. */ 522 BPFIF_DLIST_WRITER_REMOVE(d); 523 524 pserialize_perform(bpf_psz); 525 526 if (BPFIF_DLIST_WRITER_EMPTY(bp)) { 527 /* 528 * Let the driver know that there are no more listeners. 529 */ 530 *d->bd_bif->bif_driverp = NULL; 531 } 532 533 d->bd_bif = NULL; 534 535 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { 536 t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt, 537 BPF_TRACK_EVENT_DETACH); 538 } 539 } 540 541 static void 542 bpf_init(void) 543 { 544 545 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 546 bpf_psz = pserialize_create(); 547 bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET); 548 549 PSLIST_INIT(&bpf_iflist); 550 PSLIST_INIT(&bpf_dlist); 551 552 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat)); 553 554 return; 555 } 556 557 /* 558 * bpfilterattach() is called at boot time. We don't need to do anything 559 * here, since any initialization will happen as part of module init code. 560 */ 561 /* ARGSUSED */ 562 void 563 bpfilterattach(int n) 564 { 565 566 } 567 568 /* 569 * Open ethernet device. Clones. 570 */ 571 /* ARGSUSED */ 572 int 573 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 574 { 575 struct bpf_d *d; 576 struct file *fp; 577 int error, fd; 578 579 /* falloc() will fill in the descriptor for us. */ 580 if ((error = fd_allocfile(&fp, &fd)) != 0) 581 return error; 582 583 d = kmem_zalloc(sizeof(*d), KM_SLEEP); 584 d->bd_bufsize = bpf_bufsize; 585 d->bd_direction = BPF_D_INOUT; 586 d->bd_feedback = 0; 587 d->bd_pid = l->l_proc->p_pid; 588 #ifdef _LP64 589 if (curproc->p_flag & PK_32) 590 d->bd_compat32 = 1; 591 #endif 592 getnanotime(&d->bd_btime); 593 d->bd_atime = d->bd_mtime = d->bd_btime; 594 callout_init(&d->bd_callout, CALLOUT_MPSAFE); 595 selinit(&d->bd_sel); 596 d->bd_jitcode = NULL; 597 d->bd_rfilter = NULL; 598 d->bd_wfilter = NULL; 599 d->bd_locked = 0; 600 BPF_DLIST_ENTRY_INIT(d); 601 BPFIF_DLIST_ENTRY_INIT(d); 602 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 603 d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 604 cv_init(&d->bd_cv, "bpf"); 605 606 mutex_enter(&bpf_mtx); 607 BPF_DLIST_WRITER_INSERT_HEAD(d); 608 mutex_exit(&bpf_mtx); 609 610 return fd_clone(fp, fd, flag, &bpf_fileops, d); 611 } 612 613 /* 614 * Close the descriptor by detaching it from its interface, 615 * deallocating its buffers, and marking it free. 616 */ 617 /* ARGSUSED */ 618 static int 619 bpf_close(struct file *fp) 620 { 621 struct bpf_d *d; 622 623 mutex_enter(&bpf_mtx); 624 625 if ((d = fp->f_bpf) == NULL) { 626 mutex_exit(&bpf_mtx); 627 return 0; 628 } 629 630 /* 631 * Refresh the PID associated with this bpf file. 632 */ 633 d->bd_pid = curproc->p_pid; 634 635 mutex_enter(d->bd_mtx); 636 if (d->bd_state == BPF_WAITING) 637 callout_halt(&d->bd_callout, d->bd_mtx); 638 d->bd_state = BPF_IDLE; 639 if (d->bd_bif) 640 bpf_detachd(d); 641 mutex_exit(d->bd_mtx); 642 643 BPF_DLIST_WRITER_REMOVE(d); 644 645 pserialize_perform(bpf_psz); 646 mutex_exit(&bpf_mtx); 647 648 BPFIF_DLIST_ENTRY_DESTROY(d); 649 BPF_DLIST_ENTRY_DESTROY(d); 650 fp->f_bpf = NULL; 651 bpf_freed(d); 652 callout_destroy(&d->bd_callout); 653 seldestroy(&d->bd_sel); 654 mutex_obj_free(d->bd_mtx); 655 mutex_obj_free(d->bd_buf_mtx); 656 cv_destroy(&d->bd_cv); 657 658 kmem_free(d, sizeof(*d)); 659 660 return (0); 661 } 662 663 /* 664 * Rotate the packet buffers in descriptor d. Move the store buffer 665 * into the hold slot, and the free buffer into the store slot. 666 * Zero the length of the new store buffer. 667 */ 668 #define ROTATE_BUFFERS(d) \ 669 (d)->bd_hbuf = (d)->bd_sbuf; \ 670 (d)->bd_hlen = (d)->bd_slen; \ 671 (d)->bd_sbuf = (d)->bd_fbuf; \ 672 (d)->bd_slen = 0; \ 673 (d)->bd_fbuf = NULL; 674 /* 675 * bpfread - read next chunk of packets from buffers 676 */ 677 static int 678 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 679 kauth_cred_t cred, int flags) 680 { 681 struct bpf_d *d = fp->f_bpf; 682 int timed_out; 683 int error; 684 685 getnanotime(&d->bd_atime); 686 /* 687 * Restrict application to use a buffer the same size as 688 * the kernel buffers. 689 */ 690 if (uio->uio_resid != d->bd_bufsize) 691 return (EINVAL); 692 693 mutex_enter(d->bd_mtx); 694 if (d->bd_state == BPF_WAITING) 695 callout_halt(&d->bd_callout, d->bd_mtx); 696 timed_out = (d->bd_state == BPF_TIMED_OUT); 697 d->bd_state = BPF_IDLE; 698 mutex_exit(d->bd_mtx); 699 /* 700 * If the hold buffer is empty, then do a timed sleep, which 701 * ends when the timeout expires or when enough packets 702 * have arrived to fill the store buffer. 703 */ 704 mutex_enter(d->bd_buf_mtx); 705 while (d->bd_hbuf == NULL) { 706 if (fp->f_flag & FNONBLOCK) { 707 if (d->bd_slen == 0) { 708 error = EWOULDBLOCK; 709 goto out; 710 } 711 ROTATE_BUFFERS(d); 712 break; 713 } 714 715 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 716 /* 717 * A packet(s) either arrived since the previous 718 * read or arrived while we were asleep. 719 * Rotate the buffers and return what's here. 720 */ 721 ROTATE_BUFFERS(d); 722 break; 723 } 724 725 error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout); 726 727 if (error == EINTR || error == ERESTART) 728 goto out; 729 730 if (error == EWOULDBLOCK) { 731 /* 732 * On a timeout, return what's in the buffer, 733 * which may be nothing. If there is something 734 * in the store buffer, we can rotate the buffers. 735 */ 736 if (d->bd_hbuf) 737 /* 738 * We filled up the buffer in between 739 * getting the timeout and arriving 740 * here, so we don't need to rotate. 741 */ 742 break; 743 744 if (d->bd_slen == 0) { 745 error = 0; 746 goto out; 747 } 748 ROTATE_BUFFERS(d); 749 break; 750 } 751 if (error != 0) 752 goto out; 753 } 754 /* 755 * At this point, we know we have something in the hold slot. 756 */ 757 mutex_exit(d->bd_buf_mtx); 758 759 /* 760 * Move data from hold buffer into user space. 761 * We know the entire buffer is transferred since 762 * we checked above that the read buffer is bpf_bufsize bytes. 763 */ 764 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 765 766 mutex_enter(d->bd_buf_mtx); 767 d->bd_fbuf = d->bd_hbuf; 768 d->bd_hbuf = NULL; 769 d->bd_hlen = 0; 770 out: 771 mutex_exit(d->bd_buf_mtx); 772 return (error); 773 } 774 775 776 /* 777 * If there are processes sleeping on this descriptor, wake them up. 778 */ 779 static inline void 780 bpf_wakeup(struct bpf_d *d) 781 { 782 783 mutex_enter(d->bd_buf_mtx); 784 cv_broadcast(&d->bd_cv); 785 mutex_exit(d->bd_buf_mtx); 786 787 if (d->bd_async) 788 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 789 selnotify(&d->bd_sel, 0, 0); 790 } 791 792 static void 793 bpf_timed_out(void *arg) 794 { 795 struct bpf_d *d = arg; 796 797 mutex_enter(d->bd_mtx); 798 if (d->bd_state == BPF_WAITING) { 799 d->bd_state = BPF_TIMED_OUT; 800 if (d->bd_slen != 0) 801 bpf_wakeup(d); 802 } 803 mutex_exit(d->bd_mtx); 804 } 805 806 807 static int 808 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 809 kauth_cred_t cred, int flags) 810 { 811 struct bpf_d *d = fp->f_bpf; 812 struct bpf_if *bp; 813 struct ifnet *ifp; 814 struct mbuf *m, *mc; 815 int error; 816 static struct sockaddr_storage dst; 817 struct psref psref; 818 int bound; 819 820 m = NULL; /* XXX gcc */ 821 822 bound = curlwp_bind(); 823 mutex_enter(d->bd_mtx); 824 bp = d->bd_bif; 825 if (bp == NULL) { 826 mutex_exit(d->bd_mtx); 827 error = ENXIO; 828 goto out_bindx; 829 } 830 bpf_if_acquire(bp, &psref); 831 mutex_exit(d->bd_mtx); 832 833 getnanotime(&d->bd_mtime); 834 835 ifp = bp->bif_ifp; 836 if (if_is_deactivated(ifp)) { 837 error = ENXIO; 838 goto out; 839 } 840 841 if (uio->uio_resid == 0) { 842 error = 0; 843 goto out; 844 } 845 846 error = bpf_movein(uio, (int)bp->bif_dlt, ifp->if_mtu, &m, 847 (struct sockaddr *) &dst, &d->bd_wfilter); 848 if (error) 849 goto out; 850 851 if (m->m_pkthdr.len > ifp->if_mtu) { 852 m_freem(m); 853 error = EMSGSIZE; 854 goto out; 855 } 856 857 if (d->bd_hdrcmplt) 858 dst.ss_family = pseudo_AF_HDRCMPLT; 859 860 if (d->bd_feedback) { 861 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 862 if (mc != NULL) 863 m_set_rcvif(mc, ifp); 864 /* Set M_PROMISC for outgoing packets to be discarded. */ 865 if (1 /*d->bd_direction == BPF_D_INOUT*/) 866 m->m_flags |= M_PROMISC; 867 } else 868 mc = NULL; 869 870 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); 871 872 if (mc != NULL) { 873 if (error == 0) { 874 int s = splsoftnet(); 875 KERNEL_LOCK_UNLESS_IFP_MPSAFE(ifp); 876 ifp->_if_input(ifp, mc); 877 KERNEL_UNLOCK_UNLESS_IFP_MPSAFE(ifp); 878 splx(s); 879 } else 880 m_freem(mc); 881 } 882 /* 883 * The driver frees the mbuf. 884 */ 885 out: 886 bpf_if_release(bp, &psref); 887 out_bindx: 888 curlwp_bindx(bound); 889 return error; 890 } 891 892 /* 893 * Reset a descriptor by flushing its packet buffer and clearing the 894 * receive and drop counts. 895 */ 896 static void 897 reset_d(struct bpf_d *d) 898 { 899 900 KASSERT(mutex_owned(d->bd_mtx)); 901 902 mutex_enter(d->bd_buf_mtx); 903 if (d->bd_hbuf) { 904 /* Free the hold buffer. */ 905 d->bd_fbuf = d->bd_hbuf; 906 d->bd_hbuf = NULL; 907 } 908 d->bd_slen = 0; 909 d->bd_hlen = 0; 910 d->bd_rcount = 0; 911 d->bd_dcount = 0; 912 d->bd_ccount = 0; 913 mutex_exit(d->bd_buf_mtx); 914 } 915 916 /* 917 * FIONREAD Check for read packet available. 918 * BIOCGBLEN Get buffer len [for read()]. 919 * BIOCSETF Set ethernet read filter. 920 * BIOCFLUSH Flush read packet buffer. 921 * BIOCPROMISC Put interface into promiscuous mode. 922 * BIOCGDLT Get link layer type. 923 * BIOCGETIF Get interface name. 924 * BIOCSETIF Set interface. 925 * BIOCSRTIMEOUT Set read timeout. 926 * BIOCGRTIMEOUT Get read timeout. 927 * BIOCGSTATS Get packet stats. 928 * BIOCIMMEDIATE Set immediate mode. 929 * BIOCVERSION Get filter language version. 930 * BIOCGHDRCMPLT Get "header already complete" flag. 931 * BIOCSHDRCMPLT Set "header already complete" flag. 932 * BIOCSFEEDBACK Set packet feedback mode. 933 * BIOCGFEEDBACK Get packet feedback mode. 934 * BIOCGDIRECTION Get packet direction flag 935 * BIOCSDIRECTION Set packet direction flag 936 */ 937 /* ARGSUSED */ 938 static int 939 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 940 { 941 struct bpf_d *d = fp->f_bpf; 942 int error = 0; 943 944 /* 945 * Refresh the PID associated with this bpf file. 946 */ 947 d->bd_pid = curproc->p_pid; 948 #ifdef _LP64 949 if (curproc->p_flag & PK_32) 950 d->bd_compat32 = 1; 951 else 952 d->bd_compat32 = 0; 953 #endif 954 955 mutex_enter(d->bd_mtx); 956 if (d->bd_state == BPF_WAITING) 957 callout_halt(&d->bd_callout, d->bd_mtx); 958 d->bd_state = BPF_IDLE; 959 mutex_exit(d->bd_mtx); 960 961 if (d->bd_locked) { 962 switch (cmd) { 963 case BIOCGBLEN: /* FALLTHROUGH */ 964 case BIOCFLUSH: /* FALLTHROUGH */ 965 case BIOCGDLT: /* FALLTHROUGH */ 966 case BIOCGDLTLIST: /* FALLTHROUGH */ 967 case BIOCGETIF: /* FALLTHROUGH */ 968 case BIOCGRTIMEOUT: /* FALLTHROUGH */ 969 case BIOCGSTATS: /* FALLTHROUGH */ 970 case BIOCVERSION: /* FALLTHROUGH */ 971 case BIOCGHDRCMPLT: /* FALLTHROUGH */ 972 case FIONREAD: /* FALLTHROUGH */ 973 case BIOCLOCK: /* FALLTHROUGH */ 974 case BIOCSRTIMEOUT: /* FALLTHROUGH */ 975 case BIOCIMMEDIATE: /* FALLTHROUGH */ 976 case TIOCGPGRP: 977 break; 978 default: 979 return EPERM; 980 } 981 } 982 983 switch (cmd) { 984 985 default: 986 error = EINVAL; 987 break; 988 989 /* 990 * Check for read packet available. 991 */ 992 case FIONREAD: 993 { 994 int n; 995 996 mutex_enter(d->bd_buf_mtx); 997 n = d->bd_slen; 998 if (d->bd_hbuf) 999 n += d->bd_hlen; 1000 mutex_exit(d->bd_buf_mtx); 1001 1002 *(int *)addr = n; 1003 break; 1004 } 1005 1006 /* 1007 * Get buffer len [for read()]. 1008 */ 1009 case BIOCGBLEN: 1010 *(u_int *)addr = d->bd_bufsize; 1011 break; 1012 1013 /* 1014 * Set buffer length. 1015 */ 1016 case BIOCSBLEN: 1017 /* 1018 * Forbid to change the buffer length if buffers are already 1019 * allocated. 1020 */ 1021 mutex_enter(d->bd_mtx); 1022 mutex_enter(d->bd_buf_mtx); 1023 if (d->bd_bif != NULL || d->bd_sbuf != NULL) 1024 error = EINVAL; 1025 else { 1026 u_int size = *(u_int *)addr; 1027 1028 if (size > bpf_maxbufsize) 1029 *(u_int *)addr = size = bpf_maxbufsize; 1030 else if (size < BPF_MINBUFSIZE) 1031 *(u_int *)addr = size = BPF_MINBUFSIZE; 1032 d->bd_bufsize = size; 1033 } 1034 mutex_exit(d->bd_buf_mtx); 1035 mutex_exit(d->bd_mtx); 1036 break; 1037 1038 /* 1039 * Set link layer read filter. 1040 */ 1041 case BIOCSETF: /* FALLTHROUGH */ 1042 case BIOCSETWF: 1043 error = bpf_setf(d, addr, cmd); 1044 break; 1045 1046 case BIOCLOCK: 1047 d->bd_locked = 1; 1048 break; 1049 1050 /* 1051 * Flush read packet buffer. 1052 */ 1053 case BIOCFLUSH: 1054 mutex_enter(d->bd_mtx); 1055 reset_d(d); 1056 mutex_exit(d->bd_mtx); 1057 break; 1058 1059 /* 1060 * Put interface into promiscuous mode. 1061 */ 1062 case BIOCPROMISC: 1063 mutex_enter(d->bd_mtx); 1064 if (d->bd_bif == NULL) { 1065 mutex_exit(d->bd_mtx); 1066 /* 1067 * No interface attached yet. 1068 */ 1069 error = EINVAL; 1070 break; 1071 } 1072 if (d->bd_promisc == 0) { 1073 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 1074 error = ifpromisc(d->bd_bif->bif_ifp, 1); 1075 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 1076 if (error == 0) 1077 d->bd_promisc = 1; 1078 } 1079 mutex_exit(d->bd_mtx); 1080 break; 1081 1082 /* 1083 * Get device parameters. 1084 */ 1085 case BIOCGDLT: 1086 mutex_enter(d->bd_mtx); 1087 if (d->bd_bif == NULL) 1088 error = EINVAL; 1089 else 1090 *(u_int *)addr = d->bd_bif->bif_dlt; 1091 mutex_exit(d->bd_mtx); 1092 break; 1093 1094 /* 1095 * Get a list of supported device parameters. 1096 */ 1097 case BIOCGDLTLIST: 1098 mutex_enter(d->bd_mtx); 1099 if (d->bd_bif == NULL) 1100 error = EINVAL; 1101 else 1102 error = bpf_getdltlist(d, addr); 1103 mutex_exit(d->bd_mtx); 1104 break; 1105 1106 /* 1107 * Set device parameters. 1108 */ 1109 case BIOCSDLT: 1110 mutex_enter(&bpf_mtx); 1111 mutex_enter(d->bd_mtx); 1112 if (d->bd_bif == NULL) 1113 error = EINVAL; 1114 else 1115 error = bpf_setdlt(d, *(u_int *)addr); 1116 mutex_exit(d->bd_mtx); 1117 mutex_exit(&bpf_mtx); 1118 break; 1119 1120 /* 1121 * Set interface name. 1122 */ 1123 #ifdef OBIOCGETIF 1124 case OBIOCGETIF: 1125 #endif 1126 case BIOCGETIF: 1127 mutex_enter(d->bd_mtx); 1128 if (d->bd_bif == NULL) 1129 error = EINVAL; 1130 else 1131 bpf_ifname(d->bd_bif->bif_ifp, addr); 1132 mutex_exit(d->bd_mtx); 1133 break; 1134 1135 /* 1136 * Set interface. 1137 */ 1138 #ifdef OBIOCSETIF 1139 case OBIOCSETIF: 1140 #endif 1141 case BIOCSETIF: 1142 mutex_enter(&bpf_mtx); 1143 error = bpf_setif(d, addr); 1144 mutex_exit(&bpf_mtx); 1145 break; 1146 1147 /* 1148 * Set read timeout. 1149 */ 1150 case BIOCSRTIMEOUT: 1151 { 1152 struct timeval *tv = addr; 1153 1154 /* Compute number of ticks. */ 1155 if (tv->tv_sec < 0 || 1156 tv->tv_usec < 0 || tv->tv_usec >= 1000000) { 1157 error = EINVAL; 1158 break; 1159 } else if (tv->tv_sec > INT_MAX/hz - 1) { 1160 d->bd_rtout = INT_MAX; 1161 } else { 1162 d->bd_rtout = tv->tv_sec * hz 1163 + tv->tv_usec / tick; 1164 } 1165 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1166 d->bd_rtout = 1; 1167 break; 1168 } 1169 1170 #ifdef BIOCGORTIMEOUT 1171 /* 1172 * Get read timeout. 1173 */ 1174 case BIOCGORTIMEOUT: 1175 { 1176 struct timeval50 *tv = addr; 1177 1178 tv->tv_sec = d->bd_rtout / hz; 1179 tv->tv_usec = (d->bd_rtout % hz) * tick; 1180 break; 1181 } 1182 #endif 1183 1184 #ifdef BIOCSORTIMEOUT 1185 /* 1186 * Set read timeout. 1187 */ 1188 case BIOCSORTIMEOUT: 1189 { 1190 struct timeval50 *tv = addr; 1191 1192 /* Compute number of ticks. */ 1193 if (tv->tv_sec < 0 || 1194 tv->tv_usec < 0 || tv->tv_usec >= 1000000) { 1195 error = EINVAL; 1196 break; 1197 } else if (tv->tv_sec > INT_MAX/hz - 1) { 1198 d->bd_rtout = INT_MAX; 1199 } else { 1200 d->bd_rtout = tv->tv_sec * hz 1201 + tv->tv_usec / tick; 1202 } 1203 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1204 d->bd_rtout = 1; 1205 break; 1206 } 1207 #endif 1208 1209 /* 1210 * Get read timeout. 1211 */ 1212 case BIOCGRTIMEOUT: 1213 { 1214 struct timeval *tv = addr; 1215 1216 tv->tv_sec = d->bd_rtout / hz; 1217 tv->tv_usec = (d->bd_rtout % hz) * tick; 1218 break; 1219 } 1220 /* 1221 * Get packet stats. 1222 */ 1223 case BIOCGSTATS: 1224 { 1225 struct bpf_stat *bs = addr; 1226 1227 bs->bs_recv = d->bd_rcount; 1228 bs->bs_drop = d->bd_dcount; 1229 bs->bs_capt = d->bd_ccount; 1230 break; 1231 } 1232 1233 case BIOCGSTATSOLD: 1234 { 1235 struct bpf_stat_old *bs = addr; 1236 1237 bs->bs_recv = d->bd_rcount; 1238 bs->bs_drop = d->bd_dcount; 1239 break; 1240 } 1241 1242 /* 1243 * Set immediate mode. 1244 */ 1245 case BIOCIMMEDIATE: 1246 d->bd_immediate = *(u_int *)addr; 1247 break; 1248 1249 case BIOCVERSION: 1250 { 1251 struct bpf_version *bv = addr; 1252 1253 bv->bv_major = BPF_MAJOR_VERSION; 1254 bv->bv_minor = BPF_MINOR_VERSION; 1255 break; 1256 } 1257 1258 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1259 *(u_int *)addr = d->bd_hdrcmplt; 1260 break; 1261 1262 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1263 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1264 break; 1265 1266 /* 1267 * Get packet direction flag 1268 */ 1269 case BIOCGDIRECTION: 1270 *(u_int *)addr = d->bd_direction; 1271 break; 1272 1273 /* 1274 * Set packet direction flag 1275 */ 1276 case BIOCSDIRECTION: 1277 { 1278 u_int direction; 1279 1280 direction = *(u_int *)addr; 1281 switch (direction) { 1282 case BPF_D_IN: 1283 case BPF_D_INOUT: 1284 case BPF_D_OUT: 1285 d->bd_direction = direction; 1286 break; 1287 default: 1288 error = EINVAL; 1289 } 1290 } 1291 break; 1292 1293 /* 1294 * Set "feed packets from bpf back to input" mode 1295 */ 1296 case BIOCSFEEDBACK: 1297 d->bd_feedback = *(u_int *)addr; 1298 break; 1299 1300 /* 1301 * Get "feed packets from bpf back to input" mode 1302 */ 1303 case BIOCGFEEDBACK: 1304 *(u_int *)addr = d->bd_feedback; 1305 break; 1306 1307 case FIONBIO: /* Non-blocking I/O */ 1308 /* 1309 * No need to do anything special as we use IO_NDELAY in 1310 * bpfread() as an indication of whether or not to block 1311 * the read. 1312 */ 1313 break; 1314 1315 case FIOASYNC: /* Send signal on receive packets */ 1316 mutex_enter(d->bd_mtx); 1317 d->bd_async = *(int *)addr; 1318 mutex_exit(d->bd_mtx); 1319 break; 1320 1321 case TIOCSPGRP: /* Process or group to send signals to */ 1322 case FIOSETOWN: 1323 error = fsetown(&d->bd_pgid, cmd, addr); 1324 break; 1325 1326 case TIOCGPGRP: 1327 case FIOGETOWN: 1328 error = fgetown(d->bd_pgid, cmd, addr); 1329 break; 1330 } 1331 return (error); 1332 } 1333 1334 /* 1335 * Set d's packet filter program to fp. If this file already has a filter, 1336 * free it and replace it. Returns EINVAL for bogus requests. 1337 */ 1338 static int 1339 bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) 1340 { 1341 struct bpf_insn *fcode; 1342 bpfjit_func_t jcode; 1343 size_t flen, size = 0; 1344 struct bpf_filter *oldf, *newf, **storef; 1345 1346 jcode = NULL; 1347 flen = fp->bf_len; 1348 1349 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1350 return EINVAL; 1351 } 1352 1353 if (flen) { 1354 /* 1355 * Allocate the buffer, copy the byte-code from 1356 * userspace and validate it. 1357 */ 1358 size = flen * sizeof(*fp->bf_insns); 1359 fcode = kmem_alloc(size, KM_SLEEP); 1360 if (copyin(fp->bf_insns, fcode, size) != 0 || 1361 !bpf_validate(fcode, (int)flen)) { 1362 kmem_free(fcode, size); 1363 return EINVAL; 1364 } 1365 if (bpf_jit) 1366 jcode = bpf_jit_generate(NULL, fcode, flen); 1367 } else { 1368 fcode = NULL; 1369 } 1370 1371 newf = kmem_alloc(sizeof(*newf), KM_SLEEP); 1372 newf->bf_insn = fcode; 1373 newf->bf_size = size; 1374 newf->bf_jitcode = jcode; 1375 if (cmd == BIOCSETF) 1376 d->bd_jitcode = jcode; /* XXX just for kvm(3) users */ 1377 1378 /* Need to hold bpf_mtx for pserialize_perform */ 1379 mutex_enter(&bpf_mtx); 1380 mutex_enter(d->bd_mtx); 1381 if (cmd == BIOCSETWF) { 1382 oldf = d->bd_wfilter; 1383 storef = &d->bd_wfilter; 1384 } else { 1385 oldf = d->bd_rfilter; 1386 storef = &d->bd_rfilter; 1387 } 1388 atomic_store_release(storef, newf); 1389 reset_d(d); 1390 pserialize_perform(bpf_psz); 1391 mutex_exit(d->bd_mtx); 1392 mutex_exit(&bpf_mtx); 1393 1394 if (oldf != NULL) 1395 bpf_free_filter(oldf); 1396 1397 return 0; 1398 } 1399 1400 /* 1401 * Detach a file from its current interface (if attached at all) and attach 1402 * to the interface indicated by the name stored in ifr. 1403 * Return an errno or 0. 1404 */ 1405 static int 1406 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1407 { 1408 struct bpf_if *bp; 1409 char *cp; 1410 int unit_seen, i, error; 1411 1412 KASSERT(mutex_owned(&bpf_mtx)); 1413 /* 1414 * Make sure the provided name has a unit number, and default 1415 * it to '0' if not specified. 1416 * XXX This is ugly ... do this differently? 1417 */ 1418 unit_seen = 0; 1419 cp = ifr->ifr_name; 1420 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1421 while (*cp++) 1422 if (*cp >= '0' && *cp <= '9') 1423 unit_seen = 1; 1424 if (!unit_seen) { 1425 /* Make sure to leave room for the '\0'. */ 1426 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1427 if ((ifr->ifr_name[i] >= 'a' && 1428 ifr->ifr_name[i] <= 'z') || 1429 (ifr->ifr_name[i] >= 'A' && 1430 ifr->ifr_name[i] <= 'Z')) 1431 continue; 1432 ifr->ifr_name[i] = '0'; 1433 } 1434 } 1435 1436 /* 1437 * Look through attached interfaces for the named one. 1438 */ 1439 BPF_IFLIST_WRITER_FOREACH(bp) { 1440 struct ifnet *ifp = bp->bif_ifp; 1441 1442 if (ifp == NULL || 1443 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1444 continue; 1445 /* skip additional entry */ 1446 if (bp->bif_driverp != &ifp->if_bpf) 1447 continue; 1448 /* 1449 * We found the requested interface. 1450 * Allocate the packet buffers if we need to. 1451 * If we're already attached to requested interface, 1452 * just flush the buffer. 1453 */ 1454 /* 1455 * bpf_allocbufs is called only here. bpf_mtx ensures that 1456 * no race condition happen on d->bd_sbuf. 1457 */ 1458 if (d->bd_sbuf == NULL) { 1459 error = bpf_allocbufs(d); 1460 if (error != 0) 1461 return (error); 1462 } 1463 mutex_enter(d->bd_mtx); 1464 if (bp != d->bd_bif) { 1465 if (d->bd_bif) { 1466 /* 1467 * Detach if attached to something else. 1468 */ 1469 bpf_detachd(d); 1470 BPFIF_DLIST_ENTRY_INIT(d); 1471 } 1472 1473 bpf_attachd(d, bp); 1474 } 1475 reset_d(d); 1476 mutex_exit(d->bd_mtx); 1477 return (0); 1478 } 1479 /* Not found. */ 1480 return (ENXIO); 1481 } 1482 1483 /* 1484 * Copy the interface name to the ifreq. 1485 */ 1486 static void 1487 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1488 { 1489 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1490 } 1491 1492 static int 1493 bpf_stat(struct file *fp, struct stat *st) 1494 { 1495 struct bpf_d *d = fp->f_bpf; 1496 1497 (void)memset(st, 0, sizeof(*st)); 1498 mutex_enter(d->bd_mtx); 1499 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1500 st->st_atimespec = d->bd_atime; 1501 st->st_mtimespec = d->bd_mtime; 1502 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1503 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1504 st->st_gid = kauth_cred_getegid(fp->f_cred); 1505 st->st_mode = S_IFCHR; 1506 mutex_exit(d->bd_mtx); 1507 return 0; 1508 } 1509 1510 /* 1511 * Support for poll() system call 1512 * 1513 * Return true iff the specific operation will not block indefinitely - with 1514 * the assumption that it is safe to positively acknowledge a request for the 1515 * ability to write to the BPF device. 1516 * Otherwise, return false but make a note that a selnotify() must be done. 1517 */ 1518 static int 1519 bpf_poll(struct file *fp, int events) 1520 { 1521 struct bpf_d *d = fp->f_bpf; 1522 int revents; 1523 1524 /* 1525 * Refresh the PID associated with this bpf file. 1526 */ 1527 mutex_enter(&bpf_mtx); 1528 d->bd_pid = curproc->p_pid; 1529 1530 revents = events & (POLLOUT | POLLWRNORM); 1531 if (events & (POLLIN | POLLRDNORM)) { 1532 /* 1533 * An imitation of the FIONREAD ioctl code. 1534 */ 1535 mutex_enter(d->bd_mtx); 1536 if (d->bd_hlen != 0 || 1537 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1538 d->bd_slen != 0)) { 1539 revents |= events & (POLLIN | POLLRDNORM); 1540 } else { 1541 selrecord(curlwp, &d->bd_sel); 1542 /* Start the read timeout if necessary */ 1543 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1544 callout_reset(&d->bd_callout, d->bd_rtout, 1545 bpf_timed_out, d); 1546 d->bd_state = BPF_WAITING; 1547 } 1548 } 1549 mutex_exit(d->bd_mtx); 1550 } 1551 1552 mutex_exit(&bpf_mtx); 1553 return (revents); 1554 } 1555 1556 static void 1557 filt_bpfrdetach(struct knote *kn) 1558 { 1559 struct bpf_d *d = kn->kn_hook; 1560 1561 mutex_enter(d->bd_buf_mtx); 1562 selremove_knote(&d->bd_sel, kn); 1563 mutex_exit(d->bd_buf_mtx); 1564 } 1565 1566 static int 1567 filt_bpfread(struct knote *kn, long hint) 1568 { 1569 struct bpf_d *d = kn->kn_hook; 1570 int rv; 1571 1572 mutex_enter(d->bd_buf_mtx); 1573 kn->kn_data = d->bd_hlen; 1574 if (d->bd_immediate) 1575 kn->kn_data += d->bd_slen; 1576 rv = (kn->kn_data > 0); 1577 mutex_exit(d->bd_buf_mtx); 1578 return rv; 1579 } 1580 1581 static const struct filterops bpfread_filtops = { 1582 .f_flags = FILTEROP_ISFD, 1583 .f_attach = NULL, 1584 .f_detach = filt_bpfrdetach, 1585 .f_event = filt_bpfread, 1586 }; 1587 1588 static int 1589 bpf_kqfilter(struct file *fp, struct knote *kn) 1590 { 1591 struct bpf_d *d = fp->f_bpf; 1592 1593 switch (kn->kn_filter) { 1594 case EVFILT_READ: 1595 kn->kn_fop = &bpfread_filtops; 1596 break; 1597 1598 default: 1599 return (EINVAL); 1600 } 1601 1602 kn->kn_hook = d; 1603 1604 mutex_enter(d->bd_buf_mtx); 1605 selrecord_knote(&d->bd_sel, kn); 1606 mutex_exit(d->bd_buf_mtx); 1607 1608 return (0); 1609 } 1610 1611 /* 1612 * Copy data from an mbuf chain into a buffer. This code is derived 1613 * from m_copydata in sys/uipc_mbuf.c. 1614 */ 1615 static void * 1616 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1617 { 1618 const struct mbuf *m; 1619 u_int count; 1620 u_char *dst; 1621 1622 m = src_arg; 1623 dst = dst_arg; 1624 while (len > 0) { 1625 if (m == NULL) 1626 panic("bpf_mcpy"); 1627 count = uimin(m->m_len, len); 1628 memcpy(dst, mtod(m, const void *), count); 1629 m = m->m_next; 1630 dst += count; 1631 len -= count; 1632 } 1633 return dst_arg; 1634 } 1635 1636 static inline u_int 1637 bpf_xfilter(struct bpf_filter **filter, void *pkt, u_int pktlen, u_int buflen) 1638 { 1639 struct bpf_filter *filt; 1640 uint32_t mem[BPF_MEMWORDS]; 1641 bpf_args_t args = { 1642 .pkt = (const uint8_t *)pkt, 1643 .wirelen = pktlen, 1644 .buflen = buflen, 1645 .mem = mem, 1646 .arg = NULL 1647 }; 1648 u_int slen; 1649 1650 filt = atomic_load_consume(filter); 1651 if (filt == NULL) /* No filter means accept all. */ 1652 return (u_int)-1; 1653 1654 if (filt->bf_jitcode != NULL) 1655 slen = filt->bf_jitcode(NULL, &args); 1656 else 1657 slen = bpf_filter_ext(NULL, filt->bf_insn, &args); 1658 return slen; 1659 } 1660 1661 /* 1662 * Dispatch a packet to all the listeners on interface bp. 1663 * 1664 * pkt pointer to the packet, either a data buffer or an mbuf chain 1665 * buflen buffer length, if pkt is a data buffer 1666 * cpfn a function that can copy pkt into the listener's buffer 1667 * pktlen length of the packet 1668 * direction BPF_D_IN or BPF_D_OUT 1669 */ 1670 static inline void 1671 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1672 void *pkt, u_int pktlen, u_int buflen, const u_int direction) 1673 { 1674 bool gottime = false; 1675 struct timespec ts; 1676 struct bpf_d *d; 1677 int s; 1678 u_int slen; 1679 1680 KASSERT(!cpu_intr_p()); 1681 1682 /* 1683 * Note that the IPL does not have to be raised at this point. 1684 * The only problem that could arise here is that if two different 1685 * interfaces shared any data. This is not the case. 1686 */ 1687 s = pserialize_read_enter(); 1688 BPFIF_DLIST_READER_FOREACH(d, bp) { 1689 if (direction == BPF_D_IN) { 1690 if (d->bd_direction == BPF_D_OUT) 1691 continue; 1692 } else { /* BPF_D_OUT */ 1693 if (d->bd_direction == BPF_D_IN) 1694 continue; 1695 } 1696 1697 atomic_inc_ulong(&d->bd_rcount); 1698 BPF_STATINC(recv); 1699 1700 slen = bpf_xfilter(&d->bd_rfilter, pkt, pktlen, buflen); 1701 if (slen == 0) 1702 continue; 1703 1704 if (!gottime) { 1705 gottime = true; 1706 nanotime(&ts); 1707 } 1708 /* Assume catchpacket doesn't sleep */ 1709 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1710 } 1711 pserialize_read_exit(s); 1712 } 1713 1714 /* 1715 * Incoming linkage from device drivers, when the head of the packet is in 1716 * a buffer, and the tail is in an mbuf chain. 1717 */ 1718 static void 1719 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m, 1720 u_int direction) 1721 { 1722 u_int pktlen; 1723 struct mbuf mb; 1724 1725 /* Skip outgoing duplicate packets. */ 1726 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1727 m->m_flags &= ~M_PROMISC; 1728 return; 1729 } 1730 1731 pktlen = m_length(m) + dlen; 1732 1733 /* 1734 * Craft on-stack mbuf suitable for passing to bpf_filter. 1735 * Note that we cut corners here; we only setup what's 1736 * absolutely needed--this mbuf should never go anywhere else. 1737 */ 1738 (void)memset(&mb, 0, sizeof(mb)); 1739 mb.m_type = MT_DATA; 1740 mb.m_next = m; 1741 mb.m_data = data; 1742 mb.m_len = dlen; 1743 1744 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, direction); 1745 } 1746 1747 /* 1748 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1749 */ 1750 static void 1751 _bpf_mtap(struct bpf_if *bp, struct mbuf *m, u_int direction) 1752 { 1753 void *(*cpfn)(void *, const void *, size_t); 1754 u_int pktlen, buflen; 1755 void *marg; 1756 1757 /* Skip outgoing duplicate packets. */ 1758 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1759 m->m_flags &= ~M_PROMISC; 1760 return; 1761 } 1762 1763 pktlen = m_length(m); 1764 1765 /* Skip zero-sized packets. */ 1766 if (__predict_false(pktlen == 0)) { 1767 return; 1768 } 1769 1770 if (pktlen == m->m_len) { 1771 cpfn = (void *)memcpy; 1772 marg = mtod(m, void *); 1773 buflen = pktlen; 1774 KASSERT(buflen != 0); 1775 } else { 1776 cpfn = bpf_mcpy; 1777 marg = m; 1778 buflen = 0; 1779 } 1780 1781 bpf_deliver(bp, cpfn, marg, pktlen, buflen, direction); 1782 } 1783 1784 /* 1785 * We need to prepend the address family as 1786 * a four byte field. Cons up a dummy header 1787 * to pacify bpf. This is safe because bpf 1788 * will only read from the mbuf (i.e., it won't 1789 * try to free it or keep a pointer a to it). 1790 */ 1791 static void 1792 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m, u_int direction) 1793 { 1794 struct mbuf m0; 1795 1796 m0.m_type = MT_DATA; 1797 m0.m_flags = 0; 1798 m0.m_next = m; 1799 m0.m_nextpkt = NULL; 1800 m0.m_owner = NULL; 1801 m0.m_len = 4; 1802 m0.m_data = (char *)⁡ 1803 1804 _bpf_mtap(bp, &m0, direction); 1805 } 1806 1807 /* 1808 * Put the SLIP pseudo-"link header" in place. 1809 * Note this M_PREPEND() should never fail, 1810 * swince we know we always have enough space 1811 * in the input buffer. 1812 */ 1813 static void 1814 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1815 { 1816 u_char *hp; 1817 1818 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1819 if (*m == NULL) 1820 return; 1821 1822 hp = mtod(*m, u_char *); 1823 hp[SLX_DIR] = SLIPDIR_IN; 1824 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1825 1826 _bpf_mtap(bp, *m, BPF_D_IN); 1827 1828 m_adj(*m, SLIP_HDRLEN); 1829 } 1830 1831 /* 1832 * Put the SLIP pseudo-"link header" in 1833 * place. The compressed header is now 1834 * at the beginning of the mbuf. 1835 */ 1836 static void 1837 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1838 { 1839 struct mbuf m0; 1840 u_char *hp; 1841 1842 m0.m_type = MT_DATA; 1843 m0.m_flags = 0; 1844 m0.m_next = m; 1845 m0.m_nextpkt = NULL; 1846 m0.m_owner = NULL; 1847 m0.m_data = m0.m_dat; 1848 m0.m_len = SLIP_HDRLEN; 1849 1850 hp = mtod(&m0, u_char *); 1851 1852 hp[SLX_DIR] = SLIPDIR_OUT; 1853 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1854 1855 _bpf_mtap(bp, &m0, BPF_D_OUT); 1856 m_freem(m); 1857 } 1858 1859 static struct mbuf * 1860 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) 1861 { 1862 struct mbuf *dup; 1863 1864 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); 1865 if (dup == NULL) 1866 return NULL; 1867 1868 if (bp->bif_mbuf_tail != NULL) { 1869 bp->bif_mbuf_tail->m_nextpkt = dup; 1870 } else { 1871 bp->bif_mbuf_head = dup; 1872 } 1873 bp->bif_mbuf_tail = dup; 1874 #ifdef BPF_MTAP_SOFTINT_DEBUG 1875 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", 1876 __func__, dup, bp->bif_ifp->if_xname); 1877 #endif 1878 1879 return dup; 1880 } 1881 1882 static struct mbuf * 1883 bpf_mbuf_dequeue(struct bpf_if *bp) 1884 { 1885 struct mbuf *m; 1886 int s; 1887 1888 /* XXX NOMPSAFE: assumed running on one CPU */ 1889 s = splnet(); 1890 m = bp->bif_mbuf_head; 1891 if (m != NULL) { 1892 bp->bif_mbuf_head = m->m_nextpkt; 1893 m->m_nextpkt = NULL; 1894 1895 if (bp->bif_mbuf_head == NULL) 1896 bp->bif_mbuf_tail = NULL; 1897 #ifdef BPF_MTAP_SOFTINT_DEBUG 1898 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", 1899 __func__, m, bp->bif_ifp->if_xname); 1900 #endif 1901 } 1902 splx(s); 1903 1904 return m; 1905 } 1906 1907 static void 1908 bpf_mtap_si(void *arg) 1909 { 1910 struct bpf_if *bp = arg; 1911 struct mbuf *m; 1912 1913 while ((m = bpf_mbuf_dequeue(bp)) != NULL) { 1914 #ifdef BPF_MTAP_SOFTINT_DEBUG 1915 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", 1916 __func__, m, bp->bif_ifp->if_xname); 1917 #endif 1918 bpf_ops->bpf_mtap(bp, m, BPF_D_IN); 1919 m_freem(m); 1920 } 1921 } 1922 1923 static void 1924 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) 1925 { 1926 struct bpf_if *bp = ifp->if_bpf; 1927 struct mbuf *dup; 1928 1929 KASSERT(cpu_intr_p()); 1930 1931 /* To avoid extra invocations of the softint */ 1932 if (BPFIF_DLIST_READER_EMPTY(bp)) 1933 return; 1934 KASSERT(bp->bif_si != NULL); 1935 1936 dup = bpf_mbuf_enqueue(bp, m); 1937 if (dup != NULL) 1938 softint_schedule(bp->bif_si); 1939 } 1940 1941 static int 1942 bpf_hdrlen(struct bpf_d *d) 1943 { 1944 int hdrlen = d->bd_bif->bif_hdrlen; 1945 /* 1946 * Compute the length of the bpf header. This is not necessarily 1947 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1948 * that the network layer header begins on a longword boundary (for 1949 * performance reasons and to alleviate alignment restrictions). 1950 */ 1951 #ifdef _LP64 1952 if (d->bd_compat32) 1953 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1954 else 1955 #endif 1956 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1957 } 1958 1959 /* 1960 * Move the packet data from interface memory (pkt) into the 1961 * store buffer. Call the wakeup functions if it's time to wakeup 1962 * a listener (buffer full), "cpfn" is the routine called to do the 1963 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1964 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1965 * pkt is really an mbuf. 1966 */ 1967 static void 1968 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1969 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1970 { 1971 char *h; 1972 int totlen, curlen, caplen; 1973 int hdrlen = bpf_hdrlen(d); 1974 int do_wakeup = 0; 1975 1976 atomic_inc_ulong(&d->bd_ccount); 1977 BPF_STATINC(capt); 1978 /* 1979 * Figure out how many bytes to move. If the packet is 1980 * greater or equal to the snapshot length, transfer that 1981 * much. Otherwise, transfer the whole packet (unless 1982 * we hit the buffer size limit). 1983 */ 1984 totlen = hdrlen + uimin(snaplen, pktlen); 1985 if (totlen > d->bd_bufsize) 1986 totlen = d->bd_bufsize; 1987 /* 1988 * If we adjusted totlen to fit the bufsize, it could be that 1989 * totlen is smaller than hdrlen because of the link layer header. 1990 */ 1991 caplen = totlen - hdrlen; 1992 if (caplen < 0) 1993 caplen = 0; 1994 1995 mutex_enter(d->bd_buf_mtx); 1996 /* 1997 * Round up the end of the previous packet to the next longword. 1998 */ 1999 #ifdef _LP64 2000 if (d->bd_compat32) 2001 curlen = BPF_WORDALIGN32(d->bd_slen); 2002 else 2003 #endif 2004 curlen = BPF_WORDALIGN(d->bd_slen); 2005 if (curlen + totlen > d->bd_bufsize) { 2006 /* 2007 * This packet will overflow the storage buffer. 2008 * Rotate the buffers if we can, then wakeup any 2009 * pending reads. 2010 */ 2011 if (d->bd_fbuf == NULL) { 2012 mutex_exit(d->bd_buf_mtx); 2013 /* 2014 * We haven't completed the previous read yet, 2015 * so drop the packet. 2016 */ 2017 atomic_inc_ulong(&d->bd_dcount); 2018 BPF_STATINC(drop); 2019 return; 2020 } 2021 ROTATE_BUFFERS(d); 2022 do_wakeup = 1; 2023 curlen = 0; 2024 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 2025 /* 2026 * Immediate mode is set, or the read timeout has 2027 * already expired during a select call. A packet 2028 * arrived, so the reader should be woken up. 2029 */ 2030 do_wakeup = 1; 2031 } 2032 2033 /* 2034 * Append the bpf header. 2035 */ 2036 h = (char *)d->bd_sbuf + curlen; 2037 #ifdef _LP64 2038 if (d->bd_compat32) { 2039 struct bpf_hdr32 *hp32; 2040 2041 hp32 = (struct bpf_hdr32 *)h; 2042 hp32->bh_tstamp.tv_sec = ts->tv_sec; 2043 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 2044 hp32->bh_datalen = pktlen; 2045 hp32->bh_hdrlen = hdrlen; 2046 hp32->bh_caplen = caplen; 2047 } else 2048 #endif 2049 { 2050 struct bpf_hdr *hp; 2051 2052 hp = (struct bpf_hdr *)h; 2053 hp->bh_tstamp.tv_sec = ts->tv_sec; 2054 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 2055 hp->bh_datalen = pktlen; 2056 hp->bh_hdrlen = hdrlen; 2057 hp->bh_caplen = caplen; 2058 } 2059 2060 /* 2061 * Copy the packet data into the store buffer and update its length. 2062 */ 2063 (*cpfn)(h + hdrlen, pkt, caplen); 2064 d->bd_slen = curlen + totlen; 2065 mutex_exit(d->bd_buf_mtx); 2066 2067 /* 2068 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 2069 * will cause filt_bpfread() to be called with it adjusted. 2070 */ 2071 if (do_wakeup) 2072 bpf_wakeup(d); 2073 } 2074 2075 /* 2076 * Initialize all nonzero fields of a descriptor. 2077 */ 2078 static int 2079 bpf_allocbufs(struct bpf_d *d) 2080 { 2081 2082 d->bd_fbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); 2083 if (!d->bd_fbuf) 2084 return (ENOBUFS); 2085 d->bd_sbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); 2086 if (!d->bd_sbuf) { 2087 kmem_free(d->bd_fbuf, d->bd_bufsize); 2088 return (ENOBUFS); 2089 } 2090 d->bd_slen = 0; 2091 d->bd_hlen = 0; 2092 return (0); 2093 } 2094 2095 static void 2096 bpf_free_filter(struct bpf_filter *filter) 2097 { 2098 2099 KASSERT(filter != NULL); 2100 2101 if (filter->bf_insn != NULL) 2102 kmem_free(filter->bf_insn, filter->bf_size); 2103 if (filter->bf_jitcode != NULL) 2104 bpf_jit_freecode(filter->bf_jitcode); 2105 kmem_free(filter, sizeof(*filter)); 2106 } 2107 2108 /* 2109 * Free buffers currently in use by a descriptor. 2110 * Called on close. 2111 */ 2112 static void 2113 bpf_freed(struct bpf_d *d) 2114 { 2115 /* 2116 * We don't need to lock out interrupts since this descriptor has 2117 * been detached from its interface and it yet hasn't been marked 2118 * free. 2119 */ 2120 if (d->bd_sbuf != NULL) { 2121 kmem_free(d->bd_sbuf, d->bd_bufsize); 2122 if (d->bd_hbuf != NULL) 2123 kmem_free(d->bd_hbuf, d->bd_bufsize); 2124 if (d->bd_fbuf != NULL) 2125 kmem_free(d->bd_fbuf, d->bd_bufsize); 2126 } 2127 if (d->bd_rfilter != NULL) { 2128 bpf_free_filter(d->bd_rfilter); 2129 d->bd_rfilter = NULL; 2130 } 2131 if (d->bd_wfilter != NULL) { 2132 bpf_free_filter(d->bd_wfilter); 2133 d->bd_wfilter = NULL; 2134 } 2135 d->bd_jitcode = NULL; 2136 } 2137 2138 /* 2139 * Attach an interface to bpf. dlt is the link layer type; 2140 * hdrlen is the fixed size of the link header for the specified dlt 2141 * (variable length headers not yet supported). 2142 */ 2143 static void 2144 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2145 { 2146 struct bpf_if *bp; 2147 2148 bp = kmem_alloc(sizeof(*bp), KM_SLEEP); 2149 2150 mutex_enter(&bpf_mtx); 2151 bp->bif_driverp = driverp; 2152 bp->bif_ifp = ifp; 2153 bp->bif_dlt = dlt; 2154 bp->bif_si = NULL; 2155 BPF_IFLIST_ENTRY_INIT(bp); 2156 PSLIST_INIT(&bp->bif_dlist_head); 2157 psref_target_init(&bp->bif_psref, bpf_psref_class); 2158 SLIST_INIT(&bp->bif_trackers); 2159 2160 BPF_IFLIST_WRITER_INSERT_HEAD(bp); 2161 2162 *bp->bif_driverp = NULL; 2163 2164 bp->bif_hdrlen = hdrlen; 2165 mutex_exit(&bpf_mtx); 2166 #if 0 2167 printf("bpf: %s attached with dlt %x\n", ifp->if_xname, dlt); 2168 #endif 2169 } 2170 2171 static void 2172 _bpf_mtap_softint_init(struct ifnet *ifp) 2173 { 2174 struct bpf_if *bp; 2175 2176 mutex_enter(&bpf_mtx); 2177 BPF_IFLIST_WRITER_FOREACH(bp) { 2178 if (bp->bif_ifp != ifp) 2179 continue; 2180 2181 bp->bif_mbuf_head = NULL; 2182 bp->bif_mbuf_tail = NULL; 2183 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); 2184 if (bp->bif_si == NULL) 2185 panic("%s: softint_establish() failed", __func__); 2186 break; 2187 } 2188 mutex_exit(&bpf_mtx); 2189 2190 if (bp == NULL) 2191 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); 2192 } 2193 2194 /* 2195 * Remove an interface from bpf. 2196 */ 2197 static void 2198 _bpfdetach(struct ifnet *ifp) 2199 { 2200 struct bpf_if *bp; 2201 struct bpf_d *d; 2202 int s; 2203 2204 mutex_enter(&bpf_mtx); 2205 /* Nuke the vnodes for any open instances */ 2206 again_d: 2207 BPF_DLIST_WRITER_FOREACH(d) { 2208 mutex_enter(d->bd_mtx); 2209 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 2210 /* 2211 * Detach the descriptor from an interface now. 2212 * It will be free'ed later by close routine. 2213 */ 2214 bpf_detachd(d); 2215 mutex_exit(d->bd_mtx); 2216 goto again_d; 2217 } 2218 mutex_exit(d->bd_mtx); 2219 } 2220 2221 again: 2222 BPF_IFLIST_WRITER_FOREACH(bp) { 2223 if (bp->bif_ifp == ifp) { 2224 BPF_IFLIST_WRITER_REMOVE(bp); 2225 2226 pserialize_perform(bpf_psz); 2227 psref_target_destroy(&bp->bif_psref, bpf_psref_class); 2228 2229 while (!SLIST_EMPTY(&bp->bif_trackers)) { 2230 struct bpf_event_tracker *t = 2231 SLIST_FIRST(&bp->bif_trackers); 2232 SLIST_REMOVE_HEAD(&bp->bif_trackers, 2233 bet_entries); 2234 kmem_free(t, sizeof(*t)); 2235 } 2236 2237 BPF_IFLIST_ENTRY_DESTROY(bp); 2238 if (bp->bif_si != NULL) { 2239 /* XXX NOMPSAFE: assumed running on one CPU */ 2240 s = splnet(); 2241 while (bp->bif_mbuf_head != NULL) { 2242 struct mbuf *m = bp->bif_mbuf_head; 2243 bp->bif_mbuf_head = m->m_nextpkt; 2244 m_freem(m); 2245 } 2246 splx(s); 2247 softint_disestablish(bp->bif_si); 2248 } 2249 kmem_free(bp, sizeof(*bp)); 2250 goto again; 2251 } 2252 } 2253 mutex_exit(&bpf_mtx); 2254 } 2255 2256 /* 2257 * Change the data link type of a interface. 2258 */ 2259 static void 2260 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2261 { 2262 struct bpf_if *bp; 2263 2264 mutex_enter(&bpf_mtx); 2265 BPF_IFLIST_WRITER_FOREACH(bp) { 2266 if (bp->bif_driverp == &ifp->if_bpf) 2267 break; 2268 } 2269 if (bp == NULL) 2270 panic("bpf_change_type"); 2271 2272 bp->bif_dlt = dlt; 2273 2274 bp->bif_hdrlen = hdrlen; 2275 mutex_exit(&bpf_mtx); 2276 } 2277 2278 /* 2279 * Get a list of available data link type of the interface. 2280 */ 2281 static int 2282 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2283 { 2284 int n, error; 2285 struct ifnet *ifp; 2286 struct bpf_if *bp; 2287 int s, bound; 2288 2289 KASSERT(mutex_owned(d->bd_mtx)); 2290 2291 ifp = d->bd_bif->bif_ifp; 2292 n = 0; 2293 error = 0; 2294 2295 bound = curlwp_bind(); 2296 s = pserialize_read_enter(); 2297 BPF_IFLIST_READER_FOREACH(bp) { 2298 if (bp->bif_ifp != ifp) 2299 continue; 2300 if (bfl->bfl_list != NULL) { 2301 struct psref psref; 2302 2303 if (n >= bfl->bfl_len) { 2304 pserialize_read_exit(s); 2305 return ENOMEM; 2306 } 2307 2308 bpf_if_acquire(bp, &psref); 2309 pserialize_read_exit(s); 2310 2311 error = copyout(&bp->bif_dlt, 2312 bfl->bfl_list + n, sizeof(u_int)); 2313 2314 s = pserialize_read_enter(); 2315 bpf_if_release(bp, &psref); 2316 } 2317 n++; 2318 } 2319 pserialize_read_exit(s); 2320 curlwp_bindx(bound); 2321 2322 bfl->bfl_len = n; 2323 return error; 2324 } 2325 2326 /* 2327 * Set the data link type of a BPF instance. 2328 */ 2329 static int 2330 bpf_setdlt(struct bpf_d *d, u_int dlt) 2331 { 2332 int error, opromisc; 2333 struct ifnet *ifp; 2334 struct bpf_if *bp; 2335 2336 KASSERT(mutex_owned(&bpf_mtx)); 2337 KASSERT(mutex_owned(d->bd_mtx)); 2338 2339 if (d->bd_bif->bif_dlt == dlt) 2340 return 0; 2341 ifp = d->bd_bif->bif_ifp; 2342 BPF_IFLIST_WRITER_FOREACH(bp) { 2343 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2344 break; 2345 } 2346 if (bp == NULL) 2347 return EINVAL; 2348 opromisc = d->bd_promisc; 2349 bpf_detachd(d); 2350 BPFIF_DLIST_ENTRY_INIT(d); 2351 bpf_attachd(d, bp); 2352 reset_d(d); 2353 if (opromisc) { 2354 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 2355 error = ifpromisc(bp->bif_ifp, 1); 2356 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 2357 if (error) 2358 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 2359 bp->bif_ifp->if_xname, error); 2360 else 2361 d->bd_promisc = 1; 2362 } 2363 return 0; 2364 } 2365 2366 static int 2367 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 2368 { 2369 int newsize, error; 2370 struct sysctlnode node; 2371 2372 node = *rnode; 2373 node.sysctl_data = &newsize; 2374 newsize = bpf_maxbufsize; 2375 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2376 if (error || newp == NULL) 2377 return (error); 2378 2379 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 2380 return (EINVAL); 2381 2382 bpf_maxbufsize = newsize; 2383 2384 return (0); 2385 } 2386 2387 #if defined(MODULAR) || defined(BPFJIT) 2388 static int 2389 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 2390 { 2391 bool newval; 2392 int error; 2393 struct sysctlnode node; 2394 2395 node = *rnode; 2396 node.sysctl_data = &newval; 2397 newval = bpf_jit; 2398 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2399 if (error != 0 || newp == NULL) 2400 return error; 2401 2402 bpf_jit = newval; 2403 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 2404 printf("JIT compilation is postponed " 2405 "until after bpfjit module is loaded\n"); 2406 } 2407 2408 return 0; 2409 } 2410 #endif 2411 2412 static int 2413 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 2414 { 2415 int error, elem_count; 2416 struct bpf_d *dp; 2417 struct bpf_d_ext dpe; 2418 size_t len, needed, elem_size, out_size; 2419 char *sp; 2420 2421 if (namelen == 1 && name[0] == CTL_QUERY) 2422 return (sysctl_query(SYSCTLFN_CALL(rnode))); 2423 2424 if (namelen != 2) 2425 return (EINVAL); 2426 2427 /* BPF peers is privileged information. */ 2428 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 2429 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 2430 if (error) 2431 return (EPERM); 2432 2433 len = (oldp != NULL) ? *oldlenp : 0; 2434 sp = oldp; 2435 elem_size = name[0]; 2436 elem_count = name[1]; 2437 out_size = MIN(sizeof(dpe), elem_size); 2438 needed = 0; 2439 2440 if (elem_size < 1 || elem_count < 0) 2441 return (EINVAL); 2442 2443 mutex_enter(&bpf_mtx); 2444 BPF_DLIST_WRITER_FOREACH(dp) { 2445 if (len >= elem_size && elem_count > 0) { 2446 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 2447 BPF_EXT(bufsize); 2448 BPF_EXT(promisc); 2449 BPF_EXT(state); 2450 BPF_EXT(immediate); 2451 BPF_EXT(hdrcmplt); 2452 BPF_EXT(direction); 2453 BPF_EXT(pid); 2454 BPF_EXT(rcount); 2455 BPF_EXT(dcount); 2456 BPF_EXT(ccount); 2457 #undef BPF_EXT 2458 mutex_enter(dp->bd_mtx); 2459 if (dp->bd_bif) 2460 (void)strlcpy(dpe.bde_ifname, 2461 dp->bd_bif->bif_ifp->if_xname, 2462 IFNAMSIZ - 1); 2463 else 2464 dpe.bde_ifname[0] = '\0'; 2465 dpe.bde_locked = dp->bd_locked; 2466 mutex_exit(dp->bd_mtx); 2467 2468 error = copyout(&dpe, sp, out_size); 2469 if (error) 2470 break; 2471 sp += elem_size; 2472 len -= elem_size; 2473 } 2474 needed += elem_size; 2475 if (elem_count > 0 && elem_count != INT_MAX) 2476 elem_count--; 2477 } 2478 mutex_exit(&bpf_mtx); 2479 2480 *oldlenp = needed; 2481 2482 return (error); 2483 } 2484 2485 static void 2486 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused) 2487 { 2488 struct bpf_stat *const stats = p; 2489 struct bpf_stat *sum = arg; 2490 2491 int s = splnet(); 2492 2493 sum->bs_recv += stats->bs_recv; 2494 sum->bs_drop += stats->bs_drop; 2495 sum->bs_capt += stats->bs_capt; 2496 2497 splx(s); 2498 } 2499 2500 static int 2501 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS) 2502 { 2503 struct sysctlnode node; 2504 int error; 2505 struct bpf_stat sum; 2506 2507 memset(&sum, 0, sizeof(sum)); 2508 node = *rnode; 2509 2510 percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET), 2511 bpf_stats, &sum); 2512 2513 node.sysctl_data = ∑ 2514 node.sysctl_size = sizeof(sum); 2515 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2516 if (error != 0 || newp == NULL) 2517 return error; 2518 2519 return 0; 2520 } 2521 2522 SYSCTL_SETUP(sysctl_net_bpf_setup, "bpf sysctls") 2523 { 2524 const struct sysctlnode *node; 2525 2526 node = NULL; 2527 sysctl_createv(clog, 0, NULL, &node, 2528 CTLFLAG_PERMANENT, 2529 CTLTYPE_NODE, "bpf", 2530 SYSCTL_DESCR("BPF options"), 2531 NULL, 0, NULL, 0, 2532 CTL_NET, CTL_CREATE, CTL_EOL); 2533 if (node != NULL) { 2534 #if defined(MODULAR) || defined(BPFJIT) 2535 sysctl_createv(clog, 0, NULL, NULL, 2536 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2537 CTLTYPE_BOOL, "jit", 2538 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2539 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2540 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2541 #endif 2542 sysctl_createv(clog, 0, NULL, NULL, 2543 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2544 CTLTYPE_INT, "maxbufsize", 2545 SYSCTL_DESCR("Maximum size for data capture buffer"), 2546 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2547 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2548 sysctl_createv(clog, 0, NULL, NULL, 2549 CTLFLAG_PERMANENT, 2550 CTLTYPE_STRUCT, "stats", 2551 SYSCTL_DESCR("BPF stats"), 2552 bpf_sysctl_gstats_handler, 0, NULL, 0, 2553 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2554 sysctl_createv(clog, 0, NULL, NULL, 2555 CTLFLAG_PERMANENT, 2556 CTLTYPE_STRUCT, "peers", 2557 SYSCTL_DESCR("BPF peers"), 2558 sysctl_net_bpf_peers, 0, NULL, 0, 2559 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2560 } 2561 2562 } 2563 2564 static int 2565 _bpf_register_track_event(struct bpf_if **driverp, 2566 void (*_fun)(struct bpf_if *, struct ifnet *, int, int)) 2567 { 2568 struct bpf_if *bp; 2569 struct bpf_event_tracker *t; 2570 int ret = ENOENT; 2571 2572 t = kmem_zalloc(sizeof(*t), KM_SLEEP); 2573 if (!t) 2574 return ENOMEM; 2575 t->bet_notify = _fun; 2576 2577 mutex_enter(&bpf_mtx); 2578 BPF_IFLIST_WRITER_FOREACH(bp) { 2579 if (bp->bif_driverp != driverp) 2580 continue; 2581 SLIST_INSERT_HEAD(&bp->bif_trackers, t, bet_entries); 2582 ret = 0; 2583 break; 2584 } 2585 mutex_exit(&bpf_mtx); 2586 2587 return ret; 2588 } 2589 2590 static int 2591 _bpf_deregister_track_event(struct bpf_if **driverp, 2592 void (*_fun)(struct bpf_if *, struct ifnet *, int, int)) 2593 { 2594 struct bpf_if *bp; 2595 struct bpf_event_tracker *t = NULL; 2596 int ret = ENOENT; 2597 2598 mutex_enter(&bpf_mtx); 2599 BPF_IFLIST_WRITER_FOREACH(bp) { 2600 if (bp->bif_driverp != driverp) 2601 continue; 2602 SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { 2603 if (t->bet_notify == _fun) { 2604 ret = 0; 2605 break; 2606 } 2607 } 2608 if (ret == 0) 2609 break; 2610 } 2611 if (ret == 0 && t && t->bet_notify == _fun) { 2612 SLIST_REMOVE(&bp->bif_trackers, t, bpf_event_tracker, 2613 bet_entries); 2614 } 2615 mutex_exit(&bpf_mtx); 2616 if (ret == 0) 2617 kmem_free(t, sizeof(*t)); 2618 return ret; 2619 } 2620 2621 struct bpf_ops bpf_ops_kernel = { 2622 .bpf_attach = _bpfattach, 2623 .bpf_detach = _bpfdetach, 2624 .bpf_change_type = _bpf_change_type, 2625 .bpf_register_track_event = _bpf_register_track_event, 2626 .bpf_deregister_track_event = _bpf_deregister_track_event, 2627 2628 .bpf_mtap = _bpf_mtap, 2629 .bpf_mtap2 = _bpf_mtap2, 2630 .bpf_mtap_af = _bpf_mtap_af, 2631 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2632 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2633 2634 .bpf_mtap_softint = _bpf_mtap_softint, 2635 .bpf_mtap_softint_init = _bpf_mtap_softint_init, 2636 }; 2637 2638 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); 2639 2640 static int 2641 bpf_modcmd(modcmd_t cmd, void *arg) 2642 { 2643 #ifdef _MODULE 2644 devmajor_t bmajor, cmajor; 2645 #endif 2646 int error = 0; 2647 2648 switch (cmd) { 2649 case MODULE_CMD_INIT: 2650 bpf_init(); 2651 #ifdef _MODULE 2652 bmajor = cmajor = NODEVMAJOR; 2653 error = devsw_attach("bpf", NULL, &bmajor, 2654 &bpf_cdevsw, &cmajor); 2655 if (error) 2656 break; 2657 #endif 2658 2659 bpf_ops_handover_enter(&bpf_ops_kernel); 2660 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2661 bpf_ops_handover_exit(); 2662 break; 2663 2664 case MODULE_CMD_FINI: 2665 /* 2666 * While there is no reference counting for bpf callers, 2667 * unload could at least in theory be done similarly to 2668 * system call disestablishment. This should even be 2669 * a little simpler: 2670 * 2671 * 1) replace op vector with stubs 2672 * 2) post update to all cpus with xc 2673 * 3) check that nobody is in bpf anymore 2674 * (it's doubtful we'd want something like l_sysent, 2675 * but we could do something like *signed* percpu 2676 * counters. if the sum is 0, we're good). 2677 * 4) if fail, unroll changes 2678 * 2679 * NOTE: change won't be atomic to the outside. some 2680 * packets may be not captured even if unload is 2681 * not successful. I think packet capture not working 2682 * is a perfectly logical consequence of trying to 2683 * disable packet capture. 2684 */ 2685 error = EOPNOTSUPP; 2686 break; 2687 2688 default: 2689 error = ENOTTY; 2690 break; 2691 } 2692 2693 return error; 2694 } 2695