1 /* $NetBSD: bpf.c,v 1.236 2020/03/16 21:20:11 pgoyette Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.236 2020/03/16 21:20:11 pgoyette Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "opt_net_mpsafe.h" 48 #endif 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/mbuf.h> 53 #include <sys/buf.h> 54 #include <sys/time.h> 55 #include <sys/proc.h> 56 #include <sys/ioctl.h> 57 #include <sys/conf.h> 58 #include <sys/vnode.h> 59 #include <sys/queue.h> 60 #include <sys/stat.h> 61 #include <sys/module.h> 62 #include <sys/atomic.h> 63 #include <sys/cpu.h> 64 65 #include <sys/file.h> 66 #include <sys/filedesc.h> 67 #include <sys/tty.h> 68 #include <sys/uio.h> 69 70 #include <sys/protosw.h> 71 #include <sys/socket.h> 72 #include <sys/errno.h> 73 #include <sys/kernel.h> 74 #include <sys/poll.h> 75 #include <sys/sysctl.h> 76 #include <sys/kauth.h> 77 #include <sys/syslog.h> 78 #include <sys/percpu.h> 79 #include <sys/pserialize.h> 80 #include <sys/lwp.h> 81 #include <sys/xcall.h> 82 83 #include <net/if.h> 84 #include <net/slip.h> 85 86 #include <net/bpf.h> 87 #include <net/bpfdesc.h> 88 #include <net/bpfjit.h> 89 90 #include <net/if_arc.h> 91 #include <net/if_ether.h> 92 93 #include <netinet/in.h> 94 #include <netinet/if_inarp.h> 95 96 97 #include <compat/sys/sockio.h> 98 99 #ifndef BPF_BUFSIZE 100 /* 101 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 102 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 103 */ 104 # define BPF_BUFSIZE 32768 105 #endif 106 107 #define PRINET 26 /* interruptible */ 108 109 /* 110 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 111 * XXX the default values should be computed dynamically based 112 * on available memory size and available mbuf clusters. 113 */ 114 static int bpf_bufsize = BPF_BUFSIZE; 115 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 116 static bool bpf_jit = false; 117 118 struct bpfjit_ops bpfjit_module_ops = { 119 .bj_generate_code = NULL, 120 .bj_free_code = NULL 121 }; 122 123 /* 124 * Global BPF statistics returned by net.bpf.stats sysctl. 125 */ 126 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */ 127 128 #define BPF_STATINC(id) \ 129 { \ 130 struct bpf_stat *__stats = \ 131 percpu_getref(bpf_gstats_percpu); \ 132 __stats->bs_##id++; \ 133 percpu_putref(bpf_gstats_percpu); \ 134 } 135 136 /* 137 * Locking notes: 138 * - bpf_mtx (adaptive mutex) protects: 139 * - Gobal lists: bpf_iflist and bpf_dlist 140 * - struct bpf_if 141 * - bpf_close 142 * - bpf_psz (pserialize) 143 * - struct bpf_d has two mutexes: 144 * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed 145 * on packet tapping 146 * - bd_mtx (adaptive mutex) protects member variables other than the buffers 147 * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx 148 * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is 149 * never freed because struct bpf_d is only freed in bpf_close and 150 * bpf_close never be called while executing bpf_read and bpf_write 151 * - A filter that is assigned to bpf_d can be replaced with another filter 152 * while tapping packets, so it needs to be done atomically 153 * - struct bpf_d is iterated on bpf_dlist with psz 154 * - struct bpf_if is iterated on bpf_iflist with psz or psref 155 */ 156 /* 157 * Use a mutex to avoid a race condition between gathering the stats/peers 158 * and opening/closing the device. 159 */ 160 static kmutex_t bpf_mtx; 161 162 static struct psref_class *bpf_psref_class __read_mostly; 163 static pserialize_t bpf_psz; 164 165 static inline void 166 bpf_if_acquire(struct bpf_if *bp, struct psref *psref) 167 { 168 169 psref_acquire(psref, &bp->bif_psref, bpf_psref_class); 170 } 171 172 static inline void 173 bpf_if_release(struct bpf_if *bp, struct psref *psref) 174 { 175 176 psref_release(psref, &bp->bif_psref, bpf_psref_class); 177 } 178 179 /* 180 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 181 * bpf_dtab holds the descriptors, indexed by minor device # 182 */ 183 static struct pslist_head bpf_iflist; 184 static struct pslist_head bpf_dlist; 185 186 /* Macros for bpf_d on bpf_dlist */ 187 #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \ 188 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) 189 #define BPF_DLIST_READER_FOREACH(__d) \ 190 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 191 bd_bpf_dlist_entry) 192 #define BPF_DLIST_WRITER_FOREACH(__d) \ 193 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 194 bd_bpf_dlist_entry) 195 #define BPF_DLIST_ENTRY_INIT(__d) \ 196 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) 197 #define BPF_DLIST_WRITER_REMOVE(__d) \ 198 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) 199 #define BPF_DLIST_ENTRY_DESTROY(__d) \ 200 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) 201 202 /* Macros for bpf_if on bpf_iflist */ 203 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ 204 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) 205 #define BPF_IFLIST_READER_FOREACH(__bp) \ 206 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 207 bif_iflist_entry) 208 #define BPF_IFLIST_WRITER_FOREACH(__bp) \ 209 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 210 bif_iflist_entry) 211 #define BPF_IFLIST_WRITER_REMOVE(__bp) \ 212 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) 213 #define BPF_IFLIST_ENTRY_INIT(__bp) \ 214 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) 215 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ 216 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) 217 218 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ 219 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ 220 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ 221 bd_bif_dlist_entry) 222 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ 223 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ 224 bd_bif_dlist_entry) 225 #define BPFIF_DLIST_WRITER_REMOVE(__d) \ 226 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) 227 #define BPFIF_DLIST_ENTRY_INIT(__d) \ 228 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) 229 #define BPFIF_DLIST_READER_EMPTY(__bp) \ 230 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 231 bd_bif_dlist_entry) == NULL) 232 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ 233 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 234 bd_bif_dlist_entry) == NULL) 235 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ 236 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) 237 238 static int bpf_allocbufs(struct bpf_d *); 239 static void bpf_deliver(struct bpf_if *, 240 void *(*cpfn)(void *, const void *, size_t), 241 void *, u_int, u_int, const u_int); 242 static void bpf_freed(struct bpf_d *); 243 static void bpf_free_filter(struct bpf_filter *); 244 static void bpf_ifname(struct ifnet *, struct ifreq *); 245 static void *bpf_mcpy(void *, const void *, size_t); 246 static int bpf_movein(struct uio *, int, uint64_t, 247 struct mbuf **, struct sockaddr *); 248 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 249 static void bpf_detachd(struct bpf_d *); 250 static int bpf_setif(struct bpf_d *, struct ifreq *); 251 static int bpf_setf(struct bpf_d *, struct bpf_program *); 252 static void bpf_timed_out(void *); 253 static inline void 254 bpf_wakeup(struct bpf_d *); 255 static int bpf_hdrlen(struct bpf_d *); 256 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 257 void *(*)(void *, const void *, size_t), struct timespec *); 258 static void reset_d(struct bpf_d *); 259 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 260 static int bpf_setdlt(struct bpf_d *, u_int); 261 262 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 263 int); 264 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 265 int); 266 static int bpf_ioctl(struct file *, u_long, void *); 267 static int bpf_poll(struct file *, int); 268 static int bpf_stat(struct file *, struct stat *); 269 static int bpf_close(struct file *); 270 static int bpf_kqfilter(struct file *, struct knote *); 271 272 static const struct fileops bpf_fileops = { 273 .fo_name = "bpf", 274 .fo_read = bpf_read, 275 .fo_write = bpf_write, 276 .fo_ioctl = bpf_ioctl, 277 .fo_fcntl = fnullop_fcntl, 278 .fo_poll = bpf_poll, 279 .fo_stat = bpf_stat, 280 .fo_close = bpf_close, 281 .fo_kqfilter = bpf_kqfilter, 282 .fo_restart = fnullop_restart, 283 }; 284 285 dev_type_open(bpfopen); 286 287 const struct cdevsw bpf_cdevsw = { 288 .d_open = bpfopen, 289 .d_close = noclose, 290 .d_read = noread, 291 .d_write = nowrite, 292 .d_ioctl = noioctl, 293 .d_stop = nostop, 294 .d_tty = notty, 295 .d_poll = nopoll, 296 .d_mmap = nommap, 297 .d_kqfilter = nokqfilter, 298 .d_discard = nodiscard, 299 .d_flag = D_OTHER | D_MPSAFE 300 }; 301 302 bpfjit_func_t 303 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 304 { 305 struct bpfjit_ops *ops = &bpfjit_module_ops; 306 bpfjit_func_t (*generate_code)(const bpf_ctx_t *, 307 const struct bpf_insn *, size_t); 308 309 generate_code = atomic_load_acquire(&ops->bj_generate_code); 310 if (generate_code != NULL) { 311 return generate_code(bc, code, size); 312 } 313 return NULL; 314 } 315 316 void 317 bpf_jit_freecode(bpfjit_func_t jcode) 318 { 319 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 320 bpfjit_module_ops.bj_free_code(jcode); 321 } 322 323 static int 324 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 325 struct sockaddr *sockp) 326 { 327 struct mbuf *m, *m0, *n; 328 int error; 329 size_t len; 330 size_t hlen; 331 size_t align; 332 333 /* 334 * Build a sockaddr based on the data link layer type. 335 * We do this at this level because the ethernet header 336 * is copied directly into the data field of the sockaddr. 337 * In the case of SLIP, there is no header and the packet 338 * is forwarded as is. 339 * Also, we are careful to leave room at the front of the mbuf 340 * for the link level header. 341 */ 342 switch (linktype) { 343 344 case DLT_SLIP: 345 sockp->sa_family = AF_INET; 346 hlen = 0; 347 align = 0; 348 break; 349 350 case DLT_PPP: 351 sockp->sa_family = AF_UNSPEC; 352 hlen = 0; 353 align = 0; 354 break; 355 356 case DLT_EN10MB: 357 sockp->sa_family = AF_UNSPEC; 358 /* XXX Would MAXLINKHDR be better? */ 359 /* 6(dst)+6(src)+2(type) */ 360 hlen = sizeof(struct ether_header); 361 align = 2; 362 break; 363 364 case DLT_ARCNET: 365 sockp->sa_family = AF_UNSPEC; 366 hlen = ARC_HDRLEN; 367 align = 5; 368 break; 369 370 case DLT_FDDI: 371 sockp->sa_family = AF_LINK; 372 /* XXX 4(FORMAC)+6(dst)+6(src) */ 373 hlen = 16; 374 align = 0; 375 break; 376 377 case DLT_ECONET: 378 sockp->sa_family = AF_UNSPEC; 379 hlen = 6; 380 align = 2; 381 break; 382 383 case DLT_NULL: 384 sockp->sa_family = AF_UNSPEC; 385 hlen = 0; 386 align = 0; 387 break; 388 389 default: 390 return (EIO); 391 } 392 393 len = uio->uio_resid; 394 /* 395 * If there aren't enough bytes for a link level header or the 396 * packet length exceeds the interface mtu, return an error. 397 */ 398 if (len - hlen > mtu) 399 return (EMSGSIZE); 400 401 m0 = m = m_gethdr(M_WAIT, MT_DATA); 402 m_reset_rcvif(m); 403 m->m_pkthdr.len = (int)(len - hlen); 404 if (len + align > MHLEN) { 405 m_clget(m, M_WAIT); 406 if ((m->m_flags & M_EXT) == 0) { 407 error = ENOBUFS; 408 goto bad; 409 } 410 } 411 412 /* Insure the data is properly aligned */ 413 if (align > 0) 414 m->m_data += align; 415 416 for (;;) { 417 len = M_TRAILINGSPACE(m); 418 if (len > uio->uio_resid) 419 len = uio->uio_resid; 420 error = uiomove(mtod(m, void *), len, uio); 421 if (error) 422 goto bad; 423 m->m_len = len; 424 425 if (uio->uio_resid == 0) 426 break; 427 428 n = m_get(M_WAIT, MT_DATA); 429 m_clget(n, M_WAIT); /* if fails, there is no problem */ 430 m->m_next = n; 431 m = n; 432 } 433 434 if (hlen != 0) { 435 /* move link level header in the top of mbuf to sa_data */ 436 memcpy(sockp->sa_data, mtod(m0, void *), hlen); 437 m0->m_data += hlen; 438 m0->m_len -= hlen; 439 } 440 441 *mp = m0; 442 return (0); 443 444 bad: 445 m_freem(m0); 446 return (error); 447 } 448 449 /* 450 * Attach file to the bpf interface, i.e. make d listen on bp. 451 */ 452 static void 453 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 454 { 455 456 KASSERT(mutex_owned(&bpf_mtx)); 457 KASSERT(mutex_owned(d->bd_mtx)); 458 /* 459 * Point d at bp, and add d to the interface's list of listeners. 460 * Finally, point the driver's bpf cookie at the interface so 461 * it will divert packets to bpf. 462 */ 463 d->bd_bif = bp; 464 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); 465 466 *bp->bif_driverp = bp; 467 } 468 469 /* 470 * Detach a file from its interface. 471 */ 472 static void 473 bpf_detachd(struct bpf_d *d) 474 { 475 struct bpf_if *bp; 476 477 KASSERT(mutex_owned(&bpf_mtx)); 478 KASSERT(mutex_owned(d->bd_mtx)); 479 480 bp = d->bd_bif; 481 /* 482 * Check if this descriptor had requested promiscuous mode. 483 * If so, turn it off. 484 */ 485 if (d->bd_promisc) { 486 int error __diagused; 487 488 d->bd_promisc = 0; 489 /* 490 * Take device out of promiscuous mode. Since we were 491 * able to enter promiscuous mode, we should be able 492 * to turn it off. But we can get an error if 493 * the interface was configured down, so only panic 494 * if we don't get an unexpected error. 495 */ 496 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 497 error = ifpromisc(bp->bif_ifp, 0); 498 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 499 #ifdef DIAGNOSTIC 500 if (error) 501 printf("%s: ifpromisc failed: %d", __func__, error); 502 #endif 503 } 504 505 /* Remove d from the interface's descriptor list. */ 506 BPFIF_DLIST_WRITER_REMOVE(d); 507 508 pserialize_perform(bpf_psz); 509 510 if (BPFIF_DLIST_WRITER_EMPTY(bp)) { 511 /* 512 * Let the driver know that there are no more listeners. 513 */ 514 *d->bd_bif->bif_driverp = NULL; 515 } 516 d->bd_bif = NULL; 517 } 518 519 static void 520 bpf_init(void) 521 { 522 523 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 524 bpf_psz = pserialize_create(); 525 bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET); 526 527 PSLIST_INIT(&bpf_iflist); 528 PSLIST_INIT(&bpf_dlist); 529 530 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat)); 531 532 return; 533 } 534 535 /* 536 * bpfilterattach() is called at boot time. We don't need to do anything 537 * here, since any initialization will happen as part of module init code. 538 */ 539 /* ARGSUSED */ 540 void 541 bpfilterattach(int n) 542 { 543 544 } 545 546 /* 547 * Open ethernet device. Clones. 548 */ 549 /* ARGSUSED */ 550 int 551 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 552 { 553 struct bpf_d *d; 554 struct file *fp; 555 int error, fd; 556 557 /* falloc() will fill in the descriptor for us. */ 558 if ((error = fd_allocfile(&fp, &fd)) != 0) 559 return error; 560 561 d = kmem_zalloc(sizeof(*d), KM_SLEEP); 562 d->bd_bufsize = bpf_bufsize; 563 d->bd_direction = BPF_D_INOUT; 564 d->bd_feedback = 0; 565 d->bd_pid = l->l_proc->p_pid; 566 #ifdef _LP64 567 if (curproc->p_flag & PK_32) 568 d->bd_compat32 = 1; 569 #endif 570 getnanotime(&d->bd_btime); 571 d->bd_atime = d->bd_mtime = d->bd_btime; 572 callout_init(&d->bd_callout, CALLOUT_MPSAFE); 573 selinit(&d->bd_sel); 574 d->bd_jitcode = NULL; 575 d->bd_filter = NULL; 576 BPF_DLIST_ENTRY_INIT(d); 577 BPFIF_DLIST_ENTRY_INIT(d); 578 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 579 d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 580 cv_init(&d->bd_cv, "bpf"); 581 582 mutex_enter(&bpf_mtx); 583 BPF_DLIST_WRITER_INSERT_HEAD(d); 584 mutex_exit(&bpf_mtx); 585 586 return fd_clone(fp, fd, flag, &bpf_fileops, d); 587 } 588 589 /* 590 * Close the descriptor by detaching it from its interface, 591 * deallocating its buffers, and marking it free. 592 */ 593 /* ARGSUSED */ 594 static int 595 bpf_close(struct file *fp) 596 { 597 struct bpf_d *d; 598 599 mutex_enter(&bpf_mtx); 600 601 if ((d = fp->f_bpf) == NULL) { 602 mutex_exit(&bpf_mtx); 603 return 0; 604 } 605 606 /* 607 * Refresh the PID associated with this bpf file. 608 */ 609 d->bd_pid = curproc->p_pid; 610 611 mutex_enter(d->bd_mtx); 612 if (d->bd_state == BPF_WAITING) 613 callout_halt(&d->bd_callout, d->bd_mtx); 614 d->bd_state = BPF_IDLE; 615 if (d->bd_bif) 616 bpf_detachd(d); 617 mutex_exit(d->bd_mtx); 618 619 BPF_DLIST_WRITER_REMOVE(d); 620 621 pserialize_perform(bpf_psz); 622 mutex_exit(&bpf_mtx); 623 624 BPFIF_DLIST_ENTRY_DESTROY(d); 625 BPF_DLIST_ENTRY_DESTROY(d); 626 fp->f_bpf = NULL; 627 bpf_freed(d); 628 callout_destroy(&d->bd_callout); 629 seldestroy(&d->bd_sel); 630 mutex_obj_free(d->bd_mtx); 631 mutex_obj_free(d->bd_buf_mtx); 632 cv_destroy(&d->bd_cv); 633 634 kmem_free(d, sizeof(*d)); 635 636 return (0); 637 } 638 639 /* 640 * Rotate the packet buffers in descriptor d. Move the store buffer 641 * into the hold slot, and the free buffer into the store slot. 642 * Zero the length of the new store buffer. 643 */ 644 #define ROTATE_BUFFERS(d) \ 645 (d)->bd_hbuf = (d)->bd_sbuf; \ 646 (d)->bd_hlen = (d)->bd_slen; \ 647 (d)->bd_sbuf = (d)->bd_fbuf; \ 648 (d)->bd_slen = 0; \ 649 (d)->bd_fbuf = NULL; 650 /* 651 * bpfread - read next chunk of packets from buffers 652 */ 653 static int 654 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 655 kauth_cred_t cred, int flags) 656 { 657 struct bpf_d *d = fp->f_bpf; 658 int timed_out; 659 int error; 660 661 getnanotime(&d->bd_atime); 662 /* 663 * Restrict application to use a buffer the same size as 664 * the kernel buffers. 665 */ 666 if (uio->uio_resid != d->bd_bufsize) 667 return (EINVAL); 668 669 mutex_enter(d->bd_mtx); 670 if (d->bd_state == BPF_WAITING) 671 callout_halt(&d->bd_callout, d->bd_mtx); 672 timed_out = (d->bd_state == BPF_TIMED_OUT); 673 d->bd_state = BPF_IDLE; 674 mutex_exit(d->bd_mtx); 675 /* 676 * If the hold buffer is empty, then do a timed sleep, which 677 * ends when the timeout expires or when enough packets 678 * have arrived to fill the store buffer. 679 */ 680 mutex_enter(d->bd_buf_mtx); 681 while (d->bd_hbuf == NULL) { 682 if (fp->f_flag & FNONBLOCK) { 683 if (d->bd_slen == 0) { 684 error = EWOULDBLOCK; 685 goto out; 686 } 687 ROTATE_BUFFERS(d); 688 break; 689 } 690 691 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 692 /* 693 * A packet(s) either arrived since the previous 694 * read or arrived while we were asleep. 695 * Rotate the buffers and return what's here. 696 */ 697 ROTATE_BUFFERS(d); 698 break; 699 } 700 701 error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout); 702 703 if (error == EINTR || error == ERESTART) 704 goto out; 705 706 if (error == EWOULDBLOCK) { 707 /* 708 * On a timeout, return what's in the buffer, 709 * which may be nothing. If there is something 710 * in the store buffer, we can rotate the buffers. 711 */ 712 if (d->bd_hbuf) 713 /* 714 * We filled up the buffer in between 715 * getting the timeout and arriving 716 * here, so we don't need to rotate. 717 */ 718 break; 719 720 if (d->bd_slen == 0) { 721 error = 0; 722 goto out; 723 } 724 ROTATE_BUFFERS(d); 725 break; 726 } 727 if (error != 0) 728 goto out; 729 } 730 /* 731 * At this point, we know we have something in the hold slot. 732 */ 733 mutex_exit(d->bd_buf_mtx); 734 735 /* 736 * Move data from hold buffer into user space. 737 * We know the entire buffer is transferred since 738 * we checked above that the read buffer is bpf_bufsize bytes. 739 */ 740 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 741 742 mutex_enter(d->bd_buf_mtx); 743 d->bd_fbuf = d->bd_hbuf; 744 d->bd_hbuf = NULL; 745 d->bd_hlen = 0; 746 out: 747 mutex_exit(d->bd_buf_mtx); 748 return (error); 749 } 750 751 752 /* 753 * If there are processes sleeping on this descriptor, wake them up. 754 */ 755 static inline void 756 bpf_wakeup(struct bpf_d *d) 757 { 758 759 mutex_enter(d->bd_buf_mtx); 760 cv_broadcast(&d->bd_cv); 761 mutex_exit(d->bd_buf_mtx); 762 763 if (d->bd_async) 764 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 765 selnotify(&d->bd_sel, 0, 0); 766 } 767 768 static void 769 bpf_timed_out(void *arg) 770 { 771 struct bpf_d *d = arg; 772 773 mutex_enter(d->bd_mtx); 774 if (d->bd_state == BPF_WAITING) { 775 d->bd_state = BPF_TIMED_OUT; 776 if (d->bd_slen != 0) 777 bpf_wakeup(d); 778 } 779 mutex_exit(d->bd_mtx); 780 } 781 782 783 static int 784 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 785 kauth_cred_t cred, int flags) 786 { 787 struct bpf_d *d = fp->f_bpf; 788 struct bpf_if *bp; 789 struct ifnet *ifp; 790 struct mbuf *m, *mc; 791 int error; 792 static struct sockaddr_storage dst; 793 struct psref psref; 794 int bound; 795 796 m = NULL; /* XXX gcc */ 797 798 bound = curlwp_bind(); 799 mutex_enter(d->bd_mtx); 800 bp = d->bd_bif; 801 if (bp == NULL) { 802 mutex_exit(d->bd_mtx); 803 error = ENXIO; 804 goto out_bindx; 805 } 806 bpf_if_acquire(bp, &psref); 807 mutex_exit(d->bd_mtx); 808 809 getnanotime(&d->bd_mtime); 810 811 ifp = bp->bif_ifp; 812 if (if_is_deactivated(ifp)) { 813 error = ENXIO; 814 goto out; 815 } 816 817 if (uio->uio_resid == 0) { 818 error = 0; 819 goto out; 820 } 821 822 error = bpf_movein(uio, (int)bp->bif_dlt, ifp->if_mtu, &m, 823 (struct sockaddr *) &dst); 824 if (error) 825 goto out; 826 827 if (m->m_pkthdr.len > ifp->if_mtu) { 828 m_freem(m); 829 error = EMSGSIZE; 830 goto out; 831 } 832 833 if (d->bd_hdrcmplt) 834 dst.ss_family = pseudo_AF_HDRCMPLT; 835 836 if (d->bd_feedback) { 837 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 838 if (mc != NULL) 839 m_set_rcvif(mc, ifp); 840 /* Set M_PROMISC for outgoing packets to be discarded. */ 841 if (1 /*d->bd_direction == BPF_D_INOUT*/) 842 m->m_flags |= M_PROMISC; 843 } else 844 mc = NULL; 845 846 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); 847 848 if (mc != NULL) { 849 if (error == 0) { 850 int s = splsoftnet(); 851 KERNEL_LOCK_UNLESS_IFP_MPSAFE(ifp); 852 ifp->_if_input(ifp, mc); 853 KERNEL_UNLOCK_UNLESS_IFP_MPSAFE(ifp); 854 splx(s); 855 } else 856 m_freem(mc); 857 } 858 /* 859 * The driver frees the mbuf. 860 */ 861 out: 862 bpf_if_release(bp, &psref); 863 out_bindx: 864 curlwp_bindx(bound); 865 return error; 866 } 867 868 /* 869 * Reset a descriptor by flushing its packet buffer and clearing the 870 * receive and drop counts. 871 */ 872 static void 873 reset_d(struct bpf_d *d) 874 { 875 876 KASSERT(mutex_owned(d->bd_mtx)); 877 878 mutex_enter(d->bd_buf_mtx); 879 if (d->bd_hbuf) { 880 /* Free the hold buffer. */ 881 d->bd_fbuf = d->bd_hbuf; 882 d->bd_hbuf = NULL; 883 } 884 d->bd_slen = 0; 885 d->bd_hlen = 0; 886 d->bd_rcount = 0; 887 d->bd_dcount = 0; 888 d->bd_ccount = 0; 889 mutex_exit(d->bd_buf_mtx); 890 } 891 892 /* 893 * FIONREAD Check for read packet available. 894 * BIOCGBLEN Get buffer len [for read()]. 895 * BIOCSETF Set ethernet read filter. 896 * BIOCFLUSH Flush read packet buffer. 897 * BIOCPROMISC Put interface into promiscuous mode. 898 * BIOCGDLT Get link layer type. 899 * BIOCGETIF Get interface name. 900 * BIOCSETIF Set interface. 901 * BIOCSRTIMEOUT Set read timeout. 902 * BIOCGRTIMEOUT Get read timeout. 903 * BIOCGSTATS Get packet stats. 904 * BIOCIMMEDIATE Set immediate mode. 905 * BIOCVERSION Get filter language version. 906 * BIOCGHDRCMPLT Get "header already complete" flag. 907 * BIOCSHDRCMPLT Set "header already complete" flag. 908 * BIOCSFEEDBACK Set packet feedback mode. 909 * BIOCGFEEDBACK Get packet feedback mode. 910 * BIOCGDIRECTION Get packet direction flag 911 * BIOCSDIRECTION Set packet direction flag 912 */ 913 /* ARGSUSED */ 914 static int 915 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 916 { 917 struct bpf_d *d = fp->f_bpf; 918 int error = 0; 919 920 /* 921 * Refresh the PID associated with this bpf file. 922 */ 923 d->bd_pid = curproc->p_pid; 924 #ifdef _LP64 925 if (curproc->p_flag & PK_32) 926 d->bd_compat32 = 1; 927 else 928 d->bd_compat32 = 0; 929 #endif 930 931 mutex_enter(d->bd_mtx); 932 if (d->bd_state == BPF_WAITING) 933 callout_halt(&d->bd_callout, d->bd_mtx); 934 d->bd_state = BPF_IDLE; 935 mutex_exit(d->bd_mtx); 936 937 switch (cmd) { 938 939 default: 940 error = EINVAL; 941 break; 942 943 /* 944 * Check for read packet available. 945 */ 946 case FIONREAD: 947 { 948 int n; 949 950 mutex_enter(d->bd_buf_mtx); 951 n = d->bd_slen; 952 if (d->bd_hbuf) 953 n += d->bd_hlen; 954 mutex_exit(d->bd_buf_mtx); 955 956 *(int *)addr = n; 957 break; 958 } 959 960 /* 961 * Get buffer len [for read()]. 962 */ 963 case BIOCGBLEN: 964 *(u_int *)addr = d->bd_bufsize; 965 break; 966 967 /* 968 * Set buffer length. 969 */ 970 case BIOCSBLEN: 971 /* 972 * Forbid to change the buffer length if buffers are already 973 * allocated. 974 */ 975 mutex_enter(d->bd_mtx); 976 mutex_enter(d->bd_buf_mtx); 977 if (d->bd_bif != NULL || d->bd_sbuf != NULL) 978 error = EINVAL; 979 else { 980 u_int size = *(u_int *)addr; 981 982 if (size > bpf_maxbufsize) 983 *(u_int *)addr = size = bpf_maxbufsize; 984 else if (size < BPF_MINBUFSIZE) 985 *(u_int *)addr = size = BPF_MINBUFSIZE; 986 d->bd_bufsize = size; 987 } 988 mutex_exit(d->bd_buf_mtx); 989 mutex_exit(d->bd_mtx); 990 break; 991 992 /* 993 * Set link layer read filter. 994 */ 995 case BIOCSETF: 996 error = bpf_setf(d, addr); 997 break; 998 999 /* 1000 * Flush read packet buffer. 1001 */ 1002 case BIOCFLUSH: 1003 mutex_enter(d->bd_mtx); 1004 reset_d(d); 1005 mutex_exit(d->bd_mtx); 1006 break; 1007 1008 /* 1009 * Put interface into promiscuous mode. 1010 */ 1011 case BIOCPROMISC: 1012 mutex_enter(d->bd_mtx); 1013 if (d->bd_bif == NULL) { 1014 mutex_exit(d->bd_mtx); 1015 /* 1016 * No interface attached yet. 1017 */ 1018 error = EINVAL; 1019 break; 1020 } 1021 if (d->bd_promisc == 0) { 1022 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 1023 error = ifpromisc(d->bd_bif->bif_ifp, 1); 1024 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 1025 if (error == 0) 1026 d->bd_promisc = 1; 1027 } 1028 mutex_exit(d->bd_mtx); 1029 break; 1030 1031 /* 1032 * Get device parameters. 1033 */ 1034 case BIOCGDLT: 1035 mutex_enter(d->bd_mtx); 1036 if (d->bd_bif == NULL) 1037 error = EINVAL; 1038 else 1039 *(u_int *)addr = d->bd_bif->bif_dlt; 1040 mutex_exit(d->bd_mtx); 1041 break; 1042 1043 /* 1044 * Get a list of supported device parameters. 1045 */ 1046 case BIOCGDLTLIST: 1047 mutex_enter(d->bd_mtx); 1048 if (d->bd_bif == NULL) 1049 error = EINVAL; 1050 else 1051 error = bpf_getdltlist(d, addr); 1052 mutex_exit(d->bd_mtx); 1053 break; 1054 1055 /* 1056 * Set device parameters. 1057 */ 1058 case BIOCSDLT: 1059 mutex_enter(&bpf_mtx); 1060 mutex_enter(d->bd_mtx); 1061 if (d->bd_bif == NULL) 1062 error = EINVAL; 1063 else 1064 error = bpf_setdlt(d, *(u_int *)addr); 1065 mutex_exit(d->bd_mtx); 1066 mutex_exit(&bpf_mtx); 1067 break; 1068 1069 /* 1070 * Set interface name. 1071 */ 1072 #ifdef OBIOCGETIF 1073 case OBIOCGETIF: 1074 #endif 1075 case BIOCGETIF: 1076 mutex_enter(d->bd_mtx); 1077 if (d->bd_bif == NULL) 1078 error = EINVAL; 1079 else 1080 bpf_ifname(d->bd_bif->bif_ifp, addr); 1081 mutex_exit(d->bd_mtx); 1082 break; 1083 1084 /* 1085 * Set interface. 1086 */ 1087 #ifdef OBIOCSETIF 1088 case OBIOCSETIF: 1089 #endif 1090 case BIOCSETIF: 1091 mutex_enter(&bpf_mtx); 1092 error = bpf_setif(d, addr); 1093 mutex_exit(&bpf_mtx); 1094 break; 1095 1096 /* 1097 * Set read timeout. 1098 */ 1099 case BIOCSRTIMEOUT: 1100 { 1101 struct timeval *tv = addr; 1102 1103 /* Compute number of ticks. */ 1104 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1105 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1106 d->bd_rtout = 1; 1107 break; 1108 } 1109 1110 #ifdef BIOCGORTIMEOUT 1111 /* 1112 * Get read timeout. 1113 */ 1114 case BIOCGORTIMEOUT: 1115 { 1116 struct timeval50 *tv = addr; 1117 1118 tv->tv_sec = d->bd_rtout / hz; 1119 tv->tv_usec = (d->bd_rtout % hz) * tick; 1120 break; 1121 } 1122 #endif 1123 1124 #ifdef BIOCSORTIMEOUT 1125 /* 1126 * Set read timeout. 1127 */ 1128 case BIOCSORTIMEOUT: 1129 { 1130 struct timeval50 *tv = addr; 1131 1132 /* Compute number of ticks. */ 1133 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1134 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1135 d->bd_rtout = 1; 1136 break; 1137 } 1138 #endif 1139 1140 /* 1141 * Get read timeout. 1142 */ 1143 case BIOCGRTIMEOUT: 1144 { 1145 struct timeval *tv = addr; 1146 1147 tv->tv_sec = d->bd_rtout / hz; 1148 tv->tv_usec = (d->bd_rtout % hz) * tick; 1149 break; 1150 } 1151 /* 1152 * Get packet stats. 1153 */ 1154 case BIOCGSTATS: 1155 { 1156 struct bpf_stat *bs = addr; 1157 1158 bs->bs_recv = d->bd_rcount; 1159 bs->bs_drop = d->bd_dcount; 1160 bs->bs_capt = d->bd_ccount; 1161 break; 1162 } 1163 1164 case BIOCGSTATSOLD: 1165 { 1166 struct bpf_stat_old *bs = addr; 1167 1168 bs->bs_recv = d->bd_rcount; 1169 bs->bs_drop = d->bd_dcount; 1170 break; 1171 } 1172 1173 /* 1174 * Set immediate mode. 1175 */ 1176 case BIOCIMMEDIATE: 1177 d->bd_immediate = *(u_int *)addr; 1178 break; 1179 1180 case BIOCVERSION: 1181 { 1182 struct bpf_version *bv = addr; 1183 1184 bv->bv_major = BPF_MAJOR_VERSION; 1185 bv->bv_minor = BPF_MINOR_VERSION; 1186 break; 1187 } 1188 1189 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1190 *(u_int *)addr = d->bd_hdrcmplt; 1191 break; 1192 1193 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1194 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1195 break; 1196 1197 /* 1198 * Get packet direction flag 1199 */ 1200 case BIOCGDIRECTION: 1201 *(u_int *)addr = d->bd_direction; 1202 break; 1203 1204 /* 1205 * Set packet direction flag 1206 */ 1207 case BIOCSDIRECTION: 1208 { 1209 u_int direction; 1210 1211 direction = *(u_int *)addr; 1212 switch (direction) { 1213 case BPF_D_IN: 1214 case BPF_D_INOUT: 1215 case BPF_D_OUT: 1216 d->bd_direction = direction; 1217 break; 1218 default: 1219 error = EINVAL; 1220 } 1221 } 1222 break; 1223 1224 /* 1225 * Set "feed packets from bpf back to input" mode 1226 */ 1227 case BIOCSFEEDBACK: 1228 d->bd_feedback = *(u_int *)addr; 1229 break; 1230 1231 /* 1232 * Get "feed packets from bpf back to input" mode 1233 */ 1234 case BIOCGFEEDBACK: 1235 *(u_int *)addr = d->bd_feedback; 1236 break; 1237 1238 case FIONBIO: /* Non-blocking I/O */ 1239 /* 1240 * No need to do anything special as we use IO_NDELAY in 1241 * bpfread() as an indication of whether or not to block 1242 * the read. 1243 */ 1244 break; 1245 1246 case FIOASYNC: /* Send signal on receive packets */ 1247 mutex_enter(d->bd_mtx); 1248 d->bd_async = *(int *)addr; 1249 mutex_exit(d->bd_mtx); 1250 break; 1251 1252 case TIOCSPGRP: /* Process or group to send signals to */ 1253 case FIOSETOWN: 1254 error = fsetown(&d->bd_pgid, cmd, addr); 1255 break; 1256 1257 case TIOCGPGRP: 1258 case FIOGETOWN: 1259 error = fgetown(d->bd_pgid, cmd, addr); 1260 break; 1261 } 1262 return (error); 1263 } 1264 1265 /* 1266 * Set d's packet filter program to fp. If this file already has a filter, 1267 * free it and replace it. Returns EINVAL for bogus requests. 1268 */ 1269 static int 1270 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1271 { 1272 struct bpf_insn *fcode; 1273 bpfjit_func_t jcode; 1274 size_t flen, size = 0; 1275 struct bpf_filter *oldf, *newf; 1276 1277 jcode = NULL; 1278 flen = fp->bf_len; 1279 1280 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1281 return EINVAL; 1282 } 1283 1284 if (flen) { 1285 /* 1286 * Allocate the buffer, copy the byte-code from 1287 * userspace and validate it. 1288 */ 1289 size = flen * sizeof(*fp->bf_insns); 1290 fcode = kmem_alloc(size, KM_SLEEP); 1291 if (copyin(fp->bf_insns, fcode, size) != 0 || 1292 !bpf_validate(fcode, (int)flen)) { 1293 kmem_free(fcode, size); 1294 return EINVAL; 1295 } 1296 if (bpf_jit) 1297 jcode = bpf_jit_generate(NULL, fcode, flen); 1298 } else { 1299 fcode = NULL; 1300 } 1301 1302 newf = kmem_alloc(sizeof(*newf), KM_SLEEP); 1303 newf->bf_insn = fcode; 1304 newf->bf_size = size; 1305 newf->bf_jitcode = jcode; 1306 d->bd_jitcode = jcode; /* XXX just for kvm(3) users */ 1307 1308 /* Need to hold bpf_mtx for pserialize_perform */ 1309 mutex_enter(&bpf_mtx); 1310 mutex_enter(d->bd_mtx); 1311 oldf = d->bd_filter; 1312 atomic_store_release(&d->bd_filter, newf); 1313 reset_d(d); 1314 pserialize_perform(bpf_psz); 1315 mutex_exit(d->bd_mtx); 1316 mutex_exit(&bpf_mtx); 1317 1318 if (oldf != NULL) 1319 bpf_free_filter(oldf); 1320 1321 return 0; 1322 } 1323 1324 /* 1325 * Detach a file from its current interface (if attached at all) and attach 1326 * to the interface indicated by the name stored in ifr. 1327 * Return an errno or 0. 1328 */ 1329 static int 1330 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1331 { 1332 struct bpf_if *bp; 1333 char *cp; 1334 int unit_seen, i, error; 1335 1336 KASSERT(mutex_owned(&bpf_mtx)); 1337 /* 1338 * Make sure the provided name has a unit number, and default 1339 * it to '0' if not specified. 1340 * XXX This is ugly ... do this differently? 1341 */ 1342 unit_seen = 0; 1343 cp = ifr->ifr_name; 1344 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1345 while (*cp++) 1346 if (*cp >= '0' && *cp <= '9') 1347 unit_seen = 1; 1348 if (!unit_seen) { 1349 /* Make sure to leave room for the '\0'. */ 1350 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1351 if ((ifr->ifr_name[i] >= 'a' && 1352 ifr->ifr_name[i] <= 'z') || 1353 (ifr->ifr_name[i] >= 'A' && 1354 ifr->ifr_name[i] <= 'Z')) 1355 continue; 1356 ifr->ifr_name[i] = '0'; 1357 } 1358 } 1359 1360 /* 1361 * Look through attached interfaces for the named one. 1362 */ 1363 BPF_IFLIST_WRITER_FOREACH(bp) { 1364 struct ifnet *ifp = bp->bif_ifp; 1365 1366 if (ifp == NULL || 1367 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1368 continue; 1369 /* skip additional entry */ 1370 if (bp->bif_driverp != &ifp->if_bpf) 1371 continue; 1372 /* 1373 * We found the requested interface. 1374 * Allocate the packet buffers if we need to. 1375 * If we're already attached to requested interface, 1376 * just flush the buffer. 1377 */ 1378 /* 1379 * bpf_allocbufs is called only here. bpf_mtx ensures that 1380 * no race condition happen on d->bd_sbuf. 1381 */ 1382 if (d->bd_sbuf == NULL) { 1383 error = bpf_allocbufs(d); 1384 if (error != 0) 1385 return (error); 1386 } 1387 mutex_enter(d->bd_mtx); 1388 if (bp != d->bd_bif) { 1389 if (d->bd_bif) { 1390 /* 1391 * Detach if attached to something else. 1392 */ 1393 bpf_detachd(d); 1394 BPFIF_DLIST_ENTRY_INIT(d); 1395 } 1396 1397 bpf_attachd(d, bp); 1398 } 1399 reset_d(d); 1400 mutex_exit(d->bd_mtx); 1401 return (0); 1402 } 1403 /* Not found. */ 1404 return (ENXIO); 1405 } 1406 1407 /* 1408 * Copy the interface name to the ifreq. 1409 */ 1410 static void 1411 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1412 { 1413 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1414 } 1415 1416 static int 1417 bpf_stat(struct file *fp, struct stat *st) 1418 { 1419 struct bpf_d *d = fp->f_bpf; 1420 1421 (void)memset(st, 0, sizeof(*st)); 1422 mutex_enter(d->bd_mtx); 1423 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1424 st->st_atimespec = d->bd_atime; 1425 st->st_mtimespec = d->bd_mtime; 1426 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1427 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1428 st->st_gid = kauth_cred_getegid(fp->f_cred); 1429 st->st_mode = S_IFCHR; 1430 mutex_exit(d->bd_mtx); 1431 return 0; 1432 } 1433 1434 /* 1435 * Support for poll() system call 1436 * 1437 * Return true iff the specific operation will not block indefinitely - with 1438 * the assumption that it is safe to positively acknowledge a request for the 1439 * ability to write to the BPF device. 1440 * Otherwise, return false but make a note that a selnotify() must be done. 1441 */ 1442 static int 1443 bpf_poll(struct file *fp, int events) 1444 { 1445 struct bpf_d *d = fp->f_bpf; 1446 int revents; 1447 1448 /* 1449 * Refresh the PID associated with this bpf file. 1450 */ 1451 mutex_enter(&bpf_mtx); 1452 d->bd_pid = curproc->p_pid; 1453 1454 revents = events & (POLLOUT | POLLWRNORM); 1455 if (events & (POLLIN | POLLRDNORM)) { 1456 /* 1457 * An imitation of the FIONREAD ioctl code. 1458 */ 1459 mutex_enter(d->bd_mtx); 1460 if (d->bd_hlen != 0 || 1461 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1462 d->bd_slen != 0)) { 1463 revents |= events & (POLLIN | POLLRDNORM); 1464 } else { 1465 selrecord(curlwp, &d->bd_sel); 1466 /* Start the read timeout if necessary */ 1467 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1468 callout_reset(&d->bd_callout, d->bd_rtout, 1469 bpf_timed_out, d); 1470 d->bd_state = BPF_WAITING; 1471 } 1472 } 1473 mutex_exit(d->bd_mtx); 1474 } 1475 1476 mutex_exit(&bpf_mtx); 1477 return (revents); 1478 } 1479 1480 static void 1481 filt_bpfrdetach(struct knote *kn) 1482 { 1483 struct bpf_d *d = kn->kn_hook; 1484 1485 mutex_enter(d->bd_buf_mtx); 1486 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1487 mutex_exit(d->bd_buf_mtx); 1488 } 1489 1490 static int 1491 filt_bpfread(struct knote *kn, long hint) 1492 { 1493 struct bpf_d *d = kn->kn_hook; 1494 int rv; 1495 1496 mutex_enter(d->bd_buf_mtx); 1497 kn->kn_data = d->bd_hlen; 1498 if (d->bd_immediate) 1499 kn->kn_data += d->bd_slen; 1500 rv = (kn->kn_data > 0); 1501 mutex_exit(d->bd_buf_mtx); 1502 return rv; 1503 } 1504 1505 static const struct filterops bpfread_filtops = { 1506 .f_isfd = 1, 1507 .f_attach = NULL, 1508 .f_detach = filt_bpfrdetach, 1509 .f_event = filt_bpfread, 1510 }; 1511 1512 static int 1513 bpf_kqfilter(struct file *fp, struct knote *kn) 1514 { 1515 struct bpf_d *d = fp->f_bpf; 1516 struct klist *klist; 1517 1518 mutex_enter(d->bd_buf_mtx); 1519 switch (kn->kn_filter) { 1520 case EVFILT_READ: 1521 klist = &d->bd_sel.sel_klist; 1522 kn->kn_fop = &bpfread_filtops; 1523 break; 1524 1525 default: 1526 mutex_exit(d->bd_buf_mtx); 1527 return (EINVAL); 1528 } 1529 1530 kn->kn_hook = d; 1531 1532 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1533 mutex_exit(d->bd_buf_mtx); 1534 1535 return (0); 1536 } 1537 1538 /* 1539 * Copy data from an mbuf chain into a buffer. This code is derived 1540 * from m_copydata in sys/uipc_mbuf.c. 1541 */ 1542 static void * 1543 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1544 { 1545 const struct mbuf *m; 1546 u_int count; 1547 u_char *dst; 1548 1549 m = src_arg; 1550 dst = dst_arg; 1551 while (len > 0) { 1552 if (m == NULL) 1553 panic("bpf_mcpy"); 1554 count = uimin(m->m_len, len); 1555 memcpy(dst, mtod(m, const void *), count); 1556 m = m->m_next; 1557 dst += count; 1558 len -= count; 1559 } 1560 return dst_arg; 1561 } 1562 1563 /* 1564 * Dispatch a packet to all the listeners on interface bp. 1565 * 1566 * pkt pointer to the packet, either a data buffer or an mbuf chain 1567 * buflen buffer length, if pkt is a data buffer 1568 * cpfn a function that can copy pkt into the listener's buffer 1569 * pktlen length of the packet 1570 * direction BPF_D_IN or BPF_D_OUT 1571 */ 1572 static inline void 1573 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1574 void *pkt, u_int pktlen, u_int buflen, const u_int direction) 1575 { 1576 uint32_t mem[BPF_MEMWORDS]; 1577 bpf_args_t args = { 1578 .pkt = (const uint8_t *)pkt, 1579 .wirelen = pktlen, 1580 .buflen = buflen, 1581 .mem = mem, 1582 .arg = NULL 1583 }; 1584 bool gottime = false; 1585 struct timespec ts; 1586 struct bpf_d *d; 1587 int s; 1588 1589 KASSERT(!cpu_intr_p()); 1590 1591 /* 1592 * Note that the IPL does not have to be raised at this point. 1593 * The only problem that could arise here is that if two different 1594 * interfaces shared any data. This is not the case. 1595 */ 1596 s = pserialize_read_enter(); 1597 BPFIF_DLIST_READER_FOREACH(d, bp) { 1598 u_int slen = 0; 1599 struct bpf_filter *filter; 1600 1601 if (direction == BPF_D_IN) { 1602 if (d->bd_direction == BPF_D_OUT) 1603 continue; 1604 } else { /* BPF_D_OUT */ 1605 if (d->bd_direction == BPF_D_IN) 1606 continue; 1607 } 1608 1609 atomic_inc_ulong(&d->bd_rcount); 1610 BPF_STATINC(recv); 1611 1612 filter = atomic_load_consume(&d->bd_filter); 1613 if (filter != NULL) { 1614 if (filter->bf_jitcode != NULL) 1615 slen = filter->bf_jitcode(NULL, &args); 1616 else 1617 slen = bpf_filter_ext(NULL, filter->bf_insn, 1618 &args); 1619 } 1620 1621 if (!slen) { 1622 continue; 1623 } 1624 if (!gottime) { 1625 gottime = true; 1626 nanotime(&ts); 1627 } 1628 /* Assume catchpacket doesn't sleep */ 1629 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1630 } 1631 pserialize_read_exit(s); 1632 } 1633 1634 /* 1635 * Incoming linkage from device drivers, when the head of the packet is in 1636 * a buffer, and the tail is in an mbuf chain. 1637 */ 1638 static void 1639 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m, 1640 u_int direction) 1641 { 1642 u_int pktlen; 1643 struct mbuf mb; 1644 1645 /* Skip outgoing duplicate packets. */ 1646 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1647 m->m_flags &= ~M_PROMISC; 1648 return; 1649 } 1650 1651 pktlen = m_length(m) + dlen; 1652 1653 /* 1654 * Craft on-stack mbuf suitable for passing to bpf_filter. 1655 * Note that we cut corners here; we only setup what's 1656 * absolutely needed--this mbuf should never go anywhere else. 1657 */ 1658 (void)memset(&mb, 0, sizeof(mb)); 1659 mb.m_type = MT_DATA; 1660 mb.m_next = m; 1661 mb.m_data = data; 1662 mb.m_len = dlen; 1663 1664 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, direction); 1665 } 1666 1667 /* 1668 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1669 */ 1670 static void 1671 _bpf_mtap(struct bpf_if *bp, struct mbuf *m, u_int direction) 1672 { 1673 void *(*cpfn)(void *, const void *, size_t); 1674 u_int pktlen, buflen; 1675 void *marg; 1676 1677 /* Skip outgoing duplicate packets. */ 1678 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1679 m->m_flags &= ~M_PROMISC; 1680 return; 1681 } 1682 1683 pktlen = m_length(m); 1684 1685 /* Skip zero-sized packets. */ 1686 if (__predict_false(pktlen == 0)) { 1687 return; 1688 } 1689 1690 if (pktlen == m->m_len) { 1691 cpfn = (void *)memcpy; 1692 marg = mtod(m, void *); 1693 buflen = pktlen; 1694 KASSERT(buflen != 0); 1695 } else { 1696 cpfn = bpf_mcpy; 1697 marg = m; 1698 buflen = 0; 1699 } 1700 1701 bpf_deliver(bp, cpfn, marg, pktlen, buflen, direction); 1702 } 1703 1704 /* 1705 * We need to prepend the address family as 1706 * a four byte field. Cons up a dummy header 1707 * to pacify bpf. This is safe because bpf 1708 * will only read from the mbuf (i.e., it won't 1709 * try to free it or keep a pointer a to it). 1710 */ 1711 static void 1712 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m, u_int direction) 1713 { 1714 struct mbuf m0; 1715 1716 m0.m_type = MT_DATA; 1717 m0.m_flags = 0; 1718 m0.m_next = m; 1719 m0.m_nextpkt = NULL; 1720 m0.m_owner = NULL; 1721 m0.m_len = 4; 1722 m0.m_data = (char *)⁡ 1723 1724 _bpf_mtap(bp, &m0, direction); 1725 } 1726 1727 /* 1728 * Put the SLIP pseudo-"link header" in place. 1729 * Note this M_PREPEND() should never fail, 1730 * swince we know we always have enough space 1731 * in the input buffer. 1732 */ 1733 static void 1734 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1735 { 1736 u_char *hp; 1737 1738 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1739 if (*m == NULL) 1740 return; 1741 1742 hp = mtod(*m, u_char *); 1743 hp[SLX_DIR] = SLIPDIR_IN; 1744 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1745 1746 _bpf_mtap(bp, *m, BPF_D_IN); 1747 1748 m_adj(*m, SLIP_HDRLEN); 1749 } 1750 1751 /* 1752 * Put the SLIP pseudo-"link header" in 1753 * place. The compressed header is now 1754 * at the beginning of the mbuf. 1755 */ 1756 static void 1757 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1758 { 1759 struct mbuf m0; 1760 u_char *hp; 1761 1762 m0.m_type = MT_DATA; 1763 m0.m_flags = 0; 1764 m0.m_next = m; 1765 m0.m_nextpkt = NULL; 1766 m0.m_owner = NULL; 1767 m0.m_data = m0.m_dat; 1768 m0.m_len = SLIP_HDRLEN; 1769 1770 hp = mtod(&m0, u_char *); 1771 1772 hp[SLX_DIR] = SLIPDIR_OUT; 1773 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1774 1775 _bpf_mtap(bp, &m0, BPF_D_OUT); 1776 m_freem(m); 1777 } 1778 1779 static struct mbuf * 1780 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) 1781 { 1782 struct mbuf *dup; 1783 1784 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); 1785 if (dup == NULL) 1786 return NULL; 1787 1788 if (bp->bif_mbuf_tail != NULL) { 1789 bp->bif_mbuf_tail->m_nextpkt = dup; 1790 } else { 1791 bp->bif_mbuf_head = dup; 1792 } 1793 bp->bif_mbuf_tail = dup; 1794 #ifdef BPF_MTAP_SOFTINT_DEBUG 1795 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", 1796 __func__, dup, bp->bif_ifp->if_xname); 1797 #endif 1798 1799 return dup; 1800 } 1801 1802 static struct mbuf * 1803 bpf_mbuf_dequeue(struct bpf_if *bp) 1804 { 1805 struct mbuf *m; 1806 int s; 1807 1808 /* XXX NOMPSAFE: assumed running on one CPU */ 1809 s = splnet(); 1810 m = bp->bif_mbuf_head; 1811 if (m != NULL) { 1812 bp->bif_mbuf_head = m->m_nextpkt; 1813 m->m_nextpkt = NULL; 1814 1815 if (bp->bif_mbuf_head == NULL) 1816 bp->bif_mbuf_tail = NULL; 1817 #ifdef BPF_MTAP_SOFTINT_DEBUG 1818 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", 1819 __func__, m, bp->bif_ifp->if_xname); 1820 #endif 1821 } 1822 splx(s); 1823 1824 return m; 1825 } 1826 1827 static void 1828 bpf_mtap_si(void *arg) 1829 { 1830 struct bpf_if *bp = arg; 1831 struct mbuf *m; 1832 1833 while ((m = bpf_mbuf_dequeue(bp)) != NULL) { 1834 #ifdef BPF_MTAP_SOFTINT_DEBUG 1835 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", 1836 __func__, m, bp->bif_ifp->if_xname); 1837 #endif 1838 bpf_ops->bpf_mtap(bp, m, BPF_D_IN); 1839 m_freem(m); 1840 } 1841 } 1842 1843 static void 1844 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) 1845 { 1846 struct bpf_if *bp = ifp->if_bpf; 1847 struct mbuf *dup; 1848 1849 KASSERT(cpu_intr_p()); 1850 1851 /* To avoid extra invocations of the softint */ 1852 if (BPFIF_DLIST_READER_EMPTY(bp)) 1853 return; 1854 KASSERT(bp->bif_si != NULL); 1855 1856 dup = bpf_mbuf_enqueue(bp, m); 1857 if (dup != NULL) 1858 softint_schedule(bp->bif_si); 1859 } 1860 1861 static int 1862 bpf_hdrlen(struct bpf_d *d) 1863 { 1864 int hdrlen = d->bd_bif->bif_hdrlen; 1865 /* 1866 * Compute the length of the bpf header. This is not necessarily 1867 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1868 * that the network layer header begins on a longword boundary (for 1869 * performance reasons and to alleviate alignment restrictions). 1870 */ 1871 #ifdef _LP64 1872 if (d->bd_compat32) 1873 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1874 else 1875 #endif 1876 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1877 } 1878 1879 /* 1880 * Move the packet data from interface memory (pkt) into the 1881 * store buffer. Call the wakeup functions if it's time to wakeup 1882 * a listener (buffer full), "cpfn" is the routine called to do the 1883 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1884 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1885 * pkt is really an mbuf. 1886 */ 1887 static void 1888 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1889 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1890 { 1891 char *h; 1892 int totlen, curlen, caplen; 1893 int hdrlen = bpf_hdrlen(d); 1894 int do_wakeup = 0; 1895 1896 atomic_inc_ulong(&d->bd_ccount); 1897 BPF_STATINC(capt); 1898 /* 1899 * Figure out how many bytes to move. If the packet is 1900 * greater or equal to the snapshot length, transfer that 1901 * much. Otherwise, transfer the whole packet (unless 1902 * we hit the buffer size limit). 1903 */ 1904 totlen = hdrlen + uimin(snaplen, pktlen); 1905 if (totlen > d->bd_bufsize) 1906 totlen = d->bd_bufsize; 1907 /* 1908 * If we adjusted totlen to fit the bufsize, it could be that 1909 * totlen is smaller than hdrlen because of the link layer header. 1910 */ 1911 caplen = totlen - hdrlen; 1912 if (caplen < 0) 1913 caplen = 0; 1914 1915 mutex_enter(d->bd_buf_mtx); 1916 /* 1917 * Round up the end of the previous packet to the next longword. 1918 */ 1919 #ifdef _LP64 1920 if (d->bd_compat32) 1921 curlen = BPF_WORDALIGN32(d->bd_slen); 1922 else 1923 #endif 1924 curlen = BPF_WORDALIGN(d->bd_slen); 1925 if (curlen + totlen > d->bd_bufsize) { 1926 /* 1927 * This packet will overflow the storage buffer. 1928 * Rotate the buffers if we can, then wakeup any 1929 * pending reads. 1930 */ 1931 if (d->bd_fbuf == NULL) { 1932 mutex_exit(d->bd_buf_mtx); 1933 /* 1934 * We haven't completed the previous read yet, 1935 * so drop the packet. 1936 */ 1937 atomic_inc_ulong(&d->bd_dcount); 1938 BPF_STATINC(drop); 1939 return; 1940 } 1941 ROTATE_BUFFERS(d); 1942 do_wakeup = 1; 1943 curlen = 0; 1944 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1945 /* 1946 * Immediate mode is set, or the read timeout has 1947 * already expired during a select call. A packet 1948 * arrived, so the reader should be woken up. 1949 */ 1950 do_wakeup = 1; 1951 } 1952 1953 /* 1954 * Append the bpf header. 1955 */ 1956 h = (char *)d->bd_sbuf + curlen; 1957 #ifdef _LP64 1958 if (d->bd_compat32) { 1959 struct bpf_hdr32 *hp32; 1960 1961 hp32 = (struct bpf_hdr32 *)h; 1962 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1963 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1964 hp32->bh_datalen = pktlen; 1965 hp32->bh_hdrlen = hdrlen; 1966 hp32->bh_caplen = caplen; 1967 } else 1968 #endif 1969 { 1970 struct bpf_hdr *hp; 1971 1972 hp = (struct bpf_hdr *)h; 1973 hp->bh_tstamp.tv_sec = ts->tv_sec; 1974 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1975 hp->bh_datalen = pktlen; 1976 hp->bh_hdrlen = hdrlen; 1977 hp->bh_caplen = caplen; 1978 } 1979 1980 /* 1981 * Copy the packet data into the store buffer and update its length. 1982 */ 1983 (*cpfn)(h + hdrlen, pkt, caplen); 1984 d->bd_slen = curlen + totlen; 1985 mutex_exit(d->bd_buf_mtx); 1986 1987 /* 1988 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1989 * will cause filt_bpfread() to be called with it adjusted. 1990 */ 1991 if (do_wakeup) 1992 bpf_wakeup(d); 1993 } 1994 1995 /* 1996 * Initialize all nonzero fields of a descriptor. 1997 */ 1998 static int 1999 bpf_allocbufs(struct bpf_d *d) 2000 { 2001 2002 d->bd_fbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); 2003 if (!d->bd_fbuf) 2004 return (ENOBUFS); 2005 d->bd_sbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); 2006 if (!d->bd_sbuf) { 2007 kmem_free(d->bd_fbuf, d->bd_bufsize); 2008 return (ENOBUFS); 2009 } 2010 d->bd_slen = 0; 2011 d->bd_hlen = 0; 2012 return (0); 2013 } 2014 2015 static void 2016 bpf_free_filter(struct bpf_filter *filter) 2017 { 2018 2019 KASSERT(filter != NULL); 2020 KASSERT(filter->bf_insn != NULL); 2021 2022 kmem_free(filter->bf_insn, filter->bf_size); 2023 if (filter->bf_jitcode != NULL) 2024 bpf_jit_freecode(filter->bf_jitcode); 2025 kmem_free(filter, sizeof(*filter)); 2026 } 2027 2028 /* 2029 * Free buffers currently in use by a descriptor. 2030 * Called on close. 2031 */ 2032 static void 2033 bpf_freed(struct bpf_d *d) 2034 { 2035 /* 2036 * We don't need to lock out interrupts since this descriptor has 2037 * been detached from its interface and it yet hasn't been marked 2038 * free. 2039 */ 2040 if (d->bd_sbuf != NULL) { 2041 kmem_free(d->bd_sbuf, d->bd_bufsize); 2042 if (d->bd_hbuf != NULL) 2043 kmem_free(d->bd_hbuf, d->bd_bufsize); 2044 if (d->bd_fbuf != NULL) 2045 kmem_free(d->bd_fbuf, d->bd_bufsize); 2046 } 2047 if (d->bd_filter != NULL) { 2048 bpf_free_filter(d->bd_filter); 2049 d->bd_filter = NULL; 2050 } 2051 d->bd_jitcode = NULL; 2052 } 2053 2054 /* 2055 * Attach an interface to bpf. dlt is the link layer type; 2056 * hdrlen is the fixed size of the link header for the specified dlt 2057 * (variable length headers not yet supported). 2058 */ 2059 static void 2060 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2061 { 2062 struct bpf_if *bp; 2063 bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP); 2064 if (bp == NULL) 2065 panic("bpfattach"); 2066 2067 mutex_enter(&bpf_mtx); 2068 bp->bif_driverp = driverp; 2069 bp->bif_ifp = ifp; 2070 bp->bif_dlt = dlt; 2071 bp->bif_si = NULL; 2072 BPF_IFLIST_ENTRY_INIT(bp); 2073 PSLIST_INIT(&bp->bif_dlist_head); 2074 psref_target_init(&bp->bif_psref, bpf_psref_class); 2075 2076 BPF_IFLIST_WRITER_INSERT_HEAD(bp); 2077 2078 *bp->bif_driverp = NULL; 2079 2080 bp->bif_hdrlen = hdrlen; 2081 mutex_exit(&bpf_mtx); 2082 #if 0 2083 printf("bpf: %s attached\n", ifp->if_xname); 2084 #endif 2085 } 2086 2087 static void 2088 _bpf_mtap_softint_init(struct ifnet *ifp) 2089 { 2090 struct bpf_if *bp; 2091 2092 mutex_enter(&bpf_mtx); 2093 BPF_IFLIST_WRITER_FOREACH(bp) { 2094 if (bp->bif_ifp != ifp) 2095 continue; 2096 2097 bp->bif_mbuf_head = NULL; 2098 bp->bif_mbuf_tail = NULL; 2099 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); 2100 if (bp->bif_si == NULL) 2101 panic("%s: softint_establish() failed", __func__); 2102 break; 2103 } 2104 mutex_exit(&bpf_mtx); 2105 2106 if (bp == NULL) 2107 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); 2108 } 2109 2110 /* 2111 * Remove an interface from bpf. 2112 */ 2113 static void 2114 _bpfdetach(struct ifnet *ifp) 2115 { 2116 struct bpf_if *bp; 2117 struct bpf_d *d; 2118 int s; 2119 2120 mutex_enter(&bpf_mtx); 2121 /* Nuke the vnodes for any open instances */ 2122 again_d: 2123 BPF_DLIST_WRITER_FOREACH(d) { 2124 mutex_enter(d->bd_mtx); 2125 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 2126 /* 2127 * Detach the descriptor from an interface now. 2128 * It will be free'ed later by close routine. 2129 */ 2130 d->bd_promisc = 0; /* we can't touch device. */ 2131 bpf_detachd(d); 2132 mutex_exit(d->bd_mtx); 2133 goto again_d; 2134 } 2135 mutex_exit(d->bd_mtx); 2136 } 2137 2138 again: 2139 BPF_IFLIST_WRITER_FOREACH(bp) { 2140 if (bp->bif_ifp == ifp) { 2141 BPF_IFLIST_WRITER_REMOVE(bp); 2142 2143 pserialize_perform(bpf_psz); 2144 psref_target_destroy(&bp->bif_psref, bpf_psref_class); 2145 2146 BPF_IFLIST_ENTRY_DESTROY(bp); 2147 if (bp->bif_si != NULL) { 2148 /* XXX NOMPSAFE: assumed running on one CPU */ 2149 s = splnet(); 2150 while (bp->bif_mbuf_head != NULL) { 2151 struct mbuf *m = bp->bif_mbuf_head; 2152 bp->bif_mbuf_head = m->m_nextpkt; 2153 m_freem(m); 2154 } 2155 splx(s); 2156 softint_disestablish(bp->bif_si); 2157 } 2158 kmem_free(bp, sizeof(*bp)); 2159 goto again; 2160 } 2161 } 2162 mutex_exit(&bpf_mtx); 2163 } 2164 2165 /* 2166 * Change the data link type of a interface. 2167 */ 2168 static void 2169 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2170 { 2171 struct bpf_if *bp; 2172 2173 mutex_enter(&bpf_mtx); 2174 BPF_IFLIST_WRITER_FOREACH(bp) { 2175 if (bp->bif_driverp == &ifp->if_bpf) 2176 break; 2177 } 2178 if (bp == NULL) 2179 panic("bpf_change_type"); 2180 2181 bp->bif_dlt = dlt; 2182 2183 bp->bif_hdrlen = hdrlen; 2184 mutex_exit(&bpf_mtx); 2185 } 2186 2187 /* 2188 * Get a list of available data link type of the interface. 2189 */ 2190 static int 2191 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2192 { 2193 int n, error; 2194 struct ifnet *ifp; 2195 struct bpf_if *bp; 2196 int s, bound; 2197 2198 KASSERT(mutex_owned(d->bd_mtx)); 2199 2200 ifp = d->bd_bif->bif_ifp; 2201 n = 0; 2202 error = 0; 2203 2204 bound = curlwp_bind(); 2205 s = pserialize_read_enter(); 2206 BPF_IFLIST_READER_FOREACH(bp) { 2207 if (bp->bif_ifp != ifp) 2208 continue; 2209 if (bfl->bfl_list != NULL) { 2210 struct psref psref; 2211 2212 if (n >= bfl->bfl_len) { 2213 pserialize_read_exit(s); 2214 return ENOMEM; 2215 } 2216 2217 bpf_if_acquire(bp, &psref); 2218 pserialize_read_exit(s); 2219 2220 error = copyout(&bp->bif_dlt, 2221 bfl->bfl_list + n, sizeof(u_int)); 2222 2223 s = pserialize_read_enter(); 2224 bpf_if_release(bp, &psref); 2225 } 2226 n++; 2227 } 2228 pserialize_read_exit(s); 2229 curlwp_bindx(bound); 2230 2231 bfl->bfl_len = n; 2232 return error; 2233 } 2234 2235 /* 2236 * Set the data link type of a BPF instance. 2237 */ 2238 static int 2239 bpf_setdlt(struct bpf_d *d, u_int dlt) 2240 { 2241 int error, opromisc; 2242 struct ifnet *ifp; 2243 struct bpf_if *bp; 2244 2245 KASSERT(mutex_owned(&bpf_mtx)); 2246 KASSERT(mutex_owned(d->bd_mtx)); 2247 2248 if (d->bd_bif->bif_dlt == dlt) 2249 return 0; 2250 ifp = d->bd_bif->bif_ifp; 2251 BPF_IFLIST_WRITER_FOREACH(bp) { 2252 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2253 break; 2254 } 2255 if (bp == NULL) 2256 return EINVAL; 2257 opromisc = d->bd_promisc; 2258 bpf_detachd(d); 2259 BPFIF_DLIST_ENTRY_INIT(d); 2260 bpf_attachd(d, bp); 2261 reset_d(d); 2262 if (opromisc) { 2263 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 2264 error = ifpromisc(bp->bif_ifp, 1); 2265 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 2266 if (error) 2267 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 2268 bp->bif_ifp->if_xname, error); 2269 else 2270 d->bd_promisc = 1; 2271 } 2272 return 0; 2273 } 2274 2275 static int 2276 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 2277 { 2278 int newsize, error; 2279 struct sysctlnode node; 2280 2281 node = *rnode; 2282 node.sysctl_data = &newsize; 2283 newsize = bpf_maxbufsize; 2284 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2285 if (error || newp == NULL) 2286 return (error); 2287 2288 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 2289 return (EINVAL); 2290 2291 bpf_maxbufsize = newsize; 2292 2293 return (0); 2294 } 2295 2296 #if defined(MODULAR) || defined(BPFJIT) 2297 static int 2298 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 2299 { 2300 bool newval; 2301 int error; 2302 struct sysctlnode node; 2303 2304 node = *rnode; 2305 node.sysctl_data = &newval; 2306 newval = bpf_jit; 2307 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2308 if (error != 0 || newp == NULL) 2309 return error; 2310 2311 bpf_jit = newval; 2312 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 2313 printf("JIT compilation is postponed " 2314 "until after bpfjit module is loaded\n"); 2315 } 2316 2317 return 0; 2318 } 2319 #endif 2320 2321 static int 2322 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 2323 { 2324 int error, elem_count; 2325 struct bpf_d *dp; 2326 struct bpf_d_ext dpe; 2327 size_t len, needed, elem_size, out_size; 2328 char *sp; 2329 2330 if (namelen == 1 && name[0] == CTL_QUERY) 2331 return (sysctl_query(SYSCTLFN_CALL(rnode))); 2332 2333 if (namelen != 2) 2334 return (EINVAL); 2335 2336 /* BPF peers is privileged information. */ 2337 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 2338 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 2339 if (error) 2340 return (EPERM); 2341 2342 len = (oldp != NULL) ? *oldlenp : 0; 2343 sp = oldp; 2344 elem_size = name[0]; 2345 elem_count = name[1]; 2346 out_size = MIN(sizeof(dpe), elem_size); 2347 needed = 0; 2348 2349 if (elem_size < 1 || elem_count < 0) 2350 return (EINVAL); 2351 2352 mutex_enter(&bpf_mtx); 2353 BPF_DLIST_WRITER_FOREACH(dp) { 2354 if (len >= elem_size && elem_count > 0) { 2355 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 2356 BPF_EXT(bufsize); 2357 BPF_EXT(promisc); 2358 BPF_EXT(state); 2359 BPF_EXT(immediate); 2360 BPF_EXT(hdrcmplt); 2361 BPF_EXT(direction); 2362 BPF_EXT(pid); 2363 BPF_EXT(rcount); 2364 BPF_EXT(dcount); 2365 BPF_EXT(ccount); 2366 #undef BPF_EXT 2367 mutex_enter(dp->bd_mtx); 2368 if (dp->bd_bif) 2369 (void)strlcpy(dpe.bde_ifname, 2370 dp->bd_bif->bif_ifp->if_xname, 2371 IFNAMSIZ - 1); 2372 else 2373 dpe.bde_ifname[0] = '\0'; 2374 mutex_exit(dp->bd_mtx); 2375 2376 error = copyout(&dpe, sp, out_size); 2377 if (error) 2378 break; 2379 sp += elem_size; 2380 len -= elem_size; 2381 } 2382 needed += elem_size; 2383 if (elem_count > 0 && elem_count != INT_MAX) 2384 elem_count--; 2385 } 2386 mutex_exit(&bpf_mtx); 2387 2388 *oldlenp = needed; 2389 2390 return (error); 2391 } 2392 2393 static void 2394 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused) 2395 { 2396 struct bpf_stat *const stats = p; 2397 struct bpf_stat *sum = arg; 2398 2399 int s = splnet(); 2400 2401 sum->bs_recv += stats->bs_recv; 2402 sum->bs_drop += stats->bs_drop; 2403 sum->bs_capt += stats->bs_capt; 2404 2405 splx(s); 2406 } 2407 2408 static int 2409 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS) 2410 { 2411 struct sysctlnode node; 2412 int error; 2413 struct bpf_stat sum; 2414 2415 memset(&sum, 0, sizeof(sum)); 2416 node = *rnode; 2417 2418 percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET), 2419 bpf_stats, &sum); 2420 2421 node.sysctl_data = ∑ 2422 node.sysctl_size = sizeof(sum); 2423 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2424 if (error != 0 || newp == NULL) 2425 return error; 2426 2427 return 0; 2428 } 2429 2430 SYSCTL_SETUP(sysctl_net_bpf_setup, "bpf sysctls") 2431 { 2432 const struct sysctlnode *node; 2433 2434 node = NULL; 2435 sysctl_createv(clog, 0, NULL, &node, 2436 CTLFLAG_PERMANENT, 2437 CTLTYPE_NODE, "bpf", 2438 SYSCTL_DESCR("BPF options"), 2439 NULL, 0, NULL, 0, 2440 CTL_NET, CTL_CREATE, CTL_EOL); 2441 if (node != NULL) { 2442 #if defined(MODULAR) || defined(BPFJIT) 2443 sysctl_createv(clog, 0, NULL, NULL, 2444 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2445 CTLTYPE_BOOL, "jit", 2446 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2447 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2448 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2449 #endif 2450 sysctl_createv(clog, 0, NULL, NULL, 2451 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2452 CTLTYPE_INT, "maxbufsize", 2453 SYSCTL_DESCR("Maximum size for data capture buffer"), 2454 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2455 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2456 sysctl_createv(clog, 0, NULL, NULL, 2457 CTLFLAG_PERMANENT, 2458 CTLTYPE_STRUCT, "stats", 2459 SYSCTL_DESCR("BPF stats"), 2460 bpf_sysctl_gstats_handler, 0, NULL, 0, 2461 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2462 sysctl_createv(clog, 0, NULL, NULL, 2463 CTLFLAG_PERMANENT, 2464 CTLTYPE_STRUCT, "peers", 2465 SYSCTL_DESCR("BPF peers"), 2466 sysctl_net_bpf_peers, 0, NULL, 0, 2467 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2468 } 2469 2470 } 2471 2472 struct bpf_ops bpf_ops_kernel = { 2473 .bpf_attach = _bpfattach, 2474 .bpf_detach = _bpfdetach, 2475 .bpf_change_type = _bpf_change_type, 2476 2477 .bpf_mtap = _bpf_mtap, 2478 .bpf_mtap2 = _bpf_mtap2, 2479 .bpf_mtap_af = _bpf_mtap_af, 2480 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2481 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2482 2483 .bpf_mtap_softint = _bpf_mtap_softint, 2484 .bpf_mtap_softint_init = _bpf_mtap_softint_init, 2485 }; 2486 2487 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); 2488 2489 static int 2490 bpf_modcmd(modcmd_t cmd, void *arg) 2491 { 2492 #ifdef _MODULE 2493 devmajor_t bmajor, cmajor; 2494 #endif 2495 int error = 0; 2496 2497 switch (cmd) { 2498 case MODULE_CMD_INIT: 2499 bpf_init(); 2500 #ifdef _MODULE 2501 bmajor = cmajor = NODEVMAJOR; 2502 error = devsw_attach("bpf", NULL, &bmajor, 2503 &bpf_cdevsw, &cmajor); 2504 if (error) 2505 break; 2506 #endif 2507 2508 bpf_ops_handover_enter(&bpf_ops_kernel); 2509 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2510 bpf_ops_handover_exit(); 2511 break; 2512 2513 case MODULE_CMD_FINI: 2514 /* 2515 * While there is no reference counting for bpf callers, 2516 * unload could at least in theory be done similarly to 2517 * system call disestablishment. This should even be 2518 * a little simpler: 2519 * 2520 * 1) replace op vector with stubs 2521 * 2) post update to all cpus with xc 2522 * 3) check that nobody is in bpf anymore 2523 * (it's doubtful we'd want something like l_sysent, 2524 * but we could do something like *signed* percpu 2525 * counters. if the sum is 0, we're good). 2526 * 4) if fail, unroll changes 2527 * 2528 * NOTE: change won't be atomic to the outside. some 2529 * packets may be not captured even if unload is 2530 * not succesful. I think packet capture not working 2531 * is a perfectly logical consequence of trying to 2532 * disable packet capture. 2533 */ 2534 error = EOPNOTSUPP; 2535 break; 2536 2537 default: 2538 error = ENOTTY; 2539 break; 2540 } 2541 2542 return error; 2543 } 2544