1 /* $NetBSD: bpf.c,v 1.219 2017/11/17 07:37:12 ozaki-r Exp $ */ 2 3 /* 4 * Copyright (c) 1990, 1991, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from the Stanford/CMU enet packet filter, 8 * (net/enet.c) distributed as part of 4.3BSD, and code contributed 9 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence 10 * Berkeley Laboratory. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)bpf.c 8.4 (Berkeley) 1/9/95 37 * static char rcsid[] = 38 * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.219 2017/11/17 07:37:12 ozaki-r Exp $"); 43 44 #if defined(_KERNEL_OPT) 45 #include "opt_bpf.h" 46 #include "sl.h" 47 #include "strip.h" 48 #include "opt_net_mpsafe.h" 49 #endif 50 51 #include <sys/param.h> 52 #include <sys/systm.h> 53 #include <sys/mbuf.h> 54 #include <sys/buf.h> 55 #include <sys/time.h> 56 #include <sys/proc.h> 57 #include <sys/ioctl.h> 58 #include <sys/conf.h> 59 #include <sys/vnode.h> 60 #include <sys/queue.h> 61 #include <sys/stat.h> 62 #include <sys/module.h> 63 #include <sys/atomic.h> 64 #include <sys/cpu.h> 65 66 #include <sys/file.h> 67 #include <sys/filedesc.h> 68 #include <sys/tty.h> 69 #include <sys/uio.h> 70 71 #include <sys/protosw.h> 72 #include <sys/socket.h> 73 #include <sys/errno.h> 74 #include <sys/kernel.h> 75 #include <sys/poll.h> 76 #include <sys/sysctl.h> 77 #include <sys/kauth.h> 78 #include <sys/syslog.h> 79 #include <sys/percpu.h> 80 #include <sys/pserialize.h> 81 #include <sys/lwp.h> 82 83 #include <net/if.h> 84 #include <net/slip.h> 85 86 #include <net/bpf.h> 87 #include <net/bpfdesc.h> 88 #include <net/bpfjit.h> 89 90 #include <net/if_arc.h> 91 #include <net/if_ether.h> 92 93 #include <netinet/in.h> 94 #include <netinet/if_inarp.h> 95 96 97 #include <compat/sys/sockio.h> 98 99 #ifndef BPF_BUFSIZE 100 /* 101 * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet 102 * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). 103 */ 104 # define BPF_BUFSIZE 32768 105 #endif 106 107 #define PRINET 26 /* interruptible */ 108 109 /* 110 * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. 111 * XXX the default values should be computed dynamically based 112 * on available memory size and available mbuf clusters. 113 */ 114 static int bpf_bufsize = BPF_BUFSIZE; 115 static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ 116 static bool bpf_jit = false; 117 118 struct bpfjit_ops bpfjit_module_ops = { 119 .bj_generate_code = NULL, 120 .bj_free_code = NULL 121 }; 122 123 /* 124 * Global BPF statistics returned by net.bpf.stats sysctl. 125 */ 126 static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */ 127 128 #define BPF_STATINC(id) \ 129 { \ 130 struct bpf_stat *__stats = \ 131 percpu_getref(bpf_gstats_percpu); \ 132 __stats->bs_##id++; \ 133 percpu_putref(bpf_gstats_percpu); \ 134 } 135 136 /* 137 * Locking notes: 138 * - bpf_mtx (adaptive mutex) protects: 139 * - Gobal lists: bpf_iflist and bpf_dlist 140 * - struct bpf_if 141 * - bpf_close 142 * - bpf_psz (pserialize) 143 * - struct bpf_d has two mutexes: 144 * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed 145 * on packet tapping 146 * - bd_mtx (adaptive mutex) protects member variables other than the buffers 147 * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx 148 * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is 149 * never freed because struct bpf_d is only freed in bpf_close and 150 * bpf_close never be called while executing bpf_read and bpf_write 151 * - A filter that is assigned to bpf_d can be replaced with another filter 152 * while tapping packets, so it needs to be done atomically 153 * - struct bpf_d is iterated on bpf_dlist with psz 154 * - struct bpf_if is iterated on bpf_iflist with psz or psref 155 */ 156 /* 157 * Use a mutex to avoid a race condition between gathering the stats/peers 158 * and opening/closing the device. 159 */ 160 static kmutex_t bpf_mtx; 161 162 static struct psref_class *bpf_psref_class __read_mostly; 163 static pserialize_t bpf_psz; 164 165 static inline void 166 bpf_if_acquire(struct bpf_if *bp, struct psref *psref) 167 { 168 169 psref_acquire(psref, &bp->bif_psref, bpf_psref_class); 170 } 171 172 static inline void 173 bpf_if_release(struct bpf_if *bp, struct psref *psref) 174 { 175 176 psref_release(psref, &bp->bif_psref, bpf_psref_class); 177 } 178 179 /* 180 * bpf_iflist is the list of interfaces; each corresponds to an ifnet 181 * bpf_dtab holds the descriptors, indexed by minor device # 182 */ 183 static struct pslist_head bpf_iflist; 184 static struct pslist_head bpf_dlist; 185 186 /* Macros for bpf_d on bpf_dlist */ 187 #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \ 188 PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) 189 #define BPF_DLIST_READER_FOREACH(__d) \ 190 PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 191 bd_bpf_dlist_entry) 192 #define BPF_DLIST_WRITER_FOREACH(__d) \ 193 PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ 194 bd_bpf_dlist_entry) 195 #define BPF_DLIST_ENTRY_INIT(__d) \ 196 PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) 197 #define BPF_DLIST_WRITER_REMOVE(__d) \ 198 PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) 199 #define BPF_DLIST_ENTRY_DESTROY(__d) \ 200 PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) 201 202 /* Macros for bpf_if on bpf_iflist */ 203 #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ 204 PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) 205 #define BPF_IFLIST_READER_FOREACH(__bp) \ 206 PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 207 bif_iflist_entry) 208 #define BPF_IFLIST_WRITER_FOREACH(__bp) \ 209 PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ 210 bif_iflist_entry) 211 #define BPF_IFLIST_WRITER_REMOVE(__bp) \ 212 PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) 213 #define BPF_IFLIST_ENTRY_INIT(__bp) \ 214 PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) 215 #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ 216 PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) 217 218 /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ 219 #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ 220 PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ 221 bd_bif_dlist_entry) 222 #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ 223 PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ 224 bd_bif_dlist_entry) 225 #define BPFIF_DLIST_WRITER_REMOVE(__d) \ 226 PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) 227 #define BPFIF_DLIST_ENTRY_INIT(__d) \ 228 PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) 229 #define BPFIF_DLIST_READER_EMPTY(__bp) \ 230 (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 231 bd_bif_dlist_entry) == NULL) 232 #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ 233 (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ 234 bd_bif_dlist_entry) == NULL) 235 #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ 236 PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) 237 238 static int bpf_allocbufs(struct bpf_d *); 239 static void bpf_deliver(struct bpf_if *, 240 void *(*cpfn)(void *, const void *, size_t), 241 void *, u_int, u_int, const bool); 242 static void bpf_freed(struct bpf_d *); 243 static void bpf_free_filter(struct bpf_filter *); 244 static void bpf_ifname(struct ifnet *, struct ifreq *); 245 static void *bpf_mcpy(void *, const void *, size_t); 246 static int bpf_movein(struct uio *, int, uint64_t, 247 struct mbuf **, struct sockaddr *); 248 static void bpf_attachd(struct bpf_d *, struct bpf_if *); 249 static void bpf_detachd(struct bpf_d *); 250 static int bpf_setif(struct bpf_d *, struct ifreq *); 251 static int bpf_setf(struct bpf_d *, struct bpf_program *); 252 static void bpf_timed_out(void *); 253 static inline void 254 bpf_wakeup(struct bpf_d *); 255 static int bpf_hdrlen(struct bpf_d *); 256 static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, 257 void *(*)(void *, const void *, size_t), struct timespec *); 258 static void reset_d(struct bpf_d *); 259 static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); 260 static int bpf_setdlt(struct bpf_d *, u_int); 261 262 static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, 263 int); 264 static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, 265 int); 266 static int bpf_ioctl(struct file *, u_long, void *); 267 static int bpf_poll(struct file *, int); 268 static int bpf_stat(struct file *, struct stat *); 269 static int bpf_close(struct file *); 270 static int bpf_kqfilter(struct file *, struct knote *); 271 static void bpf_softintr(void *); 272 273 static const struct fileops bpf_fileops = { 274 .fo_read = bpf_read, 275 .fo_write = bpf_write, 276 .fo_ioctl = bpf_ioctl, 277 .fo_fcntl = fnullop_fcntl, 278 .fo_poll = bpf_poll, 279 .fo_stat = bpf_stat, 280 .fo_close = bpf_close, 281 .fo_kqfilter = bpf_kqfilter, 282 .fo_restart = fnullop_restart, 283 }; 284 285 dev_type_open(bpfopen); 286 287 const struct cdevsw bpf_cdevsw = { 288 .d_open = bpfopen, 289 .d_close = noclose, 290 .d_read = noread, 291 .d_write = nowrite, 292 .d_ioctl = noioctl, 293 .d_stop = nostop, 294 .d_tty = notty, 295 .d_poll = nopoll, 296 .d_mmap = nommap, 297 .d_kqfilter = nokqfilter, 298 .d_discard = nodiscard, 299 .d_flag = D_OTHER | D_MPSAFE 300 }; 301 302 bpfjit_func_t 303 bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) 304 { 305 306 membar_consumer(); 307 if (bpfjit_module_ops.bj_generate_code != NULL) { 308 return bpfjit_module_ops.bj_generate_code(bc, code, size); 309 } 310 return NULL; 311 } 312 313 void 314 bpf_jit_freecode(bpfjit_func_t jcode) 315 { 316 KASSERT(bpfjit_module_ops.bj_free_code != NULL); 317 bpfjit_module_ops.bj_free_code(jcode); 318 } 319 320 static int 321 bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, 322 struct sockaddr *sockp) 323 { 324 struct mbuf *m; 325 int error; 326 size_t len; 327 size_t hlen; 328 size_t align; 329 330 /* 331 * Build a sockaddr based on the data link layer type. 332 * We do this at this level because the ethernet header 333 * is copied directly into the data field of the sockaddr. 334 * In the case of SLIP, there is no header and the packet 335 * is forwarded as is. 336 * Also, we are careful to leave room at the front of the mbuf 337 * for the link level header. 338 */ 339 switch (linktype) { 340 341 case DLT_SLIP: 342 sockp->sa_family = AF_INET; 343 hlen = 0; 344 align = 0; 345 break; 346 347 case DLT_PPP: 348 sockp->sa_family = AF_UNSPEC; 349 hlen = 0; 350 align = 0; 351 break; 352 353 case DLT_EN10MB: 354 sockp->sa_family = AF_UNSPEC; 355 /* XXX Would MAXLINKHDR be better? */ 356 /* 6(dst)+6(src)+2(type) */ 357 hlen = sizeof(struct ether_header); 358 align = 2; 359 break; 360 361 case DLT_ARCNET: 362 sockp->sa_family = AF_UNSPEC; 363 hlen = ARC_HDRLEN; 364 align = 5; 365 break; 366 367 case DLT_FDDI: 368 sockp->sa_family = AF_LINK; 369 /* XXX 4(FORMAC)+6(dst)+6(src) */ 370 hlen = 16; 371 align = 0; 372 break; 373 374 case DLT_ECONET: 375 sockp->sa_family = AF_UNSPEC; 376 hlen = 6; 377 align = 2; 378 break; 379 380 case DLT_NULL: 381 sockp->sa_family = AF_UNSPEC; 382 hlen = 0; 383 align = 0; 384 break; 385 386 default: 387 return (EIO); 388 } 389 390 len = uio->uio_resid; 391 /* 392 * If there aren't enough bytes for a link level header or the 393 * packet length exceeds the interface mtu, return an error. 394 */ 395 if (len - hlen > mtu) 396 return (EMSGSIZE); 397 398 /* 399 * XXX Avoid complicated buffer chaining --- 400 * bail if it won't fit in a single mbuf. 401 * (Take into account possible alignment bytes) 402 */ 403 if (len + align > MCLBYTES) 404 return (EIO); 405 406 m = m_gethdr(M_WAIT, MT_DATA); 407 m_reset_rcvif(m); 408 m->m_pkthdr.len = (int)(len - hlen); 409 if (len + align > MHLEN) { 410 m_clget(m, M_WAIT); 411 if ((m->m_flags & M_EXT) == 0) { 412 error = ENOBUFS; 413 goto bad; 414 } 415 } 416 417 /* Insure the data is properly aligned */ 418 if (align > 0) { 419 m->m_data += align; 420 m->m_len -= (int)align; 421 } 422 423 error = uiomove(mtod(m, void *), len, uio); 424 if (error) 425 goto bad; 426 if (hlen != 0) { 427 memcpy(sockp->sa_data, mtod(m, void *), hlen); 428 m->m_data += hlen; /* XXX */ 429 len -= hlen; 430 } 431 m->m_len = (int)len; 432 *mp = m; 433 return (0); 434 435 bad: 436 m_freem(m); 437 return (error); 438 } 439 440 /* 441 * Attach file to the bpf interface, i.e. make d listen on bp. 442 */ 443 static void 444 bpf_attachd(struct bpf_d *d, struct bpf_if *bp) 445 { 446 447 KASSERT(mutex_owned(&bpf_mtx)); 448 KASSERT(mutex_owned(d->bd_mtx)); 449 /* 450 * Point d at bp, and add d to the interface's list of listeners. 451 * Finally, point the driver's bpf cookie at the interface so 452 * it will divert packets to bpf. 453 */ 454 d->bd_bif = bp; 455 BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); 456 457 *bp->bif_driverp = bp; 458 } 459 460 /* 461 * Detach a file from its interface. 462 */ 463 static void 464 bpf_detachd(struct bpf_d *d) 465 { 466 struct bpf_if *bp; 467 468 KASSERT(mutex_owned(&bpf_mtx)); 469 KASSERT(mutex_owned(d->bd_mtx)); 470 471 bp = d->bd_bif; 472 /* 473 * Check if this descriptor had requested promiscuous mode. 474 * If so, turn it off. 475 */ 476 if (d->bd_promisc) { 477 int error __diagused; 478 479 d->bd_promisc = 0; 480 /* 481 * Take device out of promiscuous mode. Since we were 482 * able to enter promiscuous mode, we should be able 483 * to turn it off. But we can get an error if 484 * the interface was configured down, so only panic 485 * if we don't get an unexpected error. 486 */ 487 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 488 error = ifpromisc(bp->bif_ifp, 0); 489 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 490 #ifdef DIAGNOSTIC 491 if (error) 492 printf("%s: ifpromisc failed: %d", __func__, error); 493 #endif 494 } 495 496 /* Remove d from the interface's descriptor list. */ 497 BPFIF_DLIST_WRITER_REMOVE(d); 498 499 pserialize_perform(bpf_psz); 500 501 if (BPFIF_DLIST_WRITER_EMPTY(bp)) { 502 /* 503 * Let the driver know that there are no more listeners. 504 */ 505 *d->bd_bif->bif_driverp = NULL; 506 } 507 d->bd_bif = NULL; 508 } 509 510 static void 511 bpf_init(void) 512 { 513 514 mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); 515 bpf_psz = pserialize_create(); 516 bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET); 517 518 PSLIST_INIT(&bpf_iflist); 519 PSLIST_INIT(&bpf_dlist); 520 521 bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat)); 522 523 return; 524 } 525 526 /* 527 * bpfilterattach() is called at boot time. We don't need to do anything 528 * here, since any initialization will happen as part of module init code. 529 */ 530 /* ARGSUSED */ 531 void 532 bpfilterattach(int n) 533 { 534 535 } 536 537 /* 538 * Open ethernet device. Clones. 539 */ 540 /* ARGSUSED */ 541 int 542 bpfopen(dev_t dev, int flag, int mode, struct lwp *l) 543 { 544 struct bpf_d *d; 545 struct file *fp; 546 int error, fd; 547 548 /* falloc() will fill in the descriptor for us. */ 549 if ((error = fd_allocfile(&fp, &fd)) != 0) 550 return error; 551 552 d = kmem_zalloc(sizeof(*d), KM_SLEEP); 553 d->bd_bufsize = bpf_bufsize; 554 d->bd_seesent = 1; 555 d->bd_feedback = 0; 556 d->bd_pid = l->l_proc->p_pid; 557 #ifdef _LP64 558 if (curproc->p_flag & PK_32) 559 d->bd_compat32 = 1; 560 #endif 561 getnanotime(&d->bd_btime); 562 d->bd_atime = d->bd_mtime = d->bd_btime; 563 callout_init(&d->bd_callout, 0); 564 selinit(&d->bd_sel); 565 d->bd_sih = softint_establish(SOFTINT_CLOCK, bpf_softintr, d); 566 d->bd_jitcode = NULL; 567 d->bd_filter = NULL; 568 BPF_DLIST_ENTRY_INIT(d); 569 BPFIF_DLIST_ENTRY_INIT(d); 570 d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 571 d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 572 cv_init(&d->bd_cv, "bpf"); 573 574 mutex_enter(&bpf_mtx); 575 BPF_DLIST_WRITER_INSERT_HEAD(d); 576 mutex_exit(&bpf_mtx); 577 578 return fd_clone(fp, fd, flag, &bpf_fileops, d); 579 } 580 581 /* 582 * Close the descriptor by detaching it from its interface, 583 * deallocating its buffers, and marking it free. 584 */ 585 /* ARGSUSED */ 586 static int 587 bpf_close(struct file *fp) 588 { 589 struct bpf_d *d; 590 591 mutex_enter(&bpf_mtx); 592 593 if ((d = fp->f_bpf) == NULL) { 594 mutex_exit(&bpf_mtx); 595 return 0; 596 } 597 598 /* 599 * Refresh the PID associated with this bpf file. 600 */ 601 d->bd_pid = curproc->p_pid; 602 603 mutex_enter(d->bd_mtx); 604 if (d->bd_state == BPF_WAITING) 605 callout_halt(&d->bd_callout, d->bd_mtx); 606 d->bd_state = BPF_IDLE; 607 if (d->bd_bif) 608 bpf_detachd(d); 609 mutex_exit(d->bd_mtx); 610 611 BPF_DLIST_WRITER_REMOVE(d); 612 613 pserialize_perform(bpf_psz); 614 mutex_exit(&bpf_mtx); 615 616 BPFIF_DLIST_ENTRY_DESTROY(d); 617 BPF_DLIST_ENTRY_DESTROY(d); 618 fp->f_bpf = NULL; 619 bpf_freed(d); 620 callout_destroy(&d->bd_callout); 621 seldestroy(&d->bd_sel); 622 softint_disestablish(d->bd_sih); 623 mutex_obj_free(d->bd_mtx); 624 mutex_obj_free(d->bd_buf_mtx); 625 cv_destroy(&d->bd_cv); 626 627 kmem_free(d, sizeof(*d)); 628 629 return (0); 630 } 631 632 /* 633 * Rotate the packet buffers in descriptor d. Move the store buffer 634 * into the hold slot, and the free buffer into the store slot. 635 * Zero the length of the new store buffer. 636 */ 637 #define ROTATE_BUFFERS(d) \ 638 (d)->bd_hbuf = (d)->bd_sbuf; \ 639 (d)->bd_hlen = (d)->bd_slen; \ 640 (d)->bd_sbuf = (d)->bd_fbuf; \ 641 (d)->bd_slen = 0; \ 642 (d)->bd_fbuf = NULL; 643 /* 644 * bpfread - read next chunk of packets from buffers 645 */ 646 static int 647 bpf_read(struct file *fp, off_t *offp, struct uio *uio, 648 kauth_cred_t cred, int flags) 649 { 650 struct bpf_d *d = fp->f_bpf; 651 int timed_out; 652 int error; 653 654 getnanotime(&d->bd_atime); 655 /* 656 * Restrict application to use a buffer the same size as 657 * the kernel buffers. 658 */ 659 if (uio->uio_resid != d->bd_bufsize) 660 return (EINVAL); 661 662 mutex_enter(d->bd_mtx); 663 if (d->bd_state == BPF_WAITING) 664 callout_halt(&d->bd_callout, d->bd_buf_mtx); 665 timed_out = (d->bd_state == BPF_TIMED_OUT); 666 d->bd_state = BPF_IDLE; 667 mutex_exit(d->bd_mtx); 668 /* 669 * If the hold buffer is empty, then do a timed sleep, which 670 * ends when the timeout expires or when enough packets 671 * have arrived to fill the store buffer. 672 */ 673 mutex_enter(d->bd_buf_mtx); 674 while (d->bd_hbuf == NULL) { 675 if (fp->f_flag & FNONBLOCK) { 676 if (d->bd_slen == 0) { 677 error = EWOULDBLOCK; 678 goto out; 679 } 680 ROTATE_BUFFERS(d); 681 break; 682 } 683 684 if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { 685 /* 686 * A packet(s) either arrived since the previous 687 * read or arrived while we were asleep. 688 * Rotate the buffers and return what's here. 689 */ 690 ROTATE_BUFFERS(d); 691 break; 692 } 693 694 error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout); 695 696 if (error == EINTR || error == ERESTART) 697 goto out; 698 699 if (error == EWOULDBLOCK) { 700 /* 701 * On a timeout, return what's in the buffer, 702 * which may be nothing. If there is something 703 * in the store buffer, we can rotate the buffers. 704 */ 705 if (d->bd_hbuf) 706 /* 707 * We filled up the buffer in between 708 * getting the timeout and arriving 709 * here, so we don't need to rotate. 710 */ 711 break; 712 713 if (d->bd_slen == 0) { 714 error = 0; 715 goto out; 716 } 717 ROTATE_BUFFERS(d); 718 break; 719 } 720 if (error != 0) 721 goto out; 722 } 723 /* 724 * At this point, we know we have something in the hold slot. 725 */ 726 mutex_exit(d->bd_buf_mtx); 727 728 /* 729 * Move data from hold buffer into user space. 730 * We know the entire buffer is transferred since 731 * we checked above that the read buffer is bpf_bufsize bytes. 732 */ 733 error = uiomove(d->bd_hbuf, d->bd_hlen, uio); 734 735 mutex_enter(d->bd_buf_mtx); 736 d->bd_fbuf = d->bd_hbuf; 737 d->bd_hbuf = NULL; 738 d->bd_hlen = 0; 739 out: 740 mutex_exit(d->bd_buf_mtx); 741 return (error); 742 } 743 744 745 /* 746 * If there are processes sleeping on this descriptor, wake them up. 747 */ 748 static inline void 749 bpf_wakeup(struct bpf_d *d) 750 { 751 752 mutex_enter(d->bd_buf_mtx); 753 cv_broadcast(&d->bd_cv); 754 mutex_exit(d->bd_buf_mtx); 755 756 if (d->bd_async) 757 softint_schedule(d->bd_sih); 758 selnotify(&d->bd_sel, 0, 0); 759 } 760 761 static void 762 bpf_softintr(void *cookie) 763 { 764 struct bpf_d *d; 765 766 d = cookie; 767 if (d->bd_async) 768 fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); 769 } 770 771 static void 772 bpf_timed_out(void *arg) 773 { 774 struct bpf_d *d = arg; 775 776 mutex_enter(d->bd_mtx); 777 if (d->bd_state == BPF_WAITING) { 778 d->bd_state = BPF_TIMED_OUT; 779 if (d->bd_slen != 0) 780 bpf_wakeup(d); 781 } 782 mutex_exit(d->bd_mtx); 783 } 784 785 786 static int 787 bpf_write(struct file *fp, off_t *offp, struct uio *uio, 788 kauth_cred_t cred, int flags) 789 { 790 struct bpf_d *d = fp->f_bpf; 791 struct bpf_if *bp; 792 struct ifnet *ifp; 793 struct mbuf *m, *mc; 794 int error; 795 static struct sockaddr_storage dst; 796 struct psref psref; 797 int bound; 798 799 m = NULL; /* XXX gcc */ 800 801 bound = curlwp_bind(); 802 mutex_enter(d->bd_mtx); 803 bp = d->bd_bif; 804 if (bp == NULL) { 805 mutex_exit(d->bd_mtx); 806 error = ENXIO; 807 goto out_bindx; 808 } 809 bpf_if_acquire(bp, &psref); 810 mutex_exit(d->bd_mtx); 811 812 getnanotime(&d->bd_mtime); 813 814 ifp = bp->bif_ifp; 815 if (if_is_deactivated(ifp)) { 816 error = ENXIO; 817 goto out; 818 } 819 820 if (uio->uio_resid == 0) { 821 error = 0; 822 goto out; 823 } 824 825 error = bpf_movein(uio, (int)bp->bif_dlt, ifp->if_mtu, &m, 826 (struct sockaddr *) &dst); 827 if (error) 828 goto out; 829 830 if (m->m_pkthdr.len > ifp->if_mtu) { 831 m_freem(m); 832 error = EMSGSIZE; 833 goto out; 834 } 835 836 if (d->bd_hdrcmplt) 837 dst.ss_family = pseudo_AF_HDRCMPLT; 838 839 if (d->bd_feedback) { 840 mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); 841 if (mc != NULL) 842 m_set_rcvif(mc, ifp); 843 /* Set M_PROMISC for outgoing packets to be discarded. */ 844 if (1 /*d->bd_direction == BPF_D_INOUT*/) 845 m->m_flags |= M_PROMISC; 846 } else 847 mc = NULL; 848 849 error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); 850 851 if (mc != NULL) { 852 if (error == 0) 853 ifp->_if_input(ifp, mc); 854 else 855 m_freem(mc); 856 } 857 /* 858 * The driver frees the mbuf. 859 */ 860 out: 861 bpf_if_release(bp, &psref); 862 out_bindx: 863 curlwp_bindx(bound); 864 return error; 865 } 866 867 /* 868 * Reset a descriptor by flushing its packet buffer and clearing the 869 * receive and drop counts. 870 */ 871 static void 872 reset_d(struct bpf_d *d) 873 { 874 875 KASSERT(mutex_owned(d->bd_mtx)); 876 877 mutex_enter(d->bd_buf_mtx); 878 if (d->bd_hbuf) { 879 /* Free the hold buffer. */ 880 d->bd_fbuf = d->bd_hbuf; 881 d->bd_hbuf = NULL; 882 } 883 d->bd_slen = 0; 884 d->bd_hlen = 0; 885 d->bd_rcount = 0; 886 d->bd_dcount = 0; 887 d->bd_ccount = 0; 888 mutex_exit(d->bd_buf_mtx); 889 } 890 891 /* 892 * FIONREAD Check for read packet available. 893 * BIOCGBLEN Get buffer len [for read()]. 894 * BIOCSETF Set ethernet read filter. 895 * BIOCFLUSH Flush read packet buffer. 896 * BIOCPROMISC Put interface into promiscuous mode. 897 * BIOCGDLT Get link layer type. 898 * BIOCGETIF Get interface name. 899 * BIOCSETIF Set interface. 900 * BIOCSRTIMEOUT Set read timeout. 901 * BIOCGRTIMEOUT Get read timeout. 902 * BIOCGSTATS Get packet stats. 903 * BIOCIMMEDIATE Set immediate mode. 904 * BIOCVERSION Get filter language version. 905 * BIOCGHDRCMPLT Get "header already complete" flag. 906 * BIOCSHDRCMPLT Set "header already complete" flag. 907 * BIOCSFEEDBACK Set packet feedback mode. 908 * BIOCGFEEDBACK Get packet feedback mode. 909 * BIOCGSEESENT Get "see sent packets" mode. 910 * BIOCSSEESENT Set "see sent packets" mode. 911 */ 912 /* ARGSUSED */ 913 static int 914 bpf_ioctl(struct file *fp, u_long cmd, void *addr) 915 { 916 struct bpf_d *d = fp->f_bpf; 917 int error = 0; 918 919 /* 920 * Refresh the PID associated with this bpf file. 921 */ 922 d->bd_pid = curproc->p_pid; 923 #ifdef _LP64 924 if (curproc->p_flag & PK_32) 925 d->bd_compat32 = 1; 926 else 927 d->bd_compat32 = 0; 928 #endif 929 930 mutex_enter(d->bd_mtx); 931 if (d->bd_state == BPF_WAITING) 932 callout_halt(&d->bd_callout, d->bd_mtx); 933 d->bd_state = BPF_IDLE; 934 mutex_exit(d->bd_mtx); 935 936 switch (cmd) { 937 938 default: 939 error = EINVAL; 940 break; 941 942 /* 943 * Check for read packet available. 944 */ 945 case FIONREAD: 946 { 947 int n; 948 949 mutex_enter(d->bd_buf_mtx); 950 n = d->bd_slen; 951 if (d->bd_hbuf) 952 n += d->bd_hlen; 953 mutex_exit(d->bd_buf_mtx); 954 955 *(int *)addr = n; 956 break; 957 } 958 959 /* 960 * Get buffer len [for read()]. 961 */ 962 case BIOCGBLEN: 963 *(u_int *)addr = d->bd_bufsize; 964 break; 965 966 /* 967 * Set buffer length. 968 */ 969 case BIOCSBLEN: 970 /* 971 * Forbid to change the buffer length if buffers are already 972 * allocated. 973 */ 974 mutex_enter(d->bd_mtx); 975 mutex_enter(d->bd_buf_mtx); 976 if (d->bd_bif != NULL || d->bd_sbuf != NULL) 977 error = EINVAL; 978 else { 979 u_int size = *(u_int *)addr; 980 981 if (size > bpf_maxbufsize) 982 *(u_int *)addr = size = bpf_maxbufsize; 983 else if (size < BPF_MINBUFSIZE) 984 *(u_int *)addr = size = BPF_MINBUFSIZE; 985 d->bd_bufsize = size; 986 } 987 mutex_exit(d->bd_buf_mtx); 988 mutex_exit(d->bd_mtx); 989 break; 990 991 /* 992 * Set link layer read filter. 993 */ 994 case BIOCSETF: 995 error = bpf_setf(d, addr); 996 break; 997 998 /* 999 * Flush read packet buffer. 1000 */ 1001 case BIOCFLUSH: 1002 mutex_enter(d->bd_mtx); 1003 reset_d(d); 1004 mutex_exit(d->bd_mtx); 1005 break; 1006 1007 /* 1008 * Put interface into promiscuous mode. 1009 */ 1010 case BIOCPROMISC: 1011 mutex_enter(d->bd_mtx); 1012 if (d->bd_bif == NULL) { 1013 mutex_exit(d->bd_mtx); 1014 /* 1015 * No interface attached yet. 1016 */ 1017 error = EINVAL; 1018 break; 1019 } 1020 if (d->bd_promisc == 0) { 1021 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 1022 error = ifpromisc(d->bd_bif->bif_ifp, 1); 1023 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 1024 if (error == 0) 1025 d->bd_promisc = 1; 1026 } 1027 mutex_exit(d->bd_mtx); 1028 break; 1029 1030 /* 1031 * Get device parameters. 1032 */ 1033 case BIOCGDLT: 1034 mutex_enter(d->bd_mtx); 1035 if (d->bd_bif == NULL) 1036 error = EINVAL; 1037 else 1038 *(u_int *)addr = d->bd_bif->bif_dlt; 1039 mutex_exit(d->bd_mtx); 1040 break; 1041 1042 /* 1043 * Get a list of supported device parameters. 1044 */ 1045 case BIOCGDLTLIST: 1046 mutex_enter(d->bd_mtx); 1047 if (d->bd_bif == NULL) 1048 error = EINVAL; 1049 else 1050 error = bpf_getdltlist(d, addr); 1051 mutex_exit(d->bd_mtx); 1052 break; 1053 1054 /* 1055 * Set device parameters. 1056 */ 1057 case BIOCSDLT: 1058 mutex_enter(&bpf_mtx); 1059 mutex_enter(d->bd_mtx); 1060 if (d->bd_bif == NULL) 1061 error = EINVAL; 1062 else 1063 error = bpf_setdlt(d, *(u_int *)addr); 1064 mutex_exit(d->bd_mtx); 1065 mutex_exit(&bpf_mtx); 1066 break; 1067 1068 /* 1069 * Set interface name. 1070 */ 1071 #ifdef OBIOCGETIF 1072 case OBIOCGETIF: 1073 #endif 1074 case BIOCGETIF: 1075 mutex_enter(d->bd_mtx); 1076 if (d->bd_bif == NULL) 1077 error = EINVAL; 1078 else 1079 bpf_ifname(d->bd_bif->bif_ifp, addr); 1080 mutex_exit(d->bd_mtx); 1081 break; 1082 1083 /* 1084 * Set interface. 1085 */ 1086 #ifdef OBIOCSETIF 1087 case OBIOCSETIF: 1088 #endif 1089 case BIOCSETIF: 1090 mutex_enter(&bpf_mtx); 1091 error = bpf_setif(d, addr); 1092 mutex_exit(&bpf_mtx); 1093 break; 1094 1095 /* 1096 * Set read timeout. 1097 */ 1098 case BIOCSRTIMEOUT: 1099 { 1100 struct timeval *tv = addr; 1101 1102 /* Compute number of ticks. */ 1103 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1104 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1105 d->bd_rtout = 1; 1106 break; 1107 } 1108 1109 #ifdef BIOCGORTIMEOUT 1110 /* 1111 * Get read timeout. 1112 */ 1113 case BIOCGORTIMEOUT: 1114 { 1115 struct timeval50 *tv = addr; 1116 1117 tv->tv_sec = d->bd_rtout / hz; 1118 tv->tv_usec = (d->bd_rtout % hz) * tick; 1119 break; 1120 } 1121 #endif 1122 1123 #ifdef BIOCSORTIMEOUT 1124 /* 1125 * Set read timeout. 1126 */ 1127 case BIOCSORTIMEOUT: 1128 { 1129 struct timeval50 *tv = addr; 1130 1131 /* Compute number of ticks. */ 1132 d->bd_rtout = tv->tv_sec * hz + tv->tv_usec / tick; 1133 if ((d->bd_rtout == 0) && (tv->tv_usec != 0)) 1134 d->bd_rtout = 1; 1135 break; 1136 } 1137 #endif 1138 1139 /* 1140 * Get read timeout. 1141 */ 1142 case BIOCGRTIMEOUT: 1143 { 1144 struct timeval *tv = addr; 1145 1146 tv->tv_sec = d->bd_rtout / hz; 1147 tv->tv_usec = (d->bd_rtout % hz) * tick; 1148 break; 1149 } 1150 /* 1151 * Get packet stats. 1152 */ 1153 case BIOCGSTATS: 1154 { 1155 struct bpf_stat *bs = addr; 1156 1157 bs->bs_recv = d->bd_rcount; 1158 bs->bs_drop = d->bd_dcount; 1159 bs->bs_capt = d->bd_ccount; 1160 break; 1161 } 1162 1163 case BIOCGSTATSOLD: 1164 { 1165 struct bpf_stat_old *bs = addr; 1166 1167 bs->bs_recv = d->bd_rcount; 1168 bs->bs_drop = d->bd_dcount; 1169 break; 1170 } 1171 1172 /* 1173 * Set immediate mode. 1174 */ 1175 case BIOCIMMEDIATE: 1176 d->bd_immediate = *(u_int *)addr; 1177 break; 1178 1179 case BIOCVERSION: 1180 { 1181 struct bpf_version *bv = addr; 1182 1183 bv->bv_major = BPF_MAJOR_VERSION; 1184 bv->bv_minor = BPF_MINOR_VERSION; 1185 break; 1186 } 1187 1188 case BIOCGHDRCMPLT: /* get "header already complete" flag */ 1189 *(u_int *)addr = d->bd_hdrcmplt; 1190 break; 1191 1192 case BIOCSHDRCMPLT: /* set "header already complete" flag */ 1193 d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; 1194 break; 1195 1196 /* 1197 * Get "see sent packets" flag 1198 */ 1199 case BIOCGSEESENT: 1200 *(u_int *)addr = d->bd_seesent; 1201 break; 1202 1203 /* 1204 * Set "see sent" packets flag 1205 */ 1206 case BIOCSSEESENT: 1207 d->bd_seesent = *(u_int *)addr; 1208 break; 1209 1210 /* 1211 * Set "feed packets from bpf back to input" mode 1212 */ 1213 case BIOCSFEEDBACK: 1214 d->bd_feedback = *(u_int *)addr; 1215 break; 1216 1217 /* 1218 * Get "feed packets from bpf back to input" mode 1219 */ 1220 case BIOCGFEEDBACK: 1221 *(u_int *)addr = d->bd_feedback; 1222 break; 1223 1224 case FIONBIO: /* Non-blocking I/O */ 1225 /* 1226 * No need to do anything special as we use IO_NDELAY in 1227 * bpfread() as an indication of whether or not to block 1228 * the read. 1229 */ 1230 break; 1231 1232 case FIOASYNC: /* Send signal on receive packets */ 1233 d->bd_async = *(int *)addr; 1234 break; 1235 1236 case TIOCSPGRP: /* Process or group to send signals to */ 1237 case FIOSETOWN: 1238 error = fsetown(&d->bd_pgid, cmd, addr); 1239 break; 1240 1241 case TIOCGPGRP: 1242 case FIOGETOWN: 1243 error = fgetown(d->bd_pgid, cmd, addr); 1244 break; 1245 } 1246 return (error); 1247 } 1248 1249 /* 1250 * Set d's packet filter program to fp. If this file already has a filter, 1251 * free it and replace it. Returns EINVAL for bogus requests. 1252 */ 1253 static int 1254 bpf_setf(struct bpf_d *d, struct bpf_program *fp) 1255 { 1256 struct bpf_insn *fcode; 1257 bpfjit_func_t jcode; 1258 size_t flen, size = 0; 1259 struct bpf_filter *oldf, *newf; 1260 1261 jcode = NULL; 1262 flen = fp->bf_len; 1263 1264 if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { 1265 return EINVAL; 1266 } 1267 1268 if (flen) { 1269 /* 1270 * Allocate the buffer, copy the byte-code from 1271 * userspace and validate it. 1272 */ 1273 size = flen * sizeof(*fp->bf_insns); 1274 fcode = kmem_alloc(size, KM_SLEEP); 1275 if (copyin(fp->bf_insns, fcode, size) != 0 || 1276 !bpf_validate(fcode, (int)flen)) { 1277 kmem_free(fcode, size); 1278 return EINVAL; 1279 } 1280 membar_consumer(); 1281 if (bpf_jit) 1282 jcode = bpf_jit_generate(NULL, fcode, flen); 1283 } else { 1284 fcode = NULL; 1285 } 1286 1287 newf = kmem_alloc(sizeof(*newf), KM_SLEEP); 1288 newf->bf_insn = fcode; 1289 newf->bf_size = size; 1290 newf->bf_jitcode = jcode; 1291 d->bd_jitcode = jcode; /* XXX just for kvm(3) users */ 1292 1293 /* Need to hold bpf_mtx for pserialize_perform */ 1294 mutex_enter(&bpf_mtx); 1295 mutex_enter(d->bd_mtx); 1296 oldf = d->bd_filter; 1297 d->bd_filter = newf; 1298 membar_producer(); 1299 reset_d(d); 1300 pserialize_perform(bpf_psz); 1301 mutex_exit(d->bd_mtx); 1302 mutex_exit(&bpf_mtx); 1303 1304 if (oldf != NULL) 1305 bpf_free_filter(oldf); 1306 1307 return 0; 1308 } 1309 1310 /* 1311 * Detach a file from its current interface (if attached at all) and attach 1312 * to the interface indicated by the name stored in ifr. 1313 * Return an errno or 0. 1314 */ 1315 static int 1316 bpf_setif(struct bpf_d *d, struct ifreq *ifr) 1317 { 1318 struct bpf_if *bp; 1319 char *cp; 1320 int unit_seen, i, error; 1321 1322 KASSERT(mutex_owned(&bpf_mtx)); 1323 /* 1324 * Make sure the provided name has a unit number, and default 1325 * it to '0' if not specified. 1326 * XXX This is ugly ... do this differently? 1327 */ 1328 unit_seen = 0; 1329 cp = ifr->ifr_name; 1330 cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ 1331 while (*cp++) 1332 if (*cp >= '0' && *cp <= '9') 1333 unit_seen = 1; 1334 if (!unit_seen) { 1335 /* Make sure to leave room for the '\0'. */ 1336 for (i = 0; i < (IFNAMSIZ - 1); ++i) { 1337 if ((ifr->ifr_name[i] >= 'a' && 1338 ifr->ifr_name[i] <= 'z') || 1339 (ifr->ifr_name[i] >= 'A' && 1340 ifr->ifr_name[i] <= 'Z')) 1341 continue; 1342 ifr->ifr_name[i] = '0'; 1343 } 1344 } 1345 1346 /* 1347 * Look through attached interfaces for the named one. 1348 */ 1349 BPF_IFLIST_WRITER_FOREACH(bp) { 1350 struct ifnet *ifp = bp->bif_ifp; 1351 1352 if (ifp == NULL || 1353 strcmp(ifp->if_xname, ifr->ifr_name) != 0) 1354 continue; 1355 /* skip additional entry */ 1356 if (bp->bif_driverp != &ifp->if_bpf) 1357 continue; 1358 /* 1359 * We found the requested interface. 1360 * Allocate the packet buffers if we need to. 1361 * If we're already attached to requested interface, 1362 * just flush the buffer. 1363 */ 1364 /* 1365 * bpf_allocbufs is called only here. bpf_mtx ensures that 1366 * no race condition happen on d->bd_sbuf. 1367 */ 1368 if (d->bd_sbuf == NULL) { 1369 error = bpf_allocbufs(d); 1370 if (error != 0) 1371 return (error); 1372 } 1373 mutex_enter(d->bd_mtx); 1374 if (bp != d->bd_bif) { 1375 if (d->bd_bif) { 1376 /* 1377 * Detach if attached to something else. 1378 */ 1379 bpf_detachd(d); 1380 BPFIF_DLIST_ENTRY_INIT(d); 1381 } 1382 1383 bpf_attachd(d, bp); 1384 } 1385 reset_d(d); 1386 mutex_exit(d->bd_mtx); 1387 return (0); 1388 } 1389 /* Not found. */ 1390 return (ENXIO); 1391 } 1392 1393 /* 1394 * Copy the interface name to the ifreq. 1395 */ 1396 static void 1397 bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) 1398 { 1399 memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); 1400 } 1401 1402 static int 1403 bpf_stat(struct file *fp, struct stat *st) 1404 { 1405 struct bpf_d *d = fp->f_bpf; 1406 1407 (void)memset(st, 0, sizeof(*st)); 1408 mutex_enter(d->bd_mtx); 1409 st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); 1410 st->st_atimespec = d->bd_atime; 1411 st->st_mtimespec = d->bd_mtime; 1412 st->st_ctimespec = st->st_birthtimespec = d->bd_btime; 1413 st->st_uid = kauth_cred_geteuid(fp->f_cred); 1414 st->st_gid = kauth_cred_getegid(fp->f_cred); 1415 st->st_mode = S_IFCHR; 1416 mutex_exit(d->bd_mtx); 1417 return 0; 1418 } 1419 1420 /* 1421 * Support for poll() system call 1422 * 1423 * Return true iff the specific operation will not block indefinitely - with 1424 * the assumption that it is safe to positively acknowledge a request for the 1425 * ability to write to the BPF device. 1426 * Otherwise, return false but make a note that a selnotify() must be done. 1427 */ 1428 static int 1429 bpf_poll(struct file *fp, int events) 1430 { 1431 struct bpf_d *d = fp->f_bpf; 1432 int revents; 1433 1434 /* 1435 * Refresh the PID associated with this bpf file. 1436 */ 1437 mutex_enter(&bpf_mtx); 1438 d->bd_pid = curproc->p_pid; 1439 1440 revents = events & (POLLOUT | POLLWRNORM); 1441 if (events & (POLLIN | POLLRDNORM)) { 1442 /* 1443 * An imitation of the FIONREAD ioctl code. 1444 */ 1445 mutex_enter(d->bd_mtx); 1446 if (d->bd_hlen != 0 || 1447 ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && 1448 d->bd_slen != 0)) { 1449 revents |= events & (POLLIN | POLLRDNORM); 1450 } else { 1451 selrecord(curlwp, &d->bd_sel); 1452 /* Start the read timeout if necessary */ 1453 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { 1454 callout_reset(&d->bd_callout, d->bd_rtout, 1455 bpf_timed_out, d); 1456 d->bd_state = BPF_WAITING; 1457 } 1458 } 1459 mutex_exit(d->bd_mtx); 1460 } 1461 1462 mutex_exit(&bpf_mtx); 1463 return (revents); 1464 } 1465 1466 static void 1467 filt_bpfrdetach(struct knote *kn) 1468 { 1469 struct bpf_d *d = kn->kn_hook; 1470 1471 mutex_enter(d->bd_buf_mtx); 1472 SLIST_REMOVE(&d->bd_sel.sel_klist, kn, knote, kn_selnext); 1473 mutex_exit(d->bd_buf_mtx); 1474 } 1475 1476 static int 1477 filt_bpfread(struct knote *kn, long hint) 1478 { 1479 struct bpf_d *d = kn->kn_hook; 1480 int rv; 1481 1482 mutex_enter(d->bd_buf_mtx); 1483 kn->kn_data = d->bd_hlen; 1484 if (d->bd_immediate) 1485 kn->kn_data += d->bd_slen; 1486 rv = (kn->kn_data > 0); 1487 mutex_exit(d->bd_buf_mtx); 1488 return rv; 1489 } 1490 1491 static const struct filterops bpfread_filtops = { 1492 .f_isfd = 1, 1493 .f_attach = NULL, 1494 .f_detach = filt_bpfrdetach, 1495 .f_event = filt_bpfread, 1496 }; 1497 1498 static int 1499 bpf_kqfilter(struct file *fp, struct knote *kn) 1500 { 1501 struct bpf_d *d = fp->f_bpf; 1502 struct klist *klist; 1503 1504 mutex_enter(d->bd_buf_mtx); 1505 switch (kn->kn_filter) { 1506 case EVFILT_READ: 1507 klist = &d->bd_sel.sel_klist; 1508 kn->kn_fop = &bpfread_filtops; 1509 break; 1510 1511 default: 1512 mutex_exit(d->bd_buf_mtx); 1513 return (EINVAL); 1514 } 1515 1516 kn->kn_hook = d; 1517 1518 SLIST_INSERT_HEAD(klist, kn, kn_selnext); 1519 mutex_exit(d->bd_buf_mtx); 1520 1521 return (0); 1522 } 1523 1524 /* 1525 * Copy data from an mbuf chain into a buffer. This code is derived 1526 * from m_copydata in sys/uipc_mbuf.c. 1527 */ 1528 static void * 1529 bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) 1530 { 1531 const struct mbuf *m; 1532 u_int count; 1533 u_char *dst; 1534 1535 m = src_arg; 1536 dst = dst_arg; 1537 while (len > 0) { 1538 if (m == NULL) 1539 panic("bpf_mcpy"); 1540 count = min(m->m_len, len); 1541 memcpy(dst, mtod(m, const void *), count); 1542 m = m->m_next; 1543 dst += count; 1544 len -= count; 1545 } 1546 return dst_arg; 1547 } 1548 1549 /* 1550 * Dispatch a packet to all the listeners on interface bp. 1551 * 1552 * pkt pointer to the packet, either a data buffer or an mbuf chain 1553 * buflen buffer length, if pkt is a data buffer 1554 * cpfn a function that can copy pkt into the listener's buffer 1555 * pktlen length of the packet 1556 * rcv true if packet came in 1557 */ 1558 static inline void 1559 bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), 1560 void *pkt, u_int pktlen, u_int buflen, const bool rcv) 1561 { 1562 uint32_t mem[BPF_MEMWORDS]; 1563 bpf_args_t args = { 1564 .pkt = (const uint8_t *)pkt, 1565 .wirelen = pktlen, 1566 .buflen = buflen, 1567 .mem = mem, 1568 .arg = NULL 1569 }; 1570 bool gottime = false; 1571 struct timespec ts; 1572 struct bpf_d *d; 1573 int s; 1574 1575 KASSERT(!cpu_intr_p()); 1576 1577 /* 1578 * Note that the IPL does not have to be raised at this point. 1579 * The only problem that could arise here is that if two different 1580 * interfaces shared any data. This is not the case. 1581 */ 1582 s = pserialize_read_enter(); 1583 BPFIF_DLIST_READER_FOREACH(d, bp) { 1584 u_int slen = 0; 1585 struct bpf_filter *filter; 1586 1587 if (!d->bd_seesent && !rcv) { 1588 continue; 1589 } 1590 atomic_inc_ulong(&d->bd_rcount); 1591 BPF_STATINC(recv); 1592 1593 filter = d->bd_filter; 1594 membar_datadep_consumer(); 1595 if (filter != NULL) { 1596 if (filter->bf_jitcode != NULL) 1597 slen = filter->bf_jitcode(NULL, &args); 1598 else 1599 slen = bpf_filter_ext(NULL, filter->bf_insn, 1600 &args); 1601 } 1602 1603 if (!slen) { 1604 continue; 1605 } 1606 if (!gottime) { 1607 gottime = true; 1608 nanotime(&ts); 1609 } 1610 /* Assume catchpacket doesn't sleep */ 1611 catchpacket(d, pkt, pktlen, slen, cpfn, &ts); 1612 } 1613 pserialize_read_exit(s); 1614 } 1615 1616 /* 1617 * Incoming linkage from device drivers. Process the packet pkt, of length 1618 * pktlen, which is stored in a contiguous buffer. The packet is parsed 1619 * by each process' filter, and if accepted, stashed into the corresponding 1620 * buffer. 1621 */ 1622 static void 1623 _bpf_tap(struct bpf_if *bp, u_char *pkt, u_int pktlen) 1624 { 1625 1626 bpf_deliver(bp, memcpy, pkt, pktlen, pktlen, true); 1627 } 1628 1629 /* 1630 * Incoming linkage from device drivers, when the head of the packet is in 1631 * a buffer, and the tail is in an mbuf chain. 1632 */ 1633 static void 1634 _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m) 1635 { 1636 u_int pktlen; 1637 struct mbuf mb; 1638 1639 /* Skip outgoing duplicate packets. */ 1640 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1641 m->m_flags &= ~M_PROMISC; 1642 return; 1643 } 1644 1645 pktlen = m_length(m) + dlen; 1646 1647 /* 1648 * Craft on-stack mbuf suitable for passing to bpf_filter. 1649 * Note that we cut corners here; we only setup what's 1650 * absolutely needed--this mbuf should never go anywhere else. 1651 */ 1652 (void)memset(&mb, 0, sizeof(mb)); 1653 mb.m_next = m; 1654 mb.m_data = data; 1655 mb.m_len = dlen; 1656 1657 bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, m->m_pkthdr.rcvif_index != 0); 1658 } 1659 1660 /* 1661 * Incoming linkage from device drivers, when packet is in an mbuf chain. 1662 */ 1663 static void 1664 _bpf_mtap(struct bpf_if *bp, struct mbuf *m) 1665 { 1666 void *(*cpfn)(void *, const void *, size_t); 1667 u_int pktlen, buflen; 1668 void *marg; 1669 1670 /* Skip outgoing duplicate packets. */ 1671 if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { 1672 m->m_flags &= ~M_PROMISC; 1673 return; 1674 } 1675 1676 pktlen = m_length(m); 1677 1678 if (pktlen == m->m_len) { 1679 cpfn = (void *)memcpy; 1680 marg = mtod(m, void *); 1681 buflen = pktlen; 1682 } else { 1683 cpfn = bpf_mcpy; 1684 marg = m; 1685 buflen = 0; 1686 } 1687 1688 bpf_deliver(bp, cpfn, marg, pktlen, buflen, m->m_pkthdr.rcvif_index != 0); 1689 } 1690 1691 /* 1692 * We need to prepend the address family as 1693 * a four byte field. Cons up a dummy header 1694 * to pacify bpf. This is safe because bpf 1695 * will only read from the mbuf (i.e., it won't 1696 * try to free it or keep a pointer a to it). 1697 */ 1698 static void 1699 _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m) 1700 { 1701 struct mbuf m0; 1702 1703 m0.m_flags = 0; 1704 m0.m_next = m; 1705 m0.m_len = 4; 1706 m0.m_data = (char *)⁡ 1707 1708 _bpf_mtap(bp, &m0); 1709 } 1710 1711 /* 1712 * Put the SLIP pseudo-"link header" in place. 1713 * Note this M_PREPEND() should never fail, 1714 * swince we know we always have enough space 1715 * in the input buffer. 1716 */ 1717 static void 1718 _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) 1719 { 1720 u_char *hp; 1721 1722 M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); 1723 if (*m == NULL) 1724 return; 1725 1726 hp = mtod(*m, u_char *); 1727 hp[SLX_DIR] = SLIPDIR_IN; 1728 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1729 1730 _bpf_mtap(bp, *m); 1731 1732 m_adj(*m, SLIP_HDRLEN); 1733 } 1734 1735 /* 1736 * Put the SLIP pseudo-"link header" in 1737 * place. The compressed header is now 1738 * at the beginning of the mbuf. 1739 */ 1740 static void 1741 _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) 1742 { 1743 struct mbuf m0; 1744 u_char *hp; 1745 1746 m0.m_flags = 0; 1747 m0.m_next = m; 1748 m0.m_data = m0.m_dat; 1749 m0.m_len = SLIP_HDRLEN; 1750 1751 hp = mtod(&m0, u_char *); 1752 1753 hp[SLX_DIR] = SLIPDIR_OUT; 1754 (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); 1755 1756 _bpf_mtap(bp, &m0); 1757 m_freem(m); 1758 } 1759 1760 static struct mbuf * 1761 bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) 1762 { 1763 struct mbuf *dup; 1764 1765 dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); 1766 if (dup == NULL) 1767 return NULL; 1768 1769 if (bp->bif_mbuf_tail != NULL) { 1770 bp->bif_mbuf_tail->m_nextpkt = dup; 1771 } else { 1772 bp->bif_mbuf_head = dup; 1773 } 1774 bp->bif_mbuf_tail = dup; 1775 #ifdef BPF_MTAP_SOFTINT_DEBUG 1776 log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", 1777 __func__, dup, bp->bif_ifp->if_xname); 1778 #endif 1779 1780 return dup; 1781 } 1782 1783 static struct mbuf * 1784 bpf_mbuf_dequeue(struct bpf_if *bp) 1785 { 1786 struct mbuf *m; 1787 int s; 1788 1789 /* XXX NOMPSAFE: assumed running on one CPU */ 1790 s = splnet(); 1791 m = bp->bif_mbuf_head; 1792 if (m != NULL) { 1793 bp->bif_mbuf_head = m->m_nextpkt; 1794 m->m_nextpkt = NULL; 1795 1796 if (bp->bif_mbuf_head == NULL) 1797 bp->bif_mbuf_tail = NULL; 1798 #ifdef BPF_MTAP_SOFTINT_DEBUG 1799 log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", 1800 __func__, m, bp->bif_ifp->if_xname); 1801 #endif 1802 } 1803 splx(s); 1804 1805 return m; 1806 } 1807 1808 static void 1809 bpf_mtap_si(void *arg) 1810 { 1811 struct bpf_if *bp = arg; 1812 struct mbuf *m; 1813 1814 while ((m = bpf_mbuf_dequeue(bp)) != NULL) { 1815 #ifdef BPF_MTAP_SOFTINT_DEBUG 1816 log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", 1817 __func__, m, bp->bif_ifp->if_xname); 1818 #endif 1819 bpf_ops->bpf_mtap(bp, m); 1820 m_freem(m); 1821 } 1822 } 1823 1824 static void 1825 _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) 1826 { 1827 struct bpf_if *bp = ifp->if_bpf; 1828 struct mbuf *dup; 1829 1830 KASSERT(cpu_intr_p()); 1831 1832 /* To avoid extra invocations of the softint */ 1833 if (BPFIF_DLIST_READER_EMPTY(bp)) 1834 return; 1835 KASSERT(bp->bif_si != NULL); 1836 1837 dup = bpf_mbuf_enqueue(bp, m); 1838 if (dup != NULL) 1839 softint_schedule(bp->bif_si); 1840 } 1841 1842 static int 1843 bpf_hdrlen(struct bpf_d *d) 1844 { 1845 int hdrlen = d->bd_bif->bif_hdrlen; 1846 /* 1847 * Compute the length of the bpf header. This is not necessarily 1848 * equal to SIZEOF_BPF_HDR because we want to insert spacing such 1849 * that the network layer header begins on a longword boundary (for 1850 * performance reasons and to alleviate alignment restrictions). 1851 */ 1852 #ifdef _LP64 1853 if (d->bd_compat32) 1854 return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); 1855 else 1856 #endif 1857 return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); 1858 } 1859 1860 /* 1861 * Move the packet data from interface memory (pkt) into the 1862 * store buffer. Call the wakeup functions if it's time to wakeup 1863 * a listener (buffer full), "cpfn" is the routine called to do the 1864 * actual data transfer. memcpy is passed in to copy contiguous chunks, 1865 * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, 1866 * pkt is really an mbuf. 1867 */ 1868 static void 1869 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 1870 void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) 1871 { 1872 char *h; 1873 int totlen, curlen, caplen; 1874 int hdrlen = bpf_hdrlen(d); 1875 int do_wakeup = 0; 1876 1877 atomic_inc_ulong(&d->bd_ccount); 1878 BPF_STATINC(capt); 1879 /* 1880 * Figure out how many bytes to move. If the packet is 1881 * greater or equal to the snapshot length, transfer that 1882 * much. Otherwise, transfer the whole packet (unless 1883 * we hit the buffer size limit). 1884 */ 1885 totlen = hdrlen + min(snaplen, pktlen); 1886 if (totlen > d->bd_bufsize) 1887 totlen = d->bd_bufsize; 1888 /* 1889 * If we adjusted totlen to fit the bufsize, it could be that 1890 * totlen is smaller than hdrlen because of the link layer header. 1891 */ 1892 caplen = totlen - hdrlen; 1893 if (caplen < 0) 1894 caplen = 0; 1895 1896 mutex_enter(d->bd_buf_mtx); 1897 /* 1898 * Round up the end of the previous packet to the next longword. 1899 */ 1900 #ifdef _LP64 1901 if (d->bd_compat32) 1902 curlen = BPF_WORDALIGN32(d->bd_slen); 1903 else 1904 #endif 1905 curlen = BPF_WORDALIGN(d->bd_slen); 1906 if (curlen + totlen > d->bd_bufsize) { 1907 /* 1908 * This packet will overflow the storage buffer. 1909 * Rotate the buffers if we can, then wakeup any 1910 * pending reads. 1911 */ 1912 if (d->bd_fbuf == NULL) { 1913 mutex_exit(d->bd_buf_mtx); 1914 /* 1915 * We haven't completed the previous read yet, 1916 * so drop the packet. 1917 */ 1918 atomic_inc_ulong(&d->bd_dcount); 1919 BPF_STATINC(drop); 1920 return; 1921 } 1922 ROTATE_BUFFERS(d); 1923 do_wakeup = 1; 1924 curlen = 0; 1925 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { 1926 /* 1927 * Immediate mode is set, or the read timeout has 1928 * already expired during a select call. A packet 1929 * arrived, so the reader should be woken up. 1930 */ 1931 do_wakeup = 1; 1932 } 1933 1934 /* 1935 * Append the bpf header. 1936 */ 1937 h = (char *)d->bd_sbuf + curlen; 1938 #ifdef _LP64 1939 if (d->bd_compat32) { 1940 struct bpf_hdr32 *hp32; 1941 1942 hp32 = (struct bpf_hdr32 *)h; 1943 hp32->bh_tstamp.tv_sec = ts->tv_sec; 1944 hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1945 hp32->bh_datalen = pktlen; 1946 hp32->bh_hdrlen = hdrlen; 1947 hp32->bh_caplen = caplen; 1948 } else 1949 #endif 1950 { 1951 struct bpf_hdr *hp; 1952 1953 hp = (struct bpf_hdr *)h; 1954 hp->bh_tstamp.tv_sec = ts->tv_sec; 1955 hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; 1956 hp->bh_datalen = pktlen; 1957 hp->bh_hdrlen = hdrlen; 1958 hp->bh_caplen = caplen; 1959 } 1960 1961 /* 1962 * Copy the packet data into the store buffer and update its length. 1963 */ 1964 (*cpfn)(h + hdrlen, pkt, caplen); 1965 d->bd_slen = curlen + totlen; 1966 mutex_exit(d->bd_buf_mtx); 1967 1968 /* 1969 * Call bpf_wakeup after bd_slen has been updated so that kevent(2) 1970 * will cause filt_bpfread() to be called with it adjusted. 1971 */ 1972 if (do_wakeup) 1973 bpf_wakeup(d); 1974 } 1975 1976 /* 1977 * Initialize all nonzero fields of a descriptor. 1978 */ 1979 static int 1980 bpf_allocbufs(struct bpf_d *d) 1981 { 1982 1983 d->bd_fbuf = kmem_alloc(d->bd_bufsize, KM_NOSLEEP); 1984 if (!d->bd_fbuf) 1985 return (ENOBUFS); 1986 d->bd_sbuf = kmem_alloc(d->bd_bufsize, KM_NOSLEEP); 1987 if (!d->bd_sbuf) { 1988 kmem_free(d->bd_fbuf, d->bd_bufsize); 1989 return (ENOBUFS); 1990 } 1991 d->bd_slen = 0; 1992 d->bd_hlen = 0; 1993 return (0); 1994 } 1995 1996 static void 1997 bpf_free_filter(struct bpf_filter *filter) 1998 { 1999 2000 KASSERT(filter != NULL); 2001 KASSERT(filter->bf_insn != NULL); 2002 2003 kmem_free(filter->bf_insn, filter->bf_size); 2004 if (filter->bf_jitcode != NULL) 2005 bpf_jit_freecode(filter->bf_jitcode); 2006 kmem_free(filter, sizeof(*filter)); 2007 } 2008 2009 /* 2010 * Free buffers currently in use by a descriptor. 2011 * Called on close. 2012 */ 2013 static void 2014 bpf_freed(struct bpf_d *d) 2015 { 2016 /* 2017 * We don't need to lock out interrupts since this descriptor has 2018 * been detached from its interface and it yet hasn't been marked 2019 * free. 2020 */ 2021 if (d->bd_sbuf != NULL) { 2022 kmem_free(d->bd_sbuf, d->bd_bufsize); 2023 if (d->bd_hbuf != NULL) 2024 kmem_free(d->bd_hbuf, d->bd_bufsize); 2025 if (d->bd_fbuf != NULL) 2026 kmem_free(d->bd_fbuf, d->bd_bufsize); 2027 } 2028 if (d->bd_filter != NULL) { 2029 bpf_free_filter(d->bd_filter); 2030 d->bd_filter = NULL; 2031 } 2032 d->bd_jitcode = NULL; 2033 } 2034 2035 /* 2036 * Attach an interface to bpf. dlt is the link layer type; 2037 * hdrlen is the fixed size of the link header for the specified dlt 2038 * (variable length headers not yet supported). 2039 */ 2040 static void 2041 _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) 2042 { 2043 struct bpf_if *bp; 2044 bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP); 2045 if (bp == NULL) 2046 panic("bpfattach"); 2047 2048 mutex_enter(&bpf_mtx); 2049 bp->bif_driverp = driverp; 2050 bp->bif_ifp = ifp; 2051 bp->bif_dlt = dlt; 2052 bp->bif_si = NULL; 2053 BPF_IFLIST_ENTRY_INIT(bp); 2054 PSLIST_INIT(&bp->bif_dlist_head); 2055 psref_target_init(&bp->bif_psref, bpf_psref_class); 2056 2057 BPF_IFLIST_WRITER_INSERT_HEAD(bp); 2058 2059 *bp->bif_driverp = NULL; 2060 2061 bp->bif_hdrlen = hdrlen; 2062 mutex_exit(&bpf_mtx); 2063 #if 0 2064 printf("bpf: %s attached\n", ifp->if_xname); 2065 #endif 2066 } 2067 2068 static void 2069 _bpf_mtap_softint_init(struct ifnet *ifp) 2070 { 2071 struct bpf_if *bp; 2072 2073 mutex_enter(&bpf_mtx); 2074 BPF_IFLIST_WRITER_FOREACH(bp) { 2075 if (bp->bif_ifp != ifp) 2076 continue; 2077 2078 bp->bif_mbuf_head = NULL; 2079 bp->bif_mbuf_tail = NULL; 2080 bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); 2081 if (bp->bif_si == NULL) 2082 panic("%s: softint_establish() failed", __func__); 2083 break; 2084 } 2085 mutex_exit(&bpf_mtx); 2086 2087 if (bp == NULL) 2088 panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); 2089 } 2090 2091 /* 2092 * Remove an interface from bpf. 2093 */ 2094 static void 2095 _bpfdetach(struct ifnet *ifp) 2096 { 2097 struct bpf_if *bp; 2098 struct bpf_d *d; 2099 int s; 2100 2101 mutex_enter(&bpf_mtx); 2102 /* Nuke the vnodes for any open instances */ 2103 again_d: 2104 BPF_DLIST_WRITER_FOREACH(d) { 2105 mutex_enter(d->bd_mtx); 2106 if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { 2107 /* 2108 * Detach the descriptor from an interface now. 2109 * It will be free'ed later by close routine. 2110 */ 2111 d->bd_promisc = 0; /* we can't touch device. */ 2112 bpf_detachd(d); 2113 mutex_exit(d->bd_mtx); 2114 goto again_d; 2115 } 2116 mutex_exit(d->bd_mtx); 2117 } 2118 2119 again: 2120 BPF_IFLIST_WRITER_FOREACH(bp) { 2121 if (bp->bif_ifp == ifp) { 2122 BPF_IFLIST_WRITER_REMOVE(bp); 2123 2124 pserialize_perform(bpf_psz); 2125 psref_target_destroy(&bp->bif_psref, bpf_psref_class); 2126 2127 BPF_IFLIST_ENTRY_DESTROY(bp); 2128 if (bp->bif_si != NULL) { 2129 /* XXX NOMPSAFE: assumed running on one CPU */ 2130 s = splnet(); 2131 while (bp->bif_mbuf_head != NULL) { 2132 struct mbuf *m = bp->bif_mbuf_head; 2133 bp->bif_mbuf_head = m->m_nextpkt; 2134 m_freem(m); 2135 } 2136 splx(s); 2137 softint_disestablish(bp->bif_si); 2138 } 2139 kmem_free(bp, sizeof(*bp)); 2140 goto again; 2141 } 2142 } 2143 mutex_exit(&bpf_mtx); 2144 } 2145 2146 /* 2147 * Change the data link type of a interface. 2148 */ 2149 static void 2150 _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) 2151 { 2152 struct bpf_if *bp; 2153 2154 mutex_enter(&bpf_mtx); 2155 BPF_IFLIST_WRITER_FOREACH(bp) { 2156 if (bp->bif_driverp == &ifp->if_bpf) 2157 break; 2158 } 2159 if (bp == NULL) 2160 panic("bpf_change_type"); 2161 2162 bp->bif_dlt = dlt; 2163 2164 bp->bif_hdrlen = hdrlen; 2165 mutex_exit(&bpf_mtx); 2166 } 2167 2168 /* 2169 * Get a list of available data link type of the interface. 2170 */ 2171 static int 2172 bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) 2173 { 2174 int n, error; 2175 struct ifnet *ifp; 2176 struct bpf_if *bp; 2177 int s, bound; 2178 2179 KASSERT(mutex_owned(d->bd_mtx)); 2180 2181 ifp = d->bd_bif->bif_ifp; 2182 n = 0; 2183 error = 0; 2184 2185 bound = curlwp_bind(); 2186 s = pserialize_read_enter(); 2187 BPF_IFLIST_READER_FOREACH(bp) { 2188 if (bp->bif_ifp != ifp) 2189 continue; 2190 if (bfl->bfl_list != NULL) { 2191 struct psref psref; 2192 2193 if (n >= bfl->bfl_len) { 2194 pserialize_read_exit(s); 2195 return ENOMEM; 2196 } 2197 2198 bpf_if_acquire(bp, &psref); 2199 pserialize_read_exit(s); 2200 2201 error = copyout(&bp->bif_dlt, 2202 bfl->bfl_list + n, sizeof(u_int)); 2203 2204 s = pserialize_read_enter(); 2205 bpf_if_release(bp, &psref); 2206 } 2207 n++; 2208 } 2209 pserialize_read_exit(s); 2210 curlwp_bindx(bound); 2211 2212 bfl->bfl_len = n; 2213 return error; 2214 } 2215 2216 /* 2217 * Set the data link type of a BPF instance. 2218 */ 2219 static int 2220 bpf_setdlt(struct bpf_d *d, u_int dlt) 2221 { 2222 int error, opromisc; 2223 struct ifnet *ifp; 2224 struct bpf_if *bp; 2225 2226 KASSERT(mutex_owned(&bpf_mtx)); 2227 KASSERT(mutex_owned(d->bd_mtx)); 2228 2229 if (d->bd_bif->bif_dlt == dlt) 2230 return 0; 2231 ifp = d->bd_bif->bif_ifp; 2232 BPF_IFLIST_WRITER_FOREACH(bp) { 2233 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) 2234 break; 2235 } 2236 if (bp == NULL) 2237 return EINVAL; 2238 opromisc = d->bd_promisc; 2239 bpf_detachd(d); 2240 BPFIF_DLIST_ENTRY_INIT(d); 2241 bpf_attachd(d, bp); 2242 reset_d(d); 2243 if (opromisc) { 2244 KERNEL_LOCK_UNLESS_NET_MPSAFE(); 2245 error = ifpromisc(bp->bif_ifp, 1); 2246 KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); 2247 if (error) 2248 printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", 2249 bp->bif_ifp->if_xname, error); 2250 else 2251 d->bd_promisc = 1; 2252 } 2253 return 0; 2254 } 2255 2256 static int 2257 sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) 2258 { 2259 int newsize, error; 2260 struct sysctlnode node; 2261 2262 node = *rnode; 2263 node.sysctl_data = &newsize; 2264 newsize = bpf_maxbufsize; 2265 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2266 if (error || newp == NULL) 2267 return (error); 2268 2269 if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) 2270 return (EINVAL); 2271 2272 bpf_maxbufsize = newsize; 2273 2274 return (0); 2275 } 2276 2277 #if defined(MODULAR) || defined(BPFJIT) 2278 static int 2279 sysctl_net_bpf_jit(SYSCTLFN_ARGS) 2280 { 2281 bool newval; 2282 int error; 2283 struct sysctlnode node; 2284 2285 node = *rnode; 2286 node.sysctl_data = &newval; 2287 newval = bpf_jit; 2288 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2289 if (error != 0 || newp == NULL) 2290 return error; 2291 2292 bpf_jit = newval; 2293 2294 /* 2295 * Do a full sync to publish new bpf_jit value and 2296 * update bpfjit_module_ops.bj_generate_code variable. 2297 */ 2298 membar_sync(); 2299 2300 if (newval && bpfjit_module_ops.bj_generate_code == NULL) { 2301 printf("JIT compilation is postponed " 2302 "until after bpfjit module is loaded\n"); 2303 } 2304 2305 return 0; 2306 } 2307 #endif 2308 2309 static int 2310 sysctl_net_bpf_peers(SYSCTLFN_ARGS) 2311 { 2312 int error, elem_count; 2313 struct bpf_d *dp; 2314 struct bpf_d_ext dpe; 2315 size_t len, needed, elem_size, out_size; 2316 char *sp; 2317 2318 if (namelen == 1 && name[0] == CTL_QUERY) 2319 return (sysctl_query(SYSCTLFN_CALL(rnode))); 2320 2321 if (namelen != 2) 2322 return (EINVAL); 2323 2324 /* BPF peers is privileged information. */ 2325 error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, 2326 KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); 2327 if (error) 2328 return (EPERM); 2329 2330 len = (oldp != NULL) ? *oldlenp : 0; 2331 sp = oldp; 2332 elem_size = name[0]; 2333 elem_count = name[1]; 2334 out_size = MIN(sizeof(dpe), elem_size); 2335 needed = 0; 2336 2337 if (elem_size < 1 || elem_count < 0) 2338 return (EINVAL); 2339 2340 mutex_enter(&bpf_mtx); 2341 BPF_DLIST_WRITER_FOREACH(dp) { 2342 if (len >= elem_size && elem_count > 0) { 2343 #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field 2344 BPF_EXT(bufsize); 2345 BPF_EXT(promisc); 2346 BPF_EXT(state); 2347 BPF_EXT(immediate); 2348 BPF_EXT(hdrcmplt); 2349 BPF_EXT(seesent); 2350 BPF_EXT(pid); 2351 BPF_EXT(rcount); 2352 BPF_EXT(dcount); 2353 BPF_EXT(ccount); 2354 #undef BPF_EXT 2355 mutex_enter(dp->bd_mtx); 2356 if (dp->bd_bif) 2357 (void)strlcpy(dpe.bde_ifname, 2358 dp->bd_bif->bif_ifp->if_xname, 2359 IFNAMSIZ - 1); 2360 else 2361 dpe.bde_ifname[0] = '\0'; 2362 mutex_exit(dp->bd_mtx); 2363 2364 error = copyout(&dpe, sp, out_size); 2365 if (error) 2366 break; 2367 sp += elem_size; 2368 len -= elem_size; 2369 } 2370 needed += elem_size; 2371 if (elem_count > 0 && elem_count != INT_MAX) 2372 elem_count--; 2373 } 2374 mutex_exit(&bpf_mtx); 2375 2376 *oldlenp = needed; 2377 2378 return (error); 2379 } 2380 2381 static void 2382 bpf_stats(void *p, void *arg, struct cpu_info *ci __unused) 2383 { 2384 struct bpf_stat *const stats = p; 2385 struct bpf_stat *sum = arg; 2386 2387 sum->bs_recv += stats->bs_recv; 2388 sum->bs_drop += stats->bs_drop; 2389 sum->bs_capt += stats->bs_capt; 2390 } 2391 2392 static int 2393 bpf_sysctl_gstats_handler(SYSCTLFN_ARGS) 2394 { 2395 struct sysctlnode node; 2396 int error; 2397 struct bpf_stat sum; 2398 2399 memset(&sum, 0, sizeof(sum)); 2400 node = *rnode; 2401 2402 percpu_foreach(bpf_gstats_percpu, bpf_stats, &sum); 2403 2404 node.sysctl_data = ∑ 2405 node.sysctl_size = sizeof(sum); 2406 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2407 if (error != 0 || newp == NULL) 2408 return error; 2409 2410 return 0; 2411 } 2412 2413 static struct sysctllog *bpf_sysctllog; 2414 static void 2415 sysctl_net_bpf_setup(void) 2416 { 2417 const struct sysctlnode *node; 2418 2419 node = NULL; 2420 sysctl_createv(&bpf_sysctllog, 0, NULL, &node, 2421 CTLFLAG_PERMANENT, 2422 CTLTYPE_NODE, "bpf", 2423 SYSCTL_DESCR("BPF options"), 2424 NULL, 0, NULL, 0, 2425 CTL_NET, CTL_CREATE, CTL_EOL); 2426 if (node != NULL) { 2427 #if defined(MODULAR) || defined(BPFJIT) 2428 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2429 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2430 CTLTYPE_BOOL, "jit", 2431 SYSCTL_DESCR("Toggle Just-In-Time compilation"), 2432 sysctl_net_bpf_jit, 0, &bpf_jit, 0, 2433 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2434 #endif 2435 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2436 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 2437 CTLTYPE_INT, "maxbufsize", 2438 SYSCTL_DESCR("Maximum size for data capture buffer"), 2439 sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, 2440 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2441 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2442 CTLFLAG_PERMANENT, 2443 CTLTYPE_STRUCT, "stats", 2444 SYSCTL_DESCR("BPF stats"), 2445 bpf_sysctl_gstats_handler, 0, NULL, 0, 2446 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2447 sysctl_createv(&bpf_sysctllog, 0, NULL, NULL, 2448 CTLFLAG_PERMANENT, 2449 CTLTYPE_STRUCT, "peers", 2450 SYSCTL_DESCR("BPF peers"), 2451 sysctl_net_bpf_peers, 0, NULL, 0, 2452 CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); 2453 } 2454 2455 } 2456 2457 struct bpf_ops bpf_ops_kernel = { 2458 .bpf_attach = _bpfattach, 2459 .bpf_detach = _bpfdetach, 2460 .bpf_change_type = _bpf_change_type, 2461 2462 .bpf_tap = _bpf_tap, 2463 .bpf_mtap = _bpf_mtap, 2464 .bpf_mtap2 = _bpf_mtap2, 2465 .bpf_mtap_af = _bpf_mtap_af, 2466 .bpf_mtap_sl_in = _bpf_mtap_sl_in, 2467 .bpf_mtap_sl_out = _bpf_mtap_sl_out, 2468 2469 .bpf_mtap_softint = _bpf_mtap_softint, 2470 .bpf_mtap_softint_init = _bpf_mtap_softint_init, 2471 }; 2472 2473 MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); 2474 2475 static int 2476 bpf_modcmd(modcmd_t cmd, void *arg) 2477 { 2478 #ifdef _MODULE 2479 devmajor_t bmajor, cmajor; 2480 #endif 2481 int error = 0; 2482 2483 switch (cmd) { 2484 case MODULE_CMD_INIT: 2485 bpf_init(); 2486 #ifdef _MODULE 2487 bmajor = cmajor = NODEVMAJOR; 2488 error = devsw_attach("bpf", NULL, &bmajor, 2489 &bpf_cdevsw, &cmajor); 2490 if (error) 2491 break; 2492 #endif 2493 2494 bpf_ops_handover_enter(&bpf_ops_kernel); 2495 atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); 2496 bpf_ops_handover_exit(); 2497 sysctl_net_bpf_setup(); 2498 break; 2499 2500 case MODULE_CMD_FINI: 2501 /* 2502 * While there is no reference counting for bpf callers, 2503 * unload could at least in theory be done similarly to 2504 * system call disestablishment. This should even be 2505 * a little simpler: 2506 * 2507 * 1) replace op vector with stubs 2508 * 2) post update to all cpus with xc 2509 * 3) check that nobody is in bpf anymore 2510 * (it's doubtful we'd want something like l_sysent, 2511 * but we could do something like *signed* percpu 2512 * counters. if the sum is 0, we're good). 2513 * 4) if fail, unroll changes 2514 * 2515 * NOTE: change won't be atomic to the outside. some 2516 * packets may be not captured even if unload is 2517 * not succesful. I think packet capture not working 2518 * is a perfectly logical consequence of trying to 2519 * disable packet capture. 2520 */ 2521 error = EOPNOTSUPP; 2522 /* insert sysctl teardown */ 2523 break; 2524 2525 default: 2526 error = ENOTTY; 2527 break; 2528 } 2529 2530 return error; 2531 } 2532