1 /*- 2 * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD: src/sys/kern/kern_event.c,v 1.2.2.10 2004/04/04 07:03:14 cperciva Exp $ 27 * $DragonFly: src/sys/kern/kern_event.c,v 1.33 2007/02/03 17:05:57 corecode Exp $ 28 */ 29 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/kernel.h> 33 #include <sys/proc.h> 34 #include <sys/malloc.h> 35 #include <sys/unistd.h> 36 #include <sys/file.h> 37 #include <sys/lock.h> 38 #include <sys/fcntl.h> 39 #include <sys/queue.h> 40 #include <sys/event.h> 41 #include <sys/eventvar.h> 42 #include <sys/protosw.h> 43 #include <sys/socket.h> 44 #include <sys/socketvar.h> 45 #include <sys/stat.h> 46 #include <sys/sysctl.h> 47 #include <sys/sysproto.h> 48 #include <sys/thread.h> 49 #include <sys/uio.h> 50 #include <sys/signalvar.h> 51 #include <sys/filio.h> 52 #include <sys/ktr.h> 53 54 #include <sys/thread2.h> 55 #include <sys/file2.h> 56 #include <sys/mplock2.h> 57 58 /* 59 * Global token for kqueue subsystem 60 */ 61 struct lwkt_token kq_token = LWKT_TOKEN_INITIALIZER(kq_token); 62 SYSCTL_LONG(_lwkt, OID_AUTO, kq_collisions, 63 CTLFLAG_RW, &kq_token.t_collisions, 0, 64 "Collision counter of kq_token"); 65 66 MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system"); 67 68 struct kevent_copyin_args { 69 struct kevent_args *ka; 70 int pchanges; 71 }; 72 73 static int kqueue_sleep(struct kqueue *kq, struct timespec *tsp); 74 static int kqueue_scan(struct kqueue *kq, struct kevent *kevp, int count, 75 struct knote *marker); 76 static int kqueue_read(struct file *fp, struct uio *uio, 77 struct ucred *cred, int flags); 78 static int kqueue_write(struct file *fp, struct uio *uio, 79 struct ucred *cred, int flags); 80 static int kqueue_ioctl(struct file *fp, u_long com, caddr_t data, 81 struct ucred *cred, struct sysmsg *msg); 82 static int kqueue_kqfilter(struct file *fp, struct knote *kn); 83 static int kqueue_stat(struct file *fp, struct stat *st, 84 struct ucred *cred); 85 static int kqueue_close(struct file *fp); 86 static void kqueue_wakeup(struct kqueue *kq); 87 static int filter_attach(struct knote *kn); 88 static int filter_event(struct knote *kn, long hint); 89 90 /* 91 * MPSAFE 92 */ 93 static struct fileops kqueueops = { 94 .fo_read = kqueue_read, 95 .fo_write = kqueue_write, 96 .fo_ioctl = kqueue_ioctl, 97 .fo_kqfilter = kqueue_kqfilter, 98 .fo_stat = kqueue_stat, 99 .fo_close = kqueue_close, 100 .fo_shutdown = nofo_shutdown 101 }; 102 103 static void knote_attach(struct knote *kn); 104 static void knote_drop(struct knote *kn); 105 static void knote_detach_and_drop(struct knote *kn); 106 static void knote_enqueue(struct knote *kn); 107 static void knote_dequeue(struct knote *kn); 108 static struct knote *knote_alloc(void); 109 static void knote_free(struct knote *kn); 110 111 static void filt_kqdetach(struct knote *kn); 112 static int filt_kqueue(struct knote *kn, long hint); 113 static int filt_procattach(struct knote *kn); 114 static void filt_procdetach(struct knote *kn); 115 static int filt_proc(struct knote *kn, long hint); 116 static int filt_fileattach(struct knote *kn); 117 static void filt_timerexpire(void *knx); 118 static int filt_timerattach(struct knote *kn); 119 static void filt_timerdetach(struct knote *kn); 120 static int filt_timer(struct knote *kn, long hint); 121 122 static struct filterops file_filtops = 123 { FILTEROP_ISFD, filt_fileattach, NULL, NULL }; 124 static struct filterops kqread_filtops = 125 { FILTEROP_ISFD, NULL, filt_kqdetach, filt_kqueue }; 126 static struct filterops proc_filtops = 127 { 0, filt_procattach, filt_procdetach, filt_proc }; 128 static struct filterops timer_filtops = 129 { 0, filt_timerattach, filt_timerdetach, filt_timer }; 130 131 static int kq_ncallouts = 0; 132 static int kq_calloutmax = (4 * 1024); 133 SYSCTL_INT(_kern, OID_AUTO, kq_calloutmax, CTLFLAG_RW, 134 &kq_calloutmax, 0, "Maximum number of callouts allocated for kqueue"); 135 static int kq_checkloop = 1000000; 136 SYSCTL_INT(_kern, OID_AUTO, kq_checkloop, CTLFLAG_RW, 137 &kq_checkloop, 0, "Maximum number of callouts allocated for kqueue"); 138 139 #define KNOTE_ACTIVATE(kn) do { \ 140 kn->kn_status |= KN_ACTIVE; \ 141 if ((kn->kn_status & (KN_QUEUED | KN_DISABLED)) == 0) \ 142 knote_enqueue(kn); \ 143 } while(0) 144 145 #define KN_HASHSIZE 64 /* XXX should be tunable */ 146 #define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask)) 147 148 extern struct filterops aio_filtops; 149 extern struct filterops sig_filtops; 150 151 /* 152 * Table for for all system-defined filters. 153 */ 154 static struct filterops *sysfilt_ops[] = { 155 &file_filtops, /* EVFILT_READ */ 156 &file_filtops, /* EVFILT_WRITE */ 157 &aio_filtops, /* EVFILT_AIO */ 158 &file_filtops, /* EVFILT_VNODE */ 159 &proc_filtops, /* EVFILT_PROC */ 160 &sig_filtops, /* EVFILT_SIGNAL */ 161 &timer_filtops, /* EVFILT_TIMER */ 162 &file_filtops, /* EVFILT_EXCEPT */ 163 }; 164 165 static int 166 filt_fileattach(struct knote *kn) 167 { 168 return (fo_kqfilter(kn->kn_fp, kn)); 169 } 170 171 /* 172 * MPSAFE 173 */ 174 static int 175 kqueue_kqfilter(struct file *fp, struct knote *kn) 176 { 177 struct kqueue *kq = (struct kqueue *)kn->kn_fp->f_data; 178 179 if (kn->kn_filter != EVFILT_READ) 180 return (EOPNOTSUPP); 181 182 kn->kn_fop = &kqread_filtops; 183 knote_insert(&kq->kq_kqinfo.ki_note, kn); 184 return (0); 185 } 186 187 static void 188 filt_kqdetach(struct knote *kn) 189 { 190 struct kqueue *kq = (struct kqueue *)kn->kn_fp->f_data; 191 192 knote_remove(&kq->kq_kqinfo.ki_note, kn); 193 } 194 195 /*ARGSUSED*/ 196 static int 197 filt_kqueue(struct knote *kn, long hint) 198 { 199 struct kqueue *kq = (struct kqueue *)kn->kn_fp->f_data; 200 201 kn->kn_data = kq->kq_count; 202 return (kn->kn_data > 0); 203 } 204 205 static int 206 filt_procattach(struct knote *kn) 207 { 208 struct proc *p; 209 int immediate; 210 211 immediate = 0; 212 p = pfind(kn->kn_id); 213 if (p == NULL && (kn->kn_sfflags & NOTE_EXIT)) { 214 p = zpfind(kn->kn_id); 215 immediate = 1; 216 } 217 if (p == NULL) { 218 return (ESRCH); 219 } 220 if (!PRISON_CHECK(curthread->td_ucred, p->p_ucred)) { 221 if (p) 222 PRELE(p); 223 return (EACCES); 224 } 225 226 lwkt_gettoken(&p->p_token); 227 kn->kn_ptr.p_proc = p; 228 kn->kn_flags |= EV_CLEAR; /* automatically set */ 229 230 /* 231 * internal flag indicating registration done by kernel 232 */ 233 if (kn->kn_flags & EV_FLAG1) { 234 kn->kn_data = kn->kn_sdata; /* ppid */ 235 kn->kn_fflags = NOTE_CHILD; 236 kn->kn_flags &= ~EV_FLAG1; 237 } 238 239 knote_insert(&p->p_klist, kn); 240 241 /* 242 * Immediately activate any exit notes if the target process is a 243 * zombie. This is necessary to handle the case where the target 244 * process, e.g. a child, dies before the kevent is negistered. 245 */ 246 if (immediate && filt_proc(kn, NOTE_EXIT)) 247 KNOTE_ACTIVATE(kn); 248 lwkt_reltoken(&p->p_token); 249 PRELE(p); 250 251 return (0); 252 } 253 254 /* 255 * The knote may be attached to a different process, which may exit, 256 * leaving nothing for the knote to be attached to. So when the process 257 * exits, the knote is marked as DETACHED and also flagged as ONESHOT so 258 * it will be deleted when read out. However, as part of the knote deletion, 259 * this routine is called, so a check is needed to avoid actually performing 260 * a detach, because the original process does not exist any more. 261 */ 262 static void 263 filt_procdetach(struct knote *kn) 264 { 265 struct proc *p; 266 267 if (kn->kn_status & KN_DETACHED) 268 return; 269 /* XXX locking? take proc_token here? */ 270 p = kn->kn_ptr.p_proc; 271 knote_remove(&p->p_klist, kn); 272 } 273 274 static int 275 filt_proc(struct knote *kn, long hint) 276 { 277 u_int event; 278 279 /* 280 * mask off extra data 281 */ 282 event = (u_int)hint & NOTE_PCTRLMASK; 283 284 /* 285 * if the user is interested in this event, record it. 286 */ 287 if (kn->kn_sfflags & event) 288 kn->kn_fflags |= event; 289 290 /* 291 * Process is gone, so flag the event as finished. Detach the 292 * knote from the process now because the process will be poof, 293 * gone later on. 294 */ 295 if (event == NOTE_EXIT) { 296 struct proc *p = kn->kn_ptr.p_proc; 297 if ((kn->kn_status & KN_DETACHED) == 0) { 298 knote_remove(&p->p_klist, kn); 299 kn->kn_status |= KN_DETACHED; 300 kn->kn_data = p->p_xstat; 301 kn->kn_ptr.p_proc = NULL; 302 } 303 kn->kn_flags |= (EV_EOF | EV_ONESHOT); 304 return (1); 305 } 306 307 /* 308 * process forked, and user wants to track the new process, 309 * so attach a new knote to it, and immediately report an 310 * event with the parent's pid. 311 */ 312 if ((event == NOTE_FORK) && (kn->kn_sfflags & NOTE_TRACK)) { 313 struct kevent kev; 314 int error; 315 316 /* 317 * register knote with new process. 318 */ 319 kev.ident = hint & NOTE_PDATAMASK; /* pid */ 320 kev.filter = kn->kn_filter; 321 kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_FLAG1; 322 kev.fflags = kn->kn_sfflags; 323 kev.data = kn->kn_id; /* parent */ 324 kev.udata = kn->kn_kevent.udata; /* preserve udata */ 325 error = kqueue_register(kn->kn_kq, &kev); 326 if (error) 327 kn->kn_fflags |= NOTE_TRACKERR; 328 } 329 330 return (kn->kn_fflags != 0); 331 } 332 333 /* 334 * The callout interlocks with callout_terminate() but can still 335 * race a deletion so if KN_DELETING is set we just don't touch 336 * the knote. 337 */ 338 static void 339 filt_timerexpire(void *knx) 340 { 341 struct knote *kn = knx; 342 struct callout *calloutp; 343 struct timeval tv; 344 int tticks; 345 346 lwkt_gettoken(&kq_token); 347 if ((kn->kn_status & KN_DELETING) == 0) { 348 kn->kn_data++; 349 KNOTE_ACTIVATE(kn); 350 351 if ((kn->kn_flags & EV_ONESHOT) == 0) { 352 tv.tv_sec = kn->kn_sdata / 1000; 353 tv.tv_usec = (kn->kn_sdata % 1000) * 1000; 354 tticks = tvtohz_high(&tv); 355 calloutp = (struct callout *)kn->kn_hook; 356 callout_reset(calloutp, tticks, filt_timerexpire, kn); 357 } 358 } 359 lwkt_reltoken(&kq_token); 360 } 361 362 /* 363 * data contains amount of time to sleep, in milliseconds 364 */ 365 static int 366 filt_timerattach(struct knote *kn) 367 { 368 struct callout *calloutp; 369 struct timeval tv; 370 int tticks; 371 372 if (kq_ncallouts >= kq_calloutmax) { 373 kn->kn_hook = NULL; 374 return (ENOMEM); 375 } 376 kq_ncallouts++; 377 378 tv.tv_sec = kn->kn_sdata / 1000; 379 tv.tv_usec = (kn->kn_sdata % 1000) * 1000; 380 tticks = tvtohz_high(&tv); 381 382 kn->kn_flags |= EV_CLEAR; /* automatically set */ 383 MALLOC(calloutp, struct callout *, sizeof(*calloutp), 384 M_KQUEUE, M_WAITOK); 385 callout_init(calloutp); 386 kn->kn_hook = (caddr_t)calloutp; 387 callout_reset(calloutp, tticks, filt_timerexpire, kn); 388 389 return (0); 390 } 391 392 /* 393 * This function is called with the knote flagged locked but it is 394 * still possible to race a callout event due to the callback blocking. 395 * We must call callout_terminate() instead of callout_stop() to deal 396 * with the race. 397 */ 398 static void 399 filt_timerdetach(struct knote *kn) 400 { 401 struct callout *calloutp; 402 403 calloutp = (struct callout *)kn->kn_hook; 404 callout_terminate(calloutp); 405 FREE(calloutp, M_KQUEUE); 406 kq_ncallouts--; 407 } 408 409 static int 410 filt_timer(struct knote *kn, long hint) 411 { 412 413 return (kn->kn_data != 0); 414 } 415 416 /* 417 * Acquire a knote, return non-zero on success, 0 on failure. 418 * 419 * If we cannot acquire the knote we sleep and return 0. The knote 420 * may be stale on return in this case and the caller must restart 421 * whatever loop they are in. 422 */ 423 static __inline 424 int 425 knote_acquire(struct knote *kn) 426 { 427 if (kn->kn_status & KN_PROCESSING) { 428 kn->kn_status |= KN_WAITING | KN_REPROCESS; 429 tsleep(kn, 0, "kqepts", hz); 430 /* knote may be stale now */ 431 return(0); 432 } 433 kn->kn_status |= KN_PROCESSING; 434 return(1); 435 } 436 437 /* 438 * Release an acquired knote, clearing KN_PROCESSING and handling any 439 * KN_REPROCESS events. 440 * 441 * Non-zero is returned if the knote is destroyed. 442 */ 443 static __inline 444 int 445 knote_release(struct knote *kn) 446 { 447 while (kn->kn_status & KN_REPROCESS) { 448 kn->kn_status &= ~KN_REPROCESS; 449 if (kn->kn_status & KN_WAITING) { 450 kn->kn_status &= ~KN_WAITING; 451 wakeup(kn); 452 } 453 if (kn->kn_status & KN_DELETING) { 454 knote_detach_and_drop(kn); 455 return(1); 456 /* NOT REACHED */ 457 } 458 if (filter_event(kn, 0)) 459 KNOTE_ACTIVATE(kn); 460 } 461 kn->kn_status &= ~KN_PROCESSING; 462 return(0); 463 } 464 465 /* 466 * Initialize a kqueue. 467 * 468 * NOTE: The lwp/proc code initializes a kqueue for select/poll ops. 469 * 470 * MPSAFE 471 */ 472 void 473 kqueue_init(struct kqueue *kq, struct filedesc *fdp) 474 { 475 TAILQ_INIT(&kq->kq_knpend); 476 TAILQ_INIT(&kq->kq_knlist); 477 kq->kq_count = 0; 478 kq->kq_fdp = fdp; 479 SLIST_INIT(&kq->kq_kqinfo.ki_note); 480 } 481 482 /* 483 * Terminate a kqueue. Freeing the actual kq itself is left up to the 484 * caller (it might be embedded in a lwp so we don't do it here). 485 * 486 * The kq's knlist must be completely eradicated so block on any 487 * processing races. 488 */ 489 void 490 kqueue_terminate(struct kqueue *kq) 491 { 492 struct knote *kn; 493 494 lwkt_gettoken(&kq_token); 495 while ((kn = TAILQ_FIRST(&kq->kq_knlist)) != NULL) { 496 if (knote_acquire(kn)) 497 knote_detach_and_drop(kn); 498 } 499 if (kq->kq_knhash) { 500 kfree(kq->kq_knhash, M_KQUEUE); 501 kq->kq_knhash = NULL; 502 kq->kq_knhashmask = 0; 503 } 504 lwkt_reltoken(&kq_token); 505 } 506 507 /* 508 * MPSAFE 509 */ 510 int 511 sys_kqueue(struct kqueue_args *uap) 512 { 513 struct thread *td = curthread; 514 struct kqueue *kq; 515 struct file *fp; 516 int fd, error; 517 518 error = falloc(td->td_lwp, &fp, &fd); 519 if (error) 520 return (error); 521 fp->f_flag = FREAD | FWRITE; 522 fp->f_type = DTYPE_KQUEUE; 523 fp->f_ops = &kqueueops; 524 525 kq = kmalloc(sizeof(struct kqueue), M_KQUEUE, M_WAITOK | M_ZERO); 526 kqueue_init(kq, td->td_proc->p_fd); 527 fp->f_data = kq; 528 529 fsetfd(kq->kq_fdp, fp, fd); 530 uap->sysmsg_result = fd; 531 fdrop(fp); 532 return (error); 533 } 534 535 /* 536 * Copy 'count' items into the destination list pointed to by uap->eventlist. 537 */ 538 static int 539 kevent_copyout(void *arg, struct kevent *kevp, int count, int *res) 540 { 541 struct kevent_copyin_args *kap; 542 int error; 543 544 kap = (struct kevent_copyin_args *)arg; 545 546 error = copyout(kevp, kap->ka->eventlist, count * sizeof(*kevp)); 547 if (error == 0) { 548 kap->ka->eventlist += count; 549 *res += count; 550 } else { 551 *res = -1; 552 } 553 554 return (error); 555 } 556 557 /* 558 * Copy at most 'max' items from the list pointed to by kap->changelist, 559 * return number of items in 'events'. 560 */ 561 static int 562 kevent_copyin(void *arg, struct kevent *kevp, int max, int *events) 563 { 564 struct kevent_copyin_args *kap; 565 int error, count; 566 567 kap = (struct kevent_copyin_args *)arg; 568 569 count = min(kap->ka->nchanges - kap->pchanges, max); 570 error = copyin(kap->ka->changelist, kevp, count * sizeof *kevp); 571 if (error == 0) { 572 kap->ka->changelist += count; 573 kap->pchanges += count; 574 *events = count; 575 } 576 577 return (error); 578 } 579 580 /* 581 * MPSAFE 582 */ 583 int 584 kern_kevent(struct kqueue *kq, int nevents, int *res, void *uap, 585 k_copyin_fn kevent_copyinfn, k_copyout_fn kevent_copyoutfn, 586 struct timespec *tsp_in) 587 { 588 struct kevent *kevp; 589 struct timespec *tsp; 590 int i, n, total, error, nerrors = 0; 591 int lres; 592 int limit = kq_checkloop; 593 struct kevent kev[KQ_NEVENTS]; 594 struct knote marker; 595 596 tsp = tsp_in; 597 *res = 0; 598 599 lwkt_gettoken(&kq_token); 600 for ( ;; ) { 601 n = 0; 602 error = kevent_copyinfn(uap, kev, KQ_NEVENTS, &n); 603 if (error) 604 goto done; 605 if (n == 0) 606 break; 607 for (i = 0; i < n; i++) { 608 kevp = &kev[i]; 609 kevp->flags &= ~EV_SYSFLAGS; 610 error = kqueue_register(kq, kevp); 611 612 /* 613 * If a registration returns an error we 614 * immediately post the error. The kevent() 615 * call itself will fail with the error if 616 * no space is available for posting. 617 * 618 * Such errors normally bypass the timeout/blocking 619 * code. However, if the copyoutfn function refuses 620 * to post the error (see sys_poll()), then we 621 * ignore it too. 622 */ 623 if (error) { 624 kevp->flags = EV_ERROR; 625 kevp->data = error; 626 lres = *res; 627 kevent_copyoutfn(uap, kevp, 1, res); 628 if (lres != *res) { 629 nevents--; 630 nerrors++; 631 } 632 } 633 } 634 } 635 if (nerrors) { 636 error = 0; 637 goto done; 638 } 639 640 /* 641 * Acquire/wait for events - setup timeout 642 */ 643 if (tsp != NULL) { 644 struct timespec ats; 645 646 if (tsp->tv_sec || tsp->tv_nsec) { 647 nanouptime(&ats); 648 timespecadd(tsp, &ats); /* tsp = target time */ 649 } 650 } 651 652 /* 653 * Loop as required. 654 * 655 * Collect as many events as we can. Sleeping on successive 656 * loops is disabled if copyoutfn has incremented (*res). 657 * 658 * The loop stops if an error occurs, all events have been 659 * scanned (the marker has been reached), or fewer than the 660 * maximum number of events is found. 661 * 662 * The copyoutfn function does not have to increment (*res) in 663 * order for the loop to continue. 664 * 665 * NOTE: doselect() usually passes 0x7FFFFFFF for nevents. 666 */ 667 total = 0; 668 error = 0; 669 marker.kn_filter = EVFILT_MARKER; 670 marker.kn_status = KN_PROCESSING; 671 TAILQ_INSERT_TAIL(&kq->kq_knpend, &marker, kn_tqe); 672 while ((n = nevents - total) > 0) { 673 if (n > KQ_NEVENTS) 674 n = KQ_NEVENTS; 675 676 /* 677 * If no events are pending sleep until timeout (if any) 678 * or an event occurs. 679 * 680 * After the sleep completes the marker is moved to the 681 * end of the list, making any received events available 682 * to our scan. 683 */ 684 if (kq->kq_count == 0 && *res == 0) { 685 error = kqueue_sleep(kq, tsp); 686 if (error) 687 break; 688 689 TAILQ_REMOVE(&kq->kq_knpend, &marker, kn_tqe); 690 TAILQ_INSERT_TAIL(&kq->kq_knpend, &marker, kn_tqe); 691 } 692 693 /* 694 * Process all received events 695 * Account for all non-spurious events in our total 696 */ 697 i = kqueue_scan(kq, kev, n, &marker); 698 if (i) { 699 lres = *res; 700 error = kevent_copyoutfn(uap, kev, i, res); 701 total += *res - lres; 702 if (error) 703 break; 704 } 705 if (limit && --limit == 0) 706 panic("kqueue: checkloop failed i=%d", i); 707 708 /* 709 * Normally when fewer events are returned than requested 710 * we can stop. However, if only spurious events were 711 * collected the copyout will not bump (*res) and we have 712 * to continue. 713 */ 714 if (i < n && *res) 715 break; 716 717 /* 718 * Deal with an edge case where spurious events can cause 719 * a loop to occur without moving the marker. This can 720 * prevent kqueue_scan() from picking up new events which 721 * race us. We must be sure to move the marker for this 722 * case. 723 * 724 * NOTE: We do not want to move the marker if events 725 * were scanned because normal kqueue operations 726 * may reactivate events. Moving the marker in 727 * that case could result in duplicates for the 728 * same event. 729 */ 730 if (i == 0) { 731 TAILQ_REMOVE(&kq->kq_knpend, &marker, kn_tqe); 732 TAILQ_INSERT_TAIL(&kq->kq_knpend, &marker, kn_tqe); 733 } 734 } 735 TAILQ_REMOVE(&kq->kq_knpend, &marker, kn_tqe); 736 737 /* Timeouts do not return EWOULDBLOCK. */ 738 if (error == EWOULDBLOCK) 739 error = 0; 740 741 done: 742 lwkt_reltoken(&kq_token); 743 return (error); 744 } 745 746 /* 747 * MPALMOSTSAFE 748 */ 749 int 750 sys_kevent(struct kevent_args *uap) 751 { 752 struct thread *td = curthread; 753 struct proc *p = td->td_proc; 754 struct timespec ts, *tsp; 755 struct kqueue *kq; 756 struct file *fp = NULL; 757 struct kevent_copyin_args *kap, ka; 758 int error; 759 760 if (uap->timeout) { 761 error = copyin(uap->timeout, &ts, sizeof(ts)); 762 if (error) 763 return (error); 764 tsp = &ts; 765 } else { 766 tsp = NULL; 767 } 768 769 fp = holdfp(p->p_fd, uap->fd, -1); 770 if (fp == NULL) 771 return (EBADF); 772 if (fp->f_type != DTYPE_KQUEUE) { 773 fdrop(fp); 774 return (EBADF); 775 } 776 777 kq = (struct kqueue *)fp->f_data; 778 779 kap = &ka; 780 kap->ka = uap; 781 kap->pchanges = 0; 782 783 error = kern_kevent(kq, uap->nevents, &uap->sysmsg_result, kap, 784 kevent_copyin, kevent_copyout, tsp); 785 786 fdrop(fp); 787 788 return (error); 789 } 790 791 int 792 kqueue_register(struct kqueue *kq, struct kevent *kev) 793 { 794 struct filedesc *fdp = kq->kq_fdp; 795 struct filterops *fops; 796 struct file *fp = NULL; 797 struct knote *kn = NULL; 798 int error = 0; 799 800 if (kev->filter < 0) { 801 if (kev->filter + EVFILT_SYSCOUNT < 0) 802 return (EINVAL); 803 fops = sysfilt_ops[~kev->filter]; /* to 0-base index */ 804 } else { 805 /* 806 * XXX 807 * filter attach routine is responsible for insuring that 808 * the identifier can be attached to it. 809 */ 810 kprintf("unknown filter: %d\n", kev->filter); 811 return (EINVAL); 812 } 813 814 lwkt_gettoken(&kq_token); 815 if (fops->f_flags & FILTEROP_ISFD) { 816 /* validate descriptor */ 817 fp = holdfp(fdp, kev->ident, -1); 818 if (fp == NULL) { 819 lwkt_reltoken(&kq_token); 820 return (EBADF); 821 } 822 823 again1: 824 SLIST_FOREACH(kn, &fp->f_klist, kn_link) { 825 if (kn->kn_kq == kq && 826 kn->kn_filter == kev->filter && 827 kn->kn_id == kev->ident) { 828 if (knote_acquire(kn) == 0) 829 goto again1; 830 break; 831 } 832 } 833 } else { 834 if (kq->kq_knhashmask) { 835 struct klist *list; 836 837 list = &kq->kq_knhash[ 838 KN_HASH((u_long)kev->ident, kq->kq_knhashmask)]; 839 again2: 840 SLIST_FOREACH(kn, list, kn_link) { 841 if (kn->kn_id == kev->ident && 842 kn->kn_filter == kev->filter) { 843 if (knote_acquire(kn) == 0) 844 goto again2; 845 break; 846 } 847 } 848 } 849 } 850 851 /* 852 * NOTE: At this point if kn is non-NULL we will have acquired 853 * it and set KN_PROCESSING. 854 */ 855 if (kn == NULL && ((kev->flags & EV_ADD) == 0)) { 856 error = ENOENT; 857 goto done; 858 } 859 860 /* 861 * kn now contains the matching knote, or NULL if no match 862 */ 863 if (kev->flags & EV_ADD) { 864 if (kn == NULL) { 865 kn = knote_alloc(); 866 if (kn == NULL) { 867 error = ENOMEM; 868 goto done; 869 } 870 kn->kn_fp = fp; 871 kn->kn_kq = kq; 872 kn->kn_fop = fops; 873 874 /* 875 * apply reference count to knote structure, and 876 * do not release it at the end of this routine. 877 */ 878 fp = NULL; 879 880 kn->kn_sfflags = kev->fflags; 881 kn->kn_sdata = kev->data; 882 kev->fflags = 0; 883 kev->data = 0; 884 kn->kn_kevent = *kev; 885 886 /* 887 * KN_PROCESSING prevents the knote from getting 888 * ripped out from under us while we are trying 889 * to attach it, in case the attach blocks. 890 */ 891 kn->kn_status = KN_PROCESSING; 892 knote_attach(kn); 893 if ((error = filter_attach(kn)) != 0) { 894 kn->kn_status |= KN_DELETING | KN_REPROCESS; 895 knote_drop(kn); 896 goto done; 897 } 898 899 /* 900 * Interlock against close races which either tried 901 * to remove our knote while we were blocked or missed 902 * it entirely prior to our attachment. We do not 903 * want to end up with a knote on a closed descriptor. 904 */ 905 if ((fops->f_flags & FILTEROP_ISFD) && 906 checkfdclosed(fdp, kev->ident, kn->kn_fp)) { 907 kn->kn_status |= KN_DELETING | KN_REPROCESS; 908 } 909 } else { 910 /* 911 * The user may change some filter values after the 912 * initial EV_ADD, but doing so will not reset any 913 * filter which have already been triggered. 914 */ 915 KKASSERT(kn->kn_status & KN_PROCESSING); 916 kn->kn_sfflags = kev->fflags; 917 kn->kn_sdata = kev->data; 918 kn->kn_kevent.udata = kev->udata; 919 } 920 921 /* 922 * Execute the filter event to immediately activate the 923 * knote if necessary. If reprocessing events are pending 924 * due to blocking above we do not run the filter here 925 * but instead let knote_release() do it. Otherwise we 926 * might run the filter on a deleted event. 927 */ 928 if ((kn->kn_status & KN_REPROCESS) == 0) { 929 if (filter_event(kn, 0)) 930 KNOTE_ACTIVATE(kn); 931 } 932 } else if (kev->flags & EV_DELETE) { 933 /* 934 * Delete the existing knote 935 */ 936 knote_detach_and_drop(kn); 937 goto done; 938 } 939 940 /* 941 * Disablement does not deactivate a knote here. 942 */ 943 if ((kev->flags & EV_DISABLE) && 944 ((kn->kn_status & KN_DISABLED) == 0)) { 945 kn->kn_status |= KN_DISABLED; 946 } 947 948 /* 949 * Re-enablement may have to immediately enqueue an active knote. 950 */ 951 if ((kev->flags & EV_ENABLE) && (kn->kn_status & KN_DISABLED)) { 952 kn->kn_status &= ~KN_DISABLED; 953 if ((kn->kn_status & KN_ACTIVE) && 954 ((kn->kn_status & KN_QUEUED) == 0)) { 955 knote_enqueue(kn); 956 } 957 } 958 959 /* 960 * Handle any required reprocessing 961 */ 962 knote_release(kn); 963 /* kn may be invalid now */ 964 965 done: 966 lwkt_reltoken(&kq_token); 967 if (fp != NULL) 968 fdrop(fp); 969 return (error); 970 } 971 972 /* 973 * Block as necessary until the target time is reached. 974 * If tsp is NULL we block indefinitely. If tsp->ts_secs/nsecs are both 975 * 0 we do not block at all. 976 */ 977 static int 978 kqueue_sleep(struct kqueue *kq, struct timespec *tsp) 979 { 980 int error = 0; 981 982 if (tsp == NULL) { 983 kq->kq_state |= KQ_SLEEP; 984 error = tsleep(kq, PCATCH, "kqread", 0); 985 } else if (tsp->tv_sec == 0 && tsp->tv_nsec == 0) { 986 error = EWOULDBLOCK; 987 } else { 988 struct timespec ats; 989 struct timespec atx = *tsp; 990 int timeout; 991 992 nanouptime(&ats); 993 timespecsub(&atx, &ats); 994 if (ats.tv_sec < 0) { 995 error = EWOULDBLOCK; 996 } else { 997 timeout = atx.tv_sec > 24 * 60 * 60 ? 998 24 * 60 * 60 * hz : tstohz_high(&atx); 999 kq->kq_state |= KQ_SLEEP; 1000 error = tsleep(kq, PCATCH, "kqread", timeout); 1001 } 1002 } 1003 1004 /* don't restart after signals... */ 1005 if (error == ERESTART) 1006 return (EINTR); 1007 1008 return (error); 1009 } 1010 1011 /* 1012 * Scan the kqueue, return the number of active events placed in kevp up 1013 * to count. 1014 * 1015 * Continuous mode events may get recycled, do not continue scanning past 1016 * marker unless no events have been collected. 1017 */ 1018 static int 1019 kqueue_scan(struct kqueue *kq, struct kevent *kevp, int count, 1020 struct knote *marker) 1021 { 1022 struct knote *kn, local_marker; 1023 int total; 1024 1025 total = 0; 1026 local_marker.kn_filter = EVFILT_MARKER; 1027 local_marker.kn_status = KN_PROCESSING; 1028 1029 /* 1030 * Collect events. 1031 */ 1032 TAILQ_INSERT_HEAD(&kq->kq_knpend, &local_marker, kn_tqe); 1033 while (count) { 1034 kn = TAILQ_NEXT(&local_marker, kn_tqe); 1035 if (kn->kn_filter == EVFILT_MARKER) { 1036 /* Marker reached, we are done */ 1037 if (kn == marker) 1038 break; 1039 1040 /* Move local marker past some other threads marker */ 1041 kn = TAILQ_NEXT(kn, kn_tqe); 1042 TAILQ_REMOVE(&kq->kq_knpend, &local_marker, kn_tqe); 1043 TAILQ_INSERT_BEFORE(kn, &local_marker, kn_tqe); 1044 continue; 1045 } 1046 1047 /* 1048 * We can't skip a knote undergoing processing, otherwise 1049 * we risk not returning it when the user process expects 1050 * it should be returned. Sleep and retry. 1051 */ 1052 if (knote_acquire(kn) == 0) 1053 continue; 1054 1055 /* 1056 * Remove the event for processing. 1057 * 1058 * WARNING! We must leave KN_QUEUED set to prevent the 1059 * event from being KNOTE_ACTIVATE()d while 1060 * the queue state is in limbo, in case we 1061 * block. 1062 * 1063 * WARNING! We must set KN_PROCESSING to avoid races 1064 * against deletion or another thread's 1065 * processing. 1066 */ 1067 TAILQ_REMOVE(&kq->kq_knpend, kn, kn_tqe); 1068 kq->kq_count--; 1069 1070 /* 1071 * We have to deal with an extremely important race against 1072 * file descriptor close()s here. The file descriptor can 1073 * disappear MPSAFE, and there is a small window of 1074 * opportunity between that and the call to knote_fdclose(). 1075 * 1076 * If we hit that window here while doselect or dopoll is 1077 * trying to delete a spurious event they will not be able 1078 * to match up the event against a knote and will go haywire. 1079 */ 1080 if ((kn->kn_fop->f_flags & FILTEROP_ISFD) && 1081 checkfdclosed(kq->kq_fdp, kn->kn_kevent.ident, kn->kn_fp)) { 1082 kn->kn_status |= KN_DELETING | KN_REPROCESS; 1083 } 1084 1085 if (kn->kn_status & KN_DISABLED) { 1086 /* 1087 * If disabled we ensure the event is not queued 1088 * but leave its active bit set. On re-enablement 1089 * the event may be immediately triggered. 1090 */ 1091 kn->kn_status &= ~KN_QUEUED; 1092 } else if ((kn->kn_flags & EV_ONESHOT) == 0 && 1093 (kn->kn_status & KN_DELETING) == 0 && 1094 filter_event(kn, 0) == 0) { 1095 /* 1096 * If not running in one-shot mode and the event 1097 * is no longer present we ensure it is removed 1098 * from the queue and ignore it. 1099 */ 1100 kn->kn_status &= ~(KN_QUEUED | KN_ACTIVE); 1101 } else { 1102 /* 1103 * Post the event 1104 */ 1105 *kevp++ = kn->kn_kevent; 1106 ++total; 1107 --count; 1108 1109 if (kn->kn_flags & EV_ONESHOT) { 1110 kn->kn_status &= ~KN_QUEUED; 1111 kn->kn_status |= KN_DELETING | KN_REPROCESS; 1112 } else if (kn->kn_flags & EV_CLEAR) { 1113 kn->kn_data = 0; 1114 kn->kn_fflags = 0; 1115 kn->kn_status &= ~(KN_QUEUED | KN_ACTIVE); 1116 } else { 1117 TAILQ_INSERT_TAIL(&kq->kq_knpend, kn, kn_tqe); 1118 kq->kq_count++; 1119 } 1120 } 1121 1122 /* 1123 * Handle any post-processing states 1124 */ 1125 knote_release(kn); 1126 } 1127 TAILQ_REMOVE(&kq->kq_knpend, &local_marker, kn_tqe); 1128 1129 return (total); 1130 } 1131 1132 /* 1133 * XXX 1134 * This could be expanded to call kqueue_scan, if desired. 1135 * 1136 * MPSAFE 1137 */ 1138 static int 1139 kqueue_read(struct file *fp, struct uio *uio, struct ucred *cred, int flags) 1140 { 1141 return (ENXIO); 1142 } 1143 1144 /* 1145 * MPSAFE 1146 */ 1147 static int 1148 kqueue_write(struct file *fp, struct uio *uio, struct ucred *cred, int flags) 1149 { 1150 return (ENXIO); 1151 } 1152 1153 /* 1154 * MPALMOSTSAFE 1155 */ 1156 static int 1157 kqueue_ioctl(struct file *fp, u_long com, caddr_t data, 1158 struct ucred *cred, struct sysmsg *msg) 1159 { 1160 struct kqueue *kq; 1161 int error; 1162 1163 lwkt_gettoken(&kq_token); 1164 kq = (struct kqueue *)fp->f_data; 1165 1166 switch(com) { 1167 case FIOASYNC: 1168 if (*(int *)data) 1169 kq->kq_state |= KQ_ASYNC; 1170 else 1171 kq->kq_state &= ~KQ_ASYNC; 1172 error = 0; 1173 break; 1174 case FIOSETOWN: 1175 error = fsetown(*(int *)data, &kq->kq_sigio); 1176 break; 1177 default: 1178 error = ENOTTY; 1179 break; 1180 } 1181 lwkt_reltoken(&kq_token); 1182 return (error); 1183 } 1184 1185 /* 1186 * MPSAFE 1187 */ 1188 static int 1189 kqueue_stat(struct file *fp, struct stat *st, struct ucred *cred) 1190 { 1191 struct kqueue *kq = (struct kqueue *)fp->f_data; 1192 1193 bzero((void *)st, sizeof(*st)); 1194 st->st_size = kq->kq_count; 1195 st->st_blksize = sizeof(struct kevent); 1196 st->st_mode = S_IFIFO; 1197 return (0); 1198 } 1199 1200 /* 1201 * MPSAFE 1202 */ 1203 static int 1204 kqueue_close(struct file *fp) 1205 { 1206 struct kqueue *kq = (struct kqueue *)fp->f_data; 1207 1208 kqueue_terminate(kq); 1209 1210 fp->f_data = NULL; 1211 funsetown(&kq->kq_sigio); 1212 1213 kfree(kq, M_KQUEUE); 1214 return (0); 1215 } 1216 1217 static void 1218 kqueue_wakeup(struct kqueue *kq) 1219 { 1220 if (kq->kq_state & KQ_SLEEP) { 1221 kq->kq_state &= ~KQ_SLEEP; 1222 wakeup(kq); 1223 } 1224 KNOTE(&kq->kq_kqinfo.ki_note, 0); 1225 } 1226 1227 /* 1228 * Calls filterops f_attach function, acquiring mplock if filter is not 1229 * marked as FILTEROP_MPSAFE. 1230 */ 1231 static int 1232 filter_attach(struct knote *kn) 1233 { 1234 int ret; 1235 1236 if (!(kn->kn_fop->f_flags & FILTEROP_MPSAFE)) { 1237 get_mplock(); 1238 ret = kn->kn_fop->f_attach(kn); 1239 rel_mplock(); 1240 } else { 1241 ret = kn->kn_fop->f_attach(kn); 1242 } 1243 1244 return (ret); 1245 } 1246 1247 /* 1248 * Detach the knote and drop it, destroying the knote. 1249 * 1250 * Calls filterops f_detach function, acquiring mplock if filter is not 1251 * marked as FILTEROP_MPSAFE. 1252 */ 1253 static void 1254 knote_detach_and_drop(struct knote *kn) 1255 { 1256 kn->kn_status |= KN_DELETING | KN_REPROCESS; 1257 if (kn->kn_fop->f_flags & FILTEROP_MPSAFE) { 1258 kn->kn_fop->f_detach(kn); 1259 } else { 1260 get_mplock(); 1261 kn->kn_fop->f_detach(kn); 1262 rel_mplock(); 1263 } 1264 knote_drop(kn); 1265 } 1266 1267 /* 1268 * Calls filterops f_event function, acquiring mplock if filter is not 1269 * marked as FILTEROP_MPSAFE. 1270 * 1271 * If the knote is in the middle of being created or deleted we cannot 1272 * safely call the filter op. 1273 */ 1274 static int 1275 filter_event(struct knote *kn, long hint) 1276 { 1277 int ret; 1278 1279 if (kn->kn_fop->f_flags & FILTEROP_MPSAFE) { 1280 ret = kn->kn_fop->f_event(kn, hint); 1281 } else { 1282 get_mplock(); 1283 ret = kn->kn_fop->f_event(kn, hint); 1284 rel_mplock(); 1285 } 1286 return (ret); 1287 } 1288 1289 /* 1290 * Walk down a list of knotes, activating them if their event has triggered. 1291 * 1292 * If we encounter any knotes which are undergoing processing we just mark 1293 * them for reprocessing and do not try to [re]activate the knote. However, 1294 * if a hint is being passed we have to wait and that makes things a bit 1295 * sticky. 1296 */ 1297 void 1298 knote(struct klist *list, long hint) 1299 { 1300 struct knote *kn; 1301 1302 lwkt_gettoken(&kq_token); 1303 restart: 1304 SLIST_FOREACH(kn, list, kn_next) { 1305 if (kn->kn_status & KN_PROCESSING) { 1306 /* 1307 * Someone else is processing the knote, ask the 1308 * other thread to reprocess it and don't mess 1309 * with it otherwise. 1310 */ 1311 if (hint == 0) { 1312 kn->kn_status |= KN_REPROCESS; 1313 continue; 1314 } 1315 1316 /* 1317 * If the hint is non-zero we have to wait or risk 1318 * losing the state the caller is trying to update. 1319 * 1320 * XXX This is a real problem, certain process 1321 * and signal filters will bump kn_data for 1322 * already-processed notes more than once if 1323 * we restart the list scan. FIXME. 1324 */ 1325 kn->kn_status |= KN_WAITING | KN_REPROCESS; 1326 tsleep(kn, 0, "knotec", hz); 1327 goto restart; 1328 } 1329 1330 /* 1331 * Become the reprocessing master ourselves. 1332 * 1333 * If hint is non-zer running the event is mandatory 1334 * when not deleting so do it whether reprocessing is 1335 * set or not. 1336 */ 1337 kn->kn_status |= KN_PROCESSING; 1338 if ((kn->kn_status & KN_DELETING) == 0) { 1339 if (filter_event(kn, hint)) 1340 KNOTE_ACTIVATE(kn); 1341 } 1342 if (knote_release(kn)) 1343 goto restart; 1344 } 1345 lwkt_reltoken(&kq_token); 1346 } 1347 1348 /* 1349 * Insert knote at head of klist. 1350 * 1351 * This function may only be called via a filter function and thus 1352 * kq_token should already be held and marked for processing. 1353 */ 1354 void 1355 knote_insert(struct klist *klist, struct knote *kn) 1356 { 1357 KKASSERT(kn->kn_status & KN_PROCESSING); 1358 ASSERT_LWKT_TOKEN_HELD(&kq_token); 1359 SLIST_INSERT_HEAD(klist, kn, kn_next); 1360 } 1361 1362 /* 1363 * Remove knote from a klist 1364 * 1365 * This function may only be called via a filter function and thus 1366 * kq_token should already be held and marked for processing. 1367 */ 1368 void 1369 knote_remove(struct klist *klist, struct knote *kn) 1370 { 1371 KKASSERT(kn->kn_status & KN_PROCESSING); 1372 ASSERT_LWKT_TOKEN_HELD(&kq_token); 1373 SLIST_REMOVE(klist, kn, knote, kn_next); 1374 } 1375 1376 /* 1377 * Remove all knotes from a specified klist 1378 * 1379 * Only called from aio. 1380 */ 1381 void 1382 knote_empty(struct klist *list) 1383 { 1384 struct knote *kn; 1385 1386 lwkt_gettoken(&kq_token); 1387 while ((kn = SLIST_FIRST(list)) != NULL) { 1388 if (knote_acquire(kn)) 1389 knote_detach_and_drop(kn); 1390 } 1391 lwkt_reltoken(&kq_token); 1392 } 1393 1394 void 1395 knote_assume_knotes(struct kqinfo *src, struct kqinfo *dst, 1396 struct filterops *ops, void *hook) 1397 { 1398 struct knote *kn; 1399 1400 lwkt_gettoken(&kq_token); 1401 while ((kn = SLIST_FIRST(&src->ki_note)) != NULL) { 1402 if (knote_acquire(kn)) { 1403 knote_remove(&src->ki_note, kn); 1404 kn->kn_fop = ops; 1405 kn->kn_hook = hook; 1406 knote_insert(&dst->ki_note, kn); 1407 knote_release(kn); 1408 /* kn may be invalid now */ 1409 } 1410 } 1411 lwkt_reltoken(&kq_token); 1412 } 1413 1414 /* 1415 * Remove all knotes referencing a specified fd 1416 */ 1417 void 1418 knote_fdclose(struct file *fp, struct filedesc *fdp, int fd) 1419 { 1420 struct knote *kn; 1421 1422 lwkt_gettoken(&kq_token); 1423 restart: 1424 SLIST_FOREACH(kn, &fp->f_klist, kn_link) { 1425 if (kn->kn_kq->kq_fdp == fdp && kn->kn_id == fd) { 1426 if (knote_acquire(kn)) 1427 knote_detach_and_drop(kn); 1428 goto restart; 1429 } 1430 } 1431 lwkt_reltoken(&kq_token); 1432 } 1433 1434 /* 1435 * Low level attach function. 1436 * 1437 * The knote should already be marked for processing. 1438 */ 1439 static void 1440 knote_attach(struct knote *kn) 1441 { 1442 struct klist *list; 1443 struct kqueue *kq = kn->kn_kq; 1444 1445 if (kn->kn_fop->f_flags & FILTEROP_ISFD) { 1446 KKASSERT(kn->kn_fp); 1447 list = &kn->kn_fp->f_klist; 1448 } else { 1449 if (kq->kq_knhashmask == 0) 1450 kq->kq_knhash = hashinit(KN_HASHSIZE, M_KQUEUE, 1451 &kq->kq_knhashmask); 1452 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)]; 1453 } 1454 SLIST_INSERT_HEAD(list, kn, kn_link); 1455 TAILQ_INSERT_HEAD(&kq->kq_knlist, kn, kn_kqlink); 1456 } 1457 1458 /* 1459 * Low level drop function. 1460 * 1461 * The knote should already be marked for processing. 1462 */ 1463 static void 1464 knote_drop(struct knote *kn) 1465 { 1466 struct kqueue *kq; 1467 struct klist *list; 1468 1469 kq = kn->kn_kq; 1470 1471 if (kn->kn_fop->f_flags & FILTEROP_ISFD) 1472 list = &kn->kn_fp->f_klist; 1473 else 1474 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)]; 1475 1476 SLIST_REMOVE(list, kn, knote, kn_link); 1477 TAILQ_REMOVE(&kq->kq_knlist, kn, kn_kqlink); 1478 if (kn->kn_status & KN_QUEUED) 1479 knote_dequeue(kn); 1480 if (kn->kn_fop->f_flags & FILTEROP_ISFD) { 1481 fdrop(kn->kn_fp); 1482 kn->kn_fp = NULL; 1483 } 1484 knote_free(kn); 1485 } 1486 1487 /* 1488 * Low level enqueue function. 1489 * 1490 * The knote should already be marked for processing. 1491 */ 1492 static void 1493 knote_enqueue(struct knote *kn) 1494 { 1495 struct kqueue *kq = kn->kn_kq; 1496 1497 KASSERT((kn->kn_status & KN_QUEUED) == 0, ("knote already queued")); 1498 TAILQ_INSERT_TAIL(&kq->kq_knpend, kn, kn_tqe); 1499 kn->kn_status |= KN_QUEUED; 1500 ++kq->kq_count; 1501 1502 /* 1503 * Send SIGIO on request (typically set up as a mailbox signal) 1504 */ 1505 if (kq->kq_sigio && (kq->kq_state & KQ_ASYNC) && kq->kq_count == 1) 1506 pgsigio(kq->kq_sigio, SIGIO, 0); 1507 1508 kqueue_wakeup(kq); 1509 } 1510 1511 /* 1512 * Low level dequeue function. 1513 * 1514 * The knote should already be marked for processing. 1515 */ 1516 static void 1517 knote_dequeue(struct knote *kn) 1518 { 1519 struct kqueue *kq = kn->kn_kq; 1520 1521 KASSERT(kn->kn_status & KN_QUEUED, ("knote not queued")); 1522 TAILQ_REMOVE(&kq->kq_knpend, kn, kn_tqe); 1523 kn->kn_status &= ~KN_QUEUED; 1524 kq->kq_count--; 1525 } 1526 1527 static struct knote * 1528 knote_alloc(void) 1529 { 1530 return kmalloc(sizeof(struct knote), M_KQUEUE, M_WAITOK); 1531 } 1532 1533 static void 1534 knote_free(struct knote *kn) 1535 { 1536 kfree(kn, M_KQUEUE); 1537 } 1538