1 /* $NetBSD: sys_select.c,v 1.45 2019/05/08 00:55:18 christos Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran and Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1982, 1986, 1989, 1993 34 * The Regents of the University of California. All rights reserved. 35 * (c) UNIX System Laboratories, Inc. 36 * All or some portions of this file are derived from material licensed 37 * to the University of California by American Telephone and Telegraph 38 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 39 * the permission of UNIX System Laboratories, Inc. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)sys_generic.c 8.9 (Berkeley) 2/14/95 66 */ 67 68 /* 69 * System calls of synchronous I/O multiplexing subsystem. 70 * 71 * Locking 72 * 73 * Two locks are used: <object-lock> and selcluster_t::sc_lock. 74 * 75 * The <object-lock> might be a device driver or another subsystem, e.g. 76 * socket or pipe. This lock is not exported, and thus invisible to this 77 * subsystem. Mainly, synchronisation between selrecord() and selnotify() 78 * routines depends on this lock, as it will be described in the comments. 79 * 80 * Lock order 81 * 82 * <object-lock> -> 83 * selcluster_t::sc_lock 84 */ 85 86 #include <sys/cdefs.h> 87 __KERNEL_RCSID(0, "$NetBSD: sys_select.c,v 1.45 2019/05/08 00:55:18 christos Exp $"); 88 89 #include <sys/param.h> 90 #include <sys/systm.h> 91 #include <sys/filedesc.h> 92 #include <sys/file.h> 93 #include <sys/proc.h> 94 #include <sys/socketvar.h> 95 #include <sys/signalvar.h> 96 #include <sys/uio.h> 97 #include <sys/kernel.h> 98 #include <sys/lwp.h> 99 #include <sys/poll.h> 100 #include <sys/mount.h> 101 #include <sys/syscallargs.h> 102 #include <sys/cpu.h> 103 #include <sys/atomic.h> 104 #include <sys/socketvar.h> 105 #include <sys/sleepq.h> 106 #include <sys/sysctl.h> 107 108 /* Flags for lwp::l_selflag. */ 109 #define SEL_RESET 0 /* awoken, interrupted, or not yet polling */ 110 #define SEL_SCANNING 1 /* polling descriptors */ 111 #define SEL_BLOCKING 2 /* blocking and waiting for event */ 112 #define SEL_EVENT 3 /* interrupted, events set directly */ 113 114 /* Operations: either select() or poll(). */ 115 #define SELOP_SELECT 1 116 #define SELOP_POLL 2 117 118 /* 119 * Per-cluster state for select()/poll(). For a system with fewer 120 * than 32 CPUs, this gives us per-CPU clusters. 121 */ 122 #define SELCLUSTERS 32 123 #define SELCLUSTERMASK (SELCLUSTERS - 1) 124 125 typedef struct selcluster { 126 kmutex_t *sc_lock; 127 sleepq_t sc_sleepq; 128 int sc_ncoll; 129 uint32_t sc_mask; 130 } selcluster_t; 131 132 static inline int selscan(char *, const int, const size_t, register_t *); 133 static inline int pollscan(struct pollfd *, const int, register_t *); 134 static void selclear(void); 135 136 static const int sel_flag[] = { 137 POLLRDNORM | POLLHUP | POLLERR, 138 POLLWRNORM | POLLHUP | POLLERR, 139 POLLRDBAND 140 }; 141 142 static syncobj_t select_sobj = { 143 .sobj_flag = SOBJ_SLEEPQ_FIFO, 144 .sobj_unsleep = sleepq_unsleep, 145 .sobj_changepri = sleepq_changepri, 146 .sobj_lendpri = sleepq_lendpri, 147 .sobj_owner = syncobj_noowner, 148 }; 149 150 static selcluster_t *selcluster[SELCLUSTERS] __read_mostly; 151 static int direct_select __read_mostly = 0; 152 153 /* 154 * Select system call. 155 */ 156 int 157 sys___pselect50(struct lwp *l, const struct sys___pselect50_args *uap, 158 register_t *retval) 159 { 160 /* { 161 syscallarg(int) nd; 162 syscallarg(fd_set *) in; 163 syscallarg(fd_set *) ou; 164 syscallarg(fd_set *) ex; 165 syscallarg(const struct timespec *) ts; 166 syscallarg(sigset_t *) mask; 167 } */ 168 struct timespec ats, *ts = NULL; 169 sigset_t amask, *mask = NULL; 170 int error; 171 172 if (SCARG(uap, ts)) { 173 error = copyin(SCARG(uap, ts), &ats, sizeof(ats)); 174 if (error) 175 return error; 176 ts = &ats; 177 } 178 if (SCARG(uap, mask) != NULL) { 179 error = copyin(SCARG(uap, mask), &amask, sizeof(amask)); 180 if (error) 181 return error; 182 mask = &amask; 183 } 184 185 return selcommon(retval, SCARG(uap, nd), SCARG(uap, in), 186 SCARG(uap, ou), SCARG(uap, ex), ts, mask); 187 } 188 189 int 190 sys___select50(struct lwp *l, const struct sys___select50_args *uap, 191 register_t *retval) 192 { 193 /* { 194 syscallarg(int) nd; 195 syscallarg(fd_set *) in; 196 syscallarg(fd_set *) ou; 197 syscallarg(fd_set *) ex; 198 syscallarg(struct timeval *) tv; 199 } */ 200 struct timeval atv; 201 struct timespec ats, *ts = NULL; 202 int error; 203 204 if (SCARG(uap, tv)) { 205 error = copyin(SCARG(uap, tv), (void *)&atv, sizeof(atv)); 206 if (error) 207 return error; 208 TIMEVAL_TO_TIMESPEC(&atv, &ats); 209 ts = &ats; 210 } 211 212 return selcommon(retval, SCARG(uap, nd), SCARG(uap, in), 213 SCARG(uap, ou), SCARG(uap, ex), ts, NULL); 214 } 215 216 /* 217 * sel_do_scan: common code to perform the scan on descriptors. 218 */ 219 static int 220 sel_do_scan(const int op, void *fds, const int nf, const size_t ni, 221 struct timespec *ts, sigset_t *mask, register_t *retval) 222 { 223 lwp_t * const l = curlwp; 224 selcluster_t *sc; 225 kmutex_t *lock; 226 struct timespec sleepts; 227 int error, timo; 228 229 timo = 0; 230 if (ts && inittimeleft(ts, &sleepts) == -1) { 231 return EINVAL; 232 } 233 234 if (__predict_false(mask)) 235 sigsuspendsetup(l, mask); 236 237 sc = curcpu()->ci_data.cpu_selcluster; 238 lock = sc->sc_lock; 239 l->l_selcluster = sc; 240 if (op == SELOP_SELECT) { 241 l->l_selbits = fds; 242 l->l_selni = ni; 243 } else { 244 l->l_selbits = NULL; 245 } 246 247 for (;;) { 248 int ncoll; 249 250 SLIST_INIT(&l->l_selwait); 251 l->l_selret = 0; 252 253 /* 254 * No need to lock. If this is overwritten by another value 255 * while scanning, we will retry below. We only need to see 256 * exact state from the descriptors that we are about to poll, 257 * and lock activity resulting from fo_poll is enough to 258 * provide an up to date value for new polling activity. 259 */ 260 l->l_selflag = SEL_SCANNING; 261 ncoll = sc->sc_ncoll; 262 263 if (op == SELOP_SELECT) { 264 error = selscan((char *)fds, nf, ni, retval); 265 } else { 266 error = pollscan((struct pollfd *)fds, nf, retval); 267 } 268 if (error || *retval) 269 break; 270 if (ts && (timo = gettimeleft(ts, &sleepts)) <= 0) 271 break; 272 /* 273 * Acquire the lock and perform the (re)checks. Note, if 274 * collision has occured, then our state does not matter, 275 * as we must perform re-scan. Therefore, check it first. 276 */ 277 state_check: 278 mutex_spin_enter(lock); 279 if (__predict_false(sc->sc_ncoll != ncoll)) { 280 /* Collision: perform re-scan. */ 281 mutex_spin_exit(lock); 282 selclear(); 283 continue; 284 } 285 if (__predict_true(l->l_selflag == SEL_EVENT)) { 286 /* Events occured, they are set directly. */ 287 mutex_spin_exit(lock); 288 break; 289 } 290 if (__predict_true(l->l_selflag == SEL_RESET)) { 291 /* Events occured, but re-scan is requested. */ 292 mutex_spin_exit(lock); 293 selclear(); 294 continue; 295 } 296 /* Nothing happen, therefore - sleep. */ 297 l->l_selflag = SEL_BLOCKING; 298 l->l_kpriority = true; 299 sleepq_enter(&sc->sc_sleepq, l, lock); 300 sleepq_enqueue(&sc->sc_sleepq, sc, "select", &select_sobj); 301 error = sleepq_block(timo, true); 302 if (error != 0) { 303 break; 304 } 305 /* Awoken: need to check the state. */ 306 goto state_check; 307 } 308 selclear(); 309 310 /* Add direct events if any. */ 311 if (l->l_selflag == SEL_EVENT) { 312 KASSERT(l->l_selret != 0); 313 *retval += l->l_selret; 314 } 315 316 if (__predict_false(mask)) 317 sigsuspendteardown(l); 318 319 /* select and poll are not restarted after signals... */ 320 if (error == ERESTART) 321 return EINTR; 322 if (error == EWOULDBLOCK) 323 return 0; 324 return error; 325 } 326 327 int 328 selcommon(register_t *retval, int nd, fd_set *u_in, fd_set *u_ou, 329 fd_set *u_ex, struct timespec *ts, sigset_t *mask) 330 { 331 char smallbits[howmany(FD_SETSIZE, NFDBITS) * 332 sizeof(fd_mask) * 6]; 333 char *bits; 334 int error, nf; 335 size_t ni; 336 337 if (nd < 0) 338 return (EINVAL); 339 nf = curlwp->l_fd->fd_dt->dt_nfiles; 340 if (nd > nf) { 341 /* forgiving; slightly wrong */ 342 nd = nf; 343 } 344 ni = howmany(nd, NFDBITS) * sizeof(fd_mask); 345 if (ni * 6 > sizeof(smallbits)) 346 bits = kmem_alloc(ni * 6, KM_SLEEP); 347 else 348 bits = smallbits; 349 350 #define getbits(name, x) \ 351 if (u_ ## name) { \ 352 error = copyin(u_ ## name, bits + ni * x, ni); \ 353 if (error) \ 354 goto fail; \ 355 } else \ 356 memset(bits + ni * x, 0, ni); 357 getbits(in, 0); 358 getbits(ou, 1); 359 getbits(ex, 2); 360 #undef getbits 361 362 error = sel_do_scan(SELOP_SELECT, bits, nd, ni, ts, mask, retval); 363 if (error == 0 && u_in != NULL) 364 error = copyout(bits + ni * 3, u_in, ni); 365 if (error == 0 && u_ou != NULL) 366 error = copyout(bits + ni * 4, u_ou, ni); 367 if (error == 0 && u_ex != NULL) 368 error = copyout(bits + ni * 5, u_ex, ni); 369 fail: 370 if (bits != smallbits) 371 kmem_free(bits, ni * 6); 372 return (error); 373 } 374 375 static inline int 376 selscan(char *bits, const int nfd, const size_t ni, register_t *retval) 377 { 378 fd_mask *ibitp, *obitp; 379 int msk, i, j, fd, n; 380 file_t *fp; 381 382 ibitp = (fd_mask *)(bits + ni * 0); 383 obitp = (fd_mask *)(bits + ni * 3); 384 n = 0; 385 386 memset(obitp, 0, ni * 3); 387 for (msk = 0; msk < 3; msk++) { 388 for (i = 0; i < nfd; i += NFDBITS) { 389 fd_mask ibits, obits; 390 391 ibits = *ibitp; 392 obits = 0; 393 while ((j = ffs(ibits)) && (fd = i + --j) < nfd) { 394 ibits &= ~(1 << j); 395 if ((fp = fd_getfile(fd)) == NULL) 396 return (EBADF); 397 /* 398 * Setup an argument to selrecord(), which is 399 * a file descriptor number. 400 */ 401 curlwp->l_selrec = fd; 402 if ((*fp->f_ops->fo_poll)(fp, sel_flag[msk])) { 403 obits |= (1 << j); 404 n++; 405 } 406 fd_putfile(fd); 407 } 408 if (obits != 0) { 409 if (direct_select) { 410 kmutex_t *lock; 411 lock = curlwp->l_selcluster->sc_lock; 412 mutex_spin_enter(lock); 413 *obitp |= obits; 414 mutex_spin_exit(lock); 415 } else { 416 *obitp |= obits; 417 } 418 } 419 ibitp++; 420 obitp++; 421 } 422 } 423 *retval = n; 424 return (0); 425 } 426 427 /* 428 * Poll system call. 429 */ 430 int 431 sys_poll(struct lwp *l, const struct sys_poll_args *uap, register_t *retval) 432 { 433 /* { 434 syscallarg(struct pollfd *) fds; 435 syscallarg(u_int) nfds; 436 syscallarg(int) timeout; 437 } */ 438 struct timespec ats, *ts = NULL; 439 440 if (SCARG(uap, timeout) != INFTIM) { 441 ats.tv_sec = SCARG(uap, timeout) / 1000; 442 ats.tv_nsec = (SCARG(uap, timeout) % 1000) * 1000000; 443 ts = &ats; 444 } 445 446 return pollcommon(retval, SCARG(uap, fds), SCARG(uap, nfds), ts, NULL); 447 } 448 449 /* 450 * Poll system call. 451 */ 452 int 453 sys___pollts50(struct lwp *l, const struct sys___pollts50_args *uap, 454 register_t *retval) 455 { 456 /* { 457 syscallarg(struct pollfd *) fds; 458 syscallarg(u_int) nfds; 459 syscallarg(const struct timespec *) ts; 460 syscallarg(const sigset_t *) mask; 461 } */ 462 struct timespec ats, *ts = NULL; 463 sigset_t amask, *mask = NULL; 464 int error; 465 466 if (SCARG(uap, ts)) { 467 error = copyin(SCARG(uap, ts), &ats, sizeof(ats)); 468 if (error) 469 return error; 470 ts = &ats; 471 } 472 if (SCARG(uap, mask)) { 473 error = copyin(SCARG(uap, mask), &amask, sizeof(amask)); 474 if (error) 475 return error; 476 mask = &amask; 477 } 478 479 return pollcommon(retval, SCARG(uap, fds), SCARG(uap, nfds), ts, mask); 480 } 481 482 int 483 pollcommon(register_t *retval, struct pollfd *u_fds, u_int nfds, 484 struct timespec *ts, sigset_t *mask) 485 { 486 struct pollfd smallfds[32]; 487 struct pollfd *fds; 488 int error; 489 size_t ni; 490 491 if (nfds > curlwp->l_proc->p_rlimit[RLIMIT_NOFILE].rlim_max + 1000) { 492 /* 493 * Prevent userland from causing over-allocation. 494 * Raising the default limit too high can still cause 495 * a lot of memory to be allocated, but this also means 496 * that the file descriptor array will also be large. 497 * 498 * To reduce the memory requirements here, we could 499 * process the 'fds' array in chunks, but that 500 * is a lot of code that isn't normally useful. 501 * (Or just move the copyin/out into pollscan().) 502 * 503 * Historically the code silently truncated 'fds' to 504 * dt_nfiles entries - but that does cause issues. 505 * 506 * Using the max limit equivalent to sysctl 507 * kern.maxfiles is the moral equivalent of OPEN_MAX 508 * as specified by POSIX. 509 * 510 * We add a slop of 1000 in case the resource limit was 511 * changed after opening descriptors or the same descriptor 512 * was specified more than once. 513 */ 514 return EINVAL; 515 } 516 ni = nfds * sizeof(struct pollfd); 517 if (ni > sizeof(smallfds)) 518 fds = kmem_alloc(ni, KM_SLEEP); 519 else 520 fds = smallfds; 521 522 error = copyin(u_fds, fds, ni); 523 if (error) 524 goto fail; 525 526 error = sel_do_scan(SELOP_POLL, fds, nfds, ni, ts, mask, retval); 527 if (error == 0) 528 error = copyout(fds, u_fds, ni); 529 fail: 530 if (fds != smallfds) 531 kmem_free(fds, ni); 532 return (error); 533 } 534 535 static inline int 536 pollscan(struct pollfd *fds, const int nfd, register_t *retval) 537 { 538 file_t *fp; 539 int i, n = 0, revents; 540 541 for (i = 0; i < nfd; i++, fds++) { 542 fds->revents = 0; 543 if (fds->fd < 0) { 544 revents = 0; 545 } else if ((fp = fd_getfile(fds->fd)) == NULL) { 546 revents = POLLNVAL; 547 } else { 548 /* 549 * Perform poll: registers select request or returns 550 * the events which are set. Setup an argument for 551 * selrecord(), which is a pointer to struct pollfd. 552 */ 553 curlwp->l_selrec = (uintptr_t)fds; 554 revents = (*fp->f_ops->fo_poll)(fp, 555 fds->events | POLLERR | POLLHUP); 556 fd_putfile(fds->fd); 557 } 558 if (revents) { 559 fds->revents = revents; 560 n++; 561 } 562 } 563 *retval = n; 564 return (0); 565 } 566 567 int 568 seltrue(dev_t dev, int events, lwp_t *l) 569 { 570 571 return (events & (POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM)); 572 } 573 574 /* 575 * Record a select request. Concurrency issues: 576 * 577 * The caller holds the same lock across calls to selrecord() and 578 * selnotify(), so we don't need to consider a concurrent wakeup 579 * while in this routine. 580 * 581 * The only activity we need to guard against is selclear(), called by 582 * another thread that is exiting sel_do_scan(). 583 * `sel_lwp' can only become non-NULL while the caller's lock is held, 584 * so it cannot become non-NULL due to a change made by another thread 585 * while we are in this routine. It can only become _NULL_ due to a 586 * call to selclear(). 587 * 588 * If it is non-NULL and != selector there is the potential for 589 * selclear() to be called by another thread. If either of those 590 * conditions are true, we're not interested in touching the `named 591 * waiter' part of the selinfo record because we need to record a 592 * collision. Hence there is no need for additional locking in this 593 * routine. 594 */ 595 void 596 selrecord(lwp_t *selector, struct selinfo *sip) 597 { 598 selcluster_t *sc; 599 lwp_t *other; 600 601 KASSERT(selector == curlwp); 602 603 sc = selector->l_selcluster; 604 other = sip->sel_lwp; 605 606 if (other == selector) { 607 /* 1. We (selector) already claimed to be the first LWP. */ 608 KASSERT(sip->sel_cluster == sc); 609 } else if (other == NULL) { 610 /* 611 * 2. No first LWP, therefore we (selector) are the first. 612 * 613 * There may be unnamed waiters (collisions). Issue a memory 614 * barrier to ensure that we access sel_lwp (above) before 615 * other fields - this guards against a call to selclear(). 616 */ 617 membar_enter(); 618 sip->sel_lwp = selector; 619 SLIST_INSERT_HEAD(&selector->l_selwait, sip, sel_chain); 620 /* Copy the argument, which is for selnotify(). */ 621 sip->sel_fdinfo = selector->l_selrec; 622 /* Replace selinfo's lock with the chosen cluster's lock. */ 623 sip->sel_cluster = sc; 624 } else { 625 /* 3. Multiple waiters: record a collision. */ 626 sip->sel_collision |= sc->sc_mask; 627 KASSERT(sip->sel_cluster != NULL); 628 } 629 } 630 631 /* 632 * sel_setevents: a helper function for selnotify(), to set the events 633 * for LWP sleeping in selcommon() or pollcommon(). 634 */ 635 static inline bool 636 sel_setevents(lwp_t *l, struct selinfo *sip, const int events) 637 { 638 const int oflag = l->l_selflag; 639 int ret = 0; 640 641 /* 642 * If we require re-scan or it was required by somebody else, 643 * then just (re)set SEL_RESET and return. 644 */ 645 if (__predict_false(events == 0 || oflag == SEL_RESET)) { 646 l->l_selflag = SEL_RESET; 647 return true; 648 } 649 /* 650 * Direct set. Note: select state of LWP is locked. First, 651 * determine whether it is selcommon() or pollcommon(). 652 */ 653 if (l->l_selbits != NULL) { 654 const size_t ni = l->l_selni; 655 fd_mask *fds = (fd_mask *)l->l_selbits; 656 fd_mask *ofds = (fd_mask *)((char *)fds + ni * 3); 657 const int fd = sip->sel_fdinfo, fbit = 1 << (fd & __NFDMASK); 658 const int idx = fd >> __NFDSHIFT; 659 int n; 660 661 for (n = 0; n < 3; n++) { 662 if ((fds[idx] & fbit) != 0 && 663 (ofds[idx] & fbit) == 0 && 664 (sel_flag[n] & events)) { 665 ofds[idx] |= fbit; 666 ret++; 667 } 668 fds = (fd_mask *)((char *)fds + ni); 669 ofds = (fd_mask *)((char *)ofds + ni); 670 } 671 } else { 672 struct pollfd *pfd = (void *)sip->sel_fdinfo; 673 int revents = events & (pfd->events | POLLERR | POLLHUP); 674 675 if (revents) { 676 if (pfd->revents == 0) 677 ret = 1; 678 pfd->revents |= revents; 679 } 680 } 681 /* Check whether there are any events to return. */ 682 if (!ret) { 683 return false; 684 } 685 /* Indicate direct set and note the event (cluster lock is held). */ 686 l->l_selflag = SEL_EVENT; 687 l->l_selret += ret; 688 return true; 689 } 690 691 /* 692 * Do a wakeup when a selectable event occurs. Concurrency issues: 693 * 694 * As per selrecord(), the caller's object lock is held. If there 695 * is a named waiter, we must acquire the associated selcluster's lock 696 * in order to synchronize with selclear() and pollers going to sleep 697 * in sel_do_scan(). 698 * 699 * sip->sel_cluser cannot change at this point, as it is only changed 700 * in selrecord(), and concurrent calls to selrecord() are locked 701 * out by the caller. 702 */ 703 void 704 selnotify(struct selinfo *sip, int events, long knhint) 705 { 706 selcluster_t *sc; 707 uint32_t mask; 708 int index, oflag; 709 lwp_t *l; 710 kmutex_t *lock; 711 712 KNOTE(&sip->sel_klist, knhint); 713 714 if (sip->sel_lwp != NULL) { 715 /* One named LWP is waiting. */ 716 sc = sip->sel_cluster; 717 lock = sc->sc_lock; 718 mutex_spin_enter(lock); 719 /* Still there? */ 720 if (sip->sel_lwp != NULL) { 721 /* 722 * Set the events for our LWP and indicate that. 723 * Otherwise, request for a full re-scan. 724 */ 725 l = sip->sel_lwp; 726 oflag = l->l_selflag; 727 728 if (!direct_select) { 729 l->l_selflag = SEL_RESET; 730 } else if (!sel_setevents(l, sip, events)) { 731 /* No events to return. */ 732 mutex_spin_exit(lock); 733 return; 734 } 735 736 /* 737 * If thread is sleeping, wake it up. If it's not 738 * yet asleep, it will notice the change in state 739 * and will re-poll the descriptors. 740 */ 741 if (oflag == SEL_BLOCKING && l->l_mutex == lock) { 742 KASSERT(l->l_wchan == sc); 743 sleepq_unsleep(l, false); 744 } 745 } 746 mutex_spin_exit(lock); 747 } 748 749 if ((mask = sip->sel_collision) != 0) { 750 /* 751 * There was a collision (multiple waiters): we must 752 * inform all potentially interested waiters. 753 */ 754 sip->sel_collision = 0; 755 do { 756 index = ffs(mask) - 1; 757 mask &= ~(1 << index); 758 sc = selcluster[index]; 759 lock = sc->sc_lock; 760 mutex_spin_enter(lock); 761 sc->sc_ncoll++; 762 sleepq_wake(&sc->sc_sleepq, sc, (u_int)-1, lock); 763 } while (__predict_false(mask != 0)); 764 } 765 } 766 767 /* 768 * Remove an LWP from all objects that it is waiting for. Concurrency 769 * issues: 770 * 771 * The object owner's (e.g. device driver) lock is not held here. Calls 772 * can be made to selrecord() and we do not synchronize against those 773 * directly using locks. However, we use `sel_lwp' to lock out changes. 774 * Before clearing it we must use memory barriers to ensure that we can 775 * safely traverse the list of selinfo records. 776 */ 777 static void 778 selclear(void) 779 { 780 struct selinfo *sip, *next; 781 selcluster_t *sc; 782 lwp_t *l; 783 kmutex_t *lock; 784 785 l = curlwp; 786 sc = l->l_selcluster; 787 lock = sc->sc_lock; 788 789 mutex_spin_enter(lock); 790 for (sip = SLIST_FIRST(&l->l_selwait); sip != NULL; sip = next) { 791 KASSERT(sip->sel_lwp == l); 792 KASSERT(sip->sel_cluster == l->l_selcluster); 793 794 /* 795 * Read link to next selinfo record, if any. 796 * It's no longer safe to touch `sip' after clearing 797 * `sel_lwp', so ensure that the read of `sel_chain' 798 * completes before the clearing of sel_lwp becomes 799 * globally visible. 800 */ 801 next = SLIST_NEXT(sip, sel_chain); 802 membar_exit(); 803 /* Release the record for another named waiter to use. */ 804 sip->sel_lwp = NULL; 805 } 806 mutex_spin_exit(lock); 807 } 808 809 /* 810 * Initialize the select/poll system calls. Called once for each 811 * CPU in the system, as they are attached. 812 */ 813 void 814 selsysinit(struct cpu_info *ci) 815 { 816 selcluster_t *sc; 817 u_int index; 818 819 /* If already a cluster in place for this bit, re-use. */ 820 index = cpu_index(ci) & SELCLUSTERMASK; 821 sc = selcluster[index]; 822 if (sc == NULL) { 823 sc = kmem_alloc(roundup2(sizeof(selcluster_t), 824 coherency_unit) + coherency_unit, KM_SLEEP); 825 sc = (void *)roundup2((uintptr_t)sc, coherency_unit); 826 sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED); 827 sleepq_init(&sc->sc_sleepq); 828 sc->sc_ncoll = 0; 829 sc->sc_mask = (1 << index); 830 selcluster[index] = sc; 831 } 832 ci->ci_data.cpu_selcluster = sc; 833 } 834 835 /* 836 * Initialize a selinfo record. 837 */ 838 void 839 selinit(struct selinfo *sip) 840 { 841 842 memset(sip, 0, sizeof(*sip)); 843 } 844 845 /* 846 * Destroy a selinfo record. The owning object must not gain new 847 * references while this is in progress: all activity on the record 848 * must be stopped. 849 * 850 * Concurrency issues: we only need guard against a call to selclear() 851 * by a thread exiting sel_do_scan(). The caller has prevented further 852 * references being made to the selinfo record via selrecord(), and it 853 * will not call selnotify() again. 854 */ 855 void 856 seldestroy(struct selinfo *sip) 857 { 858 selcluster_t *sc; 859 kmutex_t *lock; 860 lwp_t *l; 861 862 if (sip->sel_lwp == NULL) 863 return; 864 865 /* 866 * Lock out selclear(). The selcluster pointer can't change while 867 * we are here since it is only ever changed in selrecord(), 868 * and that will not be entered again for this record because 869 * it is dying. 870 */ 871 KASSERT(sip->sel_cluster != NULL); 872 sc = sip->sel_cluster; 873 lock = sc->sc_lock; 874 mutex_spin_enter(lock); 875 if ((l = sip->sel_lwp) != NULL) { 876 /* 877 * This should rarely happen, so although SLIST_REMOVE() 878 * is slow, using it here is not a problem. 879 */ 880 KASSERT(l->l_selcluster == sc); 881 SLIST_REMOVE(&l->l_selwait, sip, selinfo, sel_chain); 882 sip->sel_lwp = NULL; 883 } 884 mutex_spin_exit(lock); 885 } 886 887 /* 888 * System control nodes. 889 */ 890 SYSCTL_SETUP(sysctl_select_setup, "sysctl select setup") 891 { 892 893 sysctl_createv(clog, 0, NULL, NULL, 894 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 895 CTLTYPE_INT, "direct_select", 896 SYSCTL_DESCR("Enable/disable direct select (for testing)"), 897 NULL, 0, &direct_select, 0, 898 CTL_KERN, CTL_CREATE, CTL_EOL); 899 } 900