1 /* $NetBSD: sys_select.c,v 1.27 2010/07/12 11:04:25 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran and Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1982, 1986, 1989, 1993 34 * The Regents of the University of California. All rights reserved. 35 * (c) UNIX System Laboratories, Inc. 36 * All or some portions of this file are derived from material licensed 37 * to the University of California by American Telephone and Telegraph 38 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 39 * the permission of UNIX System Laboratories, Inc. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)sys_generic.c 8.9 (Berkeley) 2/14/95 66 */ 67 68 /* 69 * System calls of synchronous I/O multiplexing subsystem. 70 * 71 * Locking 72 * 73 * Two locks are used: <object-lock> and selcluster_t::sc_lock. 74 * 75 * The <object-lock> might be a device driver or another subsystem, e.g. 76 * socket or pipe. This lock is not exported, and thus invisible to this 77 * subsystem. Mainly, synchronisation between selrecord() and selnotify() 78 * routines depends on this lock, as it will be described in the comments. 79 * 80 * Lock order 81 * 82 * <object-lock> -> 83 * selcluster_t::sc_lock 84 */ 85 86 #include <sys/cdefs.h> 87 __KERNEL_RCSID(0, "$NetBSD: sys_select.c,v 1.27 2010/07/12 11:04:25 rmind Exp $"); 88 89 #include <sys/param.h> 90 #include <sys/systm.h> 91 #include <sys/filedesc.h> 92 #include <sys/ioctl.h> 93 #include <sys/file.h> 94 #include <sys/proc.h> 95 #include <sys/socketvar.h> 96 #include <sys/signalvar.h> 97 #include <sys/uio.h> 98 #include <sys/kernel.h> 99 #include <sys/stat.h> 100 #include <sys/poll.h> 101 #include <sys/vnode.h> 102 #include <sys/mount.h> 103 #include <sys/syscallargs.h> 104 #include <sys/cpu.h> 105 #include <sys/atomic.h> 106 #include <sys/socketvar.h> 107 #include <sys/sleepq.h> 108 109 /* Flags for lwp::l_selflag. */ 110 #define SEL_RESET 0 /* awoken, interrupted, or not yet polling */ 111 #define SEL_SCANNING 1 /* polling descriptors */ 112 #define SEL_BLOCKING 2 /* blocking and waiting for event */ 113 #define SEL_EVENT 3 /* interrupted, events set directly */ 114 115 /* Operations: either select() or poll(). */ 116 #define SELOP_SELECT 1 117 #define SELOP_POLL 2 118 119 /* 120 * Per-cluster state for select()/poll(). For a system with fewer 121 * than 32 CPUs, this gives us per-CPU clusters. 122 */ 123 #define SELCLUSTERS 32 124 #define SELCLUSTERMASK (SELCLUSTERS - 1) 125 126 typedef struct selcluster { 127 kmutex_t *sc_lock; 128 sleepq_t sc_sleepq; 129 int sc_ncoll; 130 uint32_t sc_mask; 131 } selcluster_t; 132 133 static inline int selscan(char *, const int, const size_t, register_t *); 134 static inline int pollscan(struct pollfd *, const int, register_t *); 135 static void selclear(void); 136 137 static const int sel_flag[] = { 138 POLLRDNORM | POLLHUP | POLLERR, 139 POLLWRNORM | POLLHUP | POLLERR, 140 POLLRDBAND 141 }; 142 143 static syncobj_t select_sobj = { 144 SOBJ_SLEEPQ_FIFO, 145 sleepq_unsleep, 146 sleepq_changepri, 147 sleepq_lendpri, 148 syncobj_noowner, 149 }; 150 151 static selcluster_t *selcluster[SELCLUSTERS] __read_mostly; 152 153 /* 154 * Select system call. 155 */ 156 int 157 sys___pselect50(struct lwp *l, const struct sys___pselect50_args *uap, 158 register_t *retval) 159 { 160 /* { 161 syscallarg(int) nd; 162 syscallarg(fd_set *) in; 163 syscallarg(fd_set *) ou; 164 syscallarg(fd_set *) ex; 165 syscallarg(const struct timespec *) ts; 166 syscallarg(sigset_t *) mask; 167 } */ 168 struct timespec ats, *ts = NULL; 169 sigset_t amask, *mask = NULL; 170 int error; 171 172 if (SCARG(uap, ts)) { 173 error = copyin(SCARG(uap, ts), &ats, sizeof(ats)); 174 if (error) 175 return error; 176 ts = &ats; 177 } 178 if (SCARG(uap, mask) != NULL) { 179 error = copyin(SCARG(uap, mask), &amask, sizeof(amask)); 180 if (error) 181 return error; 182 mask = &amask; 183 } 184 185 return selcommon(retval, SCARG(uap, nd), SCARG(uap, in), 186 SCARG(uap, ou), SCARG(uap, ex), ts, mask); 187 } 188 189 int 190 sys___select50(struct lwp *l, const struct sys___select50_args *uap, 191 register_t *retval) 192 { 193 /* { 194 syscallarg(int) nd; 195 syscallarg(fd_set *) in; 196 syscallarg(fd_set *) ou; 197 syscallarg(fd_set *) ex; 198 syscallarg(struct timeval *) tv; 199 } */ 200 struct timeval atv; 201 struct timespec ats, *ts = NULL; 202 int error; 203 204 if (SCARG(uap, tv)) { 205 error = copyin(SCARG(uap, tv), (void *)&atv, sizeof(atv)); 206 if (error) 207 return error; 208 TIMEVAL_TO_TIMESPEC(&atv, &ats); 209 ts = &ats; 210 } 211 212 return selcommon(retval, SCARG(uap, nd), SCARG(uap, in), 213 SCARG(uap, ou), SCARG(uap, ex), ts, NULL); 214 } 215 216 /* 217 * sel_do_scan: common code to perform the scan on descriptors. 218 */ 219 static int 220 sel_do_scan(const int op, void *fds, const int nf, const size_t ni, 221 struct timespec *ts, sigset_t *mask, register_t *retval) 222 { 223 lwp_t * const l = curlwp; 224 proc_t * const p = l->l_proc; 225 selcluster_t *sc; 226 kmutex_t *lock; 227 sigset_t oldmask; 228 struct timespec sleepts; 229 int error, timo; 230 231 timo = 0; 232 if (ts && inittimeleft(ts, &sleepts) == -1) { 233 return EINVAL; 234 } 235 236 if (__predict_false(mask)) { 237 sigminusset(&sigcantmask, mask); 238 mutex_enter(p->p_lock); 239 oldmask = l->l_sigmask; 240 l->l_sigmask = *mask; 241 mutex_exit(p->p_lock); 242 } else { 243 /* XXXgcc */ 244 oldmask = l->l_sigmask; 245 } 246 247 sc = curcpu()->ci_data.cpu_selcluster; 248 lock = sc->sc_lock; 249 l->l_selcluster = sc; 250 SLIST_INIT(&l->l_selwait); 251 252 l->l_selret = 0; 253 if (op == SELOP_SELECT) { 254 l->l_selbits = (char *)fds + ni * 3; 255 l->l_selni = ni; 256 } else { 257 l->l_selbits = NULL; 258 } 259 for (;;) { 260 int ncoll; 261 262 /* 263 * No need to lock. If this is overwritten by another value 264 * while scanning, we will retry below. We only need to see 265 * exact state from the descriptors that we are about to poll, 266 * and lock activity resulting from fo_poll is enough to 267 * provide an up to date value for new polling activity. 268 */ 269 l->l_selflag = SEL_SCANNING; 270 ncoll = sc->sc_ncoll; 271 272 if (op == SELOP_SELECT) { 273 error = selscan((char *)fds, nf, ni, retval); 274 } else { 275 error = pollscan((struct pollfd *)fds, nf, retval); 276 } 277 if (error || *retval) 278 break; 279 if (ts && (timo = gettimeleft(ts, &sleepts)) <= 0) 280 break; 281 /* 282 * Acquire the lock and perform the (re)checks. Note, if 283 * collision has occured, then our state does not matter, 284 * as we must perform re-scan. Therefore, check it first. 285 */ 286 state_check: 287 mutex_spin_enter(lock); 288 if (__predict_false(sc->sc_ncoll != ncoll)) { 289 /* Collision: perform re-scan. */ 290 mutex_spin_exit(lock); 291 continue; 292 } 293 if (__predict_true(l->l_selflag == SEL_EVENT)) { 294 /* Events occured, they are set directly. */ 295 mutex_spin_exit(lock); 296 KASSERT(l->l_selret != 0); 297 *retval = l->l_selret; 298 break; 299 } 300 if (__predict_true(l->l_selflag == SEL_RESET)) { 301 /* Events occured, but re-scan is requested. */ 302 mutex_spin_exit(lock); 303 continue; 304 } 305 /* Nothing happen, therefore - sleep. */ 306 l->l_selflag = SEL_BLOCKING; 307 l->l_kpriority = true; 308 sleepq_enter(&sc->sc_sleepq, l, lock); 309 sleepq_enqueue(&sc->sc_sleepq, sc, "select", &select_sobj); 310 error = sleepq_block(timo, true); 311 if (error != 0) { 312 break; 313 } 314 /* Awoken: need to check the state. */ 315 goto state_check; 316 } 317 selclear(); 318 319 if (__predict_false(mask)) { 320 mutex_enter(p->p_lock); 321 l->l_sigmask = oldmask; 322 mutex_exit(p->p_lock); 323 } 324 325 /* select and poll are not restarted after signals... */ 326 if (error == ERESTART) 327 return EINTR; 328 if (error == EWOULDBLOCK) 329 return 0; 330 return error; 331 } 332 333 int 334 selcommon(register_t *retval, int nd, fd_set *u_in, fd_set *u_ou, 335 fd_set *u_ex, struct timespec *ts, sigset_t *mask) 336 { 337 char smallbits[howmany(FD_SETSIZE, NFDBITS) * 338 sizeof(fd_mask) * 6]; 339 char *bits; 340 int error, nf; 341 size_t ni; 342 343 if (nd < 0) 344 return (EINVAL); 345 nf = curlwp->l_fd->fd_dt->dt_nfiles; 346 if (nd > nf) { 347 /* forgiving; slightly wrong */ 348 nd = nf; 349 } 350 ni = howmany(nd, NFDBITS) * sizeof(fd_mask); 351 if (ni * 6 > sizeof(smallbits)) { 352 bits = kmem_alloc(ni * 6, KM_SLEEP); 353 if (bits == NULL) 354 return ENOMEM; 355 } else 356 bits = smallbits; 357 358 #define getbits(name, x) \ 359 if (u_ ## name) { \ 360 error = copyin(u_ ## name, bits + ni * x, ni); \ 361 if (error) \ 362 goto fail; \ 363 } else \ 364 memset(bits + ni * x, 0, ni); 365 getbits(in, 0); 366 getbits(ou, 1); 367 getbits(ex, 2); 368 #undef getbits 369 370 error = sel_do_scan(SELOP_SELECT, bits, nd, ni, ts, mask, retval); 371 if (error == 0 && u_in != NULL) 372 error = copyout(bits + ni * 3, u_in, ni); 373 if (error == 0 && u_ou != NULL) 374 error = copyout(bits + ni * 4, u_ou, ni); 375 if (error == 0 && u_ex != NULL) 376 error = copyout(bits + ni * 5, u_ex, ni); 377 fail: 378 if (bits != smallbits) 379 kmem_free(bits, ni * 6); 380 return (error); 381 } 382 383 static inline int 384 selscan(char *bits, const int nfd, const size_t ni, register_t *retval) 385 { 386 fd_mask *ibitp, *obitp; 387 int msk, i, j, fd, n; 388 file_t *fp; 389 390 ibitp = (fd_mask *)(bits + ni * 0); 391 obitp = (fd_mask *)(bits + ni * 3); 392 n = 0; 393 394 for (msk = 0; msk < 3; msk++) { 395 for (i = 0; i < nfd; i += NFDBITS) { 396 fd_mask ibits, obits; 397 398 ibits = *ibitp++; 399 obits = 0; 400 while ((j = ffs(ibits)) && (fd = i + --j) < nfd) { 401 ibits &= ~(1 << j); 402 if ((fp = fd_getfile(fd)) == NULL) 403 return (EBADF); 404 /* 405 * Setup an argument to selrecord(), which is 406 * a file descriptor number. 407 */ 408 curlwp->l_selrec = fd; 409 if ((*fp->f_ops->fo_poll)(fp, sel_flag[msk])) { 410 obits |= (1 << j); 411 n++; 412 } 413 fd_putfile(fd); 414 } 415 *obitp++ = obits; 416 } 417 } 418 *retval = n; 419 return (0); 420 } 421 422 /* 423 * Poll system call. 424 */ 425 int 426 sys_poll(struct lwp *l, const struct sys_poll_args *uap, register_t *retval) 427 { 428 /* { 429 syscallarg(struct pollfd *) fds; 430 syscallarg(u_int) nfds; 431 syscallarg(int) timeout; 432 } */ 433 struct timespec ats, *ts = NULL; 434 435 if (SCARG(uap, timeout) != INFTIM) { 436 ats.tv_sec = SCARG(uap, timeout) / 1000; 437 ats.tv_nsec = (SCARG(uap, timeout) % 1000) * 1000000; 438 ts = &ats; 439 } 440 441 return pollcommon(retval, SCARG(uap, fds), SCARG(uap, nfds), ts, NULL); 442 } 443 444 /* 445 * Poll system call. 446 */ 447 int 448 sys___pollts50(struct lwp *l, const struct sys___pollts50_args *uap, 449 register_t *retval) 450 { 451 /* { 452 syscallarg(struct pollfd *) fds; 453 syscallarg(u_int) nfds; 454 syscallarg(const struct timespec *) ts; 455 syscallarg(const sigset_t *) mask; 456 } */ 457 struct timespec ats, *ts = NULL; 458 sigset_t amask, *mask = NULL; 459 int error; 460 461 if (SCARG(uap, ts)) { 462 error = copyin(SCARG(uap, ts), &ats, sizeof(ats)); 463 if (error) 464 return error; 465 ts = &ats; 466 } 467 if (SCARG(uap, mask)) { 468 error = copyin(SCARG(uap, mask), &amask, sizeof(amask)); 469 if (error) 470 return error; 471 mask = &amask; 472 } 473 474 return pollcommon(retval, SCARG(uap, fds), SCARG(uap, nfds), ts, mask); 475 } 476 477 int 478 pollcommon(register_t *retval, struct pollfd *u_fds, u_int nfds, 479 struct timespec *ts, sigset_t *mask) 480 { 481 struct pollfd smallfds[32]; 482 struct pollfd *fds; 483 int error; 484 size_t ni; 485 486 if (nfds > 1000 + curlwp->l_fd->fd_dt->dt_nfiles) { 487 /* 488 * Either the user passed in a very sparse 'fds' or junk! 489 * The kmem_alloc() call below would be bad news. 490 * We could process the 'fds' array in chunks, but that 491 * is a lot of code that isn't normally useful. 492 * (Or just move the copyin/out into pollscan().) 493 * Historically the code silently truncated 'fds' to 494 * dt_nfiles entries - but that does cause issues. 495 */ 496 return EINVAL; 497 } 498 ni = nfds * sizeof(struct pollfd); 499 if (ni > sizeof(smallfds)) { 500 fds = kmem_alloc(ni, KM_SLEEP); 501 if (fds == NULL) 502 return ENOMEM; 503 } else 504 fds = smallfds; 505 506 error = copyin(u_fds, fds, ni); 507 if (error) 508 goto fail; 509 510 error = sel_do_scan(SELOP_POLL, fds, nfds, ni, ts, mask, retval); 511 if (error == 0) 512 error = copyout(fds, u_fds, ni); 513 fail: 514 if (fds != smallfds) 515 kmem_free(fds, ni); 516 return (error); 517 } 518 519 static inline int 520 pollscan(struct pollfd *fds, const int nfd, register_t *retval) 521 { 522 file_t *fp; 523 int i, n = 0; 524 525 for (i = 0; i < nfd; i++, fds++) { 526 if (fds->fd < 0) { 527 fds->revents = 0; 528 } else if ((fp = fd_getfile(fds->fd)) == NULL) { 529 fds->revents = POLLNVAL; 530 n++; 531 } else { 532 /* 533 * Perform poll: registers select request or returns 534 * the events which are set. Setup an argument for 535 * selrecord(), which is a pointer to struct pollfd. 536 */ 537 curlwp->l_selrec = (uintptr_t)fds; 538 fds->revents = (*fp->f_ops->fo_poll)(fp, 539 fds->events | POLLERR | POLLHUP); 540 if (fds->revents != 0) 541 n++; 542 fd_putfile(fds->fd); 543 } 544 } 545 *retval = n; 546 return (0); 547 } 548 549 int 550 seltrue(dev_t dev, int events, lwp_t *l) 551 { 552 553 return (events & (POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM)); 554 } 555 556 /* 557 * Record a select request. Concurrency issues: 558 * 559 * The caller holds the same lock across calls to selrecord() and 560 * selnotify(), so we don't need to consider a concurrent wakeup 561 * while in this routine. 562 * 563 * The only activity we need to guard against is selclear(), called by 564 * another thread that is exiting sel_do_scan(). 565 * `sel_lwp' can only become non-NULL while the caller's lock is held, 566 * so it cannot become non-NULL due to a change made by another thread 567 * while we are in this routine. It can only become _NULL_ due to a 568 * call to selclear(). 569 * 570 * If it is non-NULL and != selector there is the potential for 571 * selclear() to be called by another thread. If either of those 572 * conditions are true, we're not interested in touching the `named 573 * waiter' part of the selinfo record because we need to record a 574 * collision. Hence there is no need for additional locking in this 575 * routine. 576 */ 577 void 578 selrecord(lwp_t *selector, struct selinfo *sip) 579 { 580 selcluster_t *sc; 581 lwp_t *other; 582 583 KASSERT(selector == curlwp); 584 585 sc = selector->l_selcluster; 586 other = sip->sel_lwp; 587 588 if (other == selector) { 589 /* 1. We (selector) already claimed to be the first LWP. */ 590 KASSERT(sip->sel_cluster = sc); 591 } else if (other == NULL) { 592 /* 593 * 2. No first LWP, therefore we (selector) are the first. 594 * 595 * There may be unnamed waiters (collisions). Issue a memory 596 * barrier to ensure that we access sel_lwp (above) before 597 * other fields - this guards against a call to selclear(). 598 */ 599 membar_enter(); 600 sip->sel_lwp = selector; 601 SLIST_INSERT_HEAD(&selector->l_selwait, sip, sel_chain); 602 /* Copy the argument, which is for selnotify(). */ 603 sip->sel_fdinfo = selector->l_selrec; 604 /* Replace selinfo's lock with the chosen cluster's lock. */ 605 sip->sel_cluster = sc; 606 } else { 607 /* 3. Multiple waiters: record a collision. */ 608 sip->sel_collision |= sc->sc_mask; 609 KASSERT(sip->sel_cluster != NULL); 610 } 611 } 612 613 /* 614 * sel_setevents: a helper function for selnotify(), to set the events 615 * for LWP sleeping in selcommon() or pollcommon(). 616 */ 617 static inline void 618 sel_setevents(lwp_t *l, struct selinfo *sip, const int events) 619 { 620 const int oflag = l->l_selflag; 621 622 /* 623 * If we require re-scan or it was required by somebody else, 624 * then just (re)set SEL_RESET and return. 625 */ 626 if (__predict_false(events == 0 || oflag == SEL_RESET)) { 627 l->l_selflag = SEL_RESET; 628 return; 629 } 630 /* 631 * Direct set. Note: select state of LWP is locked. First, 632 * determine whether it is selcommon() or pollcommon(). 633 */ 634 if (l->l_selbits != NULL) { 635 fd_mask *fds = (fd_mask *)l->l_selbits; 636 const size_t ni = l->l_selni; 637 const int fd = sip->sel_fdinfo; 638 const int idx = fd >> __NFDSHIFT; 639 int n; 640 641 for (n = 0; n < 3; n++) { 642 if (sel_flag[n] & events) { 643 fds[idx] |= 1 << (fd & __NFDMASK); 644 } 645 fds = (fd_mask *)((char *)fds + ni); 646 } 647 } else { 648 struct pollfd *pfd = (void *)sip->sel_fdinfo; 649 pfd->revents |= events; 650 } 651 /* Indicate direct set and note the event (cluster lock is held). */ 652 l->l_selflag = SEL_EVENT; 653 l->l_selret++; 654 } 655 656 /* 657 * Do a wakeup when a selectable event occurs. Concurrency issues: 658 * 659 * As per selrecord(), the caller's object lock is held. If there 660 * is a named waiter, we must acquire the associated selcluster's lock 661 * in order to synchronize with selclear() and pollers going to sleep 662 * in sel_do_scan(). 663 * 664 * sip->sel_cluser cannot change at this point, as it is only changed 665 * in selrecord(), and concurrent calls to selrecord() are locked 666 * out by the caller. 667 */ 668 void 669 selnotify(struct selinfo *sip, int events, long knhint) 670 { 671 selcluster_t *sc; 672 uint32_t mask; 673 int index, oflag; 674 lwp_t *l; 675 kmutex_t *lock; 676 677 KNOTE(&sip->sel_klist, knhint); 678 679 if (sip->sel_lwp != NULL) { 680 /* One named LWP is waiting. */ 681 sc = sip->sel_cluster; 682 lock = sc->sc_lock; 683 mutex_spin_enter(lock); 684 /* Still there? */ 685 if (sip->sel_lwp != NULL) { 686 /* 687 * Set the events for our LWP and indicate that. 688 * Otherwise, request for a full re-scan. 689 */ 690 l = sip->sel_lwp; 691 oflag = l->l_selflag; 692 #ifdef DIRECT_SELECT 693 sel_setevents(l, sip, events); 694 #else 695 l->l_selflag = SEL_RESET; 696 #endif 697 /* 698 * If thread is sleeping, wake it up. If it's not 699 * yet asleep, it will notice the change in state 700 * and will re-poll the descriptors. 701 */ 702 if (oflag == SEL_BLOCKING && l->l_mutex == lock) { 703 KASSERT(l->l_wchan == sc); 704 sleepq_unsleep(l, false); 705 } 706 } 707 mutex_spin_exit(lock); 708 } 709 710 if ((mask = sip->sel_collision) != 0) { 711 /* 712 * There was a collision (multiple waiters): we must 713 * inform all potentially interested waiters. 714 */ 715 sip->sel_collision = 0; 716 do { 717 index = ffs(mask) - 1; 718 mask &= ~(1 << index); 719 sc = selcluster[index]; 720 lock = sc->sc_lock; 721 mutex_spin_enter(lock); 722 sc->sc_ncoll++; 723 sleepq_wake(&sc->sc_sleepq, sc, (u_int)-1, lock); 724 } while (__predict_false(mask != 0)); 725 } 726 } 727 728 /* 729 * Remove an LWP from all objects that it is waiting for. Concurrency 730 * issues: 731 * 732 * The object owner's (e.g. device driver) lock is not held here. Calls 733 * can be made to selrecord() and we do not synchronize against those 734 * directly using locks. However, we use `sel_lwp' to lock out changes. 735 * Before clearing it we must use memory barriers to ensure that we can 736 * safely traverse the list of selinfo records. 737 */ 738 static void 739 selclear(void) 740 { 741 struct selinfo *sip, *next; 742 selcluster_t *sc; 743 lwp_t *l; 744 kmutex_t *lock; 745 746 l = curlwp; 747 sc = l->l_selcluster; 748 lock = sc->sc_lock; 749 750 mutex_spin_enter(lock); 751 for (sip = SLIST_FIRST(&l->l_selwait); sip != NULL; sip = next) { 752 KASSERT(sip->sel_lwp == l); 753 KASSERT(sip->sel_cluster == l->l_selcluster); 754 755 /* 756 * Read link to next selinfo record, if any. 757 * It's no longer safe to touch `sip' after clearing 758 * `sel_lwp', so ensure that the read of `sel_chain' 759 * completes before the clearing of sel_lwp becomes 760 * globally visible. 761 */ 762 next = SLIST_NEXT(sip, sel_chain); 763 membar_exit(); 764 /* Release the record for another named waiter to use. */ 765 sip->sel_lwp = NULL; 766 } 767 mutex_spin_exit(lock); 768 } 769 770 /* 771 * Initialize the select/poll system calls. Called once for each 772 * CPU in the system, as they are attached. 773 */ 774 void 775 selsysinit(struct cpu_info *ci) 776 { 777 selcluster_t *sc; 778 u_int index; 779 780 /* If already a cluster in place for this bit, re-use. */ 781 index = cpu_index(ci) & SELCLUSTERMASK; 782 sc = selcluster[index]; 783 if (sc == NULL) { 784 sc = kmem_alloc(roundup2(sizeof(selcluster_t), 785 coherency_unit) + coherency_unit, KM_SLEEP); 786 sc = (void *)roundup2((uintptr_t)sc, coherency_unit); 787 sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED); 788 sleepq_init(&sc->sc_sleepq); 789 sc->sc_ncoll = 0; 790 sc->sc_mask = (1 << index); 791 selcluster[index] = sc; 792 } 793 ci->ci_data.cpu_selcluster = sc; 794 } 795 796 /* 797 * Initialize a selinfo record. 798 */ 799 void 800 selinit(struct selinfo *sip) 801 { 802 803 memset(sip, 0, sizeof(*sip)); 804 } 805 806 /* 807 * Destroy a selinfo record. The owning object must not gain new 808 * references while this is in progress: all activity on the record 809 * must be stopped. 810 * 811 * Concurrency issues: we only need guard against a call to selclear() 812 * by a thread exiting sel_do_scan(). The caller has prevented further 813 * references being made to the selinfo record via selrecord(), and it 814 * will not call selnotify() again. 815 */ 816 void 817 seldestroy(struct selinfo *sip) 818 { 819 selcluster_t *sc; 820 kmutex_t *lock; 821 lwp_t *l; 822 823 if (sip->sel_lwp == NULL) 824 return; 825 826 /* 827 * Lock out selclear(). The selcluster pointer can't change while 828 * we are here since it is only ever changed in selrecord(), 829 * and that will not be entered again for this record because 830 * it is dying. 831 */ 832 KASSERT(sip->sel_cluster != NULL); 833 sc = sip->sel_cluster; 834 lock = sc->sc_lock; 835 mutex_spin_enter(lock); 836 if ((l = sip->sel_lwp) != NULL) { 837 /* 838 * This should rarely happen, so although SLIST_REMOVE() 839 * is slow, using it here is not a problem. 840 */ 841 KASSERT(l->l_selcluster == sc); 842 SLIST_REMOVE(&l->l_selwait, sip, selinfo, sel_chain); 843 sip->sel_lwp = NULL; 844 } 845 mutex_spin_exit(lock); 846 } 847 848 int 849 pollsock(struct socket *so, const struct timespec *tsp, int events) 850 { 851 int ncoll, error, timo; 852 struct timespec sleepts, ts; 853 selcluster_t *sc; 854 lwp_t *l; 855 kmutex_t *lock; 856 857 timo = 0; 858 if (tsp != NULL) { 859 ts = *tsp; 860 if (inittimeleft(&ts, &sleepts) == -1) 861 return EINVAL; 862 } 863 864 l = curlwp; 865 sc = curcpu()->ci_data.cpu_selcluster; 866 lock = sc->sc_lock; 867 l->l_selcluster = sc; 868 SLIST_INIT(&l->l_selwait); 869 error = 0; 870 for (;;) { 871 /* 872 * No need to lock. If this is overwritten by another 873 * value while scanning, we will retry below. We only 874 * need to see exact state from the descriptors that 875 * we are about to poll, and lock activity resulting 876 * from fo_poll is enough to provide an up to date value 877 * for new polling activity. 878 */ 879 ncoll = sc->sc_ncoll; 880 l->l_selflag = SEL_SCANNING; 881 if (sopoll(so, events) != 0) 882 break; 883 if (tsp && (timo = gettimeleft(&ts, &sleepts)) <= 0) 884 break; 885 mutex_spin_enter(lock); 886 if (l->l_selflag != SEL_SCANNING || sc->sc_ncoll != ncoll) { 887 mutex_spin_exit(lock); 888 continue; 889 } 890 l->l_selflag = SEL_BLOCKING; 891 sleepq_enter(&sc->sc_sleepq, l, lock); 892 sleepq_enqueue(&sc->sc_sleepq, sc, "pollsock", &select_sobj); 893 error = sleepq_block(timo, true); 894 if (error != 0) 895 break; 896 } 897 selclear(); 898 /* poll is not restarted after signals... */ 899 if (error == ERESTART) 900 error = EINTR; 901 if (error == EWOULDBLOCK) 902 error = 0; 903 return (error); 904 } 905