1 /* $NetBSD: sys_select.c,v 1.30 2011/03/06 04:41:58 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran and Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1982, 1986, 1989, 1993 34 * The Regents of the University of California. All rights reserved. 35 * (c) UNIX System Laboratories, Inc. 36 * All or some portions of this file are derived from material licensed 37 * to the University of California by American Telephone and Telegraph 38 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 39 * the permission of UNIX System Laboratories, Inc. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)sys_generic.c 8.9 (Berkeley) 2/14/95 66 */ 67 68 /* 69 * System calls of synchronous I/O multiplexing subsystem. 70 * 71 * Locking 72 * 73 * Two locks are used: <object-lock> and selcluster_t::sc_lock. 74 * 75 * The <object-lock> might be a device driver or another subsystem, e.g. 76 * socket or pipe. This lock is not exported, and thus invisible to this 77 * subsystem. Mainly, synchronisation between selrecord() and selnotify() 78 * routines depends on this lock, as it will be described in the comments. 79 * 80 * Lock order 81 * 82 * <object-lock> -> 83 * selcluster_t::sc_lock 84 */ 85 86 #include <sys/cdefs.h> 87 __KERNEL_RCSID(0, "$NetBSD: sys_select.c,v 1.30 2011/03/06 04:41:58 rmind Exp $"); 88 89 #include <sys/param.h> 90 #include <sys/systm.h> 91 #include <sys/filedesc.h> 92 #include <sys/file.h> 93 #include <sys/proc.h> 94 #include <sys/socketvar.h> 95 #include <sys/signalvar.h> 96 #include <sys/uio.h> 97 #include <sys/kernel.h> 98 #include <sys/lwp.h> 99 #include <sys/poll.h> 100 #include <sys/mount.h> 101 #include <sys/syscallargs.h> 102 #include <sys/cpu.h> 103 #include <sys/atomic.h> 104 #include <sys/socketvar.h> 105 #include <sys/sleepq.h> 106 107 /* Flags for lwp::l_selflag. */ 108 #define SEL_RESET 0 /* awoken, interrupted, or not yet polling */ 109 #define SEL_SCANNING 1 /* polling descriptors */ 110 #define SEL_BLOCKING 2 /* blocking and waiting for event */ 111 #define SEL_EVENT 3 /* interrupted, events set directly */ 112 113 /* Operations: either select() or poll(). */ 114 #define SELOP_SELECT 1 115 #define SELOP_POLL 2 116 117 /* 118 * Per-cluster state for select()/poll(). For a system with fewer 119 * than 32 CPUs, this gives us per-CPU clusters. 120 */ 121 #define SELCLUSTERS 32 122 #define SELCLUSTERMASK (SELCLUSTERS - 1) 123 124 typedef struct selcluster { 125 kmutex_t *sc_lock; 126 sleepq_t sc_sleepq; 127 int sc_ncoll; 128 uint32_t sc_mask; 129 } selcluster_t; 130 131 static inline int selscan(char *, const int, const size_t, register_t *); 132 static inline int pollscan(struct pollfd *, const int, register_t *); 133 static void selclear(void); 134 135 static const int sel_flag[] = { 136 POLLRDNORM | POLLHUP | POLLERR, 137 POLLWRNORM | POLLHUP | POLLERR, 138 POLLRDBAND 139 }; 140 141 static syncobj_t select_sobj = { 142 SOBJ_SLEEPQ_FIFO, 143 sleepq_unsleep, 144 sleepq_changepri, 145 sleepq_lendpri, 146 syncobj_noowner, 147 }; 148 149 static selcluster_t *selcluster[SELCLUSTERS] __read_mostly; 150 151 /* 152 * Select system call. 153 */ 154 int 155 sys___pselect50(struct lwp *l, const struct sys___pselect50_args *uap, 156 register_t *retval) 157 { 158 /* { 159 syscallarg(int) nd; 160 syscallarg(fd_set *) in; 161 syscallarg(fd_set *) ou; 162 syscallarg(fd_set *) ex; 163 syscallarg(const struct timespec *) ts; 164 syscallarg(sigset_t *) mask; 165 } */ 166 struct timespec ats, *ts = NULL; 167 sigset_t amask, *mask = NULL; 168 int error; 169 170 if (SCARG(uap, ts)) { 171 error = copyin(SCARG(uap, ts), &ats, sizeof(ats)); 172 if (error) 173 return error; 174 ts = &ats; 175 } 176 if (SCARG(uap, mask) != NULL) { 177 error = copyin(SCARG(uap, mask), &amask, sizeof(amask)); 178 if (error) 179 return error; 180 mask = &amask; 181 } 182 183 return selcommon(retval, SCARG(uap, nd), SCARG(uap, in), 184 SCARG(uap, ou), SCARG(uap, ex), ts, mask); 185 } 186 187 int 188 sys___select50(struct lwp *l, const struct sys___select50_args *uap, 189 register_t *retval) 190 { 191 /* { 192 syscallarg(int) nd; 193 syscallarg(fd_set *) in; 194 syscallarg(fd_set *) ou; 195 syscallarg(fd_set *) ex; 196 syscallarg(struct timeval *) tv; 197 } */ 198 struct timeval atv; 199 struct timespec ats, *ts = NULL; 200 int error; 201 202 if (SCARG(uap, tv)) { 203 error = copyin(SCARG(uap, tv), (void *)&atv, sizeof(atv)); 204 if (error) 205 return error; 206 TIMEVAL_TO_TIMESPEC(&atv, &ats); 207 ts = &ats; 208 } 209 210 return selcommon(retval, SCARG(uap, nd), SCARG(uap, in), 211 SCARG(uap, ou), SCARG(uap, ex), ts, NULL); 212 } 213 214 /* 215 * sel_do_scan: common code to perform the scan on descriptors. 216 */ 217 static int 218 sel_do_scan(const int op, void *fds, const int nf, const size_t ni, 219 struct timespec *ts, sigset_t *mask, register_t *retval) 220 { 221 lwp_t * const l = curlwp; 222 proc_t * const p = l->l_proc; 223 selcluster_t *sc; 224 kmutex_t *lock; 225 sigset_t oldmask; 226 struct timespec sleepts; 227 int error, timo; 228 229 timo = 0; 230 if (ts && inittimeleft(ts, &sleepts) == -1) { 231 return EINVAL; 232 } 233 234 if (__predict_false(mask)) { 235 sigminusset(&sigcantmask, mask); 236 mutex_enter(p->p_lock); 237 oldmask = l->l_sigmask; 238 l->l_sigmask = *mask; 239 mutex_exit(p->p_lock); 240 } else { 241 /* XXXgcc */ 242 oldmask = l->l_sigmask; 243 } 244 245 sc = curcpu()->ci_data.cpu_selcluster; 246 lock = sc->sc_lock; 247 l->l_selcluster = sc; 248 SLIST_INIT(&l->l_selwait); 249 250 l->l_selret = 0; 251 if (op == SELOP_SELECT) { 252 l->l_selbits = fds; 253 l->l_selni = ni; 254 } else { 255 l->l_selbits = NULL; 256 } 257 for (;;) { 258 int ncoll; 259 260 /* 261 * No need to lock. If this is overwritten by another value 262 * while scanning, we will retry below. We only need to see 263 * exact state from the descriptors that we are about to poll, 264 * and lock activity resulting from fo_poll is enough to 265 * provide an up to date value for new polling activity. 266 */ 267 l->l_selflag = SEL_SCANNING; 268 ncoll = sc->sc_ncoll; 269 270 if (op == SELOP_SELECT) { 271 error = selscan((char *)fds, nf, ni, retval); 272 } else { 273 error = pollscan((struct pollfd *)fds, nf, retval); 274 } 275 if (error || *retval) 276 break; 277 if (ts && (timo = gettimeleft(ts, &sleepts)) <= 0) 278 break; 279 /* 280 * Acquire the lock and perform the (re)checks. Note, if 281 * collision has occured, then our state does not matter, 282 * as we must perform re-scan. Therefore, check it first. 283 */ 284 state_check: 285 mutex_spin_enter(lock); 286 if (__predict_false(sc->sc_ncoll != ncoll)) { 287 /* Collision: perform re-scan. */ 288 mutex_spin_exit(lock); 289 continue; 290 } 291 if (__predict_true(l->l_selflag == SEL_EVENT)) { 292 /* Events occured, they are set directly. */ 293 mutex_spin_exit(lock); 294 KASSERT(l->l_selret != 0); 295 *retval = l->l_selret; 296 break; 297 } 298 if (__predict_true(l->l_selflag == SEL_RESET)) { 299 /* Events occured, but re-scan is requested. */ 300 mutex_spin_exit(lock); 301 continue; 302 } 303 /* Nothing happen, therefore - sleep. */ 304 l->l_selflag = SEL_BLOCKING; 305 l->l_kpriority = true; 306 sleepq_enter(&sc->sc_sleepq, l, lock); 307 sleepq_enqueue(&sc->sc_sleepq, sc, "select", &select_sobj); 308 error = sleepq_block(timo, true); 309 if (error != 0) { 310 break; 311 } 312 /* Awoken: need to check the state. */ 313 goto state_check; 314 } 315 selclear(); 316 317 if (__predict_false(mask)) { 318 mutex_enter(p->p_lock); 319 l->l_sigmask = oldmask; 320 mutex_exit(p->p_lock); 321 } 322 323 /* select and poll are not restarted after signals... */ 324 if (error == ERESTART) 325 return EINTR; 326 if (error == EWOULDBLOCK) 327 return 0; 328 return error; 329 } 330 331 int 332 selcommon(register_t *retval, int nd, fd_set *u_in, fd_set *u_ou, 333 fd_set *u_ex, struct timespec *ts, sigset_t *mask) 334 { 335 char smallbits[howmany(FD_SETSIZE, NFDBITS) * 336 sizeof(fd_mask) * 6]; 337 char *bits; 338 int error, nf; 339 size_t ni; 340 341 if (nd < 0) 342 return (EINVAL); 343 nf = curlwp->l_fd->fd_dt->dt_nfiles; 344 if (nd > nf) { 345 /* forgiving; slightly wrong */ 346 nd = nf; 347 } 348 ni = howmany(nd, NFDBITS) * sizeof(fd_mask); 349 if (ni * 6 > sizeof(smallbits)) { 350 bits = kmem_alloc(ni * 6, KM_SLEEP); 351 if (bits == NULL) 352 return ENOMEM; 353 } else 354 bits = smallbits; 355 356 #define getbits(name, x) \ 357 if (u_ ## name) { \ 358 error = copyin(u_ ## name, bits + ni * x, ni); \ 359 if (error) \ 360 goto fail; \ 361 } else \ 362 memset(bits + ni * x, 0, ni); 363 getbits(in, 0); 364 getbits(ou, 1); 365 getbits(ex, 2); 366 #undef getbits 367 368 error = sel_do_scan(SELOP_SELECT, bits, nd, ni, ts, mask, retval); 369 if (error == 0 && u_in != NULL) 370 error = copyout(bits + ni * 3, u_in, ni); 371 if (error == 0 && u_ou != NULL) 372 error = copyout(bits + ni * 4, u_ou, ni); 373 if (error == 0 && u_ex != NULL) 374 error = copyout(bits + ni * 5, u_ex, ni); 375 fail: 376 if (bits != smallbits) 377 kmem_free(bits, ni * 6); 378 return (error); 379 } 380 381 static inline int 382 selscan(char *bits, const int nfd, const size_t ni, register_t *retval) 383 { 384 fd_mask *ibitp, *obitp; 385 int msk, i, j, fd, n; 386 file_t *fp; 387 388 ibitp = (fd_mask *)(bits + ni * 0); 389 obitp = (fd_mask *)(bits + ni * 3); 390 n = 0; 391 392 for (msk = 0; msk < 3; msk++) { 393 for (i = 0; i < nfd; i += NFDBITS) { 394 fd_mask ibits, obits; 395 396 ibits = *ibitp++; 397 obits = 0; 398 while ((j = ffs(ibits)) && (fd = i + --j) < nfd) { 399 ibits &= ~(1 << j); 400 if ((fp = fd_getfile(fd)) == NULL) 401 return (EBADF); 402 /* 403 * Setup an argument to selrecord(), which is 404 * a file descriptor number. 405 */ 406 curlwp->l_selrec = fd; 407 if ((*fp->f_ops->fo_poll)(fp, sel_flag[msk])) { 408 obits |= (1 << j); 409 n++; 410 } 411 fd_putfile(fd); 412 } 413 *obitp++ = obits; 414 } 415 } 416 *retval = n; 417 return (0); 418 } 419 420 /* 421 * Poll system call. 422 */ 423 int 424 sys_poll(struct lwp *l, const struct sys_poll_args *uap, register_t *retval) 425 { 426 /* { 427 syscallarg(struct pollfd *) fds; 428 syscallarg(u_int) nfds; 429 syscallarg(int) timeout; 430 } */ 431 struct timespec ats, *ts = NULL; 432 433 if (SCARG(uap, timeout) != INFTIM) { 434 ats.tv_sec = SCARG(uap, timeout) / 1000; 435 ats.tv_nsec = (SCARG(uap, timeout) % 1000) * 1000000; 436 ts = &ats; 437 } 438 439 return pollcommon(retval, SCARG(uap, fds), SCARG(uap, nfds), ts, NULL); 440 } 441 442 /* 443 * Poll system call. 444 */ 445 int 446 sys___pollts50(struct lwp *l, const struct sys___pollts50_args *uap, 447 register_t *retval) 448 { 449 /* { 450 syscallarg(struct pollfd *) fds; 451 syscallarg(u_int) nfds; 452 syscallarg(const struct timespec *) ts; 453 syscallarg(const sigset_t *) mask; 454 } */ 455 struct timespec ats, *ts = NULL; 456 sigset_t amask, *mask = NULL; 457 int error; 458 459 if (SCARG(uap, ts)) { 460 error = copyin(SCARG(uap, ts), &ats, sizeof(ats)); 461 if (error) 462 return error; 463 ts = &ats; 464 } 465 if (SCARG(uap, mask)) { 466 error = copyin(SCARG(uap, mask), &amask, sizeof(amask)); 467 if (error) 468 return error; 469 mask = &amask; 470 } 471 472 return pollcommon(retval, SCARG(uap, fds), SCARG(uap, nfds), ts, mask); 473 } 474 475 int 476 pollcommon(register_t *retval, struct pollfd *u_fds, u_int nfds, 477 struct timespec *ts, sigset_t *mask) 478 { 479 struct pollfd smallfds[32]; 480 struct pollfd *fds; 481 int error; 482 size_t ni; 483 484 if (nfds > 1000 + curlwp->l_fd->fd_dt->dt_nfiles) { 485 /* 486 * Either the user passed in a very sparse 'fds' or junk! 487 * The kmem_alloc() call below would be bad news. 488 * We could process the 'fds' array in chunks, but that 489 * is a lot of code that isn't normally useful. 490 * (Or just move the copyin/out into pollscan().) 491 * Historically the code silently truncated 'fds' to 492 * dt_nfiles entries - but that does cause issues. 493 */ 494 return EINVAL; 495 } 496 ni = nfds * sizeof(struct pollfd); 497 if (ni > sizeof(smallfds)) { 498 fds = kmem_alloc(ni, KM_SLEEP); 499 if (fds == NULL) 500 return ENOMEM; 501 } else 502 fds = smallfds; 503 504 error = copyin(u_fds, fds, ni); 505 if (error) 506 goto fail; 507 508 error = sel_do_scan(SELOP_POLL, fds, nfds, ni, ts, mask, retval); 509 if (error == 0) 510 error = copyout(fds, u_fds, ni); 511 fail: 512 if (fds != smallfds) 513 kmem_free(fds, ni); 514 return (error); 515 } 516 517 static inline int 518 pollscan(struct pollfd *fds, const int nfd, register_t *retval) 519 { 520 file_t *fp; 521 int i, n = 0; 522 523 for (i = 0; i < nfd; i++, fds++) { 524 if (fds->fd < 0) { 525 fds->revents = 0; 526 } else if ((fp = fd_getfile(fds->fd)) == NULL) { 527 fds->revents = POLLNVAL; 528 n++; 529 } else { 530 /* 531 * Perform poll: registers select request or returns 532 * the events which are set. Setup an argument for 533 * selrecord(), which is a pointer to struct pollfd. 534 */ 535 curlwp->l_selrec = (uintptr_t)fds; 536 fds->revents = (*fp->f_ops->fo_poll)(fp, 537 fds->events | POLLERR | POLLHUP); 538 if (fds->revents != 0) 539 n++; 540 fd_putfile(fds->fd); 541 } 542 } 543 *retval = n; 544 return (0); 545 } 546 547 int 548 seltrue(dev_t dev, int events, lwp_t *l) 549 { 550 551 return (events & (POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM)); 552 } 553 554 /* 555 * Record a select request. Concurrency issues: 556 * 557 * The caller holds the same lock across calls to selrecord() and 558 * selnotify(), so we don't need to consider a concurrent wakeup 559 * while in this routine. 560 * 561 * The only activity we need to guard against is selclear(), called by 562 * another thread that is exiting sel_do_scan(). 563 * `sel_lwp' can only become non-NULL while the caller's lock is held, 564 * so it cannot become non-NULL due to a change made by another thread 565 * while we are in this routine. It can only become _NULL_ due to a 566 * call to selclear(). 567 * 568 * If it is non-NULL and != selector there is the potential for 569 * selclear() to be called by another thread. If either of those 570 * conditions are true, we're not interested in touching the `named 571 * waiter' part of the selinfo record because we need to record a 572 * collision. Hence there is no need for additional locking in this 573 * routine. 574 */ 575 void 576 selrecord(lwp_t *selector, struct selinfo *sip) 577 { 578 selcluster_t *sc; 579 lwp_t *other; 580 581 KASSERT(selector == curlwp); 582 583 sc = selector->l_selcluster; 584 other = sip->sel_lwp; 585 586 if (other == selector) { 587 /* 1. We (selector) already claimed to be the first LWP. */ 588 KASSERT(sip->sel_cluster = sc); 589 } else if (other == NULL) { 590 /* 591 * 2. No first LWP, therefore we (selector) are the first. 592 * 593 * There may be unnamed waiters (collisions). Issue a memory 594 * barrier to ensure that we access sel_lwp (above) before 595 * other fields - this guards against a call to selclear(). 596 */ 597 membar_enter(); 598 sip->sel_lwp = selector; 599 SLIST_INSERT_HEAD(&selector->l_selwait, sip, sel_chain); 600 /* Copy the argument, which is for selnotify(). */ 601 sip->sel_fdinfo = selector->l_selrec; 602 /* Replace selinfo's lock with the chosen cluster's lock. */ 603 sip->sel_cluster = sc; 604 } else { 605 /* 3. Multiple waiters: record a collision. */ 606 sip->sel_collision |= sc->sc_mask; 607 KASSERT(sip->sel_cluster != NULL); 608 } 609 } 610 611 /* 612 * sel_setevents: a helper function for selnotify(), to set the events 613 * for LWP sleeping in selcommon() or pollcommon(). 614 */ 615 static inline bool 616 sel_setevents(lwp_t *l, struct selinfo *sip, const int events) 617 { 618 const int oflag = l->l_selflag; 619 int ret = 0; 620 621 /* 622 * If we require re-scan or it was required by somebody else, 623 * then just (re)set SEL_RESET and return. 624 */ 625 if (__predict_false(events == 0 || oflag == SEL_RESET)) { 626 l->l_selflag = SEL_RESET; 627 return true; 628 } 629 /* 630 * Direct set. Note: select state of LWP is locked. First, 631 * determine whether it is selcommon() or pollcommon(). 632 */ 633 if (l->l_selbits != NULL) { 634 const size_t ni = l->l_selni; 635 fd_mask *fds = (fd_mask *)l->l_selbits; 636 fd_mask *ofds = (fd_mask *)((char *)fds + ni * 3); 637 const int fd = sip->sel_fdinfo, fbit = 1 << (fd & __NFDMASK); 638 const int idx = fd >> __NFDSHIFT; 639 int n; 640 641 for (n = 0; n < 3; n++) { 642 if ((fds[idx] & fbit) != 0 && (sel_flag[n] & events)) { 643 ofds[idx] |= fbit; 644 ret++; 645 } 646 fds = (fd_mask *)((char *)fds + ni); 647 ofds = (fd_mask *)((char *)ofds + ni); 648 } 649 } else { 650 struct pollfd *pfd = (void *)sip->sel_fdinfo; 651 int revents = events & (pfd->events | POLLERR | POLLHUP); 652 653 if (revents) { 654 pfd->revents |= revents; 655 ret = 1; 656 } 657 } 658 /* Check whether there are any events to return. */ 659 if (!ret) { 660 return false; 661 } 662 /* Indicate direct set and note the event (cluster lock is held). */ 663 l->l_selflag = SEL_EVENT; 664 l->l_selret += ret; 665 return true; 666 } 667 668 /* 669 * Do a wakeup when a selectable event occurs. Concurrency issues: 670 * 671 * As per selrecord(), the caller's object lock is held. If there 672 * is a named waiter, we must acquire the associated selcluster's lock 673 * in order to synchronize with selclear() and pollers going to sleep 674 * in sel_do_scan(). 675 * 676 * sip->sel_cluser cannot change at this point, as it is only changed 677 * in selrecord(), and concurrent calls to selrecord() are locked 678 * out by the caller. 679 */ 680 void 681 selnotify(struct selinfo *sip, int events, long knhint) 682 { 683 selcluster_t *sc; 684 uint32_t mask; 685 int index, oflag; 686 lwp_t *l; 687 kmutex_t *lock; 688 689 KNOTE(&sip->sel_klist, knhint); 690 691 if (sip->sel_lwp != NULL) { 692 /* One named LWP is waiting. */ 693 sc = sip->sel_cluster; 694 lock = sc->sc_lock; 695 mutex_spin_enter(lock); 696 /* Still there? */ 697 if (sip->sel_lwp != NULL) { 698 /* 699 * Set the events for our LWP and indicate that. 700 * Otherwise, request for a full re-scan. 701 */ 702 l = sip->sel_lwp; 703 oflag = l->l_selflag; 704 #ifndef NO_DIRECT_SELECT 705 if (!sel_setevents(l, sip, events)) { 706 /* No events to return. */ 707 mutex_spin_exit(lock); 708 return; 709 } 710 #else 711 l->l_selflag = SEL_RESET; 712 #endif 713 /* 714 * If thread is sleeping, wake it up. If it's not 715 * yet asleep, it will notice the change in state 716 * and will re-poll the descriptors. 717 */ 718 if (oflag == SEL_BLOCKING && l->l_mutex == lock) { 719 KASSERT(l->l_wchan == sc); 720 sleepq_unsleep(l, false); 721 } 722 } 723 mutex_spin_exit(lock); 724 } 725 726 if ((mask = sip->sel_collision) != 0) { 727 /* 728 * There was a collision (multiple waiters): we must 729 * inform all potentially interested waiters. 730 */ 731 sip->sel_collision = 0; 732 do { 733 index = ffs(mask) - 1; 734 mask &= ~(1 << index); 735 sc = selcluster[index]; 736 lock = sc->sc_lock; 737 mutex_spin_enter(lock); 738 sc->sc_ncoll++; 739 sleepq_wake(&sc->sc_sleepq, sc, (u_int)-1, lock); 740 } while (__predict_false(mask != 0)); 741 } 742 } 743 744 /* 745 * Remove an LWP from all objects that it is waiting for. Concurrency 746 * issues: 747 * 748 * The object owner's (e.g. device driver) lock is not held here. Calls 749 * can be made to selrecord() and we do not synchronize against those 750 * directly using locks. However, we use `sel_lwp' to lock out changes. 751 * Before clearing it we must use memory barriers to ensure that we can 752 * safely traverse the list of selinfo records. 753 */ 754 static void 755 selclear(void) 756 { 757 struct selinfo *sip, *next; 758 selcluster_t *sc; 759 lwp_t *l; 760 kmutex_t *lock; 761 762 l = curlwp; 763 sc = l->l_selcluster; 764 lock = sc->sc_lock; 765 766 mutex_spin_enter(lock); 767 for (sip = SLIST_FIRST(&l->l_selwait); sip != NULL; sip = next) { 768 KASSERT(sip->sel_lwp == l); 769 KASSERT(sip->sel_cluster == l->l_selcluster); 770 771 /* 772 * Read link to next selinfo record, if any. 773 * It's no longer safe to touch `sip' after clearing 774 * `sel_lwp', so ensure that the read of `sel_chain' 775 * completes before the clearing of sel_lwp becomes 776 * globally visible. 777 */ 778 next = SLIST_NEXT(sip, sel_chain); 779 membar_exit(); 780 /* Release the record for another named waiter to use. */ 781 sip->sel_lwp = NULL; 782 } 783 mutex_spin_exit(lock); 784 } 785 786 /* 787 * Initialize the select/poll system calls. Called once for each 788 * CPU in the system, as they are attached. 789 */ 790 void 791 selsysinit(struct cpu_info *ci) 792 { 793 selcluster_t *sc; 794 u_int index; 795 796 /* If already a cluster in place for this bit, re-use. */ 797 index = cpu_index(ci) & SELCLUSTERMASK; 798 sc = selcluster[index]; 799 if (sc == NULL) { 800 sc = kmem_alloc(roundup2(sizeof(selcluster_t), 801 coherency_unit) + coherency_unit, KM_SLEEP); 802 sc = (void *)roundup2((uintptr_t)sc, coherency_unit); 803 sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED); 804 sleepq_init(&sc->sc_sleepq); 805 sc->sc_ncoll = 0; 806 sc->sc_mask = (1 << index); 807 selcluster[index] = sc; 808 } 809 ci->ci_data.cpu_selcluster = sc; 810 } 811 812 /* 813 * Initialize a selinfo record. 814 */ 815 void 816 selinit(struct selinfo *sip) 817 { 818 819 memset(sip, 0, sizeof(*sip)); 820 } 821 822 /* 823 * Destroy a selinfo record. The owning object must not gain new 824 * references while this is in progress: all activity on the record 825 * must be stopped. 826 * 827 * Concurrency issues: we only need guard against a call to selclear() 828 * by a thread exiting sel_do_scan(). The caller has prevented further 829 * references being made to the selinfo record via selrecord(), and it 830 * will not call selnotify() again. 831 */ 832 void 833 seldestroy(struct selinfo *sip) 834 { 835 selcluster_t *sc; 836 kmutex_t *lock; 837 lwp_t *l; 838 839 if (sip->sel_lwp == NULL) 840 return; 841 842 /* 843 * Lock out selclear(). The selcluster pointer can't change while 844 * we are here since it is only ever changed in selrecord(), 845 * and that will not be entered again for this record because 846 * it is dying. 847 */ 848 KASSERT(sip->sel_cluster != NULL); 849 sc = sip->sel_cluster; 850 lock = sc->sc_lock; 851 mutex_spin_enter(lock); 852 if ((l = sip->sel_lwp) != NULL) { 853 /* 854 * This should rarely happen, so although SLIST_REMOVE() 855 * is slow, using it here is not a problem. 856 */ 857 KASSERT(l->l_selcluster == sc); 858 SLIST_REMOVE(&l->l_selwait, sip, selinfo, sel_chain); 859 sip->sel_lwp = NULL; 860 } 861 mutex_spin_exit(lock); 862 } 863 864 int 865 pollsock(struct socket *so, const struct timespec *tsp, int events) 866 { 867 int ncoll, error, timo; 868 struct timespec sleepts, ts; 869 selcluster_t *sc; 870 lwp_t *l; 871 kmutex_t *lock; 872 873 timo = 0; 874 if (tsp != NULL) { 875 ts = *tsp; 876 if (inittimeleft(&ts, &sleepts) == -1) 877 return EINVAL; 878 } 879 880 l = curlwp; 881 sc = curcpu()->ci_data.cpu_selcluster; 882 lock = sc->sc_lock; 883 l->l_selcluster = sc; 884 SLIST_INIT(&l->l_selwait); 885 error = 0; 886 for (;;) { 887 /* 888 * No need to lock. If this is overwritten by another 889 * value while scanning, we will retry below. We only 890 * need to see exact state from the descriptors that 891 * we are about to poll, and lock activity resulting 892 * from fo_poll is enough to provide an up to date value 893 * for new polling activity. 894 */ 895 ncoll = sc->sc_ncoll; 896 l->l_selflag = SEL_SCANNING; 897 if (sopoll(so, events) != 0) 898 break; 899 if (tsp && (timo = gettimeleft(&ts, &sleepts)) <= 0) 900 break; 901 mutex_spin_enter(lock); 902 if (l->l_selflag != SEL_SCANNING || sc->sc_ncoll != ncoll) { 903 mutex_spin_exit(lock); 904 continue; 905 } 906 l->l_selflag = SEL_BLOCKING; 907 sleepq_enter(&sc->sc_sleepq, l, lock); 908 sleepq_enqueue(&sc->sc_sleepq, sc, "pollsock", &select_sobj); 909 error = sleepq_block(timo, true); 910 if (error != 0) 911 break; 912 } 913 selclear(); 914 /* poll is not restarted after signals... */ 915 if (error == ERESTART) 916 error = EINTR; 917 if (error == EWOULDBLOCK) 918 error = 0; 919 return (error); 920 } 921