1 /* $NetBSD: kern_lwp.c,v 1.133 2009/09/13 18:45:11 pooka Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nathan J. Williams, and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Overview 34 * 35 * Lightweight processes (LWPs) are the basic unit or thread of 36 * execution within the kernel. The core state of an LWP is described 37 * by "struct lwp", also known as lwp_t. 38 * 39 * Each LWP is contained within a process (described by "struct proc"), 40 * Every process contains at least one LWP, but may contain more. The 41 * process describes attributes shared among all of its LWPs such as a 42 * private address space, global execution state (stopped, active, 43 * zombie, ...), signal disposition and so on. On a multiprocessor 44 * machine, multiple LWPs be executing concurrently in the kernel. 45 * 46 * Execution states 47 * 48 * At any given time, an LWP has overall state that is described by 49 * lwp::l_stat. The states are broken into two sets below. The first 50 * set is guaranteed to represent the absolute, current state of the 51 * LWP: 52 * 53 * LSONPROC 54 * 55 * On processor: the LWP is executing on a CPU, either in the 56 * kernel or in user space. 57 * 58 * LSRUN 59 * 60 * Runnable: the LWP is parked on a run queue, and may soon be 61 * chosen to run by an idle processor, or by a processor that 62 * has been asked to preempt a currently runnning but lower 63 * priority LWP. If the LWP is not swapped in (LW_INMEM == 0) 64 * then the LWP is not on a run queue, but may be soon. 65 * 66 * LSIDL 67 * 68 * Idle: the LWP has been created but has not yet executed, 69 * or it has ceased executing a unit of work and is waiting 70 * to be started again. 71 * 72 * LSSUSPENDED: 73 * 74 * Suspended: the LWP has had its execution suspended by 75 * another LWP in the same process using the _lwp_suspend() 76 * system call. User-level LWPs also enter the suspended 77 * state when the system is shutting down. 78 * 79 * The second set represent a "statement of intent" on behalf of the 80 * LWP. The LWP may in fact be executing on a processor, may be 81 * sleeping or idle. It is expected to take the necessary action to 82 * stop executing or become "running" again within a short timeframe. 83 * The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running. 84 * Importantly, it indicates that its state is tied to a CPU. 85 * 86 * LSZOMB: 87 * 88 * Dead or dying: the LWP has released most of its resources 89 * and is about to switch away into oblivion, or has already 90 * switched away. When it switches away, its few remaining 91 * resources can be collected. 92 * 93 * LSSLEEP: 94 * 95 * Sleeping: the LWP has entered itself onto a sleep queue, and 96 * has switched away or will switch away shortly to allow other 97 * LWPs to run on the CPU. 98 * 99 * LSSTOP: 100 * 101 * Stopped: the LWP has been stopped as a result of a job 102 * control signal, or as a result of the ptrace() interface. 103 * 104 * Stopped LWPs may run briefly within the kernel to handle 105 * signals that they receive, but will not return to user space 106 * until their process' state is changed away from stopped. 107 * 108 * Single LWPs within a process can not be set stopped 109 * selectively: all actions that can stop or continue LWPs 110 * occur at the process level. 111 * 112 * State transitions 113 * 114 * Note that the LSSTOP state may only be set when returning to 115 * user space in userret(), or when sleeping interruptably. The 116 * LSSUSPENDED state may only be set in userret(). Before setting 117 * those states, we try to ensure that the LWPs will release all 118 * locks that they hold, and at a minimum try to ensure that the 119 * LWP can be set runnable again by a signal. 120 * 121 * LWPs may transition states in the following ways: 122 * 123 * RUN -------> ONPROC ONPROC -----> RUN 124 * > SLEEP 125 * > STOPPED 126 * > SUSPENDED 127 * > ZOMB 128 * > IDL (special cases) 129 * 130 * STOPPED ---> RUN SUSPENDED --> RUN 131 * > SLEEP 132 * 133 * SLEEP -----> ONPROC IDL --------> RUN 134 * > RUN > SUSPENDED 135 * > STOPPED > STOPPED 136 * > ONPROC (special cases) 137 * 138 * Some state transitions are only possible with kernel threads (eg 139 * ONPROC -> IDL) and happen under tightly controlled circumstances 140 * free of unwanted side effects. 141 * 142 * Migration 143 * 144 * Migration of threads from one CPU to another could be performed 145 * internally by the scheduler via sched_takecpu() or sched_catchlwp() 146 * functions. The universal lwp_migrate() function should be used for 147 * any other cases. Subsystems in the kernel must be aware that CPU 148 * of LWP may change, while it is not locked. 149 * 150 * Locking 151 * 152 * The majority of fields in 'struct lwp' are covered by a single, 153 * general spin lock pointed to by lwp::l_mutex. The locks covering 154 * each field are documented in sys/lwp.h. 155 * 156 * State transitions must be made with the LWP's general lock held, 157 * and may cause the LWP's lock pointer to change. Manipulation of 158 * the general lock is not performed directly, but through calls to 159 * lwp_lock(), lwp_relock() and similar. 160 * 161 * States and their associated locks: 162 * 163 * LSONPROC, LSZOMB: 164 * 165 * Always covered by spc_lwplock, which protects running LWPs. 166 * This is a per-CPU lock and matches lwp::l_cpu. 167 * 168 * LSIDL, LSRUN: 169 * 170 * Always covered by spc_mutex, which protects the run queues. 171 * This is a per-CPU lock and matches lwp::l_cpu. 172 * 173 * LSSLEEP: 174 * 175 * Covered by a lock associated with the sleep queue that the 176 * LWP resides on. Matches lwp::l_sleepq::sq_mutex. 177 * 178 * LSSTOP, LSSUSPENDED: 179 * 180 * If the LWP was previously sleeping (l_wchan != NULL), then 181 * l_mutex references the sleep queue lock. If the LWP was 182 * runnable or on the CPU when halted, or has been removed from 183 * the sleep queue since halted, then the lock is spc_lwplock. 184 * 185 * The lock order is as follows: 186 * 187 * spc::spc_lwplock -> 188 * sleeptab::st_mutex -> 189 * tschain_t::tc_mutex -> 190 * spc::spc_mutex 191 * 192 * Each process has an scheduler state lock (proc::p_lock), and a 193 * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and 194 * so on. When an LWP is to be entered into or removed from one of the 195 * following states, p_lock must be held and the process wide counters 196 * adjusted: 197 * 198 * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED 199 * 200 * (But not always for kernel threads. There are some special cases 201 * as mentioned above. See kern_softint.c.) 202 * 203 * Note that an LWP is considered running or likely to run soon if in 204 * one of the following states. This affects the value of p_nrlwps: 205 * 206 * LSRUN, LSONPROC, LSSLEEP 207 * 208 * p_lock does not need to be held when transitioning among these 209 * three states, hence p_lock is rarely taken for state transitions. 210 */ 211 212 #include <sys/cdefs.h> 213 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.133 2009/09/13 18:45:11 pooka Exp $"); 214 215 #include "opt_ddb.h" 216 #include "opt_lockdebug.h" 217 #include "opt_sa.h" 218 219 #define _LWP_API_PRIVATE 220 221 #include <sys/param.h> 222 #include <sys/systm.h> 223 #include <sys/cpu.h> 224 #include <sys/pool.h> 225 #include <sys/proc.h> 226 #include <sys/sa.h> 227 #include <sys/savar.h> 228 #include <sys/syscallargs.h> 229 #include <sys/syscall_stats.h> 230 #include <sys/kauth.h> 231 #include <sys/sleepq.h> 232 #include <sys/user.h> 233 #include <sys/lockdebug.h> 234 #include <sys/kmem.h> 235 #include <sys/pset.h> 236 #include <sys/intr.h> 237 #include <sys/lwpctl.h> 238 #include <sys/atomic.h> 239 #include <sys/filedesc.h> 240 241 #include <uvm/uvm_extern.h> 242 #include <uvm/uvm_object.h> 243 244 struct lwplist alllwp = LIST_HEAD_INITIALIZER(alllwp); 245 246 struct pool lwp_uc_pool; 247 248 static pool_cache_t lwp_cache; 249 static specificdata_domain_t lwp_specificdata_domain; 250 251 void 252 lwpinit(void) 253 { 254 255 pool_init(&lwp_uc_pool, sizeof(ucontext_t), 0, 0, 0, "lwpucpl", 256 &pool_allocator_nointr, IPL_NONE); 257 lwp_specificdata_domain = specificdata_domain_create(); 258 KASSERT(lwp_specificdata_domain != NULL); 259 lwp_sys_init(); 260 lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0, 261 "lwppl", NULL, IPL_NONE, NULL, NULL, NULL); 262 } 263 264 /* 265 * Set an suspended. 266 * 267 * Must be called with p_lock held, and the LWP locked. Will unlock the 268 * LWP before return. 269 */ 270 int 271 lwp_suspend(struct lwp *curl, struct lwp *t) 272 { 273 int error; 274 275 KASSERT(mutex_owned(t->l_proc->p_lock)); 276 KASSERT(lwp_locked(t, NULL)); 277 278 KASSERT(curl != t || curl->l_stat == LSONPROC); 279 280 /* 281 * If the current LWP has been told to exit, we must not suspend anyone 282 * else or deadlock could occur. We won't return to userspace. 283 */ 284 if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) { 285 lwp_unlock(t); 286 return (EDEADLK); 287 } 288 289 error = 0; 290 291 switch (t->l_stat) { 292 case LSRUN: 293 case LSONPROC: 294 t->l_flag |= LW_WSUSPEND; 295 lwp_need_userret(t); 296 lwp_unlock(t); 297 break; 298 299 case LSSLEEP: 300 t->l_flag |= LW_WSUSPEND; 301 302 /* 303 * Kick the LWP and try to get it to the kernel boundary 304 * so that it will release any locks that it holds. 305 * setrunnable() will release the lock. 306 */ 307 if ((t->l_flag & LW_SINTR) != 0) 308 setrunnable(t); 309 else 310 lwp_unlock(t); 311 break; 312 313 case LSSUSPENDED: 314 lwp_unlock(t); 315 break; 316 317 case LSSTOP: 318 t->l_flag |= LW_WSUSPEND; 319 setrunnable(t); 320 break; 321 322 case LSIDL: 323 case LSZOMB: 324 error = EINTR; /* It's what Solaris does..... */ 325 lwp_unlock(t); 326 break; 327 } 328 329 return (error); 330 } 331 332 /* 333 * Restart a suspended LWP. 334 * 335 * Must be called with p_lock held, and the LWP locked. Will unlock the 336 * LWP before return. 337 */ 338 void 339 lwp_continue(struct lwp *l) 340 { 341 342 KASSERT(mutex_owned(l->l_proc->p_lock)); 343 KASSERT(lwp_locked(l, NULL)); 344 345 /* If rebooting or not suspended, then just bail out. */ 346 if ((l->l_flag & LW_WREBOOT) != 0) { 347 lwp_unlock(l); 348 return; 349 } 350 351 l->l_flag &= ~LW_WSUSPEND; 352 353 if (l->l_stat != LSSUSPENDED) { 354 lwp_unlock(l); 355 return; 356 } 357 358 /* setrunnable() will release the lock. */ 359 setrunnable(l); 360 } 361 362 /* 363 * Wait for an LWP within the current process to exit. If 'lid' is 364 * non-zero, we are waiting for a specific LWP. 365 * 366 * Must be called with p->p_lock held. 367 */ 368 int 369 lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags) 370 { 371 struct proc *p = l->l_proc; 372 struct lwp *l2; 373 int nfound, error; 374 lwpid_t curlid; 375 bool exiting; 376 377 KASSERT(mutex_owned(p->p_lock)); 378 379 p->p_nlwpwait++; 380 l->l_waitingfor = lid; 381 curlid = l->l_lid; 382 exiting = ((flags & LWPWAIT_EXITCONTROL) != 0); 383 384 for (;;) { 385 /* 386 * Avoid a race between exit1() and sigexit(): if the 387 * process is dumping core, then we need to bail out: call 388 * into lwp_userret() where we will be suspended until the 389 * deed is done. 390 */ 391 if ((p->p_sflag & PS_WCORE) != 0) { 392 mutex_exit(p->p_lock); 393 lwp_userret(l); 394 #ifdef DIAGNOSTIC 395 panic("lwp_wait1"); 396 #endif 397 /* NOTREACHED */ 398 } 399 400 /* 401 * First off, drain any detached LWP that is waiting to be 402 * reaped. 403 */ 404 while ((l2 = p->p_zomblwp) != NULL) { 405 p->p_zomblwp = NULL; 406 lwp_free(l2, false, false);/* releases proc mutex */ 407 mutex_enter(p->p_lock); 408 } 409 410 /* 411 * Now look for an LWP to collect. If the whole process is 412 * exiting, count detached LWPs as eligible to be collected, 413 * but don't drain them here. 414 */ 415 nfound = 0; 416 error = 0; 417 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 418 /* 419 * If a specific wait and the target is waiting on 420 * us, then avoid deadlock. This also traps LWPs 421 * that try to wait on themselves. 422 * 423 * Note that this does not handle more complicated 424 * cycles, like: t1 -> t2 -> t3 -> t1. The process 425 * can still be killed so it is not a major problem. 426 */ 427 if (l2->l_lid == lid && l2->l_waitingfor == curlid) { 428 error = EDEADLK; 429 break; 430 } 431 if (l2 == l) 432 continue; 433 if ((l2->l_prflag & LPR_DETACHED) != 0) { 434 nfound += exiting; 435 continue; 436 } 437 if (lid != 0) { 438 if (l2->l_lid != lid) 439 continue; 440 /* 441 * Mark this LWP as the first waiter, if there 442 * is no other. 443 */ 444 if (l2->l_waiter == 0) 445 l2->l_waiter = curlid; 446 } else if (l2->l_waiter != 0) { 447 /* 448 * It already has a waiter - so don't 449 * collect it. If the waiter doesn't 450 * grab it we'll get another chance 451 * later. 452 */ 453 nfound++; 454 continue; 455 } 456 nfound++; 457 458 /* No need to lock the LWP in order to see LSZOMB. */ 459 if (l2->l_stat != LSZOMB) 460 continue; 461 462 /* 463 * We're no longer waiting. Reset the "first waiter" 464 * pointer on the target, in case it was us. 465 */ 466 l->l_waitingfor = 0; 467 l2->l_waiter = 0; 468 p->p_nlwpwait--; 469 if (departed) 470 *departed = l2->l_lid; 471 sched_lwp_collect(l2); 472 473 /* lwp_free() releases the proc lock. */ 474 lwp_free(l2, false, false); 475 mutex_enter(p->p_lock); 476 return 0; 477 } 478 479 if (error != 0) 480 break; 481 if (nfound == 0) { 482 error = ESRCH; 483 break; 484 } 485 486 /* 487 * The kernel is careful to ensure that it can not deadlock 488 * when exiting - just keep waiting. 489 */ 490 if (exiting) { 491 KASSERT(p->p_nlwps > 1); 492 cv_wait(&p->p_lwpcv, p->p_lock); 493 continue; 494 } 495 496 /* 497 * If all other LWPs are waiting for exits or suspends 498 * and the supply of zombies and potential zombies is 499 * exhausted, then we are about to deadlock. 500 * 501 * If the process is exiting (and this LWP is not the one 502 * that is coordinating the exit) then bail out now. 503 */ 504 if ((p->p_sflag & PS_WEXIT) != 0 || 505 p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) { 506 error = EDEADLK; 507 break; 508 } 509 510 /* 511 * Sit around and wait for something to happen. We'll be 512 * awoken if any of the conditions examined change: if an 513 * LWP exits, is collected, or is detached. 514 */ 515 if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0) 516 break; 517 } 518 519 /* 520 * We didn't find any LWPs to collect, we may have received a 521 * signal, or some other condition has caused us to bail out. 522 * 523 * If waiting on a specific LWP, clear the waiters marker: some 524 * other LWP may want it. Then, kick all the remaining waiters 525 * so that they can re-check for zombies and for deadlock. 526 */ 527 if (lid != 0) { 528 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 529 if (l2->l_lid == lid) { 530 if (l2->l_waiter == curlid) 531 l2->l_waiter = 0; 532 break; 533 } 534 } 535 } 536 p->p_nlwpwait--; 537 l->l_waitingfor = 0; 538 cv_broadcast(&p->p_lwpcv); 539 540 return error; 541 } 542 543 /* 544 * Create a new LWP within process 'p2', using LWP 'l1' as a template. 545 * The new LWP is created in state LSIDL and must be set running, 546 * suspended, or stopped by the caller. 547 */ 548 int 549 lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, bool inmem, int flags, 550 void *stack, size_t stacksize, void (*func)(void *), void *arg, 551 lwp_t **rnewlwpp, int sclass) 552 { 553 struct lwp *l2, *isfree; 554 turnstile_t *ts; 555 556 KASSERT(l1 == curlwp || l1->l_proc == &proc0); 557 558 /* 559 * First off, reap any detached LWP waiting to be collected. 560 * We can re-use its LWP structure and turnstile. 561 */ 562 isfree = NULL; 563 if (p2->p_zomblwp != NULL) { 564 mutex_enter(p2->p_lock); 565 if ((isfree = p2->p_zomblwp) != NULL) { 566 p2->p_zomblwp = NULL; 567 lwp_free(isfree, true, false);/* releases proc mutex */ 568 } else 569 mutex_exit(p2->p_lock); 570 } 571 if (isfree == NULL) { 572 l2 = pool_cache_get(lwp_cache, PR_WAITOK); 573 memset(l2, 0, sizeof(*l2)); 574 l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK); 575 SLIST_INIT(&l2->l_pi_lenders); 576 } else { 577 l2 = isfree; 578 ts = l2->l_ts; 579 KASSERT(l2->l_inheritedprio == -1); 580 KASSERT(SLIST_EMPTY(&l2->l_pi_lenders)); 581 memset(l2, 0, sizeof(*l2)); 582 l2->l_ts = ts; 583 } 584 585 l2->l_stat = LSIDL; 586 l2->l_proc = p2; 587 l2->l_refcnt = 1; 588 l2->l_class = sclass; 589 590 /* 591 * If vfork(), we want the LWP to run fast and on the same CPU 592 * as its parent, so that it can reuse the VM context and cache 593 * footprint on the local CPU. 594 */ 595 l2->l_kpriority = ((flags & LWP_VFORK) ? true : false); 596 l2->l_kpribase = PRI_KERNEL; 597 l2->l_priority = l1->l_priority; 598 l2->l_inheritedprio = -1; 599 l2->l_flag = inmem ? LW_INMEM : 0; 600 l2->l_pflag = LP_MPSAFE; 601 TAILQ_INIT(&l2->l_ld_locks); 602 603 /* 604 * If not the first LWP in the process, grab a reference to the 605 * descriptor table. 606 */ 607 l2->l_fd = p2->p_fd; 608 if (p2->p_nlwps != 0) { 609 KASSERT(l1->l_proc == p2); 610 atomic_inc_uint(&l2->l_fd->fd_refcnt); 611 } else { 612 KASSERT(l1->l_proc != p2); 613 } 614 615 if (p2->p_flag & PK_SYSTEM) { 616 /* Mark it as a system LWP and not a candidate for swapping */ 617 l2->l_flag |= LW_SYSTEM; 618 } 619 620 kpreempt_disable(); 621 l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex; 622 l2->l_cpu = l1->l_cpu; 623 kpreempt_enable(); 624 625 lwp_initspecific(l2); 626 sched_lwp_fork(l1, l2); 627 lwp_update_creds(l2); 628 callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE); 629 callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2); 630 mutex_init(&l2->l_swaplock, MUTEX_DEFAULT, IPL_NONE); 631 cv_init(&l2->l_sigcv, "sigwait"); 632 l2->l_syncobj = &sched_syncobj; 633 634 if (rnewlwpp != NULL) 635 *rnewlwpp = l2; 636 637 l2->l_addr = UAREA_TO_USER(uaddr); 638 uvm_lwp_fork(l1, l2, stack, stacksize, func, 639 (arg != NULL) ? arg : l2); 640 641 mutex_enter(p2->p_lock); 642 643 if ((flags & LWP_DETACHED) != 0) { 644 l2->l_prflag = LPR_DETACHED; 645 p2->p_ndlwps++; 646 } else 647 l2->l_prflag = 0; 648 649 l2->l_sigmask = l1->l_sigmask; 650 CIRCLEQ_INIT(&l2->l_sigpend.sp_info); 651 sigemptyset(&l2->l_sigpend.sp_set); 652 653 p2->p_nlwpid++; 654 if (p2->p_nlwpid == 0) 655 p2->p_nlwpid++; 656 l2->l_lid = p2->p_nlwpid; 657 LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling); 658 p2->p_nlwps++; 659 660 if ((p2->p_flag & PK_SYSTEM) == 0) { 661 /* Inherit an affinity */ 662 if (l1->l_flag & LW_AFFINITY) { 663 /* 664 * Note that we hold the state lock while inheriting 665 * the affinity to avoid race with sched_setaffinity(). 666 */ 667 lwp_lock(l1); 668 if (l1->l_flag & LW_AFFINITY) { 669 kcpuset_use(l1->l_affinity); 670 l2->l_affinity = l1->l_affinity; 671 l2->l_flag |= LW_AFFINITY; 672 } 673 lwp_unlock(l1); 674 } 675 lwp_lock(l2); 676 /* Inherit a processor-set */ 677 l2->l_psid = l1->l_psid; 678 /* Look for a CPU to start */ 679 l2->l_cpu = sched_takecpu(l2); 680 lwp_unlock_to(l2, l2->l_cpu->ci_schedstate.spc_mutex); 681 } 682 mutex_exit(p2->p_lock); 683 684 mutex_enter(proc_lock); 685 LIST_INSERT_HEAD(&alllwp, l2, l_list); 686 mutex_exit(proc_lock); 687 688 SYSCALL_TIME_LWP_INIT(l2); 689 690 if (p2->p_emul->e_lwp_fork) 691 (*p2->p_emul->e_lwp_fork)(l1, l2); 692 693 return (0); 694 } 695 696 /* 697 * Called by MD code when a new LWP begins execution. Must be called 698 * with the previous LWP locked (so at splsched), or if there is no 699 * previous LWP, at splsched. 700 */ 701 void 702 lwp_startup(struct lwp *prev, struct lwp *new) 703 { 704 705 KASSERT(kpreempt_disabled()); 706 if (prev != NULL) { 707 /* 708 * Normalize the count of the spin-mutexes, it was 709 * increased in mi_switch(). Unmark the state of 710 * context switch - it is finished for previous LWP. 711 */ 712 curcpu()->ci_mtx_count++; 713 membar_exit(); 714 prev->l_ctxswtch = 0; 715 } 716 KPREEMPT_DISABLE(new); 717 spl0(); 718 pmap_activate(new); 719 LOCKDEBUG_BARRIER(NULL, 0); 720 KPREEMPT_ENABLE(new); 721 if ((new->l_pflag & LP_MPSAFE) == 0) { 722 KERNEL_LOCK(1, new); 723 } 724 } 725 726 /* 727 * Exit an LWP. 728 */ 729 void 730 lwp_exit(struct lwp *l) 731 { 732 struct proc *p = l->l_proc; 733 struct lwp *l2; 734 bool current; 735 736 current = (l == curlwp); 737 738 KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL)); 739 KASSERT(p == curproc); 740 741 /* 742 * Verify that we hold no locks other than the kernel lock. 743 */ 744 LOCKDEBUG_BARRIER(&kernel_lock, 0); 745 746 /* 747 * If we are the last live LWP in a process, we need to exit the 748 * entire process. We do so with an exit status of zero, because 749 * it's a "controlled" exit, and because that's what Solaris does. 750 * 751 * We are not quite a zombie yet, but for accounting purposes we 752 * must increment the count of zombies here. 753 * 754 * Note: the last LWP's specificdata will be deleted here. 755 */ 756 mutex_enter(p->p_lock); 757 if (p->p_nlwps - p->p_nzlwps == 1) { 758 KASSERT(current == true); 759 /* XXXSMP kernel_lock not held */ 760 exit1(l, 0); 761 /* NOTREACHED */ 762 } 763 p->p_nzlwps++; 764 mutex_exit(p->p_lock); 765 766 if (p->p_emul->e_lwp_exit) 767 (*p->p_emul->e_lwp_exit)(l); 768 769 /* Drop filedesc reference. */ 770 fd_free(); 771 772 /* Delete the specificdata while it's still safe to sleep. */ 773 specificdata_fini(lwp_specificdata_domain, &l->l_specdataref); 774 775 /* 776 * Release our cached credentials. 777 */ 778 kauth_cred_free(l->l_cred); 779 callout_destroy(&l->l_timeout_ch); 780 781 /* 782 * While we can still block, mark the LWP as unswappable to 783 * prevent conflicts with the with the swapper. 784 */ 785 if (current) 786 uvm_lwp_hold(l); 787 788 /* 789 * Remove the LWP from the global list. 790 */ 791 mutex_enter(proc_lock); 792 LIST_REMOVE(l, l_list); 793 mutex_exit(proc_lock); 794 795 /* 796 * Get rid of all references to the LWP that others (e.g. procfs) 797 * may have, and mark the LWP as a zombie. If the LWP is detached, 798 * mark it waiting for collection in the proc structure. Note that 799 * before we can do that, we need to free any other dead, deatched 800 * LWP waiting to meet its maker. 801 */ 802 mutex_enter(p->p_lock); 803 lwp_drainrefs(l); 804 805 if ((l->l_prflag & LPR_DETACHED) != 0) { 806 while ((l2 = p->p_zomblwp) != NULL) { 807 p->p_zomblwp = NULL; 808 lwp_free(l2, false, false);/* releases proc mutex */ 809 mutex_enter(p->p_lock); 810 l->l_refcnt++; 811 lwp_drainrefs(l); 812 } 813 p->p_zomblwp = l; 814 } 815 816 /* 817 * If we find a pending signal for the process and we have been 818 * asked to check for signals, then we loose: arrange to have 819 * all other LWPs in the process check for signals. 820 */ 821 if ((l->l_flag & LW_PENDSIG) != 0 && 822 firstsig(&p->p_sigpend.sp_set) != 0) { 823 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 824 lwp_lock(l2); 825 l2->l_flag |= LW_PENDSIG; 826 lwp_unlock(l2); 827 } 828 } 829 830 lwp_lock(l); 831 l->l_stat = LSZOMB; 832 if (l->l_name != NULL) 833 strcpy(l->l_name, "(zombie)"); 834 if (l->l_flag & LW_AFFINITY) { 835 l->l_flag &= ~LW_AFFINITY; 836 } else { 837 KASSERT(l->l_affinity == NULL); 838 } 839 lwp_unlock(l); 840 p->p_nrlwps--; 841 cv_broadcast(&p->p_lwpcv); 842 if (l->l_lwpctl != NULL) 843 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED; 844 mutex_exit(p->p_lock); 845 846 /* Safe without lock since LWP is in zombie state */ 847 if (l->l_affinity) { 848 kcpuset_unuse(l->l_affinity, NULL); 849 l->l_affinity = NULL; 850 } 851 852 /* 853 * We can no longer block. At this point, lwp_free() may already 854 * be gunning for us. On a multi-CPU system, we may be off p_lwps. 855 * 856 * Free MD LWP resources. 857 */ 858 #ifndef __NO_CPU_LWP_FREE 859 cpu_lwp_free(l, 0); 860 #endif 861 862 if (current) { 863 pmap_deactivate(l); 864 865 /* 866 * Release the kernel lock, and switch away into 867 * oblivion. 868 */ 869 #ifdef notyet 870 /* XXXSMP hold in lwp_userret() */ 871 KERNEL_UNLOCK_LAST(l); 872 #else 873 KERNEL_UNLOCK_ALL(l, NULL); 874 #endif 875 lwp_exit_switchaway(l); 876 } 877 } 878 879 /* 880 * Free a dead LWP's remaining resources. 881 * 882 * XXXLWP limits. 883 */ 884 void 885 lwp_free(struct lwp *l, bool recycle, bool last) 886 { 887 struct proc *p = l->l_proc; 888 struct rusage *ru; 889 ksiginfoq_t kq; 890 891 KASSERT(l != curlwp); 892 893 /* 894 * If this was not the last LWP in the process, then adjust 895 * counters and unlock. 896 */ 897 if (!last) { 898 /* 899 * Add the LWP's run time to the process' base value. 900 * This needs to co-incide with coming off p_lwps. 901 */ 902 bintime_add(&p->p_rtime, &l->l_rtime); 903 p->p_pctcpu += l->l_pctcpu; 904 ru = &p->p_stats->p_ru; 905 ruadd(ru, &l->l_ru); 906 ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw); 907 ru->ru_nivcsw += l->l_nivcsw; 908 LIST_REMOVE(l, l_sibling); 909 p->p_nlwps--; 910 p->p_nzlwps--; 911 if ((l->l_prflag & LPR_DETACHED) != 0) 912 p->p_ndlwps--; 913 914 /* 915 * Have any LWPs sleeping in lwp_wait() recheck for 916 * deadlock. 917 */ 918 cv_broadcast(&p->p_lwpcv); 919 mutex_exit(p->p_lock); 920 } 921 922 #ifdef MULTIPROCESSOR 923 /* 924 * In the unlikely event that the LWP is still on the CPU, 925 * then spin until it has switched away. We need to release 926 * all locks to avoid deadlock against interrupt handlers on 927 * the target CPU. 928 */ 929 if ((l->l_pflag & LP_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) { 930 int count; 931 (void)count; /* XXXgcc */ 932 KERNEL_UNLOCK_ALL(curlwp, &count); 933 while ((l->l_pflag & LP_RUNNING) != 0 || 934 l->l_cpu->ci_curlwp == l) 935 SPINLOCK_BACKOFF_HOOK; 936 KERNEL_LOCK(count, curlwp); 937 } 938 #endif 939 940 /* 941 * Destroy the LWP's remaining signal information. 942 */ 943 ksiginfo_queue_init(&kq); 944 sigclear(&l->l_sigpend, NULL, &kq); 945 ksiginfo_queue_drain(&kq); 946 cv_destroy(&l->l_sigcv); 947 mutex_destroy(&l->l_swaplock); 948 949 /* 950 * Free the LWP's turnstile and the LWP structure itself unless the 951 * caller wants to recycle them. Also, free the scheduler specific 952 * data. 953 * 954 * We can't return turnstile0 to the pool (it didn't come from it), 955 * so if it comes up just drop it quietly and move on. 956 * 957 * We don't recycle the VM resources at this time. 958 */ 959 if (l->l_lwpctl != NULL) 960 lwp_ctl_free(l); 961 962 if (!recycle && l->l_ts != &turnstile0) 963 pool_cache_put(turnstile_cache, l->l_ts); 964 if (l->l_name != NULL) 965 kmem_free(l->l_name, MAXCOMLEN); 966 #ifndef __NO_CPU_LWP_FREE 967 cpu_lwp_free2(l); 968 #endif 969 KASSERT((l->l_flag & LW_INMEM) != 0); 970 uvm_lwp_exit(l); 971 KASSERT(SLIST_EMPTY(&l->l_pi_lenders)); 972 KASSERT(l->l_inheritedprio == -1); 973 if (!recycle) 974 pool_cache_put(lwp_cache, l); 975 } 976 977 /* 978 * Migrate the LWP to the another CPU. Unlocks the LWP. 979 */ 980 void 981 lwp_migrate(lwp_t *l, struct cpu_info *tci) 982 { 983 struct schedstate_percpu *tspc; 984 int lstat = l->l_stat; 985 986 KASSERT(lwp_locked(l, NULL)); 987 KASSERT(tci != NULL); 988 989 /* If LWP is still on the CPU, it must be handled like LSONPROC */ 990 if ((l->l_pflag & LP_RUNNING) != 0) { 991 lstat = LSONPROC; 992 } 993 994 /* 995 * The destination CPU could be changed while previous migration 996 * was not finished. 997 */ 998 if (l->l_target_cpu != NULL) { 999 l->l_target_cpu = tci; 1000 lwp_unlock(l); 1001 return; 1002 } 1003 1004 /* Nothing to do if trying to migrate to the same CPU */ 1005 if (l->l_cpu == tci) { 1006 lwp_unlock(l); 1007 return; 1008 } 1009 1010 KASSERT(l->l_target_cpu == NULL); 1011 tspc = &tci->ci_schedstate; 1012 switch (lstat) { 1013 case LSRUN: 1014 if (l->l_flag & LW_INMEM) { 1015 l->l_target_cpu = tci; 1016 lwp_unlock(l); 1017 return; 1018 } 1019 case LSIDL: 1020 l->l_cpu = tci; 1021 lwp_unlock_to(l, tspc->spc_mutex); 1022 return; 1023 case LSSLEEP: 1024 l->l_cpu = tci; 1025 break; 1026 case LSSTOP: 1027 case LSSUSPENDED: 1028 l->l_cpu = tci; 1029 if (l->l_wchan == NULL) { 1030 lwp_unlock_to(l, tspc->spc_lwplock); 1031 return; 1032 } 1033 break; 1034 case LSONPROC: 1035 l->l_target_cpu = tci; 1036 spc_lock(l->l_cpu); 1037 cpu_need_resched(l->l_cpu, RESCHED_KPREEMPT); 1038 spc_unlock(l->l_cpu); 1039 break; 1040 } 1041 lwp_unlock(l); 1042 } 1043 1044 /* 1045 * Find the LWP in the process. Arguments may be zero, in such case, 1046 * the calling process and first LWP in the list will be used. 1047 * On success - returns proc locked. 1048 */ 1049 struct lwp * 1050 lwp_find2(pid_t pid, lwpid_t lid) 1051 { 1052 proc_t *p; 1053 lwp_t *l; 1054 1055 /* Find the process */ 1056 p = (pid == 0) ? curlwp->l_proc : p_find(pid, PFIND_UNLOCK_FAIL); 1057 if (p == NULL) 1058 return NULL; 1059 mutex_enter(p->p_lock); 1060 if (pid != 0) { 1061 /* Case of p_find */ 1062 mutex_exit(proc_lock); 1063 } 1064 1065 /* Find the thread */ 1066 l = (lid == 0) ? LIST_FIRST(&p->p_lwps) : lwp_find(p, lid); 1067 if (l == NULL) { 1068 mutex_exit(p->p_lock); 1069 } 1070 1071 return l; 1072 } 1073 1074 /* 1075 * Look up a live LWP within the speicifed process, and return it locked. 1076 * 1077 * Must be called with p->p_lock held. 1078 */ 1079 struct lwp * 1080 lwp_find(struct proc *p, int id) 1081 { 1082 struct lwp *l; 1083 1084 KASSERT(mutex_owned(p->p_lock)); 1085 1086 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1087 if (l->l_lid == id) 1088 break; 1089 } 1090 1091 /* 1092 * No need to lock - all of these conditions will 1093 * be visible with the process level mutex held. 1094 */ 1095 if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB)) 1096 l = NULL; 1097 1098 return l; 1099 } 1100 1101 /* 1102 * Update an LWP's cached credentials to mirror the process' master copy. 1103 * 1104 * This happens early in the syscall path, on user trap, and on LWP 1105 * creation. A long-running LWP can also voluntarily choose to update 1106 * it's credentials by calling this routine. This may be called from 1107 * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand. 1108 */ 1109 void 1110 lwp_update_creds(struct lwp *l) 1111 { 1112 kauth_cred_t oc; 1113 struct proc *p; 1114 1115 p = l->l_proc; 1116 oc = l->l_cred; 1117 1118 mutex_enter(p->p_lock); 1119 kauth_cred_hold(p->p_cred); 1120 l->l_cred = p->p_cred; 1121 l->l_prflag &= ~LPR_CRMOD; 1122 mutex_exit(p->p_lock); 1123 if (oc != NULL) 1124 kauth_cred_free(oc); 1125 } 1126 1127 /* 1128 * Verify that an LWP is locked, and optionally verify that the lock matches 1129 * one we specify. 1130 */ 1131 int 1132 lwp_locked(struct lwp *l, kmutex_t *mtx) 1133 { 1134 kmutex_t *cur = l->l_mutex; 1135 1136 return mutex_owned(cur) && (mtx == cur || mtx == NULL); 1137 } 1138 1139 /* 1140 * Lock an LWP. 1141 */ 1142 kmutex_t * 1143 lwp_lock_retry(struct lwp *l, kmutex_t *old) 1144 { 1145 1146 /* 1147 * XXXgcc ignoring kmutex_t * volatile on i386 1148 * 1149 * gcc version 4.1.2 20061021 prerelease (NetBSD nb1 20061021) 1150 */ 1151 #if 1 1152 while (l->l_mutex != old) { 1153 #else 1154 for (;;) { 1155 #endif 1156 mutex_spin_exit(old); 1157 old = l->l_mutex; 1158 mutex_spin_enter(old); 1159 1160 /* 1161 * mutex_enter() will have posted a read barrier. Re-test 1162 * l->l_mutex. If it has changed, we need to try again. 1163 */ 1164 #if 1 1165 } 1166 #else 1167 } while (__predict_false(l->l_mutex != old)); 1168 #endif 1169 1170 return old; 1171 } 1172 1173 /* 1174 * Lend a new mutex to an LWP. The old mutex must be held. 1175 */ 1176 void 1177 lwp_setlock(struct lwp *l, kmutex_t *new) 1178 { 1179 1180 KASSERT(mutex_owned(l->l_mutex)); 1181 1182 membar_exit(); 1183 l->l_mutex = new; 1184 } 1185 1186 /* 1187 * Lend a new mutex to an LWP, and release the old mutex. The old mutex 1188 * must be held. 1189 */ 1190 void 1191 lwp_unlock_to(struct lwp *l, kmutex_t *new) 1192 { 1193 kmutex_t *old; 1194 1195 KASSERT(mutex_owned(l->l_mutex)); 1196 1197 old = l->l_mutex; 1198 membar_exit(); 1199 l->l_mutex = new; 1200 mutex_spin_exit(old); 1201 } 1202 1203 /* 1204 * Acquire a new mutex, and donate it to an LWP. The LWP must already be 1205 * locked. 1206 */ 1207 void 1208 lwp_relock(struct lwp *l, kmutex_t *new) 1209 { 1210 kmutex_t *old; 1211 1212 KASSERT(mutex_owned(l->l_mutex)); 1213 1214 old = l->l_mutex; 1215 if (old != new) { 1216 mutex_spin_enter(new); 1217 l->l_mutex = new; 1218 mutex_spin_exit(old); 1219 } 1220 } 1221 1222 int 1223 lwp_trylock(struct lwp *l) 1224 { 1225 kmutex_t *old; 1226 1227 for (;;) { 1228 if (!mutex_tryenter(old = l->l_mutex)) 1229 return 0; 1230 if (__predict_true(l->l_mutex == old)) 1231 return 1; 1232 mutex_spin_exit(old); 1233 } 1234 } 1235 1236 u_int 1237 lwp_unsleep(lwp_t *l, bool cleanup) 1238 { 1239 1240 KASSERT(mutex_owned(l->l_mutex)); 1241 1242 return (*l->l_syncobj->sobj_unsleep)(l, cleanup); 1243 } 1244 1245 1246 /* 1247 * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is 1248 * set. 1249 */ 1250 void 1251 lwp_userret(struct lwp *l) 1252 { 1253 struct proc *p; 1254 void (*hook)(void); 1255 int sig; 1256 1257 KASSERT(l == curlwp); 1258 KASSERT(l->l_stat == LSONPROC); 1259 p = l->l_proc; 1260 1261 #ifndef __HAVE_FAST_SOFTINTS 1262 /* Run pending soft interrupts. */ 1263 if (l->l_cpu->ci_data.cpu_softints != 0) 1264 softint_overlay(); 1265 #endif 1266 1267 #ifdef KERN_SA 1268 /* Generate UNBLOCKED upcall if needed */ 1269 if (l->l_flag & LW_SA_BLOCKING) { 1270 sa_unblock_userret(l); 1271 /* NOTREACHED */ 1272 } 1273 #endif 1274 1275 /* 1276 * It should be safe to do this read unlocked on a multiprocessor 1277 * system.. 1278 * 1279 * LW_SA_UPCALL will be handled after the while() loop, so don't 1280 * consider it now. 1281 */ 1282 while ((l->l_flag & (LW_USERRET & ~(LW_SA_UPCALL))) != 0) { 1283 /* 1284 * Process pending signals first, unless the process 1285 * is dumping core or exiting, where we will instead 1286 * enter the LW_WSUSPEND case below. 1287 */ 1288 if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) == 1289 LW_PENDSIG) { 1290 mutex_enter(p->p_lock); 1291 while ((sig = issignal(l)) != 0) 1292 postsig(sig); 1293 mutex_exit(p->p_lock); 1294 } 1295 1296 /* 1297 * Core-dump or suspend pending. 1298 * 1299 * In case of core dump, suspend ourselves, so that the 1300 * kernel stack and therefore the userland registers saved 1301 * in the trapframe are around for coredump() to write them 1302 * out. We issue a wakeup on p->p_lwpcv so that sigexit() 1303 * will write the core file out once all other LWPs are 1304 * suspended. 1305 */ 1306 if ((l->l_flag & LW_WSUSPEND) != 0) { 1307 mutex_enter(p->p_lock); 1308 p->p_nrlwps--; 1309 cv_broadcast(&p->p_lwpcv); 1310 lwp_lock(l); 1311 l->l_stat = LSSUSPENDED; 1312 lwp_unlock(l); 1313 mutex_exit(p->p_lock); 1314 lwp_lock(l); 1315 mi_switch(l); 1316 } 1317 1318 /* Process is exiting. */ 1319 if ((l->l_flag & LW_WEXIT) != 0) { 1320 lwp_exit(l); 1321 KASSERT(0); 1322 /* NOTREACHED */ 1323 } 1324 1325 /* Call userret hook; used by Linux emulation. */ 1326 if ((l->l_flag & LW_WUSERRET) != 0) { 1327 lwp_lock(l); 1328 l->l_flag &= ~LW_WUSERRET; 1329 lwp_unlock(l); 1330 hook = p->p_userret; 1331 p->p_userret = NULL; 1332 (*hook)(); 1333 } 1334 } 1335 1336 #ifdef KERN_SA 1337 /* 1338 * Timer events are handled specially. We only try once to deliver 1339 * pending timer upcalls; if if fails, we can try again on the next 1340 * loop around. If we need to re-enter lwp_userret(), MD code will 1341 * bounce us back here through the trap path after we return. 1342 */ 1343 if (p->p_timerpend) 1344 timerupcall(l); 1345 if (l->l_flag & LW_SA_UPCALL) 1346 sa_upcall_userret(l); 1347 #endif /* KERN_SA */ 1348 } 1349 1350 /* 1351 * Force an LWP to enter the kernel, to take a trip through lwp_userret(). 1352 */ 1353 void 1354 lwp_need_userret(struct lwp *l) 1355 { 1356 KASSERT(lwp_locked(l, NULL)); 1357 1358 /* 1359 * Since the tests in lwp_userret() are done unlocked, make sure 1360 * that the condition will be seen before forcing the LWP to enter 1361 * kernel mode. 1362 */ 1363 membar_producer(); 1364 cpu_signotify(l); 1365 } 1366 1367 /* 1368 * Add one reference to an LWP. This will prevent the LWP from 1369 * exiting, thus keep the lwp structure and PCB around to inspect. 1370 */ 1371 void 1372 lwp_addref(struct lwp *l) 1373 { 1374 1375 KASSERT(mutex_owned(l->l_proc->p_lock)); 1376 KASSERT(l->l_stat != LSZOMB); 1377 KASSERT(l->l_refcnt != 0); 1378 1379 l->l_refcnt++; 1380 } 1381 1382 /* 1383 * Remove one reference to an LWP. If this is the last reference, 1384 * then we must finalize the LWP's death. 1385 */ 1386 void 1387 lwp_delref(struct lwp *l) 1388 { 1389 struct proc *p = l->l_proc; 1390 1391 mutex_enter(p->p_lock); 1392 KASSERT(l->l_stat != LSZOMB); 1393 KASSERT(l->l_refcnt > 0); 1394 if (--l->l_refcnt == 0) 1395 cv_broadcast(&p->p_lwpcv); 1396 mutex_exit(p->p_lock); 1397 } 1398 1399 /* 1400 * Drain all references to the current LWP. 1401 */ 1402 void 1403 lwp_drainrefs(struct lwp *l) 1404 { 1405 struct proc *p = l->l_proc; 1406 1407 KASSERT(mutex_owned(p->p_lock)); 1408 KASSERT(l->l_refcnt != 0); 1409 1410 l->l_refcnt--; 1411 while (l->l_refcnt != 0) 1412 cv_wait(&p->p_lwpcv, p->p_lock); 1413 } 1414 1415 /* 1416 * Return true if the specified LWP is 'alive'. Only p->p_lock need 1417 * be held. 1418 */ 1419 bool 1420 lwp_alive(lwp_t *l) 1421 { 1422 1423 KASSERT(mutex_owned(l->l_proc->p_lock)); 1424 1425 switch (l->l_stat) { 1426 case LSSLEEP: 1427 case LSRUN: 1428 case LSONPROC: 1429 case LSSTOP: 1430 case LSSUSPENDED: 1431 return true; 1432 default: 1433 return false; 1434 } 1435 } 1436 1437 /* 1438 * Return first live LWP in the process. 1439 */ 1440 lwp_t * 1441 lwp_find_first(proc_t *p) 1442 { 1443 lwp_t *l; 1444 1445 KASSERT(mutex_owned(p->p_lock)); 1446 1447 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1448 if (lwp_alive(l)) { 1449 return l; 1450 } 1451 } 1452 1453 return NULL; 1454 } 1455 1456 /* 1457 * lwp_specific_key_create -- 1458 * Create a key for subsystem lwp-specific data. 1459 */ 1460 int 1461 lwp_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor) 1462 { 1463 1464 return (specificdata_key_create(lwp_specificdata_domain, keyp, dtor)); 1465 } 1466 1467 /* 1468 * lwp_specific_key_delete -- 1469 * Delete a key for subsystem lwp-specific data. 1470 */ 1471 void 1472 lwp_specific_key_delete(specificdata_key_t key) 1473 { 1474 1475 specificdata_key_delete(lwp_specificdata_domain, key); 1476 } 1477 1478 /* 1479 * lwp_initspecific -- 1480 * Initialize an LWP's specificdata container. 1481 */ 1482 void 1483 lwp_initspecific(struct lwp *l) 1484 { 1485 int error; 1486 1487 error = specificdata_init(lwp_specificdata_domain, &l->l_specdataref); 1488 KASSERT(error == 0); 1489 } 1490 1491 /* 1492 * lwp_finispecific -- 1493 * Finalize an LWP's specificdata container. 1494 */ 1495 void 1496 lwp_finispecific(struct lwp *l) 1497 { 1498 1499 specificdata_fini(lwp_specificdata_domain, &l->l_specdataref); 1500 } 1501 1502 /* 1503 * lwp_getspecific -- 1504 * Return lwp-specific data corresponding to the specified key. 1505 * 1506 * Note: LWP specific data is NOT INTERLOCKED. An LWP should access 1507 * only its OWN SPECIFIC DATA. If it is necessary to access another 1508 * LWP's specifc data, care must be taken to ensure that doing so 1509 * would not cause internal data structure inconsistency (i.e. caller 1510 * can guarantee that the target LWP is not inside an lwp_getspecific() 1511 * or lwp_setspecific() call). 1512 */ 1513 void * 1514 lwp_getspecific(specificdata_key_t key) 1515 { 1516 1517 return (specificdata_getspecific_unlocked(lwp_specificdata_domain, 1518 &curlwp->l_specdataref, key)); 1519 } 1520 1521 void * 1522 _lwp_getspecific_by_lwp(struct lwp *l, specificdata_key_t key) 1523 { 1524 1525 return (specificdata_getspecific_unlocked(lwp_specificdata_domain, 1526 &l->l_specdataref, key)); 1527 } 1528 1529 /* 1530 * lwp_setspecific -- 1531 * Set lwp-specific data corresponding to the specified key. 1532 */ 1533 void 1534 lwp_setspecific(specificdata_key_t key, void *data) 1535 { 1536 1537 specificdata_setspecific(lwp_specificdata_domain, 1538 &curlwp->l_specdataref, key, data); 1539 } 1540 1541 /* 1542 * Allocate a new lwpctl structure for a user LWP. 1543 */ 1544 int 1545 lwp_ctl_alloc(vaddr_t *uaddr) 1546 { 1547 lcproc_t *lp; 1548 u_int bit, i, offset; 1549 struct uvm_object *uao; 1550 int error; 1551 lcpage_t *lcp; 1552 proc_t *p; 1553 lwp_t *l; 1554 1555 l = curlwp; 1556 p = l->l_proc; 1557 1558 if (l->l_lcpage != NULL) { 1559 lcp = l->l_lcpage; 1560 *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr; 1561 return (EINVAL); 1562 } 1563 1564 /* First time around, allocate header structure for the process. */ 1565 if ((lp = p->p_lwpctl) == NULL) { 1566 lp = kmem_alloc(sizeof(*lp), KM_SLEEP); 1567 mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE); 1568 lp->lp_uao = NULL; 1569 TAILQ_INIT(&lp->lp_pages); 1570 mutex_enter(p->p_lock); 1571 if (p->p_lwpctl == NULL) { 1572 p->p_lwpctl = lp; 1573 mutex_exit(p->p_lock); 1574 } else { 1575 mutex_exit(p->p_lock); 1576 mutex_destroy(&lp->lp_lock); 1577 kmem_free(lp, sizeof(*lp)); 1578 lp = p->p_lwpctl; 1579 } 1580 } 1581 1582 /* 1583 * Set up an anonymous memory region to hold the shared pages. 1584 * Map them into the process' address space. The user vmspace 1585 * gets the first reference on the UAO. 1586 */ 1587 mutex_enter(&lp->lp_lock); 1588 if (lp->lp_uao == NULL) { 1589 lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0); 1590 lp->lp_cur = 0; 1591 lp->lp_max = LWPCTL_UAREA_SZ; 1592 lp->lp_uva = p->p_emul->e_vm_default_addr(p, 1593 (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ); 1594 error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva, 1595 LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW, 1596 UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0)); 1597 if (error != 0) { 1598 uao_detach(lp->lp_uao); 1599 lp->lp_uao = NULL; 1600 mutex_exit(&lp->lp_lock); 1601 return error; 1602 } 1603 } 1604 1605 /* Get a free block and allocate for this LWP. */ 1606 TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) { 1607 if (lcp->lcp_nfree != 0) 1608 break; 1609 } 1610 if (lcp == NULL) { 1611 /* Nothing available - try to set up a free page. */ 1612 if (lp->lp_cur == lp->lp_max) { 1613 mutex_exit(&lp->lp_lock); 1614 return ENOMEM; 1615 } 1616 lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP); 1617 if (lcp == NULL) { 1618 mutex_exit(&lp->lp_lock); 1619 return ENOMEM; 1620 } 1621 /* 1622 * Wire the next page down in kernel space. Since this 1623 * is a new mapping, we must add a reference. 1624 */ 1625 uao = lp->lp_uao; 1626 (*uao->pgops->pgo_reference)(uao); 1627 lcp->lcp_kaddr = vm_map_min(kernel_map); 1628 error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE, 1629 uao, lp->lp_cur, PAGE_SIZE, 1630 UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, 1631 UVM_INH_NONE, UVM_ADV_RANDOM, 0)); 1632 if (error != 0) { 1633 mutex_exit(&lp->lp_lock); 1634 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1635 (*uao->pgops->pgo_detach)(uao); 1636 return error; 1637 } 1638 error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr, 1639 lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0); 1640 if (error != 0) { 1641 mutex_exit(&lp->lp_lock); 1642 uvm_unmap(kernel_map, lcp->lcp_kaddr, 1643 lcp->lcp_kaddr + PAGE_SIZE); 1644 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1645 return error; 1646 } 1647 /* Prepare the page descriptor and link into the list. */ 1648 lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur; 1649 lp->lp_cur += PAGE_SIZE; 1650 lcp->lcp_nfree = LWPCTL_PER_PAGE; 1651 lcp->lcp_rotor = 0; 1652 memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ); 1653 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain); 1654 } 1655 for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) { 1656 if (++i >= LWPCTL_BITMAP_ENTRIES) 1657 i = 0; 1658 } 1659 bit = ffs(lcp->lcp_bitmap[i]) - 1; 1660 lcp->lcp_bitmap[i] ^= (1 << bit); 1661 lcp->lcp_rotor = i; 1662 lcp->lcp_nfree--; 1663 l->l_lcpage = lcp; 1664 offset = (i << 5) + bit; 1665 l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset; 1666 *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t); 1667 mutex_exit(&lp->lp_lock); 1668 1669 KPREEMPT_DISABLE(l); 1670 l->l_lwpctl->lc_curcpu = (int)curcpu()->ci_data.cpu_index; 1671 KPREEMPT_ENABLE(l); 1672 1673 return 0; 1674 } 1675 1676 /* 1677 * Free an lwpctl structure back to the per-process list. 1678 */ 1679 void 1680 lwp_ctl_free(lwp_t *l) 1681 { 1682 lcproc_t *lp; 1683 lcpage_t *lcp; 1684 u_int map, offset; 1685 1686 lp = l->l_proc->p_lwpctl; 1687 KASSERT(lp != NULL); 1688 1689 lcp = l->l_lcpage; 1690 offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr); 1691 KASSERT(offset < LWPCTL_PER_PAGE); 1692 1693 mutex_enter(&lp->lp_lock); 1694 lcp->lcp_nfree++; 1695 map = offset >> 5; 1696 lcp->lcp_bitmap[map] |= (1 << (offset & 31)); 1697 if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0) 1698 lcp->lcp_rotor = map; 1699 if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) { 1700 TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain); 1701 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain); 1702 } 1703 mutex_exit(&lp->lp_lock); 1704 } 1705 1706 /* 1707 * Process is exiting; tear down lwpctl state. This can only be safely 1708 * called by the last LWP in the process. 1709 */ 1710 void 1711 lwp_ctl_exit(void) 1712 { 1713 lcpage_t *lcp, *next; 1714 lcproc_t *lp; 1715 proc_t *p; 1716 lwp_t *l; 1717 1718 l = curlwp; 1719 l->l_lwpctl = NULL; 1720 l->l_lcpage = NULL; 1721 p = l->l_proc; 1722 lp = p->p_lwpctl; 1723 1724 KASSERT(lp != NULL); 1725 KASSERT(p->p_nlwps == 1); 1726 1727 for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) { 1728 next = TAILQ_NEXT(lcp, lcp_chain); 1729 uvm_unmap(kernel_map, lcp->lcp_kaddr, 1730 lcp->lcp_kaddr + PAGE_SIZE); 1731 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1732 } 1733 1734 if (lp->lp_uao != NULL) { 1735 uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva, 1736 lp->lp_uva + LWPCTL_UAREA_SZ); 1737 } 1738 1739 mutex_destroy(&lp->lp_lock); 1740 kmem_free(lp, sizeof(*lp)); 1741 p->p_lwpctl = NULL; 1742 } 1743 1744 /* 1745 * Return the current LWP's "preemption counter". Used to detect 1746 * preemption across operations that can tolerate preemption without 1747 * crashing, but which may generate incorrect results if preempted. 1748 */ 1749 uint64_t 1750 lwp_pctr(void) 1751 { 1752 1753 return curlwp->l_ncsw; 1754 } 1755 1756 #if defined(DDB) 1757 void 1758 lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...)) 1759 { 1760 lwp_t *l; 1761 1762 LIST_FOREACH(l, &alllwp, l_list) { 1763 uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l); 1764 1765 if (addr < stack || stack + KSTACK_SIZE <= addr) { 1766 continue; 1767 } 1768 (*pr)("%p is %p+%zu, LWP %p's stack\n", 1769 (void *)addr, (void *)stack, 1770 (size_t)(addr - stack), l); 1771 } 1772 } 1773 #endif /* defined(DDB) */ 1774