1 /* $NetBSD: kern_lwp.c,v 1.131 2009/05/23 18:28:06 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nathan J. Williams, and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Overview 34 * 35 * Lightweight processes (LWPs) are the basic unit or thread of 36 * execution within the kernel. The core state of an LWP is described 37 * by "struct lwp", also known as lwp_t. 38 * 39 * Each LWP is contained within a process (described by "struct proc"), 40 * Every process contains at least one LWP, but may contain more. The 41 * process describes attributes shared among all of its LWPs such as a 42 * private address space, global execution state (stopped, active, 43 * zombie, ...), signal disposition and so on. On a multiprocessor 44 * machine, multiple LWPs be executing concurrently in the kernel. 45 * 46 * Execution states 47 * 48 * At any given time, an LWP has overall state that is described by 49 * lwp::l_stat. The states are broken into two sets below. The first 50 * set is guaranteed to represent the absolute, current state of the 51 * LWP: 52 * 53 * LSONPROC 54 * 55 * On processor: the LWP is executing on a CPU, either in the 56 * kernel or in user space. 57 * 58 * LSRUN 59 * 60 * Runnable: the LWP is parked on a run queue, and may soon be 61 * chosen to run by an idle processor, or by a processor that 62 * has been asked to preempt a currently runnning but lower 63 * priority LWP. If the LWP is not swapped in (LW_INMEM == 0) 64 * then the LWP is not on a run queue, but may be soon. 65 * 66 * LSIDL 67 * 68 * Idle: the LWP has been created but has not yet executed, 69 * or it has ceased executing a unit of work and is waiting 70 * to be started again. 71 * 72 * LSSUSPENDED: 73 * 74 * Suspended: the LWP has had its execution suspended by 75 * another LWP in the same process using the _lwp_suspend() 76 * system call. User-level LWPs also enter the suspended 77 * state when the system is shutting down. 78 * 79 * The second set represent a "statement of intent" on behalf of the 80 * LWP. The LWP may in fact be executing on a processor, may be 81 * sleeping or idle. It is expected to take the necessary action to 82 * stop executing or become "running" again within a short timeframe. 83 * The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running. 84 * Importantly, it indicates that its state is tied to a CPU. 85 * 86 * LSZOMB: 87 * 88 * Dead or dying: the LWP has released most of its resources 89 * and is about to switch away into oblivion, or has already 90 * switched away. When it switches away, its few remaining 91 * resources can be collected. 92 * 93 * LSSLEEP: 94 * 95 * Sleeping: the LWP has entered itself onto a sleep queue, and 96 * has switched away or will switch away shortly to allow other 97 * LWPs to run on the CPU. 98 * 99 * LSSTOP: 100 * 101 * Stopped: the LWP has been stopped as a result of a job 102 * control signal, or as a result of the ptrace() interface. 103 * 104 * Stopped LWPs may run briefly within the kernel to handle 105 * signals that they receive, but will not return to user space 106 * until their process' state is changed away from stopped. 107 * 108 * Single LWPs within a process can not be set stopped 109 * selectively: all actions that can stop or continue LWPs 110 * occur at the process level. 111 * 112 * State transitions 113 * 114 * Note that the LSSTOP state may only be set when returning to 115 * user space in userret(), or when sleeping interruptably. The 116 * LSSUSPENDED state may only be set in userret(). Before setting 117 * those states, we try to ensure that the LWPs will release all 118 * locks that they hold, and at a minimum try to ensure that the 119 * LWP can be set runnable again by a signal. 120 * 121 * LWPs may transition states in the following ways: 122 * 123 * RUN -------> ONPROC ONPROC -----> RUN 124 * > SLEEP 125 * > STOPPED 126 * > SUSPENDED 127 * > ZOMB 128 * > IDL (special cases) 129 * 130 * STOPPED ---> RUN SUSPENDED --> RUN 131 * > SLEEP 132 * 133 * SLEEP -----> ONPROC IDL --------> RUN 134 * > RUN > SUSPENDED 135 * > STOPPED > STOPPED 136 * > ONPROC (special cases) 137 * 138 * Some state transitions are only possible with kernel threads (eg 139 * ONPROC -> IDL) and happen under tightly controlled circumstances 140 * free of unwanted side effects. 141 * 142 * Migration 143 * 144 * Migration of threads from one CPU to another could be performed 145 * internally by the scheduler via sched_takecpu() or sched_catchlwp() 146 * functions. The universal lwp_migrate() function should be used for 147 * any other cases. Subsystems in the kernel must be aware that CPU 148 * of LWP may change, while it is not locked. 149 * 150 * Locking 151 * 152 * The majority of fields in 'struct lwp' are covered by a single, 153 * general spin lock pointed to by lwp::l_mutex. The locks covering 154 * each field are documented in sys/lwp.h. 155 * 156 * State transitions must be made with the LWP's general lock held, 157 * and may cause the LWP's lock pointer to change. Manipulation of 158 * the general lock is not performed directly, but through calls to 159 * lwp_lock(), lwp_relock() and similar. 160 * 161 * States and their associated locks: 162 * 163 * LSONPROC, LSZOMB: 164 * 165 * Always covered by spc_lwplock, which protects running LWPs. 166 * This is a per-CPU lock and matches lwp::l_cpu. 167 * 168 * LSIDL, LSRUN: 169 * 170 * Always covered by spc_mutex, which protects the run queues. 171 * This is a per-CPU lock and matches lwp::l_cpu. 172 * 173 * LSSLEEP: 174 * 175 * Covered by a lock associated with the sleep queue that the 176 * LWP resides on. Matches lwp::l_sleepq::sq_mutex. 177 * 178 * LSSTOP, LSSUSPENDED: 179 * 180 * If the LWP was previously sleeping (l_wchan != NULL), then 181 * l_mutex references the sleep queue lock. If the LWP was 182 * runnable or on the CPU when halted, or has been removed from 183 * the sleep queue since halted, then the lock is spc_lwplock. 184 * 185 * The lock order is as follows: 186 * 187 * spc::spc_lwplock -> 188 * sleeptab::st_mutex -> 189 * tschain_t::tc_mutex -> 190 * spc::spc_mutex 191 * 192 * Each process has an scheduler state lock (proc::p_lock), and a 193 * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and 194 * so on. When an LWP is to be entered into or removed from one of the 195 * following states, p_lock must be held and the process wide counters 196 * adjusted: 197 * 198 * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED 199 * 200 * (But not always for kernel threads. There are some special cases 201 * as mentioned above. See kern_softint.c.) 202 * 203 * Note that an LWP is considered running or likely to run soon if in 204 * one of the following states. This affects the value of p_nrlwps: 205 * 206 * LSRUN, LSONPROC, LSSLEEP 207 * 208 * p_lock does not need to be held when transitioning among these 209 * three states, hence p_lock is rarely taken for state transitions. 210 */ 211 212 #include <sys/cdefs.h> 213 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.131 2009/05/23 18:28:06 ad Exp $"); 214 215 #include "opt_ddb.h" 216 #include "opt_lockdebug.h" 217 #include "opt_sa.h" 218 219 #define _LWP_API_PRIVATE 220 221 #include <sys/param.h> 222 #include <sys/systm.h> 223 #include <sys/cpu.h> 224 #include <sys/pool.h> 225 #include <sys/proc.h> 226 #include <sys/sa.h> 227 #include <sys/savar.h> 228 #include <sys/syscallargs.h> 229 #include <sys/syscall_stats.h> 230 #include <sys/kauth.h> 231 #include <sys/sleepq.h> 232 #include <sys/user.h> 233 #include <sys/lockdebug.h> 234 #include <sys/kmem.h> 235 #include <sys/pset.h> 236 #include <sys/intr.h> 237 #include <sys/lwpctl.h> 238 #include <sys/atomic.h> 239 #include <sys/filedesc.h> 240 241 #include <uvm/uvm_extern.h> 242 #include <uvm/uvm_object.h> 243 244 struct lwplist alllwp = LIST_HEAD_INITIALIZER(alllwp); 245 246 POOL_INIT(lwp_uc_pool, sizeof(ucontext_t), 0, 0, 0, "lwpucpl", 247 &pool_allocator_nointr, IPL_NONE); 248 249 static pool_cache_t lwp_cache; 250 static specificdata_domain_t lwp_specificdata_domain; 251 252 void 253 lwpinit(void) 254 { 255 256 lwp_specificdata_domain = specificdata_domain_create(); 257 KASSERT(lwp_specificdata_domain != NULL); 258 lwp_sys_init(); 259 lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0, 260 "lwppl", NULL, IPL_NONE, NULL, NULL, NULL); 261 } 262 263 /* 264 * Set an suspended. 265 * 266 * Must be called with p_lock held, and the LWP locked. Will unlock the 267 * LWP before return. 268 */ 269 int 270 lwp_suspend(struct lwp *curl, struct lwp *t) 271 { 272 int error; 273 274 KASSERT(mutex_owned(t->l_proc->p_lock)); 275 KASSERT(lwp_locked(t, NULL)); 276 277 KASSERT(curl != t || curl->l_stat == LSONPROC); 278 279 /* 280 * If the current LWP has been told to exit, we must not suspend anyone 281 * else or deadlock could occur. We won't return to userspace. 282 */ 283 if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) { 284 lwp_unlock(t); 285 return (EDEADLK); 286 } 287 288 error = 0; 289 290 switch (t->l_stat) { 291 case LSRUN: 292 case LSONPROC: 293 t->l_flag |= LW_WSUSPEND; 294 lwp_need_userret(t); 295 lwp_unlock(t); 296 break; 297 298 case LSSLEEP: 299 t->l_flag |= LW_WSUSPEND; 300 301 /* 302 * Kick the LWP and try to get it to the kernel boundary 303 * so that it will release any locks that it holds. 304 * setrunnable() will release the lock. 305 */ 306 if ((t->l_flag & LW_SINTR) != 0) 307 setrunnable(t); 308 else 309 lwp_unlock(t); 310 break; 311 312 case LSSUSPENDED: 313 lwp_unlock(t); 314 break; 315 316 case LSSTOP: 317 t->l_flag |= LW_WSUSPEND; 318 setrunnable(t); 319 break; 320 321 case LSIDL: 322 case LSZOMB: 323 error = EINTR; /* It's what Solaris does..... */ 324 lwp_unlock(t); 325 break; 326 } 327 328 return (error); 329 } 330 331 /* 332 * Restart a suspended LWP. 333 * 334 * Must be called with p_lock held, and the LWP locked. Will unlock the 335 * LWP before return. 336 */ 337 void 338 lwp_continue(struct lwp *l) 339 { 340 341 KASSERT(mutex_owned(l->l_proc->p_lock)); 342 KASSERT(lwp_locked(l, NULL)); 343 344 /* If rebooting or not suspended, then just bail out. */ 345 if ((l->l_flag & LW_WREBOOT) != 0) { 346 lwp_unlock(l); 347 return; 348 } 349 350 l->l_flag &= ~LW_WSUSPEND; 351 352 if (l->l_stat != LSSUSPENDED) { 353 lwp_unlock(l); 354 return; 355 } 356 357 /* setrunnable() will release the lock. */ 358 setrunnable(l); 359 } 360 361 /* 362 * Wait for an LWP within the current process to exit. If 'lid' is 363 * non-zero, we are waiting for a specific LWP. 364 * 365 * Must be called with p->p_lock held. 366 */ 367 int 368 lwp_wait1(struct lwp *l, lwpid_t lid, lwpid_t *departed, int flags) 369 { 370 struct proc *p = l->l_proc; 371 struct lwp *l2; 372 int nfound, error; 373 lwpid_t curlid; 374 bool exiting; 375 376 KASSERT(mutex_owned(p->p_lock)); 377 378 p->p_nlwpwait++; 379 l->l_waitingfor = lid; 380 curlid = l->l_lid; 381 exiting = ((flags & LWPWAIT_EXITCONTROL) != 0); 382 383 for (;;) { 384 /* 385 * Avoid a race between exit1() and sigexit(): if the 386 * process is dumping core, then we need to bail out: call 387 * into lwp_userret() where we will be suspended until the 388 * deed is done. 389 */ 390 if ((p->p_sflag & PS_WCORE) != 0) { 391 mutex_exit(p->p_lock); 392 lwp_userret(l); 393 #ifdef DIAGNOSTIC 394 panic("lwp_wait1"); 395 #endif 396 /* NOTREACHED */ 397 } 398 399 /* 400 * First off, drain any detached LWP that is waiting to be 401 * reaped. 402 */ 403 while ((l2 = p->p_zomblwp) != NULL) { 404 p->p_zomblwp = NULL; 405 lwp_free(l2, false, false);/* releases proc mutex */ 406 mutex_enter(p->p_lock); 407 } 408 409 /* 410 * Now look for an LWP to collect. If the whole process is 411 * exiting, count detached LWPs as eligible to be collected, 412 * but don't drain them here. 413 */ 414 nfound = 0; 415 error = 0; 416 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 417 /* 418 * If a specific wait and the target is waiting on 419 * us, then avoid deadlock. This also traps LWPs 420 * that try to wait on themselves. 421 * 422 * Note that this does not handle more complicated 423 * cycles, like: t1 -> t2 -> t3 -> t1. The process 424 * can still be killed so it is not a major problem. 425 */ 426 if (l2->l_lid == lid && l2->l_waitingfor == curlid) { 427 error = EDEADLK; 428 break; 429 } 430 if (l2 == l) 431 continue; 432 if ((l2->l_prflag & LPR_DETACHED) != 0) { 433 nfound += exiting; 434 continue; 435 } 436 if (lid != 0) { 437 if (l2->l_lid != lid) 438 continue; 439 /* 440 * Mark this LWP as the first waiter, if there 441 * is no other. 442 */ 443 if (l2->l_waiter == 0) 444 l2->l_waiter = curlid; 445 } else if (l2->l_waiter != 0) { 446 /* 447 * It already has a waiter - so don't 448 * collect it. If the waiter doesn't 449 * grab it we'll get another chance 450 * later. 451 */ 452 nfound++; 453 continue; 454 } 455 nfound++; 456 457 /* No need to lock the LWP in order to see LSZOMB. */ 458 if (l2->l_stat != LSZOMB) 459 continue; 460 461 /* 462 * We're no longer waiting. Reset the "first waiter" 463 * pointer on the target, in case it was us. 464 */ 465 l->l_waitingfor = 0; 466 l2->l_waiter = 0; 467 p->p_nlwpwait--; 468 if (departed) 469 *departed = l2->l_lid; 470 sched_lwp_collect(l2); 471 472 /* lwp_free() releases the proc lock. */ 473 lwp_free(l2, false, false); 474 mutex_enter(p->p_lock); 475 return 0; 476 } 477 478 if (error != 0) 479 break; 480 if (nfound == 0) { 481 error = ESRCH; 482 break; 483 } 484 485 /* 486 * The kernel is careful to ensure that it can not deadlock 487 * when exiting - just keep waiting. 488 */ 489 if (exiting) { 490 KASSERT(p->p_nlwps > 1); 491 cv_wait(&p->p_lwpcv, p->p_lock); 492 continue; 493 } 494 495 /* 496 * If all other LWPs are waiting for exits or suspends 497 * and the supply of zombies and potential zombies is 498 * exhausted, then we are about to deadlock. 499 * 500 * If the process is exiting (and this LWP is not the one 501 * that is coordinating the exit) then bail out now. 502 */ 503 if ((p->p_sflag & PS_WEXIT) != 0 || 504 p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) { 505 error = EDEADLK; 506 break; 507 } 508 509 /* 510 * Sit around and wait for something to happen. We'll be 511 * awoken if any of the conditions examined change: if an 512 * LWP exits, is collected, or is detached. 513 */ 514 if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0) 515 break; 516 } 517 518 /* 519 * We didn't find any LWPs to collect, we may have received a 520 * signal, or some other condition has caused us to bail out. 521 * 522 * If waiting on a specific LWP, clear the waiters marker: some 523 * other LWP may want it. Then, kick all the remaining waiters 524 * so that they can re-check for zombies and for deadlock. 525 */ 526 if (lid != 0) { 527 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 528 if (l2->l_lid == lid) { 529 if (l2->l_waiter == curlid) 530 l2->l_waiter = 0; 531 break; 532 } 533 } 534 } 535 p->p_nlwpwait--; 536 l->l_waitingfor = 0; 537 cv_broadcast(&p->p_lwpcv); 538 539 return error; 540 } 541 542 /* 543 * Create a new LWP within process 'p2', using LWP 'l1' as a template. 544 * The new LWP is created in state LSIDL and must be set running, 545 * suspended, or stopped by the caller. 546 */ 547 int 548 lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, bool inmem, int flags, 549 void *stack, size_t stacksize, void (*func)(void *), void *arg, 550 lwp_t **rnewlwpp, int sclass) 551 { 552 struct lwp *l2, *isfree; 553 turnstile_t *ts; 554 555 KASSERT(l1 == curlwp || l1->l_proc == &proc0); 556 557 /* 558 * First off, reap any detached LWP waiting to be collected. 559 * We can re-use its LWP structure and turnstile. 560 */ 561 isfree = NULL; 562 if (p2->p_zomblwp != NULL) { 563 mutex_enter(p2->p_lock); 564 if ((isfree = p2->p_zomblwp) != NULL) { 565 p2->p_zomblwp = NULL; 566 lwp_free(isfree, true, false);/* releases proc mutex */ 567 } else 568 mutex_exit(p2->p_lock); 569 } 570 if (isfree == NULL) { 571 l2 = pool_cache_get(lwp_cache, PR_WAITOK); 572 memset(l2, 0, sizeof(*l2)); 573 l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK); 574 SLIST_INIT(&l2->l_pi_lenders); 575 } else { 576 l2 = isfree; 577 ts = l2->l_ts; 578 KASSERT(l2->l_inheritedprio == -1); 579 KASSERT(SLIST_EMPTY(&l2->l_pi_lenders)); 580 memset(l2, 0, sizeof(*l2)); 581 l2->l_ts = ts; 582 } 583 584 l2->l_stat = LSIDL; 585 l2->l_proc = p2; 586 l2->l_refcnt = 1; 587 l2->l_class = sclass; 588 589 /* 590 * If vfork(), we want the LWP to run fast and on the same CPU 591 * as its parent, so that it can reuse the VM context and cache 592 * footprint on the local CPU. 593 */ 594 l2->l_kpriority = ((flags & LWP_VFORK) ? true : false); 595 l2->l_kpribase = PRI_KERNEL; 596 l2->l_priority = l1->l_priority; 597 l2->l_inheritedprio = -1; 598 l2->l_flag = inmem ? LW_INMEM : 0; 599 l2->l_pflag = LP_MPSAFE; 600 TAILQ_INIT(&l2->l_ld_locks); 601 602 /* 603 * If not the first LWP in the process, grab a reference to the 604 * descriptor table. 605 */ 606 l2->l_fd = p2->p_fd; 607 if (p2->p_nlwps != 0) { 608 KASSERT(l1->l_proc == p2); 609 fd_hold(); 610 } else { 611 KASSERT(l1->l_proc != p2); 612 } 613 614 if (p2->p_flag & PK_SYSTEM) { 615 /* Mark it as a system LWP and not a candidate for swapping */ 616 l2->l_flag |= LW_SYSTEM; 617 } 618 619 kpreempt_disable(); 620 l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex; 621 l2->l_cpu = l1->l_cpu; 622 kpreempt_enable(); 623 624 lwp_initspecific(l2); 625 sched_lwp_fork(l1, l2); 626 lwp_update_creds(l2); 627 callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE); 628 callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2); 629 mutex_init(&l2->l_swaplock, MUTEX_DEFAULT, IPL_NONE); 630 cv_init(&l2->l_sigcv, "sigwait"); 631 l2->l_syncobj = &sched_syncobj; 632 633 if (rnewlwpp != NULL) 634 *rnewlwpp = l2; 635 636 l2->l_addr = UAREA_TO_USER(uaddr); 637 uvm_lwp_fork(l1, l2, stack, stacksize, func, 638 (arg != NULL) ? arg : l2); 639 640 mutex_enter(p2->p_lock); 641 642 if ((flags & LWP_DETACHED) != 0) { 643 l2->l_prflag = LPR_DETACHED; 644 p2->p_ndlwps++; 645 } else 646 l2->l_prflag = 0; 647 648 l2->l_sigmask = l1->l_sigmask; 649 CIRCLEQ_INIT(&l2->l_sigpend.sp_info); 650 sigemptyset(&l2->l_sigpend.sp_set); 651 652 p2->p_nlwpid++; 653 if (p2->p_nlwpid == 0) 654 p2->p_nlwpid++; 655 l2->l_lid = p2->p_nlwpid; 656 LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling); 657 p2->p_nlwps++; 658 659 if ((p2->p_flag & PK_SYSTEM) == 0) { 660 /* Inherit an affinity */ 661 if (l1->l_flag & LW_AFFINITY) { 662 /* 663 * Note that we hold the state lock while inheriting 664 * the affinity to avoid race with sched_setaffinity(). 665 */ 666 lwp_lock(l1); 667 if (l1->l_flag & LW_AFFINITY) { 668 kcpuset_use(l1->l_affinity); 669 l2->l_affinity = l1->l_affinity; 670 l2->l_flag |= LW_AFFINITY; 671 } 672 lwp_unlock(l1); 673 } 674 lwp_lock(l2); 675 /* Inherit a processor-set */ 676 l2->l_psid = l1->l_psid; 677 /* Look for a CPU to start */ 678 l2->l_cpu = sched_takecpu(l2); 679 lwp_unlock_to(l2, l2->l_cpu->ci_schedstate.spc_mutex); 680 } 681 mutex_exit(p2->p_lock); 682 683 mutex_enter(proc_lock); 684 LIST_INSERT_HEAD(&alllwp, l2, l_list); 685 mutex_exit(proc_lock); 686 687 SYSCALL_TIME_LWP_INIT(l2); 688 689 if (p2->p_emul->e_lwp_fork) 690 (*p2->p_emul->e_lwp_fork)(l1, l2); 691 692 return (0); 693 } 694 695 /* 696 * Called by MD code when a new LWP begins execution. Must be called 697 * with the previous LWP locked (so at splsched), or if there is no 698 * previous LWP, at splsched. 699 */ 700 void 701 lwp_startup(struct lwp *prev, struct lwp *new) 702 { 703 704 KASSERT(kpreempt_disabled()); 705 if (prev != NULL) { 706 /* 707 * Normalize the count of the spin-mutexes, it was 708 * increased in mi_switch(). Unmark the state of 709 * context switch - it is finished for previous LWP. 710 */ 711 curcpu()->ci_mtx_count++; 712 membar_exit(); 713 prev->l_ctxswtch = 0; 714 } 715 KPREEMPT_DISABLE(new); 716 spl0(); 717 pmap_activate(new); 718 LOCKDEBUG_BARRIER(NULL, 0); 719 KPREEMPT_ENABLE(new); 720 if ((new->l_pflag & LP_MPSAFE) == 0) { 721 KERNEL_LOCK(1, new); 722 } 723 } 724 725 /* 726 * Exit an LWP. 727 */ 728 void 729 lwp_exit(struct lwp *l) 730 { 731 struct proc *p = l->l_proc; 732 struct lwp *l2; 733 bool current; 734 735 current = (l == curlwp); 736 737 KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL)); 738 KASSERT(p == curproc); 739 740 /* 741 * Verify that we hold no locks other than the kernel lock. 742 */ 743 LOCKDEBUG_BARRIER(&kernel_lock, 0); 744 745 /* 746 * If we are the last live LWP in a process, we need to exit the 747 * entire process. We do so with an exit status of zero, because 748 * it's a "controlled" exit, and because that's what Solaris does. 749 * 750 * We are not quite a zombie yet, but for accounting purposes we 751 * must increment the count of zombies here. 752 * 753 * Note: the last LWP's specificdata will be deleted here. 754 */ 755 mutex_enter(p->p_lock); 756 if (p->p_nlwps - p->p_nzlwps == 1) { 757 KASSERT(current == true); 758 /* XXXSMP kernel_lock not held */ 759 exit1(l, 0); 760 /* NOTREACHED */ 761 } 762 p->p_nzlwps++; 763 mutex_exit(p->p_lock); 764 765 if (p->p_emul->e_lwp_exit) 766 (*p->p_emul->e_lwp_exit)(l); 767 768 /* Drop filedesc reference. */ 769 fd_free(); 770 771 /* Delete the specificdata while it's still safe to sleep. */ 772 specificdata_fini(lwp_specificdata_domain, &l->l_specdataref); 773 774 /* 775 * Release our cached credentials. 776 */ 777 kauth_cred_free(l->l_cred); 778 callout_destroy(&l->l_timeout_ch); 779 780 /* 781 * While we can still block, mark the LWP as unswappable to 782 * prevent conflicts with the with the swapper. 783 */ 784 if (current) 785 uvm_lwp_hold(l); 786 787 /* 788 * Remove the LWP from the global list. 789 */ 790 mutex_enter(proc_lock); 791 LIST_REMOVE(l, l_list); 792 mutex_exit(proc_lock); 793 794 /* 795 * Get rid of all references to the LWP that others (e.g. procfs) 796 * may have, and mark the LWP as a zombie. If the LWP is detached, 797 * mark it waiting for collection in the proc structure. Note that 798 * before we can do that, we need to free any other dead, deatched 799 * LWP waiting to meet its maker. 800 */ 801 mutex_enter(p->p_lock); 802 lwp_drainrefs(l); 803 804 if ((l->l_prflag & LPR_DETACHED) != 0) { 805 while ((l2 = p->p_zomblwp) != NULL) { 806 p->p_zomblwp = NULL; 807 lwp_free(l2, false, false);/* releases proc mutex */ 808 mutex_enter(p->p_lock); 809 l->l_refcnt++; 810 lwp_drainrefs(l); 811 } 812 p->p_zomblwp = l; 813 } 814 815 /* 816 * If we find a pending signal for the process and we have been 817 * asked to check for signals, then we loose: arrange to have 818 * all other LWPs in the process check for signals. 819 */ 820 if ((l->l_flag & LW_PENDSIG) != 0 && 821 firstsig(&p->p_sigpend.sp_set) != 0) { 822 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 823 lwp_lock(l2); 824 l2->l_flag |= LW_PENDSIG; 825 lwp_unlock(l2); 826 } 827 } 828 829 lwp_lock(l); 830 l->l_stat = LSZOMB; 831 if (l->l_name != NULL) 832 strcpy(l->l_name, "(zombie)"); 833 if (l->l_flag & LW_AFFINITY) { 834 l->l_flag &= ~LW_AFFINITY; 835 } else { 836 KASSERT(l->l_affinity == NULL); 837 } 838 lwp_unlock(l); 839 p->p_nrlwps--; 840 cv_broadcast(&p->p_lwpcv); 841 if (l->l_lwpctl != NULL) 842 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED; 843 mutex_exit(p->p_lock); 844 845 /* Safe without lock since LWP is in zombie state */ 846 if (l->l_affinity) { 847 kcpuset_unuse(l->l_affinity, NULL); 848 l->l_affinity = NULL; 849 } 850 851 /* 852 * We can no longer block. At this point, lwp_free() may already 853 * be gunning for us. On a multi-CPU system, we may be off p_lwps. 854 * 855 * Free MD LWP resources. 856 */ 857 #ifndef __NO_CPU_LWP_FREE 858 cpu_lwp_free(l, 0); 859 #endif 860 861 if (current) { 862 pmap_deactivate(l); 863 864 /* 865 * Release the kernel lock, and switch away into 866 * oblivion. 867 */ 868 #ifdef notyet 869 /* XXXSMP hold in lwp_userret() */ 870 KERNEL_UNLOCK_LAST(l); 871 #else 872 KERNEL_UNLOCK_ALL(l, NULL); 873 #endif 874 lwp_exit_switchaway(l); 875 } 876 } 877 878 /* 879 * Free a dead LWP's remaining resources. 880 * 881 * XXXLWP limits. 882 */ 883 void 884 lwp_free(struct lwp *l, bool recycle, bool last) 885 { 886 struct proc *p = l->l_proc; 887 struct rusage *ru; 888 ksiginfoq_t kq; 889 890 KASSERT(l != curlwp); 891 892 /* 893 * If this was not the last LWP in the process, then adjust 894 * counters and unlock. 895 */ 896 if (!last) { 897 /* 898 * Add the LWP's run time to the process' base value. 899 * This needs to co-incide with coming off p_lwps. 900 */ 901 bintime_add(&p->p_rtime, &l->l_rtime); 902 p->p_pctcpu += l->l_pctcpu; 903 ru = &p->p_stats->p_ru; 904 ruadd(ru, &l->l_ru); 905 ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw); 906 ru->ru_nivcsw += l->l_nivcsw; 907 LIST_REMOVE(l, l_sibling); 908 p->p_nlwps--; 909 p->p_nzlwps--; 910 if ((l->l_prflag & LPR_DETACHED) != 0) 911 p->p_ndlwps--; 912 913 /* 914 * Have any LWPs sleeping in lwp_wait() recheck for 915 * deadlock. 916 */ 917 cv_broadcast(&p->p_lwpcv); 918 mutex_exit(p->p_lock); 919 } 920 921 #ifdef MULTIPROCESSOR 922 /* 923 * In the unlikely event that the LWP is still on the CPU, 924 * then spin until it has switched away. We need to release 925 * all locks to avoid deadlock against interrupt handlers on 926 * the target CPU. 927 */ 928 if ((l->l_pflag & LP_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) { 929 int count; 930 (void)count; /* XXXgcc */ 931 KERNEL_UNLOCK_ALL(curlwp, &count); 932 while ((l->l_pflag & LP_RUNNING) != 0 || 933 l->l_cpu->ci_curlwp == l) 934 SPINLOCK_BACKOFF_HOOK; 935 KERNEL_LOCK(count, curlwp); 936 } 937 #endif 938 939 /* 940 * Destroy the LWP's remaining signal information. 941 */ 942 ksiginfo_queue_init(&kq); 943 sigclear(&l->l_sigpend, NULL, &kq); 944 ksiginfo_queue_drain(&kq); 945 cv_destroy(&l->l_sigcv); 946 mutex_destroy(&l->l_swaplock); 947 948 /* 949 * Free the LWP's turnstile and the LWP structure itself unless the 950 * caller wants to recycle them. Also, free the scheduler specific 951 * data. 952 * 953 * We can't return turnstile0 to the pool (it didn't come from it), 954 * so if it comes up just drop it quietly and move on. 955 * 956 * We don't recycle the VM resources at this time. 957 */ 958 if (l->l_lwpctl != NULL) 959 lwp_ctl_free(l); 960 961 if (!recycle && l->l_ts != &turnstile0) 962 pool_cache_put(turnstile_cache, l->l_ts); 963 if (l->l_name != NULL) 964 kmem_free(l->l_name, MAXCOMLEN); 965 #ifndef __NO_CPU_LWP_FREE 966 cpu_lwp_free2(l); 967 #endif 968 KASSERT((l->l_flag & LW_INMEM) != 0); 969 uvm_lwp_exit(l); 970 KASSERT(SLIST_EMPTY(&l->l_pi_lenders)); 971 KASSERT(l->l_inheritedprio == -1); 972 if (!recycle) 973 pool_cache_put(lwp_cache, l); 974 } 975 976 /* 977 * Migrate the LWP to the another CPU. Unlocks the LWP. 978 */ 979 void 980 lwp_migrate(lwp_t *l, struct cpu_info *tci) 981 { 982 struct schedstate_percpu *tspc; 983 int lstat = l->l_stat; 984 985 KASSERT(lwp_locked(l, NULL)); 986 KASSERT(tci != NULL); 987 988 /* If LWP is still on the CPU, it must be handled like LSONPROC */ 989 if ((l->l_pflag & LP_RUNNING) != 0) { 990 lstat = LSONPROC; 991 } 992 993 /* 994 * The destination CPU could be changed while previous migration 995 * was not finished. 996 */ 997 if (l->l_target_cpu != NULL) { 998 l->l_target_cpu = tci; 999 lwp_unlock(l); 1000 return; 1001 } 1002 1003 /* Nothing to do if trying to migrate to the same CPU */ 1004 if (l->l_cpu == tci) { 1005 lwp_unlock(l); 1006 return; 1007 } 1008 1009 KASSERT(l->l_target_cpu == NULL); 1010 tspc = &tci->ci_schedstate; 1011 switch (lstat) { 1012 case LSRUN: 1013 if (l->l_flag & LW_INMEM) { 1014 l->l_target_cpu = tci; 1015 lwp_unlock(l); 1016 return; 1017 } 1018 case LSIDL: 1019 l->l_cpu = tci; 1020 lwp_unlock_to(l, tspc->spc_mutex); 1021 return; 1022 case LSSLEEP: 1023 l->l_cpu = tci; 1024 break; 1025 case LSSTOP: 1026 case LSSUSPENDED: 1027 l->l_cpu = tci; 1028 if (l->l_wchan == NULL) { 1029 lwp_unlock_to(l, tspc->spc_lwplock); 1030 return; 1031 } 1032 break; 1033 case LSONPROC: 1034 l->l_target_cpu = tci; 1035 spc_lock(l->l_cpu); 1036 cpu_need_resched(l->l_cpu, RESCHED_KPREEMPT); 1037 spc_unlock(l->l_cpu); 1038 break; 1039 } 1040 lwp_unlock(l); 1041 } 1042 1043 /* 1044 * Find the LWP in the process. Arguments may be zero, in such case, 1045 * the calling process and first LWP in the list will be used. 1046 * On success - returns proc locked. 1047 */ 1048 struct lwp * 1049 lwp_find2(pid_t pid, lwpid_t lid) 1050 { 1051 proc_t *p; 1052 lwp_t *l; 1053 1054 /* Find the process */ 1055 p = (pid == 0) ? curlwp->l_proc : p_find(pid, PFIND_UNLOCK_FAIL); 1056 if (p == NULL) 1057 return NULL; 1058 mutex_enter(p->p_lock); 1059 if (pid != 0) { 1060 /* Case of p_find */ 1061 mutex_exit(proc_lock); 1062 } 1063 1064 /* Find the thread */ 1065 l = (lid == 0) ? LIST_FIRST(&p->p_lwps) : lwp_find(p, lid); 1066 if (l == NULL) { 1067 mutex_exit(p->p_lock); 1068 } 1069 1070 return l; 1071 } 1072 1073 /* 1074 * Look up a live LWP within the speicifed process, and return it locked. 1075 * 1076 * Must be called with p->p_lock held. 1077 */ 1078 struct lwp * 1079 lwp_find(struct proc *p, int id) 1080 { 1081 struct lwp *l; 1082 1083 KASSERT(mutex_owned(p->p_lock)); 1084 1085 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1086 if (l->l_lid == id) 1087 break; 1088 } 1089 1090 /* 1091 * No need to lock - all of these conditions will 1092 * be visible with the process level mutex held. 1093 */ 1094 if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB)) 1095 l = NULL; 1096 1097 return l; 1098 } 1099 1100 /* 1101 * Update an LWP's cached credentials to mirror the process' master copy. 1102 * 1103 * This happens early in the syscall path, on user trap, and on LWP 1104 * creation. A long-running LWP can also voluntarily choose to update 1105 * it's credentials by calling this routine. This may be called from 1106 * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand. 1107 */ 1108 void 1109 lwp_update_creds(struct lwp *l) 1110 { 1111 kauth_cred_t oc; 1112 struct proc *p; 1113 1114 p = l->l_proc; 1115 oc = l->l_cred; 1116 1117 mutex_enter(p->p_lock); 1118 kauth_cred_hold(p->p_cred); 1119 l->l_cred = p->p_cred; 1120 l->l_prflag &= ~LPR_CRMOD; 1121 mutex_exit(p->p_lock); 1122 if (oc != NULL) 1123 kauth_cred_free(oc); 1124 } 1125 1126 /* 1127 * Verify that an LWP is locked, and optionally verify that the lock matches 1128 * one we specify. 1129 */ 1130 int 1131 lwp_locked(struct lwp *l, kmutex_t *mtx) 1132 { 1133 kmutex_t *cur = l->l_mutex; 1134 1135 return mutex_owned(cur) && (mtx == cur || mtx == NULL); 1136 } 1137 1138 /* 1139 * Lock an LWP. 1140 */ 1141 kmutex_t * 1142 lwp_lock_retry(struct lwp *l, kmutex_t *old) 1143 { 1144 1145 /* 1146 * XXXgcc ignoring kmutex_t * volatile on i386 1147 * 1148 * gcc version 4.1.2 20061021 prerelease (NetBSD nb1 20061021) 1149 */ 1150 #if 1 1151 while (l->l_mutex != old) { 1152 #else 1153 for (;;) { 1154 #endif 1155 mutex_spin_exit(old); 1156 old = l->l_mutex; 1157 mutex_spin_enter(old); 1158 1159 /* 1160 * mutex_enter() will have posted a read barrier. Re-test 1161 * l->l_mutex. If it has changed, we need to try again. 1162 */ 1163 #if 1 1164 } 1165 #else 1166 } while (__predict_false(l->l_mutex != old)); 1167 #endif 1168 1169 return old; 1170 } 1171 1172 /* 1173 * Lend a new mutex to an LWP. The old mutex must be held. 1174 */ 1175 void 1176 lwp_setlock(struct lwp *l, kmutex_t *new) 1177 { 1178 1179 KASSERT(mutex_owned(l->l_mutex)); 1180 1181 membar_exit(); 1182 l->l_mutex = new; 1183 } 1184 1185 /* 1186 * Lend a new mutex to an LWP, and release the old mutex. The old mutex 1187 * must be held. 1188 */ 1189 void 1190 lwp_unlock_to(struct lwp *l, kmutex_t *new) 1191 { 1192 kmutex_t *old; 1193 1194 KASSERT(mutex_owned(l->l_mutex)); 1195 1196 old = l->l_mutex; 1197 membar_exit(); 1198 l->l_mutex = new; 1199 mutex_spin_exit(old); 1200 } 1201 1202 /* 1203 * Acquire a new mutex, and donate it to an LWP. The LWP must already be 1204 * locked. 1205 */ 1206 void 1207 lwp_relock(struct lwp *l, kmutex_t *new) 1208 { 1209 kmutex_t *old; 1210 1211 KASSERT(mutex_owned(l->l_mutex)); 1212 1213 old = l->l_mutex; 1214 if (old != new) { 1215 mutex_spin_enter(new); 1216 l->l_mutex = new; 1217 mutex_spin_exit(old); 1218 } 1219 } 1220 1221 int 1222 lwp_trylock(struct lwp *l) 1223 { 1224 kmutex_t *old; 1225 1226 for (;;) { 1227 if (!mutex_tryenter(old = l->l_mutex)) 1228 return 0; 1229 if (__predict_true(l->l_mutex == old)) 1230 return 1; 1231 mutex_spin_exit(old); 1232 } 1233 } 1234 1235 u_int 1236 lwp_unsleep(lwp_t *l, bool cleanup) 1237 { 1238 1239 KASSERT(mutex_owned(l->l_mutex)); 1240 1241 return (*l->l_syncobj->sobj_unsleep)(l, cleanup); 1242 } 1243 1244 1245 /* 1246 * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is 1247 * set. 1248 */ 1249 void 1250 lwp_userret(struct lwp *l) 1251 { 1252 struct proc *p; 1253 void (*hook)(void); 1254 int sig; 1255 1256 KASSERT(l == curlwp); 1257 KASSERT(l->l_stat == LSONPROC); 1258 p = l->l_proc; 1259 1260 #ifndef __HAVE_FAST_SOFTINTS 1261 /* Run pending soft interrupts. */ 1262 if (l->l_cpu->ci_data.cpu_softints != 0) 1263 softint_overlay(); 1264 #endif 1265 1266 #ifdef KERN_SA 1267 /* Generate UNBLOCKED upcall if needed */ 1268 if (l->l_flag & LW_SA_BLOCKING) { 1269 sa_unblock_userret(l); 1270 /* NOTREACHED */ 1271 } 1272 #endif 1273 1274 /* 1275 * It should be safe to do this read unlocked on a multiprocessor 1276 * system.. 1277 * 1278 * LW_SA_UPCALL will be handled after the while() loop, so don't 1279 * consider it now. 1280 */ 1281 while ((l->l_flag & (LW_USERRET & ~(LW_SA_UPCALL))) != 0) { 1282 /* 1283 * Process pending signals first, unless the process 1284 * is dumping core or exiting, where we will instead 1285 * enter the LW_WSUSPEND case below. 1286 */ 1287 if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) == 1288 LW_PENDSIG) { 1289 mutex_enter(p->p_lock); 1290 while ((sig = issignal(l)) != 0) 1291 postsig(sig); 1292 mutex_exit(p->p_lock); 1293 } 1294 1295 /* 1296 * Core-dump or suspend pending. 1297 * 1298 * In case of core dump, suspend ourselves, so that the 1299 * kernel stack and therefore the userland registers saved 1300 * in the trapframe are around for coredump() to write them 1301 * out. We issue a wakeup on p->p_lwpcv so that sigexit() 1302 * will write the core file out once all other LWPs are 1303 * suspended. 1304 */ 1305 if ((l->l_flag & LW_WSUSPEND) != 0) { 1306 mutex_enter(p->p_lock); 1307 p->p_nrlwps--; 1308 cv_broadcast(&p->p_lwpcv); 1309 lwp_lock(l); 1310 l->l_stat = LSSUSPENDED; 1311 lwp_unlock(l); 1312 mutex_exit(p->p_lock); 1313 lwp_lock(l); 1314 mi_switch(l); 1315 } 1316 1317 /* Process is exiting. */ 1318 if ((l->l_flag & LW_WEXIT) != 0) { 1319 lwp_exit(l); 1320 KASSERT(0); 1321 /* NOTREACHED */ 1322 } 1323 1324 /* Call userret hook; used by Linux emulation. */ 1325 if ((l->l_flag & LW_WUSERRET) != 0) { 1326 lwp_lock(l); 1327 l->l_flag &= ~LW_WUSERRET; 1328 lwp_unlock(l); 1329 hook = p->p_userret; 1330 p->p_userret = NULL; 1331 (*hook)(); 1332 } 1333 } 1334 1335 #ifdef KERN_SA 1336 /* 1337 * Timer events are handled specially. We only try once to deliver 1338 * pending timer upcalls; if if fails, we can try again on the next 1339 * loop around. If we need to re-enter lwp_userret(), MD code will 1340 * bounce us back here through the trap path after we return. 1341 */ 1342 if (p->p_timerpend) 1343 timerupcall(l); 1344 if (l->l_flag & LW_SA_UPCALL) 1345 sa_upcall_userret(l); 1346 #endif /* KERN_SA */ 1347 } 1348 1349 /* 1350 * Force an LWP to enter the kernel, to take a trip through lwp_userret(). 1351 */ 1352 void 1353 lwp_need_userret(struct lwp *l) 1354 { 1355 KASSERT(lwp_locked(l, NULL)); 1356 1357 /* 1358 * Since the tests in lwp_userret() are done unlocked, make sure 1359 * that the condition will be seen before forcing the LWP to enter 1360 * kernel mode. 1361 */ 1362 membar_producer(); 1363 cpu_signotify(l); 1364 } 1365 1366 /* 1367 * Add one reference to an LWP. This will prevent the LWP from 1368 * exiting, thus keep the lwp structure and PCB around to inspect. 1369 */ 1370 void 1371 lwp_addref(struct lwp *l) 1372 { 1373 1374 KASSERT(mutex_owned(l->l_proc->p_lock)); 1375 KASSERT(l->l_stat != LSZOMB); 1376 KASSERT(l->l_refcnt != 0); 1377 1378 l->l_refcnt++; 1379 } 1380 1381 /* 1382 * Remove one reference to an LWP. If this is the last reference, 1383 * then we must finalize the LWP's death. 1384 */ 1385 void 1386 lwp_delref(struct lwp *l) 1387 { 1388 struct proc *p = l->l_proc; 1389 1390 mutex_enter(p->p_lock); 1391 KASSERT(l->l_stat != LSZOMB); 1392 KASSERT(l->l_refcnt > 0); 1393 if (--l->l_refcnt == 0) 1394 cv_broadcast(&p->p_lwpcv); 1395 mutex_exit(p->p_lock); 1396 } 1397 1398 /* 1399 * Drain all references to the current LWP. 1400 */ 1401 void 1402 lwp_drainrefs(struct lwp *l) 1403 { 1404 struct proc *p = l->l_proc; 1405 1406 KASSERT(mutex_owned(p->p_lock)); 1407 KASSERT(l->l_refcnt != 0); 1408 1409 l->l_refcnt--; 1410 while (l->l_refcnt != 0) 1411 cv_wait(&p->p_lwpcv, p->p_lock); 1412 } 1413 1414 /* 1415 * Return true if the specified LWP is 'alive'. Only p->p_lock need 1416 * be held. 1417 */ 1418 bool 1419 lwp_alive(lwp_t *l) 1420 { 1421 1422 KASSERT(mutex_owned(l->l_proc->p_lock)); 1423 1424 switch (l->l_stat) { 1425 case LSSLEEP: 1426 case LSRUN: 1427 case LSONPROC: 1428 case LSSTOP: 1429 case LSSUSPENDED: 1430 return true; 1431 default: 1432 return false; 1433 } 1434 } 1435 1436 /* 1437 * Return first live LWP in the process. 1438 */ 1439 lwp_t * 1440 lwp_find_first(proc_t *p) 1441 { 1442 lwp_t *l; 1443 1444 KASSERT(mutex_owned(p->p_lock)); 1445 1446 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1447 if (lwp_alive(l)) { 1448 return l; 1449 } 1450 } 1451 1452 return NULL; 1453 } 1454 1455 /* 1456 * lwp_specific_key_create -- 1457 * Create a key for subsystem lwp-specific data. 1458 */ 1459 int 1460 lwp_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor) 1461 { 1462 1463 return (specificdata_key_create(lwp_specificdata_domain, keyp, dtor)); 1464 } 1465 1466 /* 1467 * lwp_specific_key_delete -- 1468 * Delete a key for subsystem lwp-specific data. 1469 */ 1470 void 1471 lwp_specific_key_delete(specificdata_key_t key) 1472 { 1473 1474 specificdata_key_delete(lwp_specificdata_domain, key); 1475 } 1476 1477 /* 1478 * lwp_initspecific -- 1479 * Initialize an LWP's specificdata container. 1480 */ 1481 void 1482 lwp_initspecific(struct lwp *l) 1483 { 1484 int error; 1485 1486 error = specificdata_init(lwp_specificdata_domain, &l->l_specdataref); 1487 KASSERT(error == 0); 1488 } 1489 1490 /* 1491 * lwp_finispecific -- 1492 * Finalize an LWP's specificdata container. 1493 */ 1494 void 1495 lwp_finispecific(struct lwp *l) 1496 { 1497 1498 specificdata_fini(lwp_specificdata_domain, &l->l_specdataref); 1499 } 1500 1501 /* 1502 * lwp_getspecific -- 1503 * Return lwp-specific data corresponding to the specified key. 1504 * 1505 * Note: LWP specific data is NOT INTERLOCKED. An LWP should access 1506 * only its OWN SPECIFIC DATA. If it is necessary to access another 1507 * LWP's specifc data, care must be taken to ensure that doing so 1508 * would not cause internal data structure inconsistency (i.e. caller 1509 * can guarantee that the target LWP is not inside an lwp_getspecific() 1510 * or lwp_setspecific() call). 1511 */ 1512 void * 1513 lwp_getspecific(specificdata_key_t key) 1514 { 1515 1516 return (specificdata_getspecific_unlocked(lwp_specificdata_domain, 1517 &curlwp->l_specdataref, key)); 1518 } 1519 1520 void * 1521 _lwp_getspecific_by_lwp(struct lwp *l, specificdata_key_t key) 1522 { 1523 1524 return (specificdata_getspecific_unlocked(lwp_specificdata_domain, 1525 &l->l_specdataref, key)); 1526 } 1527 1528 /* 1529 * lwp_setspecific -- 1530 * Set lwp-specific data corresponding to the specified key. 1531 */ 1532 void 1533 lwp_setspecific(specificdata_key_t key, void *data) 1534 { 1535 1536 specificdata_setspecific(lwp_specificdata_domain, 1537 &curlwp->l_specdataref, key, data); 1538 } 1539 1540 /* 1541 * Allocate a new lwpctl structure for a user LWP. 1542 */ 1543 int 1544 lwp_ctl_alloc(vaddr_t *uaddr) 1545 { 1546 lcproc_t *lp; 1547 u_int bit, i, offset; 1548 struct uvm_object *uao; 1549 int error; 1550 lcpage_t *lcp; 1551 proc_t *p; 1552 lwp_t *l; 1553 1554 l = curlwp; 1555 p = l->l_proc; 1556 1557 if (l->l_lcpage != NULL) { 1558 lcp = l->l_lcpage; 1559 *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr; 1560 return (EINVAL); 1561 } 1562 1563 /* First time around, allocate header structure for the process. */ 1564 if ((lp = p->p_lwpctl) == NULL) { 1565 lp = kmem_alloc(sizeof(*lp), KM_SLEEP); 1566 mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE); 1567 lp->lp_uao = NULL; 1568 TAILQ_INIT(&lp->lp_pages); 1569 mutex_enter(p->p_lock); 1570 if (p->p_lwpctl == NULL) { 1571 p->p_lwpctl = lp; 1572 mutex_exit(p->p_lock); 1573 } else { 1574 mutex_exit(p->p_lock); 1575 mutex_destroy(&lp->lp_lock); 1576 kmem_free(lp, sizeof(*lp)); 1577 lp = p->p_lwpctl; 1578 } 1579 } 1580 1581 /* 1582 * Set up an anonymous memory region to hold the shared pages. 1583 * Map them into the process' address space. The user vmspace 1584 * gets the first reference on the UAO. 1585 */ 1586 mutex_enter(&lp->lp_lock); 1587 if (lp->lp_uao == NULL) { 1588 lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0); 1589 lp->lp_cur = 0; 1590 lp->lp_max = LWPCTL_UAREA_SZ; 1591 lp->lp_uva = p->p_emul->e_vm_default_addr(p, 1592 (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ); 1593 error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva, 1594 LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW, 1595 UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0)); 1596 if (error != 0) { 1597 uao_detach(lp->lp_uao); 1598 lp->lp_uao = NULL; 1599 mutex_exit(&lp->lp_lock); 1600 return error; 1601 } 1602 } 1603 1604 /* Get a free block and allocate for this LWP. */ 1605 TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) { 1606 if (lcp->lcp_nfree != 0) 1607 break; 1608 } 1609 if (lcp == NULL) { 1610 /* Nothing available - try to set up a free page. */ 1611 if (lp->lp_cur == lp->lp_max) { 1612 mutex_exit(&lp->lp_lock); 1613 return ENOMEM; 1614 } 1615 lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP); 1616 if (lcp == NULL) { 1617 mutex_exit(&lp->lp_lock); 1618 return ENOMEM; 1619 } 1620 /* 1621 * Wire the next page down in kernel space. Since this 1622 * is a new mapping, we must add a reference. 1623 */ 1624 uao = lp->lp_uao; 1625 (*uao->pgops->pgo_reference)(uao); 1626 lcp->lcp_kaddr = vm_map_min(kernel_map); 1627 error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE, 1628 uao, lp->lp_cur, PAGE_SIZE, 1629 UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, 1630 UVM_INH_NONE, UVM_ADV_RANDOM, 0)); 1631 if (error != 0) { 1632 mutex_exit(&lp->lp_lock); 1633 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1634 (*uao->pgops->pgo_detach)(uao); 1635 return error; 1636 } 1637 error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr, 1638 lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0); 1639 if (error != 0) { 1640 mutex_exit(&lp->lp_lock); 1641 uvm_unmap(kernel_map, lcp->lcp_kaddr, 1642 lcp->lcp_kaddr + PAGE_SIZE); 1643 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1644 return error; 1645 } 1646 /* Prepare the page descriptor and link into the list. */ 1647 lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur; 1648 lp->lp_cur += PAGE_SIZE; 1649 lcp->lcp_nfree = LWPCTL_PER_PAGE; 1650 lcp->lcp_rotor = 0; 1651 memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ); 1652 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain); 1653 } 1654 for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) { 1655 if (++i >= LWPCTL_BITMAP_ENTRIES) 1656 i = 0; 1657 } 1658 bit = ffs(lcp->lcp_bitmap[i]) - 1; 1659 lcp->lcp_bitmap[i] ^= (1 << bit); 1660 lcp->lcp_rotor = i; 1661 lcp->lcp_nfree--; 1662 l->l_lcpage = lcp; 1663 offset = (i << 5) + bit; 1664 l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset; 1665 *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t); 1666 mutex_exit(&lp->lp_lock); 1667 1668 KPREEMPT_DISABLE(l); 1669 l->l_lwpctl->lc_curcpu = (int)curcpu()->ci_data.cpu_index; 1670 KPREEMPT_ENABLE(l); 1671 1672 return 0; 1673 } 1674 1675 /* 1676 * Free an lwpctl structure back to the per-process list. 1677 */ 1678 void 1679 lwp_ctl_free(lwp_t *l) 1680 { 1681 lcproc_t *lp; 1682 lcpage_t *lcp; 1683 u_int map, offset; 1684 1685 lp = l->l_proc->p_lwpctl; 1686 KASSERT(lp != NULL); 1687 1688 lcp = l->l_lcpage; 1689 offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr); 1690 KASSERT(offset < LWPCTL_PER_PAGE); 1691 1692 mutex_enter(&lp->lp_lock); 1693 lcp->lcp_nfree++; 1694 map = offset >> 5; 1695 lcp->lcp_bitmap[map] |= (1 << (offset & 31)); 1696 if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0) 1697 lcp->lcp_rotor = map; 1698 if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) { 1699 TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain); 1700 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain); 1701 } 1702 mutex_exit(&lp->lp_lock); 1703 } 1704 1705 /* 1706 * Process is exiting; tear down lwpctl state. This can only be safely 1707 * called by the last LWP in the process. 1708 */ 1709 void 1710 lwp_ctl_exit(void) 1711 { 1712 lcpage_t *lcp, *next; 1713 lcproc_t *lp; 1714 proc_t *p; 1715 lwp_t *l; 1716 1717 l = curlwp; 1718 l->l_lwpctl = NULL; 1719 l->l_lcpage = NULL; 1720 p = l->l_proc; 1721 lp = p->p_lwpctl; 1722 1723 KASSERT(lp != NULL); 1724 KASSERT(p->p_nlwps == 1); 1725 1726 for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) { 1727 next = TAILQ_NEXT(lcp, lcp_chain); 1728 uvm_unmap(kernel_map, lcp->lcp_kaddr, 1729 lcp->lcp_kaddr + PAGE_SIZE); 1730 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1731 } 1732 1733 if (lp->lp_uao != NULL) { 1734 uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva, 1735 lp->lp_uva + LWPCTL_UAREA_SZ); 1736 } 1737 1738 mutex_destroy(&lp->lp_lock); 1739 kmem_free(lp, sizeof(*lp)); 1740 p->p_lwpctl = NULL; 1741 } 1742 1743 /* 1744 * Return the current LWP's "preemption counter". Used to detect 1745 * preemption across operations that can tolerate preemption without 1746 * crashing, but which may generate incorrect results if preempted. 1747 */ 1748 uint64_t 1749 lwp_pctr(void) 1750 { 1751 1752 return curlwp->l_ncsw; 1753 } 1754 1755 #if defined(DDB) 1756 void 1757 lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...)) 1758 { 1759 lwp_t *l; 1760 1761 LIST_FOREACH(l, &alllwp, l_list) { 1762 uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l); 1763 1764 if (addr < stack || stack + KSTACK_SIZE <= addr) { 1765 continue; 1766 } 1767 (*pr)("%p is %p+%zu, LWP %p's stack\n", 1768 (void *)addr, (void *)stack, 1769 (size_t)(addr - stack), l); 1770 } 1771 } 1772 #endif /* defined(DDB) */ 1773