1 /* $NetBSD: kern_lwp.c,v 1.201 2019/05/17 03:34:26 ozaki-r Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nathan J. Williams, and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Overview 34 * 35 * Lightweight processes (LWPs) are the basic unit or thread of 36 * execution within the kernel. The core state of an LWP is described 37 * by "struct lwp", also known as lwp_t. 38 * 39 * Each LWP is contained within a process (described by "struct proc"), 40 * Every process contains at least one LWP, but may contain more. The 41 * process describes attributes shared among all of its LWPs such as a 42 * private address space, global execution state (stopped, active, 43 * zombie, ...), signal disposition and so on. On a multiprocessor 44 * machine, multiple LWPs be executing concurrently in the kernel. 45 * 46 * Execution states 47 * 48 * At any given time, an LWP has overall state that is described by 49 * lwp::l_stat. The states are broken into two sets below. The first 50 * set is guaranteed to represent the absolute, current state of the 51 * LWP: 52 * 53 * LSONPROC 54 * 55 * On processor: the LWP is executing on a CPU, either in the 56 * kernel or in user space. 57 * 58 * LSRUN 59 * 60 * Runnable: the LWP is parked on a run queue, and may soon be 61 * chosen to run by an idle processor, or by a processor that 62 * has been asked to preempt a currently runnning but lower 63 * priority LWP. 64 * 65 * LSIDL 66 * 67 * Idle: the LWP has been created but has not yet executed, 68 * or it has ceased executing a unit of work and is waiting 69 * to be started again. 70 * 71 * LSSUSPENDED: 72 * 73 * Suspended: the LWP has had its execution suspended by 74 * another LWP in the same process using the _lwp_suspend() 75 * system call. User-level LWPs also enter the suspended 76 * state when the system is shutting down. 77 * 78 * The second set represent a "statement of intent" on behalf of the 79 * LWP. The LWP may in fact be executing on a processor, may be 80 * sleeping or idle. It is expected to take the necessary action to 81 * stop executing or become "running" again within a short timeframe. 82 * The LP_RUNNING flag in lwp::l_pflag indicates that an LWP is running. 83 * Importantly, it indicates that its state is tied to a CPU. 84 * 85 * LSZOMB: 86 * 87 * Dead or dying: the LWP has released most of its resources 88 * and is about to switch away into oblivion, or has already 89 * switched away. When it switches away, its few remaining 90 * resources can be collected. 91 * 92 * LSSLEEP: 93 * 94 * Sleeping: the LWP has entered itself onto a sleep queue, and 95 * has switched away or will switch away shortly to allow other 96 * LWPs to run on the CPU. 97 * 98 * LSSTOP: 99 * 100 * Stopped: the LWP has been stopped as a result of a job 101 * control signal, or as a result of the ptrace() interface. 102 * 103 * Stopped LWPs may run briefly within the kernel to handle 104 * signals that they receive, but will not return to user space 105 * until their process' state is changed away from stopped. 106 * 107 * Single LWPs within a process can not be set stopped 108 * selectively: all actions that can stop or continue LWPs 109 * occur at the process level. 110 * 111 * State transitions 112 * 113 * Note that the LSSTOP state may only be set when returning to 114 * user space in userret(), or when sleeping interruptably. The 115 * LSSUSPENDED state may only be set in userret(). Before setting 116 * those states, we try to ensure that the LWPs will release all 117 * locks that they hold, and at a minimum try to ensure that the 118 * LWP can be set runnable again by a signal. 119 * 120 * LWPs may transition states in the following ways: 121 * 122 * RUN -------> ONPROC ONPROC -----> RUN 123 * > SLEEP 124 * > STOPPED 125 * > SUSPENDED 126 * > ZOMB 127 * > IDL (special cases) 128 * 129 * STOPPED ---> RUN SUSPENDED --> RUN 130 * > SLEEP 131 * 132 * SLEEP -----> ONPROC IDL --------> RUN 133 * > RUN > SUSPENDED 134 * > STOPPED > STOPPED 135 * > ONPROC (special cases) 136 * 137 * Some state transitions are only possible with kernel threads (eg 138 * ONPROC -> IDL) and happen under tightly controlled circumstances 139 * free of unwanted side effects. 140 * 141 * Migration 142 * 143 * Migration of threads from one CPU to another could be performed 144 * internally by the scheduler via sched_takecpu() or sched_catchlwp() 145 * functions. The universal lwp_migrate() function should be used for 146 * any other cases. Subsystems in the kernel must be aware that CPU 147 * of LWP may change, while it is not locked. 148 * 149 * Locking 150 * 151 * The majority of fields in 'struct lwp' are covered by a single, 152 * general spin lock pointed to by lwp::l_mutex. The locks covering 153 * each field are documented in sys/lwp.h. 154 * 155 * State transitions must be made with the LWP's general lock held, 156 * and may cause the LWP's lock pointer to change. Manipulation of 157 * the general lock is not performed directly, but through calls to 158 * lwp_lock(), lwp_unlock() and others. It should be noted that the 159 * adaptive locks are not allowed to be released while the LWP's lock 160 * is being held (unlike for other spin-locks). 161 * 162 * States and their associated locks: 163 * 164 * LSONPROC, LSZOMB: 165 * 166 * Always covered by spc_lwplock, which protects running LWPs. 167 * This is a per-CPU lock and matches lwp::l_cpu. 168 * 169 * LSIDL, LSRUN: 170 * 171 * Always covered by spc_mutex, which protects the run queues. 172 * This is a per-CPU lock and matches lwp::l_cpu. 173 * 174 * LSSLEEP: 175 * 176 * Covered by a lock associated with the sleep queue that the 177 * LWP resides on. Matches lwp::l_sleepq::sq_mutex. 178 * 179 * LSSTOP, LSSUSPENDED: 180 * 181 * If the LWP was previously sleeping (l_wchan != NULL), then 182 * l_mutex references the sleep queue lock. If the LWP was 183 * runnable or on the CPU when halted, or has been removed from 184 * the sleep queue since halted, then the lock is spc_lwplock. 185 * 186 * The lock order is as follows: 187 * 188 * spc::spc_lwplock -> 189 * sleeptab::st_mutex -> 190 * tschain_t::tc_mutex -> 191 * spc::spc_mutex 192 * 193 * Each process has an scheduler state lock (proc::p_lock), and a 194 * number of counters on LWPs and their states: p_nzlwps, p_nrlwps, and 195 * so on. When an LWP is to be entered into or removed from one of the 196 * following states, p_lock must be held and the process wide counters 197 * adjusted: 198 * 199 * LSIDL, LSZOMB, LSSTOP, LSSUSPENDED 200 * 201 * (But not always for kernel threads. There are some special cases 202 * as mentioned above. See kern_softint.c.) 203 * 204 * Note that an LWP is considered running or likely to run soon if in 205 * one of the following states. This affects the value of p_nrlwps: 206 * 207 * LSRUN, LSONPROC, LSSLEEP 208 * 209 * p_lock does not need to be held when transitioning among these 210 * three states, hence p_lock is rarely taken for state transitions. 211 */ 212 213 #include <sys/cdefs.h> 214 __KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.201 2019/05/17 03:34:26 ozaki-r Exp $"); 215 216 #include "opt_ddb.h" 217 #include "opt_lockdebug.h" 218 #include "opt_dtrace.h" 219 220 #define _LWP_API_PRIVATE 221 222 #include <sys/param.h> 223 #include <sys/systm.h> 224 #include <sys/cpu.h> 225 #include <sys/pool.h> 226 #include <sys/proc.h> 227 #include <sys/syscallargs.h> 228 #include <sys/syscall_stats.h> 229 #include <sys/kauth.h> 230 #include <sys/pserialize.h> 231 #include <sys/sleepq.h> 232 #include <sys/lockdebug.h> 233 #include <sys/kmem.h> 234 #include <sys/pset.h> 235 #include <sys/intr.h> 236 #include <sys/lwpctl.h> 237 #include <sys/atomic.h> 238 #include <sys/filedesc.h> 239 #include <sys/fstrans.h> 240 #include <sys/dtrace_bsd.h> 241 #include <sys/sdt.h> 242 #include <sys/xcall.h> 243 #include <sys/uidinfo.h> 244 #include <sys/sysctl.h> 245 #include <sys/psref.h> 246 247 #include <uvm/uvm_extern.h> 248 #include <uvm/uvm_object.h> 249 250 static pool_cache_t lwp_cache __read_mostly; 251 struct lwplist alllwp __cacheline_aligned; 252 253 static void lwp_dtor(void *, void *); 254 255 /* DTrace proc provider probes */ 256 SDT_PROVIDER_DEFINE(proc); 257 258 SDT_PROBE_DEFINE1(proc, kernel, , lwp__create, "struct lwp *"); 259 SDT_PROBE_DEFINE1(proc, kernel, , lwp__start, "struct lwp *"); 260 SDT_PROBE_DEFINE1(proc, kernel, , lwp__exit, "struct lwp *"); 261 262 struct turnstile turnstile0; 263 struct lwp lwp0 __aligned(MIN_LWP_ALIGNMENT) = { 264 #ifdef LWP0_CPU_INFO 265 .l_cpu = LWP0_CPU_INFO, 266 #endif 267 #ifdef LWP0_MD_INITIALIZER 268 .l_md = LWP0_MD_INITIALIZER, 269 #endif 270 .l_proc = &proc0, 271 .l_lid = 1, 272 .l_flag = LW_SYSTEM, 273 .l_stat = LSONPROC, 274 .l_ts = &turnstile0, 275 .l_syncobj = &sched_syncobj, 276 .l_refcnt = 1, 277 .l_priority = PRI_USER + NPRI_USER - 1, 278 .l_inheritedprio = -1, 279 .l_class = SCHED_OTHER, 280 .l_psid = PS_NONE, 281 .l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders), 282 .l_name = __UNCONST("swapper"), 283 .l_fd = &filedesc0, 284 }; 285 286 static int sysctl_kern_maxlwp(SYSCTLFN_PROTO); 287 288 /* 289 * sysctl helper routine for kern.maxlwp. Ensures that the new 290 * values are not too low or too high. 291 */ 292 static int 293 sysctl_kern_maxlwp(SYSCTLFN_ARGS) 294 { 295 int error, nmaxlwp; 296 struct sysctlnode node; 297 298 nmaxlwp = maxlwp; 299 node = *rnode; 300 node.sysctl_data = &nmaxlwp; 301 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 302 if (error || newp == NULL) 303 return error; 304 305 if (nmaxlwp < 0 || nmaxlwp >= 65536) 306 return EINVAL; 307 if (nmaxlwp > cpu_maxlwp()) 308 return EINVAL; 309 maxlwp = nmaxlwp; 310 311 return 0; 312 } 313 314 static void 315 sysctl_kern_lwp_setup(void) 316 { 317 struct sysctllog *clog = NULL; 318 319 sysctl_createv(&clog, 0, NULL, NULL, 320 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 321 CTLTYPE_INT, "maxlwp", 322 SYSCTL_DESCR("Maximum number of simultaneous threads"), 323 sysctl_kern_maxlwp, 0, NULL, 0, 324 CTL_KERN, CTL_CREATE, CTL_EOL); 325 } 326 327 void 328 lwpinit(void) 329 { 330 331 LIST_INIT(&alllwp); 332 lwpinit_specificdata(); 333 lwp_sys_init(); 334 lwp_cache = pool_cache_init(sizeof(lwp_t), MIN_LWP_ALIGNMENT, 0, 0, 335 "lwppl", NULL, IPL_NONE, NULL, lwp_dtor, NULL); 336 337 maxlwp = cpu_maxlwp(); 338 sysctl_kern_lwp_setup(); 339 } 340 341 void 342 lwp0_init(void) 343 { 344 struct lwp *l = &lwp0; 345 346 KASSERT((void *)uvm_lwp_getuarea(l) != NULL); 347 KASSERT(l->l_lid == proc0.p_nlwpid); 348 349 LIST_INSERT_HEAD(&alllwp, l, l_list); 350 351 callout_init(&l->l_timeout_ch, CALLOUT_MPSAFE); 352 callout_setfunc(&l->l_timeout_ch, sleepq_timeout, l); 353 cv_init(&l->l_sigcv, "sigwait"); 354 cv_init(&l->l_waitcv, "vfork"); 355 356 kauth_cred_hold(proc0.p_cred); 357 l->l_cred = proc0.p_cred; 358 359 kdtrace_thread_ctor(NULL, l); 360 lwp_initspecific(l); 361 362 SYSCALL_TIME_LWP_INIT(l); 363 } 364 365 static void 366 lwp_dtor(void *arg, void *obj) 367 { 368 lwp_t *l = obj; 369 uint64_t where; 370 (void)l; 371 372 /* 373 * Provide a barrier to ensure that all mutex_oncpu() and rw_oncpu() 374 * calls will exit before memory of LWP is returned to the pool, where 375 * KVA of LWP structure might be freed and re-used for other purposes. 376 * Kernel preemption is disabled around mutex_oncpu() and rw_oncpu() 377 * callers, therefore cross-call to all CPUs will do the job. Also, 378 * the value of l->l_cpu must be still valid at this point. 379 */ 380 KASSERT(l->l_cpu != NULL); 381 where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL); 382 xc_wait(where); 383 } 384 385 /* 386 * Set an suspended. 387 * 388 * Must be called with p_lock held, and the LWP locked. Will unlock the 389 * LWP before return. 390 */ 391 int 392 lwp_suspend(struct lwp *curl, struct lwp *t) 393 { 394 int error; 395 396 KASSERT(mutex_owned(t->l_proc->p_lock)); 397 KASSERT(lwp_locked(t, NULL)); 398 399 KASSERT(curl != t || curl->l_stat == LSONPROC); 400 401 /* 402 * If the current LWP has been told to exit, we must not suspend anyone 403 * else or deadlock could occur. We won't return to userspace. 404 */ 405 if ((curl->l_flag & (LW_WEXIT | LW_WCORE)) != 0) { 406 lwp_unlock(t); 407 return (EDEADLK); 408 } 409 410 error = 0; 411 412 switch (t->l_stat) { 413 case LSRUN: 414 case LSONPROC: 415 t->l_flag |= LW_WSUSPEND; 416 lwp_need_userret(t); 417 lwp_unlock(t); 418 break; 419 420 case LSSLEEP: 421 t->l_flag |= LW_WSUSPEND; 422 423 /* 424 * Kick the LWP and try to get it to the kernel boundary 425 * so that it will release any locks that it holds. 426 * setrunnable() will release the lock. 427 */ 428 if ((t->l_flag & LW_SINTR) != 0) 429 setrunnable(t); 430 else 431 lwp_unlock(t); 432 break; 433 434 case LSSUSPENDED: 435 lwp_unlock(t); 436 break; 437 438 case LSSTOP: 439 t->l_flag |= LW_WSUSPEND; 440 setrunnable(t); 441 break; 442 443 case LSIDL: 444 case LSZOMB: 445 error = EINTR; /* It's what Solaris does..... */ 446 lwp_unlock(t); 447 break; 448 } 449 450 return (error); 451 } 452 453 /* 454 * Restart a suspended LWP. 455 * 456 * Must be called with p_lock held, and the LWP locked. Will unlock the 457 * LWP before return. 458 */ 459 void 460 lwp_continue(struct lwp *l) 461 { 462 463 KASSERT(mutex_owned(l->l_proc->p_lock)); 464 KASSERT(lwp_locked(l, NULL)); 465 466 /* If rebooting or not suspended, then just bail out. */ 467 if ((l->l_flag & LW_WREBOOT) != 0) { 468 lwp_unlock(l); 469 return; 470 } 471 472 l->l_flag &= ~LW_WSUSPEND; 473 474 if (l->l_stat != LSSUSPENDED) { 475 lwp_unlock(l); 476 return; 477 } 478 479 /* setrunnable() will release the lock. */ 480 setrunnable(l); 481 } 482 483 /* 484 * Restart a stopped LWP. 485 * 486 * Must be called with p_lock held, and the LWP NOT locked. Will unlock the 487 * LWP before return. 488 */ 489 void 490 lwp_unstop(struct lwp *l) 491 { 492 struct proc *p = l->l_proc; 493 494 KASSERT(mutex_owned(proc_lock)); 495 KASSERT(mutex_owned(p->p_lock)); 496 497 lwp_lock(l); 498 499 /* If not stopped, then just bail out. */ 500 if (l->l_stat != LSSTOP) { 501 lwp_unlock(l); 502 return; 503 } 504 505 p->p_stat = SACTIVE; 506 p->p_sflag &= ~PS_STOPPING; 507 508 if (!p->p_waited) 509 p->p_pptr->p_nstopchild--; 510 511 if (l->l_wchan == NULL) { 512 /* setrunnable() will release the lock. */ 513 setrunnable(l); 514 } else if (p->p_xsig && (l->l_flag & LW_SINTR) != 0) { 515 /* setrunnable() so we can receive the signal */ 516 setrunnable(l); 517 } else { 518 l->l_stat = LSSLEEP; 519 p->p_nrlwps++; 520 lwp_unlock(l); 521 } 522 } 523 524 /* 525 * Wait for an LWP within the current process to exit. If 'lid' is 526 * non-zero, we are waiting for a specific LWP. 527 * 528 * Must be called with p->p_lock held. 529 */ 530 int 531 lwp_wait(struct lwp *l, lwpid_t lid, lwpid_t *departed, bool exiting) 532 { 533 const lwpid_t curlid = l->l_lid; 534 proc_t *p = l->l_proc; 535 lwp_t *l2; 536 int error; 537 538 KASSERT(mutex_owned(p->p_lock)); 539 540 p->p_nlwpwait++; 541 l->l_waitingfor = lid; 542 543 for (;;) { 544 int nfound; 545 546 /* 547 * Avoid a race between exit1() and sigexit(): if the 548 * process is dumping core, then we need to bail out: call 549 * into lwp_userret() where we will be suspended until the 550 * deed is done. 551 */ 552 if ((p->p_sflag & PS_WCORE) != 0) { 553 mutex_exit(p->p_lock); 554 lwp_userret(l); 555 KASSERT(false); 556 } 557 558 /* 559 * First off, drain any detached LWP that is waiting to be 560 * reaped. 561 */ 562 while ((l2 = p->p_zomblwp) != NULL) { 563 p->p_zomblwp = NULL; 564 lwp_free(l2, false, false);/* releases proc mutex */ 565 mutex_enter(p->p_lock); 566 } 567 568 /* 569 * Now look for an LWP to collect. If the whole process is 570 * exiting, count detached LWPs as eligible to be collected, 571 * but don't drain them here. 572 */ 573 nfound = 0; 574 error = 0; 575 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 576 /* 577 * If a specific wait and the target is waiting on 578 * us, then avoid deadlock. This also traps LWPs 579 * that try to wait on themselves. 580 * 581 * Note that this does not handle more complicated 582 * cycles, like: t1 -> t2 -> t3 -> t1. The process 583 * can still be killed so it is not a major problem. 584 */ 585 if (l2->l_lid == lid && l2->l_waitingfor == curlid) { 586 error = EDEADLK; 587 break; 588 } 589 if (l2 == l) 590 continue; 591 if ((l2->l_prflag & LPR_DETACHED) != 0) { 592 nfound += exiting; 593 continue; 594 } 595 if (lid != 0) { 596 if (l2->l_lid != lid) 597 continue; 598 /* 599 * Mark this LWP as the first waiter, if there 600 * is no other. 601 */ 602 if (l2->l_waiter == 0) 603 l2->l_waiter = curlid; 604 } else if (l2->l_waiter != 0) { 605 /* 606 * It already has a waiter - so don't 607 * collect it. If the waiter doesn't 608 * grab it we'll get another chance 609 * later. 610 */ 611 nfound++; 612 continue; 613 } 614 nfound++; 615 616 /* No need to lock the LWP in order to see LSZOMB. */ 617 if (l2->l_stat != LSZOMB) 618 continue; 619 620 /* 621 * We're no longer waiting. Reset the "first waiter" 622 * pointer on the target, in case it was us. 623 */ 624 l->l_waitingfor = 0; 625 l2->l_waiter = 0; 626 p->p_nlwpwait--; 627 if (departed) 628 *departed = l2->l_lid; 629 sched_lwp_collect(l2); 630 631 /* lwp_free() releases the proc lock. */ 632 lwp_free(l2, false, false); 633 mutex_enter(p->p_lock); 634 return 0; 635 } 636 637 if (error != 0) 638 break; 639 if (nfound == 0) { 640 error = ESRCH; 641 break; 642 } 643 644 /* 645 * Note: since the lock will be dropped, need to restart on 646 * wakeup to run all LWPs again, e.g. there may be new LWPs. 647 */ 648 if (exiting) { 649 KASSERT(p->p_nlwps > 1); 650 cv_wait(&p->p_lwpcv, p->p_lock); 651 error = EAGAIN; 652 break; 653 } 654 655 /* 656 * If all other LWPs are waiting for exits or suspends 657 * and the supply of zombies and potential zombies is 658 * exhausted, then we are about to deadlock. 659 * 660 * If the process is exiting (and this LWP is not the one 661 * that is coordinating the exit) then bail out now. 662 */ 663 if ((p->p_sflag & PS_WEXIT) != 0 || 664 p->p_nrlwps + p->p_nzlwps - p->p_ndlwps <= p->p_nlwpwait) { 665 error = EDEADLK; 666 break; 667 } 668 669 /* 670 * Sit around and wait for something to happen. We'll be 671 * awoken if any of the conditions examined change: if an 672 * LWP exits, is collected, or is detached. 673 */ 674 if ((error = cv_wait_sig(&p->p_lwpcv, p->p_lock)) != 0) 675 break; 676 } 677 678 /* 679 * We didn't find any LWPs to collect, we may have received a 680 * signal, or some other condition has caused us to bail out. 681 * 682 * If waiting on a specific LWP, clear the waiters marker: some 683 * other LWP may want it. Then, kick all the remaining waiters 684 * so that they can re-check for zombies and for deadlock. 685 */ 686 if (lid != 0) { 687 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 688 if (l2->l_lid == lid) { 689 if (l2->l_waiter == curlid) 690 l2->l_waiter = 0; 691 break; 692 } 693 } 694 } 695 p->p_nlwpwait--; 696 l->l_waitingfor = 0; 697 cv_broadcast(&p->p_lwpcv); 698 699 return error; 700 } 701 702 static lwpid_t 703 lwp_find_free_lid(lwpid_t try_lid, lwp_t * new_lwp, proc_t *p) 704 { 705 #define LID_SCAN (1u << 31) 706 lwp_t *scan, *free_before; 707 lwpid_t nxt_lid; 708 709 /* 710 * We want the first unused lid greater than or equal to 711 * try_lid (modulo 2^31). 712 * (If nothing else ld.elf_so doesn't want lwpid with the top bit set.) 713 * We must not return 0, and avoiding 'LID_SCAN - 1' makes 714 * the outer test easier. 715 * This would be much easier if the list were sorted in 716 * increasing order. 717 * The list is kept sorted in decreasing order. 718 * This code is only used after a process has generated 2^31 lwp. 719 * 720 * Code assumes it can always find an id. 721 */ 722 723 try_lid &= LID_SCAN - 1; 724 if (try_lid <= 1) 725 try_lid = 2; 726 727 free_before = NULL; 728 nxt_lid = LID_SCAN - 1; 729 LIST_FOREACH(scan, &p->p_lwps, l_sibling) { 730 if (scan->l_lid != nxt_lid) { 731 /* There are available lid before this entry */ 732 free_before = scan; 733 if (try_lid > scan->l_lid) 734 break; 735 } 736 if (try_lid == scan->l_lid) { 737 /* The ideal lid is busy, take a higher one */ 738 if (free_before != NULL) { 739 try_lid = free_before->l_lid + 1; 740 break; 741 } 742 /* No higher ones, reuse low numbers */ 743 try_lid = 2; 744 } 745 746 nxt_lid = scan->l_lid - 1; 747 if (LIST_NEXT(scan, l_sibling) == NULL) { 748 /* The value we have is lower than any existing lwp */ 749 LIST_INSERT_AFTER(scan, new_lwp, l_sibling); 750 return try_lid; 751 } 752 } 753 754 LIST_INSERT_BEFORE(free_before, new_lwp, l_sibling); 755 return try_lid; 756 } 757 758 /* 759 * Create a new LWP within process 'p2', using LWP 'l1' as a template. 760 * The new LWP is created in state LSIDL and must be set running, 761 * suspended, or stopped by the caller. 762 */ 763 int 764 lwp_create(lwp_t *l1, proc_t *p2, vaddr_t uaddr, int flags, 765 void *stack, size_t stacksize, void (*func)(void *), void *arg, 766 lwp_t **rnewlwpp, int sclass, const sigset_t *sigmask, 767 const stack_t *sigstk) 768 { 769 struct lwp *l2, *isfree; 770 turnstile_t *ts; 771 lwpid_t lid; 772 773 KASSERT(l1 == curlwp || l1->l_proc == &proc0); 774 775 /* 776 * Enforce limits, excluding the first lwp and kthreads. 777 */ 778 if (p2->p_nlwps != 0 && p2 != &proc0) { 779 uid_t uid = kauth_cred_getuid(l1->l_cred); 780 int count = chglwpcnt(uid, 1); 781 if (__predict_false(count > 782 p2->p_rlimit[RLIMIT_NTHR].rlim_cur)) { 783 if (kauth_authorize_process(l1->l_cred, 784 KAUTH_PROCESS_RLIMIT, p2, 785 KAUTH_ARG(KAUTH_REQ_PROCESS_RLIMIT_BYPASS), 786 &p2->p_rlimit[RLIMIT_NTHR], KAUTH_ARG(RLIMIT_NTHR)) 787 != 0) { 788 (void)chglwpcnt(uid, -1); 789 return EAGAIN; 790 } 791 } 792 } 793 794 /* 795 * First off, reap any detached LWP waiting to be collected. 796 * We can re-use its LWP structure and turnstile. 797 */ 798 isfree = NULL; 799 if (p2->p_zomblwp != NULL) { 800 mutex_enter(p2->p_lock); 801 if ((isfree = p2->p_zomblwp) != NULL) { 802 p2->p_zomblwp = NULL; 803 lwp_free(isfree, true, false);/* releases proc mutex */ 804 } else 805 mutex_exit(p2->p_lock); 806 } 807 if (isfree == NULL) { 808 l2 = pool_cache_get(lwp_cache, PR_WAITOK); 809 memset(l2, 0, sizeof(*l2)); 810 l2->l_ts = pool_cache_get(turnstile_cache, PR_WAITOK); 811 SLIST_INIT(&l2->l_pi_lenders); 812 } else { 813 l2 = isfree; 814 ts = l2->l_ts; 815 KASSERT(l2->l_inheritedprio == -1); 816 KASSERT(SLIST_EMPTY(&l2->l_pi_lenders)); 817 memset(l2, 0, sizeof(*l2)); 818 l2->l_ts = ts; 819 } 820 821 l2->l_stat = LSIDL; 822 l2->l_proc = p2; 823 l2->l_refcnt = 1; 824 l2->l_class = sclass; 825 826 /* 827 * If vfork(), we want the LWP to run fast and on the same CPU 828 * as its parent, so that it can reuse the VM context and cache 829 * footprint on the local CPU. 830 */ 831 l2->l_kpriority = ((flags & LWP_VFORK) ? true : false); 832 l2->l_kpribase = PRI_KERNEL; 833 l2->l_priority = l1->l_priority; 834 l2->l_inheritedprio = -1; 835 l2->l_protectprio = -1; 836 l2->l_auxprio = -1; 837 l2->l_flag = 0; 838 l2->l_pflag = LP_MPSAFE; 839 TAILQ_INIT(&l2->l_ld_locks); 840 l2->l_psrefs = 0; 841 842 /* 843 * For vfork, borrow parent's lwpctl context if it exists. 844 * This also causes us to return via lwp_userret. 845 */ 846 if (flags & LWP_VFORK && l1->l_lwpctl) { 847 l2->l_lwpctl = l1->l_lwpctl; 848 l2->l_flag |= LW_LWPCTL; 849 } 850 851 /* 852 * If not the first LWP in the process, grab a reference to the 853 * descriptor table. 854 */ 855 l2->l_fd = p2->p_fd; 856 if (p2->p_nlwps != 0) { 857 KASSERT(l1->l_proc == p2); 858 fd_hold(l2); 859 } else { 860 KASSERT(l1->l_proc != p2); 861 } 862 863 if (p2->p_flag & PK_SYSTEM) { 864 /* Mark it as a system LWP. */ 865 l2->l_flag |= LW_SYSTEM; 866 } 867 868 kpreempt_disable(); 869 l2->l_mutex = l1->l_cpu->ci_schedstate.spc_mutex; 870 l2->l_cpu = l1->l_cpu; 871 kpreempt_enable(); 872 873 kdtrace_thread_ctor(NULL, l2); 874 lwp_initspecific(l2); 875 sched_lwp_fork(l1, l2); 876 lwp_update_creds(l2); 877 callout_init(&l2->l_timeout_ch, CALLOUT_MPSAFE); 878 callout_setfunc(&l2->l_timeout_ch, sleepq_timeout, l2); 879 cv_init(&l2->l_sigcv, "sigwait"); 880 cv_init(&l2->l_waitcv, "vfork"); 881 l2->l_syncobj = &sched_syncobj; 882 PSREF_DEBUG_INIT_LWP(l2); 883 884 if (rnewlwpp != NULL) 885 *rnewlwpp = l2; 886 887 /* 888 * PCU state needs to be saved before calling uvm_lwp_fork() so that 889 * the MD cpu_lwp_fork() can copy the saved state to the new LWP. 890 */ 891 pcu_save_all(l1); 892 893 uvm_lwp_setuarea(l2, uaddr); 894 uvm_lwp_fork(l1, l2, stack, stacksize, func, (arg != NULL) ? arg : l2); 895 896 if ((flags & LWP_PIDLID) != 0) { 897 lid = proc_alloc_pid(p2); 898 l2->l_pflag |= LP_PIDLID; 899 } else { 900 lid = 0; 901 } 902 903 mutex_enter(p2->p_lock); 904 905 if ((flags & LWP_DETACHED) != 0) { 906 l2->l_prflag = LPR_DETACHED; 907 p2->p_ndlwps++; 908 } else 909 l2->l_prflag = 0; 910 911 l2->l_sigstk = *sigstk; 912 l2->l_sigmask = *sigmask; 913 TAILQ_INIT(&l2->l_sigpend.sp_info); 914 sigemptyset(&l2->l_sigpend.sp_set); 915 916 if (__predict_true(lid == 0)) { 917 /* 918 * XXX: l_lid are expected to be unique (for a process) 919 * if LWP_PIDLID is sometimes set this won't be true. 920 * Once 2^31 threads have been allocated we have to 921 * scan to ensure we allocate a unique value. 922 */ 923 lid = ++p2->p_nlwpid; 924 if (__predict_false(lid & LID_SCAN)) { 925 lid = lwp_find_free_lid(lid, l2, p2); 926 p2->p_nlwpid = lid | LID_SCAN; 927 /* l2 as been inserted into p_lwps in order */ 928 goto skip_insert; 929 } 930 p2->p_nlwpid = lid; 931 } 932 LIST_INSERT_HEAD(&p2->p_lwps, l2, l_sibling); 933 skip_insert: 934 l2->l_lid = lid; 935 p2->p_nlwps++; 936 p2->p_nrlwps++; 937 938 KASSERT(l2->l_affinity == NULL); 939 940 if ((p2->p_flag & PK_SYSTEM) == 0) { 941 /* Inherit the affinity mask. */ 942 if (l1->l_affinity) { 943 /* 944 * Note that we hold the state lock while inheriting 945 * the affinity to avoid race with sched_setaffinity(). 946 */ 947 lwp_lock(l1); 948 if (l1->l_affinity) { 949 kcpuset_use(l1->l_affinity); 950 l2->l_affinity = l1->l_affinity; 951 } 952 lwp_unlock(l1); 953 } 954 lwp_lock(l2); 955 /* Inherit a processor-set */ 956 l2->l_psid = l1->l_psid; 957 /* Look for a CPU to start */ 958 l2->l_cpu = sched_takecpu(l2); 959 lwp_unlock_to(l2, l2->l_cpu->ci_schedstate.spc_mutex); 960 } 961 mutex_exit(p2->p_lock); 962 963 SDT_PROBE(proc, kernel, , lwp__create, l2, 0, 0, 0, 0); 964 965 mutex_enter(proc_lock); 966 LIST_INSERT_HEAD(&alllwp, l2, l_list); 967 mutex_exit(proc_lock); 968 969 SYSCALL_TIME_LWP_INIT(l2); 970 971 if (p2->p_emul->e_lwp_fork) 972 (*p2->p_emul->e_lwp_fork)(l1, l2); 973 974 return (0); 975 } 976 977 /* 978 * Called by MD code when a new LWP begins execution. Must be called 979 * with the previous LWP locked (so at splsched), or if there is no 980 * previous LWP, at splsched. 981 */ 982 void 983 lwp_startup(struct lwp *prev, struct lwp *new_lwp) 984 { 985 KASSERTMSG(new_lwp == curlwp, "l %p curlwp %p prevlwp %p", new_lwp, curlwp, prev); 986 987 SDT_PROBE(proc, kernel, , lwp__start, new_lwp, 0, 0, 0, 0); 988 989 KASSERT(kpreempt_disabled()); 990 if (prev != NULL) { 991 /* 992 * Normalize the count of the spin-mutexes, it was 993 * increased in mi_switch(). Unmark the state of 994 * context switch - it is finished for previous LWP. 995 */ 996 curcpu()->ci_mtx_count++; 997 membar_exit(); 998 prev->l_ctxswtch = 0; 999 } 1000 KPREEMPT_DISABLE(new_lwp); 1001 if (__predict_true(new_lwp->l_proc->p_vmspace)) 1002 pmap_activate(new_lwp); 1003 spl0(); 1004 1005 /* Note trip through cpu_switchto(). */ 1006 pserialize_switchpoint(); 1007 1008 LOCKDEBUG_BARRIER(NULL, 0); 1009 KPREEMPT_ENABLE(new_lwp); 1010 if ((new_lwp->l_pflag & LP_MPSAFE) == 0) { 1011 KERNEL_LOCK(1, new_lwp); 1012 } 1013 } 1014 1015 /* 1016 * Exit an LWP. 1017 */ 1018 void 1019 lwp_exit(struct lwp *l) 1020 { 1021 struct proc *p = l->l_proc; 1022 struct lwp *l2; 1023 bool current; 1024 1025 current = (l == curlwp); 1026 1027 KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL)); 1028 KASSERT(p == curproc); 1029 1030 SDT_PROBE(proc, kernel, , lwp__exit, l, 0, 0, 0, 0); 1031 1032 /* 1033 * Verify that we hold no locks other than the kernel lock. 1034 */ 1035 LOCKDEBUG_BARRIER(&kernel_lock, 0); 1036 1037 /* 1038 * If we are the last live LWP in a process, we need to exit the 1039 * entire process. We do so with an exit status of zero, because 1040 * it's a "controlled" exit, and because that's what Solaris does. 1041 * 1042 * We are not quite a zombie yet, but for accounting purposes we 1043 * must increment the count of zombies here. 1044 * 1045 * Note: the last LWP's specificdata will be deleted here. 1046 */ 1047 mutex_enter(p->p_lock); 1048 if (p->p_nlwps - p->p_nzlwps == 1) { 1049 KASSERT(current == true); 1050 KASSERT(p != &proc0); 1051 /* XXXSMP kernel_lock not held */ 1052 exit1(l, 0, 0); 1053 /* NOTREACHED */ 1054 } 1055 p->p_nzlwps++; 1056 mutex_exit(p->p_lock); 1057 1058 if (p->p_emul->e_lwp_exit) 1059 (*p->p_emul->e_lwp_exit)(l); 1060 1061 /* Drop filedesc reference. */ 1062 fd_free(); 1063 1064 /* Release fstrans private data. */ 1065 fstrans_lwp_dtor(l); 1066 1067 /* Delete the specificdata while it's still safe to sleep. */ 1068 lwp_finispecific(l); 1069 1070 /* 1071 * Release our cached credentials. 1072 */ 1073 kauth_cred_free(l->l_cred); 1074 callout_destroy(&l->l_timeout_ch); 1075 1076 /* 1077 * If traced, report LWP exit event to the debugger. 1078 * 1079 * Remove the LWP from the global list. 1080 * Free its LID from the PID namespace if needed. 1081 */ 1082 mutex_enter(proc_lock); 1083 1084 if ((p->p_slflag & (PSL_TRACED|PSL_TRACELWP_EXIT)) == 1085 (PSL_TRACED|PSL_TRACELWP_EXIT)) { 1086 mutex_enter(p->p_lock); 1087 p->p_lwp_exited = l->l_lid; 1088 eventswitch(TRAP_LWP); 1089 mutex_enter(proc_lock); 1090 } 1091 1092 LIST_REMOVE(l, l_list); 1093 if ((l->l_pflag & LP_PIDLID) != 0 && l->l_lid != p->p_pid) { 1094 proc_free_pid(l->l_lid); 1095 } 1096 mutex_exit(proc_lock); 1097 1098 /* 1099 * Get rid of all references to the LWP that others (e.g. procfs) 1100 * may have, and mark the LWP as a zombie. If the LWP is detached, 1101 * mark it waiting for collection in the proc structure. Note that 1102 * before we can do that, we need to free any other dead, deatched 1103 * LWP waiting to meet its maker. 1104 */ 1105 mutex_enter(p->p_lock); 1106 lwp_drainrefs(l); 1107 1108 if ((l->l_prflag & LPR_DETACHED) != 0) { 1109 while ((l2 = p->p_zomblwp) != NULL) { 1110 p->p_zomblwp = NULL; 1111 lwp_free(l2, false, false);/* releases proc mutex */ 1112 mutex_enter(p->p_lock); 1113 l->l_refcnt++; 1114 lwp_drainrefs(l); 1115 } 1116 p->p_zomblwp = l; 1117 } 1118 1119 /* 1120 * If we find a pending signal for the process and we have been 1121 * asked to check for signals, then we lose: arrange to have 1122 * all other LWPs in the process check for signals. 1123 */ 1124 if ((l->l_flag & LW_PENDSIG) != 0 && 1125 firstsig(&p->p_sigpend.sp_set) != 0) { 1126 LIST_FOREACH(l2, &p->p_lwps, l_sibling) { 1127 lwp_lock(l2); 1128 l2->l_flag |= LW_PENDSIG; 1129 lwp_unlock(l2); 1130 } 1131 } 1132 1133 /* 1134 * Release any PCU resources before becoming a zombie. 1135 */ 1136 pcu_discard_all(l); 1137 1138 lwp_lock(l); 1139 l->l_stat = LSZOMB; 1140 if (l->l_name != NULL) { 1141 strcpy(l->l_name, "(zombie)"); 1142 } 1143 lwp_unlock(l); 1144 p->p_nrlwps--; 1145 cv_broadcast(&p->p_lwpcv); 1146 if (l->l_lwpctl != NULL) 1147 l->l_lwpctl->lc_curcpu = LWPCTL_CPU_EXITED; 1148 mutex_exit(p->p_lock); 1149 1150 /* 1151 * We can no longer block. At this point, lwp_free() may already 1152 * be gunning for us. On a multi-CPU system, we may be off p_lwps. 1153 * 1154 * Free MD LWP resources. 1155 */ 1156 cpu_lwp_free(l, 0); 1157 1158 if (current) { 1159 pmap_deactivate(l); 1160 1161 /* 1162 * Release the kernel lock, and switch away into 1163 * oblivion. 1164 */ 1165 #ifdef notyet 1166 /* XXXSMP hold in lwp_userret() */ 1167 KERNEL_UNLOCK_LAST(l); 1168 #else 1169 KERNEL_UNLOCK_ALL(l, NULL); 1170 #endif 1171 lwp_exit_switchaway(l); 1172 } 1173 } 1174 1175 /* 1176 * Free a dead LWP's remaining resources. 1177 * 1178 * XXXLWP limits. 1179 */ 1180 void 1181 lwp_free(struct lwp *l, bool recycle, bool last) 1182 { 1183 struct proc *p = l->l_proc; 1184 struct rusage *ru; 1185 ksiginfoq_t kq; 1186 1187 KASSERT(l != curlwp); 1188 KASSERT(last || mutex_owned(p->p_lock)); 1189 1190 /* 1191 * We use the process credentials instead of the lwp credentials here 1192 * because the lwp credentials maybe cached (just after a setuid call) 1193 * and we don't want pay for syncing, since the lwp is going away 1194 * anyway 1195 */ 1196 if (p != &proc0 && p->p_nlwps != 1) 1197 (void)chglwpcnt(kauth_cred_getuid(p->p_cred), -1); 1198 /* 1199 * If this was not the last LWP in the process, then adjust 1200 * counters and unlock. 1201 */ 1202 if (!last) { 1203 /* 1204 * Add the LWP's run time to the process' base value. 1205 * This needs to co-incide with coming off p_lwps. 1206 */ 1207 bintime_add(&p->p_rtime, &l->l_rtime); 1208 p->p_pctcpu += l->l_pctcpu; 1209 ru = &p->p_stats->p_ru; 1210 ruadd(ru, &l->l_ru); 1211 ru->ru_nvcsw += (l->l_ncsw - l->l_nivcsw); 1212 ru->ru_nivcsw += l->l_nivcsw; 1213 LIST_REMOVE(l, l_sibling); 1214 p->p_nlwps--; 1215 p->p_nzlwps--; 1216 if ((l->l_prflag & LPR_DETACHED) != 0) 1217 p->p_ndlwps--; 1218 1219 /* 1220 * Have any LWPs sleeping in lwp_wait() recheck for 1221 * deadlock. 1222 */ 1223 cv_broadcast(&p->p_lwpcv); 1224 mutex_exit(p->p_lock); 1225 } 1226 1227 #ifdef MULTIPROCESSOR 1228 /* 1229 * In the unlikely event that the LWP is still on the CPU, 1230 * then spin until it has switched away. We need to release 1231 * all locks to avoid deadlock against interrupt handlers on 1232 * the target CPU. 1233 */ 1234 if ((l->l_pflag & LP_RUNNING) != 0 || l->l_cpu->ci_curlwp == l) { 1235 int count; 1236 (void)count; /* XXXgcc */ 1237 KERNEL_UNLOCK_ALL(curlwp, &count); 1238 while ((l->l_pflag & LP_RUNNING) != 0 || 1239 l->l_cpu->ci_curlwp == l) 1240 SPINLOCK_BACKOFF_HOOK; 1241 KERNEL_LOCK(count, curlwp); 1242 } 1243 #endif 1244 1245 /* 1246 * Destroy the LWP's remaining signal information. 1247 */ 1248 ksiginfo_queue_init(&kq); 1249 sigclear(&l->l_sigpend, NULL, &kq); 1250 ksiginfo_queue_drain(&kq); 1251 cv_destroy(&l->l_sigcv); 1252 cv_destroy(&l->l_waitcv); 1253 1254 /* 1255 * Free lwpctl structure and affinity. 1256 */ 1257 if (l->l_lwpctl) { 1258 lwp_ctl_free(l); 1259 } 1260 if (l->l_affinity) { 1261 kcpuset_unuse(l->l_affinity, NULL); 1262 l->l_affinity = NULL; 1263 } 1264 1265 /* 1266 * Free the LWP's turnstile and the LWP structure itself unless the 1267 * caller wants to recycle them. Also, free the scheduler specific 1268 * data. 1269 * 1270 * We can't return turnstile0 to the pool (it didn't come from it), 1271 * so if it comes up just drop it quietly and move on. 1272 * 1273 * We don't recycle the VM resources at this time. 1274 */ 1275 1276 if (!recycle && l->l_ts != &turnstile0) 1277 pool_cache_put(turnstile_cache, l->l_ts); 1278 if (l->l_name != NULL) 1279 kmem_free(l->l_name, MAXCOMLEN); 1280 1281 cpu_lwp_free2(l); 1282 uvm_lwp_exit(l); 1283 1284 KASSERT(SLIST_EMPTY(&l->l_pi_lenders)); 1285 KASSERT(l->l_inheritedprio == -1); 1286 KASSERT(l->l_blcnt == 0); 1287 kdtrace_thread_dtor(NULL, l); 1288 if (!recycle) 1289 pool_cache_put(lwp_cache, l); 1290 } 1291 1292 /* 1293 * Migrate the LWP to the another CPU. Unlocks the LWP. 1294 */ 1295 void 1296 lwp_migrate(lwp_t *l, struct cpu_info *tci) 1297 { 1298 struct schedstate_percpu *tspc; 1299 int lstat = l->l_stat; 1300 1301 KASSERT(lwp_locked(l, NULL)); 1302 KASSERT(tci != NULL); 1303 1304 /* If LWP is still on the CPU, it must be handled like LSONPROC */ 1305 if ((l->l_pflag & LP_RUNNING) != 0) { 1306 lstat = LSONPROC; 1307 } 1308 1309 /* 1310 * The destination CPU could be changed while previous migration 1311 * was not finished. 1312 */ 1313 if (l->l_target_cpu != NULL) { 1314 l->l_target_cpu = tci; 1315 lwp_unlock(l); 1316 return; 1317 } 1318 1319 /* Nothing to do if trying to migrate to the same CPU */ 1320 if (l->l_cpu == tci) { 1321 lwp_unlock(l); 1322 return; 1323 } 1324 1325 KASSERT(l->l_target_cpu == NULL); 1326 tspc = &tci->ci_schedstate; 1327 switch (lstat) { 1328 case LSRUN: 1329 l->l_target_cpu = tci; 1330 break; 1331 case LSIDL: 1332 l->l_cpu = tci; 1333 lwp_unlock_to(l, tspc->spc_mutex); 1334 return; 1335 case LSSLEEP: 1336 l->l_cpu = tci; 1337 break; 1338 case LSSTOP: 1339 case LSSUSPENDED: 1340 l->l_cpu = tci; 1341 if (l->l_wchan == NULL) { 1342 lwp_unlock_to(l, tspc->spc_lwplock); 1343 return; 1344 } 1345 break; 1346 case LSONPROC: 1347 l->l_target_cpu = tci; 1348 spc_lock(l->l_cpu); 1349 cpu_need_resched(l->l_cpu, RESCHED_KPREEMPT); 1350 spc_unlock(l->l_cpu); 1351 break; 1352 } 1353 lwp_unlock(l); 1354 } 1355 1356 /* 1357 * Find the LWP in the process. Arguments may be zero, in such case, 1358 * the calling process and first LWP in the list will be used. 1359 * On success - returns proc locked. 1360 */ 1361 struct lwp * 1362 lwp_find2(pid_t pid, lwpid_t lid) 1363 { 1364 proc_t *p; 1365 lwp_t *l; 1366 1367 /* Find the process. */ 1368 if (pid != 0) { 1369 mutex_enter(proc_lock); 1370 p = proc_find(pid); 1371 if (p == NULL) { 1372 mutex_exit(proc_lock); 1373 return NULL; 1374 } 1375 mutex_enter(p->p_lock); 1376 mutex_exit(proc_lock); 1377 } else { 1378 p = curlwp->l_proc; 1379 mutex_enter(p->p_lock); 1380 } 1381 /* Find the thread. */ 1382 if (lid != 0) { 1383 l = lwp_find(p, lid); 1384 } else { 1385 l = LIST_FIRST(&p->p_lwps); 1386 } 1387 if (l == NULL) { 1388 mutex_exit(p->p_lock); 1389 } 1390 return l; 1391 } 1392 1393 /* 1394 * Look up a live LWP within the specified process. 1395 * 1396 * Must be called with p->p_lock held. 1397 */ 1398 struct lwp * 1399 lwp_find(struct proc *p, lwpid_t id) 1400 { 1401 struct lwp *l; 1402 1403 KASSERT(mutex_owned(p->p_lock)); 1404 1405 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1406 if (l->l_lid == id) 1407 break; 1408 } 1409 1410 /* 1411 * No need to lock - all of these conditions will 1412 * be visible with the process level mutex held. 1413 */ 1414 if (l != NULL && (l->l_stat == LSIDL || l->l_stat == LSZOMB)) 1415 l = NULL; 1416 1417 return l; 1418 } 1419 1420 /* 1421 * Update an LWP's cached credentials to mirror the process' master copy. 1422 * 1423 * This happens early in the syscall path, on user trap, and on LWP 1424 * creation. A long-running LWP can also voluntarily choose to update 1425 * its credentials by calling this routine. This may be called from 1426 * LWP_CACHE_CREDS(), which checks l->l_cred != p->p_cred beforehand. 1427 */ 1428 void 1429 lwp_update_creds(struct lwp *l) 1430 { 1431 kauth_cred_t oc; 1432 struct proc *p; 1433 1434 p = l->l_proc; 1435 oc = l->l_cred; 1436 1437 mutex_enter(p->p_lock); 1438 kauth_cred_hold(p->p_cred); 1439 l->l_cred = p->p_cred; 1440 l->l_prflag &= ~LPR_CRMOD; 1441 mutex_exit(p->p_lock); 1442 if (oc != NULL) 1443 kauth_cred_free(oc); 1444 } 1445 1446 /* 1447 * Verify that an LWP is locked, and optionally verify that the lock matches 1448 * one we specify. 1449 */ 1450 int 1451 lwp_locked(struct lwp *l, kmutex_t *mtx) 1452 { 1453 kmutex_t *cur = l->l_mutex; 1454 1455 return mutex_owned(cur) && (mtx == cur || mtx == NULL); 1456 } 1457 1458 /* 1459 * Lend a new mutex to an LWP. The old mutex must be held. 1460 */ 1461 void 1462 lwp_setlock(struct lwp *l, kmutex_t *mtx) 1463 { 1464 1465 KASSERT(mutex_owned(l->l_mutex)); 1466 1467 membar_exit(); 1468 l->l_mutex = mtx; 1469 } 1470 1471 /* 1472 * Lend a new mutex to an LWP, and release the old mutex. The old mutex 1473 * must be held. 1474 */ 1475 void 1476 lwp_unlock_to(struct lwp *l, kmutex_t *mtx) 1477 { 1478 kmutex_t *old; 1479 1480 KASSERT(lwp_locked(l, NULL)); 1481 1482 old = l->l_mutex; 1483 membar_exit(); 1484 l->l_mutex = mtx; 1485 mutex_spin_exit(old); 1486 } 1487 1488 int 1489 lwp_trylock(struct lwp *l) 1490 { 1491 kmutex_t *old; 1492 1493 for (;;) { 1494 if (!mutex_tryenter(old = l->l_mutex)) 1495 return 0; 1496 if (__predict_true(l->l_mutex == old)) 1497 return 1; 1498 mutex_spin_exit(old); 1499 } 1500 } 1501 1502 void 1503 lwp_unsleep(lwp_t *l, bool cleanup) 1504 { 1505 1506 KASSERT(mutex_owned(l->l_mutex)); 1507 (*l->l_syncobj->sobj_unsleep)(l, cleanup); 1508 } 1509 1510 /* 1511 * Handle exceptions for mi_userret(). Called if a member of LW_USERRET is 1512 * set. 1513 */ 1514 void 1515 lwp_userret(struct lwp *l) 1516 { 1517 struct proc *p; 1518 int sig; 1519 1520 KASSERT(l == curlwp); 1521 KASSERT(l->l_stat == LSONPROC); 1522 p = l->l_proc; 1523 1524 #ifndef __HAVE_FAST_SOFTINTS 1525 /* Run pending soft interrupts. */ 1526 if (l->l_cpu->ci_data.cpu_softints != 0) 1527 softint_overlay(); 1528 #endif 1529 1530 /* 1531 * It is safe to do this read unlocked on a MP system.. 1532 */ 1533 while ((l->l_flag & LW_USERRET) != 0) { 1534 /* 1535 * Process pending signals first, unless the process 1536 * is dumping core or exiting, where we will instead 1537 * enter the LW_WSUSPEND case below. 1538 */ 1539 if ((l->l_flag & (LW_PENDSIG | LW_WCORE | LW_WEXIT)) == 1540 LW_PENDSIG) { 1541 mutex_enter(p->p_lock); 1542 while ((sig = issignal(l)) != 0) 1543 postsig(sig); 1544 mutex_exit(p->p_lock); 1545 } 1546 1547 /* 1548 * Core-dump or suspend pending. 1549 * 1550 * In case of core dump, suspend ourselves, so that the kernel 1551 * stack and therefore the userland registers saved in the 1552 * trapframe are around for coredump() to write them out. 1553 * We also need to save any PCU resources that we have so that 1554 * they accessible for coredump(). We issue a wakeup on 1555 * p->p_lwpcv so that sigexit() will write the core file out 1556 * once all other LWPs are suspended. 1557 */ 1558 if ((l->l_flag & LW_WSUSPEND) != 0) { 1559 pcu_save_all(l); 1560 mutex_enter(p->p_lock); 1561 p->p_nrlwps--; 1562 cv_broadcast(&p->p_lwpcv); 1563 lwp_lock(l); 1564 l->l_stat = LSSUSPENDED; 1565 lwp_unlock(l); 1566 mutex_exit(p->p_lock); 1567 lwp_lock(l); 1568 mi_switch(l); 1569 } 1570 1571 /* Process is exiting. */ 1572 if ((l->l_flag & LW_WEXIT) != 0) { 1573 lwp_exit(l); 1574 KASSERT(0); 1575 /* NOTREACHED */ 1576 } 1577 1578 /* update lwpctl processor (for vfork child_return) */ 1579 if (l->l_flag & LW_LWPCTL) { 1580 lwp_lock(l); 1581 KASSERT(kpreempt_disabled()); 1582 l->l_lwpctl->lc_curcpu = (int)cpu_index(l->l_cpu); 1583 l->l_lwpctl->lc_pctr++; 1584 l->l_flag &= ~LW_LWPCTL; 1585 lwp_unlock(l); 1586 } 1587 } 1588 } 1589 1590 /* 1591 * Force an LWP to enter the kernel, to take a trip through lwp_userret(). 1592 */ 1593 void 1594 lwp_need_userret(struct lwp *l) 1595 { 1596 KASSERT(lwp_locked(l, NULL)); 1597 1598 /* 1599 * Since the tests in lwp_userret() are done unlocked, make sure 1600 * that the condition will be seen before forcing the LWP to enter 1601 * kernel mode. 1602 */ 1603 membar_producer(); 1604 cpu_signotify(l); 1605 } 1606 1607 /* 1608 * Add one reference to an LWP. This will prevent the LWP from 1609 * exiting, thus keep the lwp structure and PCB around to inspect. 1610 */ 1611 void 1612 lwp_addref(struct lwp *l) 1613 { 1614 1615 KASSERT(mutex_owned(l->l_proc->p_lock)); 1616 KASSERT(l->l_stat != LSZOMB); 1617 KASSERT(l->l_refcnt != 0); 1618 1619 l->l_refcnt++; 1620 } 1621 1622 /* 1623 * Remove one reference to an LWP. If this is the last reference, 1624 * then we must finalize the LWP's death. 1625 */ 1626 void 1627 lwp_delref(struct lwp *l) 1628 { 1629 struct proc *p = l->l_proc; 1630 1631 mutex_enter(p->p_lock); 1632 lwp_delref2(l); 1633 mutex_exit(p->p_lock); 1634 } 1635 1636 /* 1637 * Remove one reference to an LWP. If this is the last reference, 1638 * then we must finalize the LWP's death. The proc mutex is held 1639 * on entry. 1640 */ 1641 void 1642 lwp_delref2(struct lwp *l) 1643 { 1644 struct proc *p = l->l_proc; 1645 1646 KASSERT(mutex_owned(p->p_lock)); 1647 KASSERT(l->l_stat != LSZOMB); 1648 KASSERT(l->l_refcnt > 0); 1649 if (--l->l_refcnt == 0) 1650 cv_broadcast(&p->p_lwpcv); 1651 } 1652 1653 /* 1654 * Drain all references to the current LWP. 1655 */ 1656 void 1657 lwp_drainrefs(struct lwp *l) 1658 { 1659 struct proc *p = l->l_proc; 1660 1661 KASSERT(mutex_owned(p->p_lock)); 1662 KASSERT(l->l_refcnt != 0); 1663 1664 l->l_refcnt--; 1665 while (l->l_refcnt != 0) 1666 cv_wait(&p->p_lwpcv, p->p_lock); 1667 } 1668 1669 /* 1670 * Return true if the specified LWP is 'alive'. Only p->p_lock need 1671 * be held. 1672 */ 1673 bool 1674 lwp_alive(lwp_t *l) 1675 { 1676 1677 KASSERT(mutex_owned(l->l_proc->p_lock)); 1678 1679 switch (l->l_stat) { 1680 case LSSLEEP: 1681 case LSRUN: 1682 case LSONPROC: 1683 case LSSTOP: 1684 case LSSUSPENDED: 1685 return true; 1686 default: 1687 return false; 1688 } 1689 } 1690 1691 /* 1692 * Return first live LWP in the process. 1693 */ 1694 lwp_t * 1695 lwp_find_first(proc_t *p) 1696 { 1697 lwp_t *l; 1698 1699 KASSERT(mutex_owned(p->p_lock)); 1700 1701 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 1702 if (lwp_alive(l)) { 1703 return l; 1704 } 1705 } 1706 1707 return NULL; 1708 } 1709 1710 /* 1711 * Allocate a new lwpctl structure for a user LWP. 1712 */ 1713 int 1714 lwp_ctl_alloc(vaddr_t *uaddr) 1715 { 1716 lcproc_t *lp; 1717 u_int bit, i, offset; 1718 struct uvm_object *uao; 1719 int error; 1720 lcpage_t *lcp; 1721 proc_t *p; 1722 lwp_t *l; 1723 1724 l = curlwp; 1725 p = l->l_proc; 1726 1727 /* don't allow a vforked process to create lwp ctls */ 1728 if (p->p_lflag & PL_PPWAIT) 1729 return EBUSY; 1730 1731 if (l->l_lcpage != NULL) { 1732 lcp = l->l_lcpage; 1733 *uaddr = lcp->lcp_uaddr + (vaddr_t)l->l_lwpctl - lcp->lcp_kaddr; 1734 return 0; 1735 } 1736 1737 /* First time around, allocate header structure for the process. */ 1738 if ((lp = p->p_lwpctl) == NULL) { 1739 lp = kmem_alloc(sizeof(*lp), KM_SLEEP); 1740 mutex_init(&lp->lp_lock, MUTEX_DEFAULT, IPL_NONE); 1741 lp->lp_uao = NULL; 1742 TAILQ_INIT(&lp->lp_pages); 1743 mutex_enter(p->p_lock); 1744 if (p->p_lwpctl == NULL) { 1745 p->p_lwpctl = lp; 1746 mutex_exit(p->p_lock); 1747 } else { 1748 mutex_exit(p->p_lock); 1749 mutex_destroy(&lp->lp_lock); 1750 kmem_free(lp, sizeof(*lp)); 1751 lp = p->p_lwpctl; 1752 } 1753 } 1754 1755 /* 1756 * Set up an anonymous memory region to hold the shared pages. 1757 * Map them into the process' address space. The user vmspace 1758 * gets the first reference on the UAO. 1759 */ 1760 mutex_enter(&lp->lp_lock); 1761 if (lp->lp_uao == NULL) { 1762 lp->lp_uao = uao_create(LWPCTL_UAREA_SZ, 0); 1763 lp->lp_cur = 0; 1764 lp->lp_max = LWPCTL_UAREA_SZ; 1765 lp->lp_uva = p->p_emul->e_vm_default_addr(p, 1766 (vaddr_t)p->p_vmspace->vm_daddr, LWPCTL_UAREA_SZ, 1767 p->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN); 1768 error = uvm_map(&p->p_vmspace->vm_map, &lp->lp_uva, 1769 LWPCTL_UAREA_SZ, lp->lp_uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW, 1770 UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_NORMAL, 0)); 1771 if (error != 0) { 1772 uao_detach(lp->lp_uao); 1773 lp->lp_uao = NULL; 1774 mutex_exit(&lp->lp_lock); 1775 return error; 1776 } 1777 } 1778 1779 /* Get a free block and allocate for this LWP. */ 1780 TAILQ_FOREACH(lcp, &lp->lp_pages, lcp_chain) { 1781 if (lcp->lcp_nfree != 0) 1782 break; 1783 } 1784 if (lcp == NULL) { 1785 /* Nothing available - try to set up a free page. */ 1786 if (lp->lp_cur == lp->lp_max) { 1787 mutex_exit(&lp->lp_lock); 1788 return ENOMEM; 1789 } 1790 lcp = kmem_alloc(LWPCTL_LCPAGE_SZ, KM_SLEEP); 1791 1792 /* 1793 * Wire the next page down in kernel space. Since this 1794 * is a new mapping, we must add a reference. 1795 */ 1796 uao = lp->lp_uao; 1797 (*uao->pgops->pgo_reference)(uao); 1798 lcp->lcp_kaddr = vm_map_min(kernel_map); 1799 error = uvm_map(kernel_map, &lcp->lcp_kaddr, PAGE_SIZE, 1800 uao, lp->lp_cur, PAGE_SIZE, 1801 UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, 1802 UVM_INH_NONE, UVM_ADV_RANDOM, 0)); 1803 if (error != 0) { 1804 mutex_exit(&lp->lp_lock); 1805 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1806 (*uao->pgops->pgo_detach)(uao); 1807 return error; 1808 } 1809 error = uvm_map_pageable(kernel_map, lcp->lcp_kaddr, 1810 lcp->lcp_kaddr + PAGE_SIZE, FALSE, 0); 1811 if (error != 0) { 1812 mutex_exit(&lp->lp_lock); 1813 uvm_unmap(kernel_map, lcp->lcp_kaddr, 1814 lcp->lcp_kaddr + PAGE_SIZE); 1815 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1816 return error; 1817 } 1818 /* Prepare the page descriptor and link into the list. */ 1819 lcp->lcp_uaddr = lp->lp_uva + lp->lp_cur; 1820 lp->lp_cur += PAGE_SIZE; 1821 lcp->lcp_nfree = LWPCTL_PER_PAGE; 1822 lcp->lcp_rotor = 0; 1823 memset(lcp->lcp_bitmap, 0xff, LWPCTL_BITMAP_SZ); 1824 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain); 1825 } 1826 for (i = lcp->lcp_rotor; lcp->lcp_bitmap[i] == 0;) { 1827 if (++i >= LWPCTL_BITMAP_ENTRIES) 1828 i = 0; 1829 } 1830 bit = ffs(lcp->lcp_bitmap[i]) - 1; 1831 lcp->lcp_bitmap[i] ^= (1U << bit); 1832 lcp->lcp_rotor = i; 1833 lcp->lcp_nfree--; 1834 l->l_lcpage = lcp; 1835 offset = (i << 5) + bit; 1836 l->l_lwpctl = (lwpctl_t *)lcp->lcp_kaddr + offset; 1837 *uaddr = lcp->lcp_uaddr + offset * sizeof(lwpctl_t); 1838 mutex_exit(&lp->lp_lock); 1839 1840 KPREEMPT_DISABLE(l); 1841 l->l_lwpctl->lc_curcpu = (int)cpu_index(curcpu()); 1842 KPREEMPT_ENABLE(l); 1843 1844 return 0; 1845 } 1846 1847 /* 1848 * Free an lwpctl structure back to the per-process list. 1849 */ 1850 void 1851 lwp_ctl_free(lwp_t *l) 1852 { 1853 struct proc *p = l->l_proc; 1854 lcproc_t *lp; 1855 lcpage_t *lcp; 1856 u_int map, offset; 1857 1858 /* don't free a lwp context we borrowed for vfork */ 1859 if (p->p_lflag & PL_PPWAIT) { 1860 l->l_lwpctl = NULL; 1861 return; 1862 } 1863 1864 lp = p->p_lwpctl; 1865 KASSERT(lp != NULL); 1866 1867 lcp = l->l_lcpage; 1868 offset = (u_int)((lwpctl_t *)l->l_lwpctl - (lwpctl_t *)lcp->lcp_kaddr); 1869 KASSERT(offset < LWPCTL_PER_PAGE); 1870 1871 mutex_enter(&lp->lp_lock); 1872 lcp->lcp_nfree++; 1873 map = offset >> 5; 1874 lcp->lcp_bitmap[map] |= (1U << (offset & 31)); 1875 if (lcp->lcp_bitmap[lcp->lcp_rotor] == 0) 1876 lcp->lcp_rotor = map; 1877 if (TAILQ_FIRST(&lp->lp_pages)->lcp_nfree == 0) { 1878 TAILQ_REMOVE(&lp->lp_pages, lcp, lcp_chain); 1879 TAILQ_INSERT_HEAD(&lp->lp_pages, lcp, lcp_chain); 1880 } 1881 mutex_exit(&lp->lp_lock); 1882 } 1883 1884 /* 1885 * Process is exiting; tear down lwpctl state. This can only be safely 1886 * called by the last LWP in the process. 1887 */ 1888 void 1889 lwp_ctl_exit(void) 1890 { 1891 lcpage_t *lcp, *next; 1892 lcproc_t *lp; 1893 proc_t *p; 1894 lwp_t *l; 1895 1896 l = curlwp; 1897 l->l_lwpctl = NULL; 1898 l->l_lcpage = NULL; 1899 p = l->l_proc; 1900 lp = p->p_lwpctl; 1901 1902 KASSERT(lp != NULL); 1903 KASSERT(p->p_nlwps == 1); 1904 1905 for (lcp = TAILQ_FIRST(&lp->lp_pages); lcp != NULL; lcp = next) { 1906 next = TAILQ_NEXT(lcp, lcp_chain); 1907 uvm_unmap(kernel_map, lcp->lcp_kaddr, 1908 lcp->lcp_kaddr + PAGE_SIZE); 1909 kmem_free(lcp, LWPCTL_LCPAGE_SZ); 1910 } 1911 1912 if (lp->lp_uao != NULL) { 1913 uvm_unmap(&p->p_vmspace->vm_map, lp->lp_uva, 1914 lp->lp_uva + LWPCTL_UAREA_SZ); 1915 } 1916 1917 mutex_destroy(&lp->lp_lock); 1918 kmem_free(lp, sizeof(*lp)); 1919 p->p_lwpctl = NULL; 1920 } 1921 1922 /* 1923 * Return the current LWP's "preemption counter". Used to detect 1924 * preemption across operations that can tolerate preemption without 1925 * crashing, but which may generate incorrect results if preempted. 1926 */ 1927 uint64_t 1928 lwp_pctr(void) 1929 { 1930 1931 return curlwp->l_ncsw; 1932 } 1933 1934 /* 1935 * Set an LWP's private data pointer. 1936 */ 1937 int 1938 lwp_setprivate(struct lwp *l, void *ptr) 1939 { 1940 int error = 0; 1941 1942 l->l_private = ptr; 1943 #ifdef __HAVE_CPU_LWP_SETPRIVATE 1944 error = cpu_lwp_setprivate(l, ptr); 1945 #endif 1946 return error; 1947 } 1948 1949 #if defined(DDB) 1950 #include <machine/pcb.h> 1951 1952 void 1953 lwp_whatis(uintptr_t addr, void (*pr)(const char *, ...)) 1954 { 1955 lwp_t *l; 1956 1957 LIST_FOREACH(l, &alllwp, l_list) { 1958 uintptr_t stack = (uintptr_t)KSTACK_LOWEST_ADDR(l); 1959 1960 if (addr < stack || stack + KSTACK_SIZE <= addr) { 1961 continue; 1962 } 1963 (*pr)("%p is %p+%zu, LWP %p's stack\n", 1964 (void *)addr, (void *)stack, 1965 (size_t)(addr - stack), l); 1966 } 1967 } 1968 #endif /* defined(DDB) */ 1969