1 /* $NetBSD: pthread.c,v 1.161 2020/01/29 16:03:44 kamil Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Nathan J. Williams and Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __RCSID("$NetBSD: pthread.c,v 1.161 2020/01/29 16:03:44 kamil Exp $"); 35 36 #define __EXPOSE_STACK 1 37 38 #include <sys/param.h> 39 #include <sys/exec_elf.h> 40 #include <sys/mman.h> 41 #include <sys/lwp.h> 42 #include <sys/lwpctl.h> 43 #include <sys/resource.h> 44 #include <sys/sysctl.h> 45 #include <sys/tls.h> 46 #include <uvm/uvm_param.h> 47 48 #include <assert.h> 49 #include <dlfcn.h> 50 #include <err.h> 51 #include <errno.h> 52 #include <lwp.h> 53 #include <signal.h> 54 #include <stdio.h> 55 #include <stdlib.h> 56 #include <stddef.h> 57 #include <string.h> 58 #include <syslog.h> 59 #include <ucontext.h> 60 #include <unistd.h> 61 #include <sched.h> 62 63 #include "atexit.h" 64 #include "pthread.h" 65 #include "pthread_int.h" 66 #include "pthread_makelwp.h" 67 #include "reentrant.h" 68 69 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 70 static rb_tree_t pthread__alltree; 71 72 static signed int pthread__cmp(void *, const void *, const void *); 73 74 static const rb_tree_ops_t pthread__alltree_ops = { 75 .rbto_compare_nodes = pthread__cmp, 76 .rbto_compare_key = pthread__cmp, 77 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree), 78 .rbto_context = NULL 79 }; 80 81 static void pthread__create_tramp(void *); 82 static void pthread__initthread(pthread_t); 83 static void pthread__scrubthread(pthread_t, char *, int); 84 static void pthread__initmain(pthread_t *); 85 static void pthread__fork_callback(void); 86 static void pthread__reap(pthread_t); 87 static void pthread__child_callback(void); 88 static void pthread__start(void); 89 90 void pthread__init(void); 91 92 int pthread__started; 93 int __uselibcstub = 1; 94 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 95 pthread_queue_t pthread__deadqueue; 96 pthread_queue_t pthread__allqueue; 97 98 static pthread_attr_t pthread_default_attr; 99 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 100 101 enum { 102 DIAGASSERT_ABORT = 1<<0, 103 DIAGASSERT_STDERR = 1<<1, 104 DIAGASSERT_SYSLOG = 1<<2 105 }; 106 107 static int pthread__diagassert; 108 109 int pthread__concurrency; 110 int pthread__nspins; 111 int pthread__unpark_max = PTHREAD__UNPARK_MAX; 112 int pthread__dbg; /* set by libpthread_dbg if active */ 113 114 /* 115 * We have to initialize the pthread_stack* variables here because 116 * mutexes are used before pthread_init() and thus pthread__initmain() 117 * are called. Since mutexes only save the stack pointer and not a 118 * pointer to the thread data, it is safe to change the mapping from 119 * stack pointer to thread data afterwards. 120 */ 121 size_t pthread__stacksize; 122 size_t pthread__guardsize; 123 size_t pthread__pagesize; 124 static struct __pthread_st *pthread__main; 125 static size_t __pthread_st_size; 126 127 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 128 129 __strong_alias(__libc_thr_self,pthread_self) 130 __strong_alias(__libc_thr_create,pthread_create) 131 __strong_alias(__libc_thr_exit,pthread_exit) 132 __strong_alias(__libc_thr_errno,pthread__errno) 133 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 134 __strong_alias(__libc_thr_equal,pthread_equal) 135 __strong_alias(__libc_thr_init,pthread__init) 136 137 /* 138 * Static library kludge. Place a reference to a symbol any library 139 * file which does not already have a reference here. 140 */ 141 extern int pthread__cancel_stub_binder; 142 143 void *pthread__static_lib_binder[] = { 144 &pthread__cancel_stub_binder, 145 pthread_cond_init, 146 pthread_mutex_init, 147 pthread_rwlock_init, 148 pthread_barrier_init, 149 pthread_key_create, 150 pthread_setspecific, 151 }; 152 153 #define NHASHLOCK 64 154 155 static union hashlock { 156 pthread_mutex_t mutex; 157 char pad[64]; 158 } hashlocks[NHASHLOCK] __aligned(64); 159 160 /* 161 * This needs to be started by the library loading code, before main() 162 * gets to run, for various things that use the state of the initial thread 163 * to work properly (thread-specific data is an application-visible example; 164 * spinlock counts for mutexes is an internal example). 165 */ 166 void 167 pthread__init(void) 168 { 169 pthread_t first; 170 char *p; 171 int i; 172 int mib[2]; 173 unsigned int value; 174 size_t len; 175 extern int __isthreaded; 176 177 /* 178 * Allocate pthread_keys descriptors before 179 * reseting __uselibcstub because otherwise 180 * malloc() will call pthread_keys_create() 181 * while pthread_keys descriptors are not 182 * yet allocated. 183 */ 184 pthread__main = pthread_tsd_init(&__pthread_st_size); 185 if (pthread__main == NULL) 186 err(EXIT_FAILURE, "Cannot allocate pthread storage"); 187 188 __uselibcstub = 0; 189 190 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 191 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF); 192 193 mib[0] = CTL_VM; 194 mib[1] = VM_THREAD_GUARD_SIZE; 195 len = sizeof(value); 196 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 197 pthread__guardsize = value; 198 else 199 pthread__guardsize = pthread__pagesize; 200 201 /* Initialize locks first; they're needed elsewhere. */ 202 pthread__lockprim_init(); 203 for (i = 0; i < NHASHLOCK; i++) { 204 pthread_mutex_init(&hashlocks[i].mutex, NULL); 205 } 206 207 /* Fetch parameters. */ 208 i = (int)_lwp_unpark_all(NULL, 0, NULL); 209 if (i == -1) 210 err(EXIT_FAILURE, "_lwp_unpark_all"); 211 if (i < pthread__unpark_max) 212 pthread__unpark_max = i; 213 214 /* Basic data structure setup */ 215 pthread_attr_init(&pthread_default_attr); 216 PTQ_INIT(&pthread__allqueue); 217 PTQ_INIT(&pthread__deadqueue); 218 219 rb_tree_init(&pthread__alltree, &pthread__alltree_ops); 220 221 /* Create the thread structure corresponding to main() */ 222 pthread__initmain(&first); 223 pthread__initthread(first); 224 pthread__scrubthread(first, NULL, 0); 225 226 first->pt_lid = _lwp_self(); 227 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 228 (void)rb_tree_insert_node(&pthread__alltree, first); 229 230 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 231 err(EXIT_FAILURE, "_lwp_ctl"); 232 } 233 234 /* Start subsystems */ 235 PTHREAD_MD_INIT 236 237 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 238 switch (*p) { 239 case 'a': 240 pthread__diagassert |= DIAGASSERT_ABORT; 241 break; 242 case 'A': 243 pthread__diagassert &= ~DIAGASSERT_ABORT; 244 break; 245 case 'e': 246 pthread__diagassert |= DIAGASSERT_STDERR; 247 break; 248 case 'E': 249 pthread__diagassert &= ~DIAGASSERT_STDERR; 250 break; 251 case 'l': 252 pthread__diagassert |= DIAGASSERT_SYSLOG; 253 break; 254 case 'L': 255 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 256 break; 257 } 258 } 259 260 /* Tell libc that we're here and it should role-play accordingly. */ 261 pthread_atfork(NULL, NULL, pthread__fork_callback); 262 __isthreaded = 1; 263 } 264 265 static void 266 pthread__fork_callback(void) 267 { 268 struct __pthread_st *self = pthread__self(); 269 270 /* lwpctl state is not copied across fork. */ 271 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 272 err(EXIT_FAILURE, "_lwp_ctl"); 273 } 274 self->pt_lid = _lwp_self(); 275 } 276 277 static void 278 pthread__child_callback(void) 279 { 280 281 /* 282 * Clean up data structures that a forked child process might 283 * trip over. Note that if threads have been created (causing 284 * this handler to be registered) the standards say that the 285 * child will trigger undefined behavior if it makes any 286 * pthread_* calls (or any other calls that aren't 287 * async-signal-safe), so we don't really have to clean up 288 * much. Anything that permits some pthread_* calls to work is 289 * merely being polite. 290 */ 291 pthread__started = 0; 292 } 293 294 static void 295 pthread__start(void) 296 { 297 298 /* 299 * Per-process timers are cleared by fork(); despite the 300 * various restrictions on fork() and threads, it's legal to 301 * fork() before creating any threads. 302 */ 303 pthread_atfork(NULL, NULL, pthread__child_callback); 304 } 305 306 307 /* General-purpose thread data structure sanitization. */ 308 /* ARGSUSED */ 309 static void 310 pthread__initthread(pthread_t t) 311 { 312 313 t->pt_self = t; 314 t->pt_magic = PT_MAGIC; 315 t->pt_willpark = 0; 316 t->pt_unpark = 0; 317 t->pt_nwaiters = 0; 318 t->pt_sleepobj = NULL; 319 t->pt_signalled = 0; 320 t->pt_havespecific = 0; 321 t->pt_early = NULL; 322 t->pt_lwpctl = &pthread__dummy_lwpctl; 323 324 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 325 pthread_mutex_init(&t->pt_lock, NULL); 326 PTQ_INIT(&t->pt_cleanup_stack); 327 } 328 329 static void 330 pthread__scrubthread(pthread_t t, char *name, int flags) 331 { 332 333 t->pt_state = PT_STATE_RUNNING; 334 t->pt_exitval = NULL; 335 t->pt_flags = flags; 336 t->pt_cancel = 0; 337 t->pt_errno = 0; 338 t->pt_name = name; 339 t->pt_lid = 0; 340 } 341 342 static int 343 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 344 { 345 void *stackbase, *stackbase2, *redzone; 346 size_t stacksize, guardsize; 347 bool allocated; 348 349 if (attr != NULL) { 350 pthread_attr_getstack(attr, &stackbase, &stacksize); 351 pthread_attr_getguardsize(attr, &guardsize); 352 } else { 353 stackbase = NULL; 354 stacksize = 0; 355 guardsize = pthread__guardsize; 356 } 357 if (stacksize == 0) 358 stacksize = pthread__stacksize; 359 360 if (newthread->pt_stack_allocated) { 361 if (stackbase == NULL && 362 newthread->pt_stack.ss_size == stacksize && 363 newthread->pt_guardsize == guardsize) 364 return 0; 365 stackbase2 = newthread->pt_stack.ss_sp; 366 #ifndef __MACHINE_STACK_GROWS_UP 367 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 368 #endif 369 munmap(stackbase2, 370 newthread->pt_stack.ss_size + newthread->pt_guardsize); 371 newthread->pt_stack.ss_sp = NULL; 372 newthread->pt_stack.ss_size = 0; 373 newthread->pt_guardsize = 0; 374 newthread->pt_stack_allocated = false; 375 } 376 377 newthread->pt_stack_allocated = false; 378 379 if (stackbase == NULL) { 380 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 381 guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1; 382 stackbase = mmap(NULL, stacksize + guardsize, 383 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 384 if (stackbase == MAP_FAILED) 385 return ENOMEM; 386 allocated = true; 387 } else { 388 allocated = false; 389 } 390 #ifdef __MACHINE_STACK_GROWS_UP 391 redzone = (char *)stackbase + stacksize; 392 stackbase2 = (char *)stackbase; 393 #else 394 redzone = (char *)stackbase; 395 stackbase2 = (char *)stackbase + guardsize; 396 #endif 397 if (allocated && guardsize && 398 mprotect(redzone, guardsize, PROT_NONE) == -1) { 399 munmap(stackbase, stacksize + guardsize); 400 return EPERM; 401 } 402 newthread->pt_stack.ss_size = stacksize; 403 newthread->pt_stack.ss_sp = stackbase2; 404 newthread->pt_guardsize = guardsize; 405 newthread->pt_stack_allocated = allocated; 406 return 0; 407 } 408 409 int 410 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 411 void *(*startfunc)(void *), void *arg) 412 { 413 pthread_t newthread; 414 pthread_attr_t nattr; 415 struct pthread_attr_private *p; 416 char * volatile name; 417 unsigned long flag; 418 void *private_area; 419 int ret; 420 421 if (__predict_false(__uselibcstub)) { 422 pthread__errorfunc(__FILE__, __LINE__, __func__, 423 "pthread_create() requires linking with -lpthread"); 424 return __libc_thr_create_stub(thread, attr, startfunc, arg); 425 } 426 427 /* 428 * It's okay to check this without a lock because there can 429 * only be one thread before it becomes true. 430 */ 431 if (pthread__started == 0) { 432 pthread__start(); 433 pthread__started = 1; 434 } 435 436 if (attr == NULL) 437 nattr = pthread_default_attr; 438 else if (attr->pta_magic == PT_ATTR_MAGIC) 439 nattr = *attr; 440 else 441 return EINVAL; 442 443 /* Fetch misc. attributes from the attr structure. */ 444 name = NULL; 445 if ((p = nattr.pta_private) != NULL) 446 if (p->ptap_name[0] != '\0') 447 if ((name = strdup(p->ptap_name)) == NULL) 448 return ENOMEM; 449 450 newthread = NULL; 451 452 /* 453 * Try to reclaim a dead thread. 454 */ 455 if (!PTQ_EMPTY(&pthread__deadqueue)) { 456 pthread_mutex_lock(&pthread__deadqueue_lock); 457 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 458 /* Still busily exiting, or finished? */ 459 if (newthread->pt_lwpctl->lc_curcpu == 460 LWPCTL_CPU_EXITED) 461 break; 462 } 463 if (newthread) 464 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 465 pthread_mutex_unlock(&pthread__deadqueue_lock); 466 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 467 if (newthread && newthread->pt_tls) { 468 _rtld_tls_free(newthread->pt_tls); 469 newthread->pt_tls = NULL; 470 } 471 #endif 472 } 473 474 /* 475 * If necessary set up a stack, allocate space for a pthread_st, 476 * and initialize it. 477 */ 478 if (newthread == NULL) { 479 newthread = calloc(1, __pthread_st_size); 480 if (newthread == NULL) { 481 free(name); 482 return ENOMEM; 483 } 484 newthread->pt_stack_allocated = false; 485 486 if (pthread__getstack(newthread, attr)) { 487 free(newthread); 488 free(name); 489 return ENOMEM; 490 } 491 492 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 493 newthread->pt_tls = NULL; 494 #endif 495 496 /* Add to list of all threads. */ 497 pthread_rwlock_wrlock(&pthread__alltree_lock); 498 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 499 (void)rb_tree_insert_node(&pthread__alltree, newthread); 500 pthread_rwlock_unlock(&pthread__alltree_lock); 501 502 /* Will be reset by the thread upon exit. */ 503 pthread__initthread(newthread); 504 } else { 505 if (pthread__getstack(newthread, attr)) { 506 pthread_mutex_lock(&pthread__deadqueue_lock); 507 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 508 pthread_mutex_unlock(&pthread__deadqueue_lock); 509 return ENOMEM; 510 } 511 } 512 513 /* 514 * Create the new LWP. 515 */ 516 pthread__scrubthread(newthread, name, nattr.pta_flags); 517 newthread->pt_func = startfunc; 518 newthread->pt_arg = arg; 519 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 520 private_area = newthread->pt_tls = _rtld_tls_allocate(); 521 newthread->pt_tls->tcb_pthread = newthread; 522 #else 523 private_area = newthread; 524 #endif 525 526 flag = 0; 527 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 528 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 529 flag |= LWP_SUSPENDED; 530 if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0) 531 flag |= LWP_DETACHED; 532 533 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area, 534 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size, 535 flag, &newthread->pt_lid); 536 if (ret != 0) { 537 ret = errno; 538 pthread_mutex_lock(&newthread->pt_lock); 539 /* Will unlock and free name. */ 540 pthread__reap(newthread); 541 return ret; 542 } 543 544 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 545 if (p != NULL) { 546 (void)pthread_setschedparam(newthread, p->ptap_policy, 547 &p->ptap_sp); 548 } 549 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 550 (void)_lwp_continue(newthread->pt_lid); 551 } 552 } 553 554 *thread = newthread; 555 556 return 0; 557 } 558 559 560 __dead static void 561 pthread__create_tramp(void *cookie) 562 { 563 pthread_t self; 564 void *retval; 565 566 self = cookie; 567 568 /* 569 * Throw away some stack in a feeble attempt to reduce cache 570 * thrash. May help for SMT processors. XXX We should not 571 * be allocating stacks on fixed 2MB boundaries. Needs a 572 * thread register or decent thread local storage. 573 */ 574 (void)alloca(((unsigned)self->pt_lid & 7) << 8); 575 576 if (self->pt_name != NULL) { 577 pthread_mutex_lock(&self->pt_lock); 578 if (self->pt_name != NULL) 579 (void)_lwp_setname(0, self->pt_name); 580 pthread_mutex_unlock(&self->pt_lock); 581 } 582 583 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 584 err(EXIT_FAILURE, "_lwp_ctl"); 585 } 586 587 retval = (*self->pt_func)(self->pt_arg); 588 589 pthread_exit(retval); 590 591 /*NOTREACHED*/ 592 pthread__abort(); 593 } 594 595 int 596 pthread_suspend_np(pthread_t thread) 597 { 598 pthread_t self; 599 600 pthread__error(EINVAL, "Invalid thread", 601 thread->pt_magic == PT_MAGIC); 602 603 self = pthread__self(); 604 if (self == thread) { 605 return EDEADLK; 606 } 607 if (pthread__find(thread) != 0) 608 return ESRCH; 609 if (_lwp_suspend(thread->pt_lid) == 0) 610 return 0; 611 return errno; 612 } 613 614 int 615 pthread_resume_np(pthread_t thread) 616 { 617 618 pthread__error(EINVAL, "Invalid thread", 619 thread->pt_magic == PT_MAGIC); 620 621 if (pthread__find(thread) != 0) 622 return ESRCH; 623 if (_lwp_continue(thread->pt_lid) == 0) 624 return 0; 625 return errno; 626 } 627 628 /* 629 * In case the thread is exiting at an inopportune time leaving waiters not 630 * awoken (because cancelled, for instance) make sure we have no waiters 631 * left. 632 */ 633 static void 634 pthread__clear_waiters(pthread_t self) 635 { 636 637 if (self->pt_nwaiters != 0) { 638 (void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters, 639 NULL); 640 self->pt_nwaiters = 0; 641 } 642 self->pt_willpark = 0; 643 } 644 645 void 646 pthread_exit(void *retval) 647 { 648 pthread_t self; 649 struct pt_clean_t *cleanup; 650 651 if (__predict_false(__uselibcstub)) { 652 __libc_thr_exit_stub(retval); 653 goto out; 654 } 655 656 self = pthread__self(); 657 658 /* Disable cancellability. */ 659 pthread_mutex_lock(&self->pt_lock); 660 self->pt_flags |= PT_FLAG_CS_DISABLED; 661 self->pt_cancel = 0; 662 663 /* Call any cancellation cleanup handlers */ 664 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 665 pthread_mutex_unlock(&self->pt_lock); 666 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 667 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 668 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 669 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 670 } 671 pthread_mutex_lock(&self->pt_lock); 672 } 673 674 pthread_mutex_unlock(&self->pt_lock); 675 __cxa_thread_run_atexit(); 676 pthread_mutex_lock(&self->pt_lock); 677 678 /* Perform cleanup of thread-specific data */ 679 pthread__destroy_tsd(self); 680 681 /* 682 * Signal our exit. Our stack and pthread_t won't be reused until 683 * pthread_create() can see from kernel info that this LWP is gone. 684 */ 685 self->pt_exitval = retval; 686 if (self->pt_flags & PT_FLAG_DETACHED) { 687 /* pthread__reap() will drop the lock. */ 688 pthread__reap(self); 689 pthread__clear_waiters(self); 690 _lwp_exit(); 691 } else { 692 self->pt_state = PT_STATE_ZOMBIE; 693 pthread_mutex_unlock(&self->pt_lock); 694 pthread__clear_waiters(self); 695 /* Note: name will be freed by the joiner. */ 696 _lwp_exit(); 697 } 698 699 out: 700 /*NOTREACHED*/ 701 pthread__abort(); 702 exit(1); 703 } 704 705 706 int 707 pthread_join(pthread_t thread, void **valptr) 708 { 709 pthread_t self; 710 711 pthread__error(EINVAL, "Invalid thread", 712 thread->pt_magic == PT_MAGIC); 713 714 self = pthread__self(); 715 716 if (pthread__find(thread) != 0) 717 return ESRCH; 718 719 if (thread == self) 720 return EDEADLK; 721 722 /* XXX temporary - kernel should handle. */ 723 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) 724 return EINVAL; 725 726 /* IEEE Std 1003.1 says pthread_join() never returns EINTR. */ 727 for (;;) { 728 pthread__testcancel(self); 729 if (_lwp_wait(thread->pt_lid, NULL) == 0) 730 break; 731 if (errno != EINTR) 732 return errno; 733 } 734 735 /* 736 * Don't test for cancellation again. The spec is that if 737 * cancelled, pthread_join() must not have succeeded. 738 */ 739 pthread_mutex_lock(&thread->pt_lock); 740 if (thread->pt_state != PT_STATE_ZOMBIE) { 741 pthread__errorfunc(__FILE__, __LINE__, __func__, 742 "not a zombie"); 743 } 744 if (valptr != NULL) 745 *valptr = thread->pt_exitval; 746 747 /* pthread__reap() will drop the lock. */ 748 pthread__reap(thread); 749 return 0; 750 } 751 752 static void 753 pthread__reap(pthread_t thread) 754 { 755 char *name; 756 757 name = thread->pt_name; 758 thread->pt_name = NULL; 759 thread->pt_state = PT_STATE_DEAD; 760 pthread_mutex_unlock(&thread->pt_lock); 761 762 pthread_mutex_lock(&pthread__deadqueue_lock); 763 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 764 pthread_mutex_unlock(&pthread__deadqueue_lock); 765 766 if (name != NULL) 767 free(name); 768 } 769 770 int 771 pthread_equal(pthread_t t1, pthread_t t2) 772 { 773 774 if (__predict_false(__uselibcstub)) 775 return __libc_thr_equal_stub(t1, t2); 776 777 pthread__error(EINVAL, "Invalid thread", 778 t1->pt_magic == PT_MAGIC); 779 780 pthread__error(EINVAL, "Invalid thread", 781 t2->pt_magic == PT_MAGIC); 782 783 /* Nothing special here. */ 784 return (t1 == t2); 785 } 786 787 788 int 789 pthread_detach(pthread_t thread) 790 { 791 int error; 792 793 pthread__error(EINVAL, "Invalid thread", 794 thread->pt_magic == PT_MAGIC); 795 796 if (pthread__find(thread) != 0) 797 return ESRCH; 798 799 pthread_mutex_lock(&thread->pt_lock); 800 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 801 error = EINVAL; 802 } else { 803 error = _lwp_detach(thread->pt_lid); 804 if (error == 0) 805 thread->pt_flags |= PT_FLAG_DETACHED; 806 else 807 error = errno; 808 } 809 if (thread->pt_state == PT_STATE_ZOMBIE) { 810 /* pthread__reap() will drop the lock. */ 811 pthread__reap(thread); 812 } else 813 pthread_mutex_unlock(&thread->pt_lock); 814 return error; 815 } 816 817 818 int 819 pthread_getname_np(pthread_t thread, char *name, size_t len) 820 { 821 822 pthread__error(EINVAL, "Invalid thread", 823 thread->pt_magic == PT_MAGIC); 824 825 if (pthread__find(thread) != 0) 826 return ESRCH; 827 828 pthread_mutex_lock(&thread->pt_lock); 829 if (thread->pt_name == NULL) 830 name[0] = '\0'; 831 else 832 strlcpy(name, thread->pt_name, len); 833 pthread_mutex_unlock(&thread->pt_lock); 834 835 return 0; 836 } 837 838 839 int 840 pthread_setname_np(pthread_t thread, const char *name, void *arg) 841 { 842 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 843 int namelen; 844 845 pthread__error(EINVAL, "Invalid thread", 846 thread->pt_magic == PT_MAGIC); 847 848 if (pthread__find(thread) != 0) 849 return ESRCH; 850 851 namelen = snprintf(newname, sizeof(newname), name, arg); 852 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 853 return EINVAL; 854 855 cp = strdup(newname); 856 if (cp == NULL) 857 return ENOMEM; 858 859 pthread_mutex_lock(&thread->pt_lock); 860 oldname = thread->pt_name; 861 thread->pt_name = cp; 862 (void)_lwp_setname(thread->pt_lid, cp); 863 pthread_mutex_unlock(&thread->pt_lock); 864 865 if (oldname != NULL) 866 free(oldname); 867 868 return 0; 869 } 870 871 872 pthread_t 873 pthread_self(void) 874 { 875 if (__predict_false(__uselibcstub)) 876 return (pthread_t)__libc_thr_self_stub(); 877 878 return pthread__self(); 879 } 880 881 882 int 883 pthread_cancel(pthread_t thread) 884 { 885 886 pthread__error(EINVAL, "Invalid thread", 887 thread->pt_magic == PT_MAGIC); 888 889 if (pthread__find(thread) != 0) 890 return ESRCH; 891 pthread_mutex_lock(&thread->pt_lock); 892 thread->pt_flags |= PT_FLAG_CS_PENDING; 893 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 894 thread->pt_cancel = 1; 895 pthread_mutex_unlock(&thread->pt_lock); 896 _lwp_wakeup(thread->pt_lid); 897 } else 898 pthread_mutex_unlock(&thread->pt_lock); 899 900 return 0; 901 } 902 903 904 int 905 pthread_setcancelstate(int state, int *oldstate) 906 { 907 pthread_t self; 908 int retval; 909 910 if (__predict_false(__uselibcstub)) 911 return __libc_thr_setcancelstate_stub(state, oldstate); 912 913 self = pthread__self(); 914 retval = 0; 915 916 pthread_mutex_lock(&self->pt_lock); 917 918 if (oldstate != NULL) { 919 if (self->pt_flags & PT_FLAG_CS_DISABLED) 920 *oldstate = PTHREAD_CANCEL_DISABLE; 921 else 922 *oldstate = PTHREAD_CANCEL_ENABLE; 923 } 924 925 if (state == PTHREAD_CANCEL_DISABLE) { 926 self->pt_flags |= PT_FLAG_CS_DISABLED; 927 if (self->pt_cancel) { 928 self->pt_flags |= PT_FLAG_CS_PENDING; 929 self->pt_cancel = 0; 930 } 931 } else if (state == PTHREAD_CANCEL_ENABLE) { 932 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 933 /* 934 * If a cancellation was requested while cancellation 935 * was disabled, note that fact for future 936 * cancellation tests. 937 */ 938 if (self->pt_flags & PT_FLAG_CS_PENDING) { 939 self->pt_cancel = 1; 940 /* This is not a deferred cancellation point. */ 941 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 942 pthread_mutex_unlock(&self->pt_lock); 943 pthread__cancelled(); 944 } 945 } 946 } else 947 retval = EINVAL; 948 949 pthread_mutex_unlock(&self->pt_lock); 950 951 return retval; 952 } 953 954 955 int 956 pthread_setcanceltype(int type, int *oldtype) 957 { 958 pthread_t self; 959 int retval; 960 961 self = pthread__self(); 962 retval = 0; 963 964 pthread_mutex_lock(&self->pt_lock); 965 966 if (oldtype != NULL) { 967 if (self->pt_flags & PT_FLAG_CS_ASYNC) 968 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 969 else 970 *oldtype = PTHREAD_CANCEL_DEFERRED; 971 } 972 973 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 974 self->pt_flags |= PT_FLAG_CS_ASYNC; 975 if (self->pt_cancel) { 976 pthread_mutex_unlock(&self->pt_lock); 977 pthread__cancelled(); 978 } 979 } else if (type == PTHREAD_CANCEL_DEFERRED) 980 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 981 else 982 retval = EINVAL; 983 984 pthread_mutex_unlock(&self->pt_lock); 985 986 return retval; 987 } 988 989 990 void 991 pthread_testcancel(void) 992 { 993 pthread_t self; 994 995 self = pthread__self(); 996 if (self->pt_cancel) 997 pthread__cancelled(); 998 } 999 1000 1001 /* 1002 * POSIX requires that certain functions return an error rather than 1003 * invoking undefined behavior even when handed completely bogus 1004 * pthread_t values, e.g. stack garbage. 1005 */ 1006 int 1007 pthread__find(pthread_t id) 1008 { 1009 pthread_t target; 1010 int error; 1011 1012 pthread_rwlock_rdlock(&pthread__alltree_lock); 1013 target = rb_tree_find_node(&pthread__alltree, id); 1014 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH; 1015 pthread_rwlock_unlock(&pthread__alltree_lock); 1016 1017 return error; 1018 } 1019 1020 1021 void 1022 pthread__testcancel(pthread_t self) 1023 { 1024 1025 if (self->pt_cancel) 1026 pthread__cancelled(); 1027 } 1028 1029 1030 void 1031 pthread__cancelled(void) 1032 { 1033 1034 pthread_exit(PTHREAD_CANCELED); 1035 } 1036 1037 1038 void 1039 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 1040 { 1041 pthread_t self; 1042 struct pt_clean_t *entry; 1043 1044 self = pthread__self(); 1045 entry = store; 1046 entry->ptc_cleanup = cleanup; 1047 entry->ptc_arg = arg; 1048 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1049 } 1050 1051 1052 void 1053 pthread__cleanup_pop(int ex, void *store) 1054 { 1055 pthread_t self; 1056 struct pt_clean_t *entry; 1057 1058 self = pthread__self(); 1059 entry = store; 1060 1061 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1062 if (ex) 1063 (*entry->ptc_cleanup)(entry->ptc_arg); 1064 } 1065 1066 1067 int * 1068 pthread__errno(void) 1069 { 1070 pthread_t self; 1071 1072 if (__predict_false(__uselibcstub)) { 1073 pthread__errorfunc(__FILE__, __LINE__, __func__, 1074 "pthread__errno() requires linking with -lpthread"); 1075 return __libc_thr_errno_stub(); 1076 } 1077 1078 self = pthread__self(); 1079 1080 return &(self->pt_errno); 1081 } 1082 1083 ssize_t _sys_write(int, const void *, size_t); 1084 1085 void 1086 pthread__assertfunc(const char *file, int line, const char *function, 1087 const char *expr) 1088 { 1089 char buf[1024]; 1090 int len; 1091 1092 /* 1093 * snprintf should not acquire any locks, or we could 1094 * end up deadlocked if the assert caller held locks. 1095 */ 1096 len = snprintf(buf, 1024, 1097 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1098 expr, file, line, 1099 function ? ", function \"" : "", 1100 function ? function : "", 1101 function ? "\"" : ""); 1102 1103 _sys_write(STDERR_FILENO, buf, (size_t)len); 1104 (void)kill(getpid(), SIGABRT); 1105 1106 _exit(1); 1107 } 1108 1109 1110 void 1111 pthread__errorfunc(const char *file, int line, const char *function, 1112 const char *msg) 1113 { 1114 char buf[1024]; 1115 size_t len; 1116 1117 if (pthread__diagassert == 0) 1118 return; 1119 1120 /* 1121 * snprintf should not acquire any locks, or we could 1122 * end up deadlocked if the assert caller held locks. 1123 */ 1124 len = snprintf(buf, 1024, 1125 "%s: Error detected by libpthread: %s.\n" 1126 "Detected by file \"%s\", line %d%s%s%s.\n" 1127 "See pthread(3) for information.\n", 1128 getprogname(), msg, file, line, 1129 function ? ", function \"" : "", 1130 function ? function : "", 1131 function ? "\"" : ""); 1132 1133 if (pthread__diagassert & DIAGASSERT_STDERR) 1134 _sys_write(STDERR_FILENO, buf, len); 1135 1136 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1137 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1138 1139 if (pthread__diagassert & DIAGASSERT_ABORT) { 1140 (void)kill(getpid(), SIGABRT); 1141 _exit(1); 1142 } 1143 } 1144 1145 /* 1146 * Thread park/unpark operations. The kernel operations are 1147 * modelled after a brief description from "Multithreading in 1148 * the Solaris Operating Environment": 1149 * 1150 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1151 */ 1152 1153 #define OOPS(msg) \ 1154 pthread__errorfunc(__FILE__, __LINE__, __func__, msg) 1155 1156 int 1157 pthread__park(pthread_t self, pthread_mutex_t *lock, 1158 pthread_queue_t *queue, const struct timespec *abstime, 1159 int cancelpt, const void *hint) 1160 { 1161 int rv, error; 1162 void *obj; 1163 1164 self->pt_willpark = 1; 1165 pthread_mutex_unlock(lock); 1166 self->pt_willpark = 0; 1167 1168 /* 1169 * Wait until we are awoken by a pending unpark operation, 1170 * a signal, an unpark posted after we have gone asleep, 1171 * or an expired timeout. 1172 * 1173 * It is fine to test the value of pt_sleepobj without 1174 * holding any locks, because: 1175 * 1176 * o Only the blocking thread (this thread) ever sets them 1177 * to a non-NULL value. 1178 * 1179 * o Other threads may set them NULL, but if they do so they 1180 * must also make this thread return from _lwp_park. 1181 * 1182 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1183 * calls and all make use of spinlocks in the kernel. So 1184 * these system calls act as full memory barriers, and will 1185 * ensure that the calling CPU's store buffers are drained. 1186 * In combination with the spinlock release before unpark, 1187 * this means that modification of pt_sleepobj/onq by another 1188 * thread will become globally visible before that thread 1189 * schedules an unpark operation on this thread. 1190 * 1191 * Note: the test in the while() statement dodges the park op if 1192 * we have already been awoken, unless there is another thread to 1193 * awaken. This saves a syscall - if we were already awakened, 1194 * the next call to _lwp_park() would need to return early in order 1195 * to eat the previous wakeup. 1196 */ 1197 rv = 0; 1198 do { 1199 /* 1200 * If we deferred unparking a thread, arrange to 1201 * have _lwp_park() restart it before blocking. 1202 */ 1203 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME, 1204 __UNCONST(abstime), self->pt_unpark, hint, hint); 1205 self->pt_unpark = 0; 1206 if (error != 0) { 1207 switch (rv = errno) { 1208 case EINTR: 1209 case EALREADY: 1210 rv = 0; 1211 break; 1212 case ETIMEDOUT: 1213 break; 1214 default: 1215 OOPS("_lwp_park failed"); 1216 break; 1217 } 1218 } 1219 /* Check for cancellation. */ 1220 if (cancelpt && self->pt_cancel) 1221 rv = EINTR; 1222 } while (self->pt_sleepobj != NULL && rv == 0); 1223 1224 /* 1225 * If we have been awoken early but are still on the queue, 1226 * then remove ourself. Again, it's safe to do the test 1227 * without holding any locks. 1228 */ 1229 if (__predict_false(self->pt_sleepobj != NULL)) { 1230 pthread_mutex_lock(lock); 1231 if ((obj = self->pt_sleepobj) != NULL) { 1232 PTQ_REMOVE(queue, self, pt_sleep); 1233 self->pt_sleepobj = NULL; 1234 if (obj != NULL && self->pt_early != NULL) 1235 (*self->pt_early)(obj); 1236 } 1237 pthread_mutex_unlock(lock); 1238 } 1239 self->pt_early = NULL; 1240 1241 return rv; 1242 } 1243 1244 void 1245 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1246 pthread_mutex_t *interlock) 1247 { 1248 pthread_t target; 1249 u_int max; 1250 size_t nwaiters; 1251 1252 max = pthread__unpark_max; 1253 nwaiters = self->pt_nwaiters; 1254 target = PTQ_FIRST(queue); 1255 if (nwaiters == max) { 1256 /* Overflow. */ 1257 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1258 __UNVOLATILE(&interlock->ptm_waiters)); 1259 nwaiters = 0; 1260 } 1261 target->pt_sleepobj = NULL; 1262 self->pt_waiters[nwaiters++] = target->pt_lid; 1263 PTQ_REMOVE(queue, target, pt_sleep); 1264 self->pt_nwaiters = nwaiters; 1265 pthread__mutex_deferwake(self, interlock); 1266 } 1267 1268 void 1269 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1270 pthread_mutex_t *interlock) 1271 { 1272 pthread_t target; 1273 u_int max; 1274 size_t nwaiters; 1275 1276 max = pthread__unpark_max; 1277 nwaiters = self->pt_nwaiters; 1278 PTQ_FOREACH(target, queue, pt_sleep) { 1279 if (nwaiters == max) { 1280 /* Overflow. */ 1281 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1282 __UNVOLATILE(&interlock->ptm_waiters)); 1283 nwaiters = 0; 1284 } 1285 target->pt_sleepobj = NULL; 1286 self->pt_waiters[nwaiters++] = target->pt_lid; 1287 } 1288 self->pt_nwaiters = nwaiters; 1289 PTQ_INIT(queue); 1290 pthread__mutex_deferwake(self, interlock); 1291 } 1292 1293 #undef OOPS 1294 1295 static void 1296 pthread__initmainstack(void) 1297 { 1298 struct rlimit slimit; 1299 const AuxInfo *aux; 1300 size_t size, len; 1301 int mib[2]; 1302 unsigned int value; 1303 1304 _DIAGASSERT(_dlauxinfo() != NULL); 1305 1306 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1307 err(EXIT_FAILURE, 1308 "Couldn't get stack resource consumption limits"); 1309 size = slimit.rlim_cur; 1310 pthread__main->pt_stack.ss_size = size; 1311 pthread__main->pt_guardsize = pthread__pagesize; 1312 1313 mib[0] = CTL_VM; 1314 mib[1] = VM_GUARD_SIZE; 1315 len = sizeof(value); 1316 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 1317 pthread__main->pt_guardsize = value; 1318 1319 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1320 if (aux->a_type == AT_STACKBASE) { 1321 #ifdef __MACHINE_STACK_GROWS_UP 1322 pthread__main->pt_stack.ss_sp = (void *)aux->a_v; 1323 #else 1324 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size; 1325 #endif 1326 break; 1327 } 1328 } 1329 pthread__copy_tsd(pthread__main); 1330 } 1331 1332 /* 1333 * Set up the slightly special stack for the "initial" thread, which 1334 * runs on the normal system stack, and thus gets slightly different 1335 * treatment. 1336 */ 1337 static void 1338 pthread__initmain(pthread_t *newt) 1339 { 1340 char *value; 1341 1342 pthread__initmainstack(); 1343 1344 value = pthread__getenv("PTHREAD_STACKSIZE"); 1345 if (value != NULL) { 1346 pthread__stacksize = atoi(value) * 1024; 1347 if (pthread__stacksize > pthread__main->pt_stack.ss_size) 1348 pthread__stacksize = pthread__main->pt_stack.ss_size; 1349 } 1350 if (pthread__stacksize == 0) 1351 pthread__stacksize = pthread__main->pt_stack.ss_size; 1352 pthread__stacksize += pthread__pagesize - 1; 1353 pthread__stacksize &= ~(pthread__pagesize - 1); 1354 if (pthread__stacksize < 4 * pthread__pagesize) 1355 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1356 4 * pthread__pagesize / 1024); 1357 1358 *newt = pthread__main; 1359 #if defined(_PTHREAD_GETTCB_EXT) 1360 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT(); 1361 #elif defined(__HAVE___LWP_GETTCB_FAST) 1362 pthread__main->pt_tls = __lwp_gettcb_fast(); 1363 #else 1364 pthread__main->pt_tls = _lwp_getprivate(); 1365 #endif 1366 pthread__main->pt_tls->tcb_pthread = pthread__main; 1367 } 1368 1369 static signed int 1370 /*ARGSUSED*/ 1371 pthread__cmp(void *ctx, const void *n1, const void *n2) 1372 { 1373 const uintptr_t p1 = (const uintptr_t)n1; 1374 const uintptr_t p2 = (const uintptr_t)n2; 1375 1376 if (p1 < p2) 1377 return -1; 1378 if (p1 > p2) 1379 return 1; 1380 return 0; 1381 } 1382 1383 /* Because getenv() wants to use locks. */ 1384 char * 1385 pthread__getenv(const char *name) 1386 { 1387 extern char **environ; 1388 size_t l_name, offset; 1389 1390 if (issetugid()) 1391 return (NULL); 1392 1393 l_name = strlen(name); 1394 for (offset = 0; environ[offset] != NULL; offset++) { 1395 if (strncmp(name, environ[offset], l_name) == 0 && 1396 environ[offset][l_name] == '=') { 1397 return environ[offset] + l_name + 1; 1398 } 1399 } 1400 1401 return NULL; 1402 } 1403 1404 pthread_mutex_t * 1405 pthread__hashlock(volatile const void *p) 1406 { 1407 uintptr_t v; 1408 1409 v = (uintptr_t)p; 1410 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1411 } 1412 1413 int 1414 pthread__checkpri(int pri) 1415 { 1416 static int havepri; 1417 static long min, max; 1418 1419 if (!havepri) { 1420 min = sysconf(_SC_SCHED_PRI_MIN); 1421 max = sysconf(_SC_SCHED_PRI_MAX); 1422 havepri = 1; 1423 } 1424 return (pri < min || pri > max) ? EINVAL : 0; 1425 } 1426