1 /* $NetBSD: pthread.c,v 1.181 2022/05/31 14:23:39 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Nathan J. Williams and Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __RCSID("$NetBSD: pthread.c,v 1.181 2022/05/31 14:23:39 riastradh Exp $"); 35 36 #define __EXPOSE_STACK 1 37 38 /* Need to use libc-private names for atomic operations. */ 39 #include "../../common/lib/libc/atomic/atomic_op_namespace.h" 40 41 #include <sys/param.h> 42 #include <sys/exec_elf.h> 43 #include <sys/mman.h> 44 #include <sys/lwp.h> 45 #include <sys/lwpctl.h> 46 #include <sys/resource.h> 47 #include <sys/sysctl.h> 48 #include <sys/tls.h> 49 #include <uvm/uvm_param.h> 50 51 #include <assert.h> 52 #include <dlfcn.h> 53 #include <err.h> 54 #include <errno.h> 55 #include <lwp.h> 56 #include <signal.h> 57 #include <stdio.h> 58 #include <stdlib.h> 59 #include <stddef.h> 60 #include <string.h> 61 #include <syslog.h> 62 #include <ucontext.h> 63 #include <unistd.h> 64 #include <sched.h> 65 66 #include "atexit.h" 67 #include "pthread.h" 68 #include "pthread_int.h" 69 #include "pthread_makelwp.h" 70 #include "reentrant.h" 71 72 __BEGIN_DECLS 73 void _malloc_thread_cleanup(void) __weak; 74 __END_DECLS 75 76 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 77 static rb_tree_t pthread__alltree; 78 79 static signed int pthread__cmp(void *, const void *, const void *); 80 81 static const rb_tree_ops_t pthread__alltree_ops = { 82 .rbto_compare_nodes = pthread__cmp, 83 .rbto_compare_key = pthread__cmp, 84 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree), 85 .rbto_context = NULL 86 }; 87 88 static void pthread__create_tramp(void *); 89 static void pthread__initthread(pthread_t); 90 static void pthread__scrubthread(pthread_t, char *, int); 91 static void pthread__initmain(pthread_t *); 92 static void pthread__reap(pthread_t); 93 94 void pthread__init(void); 95 96 int pthread__started; 97 int __uselibcstub = 1; 98 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 99 pthread_queue_t pthread__deadqueue; 100 pthread_queue_t pthread__allqueue; 101 102 static pthread_attr_t pthread_default_attr; 103 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 104 105 enum { 106 DIAGASSERT_ABORT = 1<<0, 107 DIAGASSERT_STDERR = 1<<1, 108 DIAGASSERT_SYSLOG = 1<<2 109 }; 110 111 static int pthread__diagassert; 112 113 int pthread__concurrency; 114 int pthread__nspins; 115 size_t pthread__unpark_max = PTHREAD__UNPARK_MAX; 116 int pthread__dbg; /* set by libpthread_dbg if active */ 117 118 /* 119 * We have to initialize the pthread_stack* variables here because 120 * mutexes are used before pthread_init() and thus pthread__initmain() 121 * are called. Since mutexes only save the stack pointer and not a 122 * pointer to the thread data, it is safe to change the mapping from 123 * stack pointer to thread data afterwards. 124 */ 125 size_t pthread__stacksize; 126 size_t pthread__guardsize; 127 size_t pthread__pagesize; 128 static struct __pthread_st *pthread__main; 129 static size_t __pthread_st_size; 130 131 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 132 133 __strong_alias(__libc_thr_self,pthread_self) 134 __strong_alias(__libc_thr_create,pthread_create) 135 __strong_alias(__libc_thr_exit,pthread_exit) 136 __strong_alias(__libc_thr_errno,pthread__errno) 137 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 138 __strong_alias(__libc_thr_equal,pthread_equal) 139 __strong_alias(__libc_thr_init,pthread__init) 140 141 /* 142 * Static library kludge. Place a reference to a symbol any library 143 * file which does not already have a reference here. 144 */ 145 extern int pthread__cancel_stub_binder; 146 147 void *pthread__static_lib_binder[] = { 148 &pthread__cancel_stub_binder, 149 pthread_cond_init, 150 pthread_mutex_init, 151 pthread_rwlock_init, 152 pthread_barrier_init, 153 pthread_key_create, 154 pthread_setspecific, 155 }; 156 157 #define NHASHLOCK 64 158 159 static union hashlock { 160 pthread_mutex_t mutex; 161 char pad[64]; 162 } hashlocks[NHASHLOCK] __aligned(64); 163 164 static void 165 pthread__prefork(void) 166 { 167 pthread_mutex_lock(&pthread__deadqueue_lock); 168 } 169 170 static void 171 pthread__fork_parent(void) 172 { 173 pthread_mutex_unlock(&pthread__deadqueue_lock); 174 } 175 176 static void 177 pthread__fork_child(void) 178 { 179 struct __pthread_st *self = pthread__self(); 180 181 pthread_mutex_init(&pthread__deadqueue_lock, NULL); 182 183 /* lwpctl state is not copied across fork. */ 184 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 185 err(EXIT_FAILURE, "_lwp_ctl"); 186 } 187 self->pt_lid = _lwp_self(); 188 } 189 190 /* 191 * This needs to be started by the library loading code, before main() 192 * gets to run, for various things that use the state of the initial thread 193 * to work properly (thread-specific data is an application-visible example; 194 * spinlock counts for mutexes is an internal example). 195 */ 196 void 197 pthread__init(void) 198 { 199 pthread_t first; 200 char *p; 201 int mib[2]; 202 unsigned int value; 203 ssize_t slen; 204 size_t len; 205 extern int __isthreaded; 206 207 /* 208 * Allocate pthread_keys descriptors before 209 * resetting __uselibcstub because otherwise 210 * malloc() will call pthread_keys_create() 211 * while pthread_keys descriptors are not 212 * yet allocated. 213 */ 214 pthread__main = pthread_tsd_init(&__pthread_st_size); 215 if (pthread__main == NULL) 216 err(EXIT_FAILURE, "Cannot allocate pthread storage"); 217 218 __uselibcstub = 0; 219 220 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 221 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF); 222 223 mib[0] = CTL_VM; 224 mib[1] = VM_THREAD_GUARD_SIZE; 225 len = sizeof(value); 226 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 227 pthread__guardsize = value; 228 else 229 pthread__guardsize = pthread__pagesize; 230 231 /* Initialize locks first; they're needed elsewhere. */ 232 pthread__lockprim_init(); 233 for (int i = 0; i < NHASHLOCK; i++) { 234 pthread_mutex_init(&hashlocks[i].mutex, NULL); 235 } 236 237 /* Fetch parameters. */ 238 slen = _lwp_unpark_all(NULL, 0, NULL); 239 if (slen < 0) 240 err(EXIT_FAILURE, "_lwp_unpark_all"); 241 if ((size_t)slen < pthread__unpark_max) 242 pthread__unpark_max = slen; 243 244 /* Basic data structure setup */ 245 pthread_attr_init(&pthread_default_attr); 246 PTQ_INIT(&pthread__allqueue); 247 PTQ_INIT(&pthread__deadqueue); 248 249 rb_tree_init(&pthread__alltree, &pthread__alltree_ops); 250 251 /* Create the thread structure corresponding to main() */ 252 pthread__initmain(&first); 253 pthread__initthread(first); 254 pthread__scrubthread(first, NULL, 0); 255 256 first->pt_lid = _lwp_self(); 257 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 258 (void)rb_tree_insert_node(&pthread__alltree, first); 259 260 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 261 err(EXIT_FAILURE, "_lwp_ctl"); 262 } 263 264 /* Start subsystems */ 265 PTHREAD_MD_INIT 266 267 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 268 switch (*p) { 269 case 'a': 270 pthread__diagassert |= DIAGASSERT_ABORT; 271 break; 272 case 'A': 273 pthread__diagassert &= ~DIAGASSERT_ABORT; 274 break; 275 case 'e': 276 pthread__diagassert |= DIAGASSERT_STDERR; 277 break; 278 case 'E': 279 pthread__diagassert &= ~DIAGASSERT_STDERR; 280 break; 281 case 'l': 282 pthread__diagassert |= DIAGASSERT_SYSLOG; 283 break; 284 case 'L': 285 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 286 break; 287 } 288 } 289 290 /* Tell libc that we're here and it should role-play accordingly. */ 291 pthread_atfork(pthread__prefork, pthread__fork_parent, pthread__fork_child); 292 __isthreaded = 1; 293 } 294 295 /* General-purpose thread data structure sanitization. */ 296 /* ARGSUSED */ 297 static void 298 pthread__initthread(pthread_t t) 299 { 300 301 t->pt_self = t; 302 t->pt_magic = PT_MAGIC; 303 t->pt_sleepobj = NULL; 304 t->pt_havespecific = 0; 305 t->pt_lwpctl = &pthread__dummy_lwpctl; 306 307 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 308 pthread_mutex_init(&t->pt_lock, NULL); 309 PTQ_INIT(&t->pt_cleanup_stack); 310 } 311 312 static void 313 pthread__scrubthread(pthread_t t, char *name, int flags) 314 { 315 316 t->pt_state = PT_STATE_RUNNING; 317 t->pt_exitval = NULL; 318 t->pt_flags = flags; 319 t->pt_cancel = 0; 320 t->pt_errno = 0; 321 t->pt_name = name; 322 t->pt_lid = 0; 323 } 324 325 static int 326 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 327 { 328 void *stackbase, *stackbase2, *redzone; 329 size_t stacksize, guardsize; 330 bool allocated; 331 332 if (attr != NULL) { 333 pthread_attr_getstack(attr, &stackbase, &stacksize); 334 pthread_attr_getguardsize(attr, &guardsize); 335 } else { 336 stackbase = NULL; 337 stacksize = 0; 338 guardsize = pthread__guardsize; 339 } 340 if (stacksize == 0) 341 stacksize = pthread__stacksize; 342 343 if (newthread->pt_stack_allocated) { 344 if (stackbase == NULL && 345 newthread->pt_stack.ss_size == stacksize && 346 newthread->pt_guardsize == guardsize) 347 return 0; 348 stackbase2 = newthread->pt_stack.ss_sp; 349 #ifndef __MACHINE_STACK_GROWS_UP 350 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 351 #endif 352 munmap(stackbase2, 353 newthread->pt_stack.ss_size + newthread->pt_guardsize); 354 newthread->pt_stack.ss_sp = NULL; 355 newthread->pt_stack.ss_size = 0; 356 newthread->pt_guardsize = 0; 357 newthread->pt_stack_allocated = false; 358 } 359 360 newthread->pt_stack_allocated = false; 361 362 if (stackbase == NULL) { 363 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 364 guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1; 365 stackbase = mmap(NULL, stacksize + guardsize, 366 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 367 if (stackbase == MAP_FAILED) 368 return ENOMEM; 369 allocated = true; 370 } else { 371 allocated = false; 372 } 373 #ifdef __MACHINE_STACK_GROWS_UP 374 redzone = (char *)stackbase + stacksize; 375 stackbase2 = (char *)stackbase; 376 #else 377 redzone = (char *)stackbase; 378 stackbase2 = (char *)stackbase + guardsize; 379 #endif 380 if (allocated && guardsize && 381 mprotect(redzone, guardsize, PROT_NONE) == -1) { 382 munmap(stackbase, stacksize + guardsize); 383 return EPERM; 384 } 385 newthread->pt_stack.ss_size = stacksize; 386 newthread->pt_stack.ss_sp = stackbase2; 387 newthread->pt_guardsize = guardsize; 388 newthread->pt_stack_allocated = allocated; 389 return 0; 390 } 391 392 int 393 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 394 void *(*startfunc)(void *), void *arg) 395 { 396 pthread_t newthread; 397 pthread_attr_t nattr; 398 struct pthread_attr_private *p; 399 char * volatile name; 400 unsigned long flag; 401 void *private_area; 402 int ret; 403 404 if (__predict_false(__uselibcstub)) { 405 pthread__errorfunc(__FILE__, __LINE__, __func__, 406 "pthread_create() requires linking with -lpthread"); 407 return __libc_thr_create_stub(thread, attr, startfunc, arg); 408 } 409 410 if (attr == NULL) 411 nattr = pthread_default_attr; 412 else if (attr->pta_magic == PT_ATTR_MAGIC) 413 nattr = *attr; 414 else 415 return EINVAL; 416 417 if (!pthread__started) { 418 /* 419 * Force the _lwp_park symbol to be resolved before we 420 * begin any activity that might rely on concurrent 421 * wakeups. 422 * 423 * This is necessary because rtld itself uses _lwp_park 424 * and _lwp_unpark internally for its own locking: If 425 * we wait to resolve _lwp_park until there is an 426 * _lwp_unpark from another thread pending in the 427 * current lwp (for example, pthread_mutex_unlock or 428 * pthread_cond_signal), rtld's internal use of 429 * _lwp_park might consume the pending unpark. The 430 * result is a deadlock where libpthread and rtld have 431 * both correctly used _lwp_park and _lwp_unpark for 432 * themselves, but rtld has consumed the wakeup meant 433 * for libpthread so it is lost to libpthread. 434 * 435 * For the very first thread, before pthread__started 436 * is set to true, pthread__self()->pt_lid should have 437 * been initialized in pthread__init by the time we get 438 * here to the correct lid so we go to sleep and wake 439 * ourselves at the same time as a no-op. 440 */ 441 _lwp_park(CLOCK_REALTIME, 0, NULL, pthread__self()->pt_lid, 442 NULL, NULL); 443 } 444 445 pthread__started = 1; 446 447 /* Fetch misc. attributes from the attr structure. */ 448 name = NULL; 449 if ((p = nattr.pta_private) != NULL) 450 if (p->ptap_name[0] != '\0') 451 if ((name = strdup(p->ptap_name)) == NULL) 452 return ENOMEM; 453 454 newthread = NULL; 455 456 /* 457 * Try to reclaim a dead thread. 458 */ 459 if (!PTQ_EMPTY(&pthread__deadqueue)) { 460 pthread_mutex_lock(&pthread__deadqueue_lock); 461 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 462 /* Still busily exiting, or finished? */ 463 if (newthread->pt_lwpctl->lc_curcpu == 464 LWPCTL_CPU_EXITED) 465 break; 466 } 467 if (newthread) 468 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 469 pthread_mutex_unlock(&pthread__deadqueue_lock); 470 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 471 if (newthread && newthread->pt_tls) { 472 _rtld_tls_free(newthread->pt_tls); 473 newthread->pt_tls = NULL; 474 } 475 #endif 476 } 477 478 /* 479 * If necessary set up a stack, allocate space for a pthread_st, 480 * and initialize it. 481 */ 482 if (newthread == NULL) { 483 newthread = calloc(1, __pthread_st_size); 484 if (newthread == NULL) { 485 free(name); 486 return ENOMEM; 487 } 488 newthread->pt_stack_allocated = false; 489 490 if (pthread__getstack(newthread, attr)) { 491 free(newthread); 492 free(name); 493 return ENOMEM; 494 } 495 496 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 497 newthread->pt_tls = NULL; 498 #endif 499 500 /* Add to list of all threads. */ 501 pthread_rwlock_wrlock(&pthread__alltree_lock); 502 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 503 (void)rb_tree_insert_node(&pthread__alltree, newthread); 504 pthread_rwlock_unlock(&pthread__alltree_lock); 505 506 /* Will be reset by the thread upon exit. */ 507 pthread__initthread(newthread); 508 } else { 509 if (pthread__getstack(newthread, attr)) { 510 pthread_mutex_lock(&pthread__deadqueue_lock); 511 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 512 pthread_mutex_unlock(&pthread__deadqueue_lock); 513 return ENOMEM; 514 } 515 } 516 517 /* 518 * Create the new LWP. 519 */ 520 pthread__scrubthread(newthread, name, nattr.pta_flags); 521 newthread->pt_func = startfunc; 522 newthread->pt_arg = arg; 523 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 524 private_area = newthread->pt_tls = _rtld_tls_allocate(); 525 newthread->pt_tls->tcb_pthread = newthread; 526 #else 527 private_area = newthread; 528 #endif 529 530 flag = 0; 531 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 532 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 533 flag |= LWP_SUSPENDED; 534 if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0) 535 flag |= LWP_DETACHED; 536 537 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area, 538 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size, 539 flag, &newthread->pt_lid); 540 if (ret != 0) { 541 ret = errno; 542 pthread_mutex_lock(&newthread->pt_lock); 543 /* Will unlock and free name. */ 544 pthread__reap(newthread); 545 return ret; 546 } 547 548 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 549 if (p != NULL) { 550 (void)pthread_setschedparam(newthread, p->ptap_policy, 551 &p->ptap_sp); 552 } 553 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 554 (void)_lwp_continue(newthread->pt_lid); 555 } 556 } 557 558 *thread = newthread; 559 560 return 0; 561 } 562 563 564 __dead static void 565 pthread__create_tramp(void *cookie) 566 { 567 pthread_t self; 568 void *retval; 569 void *junk __unused; 570 571 self = cookie; 572 573 /* 574 * Throw away some stack in a feeble attempt to reduce cache 575 * thrash. May help for SMT processors. XXX We should not 576 * be allocating stacks on fixed 2MB boundaries. Needs a 577 * thread register or decent thread local storage. 578 */ 579 junk = alloca(((unsigned)self->pt_lid & 7) << 8); 580 581 if (self->pt_name != NULL) { 582 pthread_mutex_lock(&self->pt_lock); 583 if (self->pt_name != NULL) 584 (void)_lwp_setname(0, self->pt_name); 585 pthread_mutex_unlock(&self->pt_lock); 586 } 587 588 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 589 err(EXIT_FAILURE, "_lwp_ctl"); 590 } 591 592 retval = (*self->pt_func)(self->pt_arg); 593 594 pthread_exit(retval); 595 596 /*NOTREACHED*/ 597 pthread__abort(); 598 } 599 600 int 601 pthread_suspend_np(pthread_t thread) 602 { 603 pthread_t self; 604 605 pthread__error(EINVAL, "Invalid thread", 606 thread->pt_magic == PT_MAGIC); 607 608 self = pthread__self(); 609 if (self == thread) { 610 return EDEADLK; 611 } 612 if (pthread__find(thread) != 0) 613 return ESRCH; 614 if (_lwp_suspend(thread->pt_lid) == 0) 615 return 0; 616 return errno; 617 } 618 619 int 620 pthread_resume_np(pthread_t thread) 621 { 622 623 pthread__error(EINVAL, "Invalid thread", 624 thread->pt_magic == PT_MAGIC); 625 626 if (pthread__find(thread) != 0) 627 return ESRCH; 628 if (_lwp_continue(thread->pt_lid) == 0) 629 return 0; 630 return errno; 631 } 632 633 void 634 pthread_exit(void *retval) 635 { 636 pthread_t self; 637 struct pt_clean_t *cleanup; 638 639 if (__predict_false(__uselibcstub)) { 640 __libc_thr_exit_stub(retval); 641 goto out; 642 } 643 644 self = pthread__self(); 645 646 /* Disable cancellability. */ 647 pthread_mutex_lock(&self->pt_lock); 648 self->pt_flags |= PT_FLAG_CS_DISABLED; 649 self->pt_cancel = 0; 650 pthread_mutex_unlock(&self->pt_lock); 651 652 /* Call any cancellation cleanup handlers */ 653 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 654 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 655 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 656 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 657 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 658 } 659 } 660 661 __cxa_thread_run_atexit(); 662 663 /* Perform cleanup of thread-specific data */ 664 pthread__destroy_tsd(self); 665 666 if (_malloc_thread_cleanup) 667 _malloc_thread_cleanup(); 668 669 /* 670 * Signal our exit. Our stack and pthread_t won't be reused until 671 * pthread_create() can see from kernel info that this LWP is gone. 672 */ 673 pthread_mutex_lock(&self->pt_lock); 674 self->pt_exitval = retval; 675 if (self->pt_flags & PT_FLAG_DETACHED) { 676 /* pthread__reap() will drop the lock. */ 677 pthread__reap(self); 678 _lwp_exit(); 679 } else { 680 self->pt_state = PT_STATE_ZOMBIE; 681 pthread_mutex_unlock(&self->pt_lock); 682 /* Note: name will be freed by the joiner. */ 683 _lwp_exit(); 684 } 685 686 out: 687 /*NOTREACHED*/ 688 pthread__abort(); 689 exit(1); 690 } 691 692 693 int 694 pthread_join(pthread_t thread, void **valptr) 695 { 696 pthread_t self; 697 698 pthread__error(EINVAL, "Invalid thread", 699 thread->pt_magic == PT_MAGIC); 700 701 self = pthread__self(); 702 703 if (pthread__find(thread) != 0) 704 return ESRCH; 705 706 if (thread == self) 707 return EDEADLK; 708 709 /* IEEE Std 1003.1 says pthread_join() never returns EINTR. */ 710 for (;;) { 711 pthread__testcancel(self); 712 if (_lwp_wait(thread->pt_lid, NULL) == 0) 713 break; 714 if (errno != EINTR) 715 return errno; 716 } 717 718 /* 719 * Don't test for cancellation again. The spec is that if 720 * cancelled, pthread_join() must not have succeeded. 721 */ 722 pthread_mutex_lock(&thread->pt_lock); 723 if (thread->pt_state != PT_STATE_ZOMBIE) { 724 pthread__errorfunc(__FILE__, __LINE__, __func__, 725 "not a zombie"); 726 } 727 if (valptr != NULL) 728 *valptr = thread->pt_exitval; 729 730 /* pthread__reap() will drop the lock. */ 731 pthread__reap(thread); 732 return 0; 733 } 734 735 static void 736 pthread__reap(pthread_t thread) 737 { 738 char *name; 739 740 name = thread->pt_name; 741 thread->pt_name = NULL; 742 thread->pt_state = PT_STATE_DEAD; 743 pthread_mutex_unlock(&thread->pt_lock); 744 745 pthread_mutex_lock(&pthread__deadqueue_lock); 746 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 747 pthread_mutex_unlock(&pthread__deadqueue_lock); 748 749 if (name != NULL) 750 free(name); 751 } 752 753 int 754 pthread_equal(pthread_t t1, pthread_t t2) 755 { 756 757 if (__predict_false(__uselibcstub)) 758 return __libc_thr_equal_stub(t1, t2); 759 760 pthread__error(0, "Invalid thread", 761 (t1 != NULL) && (t1->pt_magic == PT_MAGIC)); 762 763 pthread__error(0, "Invalid thread", 764 (t2 != NULL) && (t2->pt_magic == PT_MAGIC)); 765 766 /* Nothing special here. */ 767 return (t1 == t2); 768 } 769 770 771 int 772 pthread_detach(pthread_t thread) 773 { 774 int error; 775 776 pthread__error(EINVAL, "Invalid thread", 777 thread->pt_magic == PT_MAGIC); 778 779 if (pthread__find(thread) != 0) 780 return ESRCH; 781 782 pthread_mutex_lock(&thread->pt_lock); 783 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 784 error = EINVAL; 785 } else { 786 error = _lwp_detach(thread->pt_lid); 787 if (error == 0) 788 thread->pt_flags |= PT_FLAG_DETACHED; 789 else 790 error = errno; 791 } 792 if (thread->pt_state == PT_STATE_ZOMBIE) { 793 /* pthread__reap() will drop the lock. */ 794 pthread__reap(thread); 795 } else 796 pthread_mutex_unlock(&thread->pt_lock); 797 return error; 798 } 799 800 801 int 802 pthread_getname_np(pthread_t thread, char *name, size_t len) 803 { 804 805 pthread__error(EINVAL, "Invalid thread", 806 thread->pt_magic == PT_MAGIC); 807 808 if (pthread__find(thread) != 0) 809 return ESRCH; 810 811 pthread_mutex_lock(&thread->pt_lock); 812 if (thread->pt_name == NULL) 813 name[0] = '\0'; 814 else 815 strlcpy(name, thread->pt_name, len); 816 pthread_mutex_unlock(&thread->pt_lock); 817 818 return 0; 819 } 820 821 822 int 823 pthread_setname_np(pthread_t thread, const char *name, void *arg) 824 { 825 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 826 int namelen; 827 828 pthread__error(EINVAL, "Invalid thread", 829 thread->pt_magic == PT_MAGIC); 830 831 if (pthread__find(thread) != 0) 832 return ESRCH; 833 834 namelen = snprintf(newname, sizeof(newname), name, arg); 835 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 836 return EINVAL; 837 838 cp = strdup(newname); 839 if (cp == NULL) 840 return ENOMEM; 841 842 pthread_mutex_lock(&thread->pt_lock); 843 oldname = thread->pt_name; 844 thread->pt_name = cp; 845 (void)_lwp_setname(thread->pt_lid, cp); 846 pthread_mutex_unlock(&thread->pt_lock); 847 848 if (oldname != NULL) 849 free(oldname); 850 851 return 0; 852 } 853 854 855 pthread_t 856 pthread_self(void) 857 { 858 if (__predict_false(__uselibcstub)) 859 return (pthread_t)__libc_thr_self_stub(); 860 861 return pthread__self(); 862 } 863 864 865 int 866 pthread_cancel(pthread_t thread) 867 { 868 869 pthread__error(EINVAL, "Invalid thread", 870 thread->pt_magic == PT_MAGIC); 871 872 if (pthread__find(thread) != 0) 873 return ESRCH; 874 pthread_mutex_lock(&thread->pt_lock); 875 thread->pt_flags |= PT_FLAG_CS_PENDING; 876 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 877 thread->pt_cancel = 1; 878 pthread_mutex_unlock(&thread->pt_lock); 879 _lwp_wakeup(thread->pt_lid); 880 } else 881 pthread_mutex_unlock(&thread->pt_lock); 882 883 return 0; 884 } 885 886 887 int 888 pthread_setcancelstate(int state, int *oldstate) 889 { 890 pthread_t self; 891 int retval; 892 893 if (__predict_false(__uselibcstub)) 894 return __libc_thr_setcancelstate_stub(state, oldstate); 895 896 self = pthread__self(); 897 retval = 0; 898 899 pthread_mutex_lock(&self->pt_lock); 900 901 if (oldstate != NULL) { 902 if (self->pt_flags & PT_FLAG_CS_DISABLED) 903 *oldstate = PTHREAD_CANCEL_DISABLE; 904 else 905 *oldstate = PTHREAD_CANCEL_ENABLE; 906 } 907 908 if (state == PTHREAD_CANCEL_DISABLE) { 909 self->pt_flags |= PT_FLAG_CS_DISABLED; 910 if (self->pt_cancel) { 911 self->pt_flags |= PT_FLAG_CS_PENDING; 912 self->pt_cancel = 0; 913 } 914 } else if (state == PTHREAD_CANCEL_ENABLE) { 915 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 916 /* 917 * If a cancellation was requested while cancellation 918 * was disabled, note that fact for future 919 * cancellation tests. 920 */ 921 if (self->pt_flags & PT_FLAG_CS_PENDING) { 922 self->pt_cancel = 1; 923 /* This is not a deferred cancellation point. */ 924 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 925 pthread_mutex_unlock(&self->pt_lock); 926 pthread__cancelled(); 927 } 928 } 929 } else 930 retval = EINVAL; 931 932 pthread_mutex_unlock(&self->pt_lock); 933 934 return retval; 935 } 936 937 938 int 939 pthread_setcanceltype(int type, int *oldtype) 940 { 941 pthread_t self; 942 int retval; 943 944 self = pthread__self(); 945 retval = 0; 946 947 pthread_mutex_lock(&self->pt_lock); 948 949 if (oldtype != NULL) { 950 if (self->pt_flags & PT_FLAG_CS_ASYNC) 951 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 952 else 953 *oldtype = PTHREAD_CANCEL_DEFERRED; 954 } 955 956 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 957 self->pt_flags |= PT_FLAG_CS_ASYNC; 958 if (self->pt_cancel) { 959 pthread_mutex_unlock(&self->pt_lock); 960 pthread__cancelled(); 961 } 962 } else if (type == PTHREAD_CANCEL_DEFERRED) 963 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 964 else 965 retval = EINVAL; 966 967 pthread_mutex_unlock(&self->pt_lock); 968 969 return retval; 970 } 971 972 973 void 974 pthread_testcancel(void) 975 { 976 pthread_t self; 977 978 self = pthread__self(); 979 if (self->pt_cancel) 980 pthread__cancelled(); 981 } 982 983 984 /* 985 * POSIX requires that certain functions return an error rather than 986 * invoking undefined behavior even when handed completely bogus 987 * pthread_t values, e.g. stack garbage. 988 */ 989 int 990 pthread__find(pthread_t id) 991 { 992 pthread_t target; 993 int error; 994 995 pthread_rwlock_rdlock(&pthread__alltree_lock); 996 target = rb_tree_find_node(&pthread__alltree, id); 997 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH; 998 pthread_rwlock_unlock(&pthread__alltree_lock); 999 1000 return error; 1001 } 1002 1003 1004 void 1005 pthread__testcancel(pthread_t self) 1006 { 1007 1008 if (self->pt_cancel) 1009 pthread__cancelled(); 1010 } 1011 1012 1013 void 1014 pthread__cancelled(void) 1015 { 1016 1017 pthread_exit(PTHREAD_CANCELED); 1018 } 1019 1020 1021 void 1022 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 1023 { 1024 pthread_t self; 1025 struct pt_clean_t *entry; 1026 1027 self = pthread__self(); 1028 entry = store; 1029 entry->ptc_cleanup = cleanup; 1030 entry->ptc_arg = arg; 1031 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1032 } 1033 1034 1035 void 1036 pthread__cleanup_pop(int ex, void *store) 1037 { 1038 pthread_t self; 1039 struct pt_clean_t *entry; 1040 1041 self = pthread__self(); 1042 entry = store; 1043 1044 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1045 if (ex) 1046 (*entry->ptc_cleanup)(entry->ptc_arg); 1047 } 1048 1049 1050 int * 1051 pthread__errno(void) 1052 { 1053 pthread_t self; 1054 1055 if (__predict_false(__uselibcstub)) { 1056 pthread__errorfunc(__FILE__, __LINE__, __func__, 1057 "pthread__errno() requires linking with -lpthread"); 1058 return __libc_thr_errno_stub(); 1059 } 1060 1061 self = pthread__self(); 1062 1063 return &(self->pt_errno); 1064 } 1065 1066 ssize_t _sys_write(int, const void *, size_t); 1067 1068 void 1069 pthread__assertfunc(const char *file, int line, const char *function, 1070 const char *expr) 1071 { 1072 char buf[1024]; 1073 int len; 1074 1075 /* 1076 * snprintf should not acquire any locks, or we could 1077 * end up deadlocked if the assert caller held locks. 1078 */ 1079 len = snprintf(buf, 1024, 1080 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1081 expr, file, line, 1082 function ? ", function \"" : "", 1083 function ? function : "", 1084 function ? "\"" : ""); 1085 1086 _sys_write(STDERR_FILENO, buf, (size_t)len); 1087 (void)raise(SIGABRT); 1088 _exit(1); 1089 } 1090 1091 1092 void 1093 pthread__errorfunc(const char *file, int line, const char *function, 1094 const char *msg, ...) 1095 { 1096 char buf[1024]; 1097 char buf2[1024]; 1098 size_t len; 1099 va_list ap; 1100 1101 if (pthread__diagassert == 0) 1102 return; 1103 1104 va_start(ap, msg); 1105 vsnprintf_ss(buf2, sizeof(buf2), msg, ap); 1106 va_end(ap); 1107 1108 /* 1109 * snprintf should not acquire any locks, or we could 1110 * end up deadlocked if the assert caller held locks. 1111 */ 1112 len = snprintf_ss(buf, sizeof(buf), 1113 "%s: Error detected by libpthread: %s.\n" 1114 "Detected by file \"%s\", line %d%s%s%s.\n" 1115 "See pthread(3) for information.\n", 1116 getprogname(), buf2, file, line, 1117 function ? ", function \"" : "", 1118 function ? function : "", 1119 function ? "\"" : ""); 1120 1121 if (pthread__diagassert & DIAGASSERT_STDERR) 1122 _sys_write(STDERR_FILENO, buf, len); 1123 1124 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1125 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1126 1127 if (pthread__diagassert & DIAGASSERT_ABORT) { 1128 (void)_lwp_kill(_lwp_self(), SIGABRT); 1129 _exit(1); 1130 } 1131 } 1132 1133 /* 1134 * Thread park/unpark operations. The kernel operations are 1135 * modelled after a brief description from "Multithreading in 1136 * the Solaris Operating Environment": 1137 * 1138 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1139 */ 1140 1141 int 1142 pthread__park(pthread_t self, pthread_mutex_t *lock, 1143 pthread_queue_t *queue, const struct timespec *abstime, 1144 int cancelpt) 1145 { 1146 int rv, error; 1147 1148 pthread_mutex_unlock(lock); 1149 1150 /* 1151 * Wait until we are awoken by a pending unpark operation, 1152 * a signal, an unpark posted after we have gone asleep, 1153 * or an expired timeout. 1154 * 1155 * It is fine to test the value of pt_sleepobj without 1156 * holding any locks, because: 1157 * 1158 * o Only the blocking thread (this thread) ever sets it 1159 * to a non-NULL value. 1160 * 1161 * o Other threads may set it NULL, but if they do so they 1162 * must also make this thread return from _lwp_park. 1163 * 1164 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1165 * calls and all make use of spinlocks in the kernel. So 1166 * these system calls act as full memory barriers. 1167 */ 1168 rv = 0; 1169 do { 1170 /* 1171 * If we deferred unparking a thread, arrange to 1172 * have _lwp_park() restart it before blocking. 1173 */ 1174 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME, 1175 __UNCONST(abstime), 0, NULL, NULL); 1176 if (error != 0) { 1177 switch (rv = errno) { 1178 case EINTR: 1179 case EALREADY: 1180 rv = 0; 1181 break; 1182 case ETIMEDOUT: 1183 break; 1184 default: 1185 pthread__errorfunc(__FILE__, __LINE__, 1186 __func__, "_lwp_park failed: %d", errno); 1187 break; 1188 } 1189 } 1190 /* Check for cancellation. */ 1191 if (cancelpt && self->pt_cancel) 1192 rv = EINTR; 1193 } while (self->pt_sleepobj != NULL && rv == 0); 1194 return rv; 1195 } 1196 1197 void 1198 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1199 pthread_mutex_t *interlock) 1200 { 1201 pthread_t target; 1202 1203 target = PTQ_FIRST(queue); 1204 target->pt_sleepobj = NULL; 1205 PTQ_REMOVE(queue, target, pt_sleep); 1206 (void)_lwp_unpark(target->pt_lid, NULL); 1207 } 1208 1209 void 1210 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1211 pthread_mutex_t *interlock) 1212 { 1213 lwpid_t lids[PTHREAD__UNPARK_MAX]; 1214 const size_t mlid = pthread__unpark_max; 1215 pthread_t target; 1216 size_t nlid = 0; 1217 1218 PTQ_FOREACH(target, queue, pt_sleep) { 1219 if (nlid == mlid) { 1220 (void)_lwp_unpark_all(lids, nlid, NULL); 1221 nlid = 0; 1222 } 1223 target->pt_sleepobj = NULL; 1224 lids[nlid++] = target->pt_lid; 1225 } 1226 PTQ_INIT(queue); 1227 if (nlid == 1) { 1228 (void)_lwp_unpark(lids[0], NULL); 1229 } else if (nlid > 1) { 1230 (void)_lwp_unpark_all(lids, nlid, NULL); 1231 } 1232 } 1233 1234 #undef OOPS 1235 1236 static void 1237 pthread__initmainstack(void) 1238 { 1239 struct rlimit slimit; 1240 const AuxInfo *aux; 1241 size_t size, len; 1242 int mib[2]; 1243 unsigned int value; 1244 1245 _DIAGASSERT(_dlauxinfo() != NULL); 1246 1247 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1248 err(EXIT_FAILURE, 1249 "Couldn't get stack resource consumption limits"); 1250 size = slimit.rlim_cur; 1251 pthread__main->pt_stack.ss_size = size; 1252 pthread__main->pt_guardsize = pthread__pagesize; 1253 1254 mib[0] = CTL_VM; 1255 mib[1] = VM_GUARD_SIZE; 1256 len = sizeof(value); 1257 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 1258 pthread__main->pt_guardsize = value; 1259 1260 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1261 if (aux->a_type == AT_STACKBASE) { 1262 #ifdef __MACHINE_STACK_GROWS_UP 1263 pthread__main->pt_stack.ss_sp = (void *)aux->a_v; 1264 #else 1265 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size; 1266 #endif 1267 break; 1268 } 1269 } 1270 pthread__copy_tsd(pthread__main); 1271 } 1272 1273 /* 1274 * Set up the slightly special stack for the "initial" thread, which 1275 * runs on the normal system stack, and thus gets slightly different 1276 * treatment. 1277 */ 1278 static void 1279 pthread__initmain(pthread_t *newt) 1280 { 1281 char *value; 1282 1283 pthread__initmainstack(); 1284 1285 value = pthread__getenv("PTHREAD_STACKSIZE"); 1286 if (value != NULL) { 1287 pthread__stacksize = atoi(value) * 1024; 1288 if (pthread__stacksize > pthread__main->pt_stack.ss_size) 1289 pthread__stacksize = pthread__main->pt_stack.ss_size; 1290 } 1291 if (pthread__stacksize == 0) 1292 pthread__stacksize = pthread__main->pt_stack.ss_size; 1293 pthread__stacksize += pthread__pagesize - 1; 1294 pthread__stacksize &= ~(pthread__pagesize - 1); 1295 if (pthread__stacksize < 4 * pthread__pagesize) 1296 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1297 4 * pthread__pagesize / 1024); 1298 1299 *newt = pthread__main; 1300 #if defined(_PTHREAD_GETTCB_EXT) 1301 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT(); 1302 #elif defined(__HAVE___LWP_GETTCB_FAST) 1303 pthread__main->pt_tls = __lwp_gettcb_fast(); 1304 #else 1305 pthread__main->pt_tls = _lwp_getprivate(); 1306 #endif 1307 pthread__main->pt_tls->tcb_pthread = pthread__main; 1308 } 1309 1310 static signed int 1311 /*ARGSUSED*/ 1312 pthread__cmp(void *ctx, const void *n1, const void *n2) 1313 { 1314 const uintptr_t p1 = (const uintptr_t)n1; 1315 const uintptr_t p2 = (const uintptr_t)n2; 1316 1317 if (p1 < p2) 1318 return -1; 1319 if (p1 > p2) 1320 return 1; 1321 return 0; 1322 } 1323 1324 /* Because getenv() wants to use locks. */ 1325 char * 1326 pthread__getenv(const char *name) 1327 { 1328 extern char **environ; 1329 size_t l_name, offset; 1330 1331 if (issetugid()) 1332 return (NULL); 1333 1334 l_name = strlen(name); 1335 for (offset = 0; environ[offset] != NULL; offset++) { 1336 if (strncmp(name, environ[offset], l_name) == 0 && 1337 environ[offset][l_name] == '=') { 1338 return environ[offset] + l_name + 1; 1339 } 1340 } 1341 1342 return NULL; 1343 } 1344 1345 pthread_mutex_t * 1346 pthread__hashlock(volatile const void *p) 1347 { 1348 uintptr_t v; 1349 1350 v = (uintptr_t)p; 1351 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1352 } 1353 1354 int 1355 pthread__checkpri(int pri) 1356 { 1357 static int havepri; 1358 static long min, max; 1359 1360 if (!havepri) { 1361 min = sysconf(_SC_SCHED_PRI_MIN); 1362 max = sysconf(_SC_SCHED_PRI_MAX); 1363 havepri = 1; 1364 } 1365 return (pri < min || pri > max) ? EINVAL : 0; 1366 } 1367