1 /* $NetBSD: pthread.c,v 1.179 2021/04/13 00:31:54 mrg Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Nathan J. Williams and Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __RCSID("$NetBSD: pthread.c,v 1.179 2021/04/13 00:31:54 mrg Exp $"); 35 36 #define __EXPOSE_STACK 1 37 38 #include <sys/param.h> 39 #include <sys/exec_elf.h> 40 #include <sys/mman.h> 41 #include <sys/lwp.h> 42 #include <sys/lwpctl.h> 43 #include <sys/resource.h> 44 #include <sys/sysctl.h> 45 #include <sys/tls.h> 46 #include <uvm/uvm_param.h> 47 48 #include <assert.h> 49 #include <dlfcn.h> 50 #include <err.h> 51 #include <errno.h> 52 #include <lwp.h> 53 #include <signal.h> 54 #include <stdio.h> 55 #include <stdlib.h> 56 #include <stddef.h> 57 #include <string.h> 58 #include <syslog.h> 59 #include <ucontext.h> 60 #include <unistd.h> 61 #include <sched.h> 62 63 #include "atexit.h" 64 #include "pthread.h" 65 #include "pthread_int.h" 66 #include "pthread_makelwp.h" 67 #include "reentrant.h" 68 69 __BEGIN_DECLS 70 void _malloc_thread_cleanup(void) __weak; 71 __END_DECLS 72 73 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 74 static rb_tree_t pthread__alltree; 75 76 static signed int pthread__cmp(void *, const void *, const void *); 77 78 static const rb_tree_ops_t pthread__alltree_ops = { 79 .rbto_compare_nodes = pthread__cmp, 80 .rbto_compare_key = pthread__cmp, 81 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree), 82 .rbto_context = NULL 83 }; 84 85 static void pthread__create_tramp(void *); 86 static void pthread__initthread(pthread_t); 87 static void pthread__scrubthread(pthread_t, char *, int); 88 static void pthread__initmain(pthread_t *); 89 static void pthread__reap(pthread_t); 90 91 void pthread__init(void); 92 93 int pthread__started; 94 int __uselibcstub = 1; 95 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 96 pthread_queue_t pthread__deadqueue; 97 pthread_queue_t pthread__allqueue; 98 99 static pthread_attr_t pthread_default_attr; 100 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 101 102 enum { 103 DIAGASSERT_ABORT = 1<<0, 104 DIAGASSERT_STDERR = 1<<1, 105 DIAGASSERT_SYSLOG = 1<<2 106 }; 107 108 static int pthread__diagassert; 109 110 int pthread__concurrency; 111 int pthread__nspins; 112 size_t pthread__unpark_max = PTHREAD__UNPARK_MAX; 113 int pthread__dbg; /* set by libpthread_dbg if active */ 114 115 /* 116 * We have to initialize the pthread_stack* variables here because 117 * mutexes are used before pthread_init() and thus pthread__initmain() 118 * are called. Since mutexes only save the stack pointer and not a 119 * pointer to the thread data, it is safe to change the mapping from 120 * stack pointer to thread data afterwards. 121 */ 122 size_t pthread__stacksize; 123 size_t pthread__guardsize; 124 size_t pthread__pagesize; 125 static struct __pthread_st *pthread__main; 126 static size_t __pthread_st_size; 127 128 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 129 130 __strong_alias(__libc_thr_self,pthread_self) 131 __strong_alias(__libc_thr_create,pthread_create) 132 __strong_alias(__libc_thr_exit,pthread_exit) 133 __strong_alias(__libc_thr_errno,pthread__errno) 134 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 135 __strong_alias(__libc_thr_equal,pthread_equal) 136 __strong_alias(__libc_thr_init,pthread__init) 137 138 /* 139 * Static library kludge. Place a reference to a symbol any library 140 * file which does not already have a reference here. 141 */ 142 extern int pthread__cancel_stub_binder; 143 144 void *pthread__static_lib_binder[] = { 145 &pthread__cancel_stub_binder, 146 pthread_cond_init, 147 pthread_mutex_init, 148 pthread_rwlock_init, 149 pthread_barrier_init, 150 pthread_key_create, 151 pthread_setspecific, 152 }; 153 154 #define NHASHLOCK 64 155 156 static union hashlock { 157 pthread_mutex_t mutex; 158 char pad[64]; 159 } hashlocks[NHASHLOCK] __aligned(64); 160 161 static void 162 pthread__prefork(void) 163 { 164 pthread_mutex_lock(&pthread__deadqueue_lock); 165 } 166 167 static void 168 pthread__fork_parent(void) 169 { 170 pthread_mutex_unlock(&pthread__deadqueue_lock); 171 } 172 173 static void 174 pthread__fork_child(void) 175 { 176 struct __pthread_st *self = pthread__self(); 177 178 pthread_mutex_init(&pthread__deadqueue_lock, NULL); 179 180 /* lwpctl state is not copied across fork. */ 181 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 182 err(EXIT_FAILURE, "_lwp_ctl"); 183 } 184 self->pt_lid = _lwp_self(); 185 } 186 187 /* 188 * This needs to be started by the library loading code, before main() 189 * gets to run, for various things that use the state of the initial thread 190 * to work properly (thread-specific data is an application-visible example; 191 * spinlock counts for mutexes is an internal example). 192 */ 193 void 194 pthread__init(void) 195 { 196 pthread_t first; 197 char *p; 198 int mib[2]; 199 unsigned int value; 200 ssize_t slen; 201 size_t len; 202 extern int __isthreaded; 203 204 /* 205 * Allocate pthread_keys descriptors before 206 * resetting __uselibcstub because otherwise 207 * malloc() will call pthread_keys_create() 208 * while pthread_keys descriptors are not 209 * yet allocated. 210 */ 211 pthread__main = pthread_tsd_init(&__pthread_st_size); 212 if (pthread__main == NULL) 213 err(EXIT_FAILURE, "Cannot allocate pthread storage"); 214 215 __uselibcstub = 0; 216 217 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 218 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF); 219 220 mib[0] = CTL_VM; 221 mib[1] = VM_THREAD_GUARD_SIZE; 222 len = sizeof(value); 223 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 224 pthread__guardsize = value; 225 else 226 pthread__guardsize = pthread__pagesize; 227 228 /* Initialize locks first; they're needed elsewhere. */ 229 pthread__lockprim_init(); 230 for (int i = 0; i < NHASHLOCK; i++) { 231 pthread_mutex_init(&hashlocks[i].mutex, NULL); 232 } 233 234 /* Fetch parameters. */ 235 slen = _lwp_unpark_all(NULL, 0, NULL); 236 if (slen < 0) 237 err(EXIT_FAILURE, "_lwp_unpark_all"); 238 if ((size_t)slen < pthread__unpark_max) 239 pthread__unpark_max = slen; 240 241 /* Basic data structure setup */ 242 pthread_attr_init(&pthread_default_attr); 243 PTQ_INIT(&pthread__allqueue); 244 PTQ_INIT(&pthread__deadqueue); 245 246 rb_tree_init(&pthread__alltree, &pthread__alltree_ops); 247 248 /* Create the thread structure corresponding to main() */ 249 pthread__initmain(&first); 250 pthread__initthread(first); 251 pthread__scrubthread(first, NULL, 0); 252 253 first->pt_lid = _lwp_self(); 254 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 255 (void)rb_tree_insert_node(&pthread__alltree, first); 256 257 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 258 err(EXIT_FAILURE, "_lwp_ctl"); 259 } 260 261 /* Start subsystems */ 262 PTHREAD_MD_INIT 263 264 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 265 switch (*p) { 266 case 'a': 267 pthread__diagassert |= DIAGASSERT_ABORT; 268 break; 269 case 'A': 270 pthread__diagassert &= ~DIAGASSERT_ABORT; 271 break; 272 case 'e': 273 pthread__diagassert |= DIAGASSERT_STDERR; 274 break; 275 case 'E': 276 pthread__diagassert &= ~DIAGASSERT_STDERR; 277 break; 278 case 'l': 279 pthread__diagassert |= DIAGASSERT_SYSLOG; 280 break; 281 case 'L': 282 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 283 break; 284 } 285 } 286 287 /* Tell libc that we're here and it should role-play accordingly. */ 288 pthread_atfork(pthread__prefork, pthread__fork_parent, pthread__fork_child); 289 __isthreaded = 1; 290 } 291 292 /* General-purpose thread data structure sanitization. */ 293 /* ARGSUSED */ 294 static void 295 pthread__initthread(pthread_t t) 296 { 297 298 t->pt_self = t; 299 t->pt_magic = PT_MAGIC; 300 t->pt_sleepobj = NULL; 301 t->pt_havespecific = 0; 302 t->pt_lwpctl = &pthread__dummy_lwpctl; 303 304 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 305 pthread_mutex_init(&t->pt_lock, NULL); 306 PTQ_INIT(&t->pt_cleanup_stack); 307 } 308 309 static void 310 pthread__scrubthread(pthread_t t, char *name, int flags) 311 { 312 313 t->pt_state = PT_STATE_RUNNING; 314 t->pt_exitval = NULL; 315 t->pt_flags = flags; 316 t->pt_cancel = 0; 317 t->pt_errno = 0; 318 t->pt_name = name; 319 t->pt_lid = 0; 320 } 321 322 static int 323 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 324 { 325 void *stackbase, *stackbase2, *redzone; 326 size_t stacksize, guardsize; 327 bool allocated; 328 329 if (attr != NULL) { 330 pthread_attr_getstack(attr, &stackbase, &stacksize); 331 pthread_attr_getguardsize(attr, &guardsize); 332 } else { 333 stackbase = NULL; 334 stacksize = 0; 335 guardsize = pthread__guardsize; 336 } 337 if (stacksize == 0) 338 stacksize = pthread__stacksize; 339 340 if (newthread->pt_stack_allocated) { 341 if (stackbase == NULL && 342 newthread->pt_stack.ss_size == stacksize && 343 newthread->pt_guardsize == guardsize) 344 return 0; 345 stackbase2 = newthread->pt_stack.ss_sp; 346 #ifndef __MACHINE_STACK_GROWS_UP 347 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 348 #endif 349 munmap(stackbase2, 350 newthread->pt_stack.ss_size + newthread->pt_guardsize); 351 newthread->pt_stack.ss_sp = NULL; 352 newthread->pt_stack.ss_size = 0; 353 newthread->pt_guardsize = 0; 354 newthread->pt_stack_allocated = false; 355 } 356 357 newthread->pt_stack_allocated = false; 358 359 if (stackbase == NULL) { 360 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 361 guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1; 362 stackbase = mmap(NULL, stacksize + guardsize, 363 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 364 if (stackbase == MAP_FAILED) 365 return ENOMEM; 366 allocated = true; 367 } else { 368 allocated = false; 369 } 370 #ifdef __MACHINE_STACK_GROWS_UP 371 redzone = (char *)stackbase + stacksize; 372 stackbase2 = (char *)stackbase; 373 #else 374 redzone = (char *)stackbase; 375 stackbase2 = (char *)stackbase + guardsize; 376 #endif 377 if (allocated && guardsize && 378 mprotect(redzone, guardsize, PROT_NONE) == -1) { 379 munmap(stackbase, stacksize + guardsize); 380 return EPERM; 381 } 382 newthread->pt_stack.ss_size = stacksize; 383 newthread->pt_stack.ss_sp = stackbase2; 384 newthread->pt_guardsize = guardsize; 385 newthread->pt_stack_allocated = allocated; 386 return 0; 387 } 388 389 int 390 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 391 void *(*startfunc)(void *), void *arg) 392 { 393 pthread_t newthread; 394 pthread_attr_t nattr; 395 struct pthread_attr_private *p; 396 char * volatile name; 397 unsigned long flag; 398 void *private_area; 399 int ret; 400 401 if (__predict_false(__uselibcstub)) { 402 pthread__errorfunc(__FILE__, __LINE__, __func__, 403 "pthread_create() requires linking with -lpthread"); 404 return __libc_thr_create_stub(thread, attr, startfunc, arg); 405 } 406 407 if (attr == NULL) 408 nattr = pthread_default_attr; 409 else if (attr->pta_magic == PT_ATTR_MAGIC) 410 nattr = *attr; 411 else 412 return EINVAL; 413 414 pthread__started = 1; 415 416 /* Fetch misc. attributes from the attr structure. */ 417 name = NULL; 418 if ((p = nattr.pta_private) != NULL) 419 if (p->ptap_name[0] != '\0') 420 if ((name = strdup(p->ptap_name)) == NULL) 421 return ENOMEM; 422 423 newthread = NULL; 424 425 /* 426 * Try to reclaim a dead thread. 427 */ 428 if (!PTQ_EMPTY(&pthread__deadqueue)) { 429 pthread_mutex_lock(&pthread__deadqueue_lock); 430 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 431 /* Still busily exiting, or finished? */ 432 if (newthread->pt_lwpctl->lc_curcpu == 433 LWPCTL_CPU_EXITED) 434 break; 435 } 436 if (newthread) 437 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 438 pthread_mutex_unlock(&pthread__deadqueue_lock); 439 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 440 if (newthread && newthread->pt_tls) { 441 _rtld_tls_free(newthread->pt_tls); 442 newthread->pt_tls = NULL; 443 } 444 #endif 445 } 446 447 /* 448 * If necessary set up a stack, allocate space for a pthread_st, 449 * and initialize it. 450 */ 451 if (newthread == NULL) { 452 newthread = calloc(1, __pthread_st_size); 453 if (newthread == NULL) { 454 free(name); 455 return ENOMEM; 456 } 457 newthread->pt_stack_allocated = false; 458 459 if (pthread__getstack(newthread, attr)) { 460 free(newthread); 461 free(name); 462 return ENOMEM; 463 } 464 465 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 466 newthread->pt_tls = NULL; 467 #endif 468 469 /* Add to list of all threads. */ 470 pthread_rwlock_wrlock(&pthread__alltree_lock); 471 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 472 (void)rb_tree_insert_node(&pthread__alltree, newthread); 473 pthread_rwlock_unlock(&pthread__alltree_lock); 474 475 /* Will be reset by the thread upon exit. */ 476 pthread__initthread(newthread); 477 } else { 478 if (pthread__getstack(newthread, attr)) { 479 pthread_mutex_lock(&pthread__deadqueue_lock); 480 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 481 pthread_mutex_unlock(&pthread__deadqueue_lock); 482 return ENOMEM; 483 } 484 } 485 486 /* 487 * Create the new LWP. 488 */ 489 pthread__scrubthread(newthread, name, nattr.pta_flags); 490 newthread->pt_func = startfunc; 491 newthread->pt_arg = arg; 492 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 493 private_area = newthread->pt_tls = _rtld_tls_allocate(); 494 newthread->pt_tls->tcb_pthread = newthread; 495 #else 496 private_area = newthread; 497 #endif 498 499 flag = 0; 500 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 501 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 502 flag |= LWP_SUSPENDED; 503 if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0) 504 flag |= LWP_DETACHED; 505 506 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area, 507 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size, 508 flag, &newthread->pt_lid); 509 if (ret != 0) { 510 ret = errno; 511 pthread_mutex_lock(&newthread->pt_lock); 512 /* Will unlock and free name. */ 513 pthread__reap(newthread); 514 return ret; 515 } 516 517 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 518 if (p != NULL) { 519 (void)pthread_setschedparam(newthread, p->ptap_policy, 520 &p->ptap_sp); 521 } 522 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 523 (void)_lwp_continue(newthread->pt_lid); 524 } 525 } 526 527 *thread = newthread; 528 529 return 0; 530 } 531 532 533 __dead static void 534 pthread__create_tramp(void *cookie) 535 { 536 pthread_t self; 537 void *retval; 538 void *junk __unused; 539 540 self = cookie; 541 542 /* 543 * Throw away some stack in a feeble attempt to reduce cache 544 * thrash. May help for SMT processors. XXX We should not 545 * be allocating stacks on fixed 2MB boundaries. Needs a 546 * thread register or decent thread local storage. 547 */ 548 junk = alloca(((unsigned)self->pt_lid & 7) << 8); 549 550 if (self->pt_name != NULL) { 551 pthread_mutex_lock(&self->pt_lock); 552 if (self->pt_name != NULL) 553 (void)_lwp_setname(0, self->pt_name); 554 pthread_mutex_unlock(&self->pt_lock); 555 } 556 557 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 558 err(EXIT_FAILURE, "_lwp_ctl"); 559 } 560 561 retval = (*self->pt_func)(self->pt_arg); 562 563 pthread_exit(retval); 564 565 /*NOTREACHED*/ 566 pthread__abort(); 567 } 568 569 int 570 pthread_suspend_np(pthread_t thread) 571 { 572 pthread_t self; 573 574 pthread__error(EINVAL, "Invalid thread", 575 thread->pt_magic == PT_MAGIC); 576 577 self = pthread__self(); 578 if (self == thread) { 579 return EDEADLK; 580 } 581 if (pthread__find(thread) != 0) 582 return ESRCH; 583 if (_lwp_suspend(thread->pt_lid) == 0) 584 return 0; 585 return errno; 586 } 587 588 int 589 pthread_resume_np(pthread_t thread) 590 { 591 592 pthread__error(EINVAL, "Invalid thread", 593 thread->pt_magic == PT_MAGIC); 594 595 if (pthread__find(thread) != 0) 596 return ESRCH; 597 if (_lwp_continue(thread->pt_lid) == 0) 598 return 0; 599 return errno; 600 } 601 602 void 603 pthread_exit(void *retval) 604 { 605 pthread_t self; 606 struct pt_clean_t *cleanup; 607 608 if (__predict_false(__uselibcstub)) { 609 __libc_thr_exit_stub(retval); 610 goto out; 611 } 612 613 self = pthread__self(); 614 615 /* Disable cancellability. */ 616 pthread_mutex_lock(&self->pt_lock); 617 self->pt_flags |= PT_FLAG_CS_DISABLED; 618 self->pt_cancel = 0; 619 pthread_mutex_unlock(&self->pt_lock); 620 621 /* Call any cancellation cleanup handlers */ 622 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 623 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 624 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 625 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 626 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 627 } 628 } 629 630 __cxa_thread_run_atexit(); 631 632 /* Perform cleanup of thread-specific data */ 633 pthread__destroy_tsd(self); 634 635 if (_malloc_thread_cleanup) 636 _malloc_thread_cleanup(); 637 638 /* 639 * Signal our exit. Our stack and pthread_t won't be reused until 640 * pthread_create() can see from kernel info that this LWP is gone. 641 */ 642 pthread_mutex_lock(&self->pt_lock); 643 self->pt_exitval = retval; 644 if (self->pt_flags & PT_FLAG_DETACHED) { 645 /* pthread__reap() will drop the lock. */ 646 pthread__reap(self); 647 _lwp_exit(); 648 } else { 649 self->pt_state = PT_STATE_ZOMBIE; 650 pthread_mutex_unlock(&self->pt_lock); 651 /* Note: name will be freed by the joiner. */ 652 _lwp_exit(); 653 } 654 655 out: 656 /*NOTREACHED*/ 657 pthread__abort(); 658 exit(1); 659 } 660 661 662 int 663 pthread_join(pthread_t thread, void **valptr) 664 { 665 pthread_t self; 666 667 pthread__error(EINVAL, "Invalid thread", 668 thread->pt_magic == PT_MAGIC); 669 670 self = pthread__self(); 671 672 if (pthread__find(thread) != 0) 673 return ESRCH; 674 675 if (thread == self) 676 return EDEADLK; 677 678 /* IEEE Std 1003.1 says pthread_join() never returns EINTR. */ 679 for (;;) { 680 pthread__testcancel(self); 681 if (_lwp_wait(thread->pt_lid, NULL) == 0) 682 break; 683 if (errno != EINTR) 684 return errno; 685 } 686 687 /* 688 * Don't test for cancellation again. The spec is that if 689 * cancelled, pthread_join() must not have succeeded. 690 */ 691 pthread_mutex_lock(&thread->pt_lock); 692 if (thread->pt_state != PT_STATE_ZOMBIE) { 693 pthread__errorfunc(__FILE__, __LINE__, __func__, 694 "not a zombie"); 695 } 696 if (valptr != NULL) 697 *valptr = thread->pt_exitval; 698 699 /* pthread__reap() will drop the lock. */ 700 pthread__reap(thread); 701 return 0; 702 } 703 704 static void 705 pthread__reap(pthread_t thread) 706 { 707 char *name; 708 709 name = thread->pt_name; 710 thread->pt_name = NULL; 711 thread->pt_state = PT_STATE_DEAD; 712 pthread_mutex_unlock(&thread->pt_lock); 713 714 pthread_mutex_lock(&pthread__deadqueue_lock); 715 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 716 pthread_mutex_unlock(&pthread__deadqueue_lock); 717 718 if (name != NULL) 719 free(name); 720 } 721 722 int 723 pthread_equal(pthread_t t1, pthread_t t2) 724 { 725 726 if (__predict_false(__uselibcstub)) 727 return __libc_thr_equal_stub(t1, t2); 728 729 pthread__error(0, "Invalid thread", 730 (t1 != NULL) && (t1->pt_magic == PT_MAGIC)); 731 732 pthread__error(0, "Invalid thread", 733 (t2 != NULL) && (t2->pt_magic == PT_MAGIC)); 734 735 /* Nothing special here. */ 736 return (t1 == t2); 737 } 738 739 740 int 741 pthread_detach(pthread_t thread) 742 { 743 int error; 744 745 pthread__error(EINVAL, "Invalid thread", 746 thread->pt_magic == PT_MAGIC); 747 748 if (pthread__find(thread) != 0) 749 return ESRCH; 750 751 pthread_mutex_lock(&thread->pt_lock); 752 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 753 error = EINVAL; 754 } else { 755 error = _lwp_detach(thread->pt_lid); 756 if (error == 0) 757 thread->pt_flags |= PT_FLAG_DETACHED; 758 else 759 error = errno; 760 } 761 if (thread->pt_state == PT_STATE_ZOMBIE) { 762 /* pthread__reap() will drop the lock. */ 763 pthread__reap(thread); 764 } else 765 pthread_mutex_unlock(&thread->pt_lock); 766 return error; 767 } 768 769 770 int 771 pthread_getname_np(pthread_t thread, char *name, size_t len) 772 { 773 774 pthread__error(EINVAL, "Invalid thread", 775 thread->pt_magic == PT_MAGIC); 776 777 if (pthread__find(thread) != 0) 778 return ESRCH; 779 780 pthread_mutex_lock(&thread->pt_lock); 781 if (thread->pt_name == NULL) 782 name[0] = '\0'; 783 else 784 strlcpy(name, thread->pt_name, len); 785 pthread_mutex_unlock(&thread->pt_lock); 786 787 return 0; 788 } 789 790 791 int 792 pthread_setname_np(pthread_t thread, const char *name, void *arg) 793 { 794 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 795 int namelen; 796 797 pthread__error(EINVAL, "Invalid thread", 798 thread->pt_magic == PT_MAGIC); 799 800 if (pthread__find(thread) != 0) 801 return ESRCH; 802 803 namelen = snprintf(newname, sizeof(newname), name, arg); 804 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 805 return EINVAL; 806 807 cp = strdup(newname); 808 if (cp == NULL) 809 return ENOMEM; 810 811 pthread_mutex_lock(&thread->pt_lock); 812 oldname = thread->pt_name; 813 thread->pt_name = cp; 814 (void)_lwp_setname(thread->pt_lid, cp); 815 pthread_mutex_unlock(&thread->pt_lock); 816 817 if (oldname != NULL) 818 free(oldname); 819 820 return 0; 821 } 822 823 824 pthread_t 825 pthread_self(void) 826 { 827 if (__predict_false(__uselibcstub)) 828 return (pthread_t)__libc_thr_self_stub(); 829 830 return pthread__self(); 831 } 832 833 834 int 835 pthread_cancel(pthread_t thread) 836 { 837 838 pthread__error(EINVAL, "Invalid thread", 839 thread->pt_magic == PT_MAGIC); 840 841 if (pthread__find(thread) != 0) 842 return ESRCH; 843 pthread_mutex_lock(&thread->pt_lock); 844 thread->pt_flags |= PT_FLAG_CS_PENDING; 845 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 846 thread->pt_cancel = 1; 847 pthread_mutex_unlock(&thread->pt_lock); 848 _lwp_wakeup(thread->pt_lid); 849 } else 850 pthread_mutex_unlock(&thread->pt_lock); 851 852 return 0; 853 } 854 855 856 int 857 pthread_setcancelstate(int state, int *oldstate) 858 { 859 pthread_t self; 860 int retval; 861 862 if (__predict_false(__uselibcstub)) 863 return __libc_thr_setcancelstate_stub(state, oldstate); 864 865 self = pthread__self(); 866 retval = 0; 867 868 pthread_mutex_lock(&self->pt_lock); 869 870 if (oldstate != NULL) { 871 if (self->pt_flags & PT_FLAG_CS_DISABLED) 872 *oldstate = PTHREAD_CANCEL_DISABLE; 873 else 874 *oldstate = PTHREAD_CANCEL_ENABLE; 875 } 876 877 if (state == PTHREAD_CANCEL_DISABLE) { 878 self->pt_flags |= PT_FLAG_CS_DISABLED; 879 if (self->pt_cancel) { 880 self->pt_flags |= PT_FLAG_CS_PENDING; 881 self->pt_cancel = 0; 882 } 883 } else if (state == PTHREAD_CANCEL_ENABLE) { 884 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 885 /* 886 * If a cancellation was requested while cancellation 887 * was disabled, note that fact for future 888 * cancellation tests. 889 */ 890 if (self->pt_flags & PT_FLAG_CS_PENDING) { 891 self->pt_cancel = 1; 892 /* This is not a deferred cancellation point. */ 893 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 894 pthread_mutex_unlock(&self->pt_lock); 895 pthread__cancelled(); 896 } 897 } 898 } else 899 retval = EINVAL; 900 901 pthread_mutex_unlock(&self->pt_lock); 902 903 return retval; 904 } 905 906 907 int 908 pthread_setcanceltype(int type, int *oldtype) 909 { 910 pthread_t self; 911 int retval; 912 913 self = pthread__self(); 914 retval = 0; 915 916 pthread_mutex_lock(&self->pt_lock); 917 918 if (oldtype != NULL) { 919 if (self->pt_flags & PT_FLAG_CS_ASYNC) 920 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 921 else 922 *oldtype = PTHREAD_CANCEL_DEFERRED; 923 } 924 925 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 926 self->pt_flags |= PT_FLAG_CS_ASYNC; 927 if (self->pt_cancel) { 928 pthread_mutex_unlock(&self->pt_lock); 929 pthread__cancelled(); 930 } 931 } else if (type == PTHREAD_CANCEL_DEFERRED) 932 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 933 else 934 retval = EINVAL; 935 936 pthread_mutex_unlock(&self->pt_lock); 937 938 return retval; 939 } 940 941 942 void 943 pthread_testcancel(void) 944 { 945 pthread_t self; 946 947 self = pthread__self(); 948 if (self->pt_cancel) 949 pthread__cancelled(); 950 } 951 952 953 /* 954 * POSIX requires that certain functions return an error rather than 955 * invoking undefined behavior even when handed completely bogus 956 * pthread_t values, e.g. stack garbage. 957 */ 958 int 959 pthread__find(pthread_t id) 960 { 961 pthread_t target; 962 int error; 963 964 pthread_rwlock_rdlock(&pthread__alltree_lock); 965 target = rb_tree_find_node(&pthread__alltree, id); 966 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH; 967 pthread_rwlock_unlock(&pthread__alltree_lock); 968 969 return error; 970 } 971 972 973 void 974 pthread__testcancel(pthread_t self) 975 { 976 977 if (self->pt_cancel) 978 pthread__cancelled(); 979 } 980 981 982 void 983 pthread__cancelled(void) 984 { 985 986 pthread_exit(PTHREAD_CANCELED); 987 } 988 989 990 void 991 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 992 { 993 pthread_t self; 994 struct pt_clean_t *entry; 995 996 self = pthread__self(); 997 entry = store; 998 entry->ptc_cleanup = cleanup; 999 entry->ptc_arg = arg; 1000 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1001 } 1002 1003 1004 void 1005 pthread__cleanup_pop(int ex, void *store) 1006 { 1007 pthread_t self; 1008 struct pt_clean_t *entry; 1009 1010 self = pthread__self(); 1011 entry = store; 1012 1013 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1014 if (ex) 1015 (*entry->ptc_cleanup)(entry->ptc_arg); 1016 } 1017 1018 1019 int * 1020 pthread__errno(void) 1021 { 1022 pthread_t self; 1023 1024 if (__predict_false(__uselibcstub)) { 1025 pthread__errorfunc(__FILE__, __LINE__, __func__, 1026 "pthread__errno() requires linking with -lpthread"); 1027 return __libc_thr_errno_stub(); 1028 } 1029 1030 self = pthread__self(); 1031 1032 return &(self->pt_errno); 1033 } 1034 1035 ssize_t _sys_write(int, const void *, size_t); 1036 1037 void 1038 pthread__assertfunc(const char *file, int line, const char *function, 1039 const char *expr) 1040 { 1041 char buf[1024]; 1042 int len; 1043 1044 /* 1045 * snprintf should not acquire any locks, or we could 1046 * end up deadlocked if the assert caller held locks. 1047 */ 1048 len = snprintf(buf, 1024, 1049 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1050 expr, file, line, 1051 function ? ", function \"" : "", 1052 function ? function : "", 1053 function ? "\"" : ""); 1054 1055 _sys_write(STDERR_FILENO, buf, (size_t)len); 1056 (void)raise(SIGABRT); 1057 _exit(1); 1058 } 1059 1060 1061 void 1062 pthread__errorfunc(const char *file, int line, const char *function, 1063 const char *msg, ...) 1064 { 1065 char buf[1024]; 1066 char buf2[1024]; 1067 size_t len; 1068 va_list ap; 1069 1070 if (pthread__diagassert == 0) 1071 return; 1072 1073 va_start(ap, msg); 1074 vsnprintf_ss(buf2, sizeof(buf2), msg, ap); 1075 va_end(ap); 1076 1077 /* 1078 * snprintf should not acquire any locks, or we could 1079 * end up deadlocked if the assert caller held locks. 1080 */ 1081 len = snprintf_ss(buf, sizeof(buf), 1082 "%s: Error detected by libpthread: %s.\n" 1083 "Detected by file \"%s\", line %d%s%s%s.\n" 1084 "See pthread(3) for information.\n", 1085 getprogname(), buf2, file, line, 1086 function ? ", function \"" : "", 1087 function ? function : "", 1088 function ? "\"" : ""); 1089 1090 if (pthread__diagassert & DIAGASSERT_STDERR) 1091 _sys_write(STDERR_FILENO, buf, len); 1092 1093 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1094 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1095 1096 if (pthread__diagassert & DIAGASSERT_ABORT) { 1097 (void)_lwp_kill(_lwp_self(), SIGABRT); 1098 _exit(1); 1099 } 1100 } 1101 1102 /* 1103 * Thread park/unpark operations. The kernel operations are 1104 * modelled after a brief description from "Multithreading in 1105 * the Solaris Operating Environment": 1106 * 1107 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1108 */ 1109 1110 int 1111 pthread__park(pthread_t self, pthread_mutex_t *lock, 1112 pthread_queue_t *queue, const struct timespec *abstime, 1113 int cancelpt) 1114 { 1115 int rv, error; 1116 1117 pthread_mutex_unlock(lock); 1118 1119 /* 1120 * Wait until we are awoken by a pending unpark operation, 1121 * a signal, an unpark posted after we have gone asleep, 1122 * or an expired timeout. 1123 * 1124 * It is fine to test the value of pt_sleepobj without 1125 * holding any locks, because: 1126 * 1127 * o Only the blocking thread (this thread) ever sets it 1128 * to a non-NULL value. 1129 * 1130 * o Other threads may set it NULL, but if they do so they 1131 * must also make this thread return from _lwp_park. 1132 * 1133 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1134 * calls and all make use of spinlocks in the kernel. So 1135 * these system calls act as full memory barriers. 1136 */ 1137 rv = 0; 1138 do { 1139 /* 1140 * If we deferred unparking a thread, arrange to 1141 * have _lwp_park() restart it before blocking. 1142 */ 1143 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME, 1144 __UNCONST(abstime), 0, NULL, NULL); 1145 if (error != 0) { 1146 switch (rv = errno) { 1147 case EINTR: 1148 case EALREADY: 1149 rv = 0; 1150 break; 1151 case ETIMEDOUT: 1152 break; 1153 default: 1154 pthread__errorfunc(__FILE__, __LINE__, 1155 __func__, "_lwp_park failed: %d", errno); 1156 break; 1157 } 1158 } 1159 /* Check for cancellation. */ 1160 if (cancelpt && self->pt_cancel) 1161 rv = EINTR; 1162 } while (self->pt_sleepobj != NULL && rv == 0); 1163 return rv; 1164 } 1165 1166 void 1167 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1168 pthread_mutex_t *interlock) 1169 { 1170 pthread_t target; 1171 1172 target = PTQ_FIRST(queue); 1173 target->pt_sleepobj = NULL; 1174 PTQ_REMOVE(queue, target, pt_sleep); 1175 (void)_lwp_unpark(target->pt_lid, NULL); 1176 } 1177 1178 void 1179 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1180 pthread_mutex_t *interlock) 1181 { 1182 lwpid_t lids[PTHREAD__UNPARK_MAX]; 1183 const size_t mlid = pthread__unpark_max; 1184 pthread_t target; 1185 size_t nlid = 0; 1186 1187 PTQ_FOREACH(target, queue, pt_sleep) { 1188 if (nlid == mlid) { 1189 (void)_lwp_unpark_all(lids, nlid, NULL); 1190 nlid = 0; 1191 } 1192 target->pt_sleepobj = NULL; 1193 lids[nlid++] = target->pt_lid; 1194 } 1195 PTQ_INIT(queue); 1196 if (nlid == 1) { 1197 (void)_lwp_unpark(lids[0], NULL); 1198 } else if (nlid > 1) { 1199 (void)_lwp_unpark_all(lids, nlid, NULL); 1200 } 1201 } 1202 1203 #undef OOPS 1204 1205 static void 1206 pthread__initmainstack(void) 1207 { 1208 struct rlimit slimit; 1209 const AuxInfo *aux; 1210 size_t size, len; 1211 int mib[2]; 1212 unsigned int value; 1213 1214 _DIAGASSERT(_dlauxinfo() != NULL); 1215 1216 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1217 err(EXIT_FAILURE, 1218 "Couldn't get stack resource consumption limits"); 1219 size = slimit.rlim_cur; 1220 pthread__main->pt_stack.ss_size = size; 1221 pthread__main->pt_guardsize = pthread__pagesize; 1222 1223 mib[0] = CTL_VM; 1224 mib[1] = VM_GUARD_SIZE; 1225 len = sizeof(value); 1226 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 1227 pthread__main->pt_guardsize = value; 1228 1229 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1230 if (aux->a_type == AT_STACKBASE) { 1231 #ifdef __MACHINE_STACK_GROWS_UP 1232 pthread__main->pt_stack.ss_sp = (void *)aux->a_v; 1233 #else 1234 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size; 1235 #endif 1236 break; 1237 } 1238 } 1239 pthread__copy_tsd(pthread__main); 1240 } 1241 1242 /* 1243 * Set up the slightly special stack for the "initial" thread, which 1244 * runs on the normal system stack, and thus gets slightly different 1245 * treatment. 1246 */ 1247 static void 1248 pthread__initmain(pthread_t *newt) 1249 { 1250 char *value; 1251 1252 pthread__initmainstack(); 1253 1254 value = pthread__getenv("PTHREAD_STACKSIZE"); 1255 if (value != NULL) { 1256 pthread__stacksize = atoi(value) * 1024; 1257 if (pthread__stacksize > pthread__main->pt_stack.ss_size) 1258 pthread__stacksize = pthread__main->pt_stack.ss_size; 1259 } 1260 if (pthread__stacksize == 0) 1261 pthread__stacksize = pthread__main->pt_stack.ss_size; 1262 pthread__stacksize += pthread__pagesize - 1; 1263 pthread__stacksize &= ~(pthread__pagesize - 1); 1264 if (pthread__stacksize < 4 * pthread__pagesize) 1265 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1266 4 * pthread__pagesize / 1024); 1267 1268 *newt = pthread__main; 1269 #if defined(_PTHREAD_GETTCB_EXT) 1270 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT(); 1271 #elif defined(__HAVE___LWP_GETTCB_FAST) 1272 pthread__main->pt_tls = __lwp_gettcb_fast(); 1273 #else 1274 pthread__main->pt_tls = _lwp_getprivate(); 1275 #endif 1276 pthread__main->pt_tls->tcb_pthread = pthread__main; 1277 } 1278 1279 static signed int 1280 /*ARGSUSED*/ 1281 pthread__cmp(void *ctx, const void *n1, const void *n2) 1282 { 1283 const uintptr_t p1 = (const uintptr_t)n1; 1284 const uintptr_t p2 = (const uintptr_t)n2; 1285 1286 if (p1 < p2) 1287 return -1; 1288 if (p1 > p2) 1289 return 1; 1290 return 0; 1291 } 1292 1293 /* Because getenv() wants to use locks. */ 1294 char * 1295 pthread__getenv(const char *name) 1296 { 1297 extern char **environ; 1298 size_t l_name, offset; 1299 1300 if (issetugid()) 1301 return (NULL); 1302 1303 l_name = strlen(name); 1304 for (offset = 0; environ[offset] != NULL; offset++) { 1305 if (strncmp(name, environ[offset], l_name) == 0 && 1306 environ[offset][l_name] == '=') { 1307 return environ[offset] + l_name + 1; 1308 } 1309 } 1310 1311 return NULL; 1312 } 1313 1314 pthread_mutex_t * 1315 pthread__hashlock(volatile const void *p) 1316 { 1317 uintptr_t v; 1318 1319 v = (uintptr_t)p; 1320 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1321 } 1322 1323 int 1324 pthread__checkpri(int pri) 1325 { 1326 static int havepri; 1327 static long min, max; 1328 1329 if (!havepri) { 1330 min = sysconf(_SC_SCHED_PRI_MIN); 1331 max = sysconf(_SC_SCHED_PRI_MAX); 1332 havepri = 1; 1333 } 1334 return (pri < min || pri > max) ? EINVAL : 0; 1335 } 1336