1 /* $NetBSD: pthread.c,v 1.180 2022/02/12 14:59:32 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Nathan J. Williams and Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __RCSID("$NetBSD: pthread.c,v 1.180 2022/02/12 14:59:32 riastradh Exp $"); 35 36 #define __EXPOSE_STACK 1 37 38 /* Need to use libc-private names for atomic operations. */ 39 #include "../../common/lib/libc/atomic/atomic_op_namespace.h" 40 41 #include <sys/param.h> 42 #include <sys/exec_elf.h> 43 #include <sys/mman.h> 44 #include <sys/lwp.h> 45 #include <sys/lwpctl.h> 46 #include <sys/resource.h> 47 #include <sys/sysctl.h> 48 #include <sys/tls.h> 49 #include <uvm/uvm_param.h> 50 51 #include <assert.h> 52 #include <dlfcn.h> 53 #include <err.h> 54 #include <errno.h> 55 #include <lwp.h> 56 #include <signal.h> 57 #include <stdio.h> 58 #include <stdlib.h> 59 #include <stddef.h> 60 #include <string.h> 61 #include <syslog.h> 62 #include <ucontext.h> 63 #include <unistd.h> 64 #include <sched.h> 65 66 #include "atexit.h" 67 #include "pthread.h" 68 #include "pthread_int.h" 69 #include "pthread_makelwp.h" 70 #include "reentrant.h" 71 72 __BEGIN_DECLS 73 void _malloc_thread_cleanup(void) __weak; 74 __END_DECLS 75 76 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 77 static rb_tree_t pthread__alltree; 78 79 static signed int pthread__cmp(void *, const void *, const void *); 80 81 static const rb_tree_ops_t pthread__alltree_ops = { 82 .rbto_compare_nodes = pthread__cmp, 83 .rbto_compare_key = pthread__cmp, 84 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree), 85 .rbto_context = NULL 86 }; 87 88 static void pthread__create_tramp(void *); 89 static void pthread__initthread(pthread_t); 90 static void pthread__scrubthread(pthread_t, char *, int); 91 static void pthread__initmain(pthread_t *); 92 static void pthread__reap(pthread_t); 93 94 void pthread__init(void); 95 96 int pthread__started; 97 int __uselibcstub = 1; 98 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 99 pthread_queue_t pthread__deadqueue; 100 pthread_queue_t pthread__allqueue; 101 102 static pthread_attr_t pthread_default_attr; 103 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 104 105 enum { 106 DIAGASSERT_ABORT = 1<<0, 107 DIAGASSERT_STDERR = 1<<1, 108 DIAGASSERT_SYSLOG = 1<<2 109 }; 110 111 static int pthread__diagassert; 112 113 int pthread__concurrency; 114 int pthread__nspins; 115 size_t pthread__unpark_max = PTHREAD__UNPARK_MAX; 116 int pthread__dbg; /* set by libpthread_dbg if active */ 117 118 /* 119 * We have to initialize the pthread_stack* variables here because 120 * mutexes are used before pthread_init() and thus pthread__initmain() 121 * are called. Since mutexes only save the stack pointer and not a 122 * pointer to the thread data, it is safe to change the mapping from 123 * stack pointer to thread data afterwards. 124 */ 125 size_t pthread__stacksize; 126 size_t pthread__guardsize; 127 size_t pthread__pagesize; 128 static struct __pthread_st *pthread__main; 129 static size_t __pthread_st_size; 130 131 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 132 133 __strong_alias(__libc_thr_self,pthread_self) 134 __strong_alias(__libc_thr_create,pthread_create) 135 __strong_alias(__libc_thr_exit,pthread_exit) 136 __strong_alias(__libc_thr_errno,pthread__errno) 137 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 138 __strong_alias(__libc_thr_equal,pthread_equal) 139 __strong_alias(__libc_thr_init,pthread__init) 140 141 /* 142 * Static library kludge. Place a reference to a symbol any library 143 * file which does not already have a reference here. 144 */ 145 extern int pthread__cancel_stub_binder; 146 147 void *pthread__static_lib_binder[] = { 148 &pthread__cancel_stub_binder, 149 pthread_cond_init, 150 pthread_mutex_init, 151 pthread_rwlock_init, 152 pthread_barrier_init, 153 pthread_key_create, 154 pthread_setspecific, 155 }; 156 157 #define NHASHLOCK 64 158 159 static union hashlock { 160 pthread_mutex_t mutex; 161 char pad[64]; 162 } hashlocks[NHASHLOCK] __aligned(64); 163 164 static void 165 pthread__prefork(void) 166 { 167 pthread_mutex_lock(&pthread__deadqueue_lock); 168 } 169 170 static void 171 pthread__fork_parent(void) 172 { 173 pthread_mutex_unlock(&pthread__deadqueue_lock); 174 } 175 176 static void 177 pthread__fork_child(void) 178 { 179 struct __pthread_st *self = pthread__self(); 180 181 pthread_mutex_init(&pthread__deadqueue_lock, NULL); 182 183 /* lwpctl state is not copied across fork. */ 184 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 185 err(EXIT_FAILURE, "_lwp_ctl"); 186 } 187 self->pt_lid = _lwp_self(); 188 } 189 190 /* 191 * This needs to be started by the library loading code, before main() 192 * gets to run, for various things that use the state of the initial thread 193 * to work properly (thread-specific data is an application-visible example; 194 * spinlock counts for mutexes is an internal example). 195 */ 196 void 197 pthread__init(void) 198 { 199 pthread_t first; 200 char *p; 201 int mib[2]; 202 unsigned int value; 203 ssize_t slen; 204 size_t len; 205 extern int __isthreaded; 206 207 /* 208 * Allocate pthread_keys descriptors before 209 * resetting __uselibcstub because otherwise 210 * malloc() will call pthread_keys_create() 211 * while pthread_keys descriptors are not 212 * yet allocated. 213 */ 214 pthread__main = pthread_tsd_init(&__pthread_st_size); 215 if (pthread__main == NULL) 216 err(EXIT_FAILURE, "Cannot allocate pthread storage"); 217 218 __uselibcstub = 0; 219 220 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 221 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF); 222 223 mib[0] = CTL_VM; 224 mib[1] = VM_THREAD_GUARD_SIZE; 225 len = sizeof(value); 226 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 227 pthread__guardsize = value; 228 else 229 pthread__guardsize = pthread__pagesize; 230 231 /* Initialize locks first; they're needed elsewhere. */ 232 pthread__lockprim_init(); 233 for (int i = 0; i < NHASHLOCK; i++) { 234 pthread_mutex_init(&hashlocks[i].mutex, NULL); 235 } 236 237 /* Fetch parameters. */ 238 slen = _lwp_unpark_all(NULL, 0, NULL); 239 if (slen < 0) 240 err(EXIT_FAILURE, "_lwp_unpark_all"); 241 if ((size_t)slen < pthread__unpark_max) 242 pthread__unpark_max = slen; 243 244 /* Basic data structure setup */ 245 pthread_attr_init(&pthread_default_attr); 246 PTQ_INIT(&pthread__allqueue); 247 PTQ_INIT(&pthread__deadqueue); 248 249 rb_tree_init(&pthread__alltree, &pthread__alltree_ops); 250 251 /* Create the thread structure corresponding to main() */ 252 pthread__initmain(&first); 253 pthread__initthread(first); 254 pthread__scrubthread(first, NULL, 0); 255 256 first->pt_lid = _lwp_self(); 257 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 258 (void)rb_tree_insert_node(&pthread__alltree, first); 259 260 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 261 err(EXIT_FAILURE, "_lwp_ctl"); 262 } 263 264 /* Start subsystems */ 265 PTHREAD_MD_INIT 266 267 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 268 switch (*p) { 269 case 'a': 270 pthread__diagassert |= DIAGASSERT_ABORT; 271 break; 272 case 'A': 273 pthread__diagassert &= ~DIAGASSERT_ABORT; 274 break; 275 case 'e': 276 pthread__diagassert |= DIAGASSERT_STDERR; 277 break; 278 case 'E': 279 pthread__diagassert &= ~DIAGASSERT_STDERR; 280 break; 281 case 'l': 282 pthread__diagassert |= DIAGASSERT_SYSLOG; 283 break; 284 case 'L': 285 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 286 break; 287 } 288 } 289 290 /* Tell libc that we're here and it should role-play accordingly. */ 291 pthread_atfork(pthread__prefork, pthread__fork_parent, pthread__fork_child); 292 __isthreaded = 1; 293 } 294 295 /* General-purpose thread data structure sanitization. */ 296 /* ARGSUSED */ 297 static void 298 pthread__initthread(pthread_t t) 299 { 300 301 t->pt_self = t; 302 t->pt_magic = PT_MAGIC; 303 t->pt_sleepobj = NULL; 304 t->pt_havespecific = 0; 305 t->pt_lwpctl = &pthread__dummy_lwpctl; 306 307 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 308 pthread_mutex_init(&t->pt_lock, NULL); 309 PTQ_INIT(&t->pt_cleanup_stack); 310 } 311 312 static void 313 pthread__scrubthread(pthread_t t, char *name, int flags) 314 { 315 316 t->pt_state = PT_STATE_RUNNING; 317 t->pt_exitval = NULL; 318 t->pt_flags = flags; 319 t->pt_cancel = 0; 320 t->pt_errno = 0; 321 t->pt_name = name; 322 t->pt_lid = 0; 323 } 324 325 static int 326 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 327 { 328 void *stackbase, *stackbase2, *redzone; 329 size_t stacksize, guardsize; 330 bool allocated; 331 332 if (attr != NULL) { 333 pthread_attr_getstack(attr, &stackbase, &stacksize); 334 pthread_attr_getguardsize(attr, &guardsize); 335 } else { 336 stackbase = NULL; 337 stacksize = 0; 338 guardsize = pthread__guardsize; 339 } 340 if (stacksize == 0) 341 stacksize = pthread__stacksize; 342 343 if (newthread->pt_stack_allocated) { 344 if (stackbase == NULL && 345 newthread->pt_stack.ss_size == stacksize && 346 newthread->pt_guardsize == guardsize) 347 return 0; 348 stackbase2 = newthread->pt_stack.ss_sp; 349 #ifndef __MACHINE_STACK_GROWS_UP 350 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 351 #endif 352 munmap(stackbase2, 353 newthread->pt_stack.ss_size + newthread->pt_guardsize); 354 newthread->pt_stack.ss_sp = NULL; 355 newthread->pt_stack.ss_size = 0; 356 newthread->pt_guardsize = 0; 357 newthread->pt_stack_allocated = false; 358 } 359 360 newthread->pt_stack_allocated = false; 361 362 if (stackbase == NULL) { 363 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 364 guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1; 365 stackbase = mmap(NULL, stacksize + guardsize, 366 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 367 if (stackbase == MAP_FAILED) 368 return ENOMEM; 369 allocated = true; 370 } else { 371 allocated = false; 372 } 373 #ifdef __MACHINE_STACK_GROWS_UP 374 redzone = (char *)stackbase + stacksize; 375 stackbase2 = (char *)stackbase; 376 #else 377 redzone = (char *)stackbase; 378 stackbase2 = (char *)stackbase + guardsize; 379 #endif 380 if (allocated && guardsize && 381 mprotect(redzone, guardsize, PROT_NONE) == -1) { 382 munmap(stackbase, stacksize + guardsize); 383 return EPERM; 384 } 385 newthread->pt_stack.ss_size = stacksize; 386 newthread->pt_stack.ss_sp = stackbase2; 387 newthread->pt_guardsize = guardsize; 388 newthread->pt_stack_allocated = allocated; 389 return 0; 390 } 391 392 int 393 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 394 void *(*startfunc)(void *), void *arg) 395 { 396 pthread_t newthread; 397 pthread_attr_t nattr; 398 struct pthread_attr_private *p; 399 char * volatile name; 400 unsigned long flag; 401 void *private_area; 402 int ret; 403 404 if (__predict_false(__uselibcstub)) { 405 pthread__errorfunc(__FILE__, __LINE__, __func__, 406 "pthread_create() requires linking with -lpthread"); 407 return __libc_thr_create_stub(thread, attr, startfunc, arg); 408 } 409 410 if (attr == NULL) 411 nattr = pthread_default_attr; 412 else if (attr->pta_magic == PT_ATTR_MAGIC) 413 nattr = *attr; 414 else 415 return EINVAL; 416 417 pthread__started = 1; 418 419 /* Fetch misc. attributes from the attr structure. */ 420 name = NULL; 421 if ((p = nattr.pta_private) != NULL) 422 if (p->ptap_name[0] != '\0') 423 if ((name = strdup(p->ptap_name)) == NULL) 424 return ENOMEM; 425 426 newthread = NULL; 427 428 /* 429 * Try to reclaim a dead thread. 430 */ 431 if (!PTQ_EMPTY(&pthread__deadqueue)) { 432 pthread_mutex_lock(&pthread__deadqueue_lock); 433 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 434 /* Still busily exiting, or finished? */ 435 if (newthread->pt_lwpctl->lc_curcpu == 436 LWPCTL_CPU_EXITED) 437 break; 438 } 439 if (newthread) 440 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 441 pthread_mutex_unlock(&pthread__deadqueue_lock); 442 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 443 if (newthread && newthread->pt_tls) { 444 _rtld_tls_free(newthread->pt_tls); 445 newthread->pt_tls = NULL; 446 } 447 #endif 448 } 449 450 /* 451 * If necessary set up a stack, allocate space for a pthread_st, 452 * and initialize it. 453 */ 454 if (newthread == NULL) { 455 newthread = calloc(1, __pthread_st_size); 456 if (newthread == NULL) { 457 free(name); 458 return ENOMEM; 459 } 460 newthread->pt_stack_allocated = false; 461 462 if (pthread__getstack(newthread, attr)) { 463 free(newthread); 464 free(name); 465 return ENOMEM; 466 } 467 468 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 469 newthread->pt_tls = NULL; 470 #endif 471 472 /* Add to list of all threads. */ 473 pthread_rwlock_wrlock(&pthread__alltree_lock); 474 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 475 (void)rb_tree_insert_node(&pthread__alltree, newthread); 476 pthread_rwlock_unlock(&pthread__alltree_lock); 477 478 /* Will be reset by the thread upon exit. */ 479 pthread__initthread(newthread); 480 } else { 481 if (pthread__getstack(newthread, attr)) { 482 pthread_mutex_lock(&pthread__deadqueue_lock); 483 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 484 pthread_mutex_unlock(&pthread__deadqueue_lock); 485 return ENOMEM; 486 } 487 } 488 489 /* 490 * Create the new LWP. 491 */ 492 pthread__scrubthread(newthread, name, nattr.pta_flags); 493 newthread->pt_func = startfunc; 494 newthread->pt_arg = arg; 495 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 496 private_area = newthread->pt_tls = _rtld_tls_allocate(); 497 newthread->pt_tls->tcb_pthread = newthread; 498 #else 499 private_area = newthread; 500 #endif 501 502 flag = 0; 503 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 504 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 505 flag |= LWP_SUSPENDED; 506 if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0) 507 flag |= LWP_DETACHED; 508 509 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area, 510 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size, 511 flag, &newthread->pt_lid); 512 if (ret != 0) { 513 ret = errno; 514 pthread_mutex_lock(&newthread->pt_lock); 515 /* Will unlock and free name. */ 516 pthread__reap(newthread); 517 return ret; 518 } 519 520 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 521 if (p != NULL) { 522 (void)pthread_setschedparam(newthread, p->ptap_policy, 523 &p->ptap_sp); 524 } 525 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 526 (void)_lwp_continue(newthread->pt_lid); 527 } 528 } 529 530 *thread = newthread; 531 532 return 0; 533 } 534 535 536 __dead static void 537 pthread__create_tramp(void *cookie) 538 { 539 pthread_t self; 540 void *retval; 541 void *junk __unused; 542 543 self = cookie; 544 545 /* 546 * Throw away some stack in a feeble attempt to reduce cache 547 * thrash. May help for SMT processors. XXX We should not 548 * be allocating stacks on fixed 2MB boundaries. Needs a 549 * thread register or decent thread local storage. 550 */ 551 junk = alloca(((unsigned)self->pt_lid & 7) << 8); 552 553 if (self->pt_name != NULL) { 554 pthread_mutex_lock(&self->pt_lock); 555 if (self->pt_name != NULL) 556 (void)_lwp_setname(0, self->pt_name); 557 pthread_mutex_unlock(&self->pt_lock); 558 } 559 560 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 561 err(EXIT_FAILURE, "_lwp_ctl"); 562 } 563 564 retval = (*self->pt_func)(self->pt_arg); 565 566 pthread_exit(retval); 567 568 /*NOTREACHED*/ 569 pthread__abort(); 570 } 571 572 int 573 pthread_suspend_np(pthread_t thread) 574 { 575 pthread_t self; 576 577 pthread__error(EINVAL, "Invalid thread", 578 thread->pt_magic == PT_MAGIC); 579 580 self = pthread__self(); 581 if (self == thread) { 582 return EDEADLK; 583 } 584 if (pthread__find(thread) != 0) 585 return ESRCH; 586 if (_lwp_suspend(thread->pt_lid) == 0) 587 return 0; 588 return errno; 589 } 590 591 int 592 pthread_resume_np(pthread_t thread) 593 { 594 595 pthread__error(EINVAL, "Invalid thread", 596 thread->pt_magic == PT_MAGIC); 597 598 if (pthread__find(thread) != 0) 599 return ESRCH; 600 if (_lwp_continue(thread->pt_lid) == 0) 601 return 0; 602 return errno; 603 } 604 605 void 606 pthread_exit(void *retval) 607 { 608 pthread_t self; 609 struct pt_clean_t *cleanup; 610 611 if (__predict_false(__uselibcstub)) { 612 __libc_thr_exit_stub(retval); 613 goto out; 614 } 615 616 self = pthread__self(); 617 618 /* Disable cancellability. */ 619 pthread_mutex_lock(&self->pt_lock); 620 self->pt_flags |= PT_FLAG_CS_DISABLED; 621 self->pt_cancel = 0; 622 pthread_mutex_unlock(&self->pt_lock); 623 624 /* Call any cancellation cleanup handlers */ 625 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 626 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 627 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 628 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 629 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 630 } 631 } 632 633 __cxa_thread_run_atexit(); 634 635 /* Perform cleanup of thread-specific data */ 636 pthread__destroy_tsd(self); 637 638 if (_malloc_thread_cleanup) 639 _malloc_thread_cleanup(); 640 641 /* 642 * Signal our exit. Our stack and pthread_t won't be reused until 643 * pthread_create() can see from kernel info that this LWP is gone. 644 */ 645 pthread_mutex_lock(&self->pt_lock); 646 self->pt_exitval = retval; 647 if (self->pt_flags & PT_FLAG_DETACHED) { 648 /* pthread__reap() will drop the lock. */ 649 pthread__reap(self); 650 _lwp_exit(); 651 } else { 652 self->pt_state = PT_STATE_ZOMBIE; 653 pthread_mutex_unlock(&self->pt_lock); 654 /* Note: name will be freed by the joiner. */ 655 _lwp_exit(); 656 } 657 658 out: 659 /*NOTREACHED*/ 660 pthread__abort(); 661 exit(1); 662 } 663 664 665 int 666 pthread_join(pthread_t thread, void **valptr) 667 { 668 pthread_t self; 669 670 pthread__error(EINVAL, "Invalid thread", 671 thread->pt_magic == PT_MAGIC); 672 673 self = pthread__self(); 674 675 if (pthread__find(thread) != 0) 676 return ESRCH; 677 678 if (thread == self) 679 return EDEADLK; 680 681 /* IEEE Std 1003.1 says pthread_join() never returns EINTR. */ 682 for (;;) { 683 pthread__testcancel(self); 684 if (_lwp_wait(thread->pt_lid, NULL) == 0) 685 break; 686 if (errno != EINTR) 687 return errno; 688 } 689 690 /* 691 * Don't test for cancellation again. The spec is that if 692 * cancelled, pthread_join() must not have succeeded. 693 */ 694 pthread_mutex_lock(&thread->pt_lock); 695 if (thread->pt_state != PT_STATE_ZOMBIE) { 696 pthread__errorfunc(__FILE__, __LINE__, __func__, 697 "not a zombie"); 698 } 699 if (valptr != NULL) 700 *valptr = thread->pt_exitval; 701 702 /* pthread__reap() will drop the lock. */ 703 pthread__reap(thread); 704 return 0; 705 } 706 707 static void 708 pthread__reap(pthread_t thread) 709 { 710 char *name; 711 712 name = thread->pt_name; 713 thread->pt_name = NULL; 714 thread->pt_state = PT_STATE_DEAD; 715 pthread_mutex_unlock(&thread->pt_lock); 716 717 pthread_mutex_lock(&pthread__deadqueue_lock); 718 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 719 pthread_mutex_unlock(&pthread__deadqueue_lock); 720 721 if (name != NULL) 722 free(name); 723 } 724 725 int 726 pthread_equal(pthread_t t1, pthread_t t2) 727 { 728 729 if (__predict_false(__uselibcstub)) 730 return __libc_thr_equal_stub(t1, t2); 731 732 pthread__error(0, "Invalid thread", 733 (t1 != NULL) && (t1->pt_magic == PT_MAGIC)); 734 735 pthread__error(0, "Invalid thread", 736 (t2 != NULL) && (t2->pt_magic == PT_MAGIC)); 737 738 /* Nothing special here. */ 739 return (t1 == t2); 740 } 741 742 743 int 744 pthread_detach(pthread_t thread) 745 { 746 int error; 747 748 pthread__error(EINVAL, "Invalid thread", 749 thread->pt_magic == PT_MAGIC); 750 751 if (pthread__find(thread) != 0) 752 return ESRCH; 753 754 pthread_mutex_lock(&thread->pt_lock); 755 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 756 error = EINVAL; 757 } else { 758 error = _lwp_detach(thread->pt_lid); 759 if (error == 0) 760 thread->pt_flags |= PT_FLAG_DETACHED; 761 else 762 error = errno; 763 } 764 if (thread->pt_state == PT_STATE_ZOMBIE) { 765 /* pthread__reap() will drop the lock. */ 766 pthread__reap(thread); 767 } else 768 pthread_mutex_unlock(&thread->pt_lock); 769 return error; 770 } 771 772 773 int 774 pthread_getname_np(pthread_t thread, char *name, size_t len) 775 { 776 777 pthread__error(EINVAL, "Invalid thread", 778 thread->pt_magic == PT_MAGIC); 779 780 if (pthread__find(thread) != 0) 781 return ESRCH; 782 783 pthread_mutex_lock(&thread->pt_lock); 784 if (thread->pt_name == NULL) 785 name[0] = '\0'; 786 else 787 strlcpy(name, thread->pt_name, len); 788 pthread_mutex_unlock(&thread->pt_lock); 789 790 return 0; 791 } 792 793 794 int 795 pthread_setname_np(pthread_t thread, const char *name, void *arg) 796 { 797 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 798 int namelen; 799 800 pthread__error(EINVAL, "Invalid thread", 801 thread->pt_magic == PT_MAGIC); 802 803 if (pthread__find(thread) != 0) 804 return ESRCH; 805 806 namelen = snprintf(newname, sizeof(newname), name, arg); 807 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 808 return EINVAL; 809 810 cp = strdup(newname); 811 if (cp == NULL) 812 return ENOMEM; 813 814 pthread_mutex_lock(&thread->pt_lock); 815 oldname = thread->pt_name; 816 thread->pt_name = cp; 817 (void)_lwp_setname(thread->pt_lid, cp); 818 pthread_mutex_unlock(&thread->pt_lock); 819 820 if (oldname != NULL) 821 free(oldname); 822 823 return 0; 824 } 825 826 827 pthread_t 828 pthread_self(void) 829 { 830 if (__predict_false(__uselibcstub)) 831 return (pthread_t)__libc_thr_self_stub(); 832 833 return pthread__self(); 834 } 835 836 837 int 838 pthread_cancel(pthread_t thread) 839 { 840 841 pthread__error(EINVAL, "Invalid thread", 842 thread->pt_magic == PT_MAGIC); 843 844 if (pthread__find(thread) != 0) 845 return ESRCH; 846 pthread_mutex_lock(&thread->pt_lock); 847 thread->pt_flags |= PT_FLAG_CS_PENDING; 848 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 849 thread->pt_cancel = 1; 850 pthread_mutex_unlock(&thread->pt_lock); 851 _lwp_wakeup(thread->pt_lid); 852 } else 853 pthread_mutex_unlock(&thread->pt_lock); 854 855 return 0; 856 } 857 858 859 int 860 pthread_setcancelstate(int state, int *oldstate) 861 { 862 pthread_t self; 863 int retval; 864 865 if (__predict_false(__uselibcstub)) 866 return __libc_thr_setcancelstate_stub(state, oldstate); 867 868 self = pthread__self(); 869 retval = 0; 870 871 pthread_mutex_lock(&self->pt_lock); 872 873 if (oldstate != NULL) { 874 if (self->pt_flags & PT_FLAG_CS_DISABLED) 875 *oldstate = PTHREAD_CANCEL_DISABLE; 876 else 877 *oldstate = PTHREAD_CANCEL_ENABLE; 878 } 879 880 if (state == PTHREAD_CANCEL_DISABLE) { 881 self->pt_flags |= PT_FLAG_CS_DISABLED; 882 if (self->pt_cancel) { 883 self->pt_flags |= PT_FLAG_CS_PENDING; 884 self->pt_cancel = 0; 885 } 886 } else if (state == PTHREAD_CANCEL_ENABLE) { 887 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 888 /* 889 * If a cancellation was requested while cancellation 890 * was disabled, note that fact for future 891 * cancellation tests. 892 */ 893 if (self->pt_flags & PT_FLAG_CS_PENDING) { 894 self->pt_cancel = 1; 895 /* This is not a deferred cancellation point. */ 896 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 897 pthread_mutex_unlock(&self->pt_lock); 898 pthread__cancelled(); 899 } 900 } 901 } else 902 retval = EINVAL; 903 904 pthread_mutex_unlock(&self->pt_lock); 905 906 return retval; 907 } 908 909 910 int 911 pthread_setcanceltype(int type, int *oldtype) 912 { 913 pthread_t self; 914 int retval; 915 916 self = pthread__self(); 917 retval = 0; 918 919 pthread_mutex_lock(&self->pt_lock); 920 921 if (oldtype != NULL) { 922 if (self->pt_flags & PT_FLAG_CS_ASYNC) 923 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 924 else 925 *oldtype = PTHREAD_CANCEL_DEFERRED; 926 } 927 928 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 929 self->pt_flags |= PT_FLAG_CS_ASYNC; 930 if (self->pt_cancel) { 931 pthread_mutex_unlock(&self->pt_lock); 932 pthread__cancelled(); 933 } 934 } else if (type == PTHREAD_CANCEL_DEFERRED) 935 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 936 else 937 retval = EINVAL; 938 939 pthread_mutex_unlock(&self->pt_lock); 940 941 return retval; 942 } 943 944 945 void 946 pthread_testcancel(void) 947 { 948 pthread_t self; 949 950 self = pthread__self(); 951 if (self->pt_cancel) 952 pthread__cancelled(); 953 } 954 955 956 /* 957 * POSIX requires that certain functions return an error rather than 958 * invoking undefined behavior even when handed completely bogus 959 * pthread_t values, e.g. stack garbage. 960 */ 961 int 962 pthread__find(pthread_t id) 963 { 964 pthread_t target; 965 int error; 966 967 pthread_rwlock_rdlock(&pthread__alltree_lock); 968 target = rb_tree_find_node(&pthread__alltree, id); 969 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH; 970 pthread_rwlock_unlock(&pthread__alltree_lock); 971 972 return error; 973 } 974 975 976 void 977 pthread__testcancel(pthread_t self) 978 { 979 980 if (self->pt_cancel) 981 pthread__cancelled(); 982 } 983 984 985 void 986 pthread__cancelled(void) 987 { 988 989 pthread_exit(PTHREAD_CANCELED); 990 } 991 992 993 void 994 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 995 { 996 pthread_t self; 997 struct pt_clean_t *entry; 998 999 self = pthread__self(); 1000 entry = store; 1001 entry->ptc_cleanup = cleanup; 1002 entry->ptc_arg = arg; 1003 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1004 } 1005 1006 1007 void 1008 pthread__cleanup_pop(int ex, void *store) 1009 { 1010 pthread_t self; 1011 struct pt_clean_t *entry; 1012 1013 self = pthread__self(); 1014 entry = store; 1015 1016 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1017 if (ex) 1018 (*entry->ptc_cleanup)(entry->ptc_arg); 1019 } 1020 1021 1022 int * 1023 pthread__errno(void) 1024 { 1025 pthread_t self; 1026 1027 if (__predict_false(__uselibcstub)) { 1028 pthread__errorfunc(__FILE__, __LINE__, __func__, 1029 "pthread__errno() requires linking with -lpthread"); 1030 return __libc_thr_errno_stub(); 1031 } 1032 1033 self = pthread__self(); 1034 1035 return &(self->pt_errno); 1036 } 1037 1038 ssize_t _sys_write(int, const void *, size_t); 1039 1040 void 1041 pthread__assertfunc(const char *file, int line, const char *function, 1042 const char *expr) 1043 { 1044 char buf[1024]; 1045 int len; 1046 1047 /* 1048 * snprintf should not acquire any locks, or we could 1049 * end up deadlocked if the assert caller held locks. 1050 */ 1051 len = snprintf(buf, 1024, 1052 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1053 expr, file, line, 1054 function ? ", function \"" : "", 1055 function ? function : "", 1056 function ? "\"" : ""); 1057 1058 _sys_write(STDERR_FILENO, buf, (size_t)len); 1059 (void)raise(SIGABRT); 1060 _exit(1); 1061 } 1062 1063 1064 void 1065 pthread__errorfunc(const char *file, int line, const char *function, 1066 const char *msg, ...) 1067 { 1068 char buf[1024]; 1069 char buf2[1024]; 1070 size_t len; 1071 va_list ap; 1072 1073 if (pthread__diagassert == 0) 1074 return; 1075 1076 va_start(ap, msg); 1077 vsnprintf_ss(buf2, sizeof(buf2), msg, ap); 1078 va_end(ap); 1079 1080 /* 1081 * snprintf should not acquire any locks, or we could 1082 * end up deadlocked if the assert caller held locks. 1083 */ 1084 len = snprintf_ss(buf, sizeof(buf), 1085 "%s: Error detected by libpthread: %s.\n" 1086 "Detected by file \"%s\", line %d%s%s%s.\n" 1087 "See pthread(3) for information.\n", 1088 getprogname(), buf2, file, line, 1089 function ? ", function \"" : "", 1090 function ? function : "", 1091 function ? "\"" : ""); 1092 1093 if (pthread__diagassert & DIAGASSERT_STDERR) 1094 _sys_write(STDERR_FILENO, buf, len); 1095 1096 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1097 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1098 1099 if (pthread__diagassert & DIAGASSERT_ABORT) { 1100 (void)_lwp_kill(_lwp_self(), SIGABRT); 1101 _exit(1); 1102 } 1103 } 1104 1105 /* 1106 * Thread park/unpark operations. The kernel operations are 1107 * modelled after a brief description from "Multithreading in 1108 * the Solaris Operating Environment": 1109 * 1110 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1111 */ 1112 1113 int 1114 pthread__park(pthread_t self, pthread_mutex_t *lock, 1115 pthread_queue_t *queue, const struct timespec *abstime, 1116 int cancelpt) 1117 { 1118 int rv, error; 1119 1120 pthread_mutex_unlock(lock); 1121 1122 /* 1123 * Wait until we are awoken by a pending unpark operation, 1124 * a signal, an unpark posted after we have gone asleep, 1125 * or an expired timeout. 1126 * 1127 * It is fine to test the value of pt_sleepobj without 1128 * holding any locks, because: 1129 * 1130 * o Only the blocking thread (this thread) ever sets it 1131 * to a non-NULL value. 1132 * 1133 * o Other threads may set it NULL, but if they do so they 1134 * must also make this thread return from _lwp_park. 1135 * 1136 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1137 * calls and all make use of spinlocks in the kernel. So 1138 * these system calls act as full memory barriers. 1139 */ 1140 rv = 0; 1141 do { 1142 /* 1143 * If we deferred unparking a thread, arrange to 1144 * have _lwp_park() restart it before blocking. 1145 */ 1146 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME, 1147 __UNCONST(abstime), 0, NULL, NULL); 1148 if (error != 0) { 1149 switch (rv = errno) { 1150 case EINTR: 1151 case EALREADY: 1152 rv = 0; 1153 break; 1154 case ETIMEDOUT: 1155 break; 1156 default: 1157 pthread__errorfunc(__FILE__, __LINE__, 1158 __func__, "_lwp_park failed: %d", errno); 1159 break; 1160 } 1161 } 1162 /* Check for cancellation. */ 1163 if (cancelpt && self->pt_cancel) 1164 rv = EINTR; 1165 } while (self->pt_sleepobj != NULL && rv == 0); 1166 return rv; 1167 } 1168 1169 void 1170 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1171 pthread_mutex_t *interlock) 1172 { 1173 pthread_t target; 1174 1175 target = PTQ_FIRST(queue); 1176 target->pt_sleepobj = NULL; 1177 PTQ_REMOVE(queue, target, pt_sleep); 1178 (void)_lwp_unpark(target->pt_lid, NULL); 1179 } 1180 1181 void 1182 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1183 pthread_mutex_t *interlock) 1184 { 1185 lwpid_t lids[PTHREAD__UNPARK_MAX]; 1186 const size_t mlid = pthread__unpark_max; 1187 pthread_t target; 1188 size_t nlid = 0; 1189 1190 PTQ_FOREACH(target, queue, pt_sleep) { 1191 if (nlid == mlid) { 1192 (void)_lwp_unpark_all(lids, nlid, NULL); 1193 nlid = 0; 1194 } 1195 target->pt_sleepobj = NULL; 1196 lids[nlid++] = target->pt_lid; 1197 } 1198 PTQ_INIT(queue); 1199 if (nlid == 1) { 1200 (void)_lwp_unpark(lids[0], NULL); 1201 } else if (nlid > 1) { 1202 (void)_lwp_unpark_all(lids, nlid, NULL); 1203 } 1204 } 1205 1206 #undef OOPS 1207 1208 static void 1209 pthread__initmainstack(void) 1210 { 1211 struct rlimit slimit; 1212 const AuxInfo *aux; 1213 size_t size, len; 1214 int mib[2]; 1215 unsigned int value; 1216 1217 _DIAGASSERT(_dlauxinfo() != NULL); 1218 1219 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1220 err(EXIT_FAILURE, 1221 "Couldn't get stack resource consumption limits"); 1222 size = slimit.rlim_cur; 1223 pthread__main->pt_stack.ss_size = size; 1224 pthread__main->pt_guardsize = pthread__pagesize; 1225 1226 mib[0] = CTL_VM; 1227 mib[1] = VM_GUARD_SIZE; 1228 len = sizeof(value); 1229 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 1230 pthread__main->pt_guardsize = value; 1231 1232 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1233 if (aux->a_type == AT_STACKBASE) { 1234 #ifdef __MACHINE_STACK_GROWS_UP 1235 pthread__main->pt_stack.ss_sp = (void *)aux->a_v; 1236 #else 1237 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size; 1238 #endif 1239 break; 1240 } 1241 } 1242 pthread__copy_tsd(pthread__main); 1243 } 1244 1245 /* 1246 * Set up the slightly special stack for the "initial" thread, which 1247 * runs on the normal system stack, and thus gets slightly different 1248 * treatment. 1249 */ 1250 static void 1251 pthread__initmain(pthread_t *newt) 1252 { 1253 char *value; 1254 1255 pthread__initmainstack(); 1256 1257 value = pthread__getenv("PTHREAD_STACKSIZE"); 1258 if (value != NULL) { 1259 pthread__stacksize = atoi(value) * 1024; 1260 if (pthread__stacksize > pthread__main->pt_stack.ss_size) 1261 pthread__stacksize = pthread__main->pt_stack.ss_size; 1262 } 1263 if (pthread__stacksize == 0) 1264 pthread__stacksize = pthread__main->pt_stack.ss_size; 1265 pthread__stacksize += pthread__pagesize - 1; 1266 pthread__stacksize &= ~(pthread__pagesize - 1); 1267 if (pthread__stacksize < 4 * pthread__pagesize) 1268 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1269 4 * pthread__pagesize / 1024); 1270 1271 *newt = pthread__main; 1272 #if defined(_PTHREAD_GETTCB_EXT) 1273 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT(); 1274 #elif defined(__HAVE___LWP_GETTCB_FAST) 1275 pthread__main->pt_tls = __lwp_gettcb_fast(); 1276 #else 1277 pthread__main->pt_tls = _lwp_getprivate(); 1278 #endif 1279 pthread__main->pt_tls->tcb_pthread = pthread__main; 1280 } 1281 1282 static signed int 1283 /*ARGSUSED*/ 1284 pthread__cmp(void *ctx, const void *n1, const void *n2) 1285 { 1286 const uintptr_t p1 = (const uintptr_t)n1; 1287 const uintptr_t p2 = (const uintptr_t)n2; 1288 1289 if (p1 < p2) 1290 return -1; 1291 if (p1 > p2) 1292 return 1; 1293 return 0; 1294 } 1295 1296 /* Because getenv() wants to use locks. */ 1297 char * 1298 pthread__getenv(const char *name) 1299 { 1300 extern char **environ; 1301 size_t l_name, offset; 1302 1303 if (issetugid()) 1304 return (NULL); 1305 1306 l_name = strlen(name); 1307 for (offset = 0; environ[offset] != NULL; offset++) { 1308 if (strncmp(name, environ[offset], l_name) == 0 && 1309 environ[offset][l_name] == '=') { 1310 return environ[offset] + l_name + 1; 1311 } 1312 } 1313 1314 return NULL; 1315 } 1316 1317 pthread_mutex_t * 1318 pthread__hashlock(volatile const void *p) 1319 { 1320 uintptr_t v; 1321 1322 v = (uintptr_t)p; 1323 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1324 } 1325 1326 int 1327 pthread__checkpri(int pri) 1328 { 1329 static int havepri; 1330 static long min, max; 1331 1332 if (!havepri) { 1333 min = sysconf(_SC_SCHED_PRI_MIN); 1334 max = sysconf(_SC_SCHED_PRI_MAX); 1335 havepri = 1; 1336 } 1337 return (pri < min || pri > max) ? EINVAL : 0; 1338 } 1339