1 /* $NetBSD: pthread.c,v 1.165 2020/02/15 23:59:30 kamil Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Nathan J. Williams and Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __RCSID("$NetBSD: pthread.c,v 1.165 2020/02/15 23:59:30 kamil Exp $"); 35 36 #define __EXPOSE_STACK 1 37 38 #include <sys/param.h> 39 #include <sys/exec_elf.h> 40 #include <sys/mman.h> 41 #include <sys/lwp.h> 42 #include <sys/lwpctl.h> 43 #include <sys/resource.h> 44 #include <sys/sysctl.h> 45 #include <sys/tls.h> 46 #include <uvm/uvm_param.h> 47 48 #include <assert.h> 49 #include <dlfcn.h> 50 #include <err.h> 51 #include <errno.h> 52 #include <lwp.h> 53 #include <signal.h> 54 #include <stdio.h> 55 #include <stdlib.h> 56 #include <stddef.h> 57 #include <string.h> 58 #include <syslog.h> 59 #include <ucontext.h> 60 #include <unistd.h> 61 #include <sched.h> 62 63 #include "atexit.h" 64 #include "pthread.h" 65 #include "pthread_int.h" 66 #include "pthread_makelwp.h" 67 #include "reentrant.h" 68 69 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 70 static rb_tree_t pthread__alltree; 71 72 static signed int pthread__cmp(void *, const void *, const void *); 73 74 static const rb_tree_ops_t pthread__alltree_ops = { 75 .rbto_compare_nodes = pthread__cmp, 76 .rbto_compare_key = pthread__cmp, 77 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree), 78 .rbto_context = NULL 79 }; 80 81 static void pthread__create_tramp(void *); 82 static void pthread__initthread(pthread_t); 83 static void pthread__scrubthread(pthread_t, char *, int); 84 static void pthread__initmain(pthread_t *); 85 static void pthread__fork_callback(void); 86 static void pthread__reap(pthread_t); 87 static void pthread__child_callback(void); 88 static void pthread__start(void); 89 90 void pthread__init(void); 91 92 int pthread__started; 93 int __uselibcstub = 1; 94 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 95 pthread_queue_t pthread__deadqueue; 96 pthread_queue_t pthread__allqueue; 97 98 static pthread_attr_t pthread_default_attr; 99 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 100 101 enum { 102 DIAGASSERT_ABORT = 1<<0, 103 DIAGASSERT_STDERR = 1<<1, 104 DIAGASSERT_SYSLOG = 1<<2 105 }; 106 107 static int pthread__diagassert; 108 109 int pthread__concurrency; 110 int pthread__nspins; 111 int pthread__unpark_max = PTHREAD__UNPARK_MAX; 112 int pthread__dbg; /* set by libpthread_dbg if active */ 113 114 /* 115 * We have to initialize the pthread_stack* variables here because 116 * mutexes are used before pthread_init() and thus pthread__initmain() 117 * are called. Since mutexes only save the stack pointer and not a 118 * pointer to the thread data, it is safe to change the mapping from 119 * stack pointer to thread data afterwards. 120 */ 121 size_t pthread__stacksize; 122 size_t pthread__guardsize; 123 size_t pthread__pagesize; 124 static struct __pthread_st *pthread__main; 125 static size_t __pthread_st_size; 126 127 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 128 129 __strong_alias(__libc_thr_self,pthread_self) 130 __strong_alias(__libc_thr_create,pthread_create) 131 __strong_alias(__libc_thr_exit,pthread_exit) 132 __strong_alias(__libc_thr_errno,pthread__errno) 133 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 134 __strong_alias(__libc_thr_equal,pthread_equal) 135 __strong_alias(__libc_thr_init,pthread__init) 136 137 /* 138 * Static library kludge. Place a reference to a symbol any library 139 * file which does not already have a reference here. 140 */ 141 extern int pthread__cancel_stub_binder; 142 143 void *pthread__static_lib_binder[] = { 144 &pthread__cancel_stub_binder, 145 pthread_cond_init, 146 pthread_mutex_init, 147 pthread_rwlock_init, 148 pthread_barrier_init, 149 pthread_key_create, 150 pthread_setspecific, 151 }; 152 153 #define NHASHLOCK 64 154 155 static union hashlock { 156 pthread_mutex_t mutex; 157 char pad[64]; 158 } hashlocks[NHASHLOCK] __aligned(64); 159 160 /* 161 * This needs to be started by the library loading code, before main() 162 * gets to run, for various things that use the state of the initial thread 163 * to work properly (thread-specific data is an application-visible example; 164 * spinlock counts for mutexes is an internal example). 165 */ 166 void 167 pthread__init(void) 168 { 169 pthread_t first; 170 char *p; 171 int i; 172 int mib[2]; 173 unsigned int value; 174 size_t len; 175 extern int __isthreaded; 176 177 /* 178 * Allocate pthread_keys descriptors before 179 * reseting __uselibcstub because otherwise 180 * malloc() will call pthread_keys_create() 181 * while pthread_keys descriptors are not 182 * yet allocated. 183 */ 184 pthread__main = pthread_tsd_earlyinit(&__pthread_st_size); 185 if (pthread__main == NULL) 186 err(EXIT_FAILURE, "Cannot allocate pthread storage"); 187 188 __uselibcstub = 0; 189 190 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 191 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF); 192 193 mib[0] = CTL_VM; 194 mib[1] = VM_THREAD_GUARD_SIZE; 195 len = sizeof(value); 196 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 197 pthread__guardsize = value; 198 else 199 pthread__guardsize = pthread__pagesize; 200 201 /* Initialize locks first; they're needed elsewhere. */ 202 pthread__lockprim_init(); 203 for (i = 0; i < NHASHLOCK; i++) { 204 pthread_mutex_init(&hashlocks[i].mutex, NULL); 205 } 206 207 /* Fetch parameters. */ 208 i = (int)_lwp_unpark_all(NULL, 0, NULL); 209 if (i == -1) 210 err(EXIT_FAILURE, "_lwp_unpark_all"); 211 if (i < pthread__unpark_max) 212 pthread__unpark_max = i; 213 214 /* Basic data structure setup */ 215 pthread_attr_init(&pthread_default_attr); 216 PTQ_INIT(&pthread__allqueue); 217 PTQ_INIT(&pthread__deadqueue); 218 219 rb_tree_init(&pthread__alltree, &pthread__alltree_ops); 220 221 /* Create the thread structure corresponding to main() */ 222 pthread__initmain(&first); 223 pthread__initthread(first); 224 pthread__scrubthread(first, NULL, 0); 225 226 first->pt_lid = _lwp_self(); 227 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 228 (void)rb_tree_insert_node(&pthread__alltree, first); 229 230 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 231 err(EXIT_FAILURE, "_lwp_ctl"); 232 } 233 234 /* Start subsystems */ 235 PTHREAD_MD_INIT 236 237 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 238 switch (*p) { 239 case 'a': 240 pthread__diagassert |= DIAGASSERT_ABORT; 241 break; 242 case 'A': 243 pthread__diagassert &= ~DIAGASSERT_ABORT; 244 break; 245 case 'e': 246 pthread__diagassert |= DIAGASSERT_STDERR; 247 break; 248 case 'E': 249 pthread__diagassert &= ~DIAGASSERT_STDERR; 250 break; 251 case 'l': 252 pthread__diagassert |= DIAGASSERT_SYSLOG; 253 break; 254 case 'L': 255 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 256 break; 257 } 258 } 259 260 /* 261 * Tell libc that we're here and it should role-play accordingly. 262 * 263 * pthread_atfork(3) calls malloc(3) and initializes the system malloc. 264 */ 265 pthread_atfork(NULL, NULL, pthread__fork_callback); 266 267 /* Requires functional malloc(3). */ 268 pthread_tsd_init(); 269 270 __isthreaded = 1; 271 } 272 273 static void 274 pthread__fork_callback(void) 275 { 276 struct __pthread_st *self = pthread__self(); 277 278 /* lwpctl state is not copied across fork. */ 279 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 280 err(EXIT_FAILURE, "_lwp_ctl"); 281 } 282 self->pt_lid = _lwp_self(); 283 } 284 285 static void 286 pthread__child_callback(void) 287 { 288 289 /* 290 * Clean up data structures that a forked child process might 291 * trip over. Note that if threads have been created (causing 292 * this handler to be registered) the standards say that the 293 * child will trigger undefined behavior if it makes any 294 * pthread_* calls (or any other calls that aren't 295 * async-signal-safe), so we don't really have to clean up 296 * much. Anything that permits some pthread_* calls to work is 297 * merely being polite. 298 */ 299 pthread__started = 0; 300 } 301 302 static void 303 pthread__start(void) 304 { 305 306 /* 307 * Per-process timers are cleared by fork(); despite the 308 * various restrictions on fork() and threads, it's legal to 309 * fork() before creating any threads. 310 */ 311 pthread_atfork(NULL, NULL, pthread__child_callback); 312 } 313 314 315 /* General-purpose thread data structure sanitization. */ 316 /* ARGSUSED */ 317 static void 318 pthread__initthread(pthread_t t) 319 { 320 321 t->pt_self = t; 322 t->pt_magic = PT_MAGIC; 323 t->pt_willpark = 0; 324 t->pt_unpark = 0; 325 t->pt_nwaiters = 0; 326 t->pt_sleepobj = NULL; 327 t->pt_signalled = 0; 328 t->pt_havespecific = 0; 329 t->pt_early = NULL; 330 t->pt_lwpctl = &pthread__dummy_lwpctl; 331 332 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 333 pthread_mutex_init(&t->pt_lock, NULL); 334 PTQ_INIT(&t->pt_cleanup_stack); 335 } 336 337 static void 338 pthread__scrubthread(pthread_t t, char *name, int flags) 339 { 340 341 t->pt_state = PT_STATE_RUNNING; 342 t->pt_exitval = NULL; 343 t->pt_flags = flags; 344 t->pt_cancel = 0; 345 t->pt_errno = 0; 346 t->pt_name = name; 347 t->pt_lid = 0; 348 } 349 350 static int 351 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 352 { 353 void *stackbase, *stackbase2, *redzone; 354 size_t stacksize, guardsize; 355 bool allocated; 356 357 if (attr != NULL) { 358 pthread_attr_getstack(attr, &stackbase, &stacksize); 359 pthread_attr_getguardsize(attr, &guardsize); 360 } else { 361 stackbase = NULL; 362 stacksize = 0; 363 guardsize = pthread__guardsize; 364 } 365 if (stacksize == 0) 366 stacksize = pthread__stacksize; 367 368 if (newthread->pt_stack_allocated) { 369 if (stackbase == NULL && 370 newthread->pt_stack.ss_size == stacksize && 371 newthread->pt_guardsize == guardsize) 372 return 0; 373 stackbase2 = newthread->pt_stack.ss_sp; 374 #ifndef __MACHINE_STACK_GROWS_UP 375 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 376 #endif 377 munmap(stackbase2, 378 newthread->pt_stack.ss_size + newthread->pt_guardsize); 379 newthread->pt_stack.ss_sp = NULL; 380 newthread->pt_stack.ss_size = 0; 381 newthread->pt_guardsize = 0; 382 newthread->pt_stack_allocated = false; 383 } 384 385 newthread->pt_stack_allocated = false; 386 387 if (stackbase == NULL) { 388 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 389 guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1; 390 stackbase = mmap(NULL, stacksize + guardsize, 391 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 392 if (stackbase == MAP_FAILED) 393 return ENOMEM; 394 allocated = true; 395 } else { 396 allocated = false; 397 } 398 #ifdef __MACHINE_STACK_GROWS_UP 399 redzone = (char *)stackbase + stacksize; 400 stackbase2 = (char *)stackbase; 401 #else 402 redzone = (char *)stackbase; 403 stackbase2 = (char *)stackbase + guardsize; 404 #endif 405 if (allocated && guardsize && 406 mprotect(redzone, guardsize, PROT_NONE) == -1) { 407 munmap(stackbase, stacksize + guardsize); 408 return EPERM; 409 } 410 newthread->pt_stack.ss_size = stacksize; 411 newthread->pt_stack.ss_sp = stackbase2; 412 newthread->pt_guardsize = guardsize; 413 newthread->pt_stack_allocated = allocated; 414 return 0; 415 } 416 417 int 418 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 419 void *(*startfunc)(void *), void *arg) 420 { 421 pthread_t newthread; 422 pthread_attr_t nattr; 423 struct pthread_attr_private *p; 424 char * volatile name; 425 unsigned long flag; 426 void *private_area; 427 int ret; 428 429 if (__predict_false(__uselibcstub)) { 430 pthread__errorfunc(__FILE__, __LINE__, __func__, 431 "pthread_create() requires linking with -lpthread"); 432 return __libc_thr_create_stub(thread, attr, startfunc, arg); 433 } 434 435 /* 436 * It's okay to check this without a lock because there can 437 * only be one thread before it becomes true. 438 */ 439 if (pthread__started == 0) { 440 pthread__start(); 441 pthread__started = 1; 442 } 443 444 if (attr == NULL) 445 nattr = pthread_default_attr; 446 else if (attr->pta_magic == PT_ATTR_MAGIC) 447 nattr = *attr; 448 else 449 return EINVAL; 450 451 /* Fetch misc. attributes from the attr structure. */ 452 name = NULL; 453 if ((p = nattr.pta_private) != NULL) 454 if (p->ptap_name[0] != '\0') 455 if ((name = strdup(p->ptap_name)) == NULL) 456 return ENOMEM; 457 458 newthread = NULL; 459 460 /* 461 * Try to reclaim a dead thread. 462 */ 463 if (!PTQ_EMPTY(&pthread__deadqueue)) { 464 pthread_mutex_lock(&pthread__deadqueue_lock); 465 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 466 /* Still busily exiting, or finished? */ 467 if (newthread->pt_lwpctl->lc_curcpu == 468 LWPCTL_CPU_EXITED) 469 break; 470 } 471 if (newthread) 472 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 473 pthread_mutex_unlock(&pthread__deadqueue_lock); 474 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 475 if (newthread && newthread->pt_tls) { 476 _rtld_tls_free(newthread->pt_tls); 477 newthread->pt_tls = NULL; 478 } 479 #endif 480 } 481 482 /* 483 * If necessary set up a stack, allocate space for a pthread_st, 484 * and initialize it. 485 */ 486 if (newthread == NULL) { 487 newthread = calloc(1, __pthread_st_size); 488 if (newthread == NULL) { 489 free(name); 490 return ENOMEM; 491 } 492 newthread->pt_stack_allocated = false; 493 494 if (pthread__getstack(newthread, attr)) { 495 free(newthread); 496 free(name); 497 return ENOMEM; 498 } 499 500 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 501 newthread->pt_tls = NULL; 502 #endif 503 504 /* Add to list of all threads. */ 505 pthread_rwlock_wrlock(&pthread__alltree_lock); 506 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 507 (void)rb_tree_insert_node(&pthread__alltree, newthread); 508 pthread_rwlock_unlock(&pthread__alltree_lock); 509 510 /* Will be reset by the thread upon exit. */ 511 pthread__initthread(newthread); 512 } else { 513 if (pthread__getstack(newthread, attr)) { 514 pthread_mutex_lock(&pthread__deadqueue_lock); 515 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 516 pthread_mutex_unlock(&pthread__deadqueue_lock); 517 return ENOMEM; 518 } 519 } 520 521 /* 522 * Create the new LWP. 523 */ 524 pthread__scrubthread(newthread, name, nattr.pta_flags); 525 newthread->pt_func = startfunc; 526 newthread->pt_arg = arg; 527 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 528 private_area = newthread->pt_tls = _rtld_tls_allocate(); 529 newthread->pt_tls->tcb_pthread = newthread; 530 #else 531 private_area = newthread; 532 #endif 533 534 flag = 0; 535 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 536 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 537 flag |= LWP_SUSPENDED; 538 if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0) 539 flag |= LWP_DETACHED; 540 541 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area, 542 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size, 543 flag, &newthread->pt_lid); 544 if (ret != 0) { 545 ret = errno; 546 pthread_mutex_lock(&newthread->pt_lock); 547 /* Will unlock and free name. */ 548 pthread__reap(newthread); 549 return ret; 550 } 551 552 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 553 if (p != NULL) { 554 (void)pthread_setschedparam(newthread, p->ptap_policy, 555 &p->ptap_sp); 556 } 557 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 558 (void)_lwp_continue(newthread->pt_lid); 559 } 560 } 561 562 *thread = newthread; 563 564 return 0; 565 } 566 567 568 __dead static void 569 pthread__create_tramp(void *cookie) 570 { 571 pthread_t self; 572 void *retval; 573 574 self = cookie; 575 576 /* 577 * Throw away some stack in a feeble attempt to reduce cache 578 * thrash. May help for SMT processors. XXX We should not 579 * be allocating stacks on fixed 2MB boundaries. Needs a 580 * thread register or decent thread local storage. 581 */ 582 (void)alloca(((unsigned)self->pt_lid & 7) << 8); 583 584 if (self->pt_name != NULL) { 585 pthread_mutex_lock(&self->pt_lock); 586 if (self->pt_name != NULL) 587 (void)_lwp_setname(0, self->pt_name); 588 pthread_mutex_unlock(&self->pt_lock); 589 } 590 591 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 592 err(EXIT_FAILURE, "_lwp_ctl"); 593 } 594 595 retval = (*self->pt_func)(self->pt_arg); 596 597 pthread_exit(retval); 598 599 /*NOTREACHED*/ 600 pthread__abort(); 601 } 602 603 int 604 pthread_suspend_np(pthread_t thread) 605 { 606 pthread_t self; 607 608 pthread__error(EINVAL, "Invalid thread", 609 thread->pt_magic == PT_MAGIC); 610 611 self = pthread__self(); 612 if (self == thread) { 613 return EDEADLK; 614 } 615 if (pthread__find(thread) != 0) 616 return ESRCH; 617 if (_lwp_suspend(thread->pt_lid) == 0) 618 return 0; 619 return errno; 620 } 621 622 int 623 pthread_resume_np(pthread_t thread) 624 { 625 626 pthread__error(EINVAL, "Invalid thread", 627 thread->pt_magic == PT_MAGIC); 628 629 if (pthread__find(thread) != 0) 630 return ESRCH; 631 if (_lwp_continue(thread->pt_lid) == 0) 632 return 0; 633 return errno; 634 } 635 636 /* 637 * In case the thread is exiting at an inopportune time leaving waiters not 638 * awoken (because cancelled, for instance) make sure we have no waiters 639 * left. 640 */ 641 static void 642 pthread__clear_waiters(pthread_t self) 643 { 644 645 if (self->pt_nwaiters != 0) { 646 (void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters, 647 NULL); 648 self->pt_nwaiters = 0; 649 } 650 self->pt_willpark = 0; 651 } 652 653 void 654 pthread_exit(void *retval) 655 { 656 pthread_t self; 657 struct pt_clean_t *cleanup; 658 659 if (__predict_false(__uselibcstub)) { 660 __libc_thr_exit_stub(retval); 661 goto out; 662 } 663 664 self = pthread__self(); 665 666 /* Disable cancellability. */ 667 pthread_mutex_lock(&self->pt_lock); 668 self->pt_flags |= PT_FLAG_CS_DISABLED; 669 self->pt_cancel = 0; 670 671 /* Call any cancellation cleanup handlers */ 672 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 673 pthread_mutex_unlock(&self->pt_lock); 674 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 675 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 676 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 677 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 678 } 679 pthread_mutex_lock(&self->pt_lock); 680 } 681 682 pthread_mutex_unlock(&self->pt_lock); 683 __cxa_thread_run_atexit(); 684 pthread_mutex_lock(&self->pt_lock); 685 686 /* Perform cleanup of thread-specific data */ 687 pthread__destroy_tsd(self); 688 689 /* 690 * Signal our exit. Our stack and pthread_t won't be reused until 691 * pthread_create() can see from kernel info that this LWP is gone. 692 */ 693 self->pt_exitval = retval; 694 if (self->pt_flags & PT_FLAG_DETACHED) { 695 /* pthread__reap() will drop the lock. */ 696 pthread__reap(self); 697 pthread__clear_waiters(self); 698 _lwp_exit(); 699 } else { 700 self->pt_state = PT_STATE_ZOMBIE; 701 pthread_mutex_unlock(&self->pt_lock); 702 pthread__clear_waiters(self); 703 /* Note: name will be freed by the joiner. */ 704 _lwp_exit(); 705 } 706 707 out: 708 /*NOTREACHED*/ 709 pthread__abort(); 710 exit(1); 711 } 712 713 714 int 715 pthread_join(pthread_t thread, void **valptr) 716 { 717 pthread_t self; 718 719 pthread__error(EINVAL, "Invalid thread", 720 thread->pt_magic == PT_MAGIC); 721 722 self = pthread__self(); 723 724 if (pthread__find(thread) != 0) 725 return ESRCH; 726 727 if (thread == self) 728 return EDEADLK; 729 730 /* IEEE Std 1003.1 says pthread_join() never returns EINTR. */ 731 for (;;) { 732 pthread__testcancel(self); 733 if (_lwp_wait(thread->pt_lid, NULL) == 0) 734 break; 735 if (errno != EINTR) 736 return errno; 737 } 738 739 /* 740 * Don't test for cancellation again. The spec is that if 741 * cancelled, pthread_join() must not have succeeded. 742 */ 743 pthread_mutex_lock(&thread->pt_lock); 744 if (thread->pt_state != PT_STATE_ZOMBIE) { 745 pthread__errorfunc(__FILE__, __LINE__, __func__, 746 "not a zombie"); 747 } 748 if (valptr != NULL) 749 *valptr = thread->pt_exitval; 750 751 /* pthread__reap() will drop the lock. */ 752 pthread__reap(thread); 753 return 0; 754 } 755 756 static void 757 pthread__reap(pthread_t thread) 758 { 759 char *name; 760 761 name = thread->pt_name; 762 thread->pt_name = NULL; 763 thread->pt_state = PT_STATE_DEAD; 764 pthread_mutex_unlock(&thread->pt_lock); 765 766 pthread_mutex_lock(&pthread__deadqueue_lock); 767 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 768 pthread_mutex_unlock(&pthread__deadqueue_lock); 769 770 if (name != NULL) 771 free(name); 772 } 773 774 int 775 pthread_equal(pthread_t t1, pthread_t t2) 776 { 777 778 if (__predict_false(__uselibcstub)) 779 return __libc_thr_equal_stub(t1, t2); 780 781 pthread__error(0, "Invalid thread", 782 (t1 != NULL) && (t1->pt_magic == PT_MAGIC)); 783 784 pthread__error(0, "Invalid thread", 785 (t2 != NULL) && (t2->pt_magic == PT_MAGIC)); 786 787 /* Nothing special here. */ 788 return (t1 == t2); 789 } 790 791 792 int 793 pthread_detach(pthread_t thread) 794 { 795 int error; 796 797 pthread__error(EINVAL, "Invalid thread", 798 thread->pt_magic == PT_MAGIC); 799 800 if (pthread__find(thread) != 0) 801 return ESRCH; 802 803 pthread_mutex_lock(&thread->pt_lock); 804 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 805 error = EINVAL; 806 } else { 807 error = _lwp_detach(thread->pt_lid); 808 if (error == 0) 809 thread->pt_flags |= PT_FLAG_DETACHED; 810 else 811 error = errno; 812 } 813 if (thread->pt_state == PT_STATE_ZOMBIE) { 814 /* pthread__reap() will drop the lock. */ 815 pthread__reap(thread); 816 } else 817 pthread_mutex_unlock(&thread->pt_lock); 818 return error; 819 } 820 821 822 int 823 pthread_getname_np(pthread_t thread, char *name, size_t len) 824 { 825 826 pthread__error(EINVAL, "Invalid thread", 827 thread->pt_magic == PT_MAGIC); 828 829 if (pthread__find(thread) != 0) 830 return ESRCH; 831 832 pthread_mutex_lock(&thread->pt_lock); 833 if (thread->pt_name == NULL) 834 name[0] = '\0'; 835 else 836 strlcpy(name, thread->pt_name, len); 837 pthread_mutex_unlock(&thread->pt_lock); 838 839 return 0; 840 } 841 842 843 int 844 pthread_setname_np(pthread_t thread, const char *name, void *arg) 845 { 846 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 847 int namelen; 848 849 pthread__error(EINVAL, "Invalid thread", 850 thread->pt_magic == PT_MAGIC); 851 852 if (pthread__find(thread) != 0) 853 return ESRCH; 854 855 namelen = snprintf(newname, sizeof(newname), name, arg); 856 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 857 return EINVAL; 858 859 cp = strdup(newname); 860 if (cp == NULL) 861 return ENOMEM; 862 863 pthread_mutex_lock(&thread->pt_lock); 864 oldname = thread->pt_name; 865 thread->pt_name = cp; 866 (void)_lwp_setname(thread->pt_lid, cp); 867 pthread_mutex_unlock(&thread->pt_lock); 868 869 if (oldname != NULL) 870 free(oldname); 871 872 return 0; 873 } 874 875 876 pthread_t 877 pthread_self(void) 878 { 879 if (__predict_false(__uselibcstub)) 880 return (pthread_t)__libc_thr_self_stub(); 881 882 return pthread__self(); 883 } 884 885 886 int 887 pthread_cancel(pthread_t thread) 888 { 889 890 pthread__error(EINVAL, "Invalid thread", 891 thread->pt_magic == PT_MAGIC); 892 893 if (pthread__find(thread) != 0) 894 return ESRCH; 895 pthread_mutex_lock(&thread->pt_lock); 896 thread->pt_flags |= PT_FLAG_CS_PENDING; 897 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 898 thread->pt_cancel = 1; 899 pthread_mutex_unlock(&thread->pt_lock); 900 _lwp_wakeup(thread->pt_lid); 901 } else 902 pthread_mutex_unlock(&thread->pt_lock); 903 904 return 0; 905 } 906 907 908 int 909 pthread_setcancelstate(int state, int *oldstate) 910 { 911 pthread_t self; 912 int retval; 913 914 if (__predict_false(__uselibcstub)) 915 return __libc_thr_setcancelstate_stub(state, oldstate); 916 917 self = pthread__self(); 918 retval = 0; 919 920 pthread_mutex_lock(&self->pt_lock); 921 922 if (oldstate != NULL) { 923 if (self->pt_flags & PT_FLAG_CS_DISABLED) 924 *oldstate = PTHREAD_CANCEL_DISABLE; 925 else 926 *oldstate = PTHREAD_CANCEL_ENABLE; 927 } 928 929 if (state == PTHREAD_CANCEL_DISABLE) { 930 self->pt_flags |= PT_FLAG_CS_DISABLED; 931 if (self->pt_cancel) { 932 self->pt_flags |= PT_FLAG_CS_PENDING; 933 self->pt_cancel = 0; 934 } 935 } else if (state == PTHREAD_CANCEL_ENABLE) { 936 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 937 /* 938 * If a cancellation was requested while cancellation 939 * was disabled, note that fact for future 940 * cancellation tests. 941 */ 942 if (self->pt_flags & PT_FLAG_CS_PENDING) { 943 self->pt_cancel = 1; 944 /* This is not a deferred cancellation point. */ 945 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 946 pthread_mutex_unlock(&self->pt_lock); 947 pthread__cancelled(); 948 } 949 } 950 } else 951 retval = EINVAL; 952 953 pthread_mutex_unlock(&self->pt_lock); 954 955 return retval; 956 } 957 958 959 int 960 pthread_setcanceltype(int type, int *oldtype) 961 { 962 pthread_t self; 963 int retval; 964 965 self = pthread__self(); 966 retval = 0; 967 968 pthread_mutex_lock(&self->pt_lock); 969 970 if (oldtype != NULL) { 971 if (self->pt_flags & PT_FLAG_CS_ASYNC) 972 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 973 else 974 *oldtype = PTHREAD_CANCEL_DEFERRED; 975 } 976 977 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 978 self->pt_flags |= PT_FLAG_CS_ASYNC; 979 if (self->pt_cancel) { 980 pthread_mutex_unlock(&self->pt_lock); 981 pthread__cancelled(); 982 } 983 } else if (type == PTHREAD_CANCEL_DEFERRED) 984 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 985 else 986 retval = EINVAL; 987 988 pthread_mutex_unlock(&self->pt_lock); 989 990 return retval; 991 } 992 993 994 void 995 pthread_testcancel(void) 996 { 997 pthread_t self; 998 999 self = pthread__self(); 1000 if (self->pt_cancel) 1001 pthread__cancelled(); 1002 } 1003 1004 1005 /* 1006 * POSIX requires that certain functions return an error rather than 1007 * invoking undefined behavior even when handed completely bogus 1008 * pthread_t values, e.g. stack garbage. 1009 */ 1010 int 1011 pthread__find(pthread_t id) 1012 { 1013 pthread_t target; 1014 int error; 1015 1016 pthread_rwlock_rdlock(&pthread__alltree_lock); 1017 target = rb_tree_find_node(&pthread__alltree, id); 1018 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH; 1019 pthread_rwlock_unlock(&pthread__alltree_lock); 1020 1021 return error; 1022 } 1023 1024 1025 void 1026 pthread__testcancel(pthread_t self) 1027 { 1028 1029 if (self->pt_cancel) 1030 pthread__cancelled(); 1031 } 1032 1033 1034 void 1035 pthread__cancelled(void) 1036 { 1037 1038 pthread_exit(PTHREAD_CANCELED); 1039 } 1040 1041 1042 void 1043 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 1044 { 1045 pthread_t self; 1046 struct pt_clean_t *entry; 1047 1048 self = pthread__self(); 1049 entry = store; 1050 entry->ptc_cleanup = cleanup; 1051 entry->ptc_arg = arg; 1052 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1053 } 1054 1055 1056 void 1057 pthread__cleanup_pop(int ex, void *store) 1058 { 1059 pthread_t self; 1060 struct pt_clean_t *entry; 1061 1062 self = pthread__self(); 1063 entry = store; 1064 1065 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1066 if (ex) 1067 (*entry->ptc_cleanup)(entry->ptc_arg); 1068 } 1069 1070 1071 int * 1072 pthread__errno(void) 1073 { 1074 pthread_t self; 1075 1076 if (__predict_false(__uselibcstub)) { 1077 pthread__errorfunc(__FILE__, __LINE__, __func__, 1078 "pthread__errno() requires linking with -lpthread"); 1079 return __libc_thr_errno_stub(); 1080 } 1081 1082 self = pthread__self(); 1083 1084 return &(self->pt_errno); 1085 } 1086 1087 ssize_t _sys_write(int, const void *, size_t); 1088 1089 void 1090 pthread__assertfunc(const char *file, int line, const char *function, 1091 const char *expr) 1092 { 1093 char buf[1024]; 1094 int len; 1095 1096 /* 1097 * snprintf should not acquire any locks, or we could 1098 * end up deadlocked if the assert caller held locks. 1099 */ 1100 len = snprintf(buf, 1024, 1101 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1102 expr, file, line, 1103 function ? ", function \"" : "", 1104 function ? function : "", 1105 function ? "\"" : ""); 1106 1107 _sys_write(STDERR_FILENO, buf, (size_t)len); 1108 (void)_lwp_kill(_lwp_self(), SIGABRT); 1109 _exit(1); 1110 } 1111 1112 1113 void 1114 pthread__errorfunc(const char *file, int line, const char *function, 1115 const char *msg) 1116 { 1117 char buf[1024]; 1118 size_t len; 1119 1120 if (pthread__diagassert == 0) 1121 return; 1122 1123 /* 1124 * snprintf should not acquire any locks, or we could 1125 * end up deadlocked if the assert caller held locks. 1126 */ 1127 len = snprintf(buf, 1024, 1128 "%s: Error detected by libpthread: %s.\n" 1129 "Detected by file \"%s\", line %d%s%s%s.\n" 1130 "See pthread(3) for information.\n", 1131 getprogname(), msg, file, line, 1132 function ? ", function \"" : "", 1133 function ? function : "", 1134 function ? "\"" : ""); 1135 1136 if (pthread__diagassert & DIAGASSERT_STDERR) 1137 _sys_write(STDERR_FILENO, buf, len); 1138 1139 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1140 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1141 1142 if (pthread__diagassert & DIAGASSERT_ABORT) { 1143 (void)_lwp_kill(_lwp_self(), SIGABRT); 1144 _exit(1); 1145 } 1146 } 1147 1148 /* 1149 * Thread park/unpark operations. The kernel operations are 1150 * modelled after a brief description from "Multithreading in 1151 * the Solaris Operating Environment": 1152 * 1153 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1154 */ 1155 1156 #define OOPS(msg) \ 1157 pthread__errorfunc(__FILE__, __LINE__, __func__, msg) 1158 1159 int 1160 pthread__park(pthread_t self, pthread_mutex_t *lock, 1161 pthread_queue_t *queue, const struct timespec *abstime, 1162 int cancelpt, const void *hint) 1163 { 1164 int rv, error; 1165 void *obj; 1166 1167 self->pt_willpark = 1; 1168 pthread_mutex_unlock(lock); 1169 self->pt_willpark = 0; 1170 1171 /* 1172 * Wait until we are awoken by a pending unpark operation, 1173 * a signal, an unpark posted after we have gone asleep, 1174 * or an expired timeout. 1175 * 1176 * It is fine to test the value of pt_sleepobj without 1177 * holding any locks, because: 1178 * 1179 * o Only the blocking thread (this thread) ever sets them 1180 * to a non-NULL value. 1181 * 1182 * o Other threads may set them NULL, but if they do so they 1183 * must also make this thread return from _lwp_park. 1184 * 1185 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1186 * calls and all make use of spinlocks in the kernel. So 1187 * these system calls act as full memory barriers, and will 1188 * ensure that the calling CPU's store buffers are drained. 1189 * In combination with the spinlock release before unpark, 1190 * this means that modification of pt_sleepobj/onq by another 1191 * thread will become globally visible before that thread 1192 * schedules an unpark operation on this thread. 1193 * 1194 * Note: the test in the while() statement dodges the park op if 1195 * we have already been awoken, unless there is another thread to 1196 * awaken. This saves a syscall - if we were already awakened, 1197 * the next call to _lwp_park() would need to return early in order 1198 * to eat the previous wakeup. 1199 */ 1200 rv = 0; 1201 do { 1202 /* 1203 * If we deferred unparking a thread, arrange to 1204 * have _lwp_park() restart it before blocking. 1205 */ 1206 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME, 1207 __UNCONST(abstime), self->pt_unpark, hint, hint); 1208 self->pt_unpark = 0; 1209 if (error != 0) { 1210 switch (rv = errno) { 1211 case EINTR: 1212 case EALREADY: 1213 rv = 0; 1214 break; 1215 case ETIMEDOUT: 1216 break; 1217 default: 1218 OOPS("_lwp_park failed"); 1219 break; 1220 } 1221 } 1222 /* Check for cancellation. */ 1223 if (cancelpt && self->pt_cancel) 1224 rv = EINTR; 1225 } while (self->pt_sleepobj != NULL && rv == 0); 1226 1227 /* 1228 * If we have been awoken early but are still on the queue, 1229 * then remove ourself. Again, it's safe to do the test 1230 * without holding any locks. 1231 */ 1232 if (__predict_false(self->pt_sleepobj != NULL)) { 1233 pthread_mutex_lock(lock); 1234 if ((obj = self->pt_sleepobj) != NULL) { 1235 PTQ_REMOVE(queue, self, pt_sleep); 1236 self->pt_sleepobj = NULL; 1237 if (obj != NULL && self->pt_early != NULL) 1238 (*self->pt_early)(obj); 1239 } 1240 pthread_mutex_unlock(lock); 1241 } 1242 self->pt_early = NULL; 1243 1244 return rv; 1245 } 1246 1247 void 1248 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1249 pthread_mutex_t *interlock) 1250 { 1251 pthread_t target; 1252 u_int max; 1253 size_t nwaiters; 1254 1255 max = pthread__unpark_max; 1256 nwaiters = self->pt_nwaiters; 1257 target = PTQ_FIRST(queue); 1258 if (nwaiters == max) { 1259 /* Overflow. */ 1260 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1261 __UNVOLATILE(&interlock->ptm_waiters)); 1262 nwaiters = 0; 1263 } 1264 target->pt_sleepobj = NULL; 1265 self->pt_waiters[nwaiters++] = target->pt_lid; 1266 PTQ_REMOVE(queue, target, pt_sleep); 1267 self->pt_nwaiters = nwaiters; 1268 pthread__mutex_deferwake(self, interlock); 1269 } 1270 1271 void 1272 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1273 pthread_mutex_t *interlock) 1274 { 1275 pthread_t target; 1276 u_int max; 1277 size_t nwaiters; 1278 1279 max = pthread__unpark_max; 1280 nwaiters = self->pt_nwaiters; 1281 PTQ_FOREACH(target, queue, pt_sleep) { 1282 if (nwaiters == max) { 1283 /* Overflow. */ 1284 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1285 __UNVOLATILE(&interlock->ptm_waiters)); 1286 nwaiters = 0; 1287 } 1288 target->pt_sleepobj = NULL; 1289 self->pt_waiters[nwaiters++] = target->pt_lid; 1290 } 1291 self->pt_nwaiters = nwaiters; 1292 PTQ_INIT(queue); 1293 pthread__mutex_deferwake(self, interlock); 1294 } 1295 1296 #undef OOPS 1297 1298 static void 1299 pthread__initmainstack(void) 1300 { 1301 struct rlimit slimit; 1302 const AuxInfo *aux; 1303 size_t size, len; 1304 int mib[2]; 1305 unsigned int value; 1306 1307 _DIAGASSERT(_dlauxinfo() != NULL); 1308 1309 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1310 err(EXIT_FAILURE, 1311 "Couldn't get stack resource consumption limits"); 1312 size = slimit.rlim_cur; 1313 pthread__main->pt_stack.ss_size = size; 1314 pthread__main->pt_guardsize = pthread__pagesize; 1315 1316 mib[0] = CTL_VM; 1317 mib[1] = VM_GUARD_SIZE; 1318 len = sizeof(value); 1319 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 1320 pthread__main->pt_guardsize = value; 1321 1322 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1323 if (aux->a_type == AT_STACKBASE) { 1324 #ifdef __MACHINE_STACK_GROWS_UP 1325 pthread__main->pt_stack.ss_sp = (void *)aux->a_v; 1326 #else 1327 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size; 1328 #endif 1329 break; 1330 } 1331 } 1332 pthread__copy_tsd(pthread__main); 1333 } 1334 1335 /* 1336 * Set up the slightly special stack for the "initial" thread, which 1337 * runs on the normal system stack, and thus gets slightly different 1338 * treatment. 1339 */ 1340 static void 1341 pthread__initmain(pthread_t *newt) 1342 { 1343 char *value; 1344 1345 pthread__initmainstack(); 1346 1347 value = pthread__getenv("PTHREAD_STACKSIZE"); 1348 if (value != NULL) { 1349 pthread__stacksize = atoi(value) * 1024; 1350 if (pthread__stacksize > pthread__main->pt_stack.ss_size) 1351 pthread__stacksize = pthread__main->pt_stack.ss_size; 1352 } 1353 if (pthread__stacksize == 0) 1354 pthread__stacksize = pthread__main->pt_stack.ss_size; 1355 pthread__stacksize += pthread__pagesize - 1; 1356 pthread__stacksize &= ~(pthread__pagesize - 1); 1357 if (pthread__stacksize < 4 * pthread__pagesize) 1358 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1359 4 * pthread__pagesize / 1024); 1360 1361 *newt = pthread__main; 1362 #if defined(_PTHREAD_GETTCB_EXT) 1363 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT(); 1364 #elif defined(__HAVE___LWP_GETTCB_FAST) 1365 pthread__main->pt_tls = __lwp_gettcb_fast(); 1366 #else 1367 pthread__main->pt_tls = _lwp_getprivate(); 1368 #endif 1369 pthread__main->pt_tls->tcb_pthread = pthread__main; 1370 } 1371 1372 static signed int 1373 /*ARGSUSED*/ 1374 pthread__cmp(void *ctx, const void *n1, const void *n2) 1375 { 1376 const uintptr_t p1 = (const uintptr_t)n1; 1377 const uintptr_t p2 = (const uintptr_t)n2; 1378 1379 if (p1 < p2) 1380 return -1; 1381 if (p1 > p2) 1382 return 1; 1383 return 0; 1384 } 1385 1386 /* Because getenv() wants to use locks. */ 1387 char * 1388 pthread__getenv(const char *name) 1389 { 1390 extern char **environ; 1391 size_t l_name, offset; 1392 1393 if (issetugid()) 1394 return (NULL); 1395 1396 l_name = strlen(name); 1397 for (offset = 0; environ[offset] != NULL; offset++) { 1398 if (strncmp(name, environ[offset], l_name) == 0 && 1399 environ[offset][l_name] == '=') { 1400 return environ[offset] + l_name + 1; 1401 } 1402 } 1403 1404 return NULL; 1405 } 1406 1407 pthread_mutex_t * 1408 pthread__hashlock(volatile const void *p) 1409 { 1410 uintptr_t v; 1411 1412 v = (uintptr_t)p; 1413 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1414 } 1415 1416 int 1417 pthread__checkpri(int pri) 1418 { 1419 static int havepri; 1420 static long min, max; 1421 1422 if (!havepri) { 1423 min = sysconf(_SC_SCHED_PRI_MIN); 1424 max = sysconf(_SC_SCHED_PRI_MAX); 1425 havepri = 1; 1426 } 1427 return (pri < min || pri > max) ? EINVAL : 0; 1428 } 1429