1 /* $NetBSD: pthread.c,v 1.168 2020/04/14 23:35:07 joerg Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Nathan J. Williams and Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __RCSID("$NetBSD: pthread.c,v 1.168 2020/04/14 23:35:07 joerg Exp $"); 35 36 #define __EXPOSE_STACK 1 37 38 #include <sys/param.h> 39 #include <sys/exec_elf.h> 40 #include <sys/mman.h> 41 #include <sys/lwp.h> 42 #include <sys/lwpctl.h> 43 #include <sys/resource.h> 44 #include <sys/sysctl.h> 45 #include <sys/tls.h> 46 #include <uvm/uvm_param.h> 47 48 #include <assert.h> 49 #include <dlfcn.h> 50 #include <err.h> 51 #include <errno.h> 52 #include <lwp.h> 53 #include <signal.h> 54 #include <stdio.h> 55 #include <stdlib.h> 56 #include <stddef.h> 57 #include <string.h> 58 #include <syslog.h> 59 #include <ucontext.h> 60 #include <unistd.h> 61 #include <sched.h> 62 63 #include "atexit.h" 64 #include "pthread.h" 65 #include "pthread_int.h" 66 #include "pthread_makelwp.h" 67 #include "reentrant.h" 68 69 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 70 static rb_tree_t pthread__alltree; 71 72 static signed int pthread__cmp(void *, const void *, const void *); 73 74 static const rb_tree_ops_t pthread__alltree_ops = { 75 .rbto_compare_nodes = pthread__cmp, 76 .rbto_compare_key = pthread__cmp, 77 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree), 78 .rbto_context = NULL 79 }; 80 81 static void pthread__create_tramp(void *); 82 static void pthread__initthread(pthread_t); 83 static void pthread__scrubthread(pthread_t, char *, int); 84 static void pthread__initmain(pthread_t *); 85 static void pthread__fork_callback(void); 86 static void pthread__reap(pthread_t); 87 88 void pthread__init(void); 89 90 int pthread__started; 91 int __uselibcstub = 1; 92 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 93 pthread_queue_t pthread__deadqueue; 94 pthread_queue_t pthread__allqueue; 95 96 static pthread_attr_t pthread_default_attr; 97 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 98 99 enum { 100 DIAGASSERT_ABORT = 1<<0, 101 DIAGASSERT_STDERR = 1<<1, 102 DIAGASSERT_SYSLOG = 1<<2 103 }; 104 105 static int pthread__diagassert; 106 107 int pthread__concurrency; 108 int pthread__nspins; 109 int pthread__unpark_max = PTHREAD__UNPARK_MAX; 110 int pthread__dbg; /* set by libpthread_dbg if active */ 111 112 /* 113 * We have to initialize the pthread_stack* variables here because 114 * mutexes are used before pthread_init() and thus pthread__initmain() 115 * are called. Since mutexes only save the stack pointer and not a 116 * pointer to the thread data, it is safe to change the mapping from 117 * stack pointer to thread data afterwards. 118 */ 119 size_t pthread__stacksize; 120 size_t pthread__guardsize; 121 size_t pthread__pagesize; 122 static struct __pthread_st *pthread__main; 123 static size_t __pthread_st_size; 124 125 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 126 127 __strong_alias(__libc_thr_self,pthread_self) 128 __strong_alias(__libc_thr_create,pthread_create) 129 __strong_alias(__libc_thr_exit,pthread_exit) 130 __strong_alias(__libc_thr_errno,pthread__errno) 131 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 132 __strong_alias(__libc_thr_equal,pthread_equal) 133 __strong_alias(__libc_thr_init,pthread__init) 134 135 /* 136 * Static library kludge. Place a reference to a symbol any library 137 * file which does not already have a reference here. 138 */ 139 extern int pthread__cancel_stub_binder; 140 141 void *pthread__static_lib_binder[] = { 142 &pthread__cancel_stub_binder, 143 pthread_cond_init, 144 pthread_mutex_init, 145 pthread_rwlock_init, 146 pthread_barrier_init, 147 pthread_key_create, 148 pthread_setspecific, 149 }; 150 151 #define NHASHLOCK 64 152 153 static union hashlock { 154 pthread_mutex_t mutex; 155 char pad[64]; 156 } hashlocks[NHASHLOCK] __aligned(64); 157 158 /* 159 * This needs to be started by the library loading code, before main() 160 * gets to run, for various things that use the state of the initial thread 161 * to work properly (thread-specific data is an application-visible example; 162 * spinlock counts for mutexes is an internal example). 163 */ 164 void 165 pthread__init(void) 166 { 167 pthread_t first; 168 char *p; 169 int i; 170 int mib[2]; 171 unsigned int value; 172 size_t len; 173 extern int __isthreaded; 174 175 /* 176 * Allocate pthread_keys descriptors before 177 * reseting __uselibcstub because otherwise 178 * malloc() will call pthread_keys_create() 179 * while pthread_keys descriptors are not 180 * yet allocated. 181 */ 182 pthread__main = pthread_tsd_init(&__pthread_st_size); 183 if (pthread__main == NULL) 184 err(EXIT_FAILURE, "Cannot allocate pthread storage"); 185 186 __uselibcstub = 0; 187 188 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 189 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF); 190 191 mib[0] = CTL_VM; 192 mib[1] = VM_THREAD_GUARD_SIZE; 193 len = sizeof(value); 194 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 195 pthread__guardsize = value; 196 else 197 pthread__guardsize = pthread__pagesize; 198 199 /* Initialize locks first; they're needed elsewhere. */ 200 pthread__lockprim_init(); 201 for (i = 0; i < NHASHLOCK; i++) { 202 pthread_mutex_init(&hashlocks[i].mutex, NULL); 203 } 204 205 /* Fetch parameters. */ 206 i = (int)_lwp_unpark_all(NULL, 0, NULL); 207 if (i == -1) 208 err(EXIT_FAILURE, "_lwp_unpark_all"); 209 if (i < pthread__unpark_max) 210 pthread__unpark_max = i; 211 212 /* Basic data structure setup */ 213 pthread_attr_init(&pthread_default_attr); 214 PTQ_INIT(&pthread__allqueue); 215 PTQ_INIT(&pthread__deadqueue); 216 217 rb_tree_init(&pthread__alltree, &pthread__alltree_ops); 218 219 /* Create the thread structure corresponding to main() */ 220 pthread__initmain(&first); 221 pthread__initthread(first); 222 pthread__scrubthread(first, NULL, 0); 223 224 first->pt_lid = _lwp_self(); 225 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 226 (void)rb_tree_insert_node(&pthread__alltree, first); 227 228 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 229 err(EXIT_FAILURE, "_lwp_ctl"); 230 } 231 232 /* Start subsystems */ 233 PTHREAD_MD_INIT 234 235 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 236 switch (*p) { 237 case 'a': 238 pthread__diagassert |= DIAGASSERT_ABORT; 239 break; 240 case 'A': 241 pthread__diagassert &= ~DIAGASSERT_ABORT; 242 break; 243 case 'e': 244 pthread__diagassert |= DIAGASSERT_STDERR; 245 break; 246 case 'E': 247 pthread__diagassert &= ~DIAGASSERT_STDERR; 248 break; 249 case 'l': 250 pthread__diagassert |= DIAGASSERT_SYSLOG; 251 break; 252 case 'L': 253 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 254 break; 255 } 256 } 257 258 /* Tell libc that we're here and it should role-play accordingly. */ 259 pthread_atfork(NULL, NULL, pthread__fork_callback); 260 __isthreaded = 1; 261 } 262 263 static void 264 pthread__fork_callback(void) 265 { 266 struct __pthread_st *self = pthread__self(); 267 268 /* lwpctl state is not copied across fork. */ 269 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 270 err(EXIT_FAILURE, "_lwp_ctl"); 271 } 272 self->pt_lid = _lwp_self(); 273 } 274 275 /* General-purpose thread data structure sanitization. */ 276 /* ARGSUSED */ 277 static void 278 pthread__initthread(pthread_t t) 279 { 280 281 t->pt_self = t; 282 t->pt_magic = PT_MAGIC; 283 t->pt_willpark = 0; 284 t->pt_unpark = 0; 285 t->pt_nwaiters = 0; 286 t->pt_sleepobj = NULL; 287 t->pt_signalled = 0; 288 t->pt_havespecific = 0; 289 t->pt_early = NULL; 290 t->pt_lwpctl = &pthread__dummy_lwpctl; 291 292 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 293 pthread_mutex_init(&t->pt_lock, NULL); 294 PTQ_INIT(&t->pt_cleanup_stack); 295 } 296 297 static void 298 pthread__scrubthread(pthread_t t, char *name, int flags) 299 { 300 301 t->pt_state = PT_STATE_RUNNING; 302 t->pt_exitval = NULL; 303 t->pt_flags = flags; 304 t->pt_cancel = 0; 305 t->pt_errno = 0; 306 t->pt_name = name; 307 t->pt_lid = 0; 308 } 309 310 static int 311 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 312 { 313 void *stackbase, *stackbase2, *redzone; 314 size_t stacksize, guardsize; 315 bool allocated; 316 317 if (attr != NULL) { 318 pthread_attr_getstack(attr, &stackbase, &stacksize); 319 pthread_attr_getguardsize(attr, &guardsize); 320 } else { 321 stackbase = NULL; 322 stacksize = 0; 323 guardsize = pthread__guardsize; 324 } 325 if (stacksize == 0) 326 stacksize = pthread__stacksize; 327 328 if (newthread->pt_stack_allocated) { 329 if (stackbase == NULL && 330 newthread->pt_stack.ss_size == stacksize && 331 newthread->pt_guardsize == guardsize) 332 return 0; 333 stackbase2 = newthread->pt_stack.ss_sp; 334 #ifndef __MACHINE_STACK_GROWS_UP 335 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 336 #endif 337 munmap(stackbase2, 338 newthread->pt_stack.ss_size + newthread->pt_guardsize); 339 newthread->pt_stack.ss_sp = NULL; 340 newthread->pt_stack.ss_size = 0; 341 newthread->pt_guardsize = 0; 342 newthread->pt_stack_allocated = false; 343 } 344 345 newthread->pt_stack_allocated = false; 346 347 if (stackbase == NULL) { 348 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 349 guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1; 350 stackbase = mmap(NULL, stacksize + guardsize, 351 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 352 if (stackbase == MAP_FAILED) 353 return ENOMEM; 354 allocated = true; 355 } else { 356 allocated = false; 357 } 358 #ifdef __MACHINE_STACK_GROWS_UP 359 redzone = (char *)stackbase + stacksize; 360 stackbase2 = (char *)stackbase; 361 #else 362 redzone = (char *)stackbase; 363 stackbase2 = (char *)stackbase + guardsize; 364 #endif 365 if (allocated && guardsize && 366 mprotect(redzone, guardsize, PROT_NONE) == -1) { 367 munmap(stackbase, stacksize + guardsize); 368 return EPERM; 369 } 370 newthread->pt_stack.ss_size = stacksize; 371 newthread->pt_stack.ss_sp = stackbase2; 372 newthread->pt_guardsize = guardsize; 373 newthread->pt_stack_allocated = allocated; 374 return 0; 375 } 376 377 int 378 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 379 void *(*startfunc)(void *), void *arg) 380 { 381 pthread_t newthread; 382 pthread_attr_t nattr; 383 struct pthread_attr_private *p; 384 char * volatile name; 385 unsigned long flag; 386 void *private_area; 387 int ret; 388 389 if (__predict_false(__uselibcstub)) { 390 pthread__errorfunc(__FILE__, __LINE__, __func__, 391 "pthread_create() requires linking with -lpthread"); 392 return __libc_thr_create_stub(thread, attr, startfunc, arg); 393 } 394 395 if (attr == NULL) 396 nattr = pthread_default_attr; 397 else if (attr->pta_magic == PT_ATTR_MAGIC) 398 nattr = *attr; 399 else 400 return EINVAL; 401 402 pthread__started = 1; 403 404 /* Fetch misc. attributes from the attr structure. */ 405 name = NULL; 406 if ((p = nattr.pta_private) != NULL) 407 if (p->ptap_name[0] != '\0') 408 if ((name = strdup(p->ptap_name)) == NULL) 409 return ENOMEM; 410 411 newthread = NULL; 412 413 /* 414 * Try to reclaim a dead thread. 415 */ 416 if (!PTQ_EMPTY(&pthread__deadqueue)) { 417 pthread_mutex_lock(&pthread__deadqueue_lock); 418 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 419 /* Still busily exiting, or finished? */ 420 if (newthread->pt_lwpctl->lc_curcpu == 421 LWPCTL_CPU_EXITED) 422 break; 423 } 424 if (newthread) 425 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 426 pthread_mutex_unlock(&pthread__deadqueue_lock); 427 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 428 if (newthread && newthread->pt_tls) { 429 _rtld_tls_free(newthread->pt_tls); 430 newthread->pt_tls = NULL; 431 } 432 #endif 433 } 434 435 /* 436 * If necessary set up a stack, allocate space for a pthread_st, 437 * and initialize it. 438 */ 439 if (newthread == NULL) { 440 newthread = calloc(1, __pthread_st_size); 441 if (newthread == NULL) { 442 free(name); 443 return ENOMEM; 444 } 445 newthread->pt_stack_allocated = false; 446 447 if (pthread__getstack(newthread, attr)) { 448 free(newthread); 449 free(name); 450 return ENOMEM; 451 } 452 453 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 454 newthread->pt_tls = NULL; 455 #endif 456 457 /* Add to list of all threads. */ 458 pthread_rwlock_wrlock(&pthread__alltree_lock); 459 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 460 (void)rb_tree_insert_node(&pthread__alltree, newthread); 461 pthread_rwlock_unlock(&pthread__alltree_lock); 462 463 /* Will be reset by the thread upon exit. */ 464 pthread__initthread(newthread); 465 } else { 466 if (pthread__getstack(newthread, attr)) { 467 pthread_mutex_lock(&pthread__deadqueue_lock); 468 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 469 pthread_mutex_unlock(&pthread__deadqueue_lock); 470 return ENOMEM; 471 } 472 } 473 474 /* 475 * Create the new LWP. 476 */ 477 pthread__scrubthread(newthread, name, nattr.pta_flags); 478 newthread->pt_func = startfunc; 479 newthread->pt_arg = arg; 480 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 481 private_area = newthread->pt_tls = _rtld_tls_allocate(); 482 newthread->pt_tls->tcb_pthread = newthread; 483 #else 484 private_area = newthread; 485 #endif 486 487 flag = 0; 488 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 489 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 490 flag |= LWP_SUSPENDED; 491 if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0) 492 flag |= LWP_DETACHED; 493 494 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area, 495 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size, 496 flag, &newthread->pt_lid); 497 if (ret != 0) { 498 ret = errno; 499 pthread_mutex_lock(&newthread->pt_lock); 500 /* Will unlock and free name. */ 501 pthread__reap(newthread); 502 return ret; 503 } 504 505 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 506 if (p != NULL) { 507 (void)pthread_setschedparam(newthread, p->ptap_policy, 508 &p->ptap_sp); 509 } 510 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 511 (void)_lwp_continue(newthread->pt_lid); 512 } 513 } 514 515 *thread = newthread; 516 517 return 0; 518 } 519 520 521 __dead static void 522 pthread__create_tramp(void *cookie) 523 { 524 pthread_t self; 525 void *retval; 526 527 self = cookie; 528 529 /* 530 * Throw away some stack in a feeble attempt to reduce cache 531 * thrash. May help for SMT processors. XXX We should not 532 * be allocating stacks on fixed 2MB boundaries. Needs a 533 * thread register or decent thread local storage. 534 */ 535 (void)alloca(((unsigned)self->pt_lid & 7) << 8); 536 537 if (self->pt_name != NULL) { 538 pthread_mutex_lock(&self->pt_lock); 539 if (self->pt_name != NULL) 540 (void)_lwp_setname(0, self->pt_name); 541 pthread_mutex_unlock(&self->pt_lock); 542 } 543 544 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 545 err(EXIT_FAILURE, "_lwp_ctl"); 546 } 547 548 retval = (*self->pt_func)(self->pt_arg); 549 550 pthread_exit(retval); 551 552 /*NOTREACHED*/ 553 pthread__abort(); 554 } 555 556 int 557 pthread_suspend_np(pthread_t thread) 558 { 559 pthread_t self; 560 561 pthread__error(EINVAL, "Invalid thread", 562 thread->pt_magic == PT_MAGIC); 563 564 self = pthread__self(); 565 if (self == thread) { 566 return EDEADLK; 567 } 568 if (pthread__find(thread) != 0) 569 return ESRCH; 570 if (_lwp_suspend(thread->pt_lid) == 0) 571 return 0; 572 return errno; 573 } 574 575 int 576 pthread_resume_np(pthread_t thread) 577 { 578 579 pthread__error(EINVAL, "Invalid thread", 580 thread->pt_magic == PT_MAGIC); 581 582 if (pthread__find(thread) != 0) 583 return ESRCH; 584 if (_lwp_continue(thread->pt_lid) == 0) 585 return 0; 586 return errno; 587 } 588 589 /* 590 * In case the thread is exiting at an inopportune time leaving waiters not 591 * awoken (because cancelled, for instance) make sure we have no waiters 592 * left. 593 */ 594 static void 595 pthread__clear_waiters(pthread_t self) 596 { 597 598 if (self->pt_nwaiters != 0) { 599 (void)_lwp_unpark_all(self->pt_waiters, self->pt_nwaiters, 600 NULL); 601 self->pt_nwaiters = 0; 602 } 603 self->pt_willpark = 0; 604 } 605 606 void 607 pthread_exit(void *retval) 608 { 609 pthread_t self; 610 struct pt_clean_t *cleanup; 611 612 if (__predict_false(__uselibcstub)) { 613 __libc_thr_exit_stub(retval); 614 goto out; 615 } 616 617 self = pthread__self(); 618 619 /* Disable cancellability. */ 620 pthread_mutex_lock(&self->pt_lock); 621 self->pt_flags |= PT_FLAG_CS_DISABLED; 622 self->pt_cancel = 0; 623 624 /* Call any cancellation cleanup handlers */ 625 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 626 pthread_mutex_unlock(&self->pt_lock); 627 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 628 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 629 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 630 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 631 } 632 pthread_mutex_lock(&self->pt_lock); 633 } 634 635 pthread_mutex_unlock(&self->pt_lock); 636 __cxa_thread_run_atexit(); 637 pthread_mutex_lock(&self->pt_lock); 638 639 /* Perform cleanup of thread-specific data */ 640 pthread__destroy_tsd(self); 641 642 /* 643 * Signal our exit. Our stack and pthread_t won't be reused until 644 * pthread_create() can see from kernel info that this LWP is gone. 645 */ 646 self->pt_exitval = retval; 647 if (self->pt_flags & PT_FLAG_DETACHED) { 648 /* pthread__reap() will drop the lock. */ 649 pthread__reap(self); 650 pthread__clear_waiters(self); 651 _lwp_exit(); 652 } else { 653 self->pt_state = PT_STATE_ZOMBIE; 654 pthread_mutex_unlock(&self->pt_lock); 655 pthread__clear_waiters(self); 656 /* Note: name will be freed by the joiner. */ 657 _lwp_exit(); 658 } 659 660 out: 661 /*NOTREACHED*/ 662 pthread__abort(); 663 exit(1); 664 } 665 666 667 int 668 pthread_join(pthread_t thread, void **valptr) 669 { 670 pthread_t self; 671 672 pthread__error(EINVAL, "Invalid thread", 673 thread->pt_magic == PT_MAGIC); 674 675 self = pthread__self(); 676 677 if (pthread__find(thread) != 0) 678 return ESRCH; 679 680 if (thread == self) 681 return EDEADLK; 682 683 /* IEEE Std 1003.1 says pthread_join() never returns EINTR. */ 684 for (;;) { 685 pthread__testcancel(self); 686 if (_lwp_wait(thread->pt_lid, NULL) == 0) 687 break; 688 if (errno != EINTR) 689 return errno; 690 } 691 692 /* 693 * Don't test for cancellation again. The spec is that if 694 * cancelled, pthread_join() must not have succeeded. 695 */ 696 pthread_mutex_lock(&thread->pt_lock); 697 if (thread->pt_state != PT_STATE_ZOMBIE) { 698 pthread__errorfunc(__FILE__, __LINE__, __func__, 699 "not a zombie"); 700 } 701 if (valptr != NULL) 702 *valptr = thread->pt_exitval; 703 704 /* pthread__reap() will drop the lock. */ 705 pthread__reap(thread); 706 return 0; 707 } 708 709 static void 710 pthread__reap(pthread_t thread) 711 { 712 char *name; 713 714 name = thread->pt_name; 715 thread->pt_name = NULL; 716 thread->pt_state = PT_STATE_DEAD; 717 pthread_mutex_unlock(&thread->pt_lock); 718 719 pthread_mutex_lock(&pthread__deadqueue_lock); 720 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 721 pthread_mutex_unlock(&pthread__deadqueue_lock); 722 723 if (name != NULL) 724 free(name); 725 } 726 727 int 728 pthread_equal(pthread_t t1, pthread_t t2) 729 { 730 731 if (__predict_false(__uselibcstub)) 732 return __libc_thr_equal_stub(t1, t2); 733 734 pthread__error(0, "Invalid thread", 735 (t1 != NULL) && (t1->pt_magic == PT_MAGIC)); 736 737 pthread__error(0, "Invalid thread", 738 (t2 != NULL) && (t2->pt_magic == PT_MAGIC)); 739 740 /* Nothing special here. */ 741 return (t1 == t2); 742 } 743 744 745 int 746 pthread_detach(pthread_t thread) 747 { 748 int error; 749 750 pthread__error(EINVAL, "Invalid thread", 751 thread->pt_magic == PT_MAGIC); 752 753 if (pthread__find(thread) != 0) 754 return ESRCH; 755 756 pthread_mutex_lock(&thread->pt_lock); 757 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 758 error = EINVAL; 759 } else { 760 error = _lwp_detach(thread->pt_lid); 761 if (error == 0) 762 thread->pt_flags |= PT_FLAG_DETACHED; 763 else 764 error = errno; 765 } 766 if (thread->pt_state == PT_STATE_ZOMBIE) { 767 /* pthread__reap() will drop the lock. */ 768 pthread__reap(thread); 769 } else 770 pthread_mutex_unlock(&thread->pt_lock); 771 return error; 772 } 773 774 775 int 776 pthread_getname_np(pthread_t thread, char *name, size_t len) 777 { 778 779 pthread__error(EINVAL, "Invalid thread", 780 thread->pt_magic == PT_MAGIC); 781 782 if (pthread__find(thread) != 0) 783 return ESRCH; 784 785 pthread_mutex_lock(&thread->pt_lock); 786 if (thread->pt_name == NULL) 787 name[0] = '\0'; 788 else 789 strlcpy(name, thread->pt_name, len); 790 pthread_mutex_unlock(&thread->pt_lock); 791 792 return 0; 793 } 794 795 796 int 797 pthread_setname_np(pthread_t thread, const char *name, void *arg) 798 { 799 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 800 int namelen; 801 802 pthread__error(EINVAL, "Invalid thread", 803 thread->pt_magic == PT_MAGIC); 804 805 if (pthread__find(thread) != 0) 806 return ESRCH; 807 808 namelen = snprintf(newname, sizeof(newname), name, arg); 809 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 810 return EINVAL; 811 812 cp = strdup(newname); 813 if (cp == NULL) 814 return ENOMEM; 815 816 pthread_mutex_lock(&thread->pt_lock); 817 oldname = thread->pt_name; 818 thread->pt_name = cp; 819 (void)_lwp_setname(thread->pt_lid, cp); 820 pthread_mutex_unlock(&thread->pt_lock); 821 822 if (oldname != NULL) 823 free(oldname); 824 825 return 0; 826 } 827 828 829 pthread_t 830 pthread_self(void) 831 { 832 if (__predict_false(__uselibcstub)) 833 return (pthread_t)__libc_thr_self_stub(); 834 835 return pthread__self(); 836 } 837 838 839 int 840 pthread_cancel(pthread_t thread) 841 { 842 843 pthread__error(EINVAL, "Invalid thread", 844 thread->pt_magic == PT_MAGIC); 845 846 if (pthread__find(thread) != 0) 847 return ESRCH; 848 pthread_mutex_lock(&thread->pt_lock); 849 thread->pt_flags |= PT_FLAG_CS_PENDING; 850 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 851 thread->pt_cancel = 1; 852 pthread_mutex_unlock(&thread->pt_lock); 853 _lwp_wakeup(thread->pt_lid); 854 } else 855 pthread_mutex_unlock(&thread->pt_lock); 856 857 return 0; 858 } 859 860 861 int 862 pthread_setcancelstate(int state, int *oldstate) 863 { 864 pthread_t self; 865 int retval; 866 867 if (__predict_false(__uselibcstub)) 868 return __libc_thr_setcancelstate_stub(state, oldstate); 869 870 self = pthread__self(); 871 retval = 0; 872 873 pthread_mutex_lock(&self->pt_lock); 874 875 if (oldstate != NULL) { 876 if (self->pt_flags & PT_FLAG_CS_DISABLED) 877 *oldstate = PTHREAD_CANCEL_DISABLE; 878 else 879 *oldstate = PTHREAD_CANCEL_ENABLE; 880 } 881 882 if (state == PTHREAD_CANCEL_DISABLE) { 883 self->pt_flags |= PT_FLAG_CS_DISABLED; 884 if (self->pt_cancel) { 885 self->pt_flags |= PT_FLAG_CS_PENDING; 886 self->pt_cancel = 0; 887 } 888 } else if (state == PTHREAD_CANCEL_ENABLE) { 889 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 890 /* 891 * If a cancellation was requested while cancellation 892 * was disabled, note that fact for future 893 * cancellation tests. 894 */ 895 if (self->pt_flags & PT_FLAG_CS_PENDING) { 896 self->pt_cancel = 1; 897 /* This is not a deferred cancellation point. */ 898 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 899 pthread_mutex_unlock(&self->pt_lock); 900 pthread__cancelled(); 901 } 902 } 903 } else 904 retval = EINVAL; 905 906 pthread_mutex_unlock(&self->pt_lock); 907 908 return retval; 909 } 910 911 912 int 913 pthread_setcanceltype(int type, int *oldtype) 914 { 915 pthread_t self; 916 int retval; 917 918 self = pthread__self(); 919 retval = 0; 920 921 pthread_mutex_lock(&self->pt_lock); 922 923 if (oldtype != NULL) { 924 if (self->pt_flags & PT_FLAG_CS_ASYNC) 925 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 926 else 927 *oldtype = PTHREAD_CANCEL_DEFERRED; 928 } 929 930 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 931 self->pt_flags |= PT_FLAG_CS_ASYNC; 932 if (self->pt_cancel) { 933 pthread_mutex_unlock(&self->pt_lock); 934 pthread__cancelled(); 935 } 936 } else if (type == PTHREAD_CANCEL_DEFERRED) 937 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 938 else 939 retval = EINVAL; 940 941 pthread_mutex_unlock(&self->pt_lock); 942 943 return retval; 944 } 945 946 947 void 948 pthread_testcancel(void) 949 { 950 pthread_t self; 951 952 self = pthread__self(); 953 if (self->pt_cancel) 954 pthread__cancelled(); 955 } 956 957 958 /* 959 * POSIX requires that certain functions return an error rather than 960 * invoking undefined behavior even when handed completely bogus 961 * pthread_t values, e.g. stack garbage. 962 */ 963 int 964 pthread__find(pthread_t id) 965 { 966 pthread_t target; 967 int error; 968 969 pthread_rwlock_rdlock(&pthread__alltree_lock); 970 target = rb_tree_find_node(&pthread__alltree, id); 971 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH; 972 pthread_rwlock_unlock(&pthread__alltree_lock); 973 974 return error; 975 } 976 977 978 void 979 pthread__testcancel(pthread_t self) 980 { 981 982 if (self->pt_cancel) 983 pthread__cancelled(); 984 } 985 986 987 void 988 pthread__cancelled(void) 989 { 990 991 pthread_exit(PTHREAD_CANCELED); 992 } 993 994 995 void 996 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 997 { 998 pthread_t self; 999 struct pt_clean_t *entry; 1000 1001 self = pthread__self(); 1002 entry = store; 1003 entry->ptc_cleanup = cleanup; 1004 entry->ptc_arg = arg; 1005 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1006 } 1007 1008 1009 void 1010 pthread__cleanup_pop(int ex, void *store) 1011 { 1012 pthread_t self; 1013 struct pt_clean_t *entry; 1014 1015 self = pthread__self(); 1016 entry = store; 1017 1018 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1019 if (ex) 1020 (*entry->ptc_cleanup)(entry->ptc_arg); 1021 } 1022 1023 1024 int * 1025 pthread__errno(void) 1026 { 1027 pthread_t self; 1028 1029 if (__predict_false(__uselibcstub)) { 1030 pthread__errorfunc(__FILE__, __LINE__, __func__, 1031 "pthread__errno() requires linking with -lpthread"); 1032 return __libc_thr_errno_stub(); 1033 } 1034 1035 self = pthread__self(); 1036 1037 return &(self->pt_errno); 1038 } 1039 1040 ssize_t _sys_write(int, const void *, size_t); 1041 1042 void 1043 pthread__assertfunc(const char *file, int line, const char *function, 1044 const char *expr) 1045 { 1046 char buf[1024]; 1047 int len; 1048 1049 /* 1050 * snprintf should not acquire any locks, or we could 1051 * end up deadlocked if the assert caller held locks. 1052 */ 1053 len = snprintf(buf, 1024, 1054 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1055 expr, file, line, 1056 function ? ", function \"" : "", 1057 function ? function : "", 1058 function ? "\"" : ""); 1059 1060 _sys_write(STDERR_FILENO, buf, (size_t)len); 1061 (void)_lwp_kill(_lwp_self(), SIGABRT); 1062 _exit(1); 1063 } 1064 1065 1066 void 1067 pthread__errorfunc(const char *file, int line, const char *function, 1068 const char *msg) 1069 { 1070 char buf[1024]; 1071 size_t len; 1072 1073 if (pthread__diagassert == 0) 1074 return; 1075 1076 /* 1077 * snprintf should not acquire any locks, or we could 1078 * end up deadlocked if the assert caller held locks. 1079 */ 1080 len = snprintf(buf, 1024, 1081 "%s: Error detected by libpthread: %s.\n" 1082 "Detected by file \"%s\", line %d%s%s%s.\n" 1083 "See pthread(3) for information.\n", 1084 getprogname(), msg, file, line, 1085 function ? ", function \"" : "", 1086 function ? function : "", 1087 function ? "\"" : ""); 1088 1089 if (pthread__diagassert & DIAGASSERT_STDERR) 1090 _sys_write(STDERR_FILENO, buf, len); 1091 1092 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1093 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1094 1095 if (pthread__diagassert & DIAGASSERT_ABORT) { 1096 (void)_lwp_kill(_lwp_self(), SIGABRT); 1097 _exit(1); 1098 } 1099 } 1100 1101 /* 1102 * Thread park/unpark operations. The kernel operations are 1103 * modelled after a brief description from "Multithreading in 1104 * the Solaris Operating Environment": 1105 * 1106 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1107 */ 1108 1109 #define OOPS(msg) \ 1110 pthread__errorfunc(__FILE__, __LINE__, __func__, msg) 1111 1112 int 1113 pthread__park(pthread_t self, pthread_mutex_t *lock, 1114 pthread_queue_t *queue, const struct timespec *abstime, 1115 int cancelpt, const void *hint) 1116 { 1117 int rv, error; 1118 void *obj; 1119 1120 self->pt_willpark = 1; 1121 pthread_mutex_unlock(lock); 1122 self->pt_willpark = 0; 1123 1124 /* 1125 * Wait until we are awoken by a pending unpark operation, 1126 * a signal, an unpark posted after we have gone asleep, 1127 * or an expired timeout. 1128 * 1129 * It is fine to test the value of pt_sleepobj without 1130 * holding any locks, because: 1131 * 1132 * o Only the blocking thread (this thread) ever sets them 1133 * to a non-NULL value. 1134 * 1135 * o Other threads may set them NULL, but if they do so they 1136 * must also make this thread return from _lwp_park. 1137 * 1138 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1139 * calls and all make use of spinlocks in the kernel. So 1140 * these system calls act as full memory barriers, and will 1141 * ensure that the calling CPU's store buffers are drained. 1142 * In combination with the spinlock release before unpark, 1143 * this means that modification of pt_sleepobj/onq by another 1144 * thread will become globally visible before that thread 1145 * schedules an unpark operation on this thread. 1146 * 1147 * Note: the test in the while() statement dodges the park op if 1148 * we have already been awoken, unless there is another thread to 1149 * awaken. This saves a syscall - if we were already awakened, 1150 * the next call to _lwp_park() would need to return early in order 1151 * to eat the previous wakeup. 1152 */ 1153 rv = 0; 1154 do { 1155 /* 1156 * If we deferred unparking a thread, arrange to 1157 * have _lwp_park() restart it before blocking. 1158 */ 1159 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME, 1160 __UNCONST(abstime), self->pt_unpark, hint, hint); 1161 self->pt_unpark = 0; 1162 if (error != 0) { 1163 switch (rv = errno) { 1164 case EINTR: 1165 case EALREADY: 1166 rv = 0; 1167 break; 1168 case ETIMEDOUT: 1169 break; 1170 default: 1171 OOPS("_lwp_park failed"); 1172 break; 1173 } 1174 } 1175 /* Check for cancellation. */ 1176 if (cancelpt && self->pt_cancel) 1177 rv = EINTR; 1178 } while (self->pt_sleepobj != NULL && rv == 0); 1179 1180 /* 1181 * If we have been awoken early but are still on the queue, 1182 * then remove ourself. Again, it's safe to do the test 1183 * without holding any locks. 1184 */ 1185 if (__predict_false(self->pt_sleepobj != NULL)) { 1186 pthread_mutex_lock(lock); 1187 if ((obj = self->pt_sleepobj) != NULL) { 1188 PTQ_REMOVE(queue, self, pt_sleep); 1189 self->pt_sleepobj = NULL; 1190 if (obj != NULL && self->pt_early != NULL) 1191 (*self->pt_early)(obj); 1192 } 1193 pthread_mutex_unlock(lock); 1194 } 1195 self->pt_early = NULL; 1196 1197 return rv; 1198 } 1199 1200 void 1201 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1202 pthread_mutex_t *interlock) 1203 { 1204 pthread_t target; 1205 u_int max; 1206 size_t nwaiters; 1207 1208 max = pthread__unpark_max; 1209 nwaiters = self->pt_nwaiters; 1210 target = PTQ_FIRST(queue); 1211 if (nwaiters == max) { 1212 /* Overflow. */ 1213 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1214 __UNVOLATILE(&interlock->ptm_waiters)); 1215 nwaiters = 0; 1216 } 1217 target->pt_sleepobj = NULL; 1218 self->pt_waiters[nwaiters++] = target->pt_lid; 1219 PTQ_REMOVE(queue, target, pt_sleep); 1220 self->pt_nwaiters = nwaiters; 1221 pthread__mutex_deferwake(self, interlock); 1222 } 1223 1224 void 1225 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1226 pthread_mutex_t *interlock) 1227 { 1228 pthread_t target; 1229 u_int max; 1230 size_t nwaiters; 1231 1232 max = pthread__unpark_max; 1233 nwaiters = self->pt_nwaiters; 1234 PTQ_FOREACH(target, queue, pt_sleep) { 1235 if (nwaiters == max) { 1236 /* Overflow. */ 1237 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1238 __UNVOLATILE(&interlock->ptm_waiters)); 1239 nwaiters = 0; 1240 } 1241 target->pt_sleepobj = NULL; 1242 self->pt_waiters[nwaiters++] = target->pt_lid; 1243 } 1244 self->pt_nwaiters = nwaiters; 1245 PTQ_INIT(queue); 1246 pthread__mutex_deferwake(self, interlock); 1247 } 1248 1249 #undef OOPS 1250 1251 static void 1252 pthread__initmainstack(void) 1253 { 1254 struct rlimit slimit; 1255 const AuxInfo *aux; 1256 size_t size, len; 1257 int mib[2]; 1258 unsigned int value; 1259 1260 _DIAGASSERT(_dlauxinfo() != NULL); 1261 1262 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1263 err(EXIT_FAILURE, 1264 "Couldn't get stack resource consumption limits"); 1265 size = slimit.rlim_cur; 1266 pthread__main->pt_stack.ss_size = size; 1267 pthread__main->pt_guardsize = pthread__pagesize; 1268 1269 mib[0] = CTL_VM; 1270 mib[1] = VM_GUARD_SIZE; 1271 len = sizeof(value); 1272 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 1273 pthread__main->pt_guardsize = value; 1274 1275 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1276 if (aux->a_type == AT_STACKBASE) { 1277 #ifdef __MACHINE_STACK_GROWS_UP 1278 pthread__main->pt_stack.ss_sp = (void *)aux->a_v; 1279 #else 1280 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size; 1281 #endif 1282 break; 1283 } 1284 } 1285 pthread__copy_tsd(pthread__main); 1286 } 1287 1288 /* 1289 * Set up the slightly special stack for the "initial" thread, which 1290 * runs on the normal system stack, and thus gets slightly different 1291 * treatment. 1292 */ 1293 static void 1294 pthread__initmain(pthread_t *newt) 1295 { 1296 char *value; 1297 1298 pthread__initmainstack(); 1299 1300 value = pthread__getenv("PTHREAD_STACKSIZE"); 1301 if (value != NULL) { 1302 pthread__stacksize = atoi(value) * 1024; 1303 if (pthread__stacksize > pthread__main->pt_stack.ss_size) 1304 pthread__stacksize = pthread__main->pt_stack.ss_size; 1305 } 1306 if (pthread__stacksize == 0) 1307 pthread__stacksize = pthread__main->pt_stack.ss_size; 1308 pthread__stacksize += pthread__pagesize - 1; 1309 pthread__stacksize &= ~(pthread__pagesize - 1); 1310 if (pthread__stacksize < 4 * pthread__pagesize) 1311 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1312 4 * pthread__pagesize / 1024); 1313 1314 *newt = pthread__main; 1315 #if defined(_PTHREAD_GETTCB_EXT) 1316 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT(); 1317 #elif defined(__HAVE___LWP_GETTCB_FAST) 1318 pthread__main->pt_tls = __lwp_gettcb_fast(); 1319 #else 1320 pthread__main->pt_tls = _lwp_getprivate(); 1321 #endif 1322 pthread__main->pt_tls->tcb_pthread = pthread__main; 1323 } 1324 1325 static signed int 1326 /*ARGSUSED*/ 1327 pthread__cmp(void *ctx, const void *n1, const void *n2) 1328 { 1329 const uintptr_t p1 = (const uintptr_t)n1; 1330 const uintptr_t p2 = (const uintptr_t)n2; 1331 1332 if (p1 < p2) 1333 return -1; 1334 if (p1 > p2) 1335 return 1; 1336 return 0; 1337 } 1338 1339 /* Because getenv() wants to use locks. */ 1340 char * 1341 pthread__getenv(const char *name) 1342 { 1343 extern char **environ; 1344 size_t l_name, offset; 1345 1346 if (issetugid()) 1347 return (NULL); 1348 1349 l_name = strlen(name); 1350 for (offset = 0; environ[offset] != NULL; offset++) { 1351 if (strncmp(name, environ[offset], l_name) == 0 && 1352 environ[offset][l_name] == '=') { 1353 return environ[offset] + l_name + 1; 1354 } 1355 } 1356 1357 return NULL; 1358 } 1359 1360 pthread_mutex_t * 1361 pthread__hashlock(volatile const void *p) 1362 { 1363 uintptr_t v; 1364 1365 v = (uintptr_t)p; 1366 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1367 } 1368 1369 int 1370 pthread__checkpri(int pri) 1371 { 1372 static int havepri; 1373 static long min, max; 1374 1375 if (!havepri) { 1376 min = sysconf(_SC_SCHED_PRI_MIN); 1377 max = sysconf(_SC_SCHED_PRI_MAX); 1378 havepri = 1; 1379 } 1380 return (pri < min || pri > max) ? EINVAL : 0; 1381 } 1382