1 /* $NetBSD: pthread.c,v 1.152 2018/08/19 02:10:42 kamil Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nathan J. Williams and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __RCSID("$NetBSD: pthread.c,v 1.152 2018/08/19 02:10:42 kamil Exp $"); 34 35 #define __EXPOSE_STACK 1 36 37 #include <sys/param.h> 38 #include <sys/exec_elf.h> 39 #include <sys/mman.h> 40 #include <sys/lwp.h> 41 #include <sys/lwpctl.h> 42 #include <sys/resource.h> 43 #include <sys/sysctl.h> 44 #include <sys/tls.h> 45 #include <uvm/uvm_param.h> 46 47 #include <assert.h> 48 #include <dlfcn.h> 49 #include <err.h> 50 #include <errno.h> 51 #include <lwp.h> 52 #include <signal.h> 53 #include <stdio.h> 54 #include <stdlib.h> 55 #include <stddef.h> 56 #include <string.h> 57 #include <syslog.h> 58 #include <ucontext.h> 59 #include <unistd.h> 60 #include <sched.h> 61 62 #include "atexit.h" 63 #include "pthread.h" 64 #include "pthread_int.h" 65 #include "pthread_makelwp.h" 66 #include "reentrant.h" 67 68 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 69 static rb_tree_t pthread__alltree; 70 71 static signed int pthread__cmp(void *, const void *, const void *); 72 73 static const rb_tree_ops_t pthread__alltree_ops = { 74 .rbto_compare_nodes = pthread__cmp, 75 .rbto_compare_key = pthread__cmp, 76 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree), 77 .rbto_context = NULL 78 }; 79 80 static void pthread__create_tramp(void *); 81 static void pthread__initthread(pthread_t); 82 static void pthread__scrubthread(pthread_t, char *, int); 83 static void pthread__initmain(pthread_t *); 84 static void pthread__fork_callback(void); 85 static void pthread__reap(pthread_t); 86 static void pthread__child_callback(void); 87 static void pthread__start(void); 88 89 void pthread__init(void); 90 91 int pthread__started; 92 int __uselibcstub = 1; 93 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 94 pthread_queue_t pthread__deadqueue; 95 pthread_queue_t pthread__allqueue; 96 97 static pthread_attr_t pthread_default_attr; 98 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 99 100 enum { 101 DIAGASSERT_ABORT = 1<<0, 102 DIAGASSERT_STDERR = 1<<1, 103 DIAGASSERT_SYSLOG = 1<<2 104 }; 105 106 static int pthread__diagassert; 107 108 int pthread__concurrency; 109 int pthread__nspins; 110 int pthread__unpark_max = PTHREAD__UNPARK_MAX; 111 int pthread__dbg; /* set by libpthread_dbg if active */ 112 113 /* 114 * We have to initialize the pthread_stack* variables here because 115 * mutexes are used before pthread_init() and thus pthread__initmain() 116 * are called. Since mutexes only save the stack pointer and not a 117 * pointer to the thread data, it is safe to change the mapping from 118 * stack pointer to thread data afterwards. 119 */ 120 size_t pthread__stacksize; 121 size_t pthread__guardsize; 122 size_t pthread__pagesize; 123 static struct __pthread_st *pthread__main; 124 static size_t __pthread_st_size; 125 126 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 127 128 __strong_alias(__libc_thr_self,pthread_self) 129 __strong_alias(__libc_thr_create,pthread_create) 130 __strong_alias(__libc_thr_exit,pthread_exit) 131 __strong_alias(__libc_thr_errno,pthread__errno) 132 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 133 __strong_alias(__libc_thr_equal,pthread_equal) 134 __strong_alias(__libc_thr_init,pthread__init) 135 136 /* 137 * Static library kludge. Place a reference to a symbol any library 138 * file which does not already have a reference here. 139 */ 140 extern int pthread__cancel_stub_binder; 141 142 void *pthread__static_lib_binder[] = { 143 &pthread__cancel_stub_binder, 144 pthread_cond_init, 145 pthread_mutex_init, 146 pthread_rwlock_init, 147 pthread_barrier_init, 148 pthread_key_create, 149 pthread_setspecific, 150 }; 151 152 #define NHASHLOCK 64 153 154 static union hashlock { 155 pthread_mutex_t mutex; 156 char pad[64]; 157 } hashlocks[NHASHLOCK] __aligned(64); 158 159 /* 160 * This needs to be started by the library loading code, before main() 161 * gets to run, for various things that use the state of the initial thread 162 * to work properly (thread-specific data is an application-visible example; 163 * spinlock counts for mutexes is an internal example). 164 */ 165 void 166 pthread__init(void) 167 { 168 pthread_t first; 169 char *p; 170 int i; 171 int mib[2]; 172 unsigned int value; 173 size_t len; 174 extern int __isthreaded; 175 176 /* 177 * Allocate pthread_keys descriptors before 178 * reseting __uselibcstub because otherwise 179 * malloc() will call pthread_keys_create() 180 * while pthread_keys descriptors are not 181 * yet allocated. 182 */ 183 pthread__main = pthread_tsd_init(&__pthread_st_size); 184 if (pthread__main == NULL) 185 err(EXIT_FAILURE, "Cannot allocate pthread storage"); 186 187 __uselibcstub = 0; 188 189 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 190 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF); 191 192 mib[0] = CTL_VM; 193 mib[1] = VM_THREAD_GUARD_SIZE; 194 len = sizeof(value); 195 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 196 pthread__guardsize = value; 197 else 198 pthread__guardsize = pthread__pagesize; 199 200 /* Initialize locks first; they're needed elsewhere. */ 201 pthread__lockprim_init(); 202 for (i = 0; i < NHASHLOCK; i++) { 203 pthread_mutex_init(&hashlocks[i].mutex, NULL); 204 } 205 206 /* Fetch parameters. */ 207 i = (int)_lwp_unpark_all(NULL, 0, NULL); 208 if (i == -1) 209 err(EXIT_FAILURE, "_lwp_unpark_all"); 210 if (i < pthread__unpark_max) 211 pthread__unpark_max = i; 212 213 /* Basic data structure setup */ 214 pthread_attr_init(&pthread_default_attr); 215 PTQ_INIT(&pthread__allqueue); 216 PTQ_INIT(&pthread__deadqueue); 217 218 rb_tree_init(&pthread__alltree, &pthread__alltree_ops); 219 220 /* Create the thread structure corresponding to main() */ 221 pthread__initmain(&first); 222 pthread__initthread(first); 223 pthread__scrubthread(first, NULL, 0); 224 225 first->pt_lid = _lwp_self(); 226 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 227 (void)rb_tree_insert_node(&pthread__alltree, first); 228 229 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 230 err(EXIT_FAILURE, "_lwp_ctl"); 231 } 232 233 /* Start subsystems */ 234 PTHREAD_MD_INIT 235 236 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 237 switch (*p) { 238 case 'a': 239 pthread__diagassert |= DIAGASSERT_ABORT; 240 break; 241 case 'A': 242 pthread__diagassert &= ~DIAGASSERT_ABORT; 243 break; 244 case 'e': 245 pthread__diagassert |= DIAGASSERT_STDERR; 246 break; 247 case 'E': 248 pthread__diagassert &= ~DIAGASSERT_STDERR; 249 break; 250 case 'l': 251 pthread__diagassert |= DIAGASSERT_SYSLOG; 252 break; 253 case 'L': 254 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 255 break; 256 } 257 } 258 259 /* Tell libc that we're here and it should role-play accordingly. */ 260 pthread_atfork(NULL, NULL, pthread__fork_callback); 261 __isthreaded = 1; 262 } 263 264 static void 265 pthread__fork_callback(void) 266 { 267 struct __pthread_st *self = pthread__self(); 268 269 /* lwpctl state is not copied across fork. */ 270 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 271 err(EXIT_FAILURE, "_lwp_ctl"); 272 } 273 self->pt_lid = _lwp_self(); 274 } 275 276 static void 277 pthread__child_callback(void) 278 { 279 280 /* 281 * Clean up data structures that a forked child process might 282 * trip over. Note that if threads have been created (causing 283 * this handler to be registered) the standards say that the 284 * child will trigger undefined behavior if it makes any 285 * pthread_* calls (or any other calls that aren't 286 * async-signal-safe), so we don't really have to clean up 287 * much. Anything that permits some pthread_* calls to work is 288 * merely being polite. 289 */ 290 pthread__started = 0; 291 } 292 293 static void 294 pthread__start(void) 295 { 296 297 /* 298 * Per-process timers are cleared by fork(); despite the 299 * various restrictions on fork() and threads, it's legal to 300 * fork() before creating any threads. 301 */ 302 pthread_atfork(NULL, NULL, pthread__child_callback); 303 } 304 305 306 /* General-purpose thread data structure sanitization. */ 307 /* ARGSUSED */ 308 static void 309 pthread__initthread(pthread_t t) 310 { 311 312 t->pt_self = t; 313 t->pt_magic = PT_MAGIC; 314 t->pt_willpark = 0; 315 t->pt_unpark = 0; 316 t->pt_nwaiters = 0; 317 t->pt_sleepobj = NULL; 318 t->pt_signalled = 0; 319 t->pt_havespecific = 0; 320 t->pt_early = NULL; 321 t->pt_lwpctl = &pthread__dummy_lwpctl; 322 t->pt_blocking = 0; 323 t->pt_droplock = NULL; 324 325 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 326 pthread_mutex_init(&t->pt_lock, NULL); 327 PTQ_INIT(&t->pt_cleanup_stack); 328 pthread_cond_init(&t->pt_joiners, NULL); 329 } 330 331 static void 332 pthread__scrubthread(pthread_t t, char *name, int flags) 333 { 334 335 t->pt_state = PT_STATE_RUNNING; 336 t->pt_exitval = NULL; 337 t->pt_flags = flags; 338 t->pt_cancel = 0; 339 t->pt_errno = 0; 340 t->pt_name = name; 341 t->pt_lid = 0; 342 } 343 344 static int 345 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 346 { 347 void *stackbase, *stackbase2, *redzone; 348 size_t stacksize, guardsize; 349 bool allocated; 350 351 if (attr != NULL) { 352 pthread_attr_getstack(attr, &stackbase, &stacksize); 353 pthread_attr_getguardsize(attr, &guardsize); 354 } else { 355 stackbase = NULL; 356 stacksize = 0; 357 guardsize = pthread__guardsize; 358 } 359 if (stacksize == 0) 360 stacksize = pthread__stacksize; 361 362 if (newthread->pt_stack_allocated) { 363 if (stackbase == NULL && 364 newthread->pt_stack.ss_size == stacksize && 365 newthread->pt_guardsize == guardsize) 366 return 0; 367 stackbase2 = newthread->pt_stack.ss_sp; 368 #ifndef __MACHINE_STACK_GROWS_UP 369 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 370 #endif 371 munmap(stackbase2, 372 newthread->pt_stack.ss_size + newthread->pt_guardsize); 373 newthread->pt_stack.ss_sp = NULL; 374 newthread->pt_stack.ss_size = 0; 375 newthread->pt_guardsize = 0; 376 newthread->pt_stack_allocated = false; 377 } 378 379 newthread->pt_stack_allocated = false; 380 381 if (stackbase == NULL) { 382 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 383 guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1; 384 stackbase = mmap(NULL, stacksize + guardsize, 385 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 386 if (stackbase == MAP_FAILED) 387 return ENOMEM; 388 allocated = true; 389 } else { 390 allocated = false; 391 } 392 #ifdef __MACHINE_STACK_GROWS_UP 393 redzone = (char *)stackbase + stacksize; 394 stackbase2 = (char *)stackbase; 395 #else 396 redzone = (char *)stackbase; 397 stackbase2 = (char *)stackbase + guardsize; 398 #endif 399 if (allocated && guardsize && 400 mprotect(redzone, guardsize, PROT_NONE) == -1) { 401 munmap(stackbase, stacksize + guardsize); 402 return EPERM; 403 } 404 newthread->pt_stack.ss_size = stacksize; 405 newthread->pt_stack.ss_sp = stackbase2; 406 newthread->pt_guardsize = guardsize; 407 newthread->pt_stack_allocated = allocated; 408 return 0; 409 } 410 411 int 412 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 413 void *(*startfunc)(void *), void *arg) 414 { 415 pthread_t newthread; 416 pthread_attr_t nattr; 417 struct pthread_attr_private *p; 418 char * volatile name; 419 unsigned long flag; 420 void *private_area; 421 int ret; 422 423 if (__predict_false(__uselibcstub)) { 424 pthread__errorfunc(__FILE__, __LINE__, __func__, 425 "pthread_create() requires linking with -lpthread"); 426 return __libc_thr_create_stub(thread, attr, startfunc, arg); 427 } 428 429 /* 430 * It's okay to check this without a lock because there can 431 * only be one thread before it becomes true. 432 */ 433 if (pthread__started == 0) { 434 pthread__start(); 435 pthread__started = 1; 436 } 437 438 if (attr == NULL) 439 nattr = pthread_default_attr; 440 else if (attr->pta_magic == PT_ATTR_MAGIC) 441 nattr = *attr; 442 else 443 return EINVAL; 444 445 /* Fetch misc. attributes from the attr structure. */ 446 name = NULL; 447 if ((p = nattr.pta_private) != NULL) 448 if (p->ptap_name[0] != '\0') 449 if ((name = strdup(p->ptap_name)) == NULL) 450 return ENOMEM; 451 452 newthread = NULL; 453 454 /* 455 * Try to reclaim a dead thread. 456 */ 457 if (!PTQ_EMPTY(&pthread__deadqueue)) { 458 pthread_mutex_lock(&pthread__deadqueue_lock); 459 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 460 /* Still running? */ 461 if (newthread->pt_lwpctl->lc_curcpu == 462 LWPCTL_CPU_EXITED || 463 (_lwp_kill(newthread->pt_lid, 0) == -1 && 464 errno == ESRCH)) 465 break; 466 } 467 if (newthread) 468 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 469 pthread_mutex_unlock(&pthread__deadqueue_lock); 470 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 471 if (newthread && newthread->pt_tls) { 472 _rtld_tls_free(newthread->pt_tls); 473 newthread->pt_tls = NULL; 474 } 475 #endif 476 } 477 478 /* 479 * If necessary set up a stack, allocate space for a pthread_st, 480 * and initialize it. 481 */ 482 if (newthread == NULL) { 483 newthread = calloc(1, __pthread_st_size); 484 if (newthread == NULL) { 485 free(name); 486 return ENOMEM; 487 } 488 newthread->pt_stack_allocated = false; 489 490 if (pthread__getstack(newthread, attr)) { 491 free(newthread); 492 free(name); 493 return ENOMEM; 494 } 495 496 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 497 newthread->pt_tls = NULL; 498 #endif 499 500 /* Add to list of all threads. */ 501 pthread_rwlock_wrlock(&pthread__alltree_lock); 502 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 503 (void)rb_tree_insert_node(&pthread__alltree, newthread); 504 pthread_rwlock_unlock(&pthread__alltree_lock); 505 506 /* Will be reset by the thread upon exit. */ 507 pthread__initthread(newthread); 508 } else { 509 if (pthread__getstack(newthread, attr)) { 510 pthread_mutex_lock(&pthread__deadqueue_lock); 511 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 512 pthread_mutex_unlock(&pthread__deadqueue_lock); 513 return ENOMEM; 514 } 515 } 516 517 /* 518 * Create the new LWP. 519 */ 520 pthread__scrubthread(newthread, name, nattr.pta_flags); 521 newthread->pt_func = startfunc; 522 newthread->pt_arg = arg; 523 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 524 private_area = newthread->pt_tls = _rtld_tls_allocate(); 525 newthread->pt_tls->tcb_pthread = newthread; 526 #else 527 private_area = newthread; 528 #endif 529 530 flag = LWP_DETACHED; 531 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 532 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 533 flag |= LWP_SUSPENDED; 534 535 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area, 536 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size, 537 flag, &newthread->pt_lid); 538 if (ret != 0) { 539 ret = errno; 540 pthread_mutex_lock(&newthread->pt_lock); 541 /* Will unlock and free name. */ 542 pthread__reap(newthread); 543 return ret; 544 } 545 546 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 547 if (p != NULL) { 548 (void)pthread_setschedparam(newthread, p->ptap_policy, 549 &p->ptap_sp); 550 } 551 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 552 (void)_lwp_continue(newthread->pt_lid); 553 } 554 } 555 556 *thread = newthread; 557 558 return 0; 559 } 560 561 562 __dead static void 563 pthread__create_tramp(void *cookie) 564 { 565 pthread_t self; 566 void *retval; 567 568 self = cookie; 569 570 /* 571 * Throw away some stack in a feeble attempt to reduce cache 572 * thrash. May help for SMT processors. XXX We should not 573 * be allocating stacks on fixed 2MB boundaries. Needs a 574 * thread register or decent thread local storage. 575 * 576 * Note that we may race with the kernel in _lwp_create(), 577 * and so pt_lid can be unset at this point, but we don't 578 * care. 579 */ 580 (void)alloca(((unsigned)self->pt_lid & 7) << 8); 581 582 if (self->pt_name != NULL) { 583 pthread_mutex_lock(&self->pt_lock); 584 if (self->pt_name != NULL) 585 (void)_lwp_setname(0, self->pt_name); 586 pthread_mutex_unlock(&self->pt_lock); 587 } 588 589 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 590 err(EXIT_FAILURE, "_lwp_ctl"); 591 } 592 593 retval = (*self->pt_func)(self->pt_arg); 594 595 pthread_exit(retval); 596 597 /*NOTREACHED*/ 598 pthread__abort(); 599 } 600 601 int 602 pthread_suspend_np(pthread_t thread) 603 { 604 pthread_t self; 605 606 self = pthread__self(); 607 if (self == thread) { 608 return EDEADLK; 609 } 610 if (pthread__find(thread) != 0) 611 return ESRCH; 612 if (_lwp_suspend(thread->pt_lid) == 0) 613 return 0; 614 return errno; 615 } 616 617 int 618 pthread_resume_np(pthread_t thread) 619 { 620 621 if (pthread__find(thread) != 0) 622 return ESRCH; 623 if (_lwp_continue(thread->pt_lid) == 0) 624 return 0; 625 return errno; 626 } 627 628 void 629 pthread_exit(void *retval) 630 { 631 pthread_t self; 632 struct pt_clean_t *cleanup; 633 char *name; 634 635 if (__predict_false(__uselibcstub)) { 636 __libc_thr_exit_stub(retval); 637 goto out; 638 } 639 640 self = pthread__self(); 641 642 /* Disable cancellability. */ 643 pthread_mutex_lock(&self->pt_lock); 644 self->pt_flags |= PT_FLAG_CS_DISABLED; 645 self->pt_cancel = 0; 646 647 /* Call any cancellation cleanup handlers */ 648 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 649 pthread_mutex_unlock(&self->pt_lock); 650 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 651 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 652 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 653 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 654 } 655 pthread_mutex_lock(&self->pt_lock); 656 } 657 658 pthread_mutex_unlock(&self->pt_lock); 659 __cxa_thread_run_atexit(); 660 pthread_mutex_lock(&self->pt_lock); 661 662 /* Perform cleanup of thread-specific data */ 663 pthread__destroy_tsd(self); 664 665 /* Signal our exit. */ 666 self->pt_exitval = retval; 667 if (self->pt_flags & PT_FLAG_DETACHED) { 668 self->pt_state = PT_STATE_DEAD; 669 name = self->pt_name; 670 self->pt_name = NULL; 671 pthread_mutex_unlock(&self->pt_lock); 672 if (name != NULL) 673 free(name); 674 pthread_mutex_lock(&pthread__deadqueue_lock); 675 PTQ_INSERT_TAIL(&pthread__deadqueue, self, pt_deadq); 676 pthread_mutex_unlock(&pthread__deadqueue_lock); 677 _lwp_exit(); 678 } else { 679 self->pt_state = PT_STATE_ZOMBIE; 680 pthread_cond_broadcast(&self->pt_joiners); 681 pthread_mutex_unlock(&self->pt_lock); 682 /* Note: name will be freed by the joiner. */ 683 _lwp_exit(); 684 } 685 686 out: 687 /*NOTREACHED*/ 688 pthread__abort(); 689 exit(1); 690 } 691 692 693 int 694 pthread_join(pthread_t thread, void **valptr) 695 { 696 pthread_t self; 697 int error; 698 699 self = pthread__self(); 700 701 if (pthread__find(thread) != 0) 702 return ESRCH; 703 704 if (thread->pt_magic != PT_MAGIC) 705 return EINVAL; 706 707 if (thread == self) 708 return EDEADLK; 709 710 self->pt_droplock = &thread->pt_lock; 711 pthread_mutex_lock(&thread->pt_lock); 712 for (;;) { 713 if (thread->pt_state == PT_STATE_ZOMBIE) 714 break; 715 if (thread->pt_state == PT_STATE_DEAD) { 716 pthread_mutex_unlock(&thread->pt_lock); 717 self->pt_droplock = NULL; 718 return ESRCH; 719 } 720 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 721 pthread_mutex_unlock(&thread->pt_lock); 722 self->pt_droplock = NULL; 723 return EINVAL; 724 } 725 error = pthread_cond_wait(&thread->pt_joiners, 726 &thread->pt_lock); 727 if (error != 0) { 728 pthread__errorfunc(__FILE__, __LINE__, 729 __func__, "unexpected return from cond_wait()"); 730 } 731 732 } 733 pthread__testcancel(self); 734 if (valptr != NULL) 735 *valptr = thread->pt_exitval; 736 /* pthread__reap() will drop the lock. */ 737 pthread__reap(thread); 738 self->pt_droplock = NULL; 739 740 return 0; 741 } 742 743 static void 744 pthread__reap(pthread_t thread) 745 { 746 char *name; 747 748 name = thread->pt_name; 749 thread->pt_name = NULL; 750 thread->pt_state = PT_STATE_DEAD; 751 pthread_mutex_unlock(&thread->pt_lock); 752 753 pthread_mutex_lock(&pthread__deadqueue_lock); 754 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 755 pthread_mutex_unlock(&pthread__deadqueue_lock); 756 757 if (name != NULL) 758 free(name); 759 } 760 761 int 762 pthread_equal(pthread_t t1, pthread_t t2) 763 { 764 if (__predict_false(__uselibcstub)) 765 return __libc_thr_equal_stub(t1, t2); 766 767 /* Nothing special here. */ 768 return (t1 == t2); 769 } 770 771 772 int 773 pthread_detach(pthread_t thread) 774 { 775 776 if (pthread__find(thread) != 0) 777 return ESRCH; 778 779 if (thread->pt_magic != PT_MAGIC) 780 return EINVAL; 781 782 pthread_mutex_lock(&thread->pt_lock); 783 thread->pt_flags |= PT_FLAG_DETACHED; 784 if (thread->pt_state == PT_STATE_ZOMBIE) { 785 /* pthread__reap() will drop the lock. */ 786 pthread__reap(thread); 787 } else { 788 /* 789 * Not valid for threads to be waiting in 790 * pthread_join() (there are intractable 791 * sync issues from the application 792 * perspective), but give those threads 793 * a chance anyway. 794 */ 795 pthread_cond_broadcast(&thread->pt_joiners); 796 pthread_mutex_unlock(&thread->pt_lock); 797 } 798 799 return 0; 800 } 801 802 803 int 804 pthread_getname_np(pthread_t thread, char *name, size_t len) 805 { 806 807 if (pthread__find(thread) != 0) 808 return ESRCH; 809 810 if (thread->pt_magic != PT_MAGIC) 811 return EINVAL; 812 813 pthread_mutex_lock(&thread->pt_lock); 814 if (thread->pt_name == NULL) 815 name[0] = '\0'; 816 else 817 strlcpy(name, thread->pt_name, len); 818 pthread_mutex_unlock(&thread->pt_lock); 819 820 return 0; 821 } 822 823 824 int 825 pthread_setname_np(pthread_t thread, const char *name, void *arg) 826 { 827 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 828 int namelen; 829 830 if (pthread__find(thread) != 0) 831 return ESRCH; 832 833 if (thread->pt_magic != PT_MAGIC) 834 return EINVAL; 835 836 namelen = snprintf(newname, sizeof(newname), name, arg); 837 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 838 return EINVAL; 839 840 cp = strdup(newname); 841 if (cp == NULL) 842 return ENOMEM; 843 844 pthread_mutex_lock(&thread->pt_lock); 845 oldname = thread->pt_name; 846 thread->pt_name = cp; 847 (void)_lwp_setname(thread->pt_lid, cp); 848 pthread_mutex_unlock(&thread->pt_lock); 849 850 if (oldname != NULL) 851 free(oldname); 852 853 return 0; 854 } 855 856 857 858 /* 859 * XXX There should be a way for applications to use the efficent 860 * inline version, but there are opacity/namespace issues. 861 */ 862 pthread_t 863 pthread_self(void) 864 { 865 if (__predict_false(__uselibcstub)) 866 return (pthread_t)__libc_thr_self_stub(); 867 868 return pthread__self(); 869 } 870 871 872 int 873 pthread_cancel(pthread_t thread) 874 { 875 876 if (pthread__find(thread) != 0) 877 return ESRCH; 878 pthread_mutex_lock(&thread->pt_lock); 879 thread->pt_flags |= PT_FLAG_CS_PENDING; 880 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 881 thread->pt_cancel = 1; 882 pthread_mutex_unlock(&thread->pt_lock); 883 _lwp_wakeup(thread->pt_lid); 884 } else 885 pthread_mutex_unlock(&thread->pt_lock); 886 887 return 0; 888 } 889 890 891 int 892 pthread_setcancelstate(int state, int *oldstate) 893 { 894 pthread_t self; 895 int retval; 896 897 if (__predict_false(__uselibcstub)) 898 return __libc_thr_setcancelstate_stub(state, oldstate); 899 900 self = pthread__self(); 901 retval = 0; 902 903 pthread_mutex_lock(&self->pt_lock); 904 905 if (oldstate != NULL) { 906 if (self->pt_flags & PT_FLAG_CS_DISABLED) 907 *oldstate = PTHREAD_CANCEL_DISABLE; 908 else 909 *oldstate = PTHREAD_CANCEL_ENABLE; 910 } 911 912 if (state == PTHREAD_CANCEL_DISABLE) { 913 self->pt_flags |= PT_FLAG_CS_DISABLED; 914 if (self->pt_cancel) { 915 self->pt_flags |= PT_FLAG_CS_PENDING; 916 self->pt_cancel = 0; 917 } 918 } else if (state == PTHREAD_CANCEL_ENABLE) { 919 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 920 /* 921 * If a cancellation was requested while cancellation 922 * was disabled, note that fact for future 923 * cancellation tests. 924 */ 925 if (self->pt_flags & PT_FLAG_CS_PENDING) { 926 self->pt_cancel = 1; 927 /* This is not a deferred cancellation point. */ 928 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 929 pthread_mutex_unlock(&self->pt_lock); 930 pthread__cancelled(); 931 } 932 } 933 } else 934 retval = EINVAL; 935 936 pthread_mutex_unlock(&self->pt_lock); 937 938 return retval; 939 } 940 941 942 int 943 pthread_setcanceltype(int type, int *oldtype) 944 { 945 pthread_t self; 946 int retval; 947 948 self = pthread__self(); 949 retval = 0; 950 951 pthread_mutex_lock(&self->pt_lock); 952 953 if (oldtype != NULL) { 954 if (self->pt_flags & PT_FLAG_CS_ASYNC) 955 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 956 else 957 *oldtype = PTHREAD_CANCEL_DEFERRED; 958 } 959 960 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 961 self->pt_flags |= PT_FLAG_CS_ASYNC; 962 if (self->pt_cancel) { 963 pthread_mutex_unlock(&self->pt_lock); 964 pthread__cancelled(); 965 } 966 } else if (type == PTHREAD_CANCEL_DEFERRED) 967 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 968 else 969 retval = EINVAL; 970 971 pthread_mutex_unlock(&self->pt_lock); 972 973 return retval; 974 } 975 976 977 void 978 pthread_testcancel(void) 979 { 980 pthread_t self; 981 982 self = pthread__self(); 983 if (self->pt_cancel) 984 pthread__cancelled(); 985 } 986 987 988 /* 989 * POSIX requires that certain functions return an error rather than 990 * invoking undefined behavior even when handed completely bogus 991 * pthread_t values, e.g. stack garbage. 992 */ 993 int 994 pthread__find(pthread_t id) 995 { 996 pthread_t target; 997 int error; 998 999 pthread_rwlock_rdlock(&pthread__alltree_lock); 1000 target = rb_tree_find_node(&pthread__alltree, id); 1001 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH; 1002 pthread_rwlock_unlock(&pthread__alltree_lock); 1003 1004 return error; 1005 } 1006 1007 1008 void 1009 pthread__testcancel(pthread_t self) 1010 { 1011 1012 if (self->pt_cancel) 1013 pthread__cancelled(); 1014 } 1015 1016 1017 void 1018 pthread__cancelled(void) 1019 { 1020 pthread_mutex_t *droplock; 1021 pthread_t self; 1022 1023 self = pthread__self(); 1024 droplock = self->pt_droplock; 1025 self->pt_droplock = NULL; 1026 1027 if (droplock != NULL && pthread_mutex_held_np(droplock)) 1028 pthread_mutex_unlock(droplock); 1029 1030 pthread_exit(PTHREAD_CANCELED); 1031 } 1032 1033 1034 void 1035 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 1036 { 1037 pthread_t self; 1038 struct pt_clean_t *entry; 1039 1040 self = pthread__self(); 1041 entry = store; 1042 entry->ptc_cleanup = cleanup; 1043 entry->ptc_arg = arg; 1044 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1045 } 1046 1047 1048 void 1049 pthread__cleanup_pop(int ex, void *store) 1050 { 1051 pthread_t self; 1052 struct pt_clean_t *entry; 1053 1054 self = pthread__self(); 1055 entry = store; 1056 1057 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1058 if (ex) 1059 (*entry->ptc_cleanup)(entry->ptc_arg); 1060 } 1061 1062 1063 int * 1064 pthread__errno(void) 1065 { 1066 pthread_t self; 1067 1068 if (__predict_false(__uselibcstub)) { 1069 pthread__errorfunc(__FILE__, __LINE__, __func__, 1070 "pthread__errno() requires linking with -lpthread"); 1071 return __libc_thr_errno_stub(); 1072 } 1073 1074 self = pthread__self(); 1075 1076 return &(self->pt_errno); 1077 } 1078 1079 ssize_t _sys_write(int, const void *, size_t); 1080 1081 void 1082 pthread__assertfunc(const char *file, int line, const char *function, 1083 const char *expr) 1084 { 1085 char buf[1024]; 1086 int len; 1087 1088 /* 1089 * snprintf should not acquire any locks, or we could 1090 * end up deadlocked if the assert caller held locks. 1091 */ 1092 len = snprintf(buf, 1024, 1093 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1094 expr, file, line, 1095 function ? ", function \"" : "", 1096 function ? function : "", 1097 function ? "\"" : ""); 1098 1099 _sys_write(STDERR_FILENO, buf, (size_t)len); 1100 (void)kill(getpid(), SIGABRT); 1101 1102 _exit(1); 1103 } 1104 1105 1106 void 1107 pthread__errorfunc(const char *file, int line, const char *function, 1108 const char *msg) 1109 { 1110 char buf[1024]; 1111 size_t len; 1112 1113 if (pthread__diagassert == 0) 1114 return; 1115 1116 /* 1117 * snprintf should not acquire any locks, or we could 1118 * end up deadlocked if the assert caller held locks. 1119 */ 1120 len = snprintf(buf, 1024, 1121 "%s: Error detected by libpthread: %s.\n" 1122 "Detected by file \"%s\", line %d%s%s%s.\n" 1123 "See pthread(3) for information.\n", 1124 getprogname(), msg, file, line, 1125 function ? ", function \"" : "", 1126 function ? function : "", 1127 function ? "\"" : ""); 1128 1129 if (pthread__diagassert & DIAGASSERT_STDERR) 1130 _sys_write(STDERR_FILENO, buf, len); 1131 1132 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1133 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1134 1135 if (pthread__diagassert & DIAGASSERT_ABORT) { 1136 (void)kill(getpid(), SIGABRT); 1137 _exit(1); 1138 } 1139 } 1140 1141 /* 1142 * Thread park/unpark operations. The kernel operations are 1143 * modelled after a brief description from "Multithreading in 1144 * the Solaris Operating Environment": 1145 * 1146 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1147 */ 1148 1149 #define OOPS(msg) \ 1150 pthread__errorfunc(__FILE__, __LINE__, __func__, msg) 1151 1152 int 1153 pthread__park(pthread_t self, pthread_mutex_t *lock, 1154 pthread_queue_t *queue, const struct timespec *abstime, 1155 int cancelpt, const void *hint) 1156 { 1157 int rv, error; 1158 void *obj; 1159 1160 /* 1161 * For non-interlocked release of mutexes we need a store 1162 * barrier before incrementing pt_blocking away from zero. 1163 * This is provided by pthread_mutex_unlock(). 1164 */ 1165 self->pt_willpark = 1; 1166 pthread_mutex_unlock(lock); 1167 self->pt_willpark = 0; 1168 self->pt_blocking++; 1169 1170 /* 1171 * Wait until we are awoken by a pending unpark operation, 1172 * a signal, an unpark posted after we have gone asleep, 1173 * or an expired timeout. 1174 * 1175 * It is fine to test the value of pt_sleepobj without 1176 * holding any locks, because: 1177 * 1178 * o Only the blocking thread (this thread) ever sets them 1179 * to a non-NULL value. 1180 * 1181 * o Other threads may set them NULL, but if they do so they 1182 * must also make this thread return from _lwp_park. 1183 * 1184 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1185 * calls and all make use of spinlocks in the kernel. So 1186 * these system calls act as full memory barriers, and will 1187 * ensure that the calling CPU's store buffers are drained. 1188 * In combination with the spinlock release before unpark, 1189 * this means that modification of pt_sleepobj/onq by another 1190 * thread will become globally visible before that thread 1191 * schedules an unpark operation on this thread. 1192 * 1193 * Note: the test in the while() statement dodges the park op if 1194 * we have already been awoken, unless there is another thread to 1195 * awaken. This saves a syscall - if we were already awakened, 1196 * the next call to _lwp_park() would need to return early in order 1197 * to eat the previous wakeup. 1198 */ 1199 rv = 0; 1200 do { 1201 /* 1202 * If we deferred unparking a thread, arrange to 1203 * have _lwp_park() restart it before blocking. 1204 */ 1205 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME, 1206 __UNCONST(abstime), self->pt_unpark, hint, hint); 1207 self->pt_unpark = 0; 1208 if (error != 0) { 1209 switch (rv = errno) { 1210 case EINTR: 1211 case EALREADY: 1212 rv = 0; 1213 break; 1214 case ETIMEDOUT: 1215 break; 1216 default: 1217 OOPS("_lwp_park failed"); 1218 break; 1219 } 1220 } 1221 /* Check for cancellation. */ 1222 if (cancelpt && self->pt_cancel) 1223 rv = EINTR; 1224 } while (self->pt_sleepobj != NULL && rv == 0); 1225 1226 /* 1227 * If we have been awoken early but are still on the queue, 1228 * then remove ourself. Again, it's safe to do the test 1229 * without holding any locks. 1230 */ 1231 if (__predict_false(self->pt_sleepobj != NULL)) { 1232 pthread_mutex_lock(lock); 1233 if ((obj = self->pt_sleepobj) != NULL) { 1234 PTQ_REMOVE(queue, self, pt_sleep); 1235 self->pt_sleepobj = NULL; 1236 if (obj != NULL && self->pt_early != NULL) 1237 (*self->pt_early)(obj); 1238 } 1239 pthread_mutex_unlock(lock); 1240 } 1241 self->pt_early = NULL; 1242 self->pt_blocking--; 1243 membar_sync(); 1244 1245 return rv; 1246 } 1247 1248 void 1249 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1250 pthread_mutex_t *interlock) 1251 { 1252 pthread_t target; 1253 u_int max; 1254 size_t nwaiters; 1255 1256 max = pthread__unpark_max; 1257 nwaiters = self->pt_nwaiters; 1258 target = PTQ_FIRST(queue); 1259 if (nwaiters == max) { 1260 /* Overflow. */ 1261 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1262 __UNVOLATILE(&interlock->ptm_waiters)); 1263 nwaiters = 0; 1264 } 1265 target->pt_sleepobj = NULL; 1266 self->pt_waiters[nwaiters++] = target->pt_lid; 1267 PTQ_REMOVE(queue, target, pt_sleep); 1268 self->pt_nwaiters = nwaiters; 1269 pthread__mutex_deferwake(self, interlock); 1270 } 1271 1272 void 1273 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1274 pthread_mutex_t *interlock) 1275 { 1276 pthread_t target; 1277 u_int max; 1278 size_t nwaiters; 1279 1280 max = pthread__unpark_max; 1281 nwaiters = self->pt_nwaiters; 1282 PTQ_FOREACH(target, queue, pt_sleep) { 1283 if (nwaiters == max) { 1284 /* Overflow. */ 1285 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1286 __UNVOLATILE(&interlock->ptm_waiters)); 1287 nwaiters = 0; 1288 } 1289 target->pt_sleepobj = NULL; 1290 self->pt_waiters[nwaiters++] = target->pt_lid; 1291 } 1292 self->pt_nwaiters = nwaiters; 1293 PTQ_INIT(queue); 1294 pthread__mutex_deferwake(self, interlock); 1295 } 1296 1297 #undef OOPS 1298 1299 static void 1300 pthread__initmainstack(void) 1301 { 1302 struct rlimit slimit; 1303 const AuxInfo *aux; 1304 size_t size, len; 1305 int mib[2]; 1306 unsigned int value; 1307 1308 _DIAGASSERT(_dlauxinfo() != NULL); 1309 1310 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1311 err(EXIT_FAILURE, 1312 "Couldn't get stack resource consumption limits"); 1313 size = slimit.rlim_cur; 1314 pthread__main->pt_stack.ss_size = size; 1315 pthread__main->pt_guardsize = pthread__pagesize; 1316 1317 mib[0] = CTL_VM; 1318 mib[1] = VM_GUARD_SIZE; 1319 len = sizeof(value); 1320 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 1321 pthread__main->pt_guardsize = value; 1322 1323 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1324 if (aux->a_type == AT_STACKBASE) { 1325 #ifdef __MACHINE_STACK_GROWS_UP 1326 pthread__main->pt_stack.ss_sp = (void *)aux->a_v; 1327 #else 1328 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size; 1329 #endif 1330 break; 1331 } 1332 } 1333 } 1334 1335 /* 1336 * Set up the slightly special stack for the "initial" thread, which 1337 * runs on the normal system stack, and thus gets slightly different 1338 * treatment. 1339 */ 1340 static void 1341 pthread__initmain(pthread_t *newt) 1342 { 1343 char *value; 1344 1345 pthread__initmainstack(); 1346 1347 value = pthread__getenv("PTHREAD_STACKSIZE"); 1348 if (value != NULL) { 1349 pthread__stacksize = atoi(value) * 1024; 1350 if (pthread__stacksize > pthread__main->pt_stack.ss_size) 1351 pthread__stacksize = pthread__main->pt_stack.ss_size; 1352 } 1353 if (pthread__stacksize == 0) 1354 pthread__stacksize = pthread__main->pt_stack.ss_size; 1355 pthread__stacksize += pthread__pagesize - 1; 1356 pthread__stacksize &= ~(pthread__pagesize - 1); 1357 if (pthread__stacksize < 4 * pthread__pagesize) 1358 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1359 4 * pthread__pagesize / 1024); 1360 1361 *newt = pthread__main; 1362 #if defined(_PTHREAD_GETTCB_EXT) 1363 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT(); 1364 #elif defined(__HAVE___LWP_GETTCB_FAST) 1365 pthread__main->pt_tls = __lwp_gettcb_fast(); 1366 #else 1367 pthread__main->pt_tls = _lwp_getprivate(); 1368 #endif 1369 pthread__main->pt_tls->tcb_pthread = pthread__main; 1370 } 1371 1372 static signed int 1373 /*ARGSUSED*/ 1374 pthread__cmp(void *ctx, const void *n1, const void *n2) 1375 { 1376 const uintptr_t p1 = (const uintptr_t)n1; 1377 const uintptr_t p2 = (const uintptr_t)n2; 1378 1379 if (p1 < p2) 1380 return -1; 1381 if (p1 > p2) 1382 return 1; 1383 return 0; 1384 } 1385 1386 /* Because getenv() wants to use locks. */ 1387 char * 1388 pthread__getenv(const char *name) 1389 { 1390 extern char **environ; 1391 size_t l_name, offset; 1392 1393 if (issetugid()) 1394 return (NULL); 1395 1396 l_name = strlen(name); 1397 for (offset = 0; environ[offset] != NULL; offset++) { 1398 if (strncmp(name, environ[offset], l_name) == 0 && 1399 environ[offset][l_name] == '=') { 1400 return environ[offset] + l_name + 1; 1401 } 1402 } 1403 1404 return NULL; 1405 } 1406 1407 pthread_mutex_t * 1408 pthread__hashlock(volatile const void *p) 1409 { 1410 uintptr_t v; 1411 1412 v = (uintptr_t)p; 1413 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1414 } 1415 1416 int 1417 pthread__checkpri(int pri) 1418 { 1419 static int havepri; 1420 static long min, max; 1421 1422 if (!havepri) { 1423 min = sysconf(_SC_SCHED_PRI_MIN); 1424 max = sysconf(_SC_SCHED_PRI_MAX); 1425 havepri = 1; 1426 } 1427 return (pri < min || pri > max) ? EINVAL : 0; 1428 } 1429