1 /* $NetBSD: pthread.c,v 1.147 2015/05/29 16:05:13 christos Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nathan J. Williams and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __RCSID("$NetBSD: pthread.c,v 1.147 2015/05/29 16:05:13 christos Exp $"); 34 35 #define __EXPOSE_STACK 1 36 37 #include <sys/param.h> 38 #include <sys/exec_elf.h> 39 #include <sys/mman.h> 40 #include <sys/lwp.h> 41 #include <sys/lwpctl.h> 42 #include <sys/resource.h> 43 #include <sys/tls.h> 44 45 #include <assert.h> 46 #include <dlfcn.h> 47 #include <err.h> 48 #include <errno.h> 49 #include <lwp.h> 50 #include <signal.h> 51 #include <stdio.h> 52 #include <stdlib.h> 53 #include <stddef.h> 54 #include <string.h> 55 #include <syslog.h> 56 #include <ucontext.h> 57 #include <unistd.h> 58 #include <sched.h> 59 60 #include "pthread.h" 61 #include "pthread_int.h" 62 #include "pthread_makelwp.h" 63 #include "reentrant.h" 64 65 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 66 static rb_tree_t pthread__alltree; 67 68 static signed int pthread__cmp(void *, const void *, const void *); 69 70 static const rb_tree_ops_t pthread__alltree_ops = { 71 .rbto_compare_nodes = pthread__cmp, 72 .rbto_compare_key = pthread__cmp, 73 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree), 74 .rbto_context = NULL 75 }; 76 77 static void pthread__create_tramp(void *); 78 static void pthread__initthread(pthread_t); 79 static void pthread__scrubthread(pthread_t, char *, int); 80 static void pthread__initmain(pthread_t *); 81 static void pthread__fork_callback(void); 82 static void pthread__reap(pthread_t); 83 static void pthread__child_callback(void); 84 static void pthread__start(void); 85 86 void pthread__init(void); 87 88 int pthread__started; 89 int __uselibcstub = 1; 90 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 91 pthread_queue_t pthread__deadqueue; 92 pthread_queue_t pthread__allqueue; 93 94 static pthread_attr_t pthread_default_attr; 95 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 96 97 enum { 98 DIAGASSERT_ABORT = 1<<0, 99 DIAGASSERT_STDERR = 1<<1, 100 DIAGASSERT_SYSLOG = 1<<2 101 }; 102 103 static int pthread__diagassert; 104 105 int pthread__concurrency; 106 int pthread__nspins; 107 int pthread__unpark_max = PTHREAD__UNPARK_MAX; 108 int pthread__dbg; /* set by libpthread_dbg if active */ 109 110 /* 111 * We have to initialize the pthread_stack* variables here because 112 * mutexes are used before pthread_init() and thus pthread__initmain() 113 * are called. Since mutexes only save the stack pointer and not a 114 * pointer to the thread data, it is safe to change the mapping from 115 * stack pointer to thread data afterwards. 116 */ 117 size_t pthread__stacksize; 118 size_t pthread__pagesize; 119 static struct __pthread_st *pthread__main; 120 static size_t __pthread_st_size; 121 122 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 123 124 __strong_alias(__libc_thr_self,pthread_self) 125 __strong_alias(__libc_thr_create,pthread_create) 126 __strong_alias(__libc_thr_exit,pthread_exit) 127 __strong_alias(__libc_thr_errno,pthread__errno) 128 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 129 __strong_alias(__libc_thr_equal,pthread_equal) 130 __strong_alias(__libc_thr_init,pthread__init) 131 132 /* 133 * Static library kludge. Place a reference to a symbol any library 134 * file which does not already have a reference here. 135 */ 136 extern int pthread__cancel_stub_binder; 137 138 void *pthread__static_lib_binder[] = { 139 &pthread__cancel_stub_binder, 140 pthread_cond_init, 141 pthread_mutex_init, 142 pthread_rwlock_init, 143 pthread_barrier_init, 144 pthread_key_create, 145 pthread_setspecific, 146 }; 147 148 #define NHASHLOCK 64 149 150 static union hashlock { 151 pthread_mutex_t mutex; 152 char pad[64]; 153 } hashlocks[NHASHLOCK] __aligned(64); 154 155 /* 156 * This needs to be started by the library loading code, before main() 157 * gets to run, for various things that use the state of the initial thread 158 * to work properly (thread-specific data is an application-visible example; 159 * spinlock counts for mutexes is an internal example). 160 */ 161 void 162 pthread__init(void) 163 { 164 pthread_t first; 165 char *p; 166 int i; 167 extern int __isthreaded; 168 169 /* 170 * Allocate pthread_keys descriptors before 171 * reseting __uselibcstub because otherwise 172 * malloc() will call pthread_keys_create() 173 * while pthread_keys descriptors are not 174 * yet allocated. 175 */ 176 pthread__main = pthread_tsd_init(&__pthread_st_size); 177 if (pthread__main == NULL) 178 err(EXIT_FAILURE, "Cannot allocate pthread storage"); 179 180 __uselibcstub = 0; 181 182 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 183 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF); 184 185 /* Initialize locks first; they're needed elsewhere. */ 186 pthread__lockprim_init(); 187 for (i = 0; i < NHASHLOCK; i++) { 188 pthread_mutex_init(&hashlocks[i].mutex, NULL); 189 } 190 191 /* Fetch parameters. */ 192 i = (int)_lwp_unpark_all(NULL, 0, NULL); 193 if (i == -1) 194 err(EXIT_FAILURE, "_lwp_unpark_all"); 195 if (i < pthread__unpark_max) 196 pthread__unpark_max = i; 197 198 /* Basic data structure setup */ 199 pthread_attr_init(&pthread_default_attr); 200 PTQ_INIT(&pthread__allqueue); 201 PTQ_INIT(&pthread__deadqueue); 202 203 rb_tree_init(&pthread__alltree, &pthread__alltree_ops); 204 205 /* Create the thread structure corresponding to main() */ 206 pthread__initmain(&first); 207 pthread__initthread(first); 208 pthread__scrubthread(first, NULL, 0); 209 210 first->pt_lid = _lwp_self(); 211 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 212 (void)rb_tree_insert_node(&pthread__alltree, first); 213 214 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 215 err(EXIT_FAILURE, "_lwp_ctl"); 216 } 217 218 /* Start subsystems */ 219 PTHREAD_MD_INIT 220 221 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 222 switch (*p) { 223 case 'a': 224 pthread__diagassert |= DIAGASSERT_ABORT; 225 break; 226 case 'A': 227 pthread__diagassert &= ~DIAGASSERT_ABORT; 228 break; 229 case 'e': 230 pthread__diagassert |= DIAGASSERT_STDERR; 231 break; 232 case 'E': 233 pthread__diagassert &= ~DIAGASSERT_STDERR; 234 break; 235 case 'l': 236 pthread__diagassert |= DIAGASSERT_SYSLOG; 237 break; 238 case 'L': 239 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 240 break; 241 } 242 } 243 244 /* Tell libc that we're here and it should role-play accordingly. */ 245 pthread_atfork(NULL, NULL, pthread__fork_callback); 246 __isthreaded = 1; 247 } 248 249 static void 250 pthread__fork_callback(void) 251 { 252 struct __pthread_st *self = pthread__self(); 253 254 /* lwpctl state is not copied across fork. */ 255 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 256 err(EXIT_FAILURE, "_lwp_ctl"); 257 } 258 self->pt_lid = _lwp_self(); 259 } 260 261 static void 262 pthread__child_callback(void) 263 { 264 265 /* 266 * Clean up data structures that a forked child process might 267 * trip over. Note that if threads have been created (causing 268 * this handler to be registered) the standards say that the 269 * child will trigger undefined behavior if it makes any 270 * pthread_* calls (or any other calls that aren't 271 * async-signal-safe), so we don't really have to clean up 272 * much. Anything that permits some pthread_* calls to work is 273 * merely being polite. 274 */ 275 pthread__started = 0; 276 } 277 278 static void 279 pthread__start(void) 280 { 281 282 /* 283 * Per-process timers are cleared by fork(); despite the 284 * various restrictions on fork() and threads, it's legal to 285 * fork() before creating any threads. 286 */ 287 pthread_atfork(NULL, NULL, pthread__child_callback); 288 } 289 290 291 /* General-purpose thread data structure sanitization. */ 292 /* ARGSUSED */ 293 static void 294 pthread__initthread(pthread_t t) 295 { 296 297 t->pt_self = t; 298 t->pt_magic = PT_MAGIC; 299 t->pt_willpark = 0; 300 t->pt_unpark = 0; 301 t->pt_nwaiters = 0; 302 t->pt_sleepobj = NULL; 303 t->pt_signalled = 0; 304 t->pt_havespecific = 0; 305 t->pt_early = NULL; 306 t->pt_lwpctl = &pthread__dummy_lwpctl; 307 t->pt_blocking = 0; 308 t->pt_droplock = NULL; 309 310 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 311 pthread_mutex_init(&t->pt_lock, NULL); 312 PTQ_INIT(&t->pt_cleanup_stack); 313 pthread_cond_init(&t->pt_joiners, NULL); 314 } 315 316 static void 317 pthread__scrubthread(pthread_t t, char *name, int flags) 318 { 319 320 t->pt_state = PT_STATE_RUNNING; 321 t->pt_exitval = NULL; 322 t->pt_flags = flags; 323 t->pt_cancel = 0; 324 t->pt_errno = 0; 325 t->pt_name = name; 326 t->pt_lid = 0; 327 } 328 329 static int 330 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 331 { 332 void *stackbase, *stackbase2, *redzone; 333 size_t stacksize, guardsize; 334 bool allocated; 335 336 if (attr != NULL) { 337 pthread_attr_getstack(attr, &stackbase, &stacksize); 338 } else { 339 stackbase = NULL; 340 stacksize = 0; 341 } 342 if (stacksize == 0) 343 stacksize = pthread__stacksize; 344 345 if (newthread->pt_stack_allocated) { 346 if (stackbase == NULL && 347 newthread->pt_stack.ss_size == stacksize) 348 return 0; 349 stackbase2 = newthread->pt_stack.ss_sp; 350 #ifndef __MACHINE_STACK_GROWS_UP 351 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 352 #endif 353 munmap(stackbase2, 354 newthread->pt_stack.ss_size + newthread->pt_guardsize); 355 newthread->pt_stack.ss_sp = NULL; 356 newthread->pt_stack.ss_size = 0; 357 newthread->pt_guardsize = 0; 358 newthread->pt_stack_allocated = false; 359 } 360 361 newthread->pt_stack_allocated = false; 362 363 if (stackbase == NULL) { 364 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 365 guardsize = pthread__pagesize; 366 stackbase = mmap(NULL, stacksize + guardsize, 367 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 368 if (stackbase == MAP_FAILED) 369 return ENOMEM; 370 allocated = true; 371 } else { 372 guardsize = 0; 373 allocated = false; 374 } 375 #ifdef __MACHINE_STACK_GROWS_UP 376 redzone = (char *)stackbase + stacksize; 377 stackbase2 = (char *)stackbase; 378 #else 379 redzone = (char *)stackbase; 380 stackbase2 = (char *)stackbase + guardsize; 381 #endif 382 if (allocated && guardsize && 383 mprotect(redzone, guardsize, PROT_NONE) == -1) { 384 munmap(stackbase, stacksize + guardsize); 385 return EPERM; 386 } 387 newthread->pt_stack.ss_size = stacksize; 388 newthread->pt_stack.ss_sp = stackbase2; 389 newthread->pt_guardsize = guardsize; 390 newthread->pt_stack_allocated = allocated; 391 return 0; 392 } 393 394 int 395 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 396 void *(*startfunc)(void *), void *arg) 397 { 398 pthread_t newthread; 399 pthread_attr_t nattr; 400 struct pthread_attr_private *p; 401 char * volatile name; 402 unsigned long flag; 403 void *private_area; 404 int ret; 405 406 if (__predict_false(__uselibcstub)) { 407 pthread__errorfunc(__FILE__, __LINE__, __func__, 408 "pthread_create() requires linking with -lpthread"); 409 return __libc_thr_create_stub(thread, attr, startfunc, arg); 410 } 411 412 /* 413 * It's okay to check this without a lock because there can 414 * only be one thread before it becomes true. 415 */ 416 if (pthread__started == 0) { 417 pthread__start(); 418 pthread__started = 1; 419 } 420 421 if (attr == NULL) 422 nattr = pthread_default_attr; 423 else if (attr->pta_magic == PT_ATTR_MAGIC) 424 nattr = *attr; 425 else 426 return EINVAL; 427 428 /* Fetch misc. attributes from the attr structure. */ 429 name = NULL; 430 if ((p = nattr.pta_private) != NULL) 431 if (p->ptap_name[0] != '\0') 432 if ((name = strdup(p->ptap_name)) == NULL) 433 return ENOMEM; 434 435 newthread = NULL; 436 437 /* 438 * Try to reclaim a dead thread. 439 */ 440 if (!PTQ_EMPTY(&pthread__deadqueue)) { 441 pthread_mutex_lock(&pthread__deadqueue_lock); 442 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 443 /* Still running? */ 444 if (newthread->pt_lwpctl->lc_curcpu == 445 LWPCTL_CPU_EXITED || 446 (_lwp_kill(newthread->pt_lid, 0) == -1 && 447 errno == ESRCH)) 448 break; 449 } 450 if (newthread) 451 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 452 pthread_mutex_unlock(&pthread__deadqueue_lock); 453 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 454 if (newthread && newthread->pt_tls) { 455 _rtld_tls_free(newthread->pt_tls); 456 newthread->pt_tls = NULL; 457 } 458 #endif 459 } 460 461 /* 462 * If necessary set up a stack, allocate space for a pthread_st, 463 * and initialize it. 464 */ 465 if (newthread == NULL) { 466 newthread = calloc(1, __pthread_st_size); 467 if (newthread == NULL) { 468 free(name); 469 return ENOMEM; 470 } 471 newthread->pt_stack_allocated = false; 472 473 if (pthread__getstack(newthread, attr)) { 474 free(newthread); 475 free(name); 476 return ENOMEM; 477 } 478 479 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 480 newthread->pt_tls = NULL; 481 #endif 482 483 /* Add to list of all threads. */ 484 pthread_rwlock_wrlock(&pthread__alltree_lock); 485 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 486 (void)rb_tree_insert_node(&pthread__alltree, newthread); 487 pthread_rwlock_unlock(&pthread__alltree_lock); 488 489 /* Will be reset by the thread upon exit. */ 490 pthread__initthread(newthread); 491 } else { 492 if (pthread__getstack(newthread, attr)) { 493 pthread_mutex_lock(&pthread__deadqueue_lock); 494 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 495 pthread_mutex_unlock(&pthread__deadqueue_lock); 496 return ENOMEM; 497 } 498 } 499 500 /* 501 * Create the new LWP. 502 */ 503 pthread__scrubthread(newthread, name, nattr.pta_flags); 504 newthread->pt_func = startfunc; 505 newthread->pt_arg = arg; 506 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 507 private_area = newthread->pt_tls = _rtld_tls_allocate(); 508 newthread->pt_tls->tcb_pthread = newthread; 509 #else 510 private_area = newthread; 511 #endif 512 513 flag = LWP_DETACHED; 514 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 515 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 516 flag |= LWP_SUSPENDED; 517 518 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area, 519 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size, 520 flag, &newthread->pt_lid); 521 if (ret != 0) { 522 ret = errno; 523 pthread_mutex_lock(&newthread->pt_lock); 524 /* Will unlock and free name. */ 525 pthread__reap(newthread); 526 return ret; 527 } 528 529 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 530 if (p != NULL) { 531 (void)pthread_setschedparam(newthread, p->ptap_policy, 532 &p->ptap_sp); 533 } 534 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 535 (void)_lwp_continue(newthread->pt_lid); 536 } 537 } 538 539 *thread = newthread; 540 541 return 0; 542 } 543 544 545 __dead static void 546 pthread__create_tramp(void *cookie) 547 { 548 pthread_t self; 549 void *retval; 550 551 self = cookie; 552 553 /* 554 * Throw away some stack in a feeble attempt to reduce cache 555 * thrash. May help for SMT processors. XXX We should not 556 * be allocating stacks on fixed 2MB boundaries. Needs a 557 * thread register or decent thread local storage. 558 * 559 * Note that we may race with the kernel in _lwp_create(), 560 * and so pt_lid can be unset at this point, but we don't 561 * care. 562 */ 563 (void)alloca(((unsigned)self->pt_lid & 7) << 8); 564 565 if (self->pt_name != NULL) { 566 pthread_mutex_lock(&self->pt_lock); 567 if (self->pt_name != NULL) 568 (void)_lwp_setname(0, self->pt_name); 569 pthread_mutex_unlock(&self->pt_lock); 570 } 571 572 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 573 err(EXIT_FAILURE, "_lwp_ctl"); 574 } 575 576 retval = (*self->pt_func)(self->pt_arg); 577 578 pthread_exit(retval); 579 580 /*NOTREACHED*/ 581 pthread__abort(); 582 } 583 584 int 585 pthread_suspend_np(pthread_t thread) 586 { 587 pthread_t self; 588 589 self = pthread__self(); 590 if (self == thread) { 591 return EDEADLK; 592 } 593 if (pthread__find(thread) != 0) 594 return ESRCH; 595 if (_lwp_suspend(thread->pt_lid) == 0) 596 return 0; 597 return errno; 598 } 599 600 int 601 pthread_resume_np(pthread_t thread) 602 { 603 604 if (pthread__find(thread) != 0) 605 return ESRCH; 606 if (_lwp_continue(thread->pt_lid) == 0) 607 return 0; 608 return errno; 609 } 610 611 void 612 pthread_exit(void *retval) 613 { 614 pthread_t self; 615 struct pt_clean_t *cleanup; 616 char *name; 617 618 if (__predict_false(__uselibcstub)) { 619 __libc_thr_exit_stub(retval); 620 goto out; 621 } 622 623 self = pthread__self(); 624 625 /* Disable cancellability. */ 626 pthread_mutex_lock(&self->pt_lock); 627 self->pt_flags |= PT_FLAG_CS_DISABLED; 628 self->pt_cancel = 0; 629 630 /* Call any cancellation cleanup handlers */ 631 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 632 pthread_mutex_unlock(&self->pt_lock); 633 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 634 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 635 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 636 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 637 } 638 pthread_mutex_lock(&self->pt_lock); 639 } 640 641 /* Perform cleanup of thread-specific data */ 642 pthread__destroy_tsd(self); 643 644 /* Signal our exit. */ 645 self->pt_exitval = retval; 646 if (self->pt_flags & PT_FLAG_DETACHED) { 647 self->pt_state = PT_STATE_DEAD; 648 name = self->pt_name; 649 self->pt_name = NULL; 650 pthread_mutex_unlock(&self->pt_lock); 651 if (name != NULL) 652 free(name); 653 pthread_mutex_lock(&pthread__deadqueue_lock); 654 PTQ_INSERT_TAIL(&pthread__deadqueue, self, pt_deadq); 655 pthread_mutex_unlock(&pthread__deadqueue_lock); 656 _lwp_exit(); 657 } else { 658 self->pt_state = PT_STATE_ZOMBIE; 659 pthread_cond_broadcast(&self->pt_joiners); 660 pthread_mutex_unlock(&self->pt_lock); 661 /* Note: name will be freed by the joiner. */ 662 _lwp_exit(); 663 } 664 665 out: 666 /*NOTREACHED*/ 667 pthread__abort(); 668 exit(1); 669 } 670 671 672 int 673 pthread_join(pthread_t thread, void **valptr) 674 { 675 pthread_t self; 676 int error; 677 678 self = pthread__self(); 679 680 if (pthread__find(thread) != 0) 681 return ESRCH; 682 683 if (thread->pt_magic != PT_MAGIC) 684 return EINVAL; 685 686 if (thread == self) 687 return EDEADLK; 688 689 self->pt_droplock = &thread->pt_lock; 690 pthread_mutex_lock(&thread->pt_lock); 691 for (;;) { 692 if (thread->pt_state == PT_STATE_ZOMBIE) 693 break; 694 if (thread->pt_state == PT_STATE_DEAD) { 695 pthread_mutex_unlock(&thread->pt_lock); 696 self->pt_droplock = NULL; 697 return ESRCH; 698 } 699 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 700 pthread_mutex_unlock(&thread->pt_lock); 701 self->pt_droplock = NULL; 702 return EINVAL; 703 } 704 error = pthread_cond_wait(&thread->pt_joiners, 705 &thread->pt_lock); 706 if (error != 0) { 707 pthread__errorfunc(__FILE__, __LINE__, 708 __func__, "unexpected return from cond_wait()"); 709 } 710 711 } 712 pthread__testcancel(self); 713 if (valptr != NULL) 714 *valptr = thread->pt_exitval; 715 /* pthread__reap() will drop the lock. */ 716 pthread__reap(thread); 717 self->pt_droplock = NULL; 718 719 return 0; 720 } 721 722 static void 723 pthread__reap(pthread_t thread) 724 { 725 char *name; 726 727 name = thread->pt_name; 728 thread->pt_name = NULL; 729 thread->pt_state = PT_STATE_DEAD; 730 pthread_mutex_unlock(&thread->pt_lock); 731 732 pthread_mutex_lock(&pthread__deadqueue_lock); 733 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 734 pthread_mutex_unlock(&pthread__deadqueue_lock); 735 736 if (name != NULL) 737 free(name); 738 } 739 740 int 741 pthread_equal(pthread_t t1, pthread_t t2) 742 { 743 if (__predict_false(__uselibcstub)) 744 return __libc_thr_equal_stub(t1, t2); 745 746 /* Nothing special here. */ 747 return (t1 == t2); 748 } 749 750 751 int 752 pthread_detach(pthread_t thread) 753 { 754 755 if (pthread__find(thread) != 0) 756 return ESRCH; 757 758 if (thread->pt_magic != PT_MAGIC) 759 return EINVAL; 760 761 pthread_mutex_lock(&thread->pt_lock); 762 thread->pt_flags |= PT_FLAG_DETACHED; 763 if (thread->pt_state == PT_STATE_ZOMBIE) { 764 /* pthread__reap() will drop the lock. */ 765 pthread__reap(thread); 766 } else { 767 /* 768 * Not valid for threads to be waiting in 769 * pthread_join() (there are intractable 770 * sync issues from the application 771 * perspective), but give those threads 772 * a chance anyway. 773 */ 774 pthread_cond_broadcast(&thread->pt_joiners); 775 pthread_mutex_unlock(&thread->pt_lock); 776 } 777 778 return 0; 779 } 780 781 782 int 783 pthread_getname_np(pthread_t thread, char *name, size_t len) 784 { 785 786 if (pthread__find(thread) != 0) 787 return ESRCH; 788 789 if (thread->pt_magic != PT_MAGIC) 790 return EINVAL; 791 792 pthread_mutex_lock(&thread->pt_lock); 793 if (thread->pt_name == NULL) 794 name[0] = '\0'; 795 else 796 strlcpy(name, thread->pt_name, len); 797 pthread_mutex_unlock(&thread->pt_lock); 798 799 return 0; 800 } 801 802 803 int 804 pthread_setname_np(pthread_t thread, const char *name, void *arg) 805 { 806 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 807 int namelen; 808 809 if (pthread__find(thread) != 0) 810 return ESRCH; 811 812 if (thread->pt_magic != PT_MAGIC) 813 return EINVAL; 814 815 namelen = snprintf(newname, sizeof(newname), name, arg); 816 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 817 return EINVAL; 818 819 cp = strdup(newname); 820 if (cp == NULL) 821 return ENOMEM; 822 823 pthread_mutex_lock(&thread->pt_lock); 824 oldname = thread->pt_name; 825 thread->pt_name = cp; 826 (void)_lwp_setname(thread->pt_lid, cp); 827 pthread_mutex_unlock(&thread->pt_lock); 828 829 if (oldname != NULL) 830 free(oldname); 831 832 return 0; 833 } 834 835 836 837 /* 838 * XXX There should be a way for applications to use the efficent 839 * inline version, but there are opacity/namespace issues. 840 */ 841 pthread_t 842 pthread_self(void) 843 { 844 if (__predict_false(__uselibcstub)) 845 return (pthread_t)__libc_thr_self_stub(); 846 847 return pthread__self(); 848 } 849 850 851 int 852 pthread_cancel(pthread_t thread) 853 { 854 855 if (pthread__find(thread) != 0) 856 return ESRCH; 857 pthread_mutex_lock(&thread->pt_lock); 858 thread->pt_flags |= PT_FLAG_CS_PENDING; 859 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 860 thread->pt_cancel = 1; 861 pthread_mutex_unlock(&thread->pt_lock); 862 _lwp_wakeup(thread->pt_lid); 863 } else 864 pthread_mutex_unlock(&thread->pt_lock); 865 866 return 0; 867 } 868 869 870 int 871 pthread_setcancelstate(int state, int *oldstate) 872 { 873 pthread_t self; 874 int retval; 875 876 if (__predict_false(__uselibcstub)) 877 return __libc_thr_setcancelstate_stub(state, oldstate); 878 879 self = pthread__self(); 880 retval = 0; 881 882 pthread_mutex_lock(&self->pt_lock); 883 884 if (oldstate != NULL) { 885 if (self->pt_flags & PT_FLAG_CS_DISABLED) 886 *oldstate = PTHREAD_CANCEL_DISABLE; 887 else 888 *oldstate = PTHREAD_CANCEL_ENABLE; 889 } 890 891 if (state == PTHREAD_CANCEL_DISABLE) { 892 self->pt_flags |= PT_FLAG_CS_DISABLED; 893 if (self->pt_cancel) { 894 self->pt_flags |= PT_FLAG_CS_PENDING; 895 self->pt_cancel = 0; 896 } 897 } else if (state == PTHREAD_CANCEL_ENABLE) { 898 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 899 /* 900 * If a cancellation was requested while cancellation 901 * was disabled, note that fact for future 902 * cancellation tests. 903 */ 904 if (self->pt_flags & PT_FLAG_CS_PENDING) { 905 self->pt_cancel = 1; 906 /* This is not a deferred cancellation point. */ 907 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 908 pthread_mutex_unlock(&self->pt_lock); 909 pthread__cancelled(); 910 } 911 } 912 } else 913 retval = EINVAL; 914 915 pthread_mutex_unlock(&self->pt_lock); 916 917 return retval; 918 } 919 920 921 int 922 pthread_setcanceltype(int type, int *oldtype) 923 { 924 pthread_t self; 925 int retval; 926 927 self = pthread__self(); 928 retval = 0; 929 930 pthread_mutex_lock(&self->pt_lock); 931 932 if (oldtype != NULL) { 933 if (self->pt_flags & PT_FLAG_CS_ASYNC) 934 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 935 else 936 *oldtype = PTHREAD_CANCEL_DEFERRED; 937 } 938 939 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 940 self->pt_flags |= PT_FLAG_CS_ASYNC; 941 if (self->pt_cancel) { 942 pthread_mutex_unlock(&self->pt_lock); 943 pthread__cancelled(); 944 } 945 } else if (type == PTHREAD_CANCEL_DEFERRED) 946 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 947 else 948 retval = EINVAL; 949 950 pthread_mutex_unlock(&self->pt_lock); 951 952 return retval; 953 } 954 955 956 void 957 pthread_testcancel(void) 958 { 959 pthread_t self; 960 961 self = pthread__self(); 962 if (self->pt_cancel) 963 pthread__cancelled(); 964 } 965 966 967 /* 968 * POSIX requires that certain functions return an error rather than 969 * invoking undefined behavior even when handed completely bogus 970 * pthread_t values, e.g. stack garbage. 971 */ 972 int 973 pthread__find(pthread_t id) 974 { 975 pthread_t target; 976 int error; 977 978 pthread_rwlock_rdlock(&pthread__alltree_lock); 979 target = rb_tree_find_node(&pthread__alltree, id); 980 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH; 981 pthread_rwlock_unlock(&pthread__alltree_lock); 982 983 return error; 984 } 985 986 987 void 988 pthread__testcancel(pthread_t self) 989 { 990 991 if (self->pt_cancel) 992 pthread__cancelled(); 993 } 994 995 996 void 997 pthread__cancelled(void) 998 { 999 pthread_mutex_t *droplock; 1000 pthread_t self; 1001 1002 self = pthread__self(); 1003 droplock = self->pt_droplock; 1004 self->pt_droplock = NULL; 1005 1006 if (droplock != NULL && pthread_mutex_held_np(droplock)) 1007 pthread_mutex_unlock(droplock); 1008 1009 pthread_exit(PTHREAD_CANCELED); 1010 } 1011 1012 1013 void 1014 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 1015 { 1016 pthread_t self; 1017 struct pt_clean_t *entry; 1018 1019 self = pthread__self(); 1020 entry = store; 1021 entry->ptc_cleanup = cleanup; 1022 entry->ptc_arg = arg; 1023 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1024 } 1025 1026 1027 void 1028 pthread__cleanup_pop(int ex, void *store) 1029 { 1030 pthread_t self; 1031 struct pt_clean_t *entry; 1032 1033 self = pthread__self(); 1034 entry = store; 1035 1036 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1037 if (ex) 1038 (*entry->ptc_cleanup)(entry->ptc_arg); 1039 } 1040 1041 1042 int * 1043 pthread__errno(void) 1044 { 1045 pthread_t self; 1046 1047 if (__predict_false(__uselibcstub)) { 1048 pthread__errorfunc(__FILE__, __LINE__, __func__, 1049 "pthread__errno() requires linking with -lpthread"); 1050 return __libc_thr_errno_stub(); 1051 } 1052 1053 self = pthread__self(); 1054 1055 return &(self->pt_errno); 1056 } 1057 1058 ssize_t _sys_write(int, const void *, size_t); 1059 1060 void 1061 pthread__assertfunc(const char *file, int line, const char *function, 1062 const char *expr) 1063 { 1064 char buf[1024]; 1065 int len; 1066 1067 /* 1068 * snprintf should not acquire any locks, or we could 1069 * end up deadlocked if the assert caller held locks. 1070 */ 1071 len = snprintf(buf, 1024, 1072 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1073 expr, file, line, 1074 function ? ", function \"" : "", 1075 function ? function : "", 1076 function ? "\"" : ""); 1077 1078 _sys_write(STDERR_FILENO, buf, (size_t)len); 1079 (void)kill(getpid(), SIGABRT); 1080 1081 _exit(1); 1082 } 1083 1084 1085 void 1086 pthread__errorfunc(const char *file, int line, const char *function, 1087 const char *msg) 1088 { 1089 char buf[1024]; 1090 size_t len; 1091 1092 if (pthread__diagassert == 0) 1093 return; 1094 1095 /* 1096 * snprintf should not acquire any locks, or we could 1097 * end up deadlocked if the assert caller held locks. 1098 */ 1099 len = snprintf(buf, 1024, 1100 "%s: Error detected by libpthread: %s.\n" 1101 "Detected by file \"%s\", line %d%s%s%s.\n" 1102 "See pthread(3) for information.\n", 1103 getprogname(), msg, file, line, 1104 function ? ", function \"" : "", 1105 function ? function : "", 1106 function ? "\"" : ""); 1107 1108 if (pthread__diagassert & DIAGASSERT_STDERR) 1109 _sys_write(STDERR_FILENO, buf, len); 1110 1111 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1112 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1113 1114 if (pthread__diagassert & DIAGASSERT_ABORT) { 1115 (void)kill(getpid(), SIGABRT); 1116 _exit(1); 1117 } 1118 } 1119 1120 /* 1121 * Thread park/unpark operations. The kernel operations are 1122 * modelled after a brief description from "Multithreading in 1123 * the Solaris Operating Environment": 1124 * 1125 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1126 */ 1127 1128 #define OOPS(msg) \ 1129 pthread__errorfunc(__FILE__, __LINE__, __func__, msg) 1130 1131 int 1132 pthread__park(pthread_t self, pthread_mutex_t *lock, 1133 pthread_queue_t *queue, const struct timespec *abstime, 1134 int cancelpt, const void *hint) 1135 { 1136 int rv, error; 1137 void *obj; 1138 1139 /* 1140 * For non-interlocked release of mutexes we need a store 1141 * barrier before incrementing pt_blocking away from zero. 1142 * This is provided by pthread_mutex_unlock(). 1143 */ 1144 self->pt_willpark = 1; 1145 pthread_mutex_unlock(lock); 1146 self->pt_willpark = 0; 1147 self->pt_blocking++; 1148 1149 /* 1150 * Wait until we are awoken by a pending unpark operation, 1151 * a signal, an unpark posted after we have gone asleep, 1152 * or an expired timeout. 1153 * 1154 * It is fine to test the value of pt_sleepobj without 1155 * holding any locks, because: 1156 * 1157 * o Only the blocking thread (this thread) ever sets them 1158 * to a non-NULL value. 1159 * 1160 * o Other threads may set them NULL, but if they do so they 1161 * must also make this thread return from _lwp_park. 1162 * 1163 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1164 * calls and all make use of spinlocks in the kernel. So 1165 * these system calls act as full memory barriers, and will 1166 * ensure that the calling CPU's store buffers are drained. 1167 * In combination with the spinlock release before unpark, 1168 * this means that modification of pt_sleepobj/onq by another 1169 * thread will become globally visible before that thread 1170 * schedules an unpark operation on this thread. 1171 * 1172 * Note: the test in the while() statement dodges the park op if 1173 * we have already been awoken, unless there is another thread to 1174 * awaken. This saves a syscall - if we were already awakened, 1175 * the next call to _lwp_park() would need to return early in order 1176 * to eat the previous wakeup. 1177 */ 1178 rv = 0; 1179 do { 1180 /* 1181 * If we deferred unparking a thread, arrange to 1182 * have _lwp_park() restart it before blocking. 1183 */ 1184 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME, abstime, 1185 self->pt_unpark, hint, hint); 1186 self->pt_unpark = 0; 1187 if (error != 0) { 1188 switch (rv = errno) { 1189 case EINTR: 1190 case EALREADY: 1191 rv = 0; 1192 break; 1193 case ETIMEDOUT: 1194 break; 1195 default: 1196 OOPS("_lwp_park failed"); 1197 break; 1198 } 1199 } 1200 /* Check for cancellation. */ 1201 if (cancelpt && self->pt_cancel) 1202 rv = EINTR; 1203 } while (self->pt_sleepobj != NULL && rv == 0); 1204 1205 /* 1206 * If we have been awoken early but are still on the queue, 1207 * then remove ourself. Again, it's safe to do the test 1208 * without holding any locks. 1209 */ 1210 if (__predict_false(self->pt_sleepobj != NULL)) { 1211 pthread_mutex_lock(lock); 1212 if ((obj = self->pt_sleepobj) != NULL) { 1213 PTQ_REMOVE(queue, self, pt_sleep); 1214 self->pt_sleepobj = NULL; 1215 if (obj != NULL && self->pt_early != NULL) 1216 (*self->pt_early)(obj); 1217 } 1218 pthread_mutex_unlock(lock); 1219 } 1220 self->pt_early = NULL; 1221 self->pt_blocking--; 1222 membar_sync(); 1223 1224 return rv; 1225 } 1226 1227 void 1228 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1229 pthread_mutex_t *interlock) 1230 { 1231 pthread_t target; 1232 u_int max; 1233 size_t nwaiters; 1234 1235 max = pthread__unpark_max; 1236 nwaiters = self->pt_nwaiters; 1237 target = PTQ_FIRST(queue); 1238 if (nwaiters == max) { 1239 /* Overflow. */ 1240 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1241 __UNVOLATILE(&interlock->ptm_waiters)); 1242 nwaiters = 0; 1243 } 1244 target->pt_sleepobj = NULL; 1245 self->pt_waiters[nwaiters++] = target->pt_lid; 1246 PTQ_REMOVE(queue, target, pt_sleep); 1247 self->pt_nwaiters = nwaiters; 1248 pthread__mutex_deferwake(self, interlock); 1249 } 1250 1251 void 1252 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1253 pthread_mutex_t *interlock) 1254 { 1255 pthread_t target; 1256 u_int max; 1257 size_t nwaiters; 1258 1259 max = pthread__unpark_max; 1260 nwaiters = self->pt_nwaiters; 1261 PTQ_FOREACH(target, queue, pt_sleep) { 1262 if (nwaiters == max) { 1263 /* Overflow. */ 1264 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1265 __UNVOLATILE(&interlock->ptm_waiters)); 1266 nwaiters = 0; 1267 } 1268 target->pt_sleepobj = NULL; 1269 self->pt_waiters[nwaiters++] = target->pt_lid; 1270 } 1271 self->pt_nwaiters = nwaiters; 1272 PTQ_INIT(queue); 1273 pthread__mutex_deferwake(self, interlock); 1274 } 1275 1276 #undef OOPS 1277 1278 static void 1279 pthread__initmainstack(void) 1280 { 1281 struct rlimit slimit; 1282 const AuxInfo *aux; 1283 size_t size; 1284 1285 _DIAGASSERT(_dlauxinfo() != NULL); 1286 1287 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1288 err(EXIT_FAILURE, 1289 "Couldn't get stack resource consumption limits"); 1290 size = slimit.rlim_cur; 1291 pthread__main->pt_stack.ss_size = size; 1292 1293 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1294 if (aux->a_type == AT_STACKBASE) { 1295 pthread__main->pt_stack.ss_sp = (void *)aux->a_v; 1296 #ifdef __MACHINE_STACK_GROWS_UP 1297 pthread__main->pt_stack.ss_sp = (void *)aux->a_v; 1298 #else 1299 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size; 1300 #endif 1301 break; 1302 } 1303 } 1304 } 1305 1306 /* 1307 * Set up the slightly special stack for the "initial" thread, which 1308 * runs on the normal system stack, and thus gets slightly different 1309 * treatment. 1310 */ 1311 static void 1312 pthread__initmain(pthread_t *newt) 1313 { 1314 char *value; 1315 1316 pthread__initmainstack(); 1317 1318 value = pthread__getenv("PTHREAD_STACKSIZE"); 1319 if (value != NULL) { 1320 pthread__stacksize = atoi(value) * 1024; 1321 if (pthread__stacksize > pthread__main->pt_stack.ss_size) 1322 pthread__stacksize = pthread__main->pt_stack.ss_size; 1323 } 1324 if (pthread__stacksize == 0) 1325 pthread__stacksize = pthread__main->pt_stack.ss_size; 1326 pthread__stacksize += pthread__pagesize - 1; 1327 pthread__stacksize &= ~(pthread__pagesize - 1); 1328 if (pthread__stacksize < 4 * pthread__pagesize) 1329 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1330 4 * pthread__pagesize / 1024); 1331 1332 *newt = pthread__main; 1333 #if defined(_PTHREAD_GETTCB_EXT) 1334 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT(); 1335 #elif defined(__HAVE___LWP_GETTCB_FAST) 1336 pthread__main->pt_tls = __lwp_gettcb_fast(); 1337 #else 1338 pthread__main->pt_tls = _lwp_getprivate(); 1339 #endif 1340 pthread__main->pt_tls->tcb_pthread = pthread__main; 1341 } 1342 1343 static signed int 1344 /*ARGSUSED*/ 1345 pthread__cmp(void *ctx, const void *n1, const void *n2) 1346 { 1347 const uintptr_t p1 = (const uintptr_t)n1; 1348 const uintptr_t p2 = (const uintptr_t)n2; 1349 1350 if (p1 < p2) 1351 return -1; 1352 if (p1 > p2) 1353 return 1; 1354 return 0; 1355 } 1356 1357 /* Because getenv() wants to use locks. */ 1358 char * 1359 pthread__getenv(const char *name) 1360 { 1361 extern char **environ; 1362 size_t l_name, offset; 1363 1364 l_name = strlen(name); 1365 for (offset = 0; environ[offset] != NULL; offset++) { 1366 if (strncmp(name, environ[offset], l_name) == 0 && 1367 environ[offset][l_name] == '=') { 1368 return environ[offset] + l_name + 1; 1369 } 1370 } 1371 1372 return NULL; 1373 } 1374 1375 pthread_mutex_t * 1376 pthread__hashlock(volatile const void *p) 1377 { 1378 uintptr_t v; 1379 1380 v = (uintptr_t)p; 1381 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1382 } 1383 1384 int 1385 pthread__checkpri(int pri) 1386 { 1387 static int havepri; 1388 static long min, max; 1389 1390 if (!havepri) { 1391 min = sysconf(_SC_SCHED_PRI_MIN); 1392 max = sysconf(_SC_SCHED_PRI_MAX); 1393 havepri = 1; 1394 } 1395 return (pri < min || pri > max) ? EINVAL : 0; 1396 } 1397