1 /* $NetBSD: pthread.c,v 1.135 2012/05/04 12:26:33 joerg Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nathan J. Williams and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __RCSID("$NetBSD: pthread.c,v 1.135 2012/05/04 12:26:33 joerg Exp $"); 34 35 #define __EXPOSE_STACK 1 36 37 #include <sys/param.h> 38 #include <sys/exec_elf.h> 39 #include <sys/mman.h> 40 #include <sys/sysctl.h> 41 #include <sys/lwpctl.h> 42 #include <sys/tls.h> 43 44 #include <assert.h> 45 #include <dlfcn.h> 46 #include <err.h> 47 #include <errno.h> 48 #include <lwp.h> 49 #include <signal.h> 50 #include <stdio.h> 51 #include <stdlib.h> 52 #include <string.h> 53 #include <syslog.h> 54 #include <ucontext.h> 55 #include <unistd.h> 56 #include <sched.h> 57 58 #include "pthread.h" 59 #include "pthread_int.h" 60 61 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 62 RB_HEAD(__pthread__alltree, __pthread_st) pthread__alltree; 63 64 #ifndef lint 65 static int pthread__cmp(struct __pthread_st *, struct __pthread_st *); 66 RB_PROTOTYPE_STATIC(__pthread__alltree, __pthread_st, pt_alltree, pthread__cmp) 67 #endif 68 69 static void pthread__create_tramp(void *); 70 static void pthread__initthread(pthread_t); 71 static void pthread__scrubthread(pthread_t, char *, int); 72 static void pthread__initmain(pthread_t *); 73 static void pthread__fork_callback(void); 74 static void pthread__reap(pthread_t); 75 static void pthread__child_callback(void); 76 static void pthread__start(void); 77 78 void pthread__init(void); 79 80 int pthread__started; 81 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 82 pthread_queue_t pthread__deadqueue; 83 pthread_queue_t pthread__allqueue; 84 85 static pthread_attr_t pthread_default_attr; 86 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 87 88 enum { 89 DIAGASSERT_ABORT = 1<<0, 90 DIAGASSERT_STDERR = 1<<1, 91 DIAGASSERT_SYSLOG = 1<<2 92 }; 93 94 static int pthread__diagassert; 95 96 int pthread__concurrency; 97 int pthread__nspins; 98 int pthread__unpark_max = PTHREAD__UNPARK_MAX; 99 int pthread__dbg; /* set by libpthread_dbg if active */ 100 101 /* 102 * We have to initialize the pthread_stack* variables here because 103 * mutexes are used before pthread_init() and thus pthread__initmain() 104 * are called. Since mutexes only save the stack pointer and not a 105 * pointer to the thread data, it is safe to change the mapping from 106 * stack pointer to thread data afterwards. 107 */ 108 size_t pthread__stacksize; 109 size_t pthread__pagesize; 110 static struct __pthread_st pthread__main; 111 112 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 113 114 __strong_alias(__libc_thr_self,pthread_self) 115 __strong_alias(__libc_thr_create,pthread_create) 116 __strong_alias(__libc_thr_exit,pthread_exit) 117 __strong_alias(__libc_thr_errno,pthread__errno) 118 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 119 __strong_alias(__libc_thr_equal,pthread_equal) 120 __strong_alias(__libc_thr_init,pthread__init) 121 122 /* 123 * Static library kludge. Place a reference to a symbol any library 124 * file which does not already have a reference here. 125 */ 126 extern int pthread__cancel_stub_binder; 127 128 void *pthread__static_lib_binder[] = { 129 &pthread__cancel_stub_binder, 130 pthread_cond_init, 131 pthread_mutex_init, 132 pthread_rwlock_init, 133 pthread_barrier_init, 134 pthread_key_create, 135 pthread_setspecific, 136 }; 137 138 #define NHASHLOCK 64 139 140 static union hashlock { 141 pthread_mutex_t mutex; 142 char pad[64]; 143 } hashlocks[NHASHLOCK] __aligned(64); 144 145 /* 146 * This needs to be started by the library loading code, before main() 147 * gets to run, for various things that use the state of the initial thread 148 * to work properly (thread-specific data is an application-visible example; 149 * spinlock counts for mutexes is an internal example). 150 */ 151 void 152 pthread__init(void) 153 { 154 pthread_t first; 155 char *p; 156 int i, mib[2]; 157 size_t len; 158 extern int __isthreaded; 159 160 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 161 162 mib[0] = CTL_HW; 163 mib[1] = HW_NCPU; 164 165 len = sizeof(pthread__concurrency); 166 if (sysctl(mib, 2, &pthread__concurrency, &len, NULL, 0) == -1) 167 err(1, "sysctl(hw.ncpu"); 168 169 mib[0] = CTL_KERN; 170 mib[1] = KERN_OSREV; 171 172 /* Initialize locks first; they're needed elsewhere. */ 173 pthread__lockprim_init(); 174 for (i = 0; i < NHASHLOCK; i++) { 175 pthread_mutex_init(&hashlocks[i].mutex, NULL); 176 } 177 178 /* Fetch parameters. */ 179 i = (int)_lwp_unpark_all(NULL, 0, NULL); 180 if (i == -1) 181 err(1, "_lwp_unpark_all"); 182 if (i < pthread__unpark_max) 183 pthread__unpark_max = i; 184 185 /* Basic data structure setup */ 186 pthread_attr_init(&pthread_default_attr); 187 PTQ_INIT(&pthread__allqueue); 188 PTQ_INIT(&pthread__deadqueue); 189 RB_INIT(&pthread__alltree); 190 191 /* Create the thread structure corresponding to main() */ 192 pthread__initmain(&first); 193 pthread__initthread(first); 194 pthread__scrubthread(first, NULL, 0); 195 196 first->pt_lid = _lwp_self(); 197 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 198 RB_INSERT(__pthread__alltree, &pthread__alltree, first); 199 200 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 201 err(1, "_lwp_ctl"); 202 } 203 204 /* Start subsystems */ 205 PTHREAD_MD_INIT 206 207 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 208 switch (*p) { 209 case 'a': 210 pthread__diagassert |= DIAGASSERT_ABORT; 211 break; 212 case 'A': 213 pthread__diagassert &= ~DIAGASSERT_ABORT; 214 break; 215 case 'e': 216 pthread__diagassert |= DIAGASSERT_STDERR; 217 break; 218 case 'E': 219 pthread__diagassert &= ~DIAGASSERT_STDERR; 220 break; 221 case 'l': 222 pthread__diagassert |= DIAGASSERT_SYSLOG; 223 break; 224 case 'L': 225 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 226 break; 227 } 228 } 229 230 /* Tell libc that we're here and it should role-play accordingly. */ 231 pthread_atfork(NULL, NULL, pthread__fork_callback); 232 __isthreaded = 1; 233 } 234 235 static void 236 pthread__fork_callback(void) 237 { 238 struct __pthread_st *self = pthread__self(); 239 240 /* lwpctl state is not copied across fork. */ 241 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 242 err(1, "_lwp_ctl"); 243 } 244 self->pt_lid = _lwp_self(); 245 } 246 247 static void 248 pthread__child_callback(void) 249 { 250 251 /* 252 * Clean up data structures that a forked child process might 253 * trip over. Note that if threads have been created (causing 254 * this handler to be registered) the standards say that the 255 * child will trigger undefined behavior if it makes any 256 * pthread_* calls (or any other calls that aren't 257 * async-signal-safe), so we don't really have to clean up 258 * much. Anything that permits some pthread_* calls to work is 259 * merely being polite. 260 */ 261 pthread__started = 0; 262 } 263 264 static void 265 pthread__start(void) 266 { 267 268 /* 269 * Per-process timers are cleared by fork(); despite the 270 * various restrictions on fork() and threads, it's legal to 271 * fork() before creating any threads. 272 */ 273 pthread_atfork(NULL, NULL, pthread__child_callback); 274 } 275 276 277 /* General-purpose thread data structure sanitization. */ 278 /* ARGSUSED */ 279 static void 280 pthread__initthread(pthread_t t) 281 { 282 283 t->pt_self = t; 284 t->pt_magic = PT_MAGIC; 285 t->pt_willpark = 0; 286 t->pt_unpark = 0; 287 t->pt_nwaiters = 0; 288 t->pt_sleepobj = NULL; 289 t->pt_signalled = 0; 290 t->pt_havespecific = 0; 291 t->pt_early = NULL; 292 t->pt_lwpctl = &pthread__dummy_lwpctl; 293 t->pt_blocking = 0; 294 t->pt_droplock = NULL; 295 296 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 297 pthread_mutex_init(&t->pt_lock, NULL); 298 PTQ_INIT(&t->pt_cleanup_stack); 299 pthread_cond_init(&t->pt_joiners, NULL); 300 memset(&t->pt_specific, 0, sizeof(t->pt_specific)); 301 } 302 303 static void 304 pthread__scrubthread(pthread_t t, char *name, int flags) 305 { 306 307 t->pt_state = PT_STATE_RUNNING; 308 t->pt_exitval = NULL; 309 t->pt_flags = flags; 310 t->pt_cancel = 0; 311 t->pt_errno = 0; 312 t->pt_name = name; 313 t->pt_lid = 0; 314 } 315 316 static int 317 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 318 { 319 void *stackbase, *stackbase2, *redzone; 320 size_t stacksize, guardsize; 321 bool allocated; 322 323 if (attr != NULL) { 324 pthread_attr_getstack(attr, &stackbase, &stacksize); 325 } else { 326 stackbase = NULL; 327 stacksize = 0; 328 } 329 if (stacksize == 0) 330 stacksize = pthread__stacksize; 331 332 if (newthread->pt_stack_allocated) { 333 if (stackbase == NULL && 334 newthread->pt_stack.ss_size == stacksize) 335 return 0; 336 stackbase2 = newthread->pt_stack.ss_sp; 337 #ifndef __MACHINE_STACK_GROWS_UP 338 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 339 #endif 340 munmap(stackbase2, 341 newthread->pt_stack.ss_size + newthread->pt_guardsize); 342 newthread->pt_stack.ss_sp = NULL; 343 newthread->pt_stack.ss_size = 0; 344 newthread->pt_guardsize = 0; 345 newthread->pt_stack_allocated = false; 346 } 347 348 newthread->pt_stack_allocated = false; 349 350 if (stackbase == NULL) { 351 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 352 guardsize = pthread__pagesize; 353 stackbase = mmap(NULL, stacksize + guardsize, 354 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 355 if (stackbase == MAP_FAILED) 356 return ENOMEM; 357 allocated = true; 358 } else { 359 guardsize = 0; 360 allocated = false; 361 } 362 #ifdef __MACHINE_STACK_GROWS_UP 363 redzone = (char *)stackbase + stacksize; 364 stackbase2 = (char *)stackbase; 365 #else 366 redzone = (char *)stackbase; 367 stackbase2 = (char *)stackbase + guardsize; 368 #endif 369 if (allocated && guardsize && 370 mprotect(redzone, guardsize, PROT_NONE) == -1) { 371 munmap(stackbase, stacksize + guardsize); 372 return EPERM; 373 } 374 newthread->pt_stack.ss_size = stacksize; 375 newthread->pt_stack.ss_sp = stackbase2; 376 newthread->pt_guardsize = guardsize; 377 newthread->pt_stack_allocated = allocated; 378 return 0; 379 } 380 381 int 382 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 383 void *(*startfunc)(void *), void *arg) 384 { 385 pthread_t newthread; 386 pthread_attr_t nattr; 387 struct pthread_attr_private *p; 388 char * volatile name; 389 unsigned long flag; 390 void *private_area; 391 int ret; 392 393 /* 394 * It's okay to check this without a lock because there can 395 * only be one thread before it becomes true. 396 */ 397 if (pthread__started == 0) { 398 pthread__start(); 399 pthread__started = 1; 400 } 401 402 if (attr == NULL) 403 nattr = pthread_default_attr; 404 else if (attr->pta_magic == PT_ATTR_MAGIC) 405 nattr = *attr; 406 else 407 return EINVAL; 408 409 /* Fetch misc. attributes from the attr structure. */ 410 name = NULL; 411 if ((p = nattr.pta_private) != NULL) 412 if (p->ptap_name[0] != '\0') 413 if ((name = strdup(p->ptap_name)) == NULL) 414 return ENOMEM; 415 416 newthread = NULL; 417 418 /* 419 * Try to reclaim a dead thread. 420 */ 421 if (!PTQ_EMPTY(&pthread__deadqueue)) { 422 pthread_mutex_lock(&pthread__deadqueue_lock); 423 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 424 /* Still running? */ 425 if (newthread->pt_lwpctl->lc_curcpu == 426 LWPCTL_CPU_EXITED || 427 (_lwp_kill(newthread->pt_lid, 0) == -1 && 428 errno == ESRCH)) 429 break; 430 } 431 if (newthread) 432 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 433 pthread_mutex_unlock(&pthread__deadqueue_lock); 434 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 435 if (newthread && newthread->pt_tls) { 436 _rtld_tls_free(newthread->pt_tls); 437 newthread->pt_tls = NULL; 438 } 439 #endif 440 } 441 442 /* 443 * If necessary set up a stack, allocate space for a pthread_st, 444 * and initialize it. 445 */ 446 if (newthread == NULL) { 447 newthread = malloc(sizeof(*newthread)); 448 if (newthread == NULL) { 449 free(name); 450 return ENOMEM; 451 } 452 newthread->pt_stack_allocated = false; 453 454 if (pthread__getstack(newthread, attr)) { 455 free(newthread); 456 free(name); 457 return ENOMEM; 458 } 459 460 /* This is used only when creating the thread. */ 461 _INITCONTEXT_U(&newthread->pt_uc); 462 newthread->pt_uc.uc_stack = newthread->pt_stack; 463 newthread->pt_uc.uc_link = NULL; 464 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 465 newthread->pt_tls = NULL; 466 #endif 467 468 /* Add to list of all threads. */ 469 pthread_rwlock_wrlock(&pthread__alltree_lock); 470 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 471 RB_INSERT(__pthread__alltree, &pthread__alltree, newthread); 472 pthread_rwlock_unlock(&pthread__alltree_lock); 473 474 /* Will be reset by the thread upon exit. */ 475 pthread__initthread(newthread); 476 } else { 477 if (pthread__getstack(newthread, attr)) { 478 pthread_mutex_lock(&pthread__deadqueue_lock); 479 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 480 pthread_mutex_unlock(&pthread__deadqueue_lock); 481 return ENOMEM; 482 } 483 _INITCONTEXT_U(&newthread->pt_uc); 484 newthread->pt_uc.uc_stack = newthread->pt_stack; 485 newthread->pt_uc.uc_link = NULL; 486 } 487 488 /* 489 * Create the new LWP. 490 */ 491 pthread__scrubthread(newthread, name, nattr.pta_flags); 492 newthread->pt_func = startfunc; 493 newthread->pt_arg = arg; 494 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 495 private_area = newthread->pt_tls = _rtld_tls_allocate(); 496 newthread->pt_tls->tcb_pthread = newthread; 497 #else 498 private_area = newthread; 499 #endif 500 501 _lwp_makecontext(&newthread->pt_uc, pthread__create_tramp, 502 newthread, private_area, newthread->pt_stack.ss_sp, 503 newthread->pt_stack.ss_size); 504 505 flag = LWP_DETACHED; 506 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 507 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 508 flag |= LWP_SUSPENDED; 509 ret = _lwp_create(&newthread->pt_uc, flag, &newthread->pt_lid); 510 if (ret != 0) { 511 pthread_mutex_lock(&newthread->pt_lock); 512 /* Will unlock and free name. */ 513 pthread__reap(newthread); 514 return ret; 515 } 516 517 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 518 if (p != NULL) { 519 (void)pthread_setschedparam(newthread, p->ptap_policy, 520 &p->ptap_sp); 521 } 522 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 523 (void)_lwp_continue(newthread->pt_lid); 524 } 525 } 526 527 *thread = newthread; 528 529 return 0; 530 } 531 532 533 __dead static void 534 pthread__create_tramp(void *cookie) 535 { 536 pthread_t self; 537 void *retval; 538 539 self = cookie; 540 541 /* 542 * Throw away some stack in a feeble attempt to reduce cache 543 * thrash. May help for SMT processors. XXX We should not 544 * be allocating stacks on fixed 2MB boundaries. Needs a 545 * thread register or decent thread local storage. 546 * 547 * Note that we may race with the kernel in _lwp_create(), 548 * and so pt_lid can be unset at this point, but we don't 549 * care. 550 */ 551 (void)alloca(((unsigned)self->pt_lid & 7) << 8); 552 553 if (self->pt_name != NULL) { 554 pthread_mutex_lock(&self->pt_lock); 555 if (self->pt_name != NULL) 556 (void)_lwp_setname(0, self->pt_name); 557 pthread_mutex_unlock(&self->pt_lock); 558 } 559 560 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 561 err(1, "_lwp_ctl"); 562 } 563 564 retval = (*self->pt_func)(self->pt_arg); 565 566 pthread_exit(retval); 567 568 /*NOTREACHED*/ 569 pthread__abort(); 570 } 571 572 int 573 pthread_suspend_np(pthread_t thread) 574 { 575 pthread_t self; 576 577 self = pthread__self(); 578 if (self == thread) { 579 return EDEADLK; 580 } 581 if (pthread__find(thread) != 0) 582 return ESRCH; 583 if (_lwp_suspend(thread->pt_lid) == 0) 584 return 0; 585 return errno; 586 } 587 588 int 589 pthread_resume_np(pthread_t thread) 590 { 591 592 if (pthread__find(thread) != 0) 593 return ESRCH; 594 if (_lwp_continue(thread->pt_lid) == 0) 595 return 0; 596 return errno; 597 } 598 599 void 600 pthread_exit(void *retval) 601 { 602 pthread_t self; 603 struct pt_clean_t *cleanup; 604 char *name; 605 606 self = pthread__self(); 607 608 /* Disable cancellability. */ 609 pthread_mutex_lock(&self->pt_lock); 610 self->pt_flags |= PT_FLAG_CS_DISABLED; 611 self->pt_cancel = 0; 612 613 /* Call any cancellation cleanup handlers */ 614 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 615 pthread_mutex_unlock(&self->pt_lock); 616 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 617 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 618 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 619 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 620 } 621 pthread_mutex_lock(&self->pt_lock); 622 } 623 624 /* Perform cleanup of thread-specific data */ 625 pthread__destroy_tsd(self); 626 627 /* Signal our exit. */ 628 self->pt_exitval = retval; 629 if (self->pt_flags & PT_FLAG_DETACHED) { 630 self->pt_state = PT_STATE_DEAD; 631 name = self->pt_name; 632 self->pt_name = NULL; 633 pthread_mutex_unlock(&self->pt_lock); 634 if (name != NULL) 635 free(name); 636 pthread_mutex_lock(&pthread__deadqueue_lock); 637 PTQ_INSERT_TAIL(&pthread__deadqueue, self, pt_deadq); 638 pthread_mutex_unlock(&pthread__deadqueue_lock); 639 _lwp_exit(); 640 } else { 641 self->pt_state = PT_STATE_ZOMBIE; 642 pthread_cond_broadcast(&self->pt_joiners); 643 pthread_mutex_unlock(&self->pt_lock); 644 /* Note: name will be freed by the joiner. */ 645 _lwp_exit(); 646 } 647 648 /*NOTREACHED*/ 649 pthread__abort(); 650 exit(1); 651 } 652 653 654 int 655 pthread_join(pthread_t thread, void **valptr) 656 { 657 pthread_t self; 658 int error; 659 660 self = pthread__self(); 661 662 if (pthread__find(thread) != 0) 663 return ESRCH; 664 665 if (thread->pt_magic != PT_MAGIC) 666 return EINVAL; 667 668 if (thread == self) 669 return EDEADLK; 670 671 self->pt_droplock = &thread->pt_lock; 672 pthread_mutex_lock(&thread->pt_lock); 673 for (;;) { 674 if (thread->pt_state == PT_STATE_ZOMBIE) 675 break; 676 if (thread->pt_state == PT_STATE_DEAD) { 677 pthread_mutex_unlock(&thread->pt_lock); 678 self->pt_droplock = NULL; 679 return ESRCH; 680 } 681 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 682 pthread_mutex_unlock(&thread->pt_lock); 683 self->pt_droplock = NULL; 684 return EINVAL; 685 } 686 error = pthread_cond_wait(&thread->pt_joiners, 687 &thread->pt_lock); 688 if (error != 0) { 689 pthread__errorfunc(__FILE__, __LINE__, 690 __func__, "unexpected return from cond_wait()"); 691 } 692 693 } 694 pthread__testcancel(self); 695 if (valptr != NULL) 696 *valptr = thread->pt_exitval; 697 /* pthread__reap() will drop the lock. */ 698 pthread__reap(thread); 699 self->pt_droplock = NULL; 700 701 return 0; 702 } 703 704 static void 705 pthread__reap(pthread_t thread) 706 { 707 char *name; 708 709 name = thread->pt_name; 710 thread->pt_name = NULL; 711 thread->pt_state = PT_STATE_DEAD; 712 pthread_mutex_unlock(&thread->pt_lock); 713 714 pthread_mutex_lock(&pthread__deadqueue_lock); 715 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 716 pthread_mutex_unlock(&pthread__deadqueue_lock); 717 718 if (name != NULL) 719 free(name); 720 } 721 722 int 723 pthread_equal(pthread_t t1, pthread_t t2) 724 { 725 726 /* Nothing special here. */ 727 return (t1 == t2); 728 } 729 730 731 int 732 pthread_detach(pthread_t thread) 733 { 734 735 if (pthread__find(thread) != 0) 736 return ESRCH; 737 738 if (thread->pt_magic != PT_MAGIC) 739 return EINVAL; 740 741 pthread_mutex_lock(&thread->pt_lock); 742 thread->pt_flags |= PT_FLAG_DETACHED; 743 if (thread->pt_state == PT_STATE_ZOMBIE) { 744 /* pthread__reap() will drop the lock. */ 745 pthread__reap(thread); 746 } else { 747 /* 748 * Not valid for threads to be waiting in 749 * pthread_join() (there are intractable 750 * sync issues from the application 751 * perspective), but give those threads 752 * a chance anyway. 753 */ 754 pthread_cond_broadcast(&thread->pt_joiners); 755 pthread_mutex_unlock(&thread->pt_lock); 756 } 757 758 return 0; 759 } 760 761 762 int 763 pthread_getname_np(pthread_t thread, char *name, size_t len) 764 { 765 766 if (pthread__find(thread) != 0) 767 return ESRCH; 768 769 if (thread->pt_magic != PT_MAGIC) 770 return EINVAL; 771 772 pthread_mutex_lock(&thread->pt_lock); 773 if (thread->pt_name == NULL) 774 name[0] = '\0'; 775 else 776 strlcpy(name, thread->pt_name, len); 777 pthread_mutex_unlock(&thread->pt_lock); 778 779 return 0; 780 } 781 782 783 int 784 pthread_setname_np(pthread_t thread, const char *name, void *arg) 785 { 786 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 787 int namelen; 788 789 if (pthread__find(thread) != 0) 790 return ESRCH; 791 792 if (thread->pt_magic != PT_MAGIC) 793 return EINVAL; 794 795 namelen = snprintf(newname, sizeof(newname), name, arg); 796 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 797 return EINVAL; 798 799 cp = strdup(newname); 800 if (cp == NULL) 801 return ENOMEM; 802 803 pthread_mutex_lock(&thread->pt_lock); 804 oldname = thread->pt_name; 805 thread->pt_name = cp; 806 (void)_lwp_setname(thread->pt_lid, cp); 807 pthread_mutex_unlock(&thread->pt_lock); 808 809 if (oldname != NULL) 810 free(oldname); 811 812 return 0; 813 } 814 815 816 817 /* 818 * XXX There should be a way for applications to use the efficent 819 * inline version, but there are opacity/namespace issues. 820 */ 821 pthread_t 822 pthread_self(void) 823 { 824 825 return pthread__self(); 826 } 827 828 829 int 830 pthread_cancel(pthread_t thread) 831 { 832 833 if (pthread__find(thread) != 0) 834 return ESRCH; 835 pthread_mutex_lock(&thread->pt_lock); 836 thread->pt_flags |= PT_FLAG_CS_PENDING; 837 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 838 thread->pt_cancel = 1; 839 pthread_mutex_unlock(&thread->pt_lock); 840 _lwp_wakeup(thread->pt_lid); 841 } else 842 pthread_mutex_unlock(&thread->pt_lock); 843 844 return 0; 845 } 846 847 848 int 849 pthread_setcancelstate(int state, int *oldstate) 850 { 851 pthread_t self; 852 int retval; 853 854 self = pthread__self(); 855 retval = 0; 856 857 pthread_mutex_lock(&self->pt_lock); 858 859 if (oldstate != NULL) { 860 if (self->pt_flags & PT_FLAG_CS_DISABLED) 861 *oldstate = PTHREAD_CANCEL_DISABLE; 862 else 863 *oldstate = PTHREAD_CANCEL_ENABLE; 864 } 865 866 if (state == PTHREAD_CANCEL_DISABLE) { 867 self->pt_flags |= PT_FLAG_CS_DISABLED; 868 if (self->pt_cancel) { 869 self->pt_flags |= PT_FLAG_CS_PENDING; 870 self->pt_cancel = 0; 871 } 872 } else if (state == PTHREAD_CANCEL_ENABLE) { 873 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 874 /* 875 * If a cancellation was requested while cancellation 876 * was disabled, note that fact for future 877 * cancellation tests. 878 */ 879 if (self->pt_flags & PT_FLAG_CS_PENDING) { 880 self->pt_cancel = 1; 881 /* This is not a deferred cancellation point. */ 882 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 883 pthread_mutex_unlock(&self->pt_lock); 884 pthread__cancelled(); 885 } 886 } 887 } else 888 retval = EINVAL; 889 890 pthread_mutex_unlock(&self->pt_lock); 891 892 return retval; 893 } 894 895 896 int 897 pthread_setcanceltype(int type, int *oldtype) 898 { 899 pthread_t self; 900 int retval; 901 902 self = pthread__self(); 903 retval = 0; 904 905 pthread_mutex_lock(&self->pt_lock); 906 907 if (oldtype != NULL) { 908 if (self->pt_flags & PT_FLAG_CS_ASYNC) 909 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 910 else 911 *oldtype = PTHREAD_CANCEL_DEFERRED; 912 } 913 914 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 915 self->pt_flags |= PT_FLAG_CS_ASYNC; 916 if (self->pt_cancel) { 917 pthread_mutex_unlock(&self->pt_lock); 918 pthread__cancelled(); 919 } 920 } else if (type == PTHREAD_CANCEL_DEFERRED) 921 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 922 else 923 retval = EINVAL; 924 925 pthread_mutex_unlock(&self->pt_lock); 926 927 return retval; 928 } 929 930 931 void 932 pthread_testcancel(void) 933 { 934 pthread_t self; 935 936 self = pthread__self(); 937 if (self->pt_cancel) 938 pthread__cancelled(); 939 } 940 941 942 /* 943 * POSIX requires that certain functions return an error rather than 944 * invoking undefined behavior even when handed completely bogus 945 * pthread_t values, e.g. stack garbage or (pthread_t)666. This 946 * utility routine searches the list of threads for the pthread_t 947 * value without dereferencing it. 948 */ 949 int 950 pthread__find(pthread_t id) 951 { 952 pthread_t target; 953 954 pthread_rwlock_rdlock(&pthread__alltree_lock); 955 /* LINTED */ 956 target = RB_FIND(__pthread__alltree, &pthread__alltree, id); 957 pthread_rwlock_unlock(&pthread__alltree_lock); 958 959 if (target == NULL || target->pt_state == PT_STATE_DEAD) 960 return ESRCH; 961 962 return 0; 963 } 964 965 966 void 967 pthread__testcancel(pthread_t self) 968 { 969 970 if (self->pt_cancel) 971 pthread__cancelled(); 972 } 973 974 975 void 976 pthread__cancelled(void) 977 { 978 pthread_mutex_t *droplock; 979 pthread_t self; 980 981 self = pthread__self(); 982 droplock = self->pt_droplock; 983 self->pt_droplock = NULL; 984 985 if (droplock != NULL && pthread_mutex_held_np(droplock)) 986 pthread_mutex_unlock(droplock); 987 988 pthread_exit(PTHREAD_CANCELED); 989 } 990 991 992 void 993 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 994 { 995 pthread_t self; 996 struct pt_clean_t *entry; 997 998 self = pthread__self(); 999 entry = store; 1000 entry->ptc_cleanup = cleanup; 1001 entry->ptc_arg = arg; 1002 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1003 } 1004 1005 1006 void 1007 pthread__cleanup_pop(int ex, void *store) 1008 { 1009 pthread_t self; 1010 struct pt_clean_t *entry; 1011 1012 self = pthread__self(); 1013 entry = store; 1014 1015 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1016 if (ex) 1017 (*entry->ptc_cleanup)(entry->ptc_arg); 1018 } 1019 1020 1021 int * 1022 pthread__errno(void) 1023 { 1024 pthread_t self; 1025 1026 self = pthread__self(); 1027 1028 return &(self->pt_errno); 1029 } 1030 1031 ssize_t _sys_write(int, const void *, size_t); 1032 1033 void 1034 pthread__assertfunc(const char *file, int line, const char *function, 1035 const char *expr) 1036 { 1037 char buf[1024]; 1038 int len; 1039 1040 /* 1041 * snprintf should not acquire any locks, or we could 1042 * end up deadlocked if the assert caller held locks. 1043 */ 1044 len = snprintf(buf, 1024, 1045 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1046 expr, file, line, 1047 function ? ", function \"" : "", 1048 function ? function : "", 1049 function ? "\"" : ""); 1050 1051 _sys_write(STDERR_FILENO, buf, (size_t)len); 1052 (void)kill(getpid(), SIGABRT); 1053 1054 _exit(1); 1055 } 1056 1057 1058 void 1059 pthread__errorfunc(const char *file, int line, const char *function, 1060 const char *msg) 1061 { 1062 char buf[1024]; 1063 size_t len; 1064 1065 if (pthread__diagassert == 0) 1066 return; 1067 1068 /* 1069 * snprintf should not acquire any locks, or we could 1070 * end up deadlocked if the assert caller held locks. 1071 */ 1072 len = snprintf(buf, 1024, 1073 "%s: Error detected by libpthread: %s.\n" 1074 "Detected by file \"%s\", line %d%s%s%s.\n" 1075 "See pthread(3) for information.\n", 1076 getprogname(), msg, file, line, 1077 function ? ", function \"" : "", 1078 function ? function : "", 1079 function ? "\"" : ""); 1080 1081 if (pthread__diagassert & DIAGASSERT_STDERR) 1082 _sys_write(STDERR_FILENO, buf, len); 1083 1084 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1085 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1086 1087 if (pthread__diagassert & DIAGASSERT_ABORT) { 1088 (void)kill(getpid(), SIGABRT); 1089 _exit(1); 1090 } 1091 } 1092 1093 /* 1094 * Thread park/unpark operations. The kernel operations are 1095 * modelled after a brief description from "Multithreading in 1096 * the Solaris Operating Environment": 1097 * 1098 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1099 */ 1100 1101 #define OOPS(msg) \ 1102 pthread__errorfunc(__FILE__, __LINE__, __func__, msg) 1103 1104 int 1105 pthread__park(pthread_t self, pthread_mutex_t *lock, 1106 pthread_queue_t *queue, const struct timespec *abstime, 1107 int cancelpt, const void *hint) 1108 { 1109 int rv, error; 1110 void *obj; 1111 1112 /* 1113 * For non-interlocked release of mutexes we need a store 1114 * barrier before incrementing pt_blocking away from zero. 1115 * This is provided by pthread_mutex_unlock(). 1116 */ 1117 self->pt_willpark = 1; 1118 pthread_mutex_unlock(lock); 1119 self->pt_willpark = 0; 1120 self->pt_blocking++; 1121 1122 /* 1123 * Wait until we are awoken by a pending unpark operation, 1124 * a signal, an unpark posted after we have gone asleep, 1125 * or an expired timeout. 1126 * 1127 * It is fine to test the value of pt_sleepobj without 1128 * holding any locks, because: 1129 * 1130 * o Only the blocking thread (this thread) ever sets them 1131 * to a non-NULL value. 1132 * 1133 * o Other threads may set them NULL, but if they do so they 1134 * must also make this thread return from _lwp_park. 1135 * 1136 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1137 * calls and all make use of spinlocks in the kernel. So 1138 * these system calls act as full memory barriers, and will 1139 * ensure that the calling CPU's store buffers are drained. 1140 * In combination with the spinlock release before unpark, 1141 * this means that modification of pt_sleepobj/onq by another 1142 * thread will become globally visible before that thread 1143 * schedules an unpark operation on this thread. 1144 * 1145 * Note: the test in the while() statement dodges the park op if 1146 * we have already been awoken, unless there is another thread to 1147 * awaken. This saves a syscall - if we were already awakened, 1148 * the next call to _lwp_park() would need to return early in order 1149 * to eat the previous wakeup. 1150 */ 1151 rv = 0; 1152 do { 1153 /* 1154 * If we deferred unparking a thread, arrange to 1155 * have _lwp_park() restart it before blocking. 1156 */ 1157 error = _lwp_park(abstime, self->pt_unpark, hint, hint); 1158 self->pt_unpark = 0; 1159 if (error != 0) { 1160 switch (rv = errno) { 1161 case EINTR: 1162 case EALREADY: 1163 rv = 0; 1164 break; 1165 case ETIMEDOUT: 1166 break; 1167 default: 1168 OOPS("_lwp_park failed"); 1169 break; 1170 } 1171 } 1172 /* Check for cancellation. */ 1173 if (cancelpt && self->pt_cancel) 1174 rv = EINTR; 1175 } while (self->pt_sleepobj != NULL && rv == 0); 1176 1177 /* 1178 * If we have been awoken early but are still on the queue, 1179 * then remove ourself. Again, it's safe to do the test 1180 * without holding any locks. 1181 */ 1182 if (__predict_false(self->pt_sleepobj != NULL)) { 1183 pthread_mutex_lock(lock); 1184 if ((obj = self->pt_sleepobj) != NULL) { 1185 PTQ_REMOVE(queue, self, pt_sleep); 1186 self->pt_sleepobj = NULL; 1187 if (obj != NULL && self->pt_early != NULL) 1188 (*self->pt_early)(obj); 1189 } 1190 pthread_mutex_unlock(lock); 1191 } 1192 self->pt_early = NULL; 1193 self->pt_blocking--; 1194 membar_sync(); 1195 1196 return rv; 1197 } 1198 1199 void 1200 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1201 pthread_mutex_t *interlock) 1202 { 1203 pthread_t target; 1204 u_int max; 1205 size_t nwaiters; 1206 1207 max = pthread__unpark_max; 1208 nwaiters = self->pt_nwaiters; 1209 target = PTQ_FIRST(queue); 1210 if (nwaiters == max) { 1211 /* Overflow. */ 1212 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1213 __UNVOLATILE(&interlock->ptm_waiters)); 1214 nwaiters = 0; 1215 } 1216 target->pt_sleepobj = NULL; 1217 self->pt_waiters[nwaiters++] = target->pt_lid; 1218 PTQ_REMOVE(queue, target, pt_sleep); 1219 self->pt_nwaiters = nwaiters; 1220 pthread__mutex_deferwake(self, interlock); 1221 } 1222 1223 void 1224 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1225 pthread_mutex_t *interlock) 1226 { 1227 pthread_t target; 1228 u_int max; 1229 size_t nwaiters; 1230 1231 max = pthread__unpark_max; 1232 nwaiters = self->pt_nwaiters; 1233 PTQ_FOREACH(target, queue, pt_sleep) { 1234 if (nwaiters == max) { 1235 /* Overflow. */ 1236 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1237 __UNVOLATILE(&interlock->ptm_waiters)); 1238 nwaiters = 0; 1239 } 1240 target->pt_sleepobj = NULL; 1241 self->pt_waiters[nwaiters++] = target->pt_lid; 1242 } 1243 self->pt_nwaiters = nwaiters; 1244 PTQ_INIT(queue); 1245 pthread__mutex_deferwake(self, interlock); 1246 } 1247 1248 #undef OOPS 1249 1250 static void 1251 pthread__initmainstack(void) 1252 { 1253 struct rlimit slimit; 1254 const AuxInfo *aux; 1255 size_t size; 1256 1257 _DIAGASSERT(_dlauxinfo() != NULL); 1258 1259 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1260 err(1, "Couldn't get stack resource consumption limits"); 1261 size = slimit.rlim_cur; 1262 pthread__main.pt_stack.ss_size = size; 1263 1264 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1265 if (aux->a_type == AT_STACKBASE) { 1266 pthread__main.pt_stack.ss_sp = (void *)aux->a_v; 1267 #ifdef __MACHINE_STACK_GROWS_UP 1268 pthread__main.pt_stack.ss_sp = (void *)aux->a_v; 1269 #else 1270 pthread__main.pt_stack.ss_sp = (char *)aux->a_v - size; 1271 #endif 1272 break; 1273 } 1274 } 1275 } 1276 1277 /* 1278 * Set up the slightly special stack for the "initial" thread, which 1279 * runs on the normal system stack, and thus gets slightly different 1280 * treatment. 1281 */ 1282 static void 1283 pthread__initmain(pthread_t *newt) 1284 { 1285 char *value; 1286 1287 pthread__initmainstack(); 1288 1289 value = pthread__getenv("PTHREAD_STACKSIZE"); 1290 if (value != NULL) { 1291 pthread__stacksize = atoi(value) * 1024; 1292 if (pthread__stacksize > pthread__main.pt_stack.ss_size) 1293 pthread__stacksize = pthread__main.pt_stack.ss_size; 1294 } 1295 if (pthread__stacksize == 0) 1296 pthread__stacksize = pthread__main.pt_stack.ss_size; 1297 pthread__stacksize += pthread__pagesize - 1; 1298 pthread__stacksize &= ~(pthread__pagesize - 1); 1299 if (pthread__stacksize < 4 * pthread__pagesize) 1300 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1301 4 * pthread__pagesize / 1024); 1302 1303 *newt = &pthread__main; 1304 #ifdef __HAVE___LWP_GETTCB_FAST 1305 pthread__main.pt_tls = __lwp_gettcb_fast(); 1306 #else 1307 pthread__main.pt_tls = _lwp_getprivate(); 1308 #endif 1309 pthread__main.pt_tls->tcb_pthread = &pthread__main; 1310 } 1311 1312 #ifndef lint 1313 static int 1314 pthread__cmp(struct __pthread_st *a, struct __pthread_st *b) 1315 { 1316 1317 if ((uintptr_t)a < (uintptr_t)b) 1318 return (-1); 1319 else if (a == b) 1320 return 0; 1321 else 1322 return 1; 1323 } 1324 RB_GENERATE_STATIC(__pthread__alltree, __pthread_st, pt_alltree, pthread__cmp) 1325 #endif 1326 1327 /* Because getenv() wants to use locks. */ 1328 char * 1329 pthread__getenv(const char *name) 1330 { 1331 extern char **environ; 1332 size_t l_name, offset; 1333 1334 l_name = strlen(name); 1335 for (offset = 0; environ[offset] != NULL; offset++) { 1336 if (strncmp(name, environ[offset], l_name) == 0 && 1337 environ[offset][l_name] == '=') { 1338 return environ[offset] + l_name + 1; 1339 } 1340 } 1341 1342 return NULL; 1343 } 1344 1345 pthread_mutex_t * 1346 pthread__hashlock(volatile const void *p) 1347 { 1348 uintptr_t v; 1349 1350 v = (uintptr_t)p; 1351 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1352 } 1353 1354 int 1355 pthread__checkpri(int pri) 1356 { 1357 static int havepri; 1358 static long min, max; 1359 1360 if (!havepri) { 1361 min = sysconf(_SC_SCHED_PRI_MIN); 1362 max = sysconf(_SC_SCHED_PRI_MAX); 1363 havepri = 1; 1364 } 1365 return (pri < min || pri > max) ? EINVAL : 0; 1366 } 1367