1 /* $NetBSD: pthread.c,v 1.133 2012/03/22 20:01:18 drochner Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nathan J. Williams and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __RCSID("$NetBSD: pthread.c,v 1.133 2012/03/22 20:01:18 drochner Exp $"); 34 35 #define __EXPOSE_STACK 1 36 37 #include <sys/param.h> 38 #include <sys/exec_elf.h> 39 #include <sys/mman.h> 40 #include <sys/sysctl.h> 41 #include <sys/lwpctl.h> 42 #include <sys/tls.h> 43 44 #include <assert.h> 45 #include <dlfcn.h> 46 #include <err.h> 47 #include <errno.h> 48 #include <lwp.h> 49 #include <signal.h> 50 #include <stdio.h> 51 #include <stdlib.h> 52 #include <string.h> 53 #include <syslog.h> 54 #include <ucontext.h> 55 #include <unistd.h> 56 #include <sched.h> 57 58 #include "pthread.h" 59 #include "pthread_int.h" 60 61 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 62 RB_HEAD(__pthread__alltree, __pthread_st) pthread__alltree; 63 64 #ifndef lint 65 static int pthread__cmp(struct __pthread_st *, struct __pthread_st *); 66 RB_PROTOTYPE_STATIC(__pthread__alltree, __pthread_st, pt_alltree, pthread__cmp) 67 #endif 68 69 static void pthread__create_tramp(void *); 70 static void pthread__initthread(pthread_t); 71 static void pthread__scrubthread(pthread_t, char *, int); 72 static void pthread__initmain(pthread_t *); 73 static void pthread__fork_callback(void); 74 static void pthread__reap(pthread_t); 75 static void pthread__child_callback(void); 76 static void pthread__start(void); 77 78 void pthread__init(void); 79 80 int pthread__started; 81 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 82 pthread_queue_t pthread__deadqueue; 83 pthread_queue_t pthread__allqueue; 84 85 static pthread_attr_t pthread_default_attr; 86 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 87 static pthread_t pthread__first; 88 89 enum { 90 DIAGASSERT_ABORT = 1<<0, 91 DIAGASSERT_STDERR = 1<<1, 92 DIAGASSERT_SYSLOG = 1<<2 93 }; 94 95 static int pthread__diagassert; 96 97 int pthread__concurrency; 98 int pthread__nspins; 99 int pthread__unpark_max = PTHREAD__UNPARK_MAX; 100 int pthread__dbg; /* set by libpthread_dbg if active */ 101 102 /* 103 * We have to initialize the pthread_stack* variables here because 104 * mutexes are used before pthread_init() and thus pthread__initmain() 105 * are called. Since mutexes only save the stack pointer and not a 106 * pointer to the thread data, it is safe to change the mapping from 107 * stack pointer to thread data afterwards. 108 */ 109 size_t pthread__stacksize; 110 size_t pthread__pagesize; 111 static struct __pthread_st pthread__main; 112 113 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 114 115 __strong_alias(__libc_thr_self,pthread_self) 116 __strong_alias(__libc_thr_create,pthread_create) 117 __strong_alias(__libc_thr_exit,pthread_exit) 118 __strong_alias(__libc_thr_errno,pthread__errno) 119 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 120 __strong_alias(__libc_thr_equal,pthread_equal) 121 __strong_alias(__libc_thr_init,pthread__init) 122 123 /* 124 * Static library kludge. Place a reference to a symbol any library 125 * file which does not already have a reference here. 126 */ 127 extern int pthread__cancel_stub_binder; 128 129 void *pthread__static_lib_binder[] = { 130 &pthread__cancel_stub_binder, 131 pthread_cond_init, 132 pthread_mutex_init, 133 pthread_rwlock_init, 134 pthread_barrier_init, 135 pthread_key_create, 136 pthread_setspecific, 137 }; 138 139 #define NHASHLOCK 64 140 141 static union hashlock { 142 pthread_mutex_t mutex; 143 char pad[64]; 144 } hashlocks[NHASHLOCK] __aligned(64); 145 146 /* 147 * This needs to be started by the library loading code, before main() 148 * gets to run, for various things that use the state of the initial thread 149 * to work properly (thread-specific data is an application-visible example; 150 * spinlock counts for mutexes is an internal example). 151 */ 152 void 153 pthread__init(void) 154 { 155 pthread_t first; 156 char *p; 157 int i, mib[2]; 158 size_t len; 159 extern int __isthreaded; 160 161 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 162 163 mib[0] = CTL_HW; 164 mib[1] = HW_NCPU; 165 166 len = sizeof(pthread__concurrency); 167 if (sysctl(mib, 2, &pthread__concurrency, &len, NULL, 0) == -1) 168 err(1, "sysctl(hw.ncpu"); 169 170 mib[0] = CTL_KERN; 171 mib[1] = KERN_OSREV; 172 173 /* Initialize locks first; they're needed elsewhere. */ 174 pthread__lockprim_init(); 175 for (i = 0; i < NHASHLOCK; i++) { 176 pthread_mutex_init(&hashlocks[i].mutex, NULL); 177 } 178 179 /* Fetch parameters. */ 180 i = (int)_lwp_unpark_all(NULL, 0, NULL); 181 if (i == -1) 182 err(1, "_lwp_unpark_all"); 183 if (i < pthread__unpark_max) 184 pthread__unpark_max = i; 185 186 /* Basic data structure setup */ 187 pthread_attr_init(&pthread_default_attr); 188 PTQ_INIT(&pthread__allqueue); 189 PTQ_INIT(&pthread__deadqueue); 190 RB_INIT(&pthread__alltree); 191 192 /* Create the thread structure corresponding to main() */ 193 pthread__initmain(&first); 194 pthread__initthread(first); 195 pthread__scrubthread(first, NULL, 0); 196 197 first->pt_lid = _lwp_self(); 198 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 199 RB_INSERT(__pthread__alltree, &pthread__alltree, first); 200 201 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 202 err(1, "_lwp_ctl"); 203 } 204 205 /* Start subsystems */ 206 PTHREAD_MD_INIT 207 208 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 209 switch (*p) { 210 case 'a': 211 pthread__diagassert |= DIAGASSERT_ABORT; 212 break; 213 case 'A': 214 pthread__diagassert &= ~DIAGASSERT_ABORT; 215 break; 216 case 'e': 217 pthread__diagassert |= DIAGASSERT_STDERR; 218 break; 219 case 'E': 220 pthread__diagassert &= ~DIAGASSERT_STDERR; 221 break; 222 case 'l': 223 pthread__diagassert |= DIAGASSERT_SYSLOG; 224 break; 225 case 'L': 226 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 227 break; 228 } 229 } 230 231 /* Tell libc that we're here and it should role-play accordingly. */ 232 pthread__first = first; 233 pthread_atfork(NULL, NULL, pthread__fork_callback); 234 __isthreaded = 1; 235 } 236 237 static void 238 pthread__fork_callback(void) 239 { 240 struct __pthread_st *self; 241 242 /* lwpctl state is not copied across fork. */ 243 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &pthread__first->pt_lwpctl)) { 244 err(1, "_lwp_ctl"); 245 } 246 self = pthread__self(); 247 self->pt_lid = _lwp_self(); 248 } 249 250 static void 251 pthread__child_callback(void) 252 { 253 254 /* 255 * Clean up data structures that a forked child process might 256 * trip over. Note that if threads have been created (causing 257 * this handler to be registered) the standards say that the 258 * child will trigger undefined behavior if it makes any 259 * pthread_* calls (or any other calls that aren't 260 * async-signal-safe), so we don't really have to clean up 261 * much. Anything that permits some pthread_* calls to work is 262 * merely being polite. 263 */ 264 pthread__started = 0; 265 } 266 267 static void 268 pthread__start(void) 269 { 270 271 /* 272 * Per-process timers are cleared by fork(); despite the 273 * various restrictions on fork() and threads, it's legal to 274 * fork() before creating any threads. 275 */ 276 pthread_atfork(NULL, NULL, pthread__child_callback); 277 } 278 279 280 /* General-purpose thread data structure sanitization. */ 281 /* ARGSUSED */ 282 static void 283 pthread__initthread(pthread_t t) 284 { 285 286 t->pt_self = t; 287 t->pt_magic = PT_MAGIC; 288 t->pt_willpark = 0; 289 t->pt_unpark = 0; 290 t->pt_nwaiters = 0; 291 t->pt_sleepobj = NULL; 292 t->pt_signalled = 0; 293 t->pt_havespecific = 0; 294 t->pt_early = NULL; 295 t->pt_lwpctl = &pthread__dummy_lwpctl; 296 t->pt_blocking = 0; 297 t->pt_droplock = NULL; 298 299 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 300 pthread_mutex_init(&t->pt_lock, NULL); 301 PTQ_INIT(&t->pt_cleanup_stack); 302 pthread_cond_init(&t->pt_joiners, NULL); 303 memset(&t->pt_specific, 0, sizeof(t->pt_specific)); 304 } 305 306 static void 307 pthread__scrubthread(pthread_t t, char *name, int flags) 308 { 309 310 t->pt_state = PT_STATE_RUNNING; 311 t->pt_exitval = NULL; 312 t->pt_flags = flags; 313 t->pt_cancel = 0; 314 t->pt_errno = 0; 315 t->pt_name = name; 316 t->pt_lid = 0; 317 } 318 319 static int 320 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 321 { 322 void *stackbase, *stackbase2, *redzone; 323 size_t stacksize, guardsize; 324 bool allocated; 325 326 if (attr != NULL) { 327 pthread_attr_getstack(attr, &stackbase, &stacksize); 328 } else { 329 stackbase = NULL; 330 stacksize = 0; 331 } 332 if (stacksize == 0) 333 stacksize = pthread__stacksize; 334 335 if (newthread->pt_stack_allocated) { 336 if (stackbase == NULL && 337 newthread->pt_stack.ss_size == stacksize) 338 return 0; 339 stackbase2 = newthread->pt_stack.ss_sp; 340 #ifndef __MACHINE_STACK_GROWS_UP 341 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 342 #endif 343 munmap(stackbase2, 344 newthread->pt_stack.ss_size + newthread->pt_guardsize); 345 newthread->pt_stack.ss_sp = NULL; 346 newthread->pt_stack.ss_size = 0; 347 newthread->pt_guardsize = 0; 348 newthread->pt_stack_allocated = false; 349 } 350 351 newthread->pt_stack_allocated = false; 352 353 if (stackbase == NULL) { 354 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 355 guardsize = pthread__pagesize; 356 stackbase = mmap(NULL, stacksize + guardsize, 357 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 358 if (stackbase == MAP_FAILED) 359 return ENOMEM; 360 allocated = true; 361 } else { 362 guardsize = 0; 363 allocated = false; 364 } 365 #ifdef __MACHINE_STACK_GROWS_UP 366 redzone = (char *)stackbase + stacksize; 367 stackbase2 = (char *)stackbase; 368 #else 369 redzone = (char *)stackbase; 370 stackbase2 = (char *)stackbase + guardsize; 371 #endif 372 if (allocated && guardsize && 373 mprotect(redzone, guardsize, PROT_NONE) == -1) { 374 munmap(stackbase, stacksize + guardsize); 375 return EPERM; 376 } 377 newthread->pt_stack.ss_size = stacksize; 378 newthread->pt_stack.ss_sp = stackbase2; 379 newthread->pt_guardsize = guardsize; 380 newthread->pt_stack_allocated = allocated; 381 return 0; 382 } 383 384 int 385 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 386 void *(*startfunc)(void *), void *arg) 387 { 388 pthread_t newthread; 389 pthread_attr_t nattr; 390 struct pthread_attr_private *p; 391 char * volatile name; 392 unsigned long flag; 393 void *private_area; 394 int ret; 395 396 /* 397 * It's okay to check this without a lock because there can 398 * only be one thread before it becomes true. 399 */ 400 if (pthread__started == 0) { 401 pthread__start(); 402 pthread__started = 1; 403 } 404 405 if (attr == NULL) 406 nattr = pthread_default_attr; 407 else if (attr->pta_magic == PT_ATTR_MAGIC) 408 nattr = *attr; 409 else 410 return EINVAL; 411 412 /* Fetch misc. attributes from the attr structure. */ 413 name = NULL; 414 if ((p = nattr.pta_private) != NULL) 415 if (p->ptap_name[0] != '\0') 416 if ((name = strdup(p->ptap_name)) == NULL) 417 return ENOMEM; 418 419 newthread = NULL; 420 421 /* 422 * Try to reclaim a dead thread. 423 */ 424 if (!PTQ_EMPTY(&pthread__deadqueue)) { 425 pthread_mutex_lock(&pthread__deadqueue_lock); 426 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 427 /* Still running? */ 428 if (newthread->pt_lwpctl->lc_curcpu == 429 LWPCTL_CPU_EXITED || 430 (_lwp_kill(newthread->pt_lid, 0) == -1 && 431 errno == ESRCH)) 432 break; 433 } 434 if (newthread) 435 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 436 pthread_mutex_unlock(&pthread__deadqueue_lock); 437 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 438 if (newthread && newthread->pt_tls) { 439 _rtld_tls_free(newthread->pt_tls); 440 newthread->pt_tls = NULL; 441 } 442 #endif 443 } 444 445 /* 446 * If necessary set up a stack, allocate space for a pthread_st, 447 * and initialize it. 448 */ 449 if (newthread == NULL) { 450 newthread = malloc(sizeof(*newthread)); 451 if (newthread == NULL) { 452 free(name); 453 return ENOMEM; 454 } 455 newthread->pt_stack_allocated = false; 456 457 if (pthread__getstack(newthread, attr)) { 458 free(newthread); 459 free(name); 460 return ENOMEM; 461 } 462 463 /* This is used only when creating the thread. */ 464 _INITCONTEXT_U(&newthread->pt_uc); 465 newthread->pt_uc.uc_stack = newthread->pt_stack; 466 newthread->pt_uc.uc_link = NULL; 467 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 468 newthread->pt_tls = NULL; 469 #endif 470 471 /* Add to list of all threads. */ 472 pthread_rwlock_wrlock(&pthread__alltree_lock); 473 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 474 RB_INSERT(__pthread__alltree, &pthread__alltree, newthread); 475 pthread_rwlock_unlock(&pthread__alltree_lock); 476 477 /* Will be reset by the thread upon exit. */ 478 pthread__initthread(newthread); 479 } else { 480 if (pthread__getstack(newthread, attr)) { 481 pthread_mutex_lock(&pthread__deadqueue_lock); 482 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 483 pthread_mutex_unlock(&pthread__deadqueue_lock); 484 return ENOMEM; 485 } 486 _INITCONTEXT_U(&newthread->pt_uc); 487 newthread->pt_uc.uc_stack = newthread->pt_stack; 488 newthread->pt_uc.uc_link = NULL; 489 } 490 491 /* 492 * Create the new LWP. 493 */ 494 pthread__scrubthread(newthread, name, nattr.pta_flags); 495 newthread->pt_func = startfunc; 496 newthread->pt_arg = arg; 497 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 498 private_area = newthread->pt_tls = _rtld_tls_allocate(); 499 newthread->pt_tls->tcb_pthread = newthread; 500 #else 501 private_area = newthread; 502 #endif 503 504 _lwp_makecontext(&newthread->pt_uc, pthread__create_tramp, 505 newthread, private_area, newthread->pt_stack.ss_sp, 506 newthread->pt_stack.ss_size); 507 508 flag = LWP_DETACHED; 509 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 510 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 511 flag |= LWP_SUSPENDED; 512 ret = _lwp_create(&newthread->pt_uc, flag, &newthread->pt_lid); 513 if (ret != 0) { 514 pthread_mutex_lock(&newthread->pt_lock); 515 /* Will unlock and free name. */ 516 pthread__reap(newthread); 517 return ret; 518 } 519 520 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 521 if (p != NULL) { 522 (void)pthread_setschedparam(newthread, p->ptap_policy, 523 &p->ptap_sp); 524 } 525 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 526 (void)_lwp_continue(newthread->pt_lid); 527 } 528 } 529 530 *thread = newthread; 531 532 return 0; 533 } 534 535 536 __dead static void 537 pthread__create_tramp(void *cookie) 538 { 539 pthread_t self; 540 void *retval; 541 542 self = cookie; 543 544 /* 545 * Throw away some stack in a feeble attempt to reduce cache 546 * thrash. May help for SMT processors. XXX We should not 547 * be allocating stacks on fixed 2MB boundaries. Needs a 548 * thread register or decent thread local storage. 549 * 550 * Note that we may race with the kernel in _lwp_create(), 551 * and so pt_lid can be unset at this point, but we don't 552 * care. 553 */ 554 (void)alloca(((unsigned)self->pt_lid & 7) << 8); 555 556 if (self->pt_name != NULL) { 557 pthread_mutex_lock(&self->pt_lock); 558 if (self->pt_name != NULL) 559 (void)_lwp_setname(0, self->pt_name); 560 pthread_mutex_unlock(&self->pt_lock); 561 } 562 563 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 564 err(1, "_lwp_ctl"); 565 } 566 567 retval = (*self->pt_func)(self->pt_arg); 568 569 pthread_exit(retval); 570 571 /*NOTREACHED*/ 572 pthread__abort(); 573 } 574 575 int 576 pthread_suspend_np(pthread_t thread) 577 { 578 pthread_t self; 579 580 self = pthread__self(); 581 if (self == thread) { 582 return EDEADLK; 583 } 584 if (pthread__find(thread) != 0) 585 return ESRCH; 586 if (_lwp_suspend(thread->pt_lid) == 0) 587 return 0; 588 return errno; 589 } 590 591 int 592 pthread_resume_np(pthread_t thread) 593 { 594 595 if (pthread__find(thread) != 0) 596 return ESRCH; 597 if (_lwp_continue(thread->pt_lid) == 0) 598 return 0; 599 return errno; 600 } 601 602 void 603 pthread_exit(void *retval) 604 { 605 pthread_t self; 606 struct pt_clean_t *cleanup; 607 char *name; 608 609 self = pthread__self(); 610 611 /* Disable cancellability. */ 612 pthread_mutex_lock(&self->pt_lock); 613 self->pt_flags |= PT_FLAG_CS_DISABLED; 614 self->pt_cancel = 0; 615 616 /* Call any cancellation cleanup handlers */ 617 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 618 pthread_mutex_unlock(&self->pt_lock); 619 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 620 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 621 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 622 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 623 } 624 pthread_mutex_lock(&self->pt_lock); 625 } 626 627 /* Perform cleanup of thread-specific data */ 628 pthread__destroy_tsd(self); 629 630 /* Signal our exit. */ 631 self->pt_exitval = retval; 632 if (self->pt_flags & PT_FLAG_DETACHED) { 633 self->pt_state = PT_STATE_DEAD; 634 name = self->pt_name; 635 self->pt_name = NULL; 636 pthread_mutex_unlock(&self->pt_lock); 637 if (name != NULL) 638 free(name); 639 pthread_mutex_lock(&pthread__deadqueue_lock); 640 PTQ_INSERT_TAIL(&pthread__deadqueue, self, pt_deadq); 641 pthread_mutex_unlock(&pthread__deadqueue_lock); 642 _lwp_exit(); 643 } else { 644 self->pt_state = PT_STATE_ZOMBIE; 645 pthread_cond_broadcast(&self->pt_joiners); 646 pthread_mutex_unlock(&self->pt_lock); 647 /* Note: name will be freed by the joiner. */ 648 _lwp_exit(); 649 } 650 651 /*NOTREACHED*/ 652 pthread__abort(); 653 exit(1); 654 } 655 656 657 int 658 pthread_join(pthread_t thread, void **valptr) 659 { 660 pthread_t self; 661 int error; 662 663 self = pthread__self(); 664 665 if (pthread__find(thread) != 0) 666 return ESRCH; 667 668 if (thread->pt_magic != PT_MAGIC) 669 return EINVAL; 670 671 if (thread == self) 672 return EDEADLK; 673 674 self->pt_droplock = &thread->pt_lock; 675 pthread_mutex_lock(&thread->pt_lock); 676 for (;;) { 677 if (thread->pt_state == PT_STATE_ZOMBIE) 678 break; 679 if (thread->pt_state == PT_STATE_DEAD) { 680 pthread_mutex_unlock(&thread->pt_lock); 681 self->pt_droplock = NULL; 682 return ESRCH; 683 } 684 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 685 pthread_mutex_unlock(&thread->pt_lock); 686 self->pt_droplock = NULL; 687 return EINVAL; 688 } 689 error = pthread_cond_wait(&thread->pt_joiners, 690 &thread->pt_lock); 691 if (error != 0) { 692 pthread__errorfunc(__FILE__, __LINE__, 693 __func__, "unexpected return from cond_wait()"); 694 } 695 696 } 697 pthread__testcancel(self); 698 if (valptr != NULL) 699 *valptr = thread->pt_exitval; 700 /* pthread__reap() will drop the lock. */ 701 pthread__reap(thread); 702 self->pt_droplock = NULL; 703 704 return 0; 705 } 706 707 static void 708 pthread__reap(pthread_t thread) 709 { 710 char *name; 711 712 name = thread->pt_name; 713 thread->pt_name = NULL; 714 thread->pt_state = PT_STATE_DEAD; 715 pthread_mutex_unlock(&thread->pt_lock); 716 717 pthread_mutex_lock(&pthread__deadqueue_lock); 718 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 719 pthread_mutex_unlock(&pthread__deadqueue_lock); 720 721 if (name != NULL) 722 free(name); 723 } 724 725 int 726 pthread_equal(pthread_t t1, pthread_t t2) 727 { 728 729 /* Nothing special here. */ 730 return (t1 == t2); 731 } 732 733 734 int 735 pthread_detach(pthread_t thread) 736 { 737 738 if (pthread__find(thread) != 0) 739 return ESRCH; 740 741 if (thread->pt_magic != PT_MAGIC) 742 return EINVAL; 743 744 pthread_mutex_lock(&thread->pt_lock); 745 thread->pt_flags |= PT_FLAG_DETACHED; 746 if (thread->pt_state == PT_STATE_ZOMBIE) { 747 /* pthread__reap() will drop the lock. */ 748 pthread__reap(thread); 749 } else { 750 /* 751 * Not valid for threads to be waiting in 752 * pthread_join() (there are intractable 753 * sync issues from the application 754 * perspective), but give those threads 755 * a chance anyway. 756 */ 757 pthread_cond_broadcast(&thread->pt_joiners); 758 pthread_mutex_unlock(&thread->pt_lock); 759 } 760 761 return 0; 762 } 763 764 765 int 766 pthread_getname_np(pthread_t thread, char *name, size_t len) 767 { 768 769 if (pthread__find(thread) != 0) 770 return ESRCH; 771 772 if (thread->pt_magic != PT_MAGIC) 773 return EINVAL; 774 775 pthread_mutex_lock(&thread->pt_lock); 776 if (thread->pt_name == NULL) 777 name[0] = '\0'; 778 else 779 strlcpy(name, thread->pt_name, len); 780 pthread_mutex_unlock(&thread->pt_lock); 781 782 return 0; 783 } 784 785 786 int 787 pthread_setname_np(pthread_t thread, const char *name, void *arg) 788 { 789 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 790 int namelen; 791 792 if (pthread__find(thread) != 0) 793 return ESRCH; 794 795 if (thread->pt_magic != PT_MAGIC) 796 return EINVAL; 797 798 namelen = snprintf(newname, sizeof(newname), name, arg); 799 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 800 return EINVAL; 801 802 cp = strdup(newname); 803 if (cp == NULL) 804 return ENOMEM; 805 806 pthread_mutex_lock(&thread->pt_lock); 807 oldname = thread->pt_name; 808 thread->pt_name = cp; 809 (void)_lwp_setname(thread->pt_lid, cp); 810 pthread_mutex_unlock(&thread->pt_lock); 811 812 if (oldname != NULL) 813 free(oldname); 814 815 return 0; 816 } 817 818 819 820 /* 821 * XXX There should be a way for applications to use the efficent 822 * inline version, but there are opacity/namespace issues. 823 */ 824 pthread_t 825 pthread_self(void) 826 { 827 828 return pthread__self(); 829 } 830 831 832 int 833 pthread_cancel(pthread_t thread) 834 { 835 836 if (pthread__find(thread) != 0) 837 return ESRCH; 838 pthread_mutex_lock(&thread->pt_lock); 839 thread->pt_flags |= PT_FLAG_CS_PENDING; 840 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 841 thread->pt_cancel = 1; 842 pthread_mutex_unlock(&thread->pt_lock); 843 _lwp_wakeup(thread->pt_lid); 844 } else 845 pthread_mutex_unlock(&thread->pt_lock); 846 847 return 0; 848 } 849 850 851 int 852 pthread_setcancelstate(int state, int *oldstate) 853 { 854 pthread_t self; 855 int retval; 856 857 self = pthread__self(); 858 retval = 0; 859 860 pthread_mutex_lock(&self->pt_lock); 861 862 if (oldstate != NULL) { 863 if (self->pt_flags & PT_FLAG_CS_DISABLED) 864 *oldstate = PTHREAD_CANCEL_DISABLE; 865 else 866 *oldstate = PTHREAD_CANCEL_ENABLE; 867 } 868 869 if (state == PTHREAD_CANCEL_DISABLE) { 870 self->pt_flags |= PT_FLAG_CS_DISABLED; 871 if (self->pt_cancel) { 872 self->pt_flags |= PT_FLAG_CS_PENDING; 873 self->pt_cancel = 0; 874 } 875 } else if (state == PTHREAD_CANCEL_ENABLE) { 876 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 877 /* 878 * If a cancellation was requested while cancellation 879 * was disabled, note that fact for future 880 * cancellation tests. 881 */ 882 if (self->pt_flags & PT_FLAG_CS_PENDING) { 883 self->pt_cancel = 1; 884 /* This is not a deferred cancellation point. */ 885 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 886 pthread_mutex_unlock(&self->pt_lock); 887 pthread__cancelled(); 888 } 889 } 890 } else 891 retval = EINVAL; 892 893 pthread_mutex_unlock(&self->pt_lock); 894 895 return retval; 896 } 897 898 899 int 900 pthread_setcanceltype(int type, int *oldtype) 901 { 902 pthread_t self; 903 int retval; 904 905 self = pthread__self(); 906 retval = 0; 907 908 pthread_mutex_lock(&self->pt_lock); 909 910 if (oldtype != NULL) { 911 if (self->pt_flags & PT_FLAG_CS_ASYNC) 912 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 913 else 914 *oldtype = PTHREAD_CANCEL_DEFERRED; 915 } 916 917 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 918 self->pt_flags |= PT_FLAG_CS_ASYNC; 919 if (self->pt_cancel) { 920 pthread_mutex_unlock(&self->pt_lock); 921 pthread__cancelled(); 922 } 923 } else if (type == PTHREAD_CANCEL_DEFERRED) 924 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 925 else 926 retval = EINVAL; 927 928 pthread_mutex_unlock(&self->pt_lock); 929 930 return retval; 931 } 932 933 934 void 935 pthread_testcancel(void) 936 { 937 pthread_t self; 938 939 self = pthread__self(); 940 if (self->pt_cancel) 941 pthread__cancelled(); 942 } 943 944 945 /* 946 * POSIX requires that certain functions return an error rather than 947 * invoking undefined behavior even when handed completely bogus 948 * pthread_t values, e.g. stack garbage or (pthread_t)666. This 949 * utility routine searches the list of threads for the pthread_t 950 * value without dereferencing it. 951 */ 952 int 953 pthread__find(pthread_t id) 954 { 955 pthread_t target; 956 957 pthread_rwlock_rdlock(&pthread__alltree_lock); 958 /* LINTED */ 959 target = RB_FIND(__pthread__alltree, &pthread__alltree, id); 960 pthread_rwlock_unlock(&pthread__alltree_lock); 961 962 if (target == NULL || target->pt_state == PT_STATE_DEAD) 963 return ESRCH; 964 965 return 0; 966 } 967 968 969 void 970 pthread__testcancel(pthread_t self) 971 { 972 973 if (self->pt_cancel) 974 pthread__cancelled(); 975 } 976 977 978 void 979 pthread__cancelled(void) 980 { 981 pthread_mutex_t *droplock; 982 pthread_t self; 983 984 self = pthread__self(); 985 droplock = self->pt_droplock; 986 self->pt_droplock = NULL; 987 988 if (droplock != NULL && pthread_mutex_held_np(droplock)) 989 pthread_mutex_unlock(droplock); 990 991 pthread_exit(PTHREAD_CANCELED); 992 } 993 994 995 void 996 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 997 { 998 pthread_t self; 999 struct pt_clean_t *entry; 1000 1001 self = pthread__self(); 1002 entry = store; 1003 entry->ptc_cleanup = cleanup; 1004 entry->ptc_arg = arg; 1005 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1006 } 1007 1008 1009 void 1010 pthread__cleanup_pop(int ex, void *store) 1011 { 1012 pthread_t self; 1013 struct pt_clean_t *entry; 1014 1015 self = pthread__self(); 1016 entry = store; 1017 1018 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1019 if (ex) 1020 (*entry->ptc_cleanup)(entry->ptc_arg); 1021 } 1022 1023 1024 int * 1025 pthread__errno(void) 1026 { 1027 pthread_t self; 1028 1029 self = pthread__self(); 1030 1031 return &(self->pt_errno); 1032 } 1033 1034 ssize_t _sys_write(int, const void *, size_t); 1035 1036 void 1037 pthread__assertfunc(const char *file, int line, const char *function, 1038 const char *expr) 1039 { 1040 char buf[1024]; 1041 int len; 1042 1043 /* 1044 * snprintf should not acquire any locks, or we could 1045 * end up deadlocked if the assert caller held locks. 1046 */ 1047 len = snprintf(buf, 1024, 1048 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1049 expr, file, line, 1050 function ? ", function \"" : "", 1051 function ? function : "", 1052 function ? "\"" : ""); 1053 1054 _sys_write(STDERR_FILENO, buf, (size_t)len); 1055 (void)kill(getpid(), SIGABRT); 1056 1057 _exit(1); 1058 } 1059 1060 1061 void 1062 pthread__errorfunc(const char *file, int line, const char *function, 1063 const char *msg) 1064 { 1065 char buf[1024]; 1066 size_t len; 1067 1068 if (pthread__diagassert == 0) 1069 return; 1070 1071 /* 1072 * snprintf should not acquire any locks, or we could 1073 * end up deadlocked if the assert caller held locks. 1074 */ 1075 len = snprintf(buf, 1024, 1076 "%s: Error detected by libpthread: %s.\n" 1077 "Detected by file \"%s\", line %d%s%s%s.\n" 1078 "See pthread(3) for information.\n", 1079 getprogname(), msg, file, line, 1080 function ? ", function \"" : "", 1081 function ? function : "", 1082 function ? "\"" : ""); 1083 1084 if (pthread__diagassert & DIAGASSERT_STDERR) 1085 _sys_write(STDERR_FILENO, buf, len); 1086 1087 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1088 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1089 1090 if (pthread__diagassert & DIAGASSERT_ABORT) { 1091 (void)kill(getpid(), SIGABRT); 1092 _exit(1); 1093 } 1094 } 1095 1096 /* 1097 * Thread park/unpark operations. The kernel operations are 1098 * modelled after a brief description from "Multithreading in 1099 * the Solaris Operating Environment": 1100 * 1101 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1102 */ 1103 1104 #define OOPS(msg) \ 1105 pthread__errorfunc(__FILE__, __LINE__, __func__, msg) 1106 1107 int 1108 pthread__park(pthread_t self, pthread_mutex_t *lock, 1109 pthread_queue_t *queue, const struct timespec *abstime, 1110 int cancelpt, const void *hint) 1111 { 1112 int rv, error; 1113 void *obj; 1114 1115 /* 1116 * For non-interlocked release of mutexes we need a store 1117 * barrier before incrementing pt_blocking away from zero. 1118 * This is provided by pthread_mutex_unlock(). 1119 */ 1120 self->pt_willpark = 1; 1121 pthread_mutex_unlock(lock); 1122 self->pt_willpark = 0; 1123 self->pt_blocking++; 1124 1125 /* 1126 * Wait until we are awoken by a pending unpark operation, 1127 * a signal, an unpark posted after we have gone asleep, 1128 * or an expired timeout. 1129 * 1130 * It is fine to test the value of pt_sleepobj without 1131 * holding any locks, because: 1132 * 1133 * o Only the blocking thread (this thread) ever sets them 1134 * to a non-NULL value. 1135 * 1136 * o Other threads may set them NULL, but if they do so they 1137 * must also make this thread return from _lwp_park. 1138 * 1139 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1140 * calls and all make use of spinlocks in the kernel. So 1141 * these system calls act as full memory barriers, and will 1142 * ensure that the calling CPU's store buffers are drained. 1143 * In combination with the spinlock release before unpark, 1144 * this means that modification of pt_sleepobj/onq by another 1145 * thread will become globally visible before that thread 1146 * schedules an unpark operation on this thread. 1147 * 1148 * Note: the test in the while() statement dodges the park op if 1149 * we have already been awoken, unless there is another thread to 1150 * awaken. This saves a syscall - if we were already awakened, 1151 * the next call to _lwp_park() would need to return early in order 1152 * to eat the previous wakeup. 1153 */ 1154 rv = 0; 1155 do { 1156 /* 1157 * If we deferred unparking a thread, arrange to 1158 * have _lwp_park() restart it before blocking. 1159 */ 1160 error = _lwp_park(abstime, self->pt_unpark, hint, hint); 1161 self->pt_unpark = 0; 1162 if (error != 0) { 1163 switch (rv = errno) { 1164 case EINTR: 1165 case EALREADY: 1166 rv = 0; 1167 break; 1168 case ETIMEDOUT: 1169 break; 1170 default: 1171 OOPS("_lwp_park failed"); 1172 break; 1173 } 1174 } 1175 /* Check for cancellation. */ 1176 if (cancelpt && self->pt_cancel) 1177 rv = EINTR; 1178 } while (self->pt_sleepobj != NULL && rv == 0); 1179 1180 /* 1181 * If we have been awoken early but are still on the queue, 1182 * then remove ourself. Again, it's safe to do the test 1183 * without holding any locks. 1184 */ 1185 if (__predict_false(self->pt_sleepobj != NULL)) { 1186 pthread_mutex_lock(lock); 1187 if ((obj = self->pt_sleepobj) != NULL) { 1188 PTQ_REMOVE(queue, self, pt_sleep); 1189 self->pt_sleepobj = NULL; 1190 if (obj != NULL && self->pt_early != NULL) 1191 (*self->pt_early)(obj); 1192 } 1193 pthread_mutex_unlock(lock); 1194 } 1195 self->pt_early = NULL; 1196 self->pt_blocking--; 1197 membar_sync(); 1198 1199 return rv; 1200 } 1201 1202 void 1203 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1204 pthread_mutex_t *interlock) 1205 { 1206 pthread_t target; 1207 u_int max; 1208 size_t nwaiters; 1209 1210 max = pthread__unpark_max; 1211 nwaiters = self->pt_nwaiters; 1212 target = PTQ_FIRST(queue); 1213 if (nwaiters == max) { 1214 /* Overflow. */ 1215 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1216 __UNVOLATILE(&interlock->ptm_waiters)); 1217 nwaiters = 0; 1218 } 1219 target->pt_sleepobj = NULL; 1220 self->pt_waiters[nwaiters++] = target->pt_lid; 1221 PTQ_REMOVE(queue, target, pt_sleep); 1222 self->pt_nwaiters = nwaiters; 1223 pthread__mutex_deferwake(self, interlock); 1224 } 1225 1226 void 1227 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1228 pthread_mutex_t *interlock) 1229 { 1230 pthread_t target; 1231 u_int max; 1232 size_t nwaiters; 1233 1234 max = pthread__unpark_max; 1235 nwaiters = self->pt_nwaiters; 1236 PTQ_FOREACH(target, queue, pt_sleep) { 1237 if (nwaiters == max) { 1238 /* Overflow. */ 1239 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1240 __UNVOLATILE(&interlock->ptm_waiters)); 1241 nwaiters = 0; 1242 } 1243 target->pt_sleepobj = NULL; 1244 self->pt_waiters[nwaiters++] = target->pt_lid; 1245 } 1246 self->pt_nwaiters = nwaiters; 1247 PTQ_INIT(queue); 1248 pthread__mutex_deferwake(self, interlock); 1249 } 1250 1251 #undef OOPS 1252 1253 static void 1254 pthread__initmainstack(void) 1255 { 1256 struct rlimit slimit; 1257 const AuxInfo *aux; 1258 size_t size; 1259 1260 _DIAGASSERT(_dlauxinfo() != NULL); 1261 1262 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1263 err(1, "Couldn't get stack resource consumption limits"); 1264 size = slimit.rlim_cur; 1265 pthread__main.pt_stack.ss_size = size; 1266 1267 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1268 if (aux->a_type == AT_STACKBASE) { 1269 pthread__main.pt_stack.ss_sp = (void *)aux->a_v; 1270 #ifdef __MACHINE_STACK_GROWS_UP 1271 pthread__main.pt_stack.ss_sp = (void *)aux->a_v; 1272 #else 1273 pthread__main.pt_stack.ss_sp = (char *)aux->a_v - size; 1274 #endif 1275 break; 1276 } 1277 } 1278 } 1279 1280 /* 1281 * Set up the slightly special stack for the "initial" thread, which 1282 * runs on the normal system stack, and thus gets slightly different 1283 * treatment. 1284 */ 1285 static void 1286 pthread__initmain(pthread_t *newt) 1287 { 1288 char *value; 1289 1290 pthread__initmainstack(); 1291 1292 value = pthread__getenv("PTHREAD_STACKSIZE"); 1293 if (value != NULL) { 1294 pthread__stacksize = atoi(value) * 1024; 1295 if (pthread__stacksize > pthread__main.pt_stack.ss_size) 1296 pthread__stacksize = pthread__main.pt_stack.ss_size; 1297 } 1298 if (pthread__stacksize == 0) 1299 pthread__stacksize = pthread__main.pt_stack.ss_size; 1300 pthread__stacksize += pthread__pagesize - 1; 1301 pthread__stacksize &= ~(pthread__pagesize - 1); 1302 if (pthread__stacksize < 4 * pthread__pagesize) 1303 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1304 4 * pthread__pagesize / 1024); 1305 1306 *newt = &pthread__main; 1307 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 1308 # ifdef __HAVE___LWP_GETTCB_FAST 1309 pthread__main.pt_tls = __lwp_gettcb_fast(); 1310 # else 1311 pthread__main.pt_tls = _lwp_getprivate(); 1312 # endif 1313 pthread__main.pt_tls->tcb_pthread = &pthread__main; 1314 #endif 1315 } 1316 1317 #ifndef lint 1318 static int 1319 pthread__cmp(struct __pthread_st *a, struct __pthread_st *b) 1320 { 1321 1322 if ((uintptr_t)a < (uintptr_t)b) 1323 return (-1); 1324 else if (a == b) 1325 return 0; 1326 else 1327 return 1; 1328 } 1329 RB_GENERATE_STATIC(__pthread__alltree, __pthread_st, pt_alltree, pthread__cmp) 1330 #endif 1331 1332 /* Because getenv() wants to use locks. */ 1333 char * 1334 pthread__getenv(const char *name) 1335 { 1336 extern char **environ; 1337 size_t l_name, offset; 1338 1339 l_name = strlen(name); 1340 for (offset = 0; environ[offset] != NULL; offset++) { 1341 if (strncmp(name, environ[offset], l_name) == 0 && 1342 environ[offset][l_name] == '=') { 1343 return environ[offset] + l_name + 1; 1344 } 1345 } 1346 1347 return NULL; 1348 } 1349 1350 pthread_mutex_t * 1351 pthread__hashlock(volatile const void *p) 1352 { 1353 uintptr_t v; 1354 1355 v = (uintptr_t)p; 1356 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1357 } 1358 1359 int 1360 pthread__checkpri(int pri) 1361 { 1362 static int havepri; 1363 static long min, max; 1364 1365 if (!havepri) { 1366 min = sysconf(_SC_SCHED_PRI_MIN); 1367 max = sysconf(_SC_SCHED_PRI_MAX); 1368 havepri = 1; 1369 } 1370 return (pri < min || pri > max) ? EINVAL : 0; 1371 } 1372