1 /* $NetBSD: pthread.c,v 1.144 2014/01/31 20:44:01 christos Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nathan J. Williams and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __RCSID("$NetBSD: pthread.c,v 1.144 2014/01/31 20:44:01 christos Exp $"); 34 35 #define __EXPOSE_STACK 1 36 37 #include <sys/param.h> 38 #include <sys/exec_elf.h> 39 #include <sys/mman.h> 40 #include <sys/lwp.h> 41 #include <sys/lwpctl.h> 42 #include <sys/resource.h> 43 #include <sys/tls.h> 44 45 #include <assert.h> 46 #include <dlfcn.h> 47 #include <err.h> 48 #include <errno.h> 49 #include <lwp.h> 50 #include <signal.h> 51 #include <stdio.h> 52 #include <stdlib.h> 53 #include <stddef.h> 54 #include <string.h> 55 #include <syslog.h> 56 #include <ucontext.h> 57 #include <unistd.h> 58 #include <sched.h> 59 60 #include "pthread.h" 61 #include "pthread_int.h" 62 #include "reentrant.h" 63 64 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 65 static rb_tree_t pthread__alltree; 66 67 static signed int pthread__cmp(void *, const void *, const void *); 68 69 static const rb_tree_ops_t pthread__alltree_ops = { 70 .rbto_compare_nodes = pthread__cmp, 71 .rbto_compare_key = pthread__cmp, 72 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree), 73 .rbto_context = NULL 74 }; 75 76 static void pthread__create_tramp(void *); 77 static void pthread__initthread(pthread_t); 78 static void pthread__scrubthread(pthread_t, char *, int); 79 static void pthread__initmain(pthread_t *); 80 static void pthread__fork_callback(void); 81 static void pthread__reap(pthread_t); 82 static void pthread__child_callback(void); 83 static void pthread__start(void); 84 85 void pthread__init(void); 86 87 int pthread__started; 88 int __uselibcstub = 1; 89 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 90 pthread_queue_t pthread__deadqueue; 91 pthread_queue_t pthread__allqueue; 92 93 static pthread_attr_t pthread_default_attr; 94 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 95 96 enum { 97 DIAGASSERT_ABORT = 1<<0, 98 DIAGASSERT_STDERR = 1<<1, 99 DIAGASSERT_SYSLOG = 1<<2 100 }; 101 102 static int pthread__diagassert; 103 104 int pthread__concurrency; 105 int pthread__nspins; 106 int pthread__unpark_max = PTHREAD__UNPARK_MAX; 107 int pthread__dbg; /* set by libpthread_dbg if active */ 108 109 /* 110 * We have to initialize the pthread_stack* variables here because 111 * mutexes are used before pthread_init() and thus pthread__initmain() 112 * are called. Since mutexes only save the stack pointer and not a 113 * pointer to the thread data, it is safe to change the mapping from 114 * stack pointer to thread data afterwards. 115 */ 116 size_t pthread__stacksize; 117 size_t pthread__pagesize; 118 static struct __pthread_st pthread__main; 119 120 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 121 122 __strong_alias(__libc_thr_self,pthread_self) 123 __strong_alias(__libc_thr_create,pthread_create) 124 __strong_alias(__libc_thr_exit,pthread_exit) 125 __strong_alias(__libc_thr_errno,pthread__errno) 126 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 127 __strong_alias(__libc_thr_equal,pthread_equal) 128 __strong_alias(__libc_thr_init,pthread__init) 129 130 /* 131 * Static library kludge. Place a reference to a symbol any library 132 * file which does not already have a reference here. 133 */ 134 extern int pthread__cancel_stub_binder; 135 136 void *pthread__static_lib_binder[] = { 137 &pthread__cancel_stub_binder, 138 pthread_cond_init, 139 pthread_mutex_init, 140 pthread_rwlock_init, 141 pthread_barrier_init, 142 pthread_key_create, 143 pthread_setspecific, 144 }; 145 146 #define NHASHLOCK 64 147 148 static union hashlock { 149 pthread_mutex_t mutex; 150 char pad[64]; 151 } hashlocks[NHASHLOCK] __aligned(64); 152 153 /* 154 * This needs to be started by the library loading code, before main() 155 * gets to run, for various things that use the state of the initial thread 156 * to work properly (thread-specific data is an application-visible example; 157 * spinlock counts for mutexes is an internal example). 158 */ 159 void 160 pthread__init(void) 161 { 162 pthread_t first; 163 char *p; 164 int i; 165 extern int __isthreaded; 166 167 __uselibcstub = 0; 168 169 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 170 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF); 171 172 /* Initialize locks first; they're needed elsewhere. */ 173 pthread__lockprim_init(); 174 for (i = 0; i < NHASHLOCK; i++) { 175 pthread_mutex_init(&hashlocks[i].mutex, NULL); 176 } 177 178 /* Fetch parameters. */ 179 i = (int)_lwp_unpark_all(NULL, 0, NULL); 180 if (i == -1) 181 err(1, "_lwp_unpark_all"); 182 if (i < pthread__unpark_max) 183 pthread__unpark_max = i; 184 185 /* Basic data structure setup */ 186 pthread_attr_init(&pthread_default_attr); 187 PTQ_INIT(&pthread__allqueue); 188 PTQ_INIT(&pthread__deadqueue); 189 190 rb_tree_init(&pthread__alltree, &pthread__alltree_ops); 191 192 /* Create the thread structure corresponding to main() */ 193 pthread__initmain(&first); 194 pthread__initthread(first); 195 pthread__scrubthread(first, NULL, 0); 196 197 first->pt_lid = _lwp_self(); 198 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 199 (void)rb_tree_insert_node(&pthread__alltree, first); 200 201 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 202 err(1, "_lwp_ctl"); 203 } 204 205 /* Start subsystems */ 206 PTHREAD_MD_INIT 207 208 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 209 switch (*p) { 210 case 'a': 211 pthread__diagassert |= DIAGASSERT_ABORT; 212 break; 213 case 'A': 214 pthread__diagassert &= ~DIAGASSERT_ABORT; 215 break; 216 case 'e': 217 pthread__diagassert |= DIAGASSERT_STDERR; 218 break; 219 case 'E': 220 pthread__diagassert &= ~DIAGASSERT_STDERR; 221 break; 222 case 'l': 223 pthread__diagassert |= DIAGASSERT_SYSLOG; 224 break; 225 case 'L': 226 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 227 break; 228 } 229 } 230 231 /* Tell libc that we're here and it should role-play accordingly. */ 232 pthread_atfork(NULL, NULL, pthread__fork_callback); 233 __isthreaded = 1; 234 } 235 236 static void 237 pthread__fork_callback(void) 238 { 239 struct __pthread_st *self = pthread__self(); 240 241 /* lwpctl state is not copied across fork. */ 242 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 243 err(1, "_lwp_ctl"); 244 } 245 self->pt_lid = _lwp_self(); 246 } 247 248 static void 249 pthread__child_callback(void) 250 { 251 252 /* 253 * Clean up data structures that a forked child process might 254 * trip over. Note that if threads have been created (causing 255 * this handler to be registered) the standards say that the 256 * child will trigger undefined behavior if it makes any 257 * pthread_* calls (or any other calls that aren't 258 * async-signal-safe), so we don't really have to clean up 259 * much. Anything that permits some pthread_* calls to work is 260 * merely being polite. 261 */ 262 pthread__started = 0; 263 } 264 265 static void 266 pthread__start(void) 267 { 268 269 /* 270 * Per-process timers are cleared by fork(); despite the 271 * various restrictions on fork() and threads, it's legal to 272 * fork() before creating any threads. 273 */ 274 pthread_atfork(NULL, NULL, pthread__child_callback); 275 } 276 277 278 /* General-purpose thread data structure sanitization. */ 279 /* ARGSUSED */ 280 static void 281 pthread__initthread(pthread_t t) 282 { 283 284 t->pt_self = t; 285 t->pt_magic = PT_MAGIC; 286 t->pt_willpark = 0; 287 t->pt_unpark = 0; 288 t->pt_nwaiters = 0; 289 t->pt_sleepobj = NULL; 290 t->pt_signalled = 0; 291 t->pt_havespecific = 0; 292 t->pt_early = NULL; 293 t->pt_lwpctl = &pthread__dummy_lwpctl; 294 t->pt_blocking = 0; 295 t->pt_droplock = NULL; 296 297 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 298 pthread_mutex_init(&t->pt_lock, NULL); 299 PTQ_INIT(&t->pt_cleanup_stack); 300 pthread_cond_init(&t->pt_joiners, NULL); 301 memset(&t->pt_specific, 0, sizeof(t->pt_specific)); 302 } 303 304 static void 305 pthread__scrubthread(pthread_t t, char *name, int flags) 306 { 307 308 t->pt_state = PT_STATE_RUNNING; 309 t->pt_exitval = NULL; 310 t->pt_flags = flags; 311 t->pt_cancel = 0; 312 t->pt_errno = 0; 313 t->pt_name = name; 314 t->pt_lid = 0; 315 } 316 317 static int 318 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 319 { 320 void *stackbase, *stackbase2, *redzone; 321 size_t stacksize, guardsize; 322 bool allocated; 323 324 if (attr != NULL) { 325 pthread_attr_getstack(attr, &stackbase, &stacksize); 326 } else { 327 stackbase = NULL; 328 stacksize = 0; 329 } 330 if (stacksize == 0) 331 stacksize = pthread__stacksize; 332 333 if (newthread->pt_stack_allocated) { 334 if (stackbase == NULL && 335 newthread->pt_stack.ss_size == stacksize) 336 return 0; 337 stackbase2 = newthread->pt_stack.ss_sp; 338 #ifndef __MACHINE_STACK_GROWS_UP 339 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 340 #endif 341 munmap(stackbase2, 342 newthread->pt_stack.ss_size + newthread->pt_guardsize); 343 newthread->pt_stack.ss_sp = NULL; 344 newthread->pt_stack.ss_size = 0; 345 newthread->pt_guardsize = 0; 346 newthread->pt_stack_allocated = false; 347 } 348 349 newthread->pt_stack_allocated = false; 350 351 if (stackbase == NULL) { 352 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 353 guardsize = pthread__pagesize; 354 stackbase = mmap(NULL, stacksize + guardsize, 355 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 356 if (stackbase == MAP_FAILED) 357 return ENOMEM; 358 allocated = true; 359 } else { 360 guardsize = 0; 361 allocated = false; 362 } 363 #ifdef __MACHINE_STACK_GROWS_UP 364 redzone = (char *)stackbase + stacksize; 365 stackbase2 = (char *)stackbase; 366 #else 367 redzone = (char *)stackbase; 368 stackbase2 = (char *)stackbase + guardsize; 369 #endif 370 if (allocated && guardsize && 371 mprotect(redzone, guardsize, PROT_NONE) == -1) { 372 munmap(stackbase, stacksize + guardsize); 373 return EPERM; 374 } 375 newthread->pt_stack.ss_size = stacksize; 376 newthread->pt_stack.ss_sp = stackbase2; 377 newthread->pt_guardsize = guardsize; 378 newthread->pt_stack_allocated = allocated; 379 return 0; 380 } 381 382 int 383 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 384 void *(*startfunc)(void *), void *arg) 385 { 386 pthread_t newthread; 387 pthread_attr_t nattr; 388 struct pthread_attr_private *p; 389 char * volatile name; 390 unsigned long flag; 391 void *private_area; 392 int ret; 393 394 if (__predict_false(__uselibcstub)) { 395 pthread__errorfunc(__FILE__, __LINE__, __func__, 396 "pthread_create() requires linking with -lpthread"); 397 return __libc_thr_create_stub(thread, attr, startfunc, arg); 398 } 399 400 /* 401 * It's okay to check this without a lock because there can 402 * only be one thread before it becomes true. 403 */ 404 if (pthread__started == 0) { 405 pthread__start(); 406 pthread__started = 1; 407 } 408 409 if (attr == NULL) 410 nattr = pthread_default_attr; 411 else if (attr->pta_magic == PT_ATTR_MAGIC) 412 nattr = *attr; 413 else 414 return EINVAL; 415 416 /* Fetch misc. attributes from the attr structure. */ 417 name = NULL; 418 if ((p = nattr.pta_private) != NULL) 419 if (p->ptap_name[0] != '\0') 420 if ((name = strdup(p->ptap_name)) == NULL) 421 return ENOMEM; 422 423 newthread = NULL; 424 425 /* 426 * Try to reclaim a dead thread. 427 */ 428 if (!PTQ_EMPTY(&pthread__deadqueue)) { 429 pthread_mutex_lock(&pthread__deadqueue_lock); 430 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 431 /* Still running? */ 432 if (newthread->pt_lwpctl->lc_curcpu == 433 LWPCTL_CPU_EXITED || 434 (_lwp_kill(newthread->pt_lid, 0) == -1 && 435 errno == ESRCH)) 436 break; 437 } 438 if (newthread) 439 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 440 pthread_mutex_unlock(&pthread__deadqueue_lock); 441 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 442 if (newthread && newthread->pt_tls) { 443 _rtld_tls_free(newthread->pt_tls); 444 newthread->pt_tls = NULL; 445 } 446 #endif 447 } 448 449 /* 450 * If necessary set up a stack, allocate space for a pthread_st, 451 * and initialize it. 452 */ 453 if (newthread == NULL) { 454 newthread = malloc(sizeof(*newthread)); 455 if (newthread == NULL) { 456 free(name); 457 return ENOMEM; 458 } 459 newthread->pt_stack_allocated = false; 460 461 if (pthread__getstack(newthread, attr)) { 462 free(newthread); 463 free(name); 464 return ENOMEM; 465 } 466 467 /* This is used only when creating the thread. */ 468 _INITCONTEXT_U(&newthread->pt_uc); 469 newthread->pt_uc.uc_stack = newthread->pt_stack; 470 newthread->pt_uc.uc_link = NULL; 471 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 472 newthread->pt_tls = NULL; 473 #endif 474 475 /* Add to list of all threads. */ 476 pthread_rwlock_wrlock(&pthread__alltree_lock); 477 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 478 (void)rb_tree_insert_node(&pthread__alltree, newthread); 479 pthread_rwlock_unlock(&pthread__alltree_lock); 480 481 /* Will be reset by the thread upon exit. */ 482 pthread__initthread(newthread); 483 } else { 484 if (pthread__getstack(newthread, attr)) { 485 pthread_mutex_lock(&pthread__deadqueue_lock); 486 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 487 pthread_mutex_unlock(&pthread__deadqueue_lock); 488 return ENOMEM; 489 } 490 _INITCONTEXT_U(&newthread->pt_uc); 491 newthread->pt_uc.uc_stack = newthread->pt_stack; 492 newthread->pt_uc.uc_link = NULL; 493 } 494 495 /* 496 * Create the new LWP. 497 */ 498 pthread__scrubthread(newthread, name, nattr.pta_flags); 499 newthread->pt_func = startfunc; 500 newthread->pt_arg = arg; 501 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 502 private_area = newthread->pt_tls = _rtld_tls_allocate(); 503 newthread->pt_tls->tcb_pthread = newthread; 504 #else 505 private_area = newthread; 506 #endif 507 508 _lwp_makecontext(&newthread->pt_uc, pthread__create_tramp, 509 newthread, private_area, newthread->pt_stack.ss_sp, 510 newthread->pt_stack.ss_size); 511 512 flag = LWP_DETACHED; 513 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 514 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 515 flag |= LWP_SUSPENDED; 516 ret = _lwp_create(&newthread->pt_uc, flag, &newthread->pt_lid); 517 if (ret != 0) { 518 ret = errno; 519 pthread_mutex_lock(&newthread->pt_lock); 520 /* Will unlock and free name. */ 521 pthread__reap(newthread); 522 return ret; 523 } 524 525 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 526 if (p != NULL) { 527 (void)pthread_setschedparam(newthread, p->ptap_policy, 528 &p->ptap_sp); 529 } 530 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 531 (void)_lwp_continue(newthread->pt_lid); 532 } 533 } 534 535 *thread = newthread; 536 537 return 0; 538 } 539 540 541 __dead static void 542 pthread__create_tramp(void *cookie) 543 { 544 pthread_t self; 545 void *retval; 546 547 self = cookie; 548 549 /* 550 * Throw away some stack in a feeble attempt to reduce cache 551 * thrash. May help for SMT processors. XXX We should not 552 * be allocating stacks on fixed 2MB boundaries. Needs a 553 * thread register or decent thread local storage. 554 * 555 * Note that we may race with the kernel in _lwp_create(), 556 * and so pt_lid can be unset at this point, but we don't 557 * care. 558 */ 559 (void)alloca(((unsigned)self->pt_lid & 7) << 8); 560 561 if (self->pt_name != NULL) { 562 pthread_mutex_lock(&self->pt_lock); 563 if (self->pt_name != NULL) 564 (void)_lwp_setname(0, self->pt_name); 565 pthread_mutex_unlock(&self->pt_lock); 566 } 567 568 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 569 err(1, "_lwp_ctl"); 570 } 571 572 retval = (*self->pt_func)(self->pt_arg); 573 574 pthread_exit(retval); 575 576 /*NOTREACHED*/ 577 pthread__abort(); 578 } 579 580 int 581 pthread_suspend_np(pthread_t thread) 582 { 583 pthread_t self; 584 585 self = pthread__self(); 586 if (self == thread) { 587 return EDEADLK; 588 } 589 if (pthread__find(thread) != 0) 590 return ESRCH; 591 if (_lwp_suspend(thread->pt_lid) == 0) 592 return 0; 593 return errno; 594 } 595 596 int 597 pthread_resume_np(pthread_t thread) 598 { 599 600 if (pthread__find(thread) != 0) 601 return ESRCH; 602 if (_lwp_continue(thread->pt_lid) == 0) 603 return 0; 604 return errno; 605 } 606 607 void 608 pthread_exit(void *retval) 609 { 610 pthread_t self; 611 struct pt_clean_t *cleanup; 612 char *name; 613 614 if (__predict_false(__uselibcstub)) { 615 __libc_thr_exit_stub(retval); 616 goto out; 617 } 618 619 self = pthread__self(); 620 621 /* Disable cancellability. */ 622 pthread_mutex_lock(&self->pt_lock); 623 self->pt_flags |= PT_FLAG_CS_DISABLED; 624 self->pt_cancel = 0; 625 626 /* Call any cancellation cleanup handlers */ 627 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 628 pthread_mutex_unlock(&self->pt_lock); 629 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 630 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 631 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 632 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 633 } 634 pthread_mutex_lock(&self->pt_lock); 635 } 636 637 /* Perform cleanup of thread-specific data */ 638 pthread__destroy_tsd(self); 639 640 /* Signal our exit. */ 641 self->pt_exitval = retval; 642 if (self->pt_flags & PT_FLAG_DETACHED) { 643 self->pt_state = PT_STATE_DEAD; 644 name = self->pt_name; 645 self->pt_name = NULL; 646 pthread_mutex_unlock(&self->pt_lock); 647 if (name != NULL) 648 free(name); 649 pthread_mutex_lock(&pthread__deadqueue_lock); 650 PTQ_INSERT_TAIL(&pthread__deadqueue, self, pt_deadq); 651 pthread_mutex_unlock(&pthread__deadqueue_lock); 652 _lwp_exit(); 653 } else { 654 self->pt_state = PT_STATE_ZOMBIE; 655 pthread_cond_broadcast(&self->pt_joiners); 656 pthread_mutex_unlock(&self->pt_lock); 657 /* Note: name will be freed by the joiner. */ 658 _lwp_exit(); 659 } 660 661 out: 662 /*NOTREACHED*/ 663 pthread__abort(); 664 exit(1); 665 } 666 667 668 int 669 pthread_join(pthread_t thread, void **valptr) 670 { 671 pthread_t self; 672 int error; 673 674 self = pthread__self(); 675 676 if (pthread__find(thread) != 0) 677 return ESRCH; 678 679 if (thread->pt_magic != PT_MAGIC) 680 return EINVAL; 681 682 if (thread == self) 683 return EDEADLK; 684 685 self->pt_droplock = &thread->pt_lock; 686 pthread_mutex_lock(&thread->pt_lock); 687 for (;;) { 688 if (thread->pt_state == PT_STATE_ZOMBIE) 689 break; 690 if (thread->pt_state == PT_STATE_DEAD) { 691 pthread_mutex_unlock(&thread->pt_lock); 692 self->pt_droplock = NULL; 693 return ESRCH; 694 } 695 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 696 pthread_mutex_unlock(&thread->pt_lock); 697 self->pt_droplock = NULL; 698 return EINVAL; 699 } 700 error = pthread_cond_wait(&thread->pt_joiners, 701 &thread->pt_lock); 702 if (error != 0) { 703 pthread__errorfunc(__FILE__, __LINE__, 704 __func__, "unexpected return from cond_wait()"); 705 } 706 707 } 708 pthread__testcancel(self); 709 if (valptr != NULL) 710 *valptr = thread->pt_exitval; 711 /* pthread__reap() will drop the lock. */ 712 pthread__reap(thread); 713 self->pt_droplock = NULL; 714 715 return 0; 716 } 717 718 static void 719 pthread__reap(pthread_t thread) 720 { 721 char *name; 722 723 name = thread->pt_name; 724 thread->pt_name = NULL; 725 thread->pt_state = PT_STATE_DEAD; 726 pthread_mutex_unlock(&thread->pt_lock); 727 728 pthread_mutex_lock(&pthread__deadqueue_lock); 729 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 730 pthread_mutex_unlock(&pthread__deadqueue_lock); 731 732 if (name != NULL) 733 free(name); 734 } 735 736 int 737 pthread_equal(pthread_t t1, pthread_t t2) 738 { 739 if (__predict_false(__uselibcstub)) 740 return __libc_thr_equal_stub(t1, t2); 741 742 /* Nothing special here. */ 743 return (t1 == t2); 744 } 745 746 747 int 748 pthread_detach(pthread_t thread) 749 { 750 751 if (pthread__find(thread) != 0) 752 return ESRCH; 753 754 if (thread->pt_magic != PT_MAGIC) 755 return EINVAL; 756 757 pthread_mutex_lock(&thread->pt_lock); 758 thread->pt_flags |= PT_FLAG_DETACHED; 759 if (thread->pt_state == PT_STATE_ZOMBIE) { 760 /* pthread__reap() will drop the lock. */ 761 pthread__reap(thread); 762 } else { 763 /* 764 * Not valid for threads to be waiting in 765 * pthread_join() (there are intractable 766 * sync issues from the application 767 * perspective), but give those threads 768 * a chance anyway. 769 */ 770 pthread_cond_broadcast(&thread->pt_joiners); 771 pthread_mutex_unlock(&thread->pt_lock); 772 } 773 774 return 0; 775 } 776 777 778 int 779 pthread_getname_np(pthread_t thread, char *name, size_t len) 780 { 781 782 if (pthread__find(thread) != 0) 783 return ESRCH; 784 785 if (thread->pt_magic != PT_MAGIC) 786 return EINVAL; 787 788 pthread_mutex_lock(&thread->pt_lock); 789 if (thread->pt_name == NULL) 790 name[0] = '\0'; 791 else 792 strlcpy(name, thread->pt_name, len); 793 pthread_mutex_unlock(&thread->pt_lock); 794 795 return 0; 796 } 797 798 799 int 800 pthread_setname_np(pthread_t thread, const char *name, void *arg) 801 { 802 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 803 int namelen; 804 805 if (pthread__find(thread) != 0) 806 return ESRCH; 807 808 if (thread->pt_magic != PT_MAGIC) 809 return EINVAL; 810 811 namelen = snprintf(newname, sizeof(newname), name, arg); 812 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 813 return EINVAL; 814 815 cp = strdup(newname); 816 if (cp == NULL) 817 return ENOMEM; 818 819 pthread_mutex_lock(&thread->pt_lock); 820 oldname = thread->pt_name; 821 thread->pt_name = cp; 822 (void)_lwp_setname(thread->pt_lid, cp); 823 pthread_mutex_unlock(&thread->pt_lock); 824 825 if (oldname != NULL) 826 free(oldname); 827 828 return 0; 829 } 830 831 832 833 /* 834 * XXX There should be a way for applications to use the efficent 835 * inline version, but there are opacity/namespace issues. 836 */ 837 pthread_t 838 pthread_self(void) 839 { 840 if (__predict_false(__uselibcstub)) 841 return (pthread_t)__libc_thr_self_stub(); 842 843 return pthread__self(); 844 } 845 846 847 int 848 pthread_cancel(pthread_t thread) 849 { 850 851 if (pthread__find(thread) != 0) 852 return ESRCH; 853 pthread_mutex_lock(&thread->pt_lock); 854 thread->pt_flags |= PT_FLAG_CS_PENDING; 855 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 856 thread->pt_cancel = 1; 857 pthread_mutex_unlock(&thread->pt_lock); 858 _lwp_wakeup(thread->pt_lid); 859 } else 860 pthread_mutex_unlock(&thread->pt_lock); 861 862 return 0; 863 } 864 865 866 int 867 pthread_setcancelstate(int state, int *oldstate) 868 { 869 pthread_t self; 870 int retval; 871 872 if (__predict_false(__uselibcstub)) 873 return __libc_thr_setcancelstate_stub(state, oldstate); 874 875 self = pthread__self(); 876 retval = 0; 877 878 pthread_mutex_lock(&self->pt_lock); 879 880 if (oldstate != NULL) { 881 if (self->pt_flags & PT_FLAG_CS_DISABLED) 882 *oldstate = PTHREAD_CANCEL_DISABLE; 883 else 884 *oldstate = PTHREAD_CANCEL_ENABLE; 885 } 886 887 if (state == PTHREAD_CANCEL_DISABLE) { 888 self->pt_flags |= PT_FLAG_CS_DISABLED; 889 if (self->pt_cancel) { 890 self->pt_flags |= PT_FLAG_CS_PENDING; 891 self->pt_cancel = 0; 892 } 893 } else if (state == PTHREAD_CANCEL_ENABLE) { 894 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 895 /* 896 * If a cancellation was requested while cancellation 897 * was disabled, note that fact for future 898 * cancellation tests. 899 */ 900 if (self->pt_flags & PT_FLAG_CS_PENDING) { 901 self->pt_cancel = 1; 902 /* This is not a deferred cancellation point. */ 903 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 904 pthread_mutex_unlock(&self->pt_lock); 905 pthread__cancelled(); 906 } 907 } 908 } else 909 retval = EINVAL; 910 911 pthread_mutex_unlock(&self->pt_lock); 912 913 return retval; 914 } 915 916 917 int 918 pthread_setcanceltype(int type, int *oldtype) 919 { 920 pthread_t self; 921 int retval; 922 923 self = pthread__self(); 924 retval = 0; 925 926 pthread_mutex_lock(&self->pt_lock); 927 928 if (oldtype != NULL) { 929 if (self->pt_flags & PT_FLAG_CS_ASYNC) 930 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 931 else 932 *oldtype = PTHREAD_CANCEL_DEFERRED; 933 } 934 935 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 936 self->pt_flags |= PT_FLAG_CS_ASYNC; 937 if (self->pt_cancel) { 938 pthread_mutex_unlock(&self->pt_lock); 939 pthread__cancelled(); 940 } 941 } else if (type == PTHREAD_CANCEL_DEFERRED) 942 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 943 else 944 retval = EINVAL; 945 946 pthread_mutex_unlock(&self->pt_lock); 947 948 return retval; 949 } 950 951 952 void 953 pthread_testcancel(void) 954 { 955 pthread_t self; 956 957 self = pthread__self(); 958 if (self->pt_cancel) 959 pthread__cancelled(); 960 } 961 962 963 /* 964 * POSIX requires that certain functions return an error rather than 965 * invoking undefined behavior even when handed completely bogus 966 * pthread_t values, e.g. stack garbage. 967 */ 968 int 969 pthread__find(pthread_t id) 970 { 971 pthread_t target; 972 int error; 973 974 pthread_rwlock_rdlock(&pthread__alltree_lock); 975 target = rb_tree_find_node(&pthread__alltree, id); 976 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH; 977 pthread_rwlock_unlock(&pthread__alltree_lock); 978 979 return error; 980 } 981 982 983 void 984 pthread__testcancel(pthread_t self) 985 { 986 987 if (self->pt_cancel) 988 pthread__cancelled(); 989 } 990 991 992 void 993 pthread__cancelled(void) 994 { 995 pthread_mutex_t *droplock; 996 pthread_t self; 997 998 self = pthread__self(); 999 droplock = self->pt_droplock; 1000 self->pt_droplock = NULL; 1001 1002 if (droplock != NULL && pthread_mutex_held_np(droplock)) 1003 pthread_mutex_unlock(droplock); 1004 1005 pthread_exit(PTHREAD_CANCELED); 1006 } 1007 1008 1009 void 1010 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 1011 { 1012 pthread_t self; 1013 struct pt_clean_t *entry; 1014 1015 self = pthread__self(); 1016 entry = store; 1017 entry->ptc_cleanup = cleanup; 1018 entry->ptc_arg = arg; 1019 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1020 } 1021 1022 1023 void 1024 pthread__cleanup_pop(int ex, void *store) 1025 { 1026 pthread_t self; 1027 struct pt_clean_t *entry; 1028 1029 self = pthread__self(); 1030 entry = store; 1031 1032 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1033 if (ex) 1034 (*entry->ptc_cleanup)(entry->ptc_arg); 1035 } 1036 1037 1038 int * 1039 pthread__errno(void) 1040 { 1041 pthread_t self; 1042 1043 if (__predict_false(__uselibcstub)) { 1044 pthread__errorfunc(__FILE__, __LINE__, __func__, 1045 "pthread__errno() requires linking with -lpthread"); 1046 return __libc_thr_errno_stub(); 1047 } 1048 1049 self = pthread__self(); 1050 1051 return &(self->pt_errno); 1052 } 1053 1054 ssize_t _sys_write(int, const void *, size_t); 1055 1056 void 1057 pthread__assertfunc(const char *file, int line, const char *function, 1058 const char *expr) 1059 { 1060 char buf[1024]; 1061 int len; 1062 1063 /* 1064 * snprintf should not acquire any locks, or we could 1065 * end up deadlocked if the assert caller held locks. 1066 */ 1067 len = snprintf(buf, 1024, 1068 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1069 expr, file, line, 1070 function ? ", function \"" : "", 1071 function ? function : "", 1072 function ? "\"" : ""); 1073 1074 _sys_write(STDERR_FILENO, buf, (size_t)len); 1075 (void)kill(getpid(), SIGABRT); 1076 1077 _exit(1); 1078 } 1079 1080 1081 void 1082 pthread__errorfunc(const char *file, int line, const char *function, 1083 const char *msg) 1084 { 1085 char buf[1024]; 1086 size_t len; 1087 1088 if (pthread__diagassert == 0) 1089 return; 1090 1091 /* 1092 * snprintf should not acquire any locks, or we could 1093 * end up deadlocked if the assert caller held locks. 1094 */ 1095 len = snprintf(buf, 1024, 1096 "%s: Error detected by libpthread: %s.\n" 1097 "Detected by file \"%s\", line %d%s%s%s.\n" 1098 "See pthread(3) for information.\n", 1099 getprogname(), msg, file, line, 1100 function ? ", function \"" : "", 1101 function ? function : "", 1102 function ? "\"" : ""); 1103 1104 if (pthread__diagassert & DIAGASSERT_STDERR) 1105 _sys_write(STDERR_FILENO, buf, len); 1106 1107 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1108 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1109 1110 if (pthread__diagassert & DIAGASSERT_ABORT) { 1111 (void)kill(getpid(), SIGABRT); 1112 _exit(1); 1113 } 1114 } 1115 1116 /* 1117 * Thread park/unpark operations. The kernel operations are 1118 * modelled after a brief description from "Multithreading in 1119 * the Solaris Operating Environment": 1120 * 1121 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1122 */ 1123 1124 #define OOPS(msg) \ 1125 pthread__errorfunc(__FILE__, __LINE__, __func__, msg) 1126 1127 int 1128 pthread__park(pthread_t self, pthread_mutex_t *lock, 1129 pthread_queue_t *queue, const struct timespec *abstime, 1130 int cancelpt, const void *hint) 1131 { 1132 int rv, error; 1133 void *obj; 1134 1135 /* 1136 * For non-interlocked release of mutexes we need a store 1137 * barrier before incrementing pt_blocking away from zero. 1138 * This is provided by pthread_mutex_unlock(). 1139 */ 1140 self->pt_willpark = 1; 1141 pthread_mutex_unlock(lock); 1142 self->pt_willpark = 0; 1143 self->pt_blocking++; 1144 1145 /* 1146 * Wait until we are awoken by a pending unpark operation, 1147 * a signal, an unpark posted after we have gone asleep, 1148 * or an expired timeout. 1149 * 1150 * It is fine to test the value of pt_sleepobj without 1151 * holding any locks, because: 1152 * 1153 * o Only the blocking thread (this thread) ever sets them 1154 * to a non-NULL value. 1155 * 1156 * o Other threads may set them NULL, but if they do so they 1157 * must also make this thread return from _lwp_park. 1158 * 1159 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1160 * calls and all make use of spinlocks in the kernel. So 1161 * these system calls act as full memory barriers, and will 1162 * ensure that the calling CPU's store buffers are drained. 1163 * In combination with the spinlock release before unpark, 1164 * this means that modification of pt_sleepobj/onq by another 1165 * thread will become globally visible before that thread 1166 * schedules an unpark operation on this thread. 1167 * 1168 * Note: the test in the while() statement dodges the park op if 1169 * we have already been awoken, unless there is another thread to 1170 * awaken. This saves a syscall - if we were already awakened, 1171 * the next call to _lwp_park() would need to return early in order 1172 * to eat the previous wakeup. 1173 */ 1174 rv = 0; 1175 do { 1176 /* 1177 * If we deferred unparking a thread, arrange to 1178 * have _lwp_park() restart it before blocking. 1179 */ 1180 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME, abstime, 1181 self->pt_unpark, hint, hint); 1182 self->pt_unpark = 0; 1183 if (error != 0) { 1184 switch (rv = errno) { 1185 case EINTR: 1186 case EALREADY: 1187 rv = 0; 1188 break; 1189 case ETIMEDOUT: 1190 break; 1191 default: 1192 OOPS("_lwp_park failed"); 1193 break; 1194 } 1195 } 1196 /* Check for cancellation. */ 1197 if (cancelpt && self->pt_cancel) 1198 rv = EINTR; 1199 } while (self->pt_sleepobj != NULL && rv == 0); 1200 1201 /* 1202 * If we have been awoken early but are still on the queue, 1203 * then remove ourself. Again, it's safe to do the test 1204 * without holding any locks. 1205 */ 1206 if (__predict_false(self->pt_sleepobj != NULL)) { 1207 pthread_mutex_lock(lock); 1208 if ((obj = self->pt_sleepobj) != NULL) { 1209 PTQ_REMOVE(queue, self, pt_sleep); 1210 self->pt_sleepobj = NULL; 1211 if (obj != NULL && self->pt_early != NULL) 1212 (*self->pt_early)(obj); 1213 } 1214 pthread_mutex_unlock(lock); 1215 } 1216 self->pt_early = NULL; 1217 self->pt_blocking--; 1218 membar_sync(); 1219 1220 return rv; 1221 } 1222 1223 void 1224 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1225 pthread_mutex_t *interlock) 1226 { 1227 pthread_t target; 1228 u_int max; 1229 size_t nwaiters; 1230 1231 max = pthread__unpark_max; 1232 nwaiters = self->pt_nwaiters; 1233 target = PTQ_FIRST(queue); 1234 if (nwaiters == max) { 1235 /* Overflow. */ 1236 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1237 __UNVOLATILE(&interlock->ptm_waiters)); 1238 nwaiters = 0; 1239 } 1240 target->pt_sleepobj = NULL; 1241 self->pt_waiters[nwaiters++] = target->pt_lid; 1242 PTQ_REMOVE(queue, target, pt_sleep); 1243 self->pt_nwaiters = nwaiters; 1244 pthread__mutex_deferwake(self, interlock); 1245 } 1246 1247 void 1248 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1249 pthread_mutex_t *interlock) 1250 { 1251 pthread_t target; 1252 u_int max; 1253 size_t nwaiters; 1254 1255 max = pthread__unpark_max; 1256 nwaiters = self->pt_nwaiters; 1257 PTQ_FOREACH(target, queue, pt_sleep) { 1258 if (nwaiters == max) { 1259 /* Overflow. */ 1260 (void)_lwp_unpark_all(self->pt_waiters, nwaiters, 1261 __UNVOLATILE(&interlock->ptm_waiters)); 1262 nwaiters = 0; 1263 } 1264 target->pt_sleepobj = NULL; 1265 self->pt_waiters[nwaiters++] = target->pt_lid; 1266 } 1267 self->pt_nwaiters = nwaiters; 1268 PTQ_INIT(queue); 1269 pthread__mutex_deferwake(self, interlock); 1270 } 1271 1272 #undef OOPS 1273 1274 static void 1275 pthread__initmainstack(void) 1276 { 1277 struct rlimit slimit; 1278 const AuxInfo *aux; 1279 size_t size; 1280 1281 _DIAGASSERT(_dlauxinfo() != NULL); 1282 1283 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1284 err(1, "Couldn't get stack resource consumption limits"); 1285 size = slimit.rlim_cur; 1286 pthread__main.pt_stack.ss_size = size; 1287 1288 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1289 if (aux->a_type == AT_STACKBASE) { 1290 pthread__main.pt_stack.ss_sp = (void *)aux->a_v; 1291 #ifdef __MACHINE_STACK_GROWS_UP 1292 pthread__main.pt_stack.ss_sp = (void *)aux->a_v; 1293 #else 1294 pthread__main.pt_stack.ss_sp = (char *)aux->a_v - size; 1295 #endif 1296 break; 1297 } 1298 } 1299 } 1300 1301 /* 1302 * Set up the slightly special stack for the "initial" thread, which 1303 * runs on the normal system stack, and thus gets slightly different 1304 * treatment. 1305 */ 1306 static void 1307 pthread__initmain(pthread_t *newt) 1308 { 1309 char *value; 1310 1311 pthread__initmainstack(); 1312 1313 value = pthread__getenv("PTHREAD_STACKSIZE"); 1314 if (value != NULL) { 1315 pthread__stacksize = atoi(value) * 1024; 1316 if (pthread__stacksize > pthread__main.pt_stack.ss_size) 1317 pthread__stacksize = pthread__main.pt_stack.ss_size; 1318 } 1319 if (pthread__stacksize == 0) 1320 pthread__stacksize = pthread__main.pt_stack.ss_size; 1321 pthread__stacksize += pthread__pagesize - 1; 1322 pthread__stacksize &= ~(pthread__pagesize - 1); 1323 if (pthread__stacksize < 4 * pthread__pagesize) 1324 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1325 4 * pthread__pagesize / 1024); 1326 1327 *newt = &pthread__main; 1328 #ifdef __HAVE___LWP_GETTCB_FAST 1329 pthread__main.pt_tls = __lwp_gettcb_fast(); 1330 #else 1331 pthread__main.pt_tls = _lwp_getprivate(); 1332 #endif 1333 pthread__main.pt_tls->tcb_pthread = &pthread__main; 1334 } 1335 1336 static signed int 1337 /*ARGSUSED*/ 1338 pthread__cmp(void *ctx, const void *n1, const void *n2) 1339 { 1340 const uintptr_t p1 = (const uintptr_t)n1; 1341 const uintptr_t p2 = (const uintptr_t)n2; 1342 1343 if (p1 < p2) 1344 return -1; 1345 if (p1 > p2) 1346 return 1; 1347 return 0; 1348 } 1349 1350 /* Because getenv() wants to use locks. */ 1351 char * 1352 pthread__getenv(const char *name) 1353 { 1354 extern char **environ; 1355 size_t l_name, offset; 1356 1357 l_name = strlen(name); 1358 for (offset = 0; environ[offset] != NULL; offset++) { 1359 if (strncmp(name, environ[offset], l_name) == 0 && 1360 environ[offset][l_name] == '=') { 1361 return environ[offset] + l_name + 1; 1362 } 1363 } 1364 1365 return NULL; 1366 } 1367 1368 pthread_mutex_t * 1369 pthread__hashlock(volatile const void *p) 1370 { 1371 uintptr_t v; 1372 1373 v = (uintptr_t)p; 1374 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1375 } 1376 1377 int 1378 pthread__checkpri(int pri) 1379 { 1380 static int havepri; 1381 static long min, max; 1382 1383 if (!havepri) { 1384 min = sysconf(_SC_SCHED_PRI_MIN); 1385 max = sysconf(_SC_SCHED_PRI_MAX); 1386 havepri = 1; 1387 } 1388 return (pri < min || pri > max) ? EINVAL : 0; 1389 } 1390