xref: /netbsd-src/lib/libpthread/pthread.c (revision 62e0939e7dacd599987160abb292d33cf66154d6)
1 /*	$NetBSD: pthread.c,v 1.175 2020/06/10 22:45:15 ad Exp $	*/
2 
3 /*-
4  * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020
5  *     The NetBSD Foundation, Inc.
6  * All rights reserved.
7  *
8  * This code is derived from software contributed to The NetBSD Foundation
9  * by Nathan J. Williams and Andrew Doran.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 #include <sys/cdefs.h>
34 __RCSID("$NetBSD: pthread.c,v 1.175 2020/06/10 22:45:15 ad Exp $");
35 
36 #define	__EXPOSE_STACK	1
37 
38 #include <sys/param.h>
39 #include <sys/exec_elf.h>
40 #include <sys/mman.h>
41 #include <sys/lwp.h>
42 #include <sys/lwpctl.h>
43 #include <sys/resource.h>
44 #include <sys/sysctl.h>
45 #include <sys/tls.h>
46 #include <uvm/uvm_param.h>
47 
48 #include <assert.h>
49 #include <dlfcn.h>
50 #include <err.h>
51 #include <errno.h>
52 #include <lwp.h>
53 #include <signal.h>
54 #include <stdio.h>
55 #include <stdlib.h>
56 #include <stddef.h>
57 #include <string.h>
58 #include <syslog.h>
59 #include <ucontext.h>
60 #include <unistd.h>
61 #include <sched.h>
62 
63 #include "atexit.h"
64 #include "pthread.h"
65 #include "pthread_int.h"
66 #include "pthread_makelwp.h"
67 #include "reentrant.h"
68 
69 __BEGIN_DECLS
70 void _malloc_thread_cleanup(void) __weak;
71 __END_DECLS
72 
73 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER;
74 static rb_tree_t	pthread__alltree;
75 
76 static signed int	pthread__cmp(void *, const void *, const void *);
77 
78 static const rb_tree_ops_t pthread__alltree_ops = {
79 	.rbto_compare_nodes = pthread__cmp,
80 	.rbto_compare_key = pthread__cmp,
81 	.rbto_node_offset = offsetof(struct __pthread_st, pt_alltree),
82 	.rbto_context = NULL
83 };
84 
85 static void	pthread__create_tramp(void *);
86 static void	pthread__initthread(pthread_t);
87 static void	pthread__scrubthread(pthread_t, char *, int);
88 static void	pthread__initmain(pthread_t *);
89 static void	pthread__reap(pthread_t);
90 
91 void	pthread__init(void);
92 
93 int pthread__started;
94 int __uselibcstub = 1;
95 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER;
96 pthread_queue_t pthread__deadqueue;
97 pthread_queue_t pthread__allqueue;
98 
99 static pthread_attr_t pthread_default_attr;
100 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE };
101 
102 enum {
103 	DIAGASSERT_ABORT =	1<<0,
104 	DIAGASSERT_STDERR =	1<<1,
105 	DIAGASSERT_SYSLOG =	1<<2
106 };
107 
108 static int pthread__diagassert;
109 
110 int pthread__concurrency;
111 int pthread__nspins;
112 size_t pthread__unpark_max = PTHREAD__UNPARK_MAX;
113 int pthread__dbg;	/* set by libpthread_dbg if active */
114 
115 /*
116  * We have to initialize the pthread_stack* variables here because
117  * mutexes are used before pthread_init() and thus pthread__initmain()
118  * are called.  Since mutexes only save the stack pointer and not a
119  * pointer to the thread data, it is safe to change the mapping from
120  * stack pointer to thread data afterwards.
121  */
122 size_t	pthread__stacksize;
123 size_t	pthread__guardsize;
124 size_t	pthread__pagesize;
125 static struct __pthread_st *pthread__main;
126 static size_t __pthread_st_size;
127 
128 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *);
129 
130 __strong_alias(__libc_thr_self,pthread_self)
131 __strong_alias(__libc_thr_create,pthread_create)
132 __strong_alias(__libc_thr_exit,pthread_exit)
133 __strong_alias(__libc_thr_errno,pthread__errno)
134 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate)
135 __strong_alias(__libc_thr_equal,pthread_equal)
136 __strong_alias(__libc_thr_init,pthread__init)
137 
138 /*
139  * Static library kludge.  Place a reference to a symbol any library
140  * file which does not already have a reference here.
141  */
142 extern int pthread__cancel_stub_binder;
143 
144 void *pthread__static_lib_binder[] = {
145 	&pthread__cancel_stub_binder,
146 	pthread_cond_init,
147 	pthread_mutex_init,
148 	pthread_rwlock_init,
149 	pthread_barrier_init,
150 	pthread_key_create,
151 	pthread_setspecific,
152 };
153 
154 #define	NHASHLOCK	64
155 
156 static union hashlock {
157 	pthread_mutex_t	mutex;
158 	char		pad[64];
159 } hashlocks[NHASHLOCK] __aligned(64);
160 
161 static void
162 pthread__prefork(void)
163 {
164 	pthread_mutex_lock(&pthread__deadqueue_lock);
165 }
166 
167 static void
168 pthread__fork_parent(void)
169 {
170 	pthread_mutex_unlock(&pthread__deadqueue_lock);
171 }
172 
173 static void
174 pthread__fork_child(void)
175 {
176 	struct __pthread_st *self = pthread__self();
177 
178 	pthread_mutex_init(&pthread__deadqueue_lock, NULL);
179 
180 	/* lwpctl state is not copied across fork. */
181 	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
182 		err(EXIT_FAILURE, "_lwp_ctl");
183 	}
184 	self->pt_lid = _lwp_self();
185 }
186 
187 /*
188  * This needs to be started by the library loading code, before main()
189  * gets to run, for various things that use the state of the initial thread
190  * to work properly (thread-specific data is an application-visible example;
191  * spinlock counts for mutexes is an internal example).
192  */
193 void
194 pthread__init(void)
195 {
196 	pthread_t first;
197 	char *p;
198 	int mib[2];
199 	unsigned int value;
200 	ssize_t slen;
201 	size_t len;
202 	extern int __isthreaded;
203 
204 	/*
205 	 * Allocate pthread_keys descriptors before
206 	 * reseting __uselibcstub because otherwise
207 	 * malloc() will call pthread_keys_create()
208 	 * while pthread_keys descriptors are not
209 	 * yet allocated.
210 	 */
211 	pthread__main = pthread_tsd_init(&__pthread_st_size);
212 	if (pthread__main == NULL)
213 		err(EXIT_FAILURE, "Cannot allocate pthread storage");
214 
215 	__uselibcstub = 0;
216 
217 	pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE);
218 	pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF);
219 
220 	mib[0] = CTL_VM;
221 	mib[1] = VM_THREAD_GUARD_SIZE;
222 	len = sizeof(value);
223 	if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
224 		pthread__guardsize = value;
225 	else
226 		pthread__guardsize = pthread__pagesize;
227 
228 	/* Initialize locks first; they're needed elsewhere. */
229 	pthread__lockprim_init();
230 	for (int i = 0; i < NHASHLOCK; i++) {
231 		pthread_mutex_init(&hashlocks[i].mutex, NULL);
232 	}
233 
234 	/* Fetch parameters. */
235 	slen = _lwp_unpark_all(NULL, 0, NULL);
236 	if (slen < 0)
237 		err(EXIT_FAILURE, "_lwp_unpark_all");
238 	if ((size_t)slen < pthread__unpark_max)
239 		pthread__unpark_max = slen;
240 
241 	/* Basic data structure setup */
242 	pthread_attr_init(&pthread_default_attr);
243 	PTQ_INIT(&pthread__allqueue);
244 	PTQ_INIT(&pthread__deadqueue);
245 
246 	rb_tree_init(&pthread__alltree, &pthread__alltree_ops);
247 
248 	/* Create the thread structure corresponding to main() */
249 	pthread__initmain(&first);
250 	pthread__initthread(first);
251 	pthread__scrubthread(first, NULL, 0);
252 
253 	first->pt_lid = _lwp_self();
254 	PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq);
255 	(void)rb_tree_insert_node(&pthread__alltree, first);
256 
257 	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) {
258 		err(EXIT_FAILURE, "_lwp_ctl");
259 	}
260 
261 	/* Start subsystems */
262 	PTHREAD_MD_INIT
263 
264 	for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) {
265 		switch (*p) {
266 		case 'a':
267 			pthread__diagassert |= DIAGASSERT_ABORT;
268 			break;
269 		case 'A':
270 			pthread__diagassert &= ~DIAGASSERT_ABORT;
271 			break;
272 		case 'e':
273 			pthread__diagassert |= DIAGASSERT_STDERR;
274 			break;
275 		case 'E':
276 			pthread__diagassert &= ~DIAGASSERT_STDERR;
277 			break;
278 		case 'l':
279 			pthread__diagassert |= DIAGASSERT_SYSLOG;
280 			break;
281 		case 'L':
282 			pthread__diagassert &= ~DIAGASSERT_SYSLOG;
283 			break;
284 		}
285 	}
286 
287 	/* Tell libc that we're here and it should role-play accordingly. */
288 	pthread_atfork(pthread__prefork, pthread__fork_parent, pthread__fork_child);
289 	__isthreaded = 1;
290 }
291 
292 /* General-purpose thread data structure sanitization. */
293 /* ARGSUSED */
294 static void
295 pthread__initthread(pthread_t t)
296 {
297 
298 	t->pt_self = t;
299 	t->pt_magic = PT_MAGIC;
300 	t->pt_sleepobj = NULL;
301 	t->pt_havespecific = 0;
302 	t->pt_lwpctl = &pthread__dummy_lwpctl;
303 
304 	memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops));
305 	pthread_mutex_init(&t->pt_lock, NULL);
306 	PTQ_INIT(&t->pt_cleanup_stack);
307 }
308 
309 static void
310 pthread__scrubthread(pthread_t t, char *name, int flags)
311 {
312 
313 	t->pt_state = PT_STATE_RUNNING;
314 	t->pt_exitval = NULL;
315 	t->pt_flags = flags;
316 	t->pt_cancel = 0;
317 	t->pt_errno = 0;
318 	t->pt_name = name;
319 	t->pt_lid = 0;
320 }
321 
322 static int
323 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr)
324 {
325 	void *stackbase, *stackbase2, *redzone;
326 	size_t stacksize, guardsize;
327 	bool allocated;
328 
329 	if (attr != NULL) {
330 		pthread_attr_getstack(attr, &stackbase, &stacksize);
331 		pthread_attr_getguardsize(attr, &guardsize);
332 	} else {
333 		stackbase = NULL;
334 		stacksize = 0;
335 		guardsize = pthread__guardsize;
336 	}
337 	if (stacksize == 0)
338 		stacksize = pthread__stacksize;
339 
340 	if (newthread->pt_stack_allocated) {
341 		if (stackbase == NULL &&
342 		    newthread->pt_stack.ss_size == stacksize &&
343 		    newthread->pt_guardsize == guardsize)
344 			return 0;
345 		stackbase2 = newthread->pt_stack.ss_sp;
346 #ifndef __MACHINE_STACK_GROWS_UP
347 		stackbase2 = (char *)stackbase2 - newthread->pt_guardsize;
348 #endif
349 		munmap(stackbase2,
350 		    newthread->pt_stack.ss_size + newthread->pt_guardsize);
351 		newthread->pt_stack.ss_sp = NULL;
352 		newthread->pt_stack.ss_size = 0;
353 		newthread->pt_guardsize = 0;
354 		newthread->pt_stack_allocated = false;
355 	}
356 
357 	newthread->pt_stack_allocated = false;
358 
359 	if (stackbase == NULL) {
360 		stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1;
361 		guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1;
362 		stackbase = mmap(NULL, stacksize + guardsize,
363 		    PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0);
364 		if (stackbase == MAP_FAILED)
365 			return ENOMEM;
366 		allocated = true;
367 	} else {
368 		allocated = false;
369 	}
370 #ifdef __MACHINE_STACK_GROWS_UP
371 	redzone = (char *)stackbase + stacksize;
372 	stackbase2 = (char *)stackbase;
373 #else
374 	redzone = (char *)stackbase;
375 	stackbase2 = (char *)stackbase + guardsize;
376 #endif
377 	if (allocated && guardsize &&
378 	    mprotect(redzone, guardsize, PROT_NONE) == -1) {
379 		munmap(stackbase, stacksize + guardsize);
380 		return EPERM;
381 	}
382 	newthread->pt_stack.ss_size = stacksize;
383 	newthread->pt_stack.ss_sp = stackbase2;
384 	newthread->pt_guardsize = guardsize;
385 	newthread->pt_stack_allocated = allocated;
386 	return 0;
387 }
388 
389 int
390 pthread_create(pthread_t *thread, const pthread_attr_t *attr,
391 	    void *(*startfunc)(void *), void *arg)
392 {
393 	pthread_t newthread;
394 	pthread_attr_t nattr;
395 	struct pthread_attr_private *p;
396 	char * volatile name;
397 	unsigned long flag;
398 	void *private_area;
399 	int ret;
400 
401 	if (__predict_false(__uselibcstub)) {
402     		pthread__errorfunc(__FILE__, __LINE__, __func__,
403 		    "pthread_create() requires linking with -lpthread");
404 		return __libc_thr_create_stub(thread, attr, startfunc, arg);
405 	}
406 
407 	if (attr == NULL)
408 		nattr = pthread_default_attr;
409 	else if (attr->pta_magic == PT_ATTR_MAGIC)
410 		nattr = *attr;
411 	else
412 		return EINVAL;
413 
414 	pthread__started = 1;
415 
416 	/* Fetch misc. attributes from the attr structure. */
417 	name = NULL;
418 	if ((p = nattr.pta_private) != NULL)
419 		if (p->ptap_name[0] != '\0')
420 			if ((name = strdup(p->ptap_name)) == NULL)
421 				return ENOMEM;
422 
423 	newthread = NULL;
424 
425 	/*
426 	 * Try to reclaim a dead thread.
427 	 */
428 	if (!PTQ_EMPTY(&pthread__deadqueue)) {
429 		pthread_mutex_lock(&pthread__deadqueue_lock);
430 		PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) {
431 			/* Still busily exiting, or finished? */
432 			if (newthread->pt_lwpctl->lc_curcpu ==
433 			    LWPCTL_CPU_EXITED)
434 				break;
435 		}
436 		if (newthread)
437 			PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq);
438 		pthread_mutex_unlock(&pthread__deadqueue_lock);
439 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
440 		if (newthread && newthread->pt_tls) {
441 			_rtld_tls_free(newthread->pt_tls);
442 			newthread->pt_tls = NULL;
443 		}
444 #endif
445 	}
446 
447 	/*
448 	 * If necessary set up a stack, allocate space for a pthread_st,
449 	 * and initialize it.
450 	 */
451 	if (newthread == NULL) {
452 		newthread = calloc(1, __pthread_st_size);
453 		if (newthread == NULL) {
454 			free(name);
455 			return ENOMEM;
456 		}
457 		newthread->pt_stack_allocated = false;
458 
459 		if (pthread__getstack(newthread, attr)) {
460 			free(newthread);
461 			free(name);
462 			return ENOMEM;
463 		}
464 
465 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
466 		newthread->pt_tls = NULL;
467 #endif
468 
469 		/* Add to list of all threads. */
470 		pthread_rwlock_wrlock(&pthread__alltree_lock);
471 		PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq);
472 		(void)rb_tree_insert_node(&pthread__alltree, newthread);
473 		pthread_rwlock_unlock(&pthread__alltree_lock);
474 
475 		/* Will be reset by the thread upon exit. */
476 		pthread__initthread(newthread);
477 	} else {
478 		if (pthread__getstack(newthread, attr)) {
479 			pthread_mutex_lock(&pthread__deadqueue_lock);
480 			PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq);
481 			pthread_mutex_unlock(&pthread__deadqueue_lock);
482 			return ENOMEM;
483 		}
484 	}
485 
486 	/*
487 	 * Create the new LWP.
488 	 */
489 	pthread__scrubthread(newthread, name, nattr.pta_flags);
490 	newthread->pt_func = startfunc;
491 	newthread->pt_arg = arg;
492 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
493 	private_area = newthread->pt_tls = _rtld_tls_allocate();
494 	newthread->pt_tls->tcb_pthread = newthread;
495 #else
496 	private_area = newthread;
497 #endif
498 
499 	flag = 0;
500 	if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 ||
501 	    (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0)
502 		flag |= LWP_SUSPENDED;
503 	if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0)
504 		flag |= LWP_DETACHED;
505 
506 	ret = pthread__makelwp(pthread__create_tramp, newthread, private_area,
507 	    newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size,
508 	    flag, &newthread->pt_lid);
509 	if (ret != 0) {
510 		ret = errno;
511 		pthread_mutex_lock(&newthread->pt_lock);
512 		/* Will unlock and free name. */
513 		pthread__reap(newthread);
514 		return ret;
515 	}
516 
517 	if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) {
518 		if (p != NULL) {
519 			(void)pthread_setschedparam(newthread, p->ptap_policy,
520 			    &p->ptap_sp);
521 		}
522 		if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) {
523 			(void)_lwp_continue(newthread->pt_lid);
524 		}
525 	}
526 
527 	*thread = newthread;
528 
529 	return 0;
530 }
531 
532 
533 __dead static void
534 pthread__create_tramp(void *cookie)
535 {
536 	pthread_t self;
537 	void *retval;
538 
539 	self = cookie;
540 
541 	/*
542 	 * Throw away some stack in a feeble attempt to reduce cache
543 	 * thrash.  May help for SMT processors.  XXX We should not
544 	 * be allocating stacks on fixed 2MB boundaries.  Needs a
545 	 * thread register or decent thread local storage.
546 	 */
547 	(void)alloca(((unsigned)self->pt_lid & 7) << 8);
548 
549 	if (self->pt_name != NULL) {
550 		pthread_mutex_lock(&self->pt_lock);
551 		if (self->pt_name != NULL)
552 			(void)_lwp_setname(0, self->pt_name);
553 		pthread_mutex_unlock(&self->pt_lock);
554 	}
555 
556 	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
557 		err(EXIT_FAILURE, "_lwp_ctl");
558 	}
559 
560 	retval = (*self->pt_func)(self->pt_arg);
561 
562 	pthread_exit(retval);
563 
564 	/*NOTREACHED*/
565 	pthread__abort();
566 }
567 
568 int
569 pthread_suspend_np(pthread_t thread)
570 {
571 	pthread_t self;
572 
573 	pthread__error(EINVAL, "Invalid thread",
574 	    thread->pt_magic == PT_MAGIC);
575 
576 	self = pthread__self();
577 	if (self == thread) {
578 		return EDEADLK;
579 	}
580 	if (pthread__find(thread) != 0)
581 		return ESRCH;
582 	if (_lwp_suspend(thread->pt_lid) == 0)
583 		return 0;
584 	return errno;
585 }
586 
587 int
588 pthread_resume_np(pthread_t thread)
589 {
590 
591 	pthread__error(EINVAL, "Invalid thread",
592 	    thread->pt_magic == PT_MAGIC);
593 
594 	if (pthread__find(thread) != 0)
595 		return ESRCH;
596 	if (_lwp_continue(thread->pt_lid) == 0)
597 		return 0;
598 	return errno;
599 }
600 
601 void
602 pthread_exit(void *retval)
603 {
604 	pthread_t self;
605 	struct pt_clean_t *cleanup;
606 
607 	if (__predict_false(__uselibcstub)) {
608 		__libc_thr_exit_stub(retval);
609 		goto out;
610 	}
611 
612 	self = pthread__self();
613 
614 	/* Disable cancellability. */
615 	pthread_mutex_lock(&self->pt_lock);
616 	self->pt_flags |= PT_FLAG_CS_DISABLED;
617 	self->pt_cancel = 0;
618 
619 	/* Call any cancellation cleanup handlers */
620 	if (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
621 		pthread_mutex_unlock(&self->pt_lock);
622 		while (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
623 			cleanup = PTQ_FIRST(&self->pt_cleanup_stack);
624 			PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next);
625 			(*cleanup->ptc_cleanup)(cleanup->ptc_arg);
626 		}
627 		pthread_mutex_lock(&self->pt_lock);
628 	}
629 
630 	pthread_mutex_unlock(&self->pt_lock);
631 	__cxa_thread_run_atexit();
632 	pthread_mutex_lock(&self->pt_lock);
633 
634 	/* Perform cleanup of thread-specific data */
635 	pthread__destroy_tsd(self);
636 
637 	if (_malloc_thread_cleanup)
638 		_malloc_thread_cleanup();
639 
640 	/*
641 	 * Signal our exit.  Our stack and pthread_t won't be reused until
642 	 * pthread_create() can see from kernel info that this LWP is gone.
643 	 */
644 	self->pt_exitval = retval;
645 	if (self->pt_flags & PT_FLAG_DETACHED) {
646 		/* pthread__reap() will drop the lock. */
647 		pthread__reap(self);
648 		_lwp_exit();
649 	} else {
650 		self->pt_state = PT_STATE_ZOMBIE;
651 		pthread_mutex_unlock(&self->pt_lock);
652 		/* Note: name will be freed by the joiner. */
653 		_lwp_exit();
654 	}
655 
656 out:
657 	/*NOTREACHED*/
658 	pthread__abort();
659 	exit(1);
660 }
661 
662 
663 int
664 pthread_join(pthread_t thread, void **valptr)
665 {
666 	pthread_t self;
667 
668 	pthread__error(EINVAL, "Invalid thread",
669 	    thread->pt_magic == PT_MAGIC);
670 
671 	self = pthread__self();
672 
673 	if (pthread__find(thread) != 0)
674 		return ESRCH;
675 
676 	if (thread == self)
677 		return EDEADLK;
678 
679 	/* IEEE Std 1003.1 says pthread_join() never returns EINTR. */
680 	for (;;) {
681 		pthread__testcancel(self);
682 		if (_lwp_wait(thread->pt_lid, NULL) == 0)
683 			break;
684 		if (errno != EINTR)
685 			return errno;
686 	}
687 
688 	/*
689 	 * Don't test for cancellation again.  The spec is that if
690 	 * cancelled, pthread_join() must not have succeeded.
691 	 */
692 	pthread_mutex_lock(&thread->pt_lock);
693 	if (thread->pt_state != PT_STATE_ZOMBIE) {
694 		pthread__errorfunc(__FILE__, __LINE__, __func__,
695 		    "not a zombie");
696  	}
697 	if (valptr != NULL)
698 		*valptr = thread->pt_exitval;
699 
700 	/* pthread__reap() will drop the lock. */
701 	pthread__reap(thread);
702 	return 0;
703 }
704 
705 static void
706 pthread__reap(pthread_t thread)
707 {
708 	char *name;
709 
710 	name = thread->pt_name;
711 	thread->pt_name = NULL;
712 	thread->pt_state = PT_STATE_DEAD;
713 	pthread_mutex_unlock(&thread->pt_lock);
714 
715 	pthread_mutex_lock(&pthread__deadqueue_lock);
716 	PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq);
717 	pthread_mutex_unlock(&pthread__deadqueue_lock);
718 
719 	if (name != NULL)
720 		free(name);
721 }
722 
723 int
724 pthread_equal(pthread_t t1, pthread_t t2)
725 {
726 
727 	if (__predict_false(__uselibcstub))
728 		return __libc_thr_equal_stub(t1, t2);
729 
730 	pthread__error(0, "Invalid thread",
731 	    (t1 != NULL) && (t1->pt_magic == PT_MAGIC));
732 
733 	pthread__error(0, "Invalid thread",
734 	    (t2 != NULL) && (t2->pt_magic == PT_MAGIC));
735 
736 	/* Nothing special here. */
737 	return (t1 == t2);
738 }
739 
740 
741 int
742 pthread_detach(pthread_t thread)
743 {
744 	int error;
745 
746 	pthread__error(EINVAL, "Invalid thread",
747 	    thread->pt_magic == PT_MAGIC);
748 
749 	if (pthread__find(thread) != 0)
750 		return ESRCH;
751 
752 	pthread_mutex_lock(&thread->pt_lock);
753 	if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) {
754 		error = EINVAL;
755 	} else {
756 		error = _lwp_detach(thread->pt_lid);
757 		if (error == 0)
758 			thread->pt_flags |= PT_FLAG_DETACHED;
759 		else
760 			error = errno;
761 	}
762 	if (thread->pt_state == PT_STATE_ZOMBIE) {
763 		/* pthread__reap() will drop the lock. */
764 		pthread__reap(thread);
765 	} else
766 		pthread_mutex_unlock(&thread->pt_lock);
767 	return error;
768 }
769 
770 
771 int
772 pthread_getname_np(pthread_t thread, char *name, size_t len)
773 {
774 
775 	pthread__error(EINVAL, "Invalid thread",
776 	    thread->pt_magic == PT_MAGIC);
777 
778 	if (pthread__find(thread) != 0)
779 		return ESRCH;
780 
781 	pthread_mutex_lock(&thread->pt_lock);
782 	if (thread->pt_name == NULL)
783 		name[0] = '\0';
784 	else
785 		strlcpy(name, thread->pt_name, len);
786 	pthread_mutex_unlock(&thread->pt_lock);
787 
788 	return 0;
789 }
790 
791 
792 int
793 pthread_setname_np(pthread_t thread, const char *name, void *arg)
794 {
795 	char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP];
796 	int namelen;
797 
798 	pthread__error(EINVAL, "Invalid thread",
799 	    thread->pt_magic == PT_MAGIC);
800 
801 	if (pthread__find(thread) != 0)
802 		return ESRCH;
803 
804 	namelen = snprintf(newname, sizeof(newname), name, arg);
805 	if (namelen >= PTHREAD_MAX_NAMELEN_NP)
806 		return EINVAL;
807 
808 	cp = strdup(newname);
809 	if (cp == NULL)
810 		return ENOMEM;
811 
812 	pthread_mutex_lock(&thread->pt_lock);
813 	oldname = thread->pt_name;
814 	thread->pt_name = cp;
815 	(void)_lwp_setname(thread->pt_lid, cp);
816 	pthread_mutex_unlock(&thread->pt_lock);
817 
818 	if (oldname != NULL)
819 		free(oldname);
820 
821 	return 0;
822 }
823 
824 
825 pthread_t
826 pthread_self(void)
827 {
828 	if (__predict_false(__uselibcstub))
829 		return (pthread_t)__libc_thr_self_stub();
830 
831 	return pthread__self();
832 }
833 
834 
835 int
836 pthread_cancel(pthread_t thread)
837 {
838 
839 	pthread__error(EINVAL, "Invalid thread",
840 	    thread->pt_magic == PT_MAGIC);
841 
842 	if (pthread__find(thread) != 0)
843 		return ESRCH;
844 	pthread_mutex_lock(&thread->pt_lock);
845 	thread->pt_flags |= PT_FLAG_CS_PENDING;
846 	if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) {
847 		thread->pt_cancel = 1;
848 		pthread_mutex_unlock(&thread->pt_lock);
849 		_lwp_wakeup(thread->pt_lid);
850 	} else
851 		pthread_mutex_unlock(&thread->pt_lock);
852 
853 	return 0;
854 }
855 
856 
857 int
858 pthread_setcancelstate(int state, int *oldstate)
859 {
860 	pthread_t self;
861 	int retval;
862 
863 	if (__predict_false(__uselibcstub))
864 		return __libc_thr_setcancelstate_stub(state, oldstate);
865 
866 	self = pthread__self();
867 	retval = 0;
868 
869 	pthread_mutex_lock(&self->pt_lock);
870 
871 	if (oldstate != NULL) {
872 		if (self->pt_flags & PT_FLAG_CS_DISABLED)
873 			*oldstate = PTHREAD_CANCEL_DISABLE;
874 		else
875 			*oldstate = PTHREAD_CANCEL_ENABLE;
876 	}
877 
878 	if (state == PTHREAD_CANCEL_DISABLE) {
879 		self->pt_flags |= PT_FLAG_CS_DISABLED;
880 		if (self->pt_cancel) {
881 			self->pt_flags |= PT_FLAG_CS_PENDING;
882 			self->pt_cancel = 0;
883 		}
884 	} else if (state == PTHREAD_CANCEL_ENABLE) {
885 		self->pt_flags &= ~PT_FLAG_CS_DISABLED;
886 		/*
887 		 * If a cancellation was requested while cancellation
888 		 * was disabled, note that fact for future
889 		 * cancellation tests.
890 		 */
891 		if (self->pt_flags & PT_FLAG_CS_PENDING) {
892 			self->pt_cancel = 1;
893 			/* This is not a deferred cancellation point. */
894 			if (self->pt_flags & PT_FLAG_CS_ASYNC) {
895 				pthread_mutex_unlock(&self->pt_lock);
896 				pthread__cancelled();
897 			}
898 		}
899 	} else
900 		retval = EINVAL;
901 
902 	pthread_mutex_unlock(&self->pt_lock);
903 
904 	return retval;
905 }
906 
907 
908 int
909 pthread_setcanceltype(int type, int *oldtype)
910 {
911 	pthread_t self;
912 	int retval;
913 
914 	self = pthread__self();
915 	retval = 0;
916 
917 	pthread_mutex_lock(&self->pt_lock);
918 
919 	if (oldtype != NULL) {
920 		if (self->pt_flags & PT_FLAG_CS_ASYNC)
921 			*oldtype = PTHREAD_CANCEL_ASYNCHRONOUS;
922 		else
923 			*oldtype = PTHREAD_CANCEL_DEFERRED;
924 	}
925 
926 	if (type == PTHREAD_CANCEL_ASYNCHRONOUS) {
927 		self->pt_flags |= PT_FLAG_CS_ASYNC;
928 		if (self->pt_cancel) {
929 			pthread_mutex_unlock(&self->pt_lock);
930 			pthread__cancelled();
931 		}
932 	} else if (type == PTHREAD_CANCEL_DEFERRED)
933 		self->pt_flags &= ~PT_FLAG_CS_ASYNC;
934 	else
935 		retval = EINVAL;
936 
937 	pthread_mutex_unlock(&self->pt_lock);
938 
939 	return retval;
940 }
941 
942 
943 void
944 pthread_testcancel(void)
945 {
946 	pthread_t self;
947 
948 	self = pthread__self();
949 	if (self->pt_cancel)
950 		pthread__cancelled();
951 }
952 
953 
954 /*
955  * POSIX requires that certain functions return an error rather than
956  * invoking undefined behavior even when handed completely bogus
957  * pthread_t values, e.g. stack garbage.
958  */
959 int
960 pthread__find(pthread_t id)
961 {
962 	pthread_t target;
963 	int error;
964 
965 	pthread_rwlock_rdlock(&pthread__alltree_lock);
966 	target = rb_tree_find_node(&pthread__alltree, id);
967 	error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH;
968 	pthread_rwlock_unlock(&pthread__alltree_lock);
969 
970 	return error;
971 }
972 
973 
974 void
975 pthread__testcancel(pthread_t self)
976 {
977 
978 	if (self->pt_cancel)
979 		pthread__cancelled();
980 }
981 
982 
983 void
984 pthread__cancelled(void)
985 {
986 
987 	pthread_exit(PTHREAD_CANCELED);
988 }
989 
990 
991 void
992 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store)
993 {
994 	pthread_t self;
995 	struct pt_clean_t *entry;
996 
997 	self = pthread__self();
998 	entry = store;
999 	entry->ptc_cleanup = cleanup;
1000 	entry->ptc_arg = arg;
1001 	PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next);
1002 }
1003 
1004 
1005 void
1006 pthread__cleanup_pop(int ex, void *store)
1007 {
1008 	pthread_t self;
1009 	struct pt_clean_t *entry;
1010 
1011 	self = pthread__self();
1012 	entry = store;
1013 
1014 	PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next);
1015 	if (ex)
1016 		(*entry->ptc_cleanup)(entry->ptc_arg);
1017 }
1018 
1019 
1020 int *
1021 pthread__errno(void)
1022 {
1023 	pthread_t self;
1024 
1025 	if (__predict_false(__uselibcstub)) {
1026     		pthread__errorfunc(__FILE__, __LINE__, __func__,
1027 		    "pthread__errno() requires linking with -lpthread");
1028 		return __libc_thr_errno_stub();
1029 	}
1030 
1031 	self = pthread__self();
1032 
1033 	return &(self->pt_errno);
1034 }
1035 
1036 ssize_t	_sys_write(int, const void *, size_t);
1037 
1038 void
1039 pthread__assertfunc(const char *file, int line, const char *function,
1040 		    const char *expr)
1041 {
1042 	char buf[1024];
1043 	int len;
1044 
1045 	/*
1046 	 * snprintf should not acquire any locks, or we could
1047 	 * end up deadlocked if the assert caller held locks.
1048 	 */
1049 	len = snprintf(buf, 1024,
1050 	    "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n",
1051 	    expr, file, line,
1052 	    function ? ", function \"" : "",
1053 	    function ? function : "",
1054 	    function ? "\"" : "");
1055 
1056 	_sys_write(STDERR_FILENO, buf, (size_t)len);
1057 	(void)raise(SIGABRT);
1058 	_exit(1);
1059 }
1060 
1061 
1062 void
1063 pthread__errorfunc(const char *file, int line, const char *function,
1064 		   const char *msg, ...)
1065 {
1066 	char buf[1024];
1067 	char buf2[1024];
1068 	size_t len;
1069 	va_list ap;
1070 
1071 	if (pthread__diagassert == 0)
1072 		return;
1073 
1074 	va_start(ap, msg);
1075 	vsnprintf_ss(buf2, sizeof(buf2), msg, ap);
1076 	va_end(ap);
1077 
1078 	/*
1079 	 * snprintf should not acquire any locks, or we could
1080 	 * end up deadlocked if the assert caller held locks.
1081 	 */
1082 	len = snprintf_ss(buf, sizeof(buf),
1083 	    "%s: Error detected by libpthread: %s.\n"
1084 	    "Detected by file \"%s\", line %d%s%s%s.\n"
1085 	    "See pthread(3) for information.\n",
1086 	    getprogname(), buf2, file, line,
1087 	    function ? ", function \"" : "",
1088 	    function ? function : "",
1089 	    function ? "\"" : "");
1090 
1091 	if (pthread__diagassert & DIAGASSERT_STDERR)
1092 		_sys_write(STDERR_FILENO, buf, len);
1093 
1094 	if (pthread__diagassert & DIAGASSERT_SYSLOG)
1095 		syslog(LOG_DEBUG | LOG_USER, "%s", buf);
1096 
1097 	if (pthread__diagassert & DIAGASSERT_ABORT) {
1098 		(void)_lwp_kill(_lwp_self(), SIGABRT);
1099 		_exit(1);
1100 	}
1101 }
1102 
1103 /*
1104  * Thread park/unpark operations.  The kernel operations are
1105  * modelled after a brief description from "Multithreading in
1106  * the Solaris Operating Environment":
1107  *
1108  * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf
1109  */
1110 
1111 int
1112 pthread__park(pthread_t self, pthread_mutex_t *lock,
1113 	      pthread_queue_t *queue, const struct timespec *abstime,
1114 	      int cancelpt)
1115 {
1116 	int rv, error;
1117 
1118 	pthread_mutex_unlock(lock);
1119 
1120 	/*
1121 	 * Wait until we are awoken by a pending unpark operation,
1122 	 * a signal, an unpark posted after we have gone asleep,
1123 	 * or an expired timeout.
1124 	 *
1125 	 * It is fine to test the value of pt_sleepobj without
1126 	 * holding any locks, because:
1127 	 *
1128 	 * o Only the blocking thread (this thread) ever sets it
1129 	 *   to a non-NULL value.
1130 	 *
1131 	 * o Other threads may set it NULL, but if they do so they
1132 	 *   must also make this thread return from _lwp_park.
1133 	 *
1134 	 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system
1135 	 *   calls and all make use of spinlocks in the kernel.  So
1136 	 *   these system calls act as full memory barriers.
1137 	 */
1138 	rv = 0;
1139 	do {
1140 		/*
1141 		 * If we deferred unparking a thread, arrange to
1142 		 * have _lwp_park() restart it before blocking.
1143 		 */
1144 		error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME,
1145 		    __UNCONST(abstime), 0, NULL, NULL);
1146 		if (error != 0) {
1147 			switch (rv = errno) {
1148 			case EINTR:
1149 			case EALREADY:
1150 			case ESRCH:
1151 				rv = 0;
1152 				break;
1153 			case ETIMEDOUT:
1154 				break;
1155 			default:
1156 				pthread__errorfunc(__FILE__, __LINE__,
1157 				    __func__, "_lwp_park failed: %d", errno);
1158 				break;
1159 			}
1160 		}
1161 		/* Check for cancellation. */
1162 		if (cancelpt && self->pt_cancel)
1163 			rv = EINTR;
1164 	} while (self->pt_sleepobj != NULL && rv == 0);
1165 	return rv;
1166 }
1167 
1168 void
1169 pthread__unpark(pthread_queue_t *queue, pthread_t self,
1170 		pthread_mutex_t *interlock)
1171 {
1172 	pthread_t target;
1173 
1174 	target = PTQ_FIRST(queue);
1175 	target->pt_sleepobj = NULL;
1176 	PTQ_REMOVE(queue, target, pt_sleep);
1177 	(void)_lwp_unpark(target->pt_lid, NULL);
1178 }
1179 
1180 void
1181 pthread__unpark_all(pthread_queue_t *queue, pthread_t self,
1182 		    pthread_mutex_t *interlock)
1183 {
1184 	lwpid_t lids[PTHREAD__UNPARK_MAX];
1185 	const size_t mlid = pthread__unpark_max;
1186 	pthread_t target;
1187 	size_t nlid = 0;
1188 
1189 	PTQ_FOREACH(target, queue, pt_sleep) {
1190 		if (nlid == mlid) {
1191 			(void)_lwp_unpark_all(lids, nlid, NULL);
1192 			nlid = 0;
1193 		}
1194 		target->pt_sleepobj = NULL;
1195 		lids[nlid++] = target->pt_lid;
1196 	}
1197 	PTQ_INIT(queue);
1198 	if (nlid == 1) {
1199 		(void)_lwp_unpark(lids[0], NULL);
1200 	} else if (nlid > 1) {
1201 		(void)_lwp_unpark_all(lids, nlid, NULL);
1202 	}
1203 }
1204 
1205 #undef	OOPS
1206 
1207 static void
1208 pthread__initmainstack(void)
1209 {
1210 	struct rlimit slimit;
1211 	const AuxInfo *aux;
1212 	size_t size, len;
1213 	int mib[2];
1214 	unsigned int value;
1215 
1216 	_DIAGASSERT(_dlauxinfo() != NULL);
1217 
1218 	if (getrlimit(RLIMIT_STACK, &slimit) == -1)
1219 		err(EXIT_FAILURE,
1220 		    "Couldn't get stack resource consumption limits");
1221 	size = slimit.rlim_cur;
1222 	pthread__main->pt_stack.ss_size = size;
1223 	pthread__main->pt_guardsize = pthread__pagesize;
1224 
1225 	mib[0] = CTL_VM;
1226 	mib[1] = VM_GUARD_SIZE;
1227 	len = sizeof(value);
1228 	if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
1229 		pthread__main->pt_guardsize = value;
1230 
1231 	for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) {
1232 		if (aux->a_type == AT_STACKBASE) {
1233 #ifdef __MACHINE_STACK_GROWS_UP
1234 			pthread__main->pt_stack.ss_sp = (void *)aux->a_v;
1235 #else
1236 			pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size;
1237 #endif
1238 			break;
1239 		}
1240 	}
1241 	pthread__copy_tsd(pthread__main);
1242 }
1243 
1244 /*
1245  * Set up the slightly special stack for the "initial" thread, which
1246  * runs on the normal system stack, and thus gets slightly different
1247  * treatment.
1248  */
1249 static void
1250 pthread__initmain(pthread_t *newt)
1251 {
1252 	char *value;
1253 
1254 	pthread__initmainstack();
1255 
1256 	value = pthread__getenv("PTHREAD_STACKSIZE");
1257 	if (value != NULL) {
1258 		pthread__stacksize = atoi(value) * 1024;
1259 		if (pthread__stacksize > pthread__main->pt_stack.ss_size)
1260 			pthread__stacksize = pthread__main->pt_stack.ss_size;
1261 	}
1262 	if (pthread__stacksize == 0)
1263 		pthread__stacksize = pthread__main->pt_stack.ss_size;
1264 	pthread__stacksize += pthread__pagesize - 1;
1265 	pthread__stacksize &= ~(pthread__pagesize - 1);
1266 	if (pthread__stacksize < 4 * pthread__pagesize)
1267 		errx(1, "Stacksize limit is too low, minimum %zd kbyte.",
1268 		    4 * pthread__pagesize / 1024);
1269 
1270 	*newt = pthread__main;
1271 #if defined(_PTHREAD_GETTCB_EXT)
1272 	pthread__main->pt_tls = _PTHREAD_GETTCB_EXT();
1273 #elif defined(__HAVE___LWP_GETTCB_FAST)
1274 	pthread__main->pt_tls = __lwp_gettcb_fast();
1275 #else
1276 	pthread__main->pt_tls = _lwp_getprivate();
1277 #endif
1278 	pthread__main->pt_tls->tcb_pthread = pthread__main;
1279 }
1280 
1281 static signed int
1282 /*ARGSUSED*/
1283 pthread__cmp(void *ctx, const void *n1, const void *n2)
1284 {
1285 	const uintptr_t p1 = (const uintptr_t)n1;
1286 	const uintptr_t p2 = (const uintptr_t)n2;
1287 
1288 	if (p1 < p2)
1289 		return -1;
1290 	if (p1 > p2)
1291 		return 1;
1292 	return 0;
1293 }
1294 
1295 /* Because getenv() wants to use locks. */
1296 char *
1297 pthread__getenv(const char *name)
1298 {
1299 	extern char **environ;
1300 	size_t l_name, offset;
1301 
1302 	if (issetugid())
1303 		return (NULL);
1304 
1305 	l_name = strlen(name);
1306 	for (offset = 0; environ[offset] != NULL; offset++) {
1307 		if (strncmp(name, environ[offset], l_name) == 0 &&
1308 		    environ[offset][l_name] == '=') {
1309 			return environ[offset] + l_name + 1;
1310 		}
1311 	}
1312 
1313 	return NULL;
1314 }
1315 
1316 pthread_mutex_t *
1317 pthread__hashlock(volatile const void *p)
1318 {
1319 	uintptr_t v;
1320 
1321 	v = (uintptr_t)p;
1322 	return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex;
1323 }
1324 
1325 int
1326 pthread__checkpri(int pri)
1327 {
1328 	static int havepri;
1329 	static long min, max;
1330 
1331 	if (!havepri) {
1332 		min = sysconf(_SC_SCHED_PRI_MIN);
1333 		max = sysconf(_SC_SCHED_PRI_MAX);
1334 		havepri = 1;
1335 	}
1336 	return (pri < min || pri > max) ? EINVAL : 0;
1337 }
1338