xref: /netbsd-src/lib/libpthread/pthread.c (revision 2718af68c3efc72c9769069b5c7f9ed36f6b9def)
1 /*	$NetBSD: pthread.c,v 1.180 2022/02/12 14:59:32 riastradh Exp $	*/
2 
3 /*-
4  * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020
5  *     The NetBSD Foundation, Inc.
6  * All rights reserved.
7  *
8  * This code is derived from software contributed to The NetBSD Foundation
9  * by Nathan J. Williams and Andrew Doran.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 #include <sys/cdefs.h>
34 __RCSID("$NetBSD: pthread.c,v 1.180 2022/02/12 14:59:32 riastradh Exp $");
35 
36 #define	__EXPOSE_STACK	1
37 
38 /* Need to use libc-private names for atomic operations. */
39 #include "../../common/lib/libc/atomic/atomic_op_namespace.h"
40 
41 #include <sys/param.h>
42 #include <sys/exec_elf.h>
43 #include <sys/mman.h>
44 #include <sys/lwp.h>
45 #include <sys/lwpctl.h>
46 #include <sys/resource.h>
47 #include <sys/sysctl.h>
48 #include <sys/tls.h>
49 #include <uvm/uvm_param.h>
50 
51 #include <assert.h>
52 #include <dlfcn.h>
53 #include <err.h>
54 #include <errno.h>
55 #include <lwp.h>
56 #include <signal.h>
57 #include <stdio.h>
58 #include <stdlib.h>
59 #include <stddef.h>
60 #include <string.h>
61 #include <syslog.h>
62 #include <ucontext.h>
63 #include <unistd.h>
64 #include <sched.h>
65 
66 #include "atexit.h"
67 #include "pthread.h"
68 #include "pthread_int.h"
69 #include "pthread_makelwp.h"
70 #include "reentrant.h"
71 
72 __BEGIN_DECLS
73 void _malloc_thread_cleanup(void) __weak;
74 __END_DECLS
75 
76 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER;
77 static rb_tree_t	pthread__alltree;
78 
79 static signed int	pthread__cmp(void *, const void *, const void *);
80 
81 static const rb_tree_ops_t pthread__alltree_ops = {
82 	.rbto_compare_nodes = pthread__cmp,
83 	.rbto_compare_key = pthread__cmp,
84 	.rbto_node_offset = offsetof(struct __pthread_st, pt_alltree),
85 	.rbto_context = NULL
86 };
87 
88 static void	pthread__create_tramp(void *);
89 static void	pthread__initthread(pthread_t);
90 static void	pthread__scrubthread(pthread_t, char *, int);
91 static void	pthread__initmain(pthread_t *);
92 static void	pthread__reap(pthread_t);
93 
94 void	pthread__init(void);
95 
96 int pthread__started;
97 int __uselibcstub = 1;
98 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER;
99 pthread_queue_t pthread__deadqueue;
100 pthread_queue_t pthread__allqueue;
101 
102 static pthread_attr_t pthread_default_attr;
103 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE };
104 
105 enum {
106 	DIAGASSERT_ABORT =	1<<0,
107 	DIAGASSERT_STDERR =	1<<1,
108 	DIAGASSERT_SYSLOG =	1<<2
109 };
110 
111 static int pthread__diagassert;
112 
113 int pthread__concurrency;
114 int pthread__nspins;
115 size_t pthread__unpark_max = PTHREAD__UNPARK_MAX;
116 int pthread__dbg;	/* set by libpthread_dbg if active */
117 
118 /*
119  * We have to initialize the pthread_stack* variables here because
120  * mutexes are used before pthread_init() and thus pthread__initmain()
121  * are called.  Since mutexes only save the stack pointer and not a
122  * pointer to the thread data, it is safe to change the mapping from
123  * stack pointer to thread data afterwards.
124  */
125 size_t	pthread__stacksize;
126 size_t	pthread__guardsize;
127 size_t	pthread__pagesize;
128 static struct __pthread_st *pthread__main;
129 static size_t __pthread_st_size;
130 
131 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *);
132 
133 __strong_alias(__libc_thr_self,pthread_self)
134 __strong_alias(__libc_thr_create,pthread_create)
135 __strong_alias(__libc_thr_exit,pthread_exit)
136 __strong_alias(__libc_thr_errno,pthread__errno)
137 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate)
138 __strong_alias(__libc_thr_equal,pthread_equal)
139 __strong_alias(__libc_thr_init,pthread__init)
140 
141 /*
142  * Static library kludge.  Place a reference to a symbol any library
143  * file which does not already have a reference here.
144  */
145 extern int pthread__cancel_stub_binder;
146 
147 void *pthread__static_lib_binder[] = {
148 	&pthread__cancel_stub_binder,
149 	pthread_cond_init,
150 	pthread_mutex_init,
151 	pthread_rwlock_init,
152 	pthread_barrier_init,
153 	pthread_key_create,
154 	pthread_setspecific,
155 };
156 
157 #define	NHASHLOCK	64
158 
159 static union hashlock {
160 	pthread_mutex_t	mutex;
161 	char		pad[64];
162 } hashlocks[NHASHLOCK] __aligned(64);
163 
164 static void
165 pthread__prefork(void)
166 {
167 	pthread_mutex_lock(&pthread__deadqueue_lock);
168 }
169 
170 static void
171 pthread__fork_parent(void)
172 {
173 	pthread_mutex_unlock(&pthread__deadqueue_lock);
174 }
175 
176 static void
177 pthread__fork_child(void)
178 {
179 	struct __pthread_st *self = pthread__self();
180 
181 	pthread_mutex_init(&pthread__deadqueue_lock, NULL);
182 
183 	/* lwpctl state is not copied across fork. */
184 	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
185 		err(EXIT_FAILURE, "_lwp_ctl");
186 	}
187 	self->pt_lid = _lwp_self();
188 }
189 
190 /*
191  * This needs to be started by the library loading code, before main()
192  * gets to run, for various things that use the state of the initial thread
193  * to work properly (thread-specific data is an application-visible example;
194  * spinlock counts for mutexes is an internal example).
195  */
196 void
197 pthread__init(void)
198 {
199 	pthread_t first;
200 	char *p;
201 	int mib[2];
202 	unsigned int value;
203 	ssize_t slen;
204 	size_t len;
205 	extern int __isthreaded;
206 
207 	/*
208 	 * Allocate pthread_keys descriptors before
209 	 * resetting __uselibcstub because otherwise
210 	 * malloc() will call pthread_keys_create()
211 	 * while pthread_keys descriptors are not
212 	 * yet allocated.
213 	 */
214 	pthread__main = pthread_tsd_init(&__pthread_st_size);
215 	if (pthread__main == NULL)
216 		err(EXIT_FAILURE, "Cannot allocate pthread storage");
217 
218 	__uselibcstub = 0;
219 
220 	pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE);
221 	pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF);
222 
223 	mib[0] = CTL_VM;
224 	mib[1] = VM_THREAD_GUARD_SIZE;
225 	len = sizeof(value);
226 	if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
227 		pthread__guardsize = value;
228 	else
229 		pthread__guardsize = pthread__pagesize;
230 
231 	/* Initialize locks first; they're needed elsewhere. */
232 	pthread__lockprim_init();
233 	for (int i = 0; i < NHASHLOCK; i++) {
234 		pthread_mutex_init(&hashlocks[i].mutex, NULL);
235 	}
236 
237 	/* Fetch parameters. */
238 	slen = _lwp_unpark_all(NULL, 0, NULL);
239 	if (slen < 0)
240 		err(EXIT_FAILURE, "_lwp_unpark_all");
241 	if ((size_t)slen < pthread__unpark_max)
242 		pthread__unpark_max = slen;
243 
244 	/* Basic data structure setup */
245 	pthread_attr_init(&pthread_default_attr);
246 	PTQ_INIT(&pthread__allqueue);
247 	PTQ_INIT(&pthread__deadqueue);
248 
249 	rb_tree_init(&pthread__alltree, &pthread__alltree_ops);
250 
251 	/* Create the thread structure corresponding to main() */
252 	pthread__initmain(&first);
253 	pthread__initthread(first);
254 	pthread__scrubthread(first, NULL, 0);
255 
256 	first->pt_lid = _lwp_self();
257 	PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq);
258 	(void)rb_tree_insert_node(&pthread__alltree, first);
259 
260 	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) {
261 		err(EXIT_FAILURE, "_lwp_ctl");
262 	}
263 
264 	/* Start subsystems */
265 	PTHREAD_MD_INIT
266 
267 	for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) {
268 		switch (*p) {
269 		case 'a':
270 			pthread__diagassert |= DIAGASSERT_ABORT;
271 			break;
272 		case 'A':
273 			pthread__diagassert &= ~DIAGASSERT_ABORT;
274 			break;
275 		case 'e':
276 			pthread__diagassert |= DIAGASSERT_STDERR;
277 			break;
278 		case 'E':
279 			pthread__diagassert &= ~DIAGASSERT_STDERR;
280 			break;
281 		case 'l':
282 			pthread__diagassert |= DIAGASSERT_SYSLOG;
283 			break;
284 		case 'L':
285 			pthread__diagassert &= ~DIAGASSERT_SYSLOG;
286 			break;
287 		}
288 	}
289 
290 	/* Tell libc that we're here and it should role-play accordingly. */
291 	pthread_atfork(pthread__prefork, pthread__fork_parent, pthread__fork_child);
292 	__isthreaded = 1;
293 }
294 
295 /* General-purpose thread data structure sanitization. */
296 /* ARGSUSED */
297 static void
298 pthread__initthread(pthread_t t)
299 {
300 
301 	t->pt_self = t;
302 	t->pt_magic = PT_MAGIC;
303 	t->pt_sleepobj = NULL;
304 	t->pt_havespecific = 0;
305 	t->pt_lwpctl = &pthread__dummy_lwpctl;
306 
307 	memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops));
308 	pthread_mutex_init(&t->pt_lock, NULL);
309 	PTQ_INIT(&t->pt_cleanup_stack);
310 }
311 
312 static void
313 pthread__scrubthread(pthread_t t, char *name, int flags)
314 {
315 
316 	t->pt_state = PT_STATE_RUNNING;
317 	t->pt_exitval = NULL;
318 	t->pt_flags = flags;
319 	t->pt_cancel = 0;
320 	t->pt_errno = 0;
321 	t->pt_name = name;
322 	t->pt_lid = 0;
323 }
324 
325 static int
326 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr)
327 {
328 	void *stackbase, *stackbase2, *redzone;
329 	size_t stacksize, guardsize;
330 	bool allocated;
331 
332 	if (attr != NULL) {
333 		pthread_attr_getstack(attr, &stackbase, &stacksize);
334 		pthread_attr_getguardsize(attr, &guardsize);
335 	} else {
336 		stackbase = NULL;
337 		stacksize = 0;
338 		guardsize = pthread__guardsize;
339 	}
340 	if (stacksize == 0)
341 		stacksize = pthread__stacksize;
342 
343 	if (newthread->pt_stack_allocated) {
344 		if (stackbase == NULL &&
345 		    newthread->pt_stack.ss_size == stacksize &&
346 		    newthread->pt_guardsize == guardsize)
347 			return 0;
348 		stackbase2 = newthread->pt_stack.ss_sp;
349 #ifndef __MACHINE_STACK_GROWS_UP
350 		stackbase2 = (char *)stackbase2 - newthread->pt_guardsize;
351 #endif
352 		munmap(stackbase2,
353 		    newthread->pt_stack.ss_size + newthread->pt_guardsize);
354 		newthread->pt_stack.ss_sp = NULL;
355 		newthread->pt_stack.ss_size = 0;
356 		newthread->pt_guardsize = 0;
357 		newthread->pt_stack_allocated = false;
358 	}
359 
360 	newthread->pt_stack_allocated = false;
361 
362 	if (stackbase == NULL) {
363 		stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1;
364 		guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1;
365 		stackbase = mmap(NULL, stacksize + guardsize,
366 		    PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0);
367 		if (stackbase == MAP_FAILED)
368 			return ENOMEM;
369 		allocated = true;
370 	} else {
371 		allocated = false;
372 	}
373 #ifdef __MACHINE_STACK_GROWS_UP
374 	redzone = (char *)stackbase + stacksize;
375 	stackbase2 = (char *)stackbase;
376 #else
377 	redzone = (char *)stackbase;
378 	stackbase2 = (char *)stackbase + guardsize;
379 #endif
380 	if (allocated && guardsize &&
381 	    mprotect(redzone, guardsize, PROT_NONE) == -1) {
382 		munmap(stackbase, stacksize + guardsize);
383 		return EPERM;
384 	}
385 	newthread->pt_stack.ss_size = stacksize;
386 	newthread->pt_stack.ss_sp = stackbase2;
387 	newthread->pt_guardsize = guardsize;
388 	newthread->pt_stack_allocated = allocated;
389 	return 0;
390 }
391 
392 int
393 pthread_create(pthread_t *thread, const pthread_attr_t *attr,
394 	    void *(*startfunc)(void *), void *arg)
395 {
396 	pthread_t newthread;
397 	pthread_attr_t nattr;
398 	struct pthread_attr_private *p;
399 	char * volatile name;
400 	unsigned long flag;
401 	void *private_area;
402 	int ret;
403 
404 	if (__predict_false(__uselibcstub)) {
405     		pthread__errorfunc(__FILE__, __LINE__, __func__,
406 		    "pthread_create() requires linking with -lpthread");
407 		return __libc_thr_create_stub(thread, attr, startfunc, arg);
408 	}
409 
410 	if (attr == NULL)
411 		nattr = pthread_default_attr;
412 	else if (attr->pta_magic == PT_ATTR_MAGIC)
413 		nattr = *attr;
414 	else
415 		return EINVAL;
416 
417 	pthread__started = 1;
418 
419 	/* Fetch misc. attributes from the attr structure. */
420 	name = NULL;
421 	if ((p = nattr.pta_private) != NULL)
422 		if (p->ptap_name[0] != '\0')
423 			if ((name = strdup(p->ptap_name)) == NULL)
424 				return ENOMEM;
425 
426 	newthread = NULL;
427 
428 	/*
429 	 * Try to reclaim a dead thread.
430 	 */
431 	if (!PTQ_EMPTY(&pthread__deadqueue)) {
432 		pthread_mutex_lock(&pthread__deadqueue_lock);
433 		PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) {
434 			/* Still busily exiting, or finished? */
435 			if (newthread->pt_lwpctl->lc_curcpu ==
436 			    LWPCTL_CPU_EXITED)
437 				break;
438 		}
439 		if (newthread)
440 			PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq);
441 		pthread_mutex_unlock(&pthread__deadqueue_lock);
442 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
443 		if (newthread && newthread->pt_tls) {
444 			_rtld_tls_free(newthread->pt_tls);
445 			newthread->pt_tls = NULL;
446 		}
447 #endif
448 	}
449 
450 	/*
451 	 * If necessary set up a stack, allocate space for a pthread_st,
452 	 * and initialize it.
453 	 */
454 	if (newthread == NULL) {
455 		newthread = calloc(1, __pthread_st_size);
456 		if (newthread == NULL) {
457 			free(name);
458 			return ENOMEM;
459 		}
460 		newthread->pt_stack_allocated = false;
461 
462 		if (pthread__getstack(newthread, attr)) {
463 			free(newthread);
464 			free(name);
465 			return ENOMEM;
466 		}
467 
468 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
469 		newthread->pt_tls = NULL;
470 #endif
471 
472 		/* Add to list of all threads. */
473 		pthread_rwlock_wrlock(&pthread__alltree_lock);
474 		PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq);
475 		(void)rb_tree_insert_node(&pthread__alltree, newthread);
476 		pthread_rwlock_unlock(&pthread__alltree_lock);
477 
478 		/* Will be reset by the thread upon exit. */
479 		pthread__initthread(newthread);
480 	} else {
481 		if (pthread__getstack(newthread, attr)) {
482 			pthread_mutex_lock(&pthread__deadqueue_lock);
483 			PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq);
484 			pthread_mutex_unlock(&pthread__deadqueue_lock);
485 			return ENOMEM;
486 		}
487 	}
488 
489 	/*
490 	 * Create the new LWP.
491 	 */
492 	pthread__scrubthread(newthread, name, nattr.pta_flags);
493 	newthread->pt_func = startfunc;
494 	newthread->pt_arg = arg;
495 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II)
496 	private_area = newthread->pt_tls = _rtld_tls_allocate();
497 	newthread->pt_tls->tcb_pthread = newthread;
498 #else
499 	private_area = newthread;
500 #endif
501 
502 	flag = 0;
503 	if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 ||
504 	    (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0)
505 		flag |= LWP_SUSPENDED;
506 	if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0)
507 		flag |= LWP_DETACHED;
508 
509 	ret = pthread__makelwp(pthread__create_tramp, newthread, private_area,
510 	    newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size,
511 	    flag, &newthread->pt_lid);
512 	if (ret != 0) {
513 		ret = errno;
514 		pthread_mutex_lock(&newthread->pt_lock);
515 		/* Will unlock and free name. */
516 		pthread__reap(newthread);
517 		return ret;
518 	}
519 
520 	if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) {
521 		if (p != NULL) {
522 			(void)pthread_setschedparam(newthread, p->ptap_policy,
523 			    &p->ptap_sp);
524 		}
525 		if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) {
526 			(void)_lwp_continue(newthread->pt_lid);
527 		}
528 	}
529 
530 	*thread = newthread;
531 
532 	return 0;
533 }
534 
535 
536 __dead static void
537 pthread__create_tramp(void *cookie)
538 {
539 	pthread_t self;
540 	void *retval;
541 	void *junk __unused;
542 
543 	self = cookie;
544 
545 	/*
546 	 * Throw away some stack in a feeble attempt to reduce cache
547 	 * thrash.  May help for SMT processors.  XXX We should not
548 	 * be allocating stacks on fixed 2MB boundaries.  Needs a
549 	 * thread register or decent thread local storage.
550 	 */
551 	junk = alloca(((unsigned)self->pt_lid & 7) << 8);
552 
553 	if (self->pt_name != NULL) {
554 		pthread_mutex_lock(&self->pt_lock);
555 		if (self->pt_name != NULL)
556 			(void)_lwp_setname(0, self->pt_name);
557 		pthread_mutex_unlock(&self->pt_lock);
558 	}
559 
560 	if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) {
561 		err(EXIT_FAILURE, "_lwp_ctl");
562 	}
563 
564 	retval = (*self->pt_func)(self->pt_arg);
565 
566 	pthread_exit(retval);
567 
568 	/*NOTREACHED*/
569 	pthread__abort();
570 }
571 
572 int
573 pthread_suspend_np(pthread_t thread)
574 {
575 	pthread_t self;
576 
577 	pthread__error(EINVAL, "Invalid thread",
578 	    thread->pt_magic == PT_MAGIC);
579 
580 	self = pthread__self();
581 	if (self == thread) {
582 		return EDEADLK;
583 	}
584 	if (pthread__find(thread) != 0)
585 		return ESRCH;
586 	if (_lwp_suspend(thread->pt_lid) == 0)
587 		return 0;
588 	return errno;
589 }
590 
591 int
592 pthread_resume_np(pthread_t thread)
593 {
594 
595 	pthread__error(EINVAL, "Invalid thread",
596 	    thread->pt_magic == PT_MAGIC);
597 
598 	if (pthread__find(thread) != 0)
599 		return ESRCH;
600 	if (_lwp_continue(thread->pt_lid) == 0)
601 		return 0;
602 	return errno;
603 }
604 
605 void
606 pthread_exit(void *retval)
607 {
608 	pthread_t self;
609 	struct pt_clean_t *cleanup;
610 
611 	if (__predict_false(__uselibcstub)) {
612 		__libc_thr_exit_stub(retval);
613 		goto out;
614 	}
615 
616 	self = pthread__self();
617 
618 	/* Disable cancellability. */
619 	pthread_mutex_lock(&self->pt_lock);
620 	self->pt_flags |= PT_FLAG_CS_DISABLED;
621 	self->pt_cancel = 0;
622 	pthread_mutex_unlock(&self->pt_lock);
623 
624 	/* Call any cancellation cleanup handlers */
625 	if (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
626 		while (!PTQ_EMPTY(&self->pt_cleanup_stack)) {
627 			cleanup = PTQ_FIRST(&self->pt_cleanup_stack);
628 			PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next);
629 			(*cleanup->ptc_cleanup)(cleanup->ptc_arg);
630 		}
631 	}
632 
633 	__cxa_thread_run_atexit();
634 
635 	/* Perform cleanup of thread-specific data */
636 	pthread__destroy_tsd(self);
637 
638 	if (_malloc_thread_cleanup)
639 		_malloc_thread_cleanup();
640 
641 	/*
642 	 * Signal our exit.  Our stack and pthread_t won't be reused until
643 	 * pthread_create() can see from kernel info that this LWP is gone.
644 	 */
645 	pthread_mutex_lock(&self->pt_lock);
646 	self->pt_exitval = retval;
647 	if (self->pt_flags & PT_FLAG_DETACHED) {
648 		/* pthread__reap() will drop the lock. */
649 		pthread__reap(self);
650 		_lwp_exit();
651 	} else {
652 		self->pt_state = PT_STATE_ZOMBIE;
653 		pthread_mutex_unlock(&self->pt_lock);
654 		/* Note: name will be freed by the joiner. */
655 		_lwp_exit();
656 	}
657 
658 out:
659 	/*NOTREACHED*/
660 	pthread__abort();
661 	exit(1);
662 }
663 
664 
665 int
666 pthread_join(pthread_t thread, void **valptr)
667 {
668 	pthread_t self;
669 
670 	pthread__error(EINVAL, "Invalid thread",
671 	    thread->pt_magic == PT_MAGIC);
672 
673 	self = pthread__self();
674 
675 	if (pthread__find(thread) != 0)
676 		return ESRCH;
677 
678 	if (thread == self)
679 		return EDEADLK;
680 
681 	/* IEEE Std 1003.1 says pthread_join() never returns EINTR. */
682 	for (;;) {
683 		pthread__testcancel(self);
684 		if (_lwp_wait(thread->pt_lid, NULL) == 0)
685 			break;
686 		if (errno != EINTR)
687 			return errno;
688 	}
689 
690 	/*
691 	 * Don't test for cancellation again.  The spec is that if
692 	 * cancelled, pthread_join() must not have succeeded.
693 	 */
694 	pthread_mutex_lock(&thread->pt_lock);
695 	if (thread->pt_state != PT_STATE_ZOMBIE) {
696 		pthread__errorfunc(__FILE__, __LINE__, __func__,
697 		    "not a zombie");
698  	}
699 	if (valptr != NULL)
700 		*valptr = thread->pt_exitval;
701 
702 	/* pthread__reap() will drop the lock. */
703 	pthread__reap(thread);
704 	return 0;
705 }
706 
707 static void
708 pthread__reap(pthread_t thread)
709 {
710 	char *name;
711 
712 	name = thread->pt_name;
713 	thread->pt_name = NULL;
714 	thread->pt_state = PT_STATE_DEAD;
715 	pthread_mutex_unlock(&thread->pt_lock);
716 
717 	pthread_mutex_lock(&pthread__deadqueue_lock);
718 	PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq);
719 	pthread_mutex_unlock(&pthread__deadqueue_lock);
720 
721 	if (name != NULL)
722 		free(name);
723 }
724 
725 int
726 pthread_equal(pthread_t t1, pthread_t t2)
727 {
728 
729 	if (__predict_false(__uselibcstub))
730 		return __libc_thr_equal_stub(t1, t2);
731 
732 	pthread__error(0, "Invalid thread",
733 	    (t1 != NULL) && (t1->pt_magic == PT_MAGIC));
734 
735 	pthread__error(0, "Invalid thread",
736 	    (t2 != NULL) && (t2->pt_magic == PT_MAGIC));
737 
738 	/* Nothing special here. */
739 	return (t1 == t2);
740 }
741 
742 
743 int
744 pthread_detach(pthread_t thread)
745 {
746 	int error;
747 
748 	pthread__error(EINVAL, "Invalid thread",
749 	    thread->pt_magic == PT_MAGIC);
750 
751 	if (pthread__find(thread) != 0)
752 		return ESRCH;
753 
754 	pthread_mutex_lock(&thread->pt_lock);
755 	if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) {
756 		error = EINVAL;
757 	} else {
758 		error = _lwp_detach(thread->pt_lid);
759 		if (error == 0)
760 			thread->pt_flags |= PT_FLAG_DETACHED;
761 		else
762 			error = errno;
763 	}
764 	if (thread->pt_state == PT_STATE_ZOMBIE) {
765 		/* pthread__reap() will drop the lock. */
766 		pthread__reap(thread);
767 	} else
768 		pthread_mutex_unlock(&thread->pt_lock);
769 	return error;
770 }
771 
772 
773 int
774 pthread_getname_np(pthread_t thread, char *name, size_t len)
775 {
776 
777 	pthread__error(EINVAL, "Invalid thread",
778 	    thread->pt_magic == PT_MAGIC);
779 
780 	if (pthread__find(thread) != 0)
781 		return ESRCH;
782 
783 	pthread_mutex_lock(&thread->pt_lock);
784 	if (thread->pt_name == NULL)
785 		name[0] = '\0';
786 	else
787 		strlcpy(name, thread->pt_name, len);
788 	pthread_mutex_unlock(&thread->pt_lock);
789 
790 	return 0;
791 }
792 
793 
794 int
795 pthread_setname_np(pthread_t thread, const char *name, void *arg)
796 {
797 	char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP];
798 	int namelen;
799 
800 	pthread__error(EINVAL, "Invalid thread",
801 	    thread->pt_magic == PT_MAGIC);
802 
803 	if (pthread__find(thread) != 0)
804 		return ESRCH;
805 
806 	namelen = snprintf(newname, sizeof(newname), name, arg);
807 	if (namelen >= PTHREAD_MAX_NAMELEN_NP)
808 		return EINVAL;
809 
810 	cp = strdup(newname);
811 	if (cp == NULL)
812 		return ENOMEM;
813 
814 	pthread_mutex_lock(&thread->pt_lock);
815 	oldname = thread->pt_name;
816 	thread->pt_name = cp;
817 	(void)_lwp_setname(thread->pt_lid, cp);
818 	pthread_mutex_unlock(&thread->pt_lock);
819 
820 	if (oldname != NULL)
821 		free(oldname);
822 
823 	return 0;
824 }
825 
826 
827 pthread_t
828 pthread_self(void)
829 {
830 	if (__predict_false(__uselibcstub))
831 		return (pthread_t)__libc_thr_self_stub();
832 
833 	return pthread__self();
834 }
835 
836 
837 int
838 pthread_cancel(pthread_t thread)
839 {
840 
841 	pthread__error(EINVAL, "Invalid thread",
842 	    thread->pt_magic == PT_MAGIC);
843 
844 	if (pthread__find(thread) != 0)
845 		return ESRCH;
846 	pthread_mutex_lock(&thread->pt_lock);
847 	thread->pt_flags |= PT_FLAG_CS_PENDING;
848 	if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) {
849 		thread->pt_cancel = 1;
850 		pthread_mutex_unlock(&thread->pt_lock);
851 		_lwp_wakeup(thread->pt_lid);
852 	} else
853 		pthread_mutex_unlock(&thread->pt_lock);
854 
855 	return 0;
856 }
857 
858 
859 int
860 pthread_setcancelstate(int state, int *oldstate)
861 {
862 	pthread_t self;
863 	int retval;
864 
865 	if (__predict_false(__uselibcstub))
866 		return __libc_thr_setcancelstate_stub(state, oldstate);
867 
868 	self = pthread__self();
869 	retval = 0;
870 
871 	pthread_mutex_lock(&self->pt_lock);
872 
873 	if (oldstate != NULL) {
874 		if (self->pt_flags & PT_FLAG_CS_DISABLED)
875 			*oldstate = PTHREAD_CANCEL_DISABLE;
876 		else
877 			*oldstate = PTHREAD_CANCEL_ENABLE;
878 	}
879 
880 	if (state == PTHREAD_CANCEL_DISABLE) {
881 		self->pt_flags |= PT_FLAG_CS_DISABLED;
882 		if (self->pt_cancel) {
883 			self->pt_flags |= PT_FLAG_CS_PENDING;
884 			self->pt_cancel = 0;
885 		}
886 	} else if (state == PTHREAD_CANCEL_ENABLE) {
887 		self->pt_flags &= ~PT_FLAG_CS_DISABLED;
888 		/*
889 		 * If a cancellation was requested while cancellation
890 		 * was disabled, note that fact for future
891 		 * cancellation tests.
892 		 */
893 		if (self->pt_flags & PT_FLAG_CS_PENDING) {
894 			self->pt_cancel = 1;
895 			/* This is not a deferred cancellation point. */
896 			if (self->pt_flags & PT_FLAG_CS_ASYNC) {
897 				pthread_mutex_unlock(&self->pt_lock);
898 				pthread__cancelled();
899 			}
900 		}
901 	} else
902 		retval = EINVAL;
903 
904 	pthread_mutex_unlock(&self->pt_lock);
905 
906 	return retval;
907 }
908 
909 
910 int
911 pthread_setcanceltype(int type, int *oldtype)
912 {
913 	pthread_t self;
914 	int retval;
915 
916 	self = pthread__self();
917 	retval = 0;
918 
919 	pthread_mutex_lock(&self->pt_lock);
920 
921 	if (oldtype != NULL) {
922 		if (self->pt_flags & PT_FLAG_CS_ASYNC)
923 			*oldtype = PTHREAD_CANCEL_ASYNCHRONOUS;
924 		else
925 			*oldtype = PTHREAD_CANCEL_DEFERRED;
926 	}
927 
928 	if (type == PTHREAD_CANCEL_ASYNCHRONOUS) {
929 		self->pt_flags |= PT_FLAG_CS_ASYNC;
930 		if (self->pt_cancel) {
931 			pthread_mutex_unlock(&self->pt_lock);
932 			pthread__cancelled();
933 		}
934 	} else if (type == PTHREAD_CANCEL_DEFERRED)
935 		self->pt_flags &= ~PT_FLAG_CS_ASYNC;
936 	else
937 		retval = EINVAL;
938 
939 	pthread_mutex_unlock(&self->pt_lock);
940 
941 	return retval;
942 }
943 
944 
945 void
946 pthread_testcancel(void)
947 {
948 	pthread_t self;
949 
950 	self = pthread__self();
951 	if (self->pt_cancel)
952 		pthread__cancelled();
953 }
954 
955 
956 /*
957  * POSIX requires that certain functions return an error rather than
958  * invoking undefined behavior even when handed completely bogus
959  * pthread_t values, e.g. stack garbage.
960  */
961 int
962 pthread__find(pthread_t id)
963 {
964 	pthread_t target;
965 	int error;
966 
967 	pthread_rwlock_rdlock(&pthread__alltree_lock);
968 	target = rb_tree_find_node(&pthread__alltree, id);
969 	error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH;
970 	pthread_rwlock_unlock(&pthread__alltree_lock);
971 
972 	return error;
973 }
974 
975 
976 void
977 pthread__testcancel(pthread_t self)
978 {
979 
980 	if (self->pt_cancel)
981 		pthread__cancelled();
982 }
983 
984 
985 void
986 pthread__cancelled(void)
987 {
988 
989 	pthread_exit(PTHREAD_CANCELED);
990 }
991 
992 
993 void
994 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store)
995 {
996 	pthread_t self;
997 	struct pt_clean_t *entry;
998 
999 	self = pthread__self();
1000 	entry = store;
1001 	entry->ptc_cleanup = cleanup;
1002 	entry->ptc_arg = arg;
1003 	PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next);
1004 }
1005 
1006 
1007 void
1008 pthread__cleanup_pop(int ex, void *store)
1009 {
1010 	pthread_t self;
1011 	struct pt_clean_t *entry;
1012 
1013 	self = pthread__self();
1014 	entry = store;
1015 
1016 	PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next);
1017 	if (ex)
1018 		(*entry->ptc_cleanup)(entry->ptc_arg);
1019 }
1020 
1021 
1022 int *
1023 pthread__errno(void)
1024 {
1025 	pthread_t self;
1026 
1027 	if (__predict_false(__uselibcstub)) {
1028     		pthread__errorfunc(__FILE__, __LINE__, __func__,
1029 		    "pthread__errno() requires linking with -lpthread");
1030 		return __libc_thr_errno_stub();
1031 	}
1032 
1033 	self = pthread__self();
1034 
1035 	return &(self->pt_errno);
1036 }
1037 
1038 ssize_t	_sys_write(int, const void *, size_t);
1039 
1040 void
1041 pthread__assertfunc(const char *file, int line, const char *function,
1042 		    const char *expr)
1043 {
1044 	char buf[1024];
1045 	int len;
1046 
1047 	/*
1048 	 * snprintf should not acquire any locks, or we could
1049 	 * end up deadlocked if the assert caller held locks.
1050 	 */
1051 	len = snprintf(buf, 1024,
1052 	    "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n",
1053 	    expr, file, line,
1054 	    function ? ", function \"" : "",
1055 	    function ? function : "",
1056 	    function ? "\"" : "");
1057 
1058 	_sys_write(STDERR_FILENO, buf, (size_t)len);
1059 	(void)raise(SIGABRT);
1060 	_exit(1);
1061 }
1062 
1063 
1064 void
1065 pthread__errorfunc(const char *file, int line, const char *function,
1066 		   const char *msg, ...)
1067 {
1068 	char buf[1024];
1069 	char buf2[1024];
1070 	size_t len;
1071 	va_list ap;
1072 
1073 	if (pthread__diagassert == 0)
1074 		return;
1075 
1076 	va_start(ap, msg);
1077 	vsnprintf_ss(buf2, sizeof(buf2), msg, ap);
1078 	va_end(ap);
1079 
1080 	/*
1081 	 * snprintf should not acquire any locks, or we could
1082 	 * end up deadlocked if the assert caller held locks.
1083 	 */
1084 	len = snprintf_ss(buf, sizeof(buf),
1085 	    "%s: Error detected by libpthread: %s.\n"
1086 	    "Detected by file \"%s\", line %d%s%s%s.\n"
1087 	    "See pthread(3) for information.\n",
1088 	    getprogname(), buf2, file, line,
1089 	    function ? ", function \"" : "",
1090 	    function ? function : "",
1091 	    function ? "\"" : "");
1092 
1093 	if (pthread__diagassert & DIAGASSERT_STDERR)
1094 		_sys_write(STDERR_FILENO, buf, len);
1095 
1096 	if (pthread__diagassert & DIAGASSERT_SYSLOG)
1097 		syslog(LOG_DEBUG | LOG_USER, "%s", buf);
1098 
1099 	if (pthread__diagassert & DIAGASSERT_ABORT) {
1100 		(void)_lwp_kill(_lwp_self(), SIGABRT);
1101 		_exit(1);
1102 	}
1103 }
1104 
1105 /*
1106  * Thread park/unpark operations.  The kernel operations are
1107  * modelled after a brief description from "Multithreading in
1108  * the Solaris Operating Environment":
1109  *
1110  * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf
1111  */
1112 
1113 int
1114 pthread__park(pthread_t self, pthread_mutex_t *lock,
1115 	      pthread_queue_t *queue, const struct timespec *abstime,
1116 	      int cancelpt)
1117 {
1118 	int rv, error;
1119 
1120 	pthread_mutex_unlock(lock);
1121 
1122 	/*
1123 	 * Wait until we are awoken by a pending unpark operation,
1124 	 * a signal, an unpark posted after we have gone asleep,
1125 	 * or an expired timeout.
1126 	 *
1127 	 * It is fine to test the value of pt_sleepobj without
1128 	 * holding any locks, because:
1129 	 *
1130 	 * o Only the blocking thread (this thread) ever sets it
1131 	 *   to a non-NULL value.
1132 	 *
1133 	 * o Other threads may set it NULL, but if they do so they
1134 	 *   must also make this thread return from _lwp_park.
1135 	 *
1136 	 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system
1137 	 *   calls and all make use of spinlocks in the kernel.  So
1138 	 *   these system calls act as full memory barriers.
1139 	 */
1140 	rv = 0;
1141 	do {
1142 		/*
1143 		 * If we deferred unparking a thread, arrange to
1144 		 * have _lwp_park() restart it before blocking.
1145 		 */
1146 		error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME,
1147 		    __UNCONST(abstime), 0, NULL, NULL);
1148 		if (error != 0) {
1149 			switch (rv = errno) {
1150 			case EINTR:
1151 			case EALREADY:
1152 				rv = 0;
1153 				break;
1154 			case ETIMEDOUT:
1155 				break;
1156 			default:
1157 				pthread__errorfunc(__FILE__, __LINE__,
1158 				    __func__, "_lwp_park failed: %d", errno);
1159 				break;
1160 			}
1161 		}
1162 		/* Check for cancellation. */
1163 		if (cancelpt && self->pt_cancel)
1164 			rv = EINTR;
1165 	} while (self->pt_sleepobj != NULL && rv == 0);
1166 	return rv;
1167 }
1168 
1169 void
1170 pthread__unpark(pthread_queue_t *queue, pthread_t self,
1171 		pthread_mutex_t *interlock)
1172 {
1173 	pthread_t target;
1174 
1175 	target = PTQ_FIRST(queue);
1176 	target->pt_sleepobj = NULL;
1177 	PTQ_REMOVE(queue, target, pt_sleep);
1178 	(void)_lwp_unpark(target->pt_lid, NULL);
1179 }
1180 
1181 void
1182 pthread__unpark_all(pthread_queue_t *queue, pthread_t self,
1183 		    pthread_mutex_t *interlock)
1184 {
1185 	lwpid_t lids[PTHREAD__UNPARK_MAX];
1186 	const size_t mlid = pthread__unpark_max;
1187 	pthread_t target;
1188 	size_t nlid = 0;
1189 
1190 	PTQ_FOREACH(target, queue, pt_sleep) {
1191 		if (nlid == mlid) {
1192 			(void)_lwp_unpark_all(lids, nlid, NULL);
1193 			nlid = 0;
1194 		}
1195 		target->pt_sleepobj = NULL;
1196 		lids[nlid++] = target->pt_lid;
1197 	}
1198 	PTQ_INIT(queue);
1199 	if (nlid == 1) {
1200 		(void)_lwp_unpark(lids[0], NULL);
1201 	} else if (nlid > 1) {
1202 		(void)_lwp_unpark_all(lids, nlid, NULL);
1203 	}
1204 }
1205 
1206 #undef	OOPS
1207 
1208 static void
1209 pthread__initmainstack(void)
1210 {
1211 	struct rlimit slimit;
1212 	const AuxInfo *aux;
1213 	size_t size, len;
1214 	int mib[2];
1215 	unsigned int value;
1216 
1217 	_DIAGASSERT(_dlauxinfo() != NULL);
1218 
1219 	if (getrlimit(RLIMIT_STACK, &slimit) == -1)
1220 		err(EXIT_FAILURE,
1221 		    "Couldn't get stack resource consumption limits");
1222 	size = slimit.rlim_cur;
1223 	pthread__main->pt_stack.ss_size = size;
1224 	pthread__main->pt_guardsize = pthread__pagesize;
1225 
1226 	mib[0] = CTL_VM;
1227 	mib[1] = VM_GUARD_SIZE;
1228 	len = sizeof(value);
1229 	if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0)
1230 		pthread__main->pt_guardsize = value;
1231 
1232 	for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) {
1233 		if (aux->a_type == AT_STACKBASE) {
1234 #ifdef __MACHINE_STACK_GROWS_UP
1235 			pthread__main->pt_stack.ss_sp = (void *)aux->a_v;
1236 #else
1237 			pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size;
1238 #endif
1239 			break;
1240 		}
1241 	}
1242 	pthread__copy_tsd(pthread__main);
1243 }
1244 
1245 /*
1246  * Set up the slightly special stack for the "initial" thread, which
1247  * runs on the normal system stack, and thus gets slightly different
1248  * treatment.
1249  */
1250 static void
1251 pthread__initmain(pthread_t *newt)
1252 {
1253 	char *value;
1254 
1255 	pthread__initmainstack();
1256 
1257 	value = pthread__getenv("PTHREAD_STACKSIZE");
1258 	if (value != NULL) {
1259 		pthread__stacksize = atoi(value) * 1024;
1260 		if (pthread__stacksize > pthread__main->pt_stack.ss_size)
1261 			pthread__stacksize = pthread__main->pt_stack.ss_size;
1262 	}
1263 	if (pthread__stacksize == 0)
1264 		pthread__stacksize = pthread__main->pt_stack.ss_size;
1265 	pthread__stacksize += pthread__pagesize - 1;
1266 	pthread__stacksize &= ~(pthread__pagesize - 1);
1267 	if (pthread__stacksize < 4 * pthread__pagesize)
1268 		errx(1, "Stacksize limit is too low, minimum %zd kbyte.",
1269 		    4 * pthread__pagesize / 1024);
1270 
1271 	*newt = pthread__main;
1272 #if defined(_PTHREAD_GETTCB_EXT)
1273 	pthread__main->pt_tls = _PTHREAD_GETTCB_EXT();
1274 #elif defined(__HAVE___LWP_GETTCB_FAST)
1275 	pthread__main->pt_tls = __lwp_gettcb_fast();
1276 #else
1277 	pthread__main->pt_tls = _lwp_getprivate();
1278 #endif
1279 	pthread__main->pt_tls->tcb_pthread = pthread__main;
1280 }
1281 
1282 static signed int
1283 /*ARGSUSED*/
1284 pthread__cmp(void *ctx, const void *n1, const void *n2)
1285 {
1286 	const uintptr_t p1 = (const uintptr_t)n1;
1287 	const uintptr_t p2 = (const uintptr_t)n2;
1288 
1289 	if (p1 < p2)
1290 		return -1;
1291 	if (p1 > p2)
1292 		return 1;
1293 	return 0;
1294 }
1295 
1296 /* Because getenv() wants to use locks. */
1297 char *
1298 pthread__getenv(const char *name)
1299 {
1300 	extern char **environ;
1301 	size_t l_name, offset;
1302 
1303 	if (issetugid())
1304 		return (NULL);
1305 
1306 	l_name = strlen(name);
1307 	for (offset = 0; environ[offset] != NULL; offset++) {
1308 		if (strncmp(name, environ[offset], l_name) == 0 &&
1309 		    environ[offset][l_name] == '=') {
1310 			return environ[offset] + l_name + 1;
1311 		}
1312 	}
1313 
1314 	return NULL;
1315 }
1316 
1317 pthread_mutex_t *
1318 pthread__hashlock(volatile const void *p)
1319 {
1320 	uintptr_t v;
1321 
1322 	v = (uintptr_t)p;
1323 	return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex;
1324 }
1325 
1326 int
1327 pthread__checkpri(int pri)
1328 {
1329 	static int havepri;
1330 	static long min, max;
1331 
1332 	if (!havepri) {
1333 		min = sysconf(_SC_SCHED_PRI_MIN);
1334 		max = sysconf(_SC_SCHED_PRI_MAX);
1335 		havepri = 1;
1336 	}
1337 	return (pri < min || pri > max) ? EINVAL : 0;
1338 }
1339