10Sstevel@tonic-gate /*
20Sstevel@tonic-gate * CDDL HEADER START
30Sstevel@tonic-gate *
40Sstevel@tonic-gate * The contents of this file are subject to the terms of the
51672Sraf * Common Development and Distribution License (the "License").
61672Sraf * You may not use this file except in compliance with the License.
70Sstevel@tonic-gate *
80Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
90Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
100Sstevel@tonic-gate * See the License for the specific language governing permissions
110Sstevel@tonic-gate * and limitations under the License.
120Sstevel@tonic-gate *
130Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
140Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
150Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
160Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
170Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
180Sstevel@tonic-gate *
190Sstevel@tonic-gate * CDDL HEADER END
200Sstevel@tonic-gate */
211111Sraf
220Sstevel@tonic-gate /*
2312795SRoger.Faulkner@Oracle.COM * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
240Sstevel@tonic-gate */
250Sstevel@tonic-gate
260Sstevel@tonic-gate #include "lint.h"
270Sstevel@tonic-gate #include "thr_uberdata.h"
285891Sraf #include <pthread.h>
290Sstevel@tonic-gate #include <procfs.h>
300Sstevel@tonic-gate #include <sys/uio.h>
310Sstevel@tonic-gate #include <ctype.h>
325891Sraf #include "libc.h"
330Sstevel@tonic-gate
346812Sraf /*
356812Sraf * These symbols should not be exported from libc, but
366812Sraf * /lib/libm.so.2 references _thr_main. libm needs to be fixed.
376812Sraf * Also, some older versions of the Studio compiler/debugger
386812Sraf * components reference them. These need to be fixed, too.
396812Sraf */
406812Sraf #pragma weak _thr_main = thr_main
416812Sraf #pragma weak _thr_create = thr_create
426812Sraf #pragma weak _thr_join = thr_join
436812Sraf #pragma weak _thr_self = thr_self
446812Sraf
450Sstevel@tonic-gate #undef errno
460Sstevel@tonic-gate extern int errno;
470Sstevel@tonic-gate
481672Sraf /*
491672Sraf * Between Solaris 2.5 and Solaris 9, __threaded was used to indicate
501672Sraf * "we are linked with libthread". The Sun Workshop 6 update 1 compilation
511672Sraf * system used it illegally (it is a consolidation private symbol).
521672Sraf * To accommodate this and possibly other abusers of the symbol,
531672Sraf * we make it always equal to 1 now that libthread has been folded
541672Sraf * into libc. The new __libc_threaded symbol is used to indicate
551672Sraf * the new meaning, "more than one thread exists".
561672Sraf */
571672Sraf int __threaded = 1; /* always equal to 1 */
581672Sraf int __libc_threaded = 0; /* zero until first thr_create() */
590Sstevel@tonic-gate
600Sstevel@tonic-gate /*
610Sstevel@tonic-gate * thr_concurrency and pthread_concurrency are not used by the library.
620Sstevel@tonic-gate * They exist solely to hold and return the values set by calls to
630Sstevel@tonic-gate * thr_setconcurrency() and pthread_setconcurrency().
640Sstevel@tonic-gate * Because thr_concurrency is affected by the THR_NEW_LWP flag
650Sstevel@tonic-gate * to thr_create(), thr_concurrency is protected by link_lock.
660Sstevel@tonic-gate */
670Sstevel@tonic-gate static int thr_concurrency = 1;
680Sstevel@tonic-gate static int pthread_concurrency;
690Sstevel@tonic-gate
700Sstevel@tonic-gate #define HASHTBLSZ 1024 /* must be a power of two */
710Sstevel@tonic-gate #define TIDHASH(tid, udp) (tid & (udp)->hash_mask)
720Sstevel@tonic-gate
730Sstevel@tonic-gate /* initial allocation, just enough for one lwp */
740Sstevel@tonic-gate #pragma align 64(init_hash_table)
750Sstevel@tonic-gate thr_hash_table_t init_hash_table[1] = {
760Sstevel@tonic-gate { DEFAULTMUTEX, DEFAULTCV, NULL },
770Sstevel@tonic-gate };
780Sstevel@tonic-gate
790Sstevel@tonic-gate extern const Lc_interface rtld_funcs[];
800Sstevel@tonic-gate
810Sstevel@tonic-gate /*
820Sstevel@tonic-gate * The weak version is known to libc_db and mdb.
830Sstevel@tonic-gate */
840Sstevel@tonic-gate #pragma weak _uberdata = __uberdata
850Sstevel@tonic-gate uberdata_t __uberdata = {
863512Sraf { DEFAULTMUTEX, NULL, 0 }, /* link_lock */
876515Sraf { RECURSIVEMUTEX, NULL, 0 }, /* ld_lock */
883512Sraf { RECURSIVEMUTEX, NULL, 0 }, /* fork_lock */
894843Sraf { RECURSIVEMUTEX, NULL, 0 }, /* atfork_lock */
905002Sraf { RECURSIVEMUTEX, NULL, 0 }, /* callout_lock */
913512Sraf { DEFAULTMUTEX, NULL, 0 }, /* tdb_hash_lock */
923512Sraf { 0, }, /* tdb_hash_lock_stats */
933512Sraf { { 0 }, }, /* siguaction[NSIG] */
940Sstevel@tonic-gate {{ DEFAULTMUTEX, NULL, 0 }, /* bucket[NBUCKETS] */
950Sstevel@tonic-gate { DEFAULTMUTEX, NULL, 0 },
960Sstevel@tonic-gate { DEFAULTMUTEX, NULL, 0 },
970Sstevel@tonic-gate { DEFAULTMUTEX, NULL, 0 },
980Sstevel@tonic-gate { DEFAULTMUTEX, NULL, 0 },
990Sstevel@tonic-gate { DEFAULTMUTEX, NULL, 0 },
1000Sstevel@tonic-gate { DEFAULTMUTEX, NULL, 0 },
1010Sstevel@tonic-gate { DEFAULTMUTEX, NULL, 0 },
1020Sstevel@tonic-gate { DEFAULTMUTEX, NULL, 0 },
1030Sstevel@tonic-gate { DEFAULTMUTEX, NULL, 0 }},
1040Sstevel@tonic-gate { RECURSIVEMUTEX, NULL, NULL }, /* atexit_root */
1050Sstevel@tonic-gate { DEFAULTMUTEX, 0, 0, NULL }, /* tsd_metadata */
1060Sstevel@tonic-gate { DEFAULTMUTEX, {0, 0}, {0, 0} }, /* tls_metadata */
1070Sstevel@tonic-gate 0, /* primary_map */
1080Sstevel@tonic-gate 0, /* bucket_init */
1090Sstevel@tonic-gate 0, /* pad[0] */
1100Sstevel@tonic-gate 0, /* pad[1] */
1110Sstevel@tonic-gate { 0 }, /* uberflags */
1120Sstevel@tonic-gate NULL, /* queue_head */
1130Sstevel@tonic-gate init_hash_table, /* thr_hash_table */
1140Sstevel@tonic-gate 1, /* hash_size: size of the hash table */
1150Sstevel@tonic-gate 0, /* hash_mask: hash_size - 1 */
1160Sstevel@tonic-gate NULL, /* ulwp_one */
1170Sstevel@tonic-gate NULL, /* all_lwps */
1180Sstevel@tonic-gate NULL, /* all_zombies */
1190Sstevel@tonic-gate 0, /* nthreads */
1200Sstevel@tonic-gate 0, /* nzombies */
1210Sstevel@tonic-gate 0, /* ndaemons */
1220Sstevel@tonic-gate 0, /* pid */
1230Sstevel@tonic-gate sigacthandler, /* sigacthandler */
1240Sstevel@tonic-gate NULL, /* lwp_stacks */
1250Sstevel@tonic-gate NULL, /* lwp_laststack */
1260Sstevel@tonic-gate 0, /* nfreestack */
1270Sstevel@tonic-gate 10, /* thread_stack_cache */
1280Sstevel@tonic-gate NULL, /* ulwp_freelist */
1290Sstevel@tonic-gate NULL, /* ulwp_lastfree */
1300Sstevel@tonic-gate NULL, /* ulwp_replace_free */
1310Sstevel@tonic-gate NULL, /* ulwp_replace_last */
1320Sstevel@tonic-gate NULL, /* atforklist */
1334574Sraf NULL, /* robustlocks */
1349170SRoger.Faulkner@Sun.COM NULL, /* robustlist */
135*13093SRoger.Faulkner@Oracle.COM NULL, /* progname */
1360Sstevel@tonic-gate NULL, /* __tdb_bootstrap */
1370Sstevel@tonic-gate { /* tdb */
1380Sstevel@tonic-gate NULL, /* tdb_sync_addr_hash */
1390Sstevel@tonic-gate 0, /* tdb_register_count */
1400Sstevel@tonic-gate 0, /* tdb_hash_alloc_failed */
1410Sstevel@tonic-gate NULL, /* tdb_sync_addr_free */
1420Sstevel@tonic-gate NULL, /* tdb_sync_addr_last */
1430Sstevel@tonic-gate 0, /* tdb_sync_alloc */
1440Sstevel@tonic-gate { 0, 0 }, /* tdb_ev_global_mask */
1450Sstevel@tonic-gate tdb_events, /* tdb_events array */
1460Sstevel@tonic-gate },
1470Sstevel@tonic-gate };
1480Sstevel@tonic-gate
1490Sstevel@tonic-gate /*
1500Sstevel@tonic-gate * The weak version is known to libc_db and mdb.
1510Sstevel@tonic-gate */
1520Sstevel@tonic-gate #pragma weak _tdb_bootstrap = __tdb_bootstrap
1530Sstevel@tonic-gate uberdata_t **__tdb_bootstrap = NULL;
1540Sstevel@tonic-gate
1550Sstevel@tonic-gate int thread_queue_fifo = 4;
1560Sstevel@tonic-gate int thread_queue_dump = 0;
1570Sstevel@tonic-gate int thread_cond_wait_defer = 0;
1580Sstevel@tonic-gate int thread_error_detection = 0;
1590Sstevel@tonic-gate int thread_async_safe = 0;
1600Sstevel@tonic-gate int thread_stack_cache = 10;
1610Sstevel@tonic-gate int thread_door_noreserve = 0;
1627255Sraf int thread_locks_misaligned = 0;
1630Sstevel@tonic-gate
1640Sstevel@tonic-gate static ulwp_t *ulwp_alloc(void);
1650Sstevel@tonic-gate static void ulwp_free(ulwp_t *);
1660Sstevel@tonic-gate
1670Sstevel@tonic-gate /*
1680Sstevel@tonic-gate * Insert the lwp into the hash table.
1690Sstevel@tonic-gate */
1700Sstevel@tonic-gate void
hash_in_unlocked(ulwp_t * ulwp,int ix,uberdata_t * udp)1710Sstevel@tonic-gate hash_in_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
1720Sstevel@tonic-gate {
1730Sstevel@tonic-gate ulwp->ul_hash = udp->thr_hash_table[ix].hash_bucket;
1740Sstevel@tonic-gate udp->thr_hash_table[ix].hash_bucket = ulwp;
1750Sstevel@tonic-gate ulwp->ul_ix = ix;
1760Sstevel@tonic-gate }
1770Sstevel@tonic-gate
1780Sstevel@tonic-gate void
hash_in(ulwp_t * ulwp,uberdata_t * udp)1790Sstevel@tonic-gate hash_in(ulwp_t *ulwp, uberdata_t *udp)
1800Sstevel@tonic-gate {
1810Sstevel@tonic-gate int ix = TIDHASH(ulwp->ul_lwpid, udp);
1820Sstevel@tonic-gate mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
1830Sstevel@tonic-gate
1840Sstevel@tonic-gate lmutex_lock(mp);
1850Sstevel@tonic-gate hash_in_unlocked(ulwp, ix, udp);
1860Sstevel@tonic-gate lmutex_unlock(mp);
1870Sstevel@tonic-gate }
1880Sstevel@tonic-gate
1890Sstevel@tonic-gate /*
1900Sstevel@tonic-gate * Delete the lwp from the hash table.
1910Sstevel@tonic-gate */
1920Sstevel@tonic-gate void
hash_out_unlocked(ulwp_t * ulwp,int ix,uberdata_t * udp)1930Sstevel@tonic-gate hash_out_unlocked(ulwp_t *ulwp, int ix, uberdata_t *udp)
1940Sstevel@tonic-gate {
1950Sstevel@tonic-gate ulwp_t **ulwpp;
1960Sstevel@tonic-gate
1970Sstevel@tonic-gate for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
1980Sstevel@tonic-gate ulwp != *ulwpp;
1990Sstevel@tonic-gate ulwpp = &(*ulwpp)->ul_hash)
2000Sstevel@tonic-gate ;
2010Sstevel@tonic-gate *ulwpp = ulwp->ul_hash;
2020Sstevel@tonic-gate ulwp->ul_hash = NULL;
2030Sstevel@tonic-gate ulwp->ul_ix = -1;
2040Sstevel@tonic-gate }
2050Sstevel@tonic-gate
2060Sstevel@tonic-gate void
hash_out(ulwp_t * ulwp,uberdata_t * udp)2070Sstevel@tonic-gate hash_out(ulwp_t *ulwp, uberdata_t *udp)
2080Sstevel@tonic-gate {
2090Sstevel@tonic-gate int ix;
2100Sstevel@tonic-gate
2110Sstevel@tonic-gate if ((ix = ulwp->ul_ix) >= 0) {
2120Sstevel@tonic-gate mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
2130Sstevel@tonic-gate
2140Sstevel@tonic-gate lmutex_lock(mp);
2150Sstevel@tonic-gate hash_out_unlocked(ulwp, ix, udp);
2160Sstevel@tonic-gate lmutex_unlock(mp);
2170Sstevel@tonic-gate }
2180Sstevel@tonic-gate }
2190Sstevel@tonic-gate
2205891Sraf /*
2215891Sraf * Retain stack information for thread structures that are being recycled for
2225891Sraf * new threads. All other members of the thread structure should be zeroed.
2235891Sraf */
2240Sstevel@tonic-gate static void
ulwp_clean(ulwp_t * ulwp)2250Sstevel@tonic-gate ulwp_clean(ulwp_t *ulwp)
2260Sstevel@tonic-gate {
2275891Sraf caddr_t stk = ulwp->ul_stk;
2285891Sraf size_t mapsiz = ulwp->ul_mapsiz;
2295891Sraf size_t guardsize = ulwp->ul_guardsize;
2305891Sraf uintptr_t stktop = ulwp->ul_stktop;
2315891Sraf size_t stksiz = ulwp->ul_stksiz;
2325891Sraf
2336515Sraf (void) memset(ulwp, 0, sizeof (*ulwp));
2345891Sraf
2355891Sraf ulwp->ul_stk = stk;
2365891Sraf ulwp->ul_mapsiz = mapsiz;
2375891Sraf ulwp->ul_guardsize = guardsize;
2385891Sraf ulwp->ul_stktop = stktop;
2395891Sraf ulwp->ul_stksiz = stksiz;
2400Sstevel@tonic-gate }
2410Sstevel@tonic-gate
2420Sstevel@tonic-gate static int stackprot;
2430Sstevel@tonic-gate
2440Sstevel@tonic-gate /*
2450Sstevel@tonic-gate * Answer the question, "Is the lwp in question really dead?"
2460Sstevel@tonic-gate * We must inquire of the operating system to be really sure
2470Sstevel@tonic-gate * because the lwp may have called lwp_exit() but it has not
2480Sstevel@tonic-gate * yet completed the exit.
2490Sstevel@tonic-gate */
2500Sstevel@tonic-gate static int
dead_and_buried(ulwp_t * ulwp)2510Sstevel@tonic-gate dead_and_buried(ulwp_t *ulwp)
2520Sstevel@tonic-gate {
2530Sstevel@tonic-gate if (ulwp->ul_lwpid == (lwpid_t)(-1))
2540Sstevel@tonic-gate return (1);
2550Sstevel@tonic-gate if (ulwp->ul_dead && ulwp->ul_detached &&
2566812Sraf _lwp_kill(ulwp->ul_lwpid, 0) == ESRCH) {
2570Sstevel@tonic-gate ulwp->ul_lwpid = (lwpid_t)(-1);
2580Sstevel@tonic-gate return (1);
2590Sstevel@tonic-gate }
2600Sstevel@tonic-gate return (0);
2610Sstevel@tonic-gate }
2620Sstevel@tonic-gate
2630Sstevel@tonic-gate /*
2640Sstevel@tonic-gate * Attempt to keep the stack cache within the specified cache limit.
2650Sstevel@tonic-gate */
2660Sstevel@tonic-gate static void
trim_stack_cache(int cache_limit)2670Sstevel@tonic-gate trim_stack_cache(int cache_limit)
2680Sstevel@tonic-gate {
2690Sstevel@tonic-gate ulwp_t *self = curthread;
2700Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
2710Sstevel@tonic-gate ulwp_t *prev = NULL;
2720Sstevel@tonic-gate ulwp_t **ulwpp = &udp->lwp_stacks;
2730Sstevel@tonic-gate ulwp_t *ulwp;
2740Sstevel@tonic-gate
2750Sstevel@tonic-gate ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, self));
2760Sstevel@tonic-gate
2770Sstevel@tonic-gate while (udp->nfreestack > cache_limit && (ulwp = *ulwpp) != NULL) {
2780Sstevel@tonic-gate if (dead_and_buried(ulwp)) {
2790Sstevel@tonic-gate *ulwpp = ulwp->ul_next;
2800Sstevel@tonic-gate if (ulwp == udp->lwp_laststack)
2810Sstevel@tonic-gate udp->lwp_laststack = prev;
2820Sstevel@tonic-gate hash_out(ulwp, udp);
2830Sstevel@tonic-gate udp->nfreestack--;
2846515Sraf (void) munmap(ulwp->ul_stk, ulwp->ul_mapsiz);
2850Sstevel@tonic-gate /*
2860Sstevel@tonic-gate * Now put the free ulwp on the ulwp freelist.
2870Sstevel@tonic-gate */
2880Sstevel@tonic-gate ulwp->ul_mapsiz = 0;
2890Sstevel@tonic-gate ulwp->ul_next = NULL;
2900Sstevel@tonic-gate if (udp->ulwp_freelist == NULL)
2910Sstevel@tonic-gate udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
2920Sstevel@tonic-gate else {
2930Sstevel@tonic-gate udp->ulwp_lastfree->ul_next = ulwp;
2940Sstevel@tonic-gate udp->ulwp_lastfree = ulwp;
2950Sstevel@tonic-gate }
2960Sstevel@tonic-gate } else {
2970Sstevel@tonic-gate prev = ulwp;
2980Sstevel@tonic-gate ulwpp = &ulwp->ul_next;
2990Sstevel@tonic-gate }
3000Sstevel@tonic-gate }
3010Sstevel@tonic-gate }
3020Sstevel@tonic-gate
3030Sstevel@tonic-gate /*
3040Sstevel@tonic-gate * Find an unused stack of the requested size
3050Sstevel@tonic-gate * or create a new stack of the requested size.
3060Sstevel@tonic-gate * Return a pointer to the ulwp_t structure referring to the stack, or NULL.
3070Sstevel@tonic-gate * thr_exit() stores 1 in the ul_dead member.
3080Sstevel@tonic-gate * thr_join() stores -1 in the ul_lwpid member.
3090Sstevel@tonic-gate */
3108486SRoger.Faulkner@Sun.COM static ulwp_t *
find_stack(size_t stksize,size_t guardsize)3110Sstevel@tonic-gate find_stack(size_t stksize, size_t guardsize)
3120Sstevel@tonic-gate {
3131885Sraf static size_t pagesize = 0;
3141885Sraf
3150Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
3160Sstevel@tonic-gate size_t mapsize;
3170Sstevel@tonic-gate ulwp_t *prev;
3180Sstevel@tonic-gate ulwp_t *ulwp;
3190Sstevel@tonic-gate ulwp_t **ulwpp;
3200Sstevel@tonic-gate void *stk;
3210Sstevel@tonic-gate
3220Sstevel@tonic-gate /*
3230Sstevel@tonic-gate * The stack is allocated PROT_READ|PROT_WRITE|PROT_EXEC
3240Sstevel@tonic-gate * unless overridden by the system's configuration.
3250Sstevel@tonic-gate */
3260Sstevel@tonic-gate if (stackprot == 0) { /* do this once */
3270Sstevel@tonic-gate long lprot = _sysconf(_SC_STACK_PROT);
3280Sstevel@tonic-gate if (lprot <= 0)
3290Sstevel@tonic-gate lprot = (PROT_READ|PROT_WRITE|PROT_EXEC);
3300Sstevel@tonic-gate stackprot = (int)lprot;
3310Sstevel@tonic-gate }
3321885Sraf if (pagesize == 0) /* do this once */
3331885Sraf pagesize = _sysconf(_SC_PAGESIZE);
3341885Sraf
3350Sstevel@tonic-gate /*
3360Sstevel@tonic-gate * One megabyte stacks by default, but subtract off
3370Sstevel@tonic-gate * two pages for the system-created red zones.
3380Sstevel@tonic-gate * Round up a non-zero stack size to a pagesize multiple.
3390Sstevel@tonic-gate */
3400Sstevel@tonic-gate if (stksize == 0)
3411885Sraf stksize = DEFAULTSTACK - 2 * pagesize;
3420Sstevel@tonic-gate else
3431885Sraf stksize = ((stksize + pagesize - 1) & -pagesize);
3440Sstevel@tonic-gate
3450Sstevel@tonic-gate /*
3460Sstevel@tonic-gate * Round up the mapping size to a multiple of pagesize.
3470Sstevel@tonic-gate * Note: mmap() provides at least one page of red zone
3480Sstevel@tonic-gate * so we deduct that from the value of guardsize.
3490Sstevel@tonic-gate */
3500Sstevel@tonic-gate if (guardsize != 0)
3511885Sraf guardsize = ((guardsize + pagesize - 1) & -pagesize) - pagesize;
3520Sstevel@tonic-gate mapsize = stksize + guardsize;
3530Sstevel@tonic-gate
3540Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
3550Sstevel@tonic-gate for (prev = NULL, ulwpp = &udp->lwp_stacks;
3560Sstevel@tonic-gate (ulwp = *ulwpp) != NULL;
3570Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_next) {
3580Sstevel@tonic-gate if (ulwp->ul_mapsiz == mapsize &&
3590Sstevel@tonic-gate ulwp->ul_guardsize == guardsize &&
3600Sstevel@tonic-gate dead_and_buried(ulwp)) {
3610Sstevel@tonic-gate /*
3620Sstevel@tonic-gate * The previous lwp is gone; reuse the stack.
3630Sstevel@tonic-gate * Remove the ulwp from the stack list.
3640Sstevel@tonic-gate */
3650Sstevel@tonic-gate *ulwpp = ulwp->ul_next;
3660Sstevel@tonic-gate ulwp->ul_next = NULL;
3670Sstevel@tonic-gate if (ulwp == udp->lwp_laststack)
3680Sstevel@tonic-gate udp->lwp_laststack = prev;
3690Sstevel@tonic-gate hash_out(ulwp, udp);
3700Sstevel@tonic-gate udp->nfreestack--;
3710Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
3720Sstevel@tonic-gate ulwp_clean(ulwp);
3730Sstevel@tonic-gate return (ulwp);
3740Sstevel@tonic-gate }
3750Sstevel@tonic-gate }
3760Sstevel@tonic-gate
3770Sstevel@tonic-gate /*
3780Sstevel@tonic-gate * None of the cached stacks matched our mapping size.
3790Sstevel@tonic-gate * Reduce the stack cache to get rid of possibly
3800Sstevel@tonic-gate * very old stacks that will never be reused.
3810Sstevel@tonic-gate */
3820Sstevel@tonic-gate if (udp->nfreestack > udp->thread_stack_cache)
3830Sstevel@tonic-gate trim_stack_cache(udp->thread_stack_cache);
3840Sstevel@tonic-gate else if (udp->nfreestack > 0)
3850Sstevel@tonic-gate trim_stack_cache(udp->nfreestack - 1);
3860Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
3870Sstevel@tonic-gate
3880Sstevel@tonic-gate /*
3890Sstevel@tonic-gate * Create a new stack.
3900Sstevel@tonic-gate */
3916515Sraf if ((stk = mmap(NULL, mapsize, stackprot,
3920Sstevel@tonic-gate MAP_PRIVATE|MAP_NORESERVE|MAP_ANON, -1, (off_t)0)) != MAP_FAILED) {
3930Sstevel@tonic-gate /*
3940Sstevel@tonic-gate * We have allocated our stack. Now allocate the ulwp.
3950Sstevel@tonic-gate */
3960Sstevel@tonic-gate ulwp = ulwp_alloc();
3970Sstevel@tonic-gate if (ulwp == NULL)
3986515Sraf (void) munmap(stk, mapsize);
3990Sstevel@tonic-gate else {
4000Sstevel@tonic-gate ulwp->ul_stk = stk;
4010Sstevel@tonic-gate ulwp->ul_mapsiz = mapsize;
4020Sstevel@tonic-gate ulwp->ul_guardsize = guardsize;
4030Sstevel@tonic-gate ulwp->ul_stktop = (uintptr_t)stk + mapsize;
4040Sstevel@tonic-gate ulwp->ul_stksiz = stksize;
4050Sstevel@tonic-gate if (guardsize) /* protect the extra red zone */
4066515Sraf (void) mprotect(stk, guardsize, PROT_NONE);
4070Sstevel@tonic-gate }
4080Sstevel@tonic-gate }
4090Sstevel@tonic-gate return (ulwp);
4100Sstevel@tonic-gate }
4110Sstevel@tonic-gate
4120Sstevel@tonic-gate /*
4130Sstevel@tonic-gate * Get a ulwp_t structure from the free list or allocate a new one.
4140Sstevel@tonic-gate * Such ulwp_t's do not have a stack allocated by the library.
4150Sstevel@tonic-gate */
4160Sstevel@tonic-gate static ulwp_t *
ulwp_alloc(void)4170Sstevel@tonic-gate ulwp_alloc(void)
4180Sstevel@tonic-gate {
4190Sstevel@tonic-gate ulwp_t *self = curthread;
4200Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
4210Sstevel@tonic-gate size_t tls_size;
4220Sstevel@tonic-gate ulwp_t *prev;
4230Sstevel@tonic-gate ulwp_t *ulwp;
4240Sstevel@tonic-gate ulwp_t **ulwpp;
4250Sstevel@tonic-gate caddr_t data;
4260Sstevel@tonic-gate
4270Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
4280Sstevel@tonic-gate for (prev = NULL, ulwpp = &udp->ulwp_freelist;
4290Sstevel@tonic-gate (ulwp = *ulwpp) != NULL;
4300Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_next) {
4310Sstevel@tonic-gate if (dead_and_buried(ulwp)) {
4320Sstevel@tonic-gate *ulwpp = ulwp->ul_next;
4330Sstevel@tonic-gate ulwp->ul_next = NULL;
4340Sstevel@tonic-gate if (ulwp == udp->ulwp_lastfree)
4350Sstevel@tonic-gate udp->ulwp_lastfree = prev;
4360Sstevel@tonic-gate hash_out(ulwp, udp);
4370Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
4380Sstevel@tonic-gate ulwp_clean(ulwp);
4390Sstevel@tonic-gate return (ulwp);
4400Sstevel@tonic-gate }
4410Sstevel@tonic-gate }
4420Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
4430Sstevel@tonic-gate
4440Sstevel@tonic-gate tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
4450Sstevel@tonic-gate data = lmalloc(sizeof (*ulwp) + tls_size);
4460Sstevel@tonic-gate if (data != NULL) {
4470Sstevel@tonic-gate /* LINTED pointer cast may result in improper alignment */
4480Sstevel@tonic-gate ulwp = (ulwp_t *)(data + tls_size);
4490Sstevel@tonic-gate }
4500Sstevel@tonic-gate return (ulwp);
4510Sstevel@tonic-gate }
4520Sstevel@tonic-gate
4530Sstevel@tonic-gate /*
4540Sstevel@tonic-gate * Free a ulwp structure.
4550Sstevel@tonic-gate * If there is an associated stack, put it on the stack list and
4560Sstevel@tonic-gate * munmap() previously freed stacks up to the residual cache limit.
4570Sstevel@tonic-gate * Else put it on the ulwp free list and never call lfree() on it.
4580Sstevel@tonic-gate */
4590Sstevel@tonic-gate static void
ulwp_free(ulwp_t * ulwp)4600Sstevel@tonic-gate ulwp_free(ulwp_t *ulwp)
4610Sstevel@tonic-gate {
4620Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
4630Sstevel@tonic-gate
4640Sstevel@tonic-gate ASSERT(udp->nthreads <= 1 || MUTEX_OWNED(&udp->link_lock, curthread));
4650Sstevel@tonic-gate ulwp->ul_next = NULL;
4660Sstevel@tonic-gate if (ulwp == udp->ulwp_one) /* don't reuse the primoridal stack */
4670Sstevel@tonic-gate /*EMPTY*/;
4680Sstevel@tonic-gate else if (ulwp->ul_mapsiz != 0) {
4690Sstevel@tonic-gate if (udp->lwp_stacks == NULL)
4700Sstevel@tonic-gate udp->lwp_stacks = udp->lwp_laststack = ulwp;
4710Sstevel@tonic-gate else {
4720Sstevel@tonic-gate udp->lwp_laststack->ul_next = ulwp;
4730Sstevel@tonic-gate udp->lwp_laststack = ulwp;
4740Sstevel@tonic-gate }
4750Sstevel@tonic-gate if (++udp->nfreestack > udp->thread_stack_cache)
4760Sstevel@tonic-gate trim_stack_cache(udp->thread_stack_cache);
4770Sstevel@tonic-gate } else {
4780Sstevel@tonic-gate if (udp->ulwp_freelist == NULL)
4790Sstevel@tonic-gate udp->ulwp_freelist = udp->ulwp_lastfree = ulwp;
4800Sstevel@tonic-gate else {
4810Sstevel@tonic-gate udp->ulwp_lastfree->ul_next = ulwp;
4820Sstevel@tonic-gate udp->ulwp_lastfree = ulwp;
4830Sstevel@tonic-gate }
4840Sstevel@tonic-gate }
4850Sstevel@tonic-gate }
4860Sstevel@tonic-gate
4870Sstevel@tonic-gate /*
4880Sstevel@tonic-gate * Find a named lwp and return a pointer to its hash list location.
4890Sstevel@tonic-gate * On success, returns with the hash lock held.
4900Sstevel@tonic-gate */
4910Sstevel@tonic-gate ulwp_t **
find_lwpp(thread_t tid)4920Sstevel@tonic-gate find_lwpp(thread_t tid)
4930Sstevel@tonic-gate {
4940Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
4950Sstevel@tonic-gate int ix = TIDHASH(tid, udp);
4960Sstevel@tonic-gate mutex_t *mp = &udp->thr_hash_table[ix].hash_lock;
4970Sstevel@tonic-gate ulwp_t *ulwp;
4980Sstevel@tonic-gate ulwp_t **ulwpp;
4990Sstevel@tonic-gate
5000Sstevel@tonic-gate if (tid == 0)
5010Sstevel@tonic-gate return (NULL);
5020Sstevel@tonic-gate
5030Sstevel@tonic-gate lmutex_lock(mp);
5040Sstevel@tonic-gate for (ulwpp = &udp->thr_hash_table[ix].hash_bucket;
5050Sstevel@tonic-gate (ulwp = *ulwpp) != NULL;
5060Sstevel@tonic-gate ulwpp = &ulwp->ul_hash) {
5070Sstevel@tonic-gate if (ulwp->ul_lwpid == tid)
5080Sstevel@tonic-gate return (ulwpp);
5090Sstevel@tonic-gate }
5100Sstevel@tonic-gate lmutex_unlock(mp);
5110Sstevel@tonic-gate return (NULL);
5120Sstevel@tonic-gate }
5130Sstevel@tonic-gate
5140Sstevel@tonic-gate /*
5150Sstevel@tonic-gate * Wake up all lwps waiting on this lwp for some reason.
5160Sstevel@tonic-gate */
5170Sstevel@tonic-gate void
ulwp_broadcast(ulwp_t * ulwp)5180Sstevel@tonic-gate ulwp_broadcast(ulwp_t *ulwp)
5190Sstevel@tonic-gate {
5200Sstevel@tonic-gate ulwp_t *self = curthread;
5210Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
5220Sstevel@tonic-gate
5230Sstevel@tonic-gate ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));
5246812Sraf (void) cond_broadcast(ulwp_condvar(ulwp, udp));
5250Sstevel@tonic-gate }
5260Sstevel@tonic-gate
5270Sstevel@tonic-gate /*
5280Sstevel@tonic-gate * Find a named lwp and return a pointer to it.
5290Sstevel@tonic-gate * Returns with the hash lock held.
5300Sstevel@tonic-gate */
5310Sstevel@tonic-gate ulwp_t *
find_lwp(thread_t tid)5320Sstevel@tonic-gate find_lwp(thread_t tid)
5330Sstevel@tonic-gate {
5340Sstevel@tonic-gate ulwp_t *self = curthread;
5350Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
5360Sstevel@tonic-gate ulwp_t *ulwp = NULL;
5370Sstevel@tonic-gate ulwp_t **ulwpp;
5380Sstevel@tonic-gate
5390Sstevel@tonic-gate if (self->ul_lwpid == tid) {
5400Sstevel@tonic-gate ulwp = self;
5410Sstevel@tonic-gate ulwp_lock(ulwp, udp);
5420Sstevel@tonic-gate } else if ((ulwpp = find_lwpp(tid)) != NULL) {
5430Sstevel@tonic-gate ulwp = *ulwpp;
5440Sstevel@tonic-gate }
5450Sstevel@tonic-gate
5460Sstevel@tonic-gate if (ulwp && ulwp->ul_dead) {
5470Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
5480Sstevel@tonic-gate ulwp = NULL;
5490Sstevel@tonic-gate }
5500Sstevel@tonic-gate
5510Sstevel@tonic-gate return (ulwp);
5520Sstevel@tonic-gate }
5530Sstevel@tonic-gate
5540Sstevel@tonic-gate int
_thrp_create(void * stk,size_t stksize,void * (* func)(void *),void * arg,long flags,thread_t * new_thread,size_t guardsize)5550Sstevel@tonic-gate _thrp_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
5566247Sraf long flags, thread_t *new_thread, size_t guardsize)
5570Sstevel@tonic-gate {
5580Sstevel@tonic-gate ulwp_t *self = curthread;
5590Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
5600Sstevel@tonic-gate ucontext_t uc;
5610Sstevel@tonic-gate uint_t lwp_flags;
5620Sstevel@tonic-gate thread_t tid;
5637657SRoger.Faulkner@Sun.COM int error;
5640Sstevel@tonic-gate ulwp_t *ulwp;
5650Sstevel@tonic-gate
5660Sstevel@tonic-gate /*
5670Sstevel@tonic-gate * Enforce the restriction of not creating any threads
5680Sstevel@tonic-gate * until the primary link map has been initialized.
5690Sstevel@tonic-gate * Also, disallow thread creation to a child of vfork().
5700Sstevel@tonic-gate */
5710Sstevel@tonic-gate if (!self->ul_primarymap || self->ul_vfork)
5720Sstevel@tonic-gate return (ENOTSUP);
5730Sstevel@tonic-gate
5740Sstevel@tonic-gate if (udp->hash_size == 1)
5750Sstevel@tonic-gate finish_init();
5760Sstevel@tonic-gate
5776247Sraf if ((stk || stksize) && stksize < MINSTACK)
5780Sstevel@tonic-gate return (EINVAL);
5790Sstevel@tonic-gate
5800Sstevel@tonic-gate if (stk == NULL) {
5810Sstevel@tonic-gate if ((ulwp = find_stack(stksize, guardsize)) == NULL)
5820Sstevel@tonic-gate return (ENOMEM);
5830Sstevel@tonic-gate stksize = ulwp->ul_mapsiz - ulwp->ul_guardsize;
5840Sstevel@tonic-gate } else {
5850Sstevel@tonic-gate /* initialize the private stack */
5860Sstevel@tonic-gate if ((ulwp = ulwp_alloc()) == NULL)
5870Sstevel@tonic-gate return (ENOMEM);
5880Sstevel@tonic-gate ulwp->ul_stk = stk;
5890Sstevel@tonic-gate ulwp->ul_stktop = (uintptr_t)stk + stksize;
5900Sstevel@tonic-gate ulwp->ul_stksiz = stksize;
5910Sstevel@tonic-gate }
5928486SRoger.Faulkner@Sun.COM /* ulwp is not in the hash table; make sure hash_out() doesn't fail */
5938486SRoger.Faulkner@Sun.COM ulwp->ul_ix = -1;
5940Sstevel@tonic-gate ulwp->ul_errnop = &ulwp->ul_errno;
5950Sstevel@tonic-gate
5960Sstevel@tonic-gate lwp_flags = LWP_SUSPENDED;
5970Sstevel@tonic-gate if (flags & (THR_DETACHED|THR_DAEMON)) {
5980Sstevel@tonic-gate flags |= THR_DETACHED;
5990Sstevel@tonic-gate lwp_flags |= LWP_DETACHED;
6000Sstevel@tonic-gate }
6010Sstevel@tonic-gate if (flags & THR_DAEMON)
6020Sstevel@tonic-gate lwp_flags |= LWP_DAEMON;
6030Sstevel@tonic-gate
6046812Sraf /* creating a thread: enforce mt-correctness in mutex_lock() */
6050Sstevel@tonic-gate self->ul_async_safe = 1;
6060Sstevel@tonic-gate
6070Sstevel@tonic-gate /* per-thread copies of global variables, for speed */
6080Sstevel@tonic-gate ulwp->ul_queue_fifo = self->ul_queue_fifo;
6090Sstevel@tonic-gate ulwp->ul_cond_wait_defer = self->ul_cond_wait_defer;
6100Sstevel@tonic-gate ulwp->ul_error_detection = self->ul_error_detection;
6110Sstevel@tonic-gate ulwp->ul_async_safe = self->ul_async_safe;
6120Sstevel@tonic-gate ulwp->ul_max_spinners = self->ul_max_spinners;
6130Sstevel@tonic-gate ulwp->ul_adaptive_spin = self->ul_adaptive_spin;
6140Sstevel@tonic-gate ulwp->ul_queue_spin = self->ul_queue_spin;
6150Sstevel@tonic-gate ulwp->ul_door_noreserve = self->ul_door_noreserve;
6167255Sraf ulwp->ul_misaligned = self->ul_misaligned;
6170Sstevel@tonic-gate
6186247Sraf /* new thread inherits creating thread's scheduling parameters */
6196247Sraf ulwp->ul_policy = self->ul_policy;
6206247Sraf ulwp->ul_pri = (self->ul_epri? self->ul_epri : self->ul_pri);
6216247Sraf ulwp->ul_cid = self->ul_cid;
6226247Sraf ulwp->ul_rtclassid = self->ul_rtclassid;
6236247Sraf
6240Sstevel@tonic-gate ulwp->ul_primarymap = self->ul_primarymap;
6250Sstevel@tonic-gate ulwp->ul_self = ulwp;
6260Sstevel@tonic-gate ulwp->ul_uberdata = udp;
6270Sstevel@tonic-gate
6280Sstevel@tonic-gate /* debugger support */
6290Sstevel@tonic-gate ulwp->ul_usropts = flags;
6300Sstevel@tonic-gate
6310Sstevel@tonic-gate #ifdef __sparc
6320Sstevel@tonic-gate /*
6330Sstevel@tonic-gate * We cache several instructions in the thread structure for use
6340Sstevel@tonic-gate * by the fasttrap DTrace provider. When changing this, read the
6350Sstevel@tonic-gate * comment in fasttrap.h for the all the other places that must
6360Sstevel@tonic-gate * be changed.
6370Sstevel@tonic-gate */
6380Sstevel@tonic-gate ulwp->ul_dsave = 0x9de04000; /* save %g1, %g0, %sp */
6390Sstevel@tonic-gate ulwp->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */
6400Sstevel@tonic-gate ulwp->ul_dftret = 0x91d0203a; /* ta 0x3a */
6410Sstevel@tonic-gate ulwp->ul_dreturn = 0x81ca0000; /* return %o0 */
6420Sstevel@tonic-gate #endif
6430Sstevel@tonic-gate
6440Sstevel@tonic-gate ulwp->ul_startpc = func;
6450Sstevel@tonic-gate ulwp->ul_startarg = arg;
6460Sstevel@tonic-gate _fpinherit(ulwp);
6470Sstevel@tonic-gate /*
6480Sstevel@tonic-gate * Defer signals on the new thread until its TLS constructors
6496812Sraf * have been called. _thrp_setup() will call sigon() after
6500Sstevel@tonic-gate * it has called tls_setup().
6510Sstevel@tonic-gate */
6520Sstevel@tonic-gate ulwp->ul_sigdefer = 1;
6530Sstevel@tonic-gate
6547657SRoger.Faulkner@Sun.COM error = setup_context(&uc, _thrp_setup, ulwp,
6557657SRoger.Faulkner@Sun.COM (caddr_t)ulwp->ul_stk + ulwp->ul_guardsize, stksize);
6567657SRoger.Faulkner@Sun.COM if (error != 0 && stk != NULL) /* inaccessible stack */
6577657SRoger.Faulkner@Sun.COM error = EFAULT;
6580Sstevel@tonic-gate
6590Sstevel@tonic-gate /*
6600Sstevel@tonic-gate * Call enter_critical() to avoid being suspended until we
6610Sstevel@tonic-gate * have linked the new thread into the proper lists.
6620Sstevel@tonic-gate * This is necessary because forkall() and fork1() must
6630Sstevel@tonic-gate * suspend all threads and they must see a complete list.
6640Sstevel@tonic-gate */
6650Sstevel@tonic-gate enter_critical(self);
6660Sstevel@tonic-gate uc.uc_sigmask = ulwp->ul_sigmask = self->ul_sigmask;
6670Sstevel@tonic-gate if (error != 0 ||
6680Sstevel@tonic-gate (error = __lwp_create(&uc, lwp_flags, &tid)) != 0) {
6690Sstevel@tonic-gate exit_critical(self);
6700Sstevel@tonic-gate ulwp->ul_lwpid = (lwpid_t)(-1);
6710Sstevel@tonic-gate ulwp->ul_dead = 1;
6720Sstevel@tonic-gate ulwp->ul_detached = 1;
6730Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
6740Sstevel@tonic-gate ulwp_free(ulwp);
6750Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
6760Sstevel@tonic-gate return (error);
6770Sstevel@tonic-gate }
6780Sstevel@tonic-gate self->ul_nocancel = 0; /* cancellation is now possible */
6790Sstevel@tonic-gate udp->uberflags.uf_mt = 1;
6800Sstevel@tonic-gate if (new_thread)
6810Sstevel@tonic-gate *new_thread = tid;
6820Sstevel@tonic-gate if (flags & THR_DETACHED)
6830Sstevel@tonic-gate ulwp->ul_detached = 1;
6840Sstevel@tonic-gate ulwp->ul_lwpid = tid;
6850Sstevel@tonic-gate ulwp->ul_stop = TSTP_REGULAR;
6863512Sraf if (flags & THR_SUSPENDED)
6873512Sraf ulwp->ul_created = 1;
6880Sstevel@tonic-gate
6890Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
6900Sstevel@tonic-gate ulwp->ul_forw = udp->all_lwps;
6910Sstevel@tonic-gate ulwp->ul_back = udp->all_lwps->ul_back;
6920Sstevel@tonic-gate ulwp->ul_back->ul_forw = ulwp;
6930Sstevel@tonic-gate ulwp->ul_forw->ul_back = ulwp;
6940Sstevel@tonic-gate hash_in(ulwp, udp);
6950Sstevel@tonic-gate udp->nthreads++;
6960Sstevel@tonic-gate if (flags & THR_DAEMON)
6970Sstevel@tonic-gate udp->ndaemons++;
6980Sstevel@tonic-gate if (flags & THR_NEW_LWP)
6990Sstevel@tonic-gate thr_concurrency++;
7001672Sraf __libc_threaded = 1; /* inform stdio */
7010Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
7020Sstevel@tonic-gate
7030Sstevel@tonic-gate if (__td_event_report(self, TD_CREATE, udp)) {
7040Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_CREATE;
7050Sstevel@tonic-gate self->ul_td_evbuf.eventdata = (void *)(uintptr_t)tid;
7060Sstevel@tonic-gate tdb_event(TD_CREATE, udp);
7070Sstevel@tonic-gate }
7080Sstevel@tonic-gate
7090Sstevel@tonic-gate exit_critical(self);
7103512Sraf
7113512Sraf if (!(flags & THR_SUSPENDED))
7123512Sraf (void) _thrp_continue(tid, TSTP_REGULAR);
7133512Sraf
7140Sstevel@tonic-gate return (0);
7150Sstevel@tonic-gate }
7160Sstevel@tonic-gate
7170Sstevel@tonic-gate int
thr_create(void * stk,size_t stksize,void * (* func)(void *),void * arg,long flags,thread_t * new_thread)7186812Sraf thr_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
7190Sstevel@tonic-gate long flags, thread_t *new_thread)
7200Sstevel@tonic-gate {
7216247Sraf return (_thrp_create(stk, stksize, func, arg, flags, new_thread, 0));
7220Sstevel@tonic-gate }
7230Sstevel@tonic-gate
7240Sstevel@tonic-gate /*
7250Sstevel@tonic-gate * A special cancellation cleanup hook for DCE.
7260Sstevel@tonic-gate * cleanuphndlr, when it is not NULL, will contain a callback
7270Sstevel@tonic-gate * function to be called before a thread is terminated in
7286812Sraf * thr_exit() as a result of being cancelled.
7290Sstevel@tonic-gate */
7300Sstevel@tonic-gate static void (*cleanuphndlr)(void) = NULL;
7310Sstevel@tonic-gate
7320Sstevel@tonic-gate /*
7330Sstevel@tonic-gate * _pthread_setcleanupinit: sets the cleanup hook.
7340Sstevel@tonic-gate */
7350Sstevel@tonic-gate int
_pthread_setcleanupinit(void (* func)(void))7360Sstevel@tonic-gate _pthread_setcleanupinit(void (*func)(void))
7370Sstevel@tonic-gate {
7380Sstevel@tonic-gate cleanuphndlr = func;
7390Sstevel@tonic-gate return (0);
7400Sstevel@tonic-gate }
7410Sstevel@tonic-gate
7420Sstevel@tonic-gate void
_thrp_exit()7430Sstevel@tonic-gate _thrp_exit()
7440Sstevel@tonic-gate {
7450Sstevel@tonic-gate ulwp_t *self = curthread;
7460Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
7470Sstevel@tonic-gate ulwp_t *replace = NULL;
7480Sstevel@tonic-gate
7490Sstevel@tonic-gate if (__td_event_report(self, TD_DEATH, udp)) {
7500Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_DEATH;
7510Sstevel@tonic-gate tdb_event(TD_DEATH, udp);
7520Sstevel@tonic-gate }
7530Sstevel@tonic-gate
7540Sstevel@tonic-gate ASSERT(self->ul_sigdefer != 0);
7550Sstevel@tonic-gate
7560Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
7570Sstevel@tonic-gate udp->nthreads--;
7580Sstevel@tonic-gate if (self->ul_usropts & THR_NEW_LWP)
7590Sstevel@tonic-gate thr_concurrency--;
7600Sstevel@tonic-gate if (self->ul_usropts & THR_DAEMON)
7610Sstevel@tonic-gate udp->ndaemons--;
7620Sstevel@tonic-gate else if (udp->nthreads == udp->ndaemons) {
7630Sstevel@tonic-gate /*
7640Sstevel@tonic-gate * We are the last non-daemon thread exiting.
7650Sstevel@tonic-gate * Exit the process. We retain our TSD and TLS so
7660Sstevel@tonic-gate * that atexit() application functions can use them.
7670Sstevel@tonic-gate */
7680Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
7690Sstevel@tonic-gate exit(0);
7700Sstevel@tonic-gate thr_panic("_thrp_exit(): exit(0) returned");
7710Sstevel@tonic-gate }
7720Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
7730Sstevel@tonic-gate
7744574Sraf tsd_exit(); /* deallocate thread-specific data */
7754574Sraf tls_exit(); /* deallocate thread-local storage */
7764574Sraf heldlock_exit(); /* deal with left-over held locks */
7770Sstevel@tonic-gate
7780Sstevel@tonic-gate /* block all signals to finish exiting */
7790Sstevel@tonic-gate block_all_signals(self);
7800Sstevel@tonic-gate /* also prevent ourself from being suspended */
7810Sstevel@tonic-gate enter_critical(self);
7820Sstevel@tonic-gate rwl_free(self);
7830Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
7840Sstevel@tonic-gate ulwp_free(self);
7850Sstevel@tonic-gate (void) ulwp_lock(self, udp);
7860Sstevel@tonic-gate
7870Sstevel@tonic-gate if (self->ul_mapsiz && !self->ul_detached) {
7880Sstevel@tonic-gate /*
7890Sstevel@tonic-gate * We want to free the stack for reuse but must keep
7900Sstevel@tonic-gate * the ulwp_t struct for the benefit of thr_join().
7910Sstevel@tonic-gate * For this purpose we allocate a replacement ulwp_t.
7920Sstevel@tonic-gate */
7930Sstevel@tonic-gate if ((replace = udp->ulwp_replace_free) == NULL)
7940Sstevel@tonic-gate replace = lmalloc(REPLACEMENT_SIZE);
7950Sstevel@tonic-gate else if ((udp->ulwp_replace_free = replace->ul_next) == NULL)
7960Sstevel@tonic-gate udp->ulwp_replace_last = NULL;
7970Sstevel@tonic-gate }
7980Sstevel@tonic-gate
7990Sstevel@tonic-gate if (udp->all_lwps == self)
8000Sstevel@tonic-gate udp->all_lwps = self->ul_forw;
8010Sstevel@tonic-gate if (udp->all_lwps == self)
8020Sstevel@tonic-gate udp->all_lwps = NULL;
8030Sstevel@tonic-gate else {
8040Sstevel@tonic-gate self->ul_forw->ul_back = self->ul_back;
8050Sstevel@tonic-gate self->ul_back->ul_forw = self->ul_forw;
8060Sstevel@tonic-gate }
8070Sstevel@tonic-gate self->ul_forw = self->ul_back = NULL;
8086247Sraf #if defined(THREAD_DEBUG)
8090Sstevel@tonic-gate /* collect queue lock statistics before marking ourself dead */
8100Sstevel@tonic-gate record_spin_locks(self);
8116247Sraf #endif
8120Sstevel@tonic-gate self->ul_dead = 1;
8130Sstevel@tonic-gate self->ul_pleasestop = 0;
8140Sstevel@tonic-gate if (replace != NULL) {
8150Sstevel@tonic-gate int ix = self->ul_ix; /* the hash index */
8166515Sraf (void) memcpy(replace, self, REPLACEMENT_SIZE);
8170Sstevel@tonic-gate replace->ul_self = replace;
8180Sstevel@tonic-gate replace->ul_next = NULL; /* clone not on stack list */
8190Sstevel@tonic-gate replace->ul_mapsiz = 0; /* allows clone to be freed */
8200Sstevel@tonic-gate replace->ul_replace = 1; /* requires clone to be freed */
8210Sstevel@tonic-gate hash_out_unlocked(self, ix, udp);
8220Sstevel@tonic-gate hash_in_unlocked(replace, ix, udp);
8230Sstevel@tonic-gate ASSERT(!(self->ul_detached));
8240Sstevel@tonic-gate self->ul_detached = 1; /* this frees the stack */
8250Sstevel@tonic-gate self->ul_schedctl = NULL;
8260Sstevel@tonic-gate self->ul_schedctl_called = &udp->uberflags;
8270Sstevel@tonic-gate set_curthread(self = replace);
8280Sstevel@tonic-gate /*
8290Sstevel@tonic-gate * Having just changed the address of curthread, we
8300Sstevel@tonic-gate * must reset the ownership of the locks we hold so
8310Sstevel@tonic-gate * that assertions will not fire when we release them.
8320Sstevel@tonic-gate */
8330Sstevel@tonic-gate udp->link_lock.mutex_owner = (uintptr_t)self;
8340Sstevel@tonic-gate ulwp_mutex(self, udp)->mutex_owner = (uintptr_t)self;
8350Sstevel@tonic-gate /*
8360Sstevel@tonic-gate * NOTE:
8370Sstevel@tonic-gate * On i386, %gs still references the original, not the
8380Sstevel@tonic-gate * replacement, ulwp structure. Fetching the replacement
8390Sstevel@tonic-gate * curthread pointer via %gs:0 works correctly since the
8400Sstevel@tonic-gate * original ulwp structure will not be reallocated until
8410Sstevel@tonic-gate * this lwp has completed its lwp_exit() system call (see
8420Sstevel@tonic-gate * dead_and_buried()), but from here on out, we must make
8430Sstevel@tonic-gate * no references to %gs:<offset> other than %gs:0.
8440Sstevel@tonic-gate */
8450Sstevel@tonic-gate }
8460Sstevel@tonic-gate /*
8470Sstevel@tonic-gate * Put non-detached terminated threads in the all_zombies list.
8480Sstevel@tonic-gate */
8490Sstevel@tonic-gate if (!self->ul_detached) {
8500Sstevel@tonic-gate udp->nzombies++;
8510Sstevel@tonic-gate if (udp->all_zombies == NULL) {
8520Sstevel@tonic-gate ASSERT(udp->nzombies == 1);
8530Sstevel@tonic-gate udp->all_zombies = self->ul_forw = self->ul_back = self;
8540Sstevel@tonic-gate } else {
8550Sstevel@tonic-gate self->ul_forw = udp->all_zombies;
8560Sstevel@tonic-gate self->ul_back = udp->all_zombies->ul_back;
8570Sstevel@tonic-gate self->ul_back->ul_forw = self;
8580Sstevel@tonic-gate self->ul_forw->ul_back = self;
8590Sstevel@tonic-gate }
8600Sstevel@tonic-gate }
8610Sstevel@tonic-gate /*
8620Sstevel@tonic-gate * Notify everyone waiting for this thread.
8630Sstevel@tonic-gate */
8640Sstevel@tonic-gate ulwp_broadcast(self);
8650Sstevel@tonic-gate (void) ulwp_unlock(self, udp);
8660Sstevel@tonic-gate /*
8670Sstevel@tonic-gate * Prevent any more references to the schedctl data.
8680Sstevel@tonic-gate * We are exiting and continue_fork() may not find us.
8690Sstevel@tonic-gate * Do this just before dropping link_lock, since fork
8700Sstevel@tonic-gate * serializes on link_lock.
8710Sstevel@tonic-gate */
8720Sstevel@tonic-gate self->ul_schedctl = NULL;
8730Sstevel@tonic-gate self->ul_schedctl_called = &udp->uberflags;
8740Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
8750Sstevel@tonic-gate
8760Sstevel@tonic-gate ASSERT(self->ul_critical == 1);
8770Sstevel@tonic-gate ASSERT(self->ul_preempt == 0);
8780Sstevel@tonic-gate _lwp_terminate(); /* never returns */
8790Sstevel@tonic-gate thr_panic("_thrp_exit(): _lwp_terminate() returned");
8800Sstevel@tonic-gate }
8810Sstevel@tonic-gate
8826247Sraf #if defined(THREAD_DEBUG)
8830Sstevel@tonic-gate void
collect_queue_statistics()8840Sstevel@tonic-gate collect_queue_statistics()
8850Sstevel@tonic-gate {
8860Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
8870Sstevel@tonic-gate ulwp_t *ulwp;
8880Sstevel@tonic-gate
8890Sstevel@tonic-gate if (thread_queue_dump) {
8900Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
8910Sstevel@tonic-gate if ((ulwp = udp->all_lwps) != NULL) {
8920Sstevel@tonic-gate do {
8930Sstevel@tonic-gate record_spin_locks(ulwp);
8940Sstevel@tonic-gate } while ((ulwp = ulwp->ul_forw) != udp->all_lwps);
8950Sstevel@tonic-gate }
8960Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
8970Sstevel@tonic-gate }
8980Sstevel@tonic-gate }
8996247Sraf #endif
9000Sstevel@tonic-gate
9016812Sraf static void __NORETURN
_thrp_exit_common(void * status,int unwind)9026812Sraf _thrp_exit_common(void *status, int unwind)
9030Sstevel@tonic-gate {
9040Sstevel@tonic-gate ulwp_t *self = curthread;
9050Sstevel@tonic-gate int cancelled = (self->ul_cancel_pending && status == PTHREAD_CANCELED);
9060Sstevel@tonic-gate
9070Sstevel@tonic-gate ASSERT(self->ul_critical == 0 && self->ul_preempt == 0);
9080Sstevel@tonic-gate
9090Sstevel@tonic-gate /*
9100Sstevel@tonic-gate * Disable cancellation and call the special DCE cancellation
9110Sstevel@tonic-gate * cleanup hook if it is enabled. Do nothing else before calling
9120Sstevel@tonic-gate * the DCE cancellation cleanup hook; it may call longjmp() and
9130Sstevel@tonic-gate * never return here.
9140Sstevel@tonic-gate */
9150Sstevel@tonic-gate self->ul_cancel_disabled = 1;
9160Sstevel@tonic-gate self->ul_cancel_async = 0;
9170Sstevel@tonic-gate self->ul_save_async = 0;
9180Sstevel@tonic-gate self->ul_cancelable = 0;
9190Sstevel@tonic-gate self->ul_cancel_pending = 0;
9205891Sraf set_cancel_pending_flag(self, 1);
9210Sstevel@tonic-gate if (cancelled && cleanuphndlr != NULL)
9220Sstevel@tonic-gate (*cleanuphndlr)();
9230Sstevel@tonic-gate
9240Sstevel@tonic-gate /*
9250Sstevel@tonic-gate * Block application signals while we are exiting.
9260Sstevel@tonic-gate * We call out to C++, TSD, and TLS destructors while exiting
9270Sstevel@tonic-gate * and these are application-defined, so we cannot be assured
9280Sstevel@tonic-gate * that they won't reset the signal mask. We use sigoff() to
9290Sstevel@tonic-gate * defer any signals that may be received as a result of this
9300Sstevel@tonic-gate * bad behavior. Such signals will be lost to the process
9310Sstevel@tonic-gate * when the thread finishes exiting.
9320Sstevel@tonic-gate */
9336812Sraf (void) thr_sigsetmask(SIG_SETMASK, &maskset, NULL);
9340Sstevel@tonic-gate sigoff(self);
9350Sstevel@tonic-gate
9360Sstevel@tonic-gate self->ul_rval = status;
9370Sstevel@tonic-gate
9380Sstevel@tonic-gate /*
9390Sstevel@tonic-gate * If thr_exit is being called from the places where
9400Sstevel@tonic-gate * C++ destructors are to be called such as cancellation
9410Sstevel@tonic-gate * points, then set this flag. It is checked in _t_cancel()
9420Sstevel@tonic-gate * to decide whether _ex_unwind() is to be called or not.
9430Sstevel@tonic-gate */
9440Sstevel@tonic-gate if (unwind)
9450Sstevel@tonic-gate self->ul_unwind = 1;
9460Sstevel@tonic-gate
9470Sstevel@tonic-gate /*
9480Sstevel@tonic-gate * _thrp_unwind() will eventually call _thrp_exit().
9490Sstevel@tonic-gate * It never returns.
9500Sstevel@tonic-gate */
9510Sstevel@tonic-gate _thrp_unwind(NULL);
9526812Sraf thr_panic("_thrp_exit_common(): _thrp_unwind() returned");
9536812Sraf
9546812Sraf for (;;) /* to shut the compiler up about __NORETURN */
9556812Sraf continue;
9560Sstevel@tonic-gate }
9570Sstevel@tonic-gate
9580Sstevel@tonic-gate /*
9590Sstevel@tonic-gate * Called when a thread returns from its start function.
9600Sstevel@tonic-gate * We are at the top of the stack; no unwinding is necessary.
9610Sstevel@tonic-gate */
9620Sstevel@tonic-gate void
_thrp_terminate(void * status)9636812Sraf _thrp_terminate(void *status)
9640Sstevel@tonic-gate {
9656812Sraf _thrp_exit_common(status, 0);
9660Sstevel@tonic-gate }
9670Sstevel@tonic-gate
9686812Sraf #pragma weak pthread_exit = thr_exit
9696812Sraf #pragma weak _thr_exit = thr_exit
9700Sstevel@tonic-gate void
thr_exit(void * status)9716812Sraf thr_exit(void *status)
9720Sstevel@tonic-gate {
9736812Sraf _thrp_exit_common(status, 1);
9740Sstevel@tonic-gate }
9750Sstevel@tonic-gate
9760Sstevel@tonic-gate int
_thrp_join(thread_t tid,thread_t * departed,void ** status,int do_cancel)9770Sstevel@tonic-gate _thrp_join(thread_t tid, thread_t *departed, void **status, int do_cancel)
9780Sstevel@tonic-gate {
9790Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
9800Sstevel@tonic-gate mutex_t *mp;
9810Sstevel@tonic-gate void *rval;
9820Sstevel@tonic-gate thread_t found;
9830Sstevel@tonic-gate ulwp_t *ulwp;
9840Sstevel@tonic-gate ulwp_t **ulwpp;
9850Sstevel@tonic-gate int replace;
9860Sstevel@tonic-gate int error;
9870Sstevel@tonic-gate
9880Sstevel@tonic-gate if (do_cancel)
9890Sstevel@tonic-gate error = lwp_wait(tid, &found);
9900Sstevel@tonic-gate else {
9910Sstevel@tonic-gate while ((error = __lwp_wait(tid, &found)) == EINTR)
9920Sstevel@tonic-gate ;
9930Sstevel@tonic-gate }
9940Sstevel@tonic-gate if (error)
9950Sstevel@tonic-gate return (error);
9960Sstevel@tonic-gate
9970Sstevel@tonic-gate /*
9980Sstevel@tonic-gate * We must hold link_lock to avoid a race condition with find_stack().
9990Sstevel@tonic-gate */
10000Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
10010Sstevel@tonic-gate if ((ulwpp = find_lwpp(found)) == NULL) {
10020Sstevel@tonic-gate /*
10030Sstevel@tonic-gate * lwp_wait() found an lwp that the library doesn't know
10040Sstevel@tonic-gate * about. It must have been created with _lwp_create().
10050Sstevel@tonic-gate * Just return its lwpid; we can't know its status.
10060Sstevel@tonic-gate */
10070Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
10080Sstevel@tonic-gate rval = NULL;
10090Sstevel@tonic-gate } else {
10100Sstevel@tonic-gate /*
10110Sstevel@tonic-gate * Remove ulwp from the hash table.
10120Sstevel@tonic-gate */
10130Sstevel@tonic-gate ulwp = *ulwpp;
10140Sstevel@tonic-gate *ulwpp = ulwp->ul_hash;
10150Sstevel@tonic-gate ulwp->ul_hash = NULL;
10160Sstevel@tonic-gate /*
10170Sstevel@tonic-gate * Remove ulwp from all_zombies list.
10180Sstevel@tonic-gate */
10190Sstevel@tonic-gate ASSERT(udp->nzombies >= 1);
10200Sstevel@tonic-gate if (udp->all_zombies == ulwp)
10210Sstevel@tonic-gate udp->all_zombies = ulwp->ul_forw;
10220Sstevel@tonic-gate if (udp->all_zombies == ulwp)
10230Sstevel@tonic-gate udp->all_zombies = NULL;
10240Sstevel@tonic-gate else {
10250Sstevel@tonic-gate ulwp->ul_forw->ul_back = ulwp->ul_back;
10260Sstevel@tonic-gate ulwp->ul_back->ul_forw = ulwp->ul_forw;
10270Sstevel@tonic-gate }
10280Sstevel@tonic-gate ulwp->ul_forw = ulwp->ul_back = NULL;
10290Sstevel@tonic-gate udp->nzombies--;
10300Sstevel@tonic-gate ASSERT(ulwp->ul_dead && !ulwp->ul_detached &&
10314843Sraf !(ulwp->ul_usropts & (THR_DETACHED|THR_DAEMON)));
10320Sstevel@tonic-gate /*
10330Sstevel@tonic-gate * We can't call ulwp_unlock(ulwp) after we set
10340Sstevel@tonic-gate * ulwp->ul_ix = -1 so we have to get a pointer to the
10350Sstevel@tonic-gate * ulwp's hash table mutex now in order to unlock it below.
10360Sstevel@tonic-gate */
10370Sstevel@tonic-gate mp = ulwp_mutex(ulwp, udp);
10380Sstevel@tonic-gate ulwp->ul_lwpid = (lwpid_t)(-1);
10390Sstevel@tonic-gate ulwp->ul_ix = -1;
10400Sstevel@tonic-gate rval = ulwp->ul_rval;
10410Sstevel@tonic-gate replace = ulwp->ul_replace;
10420Sstevel@tonic-gate lmutex_unlock(mp);
10430Sstevel@tonic-gate if (replace) {
10440Sstevel@tonic-gate ulwp->ul_next = NULL;
10450Sstevel@tonic-gate if (udp->ulwp_replace_free == NULL)
10460Sstevel@tonic-gate udp->ulwp_replace_free =
10474843Sraf udp->ulwp_replace_last = ulwp;
10480Sstevel@tonic-gate else {
10490Sstevel@tonic-gate udp->ulwp_replace_last->ul_next = ulwp;
10500Sstevel@tonic-gate udp->ulwp_replace_last = ulwp;
10510Sstevel@tonic-gate }
10520Sstevel@tonic-gate }
10530Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
10540Sstevel@tonic-gate }
10550Sstevel@tonic-gate
10560Sstevel@tonic-gate if (departed != NULL)
10570Sstevel@tonic-gate *departed = found;
10580Sstevel@tonic-gate if (status != NULL)
10590Sstevel@tonic-gate *status = rval;
10600Sstevel@tonic-gate return (0);
10610Sstevel@tonic-gate }
10620Sstevel@tonic-gate
10630Sstevel@tonic-gate int
thr_join(thread_t tid,thread_t * departed,void ** status)10646812Sraf thr_join(thread_t tid, thread_t *departed, void **status)
10650Sstevel@tonic-gate {
10660Sstevel@tonic-gate int error = _thrp_join(tid, departed, status, 1);
10670Sstevel@tonic-gate return ((error == EINVAL)? ESRCH : error);
10680Sstevel@tonic-gate }
10690Sstevel@tonic-gate
10700Sstevel@tonic-gate /*
10710Sstevel@tonic-gate * pthread_join() differs from Solaris thr_join():
10720Sstevel@tonic-gate * It does not return the departed thread's id
10730Sstevel@tonic-gate * and hence does not have a "departed" argument.
10740Sstevel@tonic-gate * It returns EINVAL if tid refers to a detached thread.
10750Sstevel@tonic-gate */
10766812Sraf #pragma weak _pthread_join = pthread_join
10770Sstevel@tonic-gate int
pthread_join(pthread_t tid,void ** status)10786812Sraf pthread_join(pthread_t tid, void **status)
10790Sstevel@tonic-gate {
10800Sstevel@tonic-gate return ((tid == 0)? ESRCH : _thrp_join(tid, NULL, status, 1));
10810Sstevel@tonic-gate }
10820Sstevel@tonic-gate
10830Sstevel@tonic-gate int
pthread_detach(pthread_t tid)10846812Sraf pthread_detach(pthread_t tid)
10850Sstevel@tonic-gate {
10860Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
10870Sstevel@tonic-gate ulwp_t *ulwp;
10880Sstevel@tonic-gate ulwp_t **ulwpp;
10890Sstevel@tonic-gate int error = 0;
10900Sstevel@tonic-gate
10910Sstevel@tonic-gate if ((ulwpp = find_lwpp(tid)) == NULL)
10920Sstevel@tonic-gate return (ESRCH);
10930Sstevel@tonic-gate ulwp = *ulwpp;
10940Sstevel@tonic-gate
10950Sstevel@tonic-gate if (ulwp->ul_dead) {
10960Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
10970Sstevel@tonic-gate error = _thrp_join(tid, NULL, NULL, 0);
10980Sstevel@tonic-gate } else {
10990Sstevel@tonic-gate error = __lwp_detach(tid);
11000Sstevel@tonic-gate ulwp->ul_detached = 1;
11010Sstevel@tonic-gate ulwp->ul_usropts |= THR_DETACHED;
11020Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
11030Sstevel@tonic-gate }
11040Sstevel@tonic-gate return (error);
11050Sstevel@tonic-gate }
11060Sstevel@tonic-gate
11070Sstevel@tonic-gate static const char *
ematch(const char * ev,const char * match)11080Sstevel@tonic-gate ematch(const char *ev, const char *match)
11090Sstevel@tonic-gate {
11100Sstevel@tonic-gate int c;
11110Sstevel@tonic-gate
11120Sstevel@tonic-gate while ((c = *match++) != '\0') {
11130Sstevel@tonic-gate if (*ev++ != c)
11140Sstevel@tonic-gate return (NULL);
11150Sstevel@tonic-gate }
11160Sstevel@tonic-gate if (*ev++ != '=')
11170Sstevel@tonic-gate return (NULL);
11180Sstevel@tonic-gate return (ev);
11190Sstevel@tonic-gate }
11200Sstevel@tonic-gate
11210Sstevel@tonic-gate static int
envvar(const char * ev,const char * match,int limit)11220Sstevel@tonic-gate envvar(const char *ev, const char *match, int limit)
11230Sstevel@tonic-gate {
11240Sstevel@tonic-gate int val = -1;
11250Sstevel@tonic-gate const char *ename;
11260Sstevel@tonic-gate
11270Sstevel@tonic-gate if ((ename = ematch(ev, match)) != NULL) {
11280Sstevel@tonic-gate int c;
11290Sstevel@tonic-gate for (val = 0; (c = *ename) != '\0'; ename++) {
11300Sstevel@tonic-gate if (!isdigit(c)) {
11310Sstevel@tonic-gate val = -1;
11320Sstevel@tonic-gate break;
11330Sstevel@tonic-gate }
11340Sstevel@tonic-gate val = val * 10 + (c - '0');
11350Sstevel@tonic-gate if (val > limit) {
11360Sstevel@tonic-gate val = limit;
11370Sstevel@tonic-gate break;
11380Sstevel@tonic-gate }
11390Sstevel@tonic-gate }
11400Sstevel@tonic-gate }
11410Sstevel@tonic-gate return (val);
11420Sstevel@tonic-gate }
11430Sstevel@tonic-gate
11440Sstevel@tonic-gate static void
etest(const char * ev)11450Sstevel@tonic-gate etest(const char *ev)
11460Sstevel@tonic-gate {
11470Sstevel@tonic-gate int value;
11480Sstevel@tonic-gate
11490Sstevel@tonic-gate if ((value = envvar(ev, "QUEUE_SPIN", 1000000)) >= 0)
11500Sstevel@tonic-gate thread_queue_spin = value;
11515629Sraf if ((value = envvar(ev, "ADAPTIVE_SPIN", 1000000)) >= 0)
11520Sstevel@tonic-gate thread_adaptive_spin = value;
11535629Sraf if ((value = envvar(ev, "MAX_SPINNERS", 255)) >= 0)
11540Sstevel@tonic-gate thread_max_spinners = value;
11550Sstevel@tonic-gate if ((value = envvar(ev, "QUEUE_FIFO", 8)) >= 0)
11560Sstevel@tonic-gate thread_queue_fifo = value;
11570Sstevel@tonic-gate #if defined(THREAD_DEBUG)
11580Sstevel@tonic-gate if ((value = envvar(ev, "QUEUE_VERIFY", 1)) >= 0)
11590Sstevel@tonic-gate thread_queue_verify = value;
11600Sstevel@tonic-gate if ((value = envvar(ev, "QUEUE_DUMP", 1)) >= 0)
11610Sstevel@tonic-gate thread_queue_dump = value;
11626247Sraf #endif
11630Sstevel@tonic-gate if ((value = envvar(ev, "STACK_CACHE", 10000)) >= 0)
11640Sstevel@tonic-gate thread_stack_cache = value;
11650Sstevel@tonic-gate if ((value = envvar(ev, "COND_WAIT_DEFER", 1)) >= 0)
11660Sstevel@tonic-gate thread_cond_wait_defer = value;
11670Sstevel@tonic-gate if ((value = envvar(ev, "ERROR_DETECTION", 2)) >= 0)
11680Sstevel@tonic-gate thread_error_detection = value;
11690Sstevel@tonic-gate if ((value = envvar(ev, "ASYNC_SAFE", 1)) >= 0)
11700Sstevel@tonic-gate thread_async_safe = value;
11710Sstevel@tonic-gate if ((value = envvar(ev, "DOOR_NORESERVE", 1)) >= 0)
11720Sstevel@tonic-gate thread_door_noreserve = value;
11737255Sraf if ((value = envvar(ev, "LOCKS_MISALIGNED", 1)) >= 0)
11747255Sraf thread_locks_misaligned = value;
11750Sstevel@tonic-gate }
11760Sstevel@tonic-gate
11770Sstevel@tonic-gate /*
11780Sstevel@tonic-gate * Look for and evaluate environment variables of the form "_THREAD_*".
11790Sstevel@tonic-gate * For compatibility with the past, we also look for environment
11800Sstevel@tonic-gate * names of the form "LIBTHREAD_*".
11810Sstevel@tonic-gate */
11820Sstevel@tonic-gate static void
set_thread_vars()11830Sstevel@tonic-gate set_thread_vars()
11840Sstevel@tonic-gate {
11856879Sraf extern const char **_environ;
11860Sstevel@tonic-gate const char **pev;
11870Sstevel@tonic-gate const char *ev;
11880Sstevel@tonic-gate char c;
11890Sstevel@tonic-gate
11906879Sraf if ((pev = _environ) == NULL)
11910Sstevel@tonic-gate return;
11920Sstevel@tonic-gate while ((ev = *pev++) != NULL) {
11930Sstevel@tonic-gate c = *ev;
11946812Sraf if (c == '_' && strncmp(ev, "_THREAD_", 8) == 0)
11950Sstevel@tonic-gate etest(ev + 8);
11966812Sraf if (c == 'L' && strncmp(ev, "LIBTHREAD_", 10) == 0)
11970Sstevel@tonic-gate etest(ev + 10);
11980Sstevel@tonic-gate }
11990Sstevel@tonic-gate }
12000Sstevel@tonic-gate
12010Sstevel@tonic-gate /* PROBE_SUPPORT begin */
12020Sstevel@tonic-gate #pragma weak __tnf_probe_notify
12030Sstevel@tonic-gate extern void __tnf_probe_notify(void);
12040Sstevel@tonic-gate /* PROBE_SUPPORT end */
12050Sstevel@tonic-gate
12060Sstevel@tonic-gate /* same as atexit() but private to the library */
12070Sstevel@tonic-gate extern int _atexit(void (*)(void));
12080Sstevel@tonic-gate
12090Sstevel@tonic-gate /* same as _cleanup() but private to the library */
12100Sstevel@tonic-gate extern void __cleanup(void);
12110Sstevel@tonic-gate
12120Sstevel@tonic-gate extern void atfork_init(void);
12130Sstevel@tonic-gate
12140Sstevel@tonic-gate #ifdef __amd64
12156320Sbholler extern void __proc64id(void);
12160Sstevel@tonic-gate #endif
12170Sstevel@tonic-gate
12180Sstevel@tonic-gate /*
12190Sstevel@tonic-gate * libc_init() is called by ld.so.1 for library initialization.
12200Sstevel@tonic-gate * We perform minimal initialization; enough to work with the main thread.
12210Sstevel@tonic-gate */
12220Sstevel@tonic-gate void
libc_init(void)12230Sstevel@tonic-gate libc_init(void)
12240Sstevel@tonic-gate {
12250Sstevel@tonic-gate uberdata_t *udp = &__uberdata;
12260Sstevel@tonic-gate ulwp_t *oldself = __curthread();
12270Sstevel@tonic-gate ucontext_t uc;
12280Sstevel@tonic-gate ulwp_t *self;
12290Sstevel@tonic-gate struct rlimit rl;
12300Sstevel@tonic-gate caddr_t data;
12310Sstevel@tonic-gate size_t tls_size;
12320Sstevel@tonic-gate int setmask;
12330Sstevel@tonic-gate
12340Sstevel@tonic-gate /*
12350Sstevel@tonic-gate * For the initial stage of initialization, we must be careful
12360Sstevel@tonic-gate * not to call any function that could possibly call _cerror().
12370Sstevel@tonic-gate * For this purpose, we call only the raw system call wrappers.
12380Sstevel@tonic-gate */
12390Sstevel@tonic-gate
12400Sstevel@tonic-gate #ifdef __amd64
12410Sstevel@tonic-gate /*
12420Sstevel@tonic-gate * Gather information about cache layouts for optimized
12436320Sbholler * AMD and Intel assembler strfoo() and memfoo() functions.
12440Sstevel@tonic-gate */
12456320Sbholler __proc64id();
12460Sstevel@tonic-gate #endif
12470Sstevel@tonic-gate
12480Sstevel@tonic-gate /*
12490Sstevel@tonic-gate * Every libc, regardless of which link map, must register __cleanup().
12500Sstevel@tonic-gate */
12510Sstevel@tonic-gate (void) _atexit(__cleanup);
12520Sstevel@tonic-gate
12530Sstevel@tonic-gate /*
12540Sstevel@tonic-gate * We keep our uberdata on one of (a) the first alternate link map
12550Sstevel@tonic-gate * or (b) the primary link map. We switch to the primary link map
12560Sstevel@tonic-gate * and stay there once we see it. All intermediate link maps are
12570Sstevel@tonic-gate * subject to being unloaded at any time.
12580Sstevel@tonic-gate */
12590Sstevel@tonic-gate if (oldself != NULL && (oldself->ul_primarymap || !primary_link_map)) {
12600Sstevel@tonic-gate __tdb_bootstrap = oldself->ul_uberdata->tdb_bootstrap;
12610Sstevel@tonic-gate mutex_setup();
12620Sstevel@tonic-gate atfork_init(); /* every link map needs atfork() processing */
1263*13093SRoger.Faulkner@Oracle.COM init_progname();
12640Sstevel@tonic-gate return;
12650Sstevel@tonic-gate }
12660Sstevel@tonic-gate
12670Sstevel@tonic-gate /*
12680Sstevel@tonic-gate * To establish the main stack information, we have to get our context.
12690Sstevel@tonic-gate * This is also convenient to use for getting our signal mask.
12700Sstevel@tonic-gate */
12710Sstevel@tonic-gate uc.uc_flags = UC_ALL;
12726515Sraf (void) __getcontext(&uc);
12730Sstevel@tonic-gate ASSERT(uc.uc_link == NULL);
12740Sstevel@tonic-gate
12750Sstevel@tonic-gate tls_size = roundup64(udp->tls_metadata.static_tls.tls_size);
12760Sstevel@tonic-gate ASSERT(primary_link_map || tls_size == 0);
12770Sstevel@tonic-gate data = lmalloc(sizeof (ulwp_t) + tls_size);
12780Sstevel@tonic-gate if (data == NULL)
12790Sstevel@tonic-gate thr_panic("cannot allocate thread structure for main thread");
12800Sstevel@tonic-gate /* LINTED pointer cast may result in improper alignment */
12810Sstevel@tonic-gate self = (ulwp_t *)(data + tls_size);
12820Sstevel@tonic-gate init_hash_table[0].hash_bucket = self;
12830Sstevel@tonic-gate
12840Sstevel@tonic-gate self->ul_sigmask = uc.uc_sigmask;
12850Sstevel@tonic-gate delete_reserved_signals(&self->ul_sigmask);
12860Sstevel@tonic-gate /*
12870Sstevel@tonic-gate * Are the old and new sets different?
12880Sstevel@tonic-gate * (This can happen if we are currently blocking SIGCANCEL.)
12890Sstevel@tonic-gate * If so, we must explicitly set our signal mask, below.
12900Sstevel@tonic-gate */
12910Sstevel@tonic-gate setmask =
12920Sstevel@tonic-gate ((self->ul_sigmask.__sigbits[0] ^ uc.uc_sigmask.__sigbits[0]) |
129311913SRoger.Faulkner@Sun.COM (self->ul_sigmask.__sigbits[1] ^ uc.uc_sigmask.__sigbits[1]) |
129411913SRoger.Faulkner@Sun.COM (self->ul_sigmask.__sigbits[2] ^ uc.uc_sigmask.__sigbits[2]) |
129511913SRoger.Faulkner@Sun.COM (self->ul_sigmask.__sigbits[3] ^ uc.uc_sigmask.__sigbits[3]));
12960Sstevel@tonic-gate
12970Sstevel@tonic-gate #ifdef __sparc
12980Sstevel@tonic-gate /*
12990Sstevel@tonic-gate * We cache several instructions in the thread structure for use
13000Sstevel@tonic-gate * by the fasttrap DTrace provider. When changing this, read the
13010Sstevel@tonic-gate * comment in fasttrap.h for the all the other places that must
13020Sstevel@tonic-gate * be changed.
13030Sstevel@tonic-gate */
13040Sstevel@tonic-gate self->ul_dsave = 0x9de04000; /* save %g1, %g0, %sp */
13050Sstevel@tonic-gate self->ul_drestore = 0x81e80000; /* restore %g0, %g0, %g0 */
13060Sstevel@tonic-gate self->ul_dftret = 0x91d0203a; /* ta 0x3a */
13070Sstevel@tonic-gate self->ul_dreturn = 0x81ca0000; /* return %o0 */
13080Sstevel@tonic-gate #endif
13090Sstevel@tonic-gate
13105891Sraf self->ul_stktop = (uintptr_t)uc.uc_stack.ss_sp + uc.uc_stack.ss_size;
13116515Sraf (void) getrlimit(RLIMIT_STACK, &rl);
13120Sstevel@tonic-gate self->ul_stksiz = rl.rlim_cur;
13130Sstevel@tonic-gate self->ul_stk = (caddr_t)(self->ul_stktop - self->ul_stksiz);
13140Sstevel@tonic-gate
13150Sstevel@tonic-gate self->ul_forw = self->ul_back = self;
13160Sstevel@tonic-gate self->ul_hash = NULL;
13170Sstevel@tonic-gate self->ul_ix = 0;
13186812Sraf self->ul_lwpid = 1; /* _lwp_self() */
13190Sstevel@tonic-gate self->ul_main = 1;
13200Sstevel@tonic-gate self->ul_self = self;
13216247Sraf self->ul_policy = -1; /* initialize only when needed */
13226247Sraf self->ul_pri = 0;
13236247Sraf self->ul_cid = 0;
13246296Sraf self->ul_rtclassid = -1;
13250Sstevel@tonic-gate self->ul_uberdata = udp;
13260Sstevel@tonic-gate if (oldself != NULL) {
13270Sstevel@tonic-gate int i;
13280Sstevel@tonic-gate
13290Sstevel@tonic-gate ASSERT(primary_link_map);
13300Sstevel@tonic-gate ASSERT(oldself->ul_main == 1);
13310Sstevel@tonic-gate self->ul_stsd = oldself->ul_stsd;
13320Sstevel@tonic-gate for (i = 0; i < TSD_NFAST; i++)
13330Sstevel@tonic-gate self->ul_ftsd[i] = oldself->ul_ftsd[i];
13340Sstevel@tonic-gate self->ul_tls = oldself->ul_tls;
13350Sstevel@tonic-gate /*
13360Sstevel@tonic-gate * Retrieve all pointers to uberdata allocated
13370Sstevel@tonic-gate * while running on previous link maps.
13381111Sraf * We would like to do a structure assignment here, but
13391111Sraf * gcc turns structure assignments into calls to memcpy(),
13401111Sraf * a function exported from libc. We can't call any such
13411111Sraf * external functions until we establish curthread, below,
13421111Sraf * so we just call our private version of memcpy().
13430Sstevel@tonic-gate */
13446515Sraf (void) memcpy(udp, oldself->ul_uberdata, sizeof (*udp));
13450Sstevel@tonic-gate /*
13460Sstevel@tonic-gate * These items point to global data on the primary link map.
13470Sstevel@tonic-gate */
13480Sstevel@tonic-gate udp->thr_hash_table = init_hash_table;
13490Sstevel@tonic-gate udp->sigacthandler = sigacthandler;
13500Sstevel@tonic-gate udp->tdb.tdb_events = tdb_events;
13510Sstevel@tonic-gate ASSERT(udp->nthreads == 1 && !udp->uberflags.uf_mt);
13520Sstevel@tonic-gate ASSERT(udp->lwp_stacks == NULL);
13530Sstevel@tonic-gate ASSERT(udp->ulwp_freelist == NULL);
13540Sstevel@tonic-gate ASSERT(udp->ulwp_replace_free == NULL);
13550Sstevel@tonic-gate ASSERT(udp->hash_size == 1);
13560Sstevel@tonic-gate }
13570Sstevel@tonic-gate udp->all_lwps = self;
13580Sstevel@tonic-gate udp->ulwp_one = self;
13596515Sraf udp->pid = getpid();
13600Sstevel@tonic-gate udp->nthreads = 1;
13610Sstevel@tonic-gate /*
13620Sstevel@tonic-gate * In every link map, tdb_bootstrap points to the same piece of
13630Sstevel@tonic-gate * allocated memory. When the primary link map is initialized,
13640Sstevel@tonic-gate * the allocated memory is assigned a pointer to the one true
13650Sstevel@tonic-gate * uberdata. This allows libc_db to initialize itself regardless
13660Sstevel@tonic-gate * of which instance of libc it finds in the address space.
13670Sstevel@tonic-gate */
13680Sstevel@tonic-gate if (udp->tdb_bootstrap == NULL)
13690Sstevel@tonic-gate udp->tdb_bootstrap = lmalloc(sizeof (uberdata_t *));
13700Sstevel@tonic-gate __tdb_bootstrap = udp->tdb_bootstrap;
13710Sstevel@tonic-gate if (primary_link_map) {
13720Sstevel@tonic-gate self->ul_primarymap = 1;
13730Sstevel@tonic-gate udp->primary_map = 1;
13740Sstevel@tonic-gate *udp->tdb_bootstrap = udp;
13750Sstevel@tonic-gate }
13760Sstevel@tonic-gate /*
13770Sstevel@tonic-gate * Cancellation can't happen until:
13780Sstevel@tonic-gate * pthread_cancel() is called
13790Sstevel@tonic-gate * or:
13800Sstevel@tonic-gate * another thread is created
13810Sstevel@tonic-gate * For now, as a single-threaded process, set the flag that tells
13820Sstevel@tonic-gate * PROLOGUE/EPILOGUE (in scalls.c) that cancellation can't happen.
13830Sstevel@tonic-gate */
13840Sstevel@tonic-gate self->ul_nocancel = 1;
13850Sstevel@tonic-gate
13860Sstevel@tonic-gate #if defined(__amd64)
13873446Smrj (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_FSBASE, self);
13880Sstevel@tonic-gate #elif defined(__i386)
13893446Smrj (void) ___lwp_private(_LWP_SETPRIVATE, _LWP_GSBASE, self);
13900Sstevel@tonic-gate #endif /* __i386 || __amd64 */
13910Sstevel@tonic-gate set_curthread(self); /* redundant on i386 */
13920Sstevel@tonic-gate /*
13930Sstevel@tonic-gate * Now curthread is established and it is safe to call any
13940Sstevel@tonic-gate * function in libc except one that uses thread-local storage.
13950Sstevel@tonic-gate */
13960Sstevel@tonic-gate self->ul_errnop = &errno;
13970Sstevel@tonic-gate if (oldself != NULL) {
13980Sstevel@tonic-gate /* tls_size was zero when oldself was allocated */
13990Sstevel@tonic-gate lfree(oldself, sizeof (ulwp_t));
14000Sstevel@tonic-gate }
14010Sstevel@tonic-gate mutex_setup();
14020Sstevel@tonic-gate atfork_init();
14030Sstevel@tonic-gate signal_init();
14040Sstevel@tonic-gate
14050Sstevel@tonic-gate /*
14060Sstevel@tonic-gate * If the stack is unlimited, we set the size to zero to disable
14070Sstevel@tonic-gate * stack checking.
14080Sstevel@tonic-gate * XXX: Work harder here. Get the stack size from /proc/self/rmap
14090Sstevel@tonic-gate */
14100Sstevel@tonic-gate if (self->ul_stksiz == RLIM_INFINITY) {
14110Sstevel@tonic-gate self->ul_ustack.ss_sp = (void *)self->ul_stktop;
14120Sstevel@tonic-gate self->ul_ustack.ss_size = 0;
14130Sstevel@tonic-gate } else {
14140Sstevel@tonic-gate self->ul_ustack.ss_sp = self->ul_stk;
14150Sstevel@tonic-gate self->ul_ustack.ss_size = self->ul_stksiz;
14160Sstevel@tonic-gate }
14170Sstevel@tonic-gate self->ul_ustack.ss_flags = 0;
14186515Sraf (void) setustack(&self->ul_ustack);
14190Sstevel@tonic-gate
14200Sstevel@tonic-gate /*
14210Sstevel@tonic-gate * Get the variables that affect thread behavior from the environment.
14220Sstevel@tonic-gate */
14230Sstevel@tonic-gate set_thread_vars();
14240Sstevel@tonic-gate udp->uberflags.uf_thread_error_detection = (char)thread_error_detection;
14250Sstevel@tonic-gate udp->thread_stack_cache = thread_stack_cache;
14260Sstevel@tonic-gate
14270Sstevel@tonic-gate /*
14280Sstevel@tonic-gate * Make per-thread copies of global variables, for speed.
14290Sstevel@tonic-gate */
14300Sstevel@tonic-gate self->ul_queue_fifo = (char)thread_queue_fifo;
14310Sstevel@tonic-gate self->ul_cond_wait_defer = (char)thread_cond_wait_defer;
14320Sstevel@tonic-gate self->ul_error_detection = (char)thread_error_detection;
14330Sstevel@tonic-gate self->ul_async_safe = (char)thread_async_safe;
14340Sstevel@tonic-gate self->ul_door_noreserve = (char)thread_door_noreserve;
14357255Sraf self->ul_misaligned = (char)thread_locks_misaligned;
14365629Sraf self->ul_max_spinners = (uint8_t)thread_max_spinners;
14370Sstevel@tonic-gate self->ul_adaptive_spin = thread_adaptive_spin;
14380Sstevel@tonic-gate self->ul_queue_spin = thread_queue_spin;
14390Sstevel@tonic-gate
144010629SRoger.Faulkner@Sun.COM #if defined(__sparc) && !defined(_LP64)
144110629SRoger.Faulkner@Sun.COM if (self->ul_misaligned) {
144210629SRoger.Faulkner@Sun.COM /*
144310629SRoger.Faulkner@Sun.COM * Tell the kernel to fix up ldx/stx instructions that
144410629SRoger.Faulkner@Sun.COM * refer to non-8-byte aligned data instead of giving
144510629SRoger.Faulkner@Sun.COM * the process an alignment trap and generating SIGBUS.
144610629SRoger.Faulkner@Sun.COM *
144710629SRoger.Faulkner@Sun.COM * Programs compiled for 32-bit sparc with the Studio SS12
144810629SRoger.Faulkner@Sun.COM * compiler get this done for them automatically (in _init()).
144910629SRoger.Faulkner@Sun.COM * We do it here for the benefit of programs compiled with
145010629SRoger.Faulkner@Sun.COM * other compilers, like gcc.
145110629SRoger.Faulkner@Sun.COM *
145210629SRoger.Faulkner@Sun.COM * This is necessary for the _THREAD_LOCKS_MISALIGNED=1
145310629SRoger.Faulkner@Sun.COM * environment variable horrible hack to work.
145410629SRoger.Faulkner@Sun.COM */
145510629SRoger.Faulkner@Sun.COM extern void _do_fix_align(void);
145610629SRoger.Faulkner@Sun.COM _do_fix_align();
145710629SRoger.Faulkner@Sun.COM }
145810629SRoger.Faulkner@Sun.COM #endif
145910629SRoger.Faulkner@Sun.COM
14600Sstevel@tonic-gate /*
14610Sstevel@tonic-gate * When we have initialized the primary link map, inform
14620Sstevel@tonic-gate * the dynamic linker about our interface functions.
1463*13093SRoger.Faulkner@Oracle.COM * Set up our pointer to the program name.
14640Sstevel@tonic-gate */
14650Sstevel@tonic-gate if (self->ul_primarymap)
14660Sstevel@tonic-gate _ld_libc((void *)rtld_funcs);
1467*13093SRoger.Faulkner@Oracle.COM init_progname();
14680Sstevel@tonic-gate
14690Sstevel@tonic-gate /*
14700Sstevel@tonic-gate * Defer signals until TLS constructors have been called.
14710Sstevel@tonic-gate */
14720Sstevel@tonic-gate sigoff(self);
14730Sstevel@tonic-gate tls_setup();
14740Sstevel@tonic-gate sigon(self);
14750Sstevel@tonic-gate if (setmask)
14760Sstevel@tonic-gate (void) restore_signals(self);
14770Sstevel@tonic-gate
14785891Sraf /*
14795891Sraf * Make private copies of __xpg4 and __xpg6 so libc can test
14805891Sraf * them after this point without invoking the dynamic linker.
14815891Sraf */
14825891Sraf libc__xpg4 = __xpg4;
14835891Sraf libc__xpg6 = __xpg6;
14845891Sraf
14850Sstevel@tonic-gate /* PROBE_SUPPORT begin */
14860Sstevel@tonic-gate if (self->ul_primarymap && __tnf_probe_notify != NULL)
14870Sstevel@tonic-gate __tnf_probe_notify();
14880Sstevel@tonic-gate /* PROBE_SUPPORT end */
14892248Sraf
14902248Sraf init_sigev_thread();
14912248Sraf init_aio();
14922692Snakanon
14932692Snakanon /*
14942692Snakanon * We need to reset __threaded dynamically at runtime, so that
14952692Snakanon * __threaded can be bound to __threaded outside libc which may not
14962692Snakanon * have initial value of 1 (without a copy relocation in a.out).
14972692Snakanon */
14982692Snakanon __threaded = 1;
14990Sstevel@tonic-gate }
15000Sstevel@tonic-gate
15010Sstevel@tonic-gate #pragma fini(libc_fini)
15020Sstevel@tonic-gate void
libc_fini()15030Sstevel@tonic-gate libc_fini()
15040Sstevel@tonic-gate {
15050Sstevel@tonic-gate /*
15060Sstevel@tonic-gate * If we are doing fini processing for the instance of libc
15070Sstevel@tonic-gate * on the first alternate link map (this happens only when
15080Sstevel@tonic-gate * the dynamic linker rejects a bad audit library), then clear
15090Sstevel@tonic-gate * __curthread(). We abandon whatever memory was allocated by
15100Sstevel@tonic-gate * lmalloc() while running on this alternate link-map but we
15110Sstevel@tonic-gate * don't care (and can't find the memory in any case); we just
15120Sstevel@tonic-gate * want to protect the application from this bad audit library.
15130Sstevel@tonic-gate * No fini processing is done by libc in the normal case.
15140Sstevel@tonic-gate */
15150Sstevel@tonic-gate
15160Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
15170Sstevel@tonic-gate
15180Sstevel@tonic-gate if (udp->primary_map == 0 && udp == &__uberdata)
15190Sstevel@tonic-gate set_curthread(NULL);
15200Sstevel@tonic-gate }
15210Sstevel@tonic-gate
15220Sstevel@tonic-gate /*
15230Sstevel@tonic-gate * finish_init is called when we are about to become multi-threaded,
15240Sstevel@tonic-gate * that is, on the first call to thr_create().
15250Sstevel@tonic-gate */
15260Sstevel@tonic-gate void
finish_init()15270Sstevel@tonic-gate finish_init()
15280Sstevel@tonic-gate {
15290Sstevel@tonic-gate ulwp_t *self = curthread;
15300Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
15310Sstevel@tonic-gate thr_hash_table_t *htp;
15320Sstevel@tonic-gate void *data;
15330Sstevel@tonic-gate int i;
15340Sstevel@tonic-gate
15350Sstevel@tonic-gate /*
15360Sstevel@tonic-gate * No locks needed here; we are single-threaded on the first call.
15370Sstevel@tonic-gate * We can be called only after the primary link map has been set up.
15380Sstevel@tonic-gate */
15390Sstevel@tonic-gate ASSERT(self->ul_primarymap);
15400Sstevel@tonic-gate ASSERT(self == udp->ulwp_one);
15410Sstevel@tonic-gate ASSERT(!udp->uberflags.uf_mt);
15420Sstevel@tonic-gate ASSERT(udp->hash_size == 1);
15430Sstevel@tonic-gate
15440Sstevel@tonic-gate /*
15456247Sraf * Initialize self->ul_policy, self->ul_cid, and self->ul_pri.
15466247Sraf */
15476247Sraf update_sched(self);
15486247Sraf
15496247Sraf /*
15506247Sraf * Allocate the queue_head array if not already allocated.
15510Sstevel@tonic-gate */
15520Sstevel@tonic-gate if (udp->queue_head == NULL)
15530Sstevel@tonic-gate queue_alloc();
15540Sstevel@tonic-gate
15550Sstevel@tonic-gate /*
15560Sstevel@tonic-gate * Now allocate the thread hash table.
15570Sstevel@tonic-gate */
15586515Sraf if ((data = mmap(NULL, HASHTBLSZ * sizeof (thr_hash_table_t),
15590Sstevel@tonic-gate PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0))
15600Sstevel@tonic-gate == MAP_FAILED)
15610Sstevel@tonic-gate thr_panic("cannot allocate thread hash table");
15620Sstevel@tonic-gate
15630Sstevel@tonic-gate udp->thr_hash_table = htp = (thr_hash_table_t *)data;
15640Sstevel@tonic-gate udp->hash_size = HASHTBLSZ;
15650Sstevel@tonic-gate udp->hash_mask = HASHTBLSZ - 1;
15660Sstevel@tonic-gate
15670Sstevel@tonic-gate for (i = 0; i < HASHTBLSZ; i++, htp++) {
15684574Sraf htp->hash_lock.mutex_flag = LOCK_INITED;
15690Sstevel@tonic-gate htp->hash_lock.mutex_magic = MUTEX_MAGIC;
15700Sstevel@tonic-gate htp->hash_cond.cond_magic = COND_MAGIC;
15710Sstevel@tonic-gate }
15720Sstevel@tonic-gate hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);
15730Sstevel@tonic-gate
15740Sstevel@tonic-gate /*
15750Sstevel@tonic-gate * Set up the SIGCANCEL handler for threads cancellation.
15760Sstevel@tonic-gate */
15772248Sraf setup_cancelsig(SIGCANCEL);
15780Sstevel@tonic-gate
15790Sstevel@tonic-gate /*
15800Sstevel@tonic-gate * Arrange to do special things on exit --
15810Sstevel@tonic-gate * - collect queue statistics from all remaining active threads.
15826247Sraf * - dump queue statistics to stderr if _THREAD_QUEUE_DUMP is set.
15830Sstevel@tonic-gate * - grab assert_lock to ensure that assertion failures
15840Sstevel@tonic-gate * and a core dump take precedence over _exit().
15850Sstevel@tonic-gate * (Functions are called in the reverse order of their registration.)
15860Sstevel@tonic-gate */
15876247Sraf (void) _atexit(grab_assert_lock);
15886247Sraf #if defined(THREAD_DEBUG)
15890Sstevel@tonic-gate (void) _atexit(dump_queue_statistics);
15900Sstevel@tonic-gate (void) _atexit(collect_queue_statistics);
15916247Sraf #endif
15920Sstevel@tonic-gate }
15930Sstevel@tonic-gate
15940Sstevel@tonic-gate /*
15953235Sraf * Used only by postfork1_child(), below.
15960Sstevel@tonic-gate */
15970Sstevel@tonic-gate static void
mark_dead_and_buried(ulwp_t * ulwp)15980Sstevel@tonic-gate mark_dead_and_buried(ulwp_t *ulwp)
15990Sstevel@tonic-gate {
16000Sstevel@tonic-gate ulwp->ul_dead = 1;
16010Sstevel@tonic-gate ulwp->ul_lwpid = (lwpid_t)(-1);
16020Sstevel@tonic-gate ulwp->ul_hash = NULL;
16030Sstevel@tonic-gate ulwp->ul_ix = -1;
16040Sstevel@tonic-gate ulwp->ul_schedctl = NULL;
16050Sstevel@tonic-gate ulwp->ul_schedctl_called = NULL;
16060Sstevel@tonic-gate }
16070Sstevel@tonic-gate
16080Sstevel@tonic-gate /*
16090Sstevel@tonic-gate * This is called from fork1() in the child.
16100Sstevel@tonic-gate * Reset our data structures to reflect one lwp.
16110Sstevel@tonic-gate */
16120Sstevel@tonic-gate void
postfork1_child()16132248Sraf postfork1_child()
16140Sstevel@tonic-gate {
16150Sstevel@tonic-gate ulwp_t *self = curthread;
16160Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
16176247Sraf queue_head_t *qp;
16180Sstevel@tonic-gate ulwp_t *next;
16190Sstevel@tonic-gate ulwp_t *ulwp;
16200Sstevel@tonic-gate int i;
16210Sstevel@tonic-gate
16220Sstevel@tonic-gate /* daemon threads shouldn't call fork1(), but oh well... */
16230Sstevel@tonic-gate self->ul_usropts &= ~THR_DAEMON;
16240Sstevel@tonic-gate udp->nthreads = 1;
16250Sstevel@tonic-gate udp->ndaemons = 0;
16260Sstevel@tonic-gate udp->uberflags.uf_mt = 0;
16271672Sraf __libc_threaded = 0;
16280Sstevel@tonic-gate for (i = 0; i < udp->hash_size; i++)
16290Sstevel@tonic-gate udp->thr_hash_table[i].hash_bucket = NULL;
16306812Sraf self->ul_lwpid = _lwp_self();
16310Sstevel@tonic-gate hash_in_unlocked(self, TIDHASH(self->ul_lwpid, udp), udp);
16320Sstevel@tonic-gate
16335002Sraf /*
16346515Sraf * Some thread in the parent might have been suspended
16356515Sraf * while holding udp->callout_lock or udp->ld_lock.
16366515Sraf * Reinitialize the child's copies.
16375002Sraf */
16386812Sraf (void) mutex_init(&udp->callout_lock,
16396812Sraf USYNC_THREAD | LOCK_RECURSIVE, NULL);
16406812Sraf (void) mutex_init(&udp->ld_lock,
16416812Sraf USYNC_THREAD | LOCK_RECURSIVE, NULL);
16425002Sraf
16430Sstevel@tonic-gate /* no one in the child is on a sleep queue; reinitialize */
16446247Sraf if ((qp = udp->queue_head) != NULL) {
16456515Sraf (void) memset(qp, 0, 2 * QHASHSIZE * sizeof (queue_head_t));
16466247Sraf for (i = 0; i < 2 * QHASHSIZE; qp++, i++) {
16476247Sraf qp->qh_type = (i < QHASHSIZE)? MX : CV;
16486247Sraf qp->qh_lock.mutex_flag = LOCK_INITED;
16496247Sraf qp->qh_lock.mutex_magic = MUTEX_MAGIC;
16506247Sraf qp->qh_hlist = &qp->qh_def_root;
16516247Sraf #if defined(THREAD_DEBUG)
16526247Sraf qp->qh_hlen = 1;
16536247Sraf qp->qh_hmax = 1;
16546247Sraf #endif
16554574Sraf }
16560Sstevel@tonic-gate }
16570Sstevel@tonic-gate
16580Sstevel@tonic-gate /*
165912795SRoger.Faulkner@Oracle.COM * Do post-fork1 processing for subsystems that need it.
166012795SRoger.Faulkner@Oracle.COM * We need to do this before unmapping all of the abandoned
166112795SRoger.Faulkner@Oracle.COM * threads' stacks, below(), because the post-fork1 actions
166212795SRoger.Faulkner@Oracle.COM * might require access to those stacks.
166312795SRoger.Faulkner@Oracle.COM */
166412795SRoger.Faulkner@Oracle.COM postfork1_child_sigev_aio();
166512795SRoger.Faulkner@Oracle.COM postfork1_child_sigev_mq();
166612795SRoger.Faulkner@Oracle.COM postfork1_child_sigev_timer();
166712795SRoger.Faulkner@Oracle.COM postfork1_child_aio();
166812795SRoger.Faulkner@Oracle.COM /*
166912795SRoger.Faulkner@Oracle.COM * The above subsystems use thread pools, so this action
167012795SRoger.Faulkner@Oracle.COM * must be performed after those actions.
167112795SRoger.Faulkner@Oracle.COM */
167212795SRoger.Faulkner@Oracle.COM postfork1_child_tpool();
167312795SRoger.Faulkner@Oracle.COM
167412795SRoger.Faulkner@Oracle.COM /*
16750Sstevel@tonic-gate * All lwps except ourself are gone. Mark them so.
16760Sstevel@tonic-gate * First mark all of the lwps that have already been freed.
16770Sstevel@tonic-gate * Then mark and free all of the active lwps except ourself.
16780Sstevel@tonic-gate * Since we are single-threaded, no locks are required here.
16790Sstevel@tonic-gate */
16800Sstevel@tonic-gate for (ulwp = udp->lwp_stacks; ulwp != NULL; ulwp = ulwp->ul_next)
16810Sstevel@tonic-gate mark_dead_and_buried(ulwp);
16820Sstevel@tonic-gate for (ulwp = udp->ulwp_freelist; ulwp != NULL; ulwp = ulwp->ul_next)
16830Sstevel@tonic-gate mark_dead_and_buried(ulwp);
16840Sstevel@tonic-gate for (ulwp = self->ul_forw; ulwp != self; ulwp = next) {
16850Sstevel@tonic-gate next = ulwp->ul_forw;
16860Sstevel@tonic-gate ulwp->ul_forw = ulwp->ul_back = NULL;
16870Sstevel@tonic-gate mark_dead_and_buried(ulwp);
16880Sstevel@tonic-gate tsd_free(ulwp);
16890Sstevel@tonic-gate tls_free(ulwp);
16900Sstevel@tonic-gate rwl_free(ulwp);
16914574Sraf heldlock_free(ulwp);
16920Sstevel@tonic-gate ulwp_free(ulwp);
16930Sstevel@tonic-gate }
16940Sstevel@tonic-gate self->ul_forw = self->ul_back = udp->all_lwps = self;
16950Sstevel@tonic-gate if (self != udp->ulwp_one)
16960Sstevel@tonic-gate mark_dead_and_buried(udp->ulwp_one);
16970Sstevel@tonic-gate if ((ulwp = udp->all_zombies) != NULL) {
16980Sstevel@tonic-gate ASSERT(udp->nzombies != 0);
16990Sstevel@tonic-gate do {
17000Sstevel@tonic-gate next = ulwp->ul_forw;
17010Sstevel@tonic-gate ulwp->ul_forw = ulwp->ul_back = NULL;
17020Sstevel@tonic-gate mark_dead_and_buried(ulwp);
17030Sstevel@tonic-gate udp->nzombies--;
17040Sstevel@tonic-gate if (ulwp->ul_replace) {
17050Sstevel@tonic-gate ulwp->ul_next = NULL;
17060Sstevel@tonic-gate if (udp->ulwp_replace_free == NULL) {
17070Sstevel@tonic-gate udp->ulwp_replace_free =
17084843Sraf udp->ulwp_replace_last = ulwp;
17090Sstevel@tonic-gate } else {
17100Sstevel@tonic-gate udp->ulwp_replace_last->ul_next = ulwp;
17110Sstevel@tonic-gate udp->ulwp_replace_last = ulwp;
17120Sstevel@tonic-gate }
17130Sstevel@tonic-gate }
17140Sstevel@tonic-gate } while ((ulwp = next) != udp->all_zombies);
17150Sstevel@tonic-gate ASSERT(udp->nzombies == 0);
17160Sstevel@tonic-gate udp->all_zombies = NULL;
17170Sstevel@tonic-gate udp->nzombies = 0;
17180Sstevel@tonic-gate }
17190Sstevel@tonic-gate trim_stack_cache(0);
17200Sstevel@tonic-gate }
17210Sstevel@tonic-gate
17220Sstevel@tonic-gate lwpid_t
lwp_self(void)17230Sstevel@tonic-gate lwp_self(void)
17240Sstevel@tonic-gate {
17250Sstevel@tonic-gate return (curthread->ul_lwpid);
17260Sstevel@tonic-gate }
17270Sstevel@tonic-gate
17286812Sraf #pragma weak _ti_thr_self = thr_self
17296812Sraf #pragma weak pthread_self = thr_self
17300Sstevel@tonic-gate thread_t
thr_self()17316812Sraf thr_self()
17320Sstevel@tonic-gate {
17330Sstevel@tonic-gate return (curthread->ul_lwpid);
17340Sstevel@tonic-gate }
17350Sstevel@tonic-gate
17360Sstevel@tonic-gate int
thr_main()17376812Sraf thr_main()
17380Sstevel@tonic-gate {
17390Sstevel@tonic-gate ulwp_t *self = __curthread();
17400Sstevel@tonic-gate
17410Sstevel@tonic-gate return ((self == NULL)? -1 : self->ul_main);
17420Sstevel@tonic-gate }
17430Sstevel@tonic-gate
17440Sstevel@tonic-gate int
_thrp_cancelled(void)17453235Sraf _thrp_cancelled(void)
17463235Sraf {
17473235Sraf return (curthread->ul_rval == PTHREAD_CANCELED);
17483235Sraf }
17493235Sraf
17503235Sraf int
_thrp_stksegment(ulwp_t * ulwp,stack_t * stk)17510Sstevel@tonic-gate _thrp_stksegment(ulwp_t *ulwp, stack_t *stk)
17520Sstevel@tonic-gate {
17530Sstevel@tonic-gate stk->ss_sp = (void *)ulwp->ul_stktop;
17540Sstevel@tonic-gate stk->ss_size = ulwp->ul_stksiz;
17550Sstevel@tonic-gate stk->ss_flags = 0;
17560Sstevel@tonic-gate return (0);
17570Sstevel@tonic-gate }
17580Sstevel@tonic-gate
17596812Sraf #pragma weak _thr_stksegment = thr_stksegment
17600Sstevel@tonic-gate int
thr_stksegment(stack_t * stk)17616812Sraf thr_stksegment(stack_t *stk)
17620Sstevel@tonic-gate {
17630Sstevel@tonic-gate return (_thrp_stksegment(curthread, stk));
17640Sstevel@tonic-gate }
17650Sstevel@tonic-gate
17660Sstevel@tonic-gate void
force_continue(ulwp_t * ulwp)17670Sstevel@tonic-gate force_continue(ulwp_t *ulwp)
17680Sstevel@tonic-gate {
17690Sstevel@tonic-gate #if defined(THREAD_DEBUG)
17700Sstevel@tonic-gate ulwp_t *self = curthread;
17710Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
17720Sstevel@tonic-gate #endif
17730Sstevel@tonic-gate int error;
17740Sstevel@tonic-gate timespec_t ts;
17750Sstevel@tonic-gate
17763512Sraf ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
17770Sstevel@tonic-gate ASSERT(MUTEX_OWNED(ulwp_mutex(ulwp, udp), self));
17780Sstevel@tonic-gate
17790Sstevel@tonic-gate for (;;) {
17806812Sraf error = _lwp_continue(ulwp->ul_lwpid);
17810Sstevel@tonic-gate if (error != 0 && error != EINTR)
17820Sstevel@tonic-gate break;
17830Sstevel@tonic-gate error = 0;
17840Sstevel@tonic-gate if (ulwp->ul_stopping) { /* he is stopping himself */
17850Sstevel@tonic-gate ts.tv_sec = 0; /* give him a chance to run */
17860Sstevel@tonic-gate ts.tv_nsec = 100000; /* 100 usecs or clock tick */
17872248Sraf (void) __nanosleep(&ts, NULL);
17880Sstevel@tonic-gate }
17890Sstevel@tonic-gate if (!ulwp->ul_stopping) /* he is running now */
17900Sstevel@tonic-gate break; /* so we are done */
17910Sstevel@tonic-gate /*
17920Sstevel@tonic-gate * He is marked as being in the process of stopping
17930Sstevel@tonic-gate * himself. Loop around and continue him again.
17940Sstevel@tonic-gate * He may not have been stopped the first time.
17950Sstevel@tonic-gate */
17960Sstevel@tonic-gate }
17970Sstevel@tonic-gate }
17980Sstevel@tonic-gate
17990Sstevel@tonic-gate /*
180010637SRoger.Faulkner@Sun.COM * Suspend an lwp with lwp_suspend(), then move it to a safe point,
180110637SRoger.Faulkner@Sun.COM * that is, to a point where ul_critical and ul_rtld are both zero.
18020Sstevel@tonic-gate * On return, the ulwp_lock() is dropped as with ulwp_unlock().
18030Sstevel@tonic-gate * If 'link_dropped' is non-NULL, then 'link_lock' is held on entry.
18040Sstevel@tonic-gate * If we have to drop link_lock, we store 1 through link_dropped.
18050Sstevel@tonic-gate * If the lwp exits before it can be suspended, we return ESRCH.
18060Sstevel@tonic-gate */
18070Sstevel@tonic-gate int
safe_suspend(ulwp_t * ulwp,uchar_t whystopped,int * link_dropped)18080Sstevel@tonic-gate safe_suspend(ulwp_t *ulwp, uchar_t whystopped, int *link_dropped)
18090Sstevel@tonic-gate {
18100Sstevel@tonic-gate ulwp_t *self = curthread;
18110Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
18120Sstevel@tonic-gate cond_t *cvp = ulwp_condvar(ulwp, udp);
18130Sstevel@tonic-gate mutex_t *mp = ulwp_mutex(ulwp, udp);
18140Sstevel@tonic-gate thread_t tid = ulwp->ul_lwpid;
18150Sstevel@tonic-gate int ix = ulwp->ul_ix;
18160Sstevel@tonic-gate int error = 0;
18170Sstevel@tonic-gate
18180Sstevel@tonic-gate ASSERT(whystopped == TSTP_REGULAR ||
18190Sstevel@tonic-gate whystopped == TSTP_MUTATOR ||
18200Sstevel@tonic-gate whystopped == TSTP_FORK);
18210Sstevel@tonic-gate ASSERT(ulwp != self);
18220Sstevel@tonic-gate ASSERT(!ulwp->ul_stop);
18233512Sraf ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
18240Sstevel@tonic-gate ASSERT(MUTEX_OWNED(mp, self));
18250Sstevel@tonic-gate
18260Sstevel@tonic-gate if (link_dropped != NULL)
18270Sstevel@tonic-gate *link_dropped = 0;
18280Sstevel@tonic-gate
18290Sstevel@tonic-gate /*
18300Sstevel@tonic-gate * We must grab the target's spin lock before suspending it.
18310Sstevel@tonic-gate * See the comments below and in _thrp_suspend() for why.
18320Sstevel@tonic-gate */
18330Sstevel@tonic-gate spin_lock_set(&ulwp->ul_spinlock);
18340Sstevel@tonic-gate (void) ___lwp_suspend(tid);
18350Sstevel@tonic-gate spin_lock_clear(&ulwp->ul_spinlock);
18360Sstevel@tonic-gate
18370Sstevel@tonic-gate top:
183810637SRoger.Faulkner@Sun.COM if ((ulwp->ul_critical == 0 && ulwp->ul_rtld == 0) ||
183910637SRoger.Faulkner@Sun.COM ulwp->ul_stopping) {
18400Sstevel@tonic-gate /* thread is already safe */
18410Sstevel@tonic-gate ulwp->ul_stop |= whystopped;
18420Sstevel@tonic-gate } else {
18430Sstevel@tonic-gate /*
18440Sstevel@tonic-gate * Setting ul_pleasestop causes the target thread to stop
18450Sstevel@tonic-gate * itself in _thrp_suspend(), below, after we drop its lock.
18460Sstevel@tonic-gate * We must continue the critical thread before dropping
18470Sstevel@tonic-gate * link_lock because the critical thread may be holding
18480Sstevel@tonic-gate * the queue lock for link_lock. This is delicate.
18490Sstevel@tonic-gate */
18500Sstevel@tonic-gate ulwp->ul_pleasestop |= whystopped;
18510Sstevel@tonic-gate force_continue(ulwp);
18520Sstevel@tonic-gate if (link_dropped != NULL) {
18530Sstevel@tonic-gate *link_dropped = 1;
18540Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
18550Sstevel@tonic-gate /* be sure to drop link_lock only once */
18560Sstevel@tonic-gate link_dropped = NULL;
18570Sstevel@tonic-gate }
18580Sstevel@tonic-gate
18590Sstevel@tonic-gate /*
18600Sstevel@tonic-gate * The thread may disappear by calling thr_exit() so we
18610Sstevel@tonic-gate * cannot rely on the ulwp pointer after dropping the lock.
18620Sstevel@tonic-gate * Instead, we search the hash table to find it again.
18630Sstevel@tonic-gate * When we return, we may find that the thread has been
18640Sstevel@tonic-gate * continued by some other thread. The suspend/continue
18650Sstevel@tonic-gate * interfaces are prone to such race conditions by design.
18660Sstevel@tonic-gate */
18670Sstevel@tonic-gate while (ulwp && !ulwp->ul_dead && !ulwp->ul_stop &&
18680Sstevel@tonic-gate (ulwp->ul_pleasestop & whystopped)) {
18695891Sraf (void) __cond_wait(cvp, mp);
18700Sstevel@tonic-gate for (ulwp = udp->thr_hash_table[ix].hash_bucket;
18710Sstevel@tonic-gate ulwp != NULL; ulwp = ulwp->ul_hash) {
18720Sstevel@tonic-gate if (ulwp->ul_lwpid == tid)
18730Sstevel@tonic-gate break;
18740Sstevel@tonic-gate }
18750Sstevel@tonic-gate }
18760Sstevel@tonic-gate
18770Sstevel@tonic-gate if (ulwp == NULL || ulwp->ul_dead)
18780Sstevel@tonic-gate error = ESRCH;
18790Sstevel@tonic-gate else {
18800Sstevel@tonic-gate /*
18810Sstevel@tonic-gate * Do another lwp_suspend() to make sure we don't
18820Sstevel@tonic-gate * return until the target thread is fully stopped
18830Sstevel@tonic-gate * in the kernel. Don't apply lwp_suspend() until
18840Sstevel@tonic-gate * we know that the target is not holding any
18850Sstevel@tonic-gate * queue locks, that is, that it has completed
18860Sstevel@tonic-gate * ulwp_unlock(self) and has, or at least is
18870Sstevel@tonic-gate * about to, call lwp_suspend() on itself. We do
18880Sstevel@tonic-gate * this by grabbing the target's spin lock.
18890Sstevel@tonic-gate */
18900Sstevel@tonic-gate ASSERT(ulwp->ul_lwpid == tid);
18910Sstevel@tonic-gate spin_lock_set(&ulwp->ul_spinlock);
18920Sstevel@tonic-gate (void) ___lwp_suspend(tid);
18930Sstevel@tonic-gate spin_lock_clear(&ulwp->ul_spinlock);
18940Sstevel@tonic-gate /*
18950Sstevel@tonic-gate * If some other thread did a thr_continue()
18960Sstevel@tonic-gate * on the target thread we have to start over.
18970Sstevel@tonic-gate */
18980Sstevel@tonic-gate if (!ulwp->ul_stopping || !(ulwp->ul_stop & whystopped))
18990Sstevel@tonic-gate goto top;
19000Sstevel@tonic-gate }
19010Sstevel@tonic-gate }
19020Sstevel@tonic-gate
19036812Sraf (void) cond_broadcast(cvp);
19040Sstevel@tonic-gate lmutex_unlock(mp);
19050Sstevel@tonic-gate return (error);
19060Sstevel@tonic-gate }
19070Sstevel@tonic-gate
19080Sstevel@tonic-gate int
_thrp_suspend(thread_t tid,uchar_t whystopped)19090Sstevel@tonic-gate _thrp_suspend(thread_t tid, uchar_t whystopped)
19100Sstevel@tonic-gate {
19110Sstevel@tonic-gate ulwp_t *self = curthread;
19120Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
19130Sstevel@tonic-gate ulwp_t *ulwp;
19140Sstevel@tonic-gate int error = 0;
19150Sstevel@tonic-gate
19160Sstevel@tonic-gate ASSERT((whystopped & (TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) != 0);
19170Sstevel@tonic-gate ASSERT((whystopped & ~(TSTP_REGULAR|TSTP_MUTATOR|TSTP_FORK)) == 0);
19180Sstevel@tonic-gate
19190Sstevel@tonic-gate /*
19204561Sraf * We can't suspend anyone except ourself while
19214561Sraf * some other thread is performing a fork.
19224561Sraf * This also allows only one suspension at a time.
19230Sstevel@tonic-gate */
19244561Sraf if (tid != self->ul_lwpid)
19254843Sraf fork_lock_enter();
19260Sstevel@tonic-gate
19270Sstevel@tonic-gate if ((ulwp = find_lwp(tid)) == NULL)
19280Sstevel@tonic-gate error = ESRCH;
19290Sstevel@tonic-gate else if (whystopped == TSTP_MUTATOR && !ulwp->ul_mutator) {
19300Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
19310Sstevel@tonic-gate error = EINVAL;
19320Sstevel@tonic-gate } else if (ulwp->ul_stop) { /* already stopped */
19330Sstevel@tonic-gate ulwp->ul_stop |= whystopped;
19340Sstevel@tonic-gate ulwp_broadcast(ulwp);
19350Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
19360Sstevel@tonic-gate } else if (ulwp != self) {
19370Sstevel@tonic-gate /*
19380Sstevel@tonic-gate * After suspending the other thread, move it out of a
19390Sstevel@tonic-gate * critical section and deal with the schedctl mappings.
19400Sstevel@tonic-gate * safe_suspend() suspends the other thread, calls
19410Sstevel@tonic-gate * ulwp_broadcast(ulwp) and drops the ulwp lock.
19420Sstevel@tonic-gate */
19430Sstevel@tonic-gate error = safe_suspend(ulwp, whystopped, NULL);
19440Sstevel@tonic-gate } else {
19450Sstevel@tonic-gate int schedctl_after_fork = 0;
19460Sstevel@tonic-gate
19470Sstevel@tonic-gate /*
19480Sstevel@tonic-gate * We are suspending ourself. We must not take a signal
19490Sstevel@tonic-gate * until we return from lwp_suspend() and clear ul_stopping.
19500Sstevel@tonic-gate * This is to guard against siglongjmp().
19510Sstevel@tonic-gate */
19520Sstevel@tonic-gate enter_critical(self);
19530Sstevel@tonic-gate self->ul_sp = stkptr();
19540Sstevel@tonic-gate _flush_windows(); /* sparc */
19550Sstevel@tonic-gate self->ul_pleasestop = 0;
19560Sstevel@tonic-gate self->ul_stop |= whystopped;
19570Sstevel@tonic-gate /*
19580Sstevel@tonic-gate * Grab our spin lock before dropping ulwp_mutex(self).
19590Sstevel@tonic-gate * This prevents the suspending thread from applying
19600Sstevel@tonic-gate * lwp_suspend() to us before we emerge from
19610Sstevel@tonic-gate * lmutex_unlock(mp) and have dropped mp's queue lock.
19620Sstevel@tonic-gate */
19630Sstevel@tonic-gate spin_lock_set(&self->ul_spinlock);
19640Sstevel@tonic-gate self->ul_stopping = 1;
19650Sstevel@tonic-gate ulwp_broadcast(self);
19660Sstevel@tonic-gate ulwp_unlock(self, udp);
19670Sstevel@tonic-gate /*
19680Sstevel@tonic-gate * From this point until we return from lwp_suspend(),
19690Sstevel@tonic-gate * we must not call any function that might invoke the
19700Sstevel@tonic-gate * dynamic linker, that is, we can only call functions
19710Sstevel@tonic-gate * private to the library.
19720Sstevel@tonic-gate *
19730Sstevel@tonic-gate * Also, this is a nasty race condition for a process
19740Sstevel@tonic-gate * that is undergoing a forkall() operation:
19750Sstevel@tonic-gate * Once we clear our spinlock (below), we are vulnerable
19760Sstevel@tonic-gate * to being suspended by the forkall() thread before
19770Sstevel@tonic-gate * we manage to suspend ourself in ___lwp_suspend().
19780Sstevel@tonic-gate * See safe_suspend() and force_continue().
19790Sstevel@tonic-gate *
19800Sstevel@tonic-gate * To avoid a SIGSEGV due to the disappearance
19810Sstevel@tonic-gate * of the schedctl mappings in the child process,
19820Sstevel@tonic-gate * which can happen in spin_lock_clear() if we
19830Sstevel@tonic-gate * are suspended while we are in the middle of
19840Sstevel@tonic-gate * its call to preempt(), we preemptively clear
19850Sstevel@tonic-gate * our own schedctl pointer before dropping our
19860Sstevel@tonic-gate * spinlock. We reinstate it, in both the parent
19870Sstevel@tonic-gate * and (if this really is a forkall()) the child.
19880Sstevel@tonic-gate */
19890Sstevel@tonic-gate if (whystopped & TSTP_FORK) {
19900Sstevel@tonic-gate schedctl_after_fork = 1;
19910Sstevel@tonic-gate self->ul_schedctl = NULL;
19920Sstevel@tonic-gate self->ul_schedctl_called = &udp->uberflags;
19930Sstevel@tonic-gate }
19940Sstevel@tonic-gate spin_lock_clear(&self->ul_spinlock);
19950Sstevel@tonic-gate (void) ___lwp_suspend(tid);
19960Sstevel@tonic-gate /*
19970Sstevel@tonic-gate * Somebody else continued us.
19980Sstevel@tonic-gate * We can't grab ulwp_lock(self)
19990Sstevel@tonic-gate * until after clearing ul_stopping.
20000Sstevel@tonic-gate * force_continue() relies on this.
20010Sstevel@tonic-gate */
20020Sstevel@tonic-gate self->ul_stopping = 0;
20030Sstevel@tonic-gate self->ul_sp = 0;
20040Sstevel@tonic-gate if (schedctl_after_fork) {
20050Sstevel@tonic-gate self->ul_schedctl_called = NULL;
20060Sstevel@tonic-gate self->ul_schedctl = NULL;
20070Sstevel@tonic-gate (void) setup_schedctl();
20080Sstevel@tonic-gate }
20090Sstevel@tonic-gate ulwp_lock(self, udp);
20100Sstevel@tonic-gate ulwp_broadcast(self);
20110Sstevel@tonic-gate ulwp_unlock(self, udp);
20120Sstevel@tonic-gate exit_critical(self);
20130Sstevel@tonic-gate }
20140Sstevel@tonic-gate
20150Sstevel@tonic-gate if (tid != self->ul_lwpid)
20160Sstevel@tonic-gate fork_lock_exit();
20170Sstevel@tonic-gate
20180Sstevel@tonic-gate return (error);
20190Sstevel@tonic-gate }
20200Sstevel@tonic-gate
20210Sstevel@tonic-gate /*
20220Sstevel@tonic-gate * Suspend all lwps other than ourself in preparation for fork.
20230Sstevel@tonic-gate */
20240Sstevel@tonic-gate void
suspend_fork()20250Sstevel@tonic-gate suspend_fork()
20260Sstevel@tonic-gate {
20270Sstevel@tonic-gate ulwp_t *self = curthread;
20280Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
20290Sstevel@tonic-gate ulwp_t *ulwp;
20300Sstevel@tonic-gate int link_dropped;
20310Sstevel@tonic-gate
20323512Sraf ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
20330Sstevel@tonic-gate top:
20340Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
20350Sstevel@tonic-gate
20360Sstevel@tonic-gate for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
20370Sstevel@tonic-gate ulwp_lock(ulwp, udp);
20380Sstevel@tonic-gate if (ulwp->ul_stop) { /* already stopped */
20390Sstevel@tonic-gate ulwp->ul_stop |= TSTP_FORK;
20400Sstevel@tonic-gate ulwp_broadcast(ulwp);
20410Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
20420Sstevel@tonic-gate } else {
20430Sstevel@tonic-gate /*
20440Sstevel@tonic-gate * Move the stopped lwp out of a critical section.
20450Sstevel@tonic-gate */
20460Sstevel@tonic-gate if (safe_suspend(ulwp, TSTP_FORK, &link_dropped) ||
20470Sstevel@tonic-gate link_dropped)
20480Sstevel@tonic-gate goto top;
20490Sstevel@tonic-gate }
20500Sstevel@tonic-gate }
20510Sstevel@tonic-gate
20520Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
20530Sstevel@tonic-gate }
20540Sstevel@tonic-gate
20550Sstevel@tonic-gate void
continue_fork(int child)20560Sstevel@tonic-gate continue_fork(int child)
20570Sstevel@tonic-gate {
20580Sstevel@tonic-gate ulwp_t *self = curthread;
20590Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
20600Sstevel@tonic-gate ulwp_t *ulwp;
20610Sstevel@tonic-gate
20623512Sraf ASSERT(MUTEX_OWNED(&udp->fork_lock, self));
20633512Sraf
20640Sstevel@tonic-gate /*
20650Sstevel@tonic-gate * Clear the schedctl pointers in the child of forkall().
20660Sstevel@tonic-gate */
20670Sstevel@tonic-gate if (child) {
20680Sstevel@tonic-gate for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
20690Sstevel@tonic-gate ulwp->ul_schedctl_called =
20704843Sraf ulwp->ul_dead? &udp->uberflags : NULL;
20710Sstevel@tonic-gate ulwp->ul_schedctl = NULL;
20720Sstevel@tonic-gate }
20730Sstevel@tonic-gate }
20740Sstevel@tonic-gate
20750Sstevel@tonic-gate /*
20760Sstevel@tonic-gate * Set all lwps that were stopped for fork() running again.
20770Sstevel@tonic-gate */
20780Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
20790Sstevel@tonic-gate for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
20800Sstevel@tonic-gate mutex_t *mp = ulwp_mutex(ulwp, udp);
20810Sstevel@tonic-gate lmutex_lock(mp);
20820Sstevel@tonic-gate ASSERT(ulwp->ul_stop & TSTP_FORK);
20830Sstevel@tonic-gate ulwp->ul_stop &= ~TSTP_FORK;
20840Sstevel@tonic-gate ulwp_broadcast(ulwp);
20850Sstevel@tonic-gate if (!ulwp->ul_stop)
20860Sstevel@tonic-gate force_continue(ulwp);
20870Sstevel@tonic-gate lmutex_unlock(mp);
20880Sstevel@tonic-gate }
20890Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
20900Sstevel@tonic-gate }
20910Sstevel@tonic-gate
20920Sstevel@tonic-gate int
_thrp_continue(thread_t tid,uchar_t whystopped)20930Sstevel@tonic-gate _thrp_continue(thread_t tid, uchar_t whystopped)
20940Sstevel@tonic-gate {
20950Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
20960Sstevel@tonic-gate ulwp_t *ulwp;
20970Sstevel@tonic-gate mutex_t *mp;
20980Sstevel@tonic-gate int error = 0;
20990Sstevel@tonic-gate
21000Sstevel@tonic-gate ASSERT(whystopped == TSTP_REGULAR ||
21010Sstevel@tonic-gate whystopped == TSTP_MUTATOR);
21020Sstevel@tonic-gate
21033512Sraf /*
21043512Sraf * We single-thread the entire thread suspend/continue mechanism.
21053512Sraf */
21064843Sraf fork_lock_enter();
21073512Sraf
21083512Sraf if ((ulwp = find_lwp(tid)) == NULL) {
21093512Sraf fork_lock_exit();
21100Sstevel@tonic-gate return (ESRCH);
21113512Sraf }
21120Sstevel@tonic-gate
21130Sstevel@tonic-gate mp = ulwp_mutex(ulwp, udp);
21140Sstevel@tonic-gate if ((whystopped == TSTP_MUTATOR && !ulwp->ul_mutator)) {
21150Sstevel@tonic-gate error = EINVAL;
21160Sstevel@tonic-gate } else if (ulwp->ul_stop & whystopped) {
21170Sstevel@tonic-gate ulwp->ul_stop &= ~whystopped;
21180Sstevel@tonic-gate ulwp_broadcast(ulwp);
21190Sstevel@tonic-gate if (!ulwp->ul_stop) {
21200Sstevel@tonic-gate if (whystopped == TSTP_REGULAR && ulwp->ul_created) {
21210Sstevel@tonic-gate ulwp->ul_sp = 0;
21220Sstevel@tonic-gate ulwp->ul_created = 0;
21230Sstevel@tonic-gate }
21240Sstevel@tonic-gate force_continue(ulwp);
21250Sstevel@tonic-gate }
21260Sstevel@tonic-gate }
21273512Sraf lmutex_unlock(mp);
21280Sstevel@tonic-gate
21293512Sraf fork_lock_exit();
21300Sstevel@tonic-gate return (error);
21310Sstevel@tonic-gate }
21320Sstevel@tonic-gate
21330Sstevel@tonic-gate int
thr_suspend(thread_t tid)21346812Sraf thr_suspend(thread_t tid)
21350Sstevel@tonic-gate {
21360Sstevel@tonic-gate return (_thrp_suspend(tid, TSTP_REGULAR));
21370Sstevel@tonic-gate }
21380Sstevel@tonic-gate
21390Sstevel@tonic-gate int
thr_continue(thread_t tid)21406812Sraf thr_continue(thread_t tid)
21410Sstevel@tonic-gate {
21420Sstevel@tonic-gate return (_thrp_continue(tid, TSTP_REGULAR));
21430Sstevel@tonic-gate }
21440Sstevel@tonic-gate
21450Sstevel@tonic-gate void
thr_yield()21466812Sraf thr_yield()
21470Sstevel@tonic-gate {
21486515Sraf yield();
21490Sstevel@tonic-gate }
21500Sstevel@tonic-gate
21516812Sraf #pragma weak pthread_kill = thr_kill
21526812Sraf #pragma weak _thr_kill = thr_kill
21530Sstevel@tonic-gate int
thr_kill(thread_t tid,int sig)21546812Sraf thr_kill(thread_t tid, int sig)
21550Sstevel@tonic-gate {
21560Sstevel@tonic-gate if (sig == SIGCANCEL)
21570Sstevel@tonic-gate return (EINVAL);
21586812Sraf return (_lwp_kill(tid, sig));
21590Sstevel@tonic-gate }
21600Sstevel@tonic-gate
21610Sstevel@tonic-gate /*
21620Sstevel@tonic-gate * Exit a critical section, take deferred actions if necessary.
216310637SRoger.Faulkner@Sun.COM * Called from exit_critical() and from sigon().
21640Sstevel@tonic-gate */
21650Sstevel@tonic-gate void
do_exit_critical()21660Sstevel@tonic-gate do_exit_critical()
21670Sstevel@tonic-gate {
21680Sstevel@tonic-gate ulwp_t *self = curthread;
21690Sstevel@tonic-gate int sig;
21700Sstevel@tonic-gate
21710Sstevel@tonic-gate ASSERT(self->ul_critical == 0);
217210637SRoger.Faulkner@Sun.COM
217310637SRoger.Faulkner@Sun.COM /*
217410637SRoger.Faulkner@Sun.COM * Don't suspend ourself or take a deferred signal while dying
217510637SRoger.Faulkner@Sun.COM * or while executing inside the dynamic linker (ld.so.1).
217610637SRoger.Faulkner@Sun.COM */
217710637SRoger.Faulkner@Sun.COM if (self->ul_dead || self->ul_rtld)
21780Sstevel@tonic-gate return;
21790Sstevel@tonic-gate
21800Sstevel@tonic-gate while (self->ul_pleasestop ||
21810Sstevel@tonic-gate (self->ul_cursig != 0 && self->ul_sigdefer == 0)) {
21820Sstevel@tonic-gate /*
21830Sstevel@tonic-gate * Avoid a recursive call to exit_critical() in _thrp_suspend()
21840Sstevel@tonic-gate * by keeping self->ul_critical == 1 here.
21850Sstevel@tonic-gate */
21860Sstevel@tonic-gate self->ul_critical++;
21870Sstevel@tonic-gate while (self->ul_pleasestop) {
21880Sstevel@tonic-gate /*
21890Sstevel@tonic-gate * Guard against suspending ourself while on a sleep
21900Sstevel@tonic-gate * queue. See the comments in call_user_handler().
21910Sstevel@tonic-gate */
21920Sstevel@tonic-gate unsleep_self();
21930Sstevel@tonic-gate set_parking_flag(self, 0);
21940Sstevel@tonic-gate (void) _thrp_suspend(self->ul_lwpid,
21954843Sraf self->ul_pleasestop);
21960Sstevel@tonic-gate }
21970Sstevel@tonic-gate self->ul_critical--;
21980Sstevel@tonic-gate
21990Sstevel@tonic-gate if ((sig = self->ul_cursig) != 0 && self->ul_sigdefer == 0) {
22000Sstevel@tonic-gate /*
22010Sstevel@tonic-gate * Clear ul_cursig before proceeding.
22020Sstevel@tonic-gate * This protects us from the dynamic linker's
22030Sstevel@tonic-gate * calls to bind_guard()/bind_clear() in the
22040Sstevel@tonic-gate * event that it is invoked to resolve a symbol
22050Sstevel@tonic-gate * like take_deferred_signal() below.
22060Sstevel@tonic-gate */
22070Sstevel@tonic-gate self->ul_cursig = 0;
22080Sstevel@tonic-gate take_deferred_signal(sig);
22090Sstevel@tonic-gate ASSERT(self->ul_cursig == 0);
22100Sstevel@tonic-gate }
22110Sstevel@tonic-gate }
22120Sstevel@tonic-gate ASSERT(self->ul_critical == 0);
22130Sstevel@tonic-gate }
22140Sstevel@tonic-gate
22155891Sraf /*
22165891Sraf * _ti_bind_guard() and _ti_bind_clear() are called by the dynamic linker
22175891Sraf * (ld.so.1) when it has do do something, like resolve a symbol to be called
22185891Sraf * by the application or one of its libraries. _ti_bind_guard() is called
22195891Sraf * on entry to ld.so.1, _ti_bind_clear() on exit from ld.so.1 back to the
22205891Sraf * application. The dynamic linker gets special dispensation from libc to
22215891Sraf * run in a critical region (all signals deferred and no thread suspension
22225891Sraf * or forking allowed), and to be immune from cancellation for the duration.
22235891Sraf */
22240Sstevel@tonic-gate int
_ti_bind_guard(int flags)22256515Sraf _ti_bind_guard(int flags)
22260Sstevel@tonic-gate {
22270Sstevel@tonic-gate ulwp_t *self = curthread;
22286515Sraf uberdata_t *udp = self->ul_uberdata;
22296515Sraf int bindflag = (flags & THR_FLG_RTLD);
22300Sstevel@tonic-gate
22310Sstevel@tonic-gate if ((self->ul_bindflags & bindflag) == bindflag)
22320Sstevel@tonic-gate return (0);
22339569SRod.Evans@Sun.COM self->ul_bindflags |= bindflag;
22346515Sraf if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
22356515Sraf sigoff(self); /* see no signals while holding ld_lock */
223610637SRoger.Faulkner@Sun.COM self->ul_rtld++; /* don't suspend while in ld.so.1 */
22376812Sraf (void) mutex_lock(&udp->ld_lock);
22386515Sraf }
22390Sstevel@tonic-gate enter_critical(self);
22405891Sraf self->ul_save_state = self->ul_cancel_disabled;
22415891Sraf self->ul_cancel_disabled = 1;
22425891Sraf set_cancel_pending_flag(self, 0);
22430Sstevel@tonic-gate return (1);
22440Sstevel@tonic-gate }
22450Sstevel@tonic-gate
22460Sstevel@tonic-gate int
_ti_bind_clear(int flags)22476515Sraf _ti_bind_clear(int flags)
22480Sstevel@tonic-gate {
22490Sstevel@tonic-gate ulwp_t *self = curthread;
22506515Sraf uberdata_t *udp = self->ul_uberdata;
22516515Sraf int bindflag = (flags & THR_FLG_RTLD);
22520Sstevel@tonic-gate
22530Sstevel@tonic-gate if ((self->ul_bindflags & bindflag) == 0)
22540Sstevel@tonic-gate return (self->ul_bindflags);
22550Sstevel@tonic-gate self->ul_bindflags &= ~bindflag;
22565891Sraf self->ul_cancel_disabled = self->ul_save_state;
22575891Sraf set_cancel_pending_flag(self, 0);
22580Sstevel@tonic-gate exit_critical(self);
22596515Sraf if ((flags & (THR_FLG_NOLOCK | THR_FLG_REENTER)) == THR_FLG_NOLOCK) {
22606515Sraf if (MUTEX_OWNED(&udp->ld_lock, self)) {
22616812Sraf (void) mutex_unlock(&udp->ld_lock);
226210637SRoger.Faulkner@Sun.COM self->ul_rtld--;
22636515Sraf sigon(self); /* reenable signals */
22646515Sraf }
22656515Sraf }
22660Sstevel@tonic-gate return (self->ul_bindflags);
22670Sstevel@tonic-gate }
22680Sstevel@tonic-gate
22690Sstevel@tonic-gate /*
22709569SRod.Evans@Sun.COM * Tell the dynamic linker (ld.so.1) whether or not it was entered from
22719569SRod.Evans@Sun.COM * a critical region in libc. Return zero if not, else return non-zero.
22729569SRod.Evans@Sun.COM */
22739569SRod.Evans@Sun.COM int
_ti_critical(void)22749569SRod.Evans@Sun.COM _ti_critical(void)
22759569SRod.Evans@Sun.COM {
22769569SRod.Evans@Sun.COM ulwp_t *self = curthread;
22779569SRod.Evans@Sun.COM int level = self->ul_critical;
22789569SRod.Evans@Sun.COM
22799569SRod.Evans@Sun.COM if ((self->ul_bindflags & THR_FLG_RTLD) == 0 || level == 0)
22809569SRod.Evans@Sun.COM return (level); /* ld.so.1 hasn't (yet) called enter() */
22819569SRod.Evans@Sun.COM return (level - 1);
22829569SRod.Evans@Sun.COM }
22839569SRod.Evans@Sun.COM
22849569SRod.Evans@Sun.COM /*
22850Sstevel@tonic-gate * sigoff() and sigon() enable cond_wait() to behave (optionally) like
22860Sstevel@tonic-gate * it does in the old libthread (see the comments in cond_wait_queue()).
22870Sstevel@tonic-gate * Also, signals are deferred at thread startup until TLS constructors
22886812Sraf * have all been called, at which time _thrp_setup() calls sigon().
22891885Sraf *
22902248Sraf * _sigoff() and _sigon() are external consolidation-private interfaces to
22912248Sraf * sigoff() and sigon(), respectively, in libc. These are used in libnsl.
22920Sstevel@tonic-gate * Also, _sigoff() and _sigon() are called from dbx's run-time checking
22930Sstevel@tonic-gate * (librtc.so) to defer signals during its critical sections (not to be
22940Sstevel@tonic-gate * confused with libc critical sections [see exit_critical() above]).
22950Sstevel@tonic-gate */
22960Sstevel@tonic-gate void
_sigoff(void)22970Sstevel@tonic-gate _sigoff(void)
22980Sstevel@tonic-gate {
229910637SRoger.Faulkner@Sun.COM ulwp_t *self = curthread;
230010637SRoger.Faulkner@Sun.COM
230110637SRoger.Faulkner@Sun.COM sigoff(self);
23020Sstevel@tonic-gate }
23030Sstevel@tonic-gate
23040Sstevel@tonic-gate void
_sigon(void)23050Sstevel@tonic-gate _sigon(void)
23060Sstevel@tonic-gate {
230710637SRoger.Faulkner@Sun.COM ulwp_t *self = curthread;
23080Sstevel@tonic-gate
23090Sstevel@tonic-gate ASSERT(self->ul_sigdefer > 0);
231010637SRoger.Faulkner@Sun.COM sigon(self);
23110Sstevel@tonic-gate }
23120Sstevel@tonic-gate
23130Sstevel@tonic-gate int
thr_getconcurrency()23146812Sraf thr_getconcurrency()
23150Sstevel@tonic-gate {
23160Sstevel@tonic-gate return (thr_concurrency);
23170Sstevel@tonic-gate }
23180Sstevel@tonic-gate
23190Sstevel@tonic-gate int
pthread_getconcurrency()23206812Sraf pthread_getconcurrency()
23210Sstevel@tonic-gate {
23220Sstevel@tonic-gate return (pthread_concurrency);
23230Sstevel@tonic-gate }
23240Sstevel@tonic-gate
23250Sstevel@tonic-gate int
thr_setconcurrency(int new_level)23266812Sraf thr_setconcurrency(int new_level)
23270Sstevel@tonic-gate {
23280Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
23290Sstevel@tonic-gate
23300Sstevel@tonic-gate if (new_level < 0)
23310Sstevel@tonic-gate return (EINVAL);
23320Sstevel@tonic-gate if (new_level > 65536) /* 65536 is totally arbitrary */
23330Sstevel@tonic-gate return (EAGAIN);
23340Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
23350Sstevel@tonic-gate if (new_level > thr_concurrency)
23360Sstevel@tonic-gate thr_concurrency = new_level;
23370Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
23380Sstevel@tonic-gate return (0);
23390Sstevel@tonic-gate }
23400Sstevel@tonic-gate
23410Sstevel@tonic-gate int
pthread_setconcurrency(int new_level)23426812Sraf pthread_setconcurrency(int new_level)
23430Sstevel@tonic-gate {
23440Sstevel@tonic-gate if (new_level < 0)
23450Sstevel@tonic-gate return (EINVAL);
23460Sstevel@tonic-gate if (new_level > 65536) /* 65536 is totally arbitrary */
23470Sstevel@tonic-gate return (EAGAIN);
23480Sstevel@tonic-gate pthread_concurrency = new_level;
23490Sstevel@tonic-gate return (0);
23500Sstevel@tonic-gate }
23510Sstevel@tonic-gate
23520Sstevel@tonic-gate size_t
thr_min_stack(void)23536812Sraf thr_min_stack(void)
23540Sstevel@tonic-gate {
23550Sstevel@tonic-gate return (MINSTACK);
23560Sstevel@tonic-gate }
23570Sstevel@tonic-gate
23580Sstevel@tonic-gate int
__nthreads(void)23590Sstevel@tonic-gate __nthreads(void)
23600Sstevel@tonic-gate {
23610Sstevel@tonic-gate return (curthread->ul_uberdata->nthreads);
23620Sstevel@tonic-gate }
23630Sstevel@tonic-gate
23640Sstevel@tonic-gate /*
23650Sstevel@tonic-gate * XXX
23660Sstevel@tonic-gate * The remainder of this file implements the private interfaces to java for
23670Sstevel@tonic-gate * garbage collection. It is no longer used, at least by java 1.2.
23680Sstevel@tonic-gate * It can all go away once all old JVMs have disappeared.
23690Sstevel@tonic-gate */
23700Sstevel@tonic-gate
23710Sstevel@tonic-gate int suspendingallmutators; /* when non-zero, suspending all mutators. */
23720Sstevel@tonic-gate int suspendedallmutators; /* when non-zero, all mutators suspended. */
23730Sstevel@tonic-gate int mutatorsbarrier; /* when non-zero, mutators barrier imposed. */
23740Sstevel@tonic-gate mutex_t mutatorslock = DEFAULTMUTEX; /* used to enforce mutators barrier. */
23750Sstevel@tonic-gate cond_t mutatorscv = DEFAULTCV; /* where non-mutators sleep. */
23760Sstevel@tonic-gate
23770Sstevel@tonic-gate /*
23780Sstevel@tonic-gate * Get the available register state for the target thread.
23790Sstevel@tonic-gate * Return non-volatile registers: TRS_NONVOLATILE
23800Sstevel@tonic-gate */
23816812Sraf #pragma weak _thr_getstate = thr_getstate
23820Sstevel@tonic-gate int
thr_getstate(thread_t tid,int * flag,lwpid_t * lwp,stack_t * ss,gregset_t rs)23836812Sraf thr_getstate(thread_t tid, int *flag, lwpid_t *lwp, stack_t *ss, gregset_t rs)
23840Sstevel@tonic-gate {
23850Sstevel@tonic-gate ulwp_t *self = curthread;
23860Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
23870Sstevel@tonic-gate ulwp_t **ulwpp;
23880Sstevel@tonic-gate ulwp_t *ulwp;
23890Sstevel@tonic-gate int error = 0;
23900Sstevel@tonic-gate int trs_flag = TRS_LWPID;
23910Sstevel@tonic-gate
23920Sstevel@tonic-gate if (tid == 0 || self->ul_lwpid == tid) {
23930Sstevel@tonic-gate ulwp = self;
23940Sstevel@tonic-gate ulwp_lock(ulwp, udp);
23950Sstevel@tonic-gate } else if ((ulwpp = find_lwpp(tid)) != NULL) {
23960Sstevel@tonic-gate ulwp = *ulwpp;
23970Sstevel@tonic-gate } else {
23980Sstevel@tonic-gate if (flag)
23990Sstevel@tonic-gate *flag = TRS_INVALID;
24000Sstevel@tonic-gate return (ESRCH);
24010Sstevel@tonic-gate }
24020Sstevel@tonic-gate
24030Sstevel@tonic-gate if (ulwp->ul_dead) {
24040Sstevel@tonic-gate trs_flag = TRS_INVALID;
24050Sstevel@tonic-gate } else if (!ulwp->ul_stop && !suspendedallmutators) {
24060Sstevel@tonic-gate error = EINVAL;
24070Sstevel@tonic-gate trs_flag = TRS_INVALID;
24080Sstevel@tonic-gate } else if (ulwp->ul_stop) {
24090Sstevel@tonic-gate trs_flag = TRS_NONVOLATILE;
24100Sstevel@tonic-gate getgregs(ulwp, rs);
24110Sstevel@tonic-gate }
24120Sstevel@tonic-gate
24130Sstevel@tonic-gate if (flag)
24140Sstevel@tonic-gate *flag = trs_flag;
24150Sstevel@tonic-gate if (lwp)
24160Sstevel@tonic-gate *lwp = tid;
24170Sstevel@tonic-gate if (ss != NULL)
24180Sstevel@tonic-gate (void) _thrp_stksegment(ulwp, ss);
24190Sstevel@tonic-gate
24200Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
24210Sstevel@tonic-gate return (error);
24220Sstevel@tonic-gate }
24230Sstevel@tonic-gate
24240Sstevel@tonic-gate /*
24250Sstevel@tonic-gate * Set the appropriate register state for the target thread.
24260Sstevel@tonic-gate * This is not used by java. It exists solely for the MSTC test suite.
24270Sstevel@tonic-gate */
24286812Sraf #pragma weak _thr_setstate = thr_setstate
24290Sstevel@tonic-gate int
thr_setstate(thread_t tid,int flag,gregset_t rs)24306812Sraf thr_setstate(thread_t tid, int flag, gregset_t rs)
24310Sstevel@tonic-gate {
24320Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
24330Sstevel@tonic-gate ulwp_t *ulwp;
24340Sstevel@tonic-gate int error = 0;
24350Sstevel@tonic-gate
24360Sstevel@tonic-gate if ((ulwp = find_lwp(tid)) == NULL)
24370Sstevel@tonic-gate return (ESRCH);
24380Sstevel@tonic-gate
24390Sstevel@tonic-gate if (!ulwp->ul_stop && !suspendedallmutators)
24400Sstevel@tonic-gate error = EINVAL;
24410Sstevel@tonic-gate else if (rs != NULL) {
24420Sstevel@tonic-gate switch (flag) {
24430Sstevel@tonic-gate case TRS_NONVOLATILE:
24440Sstevel@tonic-gate /* do /proc stuff here? */
24450Sstevel@tonic-gate if (ulwp->ul_stop)
24460Sstevel@tonic-gate setgregs(ulwp, rs);
24470Sstevel@tonic-gate else
24480Sstevel@tonic-gate error = EINVAL;
24490Sstevel@tonic-gate break;
24500Sstevel@tonic-gate case TRS_LWPID: /* do /proc stuff here? */
24510Sstevel@tonic-gate default:
24520Sstevel@tonic-gate error = EINVAL;
24530Sstevel@tonic-gate break;
24540Sstevel@tonic-gate }
24550Sstevel@tonic-gate }
24560Sstevel@tonic-gate
24570Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
24580Sstevel@tonic-gate return (error);
24590Sstevel@tonic-gate }
24600Sstevel@tonic-gate
24610Sstevel@tonic-gate int
getlwpstatus(thread_t tid,struct lwpstatus * sp)24620Sstevel@tonic-gate getlwpstatus(thread_t tid, struct lwpstatus *sp)
24630Sstevel@tonic-gate {
24645891Sraf extern ssize_t __pread(int, void *, size_t, off_t);
24650Sstevel@tonic-gate char buf[100];
24660Sstevel@tonic-gate int fd;
24670Sstevel@tonic-gate
24680Sstevel@tonic-gate /* "/proc/self/lwp/%u/lwpstatus" w/o stdio */
24690Sstevel@tonic-gate (void) strcpy(buf, "/proc/self/lwp/");
24700Sstevel@tonic-gate ultos((uint64_t)tid, 10, buf + strlen(buf));
24710Sstevel@tonic-gate (void) strcat(buf, "/lwpstatus");
24726515Sraf if ((fd = __open(buf, O_RDONLY, 0)) >= 0) {
24735891Sraf while (__pread(fd, sp, sizeof (*sp), 0) == sizeof (*sp)) {
24740Sstevel@tonic-gate if (sp->pr_flags & PR_STOPPED) {
24756515Sraf (void) __close(fd);
24760Sstevel@tonic-gate return (0);
24770Sstevel@tonic-gate }
24786515Sraf yield(); /* give him a chance to stop */
24790Sstevel@tonic-gate }
24806515Sraf (void) __close(fd);
24810Sstevel@tonic-gate }
24820Sstevel@tonic-gate return (-1);
24830Sstevel@tonic-gate }
24840Sstevel@tonic-gate
24850Sstevel@tonic-gate int
putlwpregs(thread_t tid,prgregset_t prp)24860Sstevel@tonic-gate putlwpregs(thread_t tid, prgregset_t prp)
24870Sstevel@tonic-gate {
24885891Sraf extern ssize_t __writev(int, const struct iovec *, int);
24890Sstevel@tonic-gate char buf[100];
24900Sstevel@tonic-gate int fd;
24910Sstevel@tonic-gate long dstop_sreg[2];
24920Sstevel@tonic-gate long run_null[2];
24930Sstevel@tonic-gate iovec_t iov[3];
24940Sstevel@tonic-gate
24950Sstevel@tonic-gate /* "/proc/self/lwp/%u/lwpctl" w/o stdio */
24960Sstevel@tonic-gate (void) strcpy(buf, "/proc/self/lwp/");
24970Sstevel@tonic-gate ultos((uint64_t)tid, 10, buf + strlen(buf));
24980Sstevel@tonic-gate (void) strcat(buf, "/lwpctl");
24996515Sraf if ((fd = __open(buf, O_WRONLY, 0)) >= 0) {
25000Sstevel@tonic-gate dstop_sreg[0] = PCDSTOP; /* direct it to stop */
25010Sstevel@tonic-gate dstop_sreg[1] = PCSREG; /* set the registers */
25020Sstevel@tonic-gate iov[0].iov_base = (caddr_t)dstop_sreg;
25030Sstevel@tonic-gate iov[0].iov_len = sizeof (dstop_sreg);
25040Sstevel@tonic-gate iov[1].iov_base = (caddr_t)prp; /* from the register set */
25050Sstevel@tonic-gate iov[1].iov_len = sizeof (prgregset_t);
25060Sstevel@tonic-gate run_null[0] = PCRUN; /* make it runnable again */
25070Sstevel@tonic-gate run_null[1] = 0;
25080Sstevel@tonic-gate iov[2].iov_base = (caddr_t)run_null;
25090Sstevel@tonic-gate iov[2].iov_len = sizeof (run_null);
25105891Sraf if (__writev(fd, iov, 3) >= 0) {
25116515Sraf (void) __close(fd);
25120Sstevel@tonic-gate return (0);
25130Sstevel@tonic-gate }
25146515Sraf (void) __close(fd);
25150Sstevel@tonic-gate }
25160Sstevel@tonic-gate return (-1);
25170Sstevel@tonic-gate }
25180Sstevel@tonic-gate
25190Sstevel@tonic-gate static ulong_t
gettsp_slow(thread_t tid)25200Sstevel@tonic-gate gettsp_slow(thread_t tid)
25210Sstevel@tonic-gate {
25220Sstevel@tonic-gate char buf[100];
25230Sstevel@tonic-gate struct lwpstatus status;
25240Sstevel@tonic-gate
25250Sstevel@tonic-gate if (getlwpstatus(tid, &status) != 0) {
25260Sstevel@tonic-gate /* "__gettsp(%u): can't read lwpstatus" w/o stdio */
25270Sstevel@tonic-gate (void) strcpy(buf, "__gettsp(");
25280Sstevel@tonic-gate ultos((uint64_t)tid, 10, buf + strlen(buf));
25290Sstevel@tonic-gate (void) strcat(buf, "): can't read lwpstatus");
25300Sstevel@tonic-gate thr_panic(buf);
25310Sstevel@tonic-gate }
25320Sstevel@tonic-gate return (status.pr_reg[R_SP]);
25330Sstevel@tonic-gate }
25340Sstevel@tonic-gate
25350Sstevel@tonic-gate ulong_t
__gettsp(thread_t tid)25360Sstevel@tonic-gate __gettsp(thread_t tid)
25370Sstevel@tonic-gate {
25380Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
25390Sstevel@tonic-gate ulwp_t *ulwp;
25400Sstevel@tonic-gate ulong_t result;
25410Sstevel@tonic-gate
25420Sstevel@tonic-gate if ((ulwp = find_lwp(tid)) == NULL)
25430Sstevel@tonic-gate return (0);
25440Sstevel@tonic-gate
25450Sstevel@tonic-gate if (ulwp->ul_stop && (result = ulwp->ul_sp) != 0) {
25460Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
25470Sstevel@tonic-gate return (result);
25480Sstevel@tonic-gate }
25490Sstevel@tonic-gate
25500Sstevel@tonic-gate result = gettsp_slow(tid);
25510Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
25520Sstevel@tonic-gate return (result);
25530Sstevel@tonic-gate }
25540Sstevel@tonic-gate
25550Sstevel@tonic-gate /*
25560Sstevel@tonic-gate * This tells java stack walkers how to find the ucontext
25570Sstevel@tonic-gate * structure passed to signal handlers.
25580Sstevel@tonic-gate */
25596812Sraf #pragma weak _thr_sighndlrinfo = thr_sighndlrinfo
25600Sstevel@tonic-gate void
thr_sighndlrinfo(void (** func)(),int * funcsize)25616812Sraf thr_sighndlrinfo(void (**func)(), int *funcsize)
25620Sstevel@tonic-gate {
25630Sstevel@tonic-gate *func = &__sighndlr;
25640Sstevel@tonic-gate *funcsize = (char *)&__sighndlrend - (char *)&__sighndlr;
25650Sstevel@tonic-gate }
25660Sstevel@tonic-gate
25670Sstevel@tonic-gate /*
25680Sstevel@tonic-gate * Mark a thread a mutator or reset a mutator to being a default,
25690Sstevel@tonic-gate * non-mutator thread.
25700Sstevel@tonic-gate */
25716812Sraf #pragma weak _thr_setmutator = thr_setmutator
25720Sstevel@tonic-gate int
thr_setmutator(thread_t tid,int enabled)25736812Sraf thr_setmutator(thread_t tid, int enabled)
25740Sstevel@tonic-gate {
25750Sstevel@tonic-gate ulwp_t *self = curthread;
25760Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
25770Sstevel@tonic-gate ulwp_t *ulwp;
25780Sstevel@tonic-gate int error;
25795891Sraf int cancel_state;
25800Sstevel@tonic-gate
25815891Sraf enabled = enabled? 1 : 0;
25820Sstevel@tonic-gate top:
25830Sstevel@tonic-gate if (tid == 0) {
25840Sstevel@tonic-gate ulwp = self;
25850Sstevel@tonic-gate ulwp_lock(ulwp, udp);
25860Sstevel@tonic-gate } else if ((ulwp = find_lwp(tid)) == NULL) {
25870Sstevel@tonic-gate return (ESRCH);
25880Sstevel@tonic-gate }
25890Sstevel@tonic-gate
25900Sstevel@tonic-gate /*
25910Sstevel@tonic-gate * The target thread should be the caller itself or a suspended thread.
25920Sstevel@tonic-gate * This prevents the target from also changing its ul_mutator field.
25930Sstevel@tonic-gate */
25940Sstevel@tonic-gate error = 0;
25950Sstevel@tonic-gate if (ulwp != self && !ulwp->ul_stop && enabled)
25960Sstevel@tonic-gate error = EINVAL;
25970Sstevel@tonic-gate else if (ulwp->ul_mutator != enabled) {
25980Sstevel@tonic-gate lmutex_lock(&mutatorslock);
25990Sstevel@tonic-gate if (mutatorsbarrier) {
26000Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
26016812Sraf (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
26025891Sraf &cancel_state);
26030Sstevel@tonic-gate while (mutatorsbarrier)
26046812Sraf (void) cond_wait(&mutatorscv, &mutatorslock);
26056812Sraf (void) pthread_setcancelstate(cancel_state, NULL);
26060Sstevel@tonic-gate lmutex_unlock(&mutatorslock);
26070Sstevel@tonic-gate goto top;
26080Sstevel@tonic-gate }
26090Sstevel@tonic-gate ulwp->ul_mutator = enabled;
26100Sstevel@tonic-gate lmutex_unlock(&mutatorslock);
26110Sstevel@tonic-gate }
26120Sstevel@tonic-gate
26130Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
26140Sstevel@tonic-gate return (error);
26150Sstevel@tonic-gate }
26160Sstevel@tonic-gate
26170Sstevel@tonic-gate /*
26180Sstevel@tonic-gate * Establish a barrier against new mutators. Any non-mutator trying
26190Sstevel@tonic-gate * to become a mutator is suspended until the barrier is removed.
26200Sstevel@tonic-gate */
26216812Sraf #pragma weak _thr_mutators_barrier = thr_mutators_barrier
26220Sstevel@tonic-gate void
thr_mutators_barrier(int enabled)26236812Sraf thr_mutators_barrier(int enabled)
26240Sstevel@tonic-gate {
26250Sstevel@tonic-gate int oldvalue;
26265891Sraf int cancel_state;
26270Sstevel@tonic-gate
26280Sstevel@tonic-gate lmutex_lock(&mutatorslock);
26290Sstevel@tonic-gate
26300Sstevel@tonic-gate /*
26310Sstevel@tonic-gate * Wait if trying to set the barrier while it is already set.
26320Sstevel@tonic-gate */
26336812Sraf (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
26340Sstevel@tonic-gate while (mutatorsbarrier && enabled)
26356812Sraf (void) cond_wait(&mutatorscv, &mutatorslock);
26366812Sraf (void) pthread_setcancelstate(cancel_state, NULL);
26370Sstevel@tonic-gate
26380Sstevel@tonic-gate oldvalue = mutatorsbarrier;
26390Sstevel@tonic-gate mutatorsbarrier = enabled;
26400Sstevel@tonic-gate /*
26410Sstevel@tonic-gate * Wakeup any blocked non-mutators when barrier is removed.
26420Sstevel@tonic-gate */
26430Sstevel@tonic-gate if (oldvalue && !enabled)
26446812Sraf (void) cond_broadcast(&mutatorscv);
26450Sstevel@tonic-gate lmutex_unlock(&mutatorslock);
26460Sstevel@tonic-gate }
26470Sstevel@tonic-gate
26480Sstevel@tonic-gate /*
26490Sstevel@tonic-gate * Suspend the set of all mutators except for the caller. The list
26500Sstevel@tonic-gate * of actively running threads is searched and only the mutators
26510Sstevel@tonic-gate * in this list are suspended. Actively running non-mutators remain
26520Sstevel@tonic-gate * running. Any other thread is suspended.
26530Sstevel@tonic-gate */
26546812Sraf #pragma weak _thr_suspend_allmutators = thr_suspend_allmutators
26550Sstevel@tonic-gate int
thr_suspend_allmutators(void)26566812Sraf thr_suspend_allmutators(void)
26570Sstevel@tonic-gate {
26580Sstevel@tonic-gate ulwp_t *self = curthread;
26590Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
26600Sstevel@tonic-gate ulwp_t *ulwp;
26610Sstevel@tonic-gate int link_dropped;
26620Sstevel@tonic-gate
26630Sstevel@tonic-gate /*
26643512Sraf * We single-thread the entire thread suspend/continue mechanism.
26650Sstevel@tonic-gate */
26664843Sraf fork_lock_enter();
26673512Sraf
26680Sstevel@tonic-gate top:
26690Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
26700Sstevel@tonic-gate
26710Sstevel@tonic-gate if (suspendingallmutators || suspendedallmutators) {
26720Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
26730Sstevel@tonic-gate fork_lock_exit();
26740Sstevel@tonic-gate return (EINVAL);
26750Sstevel@tonic-gate }
26760Sstevel@tonic-gate suspendingallmutators = 1;
26770Sstevel@tonic-gate
26780Sstevel@tonic-gate for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
26790Sstevel@tonic-gate ulwp_lock(ulwp, udp);
26800Sstevel@tonic-gate if (!ulwp->ul_mutator) {
26810Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
26820Sstevel@tonic-gate } else if (ulwp->ul_stop) { /* already stopped */
26830Sstevel@tonic-gate ulwp->ul_stop |= TSTP_MUTATOR;
26840Sstevel@tonic-gate ulwp_broadcast(ulwp);
26850Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
26860Sstevel@tonic-gate } else {
26870Sstevel@tonic-gate /*
26880Sstevel@tonic-gate * Move the stopped lwp out of a critical section.
26890Sstevel@tonic-gate */
26900Sstevel@tonic-gate if (safe_suspend(ulwp, TSTP_MUTATOR, &link_dropped) ||
26910Sstevel@tonic-gate link_dropped) {
26920Sstevel@tonic-gate suspendingallmutators = 0;
26930Sstevel@tonic-gate goto top;
26940Sstevel@tonic-gate }
26950Sstevel@tonic-gate }
26960Sstevel@tonic-gate }
26970Sstevel@tonic-gate
26980Sstevel@tonic-gate suspendedallmutators = 1;
26990Sstevel@tonic-gate suspendingallmutators = 0;
27000Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
27010Sstevel@tonic-gate fork_lock_exit();
27020Sstevel@tonic-gate return (0);
27030Sstevel@tonic-gate }
27040Sstevel@tonic-gate
27050Sstevel@tonic-gate /*
27060Sstevel@tonic-gate * Suspend the target mutator. The caller is permitted to suspend
27070Sstevel@tonic-gate * itself. If a mutator barrier is enabled, the caller will suspend
27080Sstevel@tonic-gate * itself as though it had been suspended by thr_suspend_allmutators().
27090Sstevel@tonic-gate * When the barrier is removed, this thread will be resumed. Any
27100Sstevel@tonic-gate * suspended mutator, whether suspended by thr_suspend_mutator(), or by
27110Sstevel@tonic-gate * thr_suspend_allmutators(), can be resumed by thr_continue_mutator().
27120Sstevel@tonic-gate */
27136812Sraf #pragma weak _thr_suspend_mutator = thr_suspend_mutator
27140Sstevel@tonic-gate int
thr_suspend_mutator(thread_t tid)27156812Sraf thr_suspend_mutator(thread_t tid)
27160Sstevel@tonic-gate {
27170Sstevel@tonic-gate if (tid == 0)
27180Sstevel@tonic-gate tid = curthread->ul_lwpid;
27190Sstevel@tonic-gate return (_thrp_suspend(tid, TSTP_MUTATOR));
27200Sstevel@tonic-gate }
27210Sstevel@tonic-gate
27220Sstevel@tonic-gate /*
27230Sstevel@tonic-gate * Resume the set of all suspended mutators.
27240Sstevel@tonic-gate */
27256812Sraf #pragma weak _thr_continue_allmutators = thr_continue_allmutators
27260Sstevel@tonic-gate int
thr_continue_allmutators()27276812Sraf thr_continue_allmutators()
27280Sstevel@tonic-gate {
27290Sstevel@tonic-gate ulwp_t *self = curthread;
27300Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata;
27310Sstevel@tonic-gate ulwp_t *ulwp;
27323512Sraf
27333512Sraf /*
27343512Sraf * We single-thread the entire thread suspend/continue mechanism.
27353512Sraf */
27364843Sraf fork_lock_enter();
27370Sstevel@tonic-gate
27380Sstevel@tonic-gate lmutex_lock(&udp->link_lock);
27390Sstevel@tonic-gate if (!suspendedallmutators) {
27400Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
27413512Sraf fork_lock_exit();
27420Sstevel@tonic-gate return (EINVAL);
27430Sstevel@tonic-gate }
27440Sstevel@tonic-gate suspendedallmutators = 0;
27450Sstevel@tonic-gate
27460Sstevel@tonic-gate for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
27470Sstevel@tonic-gate mutex_t *mp = ulwp_mutex(ulwp, udp);
27480Sstevel@tonic-gate lmutex_lock(mp);
27490Sstevel@tonic-gate if (ulwp->ul_stop & TSTP_MUTATOR) {
27500Sstevel@tonic-gate ulwp->ul_stop &= ~TSTP_MUTATOR;
27510Sstevel@tonic-gate ulwp_broadcast(ulwp);
27520Sstevel@tonic-gate if (!ulwp->ul_stop)
27530Sstevel@tonic-gate force_continue(ulwp);
27540Sstevel@tonic-gate }
27550Sstevel@tonic-gate lmutex_unlock(mp);
27560Sstevel@tonic-gate }
27570Sstevel@tonic-gate
27580Sstevel@tonic-gate lmutex_unlock(&udp->link_lock);
27593512Sraf fork_lock_exit();
27600Sstevel@tonic-gate return (0);
27610Sstevel@tonic-gate }
27620Sstevel@tonic-gate
27630Sstevel@tonic-gate /*
27640Sstevel@tonic-gate * Resume a suspended mutator.
27650Sstevel@tonic-gate */
27666812Sraf #pragma weak _thr_continue_mutator = thr_continue_mutator
27670Sstevel@tonic-gate int
thr_continue_mutator(thread_t tid)27686812Sraf thr_continue_mutator(thread_t tid)
27690Sstevel@tonic-gate {
27700Sstevel@tonic-gate return (_thrp_continue(tid, TSTP_MUTATOR));
27710Sstevel@tonic-gate }
27720Sstevel@tonic-gate
27736812Sraf #pragma weak _thr_wait_mutator = thr_wait_mutator
27740Sstevel@tonic-gate int
thr_wait_mutator(thread_t tid,int dontwait)27756812Sraf thr_wait_mutator(thread_t tid, int dontwait)
27760Sstevel@tonic-gate {
27770Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata;
27780Sstevel@tonic-gate ulwp_t *ulwp;
27795891Sraf int cancel_state;
27800Sstevel@tonic-gate int error = 0;
27810Sstevel@tonic-gate
27826812Sraf (void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
27830Sstevel@tonic-gate top:
27845891Sraf if ((ulwp = find_lwp(tid)) == NULL) {
27856812Sraf (void) pthread_setcancelstate(cancel_state, NULL);
27860Sstevel@tonic-gate return (ESRCH);
27875891Sraf }
27880Sstevel@tonic-gate
27890Sstevel@tonic-gate if (!ulwp->ul_mutator)
27900Sstevel@tonic-gate error = EINVAL;
27910Sstevel@tonic-gate else if (dontwait) {
27920Sstevel@tonic-gate if (!(ulwp->ul_stop & TSTP_MUTATOR))
27930Sstevel@tonic-gate error = EWOULDBLOCK;
27940Sstevel@tonic-gate } else if (!(ulwp->ul_stop & TSTP_MUTATOR)) {
27950Sstevel@tonic-gate cond_t *cvp = ulwp_condvar(ulwp, udp);
27960Sstevel@tonic-gate mutex_t *mp = ulwp_mutex(ulwp, udp);
27970Sstevel@tonic-gate
27986812Sraf (void) cond_wait(cvp, mp);
27990Sstevel@tonic-gate (void) lmutex_unlock(mp);
28000Sstevel@tonic-gate goto top;
28010Sstevel@tonic-gate }
28020Sstevel@tonic-gate
28030Sstevel@tonic-gate ulwp_unlock(ulwp, udp);
28046812Sraf (void) pthread_setcancelstate(cancel_state, NULL);
28050Sstevel@tonic-gate return (error);
28060Sstevel@tonic-gate }
28070Sstevel@tonic-gate
28080Sstevel@tonic-gate /* PROBE_SUPPORT begin */
28090Sstevel@tonic-gate
28100Sstevel@tonic-gate void
thr_probe_setup(void * data)28110Sstevel@tonic-gate thr_probe_setup(void *data)
28120Sstevel@tonic-gate {
28130Sstevel@tonic-gate curthread->ul_tpdp = data;
28140Sstevel@tonic-gate }
28150Sstevel@tonic-gate
28160Sstevel@tonic-gate static void *
_thread_probe_getfunc()28170Sstevel@tonic-gate _thread_probe_getfunc()
28180Sstevel@tonic-gate {
28190Sstevel@tonic-gate return (curthread->ul_tpdp);
28200Sstevel@tonic-gate }
28210Sstevel@tonic-gate
28220Sstevel@tonic-gate void * (*thr_probe_getfunc_addr)(void) = _thread_probe_getfunc;
28230Sstevel@tonic-gate
28240Sstevel@tonic-gate /* ARGSUSED */
28250Sstevel@tonic-gate void
_resume(ulwp_t * ulwp,caddr_t sp,int dontsave)28260Sstevel@tonic-gate _resume(ulwp_t *ulwp, caddr_t sp, int dontsave)
28270Sstevel@tonic-gate {
28280Sstevel@tonic-gate /* never called */
28290Sstevel@tonic-gate }
28300Sstevel@tonic-gate
28310Sstevel@tonic-gate /* ARGSUSED */
28320Sstevel@tonic-gate void
_resume_ret(ulwp_t * oldlwp)28330Sstevel@tonic-gate _resume_ret(ulwp_t *oldlwp)
28340Sstevel@tonic-gate {
28350Sstevel@tonic-gate /* never called */
28360Sstevel@tonic-gate }
28370Sstevel@tonic-gate
28380Sstevel@tonic-gate /* PROBE_SUPPORT end */
2839