1 /* $NetBSD: pthread.c,v 1.170 2020/05/16 22:53:37 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2003, 2006, 2007, 2008, 2020 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Nathan J. Williams and Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __RCSID("$NetBSD: pthread.c,v 1.170 2020/05/16 22:53:37 ad Exp $"); 35 36 #define __EXPOSE_STACK 1 37 38 #include <sys/param.h> 39 #include <sys/exec_elf.h> 40 #include <sys/mman.h> 41 #include <sys/lwp.h> 42 #include <sys/lwpctl.h> 43 #include <sys/resource.h> 44 #include <sys/sysctl.h> 45 #include <sys/tls.h> 46 #include <uvm/uvm_param.h> 47 48 #include <assert.h> 49 #include <dlfcn.h> 50 #include <err.h> 51 #include <errno.h> 52 #include <lwp.h> 53 #include <signal.h> 54 #include <stdio.h> 55 #include <stdlib.h> 56 #include <stddef.h> 57 #include <string.h> 58 #include <syslog.h> 59 #include <ucontext.h> 60 #include <unistd.h> 61 #include <sched.h> 62 63 #include "atexit.h" 64 #include "pthread.h" 65 #include "pthread_int.h" 66 #include "pthread_makelwp.h" 67 #include "reentrant.h" 68 69 pthread_rwlock_t pthread__alltree_lock = PTHREAD_RWLOCK_INITIALIZER; 70 static rb_tree_t pthread__alltree; 71 72 static signed int pthread__cmp(void *, const void *, const void *); 73 74 static const rb_tree_ops_t pthread__alltree_ops = { 75 .rbto_compare_nodes = pthread__cmp, 76 .rbto_compare_key = pthread__cmp, 77 .rbto_node_offset = offsetof(struct __pthread_st, pt_alltree), 78 .rbto_context = NULL 79 }; 80 81 static void pthread__create_tramp(void *); 82 static void pthread__initthread(pthread_t); 83 static void pthread__scrubthread(pthread_t, char *, int); 84 static void pthread__initmain(pthread_t *); 85 static void pthread__reap(pthread_t); 86 87 void pthread__init(void); 88 89 int pthread__started; 90 int __uselibcstub = 1; 91 pthread_mutex_t pthread__deadqueue_lock = PTHREAD_MUTEX_INITIALIZER; 92 pthread_queue_t pthread__deadqueue; 93 pthread_queue_t pthread__allqueue; 94 95 static pthread_attr_t pthread_default_attr; 96 static lwpctl_t pthread__dummy_lwpctl = { .lc_curcpu = LWPCTL_CPU_NONE }; 97 98 enum { 99 DIAGASSERT_ABORT = 1<<0, 100 DIAGASSERT_STDERR = 1<<1, 101 DIAGASSERT_SYSLOG = 1<<2 102 }; 103 104 static int pthread__diagassert; 105 106 int pthread__concurrency; 107 int pthread__nspins; 108 size_t pthread__unpark_max = PTHREAD__UNPARK_MAX; 109 int pthread__dbg; /* set by libpthread_dbg if active */ 110 111 /* 112 * We have to initialize the pthread_stack* variables here because 113 * mutexes are used before pthread_init() and thus pthread__initmain() 114 * are called. Since mutexes only save the stack pointer and not a 115 * pointer to the thread data, it is safe to change the mapping from 116 * stack pointer to thread data afterwards. 117 */ 118 size_t pthread__stacksize; 119 size_t pthread__guardsize; 120 size_t pthread__pagesize; 121 static struct __pthread_st *pthread__main; 122 static size_t __pthread_st_size; 123 124 int _sys___sigprocmask14(int, const sigset_t *, sigset_t *); 125 126 __strong_alias(__libc_thr_self,pthread_self) 127 __strong_alias(__libc_thr_create,pthread_create) 128 __strong_alias(__libc_thr_exit,pthread_exit) 129 __strong_alias(__libc_thr_errno,pthread__errno) 130 __strong_alias(__libc_thr_setcancelstate,pthread_setcancelstate) 131 __strong_alias(__libc_thr_equal,pthread_equal) 132 __strong_alias(__libc_thr_init,pthread__init) 133 134 /* 135 * Static library kludge. Place a reference to a symbol any library 136 * file which does not already have a reference here. 137 */ 138 extern int pthread__cancel_stub_binder; 139 140 void *pthread__static_lib_binder[] = { 141 &pthread__cancel_stub_binder, 142 pthread_cond_init, 143 pthread_mutex_init, 144 pthread_rwlock_init, 145 pthread_barrier_init, 146 pthread_key_create, 147 pthread_setspecific, 148 }; 149 150 #define NHASHLOCK 64 151 152 static union hashlock { 153 pthread_mutex_t mutex; 154 char pad[64]; 155 } hashlocks[NHASHLOCK] __aligned(64); 156 157 static void 158 pthread__prefork(void) 159 { 160 pthread_mutex_lock(&pthread__deadqueue_lock); 161 } 162 163 static void 164 pthread__fork_parent(void) 165 { 166 pthread_mutex_unlock(&pthread__deadqueue_lock); 167 } 168 169 static void 170 pthread__fork_child(void) 171 { 172 struct __pthread_st *self = pthread__self(); 173 174 pthread_mutex_init(&pthread__deadqueue_lock, NULL); 175 176 /* lwpctl state is not copied across fork. */ 177 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 178 err(EXIT_FAILURE, "_lwp_ctl"); 179 } 180 self->pt_lid = _lwp_self(); 181 } 182 183 /* 184 * This needs to be started by the library loading code, before main() 185 * gets to run, for various things that use the state of the initial thread 186 * to work properly (thread-specific data is an application-visible example; 187 * spinlock counts for mutexes is an internal example). 188 */ 189 void 190 pthread__init(void) 191 { 192 pthread_t first; 193 char *p; 194 int mib[2]; 195 unsigned int value; 196 ssize_t slen; 197 size_t len; 198 extern int __isthreaded; 199 200 /* 201 * Allocate pthread_keys descriptors before 202 * reseting __uselibcstub because otherwise 203 * malloc() will call pthread_keys_create() 204 * while pthread_keys descriptors are not 205 * yet allocated. 206 */ 207 pthread__main = pthread_tsd_init(&__pthread_st_size); 208 if (pthread__main == NULL) 209 err(EXIT_FAILURE, "Cannot allocate pthread storage"); 210 211 __uselibcstub = 0; 212 213 pthread__pagesize = (size_t)sysconf(_SC_PAGESIZE); 214 pthread__concurrency = (int)sysconf(_SC_NPROCESSORS_CONF); 215 216 mib[0] = CTL_VM; 217 mib[1] = VM_THREAD_GUARD_SIZE; 218 len = sizeof(value); 219 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 220 pthread__guardsize = value; 221 else 222 pthread__guardsize = pthread__pagesize; 223 224 /* Initialize locks first; they're needed elsewhere. */ 225 pthread__lockprim_init(); 226 for (int i = 0; i < NHASHLOCK; i++) { 227 pthread_mutex_init(&hashlocks[i].mutex, NULL); 228 } 229 230 /* Fetch parameters. */ 231 slen = _lwp_unpark_all(NULL, 0, NULL); 232 if (slen < 0) 233 err(EXIT_FAILURE, "_lwp_unpark_all"); 234 if ((size_t)slen < pthread__unpark_max) 235 pthread__unpark_max = slen; 236 237 /* Basic data structure setup */ 238 pthread_attr_init(&pthread_default_attr); 239 PTQ_INIT(&pthread__allqueue); 240 PTQ_INIT(&pthread__deadqueue); 241 242 rb_tree_init(&pthread__alltree, &pthread__alltree_ops); 243 244 /* Create the thread structure corresponding to main() */ 245 pthread__initmain(&first); 246 pthread__initthread(first); 247 pthread__scrubthread(first, NULL, 0); 248 249 first->pt_lid = _lwp_self(); 250 PTQ_INSERT_HEAD(&pthread__allqueue, first, pt_allq); 251 (void)rb_tree_insert_node(&pthread__alltree, first); 252 253 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &first->pt_lwpctl) != 0) { 254 err(EXIT_FAILURE, "_lwp_ctl"); 255 } 256 257 /* Start subsystems */ 258 PTHREAD_MD_INIT 259 260 for (p = pthread__getenv("PTHREAD_DIAGASSERT"); p && *p; p++) { 261 switch (*p) { 262 case 'a': 263 pthread__diagassert |= DIAGASSERT_ABORT; 264 break; 265 case 'A': 266 pthread__diagassert &= ~DIAGASSERT_ABORT; 267 break; 268 case 'e': 269 pthread__diagassert |= DIAGASSERT_STDERR; 270 break; 271 case 'E': 272 pthread__diagassert &= ~DIAGASSERT_STDERR; 273 break; 274 case 'l': 275 pthread__diagassert |= DIAGASSERT_SYSLOG; 276 break; 277 case 'L': 278 pthread__diagassert &= ~DIAGASSERT_SYSLOG; 279 break; 280 } 281 } 282 283 /* Tell libc that we're here and it should role-play accordingly. */ 284 pthread_atfork(pthread__prefork, pthread__fork_parent, pthread__fork_child); 285 __isthreaded = 1; 286 } 287 288 /* General-purpose thread data structure sanitization. */ 289 /* ARGSUSED */ 290 static void 291 pthread__initthread(pthread_t t) 292 { 293 294 t->pt_self = t; 295 t->pt_magic = PT_MAGIC; 296 t->pt_willpark = 0; 297 t->pt_unpark = 0; 298 t->pt_nwaiters = 0; 299 t->pt_sleepobj = NULL; 300 t->pt_signalled = 0; 301 t->pt_havespecific = 0; 302 t->pt_early = NULL; 303 t->pt_lwpctl = &pthread__dummy_lwpctl; 304 305 memcpy(&t->pt_lockops, pthread__lock_ops, sizeof(t->pt_lockops)); 306 pthread_mutex_init(&t->pt_lock, NULL); 307 PTQ_INIT(&t->pt_cleanup_stack); 308 } 309 310 static void 311 pthread__scrubthread(pthread_t t, char *name, int flags) 312 { 313 314 t->pt_state = PT_STATE_RUNNING; 315 t->pt_exitval = NULL; 316 t->pt_flags = flags; 317 t->pt_cancel = 0; 318 t->pt_errno = 0; 319 t->pt_name = name; 320 t->pt_lid = 0; 321 } 322 323 static int 324 pthread__getstack(pthread_t newthread, const pthread_attr_t *attr) 325 { 326 void *stackbase, *stackbase2, *redzone; 327 size_t stacksize, guardsize; 328 bool allocated; 329 330 if (attr != NULL) { 331 pthread_attr_getstack(attr, &stackbase, &stacksize); 332 pthread_attr_getguardsize(attr, &guardsize); 333 } else { 334 stackbase = NULL; 335 stacksize = 0; 336 guardsize = pthread__guardsize; 337 } 338 if (stacksize == 0) 339 stacksize = pthread__stacksize; 340 341 if (newthread->pt_stack_allocated) { 342 if (stackbase == NULL && 343 newthread->pt_stack.ss_size == stacksize && 344 newthread->pt_guardsize == guardsize) 345 return 0; 346 stackbase2 = newthread->pt_stack.ss_sp; 347 #ifndef __MACHINE_STACK_GROWS_UP 348 stackbase2 = (char *)stackbase2 - newthread->pt_guardsize; 349 #endif 350 munmap(stackbase2, 351 newthread->pt_stack.ss_size + newthread->pt_guardsize); 352 newthread->pt_stack.ss_sp = NULL; 353 newthread->pt_stack.ss_size = 0; 354 newthread->pt_guardsize = 0; 355 newthread->pt_stack_allocated = false; 356 } 357 358 newthread->pt_stack_allocated = false; 359 360 if (stackbase == NULL) { 361 stacksize = ((stacksize - 1) | (pthread__pagesize - 1)) + 1; 362 guardsize = ((guardsize - 1) | (pthread__pagesize - 1)) + 1; 363 stackbase = mmap(NULL, stacksize + guardsize, 364 PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, (off_t)0); 365 if (stackbase == MAP_FAILED) 366 return ENOMEM; 367 allocated = true; 368 } else { 369 allocated = false; 370 } 371 #ifdef __MACHINE_STACK_GROWS_UP 372 redzone = (char *)stackbase + stacksize; 373 stackbase2 = (char *)stackbase; 374 #else 375 redzone = (char *)stackbase; 376 stackbase2 = (char *)stackbase + guardsize; 377 #endif 378 if (allocated && guardsize && 379 mprotect(redzone, guardsize, PROT_NONE) == -1) { 380 munmap(stackbase, stacksize + guardsize); 381 return EPERM; 382 } 383 newthread->pt_stack.ss_size = stacksize; 384 newthread->pt_stack.ss_sp = stackbase2; 385 newthread->pt_guardsize = guardsize; 386 newthread->pt_stack_allocated = allocated; 387 return 0; 388 } 389 390 int 391 pthread_create(pthread_t *thread, const pthread_attr_t *attr, 392 void *(*startfunc)(void *), void *arg) 393 { 394 pthread_t newthread; 395 pthread_attr_t nattr; 396 struct pthread_attr_private *p; 397 char * volatile name; 398 unsigned long flag; 399 void *private_area; 400 int ret; 401 402 if (__predict_false(__uselibcstub)) { 403 pthread__errorfunc(__FILE__, __LINE__, __func__, 404 "pthread_create() requires linking with -lpthread"); 405 return __libc_thr_create_stub(thread, attr, startfunc, arg); 406 } 407 408 if (attr == NULL) 409 nattr = pthread_default_attr; 410 else if (attr->pta_magic == PT_ATTR_MAGIC) 411 nattr = *attr; 412 else 413 return EINVAL; 414 415 pthread__started = 1; 416 417 /* Fetch misc. attributes from the attr structure. */ 418 name = NULL; 419 if ((p = nattr.pta_private) != NULL) 420 if (p->ptap_name[0] != '\0') 421 if ((name = strdup(p->ptap_name)) == NULL) 422 return ENOMEM; 423 424 newthread = NULL; 425 426 /* 427 * Try to reclaim a dead thread. 428 */ 429 if (!PTQ_EMPTY(&pthread__deadqueue)) { 430 pthread_mutex_lock(&pthread__deadqueue_lock); 431 PTQ_FOREACH(newthread, &pthread__deadqueue, pt_deadq) { 432 /* Still busily exiting, or finished? */ 433 if (newthread->pt_lwpctl->lc_curcpu == 434 LWPCTL_CPU_EXITED) 435 break; 436 } 437 if (newthread) 438 PTQ_REMOVE(&pthread__deadqueue, newthread, pt_deadq); 439 pthread_mutex_unlock(&pthread__deadqueue_lock); 440 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 441 if (newthread && newthread->pt_tls) { 442 _rtld_tls_free(newthread->pt_tls); 443 newthread->pt_tls = NULL; 444 } 445 #endif 446 } 447 448 /* 449 * If necessary set up a stack, allocate space for a pthread_st, 450 * and initialize it. 451 */ 452 if (newthread == NULL) { 453 newthread = calloc(1, __pthread_st_size); 454 if (newthread == NULL) { 455 free(name); 456 return ENOMEM; 457 } 458 newthread->pt_stack_allocated = false; 459 460 if (pthread__getstack(newthread, attr)) { 461 free(newthread); 462 free(name); 463 return ENOMEM; 464 } 465 466 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 467 newthread->pt_tls = NULL; 468 #endif 469 470 /* Add to list of all threads. */ 471 pthread_rwlock_wrlock(&pthread__alltree_lock); 472 PTQ_INSERT_TAIL(&pthread__allqueue, newthread, pt_allq); 473 (void)rb_tree_insert_node(&pthread__alltree, newthread); 474 pthread_rwlock_unlock(&pthread__alltree_lock); 475 476 /* Will be reset by the thread upon exit. */ 477 pthread__initthread(newthread); 478 } else { 479 if (pthread__getstack(newthread, attr)) { 480 pthread_mutex_lock(&pthread__deadqueue_lock); 481 PTQ_INSERT_TAIL(&pthread__deadqueue, newthread, pt_deadq); 482 pthread_mutex_unlock(&pthread__deadqueue_lock); 483 return ENOMEM; 484 } 485 } 486 487 /* 488 * Create the new LWP. 489 */ 490 pthread__scrubthread(newthread, name, nattr.pta_flags); 491 newthread->pt_func = startfunc; 492 newthread->pt_arg = arg; 493 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 494 private_area = newthread->pt_tls = _rtld_tls_allocate(); 495 newthread->pt_tls->tcb_pthread = newthread; 496 #else 497 private_area = newthread; 498 #endif 499 500 flag = 0; 501 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) != 0 || 502 (nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) 503 flag |= LWP_SUSPENDED; 504 if ((newthread->pt_flags & PT_FLAG_DETACHED) != 0) 505 flag |= LWP_DETACHED; 506 507 ret = pthread__makelwp(pthread__create_tramp, newthread, private_area, 508 newthread->pt_stack.ss_sp, newthread->pt_stack.ss_size, 509 flag, &newthread->pt_lid); 510 if (ret != 0) { 511 ret = errno; 512 pthread_mutex_lock(&newthread->pt_lock); 513 /* Will unlock and free name. */ 514 pthread__reap(newthread); 515 return ret; 516 } 517 518 if ((nattr.pta_flags & PT_FLAG_EXPLICIT_SCHED) != 0) { 519 if (p != NULL) { 520 (void)pthread_setschedparam(newthread, p->ptap_policy, 521 &p->ptap_sp); 522 } 523 if ((newthread->pt_flags & PT_FLAG_SUSPENDED) == 0) { 524 (void)_lwp_continue(newthread->pt_lid); 525 } 526 } 527 528 *thread = newthread; 529 530 return 0; 531 } 532 533 534 __dead static void 535 pthread__create_tramp(void *cookie) 536 { 537 pthread_t self; 538 void *retval; 539 540 self = cookie; 541 542 /* 543 * Throw away some stack in a feeble attempt to reduce cache 544 * thrash. May help for SMT processors. XXX We should not 545 * be allocating stacks on fixed 2MB boundaries. Needs a 546 * thread register or decent thread local storage. 547 */ 548 (void)alloca(((unsigned)self->pt_lid & 7) << 8); 549 550 if (self->pt_name != NULL) { 551 pthread_mutex_lock(&self->pt_lock); 552 if (self->pt_name != NULL) 553 (void)_lwp_setname(0, self->pt_name); 554 pthread_mutex_unlock(&self->pt_lock); 555 } 556 557 if (_lwp_ctl(LWPCTL_FEATURE_CURCPU, &self->pt_lwpctl)) { 558 err(EXIT_FAILURE, "_lwp_ctl"); 559 } 560 561 retval = (*self->pt_func)(self->pt_arg); 562 563 pthread_exit(retval); 564 565 /*NOTREACHED*/ 566 pthread__abort(); 567 } 568 569 int 570 pthread_suspend_np(pthread_t thread) 571 { 572 pthread_t self; 573 574 pthread__error(EINVAL, "Invalid thread", 575 thread->pt_magic == PT_MAGIC); 576 577 self = pthread__self(); 578 if (self == thread) { 579 return EDEADLK; 580 } 581 if (pthread__find(thread) != 0) 582 return ESRCH; 583 if (_lwp_suspend(thread->pt_lid) == 0) 584 return 0; 585 return errno; 586 } 587 588 int 589 pthread_resume_np(pthread_t thread) 590 { 591 592 pthread__error(EINVAL, "Invalid thread", 593 thread->pt_magic == PT_MAGIC); 594 595 if (pthread__find(thread) != 0) 596 return ESRCH; 597 if (_lwp_continue(thread->pt_lid) == 0) 598 return 0; 599 return errno; 600 } 601 602 /* 603 * In case the thread is exiting at an inopportune time leaving waiters not 604 * awoken (because cancelled, for instance) make sure we have no waiters 605 * left. 606 */ 607 void 608 pthread__clear_waiters(pthread_t self) 609 { 610 int rv; 611 612 /* Zero waiters or one waiter in error case (pthread_exit()). */ 613 if (self->pt_nwaiters == 0) { 614 if (self->pt_unpark != 0 && self->pt_willpark == 0) { 615 rv = (ssize_t)_lwp_unpark(self->pt_unpark, NULL); 616 self->pt_unpark = 0; 617 if (rv != 0 && errno != EALREADY && errno != EINTR && 618 errno != ESRCH) { 619 pthread__errorfunc(__FILE__, __LINE__, __func__, 620 "_lwp_unpark failed"); 621 } 622 } 623 return; 624 } 625 626 /* One waiter or two waiters (the second being a deferred wakeup). */ 627 if (self->pt_nwaiters == 1) { 628 if (self->pt_unpark != 0) { 629 /* Fall through to multiple waiters case. */ 630 self->pt_waiters[1] = self->pt_unpark; 631 self->pt_nwaiters = 2; 632 self->pt_unpark = 0; 633 } else if (self->pt_willpark) { 634 /* Defer to _lwp_park(). */ 635 self->pt_unpark = self->pt_waiters[0]; 636 self->pt_nwaiters = 0; 637 return; 638 } else { 639 /* Wake one now. */ 640 rv = (ssize_t)_lwp_unpark(self->pt_waiters[0], NULL); 641 self->pt_nwaiters = 0; 642 if (rv != 0 && errno != EALREADY && errno != EINTR && 643 errno != ESRCH) { 644 pthread__errorfunc(__FILE__, __LINE__, __func__, 645 "_lwp_unpark failed"); 646 } 647 return; 648 } 649 } 650 651 /* Multiple waiters. */ 652 rv = _lwp_unpark_all(self->pt_waiters, self->pt_nwaiters, NULL); 653 self->pt_nwaiters = 0; 654 if (rv != 0 && errno != EINTR) { 655 pthread__errorfunc(__FILE__, __LINE__, __func__, 656 "_lwp_unpark_all failed"); 657 } 658 } 659 660 void 661 pthread_exit(void *retval) 662 { 663 pthread_t self; 664 struct pt_clean_t *cleanup; 665 666 if (__predict_false(__uselibcstub)) { 667 __libc_thr_exit_stub(retval); 668 goto out; 669 } 670 671 self = pthread__self(); 672 673 /* Disable cancellability. */ 674 self->pt_willpark = 0; 675 pthread_mutex_lock(&self->pt_lock); 676 self->pt_flags |= PT_FLAG_CS_DISABLED; 677 self->pt_cancel = 0; 678 679 /* Call any cancellation cleanup handlers */ 680 if (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 681 pthread_mutex_unlock(&self->pt_lock); 682 while (!PTQ_EMPTY(&self->pt_cleanup_stack)) { 683 cleanup = PTQ_FIRST(&self->pt_cleanup_stack); 684 PTQ_REMOVE(&self->pt_cleanup_stack, cleanup, ptc_next); 685 (*cleanup->ptc_cleanup)(cleanup->ptc_arg); 686 } 687 pthread_mutex_lock(&self->pt_lock); 688 } 689 690 pthread_mutex_unlock(&self->pt_lock); 691 __cxa_thread_run_atexit(); 692 pthread_mutex_lock(&self->pt_lock); 693 694 /* Perform cleanup of thread-specific data */ 695 pthread__destroy_tsd(self); 696 697 /* 698 * Signal our exit. Our stack and pthread_t won't be reused until 699 * pthread_create() can see from kernel info that this LWP is gone. 700 */ 701 self->pt_exitval = retval; 702 if (self->pt_flags & PT_FLAG_DETACHED) { 703 /* pthread__reap() will drop the lock. */ 704 pthread__reap(self); 705 pthread__assert(!self->pt_willpark); 706 pthread__clear_waiters(self); 707 _lwp_exit(); 708 } else { 709 self->pt_state = PT_STATE_ZOMBIE; 710 pthread__assert(!self->pt_willpark); 711 pthread_mutex_unlock(&self->pt_lock); 712 pthread__clear_waiters(self); 713 /* Note: name will be freed by the joiner. */ 714 _lwp_exit(); 715 } 716 717 out: 718 /*NOTREACHED*/ 719 pthread__abort(); 720 exit(1); 721 } 722 723 724 int 725 pthread_join(pthread_t thread, void **valptr) 726 { 727 pthread_t self; 728 729 pthread__error(EINVAL, "Invalid thread", 730 thread->pt_magic == PT_MAGIC); 731 732 self = pthread__self(); 733 734 if (pthread__find(thread) != 0) 735 return ESRCH; 736 737 if (thread == self) 738 return EDEADLK; 739 740 /* IEEE Std 1003.1 says pthread_join() never returns EINTR. */ 741 for (;;) { 742 pthread__testcancel(self); 743 if (_lwp_wait(thread->pt_lid, NULL) == 0) 744 break; 745 if (errno != EINTR) 746 return errno; 747 } 748 749 /* 750 * Don't test for cancellation again. The spec is that if 751 * cancelled, pthread_join() must not have succeeded. 752 */ 753 pthread_mutex_lock(&thread->pt_lock); 754 if (thread->pt_state != PT_STATE_ZOMBIE) { 755 pthread__errorfunc(__FILE__, __LINE__, __func__, 756 "not a zombie"); 757 } 758 if (valptr != NULL) 759 *valptr = thread->pt_exitval; 760 761 /* pthread__reap() will drop the lock. */ 762 pthread__reap(thread); 763 return 0; 764 } 765 766 static void 767 pthread__reap(pthread_t thread) 768 { 769 char *name; 770 771 name = thread->pt_name; 772 thread->pt_name = NULL; 773 thread->pt_state = PT_STATE_DEAD; 774 pthread_mutex_unlock(&thread->pt_lock); 775 776 pthread_mutex_lock(&pthread__deadqueue_lock); 777 PTQ_INSERT_HEAD(&pthread__deadqueue, thread, pt_deadq); 778 pthread_mutex_unlock(&pthread__deadqueue_lock); 779 780 if (name != NULL) 781 free(name); 782 } 783 784 int 785 pthread_equal(pthread_t t1, pthread_t t2) 786 { 787 788 if (__predict_false(__uselibcstub)) 789 return __libc_thr_equal_stub(t1, t2); 790 791 pthread__error(0, "Invalid thread", 792 (t1 != NULL) && (t1->pt_magic == PT_MAGIC)); 793 794 pthread__error(0, "Invalid thread", 795 (t2 != NULL) && (t2->pt_magic == PT_MAGIC)); 796 797 /* Nothing special here. */ 798 return (t1 == t2); 799 } 800 801 802 int 803 pthread_detach(pthread_t thread) 804 { 805 int error; 806 807 pthread__error(EINVAL, "Invalid thread", 808 thread->pt_magic == PT_MAGIC); 809 810 if (pthread__find(thread) != 0) 811 return ESRCH; 812 813 pthread_mutex_lock(&thread->pt_lock); 814 if ((thread->pt_flags & PT_FLAG_DETACHED) != 0) { 815 error = EINVAL; 816 } else { 817 error = _lwp_detach(thread->pt_lid); 818 if (error == 0) 819 thread->pt_flags |= PT_FLAG_DETACHED; 820 else 821 error = errno; 822 } 823 if (thread->pt_state == PT_STATE_ZOMBIE) { 824 /* pthread__reap() will drop the lock. */ 825 pthread__reap(thread); 826 } else 827 pthread_mutex_unlock(&thread->pt_lock); 828 return error; 829 } 830 831 832 int 833 pthread_getname_np(pthread_t thread, char *name, size_t len) 834 { 835 836 pthread__error(EINVAL, "Invalid thread", 837 thread->pt_magic == PT_MAGIC); 838 839 if (pthread__find(thread) != 0) 840 return ESRCH; 841 842 pthread_mutex_lock(&thread->pt_lock); 843 if (thread->pt_name == NULL) 844 name[0] = '\0'; 845 else 846 strlcpy(name, thread->pt_name, len); 847 pthread_mutex_unlock(&thread->pt_lock); 848 849 return 0; 850 } 851 852 853 int 854 pthread_setname_np(pthread_t thread, const char *name, void *arg) 855 { 856 char *oldname, *cp, newname[PTHREAD_MAX_NAMELEN_NP]; 857 int namelen; 858 859 pthread__error(EINVAL, "Invalid thread", 860 thread->pt_magic == PT_MAGIC); 861 862 if (pthread__find(thread) != 0) 863 return ESRCH; 864 865 namelen = snprintf(newname, sizeof(newname), name, arg); 866 if (namelen >= PTHREAD_MAX_NAMELEN_NP) 867 return EINVAL; 868 869 cp = strdup(newname); 870 if (cp == NULL) 871 return ENOMEM; 872 873 pthread_mutex_lock(&thread->pt_lock); 874 oldname = thread->pt_name; 875 thread->pt_name = cp; 876 (void)_lwp_setname(thread->pt_lid, cp); 877 pthread_mutex_unlock(&thread->pt_lock); 878 879 if (oldname != NULL) 880 free(oldname); 881 882 return 0; 883 } 884 885 886 pthread_t 887 pthread_self(void) 888 { 889 if (__predict_false(__uselibcstub)) 890 return (pthread_t)__libc_thr_self_stub(); 891 892 return pthread__self(); 893 } 894 895 896 int 897 pthread_cancel(pthread_t thread) 898 { 899 900 pthread__error(EINVAL, "Invalid thread", 901 thread->pt_magic == PT_MAGIC); 902 903 if (pthread__find(thread) != 0) 904 return ESRCH; 905 pthread_mutex_lock(&thread->pt_lock); 906 thread->pt_flags |= PT_FLAG_CS_PENDING; 907 if ((thread->pt_flags & PT_FLAG_CS_DISABLED) == 0) { 908 thread->pt_cancel = 1; 909 pthread_mutex_unlock(&thread->pt_lock); 910 _lwp_wakeup(thread->pt_lid); 911 } else 912 pthread_mutex_unlock(&thread->pt_lock); 913 914 return 0; 915 } 916 917 918 int 919 pthread_setcancelstate(int state, int *oldstate) 920 { 921 pthread_t self; 922 int retval; 923 924 if (__predict_false(__uselibcstub)) 925 return __libc_thr_setcancelstate_stub(state, oldstate); 926 927 self = pthread__self(); 928 retval = 0; 929 930 pthread_mutex_lock(&self->pt_lock); 931 932 if (oldstate != NULL) { 933 if (self->pt_flags & PT_FLAG_CS_DISABLED) 934 *oldstate = PTHREAD_CANCEL_DISABLE; 935 else 936 *oldstate = PTHREAD_CANCEL_ENABLE; 937 } 938 939 if (state == PTHREAD_CANCEL_DISABLE) { 940 self->pt_flags |= PT_FLAG_CS_DISABLED; 941 if (self->pt_cancel) { 942 self->pt_flags |= PT_FLAG_CS_PENDING; 943 self->pt_cancel = 0; 944 } 945 } else if (state == PTHREAD_CANCEL_ENABLE) { 946 self->pt_flags &= ~PT_FLAG_CS_DISABLED; 947 /* 948 * If a cancellation was requested while cancellation 949 * was disabled, note that fact for future 950 * cancellation tests. 951 */ 952 if (self->pt_flags & PT_FLAG_CS_PENDING) { 953 self->pt_cancel = 1; 954 /* This is not a deferred cancellation point. */ 955 if (self->pt_flags & PT_FLAG_CS_ASYNC) { 956 pthread_mutex_unlock(&self->pt_lock); 957 pthread__cancelled(); 958 } 959 } 960 } else 961 retval = EINVAL; 962 963 pthread_mutex_unlock(&self->pt_lock); 964 965 return retval; 966 } 967 968 969 int 970 pthread_setcanceltype(int type, int *oldtype) 971 { 972 pthread_t self; 973 int retval; 974 975 self = pthread__self(); 976 retval = 0; 977 978 pthread_mutex_lock(&self->pt_lock); 979 980 if (oldtype != NULL) { 981 if (self->pt_flags & PT_FLAG_CS_ASYNC) 982 *oldtype = PTHREAD_CANCEL_ASYNCHRONOUS; 983 else 984 *oldtype = PTHREAD_CANCEL_DEFERRED; 985 } 986 987 if (type == PTHREAD_CANCEL_ASYNCHRONOUS) { 988 self->pt_flags |= PT_FLAG_CS_ASYNC; 989 if (self->pt_cancel) { 990 pthread_mutex_unlock(&self->pt_lock); 991 pthread__cancelled(); 992 } 993 } else if (type == PTHREAD_CANCEL_DEFERRED) 994 self->pt_flags &= ~PT_FLAG_CS_ASYNC; 995 else 996 retval = EINVAL; 997 998 pthread_mutex_unlock(&self->pt_lock); 999 1000 return retval; 1001 } 1002 1003 1004 void 1005 pthread_testcancel(void) 1006 { 1007 pthread_t self; 1008 1009 self = pthread__self(); 1010 if (self->pt_cancel) 1011 pthread__cancelled(); 1012 } 1013 1014 1015 /* 1016 * POSIX requires that certain functions return an error rather than 1017 * invoking undefined behavior even when handed completely bogus 1018 * pthread_t values, e.g. stack garbage. 1019 */ 1020 int 1021 pthread__find(pthread_t id) 1022 { 1023 pthread_t target; 1024 int error; 1025 1026 pthread_rwlock_rdlock(&pthread__alltree_lock); 1027 target = rb_tree_find_node(&pthread__alltree, id); 1028 error = (target && target->pt_state != PT_STATE_DEAD) ? 0 : ESRCH; 1029 pthread_rwlock_unlock(&pthread__alltree_lock); 1030 1031 return error; 1032 } 1033 1034 1035 void 1036 pthread__testcancel(pthread_t self) 1037 { 1038 1039 if (self->pt_cancel) 1040 pthread__cancelled(); 1041 } 1042 1043 1044 void 1045 pthread__cancelled(void) 1046 { 1047 1048 pthread_exit(PTHREAD_CANCELED); 1049 } 1050 1051 1052 void 1053 pthread__cleanup_push(void (*cleanup)(void *), void *arg, void *store) 1054 { 1055 pthread_t self; 1056 struct pt_clean_t *entry; 1057 1058 self = pthread__self(); 1059 entry = store; 1060 entry->ptc_cleanup = cleanup; 1061 entry->ptc_arg = arg; 1062 PTQ_INSERT_HEAD(&self->pt_cleanup_stack, entry, ptc_next); 1063 } 1064 1065 1066 void 1067 pthread__cleanup_pop(int ex, void *store) 1068 { 1069 pthread_t self; 1070 struct pt_clean_t *entry; 1071 1072 self = pthread__self(); 1073 entry = store; 1074 1075 PTQ_REMOVE(&self->pt_cleanup_stack, entry, ptc_next); 1076 if (ex) 1077 (*entry->ptc_cleanup)(entry->ptc_arg); 1078 } 1079 1080 1081 int * 1082 pthread__errno(void) 1083 { 1084 pthread_t self; 1085 1086 if (__predict_false(__uselibcstub)) { 1087 pthread__errorfunc(__FILE__, __LINE__, __func__, 1088 "pthread__errno() requires linking with -lpthread"); 1089 return __libc_thr_errno_stub(); 1090 } 1091 1092 self = pthread__self(); 1093 1094 return &(self->pt_errno); 1095 } 1096 1097 ssize_t _sys_write(int, const void *, size_t); 1098 1099 void 1100 pthread__assertfunc(const char *file, int line, const char *function, 1101 const char *expr) 1102 { 1103 char buf[1024]; 1104 int len; 1105 1106 /* 1107 * snprintf should not acquire any locks, or we could 1108 * end up deadlocked if the assert caller held locks. 1109 */ 1110 len = snprintf(buf, 1024, 1111 "assertion \"%s\" failed: file \"%s\", line %d%s%s%s\n", 1112 expr, file, line, 1113 function ? ", function \"" : "", 1114 function ? function : "", 1115 function ? "\"" : ""); 1116 1117 _sys_write(STDERR_FILENO, buf, (size_t)len); 1118 (void)_lwp_kill(_lwp_self(), SIGABRT); 1119 _exit(1); 1120 } 1121 1122 1123 void 1124 pthread__errorfunc(const char *file, int line, const char *function, 1125 const char *msg) 1126 { 1127 char buf[1024]; 1128 size_t len; 1129 1130 if (pthread__diagassert == 0) 1131 return; 1132 1133 /* 1134 * snprintf should not acquire any locks, or we could 1135 * end up deadlocked if the assert caller held locks. 1136 */ 1137 len = snprintf(buf, 1024, 1138 "%s: Error detected by libpthread: %s.\n" 1139 "Detected by file \"%s\", line %d%s%s%s.\n" 1140 "See pthread(3) for information.\n", 1141 getprogname(), msg, file, line, 1142 function ? ", function \"" : "", 1143 function ? function : "", 1144 function ? "\"" : ""); 1145 1146 if (pthread__diagassert & DIAGASSERT_STDERR) 1147 _sys_write(STDERR_FILENO, buf, len); 1148 1149 if (pthread__diagassert & DIAGASSERT_SYSLOG) 1150 syslog(LOG_DEBUG | LOG_USER, "%s", buf); 1151 1152 if (pthread__diagassert & DIAGASSERT_ABORT) { 1153 (void)_lwp_kill(_lwp_self(), SIGABRT); 1154 _exit(1); 1155 } 1156 } 1157 1158 /* 1159 * Thread park/unpark operations. The kernel operations are 1160 * modelled after a brief description from "Multithreading in 1161 * the Solaris Operating Environment": 1162 * 1163 * http://www.sun.com/software/whitepapers/solaris9/multithread.pdf 1164 */ 1165 1166 #define OOPS(msg) \ 1167 pthread__errorfunc(__FILE__, __LINE__, __func__, msg) 1168 1169 int 1170 pthread__park(pthread_t self, pthread_mutex_t *lock, 1171 pthread_queue_t *queue, const struct timespec *abstime, 1172 int cancelpt) 1173 { 1174 int rv, error; 1175 void *obj; 1176 1177 self->pt_willpark = 1; 1178 pthread_mutex_unlock(lock); 1179 self->pt_willpark = 0; 1180 1181 /* 1182 * Wait until we are awoken by a pending unpark operation, 1183 * a signal, an unpark posted after we have gone asleep, 1184 * or an expired timeout. 1185 * 1186 * It is fine to test the value of pt_sleepobj without 1187 * holding any locks, because: 1188 * 1189 * o Only the blocking thread (this thread) ever sets them 1190 * to a non-NULL value. 1191 * 1192 * o Other threads may set them NULL, but if they do so they 1193 * must also make this thread return from _lwp_park. 1194 * 1195 * o _lwp_park, _lwp_unpark and _lwp_unpark_all are system 1196 * calls and all make use of spinlocks in the kernel. So 1197 * these system calls act as full memory barriers, and will 1198 * ensure that the calling CPU's store buffers are drained. 1199 * In combination with the spinlock release before unpark, 1200 * this means that modification of pt_sleepobj/onq by another 1201 * thread will become globally visible before that thread 1202 * schedules an unpark operation on this thread. 1203 * 1204 * Note: the test in the while() statement dodges the park op if 1205 * we have already been awoken, unless there is another thread to 1206 * awaken. This saves a syscall - if we were already awakened, 1207 * the next call to _lwp_park() would need to return early in order 1208 * to eat the previous wakeup. 1209 */ 1210 rv = 0; 1211 do { 1212 /* 1213 * If we deferred unparking a thread, arrange to 1214 * have _lwp_park() restart it before blocking. 1215 */ 1216 error = _lwp_park(CLOCK_REALTIME, TIMER_ABSTIME, 1217 __UNCONST(abstime), self->pt_unpark, NULL, NULL); 1218 self->pt_unpark = 0; 1219 if (error != 0) { 1220 switch (rv = errno) { 1221 case EINTR: 1222 case EALREADY: 1223 rv = 0; 1224 break; 1225 case ETIMEDOUT: 1226 break; 1227 default: 1228 OOPS("_lwp_park failed"); 1229 break; 1230 } 1231 } 1232 /* Check for cancellation. */ 1233 if (cancelpt && self->pt_cancel) 1234 rv = EINTR; 1235 } while (self->pt_sleepobj != NULL && rv == 0); 1236 1237 /* 1238 * If we have been awoken early but are still on the queue, 1239 * then remove ourself. Again, it's safe to do the test 1240 * without holding any locks. 1241 */ 1242 if (__predict_false(self->pt_sleepobj != NULL)) { 1243 pthread_mutex_lock(lock); 1244 if ((obj = self->pt_sleepobj) != NULL) { 1245 PTQ_REMOVE(queue, self, pt_sleep); 1246 self->pt_sleepobj = NULL; 1247 if (obj != NULL && self->pt_early != NULL) 1248 (*self->pt_early)(obj); 1249 } 1250 pthread_mutex_unlock(lock); 1251 } 1252 self->pt_early = NULL; 1253 1254 return rv; 1255 } 1256 1257 void 1258 pthread__unpark(pthread_queue_t *queue, pthread_t self, 1259 pthread_mutex_t *interlock) 1260 { 1261 pthread_t target; 1262 1263 target = PTQ_FIRST(queue); 1264 if (self->pt_nwaiters == pthread__unpark_max) { 1265 pthread__clear_waiters(self); 1266 } 1267 target->pt_sleepobj = NULL; 1268 self->pt_waiters[self->pt_nwaiters++] = target->pt_lid; 1269 PTQ_REMOVE(queue, target, pt_sleep); 1270 pthread__mutex_deferwake(self, interlock); 1271 } 1272 1273 void 1274 pthread__unpark_all(pthread_queue_t *queue, pthread_t self, 1275 pthread_mutex_t *interlock) 1276 { 1277 const size_t max = pthread__unpark_max; 1278 pthread_t target; 1279 1280 PTQ_FOREACH(target, queue, pt_sleep) { 1281 if (self->pt_nwaiters == max) { 1282 pthread__clear_waiters(self); 1283 } 1284 target->pt_sleepobj = NULL; 1285 self->pt_waiters[self->pt_nwaiters++] = target->pt_lid; 1286 } 1287 PTQ_INIT(queue); 1288 pthread__mutex_deferwake(self, interlock); 1289 } 1290 1291 #undef OOPS 1292 1293 static void 1294 pthread__initmainstack(void) 1295 { 1296 struct rlimit slimit; 1297 const AuxInfo *aux; 1298 size_t size, len; 1299 int mib[2]; 1300 unsigned int value; 1301 1302 _DIAGASSERT(_dlauxinfo() != NULL); 1303 1304 if (getrlimit(RLIMIT_STACK, &slimit) == -1) 1305 err(EXIT_FAILURE, 1306 "Couldn't get stack resource consumption limits"); 1307 size = slimit.rlim_cur; 1308 pthread__main->pt_stack.ss_size = size; 1309 pthread__main->pt_guardsize = pthread__pagesize; 1310 1311 mib[0] = CTL_VM; 1312 mib[1] = VM_GUARD_SIZE; 1313 len = sizeof(value); 1314 if (sysctl(mib, __arraycount(mib), &value, &len, NULL, 0) == 0) 1315 pthread__main->pt_guardsize = value; 1316 1317 for (aux = _dlauxinfo(); aux->a_type != AT_NULL; ++aux) { 1318 if (aux->a_type == AT_STACKBASE) { 1319 #ifdef __MACHINE_STACK_GROWS_UP 1320 pthread__main->pt_stack.ss_sp = (void *)aux->a_v; 1321 #else 1322 pthread__main->pt_stack.ss_sp = (char *)aux->a_v - size; 1323 #endif 1324 break; 1325 } 1326 } 1327 pthread__copy_tsd(pthread__main); 1328 } 1329 1330 /* 1331 * Set up the slightly special stack for the "initial" thread, which 1332 * runs on the normal system stack, and thus gets slightly different 1333 * treatment. 1334 */ 1335 static void 1336 pthread__initmain(pthread_t *newt) 1337 { 1338 char *value; 1339 1340 pthread__initmainstack(); 1341 1342 value = pthread__getenv("PTHREAD_STACKSIZE"); 1343 if (value != NULL) { 1344 pthread__stacksize = atoi(value) * 1024; 1345 if (pthread__stacksize > pthread__main->pt_stack.ss_size) 1346 pthread__stacksize = pthread__main->pt_stack.ss_size; 1347 } 1348 if (pthread__stacksize == 0) 1349 pthread__stacksize = pthread__main->pt_stack.ss_size; 1350 pthread__stacksize += pthread__pagesize - 1; 1351 pthread__stacksize &= ~(pthread__pagesize - 1); 1352 if (pthread__stacksize < 4 * pthread__pagesize) 1353 errx(1, "Stacksize limit is too low, minimum %zd kbyte.", 1354 4 * pthread__pagesize / 1024); 1355 1356 *newt = pthread__main; 1357 #if defined(_PTHREAD_GETTCB_EXT) 1358 pthread__main->pt_tls = _PTHREAD_GETTCB_EXT(); 1359 #elif defined(__HAVE___LWP_GETTCB_FAST) 1360 pthread__main->pt_tls = __lwp_gettcb_fast(); 1361 #else 1362 pthread__main->pt_tls = _lwp_getprivate(); 1363 #endif 1364 pthread__main->pt_tls->tcb_pthread = pthread__main; 1365 } 1366 1367 static signed int 1368 /*ARGSUSED*/ 1369 pthread__cmp(void *ctx, const void *n1, const void *n2) 1370 { 1371 const uintptr_t p1 = (const uintptr_t)n1; 1372 const uintptr_t p2 = (const uintptr_t)n2; 1373 1374 if (p1 < p2) 1375 return -1; 1376 if (p1 > p2) 1377 return 1; 1378 return 0; 1379 } 1380 1381 /* Because getenv() wants to use locks. */ 1382 char * 1383 pthread__getenv(const char *name) 1384 { 1385 extern char **environ; 1386 size_t l_name, offset; 1387 1388 if (issetugid()) 1389 return (NULL); 1390 1391 l_name = strlen(name); 1392 for (offset = 0; environ[offset] != NULL; offset++) { 1393 if (strncmp(name, environ[offset], l_name) == 0 && 1394 environ[offset][l_name] == '=') { 1395 return environ[offset] + l_name + 1; 1396 } 1397 } 1398 1399 return NULL; 1400 } 1401 1402 pthread_mutex_t * 1403 pthread__hashlock(volatile const void *p) 1404 { 1405 uintptr_t v; 1406 1407 v = (uintptr_t)p; 1408 return &hashlocks[((v >> 9) ^ (v >> 3)) & (NHASHLOCK - 1)].mutex; 1409 } 1410 1411 int 1412 pthread__checkpri(int pri) 1413 { 1414 static int havepri; 1415 static long min, max; 1416 1417 if (!havepri) { 1418 min = sysconf(_SC_SCHED_PRI_MIN); 1419 max = sysconf(_SC_SCHED_PRI_MAX); 1420 havepri = 1; 1421 } 1422 return (pri < min || pri > max) ? EINVAL : 0; 1423 } 1424