1 /* 2 * z_Linux_util.cpp -- platform specific routines. 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_affinity.h" 15 #include "kmp_i18n.h" 16 #include "kmp_io.h" 17 #include "kmp_itt.h" 18 #include "kmp_lock.h" 19 #include "kmp_stats.h" 20 #include "kmp_str.h" 21 #include "kmp_wait_release.h" 22 #include "kmp_wrapper_getpid.h" 23 24 #if !KMP_OS_DRAGONFLY && !KMP_OS_FREEBSD && !KMP_OS_NETBSD && !KMP_OS_OPENBSD 25 #include <alloca.h> 26 #endif 27 #include <math.h> // HUGE_VAL. 28 #if KMP_OS_LINUX 29 #include <semaphore.h> 30 #endif // KMP_OS_LINUX 31 #include <sys/resource.h> 32 #if !KMP_OS_AIX 33 #include <sys/syscall.h> 34 #endif 35 #include <sys/time.h> 36 #include <sys/times.h> 37 #include <unistd.h> 38 39 #if KMP_OS_LINUX 40 #include <sys/sysinfo.h> 41 #if KMP_USE_FUTEX 42 // We should really include <futex.h>, but that causes compatibility problems on 43 // different Linux* OS distributions that either require that you include (or 44 // break when you try to include) <pci/types.h>. Since all we need is the two 45 // macros below (which are part of the kernel ABI, so can't change) we just 46 // define the constants here and don't include <futex.h> 47 #ifndef FUTEX_WAIT 48 #define FUTEX_WAIT 0 49 #endif 50 #ifndef FUTEX_WAKE 51 #define FUTEX_WAKE 1 52 #endif 53 #endif 54 #elif KMP_OS_DARWIN 55 #include <mach/mach.h> 56 #include <sys/sysctl.h> 57 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD 58 #include <sys/types.h> 59 #include <sys/sysctl.h> 60 #include <sys/user.h> 61 #include <pthread_np.h> 62 #if KMP_OS_DRAGONFLY 63 #include <kvm.h> 64 #endif 65 #elif KMP_OS_NETBSD || KMP_OS_OPENBSD 66 #include <sys/types.h> 67 #include <sys/sysctl.h> 68 #elif KMP_OS_SOLARIS 69 #include <libproc.h> 70 #include <procfs.h> 71 #include <thread.h> 72 #include <sys/loadavg.h> 73 #endif 74 75 #include <ctype.h> 76 #include <dirent.h> 77 #include <fcntl.h> 78 79 struct kmp_sys_timer { 80 struct timespec start; 81 }; 82 83 #ifndef TIMEVAL_TO_TIMESPEC 84 // Convert timeval to timespec. 85 #define TIMEVAL_TO_TIMESPEC(tv, ts) \ 86 do { \ 87 (ts)->tv_sec = (tv)->tv_sec; \ 88 (ts)->tv_nsec = (tv)->tv_usec * 1000; \ 89 } while (0) 90 #endif 91 92 // Convert timespec to nanoseconds. 93 #define TS2NS(timespec) \ 94 (((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec) 95 96 static struct kmp_sys_timer __kmp_sys_timer_data; 97 98 #if KMP_HANDLE_SIGNALS 99 typedef void (*sig_func_t)(int); 100 STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[NSIG]; 101 static sigset_t __kmp_sigset; 102 #endif 103 104 static int __kmp_init_runtime = FALSE; 105 106 static int __kmp_fork_count = 0; 107 108 static pthread_condattr_t __kmp_suspend_cond_attr; 109 static pthread_mutexattr_t __kmp_suspend_mutex_attr; 110 111 static kmp_cond_align_t __kmp_wait_cv; 112 static kmp_mutex_align_t __kmp_wait_mx; 113 114 kmp_uint64 __kmp_ticks_per_msec = 1000000; 115 kmp_uint64 __kmp_ticks_per_usec = 1000; 116 117 #ifdef DEBUG_SUSPEND 118 static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) { 119 KMP_SNPRINTF(buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))", 120 cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock, 121 cond->c_cond.__c_waiting); 122 } 123 #endif 124 125 #if ((KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED) 126 127 /* Affinity support */ 128 129 void __kmp_affinity_bind_thread(int which) { 130 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(), 131 "Illegal set affinity operation when not capable"); 132 133 kmp_affin_mask_t *mask; 134 KMP_CPU_ALLOC_ON_STACK(mask); 135 KMP_CPU_ZERO(mask); 136 KMP_CPU_SET(which, mask); 137 __kmp_set_system_affinity(mask, TRUE); 138 KMP_CPU_FREE_FROM_STACK(mask); 139 } 140 141 /* Determine if we can access affinity functionality on this version of 142 * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set 143 * __kmp_affin_mask_size to the appropriate value (0 means not capable). */ 144 void __kmp_affinity_determine_capable(const char *env_var) { 145 // Check and see if the OS supports thread affinity. 146 147 #if KMP_OS_LINUX 148 #define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024) 149 #define KMP_CPU_SET_TRY_SIZE CACHE_LINE 150 #elif KMP_OS_FREEBSD 151 #define KMP_CPU_SET_SIZE_LIMIT (sizeof(cpuset_t)) 152 #endif 153 154 int verbose = __kmp_affinity.flags.verbose; 155 int warnings = __kmp_affinity.flags.warnings; 156 enum affinity_type type = __kmp_affinity.type; 157 158 #if KMP_OS_LINUX 159 long gCode; 160 unsigned char *buf; 161 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT); 162 163 // If the syscall returns a suggestion for the size, 164 // then we don't have to search for an appropriate size. 165 gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_TRY_SIZE, buf); 166 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 167 "initial getaffinity call returned %ld errno = %d\n", 168 gCode, errno)); 169 170 if (gCode < 0 && errno != EINVAL) { 171 // System call not supported 172 if (verbose || 173 (warnings && (type != affinity_none) && (type != affinity_default) && 174 (type != affinity_disabled))) { 175 int error = errno; 176 kmp_msg_t err_code = KMP_ERR(error); 177 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var), 178 err_code, __kmp_msg_null); 179 if (__kmp_generate_warnings == kmp_warnings_off) { 180 __kmp_str_free(&err_code.str); 181 } 182 } 183 KMP_AFFINITY_DISABLE(); 184 KMP_INTERNAL_FREE(buf); 185 return; 186 } else if (gCode > 0) { 187 // The optimal situation: the OS returns the size of the buffer it expects. 188 KMP_AFFINITY_ENABLE(gCode); 189 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 190 "affinity supported (mask size %d)\n", 191 (int)__kmp_affin_mask_size)); 192 KMP_INTERNAL_FREE(buf); 193 return; 194 } 195 196 // Call the getaffinity system call repeatedly with increasing set sizes 197 // until we succeed, or reach an upper bound on the search. 198 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 199 "searching for proper set size\n")); 200 int size; 201 for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) { 202 gCode = syscall(__NR_sched_getaffinity, 0, size, buf); 203 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 204 "getaffinity for mask size %ld returned %ld errno = %d\n", 205 size, gCode, errno)); 206 207 if (gCode < 0) { 208 if (errno == ENOSYS) { 209 // We shouldn't get here 210 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 211 "inconsistent OS call behavior: errno == ENOSYS for mask " 212 "size %d\n", 213 size)); 214 if (verbose || 215 (warnings && (type != affinity_none) && 216 (type != affinity_default) && (type != affinity_disabled))) { 217 int error = errno; 218 kmp_msg_t err_code = KMP_ERR(error); 219 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var), 220 err_code, __kmp_msg_null); 221 if (__kmp_generate_warnings == kmp_warnings_off) { 222 __kmp_str_free(&err_code.str); 223 } 224 } 225 KMP_AFFINITY_DISABLE(); 226 KMP_INTERNAL_FREE(buf); 227 return; 228 } 229 continue; 230 } 231 232 KMP_AFFINITY_ENABLE(gCode); 233 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 234 "affinity supported (mask size %d)\n", 235 (int)__kmp_affin_mask_size)); 236 KMP_INTERNAL_FREE(buf); 237 return; 238 } 239 #elif KMP_OS_FREEBSD 240 long gCode; 241 unsigned char *buf; 242 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT); 243 gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT, 244 reinterpret_cast<cpuset_t *>(buf)); 245 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 246 "initial getaffinity call returned %d errno = %d\n", 247 gCode, errno)); 248 if (gCode == 0) { 249 KMP_AFFINITY_ENABLE(KMP_CPU_SET_SIZE_LIMIT); 250 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 251 "affinity supported (mask size %d)\n", 252 (int)__kmp_affin_mask_size)); 253 KMP_INTERNAL_FREE(buf); 254 return; 255 } 256 #endif 257 KMP_INTERNAL_FREE(buf); 258 259 // Affinity is not supported 260 KMP_AFFINITY_DISABLE(); 261 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 262 "cannot determine mask size - affinity not supported\n")); 263 if (verbose || (warnings && (type != affinity_none) && 264 (type != affinity_default) && (type != affinity_disabled))) { 265 KMP_WARNING(AffCantGetMaskSize, env_var); 266 } 267 } 268 269 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 270 271 #if KMP_USE_FUTEX 272 273 int __kmp_futex_determine_capable() { 274 int loc = 0; 275 long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0); 276 int retval = (rc == 0) || (errno != ENOSYS); 277 278 KA_TRACE(10, 279 ("__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno)); 280 KA_TRACE(10, ("__kmp_futex_determine_capable: futex syscall%s supported\n", 281 retval ? "" : " not")); 282 283 return retval; 284 } 285 286 #endif // KMP_USE_FUTEX 287 288 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS) 289 /* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to 290 use compare_and_store for these routines */ 291 292 kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 d) { 293 kmp_int8 old_value, new_value; 294 295 old_value = TCR_1(*p); 296 new_value = old_value | d; 297 298 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) { 299 KMP_CPU_PAUSE(); 300 old_value = TCR_1(*p); 301 new_value = old_value | d; 302 } 303 return old_value; 304 } 305 306 kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 d) { 307 kmp_int8 old_value, new_value; 308 309 old_value = TCR_1(*p); 310 new_value = old_value & d; 311 312 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) { 313 KMP_CPU_PAUSE(); 314 old_value = TCR_1(*p); 315 new_value = old_value & d; 316 } 317 return old_value; 318 } 319 320 kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 d) { 321 kmp_uint32 old_value, new_value; 322 323 old_value = TCR_4(*p); 324 new_value = old_value | d; 325 326 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) { 327 KMP_CPU_PAUSE(); 328 old_value = TCR_4(*p); 329 new_value = old_value | d; 330 } 331 return old_value; 332 } 333 334 kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) { 335 kmp_uint32 old_value, new_value; 336 337 old_value = TCR_4(*p); 338 new_value = old_value & d; 339 340 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) { 341 KMP_CPU_PAUSE(); 342 old_value = TCR_4(*p); 343 new_value = old_value & d; 344 } 345 return old_value; 346 } 347 348 #if KMP_ARCH_X86 || KMP_ARCH_WASM 349 kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) { 350 kmp_int8 old_value, new_value; 351 352 old_value = TCR_1(*p); 353 new_value = old_value + d; 354 355 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) { 356 KMP_CPU_PAUSE(); 357 old_value = TCR_1(*p); 358 new_value = old_value + d; 359 } 360 return old_value; 361 } 362 363 kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 d) { 364 kmp_int64 old_value, new_value; 365 366 old_value = TCR_8(*p); 367 new_value = old_value + d; 368 369 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) { 370 KMP_CPU_PAUSE(); 371 old_value = TCR_8(*p); 372 new_value = old_value + d; 373 } 374 return old_value; 375 } 376 #endif /* KMP_ARCH_X86 */ 377 378 kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 d) { 379 kmp_uint64 old_value, new_value; 380 381 old_value = TCR_8(*p); 382 new_value = old_value | d; 383 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) { 384 KMP_CPU_PAUSE(); 385 old_value = TCR_8(*p); 386 new_value = old_value | d; 387 } 388 return old_value; 389 } 390 391 kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 d) { 392 kmp_uint64 old_value, new_value; 393 394 old_value = TCR_8(*p); 395 new_value = old_value & d; 396 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) { 397 KMP_CPU_PAUSE(); 398 old_value = TCR_8(*p); 399 new_value = old_value & d; 400 } 401 return old_value; 402 } 403 404 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */ 405 406 void __kmp_terminate_thread(int gtid) { 407 int status; 408 kmp_info_t *th = __kmp_threads[gtid]; 409 410 if (!th) 411 return; 412 413 #ifdef KMP_CANCEL_THREADS 414 KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid)); 415 status = pthread_cancel(th->th.th_info.ds.ds_thread); 416 if (status != 0 && status != ESRCH) { 417 __kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status), 418 __kmp_msg_null); 419 } 420 #endif 421 KMP_YIELD(TRUE); 422 } // 423 424 /* Set thread stack info. 425 If values are unreasonable, assume call failed and use incremental stack 426 refinement method instead. Returns TRUE if the stack parameters could be 427 determined exactly, FALSE if incremental refinement is necessary. */ 428 static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) { 429 int stack_data; 430 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 431 KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX 432 int status; 433 size_t size = 0; 434 void *addr = 0; 435 436 /* Always do incremental stack refinement for ubermaster threads since the 437 initial thread stack range can be reduced by sibling thread creation so 438 pthread_attr_getstack may cause thread gtid aliasing */ 439 if (!KMP_UBER_GTID(gtid)) { 440 441 #if KMP_OS_SOLARIS 442 stack_t s; 443 if ((status = thr_stksegment(&s)) < 0) { 444 KMP_CHECK_SYSFAIL("thr_stksegment", status); 445 } 446 447 addr = s.ss_sp; 448 size = s.ss_size; 449 KA_TRACE(60, ("__kmp_set_stack_info: T#%d thr_stksegment returned size:" 450 " %lu, low addr: %p\n", 451 gtid, size, addr)); 452 #else 453 pthread_attr_t attr; 454 /* Fetch the real thread attributes */ 455 status = pthread_attr_init(&attr); 456 KMP_CHECK_SYSFAIL("pthread_attr_init", status); 457 #if KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD 458 status = pthread_attr_get_np(pthread_self(), &attr); 459 KMP_CHECK_SYSFAIL("pthread_attr_get_np", status); 460 #else 461 status = pthread_getattr_np(pthread_self(), &attr); 462 KMP_CHECK_SYSFAIL("pthread_getattr_np", status); 463 #endif 464 status = pthread_attr_getstack(&attr, &addr, &size); 465 KMP_CHECK_SYSFAIL("pthread_attr_getstack", status); 466 KA_TRACE(60, 467 ("__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:" 468 " %lu, low addr: %p\n", 469 gtid, size, addr)); 470 status = pthread_attr_destroy(&attr); 471 KMP_CHECK_SYSFAIL("pthread_attr_destroy", status); 472 #endif 473 } 474 475 if (size != 0 && addr != 0) { // was stack parameter determination successful? 476 /* Store the correct base and size */ 477 TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size)); 478 TCW_PTR(th->th.th_info.ds.ds_stacksize, size); 479 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE); 480 return TRUE; 481 } 482 #endif /* KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD \ 483 || KMP_OS_HURD || KMP_OS_SOLARIS */ 484 /* Use incremental refinement starting from initial conservative estimate */ 485 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0); 486 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data); 487 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE); 488 return FALSE; 489 } 490 491 static void *__kmp_launch_worker(void *thr) { 492 int status, old_type, old_state; 493 #ifdef KMP_BLOCK_SIGNALS 494 sigset_t new_set, old_set; 495 #endif /* KMP_BLOCK_SIGNALS */ 496 void *exit_val; 497 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 498 KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS 499 void *volatile padding = 0; 500 #endif 501 int gtid; 502 503 gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid; 504 __kmp_gtid_set_specific(gtid); 505 #ifdef KMP_TDATA_GTID 506 __kmp_gtid = gtid; 507 #endif 508 #if KMP_STATS_ENABLED 509 // set thread local index to point to thread-specific stats 510 __kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats; 511 __kmp_stats_thread_ptr->startLife(); 512 KMP_SET_THREAD_STATE(IDLE); 513 KMP_INIT_PARTITIONED_TIMERS(OMP_idle); 514 #endif 515 516 #if USE_ITT_BUILD 517 __kmp_itt_thread_name(gtid); 518 #endif /* USE_ITT_BUILD */ 519 520 #if KMP_AFFINITY_SUPPORTED 521 __kmp_affinity_bind_init_mask(gtid); 522 #endif 523 524 #ifdef KMP_CANCEL_THREADS 525 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type); 526 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status); 527 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? 528 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state); 529 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 530 #endif 531 532 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 533 // Set FP control regs to be a copy of the parallel initialization thread's. 534 __kmp_clear_x87_fpu_status_word(); 535 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word); 536 __kmp_load_mxcsr(&__kmp_init_mxcsr); 537 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 538 539 #ifdef KMP_BLOCK_SIGNALS 540 status = sigfillset(&new_set); 541 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status); 542 status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set); 543 KMP_CHECK_SYSFAIL("pthread_sigmask", status); 544 #endif /* KMP_BLOCK_SIGNALS */ 545 546 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 547 KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS 548 if (__kmp_stkoffset > 0 && gtid > 0) { 549 padding = KMP_ALLOCA(gtid * __kmp_stkoffset); 550 (void)padding; 551 } 552 #endif 553 554 KMP_MB(); 555 __kmp_set_stack_info(gtid, (kmp_info_t *)thr); 556 557 __kmp_check_stack_overlap((kmp_info_t *)thr); 558 559 exit_val = __kmp_launch_thread((kmp_info_t *)thr); 560 561 #ifdef KMP_BLOCK_SIGNALS 562 status = pthread_sigmask(SIG_SETMASK, &old_set, NULL); 563 KMP_CHECK_SYSFAIL("pthread_sigmask", status); 564 #endif /* KMP_BLOCK_SIGNALS */ 565 566 return exit_val; 567 } 568 569 #if KMP_USE_MONITOR 570 /* The monitor thread controls all of the threads in the complex */ 571 572 static void *__kmp_launch_monitor(void *thr) { 573 int status, old_type, old_state; 574 #ifdef KMP_BLOCK_SIGNALS 575 sigset_t new_set; 576 #endif /* KMP_BLOCK_SIGNALS */ 577 struct timespec interval; 578 579 KMP_MB(); /* Flush all pending memory write invalidates. */ 580 581 KA_TRACE(10, ("__kmp_launch_monitor: #1 launched\n")); 582 583 /* register us as the monitor thread */ 584 __kmp_gtid_set_specific(KMP_GTID_MONITOR); 585 #ifdef KMP_TDATA_GTID 586 __kmp_gtid = KMP_GTID_MONITOR; 587 #endif 588 589 KMP_MB(); 590 591 #if USE_ITT_BUILD 592 // Instruct Intel(R) Threading Tools to ignore monitor thread. 593 __kmp_itt_thread_ignore(); 594 #endif /* USE_ITT_BUILD */ 595 596 __kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid, 597 (kmp_info_t *)thr); 598 599 __kmp_check_stack_overlap((kmp_info_t *)thr); 600 601 #ifdef KMP_CANCEL_THREADS 602 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type); 603 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status); 604 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? 605 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state); 606 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 607 #endif 608 609 #if KMP_REAL_TIME_FIX 610 // This is a potential fix which allows application with real-time scheduling 611 // policy work. However, decision about the fix is not made yet, so it is 612 // disabled by default. 613 { // Are program started with real-time scheduling policy? 614 int sched = sched_getscheduler(0); 615 if (sched == SCHED_FIFO || sched == SCHED_RR) { 616 // Yes, we are a part of real-time application. Try to increase the 617 // priority of the monitor. 618 struct sched_param param; 619 int max_priority = sched_get_priority_max(sched); 620 int rc; 621 KMP_WARNING(RealTimeSchedNotSupported); 622 sched_getparam(0, ¶m); 623 if (param.sched_priority < max_priority) { 624 param.sched_priority += 1; 625 rc = sched_setscheduler(0, sched, ¶m); 626 if (rc != 0) { 627 int error = errno; 628 kmp_msg_t err_code = KMP_ERR(error); 629 __kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority), 630 err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null); 631 if (__kmp_generate_warnings == kmp_warnings_off) { 632 __kmp_str_free(&err_code.str); 633 } 634 } 635 } else { 636 // We cannot abort here, because number of CPUs may be enough for all 637 // the threads, including the monitor thread, so application could 638 // potentially work... 639 __kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority), 640 KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority), 641 __kmp_msg_null); 642 } 643 } 644 // AC: free thread that waits for monitor started 645 TCW_4(__kmp_global.g.g_time.dt.t_value, 0); 646 } 647 #endif // KMP_REAL_TIME_FIX 648 649 KMP_MB(); /* Flush all pending memory write invalidates. */ 650 651 if (__kmp_monitor_wakeups == 1) { 652 interval.tv_sec = 1; 653 interval.tv_nsec = 0; 654 } else { 655 interval.tv_sec = 0; 656 interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups); 657 } 658 659 KA_TRACE(10, ("__kmp_launch_monitor: #2 monitor\n")); 660 661 while (!TCR_4(__kmp_global.g.g_done)) { 662 struct timespec now; 663 struct timeval tval; 664 665 /* This thread monitors the state of the system */ 666 667 KA_TRACE(15, ("__kmp_launch_monitor: update\n")); 668 669 status = gettimeofday(&tval, NULL); 670 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 671 TIMEVAL_TO_TIMESPEC(&tval, &now); 672 673 now.tv_sec += interval.tv_sec; 674 now.tv_nsec += interval.tv_nsec; 675 676 if (now.tv_nsec >= KMP_NSEC_PER_SEC) { 677 now.tv_sec += 1; 678 now.tv_nsec -= KMP_NSEC_PER_SEC; 679 } 680 681 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex); 682 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 683 // AC: the monitor should not fall asleep if g_done has been set 684 if (!TCR_4(__kmp_global.g.g_done)) { // check once more under mutex 685 status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond, 686 &__kmp_wait_mx.m_mutex, &now); 687 if (status != 0) { 688 if (status != ETIMEDOUT && status != EINTR) { 689 KMP_SYSFAIL("pthread_cond_timedwait", status); 690 } 691 } 692 } 693 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex); 694 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 695 696 TCW_4(__kmp_global.g.g_time.dt.t_value, 697 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1); 698 699 KMP_MB(); /* Flush all pending memory write invalidates. */ 700 } 701 702 KA_TRACE(10, ("__kmp_launch_monitor: #3 cleanup\n")); 703 704 #ifdef KMP_BLOCK_SIGNALS 705 status = sigfillset(&new_set); 706 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status); 707 status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL); 708 KMP_CHECK_SYSFAIL("pthread_sigmask", status); 709 #endif /* KMP_BLOCK_SIGNALS */ 710 711 KA_TRACE(10, ("__kmp_launch_monitor: #4 finished\n")); 712 713 if (__kmp_global.g.g_abort != 0) { 714 /* now we need to terminate the worker threads */ 715 /* the value of t_abort is the signal we caught */ 716 717 int gtid; 718 719 KA_TRACE(10, ("__kmp_launch_monitor: #5 terminate sig=%d\n", 720 __kmp_global.g.g_abort)); 721 722 /* terminate the OpenMP worker threads */ 723 /* TODO this is not valid for sibling threads!! 724 * the uber master might not be 0 anymore.. */ 725 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid) 726 __kmp_terminate_thread(gtid); 727 728 __kmp_cleanup(); 729 730 KA_TRACE(10, ("__kmp_launch_monitor: #6 raise sig=%d\n", 731 __kmp_global.g.g_abort)); 732 733 if (__kmp_global.g.g_abort > 0) 734 raise(__kmp_global.g.g_abort); 735 } 736 737 KA_TRACE(10, ("__kmp_launch_monitor: #7 exit\n")); 738 739 return thr; 740 } 741 #endif // KMP_USE_MONITOR 742 743 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) { 744 pthread_t handle; 745 pthread_attr_t thread_attr; 746 int status; 747 748 th->th.th_info.ds.ds_gtid = gtid; 749 750 #if KMP_STATS_ENABLED 751 // sets up worker thread stats 752 __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid); 753 754 // th->th.th_stats is used to transfer thread-specific stats-pointer to 755 // __kmp_launch_worker. So when thread is created (goes into 756 // __kmp_launch_worker) it will set its thread local pointer to 757 // th->th.th_stats 758 if (!KMP_UBER_GTID(gtid)) { 759 th->th.th_stats = __kmp_stats_list->push_back(gtid); 760 } else { 761 // For root threads, __kmp_stats_thread_ptr is set in __kmp_register_root(), 762 // so set the th->th.th_stats field to it. 763 th->th.th_stats = __kmp_stats_thread_ptr; 764 } 765 __kmp_release_tas_lock(&__kmp_stats_lock, gtid); 766 767 #endif // KMP_STATS_ENABLED 768 769 if (KMP_UBER_GTID(gtid)) { 770 KA_TRACE(10, ("__kmp_create_worker: uber thread (%d)\n", gtid)); 771 th->th.th_info.ds.ds_thread = pthread_self(); 772 __kmp_set_stack_info(gtid, th); 773 __kmp_check_stack_overlap(th); 774 return; 775 } 776 777 KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid)); 778 779 KMP_MB(); /* Flush all pending memory write invalidates. */ 780 781 #ifdef KMP_THREAD_ATTR 782 status = pthread_attr_init(&thread_attr); 783 if (status != 0) { 784 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null); 785 } 786 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE); 787 if (status != 0) { 788 __kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null); 789 } 790 791 /* Set stack size for this thread now. 792 The multiple of 2 is there because on some machines, requesting an unusual 793 stacksize causes the thread to have an offset before the dummy alloca() 794 takes place to create the offset. Since we want the user to have a 795 sufficient stacksize AND support a stack offset, we alloca() twice the 796 offset so that the upcoming alloca() does not eliminate any premade offset, 797 and also gives the user the stack space they requested for all threads */ 798 stack_size += gtid * __kmp_stkoffset * 2; 799 800 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, " 801 "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n", 802 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size)); 803 804 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 805 status = pthread_attr_setstacksize(&thread_attr, stack_size); 806 #ifdef KMP_BACKUP_STKSIZE 807 if (status != 0) { 808 if (!__kmp_env_stksize) { 809 stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset; 810 __kmp_stksize = KMP_BACKUP_STKSIZE; 811 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, " 812 "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu " 813 "bytes\n", 814 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size)); 815 status = pthread_attr_setstacksize(&thread_attr, stack_size); 816 } 817 } 818 #endif /* KMP_BACKUP_STKSIZE */ 819 if (status != 0) { 820 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status), 821 KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null); 822 } 823 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 824 825 #endif /* KMP_THREAD_ATTR */ 826 827 status = 828 pthread_create(&handle, &thread_attr, __kmp_launch_worker, (void *)th); 829 if (status != 0 || !handle) { // ??? Why do we check handle?? 830 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 831 if (status == EINVAL) { 832 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status), 833 KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null); 834 } 835 if (status == ENOMEM) { 836 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status), 837 KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null); 838 } 839 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 840 if (status == EAGAIN) { 841 __kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status), 842 KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null); 843 } 844 KMP_SYSFAIL("pthread_create", status); 845 } 846 847 th->th.th_info.ds.ds_thread = handle; 848 849 #ifdef KMP_THREAD_ATTR 850 status = pthread_attr_destroy(&thread_attr); 851 if (status) { 852 kmp_msg_t err_code = KMP_ERR(status); 853 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code, 854 __kmp_msg_null); 855 if (__kmp_generate_warnings == kmp_warnings_off) { 856 __kmp_str_free(&err_code.str); 857 } 858 } 859 #endif /* KMP_THREAD_ATTR */ 860 861 KMP_MB(); /* Flush all pending memory write invalidates. */ 862 863 KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid)); 864 865 } // __kmp_create_worker 866 867 #if KMP_USE_MONITOR 868 void __kmp_create_monitor(kmp_info_t *th) { 869 pthread_t handle; 870 pthread_attr_t thread_attr; 871 size_t size; 872 int status; 873 int auto_adj_size = FALSE; 874 875 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) { 876 // We don't need monitor thread in case of MAX_BLOCKTIME 877 KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of " 878 "MAX blocktime\n")); 879 th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op 880 th->th.th_info.ds.ds_gtid = 0; 881 return; 882 } 883 KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n")); 884 885 KMP_MB(); /* Flush all pending memory write invalidates. */ 886 887 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR; 888 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR; 889 #if KMP_REAL_TIME_FIX 890 TCW_4(__kmp_global.g.g_time.dt.t_value, 891 -1); // Will use it for synchronization a bit later. 892 #else 893 TCW_4(__kmp_global.g.g_time.dt.t_value, 0); 894 #endif // KMP_REAL_TIME_FIX 895 896 #ifdef KMP_THREAD_ATTR 897 if (__kmp_monitor_stksize == 0) { 898 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE; 899 auto_adj_size = TRUE; 900 } 901 status = pthread_attr_init(&thread_attr); 902 if (status != 0) { 903 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null); 904 } 905 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE); 906 if (status != 0) { 907 __kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null); 908 } 909 910 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 911 status = pthread_attr_getstacksize(&thread_attr, &size); 912 KMP_CHECK_SYSFAIL("pthread_attr_getstacksize", status); 913 #else 914 size = __kmp_sys_min_stksize; 915 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 916 #endif /* KMP_THREAD_ATTR */ 917 918 if (__kmp_monitor_stksize == 0) { 919 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE; 920 } 921 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) { 922 __kmp_monitor_stksize = __kmp_sys_min_stksize; 923 } 924 925 KA_TRACE(10, ("__kmp_create_monitor: default stacksize = %lu bytes," 926 "requested stacksize = %lu bytes\n", 927 size, __kmp_monitor_stksize)); 928 929 retry: 930 931 /* Set stack size for this thread now. */ 932 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 933 KA_TRACE(10, ("__kmp_create_monitor: setting stacksize = %lu bytes,", 934 __kmp_monitor_stksize)); 935 status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize); 936 if (status != 0) { 937 if (auto_adj_size) { 938 __kmp_monitor_stksize *= 2; 939 goto retry; 940 } 941 kmp_msg_t err_code = KMP_ERR(status); 942 __kmp_msg(kmp_ms_warning, // should this be fatal? BB 943 KMP_MSG(CantSetMonitorStackSize, (long int)__kmp_monitor_stksize), 944 err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null); 945 if (__kmp_generate_warnings == kmp_warnings_off) { 946 __kmp_str_free(&err_code.str); 947 } 948 } 949 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 950 951 status = 952 pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (void *)th); 953 954 if (status != 0) { 955 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 956 if (status == EINVAL) { 957 if (auto_adj_size && (__kmp_monitor_stksize < (size_t)0x40000000)) { 958 __kmp_monitor_stksize *= 2; 959 goto retry; 960 } 961 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize), 962 KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize), 963 __kmp_msg_null); 964 } 965 if (status == ENOMEM) { 966 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize), 967 KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize), 968 __kmp_msg_null); 969 } 970 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 971 if (status == EAGAIN) { 972 __kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status), 973 KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null); 974 } 975 KMP_SYSFAIL("pthread_create", status); 976 } 977 978 th->th.th_info.ds.ds_thread = handle; 979 980 #if KMP_REAL_TIME_FIX 981 // Wait for the monitor thread is really started and set its *priority*. 982 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 983 sizeof(__kmp_global.g.g_time.dt.t_value)); 984 __kmp_wait_4((kmp_uint32 volatile *)&__kmp_global.g.g_time.dt.t_value, -1, 985 &__kmp_neq_4, NULL); 986 #endif // KMP_REAL_TIME_FIX 987 988 #ifdef KMP_THREAD_ATTR 989 status = pthread_attr_destroy(&thread_attr); 990 if (status != 0) { 991 kmp_msg_t err_code = KMP_ERR(status); 992 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code, 993 __kmp_msg_null); 994 if (__kmp_generate_warnings == kmp_warnings_off) { 995 __kmp_str_free(&err_code.str); 996 } 997 } 998 #endif 999 1000 KMP_MB(); /* Flush all pending memory write invalidates. */ 1001 1002 KA_TRACE(10, ("__kmp_create_monitor: monitor created %#.8lx\n", 1003 th->th.th_info.ds.ds_thread)); 1004 1005 } // __kmp_create_monitor 1006 #endif // KMP_USE_MONITOR 1007 1008 void __kmp_exit_thread(int exit_status) { 1009 #if KMP_OS_WASI 1010 // TODO: the wasm32-wasi-threads target does not yet support pthread_exit. 1011 #else 1012 pthread_exit((void *)(intptr_t)exit_status); 1013 #endif 1014 } // __kmp_exit_thread 1015 1016 #if KMP_USE_MONITOR 1017 void __kmp_resume_monitor(); 1018 1019 extern "C" void __kmp_reap_monitor(kmp_info_t *th) { 1020 int status; 1021 void *exit_val; 1022 1023 KA_TRACE(10, ("__kmp_reap_monitor: try to reap monitor thread with handle" 1024 " %#.8lx\n", 1025 th->th.th_info.ds.ds_thread)); 1026 1027 // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR. 1028 // If both tid and gtid are 0, it means the monitor did not ever start. 1029 // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down. 1030 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid); 1031 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) { 1032 KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n")); 1033 return; 1034 } 1035 1036 KMP_MB(); /* Flush all pending memory write invalidates. */ 1037 1038 /* First, check to see whether the monitor thread exists to wake it up. This 1039 is to avoid performance problem when the monitor sleeps during 1040 blocktime-size interval */ 1041 1042 status = pthread_kill(th->th.th_info.ds.ds_thread, 0); 1043 if (status != ESRCH) { 1044 __kmp_resume_monitor(); // Wake up the monitor thread 1045 } 1046 KA_TRACE(10, ("__kmp_reap_monitor: try to join with monitor\n")); 1047 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val); 1048 if (exit_val != th) { 1049 __kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null); 1050 } 1051 1052 th->th.th_info.ds.ds_tid = KMP_GTID_DNE; 1053 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE; 1054 1055 KA_TRACE(10, ("__kmp_reap_monitor: done reaping monitor thread with handle" 1056 " %#.8lx\n", 1057 th->th.th_info.ds.ds_thread)); 1058 1059 KMP_MB(); /* Flush all pending memory write invalidates. */ 1060 } 1061 #else 1062 // Empty symbol to export (see exports_so.txt) when 1063 // monitor thread feature is disabled 1064 extern "C" void __kmp_reap_monitor(kmp_info_t *th) { (void)th; } 1065 #endif // KMP_USE_MONITOR 1066 1067 void __kmp_reap_worker(kmp_info_t *th) { 1068 int status; 1069 void *exit_val; 1070 1071 KMP_MB(); /* Flush all pending memory write invalidates. */ 1072 1073 KA_TRACE( 1074 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid)); 1075 1076 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val); 1077 #ifdef KMP_DEBUG 1078 /* Don't expose these to the user until we understand when they trigger */ 1079 if (status != 0) { 1080 __kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null); 1081 } 1082 if (exit_val != th) { 1083 KA_TRACE(10, ("__kmp_reap_worker: worker T#%d did not reap properly, " 1084 "exit_val = %p\n", 1085 th->th.th_info.ds.ds_gtid, exit_val)); 1086 } 1087 #else 1088 (void)status; // unused variable 1089 #endif /* KMP_DEBUG */ 1090 1091 KA_TRACE(10, ("__kmp_reap_worker: done reaping T#%d\n", 1092 th->th.th_info.ds.ds_gtid)); 1093 1094 KMP_MB(); /* Flush all pending memory write invalidates. */ 1095 } 1096 1097 #if KMP_HANDLE_SIGNALS 1098 1099 static void __kmp_null_handler(int signo) { 1100 // Do nothing, for doing SIG_IGN-type actions. 1101 } // __kmp_null_handler 1102 1103 static void __kmp_team_handler(int signo) { 1104 if (__kmp_global.g.g_abort == 0) { 1105 /* Stage 1 signal handler, let's shut down all of the threads */ 1106 #ifdef KMP_DEBUG 1107 __kmp_debug_printf("__kmp_team_handler: caught signal = %d\n", signo); 1108 #endif 1109 switch (signo) { 1110 case SIGHUP: 1111 case SIGINT: 1112 case SIGQUIT: 1113 case SIGILL: 1114 case SIGABRT: 1115 case SIGFPE: 1116 case SIGBUS: 1117 case SIGSEGV: 1118 #ifdef SIGSYS 1119 case SIGSYS: 1120 #endif 1121 case SIGTERM: 1122 if (__kmp_debug_buf) { 1123 __kmp_dump_debug_buffer(); 1124 } 1125 __kmp_unregister_library(); // cleanup shared memory 1126 KMP_MB(); // Flush all pending memory write invalidates. 1127 TCW_4(__kmp_global.g.g_abort, signo); 1128 KMP_MB(); // Flush all pending memory write invalidates. 1129 TCW_4(__kmp_global.g.g_done, TRUE); 1130 KMP_MB(); // Flush all pending memory write invalidates. 1131 break; 1132 default: 1133 #ifdef KMP_DEBUG 1134 __kmp_debug_printf("__kmp_team_handler: unknown signal type"); 1135 #endif 1136 break; 1137 } 1138 } 1139 } // __kmp_team_handler 1140 1141 static void __kmp_sigaction(int signum, const struct sigaction *act, 1142 struct sigaction *oldact) { 1143 int rc = sigaction(signum, act, oldact); 1144 KMP_CHECK_SYSFAIL_ERRNO("sigaction", rc); 1145 } 1146 1147 static void __kmp_install_one_handler(int sig, sig_func_t handler_func, 1148 int parallel_init) { 1149 KMP_MB(); // Flush all pending memory write invalidates. 1150 KB_TRACE(60, 1151 ("__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init)); 1152 if (parallel_init) { 1153 struct sigaction new_action; 1154 struct sigaction old_action; 1155 new_action.sa_handler = handler_func; 1156 new_action.sa_flags = 0; 1157 sigfillset(&new_action.sa_mask); 1158 __kmp_sigaction(sig, &new_action, &old_action); 1159 if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) { 1160 sigaddset(&__kmp_sigset, sig); 1161 } else { 1162 // Restore/keep user's handler if one previously installed. 1163 __kmp_sigaction(sig, &old_action, NULL); 1164 } 1165 } else { 1166 // Save initial/system signal handlers to see if user handlers installed. 1167 __kmp_sigaction(sig, NULL, &__kmp_sighldrs[sig]); 1168 } 1169 KMP_MB(); // Flush all pending memory write invalidates. 1170 } // __kmp_install_one_handler 1171 1172 static void __kmp_remove_one_handler(int sig) { 1173 KB_TRACE(60, ("__kmp_remove_one_handler( %d )\n", sig)); 1174 if (sigismember(&__kmp_sigset, sig)) { 1175 struct sigaction old; 1176 KMP_MB(); // Flush all pending memory write invalidates. 1177 __kmp_sigaction(sig, &__kmp_sighldrs[sig], &old); 1178 if ((old.sa_handler != __kmp_team_handler) && 1179 (old.sa_handler != __kmp_null_handler)) { 1180 // Restore the users signal handler. 1181 KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, " 1182 "restoring: sig=%d\n", 1183 sig)); 1184 __kmp_sigaction(sig, &old, NULL); 1185 } 1186 sigdelset(&__kmp_sigset, sig); 1187 KMP_MB(); // Flush all pending memory write invalidates. 1188 } 1189 } // __kmp_remove_one_handler 1190 1191 void __kmp_install_signals(int parallel_init) { 1192 KB_TRACE(10, ("__kmp_install_signals( %d )\n", parallel_init)); 1193 if (__kmp_handle_signals || !parallel_init) { 1194 // If ! parallel_init, we do not install handlers, just save original 1195 // handlers. Let us do it even __handle_signals is 0. 1196 sigemptyset(&__kmp_sigset); 1197 __kmp_install_one_handler(SIGHUP, __kmp_team_handler, parallel_init); 1198 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init); 1199 __kmp_install_one_handler(SIGQUIT, __kmp_team_handler, parallel_init); 1200 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init); 1201 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init); 1202 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init); 1203 __kmp_install_one_handler(SIGBUS, __kmp_team_handler, parallel_init); 1204 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init); 1205 #ifdef SIGSYS 1206 __kmp_install_one_handler(SIGSYS, __kmp_team_handler, parallel_init); 1207 #endif // SIGSYS 1208 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init); 1209 #ifdef SIGPIPE 1210 __kmp_install_one_handler(SIGPIPE, __kmp_team_handler, parallel_init); 1211 #endif // SIGPIPE 1212 } 1213 } // __kmp_install_signals 1214 1215 void __kmp_remove_signals(void) { 1216 int sig; 1217 KB_TRACE(10, ("__kmp_remove_signals()\n")); 1218 for (sig = 1; sig < NSIG; ++sig) { 1219 __kmp_remove_one_handler(sig); 1220 } 1221 } // __kmp_remove_signals 1222 1223 #endif // KMP_HANDLE_SIGNALS 1224 1225 void __kmp_enable(int new_state) { 1226 #ifdef KMP_CANCEL_THREADS 1227 int status, old_state; 1228 status = pthread_setcancelstate(new_state, &old_state); 1229 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 1230 KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE); 1231 #endif 1232 } 1233 1234 void __kmp_disable(int *old_state) { 1235 #ifdef KMP_CANCEL_THREADS 1236 int status; 1237 status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state); 1238 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 1239 #endif 1240 } 1241 1242 static void __kmp_atfork_prepare(void) { 1243 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 1244 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1245 } 1246 1247 static void __kmp_atfork_parent(void) { 1248 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1249 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 1250 } 1251 1252 /* Reset the library so execution in the child starts "all over again" with 1253 clean data structures in initial states. Don't worry about freeing memory 1254 allocated by parent, just abandon it to be safe. */ 1255 static void __kmp_atfork_child(void) { 1256 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1257 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 1258 /* TODO make sure this is done right for nested/sibling */ 1259 // ATT: Memory leaks are here? TODO: Check it and fix. 1260 /* KMP_ASSERT( 0 ); */ 1261 1262 ++__kmp_fork_count; 1263 1264 #if KMP_AFFINITY_SUPPORTED 1265 #if KMP_OS_LINUX || KMP_OS_FREEBSD 1266 // reset the affinity in the child to the initial thread 1267 // affinity in the parent 1268 kmp_set_thread_affinity_mask_initial(); 1269 #endif 1270 // Set default not to bind threads tightly in the child (we're expecting 1271 // over-subscription after the fork and this can improve things for 1272 // scripting languages that use OpenMP inside process-parallel code). 1273 if (__kmp_nested_proc_bind.bind_types != NULL) { 1274 __kmp_nested_proc_bind.bind_types[0] = proc_bind_false; 1275 } 1276 for (kmp_affinity_t *affinity : __kmp_affinities) 1277 *affinity = KMP_AFFINITY_INIT(affinity->env_var); 1278 __kmp_affin_fullMask = nullptr; 1279 __kmp_affin_origMask = nullptr; 1280 __kmp_topology = nullptr; 1281 #endif // KMP_AFFINITY_SUPPORTED 1282 1283 #if KMP_USE_MONITOR 1284 __kmp_init_monitor = 0; 1285 #endif 1286 __kmp_init_parallel = FALSE; 1287 __kmp_init_middle = FALSE; 1288 __kmp_init_serial = FALSE; 1289 TCW_4(__kmp_init_gtid, FALSE); 1290 __kmp_init_common = FALSE; 1291 1292 TCW_4(__kmp_init_user_locks, FALSE); 1293 #if !KMP_USE_DYNAMIC_LOCK 1294 __kmp_user_lock_table.used = 1; 1295 __kmp_user_lock_table.allocated = 0; 1296 __kmp_user_lock_table.table = NULL; 1297 __kmp_lock_blocks = NULL; 1298 #endif 1299 1300 __kmp_all_nth = 0; 1301 TCW_4(__kmp_nth, 0); 1302 1303 __kmp_thread_pool = NULL; 1304 __kmp_thread_pool_insert_pt = NULL; 1305 __kmp_team_pool = NULL; 1306 1307 /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate 1308 here so threadprivate doesn't use stale data */ 1309 KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n", 1310 __kmp_threadpriv_cache_list)); 1311 1312 while (__kmp_threadpriv_cache_list != NULL) { 1313 1314 if (*__kmp_threadpriv_cache_list->addr != NULL) { 1315 KC_TRACE(50, ("__kmp_atfork_child: zeroing cache at address %p\n", 1316 &(*__kmp_threadpriv_cache_list->addr))); 1317 1318 *__kmp_threadpriv_cache_list->addr = NULL; 1319 } 1320 __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next; 1321 } 1322 1323 __kmp_init_runtime = FALSE; 1324 1325 /* reset statically initialized locks */ 1326 __kmp_init_bootstrap_lock(&__kmp_initz_lock); 1327 __kmp_init_bootstrap_lock(&__kmp_stdio_lock); 1328 __kmp_init_bootstrap_lock(&__kmp_console_lock); 1329 __kmp_init_bootstrap_lock(&__kmp_task_team_lock); 1330 1331 #if USE_ITT_BUILD 1332 __kmp_itt_reset(); // reset ITT's global state 1333 #endif /* USE_ITT_BUILD */ 1334 1335 { 1336 // Child process often get terminated without any use of OpenMP. That might 1337 // cause mapped shared memory file to be left unattended. Thus we postpone 1338 // library registration till middle initialization in the child process. 1339 __kmp_need_register_serial = FALSE; 1340 __kmp_serial_initialize(); 1341 } 1342 1343 /* This is necessary to make sure no stale data is left around */ 1344 /* AC: customers complain that we use unsafe routines in the atfork 1345 handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen 1346 in dynamic_link when check the presence of shared tbbmalloc library. 1347 Suggestion is to make the library initialization lazier, similar 1348 to what done for __kmpc_begin(). */ 1349 // TODO: synchronize all static initializations with regular library 1350 // startup; look at kmp_global.cpp and etc. 1351 //__kmp_internal_begin (); 1352 } 1353 1354 void __kmp_register_atfork(void) { 1355 if (__kmp_need_register_atfork) { 1356 #if !KMP_OS_WASI 1357 int status = pthread_atfork(__kmp_atfork_prepare, __kmp_atfork_parent, 1358 __kmp_atfork_child); 1359 KMP_CHECK_SYSFAIL("pthread_atfork", status); 1360 #endif 1361 __kmp_need_register_atfork = FALSE; 1362 } 1363 } 1364 1365 void __kmp_suspend_initialize(void) { 1366 int status; 1367 status = pthread_mutexattr_init(&__kmp_suspend_mutex_attr); 1368 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status); 1369 status = pthread_condattr_init(&__kmp_suspend_cond_attr); 1370 KMP_CHECK_SYSFAIL("pthread_condattr_init", status); 1371 } 1372 1373 void __kmp_suspend_initialize_thread(kmp_info_t *th) { 1374 int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count); 1375 int new_value = __kmp_fork_count + 1; 1376 // Return if already initialized 1377 if (old_value == new_value) 1378 return; 1379 // Wait, then return if being initialized 1380 if (old_value == -1 || !__kmp_atomic_compare_store( 1381 &th->th.th_suspend_init_count, old_value, -1)) { 1382 while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) { 1383 KMP_CPU_PAUSE(); 1384 } 1385 } else { 1386 // Claim to be the initializer and do initializations 1387 int status; 1388 status = pthread_cond_init(&th->th.th_suspend_cv.c_cond, 1389 &__kmp_suspend_cond_attr); 1390 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 1391 status = pthread_mutex_init(&th->th.th_suspend_mx.m_mutex, 1392 &__kmp_suspend_mutex_attr); 1393 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 1394 KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value); 1395 } 1396 } 1397 1398 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) { 1399 if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) { 1400 /* this means we have initialize the suspension pthread objects for this 1401 thread in this instance of the process */ 1402 int status; 1403 1404 status = pthread_cond_destroy(&th->th.th_suspend_cv.c_cond); 1405 if (status != 0 && status != EBUSY) { 1406 KMP_SYSFAIL("pthread_cond_destroy", status); 1407 } 1408 status = pthread_mutex_destroy(&th->th.th_suspend_mx.m_mutex); 1409 if (status != 0 && status != EBUSY) { 1410 KMP_SYSFAIL("pthread_mutex_destroy", status); 1411 } 1412 --th->th.th_suspend_init_count; 1413 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count) == 1414 __kmp_fork_count); 1415 } 1416 } 1417 1418 // return true if lock obtained, false otherwise 1419 int __kmp_try_suspend_mx(kmp_info_t *th) { 1420 return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0); 1421 } 1422 1423 void __kmp_lock_suspend_mx(kmp_info_t *th) { 1424 int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex); 1425 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 1426 } 1427 1428 void __kmp_unlock_suspend_mx(kmp_info_t *th) { 1429 int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex); 1430 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 1431 } 1432 1433 /* This routine puts the calling thread to sleep after setting the 1434 sleep bit for the indicated flag variable to true. */ 1435 template <class C> 1436 static inline void __kmp_suspend_template(int th_gtid, C *flag) { 1437 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend); 1438 kmp_info_t *th = __kmp_threads[th_gtid]; 1439 int status; 1440 typename C::flag_t old_spin; 1441 1442 KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, 1443 flag->get())); 1444 1445 __kmp_suspend_initialize_thread(th); 1446 1447 __kmp_lock_suspend_mx(th); 1448 1449 KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n", 1450 th_gtid, flag->get())); 1451 1452 /* TODO: shouldn't this use release semantics to ensure that 1453 __kmp_suspend_initialize_thread gets called first? */ 1454 old_spin = flag->set_sleeping(); 1455 TCW_PTR(th->th.th_sleep_loc, (void *)flag); 1456 th->th.th_sleep_loc_type = flag->get_type(); 1457 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && 1458 __kmp_pause_status != kmp_soft_paused) { 1459 flag->unset_sleeping(); 1460 TCW_PTR(th->th.th_sleep_loc, NULL); 1461 th->th.th_sleep_loc_type = flag_unset; 1462 __kmp_unlock_suspend_mx(th); 1463 return; 1464 } 1465 KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x," 1466 " was %x\n", 1467 th_gtid, flag->get(), flag->load(), old_spin)); 1468 1469 if (flag->done_check_val(old_spin) || flag->done_check()) { 1470 flag->unset_sleeping(); 1471 TCW_PTR(th->th.th_sleep_loc, NULL); 1472 th->th.th_sleep_loc_type = flag_unset; 1473 KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit " 1474 "for spin(%p)\n", 1475 th_gtid, flag->get())); 1476 } else { 1477 /* Encapsulate in a loop as the documentation states that this may 1478 "with low probability" return when the condition variable has 1479 not been signaled or broadcast */ 1480 int deactivated = FALSE; 1481 1482 while (flag->is_sleeping()) { 1483 #ifdef DEBUG_SUSPEND 1484 char buffer[128]; 1485 __kmp_suspend_count++; 1486 __kmp_print_cond(buffer, &th->th.th_suspend_cv); 1487 __kmp_printf("__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, 1488 buffer); 1489 #endif 1490 // Mark the thread as no longer active (only in the first iteration of the 1491 // loop). 1492 if (!deactivated) { 1493 th->th.th_active = FALSE; 1494 if (th->th.th_active_in_pool) { 1495 th->th.th_active_in_pool = FALSE; 1496 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth); 1497 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0); 1498 } 1499 deactivated = TRUE; 1500 } 1501 1502 KMP_DEBUG_ASSERT(th->th.th_sleep_loc); 1503 KMP_DEBUG_ASSERT(flag->get_type() == th->th.th_sleep_loc_type); 1504 1505 #if USE_SUSPEND_TIMEOUT 1506 struct timespec now; 1507 struct timeval tval; 1508 int msecs; 1509 1510 status = gettimeofday(&tval, NULL); 1511 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 1512 TIMEVAL_TO_TIMESPEC(&tval, &now); 1513 1514 msecs = (4 * __kmp_dflt_blocktime) + 200; 1515 now.tv_sec += msecs / 1000; 1516 now.tv_nsec += (msecs % 1000) * 1000; 1517 1518 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform " 1519 "pthread_cond_timedwait\n", 1520 th_gtid)); 1521 status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond, 1522 &th->th.th_suspend_mx.m_mutex, &now); 1523 #else 1524 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform" 1525 " pthread_cond_wait\n", 1526 th_gtid)); 1527 status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond, 1528 &th->th.th_suspend_mx.m_mutex); 1529 #endif // USE_SUSPEND_TIMEOUT 1530 1531 if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) { 1532 KMP_SYSFAIL("pthread_cond_wait", status); 1533 } 1534 1535 KMP_DEBUG_ASSERT(flag->get_type() == flag->get_ptr_type()); 1536 1537 if (!flag->is_sleeping() && 1538 ((status == EINTR) || (status == ETIMEDOUT))) { 1539 // if interrupt or timeout, and thread is no longer sleeping, we need to 1540 // make sure sleep_loc gets reset; however, this shouldn't be needed if 1541 // we woke up with resume 1542 flag->unset_sleeping(); 1543 TCW_PTR(th->th.th_sleep_loc, NULL); 1544 th->th.th_sleep_loc_type = flag_unset; 1545 } 1546 #ifdef KMP_DEBUG 1547 if (status == ETIMEDOUT) { 1548 if (flag->is_sleeping()) { 1549 KF_TRACE(100, 1550 ("__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid)); 1551 } else { 1552 KF_TRACE(2, ("__kmp_suspend_template: T#%d timeout wakeup, sleep bit " 1553 "not set!\n", 1554 th_gtid)); 1555 TCW_PTR(th->th.th_sleep_loc, NULL); 1556 th->th.th_sleep_loc_type = flag_unset; 1557 } 1558 } else if (flag->is_sleeping()) { 1559 KF_TRACE(100, 1560 ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid)); 1561 } 1562 #endif 1563 } // while 1564 1565 // Mark the thread as active again (if it was previous marked as inactive) 1566 if (deactivated) { 1567 th->th.th_active = TRUE; 1568 if (TCR_4(th->th.th_in_pool)) { 1569 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth); 1570 th->th.th_active_in_pool = TRUE; 1571 } 1572 } 1573 } 1574 // We may have had the loop variable set before entering the loop body; 1575 // so we need to reset sleep_loc. 1576 TCW_PTR(th->th.th_sleep_loc, NULL); 1577 th->th.th_sleep_loc_type = flag_unset; 1578 1579 KMP_DEBUG_ASSERT(!flag->is_sleeping()); 1580 KMP_DEBUG_ASSERT(!th->th.th_sleep_loc); 1581 #ifdef DEBUG_SUSPEND 1582 { 1583 char buffer[128]; 1584 __kmp_print_cond(buffer, &th->th.th_suspend_cv); 1585 __kmp_printf("__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, 1586 buffer); 1587 } 1588 #endif 1589 1590 __kmp_unlock_suspend_mx(th); 1591 KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid)); 1592 } 1593 1594 template <bool C, bool S> 1595 void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) { 1596 __kmp_suspend_template(th_gtid, flag); 1597 } 1598 template <bool C, bool S> 1599 void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) { 1600 __kmp_suspend_template(th_gtid, flag); 1601 } 1602 template <bool C, bool S> 1603 void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag) { 1604 __kmp_suspend_template(th_gtid, flag); 1605 } 1606 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) { 1607 __kmp_suspend_template(th_gtid, flag); 1608 } 1609 1610 template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *); 1611 template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *); 1612 template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *); 1613 template void 1614 __kmp_atomic_suspend_64<false, true>(int, kmp_atomic_flag_64<false, true> *); 1615 template void 1616 __kmp_atomic_suspend_64<true, false>(int, kmp_atomic_flag_64<true, false> *); 1617 1618 /* This routine signals the thread specified by target_gtid to wake up 1619 after setting the sleep bit indicated by the flag argument to FALSE. 1620 The target thread must already have called __kmp_suspend_template() */ 1621 template <class C> 1622 static inline void __kmp_resume_template(int target_gtid, C *flag) { 1623 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume); 1624 kmp_info_t *th = __kmp_threads[target_gtid]; 1625 int status; 1626 1627 #ifdef KMP_DEBUG 1628 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1; 1629 #endif 1630 1631 KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", 1632 gtid, target_gtid)); 1633 KMP_DEBUG_ASSERT(gtid != target_gtid); 1634 1635 __kmp_suspend_initialize_thread(th); 1636 1637 __kmp_lock_suspend_mx(th); 1638 1639 if (!flag || flag != th->th.th_sleep_loc) { 1640 // coming from __kmp_null_resume_wrapper, or thread is now sleeping on a 1641 // different location; wake up at new location 1642 flag = (C *)CCAST(void *, th->th.th_sleep_loc); 1643 } 1644 1645 // First, check if the flag is null or its type has changed. If so, someone 1646 // else woke it up. 1647 if (!flag) { // Thread doesn't appear to be sleeping on anything 1648 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already " 1649 "awake: flag(%p)\n", 1650 gtid, target_gtid, (void *)NULL)); 1651 __kmp_unlock_suspend_mx(th); 1652 return; 1653 } else if (flag->get_type() != th->th.th_sleep_loc_type) { 1654 // Flag type does not appear to match this function template; possibly the 1655 // thread is sleeping on something else. Try null resume again. 1656 KF_TRACE( 1657 5, 1658 ("__kmp_resume_template: T#%d retrying, thread T#%d Mismatch flag(%p), " 1659 "spin(%p) type=%d ptr_type=%d\n", 1660 gtid, target_gtid, flag, flag->get(), flag->get_type(), 1661 th->th.th_sleep_loc_type)); 1662 __kmp_unlock_suspend_mx(th); 1663 __kmp_null_resume_wrapper(th); 1664 return; 1665 } else { // if multiple threads are sleeping, flag should be internally 1666 // referring to a specific thread here 1667 if (!flag->is_sleeping()) { 1668 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already " 1669 "awake: flag(%p): %u\n", 1670 gtid, target_gtid, flag->get(), (unsigned int)flag->load())); 1671 __kmp_unlock_suspend_mx(th); 1672 return; 1673 } 1674 } 1675 KMP_DEBUG_ASSERT(flag); 1676 flag->unset_sleeping(); 1677 TCW_PTR(th->th.th_sleep_loc, NULL); 1678 th->th.th_sleep_loc_type = flag_unset; 1679 1680 KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset " 1681 "sleep bit for flag's loc(%p): %u\n", 1682 gtid, target_gtid, flag->get(), (unsigned int)flag->load())); 1683 1684 #ifdef DEBUG_SUSPEND 1685 { 1686 char buffer[128]; 1687 __kmp_print_cond(buffer, &th->th.th_suspend_cv); 1688 __kmp_printf("__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, 1689 target_gtid, buffer); 1690 } 1691 #endif 1692 status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond); 1693 KMP_CHECK_SYSFAIL("pthread_cond_signal", status); 1694 __kmp_unlock_suspend_mx(th); 1695 KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up" 1696 " for T#%d\n", 1697 gtid, target_gtid)); 1698 } 1699 1700 template <bool C, bool S> 1701 void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) { 1702 __kmp_resume_template(target_gtid, flag); 1703 } 1704 template <bool C, bool S> 1705 void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) { 1706 __kmp_resume_template(target_gtid, flag); 1707 } 1708 template <bool C, bool S> 1709 void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64<C, S> *flag) { 1710 __kmp_resume_template(target_gtid, flag); 1711 } 1712 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) { 1713 __kmp_resume_template(target_gtid, flag); 1714 } 1715 1716 template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *); 1717 template void __kmp_resume_32<false, false>(int, kmp_flag_32<false, false> *); 1718 template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *); 1719 template void 1720 __kmp_atomic_resume_64<false, true>(int, kmp_atomic_flag_64<false, true> *); 1721 1722 #if KMP_USE_MONITOR 1723 void __kmp_resume_monitor() { 1724 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume); 1725 int status; 1726 #ifdef KMP_DEBUG 1727 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1; 1728 KF_TRACE(30, ("__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid, 1729 KMP_GTID_MONITOR)); 1730 KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR); 1731 #endif 1732 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex); 1733 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 1734 #ifdef DEBUG_SUSPEND 1735 { 1736 char buffer[128]; 1737 __kmp_print_cond(buffer, &__kmp_wait_cv.c_cond); 1738 __kmp_printf("__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, 1739 KMP_GTID_MONITOR, buffer); 1740 } 1741 #endif 1742 status = pthread_cond_signal(&__kmp_wait_cv.c_cond); 1743 KMP_CHECK_SYSFAIL("pthread_cond_signal", status); 1744 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex); 1745 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 1746 KF_TRACE(30, ("__kmp_resume_monitor: T#%d exiting after signaling wake up" 1747 " for T#%d\n", 1748 gtid, KMP_GTID_MONITOR)); 1749 } 1750 #endif // KMP_USE_MONITOR 1751 1752 void __kmp_yield() { sched_yield(); } 1753 1754 void __kmp_gtid_set_specific(int gtid) { 1755 if (__kmp_init_gtid) { 1756 int status; 1757 status = pthread_setspecific(__kmp_gtid_threadprivate_key, 1758 (void *)(intptr_t)(gtid + 1)); 1759 KMP_CHECK_SYSFAIL("pthread_setspecific", status); 1760 } else { 1761 KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n")); 1762 } 1763 } 1764 1765 int __kmp_gtid_get_specific() { 1766 int gtid; 1767 if (!__kmp_init_gtid) { 1768 KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning " 1769 "KMP_GTID_SHUTDOWN\n")); 1770 return KMP_GTID_SHUTDOWN; 1771 } 1772 gtid = (int)(size_t)pthread_getspecific(__kmp_gtid_threadprivate_key); 1773 if (gtid == 0) { 1774 gtid = KMP_GTID_DNE; 1775 } else { 1776 gtid--; 1777 } 1778 KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n", 1779 __kmp_gtid_threadprivate_key, gtid)); 1780 return gtid; 1781 } 1782 1783 double __kmp_read_cpu_time(void) { 1784 /*clock_t t;*/ 1785 struct tms buffer; 1786 1787 /*t =*/times(&buffer); 1788 1789 return (double)(buffer.tms_utime + buffer.tms_cutime) / 1790 (double)CLOCKS_PER_SEC; 1791 } 1792 1793 int __kmp_read_system_info(struct kmp_sys_info *info) { 1794 int status; 1795 struct rusage r_usage; 1796 1797 memset(info, 0, sizeof(*info)); 1798 1799 status = getrusage(RUSAGE_SELF, &r_usage); 1800 KMP_CHECK_SYSFAIL_ERRNO("getrusage", status); 1801 1802 #if !KMP_OS_WASI 1803 // The maximum resident set size utilized (in kilobytes) 1804 info->maxrss = r_usage.ru_maxrss; 1805 // The number of page faults serviced without any I/O 1806 info->minflt = r_usage.ru_minflt; 1807 // The number of page faults serviced that required I/O 1808 info->majflt = r_usage.ru_majflt; 1809 // The number of times a process was "swapped" out of memory 1810 info->nswap = r_usage.ru_nswap; 1811 // The number of times the file system had to perform input 1812 info->inblock = r_usage.ru_inblock; 1813 // The number of times the file system had to perform output 1814 info->oublock = r_usage.ru_oublock; 1815 // The number of times a context switch was voluntarily 1816 info->nvcsw = r_usage.ru_nvcsw; 1817 // The number of times a context switch was forced 1818 info->nivcsw = r_usage.ru_nivcsw; 1819 #endif 1820 1821 return (status != 0); 1822 } 1823 1824 void __kmp_read_system_time(double *delta) { 1825 double t_ns; 1826 struct timeval tval; 1827 struct timespec stop; 1828 int status; 1829 1830 status = gettimeofday(&tval, NULL); 1831 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 1832 TIMEVAL_TO_TIMESPEC(&tval, &stop); 1833 t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start)); 1834 *delta = (t_ns * 1e-9); 1835 } 1836 1837 void __kmp_clear_system_time(void) { 1838 struct timeval tval; 1839 int status; 1840 status = gettimeofday(&tval, NULL); 1841 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 1842 TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start); 1843 } 1844 1845 static int __kmp_get_xproc(void) { 1846 1847 int r = 0; 1848 1849 #if KMP_OS_LINUX 1850 1851 __kmp_type_convert(sysconf(_SC_NPROCESSORS_CONF), &(r)); 1852 1853 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || \ 1854 KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX 1855 1856 __kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r)); 1857 1858 #elif KMP_OS_DARWIN 1859 1860 // Bug C77011 High "OpenMP Threads and number of active cores". 1861 1862 // Find the number of available CPUs. 1863 kern_return_t rc; 1864 host_basic_info_data_t info; 1865 mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT; 1866 rc = host_info(mach_host_self(), HOST_BASIC_INFO, (host_info_t)&info, &num); 1867 if (rc == 0 && num == HOST_BASIC_INFO_COUNT) { 1868 // Cannot use KA_TRACE() here because this code works before trace support 1869 // is initialized. 1870 r = info.avail_cpus; 1871 } else { 1872 KMP_WARNING(CantGetNumAvailCPU); 1873 KMP_INFORM(AssumedNumCPU); 1874 } 1875 1876 #else 1877 1878 #error "Unknown or unsupported OS." 1879 1880 #endif 1881 1882 return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */ 1883 1884 } // __kmp_get_xproc 1885 1886 int __kmp_read_from_file(char const *path, char const *format, ...) { 1887 int result; 1888 va_list args; 1889 1890 va_start(args, format); 1891 FILE *f = fopen(path, "rb"); 1892 if (f == NULL) { 1893 va_end(args); 1894 return 0; 1895 } 1896 result = vfscanf(f, format, args); 1897 fclose(f); 1898 va_end(args); 1899 1900 return result; 1901 } 1902 1903 void __kmp_runtime_initialize(void) { 1904 int status; 1905 pthread_mutexattr_t mutex_attr; 1906 pthread_condattr_t cond_attr; 1907 1908 if (__kmp_init_runtime) { 1909 return; 1910 } 1911 1912 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 1913 if (!__kmp_cpuinfo.initialized) { 1914 __kmp_query_cpuid(&__kmp_cpuinfo); 1915 } 1916 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 1917 1918 __kmp_xproc = __kmp_get_xproc(); 1919 1920 #if !KMP_32_BIT_ARCH 1921 struct rlimit rlim; 1922 // read stack size of calling thread, save it as default for worker threads; 1923 // this should be done before reading environment variables 1924 status = getrlimit(RLIMIT_STACK, &rlim); 1925 if (status == 0) { // success? 1926 __kmp_stksize = rlim.rlim_cur; 1927 __kmp_check_stksize(&__kmp_stksize); // check value and adjust if needed 1928 } 1929 #endif /* KMP_32_BIT_ARCH */ 1930 1931 if (sysconf(_SC_THREADS)) { 1932 1933 /* Query the maximum number of threads */ 1934 __kmp_type_convert(sysconf(_SC_THREAD_THREADS_MAX), &(__kmp_sys_max_nth)); 1935 #ifdef __ve__ 1936 if (__kmp_sys_max_nth == -1) { 1937 // VE's pthread supports only up to 64 threads per a VE process. 1938 // So we use that KMP_MAX_NTH (predefined as 64) here. 1939 __kmp_sys_max_nth = KMP_MAX_NTH; 1940 } 1941 #else 1942 if (__kmp_sys_max_nth == -1) { 1943 /* Unlimited threads for NPTL */ 1944 __kmp_sys_max_nth = INT_MAX; 1945 } else if (__kmp_sys_max_nth <= 1) { 1946 /* Can't tell, just use PTHREAD_THREADS_MAX */ 1947 __kmp_sys_max_nth = KMP_MAX_NTH; 1948 } 1949 #endif 1950 1951 /* Query the minimum stack size */ 1952 __kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN); 1953 if (__kmp_sys_min_stksize <= 1) { 1954 __kmp_sys_min_stksize = KMP_MIN_STKSIZE; 1955 } 1956 } 1957 1958 /* Set up minimum number of threads to switch to TLS gtid */ 1959 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN; 1960 1961 status = pthread_key_create(&__kmp_gtid_threadprivate_key, 1962 __kmp_internal_end_dest); 1963 KMP_CHECK_SYSFAIL("pthread_key_create", status); 1964 status = pthread_mutexattr_init(&mutex_attr); 1965 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status); 1966 status = pthread_mutex_init(&__kmp_wait_mx.m_mutex, &mutex_attr); 1967 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 1968 status = pthread_mutexattr_destroy(&mutex_attr); 1969 KMP_CHECK_SYSFAIL("pthread_mutexattr_destroy", status); 1970 status = pthread_condattr_init(&cond_attr); 1971 KMP_CHECK_SYSFAIL("pthread_condattr_init", status); 1972 status = pthread_cond_init(&__kmp_wait_cv.c_cond, &cond_attr); 1973 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 1974 status = pthread_condattr_destroy(&cond_attr); 1975 KMP_CHECK_SYSFAIL("pthread_condattr_destroy", status); 1976 #if USE_ITT_BUILD 1977 __kmp_itt_initialize(); 1978 #endif /* USE_ITT_BUILD */ 1979 1980 __kmp_init_runtime = TRUE; 1981 } 1982 1983 void __kmp_runtime_destroy(void) { 1984 int status; 1985 1986 if (!__kmp_init_runtime) { 1987 return; // Nothing to do. 1988 } 1989 1990 #if USE_ITT_BUILD 1991 __kmp_itt_destroy(); 1992 #endif /* USE_ITT_BUILD */ 1993 1994 status = pthread_key_delete(__kmp_gtid_threadprivate_key); 1995 KMP_CHECK_SYSFAIL("pthread_key_delete", status); 1996 1997 status = pthread_mutex_destroy(&__kmp_wait_mx.m_mutex); 1998 if (status != 0 && status != EBUSY) { 1999 KMP_SYSFAIL("pthread_mutex_destroy", status); 2000 } 2001 status = pthread_cond_destroy(&__kmp_wait_cv.c_cond); 2002 if (status != 0 && status != EBUSY) { 2003 KMP_SYSFAIL("pthread_cond_destroy", status); 2004 } 2005 #if KMP_AFFINITY_SUPPORTED 2006 __kmp_affinity_uninitialize(); 2007 #endif 2008 2009 __kmp_init_runtime = FALSE; 2010 } 2011 2012 /* Put the thread to sleep for a time period */ 2013 /* NOTE: not currently used anywhere */ 2014 void __kmp_thread_sleep(int millis) { sleep((millis + 500) / 1000); } 2015 2016 /* Calculate the elapsed wall clock time for the user */ 2017 void __kmp_elapsed(double *t) { 2018 int status; 2019 #ifdef FIX_SGI_CLOCK 2020 struct timespec ts; 2021 2022 status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); 2023 KMP_CHECK_SYSFAIL_ERRNO("clock_gettime", status); 2024 *t = 2025 (double)ts.tv_nsec * (1.0 / (double)KMP_NSEC_PER_SEC) + (double)ts.tv_sec; 2026 #else 2027 struct timeval tv; 2028 2029 status = gettimeofday(&tv, NULL); 2030 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 2031 *t = 2032 (double)tv.tv_usec * (1.0 / (double)KMP_USEC_PER_SEC) + (double)tv.tv_sec; 2033 #endif 2034 } 2035 2036 /* Calculate the elapsed wall clock tick for the user */ 2037 void __kmp_elapsed_tick(double *t) { *t = 1 / (double)CLOCKS_PER_SEC; } 2038 2039 /* Return the current time stamp in nsec */ 2040 kmp_uint64 __kmp_now_nsec() { 2041 struct timeval t; 2042 gettimeofday(&t, NULL); 2043 kmp_uint64 nsec = (kmp_uint64)KMP_NSEC_PER_SEC * (kmp_uint64)t.tv_sec + 2044 (kmp_uint64)1000 * (kmp_uint64)t.tv_usec; 2045 return nsec; 2046 } 2047 2048 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 2049 /* Measure clock ticks per millisecond */ 2050 void __kmp_initialize_system_tick() { 2051 kmp_uint64 now, nsec2, diff; 2052 kmp_uint64 delay = 1000000; // ~450 usec on most machines. 2053 kmp_uint64 nsec = __kmp_now_nsec(); 2054 kmp_uint64 goal = __kmp_hardware_timestamp() + delay; 2055 while ((now = __kmp_hardware_timestamp()) < goal) 2056 ; 2057 nsec2 = __kmp_now_nsec(); 2058 diff = nsec2 - nsec; 2059 if (diff > 0) { 2060 double tpus = 1000.0 * (double)(delay + (now - goal)) / (double)diff; 2061 if (tpus > 0.0) { 2062 __kmp_ticks_per_msec = (kmp_uint64)(tpus * 1000.0); 2063 __kmp_ticks_per_usec = (kmp_uint64)tpus; 2064 } 2065 } 2066 } 2067 #endif 2068 2069 /* Determine whether the given address is mapped into the current address 2070 space. */ 2071 2072 int __kmp_is_address_mapped(void *addr) { 2073 2074 int found = 0; 2075 int rc; 2076 2077 #if KMP_OS_LINUX || KMP_OS_HURD 2078 2079 /* On GNUish OSes, read the /proc/<pid>/maps pseudo-file to get all the 2080 address ranges mapped into the address space. */ 2081 2082 char *name = __kmp_str_format("/proc/%d/maps", getpid()); 2083 FILE *file = NULL; 2084 2085 file = fopen(name, "r"); 2086 KMP_ASSERT(file != NULL); 2087 2088 for (;;) { 2089 2090 void *beginning = NULL; 2091 void *ending = NULL; 2092 char perms[5]; 2093 2094 rc = fscanf(file, "%p-%p %4s %*[^\n]\n", &beginning, &ending, perms); 2095 if (rc == EOF) { 2096 break; 2097 } 2098 KMP_ASSERT(rc == 3 && 2099 KMP_STRLEN(perms) == 4); // Make sure all fields are read. 2100 2101 // Ending address is not included in the region, but beginning is. 2102 if ((addr >= beginning) && (addr < ending)) { 2103 perms[2] = 0; // 3th and 4th character does not matter. 2104 if (strcmp(perms, "rw") == 0) { 2105 // Memory we are looking for should be readable and writable. 2106 found = 1; 2107 } 2108 break; 2109 } 2110 } 2111 2112 // Free resources. 2113 fclose(file); 2114 KMP_INTERNAL_FREE(name); 2115 #elif KMP_OS_FREEBSD 2116 char *buf; 2117 size_t lstsz; 2118 int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()}; 2119 rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0); 2120 if (rc < 0) 2121 return 0; 2122 // We pass from number of vm entry's semantic 2123 // to size of whole entry map list. 2124 lstsz = lstsz * 4 / 3; 2125 buf = reinterpret_cast<char *>(kmpc_malloc(lstsz)); 2126 rc = sysctl(mib, 4, buf, &lstsz, NULL, 0); 2127 if (rc < 0) { 2128 kmpc_free(buf); 2129 return 0; 2130 } 2131 2132 char *lw = buf; 2133 char *up = buf + lstsz; 2134 2135 while (lw < up) { 2136 struct kinfo_vmentry *cur = reinterpret_cast<struct kinfo_vmentry *>(lw); 2137 size_t cursz = cur->kve_structsize; 2138 if (cursz == 0) 2139 break; 2140 void *start = reinterpret_cast<void *>(cur->kve_start); 2141 void *end = reinterpret_cast<void *>(cur->kve_end); 2142 // Readable/Writable addresses within current map entry 2143 if ((addr >= start) && (addr < end)) { 2144 if ((cur->kve_protection & KVME_PROT_READ) != 0 && 2145 (cur->kve_protection & KVME_PROT_WRITE) != 0) { 2146 found = 1; 2147 break; 2148 } 2149 } 2150 lw += cursz; 2151 } 2152 kmpc_free(buf); 2153 #elif KMP_OS_DRAGONFLY 2154 char err[_POSIX2_LINE_MAX]; 2155 kinfo_proc *proc; 2156 vmspace sp; 2157 vm_map *cur; 2158 vm_map_entry entry, *c; 2159 struct proc p; 2160 kvm_t *fd; 2161 uintptr_t uaddr; 2162 int num; 2163 2164 fd = kvm_openfiles(nullptr, nullptr, nullptr, O_RDONLY, err); 2165 if (!fd) { 2166 return 0; 2167 } 2168 2169 proc = kvm_getprocs(fd, KERN_PROC_PID, getpid(), &num); 2170 2171 if (kvm_read(fd, static_cast<uintptr_t>(proc->kp_paddr), &p, sizeof(p)) != 2172 sizeof(p) || 2173 kvm_read(fd, reinterpret_cast<uintptr_t>(p.p_vmspace), &sp, sizeof(sp)) != 2174 sizeof(sp)) { 2175 kvm_close(fd); 2176 return 0; 2177 } 2178 2179 (void)rc; 2180 cur = &sp.vm_map; 2181 uaddr = reinterpret_cast<uintptr_t>(addr); 2182 for (c = kvm_vm_map_entry_first(fd, cur, &entry); c; 2183 c = kvm_vm_map_entry_next(fd, c, &entry)) { 2184 if ((uaddr >= entry.ba.start) && (uaddr <= entry.ba.end)) { 2185 if ((entry.protection & VM_PROT_READ) != 0 && 2186 (entry.protection & VM_PROT_WRITE) != 0) { 2187 found = 1; 2188 break; 2189 } 2190 } 2191 } 2192 2193 kvm_close(fd); 2194 #elif KMP_OS_SOLARIS 2195 prmap_t *cur, *map; 2196 void *buf; 2197 uintptr_t uaddr; 2198 ssize_t rd; 2199 int err; 2200 int file; 2201 2202 pid_t pid = getpid(); 2203 struct ps_prochandle *fd = Pgrab(pid, PGRAB_RDONLY, &err); 2204 ; 2205 2206 if (!fd) { 2207 return 0; 2208 } 2209 2210 char *name = __kmp_str_format("/proc/%d/map", pid); 2211 size_t sz = (1 << 20); 2212 file = open(name, O_RDONLY); 2213 if (file == -1) { 2214 KMP_INTERNAL_FREE(name); 2215 return 0; 2216 } 2217 2218 buf = kmpc_malloc(sz); 2219 2220 while (sz > 0 && (rd = pread(file, buf, sz, 0)) == sz) { 2221 void *newbuf; 2222 sz <<= 1; 2223 newbuf = kmpc_realloc(buf, sz); 2224 buf = newbuf; 2225 } 2226 2227 map = reinterpret_cast<prmap_t *>(buf); 2228 uaddr = reinterpret_cast<uintptr_t>(addr); 2229 2230 for (cur = map; rd > 0; cur++, rd = -sizeof(*map)) { 2231 if ((uaddr >= cur->pr_vaddr) && (uaddr < cur->pr_vaddr)) { 2232 if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) { 2233 found = 1; 2234 break; 2235 } 2236 } 2237 } 2238 2239 kmpc_free(map); 2240 close(file); 2241 KMP_INTERNAL_FREE(name); 2242 #elif KMP_OS_DARWIN 2243 2244 /* On OS X*, /proc pseudo filesystem is not available. Try to read memory 2245 using vm interface. */ 2246 2247 int buffer; 2248 vm_size_t count; 2249 rc = vm_read_overwrite( 2250 mach_task_self(), // Task to read memory of. 2251 (vm_address_t)(addr), // Address to read from. 2252 1, // Number of bytes to be read. 2253 (vm_address_t)(&buffer), // Address of buffer to save read bytes in. 2254 &count // Address of var to save number of read bytes in. 2255 ); 2256 if (rc == 0) { 2257 // Memory successfully read. 2258 found = 1; 2259 } 2260 2261 #elif KMP_OS_NETBSD 2262 2263 int mib[5]; 2264 mib[0] = CTL_VM; 2265 mib[1] = VM_PROC; 2266 mib[2] = VM_PROC_MAP; 2267 mib[3] = getpid(); 2268 mib[4] = sizeof(struct kinfo_vmentry); 2269 2270 size_t size; 2271 rc = sysctl(mib, __arraycount(mib), NULL, &size, NULL, 0); 2272 KMP_ASSERT(!rc); 2273 KMP_ASSERT(size); 2274 2275 size = size * 4 / 3; 2276 struct kinfo_vmentry *kiv = (struct kinfo_vmentry *)KMP_INTERNAL_MALLOC(size); 2277 KMP_ASSERT(kiv); 2278 2279 rc = sysctl(mib, __arraycount(mib), kiv, &size, NULL, 0); 2280 KMP_ASSERT(!rc); 2281 KMP_ASSERT(size); 2282 2283 for (size_t i = 0; i < size; i++) { 2284 if (kiv[i].kve_start >= (uint64_t)addr && 2285 kiv[i].kve_end <= (uint64_t)addr) { 2286 found = 1; 2287 break; 2288 } 2289 } 2290 KMP_INTERNAL_FREE(kiv); 2291 #elif KMP_OS_OPENBSD 2292 2293 int mib[3]; 2294 mib[0] = CTL_KERN; 2295 mib[1] = KERN_PROC_VMMAP; 2296 mib[2] = getpid(); 2297 2298 size_t size; 2299 uint64_t end; 2300 rc = sysctl(mib, 3, NULL, &size, NULL, 0); 2301 KMP_ASSERT(!rc); 2302 KMP_ASSERT(size); 2303 end = size; 2304 2305 struct kinfo_vmentry kiv = {.kve_start = 0}; 2306 2307 while ((rc = sysctl(mib, 3, &kiv, &size, NULL, 0)) == 0) { 2308 KMP_ASSERT(size); 2309 if (kiv.kve_end == end) 2310 break; 2311 2312 if (kiv.kve_start >= (uint64_t)addr && kiv.kve_end <= (uint64_t)addr) { 2313 found = 1; 2314 break; 2315 } 2316 kiv.kve_start += 1; 2317 } 2318 #elif KMP_OS_WASI 2319 found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE); 2320 #elif KMP_OS_AIX 2321 2322 // FIXME(AIX): Implement this 2323 found = 1; 2324 2325 #else 2326 2327 #error "Unknown or unsupported OS" 2328 2329 #endif 2330 2331 return found; 2332 2333 } // __kmp_is_address_mapped 2334 2335 #ifdef USE_LOAD_BALANCE 2336 2337 #if KMP_OS_DARWIN || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 2338 KMP_OS_OPENBSD || KMP_OS_SOLARIS 2339 2340 // The function returns the rounded value of the system load average 2341 // during given time interval which depends on the value of 2342 // __kmp_load_balance_interval variable (default is 60 sec, other values 2343 // may be 300 sec or 900 sec). 2344 // It returns -1 in case of error. 2345 int __kmp_get_load_balance(int max) { 2346 double averages[3]; 2347 int ret_avg = 0; 2348 2349 int res = getloadavg(averages, 3); 2350 2351 // Check __kmp_load_balance_interval to determine which of averages to use. 2352 // getloadavg() may return the number of samples less than requested that is 2353 // less than 3. 2354 if (__kmp_load_balance_interval < 180 && (res >= 1)) { 2355 ret_avg = (int)averages[0]; // 1 min 2356 } else if ((__kmp_load_balance_interval >= 180 && 2357 __kmp_load_balance_interval < 600) && 2358 (res >= 2)) { 2359 ret_avg = (int)averages[1]; // 5 min 2360 } else if ((__kmp_load_balance_interval >= 600) && (res == 3)) { 2361 ret_avg = (int)averages[2]; // 15 min 2362 } else { // Error occurred 2363 return -1; 2364 } 2365 2366 return ret_avg; 2367 } 2368 2369 #else // Linux* OS 2370 2371 // The function returns number of running (not sleeping) threads, or -1 in case 2372 // of error. Error could be reported if Linux* OS kernel too old (without 2373 // "/proc" support). Counting running threads stops if max running threads 2374 // encountered. 2375 int __kmp_get_load_balance(int max) { 2376 static int permanent_error = 0; 2377 static int glb_running_threads = 0; // Saved count of the running threads for 2378 // the thread balance algorithm 2379 static double glb_call_time = 0; /* Thread balance algorithm call time */ 2380 2381 int running_threads = 0; // Number of running threads in the system. 2382 2383 DIR *proc_dir = NULL; // Handle of "/proc/" directory. 2384 struct dirent *proc_entry = NULL; 2385 2386 kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path. 2387 DIR *task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory. 2388 struct dirent *task_entry = NULL; 2389 int task_path_fixed_len; 2390 2391 kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path. 2392 int stat_file = -1; 2393 int stat_path_fixed_len; 2394 2395 #ifdef KMP_DEBUG 2396 int total_processes = 0; // Total number of processes in system. 2397 #endif 2398 2399 double call_time = 0.0; 2400 2401 __kmp_str_buf_init(&task_path); 2402 __kmp_str_buf_init(&stat_path); 2403 2404 __kmp_elapsed(&call_time); 2405 2406 if (glb_call_time && 2407 (call_time - glb_call_time < __kmp_load_balance_interval)) { 2408 running_threads = glb_running_threads; 2409 goto finish; 2410 } 2411 2412 glb_call_time = call_time; 2413 2414 // Do not spend time on scanning "/proc/" if we have a permanent error. 2415 if (permanent_error) { 2416 running_threads = -1; 2417 goto finish; 2418 } 2419 2420 if (max <= 0) { 2421 max = INT_MAX; 2422 } 2423 2424 // Open "/proc/" directory. 2425 proc_dir = opendir("/proc"); 2426 if (proc_dir == NULL) { 2427 // Cannot open "/proc/". Probably the kernel does not support it. Return an 2428 // error now and in subsequent calls. 2429 running_threads = -1; 2430 permanent_error = 1; 2431 goto finish; 2432 } 2433 2434 // Initialize fixed part of task_path. This part will not change. 2435 __kmp_str_buf_cat(&task_path, "/proc/", 6); 2436 task_path_fixed_len = task_path.used; // Remember number of used characters. 2437 2438 proc_entry = readdir(proc_dir); 2439 while (proc_entry != NULL) { 2440 #if KMP_OS_AIX 2441 // Proc entry name starts with a digit. Assume it is a process' directory. 2442 if (isdigit(proc_entry->d_name[0])) { 2443 #else 2444 // Proc entry is a directory and name starts with a digit. Assume it is a 2445 // process' directory. 2446 if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) { 2447 #endif 2448 2449 #ifdef KMP_DEBUG 2450 ++total_processes; 2451 #endif 2452 // Make sure init process is the very first in "/proc", so we can replace 2453 // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes == 2454 // 1. We are going to check that total_processes == 1 => d_name == "1" is 2455 // true (where "=>" is implication). Since C++ does not have => operator, 2456 // let us replace it with its equivalent: a => b == ! a || b. 2457 KMP_DEBUG_ASSERT(total_processes != 1 || 2458 strcmp(proc_entry->d_name, "1") == 0); 2459 2460 // Construct task_path. 2461 task_path.used = task_path_fixed_len; // Reset task_path to "/proc/". 2462 __kmp_str_buf_cat(&task_path, proc_entry->d_name, 2463 KMP_STRLEN(proc_entry->d_name)); 2464 __kmp_str_buf_cat(&task_path, "/task", 5); 2465 2466 task_dir = opendir(task_path.str); 2467 if (task_dir == NULL) { 2468 // Process can finish between reading "/proc/" directory entry and 2469 // opening process' "task/" directory. So, in general case we should not 2470 // complain, but have to skip this process and read the next one. But on 2471 // systems with no "task/" support we will spend lot of time to scan 2472 // "/proc/" tree again and again without any benefit. "init" process 2473 // (its pid is 1) should exist always, so, if we cannot open 2474 // "/proc/1/task/" directory, it means "task/" is not supported by 2475 // kernel. Report an error now and in the future. 2476 if (strcmp(proc_entry->d_name, "1") == 0) { 2477 running_threads = -1; 2478 permanent_error = 1; 2479 goto finish; 2480 } 2481 } else { 2482 // Construct fixed part of stat file path. 2483 __kmp_str_buf_clear(&stat_path); 2484 __kmp_str_buf_cat(&stat_path, task_path.str, task_path.used); 2485 __kmp_str_buf_cat(&stat_path, "/", 1); 2486 stat_path_fixed_len = stat_path.used; 2487 2488 task_entry = readdir(task_dir); 2489 while (task_entry != NULL) { 2490 // It is a directory and name starts with a digit. 2491 #if KMP_OS_AIX 2492 if (isdigit(task_entry->d_name[0])) { 2493 #else 2494 if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) { 2495 #endif 2496 2497 // Construct complete stat file path. Easiest way would be: 2498 // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str, 2499 // task_entry->d_name ); 2500 // but seriae of __kmp_str_buf_cat works a bit faster. 2501 stat_path.used = 2502 stat_path_fixed_len; // Reset stat path to its fixed part. 2503 __kmp_str_buf_cat(&stat_path, task_entry->d_name, 2504 KMP_STRLEN(task_entry->d_name)); 2505 __kmp_str_buf_cat(&stat_path, "/stat", 5); 2506 2507 // Note: Low-level API (open/read/close) is used. High-level API 2508 // (fopen/fclose) works ~ 30 % slower. 2509 stat_file = open(stat_path.str, O_RDONLY); 2510 if (stat_file == -1) { 2511 // We cannot report an error because task (thread) can terminate 2512 // just before reading this file. 2513 } else { 2514 /* Content of "stat" file looks like: 2515 24285 (program) S ... 2516 2517 It is a single line (if program name does not include funny 2518 symbols). First number is a thread id, then name of executable 2519 file name in paretheses, then state of the thread. We need just 2520 thread state. 2521 2522 Good news: Length of program name is 15 characters max. Longer 2523 names are truncated. 2524 2525 Thus, we need rather short buffer: 15 chars for program name + 2526 2 parenthesis, + 3 spaces + ~7 digits of pid = 37. 2527 2528 Bad news: Program name may contain special symbols like space, 2529 closing parenthesis, or even new line. This makes parsing 2530 "stat" file not 100 % reliable. In case of fanny program names 2531 parsing may fail (report incorrect thread state). 2532 2533 Parsing "status" file looks more promissing (due to different 2534 file structure and escaping special symbols) but reading and 2535 parsing of "status" file works slower. 2536 -- ln 2537 */ 2538 char buffer[65]; 2539 ssize_t len; 2540 len = read(stat_file, buffer, sizeof(buffer) - 1); 2541 if (len >= 0) { 2542 buffer[len] = 0; 2543 // Using scanf: 2544 // sscanf( buffer, "%*d (%*s) %c ", & state ); 2545 // looks very nice, but searching for a closing parenthesis 2546 // works a bit faster. 2547 char *close_parent = strstr(buffer, ") "); 2548 if (close_parent != NULL) { 2549 char state = *(close_parent + 2); 2550 if (state == 'R') { 2551 ++running_threads; 2552 if (running_threads >= max) { 2553 goto finish; 2554 } 2555 } 2556 } 2557 } 2558 close(stat_file); 2559 stat_file = -1; 2560 } 2561 } 2562 task_entry = readdir(task_dir); 2563 } 2564 closedir(task_dir); 2565 task_dir = NULL; 2566 } 2567 } 2568 proc_entry = readdir(proc_dir); 2569 } 2570 2571 // There _might_ be a timing hole where the thread executing this 2572 // code get skipped in the load balance, and running_threads is 0. 2573 // Assert in the debug builds only!!! 2574 KMP_DEBUG_ASSERT(running_threads > 0); 2575 if (running_threads <= 0) { 2576 running_threads = 1; 2577 } 2578 2579 finish: // Clean up and exit. 2580 if (proc_dir != NULL) { 2581 closedir(proc_dir); 2582 } 2583 __kmp_str_buf_free(&task_path); 2584 if (task_dir != NULL) { 2585 closedir(task_dir); 2586 } 2587 __kmp_str_buf_free(&stat_path); 2588 if (stat_file != -1) { 2589 close(stat_file); 2590 } 2591 2592 glb_running_threads = running_threads; 2593 2594 return running_threads; 2595 2596 } // __kmp_get_load_balance 2597 2598 #endif // KMP_OS_DARWIN 2599 2600 #endif // USE_LOAD_BALANCE 2601 2602 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \ 2603 ((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || \ 2604 KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || \ 2605 KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF) 2606 2607 // we really only need the case with 1 argument, because CLANG always build 2608 // a struct of pointers to shared variables referenced in the outlined function 2609 int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc, 2610 void *p_argv[] 2611 #if OMPT_SUPPORT 2612 , 2613 void **exit_frame_ptr 2614 #endif 2615 ) { 2616 #if OMPT_SUPPORT 2617 *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); 2618 #endif 2619 2620 switch (argc) { 2621 default: 2622 fprintf(stderr, "Too many args to microtask: %d!\n", argc); 2623 fflush(stderr); 2624 exit(-1); 2625 case 0: 2626 (*pkfn)(>id, &tid); 2627 break; 2628 case 1: 2629 (*pkfn)(>id, &tid, p_argv[0]); 2630 break; 2631 case 2: 2632 (*pkfn)(>id, &tid, p_argv[0], p_argv[1]); 2633 break; 2634 case 3: 2635 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2]); 2636 break; 2637 case 4: 2638 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]); 2639 break; 2640 case 5: 2641 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]); 2642 break; 2643 case 6: 2644 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2645 p_argv[5]); 2646 break; 2647 case 7: 2648 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2649 p_argv[5], p_argv[6]); 2650 break; 2651 case 8: 2652 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2653 p_argv[5], p_argv[6], p_argv[7]); 2654 break; 2655 case 9: 2656 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2657 p_argv[5], p_argv[6], p_argv[7], p_argv[8]); 2658 break; 2659 case 10: 2660 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2661 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]); 2662 break; 2663 case 11: 2664 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2665 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]); 2666 break; 2667 case 12: 2668 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2669 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2670 p_argv[11]); 2671 break; 2672 case 13: 2673 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2674 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2675 p_argv[11], p_argv[12]); 2676 break; 2677 case 14: 2678 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2679 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2680 p_argv[11], p_argv[12], p_argv[13]); 2681 break; 2682 case 15: 2683 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4], 2684 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2685 p_argv[11], p_argv[12], p_argv[13], p_argv[14]); 2686 break; 2687 } 2688 2689 return 1; 2690 } 2691 2692 #endif 2693 2694 #if KMP_OS_LINUX 2695 // Functions for hidden helper task 2696 namespace { 2697 // Condition variable for initializing hidden helper team 2698 pthread_cond_t hidden_helper_threads_initz_cond_var; 2699 pthread_mutex_t hidden_helper_threads_initz_lock; 2700 volatile int hidden_helper_initz_signaled = FALSE; 2701 2702 // Condition variable for deinitializing hidden helper team 2703 pthread_cond_t hidden_helper_threads_deinitz_cond_var; 2704 pthread_mutex_t hidden_helper_threads_deinitz_lock; 2705 volatile int hidden_helper_deinitz_signaled = FALSE; 2706 2707 // Condition variable for the wrapper function of main thread 2708 pthread_cond_t hidden_helper_main_thread_cond_var; 2709 pthread_mutex_t hidden_helper_main_thread_lock; 2710 volatile int hidden_helper_main_thread_signaled = FALSE; 2711 2712 // Semaphore for worker threads. We don't use condition variable here in case 2713 // that when multiple signals are sent at the same time, only one thread might 2714 // be waken. 2715 sem_t hidden_helper_task_sem; 2716 } // namespace 2717 2718 void __kmp_hidden_helper_worker_thread_wait() { 2719 int status = sem_wait(&hidden_helper_task_sem); 2720 KMP_CHECK_SYSFAIL("sem_wait", status); 2721 } 2722 2723 void __kmp_do_initialize_hidden_helper_threads() { 2724 // Initialize condition variable 2725 int status = 2726 pthread_cond_init(&hidden_helper_threads_initz_cond_var, nullptr); 2727 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 2728 2729 status = pthread_cond_init(&hidden_helper_threads_deinitz_cond_var, nullptr); 2730 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 2731 2732 status = pthread_cond_init(&hidden_helper_main_thread_cond_var, nullptr); 2733 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 2734 2735 status = pthread_mutex_init(&hidden_helper_threads_initz_lock, nullptr); 2736 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 2737 2738 status = pthread_mutex_init(&hidden_helper_threads_deinitz_lock, nullptr); 2739 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 2740 2741 status = pthread_mutex_init(&hidden_helper_main_thread_lock, nullptr); 2742 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 2743 2744 // Initialize the semaphore 2745 status = sem_init(&hidden_helper_task_sem, 0, 0); 2746 KMP_CHECK_SYSFAIL("sem_init", status); 2747 2748 // Create a new thread to finish initialization 2749 pthread_t handle; 2750 status = pthread_create( 2751 &handle, nullptr, 2752 [](void *) -> void * { 2753 __kmp_hidden_helper_threads_initz_routine(); 2754 return nullptr; 2755 }, 2756 nullptr); 2757 KMP_CHECK_SYSFAIL("pthread_create", status); 2758 } 2759 2760 void __kmp_hidden_helper_threads_initz_wait() { 2761 // Initial thread waits here for the completion of the initialization. The 2762 // condition variable will be notified by main thread of hidden helper teams. 2763 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock); 2764 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2765 2766 if (!TCR_4(hidden_helper_initz_signaled)) { 2767 status = pthread_cond_wait(&hidden_helper_threads_initz_cond_var, 2768 &hidden_helper_threads_initz_lock); 2769 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2770 } 2771 2772 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock); 2773 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2774 } 2775 2776 void __kmp_hidden_helper_initz_release() { 2777 // After all initialization, reset __kmp_init_hidden_helper_threads to false. 2778 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock); 2779 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2780 2781 status = pthread_cond_signal(&hidden_helper_threads_initz_cond_var); 2782 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2783 2784 TCW_SYNC_4(hidden_helper_initz_signaled, TRUE); 2785 2786 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock); 2787 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2788 } 2789 2790 void __kmp_hidden_helper_main_thread_wait() { 2791 // The main thread of hidden helper team will be blocked here. The 2792 // condition variable can only be signal in the destructor of RTL. 2793 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock); 2794 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2795 2796 if (!TCR_4(hidden_helper_main_thread_signaled)) { 2797 status = pthread_cond_wait(&hidden_helper_main_thread_cond_var, 2798 &hidden_helper_main_thread_lock); 2799 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2800 } 2801 2802 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock); 2803 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2804 } 2805 2806 void __kmp_hidden_helper_main_thread_release() { 2807 // The initial thread of OpenMP RTL should call this function to wake up the 2808 // main thread of hidden helper team. 2809 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock); 2810 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2811 2812 status = pthread_cond_signal(&hidden_helper_main_thread_cond_var); 2813 KMP_CHECK_SYSFAIL("pthread_cond_signal", status); 2814 2815 // The hidden helper team is done here 2816 TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE); 2817 2818 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock); 2819 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2820 } 2821 2822 void __kmp_hidden_helper_worker_thread_signal() { 2823 int status = sem_post(&hidden_helper_task_sem); 2824 KMP_CHECK_SYSFAIL("sem_post", status); 2825 } 2826 2827 void __kmp_hidden_helper_threads_deinitz_wait() { 2828 // Initial thread waits here for the completion of the deinitialization. The 2829 // condition variable will be notified by main thread of hidden helper teams. 2830 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock); 2831 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2832 2833 if (!TCR_4(hidden_helper_deinitz_signaled)) { 2834 status = pthread_cond_wait(&hidden_helper_threads_deinitz_cond_var, 2835 &hidden_helper_threads_deinitz_lock); 2836 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2837 } 2838 2839 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock); 2840 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2841 } 2842 2843 void __kmp_hidden_helper_threads_deinitz_release() { 2844 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock); 2845 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2846 2847 status = pthread_cond_signal(&hidden_helper_threads_deinitz_cond_var); 2848 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2849 2850 TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE); 2851 2852 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock); 2853 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2854 } 2855 #else // KMP_OS_LINUX 2856 void __kmp_hidden_helper_worker_thread_wait() { 2857 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2858 } 2859 2860 void __kmp_do_initialize_hidden_helper_threads() { 2861 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2862 } 2863 2864 void __kmp_hidden_helper_threads_initz_wait() { 2865 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2866 } 2867 2868 void __kmp_hidden_helper_initz_release() { 2869 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2870 } 2871 2872 void __kmp_hidden_helper_main_thread_wait() { 2873 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2874 } 2875 2876 void __kmp_hidden_helper_main_thread_release() { 2877 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2878 } 2879 2880 void __kmp_hidden_helper_worker_thread_signal() { 2881 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2882 } 2883 2884 void __kmp_hidden_helper_threads_deinitz_wait() { 2885 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2886 } 2887 2888 void __kmp_hidden_helper_threads_deinitz_release() { 2889 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2890 } 2891 #endif // KMP_OS_LINUX 2892 2893 bool __kmp_detect_shm() { 2894 DIR *dir = opendir("/dev/shm"); 2895 if (dir) { // /dev/shm exists 2896 closedir(dir); 2897 return true; 2898 } else if (ENOENT == errno) { // /dev/shm does not exist 2899 return false; 2900 } else { // opendir() failed 2901 return false; 2902 } 2903 } 2904 2905 bool __kmp_detect_tmp() { 2906 DIR *dir = opendir("/tmp"); 2907 if (dir) { // /tmp exists 2908 closedir(dir); 2909 return true; 2910 } else if (ENOENT == errno) { // /tmp does not exist 2911 return false; 2912 } else { // opendir() failed 2913 return false; 2914 } 2915 } 2916 2917 // end of file // 2918