1 /* 2 * z_Linux_util.cpp -- platform specific routines. 3 */ 4 5 //===----------------------------------------------------------------------===// 6 // 7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8 // See https://llvm.org/LICENSE.txt for license information. 9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "kmp.h" 14 #include "kmp_affinity.h" 15 #include "kmp_i18n.h" 16 #include "kmp_io.h" 17 #include "kmp_itt.h" 18 #include "kmp_lock.h" 19 #include "kmp_stats.h" 20 #include "kmp_str.h" 21 #include "kmp_wait_release.h" 22 #include "kmp_wrapper_getpid.h" 23 24 #if !KMP_OS_DRAGONFLY && !KMP_OS_FREEBSD && !KMP_OS_NETBSD && !KMP_OS_OPENBSD 25 #include <alloca.h> 26 #endif 27 #include <math.h> // HUGE_VAL. 28 #if KMP_OS_LINUX 29 #include <semaphore.h> 30 #endif // KMP_OS_LINUX 31 #include <sys/resource.h> 32 #if KMP_OS_AIX 33 #include <sys/ldr.h> 34 #else 35 #include <sys/syscall.h> 36 #endif 37 #include <sys/time.h> 38 #include <sys/times.h> 39 #include <unistd.h> 40 41 #if KMP_OS_LINUX 42 #include <sys/sysinfo.h> 43 #if KMP_USE_FUTEX 44 // We should really include <futex.h>, but that causes compatibility problems on 45 // different Linux* OS distributions that either require that you include (or 46 // break when you try to include) <pci/types.h>. Since all we need is the two 47 // macros below (which are part of the kernel ABI, so can't change) we just 48 // define the constants here and don't include <futex.h> 49 #ifndef FUTEX_WAIT 50 #define FUTEX_WAIT 0 51 #endif 52 #ifndef FUTEX_WAKE 53 #define FUTEX_WAKE 1 54 #endif 55 #endif 56 #elif KMP_OS_DARWIN 57 #include <mach/mach.h> 58 #include <sys/sysctl.h> 59 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD 60 #include <sys/types.h> 61 #include <sys/sysctl.h> 62 #include <sys/user.h> 63 #include <pthread_np.h> 64 #if KMP_OS_DRAGONFLY 65 #include <kvm.h> 66 #endif 67 #elif KMP_OS_NETBSD || KMP_OS_OPENBSD 68 #include <sys/types.h> 69 #include <sys/sysctl.h> 70 #if KMP_OS_NETBSD 71 #include <sched.h> 72 #endif 73 #elif KMP_OS_SOLARIS 74 #include <libproc.h> 75 #include <procfs.h> 76 #include <thread.h> 77 #include <sys/loadavg.h> 78 #endif 79 80 #include <ctype.h> 81 #include <dirent.h> 82 #include <fcntl.h> 83 84 struct kmp_sys_timer { 85 struct timespec start; 86 }; 87 88 #ifndef TIMEVAL_TO_TIMESPEC 89 // Convert timeval to timespec. 90 #define TIMEVAL_TO_TIMESPEC(tv, ts) \ 91 do { \ 92 (ts)->tv_sec = (tv)->tv_sec; \ 93 (ts)->tv_nsec = (tv)->tv_usec * 1000; \ 94 } while (0) 95 #endif 96 97 // Convert timespec to nanoseconds. 98 #define TS2NS(timespec) \ 99 (((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec) 100 101 static struct kmp_sys_timer __kmp_sys_timer_data; 102 103 #if KMP_HANDLE_SIGNALS 104 typedef void (*sig_func_t)(int); 105 STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[NSIG]; 106 static sigset_t __kmp_sigset; 107 #endif 108 109 static int __kmp_init_runtime = FALSE; 110 111 static int __kmp_fork_count = 0; 112 113 static pthread_condattr_t __kmp_suspend_cond_attr; 114 static pthread_mutexattr_t __kmp_suspend_mutex_attr; 115 116 static kmp_cond_align_t __kmp_wait_cv; 117 static kmp_mutex_align_t __kmp_wait_mx; 118 119 kmp_uint64 __kmp_ticks_per_msec = 1000000; 120 kmp_uint64 __kmp_ticks_per_usec = 1000; 121 122 #ifdef DEBUG_SUSPEND 123 static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) { 124 KMP_SNPRINTF(buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))", 125 cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock, 126 cond->c_cond.__c_waiting); 127 } 128 #endif 129 130 #if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ 131 KMP_OS_AIX) && \ 132 KMP_AFFINITY_SUPPORTED) 133 134 /* Affinity support */ 135 136 void __kmp_affinity_bind_thread(int which) { 137 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(), 138 "Illegal set affinity operation when not capable"); 139 140 kmp_affin_mask_t *mask; 141 KMP_CPU_ALLOC_ON_STACK(mask); 142 KMP_CPU_ZERO(mask); 143 KMP_CPU_SET(which, mask); 144 __kmp_set_system_affinity(mask, TRUE); 145 KMP_CPU_FREE_FROM_STACK(mask); 146 } 147 148 #if KMP_OS_AIX 149 void __kmp_affinity_determine_capable(const char *env_var) { 150 // All versions of AIX support bindprocessor(). 151 152 size_t mask_size = __kmp_xproc / CHAR_BIT; 153 // Round up to byte boundary. 154 if (__kmp_xproc % CHAR_BIT) 155 ++mask_size; 156 157 // Round up to the mask_size_type boundary. 158 if (mask_size % sizeof(__kmp_affin_mask_size)) 159 mask_size += sizeof(__kmp_affin_mask_size) - 160 mask_size % sizeof(__kmp_affin_mask_size); 161 KMP_AFFINITY_ENABLE(mask_size); 162 KA_TRACE(10, 163 ("__kmp_affinity_determine_capable: " 164 "AIX OS affinity interface bindprocessor functional (mask size = " 165 "%" KMP_SIZE_T_SPEC ").\n", 166 __kmp_affin_mask_size)); 167 } 168 169 #else // !KMP_OS_AIX 170 171 /* Determine if we can access affinity functionality on this version of 172 * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set 173 * __kmp_affin_mask_size to the appropriate value (0 means not capable). */ 174 void __kmp_affinity_determine_capable(const char *env_var) { 175 // Check and see if the OS supports thread affinity. 176 177 #if KMP_OS_LINUX 178 #define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024) 179 #define KMP_CPU_SET_TRY_SIZE CACHE_LINE 180 #elif KMP_OS_FREEBSD || KMP_OS_DRAGONFLY 181 #define KMP_CPU_SET_SIZE_LIMIT (sizeof(cpuset_t)) 182 #elif KMP_OS_NETBSD 183 #define KMP_CPU_SET_SIZE_LIMIT (256) 184 #endif 185 186 int verbose = __kmp_affinity.flags.verbose; 187 int warnings = __kmp_affinity.flags.warnings; 188 enum affinity_type type = __kmp_affinity.type; 189 190 #if KMP_OS_LINUX 191 long gCode; 192 unsigned char *buf; 193 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT); 194 195 // If the syscall returns a suggestion for the size, 196 // then we don't have to search for an appropriate size. 197 gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_TRY_SIZE, buf); 198 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 199 "initial getaffinity call returned %ld errno = %d\n", 200 gCode, errno)); 201 202 if (gCode < 0 && errno != EINVAL) { 203 // System call not supported 204 if (verbose || 205 (warnings && (type != affinity_none) && (type != affinity_default) && 206 (type != affinity_disabled))) { 207 int error = errno; 208 kmp_msg_t err_code = KMP_ERR(error); 209 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var), 210 err_code, __kmp_msg_null); 211 if (__kmp_generate_warnings == kmp_warnings_off) { 212 __kmp_str_free(&err_code.str); 213 } 214 } 215 KMP_AFFINITY_DISABLE(); 216 KMP_INTERNAL_FREE(buf); 217 return; 218 } else if (gCode > 0) { 219 // The optimal situation: the OS returns the size of the buffer it expects. 220 KMP_AFFINITY_ENABLE(gCode); 221 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 222 "affinity supported (mask size %d)\n", 223 (int)__kmp_affin_mask_size)); 224 KMP_INTERNAL_FREE(buf); 225 return; 226 } 227 228 // Call the getaffinity system call repeatedly with increasing set sizes 229 // until we succeed, or reach an upper bound on the search. 230 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 231 "searching for proper set size\n")); 232 int size; 233 for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) { 234 gCode = syscall(__NR_sched_getaffinity, 0, size, buf); 235 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 236 "getaffinity for mask size %ld returned %ld errno = %d\n", 237 size, gCode, errno)); 238 239 if (gCode < 0) { 240 if (errno == ENOSYS) { 241 // We shouldn't get here 242 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 243 "inconsistent OS call behavior: errno == ENOSYS for mask " 244 "size %d\n", 245 size)); 246 if (verbose || 247 (warnings && (type != affinity_none) && 248 (type != affinity_default) && (type != affinity_disabled))) { 249 int error = errno; 250 kmp_msg_t err_code = KMP_ERR(error); 251 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var), 252 err_code, __kmp_msg_null); 253 if (__kmp_generate_warnings == kmp_warnings_off) { 254 __kmp_str_free(&err_code.str); 255 } 256 } 257 KMP_AFFINITY_DISABLE(); 258 KMP_INTERNAL_FREE(buf); 259 return; 260 } 261 continue; 262 } 263 264 KMP_AFFINITY_ENABLE(gCode); 265 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 266 "affinity supported (mask size %d)\n", 267 (int)__kmp_affin_mask_size)); 268 KMP_INTERNAL_FREE(buf); 269 return; 270 } 271 #elif KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY 272 long gCode; 273 unsigned char *buf; 274 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT); 275 gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT, 276 reinterpret_cast<cpuset_t *>(buf)); 277 KA_TRACE(30, ("__kmp_affinity_determine_capable: " 278 "initial getaffinity call returned %d errno = %d\n", 279 gCode, errno)); 280 if (gCode == 0) { 281 KMP_AFFINITY_ENABLE(KMP_CPU_SET_SIZE_LIMIT); 282 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 283 "affinity supported (mask size %d)\n", 284 (int)__kmp_affin_mask_size)); 285 KMP_INTERNAL_FREE(buf); 286 return; 287 } 288 #endif 289 KMP_INTERNAL_FREE(buf); 290 291 // Affinity is not supported 292 KMP_AFFINITY_DISABLE(); 293 KA_TRACE(10, ("__kmp_affinity_determine_capable: " 294 "cannot determine mask size - affinity not supported\n")); 295 if (verbose || (warnings && (type != affinity_none) && 296 (type != affinity_default) && (type != affinity_disabled))) { 297 KMP_WARNING(AffCantGetMaskSize, env_var); 298 } 299 } 300 #endif // KMP_OS_AIX 301 #endif // (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 302 KMP_OS_DRAGONFLY || KMP_OS_AIX) && KMP_AFFINITY_SUPPORTED 303 304 #if KMP_USE_FUTEX 305 306 int __kmp_futex_determine_capable() { 307 int loc = 0; 308 long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0); 309 int retval = (rc == 0) || (errno != ENOSYS); 310 311 KA_TRACE(10, 312 ("__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno)); 313 KA_TRACE(10, ("__kmp_futex_determine_capable: futex syscall%s supported\n", 314 retval ? "" : " not")); 315 316 return retval; 317 } 318 319 #endif // KMP_USE_FUTEX 320 321 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS) 322 /* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to 323 use compare_and_store for these routines */ 324 325 kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 d) { 326 kmp_int8 old_value, new_value; 327 328 old_value = TCR_1(*p); 329 new_value = old_value | d; 330 331 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) { 332 KMP_CPU_PAUSE(); 333 old_value = TCR_1(*p); 334 new_value = old_value | d; 335 } 336 return old_value; 337 } 338 339 kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 d) { 340 kmp_int8 old_value, new_value; 341 342 old_value = TCR_1(*p); 343 new_value = old_value & d; 344 345 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) { 346 KMP_CPU_PAUSE(); 347 old_value = TCR_1(*p); 348 new_value = old_value & d; 349 } 350 return old_value; 351 } 352 353 kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 d) { 354 kmp_uint32 old_value, new_value; 355 356 old_value = TCR_4(*p); 357 new_value = old_value | d; 358 359 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) { 360 KMP_CPU_PAUSE(); 361 old_value = TCR_4(*p); 362 new_value = old_value | d; 363 } 364 return old_value; 365 } 366 367 kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) { 368 kmp_uint32 old_value, new_value; 369 370 old_value = TCR_4(*p); 371 new_value = old_value & d; 372 373 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) { 374 KMP_CPU_PAUSE(); 375 old_value = TCR_4(*p); 376 new_value = old_value & d; 377 } 378 return old_value; 379 } 380 381 #if KMP_ARCH_X86 || KMP_ARCH_WASM 382 kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) { 383 kmp_int8 old_value, new_value; 384 385 old_value = TCR_1(*p); 386 new_value = old_value + d; 387 388 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) { 389 KMP_CPU_PAUSE(); 390 old_value = TCR_1(*p); 391 new_value = old_value + d; 392 } 393 return old_value; 394 } 395 396 kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 d) { 397 kmp_int64 old_value, new_value; 398 399 old_value = TCR_8(*p); 400 new_value = old_value + d; 401 402 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) { 403 KMP_CPU_PAUSE(); 404 old_value = TCR_8(*p); 405 new_value = old_value + d; 406 } 407 return old_value; 408 } 409 #endif /* KMP_ARCH_X86 */ 410 411 kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 d) { 412 kmp_uint64 old_value, new_value; 413 414 old_value = TCR_8(*p); 415 new_value = old_value | d; 416 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) { 417 KMP_CPU_PAUSE(); 418 old_value = TCR_8(*p); 419 new_value = old_value | d; 420 } 421 return old_value; 422 } 423 424 kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 d) { 425 kmp_uint64 old_value, new_value; 426 427 old_value = TCR_8(*p); 428 new_value = old_value & d; 429 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) { 430 KMP_CPU_PAUSE(); 431 old_value = TCR_8(*p); 432 new_value = old_value & d; 433 } 434 return old_value; 435 } 436 437 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */ 438 439 void __kmp_terminate_thread(int gtid) { 440 int status; 441 kmp_info_t *th = __kmp_threads[gtid]; 442 443 if (!th) 444 return; 445 446 #ifdef KMP_CANCEL_THREADS 447 KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid)); 448 status = pthread_cancel(th->th.th_info.ds.ds_thread); 449 if (status != 0 && status != ESRCH) { 450 __kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status), 451 __kmp_msg_null); 452 } 453 #endif 454 KMP_YIELD(TRUE); 455 } // 456 457 /* Set thread stack info. 458 If values are unreasonable, assume call failed and use incremental stack 459 refinement method instead. Returns TRUE if the stack parameters could be 460 determined exactly, FALSE if incremental refinement is necessary. */ 461 static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) { 462 int stack_data; 463 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 464 KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX 465 int status; 466 size_t size = 0; 467 void *addr = 0; 468 469 /* Always do incremental stack refinement for ubermaster threads since the 470 initial thread stack range can be reduced by sibling thread creation so 471 pthread_attr_getstack may cause thread gtid aliasing */ 472 if (!KMP_UBER_GTID(gtid)) { 473 474 #if KMP_OS_SOLARIS 475 stack_t s; 476 if ((status = thr_stksegment(&s)) < 0) { 477 KMP_CHECK_SYSFAIL("thr_stksegment", status); 478 } 479 480 addr = s.ss_sp; 481 size = s.ss_size; 482 KA_TRACE(60, ("__kmp_set_stack_info: T#%d thr_stksegment returned size:" 483 " %lu, low addr: %p\n", 484 gtid, size, addr)); 485 #else 486 pthread_attr_t attr; 487 /* Fetch the real thread attributes */ 488 status = pthread_attr_init(&attr); 489 KMP_CHECK_SYSFAIL("pthread_attr_init", status); 490 #if KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD 491 status = pthread_attr_get_np(pthread_self(), &attr); 492 KMP_CHECK_SYSFAIL("pthread_attr_get_np", status); 493 #else 494 status = pthread_getattr_np(pthread_self(), &attr); 495 KMP_CHECK_SYSFAIL("pthread_getattr_np", status); 496 #endif 497 status = pthread_attr_getstack(&attr, &addr, &size); 498 KMP_CHECK_SYSFAIL("pthread_attr_getstack", status); 499 KA_TRACE(60, 500 ("__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:" 501 " %lu, low addr: %p\n", 502 gtid, size, addr)); 503 status = pthread_attr_destroy(&attr); 504 KMP_CHECK_SYSFAIL("pthread_attr_destroy", status); 505 #endif 506 } 507 508 if (size != 0 && addr != 0) { // was stack parameter determination successful? 509 /* Store the correct base and size */ 510 TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size)); 511 TCW_PTR(th->th.th_info.ds.ds_stacksize, size); 512 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE); 513 return TRUE; 514 } 515 #endif /* KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD \ 516 || KMP_OS_HURD || KMP_OS_SOLARIS */ 517 /* Use incremental refinement starting from initial conservative estimate */ 518 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0); 519 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data); 520 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE); 521 return FALSE; 522 } 523 524 static void *__kmp_launch_worker(void *thr) { 525 int status, old_type, old_state; 526 #ifdef KMP_BLOCK_SIGNALS 527 sigset_t new_set, old_set; 528 #endif /* KMP_BLOCK_SIGNALS */ 529 void *exit_val; 530 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 531 KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX 532 void *volatile padding = 0; 533 #endif 534 int gtid; 535 536 gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid; 537 __kmp_gtid_set_specific(gtid); 538 #ifdef KMP_TDATA_GTID 539 __kmp_gtid = gtid; 540 #endif 541 #if KMP_STATS_ENABLED 542 // set thread local index to point to thread-specific stats 543 __kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats; 544 __kmp_stats_thread_ptr->startLife(); 545 KMP_SET_THREAD_STATE(IDLE); 546 KMP_INIT_PARTITIONED_TIMERS(OMP_idle); 547 #endif 548 549 #if USE_ITT_BUILD 550 __kmp_itt_thread_name(gtid); 551 #endif /* USE_ITT_BUILD */ 552 553 #if KMP_AFFINITY_SUPPORTED 554 __kmp_affinity_bind_init_mask(gtid); 555 #endif 556 557 #ifdef KMP_CANCEL_THREADS 558 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type); 559 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status); 560 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? 561 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state); 562 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 563 #endif 564 565 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 566 // Set FP control regs to be a copy of the parallel initialization thread's. 567 __kmp_clear_x87_fpu_status_word(); 568 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word); 569 __kmp_load_mxcsr(&__kmp_init_mxcsr); 570 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 571 572 #ifdef KMP_BLOCK_SIGNALS 573 status = sigfillset(&new_set); 574 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status); 575 status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set); 576 KMP_CHECK_SYSFAIL("pthread_sigmask", status); 577 #endif /* KMP_BLOCK_SIGNALS */ 578 579 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 580 KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX 581 if (__kmp_stkoffset > 0 && gtid > 0) { 582 padding = KMP_ALLOCA(gtid * __kmp_stkoffset); 583 (void)padding; 584 } 585 #endif 586 587 KMP_MB(); 588 __kmp_set_stack_info(gtid, (kmp_info_t *)thr); 589 590 __kmp_check_stack_overlap((kmp_info_t *)thr); 591 592 exit_val = __kmp_launch_thread((kmp_info_t *)thr); 593 594 #ifdef KMP_BLOCK_SIGNALS 595 status = pthread_sigmask(SIG_SETMASK, &old_set, NULL); 596 KMP_CHECK_SYSFAIL("pthread_sigmask", status); 597 #endif /* KMP_BLOCK_SIGNALS */ 598 599 return exit_val; 600 } 601 602 #if KMP_USE_MONITOR 603 /* The monitor thread controls all of the threads in the complex */ 604 605 static void *__kmp_launch_monitor(void *thr) { 606 int status, old_type, old_state; 607 #ifdef KMP_BLOCK_SIGNALS 608 sigset_t new_set; 609 #endif /* KMP_BLOCK_SIGNALS */ 610 struct timespec interval; 611 612 KMP_MB(); /* Flush all pending memory write invalidates. */ 613 614 KA_TRACE(10, ("__kmp_launch_monitor: #1 launched\n")); 615 616 /* register us as the monitor thread */ 617 __kmp_gtid_set_specific(KMP_GTID_MONITOR); 618 #ifdef KMP_TDATA_GTID 619 __kmp_gtid = KMP_GTID_MONITOR; 620 #endif 621 622 KMP_MB(); 623 624 #if USE_ITT_BUILD 625 // Instruct Intel(R) Threading Tools to ignore monitor thread. 626 __kmp_itt_thread_ignore(); 627 #endif /* USE_ITT_BUILD */ 628 629 __kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid, 630 (kmp_info_t *)thr); 631 632 __kmp_check_stack_overlap((kmp_info_t *)thr); 633 634 #ifdef KMP_CANCEL_THREADS 635 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type); 636 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status); 637 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? 638 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state); 639 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 640 #endif 641 642 #if KMP_REAL_TIME_FIX 643 // This is a potential fix which allows application with real-time scheduling 644 // policy work. However, decision about the fix is not made yet, so it is 645 // disabled by default. 646 { // Are program started with real-time scheduling policy? 647 int sched = sched_getscheduler(0); 648 if (sched == SCHED_FIFO || sched == SCHED_RR) { 649 // Yes, we are a part of real-time application. Try to increase the 650 // priority of the monitor. 651 struct sched_param param; 652 int max_priority = sched_get_priority_max(sched); 653 int rc; 654 KMP_WARNING(RealTimeSchedNotSupported); 655 sched_getparam(0, ¶m); 656 if (param.sched_priority < max_priority) { 657 param.sched_priority += 1; 658 rc = sched_setscheduler(0, sched, ¶m); 659 if (rc != 0) { 660 int error = errno; 661 kmp_msg_t err_code = KMP_ERR(error); 662 __kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority), 663 err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null); 664 if (__kmp_generate_warnings == kmp_warnings_off) { 665 __kmp_str_free(&err_code.str); 666 } 667 } 668 } else { 669 // We cannot abort here, because number of CPUs may be enough for all 670 // the threads, including the monitor thread, so application could 671 // potentially work... 672 __kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority), 673 KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority), 674 __kmp_msg_null); 675 } 676 } 677 // AC: free thread that waits for monitor started 678 TCW_4(__kmp_global.g.g_time.dt.t_value, 0); 679 } 680 #endif // KMP_REAL_TIME_FIX 681 682 KMP_MB(); /* Flush all pending memory write invalidates. */ 683 684 if (__kmp_monitor_wakeups == 1) { 685 interval.tv_sec = 1; 686 interval.tv_nsec = 0; 687 } else { 688 interval.tv_sec = 0; 689 interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups); 690 } 691 692 KA_TRACE(10, ("__kmp_launch_monitor: #2 monitor\n")); 693 694 while (!TCR_4(__kmp_global.g.g_done)) { 695 struct timespec now; 696 struct timeval tval; 697 698 /* This thread monitors the state of the system */ 699 700 KA_TRACE(15, ("__kmp_launch_monitor: update\n")); 701 702 status = gettimeofday(&tval, NULL); 703 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 704 TIMEVAL_TO_TIMESPEC(&tval, &now); 705 706 now.tv_sec += interval.tv_sec; 707 now.tv_nsec += interval.tv_nsec; 708 709 if (now.tv_nsec >= KMP_NSEC_PER_SEC) { 710 now.tv_sec += 1; 711 now.tv_nsec -= KMP_NSEC_PER_SEC; 712 } 713 714 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex); 715 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 716 // AC: the monitor should not fall asleep if g_done has been set 717 if (!TCR_4(__kmp_global.g.g_done)) { // check once more under mutex 718 status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond, 719 &__kmp_wait_mx.m_mutex, &now); 720 if (status != 0) { 721 if (status != ETIMEDOUT && status != EINTR) { 722 KMP_SYSFAIL("pthread_cond_timedwait", status); 723 } 724 } 725 } 726 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex); 727 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 728 729 TCW_4(__kmp_global.g.g_time.dt.t_value, 730 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1); 731 732 KMP_MB(); /* Flush all pending memory write invalidates. */ 733 } 734 735 KA_TRACE(10, ("__kmp_launch_monitor: #3 cleanup\n")); 736 737 #ifdef KMP_BLOCK_SIGNALS 738 status = sigfillset(&new_set); 739 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status); 740 status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL); 741 KMP_CHECK_SYSFAIL("pthread_sigmask", status); 742 #endif /* KMP_BLOCK_SIGNALS */ 743 744 KA_TRACE(10, ("__kmp_launch_monitor: #4 finished\n")); 745 746 if (__kmp_global.g.g_abort != 0) { 747 /* now we need to terminate the worker threads */ 748 /* the value of t_abort is the signal we caught */ 749 750 int gtid; 751 752 KA_TRACE(10, ("__kmp_launch_monitor: #5 terminate sig=%d\n", 753 __kmp_global.g.g_abort)); 754 755 /* terminate the OpenMP worker threads */ 756 /* TODO this is not valid for sibling threads!! 757 * the uber master might not be 0 anymore.. */ 758 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid) 759 __kmp_terminate_thread(gtid); 760 761 __kmp_cleanup(); 762 763 KA_TRACE(10, ("__kmp_launch_monitor: #6 raise sig=%d\n", 764 __kmp_global.g.g_abort)); 765 766 if (__kmp_global.g.g_abort > 0) 767 raise(__kmp_global.g.g_abort); 768 } 769 770 KA_TRACE(10, ("__kmp_launch_monitor: #7 exit\n")); 771 772 return thr; 773 } 774 #endif // KMP_USE_MONITOR 775 776 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) { 777 pthread_t handle; 778 pthread_attr_t thread_attr; 779 int status; 780 781 th->th.th_info.ds.ds_gtid = gtid; 782 783 #if KMP_STATS_ENABLED 784 // sets up worker thread stats 785 __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid); 786 787 // th->th.th_stats is used to transfer thread-specific stats-pointer to 788 // __kmp_launch_worker. So when thread is created (goes into 789 // __kmp_launch_worker) it will set its thread local pointer to 790 // th->th.th_stats 791 if (!KMP_UBER_GTID(gtid)) { 792 th->th.th_stats = __kmp_stats_list->push_back(gtid); 793 } else { 794 // For root threads, __kmp_stats_thread_ptr is set in __kmp_register_root(), 795 // so set the th->th.th_stats field to it. 796 th->th.th_stats = __kmp_stats_thread_ptr; 797 } 798 __kmp_release_tas_lock(&__kmp_stats_lock, gtid); 799 800 #endif // KMP_STATS_ENABLED 801 802 if (KMP_UBER_GTID(gtid)) { 803 KA_TRACE(10, ("__kmp_create_worker: uber thread (%d)\n", gtid)); 804 th->th.th_info.ds.ds_thread = pthread_self(); 805 __kmp_set_stack_info(gtid, th); 806 __kmp_check_stack_overlap(th); 807 return; 808 } 809 810 KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid)); 811 812 KMP_MB(); /* Flush all pending memory write invalidates. */ 813 814 #ifdef KMP_THREAD_ATTR 815 status = pthread_attr_init(&thread_attr); 816 if (status != 0) { 817 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null); 818 } 819 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE); 820 if (status != 0) { 821 __kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null); 822 } 823 824 /* Set stack size for this thread now. 825 The multiple of 2 is there because on some machines, requesting an unusual 826 stacksize causes the thread to have an offset before the dummy alloca() 827 takes place to create the offset. Since we want the user to have a 828 sufficient stacksize AND support a stack offset, we alloca() twice the 829 offset so that the upcoming alloca() does not eliminate any premade offset, 830 and also gives the user the stack space they requested for all threads */ 831 stack_size += gtid * __kmp_stkoffset * 2; 832 833 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, " 834 "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n", 835 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size)); 836 837 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 838 status = pthread_attr_setstacksize(&thread_attr, stack_size); 839 #ifdef KMP_BACKUP_STKSIZE 840 if (status != 0) { 841 if (!__kmp_env_stksize) { 842 stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset; 843 __kmp_stksize = KMP_BACKUP_STKSIZE; 844 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, " 845 "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu " 846 "bytes\n", 847 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size)); 848 status = pthread_attr_setstacksize(&thread_attr, stack_size); 849 } 850 } 851 #endif /* KMP_BACKUP_STKSIZE */ 852 if (status != 0) { 853 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status), 854 KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null); 855 } 856 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 857 858 #endif /* KMP_THREAD_ATTR */ 859 860 status = 861 pthread_create(&handle, &thread_attr, __kmp_launch_worker, (void *)th); 862 if (status != 0 || !handle) { // ??? Why do we check handle?? 863 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 864 if (status == EINVAL) { 865 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status), 866 KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null); 867 } 868 if (status == ENOMEM) { 869 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status), 870 KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null); 871 } 872 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 873 if (status == EAGAIN) { 874 __kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status), 875 KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null); 876 } 877 KMP_SYSFAIL("pthread_create", status); 878 } 879 880 th->th.th_info.ds.ds_thread = handle; 881 882 #ifdef KMP_THREAD_ATTR 883 status = pthread_attr_destroy(&thread_attr); 884 if (status) { 885 kmp_msg_t err_code = KMP_ERR(status); 886 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code, 887 __kmp_msg_null); 888 if (__kmp_generate_warnings == kmp_warnings_off) { 889 __kmp_str_free(&err_code.str); 890 } 891 } 892 #endif /* KMP_THREAD_ATTR */ 893 894 KMP_MB(); /* Flush all pending memory write invalidates. */ 895 896 KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid)); 897 898 } // __kmp_create_worker 899 900 #if KMP_USE_MONITOR 901 void __kmp_create_monitor(kmp_info_t *th) { 902 pthread_t handle; 903 pthread_attr_t thread_attr; 904 size_t size; 905 int status; 906 int auto_adj_size = FALSE; 907 908 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) { 909 // We don't need monitor thread in case of MAX_BLOCKTIME 910 KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of " 911 "MAX blocktime\n")); 912 th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op 913 th->th.th_info.ds.ds_gtid = 0; 914 return; 915 } 916 KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n")); 917 918 KMP_MB(); /* Flush all pending memory write invalidates. */ 919 920 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR; 921 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR; 922 #if KMP_REAL_TIME_FIX 923 TCW_4(__kmp_global.g.g_time.dt.t_value, 924 -1); // Will use it for synchronization a bit later. 925 #else 926 TCW_4(__kmp_global.g.g_time.dt.t_value, 0); 927 #endif // KMP_REAL_TIME_FIX 928 929 #ifdef KMP_THREAD_ATTR 930 if (__kmp_monitor_stksize == 0) { 931 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE; 932 auto_adj_size = TRUE; 933 } 934 status = pthread_attr_init(&thread_attr); 935 if (status != 0) { 936 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null); 937 } 938 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE); 939 if (status != 0) { 940 __kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null); 941 } 942 943 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 944 status = pthread_attr_getstacksize(&thread_attr, &size); 945 KMP_CHECK_SYSFAIL("pthread_attr_getstacksize", status); 946 #else 947 size = __kmp_sys_min_stksize; 948 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 949 #endif /* KMP_THREAD_ATTR */ 950 951 if (__kmp_monitor_stksize == 0) { 952 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE; 953 } 954 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) { 955 __kmp_monitor_stksize = __kmp_sys_min_stksize; 956 } 957 958 KA_TRACE(10, ("__kmp_create_monitor: default stacksize = %lu bytes," 959 "requested stacksize = %lu bytes\n", 960 size, __kmp_monitor_stksize)); 961 962 retry: 963 964 /* Set stack size for this thread now. */ 965 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 966 KA_TRACE(10, ("__kmp_create_monitor: setting stacksize = %lu bytes,", 967 __kmp_monitor_stksize)); 968 status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize); 969 if (status != 0) { 970 if (auto_adj_size) { 971 __kmp_monitor_stksize *= 2; 972 goto retry; 973 } 974 kmp_msg_t err_code = KMP_ERR(status); 975 __kmp_msg(kmp_ms_warning, // should this be fatal? BB 976 KMP_MSG(CantSetMonitorStackSize, (long int)__kmp_monitor_stksize), 977 err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null); 978 if (__kmp_generate_warnings == kmp_warnings_off) { 979 __kmp_str_free(&err_code.str); 980 } 981 } 982 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 983 984 status = 985 pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (void *)th); 986 987 if (status != 0) { 988 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 989 if (status == EINVAL) { 990 if (auto_adj_size && (__kmp_monitor_stksize < (size_t)0x40000000)) { 991 __kmp_monitor_stksize *= 2; 992 goto retry; 993 } 994 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize), 995 KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize), 996 __kmp_msg_null); 997 } 998 if (status == ENOMEM) { 999 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize), 1000 KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize), 1001 __kmp_msg_null); 1002 } 1003 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */ 1004 if (status == EAGAIN) { 1005 __kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status), 1006 KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null); 1007 } 1008 KMP_SYSFAIL("pthread_create", status); 1009 } 1010 1011 th->th.th_info.ds.ds_thread = handle; 1012 1013 #if KMP_REAL_TIME_FIX 1014 // Wait for the monitor thread is really started and set its *priority*. 1015 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 1016 sizeof(__kmp_global.g.g_time.dt.t_value)); 1017 __kmp_wait_4((kmp_uint32 volatile *)&__kmp_global.g.g_time.dt.t_value, -1, 1018 &__kmp_neq_4, NULL); 1019 #endif // KMP_REAL_TIME_FIX 1020 1021 #ifdef KMP_THREAD_ATTR 1022 status = pthread_attr_destroy(&thread_attr); 1023 if (status != 0) { 1024 kmp_msg_t err_code = KMP_ERR(status); 1025 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code, 1026 __kmp_msg_null); 1027 if (__kmp_generate_warnings == kmp_warnings_off) { 1028 __kmp_str_free(&err_code.str); 1029 } 1030 } 1031 #endif 1032 1033 KMP_MB(); /* Flush all pending memory write invalidates. */ 1034 1035 KA_TRACE(10, ("__kmp_create_monitor: monitor created %#.8lx\n", 1036 th->th.th_info.ds.ds_thread)); 1037 1038 } // __kmp_create_monitor 1039 #endif // KMP_USE_MONITOR 1040 1041 void __kmp_exit_thread(int exit_status) { 1042 #if KMP_OS_WASI 1043 // TODO: the wasm32-wasi-threads target does not yet support pthread_exit. 1044 #else 1045 pthread_exit((void *)(intptr_t)exit_status); 1046 #endif 1047 } // __kmp_exit_thread 1048 1049 #if KMP_USE_MONITOR 1050 void __kmp_resume_monitor(); 1051 1052 extern "C" void __kmp_reap_monitor(kmp_info_t *th) { 1053 int status; 1054 void *exit_val; 1055 1056 KA_TRACE(10, ("__kmp_reap_monitor: try to reap monitor thread with handle" 1057 " %#.8lx\n", 1058 th->th.th_info.ds.ds_thread)); 1059 1060 // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR. 1061 // If both tid and gtid are 0, it means the monitor did not ever start. 1062 // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down. 1063 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid); 1064 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) { 1065 KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n")); 1066 return; 1067 } 1068 1069 KMP_MB(); /* Flush all pending memory write invalidates. */ 1070 1071 /* First, check to see whether the monitor thread exists to wake it up. This 1072 is to avoid performance problem when the monitor sleeps during 1073 blocktime-size interval */ 1074 1075 status = pthread_kill(th->th.th_info.ds.ds_thread, 0); 1076 if (status != ESRCH) { 1077 __kmp_resume_monitor(); // Wake up the monitor thread 1078 } 1079 KA_TRACE(10, ("__kmp_reap_monitor: try to join with monitor\n")); 1080 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val); 1081 if (exit_val != th) { 1082 __kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null); 1083 } 1084 1085 th->th.th_info.ds.ds_tid = KMP_GTID_DNE; 1086 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE; 1087 1088 KA_TRACE(10, ("__kmp_reap_monitor: done reaping monitor thread with handle" 1089 " %#.8lx\n", 1090 th->th.th_info.ds.ds_thread)); 1091 1092 KMP_MB(); /* Flush all pending memory write invalidates. */ 1093 } 1094 #else 1095 // Empty symbol to export (see exports_so.txt) when 1096 // monitor thread feature is disabled 1097 extern "C" void __kmp_reap_monitor(kmp_info_t *th) { (void)th; } 1098 #endif // KMP_USE_MONITOR 1099 1100 void __kmp_reap_worker(kmp_info_t *th) { 1101 int status; 1102 void *exit_val; 1103 1104 KMP_MB(); /* Flush all pending memory write invalidates. */ 1105 1106 KA_TRACE( 1107 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid)); 1108 1109 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val); 1110 #ifdef KMP_DEBUG 1111 /* Don't expose these to the user until we understand when they trigger */ 1112 if (status != 0) { 1113 __kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null); 1114 } 1115 if (exit_val != th) { 1116 KA_TRACE(10, ("__kmp_reap_worker: worker T#%d did not reap properly, " 1117 "exit_val = %p\n", 1118 th->th.th_info.ds.ds_gtid, exit_val)); 1119 } 1120 #else 1121 (void)status; // unused variable 1122 #endif /* KMP_DEBUG */ 1123 1124 KA_TRACE(10, ("__kmp_reap_worker: done reaping T#%d\n", 1125 th->th.th_info.ds.ds_gtid)); 1126 1127 KMP_MB(); /* Flush all pending memory write invalidates. */ 1128 } 1129 1130 #if KMP_HANDLE_SIGNALS 1131 1132 static void __kmp_null_handler(int signo) { 1133 // Do nothing, for doing SIG_IGN-type actions. 1134 } // __kmp_null_handler 1135 1136 static void __kmp_team_handler(int signo) { 1137 if (__kmp_global.g.g_abort == 0) { 1138 /* Stage 1 signal handler, let's shut down all of the threads */ 1139 #ifdef KMP_DEBUG 1140 __kmp_debug_printf("__kmp_team_handler: caught signal = %d\n", signo); 1141 #endif 1142 switch (signo) { 1143 case SIGHUP: 1144 case SIGINT: 1145 case SIGQUIT: 1146 case SIGILL: 1147 case SIGABRT: 1148 case SIGFPE: 1149 case SIGBUS: 1150 case SIGSEGV: 1151 #ifdef SIGSYS 1152 case SIGSYS: 1153 #endif 1154 case SIGTERM: 1155 if (__kmp_debug_buf) { 1156 __kmp_dump_debug_buffer(); 1157 } 1158 __kmp_unregister_library(); // cleanup shared memory 1159 KMP_MB(); // Flush all pending memory write invalidates. 1160 TCW_4(__kmp_global.g.g_abort, signo); 1161 KMP_MB(); // Flush all pending memory write invalidates. 1162 TCW_4(__kmp_global.g.g_done, TRUE); 1163 KMP_MB(); // Flush all pending memory write invalidates. 1164 break; 1165 default: 1166 #ifdef KMP_DEBUG 1167 __kmp_debug_printf("__kmp_team_handler: unknown signal type"); 1168 #endif 1169 break; 1170 } 1171 } 1172 } // __kmp_team_handler 1173 1174 static void __kmp_sigaction(int signum, const struct sigaction *act, 1175 struct sigaction *oldact) { 1176 int rc = sigaction(signum, act, oldact); 1177 KMP_CHECK_SYSFAIL_ERRNO("sigaction", rc); 1178 } 1179 1180 static void __kmp_install_one_handler(int sig, sig_func_t handler_func, 1181 int parallel_init) { 1182 KMP_MB(); // Flush all pending memory write invalidates. 1183 KB_TRACE(60, 1184 ("__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init)); 1185 if (parallel_init) { 1186 struct sigaction new_action; 1187 struct sigaction old_action; 1188 new_action.sa_handler = handler_func; 1189 new_action.sa_flags = 0; 1190 sigfillset(&new_action.sa_mask); 1191 __kmp_sigaction(sig, &new_action, &old_action); 1192 if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) { 1193 sigaddset(&__kmp_sigset, sig); 1194 } else { 1195 // Restore/keep user's handler if one previously installed. 1196 __kmp_sigaction(sig, &old_action, NULL); 1197 } 1198 } else { 1199 // Save initial/system signal handlers to see if user handlers installed. 1200 __kmp_sigaction(sig, NULL, &__kmp_sighldrs[sig]); 1201 } 1202 KMP_MB(); // Flush all pending memory write invalidates. 1203 } // __kmp_install_one_handler 1204 1205 static void __kmp_remove_one_handler(int sig) { 1206 KB_TRACE(60, ("__kmp_remove_one_handler( %d )\n", sig)); 1207 if (sigismember(&__kmp_sigset, sig)) { 1208 struct sigaction old; 1209 KMP_MB(); // Flush all pending memory write invalidates. 1210 __kmp_sigaction(sig, &__kmp_sighldrs[sig], &old); 1211 if ((old.sa_handler != __kmp_team_handler) && 1212 (old.sa_handler != __kmp_null_handler)) { 1213 // Restore the users signal handler. 1214 KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, " 1215 "restoring: sig=%d\n", 1216 sig)); 1217 __kmp_sigaction(sig, &old, NULL); 1218 } 1219 sigdelset(&__kmp_sigset, sig); 1220 KMP_MB(); // Flush all pending memory write invalidates. 1221 } 1222 } // __kmp_remove_one_handler 1223 1224 void __kmp_install_signals(int parallel_init) { 1225 KB_TRACE(10, ("__kmp_install_signals( %d )\n", parallel_init)); 1226 if (__kmp_handle_signals || !parallel_init) { 1227 // If ! parallel_init, we do not install handlers, just save original 1228 // handlers. Let us do it even __handle_signals is 0. 1229 sigemptyset(&__kmp_sigset); 1230 __kmp_install_one_handler(SIGHUP, __kmp_team_handler, parallel_init); 1231 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init); 1232 __kmp_install_one_handler(SIGQUIT, __kmp_team_handler, parallel_init); 1233 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init); 1234 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init); 1235 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init); 1236 __kmp_install_one_handler(SIGBUS, __kmp_team_handler, parallel_init); 1237 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init); 1238 #ifdef SIGSYS 1239 __kmp_install_one_handler(SIGSYS, __kmp_team_handler, parallel_init); 1240 #endif // SIGSYS 1241 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init); 1242 #ifdef SIGPIPE 1243 __kmp_install_one_handler(SIGPIPE, __kmp_team_handler, parallel_init); 1244 #endif // SIGPIPE 1245 } 1246 } // __kmp_install_signals 1247 1248 void __kmp_remove_signals(void) { 1249 int sig; 1250 KB_TRACE(10, ("__kmp_remove_signals()\n")); 1251 for (sig = 1; sig < NSIG; ++sig) { 1252 __kmp_remove_one_handler(sig); 1253 } 1254 } // __kmp_remove_signals 1255 1256 #endif // KMP_HANDLE_SIGNALS 1257 1258 void __kmp_enable(int new_state) { 1259 #ifdef KMP_CANCEL_THREADS 1260 int status, old_state; 1261 status = pthread_setcancelstate(new_state, &old_state); 1262 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 1263 KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE); 1264 #endif 1265 } 1266 1267 void __kmp_disable(int *old_state) { 1268 #ifdef KMP_CANCEL_THREADS 1269 int status; 1270 status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state); 1271 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status); 1272 #endif 1273 } 1274 1275 static void __kmp_atfork_prepare(void) { 1276 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); 1277 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock); 1278 } 1279 1280 static void __kmp_atfork_parent(void) { 1281 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1282 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 1283 } 1284 1285 /* Reset the library so execution in the child starts "all over again" with 1286 clean data structures in initial states. Don't worry about freeing memory 1287 allocated by parent, just abandon it to be safe. */ 1288 static void __kmp_atfork_child(void) { 1289 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); 1290 __kmp_release_bootstrap_lock(&__kmp_initz_lock); 1291 /* TODO make sure this is done right for nested/sibling */ 1292 // ATT: Memory leaks are here? TODO: Check it and fix. 1293 /* KMP_ASSERT( 0 ); */ 1294 1295 ++__kmp_fork_count; 1296 1297 #if KMP_AFFINITY_SUPPORTED 1298 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \ 1299 KMP_OS_AIX 1300 // reset the affinity in the child to the initial thread 1301 // affinity in the parent 1302 kmp_set_thread_affinity_mask_initial(); 1303 #endif 1304 // Set default not to bind threads tightly in the child (we're expecting 1305 // over-subscription after the fork and this can improve things for 1306 // scripting languages that use OpenMP inside process-parallel code). 1307 if (__kmp_nested_proc_bind.bind_types != NULL) { 1308 __kmp_nested_proc_bind.bind_types[0] = proc_bind_false; 1309 } 1310 for (kmp_affinity_t *affinity : __kmp_affinities) 1311 *affinity = KMP_AFFINITY_INIT(affinity->env_var); 1312 __kmp_affin_fullMask = nullptr; 1313 __kmp_affin_origMask = nullptr; 1314 __kmp_topology = nullptr; 1315 #endif // KMP_AFFINITY_SUPPORTED 1316 1317 #if KMP_USE_MONITOR 1318 __kmp_init_monitor = 0; 1319 #endif 1320 __kmp_init_parallel = FALSE; 1321 __kmp_init_middle = FALSE; 1322 __kmp_init_serial = FALSE; 1323 TCW_4(__kmp_init_gtid, FALSE); 1324 __kmp_init_common = FALSE; 1325 1326 TCW_4(__kmp_init_user_locks, FALSE); 1327 #if !KMP_USE_DYNAMIC_LOCK 1328 __kmp_user_lock_table.used = 1; 1329 __kmp_user_lock_table.allocated = 0; 1330 __kmp_user_lock_table.table = NULL; 1331 __kmp_lock_blocks = NULL; 1332 #endif 1333 1334 __kmp_all_nth = 0; 1335 TCW_4(__kmp_nth, 0); 1336 1337 __kmp_thread_pool = NULL; 1338 __kmp_thread_pool_insert_pt = NULL; 1339 __kmp_team_pool = NULL; 1340 1341 /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate 1342 here so threadprivate doesn't use stale data */ 1343 KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n", 1344 __kmp_threadpriv_cache_list)); 1345 1346 while (__kmp_threadpriv_cache_list != NULL) { 1347 1348 if (*__kmp_threadpriv_cache_list->addr != NULL) { 1349 KC_TRACE(50, ("__kmp_atfork_child: zeroing cache at address %p\n", 1350 &(*__kmp_threadpriv_cache_list->addr))); 1351 1352 *__kmp_threadpriv_cache_list->addr = NULL; 1353 } 1354 __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next; 1355 } 1356 1357 __kmp_init_runtime = FALSE; 1358 1359 /* reset statically initialized locks */ 1360 __kmp_init_bootstrap_lock(&__kmp_initz_lock); 1361 __kmp_init_bootstrap_lock(&__kmp_stdio_lock); 1362 __kmp_init_bootstrap_lock(&__kmp_console_lock); 1363 __kmp_init_bootstrap_lock(&__kmp_task_team_lock); 1364 1365 #if USE_ITT_BUILD 1366 __kmp_itt_reset(); // reset ITT's global state 1367 #endif /* USE_ITT_BUILD */ 1368 1369 { 1370 // Child process often get terminated without any use of OpenMP. That might 1371 // cause mapped shared memory file to be left unattended. Thus we postpone 1372 // library registration till middle initialization in the child process. 1373 __kmp_need_register_serial = FALSE; 1374 __kmp_serial_initialize(); 1375 } 1376 1377 /* This is necessary to make sure no stale data is left around */ 1378 /* AC: customers complain that we use unsafe routines in the atfork 1379 handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen 1380 in dynamic_link when check the presence of shared tbbmalloc library. 1381 Suggestion is to make the library initialization lazier, similar 1382 to what done for __kmpc_begin(). */ 1383 // TODO: synchronize all static initializations with regular library 1384 // startup; look at kmp_global.cpp and etc. 1385 //__kmp_internal_begin (); 1386 } 1387 1388 void __kmp_register_atfork(void) { 1389 if (__kmp_need_register_atfork) { 1390 #if !KMP_OS_WASI 1391 int status = pthread_atfork(__kmp_atfork_prepare, __kmp_atfork_parent, 1392 __kmp_atfork_child); 1393 KMP_CHECK_SYSFAIL("pthread_atfork", status); 1394 #endif 1395 __kmp_need_register_atfork = FALSE; 1396 } 1397 } 1398 1399 void __kmp_suspend_initialize(void) { 1400 int status; 1401 status = pthread_mutexattr_init(&__kmp_suspend_mutex_attr); 1402 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status); 1403 status = pthread_condattr_init(&__kmp_suspend_cond_attr); 1404 KMP_CHECK_SYSFAIL("pthread_condattr_init", status); 1405 } 1406 1407 void __kmp_suspend_initialize_thread(kmp_info_t *th) { 1408 int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count); 1409 int new_value = __kmp_fork_count + 1; 1410 // Return if already initialized 1411 if (old_value == new_value) 1412 return; 1413 // Wait, then return if being initialized 1414 if (old_value == -1 || !__kmp_atomic_compare_store( 1415 &th->th.th_suspend_init_count, old_value, -1)) { 1416 while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) { 1417 KMP_CPU_PAUSE(); 1418 } 1419 } else { 1420 // Claim to be the initializer and do initializations 1421 int status; 1422 status = pthread_cond_init(&th->th.th_suspend_cv.c_cond, 1423 &__kmp_suspend_cond_attr); 1424 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 1425 status = pthread_mutex_init(&th->th.th_suspend_mx.m_mutex, 1426 &__kmp_suspend_mutex_attr); 1427 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 1428 KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value); 1429 } 1430 } 1431 1432 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) { 1433 if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) { 1434 /* this means we have initialize the suspension pthread objects for this 1435 thread in this instance of the process */ 1436 int status; 1437 1438 status = pthread_cond_destroy(&th->th.th_suspend_cv.c_cond); 1439 if (status != 0 && status != EBUSY) { 1440 KMP_SYSFAIL("pthread_cond_destroy", status); 1441 } 1442 status = pthread_mutex_destroy(&th->th.th_suspend_mx.m_mutex); 1443 if (status != 0 && status != EBUSY) { 1444 KMP_SYSFAIL("pthread_mutex_destroy", status); 1445 } 1446 --th->th.th_suspend_init_count; 1447 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count) == 1448 __kmp_fork_count); 1449 } 1450 } 1451 1452 // return true if lock obtained, false otherwise 1453 int __kmp_try_suspend_mx(kmp_info_t *th) { 1454 return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0); 1455 } 1456 1457 void __kmp_lock_suspend_mx(kmp_info_t *th) { 1458 int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex); 1459 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 1460 } 1461 1462 void __kmp_unlock_suspend_mx(kmp_info_t *th) { 1463 int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex); 1464 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 1465 } 1466 1467 /* This routine puts the calling thread to sleep after setting the 1468 sleep bit for the indicated flag variable to true. */ 1469 template <class C> 1470 static inline void __kmp_suspend_template(int th_gtid, C *flag) { 1471 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend); 1472 kmp_info_t *th = __kmp_threads[th_gtid]; 1473 int status; 1474 typename C::flag_t old_spin; 1475 1476 KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, 1477 flag->get())); 1478 1479 __kmp_suspend_initialize_thread(th); 1480 1481 __kmp_lock_suspend_mx(th); 1482 1483 KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n", 1484 th_gtid, flag->get())); 1485 1486 /* TODO: shouldn't this use release semantics to ensure that 1487 __kmp_suspend_initialize_thread gets called first? */ 1488 old_spin = flag->set_sleeping(); 1489 TCW_PTR(th->th.th_sleep_loc, (void *)flag); 1490 th->th.th_sleep_loc_type = flag->get_type(); 1491 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && 1492 __kmp_pause_status != kmp_soft_paused) { 1493 flag->unset_sleeping(); 1494 TCW_PTR(th->th.th_sleep_loc, NULL); 1495 th->th.th_sleep_loc_type = flag_unset; 1496 __kmp_unlock_suspend_mx(th); 1497 return; 1498 } 1499 KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x," 1500 " was %x\n", 1501 th_gtid, flag->get(), flag->load(), old_spin)); 1502 1503 if (flag->done_check_val(old_spin) || flag->done_check()) { 1504 flag->unset_sleeping(); 1505 TCW_PTR(th->th.th_sleep_loc, NULL); 1506 th->th.th_sleep_loc_type = flag_unset; 1507 KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit " 1508 "for spin(%p)\n", 1509 th_gtid, flag->get())); 1510 } else { 1511 /* Encapsulate in a loop as the documentation states that this may 1512 "with low probability" return when the condition variable has 1513 not been signaled or broadcast */ 1514 int deactivated = FALSE; 1515 1516 while (flag->is_sleeping()) { 1517 #ifdef DEBUG_SUSPEND 1518 char buffer[128]; 1519 __kmp_suspend_count++; 1520 __kmp_print_cond(buffer, &th->th.th_suspend_cv); 1521 __kmp_printf("__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, 1522 buffer); 1523 #endif 1524 // Mark the thread as no longer active (only in the first iteration of the 1525 // loop). 1526 if (!deactivated) { 1527 th->th.th_active = FALSE; 1528 if (th->th.th_active_in_pool) { 1529 th->th.th_active_in_pool = FALSE; 1530 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth); 1531 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0); 1532 } 1533 deactivated = TRUE; 1534 } 1535 1536 KMP_DEBUG_ASSERT(th->th.th_sleep_loc); 1537 KMP_DEBUG_ASSERT(flag->get_type() == th->th.th_sleep_loc_type); 1538 1539 #if USE_SUSPEND_TIMEOUT 1540 struct timespec now; 1541 struct timeval tval; 1542 int msecs; 1543 1544 status = gettimeofday(&tval, NULL); 1545 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 1546 TIMEVAL_TO_TIMESPEC(&tval, &now); 1547 1548 msecs = (4 * __kmp_dflt_blocktime) + 200; 1549 now.tv_sec += msecs / 1000; 1550 now.tv_nsec += (msecs % 1000) * 1000; 1551 1552 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform " 1553 "pthread_cond_timedwait\n", 1554 th_gtid)); 1555 status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond, 1556 &th->th.th_suspend_mx.m_mutex, &now); 1557 #else 1558 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform" 1559 " pthread_cond_wait\n", 1560 th_gtid)); 1561 status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond, 1562 &th->th.th_suspend_mx.m_mutex); 1563 #endif // USE_SUSPEND_TIMEOUT 1564 1565 if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) { 1566 KMP_SYSFAIL("pthread_cond_wait", status); 1567 } 1568 1569 KMP_DEBUG_ASSERT(flag->get_type() == flag->get_ptr_type()); 1570 1571 if (!flag->is_sleeping() && 1572 ((status == EINTR) || (status == ETIMEDOUT))) { 1573 // if interrupt or timeout, and thread is no longer sleeping, we need to 1574 // make sure sleep_loc gets reset; however, this shouldn't be needed if 1575 // we woke up with resume 1576 flag->unset_sleeping(); 1577 TCW_PTR(th->th.th_sleep_loc, NULL); 1578 th->th.th_sleep_loc_type = flag_unset; 1579 } 1580 #ifdef KMP_DEBUG 1581 if (status == ETIMEDOUT) { 1582 if (flag->is_sleeping()) { 1583 KF_TRACE(100, 1584 ("__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid)); 1585 } else { 1586 KF_TRACE(2, ("__kmp_suspend_template: T#%d timeout wakeup, sleep bit " 1587 "not set!\n", 1588 th_gtid)); 1589 TCW_PTR(th->th.th_sleep_loc, NULL); 1590 th->th.th_sleep_loc_type = flag_unset; 1591 } 1592 } else if (flag->is_sleeping()) { 1593 KF_TRACE(100, 1594 ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid)); 1595 } 1596 #endif 1597 } // while 1598 1599 // Mark the thread as active again (if it was previous marked as inactive) 1600 if (deactivated) { 1601 th->th.th_active = TRUE; 1602 if (TCR_4(th->th.th_in_pool)) { 1603 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth); 1604 th->th.th_active_in_pool = TRUE; 1605 } 1606 } 1607 } 1608 // We may have had the loop variable set before entering the loop body; 1609 // so we need to reset sleep_loc. 1610 TCW_PTR(th->th.th_sleep_loc, NULL); 1611 th->th.th_sleep_loc_type = flag_unset; 1612 1613 KMP_DEBUG_ASSERT(!flag->is_sleeping()); 1614 KMP_DEBUG_ASSERT(!th->th.th_sleep_loc); 1615 #ifdef DEBUG_SUSPEND 1616 { 1617 char buffer[128]; 1618 __kmp_print_cond(buffer, &th->th.th_suspend_cv); 1619 __kmp_printf("__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, 1620 buffer); 1621 } 1622 #endif 1623 1624 __kmp_unlock_suspend_mx(th); 1625 KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid)); 1626 } 1627 1628 template <bool C, bool S> 1629 void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) { 1630 __kmp_suspend_template(th_gtid, flag); 1631 } 1632 template <bool C, bool S> 1633 void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) { 1634 __kmp_suspend_template(th_gtid, flag); 1635 } 1636 template <bool C, bool S> 1637 void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag) { 1638 __kmp_suspend_template(th_gtid, flag); 1639 } 1640 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) { 1641 __kmp_suspend_template(th_gtid, flag); 1642 } 1643 1644 template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *); 1645 template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *); 1646 template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *); 1647 template void 1648 __kmp_atomic_suspend_64<false, true>(int, kmp_atomic_flag_64<false, true> *); 1649 template void 1650 __kmp_atomic_suspend_64<true, false>(int, kmp_atomic_flag_64<true, false> *); 1651 1652 /* This routine signals the thread specified by target_gtid to wake up 1653 after setting the sleep bit indicated by the flag argument to FALSE. 1654 The target thread must already have called __kmp_suspend_template() */ 1655 template <class C> 1656 static inline void __kmp_resume_template(int target_gtid, C *flag) { 1657 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume); 1658 kmp_info_t *th = __kmp_threads[target_gtid]; 1659 int status; 1660 1661 #ifdef KMP_DEBUG 1662 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1; 1663 #endif 1664 1665 KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", 1666 gtid, target_gtid)); 1667 KMP_DEBUG_ASSERT(gtid != target_gtid); 1668 1669 __kmp_suspend_initialize_thread(th); 1670 1671 __kmp_lock_suspend_mx(th); 1672 1673 if (!flag || flag != th->th.th_sleep_loc) { 1674 // coming from __kmp_null_resume_wrapper, or thread is now sleeping on a 1675 // different location; wake up at new location 1676 flag = (C *)CCAST(void *, th->th.th_sleep_loc); 1677 } 1678 1679 // First, check if the flag is null or its type has changed. If so, someone 1680 // else woke it up. 1681 if (!flag) { // Thread doesn't appear to be sleeping on anything 1682 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already " 1683 "awake: flag(%p)\n", 1684 gtid, target_gtid, (void *)NULL)); 1685 __kmp_unlock_suspend_mx(th); 1686 return; 1687 } else if (flag->get_type() != th->th.th_sleep_loc_type) { 1688 // Flag type does not appear to match this function template; possibly the 1689 // thread is sleeping on something else. Try null resume again. 1690 KF_TRACE( 1691 5, 1692 ("__kmp_resume_template: T#%d retrying, thread T#%d Mismatch flag(%p), " 1693 "spin(%p) type=%d ptr_type=%d\n", 1694 gtid, target_gtid, flag, flag->get(), flag->get_type(), 1695 th->th.th_sleep_loc_type)); 1696 __kmp_unlock_suspend_mx(th); 1697 __kmp_null_resume_wrapper(th); 1698 return; 1699 } else { // if multiple threads are sleeping, flag should be internally 1700 // referring to a specific thread here 1701 if (!flag->is_sleeping()) { 1702 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already " 1703 "awake: flag(%p): %u\n", 1704 gtid, target_gtid, flag->get(), (unsigned int)flag->load())); 1705 __kmp_unlock_suspend_mx(th); 1706 return; 1707 } 1708 } 1709 KMP_DEBUG_ASSERT(flag); 1710 flag->unset_sleeping(); 1711 TCW_PTR(th->th.th_sleep_loc, NULL); 1712 th->th.th_sleep_loc_type = flag_unset; 1713 1714 KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset " 1715 "sleep bit for flag's loc(%p): %u\n", 1716 gtid, target_gtid, flag->get(), (unsigned int)flag->load())); 1717 1718 #ifdef DEBUG_SUSPEND 1719 { 1720 char buffer[128]; 1721 __kmp_print_cond(buffer, &th->th.th_suspend_cv); 1722 __kmp_printf("__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, 1723 target_gtid, buffer); 1724 } 1725 #endif 1726 status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond); 1727 KMP_CHECK_SYSFAIL("pthread_cond_signal", status); 1728 __kmp_unlock_suspend_mx(th); 1729 KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up" 1730 " for T#%d\n", 1731 gtid, target_gtid)); 1732 } 1733 1734 template <bool C, bool S> 1735 void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) { 1736 __kmp_resume_template(target_gtid, flag); 1737 } 1738 template <bool C, bool S> 1739 void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) { 1740 __kmp_resume_template(target_gtid, flag); 1741 } 1742 template <bool C, bool S> 1743 void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64<C, S> *flag) { 1744 __kmp_resume_template(target_gtid, flag); 1745 } 1746 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) { 1747 __kmp_resume_template(target_gtid, flag); 1748 } 1749 1750 template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *); 1751 template void __kmp_resume_32<false, false>(int, kmp_flag_32<false, false> *); 1752 template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *); 1753 template void 1754 __kmp_atomic_resume_64<false, true>(int, kmp_atomic_flag_64<false, true> *); 1755 1756 #if KMP_USE_MONITOR 1757 void __kmp_resume_monitor() { 1758 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume); 1759 int status; 1760 #ifdef KMP_DEBUG 1761 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1; 1762 KF_TRACE(30, ("__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid, 1763 KMP_GTID_MONITOR)); 1764 KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR); 1765 #endif 1766 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex); 1767 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 1768 #ifdef DEBUG_SUSPEND 1769 { 1770 char buffer[128]; 1771 __kmp_print_cond(buffer, &__kmp_wait_cv.c_cond); 1772 __kmp_printf("__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, 1773 KMP_GTID_MONITOR, buffer); 1774 } 1775 #endif 1776 status = pthread_cond_signal(&__kmp_wait_cv.c_cond); 1777 KMP_CHECK_SYSFAIL("pthread_cond_signal", status); 1778 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex); 1779 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 1780 KF_TRACE(30, ("__kmp_resume_monitor: T#%d exiting after signaling wake up" 1781 " for T#%d\n", 1782 gtid, KMP_GTID_MONITOR)); 1783 } 1784 #endif // KMP_USE_MONITOR 1785 1786 void __kmp_yield() { sched_yield(); } 1787 1788 void __kmp_gtid_set_specific(int gtid) { 1789 if (__kmp_init_gtid) { 1790 int status; 1791 status = pthread_setspecific(__kmp_gtid_threadprivate_key, 1792 (void *)(intptr_t)(gtid + 1)); 1793 KMP_CHECK_SYSFAIL("pthread_setspecific", status); 1794 } else { 1795 KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n")); 1796 } 1797 } 1798 1799 int __kmp_gtid_get_specific() { 1800 int gtid; 1801 if (!__kmp_init_gtid) { 1802 KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning " 1803 "KMP_GTID_SHUTDOWN\n")); 1804 return KMP_GTID_SHUTDOWN; 1805 } 1806 gtid = (int)(size_t)pthread_getspecific(__kmp_gtid_threadprivate_key); 1807 if (gtid == 0) { 1808 gtid = KMP_GTID_DNE; 1809 } else { 1810 gtid--; 1811 } 1812 KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n", 1813 __kmp_gtid_threadprivate_key, gtid)); 1814 return gtid; 1815 } 1816 1817 double __kmp_read_cpu_time(void) { 1818 /*clock_t t;*/ 1819 struct tms buffer; 1820 1821 /*t =*/times(&buffer); 1822 1823 return (double)(buffer.tms_utime + buffer.tms_cutime) / 1824 (double)CLOCKS_PER_SEC; 1825 } 1826 1827 int __kmp_read_system_info(struct kmp_sys_info *info) { 1828 int status; 1829 struct rusage r_usage; 1830 1831 memset(info, 0, sizeof(*info)); 1832 1833 status = getrusage(RUSAGE_SELF, &r_usage); 1834 KMP_CHECK_SYSFAIL_ERRNO("getrusage", status); 1835 1836 #if !KMP_OS_WASI 1837 // The maximum resident set size utilized (in kilobytes) 1838 info->maxrss = r_usage.ru_maxrss; 1839 // The number of page faults serviced without any I/O 1840 info->minflt = r_usage.ru_minflt; 1841 // The number of page faults serviced that required I/O 1842 info->majflt = r_usage.ru_majflt; 1843 // The number of times a process was "swapped" out of memory 1844 info->nswap = r_usage.ru_nswap; 1845 // The number of times the file system had to perform input 1846 info->inblock = r_usage.ru_inblock; 1847 // The number of times the file system had to perform output 1848 info->oublock = r_usage.ru_oublock; 1849 // The number of times a context switch was voluntarily 1850 info->nvcsw = r_usage.ru_nvcsw; 1851 // The number of times a context switch was forced 1852 info->nivcsw = r_usage.ru_nivcsw; 1853 #endif 1854 1855 return (status != 0); 1856 } 1857 1858 void __kmp_read_system_time(double *delta) { 1859 double t_ns; 1860 struct timeval tval; 1861 struct timespec stop; 1862 int status; 1863 1864 status = gettimeofday(&tval, NULL); 1865 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 1866 TIMEVAL_TO_TIMESPEC(&tval, &stop); 1867 t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start)); 1868 *delta = (t_ns * 1e-9); 1869 } 1870 1871 void __kmp_clear_system_time(void) { 1872 struct timeval tval; 1873 int status; 1874 status = gettimeofday(&tval, NULL); 1875 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 1876 TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start); 1877 } 1878 1879 static int __kmp_get_xproc(void) { 1880 1881 int r = 0; 1882 1883 #if KMP_OS_LINUX 1884 1885 __kmp_type_convert(sysconf(_SC_NPROCESSORS_CONF), &(r)); 1886 1887 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || \ 1888 KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX 1889 1890 __kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r)); 1891 1892 #elif KMP_OS_DARWIN 1893 1894 size_t len = sizeof(r); 1895 sysctlbyname("hw.logicalcpu", &r, &len, NULL, 0); 1896 1897 #else 1898 1899 #error "Unknown or unsupported OS." 1900 1901 #endif 1902 1903 return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */ 1904 1905 } // __kmp_get_xproc 1906 1907 int __kmp_read_from_file(char const *path, char const *format, ...) { 1908 int result; 1909 va_list args; 1910 1911 va_start(args, format); 1912 FILE *f = fopen(path, "rb"); 1913 if (f == NULL) { 1914 va_end(args); 1915 return 0; 1916 } 1917 result = vfscanf(f, format, args); 1918 fclose(f); 1919 va_end(args); 1920 1921 return result; 1922 } 1923 1924 void __kmp_runtime_initialize(void) { 1925 int status; 1926 pthread_mutexattr_t mutex_attr; 1927 pthread_condattr_t cond_attr; 1928 1929 if (__kmp_init_runtime) { 1930 return; 1931 } 1932 1933 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 1934 if (!__kmp_cpuinfo.initialized) { 1935 __kmp_query_cpuid(&__kmp_cpuinfo); 1936 } 1937 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ 1938 1939 __kmp_xproc = __kmp_get_xproc(); 1940 1941 #if !KMP_32_BIT_ARCH 1942 struct rlimit rlim; 1943 // read stack size of calling thread, save it as default for worker threads; 1944 // this should be done before reading environment variables 1945 status = getrlimit(RLIMIT_STACK, &rlim); 1946 if (status == 0) { // success? 1947 __kmp_stksize = rlim.rlim_cur; 1948 __kmp_check_stksize(&__kmp_stksize); // check value and adjust if needed 1949 } 1950 #endif /* KMP_32_BIT_ARCH */ 1951 1952 if (sysconf(_SC_THREADS)) { 1953 1954 /* Query the maximum number of threads */ 1955 __kmp_type_convert(sysconf(_SC_THREAD_THREADS_MAX), &(__kmp_sys_max_nth)); 1956 #ifdef __ve__ 1957 if (__kmp_sys_max_nth == -1) { 1958 // VE's pthread supports only up to 64 threads per a VE process. 1959 // So we use that KMP_MAX_NTH (predefined as 64) here. 1960 __kmp_sys_max_nth = KMP_MAX_NTH; 1961 } 1962 #else 1963 if (__kmp_sys_max_nth == -1) { 1964 /* Unlimited threads for NPTL */ 1965 __kmp_sys_max_nth = INT_MAX; 1966 } else if (__kmp_sys_max_nth <= 1) { 1967 /* Can't tell, just use PTHREAD_THREADS_MAX */ 1968 __kmp_sys_max_nth = KMP_MAX_NTH; 1969 } 1970 #endif 1971 1972 /* Query the minimum stack size */ 1973 __kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN); 1974 if (__kmp_sys_min_stksize <= 1) { 1975 __kmp_sys_min_stksize = KMP_MIN_STKSIZE; 1976 } 1977 } 1978 1979 /* Set up minimum number of threads to switch to TLS gtid */ 1980 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN; 1981 1982 status = pthread_key_create(&__kmp_gtid_threadprivate_key, 1983 __kmp_internal_end_dest); 1984 KMP_CHECK_SYSFAIL("pthread_key_create", status); 1985 status = pthread_mutexattr_init(&mutex_attr); 1986 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status); 1987 status = pthread_mutex_init(&__kmp_wait_mx.m_mutex, &mutex_attr); 1988 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 1989 status = pthread_mutexattr_destroy(&mutex_attr); 1990 KMP_CHECK_SYSFAIL("pthread_mutexattr_destroy", status); 1991 status = pthread_condattr_init(&cond_attr); 1992 KMP_CHECK_SYSFAIL("pthread_condattr_init", status); 1993 status = pthread_cond_init(&__kmp_wait_cv.c_cond, &cond_attr); 1994 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 1995 status = pthread_condattr_destroy(&cond_attr); 1996 KMP_CHECK_SYSFAIL("pthread_condattr_destroy", status); 1997 #if USE_ITT_BUILD 1998 __kmp_itt_initialize(); 1999 #endif /* USE_ITT_BUILD */ 2000 2001 __kmp_init_runtime = TRUE; 2002 } 2003 2004 void __kmp_runtime_destroy(void) { 2005 int status; 2006 2007 if (!__kmp_init_runtime) { 2008 return; // Nothing to do. 2009 } 2010 2011 #if USE_ITT_BUILD 2012 __kmp_itt_destroy(); 2013 #endif /* USE_ITT_BUILD */ 2014 2015 status = pthread_key_delete(__kmp_gtid_threadprivate_key); 2016 KMP_CHECK_SYSFAIL("pthread_key_delete", status); 2017 2018 status = pthread_mutex_destroy(&__kmp_wait_mx.m_mutex); 2019 if (status != 0 && status != EBUSY) { 2020 KMP_SYSFAIL("pthread_mutex_destroy", status); 2021 } 2022 status = pthread_cond_destroy(&__kmp_wait_cv.c_cond); 2023 if (status != 0 && status != EBUSY) { 2024 KMP_SYSFAIL("pthread_cond_destroy", status); 2025 } 2026 #if KMP_AFFINITY_SUPPORTED 2027 __kmp_affinity_uninitialize(); 2028 #endif 2029 2030 __kmp_init_runtime = FALSE; 2031 } 2032 2033 /* Put the thread to sleep for a time period */ 2034 /* NOTE: not currently used anywhere */ 2035 void __kmp_thread_sleep(int millis) { sleep((millis + 500) / 1000); } 2036 2037 /* Calculate the elapsed wall clock time for the user */ 2038 void __kmp_elapsed(double *t) { 2039 int status; 2040 #ifdef FIX_SGI_CLOCK 2041 struct timespec ts; 2042 2043 status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); 2044 KMP_CHECK_SYSFAIL_ERRNO("clock_gettime", status); 2045 *t = 2046 (double)ts.tv_nsec * (1.0 / (double)KMP_NSEC_PER_SEC) + (double)ts.tv_sec; 2047 #else 2048 struct timeval tv; 2049 2050 status = gettimeofday(&tv, NULL); 2051 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); 2052 *t = 2053 (double)tv.tv_usec * (1.0 / (double)KMP_USEC_PER_SEC) + (double)tv.tv_sec; 2054 #endif 2055 } 2056 2057 /* Calculate the elapsed wall clock tick for the user */ 2058 void __kmp_elapsed_tick(double *t) { *t = 1 / (double)CLOCKS_PER_SEC; } 2059 2060 /* Return the current time stamp in nsec */ 2061 kmp_uint64 __kmp_now_nsec() { 2062 struct timeval t; 2063 gettimeofday(&t, NULL); 2064 kmp_uint64 nsec = (kmp_uint64)KMP_NSEC_PER_SEC * (kmp_uint64)t.tv_sec + 2065 (kmp_uint64)1000 * (kmp_uint64)t.tv_usec; 2066 return nsec; 2067 } 2068 2069 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 2070 /* Measure clock ticks per millisecond */ 2071 void __kmp_initialize_system_tick() { 2072 kmp_uint64 now, nsec2, diff; 2073 kmp_uint64 delay = 1000000; // ~450 usec on most machines. 2074 kmp_uint64 nsec = __kmp_now_nsec(); 2075 kmp_uint64 goal = __kmp_hardware_timestamp() + delay; 2076 while ((now = __kmp_hardware_timestamp()) < goal) 2077 ; 2078 nsec2 = __kmp_now_nsec(); 2079 diff = nsec2 - nsec; 2080 if (diff > 0) { 2081 double tpus = 1000.0 * (double)(delay + (now - goal)) / (double)diff; 2082 if (tpus > 0.0) { 2083 __kmp_ticks_per_msec = (kmp_uint64)(tpus * 1000.0); 2084 __kmp_ticks_per_usec = (kmp_uint64)tpus; 2085 } 2086 } 2087 } 2088 #endif 2089 2090 /* Determine whether the given address is mapped into the current address 2091 space. */ 2092 2093 int __kmp_is_address_mapped(void *addr) { 2094 2095 int found = 0; 2096 int rc; 2097 2098 #if KMP_OS_LINUX || KMP_OS_HURD 2099 2100 /* On GNUish OSes, read the /proc/<pid>/maps pseudo-file to get all the 2101 address ranges mapped into the address space. */ 2102 2103 char *name = __kmp_str_format("/proc/%d/maps", getpid()); 2104 FILE *file = NULL; 2105 2106 file = fopen(name, "r"); 2107 KMP_ASSERT(file != NULL); 2108 2109 for (;;) { 2110 2111 void *beginning = NULL; 2112 void *ending = NULL; 2113 char perms[5]; 2114 2115 rc = fscanf(file, "%p-%p %4s %*[^\n]\n", &beginning, &ending, perms); 2116 if (rc == EOF) { 2117 break; 2118 } 2119 KMP_ASSERT(rc == 3 && 2120 KMP_STRLEN(perms) == 4); // Make sure all fields are read. 2121 2122 // Ending address is not included in the region, but beginning is. 2123 if ((addr >= beginning) && (addr < ending)) { 2124 perms[2] = 0; // 3th and 4th character does not matter. 2125 if (strcmp(perms, "rw") == 0) { 2126 // Memory we are looking for should be readable and writable. 2127 found = 1; 2128 } 2129 break; 2130 } 2131 } 2132 2133 // Free resources. 2134 fclose(file); 2135 KMP_INTERNAL_FREE(name); 2136 #elif KMP_OS_FREEBSD 2137 char *buf; 2138 size_t lstsz; 2139 int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()}; 2140 rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0); 2141 if (rc < 0) 2142 return 0; 2143 // We pass from number of vm entry's semantic 2144 // to size of whole entry map list. 2145 lstsz = lstsz * 4 / 3; 2146 buf = reinterpret_cast<char *>(KMP_INTERNAL_MALLOC(lstsz)); 2147 rc = sysctl(mib, 4, buf, &lstsz, NULL, 0); 2148 if (rc < 0) { 2149 KMP_INTERNAL_FREE(buf); 2150 return 0; 2151 } 2152 2153 char *lw = buf; 2154 char *up = buf + lstsz; 2155 2156 while (lw < up) { 2157 struct kinfo_vmentry *cur = reinterpret_cast<struct kinfo_vmentry *>(lw); 2158 size_t cursz = cur->kve_structsize; 2159 if (cursz == 0) 2160 break; 2161 void *start = reinterpret_cast<void *>(cur->kve_start); 2162 void *end = reinterpret_cast<void *>(cur->kve_end); 2163 // Readable/Writable addresses within current map entry 2164 if ((addr >= start) && (addr < end)) { 2165 if ((cur->kve_protection & KVME_PROT_READ) != 0 && 2166 (cur->kve_protection & KVME_PROT_WRITE) != 0) { 2167 found = 1; 2168 break; 2169 } 2170 } 2171 lw += cursz; 2172 } 2173 KMP_INTERNAL_FREE(buf); 2174 #elif KMP_OS_DRAGONFLY 2175 char err[_POSIX2_LINE_MAX]; 2176 kinfo_proc *proc; 2177 vmspace sp; 2178 vm_map *cur; 2179 vm_map_entry entry, *c; 2180 struct proc p; 2181 kvm_t *fd; 2182 uintptr_t uaddr; 2183 int num; 2184 2185 fd = kvm_openfiles(nullptr, nullptr, nullptr, O_RDONLY, err); 2186 if (!fd) { 2187 return 0; 2188 } 2189 2190 proc = kvm_getprocs(fd, KERN_PROC_PID, getpid(), &num); 2191 2192 if (kvm_read(fd, static_cast<uintptr_t>(proc->kp_paddr), &p, sizeof(p)) != 2193 sizeof(p) || 2194 kvm_read(fd, reinterpret_cast<uintptr_t>(p.p_vmspace), &sp, sizeof(sp)) != 2195 sizeof(sp)) { 2196 kvm_close(fd); 2197 return 0; 2198 } 2199 2200 (void)rc; 2201 cur = &sp.vm_map; 2202 uaddr = reinterpret_cast<uintptr_t>(addr); 2203 for (c = kvm_vm_map_entry_first(fd, cur, &entry); c; 2204 c = kvm_vm_map_entry_next(fd, c, &entry)) { 2205 if ((uaddr >= entry.ba.start) && (uaddr <= entry.ba.end)) { 2206 if ((entry.protection & VM_PROT_READ) != 0 && 2207 (entry.protection & VM_PROT_WRITE) != 0) { 2208 found = 1; 2209 break; 2210 } 2211 } 2212 } 2213 2214 kvm_close(fd); 2215 #elif KMP_OS_SOLARIS 2216 prmap_t *cur, *map; 2217 void *buf; 2218 uintptr_t uaddr; 2219 ssize_t rd; 2220 int err; 2221 int file; 2222 2223 pid_t pid = getpid(); 2224 struct ps_prochandle *fd = Pgrab(pid, PGRAB_RDONLY, &err); 2225 ; 2226 2227 if (!fd) { 2228 return 0; 2229 } 2230 2231 char *name = __kmp_str_format("/proc/%d/map", pid); 2232 size_t sz = (1 << 20); 2233 file = open(name, O_RDONLY); 2234 if (file == -1) { 2235 KMP_INTERNAL_FREE(name); 2236 return 0; 2237 } 2238 2239 buf = KMP_INTERNAL_MALLOC(sz); 2240 2241 while (sz > 0 && (rd = pread(file, buf, sz, 0)) == sz) { 2242 void *newbuf; 2243 sz <<= 1; 2244 newbuf = KMP_INTERNAL_REALLOC(buf, sz); 2245 buf = newbuf; 2246 } 2247 2248 map = reinterpret_cast<prmap_t *>(buf); 2249 uaddr = reinterpret_cast<uintptr_t>(addr); 2250 2251 for (cur = map; rd > 0; cur++, rd = -sizeof(*map)) { 2252 if ((uaddr >= cur->pr_vaddr) && (uaddr < cur->pr_vaddr)) { 2253 if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) { 2254 found = 1; 2255 break; 2256 } 2257 } 2258 } 2259 2260 KMP_INTERNAL_FREE(map); 2261 close(file); 2262 KMP_INTERNAL_FREE(name); 2263 #elif KMP_OS_DARWIN 2264 2265 /* On OS X*, /proc pseudo filesystem is not available. Try to read memory 2266 using vm interface. */ 2267 2268 int buffer; 2269 vm_size_t count; 2270 rc = vm_read_overwrite( 2271 mach_task_self(), // Task to read memory of. 2272 (vm_address_t)(addr), // Address to read from. 2273 1, // Number of bytes to be read. 2274 (vm_address_t)(&buffer), // Address of buffer to save read bytes in. 2275 &count // Address of var to save number of read bytes in. 2276 ); 2277 if (rc == 0) { 2278 // Memory successfully read. 2279 found = 1; 2280 } 2281 2282 #elif KMP_OS_NETBSD 2283 2284 int mib[5]; 2285 mib[0] = CTL_VM; 2286 mib[1] = VM_PROC; 2287 mib[2] = VM_PROC_MAP; 2288 mib[3] = getpid(); 2289 mib[4] = sizeof(struct kinfo_vmentry); 2290 2291 size_t size; 2292 rc = sysctl(mib, __arraycount(mib), NULL, &size, NULL, 0); 2293 KMP_ASSERT(!rc); 2294 KMP_ASSERT(size); 2295 2296 size = size * 4 / 3; 2297 struct kinfo_vmentry *kiv = (struct kinfo_vmentry *)KMP_INTERNAL_MALLOC(size); 2298 KMP_ASSERT(kiv); 2299 2300 rc = sysctl(mib, __arraycount(mib), kiv, &size, NULL, 0); 2301 KMP_ASSERT(!rc); 2302 KMP_ASSERT(size); 2303 2304 for (size_t i = 0; i < size; i++) { 2305 if (kiv[i].kve_start >= (uint64_t)addr && 2306 kiv[i].kve_end <= (uint64_t)addr) { 2307 found = 1; 2308 break; 2309 } 2310 } 2311 KMP_INTERNAL_FREE(kiv); 2312 #elif KMP_OS_OPENBSD 2313 2314 int mib[3]; 2315 mib[0] = CTL_KERN; 2316 mib[1] = KERN_PROC_VMMAP; 2317 mib[2] = getpid(); 2318 2319 size_t size; 2320 uint64_t end; 2321 rc = sysctl(mib, 3, NULL, &size, NULL, 0); 2322 KMP_ASSERT(!rc); 2323 KMP_ASSERT(size); 2324 end = size; 2325 2326 struct kinfo_vmentry kiv = {.kve_start = 0}; 2327 2328 while ((rc = sysctl(mib, 3, &kiv, &size, NULL, 0)) == 0) { 2329 KMP_ASSERT(size); 2330 if (kiv.kve_end == end) 2331 break; 2332 2333 if (kiv.kve_start >= (uint64_t)addr && kiv.kve_end <= (uint64_t)addr) { 2334 found = 1; 2335 break; 2336 } 2337 kiv.kve_start += 1; 2338 } 2339 #elif KMP_OS_WASI 2340 found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE); 2341 #elif KMP_OS_AIX 2342 2343 uint32_t loadQueryBufSize = 4096u; // Default loadquery buffer size. 2344 char *loadQueryBuf; 2345 2346 for (;;) { 2347 loadQueryBuf = (char *)KMP_INTERNAL_MALLOC(loadQueryBufSize); 2348 if (loadQueryBuf == NULL) { 2349 return 0; 2350 } 2351 2352 rc = loadquery(L_GETXINFO | L_IGNOREUNLOAD, loadQueryBuf, loadQueryBufSize); 2353 if (rc < 0) { 2354 KMP_INTERNAL_FREE(loadQueryBuf); 2355 if (errno != ENOMEM) { 2356 return 0; 2357 } 2358 // errno == ENOMEM; double the size. 2359 loadQueryBufSize <<= 1; 2360 continue; 2361 } 2362 // Obtained the load info successfully. 2363 break; 2364 } 2365 2366 struct ld_xinfo *curLdInfo = (struct ld_xinfo *)loadQueryBuf; 2367 2368 // Loop through the load info to find if there is a match. 2369 for (;;) { 2370 uintptr_t curDataStart = (uintptr_t)curLdInfo->ldinfo_dataorg; 2371 uintptr_t curDataEnd = curDataStart + curLdInfo->ldinfo_datasize; 2372 2373 // The data segment is readable and writable. 2374 if (curDataStart <= (uintptr_t)addr && (uintptr_t)addr < curDataEnd) { 2375 found = 1; 2376 break; 2377 } 2378 if (curLdInfo->ldinfo_next == 0u) { 2379 // Reached the end of load info. 2380 break; 2381 } 2382 curLdInfo = (struct ld_xinfo *)((char *)curLdInfo + curLdInfo->ldinfo_next); 2383 } 2384 KMP_INTERNAL_FREE(loadQueryBuf); 2385 2386 #else 2387 2388 #error "Unknown or unsupported OS" 2389 2390 #endif 2391 2392 return found; 2393 2394 } // __kmp_is_address_mapped 2395 2396 #ifdef USE_LOAD_BALANCE 2397 2398 #if KMP_OS_DARWIN || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ 2399 KMP_OS_OPENBSD || KMP_OS_SOLARIS 2400 2401 // The function returns the rounded value of the system load average 2402 // during given time interval which depends on the value of 2403 // __kmp_load_balance_interval variable (default is 60 sec, other values 2404 // may be 300 sec or 900 sec). 2405 // It returns -1 in case of error. 2406 int __kmp_get_load_balance(int max) { 2407 double averages[3]; 2408 int ret_avg = 0; 2409 2410 int res = getloadavg(averages, 3); 2411 2412 // Check __kmp_load_balance_interval to determine which of averages to use. 2413 // getloadavg() may return the number of samples less than requested that is 2414 // less than 3. 2415 if (__kmp_load_balance_interval < 180 && (res >= 1)) { 2416 ret_avg = (int)averages[0]; // 1 min 2417 } else if ((__kmp_load_balance_interval >= 180 && 2418 __kmp_load_balance_interval < 600) && 2419 (res >= 2)) { 2420 ret_avg = (int)averages[1]; // 5 min 2421 } else if ((__kmp_load_balance_interval >= 600) && (res == 3)) { 2422 ret_avg = (int)averages[2]; // 15 min 2423 } else { // Error occurred 2424 return -1; 2425 } 2426 2427 return ret_avg; 2428 } 2429 2430 #else // Linux* OS 2431 2432 // The function returns number of running (not sleeping) threads, or -1 in case 2433 // of error. Error could be reported if Linux* OS kernel too old (without 2434 // "/proc" support). Counting running threads stops if max running threads 2435 // encountered. 2436 int __kmp_get_load_balance(int max) { 2437 static int permanent_error = 0; 2438 static int glb_running_threads = 0; // Saved count of the running threads for 2439 // the thread balance algorithm 2440 static double glb_call_time = 0; /* Thread balance algorithm call time */ 2441 2442 int running_threads = 0; // Number of running threads in the system. 2443 2444 DIR *proc_dir = NULL; // Handle of "/proc/" directory. 2445 struct dirent *proc_entry = NULL; 2446 2447 kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path. 2448 DIR *task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory. 2449 struct dirent *task_entry = NULL; 2450 int task_path_fixed_len; 2451 2452 kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path. 2453 int stat_file = -1; 2454 int stat_path_fixed_len; 2455 2456 #ifdef KMP_DEBUG 2457 int total_processes = 0; // Total number of processes in system. 2458 #endif 2459 2460 double call_time = 0.0; 2461 2462 __kmp_str_buf_init(&task_path); 2463 __kmp_str_buf_init(&stat_path); 2464 2465 __kmp_elapsed(&call_time); 2466 2467 if (glb_call_time && 2468 (call_time - glb_call_time < __kmp_load_balance_interval)) { 2469 running_threads = glb_running_threads; 2470 goto finish; 2471 } 2472 2473 glb_call_time = call_time; 2474 2475 // Do not spend time on scanning "/proc/" if we have a permanent error. 2476 if (permanent_error) { 2477 running_threads = -1; 2478 goto finish; 2479 } 2480 2481 if (max <= 0) { 2482 max = INT_MAX; 2483 } 2484 2485 // Open "/proc/" directory. 2486 proc_dir = opendir("/proc"); 2487 if (proc_dir == NULL) { 2488 // Cannot open "/proc/". Probably the kernel does not support it. Return an 2489 // error now and in subsequent calls. 2490 running_threads = -1; 2491 permanent_error = 1; 2492 goto finish; 2493 } 2494 2495 // Initialize fixed part of task_path. This part will not change. 2496 __kmp_str_buf_cat(&task_path, "/proc/", 6); 2497 task_path_fixed_len = task_path.used; // Remember number of used characters. 2498 2499 proc_entry = readdir(proc_dir); 2500 while (proc_entry != NULL) { 2501 #if KMP_OS_AIX 2502 // Proc entry name starts with a digit. Assume it is a process' directory. 2503 if (isdigit(proc_entry->d_name[0])) { 2504 #else 2505 // Proc entry is a directory and name starts with a digit. Assume it is a 2506 // process' directory. 2507 if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) { 2508 #endif 2509 2510 #ifdef KMP_DEBUG 2511 ++total_processes; 2512 #endif 2513 // Make sure init process is the very first in "/proc", so we can replace 2514 // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes == 2515 // 1. We are going to check that total_processes == 1 => d_name == "1" is 2516 // true (where "=>" is implication). Since C++ does not have => operator, 2517 // let us replace it with its equivalent: a => b == ! a || b. 2518 KMP_DEBUG_ASSERT(total_processes != 1 || 2519 strcmp(proc_entry->d_name, "1") == 0); 2520 2521 // Construct task_path. 2522 task_path.used = task_path_fixed_len; // Reset task_path to "/proc/". 2523 __kmp_str_buf_cat(&task_path, proc_entry->d_name, 2524 KMP_STRLEN(proc_entry->d_name)); 2525 __kmp_str_buf_cat(&task_path, "/task", 5); 2526 2527 task_dir = opendir(task_path.str); 2528 if (task_dir == NULL) { 2529 // Process can finish between reading "/proc/" directory entry and 2530 // opening process' "task/" directory. So, in general case we should not 2531 // complain, but have to skip this process and read the next one. But on 2532 // systems with no "task/" support we will spend lot of time to scan 2533 // "/proc/" tree again and again without any benefit. "init" process 2534 // (its pid is 1) should exist always, so, if we cannot open 2535 // "/proc/1/task/" directory, it means "task/" is not supported by 2536 // kernel. Report an error now and in the future. 2537 if (strcmp(proc_entry->d_name, "1") == 0) { 2538 running_threads = -1; 2539 permanent_error = 1; 2540 goto finish; 2541 } 2542 } else { 2543 // Construct fixed part of stat file path. 2544 __kmp_str_buf_clear(&stat_path); 2545 __kmp_str_buf_cat(&stat_path, task_path.str, task_path.used); 2546 __kmp_str_buf_cat(&stat_path, "/", 1); 2547 stat_path_fixed_len = stat_path.used; 2548 2549 task_entry = readdir(task_dir); 2550 while (task_entry != NULL) { 2551 // It is a directory and name starts with a digit. 2552 #if KMP_OS_AIX 2553 if (isdigit(task_entry->d_name[0])) { 2554 #else 2555 if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) { 2556 #endif 2557 2558 // Construct complete stat file path. Easiest way would be: 2559 // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str, 2560 // task_entry->d_name ); 2561 // but seriae of __kmp_str_buf_cat works a bit faster. 2562 stat_path.used = 2563 stat_path_fixed_len; // Reset stat path to its fixed part. 2564 __kmp_str_buf_cat(&stat_path, task_entry->d_name, 2565 KMP_STRLEN(task_entry->d_name)); 2566 __kmp_str_buf_cat(&stat_path, "/stat", 5); 2567 2568 // Note: Low-level API (open/read/close) is used. High-level API 2569 // (fopen/fclose) works ~ 30 % slower. 2570 stat_file = open(stat_path.str, O_RDONLY); 2571 if (stat_file == -1) { 2572 // We cannot report an error because task (thread) can terminate 2573 // just before reading this file. 2574 } else { 2575 /* Content of "stat" file looks like: 2576 24285 (program) S ... 2577 2578 It is a single line (if program name does not include funny 2579 symbols). First number is a thread id, then name of executable 2580 file name in paretheses, then state of the thread. We need just 2581 thread state. 2582 2583 Good news: Length of program name is 15 characters max. Longer 2584 names are truncated. 2585 2586 Thus, we need rather short buffer: 15 chars for program name + 2587 2 parenthesis, + 3 spaces + ~7 digits of pid = 37. 2588 2589 Bad news: Program name may contain special symbols like space, 2590 closing parenthesis, or even new line. This makes parsing 2591 "stat" file not 100 % reliable. In case of fanny program names 2592 parsing may fail (report incorrect thread state). 2593 2594 Parsing "status" file looks more promissing (due to different 2595 file structure and escaping special symbols) but reading and 2596 parsing of "status" file works slower. 2597 -- ln 2598 */ 2599 char buffer[65]; 2600 ssize_t len; 2601 len = read(stat_file, buffer, sizeof(buffer) - 1); 2602 if (len >= 0) { 2603 buffer[len] = 0; 2604 // Using scanf: 2605 // sscanf( buffer, "%*d (%*s) %c ", & state ); 2606 // looks very nice, but searching for a closing parenthesis 2607 // works a bit faster. 2608 char *close_parent = strstr(buffer, ") "); 2609 if (close_parent != NULL) { 2610 char state = *(close_parent + 2); 2611 if (state == 'R') { 2612 ++running_threads; 2613 if (running_threads >= max) { 2614 goto finish; 2615 } 2616 } 2617 } 2618 } 2619 close(stat_file); 2620 stat_file = -1; 2621 } 2622 } 2623 task_entry = readdir(task_dir); 2624 } 2625 closedir(task_dir); 2626 task_dir = NULL; 2627 } 2628 } 2629 proc_entry = readdir(proc_dir); 2630 } 2631 2632 // There _might_ be a timing hole where the thread executing this 2633 // code get skipped in the load balance, and running_threads is 0. 2634 // Assert in the debug builds only!!! 2635 KMP_DEBUG_ASSERT(running_threads > 0); 2636 if (running_threads <= 0) { 2637 running_threads = 1; 2638 } 2639 2640 finish: // Clean up and exit. 2641 if (proc_dir != NULL) { 2642 closedir(proc_dir); 2643 } 2644 __kmp_str_buf_free(&task_path); 2645 if (task_dir != NULL) { 2646 closedir(task_dir); 2647 } 2648 __kmp_str_buf_free(&stat_path); 2649 if (stat_file != -1) { 2650 close(stat_file); 2651 } 2652 2653 glb_running_threads = running_threads; 2654 2655 return running_threads; 2656 2657 } // __kmp_get_load_balance 2658 2659 #endif // KMP_OS_DARWIN 2660 2661 #endif // USE_LOAD_BALANCE 2662 2663 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \ 2664 ((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || \ 2665 KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || \ 2666 KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF || \ 2667 KMP_ARCH_AARCH64_32) 2668 2669 // Because WebAssembly will use `call_indirect` to invoke the microtask and 2670 // WebAssembly indirect calls check that the called signature is a precise 2671 // match, we need to cast each microtask function pointer back from `void *` to 2672 // its original type. 2673 typedef void (*microtask_t0)(int *, int *); 2674 typedef void (*microtask_t1)(int *, int *, void *); 2675 typedef void (*microtask_t2)(int *, int *, void *, void *); 2676 typedef void (*microtask_t3)(int *, int *, void *, void *, void *); 2677 typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *); 2678 typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *, 2679 void *); 2680 typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *, 2681 void *, void *); 2682 typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *, 2683 void *, void *, void *); 2684 typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *, 2685 void *, void *, void *, void *); 2686 typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *, 2687 void *, void *, void *, void *, void *); 2688 typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *, 2689 void *, void *, void *, void *, void *, void *); 2690 typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *, 2691 void *, void *, void *, void *, void *, void *, 2692 void *); 2693 typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *, 2694 void *, void *, void *, void *, void *, void *, 2695 void *, void *); 2696 typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *, 2697 void *, void *, void *, void *, void *, void *, 2698 void *, void *, void *); 2699 typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *, 2700 void *, void *, void *, void *, void *, void *, 2701 void *, void *, void *, void *); 2702 typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *, 2703 void *, void *, void *, void *, void *, void *, 2704 void *, void *, void *, void *, void *); 2705 2706 // we really only need the case with 1 argument, because CLANG always build 2707 // a struct of pointers to shared variables referenced in the outlined function 2708 int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc, 2709 void *p_argv[] 2710 #if OMPT_SUPPORT 2711 , 2712 void **exit_frame_ptr 2713 #endif 2714 ) { 2715 #if OMPT_SUPPORT 2716 *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); 2717 #endif 2718 2719 switch (argc) { 2720 default: 2721 fprintf(stderr, "Too many args to microtask: %d!\n", argc); 2722 fflush(stderr); 2723 exit(-1); 2724 case 0: 2725 (*(microtask_t0)pkfn)(>id, &tid); 2726 break; 2727 case 1: 2728 (*(microtask_t1)pkfn)(>id, &tid, p_argv[0]); 2729 break; 2730 case 2: 2731 (*(microtask_t2)pkfn)(>id, &tid, p_argv[0], p_argv[1]); 2732 break; 2733 case 3: 2734 (*(microtask_t3)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2]); 2735 break; 2736 case 4: 2737 (*(microtask_t4)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2738 p_argv[3]); 2739 break; 2740 case 5: 2741 (*(microtask_t5)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2742 p_argv[3], p_argv[4]); 2743 break; 2744 case 6: 2745 (*(microtask_t6)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2746 p_argv[3], p_argv[4], p_argv[5]); 2747 break; 2748 case 7: 2749 (*(microtask_t7)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2750 p_argv[3], p_argv[4], p_argv[5], p_argv[6]); 2751 break; 2752 case 8: 2753 (*(microtask_t8)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2754 p_argv[3], p_argv[4], p_argv[5], p_argv[6], 2755 p_argv[7]); 2756 break; 2757 case 9: 2758 (*(microtask_t9)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2759 p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7], 2760 p_argv[8]); 2761 break; 2762 case 10: 2763 (*(microtask_t10)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2764 p_argv[3], p_argv[4], p_argv[5], p_argv[6], 2765 p_argv[7], p_argv[8], p_argv[9]); 2766 break; 2767 case 11: 2768 (*(microtask_t11)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2769 p_argv[3], p_argv[4], p_argv[5], p_argv[6], 2770 p_argv[7], p_argv[8], p_argv[9], p_argv[10]); 2771 break; 2772 case 12: 2773 (*(microtask_t12)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2774 p_argv[3], p_argv[4], p_argv[5], p_argv[6], 2775 p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2776 p_argv[11]); 2777 break; 2778 case 13: 2779 (*(microtask_t13)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2780 p_argv[3], p_argv[4], p_argv[5], p_argv[6], 2781 p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2782 p_argv[11], p_argv[12]); 2783 break; 2784 case 14: 2785 (*(microtask_t14)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2786 p_argv[3], p_argv[4], p_argv[5], p_argv[6], 2787 p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2788 p_argv[11], p_argv[12], p_argv[13]); 2789 break; 2790 case 15: 2791 (*(microtask_t15)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], 2792 p_argv[3], p_argv[4], p_argv[5], p_argv[6], 2793 p_argv[7], p_argv[8], p_argv[9], p_argv[10], 2794 p_argv[11], p_argv[12], p_argv[13], p_argv[14]); 2795 break; 2796 } 2797 2798 return 1; 2799 } 2800 2801 #endif 2802 2803 #if KMP_OS_LINUX 2804 // Functions for hidden helper task 2805 namespace { 2806 // Condition variable for initializing hidden helper team 2807 pthread_cond_t hidden_helper_threads_initz_cond_var; 2808 pthread_mutex_t hidden_helper_threads_initz_lock; 2809 volatile int hidden_helper_initz_signaled = FALSE; 2810 2811 // Condition variable for deinitializing hidden helper team 2812 pthread_cond_t hidden_helper_threads_deinitz_cond_var; 2813 pthread_mutex_t hidden_helper_threads_deinitz_lock; 2814 volatile int hidden_helper_deinitz_signaled = FALSE; 2815 2816 // Condition variable for the wrapper function of main thread 2817 pthread_cond_t hidden_helper_main_thread_cond_var; 2818 pthread_mutex_t hidden_helper_main_thread_lock; 2819 volatile int hidden_helper_main_thread_signaled = FALSE; 2820 2821 // Semaphore for worker threads. We don't use condition variable here in case 2822 // that when multiple signals are sent at the same time, only one thread might 2823 // be waken. 2824 sem_t hidden_helper_task_sem; 2825 } // namespace 2826 2827 void __kmp_hidden_helper_worker_thread_wait() { 2828 int status = sem_wait(&hidden_helper_task_sem); 2829 KMP_CHECK_SYSFAIL("sem_wait", status); 2830 } 2831 2832 void __kmp_do_initialize_hidden_helper_threads() { 2833 // Initialize condition variable 2834 int status = 2835 pthread_cond_init(&hidden_helper_threads_initz_cond_var, nullptr); 2836 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 2837 2838 status = pthread_cond_init(&hidden_helper_threads_deinitz_cond_var, nullptr); 2839 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 2840 2841 status = pthread_cond_init(&hidden_helper_main_thread_cond_var, nullptr); 2842 KMP_CHECK_SYSFAIL("pthread_cond_init", status); 2843 2844 status = pthread_mutex_init(&hidden_helper_threads_initz_lock, nullptr); 2845 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 2846 2847 status = pthread_mutex_init(&hidden_helper_threads_deinitz_lock, nullptr); 2848 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 2849 2850 status = pthread_mutex_init(&hidden_helper_main_thread_lock, nullptr); 2851 KMP_CHECK_SYSFAIL("pthread_mutex_init", status); 2852 2853 // Initialize the semaphore 2854 status = sem_init(&hidden_helper_task_sem, 0, 0); 2855 KMP_CHECK_SYSFAIL("sem_init", status); 2856 2857 // Create a new thread to finish initialization 2858 pthread_t handle; 2859 status = pthread_create( 2860 &handle, nullptr, 2861 [](void *) -> void * { 2862 __kmp_hidden_helper_threads_initz_routine(); 2863 return nullptr; 2864 }, 2865 nullptr); 2866 KMP_CHECK_SYSFAIL("pthread_create", status); 2867 } 2868 2869 void __kmp_hidden_helper_threads_initz_wait() { 2870 // Initial thread waits here for the completion of the initialization. The 2871 // condition variable will be notified by main thread of hidden helper teams. 2872 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock); 2873 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2874 2875 if (!TCR_4(hidden_helper_initz_signaled)) { 2876 status = pthread_cond_wait(&hidden_helper_threads_initz_cond_var, 2877 &hidden_helper_threads_initz_lock); 2878 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2879 } 2880 2881 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock); 2882 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2883 } 2884 2885 void __kmp_hidden_helper_initz_release() { 2886 // After all initialization, reset __kmp_init_hidden_helper_threads to false. 2887 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock); 2888 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2889 2890 status = pthread_cond_signal(&hidden_helper_threads_initz_cond_var); 2891 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2892 2893 TCW_SYNC_4(hidden_helper_initz_signaled, TRUE); 2894 2895 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock); 2896 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2897 } 2898 2899 void __kmp_hidden_helper_main_thread_wait() { 2900 // The main thread of hidden helper team will be blocked here. The 2901 // condition variable can only be signal in the destructor of RTL. 2902 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock); 2903 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2904 2905 if (!TCR_4(hidden_helper_main_thread_signaled)) { 2906 status = pthread_cond_wait(&hidden_helper_main_thread_cond_var, 2907 &hidden_helper_main_thread_lock); 2908 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2909 } 2910 2911 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock); 2912 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2913 } 2914 2915 void __kmp_hidden_helper_main_thread_release() { 2916 // The initial thread of OpenMP RTL should call this function to wake up the 2917 // main thread of hidden helper team. 2918 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock); 2919 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2920 2921 status = pthread_cond_signal(&hidden_helper_main_thread_cond_var); 2922 KMP_CHECK_SYSFAIL("pthread_cond_signal", status); 2923 2924 // The hidden helper team is done here 2925 TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE); 2926 2927 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock); 2928 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2929 } 2930 2931 void __kmp_hidden_helper_worker_thread_signal() { 2932 int status = sem_post(&hidden_helper_task_sem); 2933 KMP_CHECK_SYSFAIL("sem_post", status); 2934 } 2935 2936 void __kmp_hidden_helper_threads_deinitz_wait() { 2937 // Initial thread waits here for the completion of the deinitialization. The 2938 // condition variable will be notified by main thread of hidden helper teams. 2939 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock); 2940 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2941 2942 if (!TCR_4(hidden_helper_deinitz_signaled)) { 2943 status = pthread_cond_wait(&hidden_helper_threads_deinitz_cond_var, 2944 &hidden_helper_threads_deinitz_lock); 2945 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2946 } 2947 2948 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock); 2949 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2950 } 2951 2952 void __kmp_hidden_helper_threads_deinitz_release() { 2953 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock); 2954 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); 2955 2956 status = pthread_cond_signal(&hidden_helper_threads_deinitz_cond_var); 2957 KMP_CHECK_SYSFAIL("pthread_cond_wait", status); 2958 2959 TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE); 2960 2961 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock); 2962 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); 2963 } 2964 #else // KMP_OS_LINUX 2965 void __kmp_hidden_helper_worker_thread_wait() { 2966 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2967 } 2968 2969 void __kmp_do_initialize_hidden_helper_threads() { 2970 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2971 } 2972 2973 void __kmp_hidden_helper_threads_initz_wait() { 2974 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2975 } 2976 2977 void __kmp_hidden_helper_initz_release() { 2978 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2979 } 2980 2981 void __kmp_hidden_helper_main_thread_wait() { 2982 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2983 } 2984 2985 void __kmp_hidden_helper_main_thread_release() { 2986 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2987 } 2988 2989 void __kmp_hidden_helper_worker_thread_signal() { 2990 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2991 } 2992 2993 void __kmp_hidden_helper_threads_deinitz_wait() { 2994 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2995 } 2996 2997 void __kmp_hidden_helper_threads_deinitz_release() { 2998 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS"); 2999 } 3000 #endif // KMP_OS_LINUX 3001 3002 bool __kmp_detect_shm() { 3003 DIR *dir = opendir("/dev/shm"); 3004 if (dir) { // /dev/shm exists 3005 closedir(dir); 3006 return true; 3007 } else if (ENOENT == errno) { // /dev/shm does not exist 3008 return false; 3009 } else { // opendir() failed 3010 return false; 3011 } 3012 } 3013 3014 bool __kmp_detect_tmp() { 3015 DIR *dir = opendir("/tmp"); 3016 if (dir) { // /tmp exists 3017 closedir(dir); 3018 return true; 3019 } else if (ENOENT == errno) { // /tmp does not exist 3020 return false; 3021 } else { // opendir() failed 3022 return false; 3023 } 3024 } 3025 3026 // end of file // 3027