LLVM OpenMP* Runtime Library
z_Linux_util.cpp
1 /*
2  * z_Linux_util.cpp -- platform specific routines.
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "kmp.h"
14 #include "kmp_affinity.h"
15 #include "kmp_i18n.h"
16 #include "kmp_io.h"
17 #include "kmp_itt.h"
18 #include "kmp_lock.h"
19 #include "kmp_stats.h"
20 #include "kmp_str.h"
21 #include "kmp_wait_release.h"
22 #include "kmp_wrapper_getpid.h"
23 
24 #if !KMP_OS_DRAGONFLY && !KMP_OS_FREEBSD && !KMP_OS_NETBSD && !KMP_OS_OPENBSD
25 #include <alloca.h>
26 #endif
27 #include <math.h> // HUGE_VAL.
28 #if KMP_OS_LINUX
29 #include <semaphore.h>
30 #endif // KMP_OS_LINUX
31 #include <sys/resource.h>
32 #if KMP_OS_AIX
33 #include <sys/ldr.h>
34 #include <libperfstat.h>
35 #else
36 #include <sys/syscall.h>
37 #endif
38 #include <sys/time.h>
39 #include <sys/times.h>
40 #include <unistd.h>
41 
42 #if KMP_OS_LINUX
43 #include <sys/sysinfo.h>
44 #if KMP_USE_FUTEX
45 // We should really include <futex.h>, but that causes compatibility problems on
46 // different Linux* OS distributions that either require that you include (or
47 // break when you try to include) <pci/types.h>. Since all we need is the two
48 // macros below (which are part of the kernel ABI, so can't change) we just
49 // define the constants here and don't include <futex.h>
50 #ifndef FUTEX_WAIT
51 #define FUTEX_WAIT 0
52 #endif
53 #ifndef FUTEX_WAKE
54 #define FUTEX_WAKE 1
55 #endif
56 #endif
57 #elif KMP_OS_DARWIN
58 #include <mach/mach.h>
59 #include <sys/sysctl.h>
60 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD
61 #include <sys/types.h>
62 #include <sys/sysctl.h>
63 #include <sys/user.h>
64 #include <pthread_np.h>
65 #if KMP_OS_DRAGONFLY
66 #include <kvm.h>
67 #endif
68 #elif KMP_OS_NETBSD || KMP_OS_OPENBSD
69 #include <sys/types.h>
70 #include <sys/sysctl.h>
71 #if KMP_OS_NETBSD
72 #include <sched.h>
73 #endif
74 #elif KMP_OS_SOLARIS
75 #include <libproc.h>
76 #include <procfs.h>
77 #include <thread.h>
78 #include <sys/loadavg.h>
79 #endif
80 
81 #include <ctype.h>
82 #include <dirent.h>
83 #include <fcntl.h>
84 
85 struct kmp_sys_timer {
86  struct timespec start;
87 };
88 
89 #ifndef TIMEVAL_TO_TIMESPEC
90 // Convert timeval to timespec.
91 #define TIMEVAL_TO_TIMESPEC(tv, ts) \
92  do { \
93  (ts)->tv_sec = (tv)->tv_sec; \
94  (ts)->tv_nsec = (tv)->tv_usec * 1000; \
95  } while (0)
96 #endif
97 
98 // Convert timespec to nanoseconds.
99 #define TS2NS(timespec) \
100  (((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec)
101 
102 static struct kmp_sys_timer __kmp_sys_timer_data;
103 
104 #if KMP_HANDLE_SIGNALS
105 typedef void (*sig_func_t)(int);
106 STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[NSIG];
107 static sigset_t __kmp_sigset;
108 #endif
109 
110 static int __kmp_init_runtime = FALSE;
111 
112 static int __kmp_fork_count = 0;
113 
114 static pthread_condattr_t __kmp_suspend_cond_attr;
115 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
116 
117 static kmp_cond_align_t __kmp_wait_cv;
118 static kmp_mutex_align_t __kmp_wait_mx;
119 
120 kmp_uint64 __kmp_ticks_per_msec = 1000000;
121 kmp_uint64 __kmp_ticks_per_usec = 1000;
122 
123 #ifdef DEBUG_SUSPEND
124 static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) {
125  KMP_SNPRINTF(buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
126  cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
127  cond->c_cond.__c_waiting);
128 }
129 #endif
130 
131 #if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
132  KMP_OS_AIX) && \
133  KMP_AFFINITY_SUPPORTED)
134 
135 /* Affinity support */
136 
137 void __kmp_affinity_bind_thread(int which) {
138  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
139  "Illegal set affinity operation when not capable");
140 
141  kmp_affin_mask_t *mask;
142  KMP_CPU_ALLOC_ON_STACK(mask);
143  KMP_CPU_ZERO(mask);
144  KMP_CPU_SET(which, mask);
145  __kmp_set_system_affinity(mask, TRUE);
146  KMP_CPU_FREE_FROM_STACK(mask);
147 }
148 
149 #if KMP_OS_AIX
150 void __kmp_affinity_determine_capable(const char *env_var) {
151  // All versions of AIX support bindprocessor().
152 
153  size_t mask_size = __kmp_xproc / CHAR_BIT;
154  // Round up to byte boundary.
155  if (__kmp_xproc % CHAR_BIT)
156  ++mask_size;
157 
158  // Round up to the mask_size_type boundary.
159  if (mask_size % sizeof(__kmp_affin_mask_size))
160  mask_size += sizeof(__kmp_affin_mask_size) -
161  mask_size % sizeof(__kmp_affin_mask_size);
162  KMP_AFFINITY_ENABLE(mask_size);
163  KA_TRACE(10,
164  ("__kmp_affinity_determine_capable: "
165  "AIX OS affinity interface bindprocessor functional (mask size = "
166  "%" KMP_SIZE_T_SPEC ").\n",
167  __kmp_affin_mask_size));
168 }
169 
170 #else // !KMP_OS_AIX
171 
172 /* Determine if we can access affinity functionality on this version of
173  * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
174  * __kmp_affin_mask_size to the appropriate value (0 means not capable). */
175 void __kmp_affinity_determine_capable(const char *env_var) {
176  // Check and see if the OS supports thread affinity.
177 
178 #if KMP_OS_LINUX
179 #define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024)
180 #define KMP_CPU_SET_TRY_SIZE CACHE_LINE
181 #elif KMP_OS_FREEBSD || KMP_OS_DRAGONFLY
182 #define KMP_CPU_SET_SIZE_LIMIT (sizeof(cpuset_t))
183 #elif KMP_OS_NETBSD
184 #define KMP_CPU_SET_SIZE_LIMIT (256)
185 #endif
186 
187  int verbose = __kmp_affinity.flags.verbose;
188  int warnings = __kmp_affinity.flags.warnings;
189  enum affinity_type type = __kmp_affinity.type;
190 
191 #if KMP_OS_LINUX
192  long gCode;
193  unsigned char *buf;
194  buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
195 
196  // If the syscall returns a suggestion for the size,
197  // then we don't have to search for an appropriate size.
198  gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_TRY_SIZE, buf);
199  KA_TRACE(30, ("__kmp_affinity_determine_capable: "
200  "initial getaffinity call returned %ld errno = %d\n",
201  gCode, errno));
202 
203  if (gCode < 0 && errno != EINVAL) {
204  // System call not supported
205  if (verbose ||
206  (warnings && (type != affinity_none) && (type != affinity_default) &&
207  (type != affinity_disabled))) {
208  int error = errno;
209  kmp_msg_t err_code = KMP_ERR(error);
210  __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
211  err_code, __kmp_msg_null);
212  if (__kmp_generate_warnings == kmp_warnings_off) {
213  __kmp_str_free(&err_code.str);
214  }
215  }
216  KMP_AFFINITY_DISABLE();
217  KMP_INTERNAL_FREE(buf);
218  return;
219  } else if (gCode > 0) {
220  // The optimal situation: the OS returns the size of the buffer it expects.
221  KMP_AFFINITY_ENABLE(gCode);
222  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
223  "affinity supported (mask size %d)\n",
224  (int)__kmp_affin_mask_size));
225  KMP_INTERNAL_FREE(buf);
226  return;
227  }
228 
229  // Call the getaffinity system call repeatedly with increasing set sizes
230  // until we succeed, or reach an upper bound on the search.
231  KA_TRACE(30, ("__kmp_affinity_determine_capable: "
232  "searching for proper set size\n"));
233  int size;
234  for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
235  gCode = syscall(__NR_sched_getaffinity, 0, size, buf);
236  KA_TRACE(30, ("__kmp_affinity_determine_capable: "
237  "getaffinity for mask size %ld returned %ld errno = %d\n",
238  size, gCode, errno));
239 
240  if (gCode < 0) {
241  if (errno == ENOSYS) {
242  // We shouldn't get here
243  KA_TRACE(30, ("__kmp_affinity_determine_capable: "
244  "inconsistent OS call behavior: errno == ENOSYS for mask "
245  "size %d\n",
246  size));
247  if (verbose ||
248  (warnings && (type != affinity_none) &&
249  (type != affinity_default) && (type != affinity_disabled))) {
250  int error = errno;
251  kmp_msg_t err_code = KMP_ERR(error);
252  __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
253  err_code, __kmp_msg_null);
254  if (__kmp_generate_warnings == kmp_warnings_off) {
255  __kmp_str_free(&err_code.str);
256  }
257  }
258  KMP_AFFINITY_DISABLE();
259  KMP_INTERNAL_FREE(buf);
260  return;
261  }
262  continue;
263  }
264 
265  KMP_AFFINITY_ENABLE(gCode);
266  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
267  "affinity supported (mask size %d)\n",
268  (int)__kmp_affin_mask_size));
269  KMP_INTERNAL_FREE(buf);
270  return;
271  }
272 #elif KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY
273  long gCode;
274  unsigned char *buf;
275  buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
276  gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT,
277  reinterpret_cast<cpuset_t *>(buf));
278  KA_TRACE(30, ("__kmp_affinity_determine_capable: "
279  "initial getaffinity call returned %d errno = %d\n",
280  gCode, errno));
281  if (gCode == 0) {
282  KMP_AFFINITY_ENABLE(KMP_CPU_SET_SIZE_LIMIT);
283  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
284  "affinity supported (mask size %d)\n",
285  (int)__kmp_affin_mask_size));
286  KMP_INTERNAL_FREE(buf);
287  return;
288  }
289 #endif
290  KMP_INTERNAL_FREE(buf);
291 
292  // Affinity is not supported
293  KMP_AFFINITY_DISABLE();
294  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
295  "cannot determine mask size - affinity not supported\n"));
296  if (verbose || (warnings && (type != affinity_none) &&
297  (type != affinity_default) && (type != affinity_disabled))) {
298  KMP_WARNING(AffCantGetMaskSize, env_var);
299  }
300 }
301 #endif // KMP_OS_AIX
302 #endif // (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
303  KMP_OS_DRAGONFLY || KMP_OS_AIX) && KMP_AFFINITY_SUPPORTED
304 
305 #if KMP_USE_FUTEX
306 
307 int __kmp_futex_determine_capable() {
308  int loc = 0;
309  long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0);
310  int retval = (rc == 0) || (errno != ENOSYS);
311 
312  KA_TRACE(10,
313  ("__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno));
314  KA_TRACE(10, ("__kmp_futex_determine_capable: futex syscall%s supported\n",
315  retval ? "" : " not"));
316 
317  return retval;
318 }
319 
320 #endif // KMP_USE_FUTEX
321 
322 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS)
323 /* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
324  use compare_and_store for these routines */
325 
326 kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 d) {
327  kmp_int8 old_value, new_value;
328 
329  old_value = TCR_1(*p);
330  new_value = old_value | d;
331 
332  while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
333  KMP_CPU_PAUSE();
334  old_value = TCR_1(*p);
335  new_value = old_value | d;
336  }
337  return old_value;
338 }
339 
340 kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 d) {
341  kmp_int8 old_value, new_value;
342 
343  old_value = TCR_1(*p);
344  new_value = old_value & d;
345 
346  while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
347  KMP_CPU_PAUSE();
348  old_value = TCR_1(*p);
349  new_value = old_value & d;
350  }
351  return old_value;
352 }
353 
354 kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 d) {
355  kmp_uint32 old_value, new_value;
356 
357  old_value = TCR_4(*p);
358  new_value = old_value | d;
359 
360  while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
361  KMP_CPU_PAUSE();
362  old_value = TCR_4(*p);
363  new_value = old_value | d;
364  }
365  return old_value;
366 }
367 
368 kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) {
369  kmp_uint32 old_value, new_value;
370 
371  old_value = TCR_4(*p);
372  new_value = old_value & d;
373 
374  while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
375  KMP_CPU_PAUSE();
376  old_value = TCR_4(*p);
377  new_value = old_value & d;
378  }
379  return old_value;
380 }
381 
382 #if KMP_ARCH_X86 || KMP_ARCH_WASM
383 kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) {
384  kmp_int8 old_value, new_value;
385 
386  old_value = TCR_1(*p);
387  new_value = old_value + d;
388 
389  while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
390  KMP_CPU_PAUSE();
391  old_value = TCR_1(*p);
392  new_value = old_value + d;
393  }
394  return old_value;
395 }
396 
397 kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 d) {
398  kmp_int64 old_value, new_value;
399 
400  old_value = TCR_8(*p);
401  new_value = old_value + d;
402 
403  while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
404  KMP_CPU_PAUSE();
405  old_value = TCR_8(*p);
406  new_value = old_value + d;
407  }
408  return old_value;
409 }
410 #endif /* KMP_ARCH_X86 */
411 
412 kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 d) {
413  kmp_uint64 old_value, new_value;
414 
415  old_value = TCR_8(*p);
416  new_value = old_value | d;
417  while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
418  KMP_CPU_PAUSE();
419  old_value = TCR_8(*p);
420  new_value = old_value | d;
421  }
422  return old_value;
423 }
424 
425 kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 d) {
426  kmp_uint64 old_value, new_value;
427 
428  old_value = TCR_8(*p);
429  new_value = old_value & d;
430  while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
431  KMP_CPU_PAUSE();
432  old_value = TCR_8(*p);
433  new_value = old_value & d;
434  }
435  return old_value;
436 }
437 
438 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
439 
440 void __kmp_terminate_thread(int gtid) {
441  int status;
442  kmp_info_t *th = __kmp_threads[gtid];
443 
444  if (!th)
445  return;
446 
447 #ifdef KMP_CANCEL_THREADS
448  KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
449  status = pthread_cancel(th->th.th_info.ds.ds_thread);
450  if (status != 0 && status != ESRCH) {
451  __kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status),
452  __kmp_msg_null);
453  }
454 #endif
455  KMP_YIELD(TRUE);
456 } //
457 
458 /* Set thread stack info.
459  If values are unreasonable, assume call failed and use incremental stack
460  refinement method instead. Returns TRUE if the stack parameters could be
461  determined exactly, FALSE if incremental refinement is necessary. */
462 static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
463  int stack_data;
464 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
465  KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
466  int status;
467  size_t size = 0;
468  void *addr = 0;
469 
470  /* Always do incremental stack refinement for ubermaster threads since the
471  initial thread stack range can be reduced by sibling thread creation so
472  pthread_attr_getstack may cause thread gtid aliasing */
473  if (!KMP_UBER_GTID(gtid)) {
474 
475 #if KMP_OS_SOLARIS
476  stack_t s;
477  if ((status = thr_stksegment(&s)) < 0) {
478  KMP_CHECK_SYSFAIL("thr_stksegment", status);
479  }
480 
481  addr = s.ss_sp;
482  size = s.ss_size;
483  KA_TRACE(60, ("__kmp_set_stack_info: T#%d thr_stksegment returned size:"
484  " %lu, low addr: %p\n",
485  gtid, size, addr));
486 #else
487  pthread_attr_t attr;
488  /* Fetch the real thread attributes */
489  status = pthread_attr_init(&attr);
490  KMP_CHECK_SYSFAIL("pthread_attr_init", status);
491 #if KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD
492  status = pthread_attr_get_np(pthread_self(), &attr);
493  KMP_CHECK_SYSFAIL("pthread_attr_get_np", status);
494 #else
495  status = pthread_getattr_np(pthread_self(), &attr);
496  KMP_CHECK_SYSFAIL("pthread_getattr_np", status);
497 #endif
498  status = pthread_attr_getstack(&attr, &addr, &size);
499  KMP_CHECK_SYSFAIL("pthread_attr_getstack", status);
500  KA_TRACE(60,
501  ("__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:"
502  " %lu, low addr: %p\n",
503  gtid, size, addr));
504  status = pthread_attr_destroy(&attr);
505  KMP_CHECK_SYSFAIL("pthread_attr_destroy", status);
506 #endif
507  }
508 
509  if (size != 0 && addr != 0) { // was stack parameter determination successful?
510  /* Store the correct base and size */
511  TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
512  TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
513  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
514  return TRUE;
515  }
516 #endif /* KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD \
517  || KMP_OS_HURD || KMP_OS_SOLARIS */
518  /* Use incremental refinement starting from initial conservative estimate */
519  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
520  TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
521  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
522  return FALSE;
523 }
524 
525 static void *__kmp_launch_worker(void *thr) {
526  int status, old_type, old_state;
527 #ifdef KMP_BLOCK_SIGNALS
528  sigset_t new_set, old_set;
529 #endif /* KMP_BLOCK_SIGNALS */
530  void *exit_val;
531 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
532  KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
533  void *volatile padding = 0;
534 #endif
535  int gtid;
536 
537  gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid;
538  __kmp_gtid_set_specific(gtid);
539 #ifdef KMP_TDATA_GTID
540  __kmp_gtid = gtid;
541 #endif
542 #if KMP_STATS_ENABLED
543  // set thread local index to point to thread-specific stats
544  __kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats;
545  __kmp_stats_thread_ptr->startLife();
546  KMP_SET_THREAD_STATE(IDLE);
547  KMP_INIT_PARTITIONED_TIMERS(OMP_idle);
548 #endif
549 
550 #if USE_ITT_BUILD
551  __kmp_itt_thread_name(gtid);
552 #endif /* USE_ITT_BUILD */
553 
554 #if KMP_AFFINITY_SUPPORTED
555  __kmp_affinity_bind_init_mask(gtid);
556 #endif
557 
558 #ifdef KMP_CANCEL_THREADS
559  status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
560  KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
561  // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
562  status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
563  KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
564 #endif
565 
566 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
567  // Set FP control regs to be a copy of the parallel initialization thread's.
568  __kmp_clear_x87_fpu_status_word();
569  __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
570  __kmp_load_mxcsr(&__kmp_init_mxcsr);
571 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
572 
573 #ifdef KMP_BLOCK_SIGNALS
574  status = sigfillset(&new_set);
575  KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
576  status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set);
577  KMP_CHECK_SYSFAIL("pthread_sigmask", status);
578 #endif /* KMP_BLOCK_SIGNALS */
579 
580 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
581  KMP_OS_OPENBSD || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
582  if (__kmp_stkoffset > 0 && gtid > 0) {
583  padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
584  (void)padding;
585  }
586 #endif
587 
588  KMP_MB();
589  __kmp_set_stack_info(gtid, (kmp_info_t *)thr);
590 
591  __kmp_check_stack_overlap((kmp_info_t *)thr);
592 
593  exit_val = __kmp_launch_thread((kmp_info_t *)thr);
594 
595 #ifdef KMP_BLOCK_SIGNALS
596  status = pthread_sigmask(SIG_SETMASK, &old_set, NULL);
597  KMP_CHECK_SYSFAIL("pthread_sigmask", status);
598 #endif /* KMP_BLOCK_SIGNALS */
599 
600  return exit_val;
601 }
602 
603 #if KMP_USE_MONITOR
604 /* The monitor thread controls all of the threads in the complex */
605 
606 static void *__kmp_launch_monitor(void *thr) {
607  int status, old_type, old_state;
608 #ifdef KMP_BLOCK_SIGNALS
609  sigset_t new_set;
610 #endif /* KMP_BLOCK_SIGNALS */
611  struct timespec interval;
612 
613  KMP_MB(); /* Flush all pending memory write invalidates. */
614 
615  KA_TRACE(10, ("__kmp_launch_monitor: #1 launched\n"));
616 
617  /* register us as the monitor thread */
618  __kmp_gtid_set_specific(KMP_GTID_MONITOR);
619 #ifdef KMP_TDATA_GTID
620  __kmp_gtid = KMP_GTID_MONITOR;
621 #endif
622 
623  KMP_MB();
624 
625 #if USE_ITT_BUILD
626  // Instruct Intel(R) Threading Tools to ignore monitor thread.
627  __kmp_itt_thread_ignore();
628 #endif /* USE_ITT_BUILD */
629 
630  __kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid,
631  (kmp_info_t *)thr);
632 
633  __kmp_check_stack_overlap((kmp_info_t *)thr);
634 
635 #ifdef KMP_CANCEL_THREADS
636  status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
637  KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
638  // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
639  status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
640  KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
641 #endif
642 
643 #if KMP_REAL_TIME_FIX
644  // This is a potential fix which allows application with real-time scheduling
645  // policy work. However, decision about the fix is not made yet, so it is
646  // disabled by default.
647  { // Are program started with real-time scheduling policy?
648  int sched = sched_getscheduler(0);
649  if (sched == SCHED_FIFO || sched == SCHED_RR) {
650  // Yes, we are a part of real-time application. Try to increase the
651  // priority of the monitor.
652  struct sched_param param;
653  int max_priority = sched_get_priority_max(sched);
654  int rc;
655  KMP_WARNING(RealTimeSchedNotSupported);
656  sched_getparam(0, &param);
657  if (param.sched_priority < max_priority) {
658  param.sched_priority += 1;
659  rc = sched_setscheduler(0, sched, &param);
660  if (rc != 0) {
661  int error = errno;
662  kmp_msg_t err_code = KMP_ERR(error);
663  __kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority),
664  err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null);
665  if (__kmp_generate_warnings == kmp_warnings_off) {
666  __kmp_str_free(&err_code.str);
667  }
668  }
669  } else {
670  // We cannot abort here, because number of CPUs may be enough for all
671  // the threads, including the monitor thread, so application could
672  // potentially work...
673  __kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority),
674  KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority),
675  __kmp_msg_null);
676  }
677  }
678  // AC: free thread that waits for monitor started
679  TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
680  }
681 #endif // KMP_REAL_TIME_FIX
682 
683  KMP_MB(); /* Flush all pending memory write invalidates. */
684 
685  if (__kmp_monitor_wakeups == 1) {
686  interval.tv_sec = 1;
687  interval.tv_nsec = 0;
688  } else {
689  interval.tv_sec = 0;
690  interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
691  }
692 
693  KA_TRACE(10, ("__kmp_launch_monitor: #2 monitor\n"));
694 
695  while (!TCR_4(__kmp_global.g.g_done)) {
696  struct timespec now;
697  struct timeval tval;
698 
699  /* This thread monitors the state of the system */
700 
701  KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
702 
703  status = gettimeofday(&tval, NULL);
704  KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
705  TIMEVAL_TO_TIMESPEC(&tval, &now);
706 
707  now.tv_sec += interval.tv_sec;
708  now.tv_nsec += interval.tv_nsec;
709 
710  if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
711  now.tv_sec += 1;
712  now.tv_nsec -= KMP_NSEC_PER_SEC;
713  }
714 
715  status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
716  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
717  // AC: the monitor should not fall asleep if g_done has been set
718  if (!TCR_4(__kmp_global.g.g_done)) { // check once more under mutex
719  status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond,
720  &__kmp_wait_mx.m_mutex, &now);
721  if (status != 0) {
722  if (status != ETIMEDOUT && status != EINTR) {
723  KMP_SYSFAIL("pthread_cond_timedwait", status);
724  }
725  }
726  }
727  status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
728  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
729 
730  TCW_4(__kmp_global.g.g_time.dt.t_value,
731  TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
732 
733  KMP_MB(); /* Flush all pending memory write invalidates. */
734  }
735 
736  KA_TRACE(10, ("__kmp_launch_monitor: #3 cleanup\n"));
737 
738 #ifdef KMP_BLOCK_SIGNALS
739  status = sigfillset(&new_set);
740  KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
741  status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL);
742  KMP_CHECK_SYSFAIL("pthread_sigmask", status);
743 #endif /* KMP_BLOCK_SIGNALS */
744 
745  KA_TRACE(10, ("__kmp_launch_monitor: #4 finished\n"));
746 
747  if (__kmp_global.g.g_abort != 0) {
748  /* now we need to terminate the worker threads */
749  /* the value of t_abort is the signal we caught */
750 
751  int gtid;
752 
753  KA_TRACE(10, ("__kmp_launch_monitor: #5 terminate sig=%d\n",
754  __kmp_global.g.g_abort));
755 
756  /* terminate the OpenMP worker threads */
757  /* TODO this is not valid for sibling threads!!
758  * the uber master might not be 0 anymore.. */
759  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
760  __kmp_terminate_thread(gtid);
761 
762  __kmp_cleanup();
763 
764  KA_TRACE(10, ("__kmp_launch_monitor: #6 raise sig=%d\n",
765  __kmp_global.g.g_abort));
766 
767  if (__kmp_global.g.g_abort > 0)
768  raise(__kmp_global.g.g_abort);
769  }
770 
771  KA_TRACE(10, ("__kmp_launch_monitor: #7 exit\n"));
772 
773  return thr;
774 }
775 #endif // KMP_USE_MONITOR
776 
777 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
778  pthread_t handle;
779  pthread_attr_t thread_attr;
780  int status;
781 
782  th->th.th_info.ds.ds_gtid = gtid;
783 
784 #if KMP_STATS_ENABLED
785  // sets up worker thread stats
786  __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
787 
788  // th->th.th_stats is used to transfer thread-specific stats-pointer to
789  // __kmp_launch_worker. So when thread is created (goes into
790  // __kmp_launch_worker) it will set its thread local pointer to
791  // th->th.th_stats
792  if (!KMP_UBER_GTID(gtid)) {
793  th->th.th_stats = __kmp_stats_list->push_back(gtid);
794  } else {
795  // For root threads, __kmp_stats_thread_ptr is set in __kmp_register_root(),
796  // so set the th->th.th_stats field to it.
797  th->th.th_stats = __kmp_stats_thread_ptr;
798  }
799  __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
800 
801 #endif // KMP_STATS_ENABLED
802 
803  if (KMP_UBER_GTID(gtid)) {
804  KA_TRACE(10, ("__kmp_create_worker: uber thread (%d)\n", gtid));
805  th->th.th_info.ds.ds_thread = pthread_self();
806  __kmp_set_stack_info(gtid, th);
807  __kmp_check_stack_overlap(th);
808  return;
809  }
810 
811  KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
812 
813  KMP_MB(); /* Flush all pending memory write invalidates. */
814 
815 #ifdef KMP_THREAD_ATTR
816  status = pthread_attr_init(&thread_attr);
817  if (status != 0) {
818  __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
819  }
820  status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
821  if (status != 0) {
822  __kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null);
823  }
824 
825  /* Set stack size for this thread now.
826  The multiple of 2 is there because on some machines, requesting an unusual
827  stacksize causes the thread to have an offset before the dummy alloca()
828  takes place to create the offset. Since we want the user to have a
829  sufficient stacksize AND support a stack offset, we alloca() twice the
830  offset so that the upcoming alloca() does not eliminate any premade offset,
831  and also gives the user the stack space they requested for all threads */
832  stack_size += gtid * __kmp_stkoffset * 2;
833 
834  KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
835  "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
836  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
837 
838 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
839  status = pthread_attr_setstacksize(&thread_attr, stack_size);
840 #ifdef KMP_BACKUP_STKSIZE
841  if (status != 0) {
842  if (!__kmp_env_stksize) {
843  stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
844  __kmp_stksize = KMP_BACKUP_STKSIZE;
845  KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
846  "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
847  "bytes\n",
848  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
849  status = pthread_attr_setstacksize(&thread_attr, stack_size);
850  }
851  }
852 #endif /* KMP_BACKUP_STKSIZE */
853  if (status != 0) {
854  __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
855  KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null);
856  }
857 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
858 
859 #endif /* KMP_THREAD_ATTR */
860 
861  status =
862  pthread_create(&handle, &thread_attr, __kmp_launch_worker, (void *)th);
863  if (status != 0 || !handle) { // ??? Why do we check handle??
864 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
865  if (status == EINVAL) {
866  __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
867  KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null);
868  }
869  if (status == ENOMEM) {
870  __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
871  KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null);
872  }
873 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
874  if (status == EAGAIN) {
875  __kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status),
876  KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null);
877  }
878  KMP_SYSFAIL("pthread_create", status);
879  }
880 
881  th->th.th_info.ds.ds_thread = handle;
882 
883 #ifdef KMP_THREAD_ATTR
884  status = pthread_attr_destroy(&thread_attr);
885  if (status) {
886  kmp_msg_t err_code = KMP_ERR(status);
887  __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
888  __kmp_msg_null);
889  if (__kmp_generate_warnings == kmp_warnings_off) {
890  __kmp_str_free(&err_code.str);
891  }
892  }
893 #endif /* KMP_THREAD_ATTR */
894 
895  KMP_MB(); /* Flush all pending memory write invalidates. */
896 
897  KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
898 
899 } // __kmp_create_worker
900 
901 #if KMP_USE_MONITOR
902 void __kmp_create_monitor(kmp_info_t *th) {
903  pthread_t handle;
904  pthread_attr_t thread_attr;
905  size_t size;
906  int status;
907  int auto_adj_size = FALSE;
908 
909  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
910  // We don't need monitor thread in case of MAX_BLOCKTIME
911  KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
912  "MAX blocktime\n"));
913  th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
914  th->th.th_info.ds.ds_gtid = 0;
915  return;
916  }
917  KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
918 
919  KMP_MB(); /* Flush all pending memory write invalidates. */
920 
921  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
922  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
923 #if KMP_REAL_TIME_FIX
924  TCW_4(__kmp_global.g.g_time.dt.t_value,
925  -1); // Will use it for synchronization a bit later.
926 #else
927  TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
928 #endif // KMP_REAL_TIME_FIX
929 
930 #ifdef KMP_THREAD_ATTR
931  if (__kmp_monitor_stksize == 0) {
932  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
933  auto_adj_size = TRUE;
934  }
935  status = pthread_attr_init(&thread_attr);
936  if (status != 0) {
937  __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
938  }
939  status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
940  if (status != 0) {
941  __kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null);
942  }
943 
944 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
945  status = pthread_attr_getstacksize(&thread_attr, &size);
946  KMP_CHECK_SYSFAIL("pthread_attr_getstacksize", status);
947 #else
948  size = __kmp_sys_min_stksize;
949 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
950 #endif /* KMP_THREAD_ATTR */
951 
952  if (__kmp_monitor_stksize == 0) {
953  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
954  }
955  if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
956  __kmp_monitor_stksize = __kmp_sys_min_stksize;
957  }
958 
959  KA_TRACE(10, ("__kmp_create_monitor: default stacksize = %lu bytes,"
960  "requested stacksize = %lu bytes\n",
961  size, __kmp_monitor_stksize));
962 
963 retry:
964 
965 /* Set stack size for this thread now. */
966 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
967  KA_TRACE(10, ("__kmp_create_monitor: setting stacksize = %lu bytes,",
968  __kmp_monitor_stksize));
969  status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize);
970  if (status != 0) {
971  if (auto_adj_size) {
972  __kmp_monitor_stksize *= 2;
973  goto retry;
974  }
975  kmp_msg_t err_code = KMP_ERR(status);
976  __kmp_msg(kmp_ms_warning, // should this be fatal? BB
977  KMP_MSG(CantSetMonitorStackSize, (long int)__kmp_monitor_stksize),
978  err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null);
979  if (__kmp_generate_warnings == kmp_warnings_off) {
980  __kmp_str_free(&err_code.str);
981  }
982  }
983 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
984 
985  status =
986  pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (void *)th);
987 
988  if (status != 0) {
989 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
990  if (status == EINVAL) {
991  if (auto_adj_size && (__kmp_monitor_stksize < (size_t)0x40000000)) {
992  __kmp_monitor_stksize *= 2;
993  goto retry;
994  }
995  __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
996  KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize),
997  __kmp_msg_null);
998  }
999  if (status == ENOMEM) {
1000  __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
1001  KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize),
1002  __kmp_msg_null);
1003  }
1004 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1005  if (status == EAGAIN) {
1006  __kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status),
1007  KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null);
1008  }
1009  KMP_SYSFAIL("pthread_create", status);
1010  }
1011 
1012  th->th.th_info.ds.ds_thread = handle;
1013 
1014 #if KMP_REAL_TIME_FIX
1015  // Wait for the monitor thread is really started and set its *priority*.
1016  KMP_DEBUG_ASSERT(sizeof(kmp_uint32) ==
1017  sizeof(__kmp_global.g.g_time.dt.t_value));
1018  __kmp_wait_4((kmp_uint32 volatile *)&__kmp_global.g.g_time.dt.t_value, -1,
1019  &__kmp_neq_4, NULL);
1020 #endif // KMP_REAL_TIME_FIX
1021 
1022 #ifdef KMP_THREAD_ATTR
1023  status = pthread_attr_destroy(&thread_attr);
1024  if (status != 0) {
1025  kmp_msg_t err_code = KMP_ERR(status);
1026  __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
1027  __kmp_msg_null);
1028  if (__kmp_generate_warnings == kmp_warnings_off) {
1029  __kmp_str_free(&err_code.str);
1030  }
1031  }
1032 #endif
1033 
1034  KMP_MB(); /* Flush all pending memory write invalidates. */
1035 
1036  KA_TRACE(10, ("__kmp_create_monitor: monitor created %#.8lx\n",
1037  th->th.th_info.ds.ds_thread));
1038 
1039 } // __kmp_create_monitor
1040 #endif // KMP_USE_MONITOR
1041 
1042 void __kmp_exit_thread(int exit_status) {
1043 #if KMP_OS_WASI
1044 // TODO: the wasm32-wasi-threads target does not yet support pthread_exit.
1045 #else
1046  pthread_exit((void *)(intptr_t)exit_status);
1047 #endif
1048 } // __kmp_exit_thread
1049 
1050 #if KMP_USE_MONITOR
1051 void __kmp_resume_monitor();
1052 
1053 extern "C" void __kmp_reap_monitor(kmp_info_t *th) {
1054  int status;
1055  void *exit_val;
1056 
1057  KA_TRACE(10, ("__kmp_reap_monitor: try to reap monitor thread with handle"
1058  " %#.8lx\n",
1059  th->th.th_info.ds.ds_thread));
1060 
1061  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1062  // If both tid and gtid are 0, it means the monitor did not ever start.
1063  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1064  KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1065  if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1066  KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1067  return;
1068  }
1069 
1070  KMP_MB(); /* Flush all pending memory write invalidates. */
1071 
1072  /* First, check to see whether the monitor thread exists to wake it up. This
1073  is to avoid performance problem when the monitor sleeps during
1074  blocktime-size interval */
1075 
1076  status = pthread_kill(th->th.th_info.ds.ds_thread, 0);
1077  if (status != ESRCH) {
1078  __kmp_resume_monitor(); // Wake up the monitor thread
1079  }
1080  KA_TRACE(10, ("__kmp_reap_monitor: try to join with monitor\n"));
1081  status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1082  if (exit_val != th) {
1083  __kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null);
1084  }
1085 
1086  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1087  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1088 
1089  KA_TRACE(10, ("__kmp_reap_monitor: done reaping monitor thread with handle"
1090  " %#.8lx\n",
1091  th->th.th_info.ds.ds_thread));
1092 
1093  KMP_MB(); /* Flush all pending memory write invalidates. */
1094 }
1095 #else
1096 // Empty symbol to export (see exports_so.txt) when
1097 // monitor thread feature is disabled
1098 extern "C" void __kmp_reap_monitor(kmp_info_t *th) { (void)th; }
1099 #endif // KMP_USE_MONITOR
1100 
1101 void __kmp_reap_worker(kmp_info_t *th) {
1102  int status;
1103  void *exit_val;
1104 
1105  KMP_MB(); /* Flush all pending memory write invalidates. */
1106 
1107  KA_TRACE(
1108  10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid));
1109 
1110  status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1111 #ifdef KMP_DEBUG
1112  /* Don't expose these to the user until we understand when they trigger */
1113  if (status != 0) {
1114  __kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null);
1115  }
1116  if (exit_val != th) {
1117  KA_TRACE(10, ("__kmp_reap_worker: worker T#%d did not reap properly, "
1118  "exit_val = %p\n",
1119  th->th.th_info.ds.ds_gtid, exit_val));
1120  }
1121 #else
1122  (void)status; // unused variable
1123 #endif /* KMP_DEBUG */
1124 
1125  KA_TRACE(10, ("__kmp_reap_worker: done reaping T#%d\n",
1126  th->th.th_info.ds.ds_gtid));
1127 
1128  KMP_MB(); /* Flush all pending memory write invalidates. */
1129 }
1130 
1131 #if KMP_HANDLE_SIGNALS
1132 
1133 static void __kmp_null_handler(int signo) {
1134  // Do nothing, for doing SIG_IGN-type actions.
1135 } // __kmp_null_handler
1136 
1137 static void __kmp_team_handler(int signo) {
1138  if (__kmp_global.g.g_abort == 0) {
1139 /* Stage 1 signal handler, let's shut down all of the threads */
1140 #ifdef KMP_DEBUG
1141  __kmp_debug_printf("__kmp_team_handler: caught signal = %d\n", signo);
1142 #endif
1143  switch (signo) {
1144  case SIGHUP:
1145  case SIGINT:
1146  case SIGQUIT:
1147  case SIGILL:
1148  case SIGABRT:
1149  case SIGFPE:
1150  case SIGBUS:
1151  case SIGSEGV:
1152 #ifdef SIGSYS
1153  case SIGSYS:
1154 #endif
1155  case SIGTERM:
1156  if (__kmp_debug_buf) {
1157  __kmp_dump_debug_buffer();
1158  }
1159  __kmp_unregister_library(); // cleanup shared memory
1160  KMP_MB(); // Flush all pending memory write invalidates.
1161  TCW_4(__kmp_global.g.g_abort, signo);
1162  KMP_MB(); // Flush all pending memory write invalidates.
1163  TCW_4(__kmp_global.g.g_done, TRUE);
1164  KMP_MB(); // Flush all pending memory write invalidates.
1165  break;
1166  default:
1167 #ifdef KMP_DEBUG
1168  __kmp_debug_printf("__kmp_team_handler: unknown signal type");
1169 #endif
1170  break;
1171  }
1172  }
1173 } // __kmp_team_handler
1174 
1175 static void __kmp_sigaction(int signum, const struct sigaction *act,
1176  struct sigaction *oldact) {
1177  int rc = sigaction(signum, act, oldact);
1178  KMP_CHECK_SYSFAIL_ERRNO("sigaction", rc);
1179 }
1180 
1181 static void __kmp_install_one_handler(int sig, sig_func_t handler_func,
1182  int parallel_init) {
1183  KMP_MB(); // Flush all pending memory write invalidates.
1184  KB_TRACE(60,
1185  ("__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init));
1186  if (parallel_init) {
1187  struct sigaction new_action;
1188  struct sigaction old_action;
1189  new_action.sa_handler = handler_func;
1190  new_action.sa_flags = 0;
1191  sigfillset(&new_action.sa_mask);
1192  __kmp_sigaction(sig, &new_action, &old_action);
1193  if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) {
1194  sigaddset(&__kmp_sigset, sig);
1195  } else {
1196  // Restore/keep user's handler if one previously installed.
1197  __kmp_sigaction(sig, &old_action, NULL);
1198  }
1199  } else {
1200  // Save initial/system signal handlers to see if user handlers installed.
1201  __kmp_sigaction(sig, NULL, &__kmp_sighldrs[sig]);
1202  }
1203  KMP_MB(); // Flush all pending memory write invalidates.
1204 } // __kmp_install_one_handler
1205 
1206 static void __kmp_remove_one_handler(int sig) {
1207  KB_TRACE(60, ("__kmp_remove_one_handler( %d )\n", sig));
1208  if (sigismember(&__kmp_sigset, sig)) {
1209  struct sigaction old;
1210  KMP_MB(); // Flush all pending memory write invalidates.
1211  __kmp_sigaction(sig, &__kmp_sighldrs[sig], &old);
1212  if ((old.sa_handler != __kmp_team_handler) &&
1213  (old.sa_handler != __kmp_null_handler)) {
1214  // Restore the users signal handler.
1215  KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1216  "restoring: sig=%d\n",
1217  sig));
1218  __kmp_sigaction(sig, &old, NULL);
1219  }
1220  sigdelset(&__kmp_sigset, sig);
1221  KMP_MB(); // Flush all pending memory write invalidates.
1222  }
1223 } // __kmp_remove_one_handler
1224 
1225 void __kmp_install_signals(int parallel_init) {
1226  KB_TRACE(10, ("__kmp_install_signals( %d )\n", parallel_init));
1227  if (__kmp_handle_signals || !parallel_init) {
1228  // If ! parallel_init, we do not install handlers, just save original
1229  // handlers. Let us do it even __handle_signals is 0.
1230  sigemptyset(&__kmp_sigset);
1231  __kmp_install_one_handler(SIGHUP, __kmp_team_handler, parallel_init);
1232  __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1233  __kmp_install_one_handler(SIGQUIT, __kmp_team_handler, parallel_init);
1234  __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1235  __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1236  __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1237  __kmp_install_one_handler(SIGBUS, __kmp_team_handler, parallel_init);
1238  __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1239 #ifdef SIGSYS
1240  __kmp_install_one_handler(SIGSYS, __kmp_team_handler, parallel_init);
1241 #endif // SIGSYS
1242  __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1243 #ifdef SIGPIPE
1244  __kmp_install_one_handler(SIGPIPE, __kmp_team_handler, parallel_init);
1245 #endif // SIGPIPE
1246  }
1247 } // __kmp_install_signals
1248 
1249 void __kmp_remove_signals(void) {
1250  int sig;
1251  KB_TRACE(10, ("__kmp_remove_signals()\n"));
1252  for (sig = 1; sig < NSIG; ++sig) {
1253  __kmp_remove_one_handler(sig);
1254  }
1255 } // __kmp_remove_signals
1256 
1257 #endif // KMP_HANDLE_SIGNALS
1258 
1259 void __kmp_enable(int new_state) {
1260 #ifdef KMP_CANCEL_THREADS
1261  int status, old_state;
1262  status = pthread_setcancelstate(new_state, &old_state);
1263  KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1264  KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE);
1265 #endif
1266 }
1267 
1268 void __kmp_disable(int *old_state) {
1269 #ifdef KMP_CANCEL_THREADS
1270  int status;
1271  status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state);
1272  KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1273 #endif
1274 }
1275 
1276 static void __kmp_atfork_prepare(void) {
1277  __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
1278  __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1279 }
1280 
1281 static void __kmp_atfork_parent(void) {
1282  __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1283  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1284 }
1285 
1286 /* Reset the library so execution in the child starts "all over again" with
1287  clean data structures in initial states. Don't worry about freeing memory
1288  allocated by parent, just abandon it to be safe. */
1289 static void __kmp_atfork_child(void) {
1290  __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1291  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1292  /* TODO make sure this is done right for nested/sibling */
1293  // ATT: Memory leaks are here? TODO: Check it and fix.
1294  /* KMP_ASSERT( 0 ); */
1295 
1296  ++__kmp_fork_count;
1297 
1298 #if KMP_AFFINITY_SUPPORTED
1299 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
1300  KMP_OS_AIX
1301  // reset the affinity in the child to the initial thread
1302  // affinity in the parent
1303  kmp_set_thread_affinity_mask_initial();
1304 #endif
1305  // Set default not to bind threads tightly in the child (we're expecting
1306  // over-subscription after the fork and this can improve things for
1307  // scripting languages that use OpenMP inside process-parallel code).
1308  if (__kmp_nested_proc_bind.bind_types != NULL) {
1309  __kmp_nested_proc_bind.bind_types[0] = proc_bind_false;
1310  }
1311  for (kmp_affinity_t *affinity : __kmp_affinities)
1312  *affinity = KMP_AFFINITY_INIT(affinity->env_var);
1313  __kmp_affin_fullMask = nullptr;
1314  __kmp_affin_origMask = nullptr;
1315  __kmp_topology = nullptr;
1316 #endif // KMP_AFFINITY_SUPPORTED
1317 
1318 #if KMP_USE_MONITOR
1319  __kmp_init_monitor = 0;
1320 #endif
1321  __kmp_init_parallel = FALSE;
1322  __kmp_init_middle = FALSE;
1323  __kmp_init_serial = FALSE;
1324  TCW_4(__kmp_init_gtid, FALSE);
1325  __kmp_init_common = FALSE;
1326 
1327  TCW_4(__kmp_init_user_locks, FALSE);
1328 #if !KMP_USE_DYNAMIC_LOCK
1329  __kmp_user_lock_table.used = 1;
1330  __kmp_user_lock_table.allocated = 0;
1331  __kmp_user_lock_table.table = NULL;
1332  __kmp_lock_blocks = NULL;
1333 #endif
1334 
1335  __kmp_all_nth = 0;
1336  TCW_4(__kmp_nth, 0);
1337 
1338  __kmp_thread_pool = NULL;
1339  __kmp_thread_pool_insert_pt = NULL;
1340  __kmp_team_pool = NULL;
1341 
1342  /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate
1343  here so threadprivate doesn't use stale data */
1344  KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n",
1345  __kmp_threadpriv_cache_list));
1346 
1347  while (__kmp_threadpriv_cache_list != NULL) {
1348 
1349  if (*__kmp_threadpriv_cache_list->addr != NULL) {
1350  KC_TRACE(50, ("__kmp_atfork_child: zeroing cache at address %p\n",
1351  &(*__kmp_threadpriv_cache_list->addr)));
1352 
1353  *__kmp_threadpriv_cache_list->addr = NULL;
1354  }
1355  __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next;
1356  }
1357 
1358  __kmp_init_runtime = FALSE;
1359 
1360  /* reset statically initialized locks */
1361  __kmp_init_bootstrap_lock(&__kmp_initz_lock);
1362  __kmp_init_bootstrap_lock(&__kmp_stdio_lock);
1363  __kmp_init_bootstrap_lock(&__kmp_console_lock);
1364  __kmp_init_bootstrap_lock(&__kmp_task_team_lock);
1365 
1366 #if USE_ITT_BUILD
1367  __kmp_itt_reset(); // reset ITT's global state
1368 #endif /* USE_ITT_BUILD */
1369 
1370  {
1371  // Child process often get terminated without any use of OpenMP. That might
1372  // cause mapped shared memory file to be left unattended. Thus we postpone
1373  // library registration till middle initialization in the child process.
1374  __kmp_need_register_serial = FALSE;
1375  __kmp_serial_initialize();
1376  }
1377 
1378  /* This is necessary to make sure no stale data is left around */
1379  /* AC: customers complain that we use unsafe routines in the atfork
1380  handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1381  in dynamic_link when check the presence of shared tbbmalloc library.
1382  Suggestion is to make the library initialization lazier, similar
1383  to what done for __kmpc_begin(). */
1384  // TODO: synchronize all static initializations with regular library
1385  // startup; look at kmp_global.cpp and etc.
1386  //__kmp_internal_begin ();
1387 }
1388 
1389 void __kmp_register_atfork(void) {
1390  if (__kmp_need_register_atfork) {
1391 #if !KMP_OS_WASI
1392  int status = pthread_atfork(__kmp_atfork_prepare, __kmp_atfork_parent,
1393  __kmp_atfork_child);
1394  KMP_CHECK_SYSFAIL("pthread_atfork", status);
1395 #endif
1396  __kmp_need_register_atfork = FALSE;
1397  }
1398 }
1399 
1400 void __kmp_suspend_initialize(void) {
1401  int status;
1402  status = pthread_mutexattr_init(&__kmp_suspend_mutex_attr);
1403  KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
1404  status = pthread_condattr_init(&__kmp_suspend_cond_attr);
1405  KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
1406 }
1407 
1408 void __kmp_suspend_initialize_thread(kmp_info_t *th) {
1409  int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count);
1410  int new_value = __kmp_fork_count + 1;
1411  // Return if already initialized
1412  if (old_value == new_value)
1413  return;
1414  // Wait, then return if being initialized
1415  if (old_value == -1 || !__kmp_atomic_compare_store(
1416  &th->th.th_suspend_init_count, old_value, -1)) {
1417  while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) {
1418  KMP_CPU_PAUSE();
1419  }
1420  } else {
1421  // Claim to be the initializer and do initializations
1422  int status;
1423  status = pthread_cond_init(&th->th.th_suspend_cv.c_cond,
1424  &__kmp_suspend_cond_attr);
1425  KMP_CHECK_SYSFAIL("pthread_cond_init", status);
1426  status = pthread_mutex_init(&th->th.th_suspend_mx.m_mutex,
1427  &__kmp_suspend_mutex_attr);
1428  KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
1429  KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value);
1430  }
1431 }
1432 
1433 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
1434  if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) {
1435  /* this means we have initialize the suspension pthread objects for this
1436  thread in this instance of the process */
1437  int status;
1438 
1439  status = pthread_cond_destroy(&th->th.th_suspend_cv.c_cond);
1440  if (status != 0 && status != EBUSY) {
1441  KMP_SYSFAIL("pthread_cond_destroy", status);
1442  }
1443  status = pthread_mutex_destroy(&th->th.th_suspend_mx.m_mutex);
1444  if (status != 0 && status != EBUSY) {
1445  KMP_SYSFAIL("pthread_mutex_destroy", status);
1446  }
1447  --th->th.th_suspend_init_count;
1448  KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count) ==
1449  __kmp_fork_count);
1450  }
1451 }
1452 
1453 // return true if lock obtained, false otherwise
1454 int __kmp_try_suspend_mx(kmp_info_t *th) {
1455  return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0);
1456 }
1457 
1458 void __kmp_lock_suspend_mx(kmp_info_t *th) {
1459  int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
1460  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1461 }
1462 
1463 void __kmp_unlock_suspend_mx(kmp_info_t *th) {
1464  int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
1465  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1466 }
1467 
1468 /* This routine puts the calling thread to sleep after setting the
1469  sleep bit for the indicated flag variable to true. */
1470 template <class C>
1471 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
1472  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
1473  kmp_info_t *th = __kmp_threads[th_gtid];
1474  int status;
1475  typename C::flag_t old_spin;
1476 
1477  KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid,
1478  flag->get()));
1479 
1480  __kmp_suspend_initialize_thread(th);
1481 
1482  __kmp_lock_suspend_mx(th);
1483 
1484  KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1485  th_gtid, flag->get()));
1486 
1487  /* TODO: shouldn't this use release semantics to ensure that
1488  __kmp_suspend_initialize_thread gets called first? */
1489  old_spin = flag->set_sleeping();
1490  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1491  th->th.th_sleep_loc_type = flag->get_type();
1492  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
1493  __kmp_pause_status != kmp_soft_paused) {
1494  flag->unset_sleeping();
1495  TCW_PTR(th->th.th_sleep_loc, NULL);
1496  th->th.th_sleep_loc_type = flag_unset;
1497  __kmp_unlock_suspend_mx(th);
1498  return;
1499  }
1500  KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x,"
1501  " was %x\n",
1502  th_gtid, flag->get(), flag->load(), old_spin));
1503 
1504  if (flag->done_check_val(old_spin) || flag->done_check()) {
1505  flag->unset_sleeping();
1506  TCW_PTR(th->th.th_sleep_loc, NULL);
1507  th->th.th_sleep_loc_type = flag_unset;
1508  KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
1509  "for spin(%p)\n",
1510  th_gtid, flag->get()));
1511  } else {
1512  /* Encapsulate in a loop as the documentation states that this may
1513  "with low probability" return when the condition variable has
1514  not been signaled or broadcast */
1515  int deactivated = FALSE;
1516 
1517  while (flag->is_sleeping()) {
1518 #ifdef DEBUG_SUSPEND
1519  char buffer[128];
1520  __kmp_suspend_count++;
1521  __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1522  __kmp_printf("__kmp_suspend_template: suspending T#%d: %s\n", th_gtid,
1523  buffer);
1524 #endif
1525  // Mark the thread as no longer active (only in the first iteration of the
1526  // loop).
1527  if (!deactivated) {
1528  th->th.th_active = FALSE;
1529  if (th->th.th_active_in_pool) {
1530  th->th.th_active_in_pool = FALSE;
1531  KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
1532  KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
1533  }
1534  deactivated = TRUE;
1535  }
1536 
1537  KMP_DEBUG_ASSERT(th->th.th_sleep_loc);
1538  KMP_DEBUG_ASSERT(flag->get_type() == th->th.th_sleep_loc_type);
1539 
1540 #if USE_SUSPEND_TIMEOUT
1541  struct timespec now;
1542  struct timeval tval;
1543  int msecs;
1544 
1545  status = gettimeofday(&tval, NULL);
1546  KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1547  TIMEVAL_TO_TIMESPEC(&tval, &now);
1548 
1549  msecs = (4 * __kmp_dflt_blocktime) + 200;
1550  now.tv_sec += msecs / 1000;
1551  now.tv_nsec += (msecs % 1000) * 1000;
1552 
1553  KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
1554  "pthread_cond_timedwait\n",
1555  th_gtid));
1556  status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond,
1557  &th->th.th_suspend_mx.m_mutex, &now);
1558 #else
1559  KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform"
1560  " pthread_cond_wait\n",
1561  th_gtid));
1562  status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond,
1563  &th->th.th_suspend_mx.m_mutex);
1564 #endif // USE_SUSPEND_TIMEOUT
1565 
1566  if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) {
1567  KMP_SYSFAIL("pthread_cond_wait", status);
1568  }
1569 
1570  KMP_DEBUG_ASSERT(flag->get_type() == flag->get_ptr_type());
1571 
1572  if (!flag->is_sleeping() &&
1573  ((status == EINTR) || (status == ETIMEDOUT))) {
1574  // if interrupt or timeout, and thread is no longer sleeping, we need to
1575  // make sure sleep_loc gets reset; however, this shouldn't be needed if
1576  // we woke up with resume
1577  flag->unset_sleeping();
1578  TCW_PTR(th->th.th_sleep_loc, NULL);
1579  th->th.th_sleep_loc_type = flag_unset;
1580  }
1581 #ifdef KMP_DEBUG
1582  if (status == ETIMEDOUT) {
1583  if (flag->is_sleeping()) {
1584  KF_TRACE(100,
1585  ("__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid));
1586  } else {
1587  KF_TRACE(2, ("__kmp_suspend_template: T#%d timeout wakeup, sleep bit "
1588  "not set!\n",
1589  th_gtid));
1590  TCW_PTR(th->th.th_sleep_loc, NULL);
1591  th->th.th_sleep_loc_type = flag_unset;
1592  }
1593  } else if (flag->is_sleeping()) {
1594  KF_TRACE(100,
1595  ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
1596  }
1597 #endif
1598  } // while
1599 
1600  // Mark the thread as active again (if it was previous marked as inactive)
1601  if (deactivated) {
1602  th->th.th_active = TRUE;
1603  if (TCR_4(th->th.th_in_pool)) {
1604  KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
1605  th->th.th_active_in_pool = TRUE;
1606  }
1607  }
1608  }
1609  // We may have had the loop variable set before entering the loop body;
1610  // so we need to reset sleep_loc.
1611  TCW_PTR(th->th.th_sleep_loc, NULL);
1612  th->th.th_sleep_loc_type = flag_unset;
1613 
1614  KMP_DEBUG_ASSERT(!flag->is_sleeping());
1615  KMP_DEBUG_ASSERT(!th->th.th_sleep_loc);
1616 #ifdef DEBUG_SUSPEND
1617  {
1618  char buffer[128];
1619  __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1620  __kmp_printf("__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid,
1621  buffer);
1622  }
1623 #endif
1624 
1625  __kmp_unlock_suspend_mx(th);
1626  KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
1627 }
1628 
1629 template <bool C, bool S>
1630 void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
1631  __kmp_suspend_template(th_gtid, flag);
1632 }
1633 template <bool C, bool S>
1634 void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
1635  __kmp_suspend_template(th_gtid, flag);
1636 }
1637 template <bool C, bool S>
1638 void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag) {
1639  __kmp_suspend_template(th_gtid, flag);
1640 }
1641 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1642  __kmp_suspend_template(th_gtid, flag);
1643 }
1644 
1645 template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
1646 template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
1647 template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
1648 template void
1649 __kmp_atomic_suspend_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1650 template void
1651 __kmp_atomic_suspend_64<true, false>(int, kmp_atomic_flag_64<true, false> *);
1652 
1653 /* This routine signals the thread specified by target_gtid to wake up
1654  after setting the sleep bit indicated by the flag argument to FALSE.
1655  The target thread must already have called __kmp_suspend_template() */
1656 template <class C>
1657 static inline void __kmp_resume_template(int target_gtid, C *flag) {
1658  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1659  kmp_info_t *th = __kmp_threads[target_gtid];
1660  int status;
1661 
1662 #ifdef KMP_DEBUG
1663  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1664 #endif
1665 
1666  KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
1667  gtid, target_gtid));
1668  KMP_DEBUG_ASSERT(gtid != target_gtid);
1669 
1670  __kmp_suspend_initialize_thread(th);
1671 
1672  __kmp_lock_suspend_mx(th);
1673 
1674  if (!flag || flag != th->th.th_sleep_loc) {
1675  // coming from __kmp_null_resume_wrapper, or thread is now sleeping on a
1676  // different location; wake up at new location
1677  flag = (C *)CCAST(void *, th->th.th_sleep_loc);
1678  }
1679 
1680  // First, check if the flag is null or its type has changed. If so, someone
1681  // else woke it up.
1682  if (!flag) { // Thread doesn't appear to be sleeping on anything
1683  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1684  "awake: flag(%p)\n",
1685  gtid, target_gtid, (void *)NULL));
1686  __kmp_unlock_suspend_mx(th);
1687  return;
1688  } else if (flag->get_type() != th->th.th_sleep_loc_type) {
1689  // Flag type does not appear to match this function template; possibly the
1690  // thread is sleeping on something else. Try null resume again.
1691  KF_TRACE(
1692  5,
1693  ("__kmp_resume_template: T#%d retrying, thread T#%d Mismatch flag(%p), "
1694  "spin(%p) type=%d ptr_type=%d\n",
1695  gtid, target_gtid, flag, flag->get(), flag->get_type(),
1696  th->th.th_sleep_loc_type));
1697  __kmp_unlock_suspend_mx(th);
1698  __kmp_null_resume_wrapper(th);
1699  return;
1700  } else { // if multiple threads are sleeping, flag should be internally
1701  // referring to a specific thread here
1702  if (!flag->is_sleeping()) {
1703  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1704  "awake: flag(%p): %u\n",
1705  gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1706  __kmp_unlock_suspend_mx(th);
1707  return;
1708  }
1709  }
1710  KMP_DEBUG_ASSERT(flag);
1711  flag->unset_sleeping();
1712  TCW_PTR(th->th.th_sleep_loc, NULL);
1713  th->th.th_sleep_loc_type = flag_unset;
1714 
1715  KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset "
1716  "sleep bit for flag's loc(%p): %u\n",
1717  gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1718 
1719 #ifdef DEBUG_SUSPEND
1720  {
1721  char buffer[128];
1722  __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1723  __kmp_printf("__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid,
1724  target_gtid, buffer);
1725  }
1726 #endif
1727  status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond);
1728  KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1729  __kmp_unlock_suspend_mx(th);
1730  KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
1731  " for T#%d\n",
1732  gtid, target_gtid));
1733 }
1734 
1735 template <bool C, bool S>
1736 void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
1737  __kmp_resume_template(target_gtid, flag);
1738 }
1739 template <bool C, bool S>
1740 void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
1741  __kmp_resume_template(target_gtid, flag);
1742 }
1743 template <bool C, bool S>
1744 void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64<C, S> *flag) {
1745  __kmp_resume_template(target_gtid, flag);
1746 }
1747 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1748  __kmp_resume_template(target_gtid, flag);
1749 }
1750 
1751 template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
1752 template void __kmp_resume_32<false, false>(int, kmp_flag_32<false, false> *);
1753 template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
1754 template void
1755 __kmp_atomic_resume_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1756 
1757 #if KMP_USE_MONITOR
1758 void __kmp_resume_monitor() {
1759  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1760  int status;
1761 #ifdef KMP_DEBUG
1762  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1763  KF_TRACE(30, ("__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid,
1764  KMP_GTID_MONITOR));
1765  KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR);
1766 #endif
1767  status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
1768  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1769 #ifdef DEBUG_SUSPEND
1770  {
1771  char buffer[128];
1772  __kmp_print_cond(buffer, &__kmp_wait_cv.c_cond);
1773  __kmp_printf("__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid,
1774  KMP_GTID_MONITOR, buffer);
1775  }
1776 #endif
1777  status = pthread_cond_signal(&__kmp_wait_cv.c_cond);
1778  KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1779  status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
1780  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1781  KF_TRACE(30, ("__kmp_resume_monitor: T#%d exiting after signaling wake up"
1782  " for T#%d\n",
1783  gtid, KMP_GTID_MONITOR));
1784 }
1785 #endif // KMP_USE_MONITOR
1786 
1787 void __kmp_yield() { sched_yield(); }
1788 
1789 void __kmp_gtid_set_specific(int gtid) {
1790  if (__kmp_init_gtid) {
1791  int status;
1792  status = pthread_setspecific(__kmp_gtid_threadprivate_key,
1793  (void *)(intptr_t)(gtid + 1));
1794  KMP_CHECK_SYSFAIL("pthread_setspecific", status);
1795  } else {
1796  KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
1797  }
1798 }
1799 
1800 int __kmp_gtid_get_specific() {
1801  int gtid;
1802  if (!__kmp_init_gtid) {
1803  KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
1804  "KMP_GTID_SHUTDOWN\n"));
1805  return KMP_GTID_SHUTDOWN;
1806  }
1807  gtid = (int)(size_t)pthread_getspecific(__kmp_gtid_threadprivate_key);
1808  if (gtid == 0) {
1809  gtid = KMP_GTID_DNE;
1810  } else {
1811  gtid--;
1812  }
1813  KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1814  __kmp_gtid_threadprivate_key, gtid));
1815  return gtid;
1816 }
1817 
1818 double __kmp_read_cpu_time(void) {
1819  /*clock_t t;*/
1820  struct tms buffer;
1821 
1822  /*t =*/times(&buffer);
1823 
1824  return (double)(buffer.tms_utime + buffer.tms_cutime) /
1825  (double)CLOCKS_PER_SEC;
1826 }
1827 
1828 int __kmp_read_system_info(struct kmp_sys_info *info) {
1829  int status;
1830  struct rusage r_usage;
1831 
1832  memset(info, 0, sizeof(*info));
1833 
1834  status = getrusage(RUSAGE_SELF, &r_usage);
1835  KMP_CHECK_SYSFAIL_ERRNO("getrusage", status);
1836 
1837 #if !KMP_OS_WASI
1838  // The maximum resident set size utilized (in kilobytes)
1839  info->maxrss = r_usage.ru_maxrss;
1840  // The number of page faults serviced without any I/O
1841  info->minflt = r_usage.ru_minflt;
1842  // The number of page faults serviced that required I/O
1843  info->majflt = r_usage.ru_majflt;
1844  // The number of times a process was "swapped" out of memory
1845  info->nswap = r_usage.ru_nswap;
1846  // The number of times the file system had to perform input
1847  info->inblock = r_usage.ru_inblock;
1848  // The number of times the file system had to perform output
1849  info->oublock = r_usage.ru_oublock;
1850  // The number of times a context switch was voluntarily
1851  info->nvcsw = r_usage.ru_nvcsw;
1852  // The number of times a context switch was forced
1853  info->nivcsw = r_usage.ru_nivcsw;
1854 #endif
1855 
1856  return (status != 0);
1857 }
1858 
1859 void __kmp_read_system_time(double *delta) {
1860  double t_ns;
1861  struct timeval tval;
1862  struct timespec stop;
1863  int status;
1864 
1865  status = gettimeofday(&tval, NULL);
1866  KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1867  TIMEVAL_TO_TIMESPEC(&tval, &stop);
1868  t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start));
1869  *delta = (t_ns * 1e-9);
1870 }
1871 
1872 void __kmp_clear_system_time(void) {
1873  struct timeval tval;
1874  int status;
1875  status = gettimeofday(&tval, NULL);
1876  KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1877  TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start);
1878 }
1879 
1880 static int __kmp_get_xproc(void) {
1881 
1882  int r = 0;
1883 
1884 #if KMP_OS_LINUX
1885 
1886  __kmp_type_convert(sysconf(_SC_NPROCESSORS_CONF), &(r));
1887 
1888 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || \
1889  KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
1890 
1891  __kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r));
1892 
1893 #elif KMP_OS_DARWIN
1894 
1895  size_t len = sizeof(r);
1896  sysctlbyname("hw.logicalcpu", &r, &len, NULL, 0);
1897 
1898 #else
1899 
1900 #error "Unknown or unsupported OS."
1901 
1902 #endif
1903 
1904  return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
1905 
1906 } // __kmp_get_xproc
1907 
1908 int __kmp_read_from_file(char const *path, char const *format, ...) {
1909  int result;
1910  va_list args;
1911 
1912  va_start(args, format);
1913  FILE *f = fopen(path, "rb");
1914  if (f == NULL) {
1915  va_end(args);
1916  return 0;
1917  }
1918  result = vfscanf(f, format, args);
1919  fclose(f);
1920  va_end(args);
1921 
1922  return result;
1923 }
1924 
1925 void __kmp_runtime_initialize(void) {
1926  int status;
1927  pthread_mutexattr_t mutex_attr;
1928  pthread_condattr_t cond_attr;
1929 
1930  if (__kmp_init_runtime) {
1931  return;
1932  }
1933 
1934 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1935  if (!__kmp_cpuinfo.initialized) {
1936  __kmp_query_cpuid(&__kmp_cpuinfo);
1937  }
1938 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1939 
1940  __kmp_xproc = __kmp_get_xproc();
1941 
1942 #if !KMP_32_BIT_ARCH
1943  struct rlimit rlim;
1944  // read stack size of calling thread, save it as default for worker threads;
1945  // this should be done before reading environment variables
1946  status = getrlimit(RLIMIT_STACK, &rlim);
1947  if (status == 0) { // success?
1948  __kmp_stksize = rlim.rlim_cur;
1949  __kmp_check_stksize(&__kmp_stksize); // check value and adjust if needed
1950  }
1951 #endif /* KMP_32_BIT_ARCH */
1952 
1953  if (sysconf(_SC_THREADS)) {
1954 
1955  /* Query the maximum number of threads */
1956  __kmp_type_convert(sysconf(_SC_THREAD_THREADS_MAX), &(__kmp_sys_max_nth));
1957 #ifdef __ve__
1958  if (__kmp_sys_max_nth == -1) {
1959  // VE's pthread supports only up to 64 threads per a VE process.
1960  // So we use that KMP_MAX_NTH (predefined as 64) here.
1961  __kmp_sys_max_nth = KMP_MAX_NTH;
1962  }
1963 #else
1964  if (__kmp_sys_max_nth == -1) {
1965  /* Unlimited threads for NPTL */
1966  __kmp_sys_max_nth = INT_MAX;
1967  } else if (__kmp_sys_max_nth <= 1) {
1968  /* Can't tell, just use PTHREAD_THREADS_MAX */
1969  __kmp_sys_max_nth = KMP_MAX_NTH;
1970  }
1971 #endif
1972 
1973  /* Query the minimum stack size */
1974  __kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN);
1975  if (__kmp_sys_min_stksize <= 1) {
1976  __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
1977  }
1978  }
1979 
1980  /* Set up minimum number of threads to switch to TLS gtid */
1981  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
1982 
1983  status = pthread_key_create(&__kmp_gtid_threadprivate_key,
1984  __kmp_internal_end_dest);
1985  KMP_CHECK_SYSFAIL("pthread_key_create", status);
1986  status = pthread_mutexattr_init(&mutex_attr);
1987  KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
1988  status = pthread_mutex_init(&__kmp_wait_mx.m_mutex, &mutex_attr);
1989  KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
1990  status = pthread_mutexattr_destroy(&mutex_attr);
1991  KMP_CHECK_SYSFAIL("pthread_mutexattr_destroy", status);
1992  status = pthread_condattr_init(&cond_attr);
1993  KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
1994  status = pthread_cond_init(&__kmp_wait_cv.c_cond, &cond_attr);
1995  KMP_CHECK_SYSFAIL("pthread_cond_init", status);
1996  status = pthread_condattr_destroy(&cond_attr);
1997  KMP_CHECK_SYSFAIL("pthread_condattr_destroy", status);
1998 #if USE_ITT_BUILD
1999  __kmp_itt_initialize();
2000 #endif /* USE_ITT_BUILD */
2001 
2002  __kmp_init_runtime = TRUE;
2003 }
2004 
2005 void __kmp_runtime_destroy(void) {
2006  int status;
2007 
2008  if (!__kmp_init_runtime) {
2009  return; // Nothing to do.
2010  }
2011 
2012 #if USE_ITT_BUILD
2013  __kmp_itt_destroy();
2014 #endif /* USE_ITT_BUILD */
2015 
2016  status = pthread_key_delete(__kmp_gtid_threadprivate_key);
2017  KMP_CHECK_SYSFAIL("pthread_key_delete", status);
2018 
2019  status = pthread_mutex_destroy(&__kmp_wait_mx.m_mutex);
2020  if (status != 0 && status != EBUSY) {
2021  KMP_SYSFAIL("pthread_mutex_destroy", status);
2022  }
2023  status = pthread_cond_destroy(&__kmp_wait_cv.c_cond);
2024  if (status != 0 && status != EBUSY) {
2025  KMP_SYSFAIL("pthread_cond_destroy", status);
2026  }
2027 #if KMP_AFFINITY_SUPPORTED
2028  __kmp_affinity_uninitialize();
2029 #endif
2030 
2031  __kmp_init_runtime = FALSE;
2032 }
2033 
2034 /* Put the thread to sleep for a time period */
2035 /* NOTE: not currently used anywhere */
2036 void __kmp_thread_sleep(int millis) { sleep((millis + 500) / 1000); }
2037 
2038 /* Calculate the elapsed wall clock time for the user */
2039 void __kmp_elapsed(double *t) {
2040  int status;
2041 #ifdef FIX_SGI_CLOCK
2042  struct timespec ts;
2043 
2044  status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
2045  KMP_CHECK_SYSFAIL_ERRNO("clock_gettime", status);
2046  *t =
2047  (double)ts.tv_nsec * (1.0 / (double)KMP_NSEC_PER_SEC) + (double)ts.tv_sec;
2048 #else
2049  struct timeval tv;
2050 
2051  status = gettimeofday(&tv, NULL);
2052  KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
2053  *t =
2054  (double)tv.tv_usec * (1.0 / (double)KMP_USEC_PER_SEC) + (double)tv.tv_sec;
2055 #endif
2056 }
2057 
2058 /* Calculate the elapsed wall clock tick for the user */
2059 void __kmp_elapsed_tick(double *t) { *t = 1 / (double)CLOCKS_PER_SEC; }
2060 
2061 /* Return the current time stamp in nsec */
2062 kmp_uint64 __kmp_now_nsec() {
2063  struct timeval t;
2064  gettimeofday(&t, NULL);
2065  kmp_uint64 nsec = (kmp_uint64)KMP_NSEC_PER_SEC * (kmp_uint64)t.tv_sec +
2066  (kmp_uint64)1000 * (kmp_uint64)t.tv_usec;
2067  return nsec;
2068 }
2069 
2070 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
2071 /* Measure clock ticks per millisecond */
2072 void __kmp_initialize_system_tick() {
2073  kmp_uint64 now, nsec2, diff;
2074  kmp_uint64 delay = 1000000; // ~450 usec on most machines.
2075  kmp_uint64 nsec = __kmp_now_nsec();
2076  kmp_uint64 goal = __kmp_hardware_timestamp() + delay;
2077  while ((now = __kmp_hardware_timestamp()) < goal)
2078  ;
2079  nsec2 = __kmp_now_nsec();
2080  diff = nsec2 - nsec;
2081  if (diff > 0) {
2082  double tpus = 1000.0 * (double)(delay + (now - goal)) / (double)diff;
2083  if (tpus > 0.0) {
2084  __kmp_ticks_per_msec = (kmp_uint64)(tpus * 1000.0);
2085  __kmp_ticks_per_usec = (kmp_uint64)tpus;
2086  }
2087  }
2088 }
2089 #endif
2090 
2091 /* Determine whether the given address is mapped into the current address
2092  space. */
2093 
2094 int __kmp_is_address_mapped(void *addr) {
2095 
2096  int found = 0;
2097  int rc;
2098 
2099 #if KMP_OS_LINUX || KMP_OS_HURD
2100 
2101  /* On GNUish OSes, read the /proc/<pid>/maps pseudo-file to get all the
2102  address ranges mapped into the address space. */
2103 
2104  char *name = __kmp_str_format("/proc/%d/maps", getpid());
2105  FILE *file = NULL;
2106 
2107  file = fopen(name, "r");
2108  KMP_ASSERT(file != NULL);
2109 
2110  for (;;) {
2111 
2112  void *beginning = NULL;
2113  void *ending = NULL;
2114  char perms[5];
2115 
2116  rc = fscanf(file, "%p-%p %4s %*[^\n]\n", &beginning, &ending, perms);
2117  if (rc == EOF) {
2118  break;
2119  }
2120  KMP_ASSERT(rc == 3 &&
2121  KMP_STRLEN(perms) == 4); // Make sure all fields are read.
2122 
2123  // Ending address is not included in the region, but beginning is.
2124  if ((addr >= beginning) && (addr < ending)) {
2125  perms[2] = 0; // 3th and 4th character does not matter.
2126  if (strcmp(perms, "rw") == 0) {
2127  // Memory we are looking for should be readable and writable.
2128  found = 1;
2129  }
2130  break;
2131  }
2132  }
2133 
2134  // Free resources.
2135  fclose(file);
2136  KMP_INTERNAL_FREE(name);
2137 #elif KMP_OS_FREEBSD
2138  char *buf;
2139  size_t lstsz;
2140  int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
2141  rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0);
2142  if (rc < 0)
2143  return 0;
2144  // We pass from number of vm entry's semantic
2145  // to size of whole entry map list.
2146  lstsz = lstsz * 4 / 3;
2147  buf = reinterpret_cast<char *>(KMP_INTERNAL_MALLOC(lstsz));
2148  rc = sysctl(mib, 4, buf, &lstsz, NULL, 0);
2149  if (rc < 0) {
2150  KMP_INTERNAL_FREE(buf);
2151  return 0;
2152  }
2153 
2154  char *lw = buf;
2155  char *up = buf + lstsz;
2156 
2157  while (lw < up) {
2158  struct kinfo_vmentry *cur = reinterpret_cast<struct kinfo_vmentry *>(lw);
2159  size_t cursz = cur->kve_structsize;
2160  if (cursz == 0)
2161  break;
2162  void *start = reinterpret_cast<void *>(cur->kve_start);
2163  void *end = reinterpret_cast<void *>(cur->kve_end);
2164  // Readable/Writable addresses within current map entry
2165  if ((addr >= start) && (addr < end)) {
2166  if ((cur->kve_protection & KVME_PROT_READ) != 0 &&
2167  (cur->kve_protection & KVME_PROT_WRITE) != 0) {
2168  found = 1;
2169  break;
2170  }
2171  }
2172  lw += cursz;
2173  }
2174  KMP_INTERNAL_FREE(buf);
2175 #elif KMP_OS_DRAGONFLY
2176  char err[_POSIX2_LINE_MAX];
2177  kinfo_proc *proc;
2178  vmspace sp;
2179  vm_map *cur;
2180  vm_map_entry entry, *c;
2181  struct proc p;
2182  kvm_t *fd;
2183  uintptr_t uaddr;
2184  int num;
2185 
2186  fd = kvm_openfiles(nullptr, nullptr, nullptr, O_RDONLY, err);
2187  if (!fd) {
2188  return 0;
2189  }
2190 
2191  proc = kvm_getprocs(fd, KERN_PROC_PID, getpid(), &num);
2192 
2193  if (kvm_read(fd, static_cast<uintptr_t>(proc->kp_paddr), &p, sizeof(p)) !=
2194  sizeof(p) ||
2195  kvm_read(fd, reinterpret_cast<uintptr_t>(p.p_vmspace), &sp, sizeof(sp)) !=
2196  sizeof(sp)) {
2197  kvm_close(fd);
2198  return 0;
2199  }
2200 
2201  (void)rc;
2202  cur = &sp.vm_map;
2203  uaddr = reinterpret_cast<uintptr_t>(addr);
2204  for (c = kvm_vm_map_entry_first(fd, cur, &entry); c;
2205  c = kvm_vm_map_entry_next(fd, c, &entry)) {
2206  if ((uaddr >= entry.ba.start) && (uaddr <= entry.ba.end)) {
2207  if ((entry.protection & VM_PROT_READ) != 0 &&
2208  (entry.protection & VM_PROT_WRITE) != 0) {
2209  found = 1;
2210  break;
2211  }
2212  }
2213  }
2214 
2215  kvm_close(fd);
2216 #elif KMP_OS_SOLARIS
2217  prmap_t *cur, *map;
2218  void *buf;
2219  uintptr_t uaddr;
2220  ssize_t rd;
2221  int err;
2222  int file;
2223 
2224  pid_t pid = getpid();
2225  struct ps_prochandle *fd = Pgrab(pid, PGRAB_RDONLY, &err);
2226  ;
2227 
2228  if (!fd) {
2229  return 0;
2230  }
2231 
2232  char *name = __kmp_str_format("/proc/%d/map", pid);
2233  size_t sz = (1 << 20);
2234  file = open(name, O_RDONLY);
2235  if (file == -1) {
2236  KMP_INTERNAL_FREE(name);
2237  return 0;
2238  }
2239 
2240  buf = KMP_INTERNAL_MALLOC(sz);
2241 
2242  while (sz > 0 && (rd = pread(file, buf, sz, 0)) == sz) {
2243  void *newbuf;
2244  sz <<= 1;
2245  newbuf = KMP_INTERNAL_REALLOC(buf, sz);
2246  buf = newbuf;
2247  }
2248 
2249  map = reinterpret_cast<prmap_t *>(buf);
2250  uaddr = reinterpret_cast<uintptr_t>(addr);
2251 
2252  for (cur = map; rd > 0; cur++, rd = -sizeof(*map)) {
2253  if ((uaddr >= cur->pr_vaddr) && (uaddr < cur->pr_vaddr)) {
2254  if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) {
2255  found = 1;
2256  break;
2257  }
2258  }
2259  }
2260 
2261  KMP_INTERNAL_FREE(map);
2262  close(file);
2263  KMP_INTERNAL_FREE(name);
2264 #elif KMP_OS_DARWIN
2265 
2266  /* On OS X*, /proc pseudo filesystem is not available. Try to read memory
2267  using vm interface. */
2268 
2269  int buffer;
2270  vm_size_t count;
2271  rc = vm_read_overwrite(
2272  mach_task_self(), // Task to read memory of.
2273  (vm_address_t)(addr), // Address to read from.
2274  1, // Number of bytes to be read.
2275  (vm_address_t)(&buffer), // Address of buffer to save read bytes in.
2276  &count // Address of var to save number of read bytes in.
2277  );
2278  if (rc == 0) {
2279  // Memory successfully read.
2280  found = 1;
2281  }
2282 
2283 #elif KMP_OS_NETBSD
2284 
2285  int mib[5];
2286  mib[0] = CTL_VM;
2287  mib[1] = VM_PROC;
2288  mib[2] = VM_PROC_MAP;
2289  mib[3] = getpid();
2290  mib[4] = sizeof(struct kinfo_vmentry);
2291 
2292  size_t size;
2293  rc = sysctl(mib, __arraycount(mib), NULL, &size, NULL, 0);
2294  KMP_ASSERT(!rc);
2295  KMP_ASSERT(size);
2296 
2297  size = size * 4 / 3;
2298  struct kinfo_vmentry *kiv = (struct kinfo_vmentry *)KMP_INTERNAL_MALLOC(size);
2299  KMP_ASSERT(kiv);
2300 
2301  rc = sysctl(mib, __arraycount(mib), kiv, &size, NULL, 0);
2302  KMP_ASSERT(!rc);
2303  KMP_ASSERT(size);
2304 
2305  for (size_t i = 0; i < size; i++) {
2306  if (kiv[i].kve_start >= (uint64_t)addr &&
2307  kiv[i].kve_end <= (uint64_t)addr) {
2308  found = 1;
2309  break;
2310  }
2311  }
2312  KMP_INTERNAL_FREE(kiv);
2313 #elif KMP_OS_OPENBSD
2314 
2315  int mib[3];
2316  mib[0] = CTL_KERN;
2317  mib[1] = KERN_PROC_VMMAP;
2318  mib[2] = getpid();
2319 
2320  size_t size;
2321  uint64_t end;
2322  rc = sysctl(mib, 3, NULL, &size, NULL, 0);
2323  KMP_ASSERT(!rc);
2324  KMP_ASSERT(size);
2325  end = size;
2326 
2327  struct kinfo_vmentry kiv = {.kve_start = 0};
2328 
2329  while ((rc = sysctl(mib, 3, &kiv, &size, NULL, 0)) == 0) {
2330  KMP_ASSERT(size);
2331  if (kiv.kve_end == end)
2332  break;
2333 
2334  if (kiv.kve_start >= (uint64_t)addr && kiv.kve_end <= (uint64_t)addr) {
2335  found = 1;
2336  break;
2337  }
2338  kiv.kve_start += 1;
2339  }
2340 #elif KMP_OS_WASI
2341  found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE);
2342 #elif KMP_OS_AIX
2343 
2344  uint32_t loadQueryBufSize = 4096u; // Default loadquery buffer size.
2345  char *loadQueryBuf;
2346 
2347  for (;;) {
2348  loadQueryBuf = (char *)KMP_INTERNAL_MALLOC(loadQueryBufSize);
2349  if (loadQueryBuf == NULL) {
2350  return 0;
2351  }
2352 
2353  rc = loadquery(L_GETXINFO | L_IGNOREUNLOAD, loadQueryBuf, loadQueryBufSize);
2354  if (rc < 0) {
2355  KMP_INTERNAL_FREE(loadQueryBuf);
2356  if (errno != ENOMEM) {
2357  return 0;
2358  }
2359  // errno == ENOMEM; double the size.
2360  loadQueryBufSize <<= 1;
2361  continue;
2362  }
2363  // Obtained the load info successfully.
2364  break;
2365  }
2366 
2367  struct ld_xinfo *curLdInfo = (struct ld_xinfo *)loadQueryBuf;
2368 
2369  // Loop through the load info to find if there is a match.
2370  for (;;) {
2371  uintptr_t curDataStart = (uintptr_t)curLdInfo->ldinfo_dataorg;
2372  uintptr_t curDataEnd = curDataStart + curLdInfo->ldinfo_datasize;
2373 
2374  // The data segment is readable and writable.
2375  if (curDataStart <= (uintptr_t)addr && (uintptr_t)addr < curDataEnd) {
2376  found = 1;
2377  break;
2378  }
2379  if (curLdInfo->ldinfo_next == 0u) {
2380  // Reached the end of load info.
2381  break;
2382  }
2383  curLdInfo = (struct ld_xinfo *)((char *)curLdInfo + curLdInfo->ldinfo_next);
2384  }
2385  KMP_INTERNAL_FREE(loadQueryBuf);
2386 
2387 #else
2388 
2389 #error "Unknown or unsupported OS"
2390 
2391 #endif
2392 
2393  return found;
2394 
2395 } // __kmp_is_address_mapped
2396 
2397 #ifdef USE_LOAD_BALANCE
2398 
2399 #if KMP_OS_DARWIN || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
2400  KMP_OS_OPENBSD || KMP_OS_SOLARIS
2401 
2402 // The function returns the rounded value of the system load average
2403 // during given time interval which depends on the value of
2404 // __kmp_load_balance_interval variable (default is 60 sec, other values
2405 // may be 300 sec or 900 sec).
2406 // It returns -1 in case of error.
2407 int __kmp_get_load_balance(int max) {
2408  double averages[3];
2409  int ret_avg = 0;
2410 
2411  int res = getloadavg(averages, 3);
2412 
2413  // Check __kmp_load_balance_interval to determine which of averages to use.
2414  // getloadavg() may return the number of samples less than requested that is
2415  // less than 3.
2416  if (__kmp_load_balance_interval < 180 && (res >= 1)) {
2417  ret_avg = (int)averages[0]; // 1 min
2418  } else if ((__kmp_load_balance_interval >= 180 &&
2419  __kmp_load_balance_interval < 600) &&
2420  (res >= 2)) {
2421  ret_avg = (int)averages[1]; // 5 min
2422  } else if ((__kmp_load_balance_interval >= 600) && (res == 3)) {
2423  ret_avg = (int)averages[2]; // 15 min
2424  } else { // Error occurred
2425  return -1;
2426  }
2427 
2428  return ret_avg;
2429 }
2430 
2431 #elif KMP_OS_AIX
2432 
2433 // The function returns number of running (not sleeping) threads, or -1 in case
2434 // of error.
2435 int __kmp_get_load_balance(int max) {
2436 
2437  static int glb_running_threads = 0; // Saved count of the running threads for
2438  // the thread balance algorithm.
2439  static double glb_call_time = 0; // Thread balance algorithm call time.
2440  int running_threads = 0; // Number of running threads in the system.
2441 
2442  double call_time = 0.0;
2443 
2444  __kmp_elapsed(&call_time);
2445 
2446  if (glb_call_time &&
2447  (call_time - glb_call_time < __kmp_load_balance_interval))
2448  return glb_running_threads;
2449 
2450  glb_call_time = call_time;
2451 
2452  if (max <= 0) {
2453  max = INT_MAX;
2454  }
2455 
2456  // Check how many perfstat_cpu_t structures are available.
2457  int logical_cpus = perfstat_cpu(NULL, NULL, sizeof(perfstat_cpu_t), 0);
2458  if (logical_cpus <= 0) {
2459  glb_call_time = -1;
2460  return -1;
2461  }
2462 
2463  perfstat_cpu_t *cpu_stat = (perfstat_cpu_t *)KMP_INTERNAL_MALLOC(
2464  logical_cpus * sizeof(perfstat_cpu_t));
2465  if (cpu_stat == NULL) {
2466  glb_call_time = -1;
2467  return -1;
2468  }
2469 
2470  // Set first CPU as the name of the first logical CPU for which the info is
2471  // desired.
2472  perfstat_id_t first_cpu_name;
2473  strcpy(first_cpu_name.name, FIRST_CPU);
2474 
2475  // Get the stat info of logical CPUs.
2476  int rc = perfstat_cpu(&first_cpu_name, cpu_stat, sizeof(perfstat_cpu_t),
2477  logical_cpus);
2478  KMP_DEBUG_ASSERT(rc == logical_cpus);
2479  if (rc <= 0) {
2480  KMP_INTERNAL_FREE(cpu_stat);
2481  glb_call_time = -1;
2482  return -1;
2483  }
2484  for (int i = 0; i < logical_cpus; ++i) {
2485  running_threads += cpu_stat[i].runque;
2486  if (running_threads >= max)
2487  break;
2488  }
2489 
2490  // There _might_ be a timing hole where the thread executing this
2491  // code gets skipped in the load balance, and running_threads is 0.
2492  // Assert in the debug builds only!!!
2493  KMP_DEBUG_ASSERT(running_threads > 0);
2494  if (running_threads <= 0)
2495  running_threads = 1;
2496 
2497  KMP_INTERNAL_FREE(cpu_stat);
2498 
2499  glb_running_threads = running_threads;
2500 
2501  return running_threads;
2502 }
2503 
2504 #else // Linux* OS
2505 
2506 // The function returns number of running (not sleeping) threads, or -1 in case
2507 // of error. Error could be reported if Linux* OS kernel too old (without
2508 // "/proc" support). Counting running threads stops if max running threads
2509 // encountered.
2510 int __kmp_get_load_balance(int max) {
2511  static int permanent_error = 0;
2512  static int glb_running_threads = 0; // Saved count of the running threads for
2513  // the thread balance algorithm
2514  static double glb_call_time = 0; /* Thread balance algorithm call time */
2515 
2516  int running_threads = 0; // Number of running threads in the system.
2517 
2518  DIR *proc_dir = NULL; // Handle of "/proc/" directory.
2519  struct dirent *proc_entry = NULL;
2520 
2521  kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2522  DIR *task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2523  struct dirent *task_entry = NULL;
2524  int task_path_fixed_len;
2525 
2526  kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2527  int stat_file = -1;
2528  int stat_path_fixed_len;
2529 
2530 #ifdef KMP_DEBUG
2531  int total_processes = 0; // Total number of processes in system.
2532 #endif
2533 
2534  double call_time = 0.0;
2535 
2536  __kmp_str_buf_init(&task_path);
2537  __kmp_str_buf_init(&stat_path);
2538 
2539  __kmp_elapsed(&call_time);
2540 
2541  if (glb_call_time &&
2542  (call_time - glb_call_time < __kmp_load_balance_interval)) {
2543  running_threads = glb_running_threads;
2544  goto finish;
2545  }
2546 
2547  glb_call_time = call_time;
2548 
2549  // Do not spend time on scanning "/proc/" if we have a permanent error.
2550  if (permanent_error) {
2551  running_threads = -1;
2552  goto finish;
2553  }
2554 
2555  if (max <= 0) {
2556  max = INT_MAX;
2557  }
2558 
2559  // Open "/proc/" directory.
2560  proc_dir = opendir("/proc");
2561  if (proc_dir == NULL) {
2562  // Cannot open "/proc/". Probably the kernel does not support it. Return an
2563  // error now and in subsequent calls.
2564  running_threads = -1;
2565  permanent_error = 1;
2566  goto finish;
2567  }
2568 
2569  // Initialize fixed part of task_path. This part will not change.
2570  __kmp_str_buf_cat(&task_path, "/proc/", 6);
2571  task_path_fixed_len = task_path.used; // Remember number of used characters.
2572 
2573  proc_entry = readdir(proc_dir);
2574  while (proc_entry != NULL) {
2575  // Proc entry is a directory and name starts with a digit. Assume it is a
2576  // process' directory.
2577  if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) {
2578 
2579 #ifdef KMP_DEBUG
2580  ++total_processes;
2581 #endif
2582  // Make sure init process is the very first in "/proc", so we can replace
2583  // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes ==
2584  // 1. We are going to check that total_processes == 1 => d_name == "1" is
2585  // true (where "=>" is implication). Since C++ does not have => operator,
2586  // let us replace it with its equivalent: a => b == ! a || b.
2587  KMP_DEBUG_ASSERT(total_processes != 1 ||
2588  strcmp(proc_entry->d_name, "1") == 0);
2589 
2590  // Construct task_path.
2591  task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2592  __kmp_str_buf_cat(&task_path, proc_entry->d_name,
2593  KMP_STRLEN(proc_entry->d_name));
2594  __kmp_str_buf_cat(&task_path, "/task", 5);
2595 
2596  task_dir = opendir(task_path.str);
2597  if (task_dir == NULL) {
2598  // Process can finish between reading "/proc/" directory entry and
2599  // opening process' "task/" directory. So, in general case we should not
2600  // complain, but have to skip this process and read the next one. But on
2601  // systems with no "task/" support we will spend lot of time to scan
2602  // "/proc/" tree again and again without any benefit. "init" process
2603  // (its pid is 1) should exist always, so, if we cannot open
2604  // "/proc/1/task/" directory, it means "task/" is not supported by
2605  // kernel. Report an error now and in the future.
2606  if (strcmp(proc_entry->d_name, "1") == 0) {
2607  running_threads = -1;
2608  permanent_error = 1;
2609  goto finish;
2610  }
2611  } else {
2612  // Construct fixed part of stat file path.
2613  __kmp_str_buf_clear(&stat_path);
2614  __kmp_str_buf_cat(&stat_path, task_path.str, task_path.used);
2615  __kmp_str_buf_cat(&stat_path, "/", 1);
2616  stat_path_fixed_len = stat_path.used;
2617 
2618  task_entry = readdir(task_dir);
2619  while (task_entry != NULL) {
2620  // It is a directory and name starts with a digit.
2621  if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) {
2622 
2623  // Construct complete stat file path. Easiest way would be:
2624  // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str,
2625  // task_entry->d_name );
2626  // but seriae of __kmp_str_buf_cat works a bit faster.
2627  stat_path.used =
2628  stat_path_fixed_len; // Reset stat path to its fixed part.
2629  __kmp_str_buf_cat(&stat_path, task_entry->d_name,
2630  KMP_STRLEN(task_entry->d_name));
2631  __kmp_str_buf_cat(&stat_path, "/stat", 5);
2632 
2633  // Note: Low-level API (open/read/close) is used. High-level API
2634  // (fopen/fclose) works ~ 30 % slower.
2635  stat_file = open(stat_path.str, O_RDONLY);
2636  if (stat_file == -1) {
2637  // We cannot report an error because task (thread) can terminate
2638  // just before reading this file.
2639  } else {
2640  /* Content of "stat" file looks like:
2641  24285 (program) S ...
2642 
2643  It is a single line (if program name does not include funny
2644  symbols). First number is a thread id, then name of executable
2645  file name in paretheses, then state of the thread. We need just
2646  thread state.
2647 
2648  Good news: Length of program name is 15 characters max. Longer
2649  names are truncated.
2650 
2651  Thus, we need rather short buffer: 15 chars for program name +
2652  2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2653 
2654  Bad news: Program name may contain special symbols like space,
2655  closing parenthesis, or even new line. This makes parsing
2656  "stat" file not 100 % reliable. In case of fanny program names
2657  parsing may fail (report incorrect thread state).
2658 
2659  Parsing "status" file looks more promissing (due to different
2660  file structure and escaping special symbols) but reading and
2661  parsing of "status" file works slower.
2662  -- ln
2663  */
2664  char buffer[65];
2665  ssize_t len;
2666  len = read(stat_file, buffer, sizeof(buffer) - 1);
2667  if (len >= 0) {
2668  buffer[len] = 0;
2669  // Using scanf:
2670  // sscanf( buffer, "%*d (%*s) %c ", & state );
2671  // looks very nice, but searching for a closing parenthesis
2672  // works a bit faster.
2673  char *close_parent = strstr(buffer, ") ");
2674  if (close_parent != NULL) {
2675  char state = *(close_parent + 2);
2676  if (state == 'R') {
2677  ++running_threads;
2678  if (running_threads >= max) {
2679  goto finish;
2680  }
2681  }
2682  }
2683  }
2684  close(stat_file);
2685  stat_file = -1;
2686  }
2687  }
2688  task_entry = readdir(task_dir);
2689  }
2690  closedir(task_dir);
2691  task_dir = NULL;
2692  }
2693  }
2694  proc_entry = readdir(proc_dir);
2695  }
2696 
2697  // There _might_ be a timing hole where the thread executing this
2698  // code get skipped in the load balance, and running_threads is 0.
2699  // Assert in the debug builds only!!!
2700  KMP_DEBUG_ASSERT(running_threads > 0);
2701  if (running_threads <= 0) {
2702  running_threads = 1;
2703  }
2704 
2705 finish: // Clean up and exit.
2706  if (proc_dir != NULL) {
2707  closedir(proc_dir);
2708  }
2709  __kmp_str_buf_free(&task_path);
2710  if (task_dir != NULL) {
2711  closedir(task_dir);
2712  }
2713  __kmp_str_buf_free(&stat_path);
2714  if (stat_file != -1) {
2715  close(stat_file);
2716  }
2717 
2718  glb_running_threads = running_threads;
2719 
2720  return running_threads;
2721 
2722 } // __kmp_get_load_balance
2723 
2724 #endif // KMP_OS_DARWIN
2725 
2726 #endif // USE_LOAD_BALANCE
2727 
2728 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \
2729  ((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || \
2730  KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || \
2731  KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF || \
2732  KMP_ARCH_AARCH64_32)
2733 
2734 // Because WebAssembly will use `call_indirect` to invoke the microtask and
2735 // WebAssembly indirect calls check that the called signature is a precise
2736 // match, we need to cast each microtask function pointer back from `void *` to
2737 // its original type.
2738 typedef void (*microtask_t0)(int *, int *);
2739 typedef void (*microtask_t1)(int *, int *, void *);
2740 typedef void (*microtask_t2)(int *, int *, void *, void *);
2741 typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
2742 typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
2743 typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
2744  void *);
2745 typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
2746  void *, void *);
2747 typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
2748  void *, void *, void *);
2749 typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
2750  void *, void *, void *, void *);
2751 typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
2752  void *, void *, void *, void *, void *);
2753 typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
2754  void *, void *, void *, void *, void *, void *);
2755 typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
2756  void *, void *, void *, void *, void *, void *,
2757  void *);
2758 typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
2759  void *, void *, void *, void *, void *, void *,
2760  void *, void *);
2761 typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
2762  void *, void *, void *, void *, void *, void *,
2763  void *, void *, void *);
2764 typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
2765  void *, void *, void *, void *, void *, void *,
2766  void *, void *, void *, void *);
2767 typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
2768  void *, void *, void *, void *, void *, void *,
2769  void *, void *, void *, void *, void *);
2770 
2771 // we really only need the case with 1 argument, because CLANG always build
2772 // a struct of pointers to shared variables referenced in the outlined function
2773 int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
2774  void *p_argv[]
2775 #if OMPT_SUPPORT
2776  ,
2777  void **exit_frame_ptr
2778 #endif
2779 ) {
2780 #if OMPT_SUPPORT
2781  *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2782 #endif
2783 
2784  switch (argc) {
2785  default:
2786  fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2787  fflush(stderr);
2788  exit(-1);
2789  case 0:
2790  (*(microtask_t0)pkfn)(&gtid, &tid);
2791  break;
2792  case 1:
2793  (*(microtask_t1)pkfn)(&gtid, &tid, p_argv[0]);
2794  break;
2795  case 2:
2796  (*(microtask_t2)pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2797  break;
2798  case 3:
2799  (*(microtask_t3)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2800  break;
2801  case 4:
2802  (*(microtask_t4)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2803  p_argv[3]);
2804  break;
2805  case 5:
2806  (*(microtask_t5)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2807  p_argv[3], p_argv[4]);
2808  break;
2809  case 6:
2810  (*(microtask_t6)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2811  p_argv[3], p_argv[4], p_argv[5]);
2812  break;
2813  case 7:
2814  (*(microtask_t7)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2815  p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
2816  break;
2817  case 8:
2818  (*(microtask_t8)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2819  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2820  p_argv[7]);
2821  break;
2822  case 9:
2823  (*(microtask_t9)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2824  p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
2825  p_argv[8]);
2826  break;
2827  case 10:
2828  (*(microtask_t10)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2829  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2830  p_argv[7], p_argv[8], p_argv[9]);
2831  break;
2832  case 11:
2833  (*(microtask_t11)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2834  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2835  p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2836  break;
2837  case 12:
2838  (*(microtask_t12)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2839  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2840  p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2841  p_argv[11]);
2842  break;
2843  case 13:
2844  (*(microtask_t13)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2845  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2846  p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2847  p_argv[11], p_argv[12]);
2848  break;
2849  case 14:
2850  (*(microtask_t14)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2851  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2852  p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2853  p_argv[11], p_argv[12], p_argv[13]);
2854  break;
2855  case 15:
2856  (*(microtask_t15)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2857  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2858  p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2859  p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2860  break;
2861  }
2862 
2863  return 1;
2864 }
2865 
2866 #endif
2867 
2868 #if KMP_OS_LINUX
2869 // Functions for hidden helper task
2870 namespace {
2871 // Condition variable for initializing hidden helper team
2872 pthread_cond_t hidden_helper_threads_initz_cond_var;
2873 pthread_mutex_t hidden_helper_threads_initz_lock;
2874 volatile int hidden_helper_initz_signaled = FALSE;
2875 
2876 // Condition variable for deinitializing hidden helper team
2877 pthread_cond_t hidden_helper_threads_deinitz_cond_var;
2878 pthread_mutex_t hidden_helper_threads_deinitz_lock;
2879 volatile int hidden_helper_deinitz_signaled = FALSE;
2880 
2881 // Condition variable for the wrapper function of main thread
2882 pthread_cond_t hidden_helper_main_thread_cond_var;
2883 pthread_mutex_t hidden_helper_main_thread_lock;
2884 volatile int hidden_helper_main_thread_signaled = FALSE;
2885 
2886 // Semaphore for worker threads. We don't use condition variable here in case
2887 // that when multiple signals are sent at the same time, only one thread might
2888 // be waken.
2889 sem_t hidden_helper_task_sem;
2890 } // namespace
2891 
2892 void __kmp_hidden_helper_worker_thread_wait() {
2893  int status = sem_wait(&hidden_helper_task_sem);
2894  KMP_CHECK_SYSFAIL("sem_wait", status);
2895 }
2896 
2897 void __kmp_do_initialize_hidden_helper_threads() {
2898  // Initialize condition variable
2899  int status =
2900  pthread_cond_init(&hidden_helper_threads_initz_cond_var, nullptr);
2901  KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2902 
2903  status = pthread_cond_init(&hidden_helper_threads_deinitz_cond_var, nullptr);
2904  KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2905 
2906  status = pthread_cond_init(&hidden_helper_main_thread_cond_var, nullptr);
2907  KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2908 
2909  status = pthread_mutex_init(&hidden_helper_threads_initz_lock, nullptr);
2910  KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2911 
2912  status = pthread_mutex_init(&hidden_helper_threads_deinitz_lock, nullptr);
2913  KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2914 
2915  status = pthread_mutex_init(&hidden_helper_main_thread_lock, nullptr);
2916  KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2917 
2918  // Initialize the semaphore
2919  status = sem_init(&hidden_helper_task_sem, 0, 0);
2920  KMP_CHECK_SYSFAIL("sem_init", status);
2921 
2922  // Create a new thread to finish initialization
2923  pthread_t handle;
2924  status = pthread_create(
2925  &handle, nullptr,
2926  [](void *) -> void * {
2927  __kmp_hidden_helper_threads_initz_routine();
2928  return nullptr;
2929  },
2930  nullptr);
2931  KMP_CHECK_SYSFAIL("pthread_create", status);
2932 }
2933 
2934 void __kmp_hidden_helper_threads_initz_wait() {
2935  // Initial thread waits here for the completion of the initialization. The
2936  // condition variable will be notified by main thread of hidden helper teams.
2937  int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2938  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2939 
2940  if (!TCR_4(hidden_helper_initz_signaled)) {
2941  status = pthread_cond_wait(&hidden_helper_threads_initz_cond_var,
2942  &hidden_helper_threads_initz_lock);
2943  KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2944  }
2945 
2946  status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2947  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2948 }
2949 
2950 void __kmp_hidden_helper_initz_release() {
2951  // After all initialization, reset __kmp_init_hidden_helper_threads to false.
2952  int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2953  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2954 
2955  status = pthread_cond_signal(&hidden_helper_threads_initz_cond_var);
2956  KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2957 
2958  TCW_SYNC_4(hidden_helper_initz_signaled, TRUE);
2959 
2960  status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2961  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2962 }
2963 
2964 void __kmp_hidden_helper_main_thread_wait() {
2965  // The main thread of hidden helper team will be blocked here. The
2966  // condition variable can only be signal in the destructor of RTL.
2967  int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
2968  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2969 
2970  if (!TCR_4(hidden_helper_main_thread_signaled)) {
2971  status = pthread_cond_wait(&hidden_helper_main_thread_cond_var,
2972  &hidden_helper_main_thread_lock);
2973  KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2974  }
2975 
2976  status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
2977  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2978 }
2979 
2980 void __kmp_hidden_helper_main_thread_release() {
2981  // The initial thread of OpenMP RTL should call this function to wake up the
2982  // main thread of hidden helper team.
2983  int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
2984  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2985 
2986  status = pthread_cond_signal(&hidden_helper_main_thread_cond_var);
2987  KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
2988 
2989  // The hidden helper team is done here
2990  TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE);
2991 
2992  status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
2993  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2994 }
2995 
2996 void __kmp_hidden_helper_worker_thread_signal() {
2997  int status = sem_post(&hidden_helper_task_sem);
2998  KMP_CHECK_SYSFAIL("sem_post", status);
2999 }
3000 
3001 void __kmp_hidden_helper_threads_deinitz_wait() {
3002  // Initial thread waits here for the completion of the deinitialization. The
3003  // condition variable will be notified by main thread of hidden helper teams.
3004  int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
3005  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3006 
3007  if (!TCR_4(hidden_helper_deinitz_signaled)) {
3008  status = pthread_cond_wait(&hidden_helper_threads_deinitz_cond_var,
3009  &hidden_helper_threads_deinitz_lock);
3010  KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
3011  }
3012 
3013  status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
3014  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3015 }
3016 
3017 void __kmp_hidden_helper_threads_deinitz_release() {
3018  int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
3019  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3020 
3021  status = pthread_cond_signal(&hidden_helper_threads_deinitz_cond_var);
3022  KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
3023 
3024  TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE);
3025 
3026  status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
3027  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3028 }
3029 #else // KMP_OS_LINUX
3030 void __kmp_hidden_helper_worker_thread_wait() {
3031  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3032 }
3033 
3034 void __kmp_do_initialize_hidden_helper_threads() {
3035  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3036 }
3037 
3038 void __kmp_hidden_helper_threads_initz_wait() {
3039  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3040 }
3041 
3042 void __kmp_hidden_helper_initz_release() {
3043  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3044 }
3045 
3046 void __kmp_hidden_helper_main_thread_wait() {
3047  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3048 }
3049 
3050 void __kmp_hidden_helper_main_thread_release() {
3051  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3052 }
3053 
3054 void __kmp_hidden_helper_worker_thread_signal() {
3055  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3056 }
3057 
3058 void __kmp_hidden_helper_threads_deinitz_wait() {
3059  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3060 }
3061 
3062 void __kmp_hidden_helper_threads_deinitz_release() {
3063  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3064 }
3065 #endif // KMP_OS_LINUX
3066 
3067 bool __kmp_detect_shm() {
3068  DIR *dir = opendir("/dev/shm");
3069  if (dir) { // /dev/shm exists
3070  closedir(dir);
3071  return true;
3072  } else if (ENOENT == errno) { // /dev/shm does not exist
3073  return false;
3074  } else { // opendir() failed
3075  return false;
3076  }
3077 }
3078 
3079 bool __kmp_detect_tmp() {
3080  DIR *dir = opendir("/tmp");
3081  if (dir) { // /tmp exists
3082  closedir(dir);
3083  return true;
3084  } else if (ENOENT == errno) { // /tmp does not exist
3085  return false;
3086  } else { // opendir() failed
3087  return false;
3088  }
3089 }
3090 
3091 // end of file //
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the partitioned timers to begin with name.
Definition: kmp_stats.h:940