LLVM OpenMP* Runtime Library
z_Linux_util.c
1 /*
2  * z_Linux_util.c -- platform specific routines.
3  */
4 
5 
6 //===----------------------------------------------------------------------===//
7 //
8 // The LLVM Compiler Infrastructure
9 //
10 // This file is dual licensed under the MIT and the University of Illinois Open
11 // Source Licenses. See LICENSE.txt for details.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 
16 #include "kmp.h"
17 #include "kmp_wrapper_getpid.h"
18 #include "kmp_itt.h"
19 #include "kmp_str.h"
20 #include "kmp_i18n.h"
21 #include "kmp_io.h"
22 #include "kmp_stats.h"
23 #include "kmp_wait_release.h"
24 
25 #if !KMP_OS_FREEBSD
26 # include <alloca.h>
27 #endif
28 #include <unistd.h>
29 #include <math.h> // HUGE_VAL.
30 #include <sys/time.h>
31 #include <sys/times.h>
32 #include <sys/resource.h>
33 #include <sys/syscall.h>
34 
35 #if KMP_OS_LINUX && !KMP_OS_CNK
36 # include <sys/sysinfo.h>
37 # if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
38 // We should really include <futex.h>, but that causes compatibility problems on different
39 // Linux* OS distributions that either require that you include (or break when you try to include)
40 // <pci/types.h>.
41 // Since all we need is the two macros below (which are part of the kernel ABI, so can't change)
42 // we just define the constants here and don't include <futex.h>
43 # ifndef FUTEX_WAIT
44 # define FUTEX_WAIT 0
45 # endif
46 # ifndef FUTEX_WAKE
47 # define FUTEX_WAKE 1
48 # endif
49 # endif
50 #elif KMP_OS_DARWIN
51 # include <sys/sysctl.h>
52 # include <mach/mach.h>
53 #elif KMP_OS_FREEBSD
54 # include <sys/sysctl.h>
55 # include <pthread_np.h>
56 #endif
57 
58 
59 #include <dirent.h>
60 #include <ctype.h>
61 #include <fcntl.h>
62 
63 // For non-x86 architecture
64 #if KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64)
65 # include <stdbool.h>
66 # include <ffi.h>
67 #endif
68 
69 /* ------------------------------------------------------------------------ */
70 /* ------------------------------------------------------------------------ */
71 
72 struct kmp_sys_timer {
73  struct timespec start;
74 };
75 
76 // Convert timespec to nanoseconds.
77 #define TS2NS(timespec) (((timespec).tv_sec * 1e9) + (timespec).tv_nsec)
78 
79 static struct kmp_sys_timer __kmp_sys_timer_data;
80 
81 #if KMP_HANDLE_SIGNALS
82  typedef void (* sig_func_t )( int );
83  STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[ NSIG ];
84  static sigset_t __kmp_sigset;
85 #endif
86 
87 static int __kmp_init_runtime = FALSE;
88 
89 static int __kmp_fork_count = 0;
90 
91 static pthread_condattr_t __kmp_suspend_cond_attr;
92 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
93 
94 static kmp_cond_align_t __kmp_wait_cv;
95 static kmp_mutex_align_t __kmp_wait_mx;
96 
97 /* ------------------------------------------------------------------------ */
98 /* ------------------------------------------------------------------------ */
99 
100 #ifdef DEBUG_SUSPEND
101 static void
102 __kmp_print_cond( char *buffer, kmp_cond_align_t *cond )
103 {
104  KMP_SNPRINTF( buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
105  cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
106  cond->c_cond.__c_waiting );
107 }
108 #endif
109 
110 /* ------------------------------------------------------------------------ */
111 /* ------------------------------------------------------------------------ */
112 
113 #if ( KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED)
114 
115 /*
116  * Affinity support
117  */
118 
119 /*
120  * On some of the older OS's that we build on, these constants aren't present
121  * in <asm/unistd.h> #included from <sys.syscall.h>. They must be the same on
122  * all systems of the same arch where they are defined, and they cannot change.
123  * stone forever.
124  */
125 
126 # if KMP_ARCH_X86 || KMP_ARCH_ARM
127 # ifndef __NR_sched_setaffinity
128 # define __NR_sched_setaffinity 241
129 # elif __NR_sched_setaffinity != 241
130 # error Wrong code for setaffinity system call.
131 # endif /* __NR_sched_setaffinity */
132 # ifndef __NR_sched_getaffinity
133 # define __NR_sched_getaffinity 242
134 # elif __NR_sched_getaffinity != 242
135 # error Wrong code for getaffinity system call.
136 # endif /* __NR_sched_getaffinity */
137 
138 # elif KMP_ARCH_AARCH64
139 # ifndef __NR_sched_setaffinity
140 # define __NR_sched_setaffinity 122
141 # elif __NR_sched_setaffinity != 122
142 # error Wrong code for setaffinity system call.
143 # endif /* __NR_sched_setaffinity */
144 # ifndef __NR_sched_getaffinity
145 # define __NR_sched_getaffinity 123
146 # elif __NR_sched_getaffinity != 123
147 # error Wrong code for getaffinity system call.
148 # endif /* __NR_sched_getaffinity */
149 
150 # elif KMP_ARCH_X86_64
151 # ifndef __NR_sched_setaffinity
152 # define __NR_sched_setaffinity 203
153 # elif __NR_sched_setaffinity != 203
154 # error Wrong code for setaffinity system call.
155 # endif /* __NR_sched_setaffinity */
156 # ifndef __NR_sched_getaffinity
157 # define __NR_sched_getaffinity 204
158 # elif __NR_sched_getaffinity != 204
159 # error Wrong code for getaffinity system call.
160 # endif /* __NR_sched_getaffinity */
161 
162 # elif KMP_ARCH_PPC64
163 # ifndef __NR_sched_setaffinity
164 # define __NR_sched_setaffinity 222
165 # elif __NR_sched_setaffinity != 222
166 # error Wrong code for setaffinity system call.
167 # endif /* __NR_sched_setaffinity */
168 # ifndef __NR_sched_getaffinity
169 # define __NR_sched_getaffinity 223
170 # elif __NR_sched_getaffinity != 223
171 # error Wrong code for getaffinity system call.
172 # endif /* __NR_sched_getaffinity */
173 
174 # elif KMP_ARCH_MIPS
175 # ifndef __NR_sched_setaffinity
176 # define __NR_sched_setaffinity 4239
177 # elif __NR_sched_setaffinity != 4239
178 # error Wrong code for setaffinity system call.
179 # endif /* __NR_sched_setaffinity */
180 # ifndef __NR_sched_getaffinity
181 # define __NR_sched_getaffinity 4240
182 # elif __NR_sched_getaffinity != 4240
183 # error Wrong code for getaffinity system call.
184 # endif /* __NR_sched_getaffinity */
185 
186 # elif KMP_ARCH_MIPS64
187 # ifndef __NR_sched_setaffinity
188 # define __NR_sched_setaffinity 5195
189 # elif __NR_sched_setaffinity != 5195
190 # error Wrong code for setaffinity system call.
191 # endif /* __NR_sched_setaffinity */
192 # ifndef __NR_sched_getaffinity
193 # define __NR_sched_getaffinity 5196
194 # elif __NR_sched_getaffinity != 5196
195 # error Wrong code for getaffinity system call.
196 # endif /* __NR_sched_getaffinity */
197 
198 # else
199 # error Unknown or unsupported architecture
200 
201 # endif /* KMP_ARCH_* */
202 
203 int
204 __kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
205 {
206  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
207  "Illegal set affinity operation when not capable");
208 
209  int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
210  if (retval >= 0) {
211  return 0;
212  }
213  int error = errno;
214  if (abort_on_error) {
215  __kmp_msg(
216  kmp_ms_fatal,
217  KMP_MSG( FatalSysError ),
218  KMP_ERR( error ),
219  __kmp_msg_null
220  );
221  }
222  return error;
223 }
224 
225 int
226 __kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
227 {
228  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
229  "Illegal get affinity operation when not capable");
230 
231  int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
232  if (retval >= 0) {
233  return 0;
234  }
235  int error = errno;
236  if (abort_on_error) {
237  __kmp_msg(
238  kmp_ms_fatal,
239  KMP_MSG( FatalSysError ),
240  KMP_ERR( error ),
241  __kmp_msg_null
242  );
243  }
244  return error;
245 }
246 
247 void
248 __kmp_affinity_bind_thread( int which )
249 {
250  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
251  "Illegal set affinity operation when not capable");
252 
253  kmp_affin_mask_t *mask = (kmp_affin_mask_t *)KMP_ALLOCA(__kmp_affin_mask_size);
254  KMP_CPU_ZERO(mask);
255  KMP_CPU_SET(which, mask);
256  __kmp_set_system_affinity(mask, TRUE);
257 }
258 
259 /*
260  * Determine if we can access affinity functionality on this version of
261  * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
262  * __kmp_affin_mask_size to the appropriate value (0 means not capable).
263  */
264 void
265 __kmp_affinity_determine_capable(const char *env_var)
266 {
267  //
268  // Check and see if the OS supports thread affinity.
269  //
270 
271 # define KMP_CPU_SET_SIZE_LIMIT (1024*1024)
272 
273  int gCode;
274  int sCode;
275  kmp_affin_mask_t *buf;
276  buf = ( kmp_affin_mask_t * ) KMP_INTERNAL_MALLOC( KMP_CPU_SET_SIZE_LIMIT );
277 
278  // If Linux* OS:
279  // If the syscall fails or returns a suggestion for the size,
280  // then we don't have to search for an appropriate size.
281  gCode = syscall( __NR_sched_getaffinity, 0, KMP_CPU_SET_SIZE_LIMIT, buf );
282  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
283  "initial getaffinity call returned %d errno = %d\n",
284  gCode, errno));
285 
286  //if ((gCode < 0) && (errno == ENOSYS))
287  if (gCode < 0) {
288  //
289  // System call not supported
290  //
291  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
292  && (__kmp_affinity_type != affinity_none)
293  && (__kmp_affinity_type != affinity_default)
294  && (__kmp_affinity_type != affinity_disabled))) {
295  int error = errno;
296  __kmp_msg(
297  kmp_ms_warning,
298  KMP_MSG( GetAffSysCallNotSupported, env_var ),
299  KMP_ERR( error ),
300  __kmp_msg_null
301  );
302  }
303  KMP_AFFINITY_DISABLE();
304  KMP_INTERNAL_FREE(buf);
305  return;
306  }
307  if (gCode > 0) { // Linux* OS only
308  // The optimal situation: the OS returns the size of the buffer
309  // it expects.
310  //
311  // A verification of correct behavior is that Isetaffinity on a NULL
312  // buffer with the same size fails with errno set to EFAULT.
313  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
314  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
315  "setaffinity for mask size %d returned %d errno = %d\n",
316  gCode, sCode, errno));
317  if (sCode < 0) {
318  if (errno == ENOSYS) {
319  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
320  && (__kmp_affinity_type != affinity_none)
321  && (__kmp_affinity_type != affinity_default)
322  && (__kmp_affinity_type != affinity_disabled))) {
323  int error = errno;
324  __kmp_msg(
325  kmp_ms_warning,
326  KMP_MSG( SetAffSysCallNotSupported, env_var ),
327  KMP_ERR( error ),
328  __kmp_msg_null
329  );
330  }
331  KMP_AFFINITY_DISABLE();
332  KMP_INTERNAL_FREE(buf);
333  }
334  if (errno == EFAULT) {
335  KMP_AFFINITY_ENABLE(gCode);
336  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
337  "affinity supported (mask size %d)\n",
338  (int)__kmp_affin_mask_size));
339  KMP_INTERNAL_FREE(buf);
340  return;
341  }
342  }
343  }
344 
345  //
346  // Call the getaffinity system call repeatedly with increasing set sizes
347  // until we succeed, or reach an upper bound on the search.
348  //
349  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
350  "searching for proper set size\n"));
351  int size;
352  for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
353  gCode = syscall( __NR_sched_getaffinity, 0, size, buf );
354  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
355  "getaffinity for mask size %d returned %d errno = %d\n", size,
356  gCode, errno));
357 
358  if (gCode < 0) {
359  if ( errno == ENOSYS )
360  {
361  //
362  // We shouldn't get here
363  //
364  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
365  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
366  size));
367  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
368  && (__kmp_affinity_type != affinity_none)
369  && (__kmp_affinity_type != affinity_default)
370  && (__kmp_affinity_type != affinity_disabled))) {
371  int error = errno;
372  __kmp_msg(
373  kmp_ms_warning,
374  KMP_MSG( GetAffSysCallNotSupported, env_var ),
375  KMP_ERR( error ),
376  __kmp_msg_null
377  );
378  }
379  KMP_AFFINITY_DISABLE();
380  KMP_INTERNAL_FREE(buf);
381  return;
382  }
383  continue;
384  }
385 
386  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
387  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
388  "setaffinity for mask size %d returned %d errno = %d\n",
389  gCode, sCode, errno));
390  if (sCode < 0) {
391  if (errno == ENOSYS) { // Linux* OS only
392  //
393  // We shouldn't get here
394  //
395  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
396  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
397  size));
398  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
399  && (__kmp_affinity_type != affinity_none)
400  && (__kmp_affinity_type != affinity_default)
401  && (__kmp_affinity_type != affinity_disabled))) {
402  int error = errno;
403  __kmp_msg(
404  kmp_ms_warning,
405  KMP_MSG( SetAffSysCallNotSupported, env_var ),
406  KMP_ERR( error ),
407  __kmp_msg_null
408  );
409  }
410  KMP_AFFINITY_DISABLE();
411  KMP_INTERNAL_FREE(buf);
412  return;
413  }
414  if (errno == EFAULT) {
415  KMP_AFFINITY_ENABLE(gCode);
416  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
417  "affinity supported (mask size %d)\n",
418  (int)__kmp_affin_mask_size));
419  KMP_INTERNAL_FREE(buf);
420  return;
421  }
422  }
423  }
424  //int error = errno; // save uncaught error code
425  KMP_INTERNAL_FREE(buf);
426  // errno = error; // restore uncaught error code, will be printed at the next KMP_WARNING below
427 
428  //
429  // Affinity is not supported
430  //
431  KMP_AFFINITY_DISABLE();
432  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
433  "cannot determine mask size - affinity not supported\n"));
434  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
435  && (__kmp_affinity_type != affinity_none)
436  && (__kmp_affinity_type != affinity_default)
437  && (__kmp_affinity_type != affinity_disabled))) {
438  KMP_WARNING( AffCantGetMaskSize, env_var );
439  }
440 }
441 
442 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
443 
444 /* ------------------------------------------------------------------------ */
445 /* ------------------------------------------------------------------------ */
446 
447 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && !KMP_OS_CNK
448 
449 int
450 __kmp_futex_determine_capable()
451 {
452  int loc = 0;
453  int rc = syscall( __NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0 );
454  int retval = ( rc == 0 ) || ( errno != ENOSYS );
455 
456  KA_TRACE(10, ( "__kmp_futex_determine_capable: rc = %d errno = %d\n", rc,
457  errno ) );
458  KA_TRACE(10, ( "__kmp_futex_determine_capable: futex syscall%s supported\n",
459  retval ? "" : " not" ) );
460 
461  return retval;
462 }
463 
464 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) && !KMP_OS_CNK
465 
466 /* ------------------------------------------------------------------------ */
467 /* ------------------------------------------------------------------------ */
468 
469 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS)
470 /*
471  * Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
472  * use compare_and_store for these routines
473  */
474 
475 kmp_int8
476 __kmp_test_then_or8( volatile kmp_int8 *p, kmp_int8 d )
477 {
478  kmp_int8 old_value, new_value;
479 
480  old_value = TCR_1( *p );
481  new_value = old_value | d;
482 
483  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
484  {
485  KMP_CPU_PAUSE();
486  old_value = TCR_1( *p );
487  new_value = old_value | d;
488  }
489  return old_value;
490 }
491 
492 kmp_int8
493 __kmp_test_then_and8( volatile kmp_int8 *p, kmp_int8 d )
494 {
495  kmp_int8 old_value, new_value;
496 
497  old_value = TCR_1( *p );
498  new_value = old_value & d;
499 
500  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
501  {
502  KMP_CPU_PAUSE();
503  old_value = TCR_1( *p );
504  new_value = old_value & d;
505  }
506  return old_value;
507 }
508 
509 kmp_int32
510 __kmp_test_then_or32( volatile kmp_int32 *p, kmp_int32 d )
511 {
512  kmp_int32 old_value, new_value;
513 
514  old_value = TCR_4( *p );
515  new_value = old_value | d;
516 
517  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
518  {
519  KMP_CPU_PAUSE();
520  old_value = TCR_4( *p );
521  new_value = old_value | d;
522  }
523  return old_value;
524 }
525 
526 kmp_int32
527 __kmp_test_then_and32( volatile kmp_int32 *p, kmp_int32 d )
528 {
529  kmp_int32 old_value, new_value;
530 
531  old_value = TCR_4( *p );
532  new_value = old_value & d;
533 
534  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
535  {
536  KMP_CPU_PAUSE();
537  old_value = TCR_4( *p );
538  new_value = old_value & d;
539  }
540  return old_value;
541 }
542 
543 # if KMP_ARCH_X86 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
544 kmp_int8
545 __kmp_test_then_add8( volatile kmp_int8 *p, kmp_int8 d )
546 {
547  kmp_int8 old_value, new_value;
548 
549  old_value = TCR_1( *p );
550  new_value = old_value + d;
551 
552  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
553  {
554  KMP_CPU_PAUSE();
555  old_value = TCR_1( *p );
556  new_value = old_value + d;
557  }
558  return old_value;
559 }
560 
561 kmp_int64
562 __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 d )
563 {
564  kmp_int64 old_value, new_value;
565 
566  old_value = TCR_8( *p );
567  new_value = old_value + d;
568 
569  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
570  {
571  KMP_CPU_PAUSE();
572  old_value = TCR_8( *p );
573  new_value = old_value + d;
574  }
575  return old_value;
576 }
577 # endif /* KMP_ARCH_X86 */
578 
579 kmp_int64
580 __kmp_test_then_or64( volatile kmp_int64 *p, kmp_int64 d )
581 {
582  kmp_int64 old_value, new_value;
583 
584  old_value = TCR_8( *p );
585  new_value = old_value | d;
586  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
587  {
588  KMP_CPU_PAUSE();
589  old_value = TCR_8( *p );
590  new_value = old_value | d;
591  }
592  return old_value;
593 }
594 
595 kmp_int64
596 __kmp_test_then_and64( volatile kmp_int64 *p, kmp_int64 d )
597 {
598  kmp_int64 old_value, new_value;
599 
600  old_value = TCR_8( *p );
601  new_value = old_value & d;
602  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
603  {
604  KMP_CPU_PAUSE();
605  old_value = TCR_8( *p );
606  new_value = old_value & d;
607  }
608  return old_value;
609 }
610 
611 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
612 
613 void
614 __kmp_terminate_thread( int gtid )
615 {
616  int status;
617  kmp_info_t *th = __kmp_threads[ gtid ];
618 
619  if ( !th ) return;
620 
621  #ifdef KMP_CANCEL_THREADS
622  KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) );
623  status = pthread_cancel( th->th.th_info.ds.ds_thread );
624  if ( status != 0 && status != ESRCH ) {
625  __kmp_msg(
626  kmp_ms_fatal,
627  KMP_MSG( CantTerminateWorkerThread ),
628  KMP_ERR( status ),
629  __kmp_msg_null
630  );
631  }; // if
632  #endif
633  __kmp_yield( TRUE );
634 } //
635 
636 /* ------------------------------------------------------------------------ */
637 /* ------------------------------------------------------------------------ */
638 
639 /* ------------------------------------------------------------------------ */
640 /* ------------------------------------------------------------------------ */
641 
642 /*
643  * Set thread stack info according to values returned by
644  * pthread_getattr_np().
645  * If values are unreasonable, assume call failed and use
646  * incremental stack refinement method instead.
647  * Returns TRUE if the stack parameters could be determined exactly,
648  * FALSE if incremental refinement is necessary.
649  */
650 static kmp_int32
651 __kmp_set_stack_info( int gtid, kmp_info_t *th )
652 {
653  int stack_data;
654 #if KMP_OS_LINUX || KMP_OS_FREEBSD
655  /* Linux* OS only -- no pthread_getattr_np support on OS X* */
656  pthread_attr_t attr;
657  int status;
658  size_t size = 0;
659  void * addr = 0;
660 
661  /* Always do incremental stack refinement for ubermaster threads since the initial
662  thread stack range can be reduced by sibling thread creation so pthread_attr_getstack
663  may cause thread gtid aliasing */
664  if ( ! KMP_UBER_GTID(gtid) ) {
665 
666  /* Fetch the real thread attributes */
667  status = pthread_attr_init( &attr );
668  KMP_CHECK_SYSFAIL( "pthread_attr_init", status );
669 #if KMP_OS_FREEBSD
670  status = pthread_attr_get_np( pthread_self(), &attr );
671  KMP_CHECK_SYSFAIL( "pthread_attr_get_np", status );
672 #else
673  status = pthread_getattr_np( pthread_self(), &attr );
674  KMP_CHECK_SYSFAIL( "pthread_getattr_np", status );
675 #endif
676  status = pthread_attr_getstack( &attr, &addr, &size );
677  KMP_CHECK_SYSFAIL( "pthread_attr_getstack", status );
678  KA_TRACE( 60, ( "__kmp_set_stack_info: T#%d pthread_attr_getstack returned size: %lu, "
679  "low addr: %p\n",
680  gtid, size, addr ));
681 
682  status = pthread_attr_destroy( &attr );
683  KMP_CHECK_SYSFAIL( "pthread_attr_destroy", status );
684  }
685 
686  if ( size != 0 && addr != 0 ) { /* was stack parameter determination successful? */
687  /* Store the correct base and size */
688  TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
689  TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
690  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
691  return TRUE;
692  }
693 #endif /* KMP_OS_LINUX || KMP_OS_FREEBSD */
694  /* Use incremental refinement starting from initial conservative estimate */
695  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
696  TCW_PTR(th -> th.th_info.ds.ds_stackbase, &stack_data);
697  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
698  return FALSE;
699 }
700 
701 static void*
702 __kmp_launch_worker( void *thr )
703 {
704  int status, old_type, old_state;
705 #ifdef KMP_BLOCK_SIGNALS
706  sigset_t new_set, old_set;
707 #endif /* KMP_BLOCK_SIGNALS */
708  void *exit_val;
709 #if KMP_OS_LINUX || KMP_OS_FREEBSD
710  void *padding = 0;
711 #endif
712  int gtid;
713 
714  gtid = ((kmp_info_t*)thr) -> th.th_info.ds.ds_gtid;
715  __kmp_gtid_set_specific( gtid );
716 #ifdef KMP_TDATA_GTID
717  __kmp_gtid = gtid;
718 #endif
719 #if KMP_STATS_ENABLED
720  // set __thread local index to point to thread-specific stats
721  __kmp_stats_thread_ptr = ((kmp_info_t*)thr)->th.th_stats;
722 #endif
723 
724 #if USE_ITT_BUILD
725  __kmp_itt_thread_name( gtid );
726 #endif /* USE_ITT_BUILD */
727 
728 #if KMP_AFFINITY_SUPPORTED
729  __kmp_affinity_set_init_mask( gtid, FALSE );
730 #endif
731 
732 #ifdef KMP_CANCEL_THREADS
733  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
734  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
735  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
736  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
737  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
738 #endif
739 
740 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
741  //
742  // Set the FP control regs to be a copy of
743  // the parallel initialization thread's.
744  //
745  __kmp_clear_x87_fpu_status_word();
746  __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
747  __kmp_load_mxcsr( &__kmp_init_mxcsr );
748 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
749 
750 #ifdef KMP_BLOCK_SIGNALS
751  status = sigfillset( & new_set );
752  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
753  status = pthread_sigmask( SIG_BLOCK, & new_set, & old_set );
754  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
755 #endif /* KMP_BLOCK_SIGNALS */
756 
757 #if KMP_OS_LINUX || KMP_OS_FREEBSD
758  if ( __kmp_stkoffset > 0 && gtid > 0 ) {
759  padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
760  }
761 #endif
762 
763  KMP_MB();
764  __kmp_set_stack_info( gtid, (kmp_info_t*)thr );
765 
766  __kmp_check_stack_overlap( (kmp_info_t*)thr );
767 
768  exit_val = __kmp_launch_thread( (kmp_info_t *) thr );
769 
770 #ifdef KMP_BLOCK_SIGNALS
771  status = pthread_sigmask( SIG_SETMASK, & old_set, NULL );
772  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
773 #endif /* KMP_BLOCK_SIGNALS */
774 
775  return exit_val;
776 }
777 
778 
779 /* The monitor thread controls all of the threads in the complex */
780 
781 static void*
782 __kmp_launch_monitor( void *thr )
783 {
784  int status, old_type, old_state;
785 #ifdef KMP_BLOCK_SIGNALS
786  sigset_t new_set;
787 #endif /* KMP_BLOCK_SIGNALS */
788  struct timespec interval;
789  int yield_count;
790  int yield_cycles = 0;
791 
792  KMP_MB(); /* Flush all pending memory write invalidates. */
793 
794  KA_TRACE( 10, ("__kmp_launch_monitor: #1 launched\n" ) );
795 
796  /* register us as the monitor thread */
797  __kmp_gtid_set_specific( KMP_GTID_MONITOR );
798 #ifdef KMP_TDATA_GTID
799  __kmp_gtid = KMP_GTID_MONITOR;
800 #endif
801 
802  KMP_MB();
803 
804 #if USE_ITT_BUILD
805  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread.
806 #endif /* USE_ITT_BUILD */
807 
808  __kmp_set_stack_info( ((kmp_info_t*)thr)->th.th_info.ds.ds_gtid, (kmp_info_t*)thr );
809 
810  __kmp_check_stack_overlap( (kmp_info_t*)thr );
811 
812 #ifdef KMP_CANCEL_THREADS
813  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
814  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
815  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
816  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
817  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
818 #endif
819 
820  #if KMP_REAL_TIME_FIX
821  // This is a potential fix which allows application with real-time scheduling policy work.
822  // However, decision about the fix is not made yet, so it is disabled by default.
823  { // Are program started with real-time scheduling policy?
824  int sched = sched_getscheduler( 0 );
825  if ( sched == SCHED_FIFO || sched == SCHED_RR ) {
826  // Yes, we are a part of real-time application. Try to increase the priority of the
827  // monitor.
828  struct sched_param param;
829  int max_priority = sched_get_priority_max( sched );
830  int rc;
831  KMP_WARNING( RealTimeSchedNotSupported );
832  sched_getparam( 0, & param );
833  if ( param.sched_priority < max_priority ) {
834  param.sched_priority += 1;
835  rc = sched_setscheduler( 0, sched, & param );
836  if ( rc != 0 ) {
837  int error = errno;
838  __kmp_msg(
839  kmp_ms_warning,
840  KMP_MSG( CantChangeMonitorPriority ),
841  KMP_ERR( error ),
842  KMP_MSG( MonitorWillStarve ),
843  __kmp_msg_null
844  );
845  }; // if
846  } else {
847  // We cannot abort here, because number of CPUs may be enough for all the threads,
848  // including the monitor thread, so application could potentially work...
849  __kmp_msg(
850  kmp_ms_warning,
851  KMP_MSG( RunningAtMaxPriority ),
852  KMP_MSG( MonitorWillStarve ),
853  KMP_HNT( RunningAtMaxPriority ),
854  __kmp_msg_null
855  );
856  }; // if
857  }; // if
858  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 ); // AC: free thread that waits for monitor started
859  }
860  #endif // KMP_REAL_TIME_FIX
861 
862  KMP_MB(); /* Flush all pending memory write invalidates. */
863 
864  if ( __kmp_monitor_wakeups == 1 ) {
865  interval.tv_sec = 1;
866  interval.tv_nsec = 0;
867  } else {
868  interval.tv_sec = 0;
869  interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
870  }
871 
872  KA_TRACE( 10, ("__kmp_launch_monitor: #2 monitor\n" ) );
873 
874  if (__kmp_yield_cycle) {
875  __kmp_yielding_on = 0; /* Start out with yielding shut off */
876  yield_count = __kmp_yield_off_count;
877  } else {
878  __kmp_yielding_on = 1; /* Yielding is on permanently */
879  }
880 
881  while( ! TCR_4( __kmp_global.g.g_done ) ) {
882  struct timespec now;
883  struct timeval tval;
884 
885  /* This thread monitors the state of the system */
886 
887  KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) );
888 
889  status = gettimeofday( &tval, NULL );
890  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
891  TIMEVAL_TO_TIMESPEC( &tval, &now );
892 
893  now.tv_sec += interval.tv_sec;
894  now.tv_nsec += interval.tv_nsec;
895 
896  if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
897  now.tv_sec += 1;
898  now.tv_nsec -= KMP_NSEC_PER_SEC;
899  }
900 
901  status = pthread_mutex_lock( & __kmp_wait_mx.m_mutex );
902  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
903  // AC: the monitor should not fall asleep if g_done has been set
904  if ( !TCR_4(__kmp_global.g.g_done) ) { // check once more under mutex
905  status = pthread_cond_timedwait( &__kmp_wait_cv.c_cond, &__kmp_wait_mx.m_mutex, &now );
906  if ( status != 0 ) {
907  if ( status != ETIMEDOUT && status != EINTR ) {
908  KMP_SYSFAIL( "pthread_cond_timedwait", status );
909  };
910  };
911  };
912  status = pthread_mutex_unlock( & __kmp_wait_mx.m_mutex );
913  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
914 
915  if (__kmp_yield_cycle) {
916  yield_cycles++;
917  if ( (yield_cycles % yield_count) == 0 ) {
918  if (__kmp_yielding_on) {
919  __kmp_yielding_on = 0; /* Turn it off now */
920  yield_count = __kmp_yield_off_count;
921  } else {
922  __kmp_yielding_on = 1; /* Turn it on now */
923  yield_count = __kmp_yield_on_count;
924  }
925  yield_cycles = 0;
926  }
927  } else {
928  __kmp_yielding_on = 1;
929  }
930 
931  TCW_4( __kmp_global.g.g_time.dt.t_value,
932  TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
933 
934  KMP_MB(); /* Flush all pending memory write invalidates. */
935  }
936 
937  KA_TRACE( 10, ("__kmp_launch_monitor: #3 cleanup\n" ) );
938 
939 #ifdef KMP_BLOCK_SIGNALS
940  status = sigfillset( & new_set );
941  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
942  status = pthread_sigmask( SIG_UNBLOCK, & new_set, NULL );
943  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
944 #endif /* KMP_BLOCK_SIGNALS */
945 
946  KA_TRACE( 10, ("__kmp_launch_monitor: #4 finished\n" ) );
947 
948  if( __kmp_global.g.g_abort != 0 ) {
949  /* now we need to terminate the worker threads */
950  /* the value of t_abort is the signal we caught */
951 
952  int gtid;
953 
954  KA_TRACE( 10, ("__kmp_launch_monitor: #5 terminate sig=%d\n", __kmp_global.g.g_abort ) );
955 
956  /* terminate the OpenMP worker threads */
957  /* TODO this is not valid for sibling threads!!
958  * the uber master might not be 0 anymore.. */
959  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
960  __kmp_terminate_thread( gtid );
961 
962  __kmp_cleanup();
963 
964  KA_TRACE( 10, ("__kmp_launch_monitor: #6 raise sig=%d\n", __kmp_global.g.g_abort ) );
965 
966  if (__kmp_global.g.g_abort > 0)
967  raise( __kmp_global.g.g_abort );
968 
969  }
970 
971  KA_TRACE( 10, ("__kmp_launch_monitor: #7 exit\n" ) );
972 
973  return thr;
974 }
975 
976 void
977 __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
978 {
979  pthread_t handle;
980  pthread_attr_t thread_attr;
981  int status;
982 
983 
984  th->th.th_info.ds.ds_gtid = gtid;
985 
986 #if KMP_STATS_ENABLED
987  // sets up worker thread stats
988  __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
989 
990  // th->th.th_stats is used to transfer thread specific stats-pointer to __kmp_launch_worker
991  // So when thread is created (goes into __kmp_launch_worker) it will
992  // set it's __thread local pointer to th->th.th_stats
993  th->th.th_stats = __kmp_stats_list.push_back(gtid);
994  if(KMP_UBER_GTID(gtid)) {
995  __kmp_stats_start_time = tsc_tick_count::now();
996  __kmp_stats_thread_ptr = th->th.th_stats;
997  __kmp_stats_init();
998  KMP_START_EXPLICIT_TIMER(OMP_serial);
999  KMP_START_EXPLICIT_TIMER(OMP_start_end);
1000  }
1001  __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
1002 
1003 #endif // KMP_STATS_ENABLED
1004 
1005  if ( KMP_UBER_GTID(gtid) ) {
1006  KA_TRACE( 10, ("__kmp_create_worker: uber thread (%d)\n", gtid ) );
1007  th -> th.th_info.ds.ds_thread = pthread_self();
1008  __kmp_set_stack_info( gtid, th );
1009  __kmp_check_stack_overlap( th );
1010  return;
1011  }; // if
1012 
1013  KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1014 
1015  KMP_MB(); /* Flush all pending memory write invalidates. */
1016 
1017 #ifdef KMP_THREAD_ATTR
1018  {
1019  status = pthread_attr_init( &thread_attr );
1020  if ( status != 0 ) {
1021  __kmp_msg(
1022  kmp_ms_fatal,
1023  KMP_MSG( CantInitThreadAttrs ),
1024  KMP_ERR( status ),
1025  __kmp_msg_null
1026  );
1027  }; // if
1028  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1029  if ( status != 0 ) {
1030  __kmp_msg(
1031  kmp_ms_fatal,
1032  KMP_MSG( CantSetWorkerState ),
1033  KMP_ERR( status ),
1034  __kmp_msg_null
1035  );
1036  }; // if
1037 
1038  /* Set stack size for this thread now. */
1039  stack_size += gtid * __kmp_stkoffset;
1040 
1041  KA_TRACE( 10, ( "__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1042  "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
1043  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size ) );
1044 
1045 # ifdef _POSIX_THREAD_ATTR_STACKSIZE
1046  status = pthread_attr_setstacksize( & thread_attr, stack_size );
1047 # ifdef KMP_BACKUP_STKSIZE
1048  if ( status != 0 ) {
1049  if ( ! __kmp_env_stksize ) {
1050  stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
1051  __kmp_stksize = KMP_BACKUP_STKSIZE;
1052  KA_TRACE( 10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1053  "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
1054  "bytes\n",
1055  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size )
1056  );
1057  status = pthread_attr_setstacksize( &thread_attr, stack_size );
1058  }; // if
1059  }; // if
1060 # endif /* KMP_BACKUP_STKSIZE */
1061  if ( status != 0 ) {
1062  __kmp_msg(
1063  kmp_ms_fatal,
1064  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1065  KMP_ERR( status ),
1066  KMP_HNT( ChangeWorkerStackSize ),
1067  __kmp_msg_null
1068  );
1069  }; // if
1070 # endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1071  }
1072 #endif /* KMP_THREAD_ATTR */
1073 
1074  {
1075  status = pthread_create( & handle, & thread_attr, __kmp_launch_worker, (void *) th );
1076  if ( status != 0 || ! handle ) { // ??? Why do we check handle??
1077 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1078  if ( status == EINVAL ) {
1079  __kmp_msg(
1080  kmp_ms_fatal,
1081  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1082  KMP_ERR( status ),
1083  KMP_HNT( IncreaseWorkerStackSize ),
1084  __kmp_msg_null
1085  );
1086  };
1087  if ( status == ENOMEM ) {
1088  __kmp_msg(
1089  kmp_ms_fatal,
1090  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1091  KMP_ERR( status ),
1092  KMP_HNT( DecreaseWorkerStackSize ),
1093  __kmp_msg_null
1094  );
1095  };
1096 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1097  if ( status == EAGAIN ) {
1098  __kmp_msg(
1099  kmp_ms_fatal,
1100  KMP_MSG( NoResourcesForWorkerThread ),
1101  KMP_ERR( status ),
1102  KMP_HNT( Decrease_NUM_THREADS ),
1103  __kmp_msg_null
1104  );
1105  }; // if
1106  KMP_SYSFAIL( "pthread_create", status );
1107  }; // if
1108 
1109  th->th.th_info.ds.ds_thread = handle;
1110  }
1111 
1112 #ifdef KMP_THREAD_ATTR
1113  {
1114  status = pthread_attr_destroy( & thread_attr );
1115  if ( status ) {
1116  __kmp_msg(
1117  kmp_ms_warning,
1118  KMP_MSG( CantDestroyThreadAttrs ),
1119  KMP_ERR( status ),
1120  __kmp_msg_null
1121  );
1122  }; // if
1123  }
1124 #endif /* KMP_THREAD_ATTR */
1125 
1126  KMP_MB(); /* Flush all pending memory write invalidates. */
1127 
1128  KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1129 
1130 } // __kmp_create_worker
1131 
1132 
1133 void
1134 __kmp_create_monitor( kmp_info_t *th )
1135 {
1136  pthread_t handle;
1137  pthread_attr_t thread_attr;
1138  size_t size;
1139  int status;
1140  int caller_gtid;
1141  int auto_adj_size = FALSE;
1142 
1143  caller_gtid = __kmp_get_gtid();
1144 
1145  KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) );
1146 
1147  KMP_MB(); /* Flush all pending memory write invalidates. */
1148 
1149  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1150  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1151  #if KMP_REAL_TIME_FIX
1152  TCW_4( __kmp_global.g.g_time.dt.t_value, -1 ); // Will use it for synchronization a bit later.
1153  #else
1154  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1155  #endif // KMP_REAL_TIME_FIX
1156 
1157  #ifdef KMP_THREAD_ATTR
1158  if ( __kmp_monitor_stksize == 0 ) {
1159  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1160  auto_adj_size = TRUE;
1161  }
1162  status = pthread_attr_init( &thread_attr );
1163  if ( status != 0 ) {
1164  __kmp_msg(
1165  kmp_ms_fatal,
1166  KMP_MSG( CantInitThreadAttrs ),
1167  KMP_ERR( status ),
1168  __kmp_msg_null
1169  );
1170  }; // if
1171  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1172  if ( status != 0 ) {
1173  __kmp_msg(
1174  kmp_ms_fatal,
1175  KMP_MSG( CantSetMonitorState ),
1176  KMP_ERR( status ),
1177  __kmp_msg_null
1178  );
1179  }; // if
1180 
1181  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1182  status = pthread_attr_getstacksize( & thread_attr, & size );
1183  KMP_CHECK_SYSFAIL( "pthread_attr_getstacksize", status );
1184  #else
1185  size = __kmp_sys_min_stksize;
1186  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1187  #endif /* KMP_THREAD_ATTR */
1188 
1189  if ( __kmp_monitor_stksize == 0 ) {
1190  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1191  }
1192  if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1193  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1194  }
1195 
1196  KA_TRACE( 10, ( "__kmp_create_monitor: default stacksize = %lu bytes,"
1197  "requested stacksize = %lu bytes\n",
1198  size, __kmp_monitor_stksize ) );
1199 
1200  retry:
1201 
1202  /* Set stack size for this thread now. */
1203 
1204  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1205  KA_TRACE( 10, ( "__kmp_create_monitor: setting stacksize = %lu bytes,",
1206  __kmp_monitor_stksize ) );
1207  status = pthread_attr_setstacksize( & thread_attr, __kmp_monitor_stksize );
1208  if ( status != 0 ) {
1209  if ( auto_adj_size ) {
1210  __kmp_monitor_stksize *= 2;
1211  goto retry;
1212  }
1213  __kmp_msg(
1214  kmp_ms_warning, // should this be fatal? BB
1215  KMP_MSG( CantSetMonitorStackSize, (long int) __kmp_monitor_stksize ),
1216  KMP_ERR( status ),
1217  KMP_HNT( ChangeMonitorStackSize ),
1218  __kmp_msg_null
1219  );
1220  }; // if
1221  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1222 
1223  status = pthread_create( &handle, & thread_attr, __kmp_launch_monitor, (void *) th );
1224 
1225  if ( status != 0 ) {
1226  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1227  if ( status == EINVAL ) {
1228  if ( auto_adj_size && ( __kmp_monitor_stksize < (size_t)0x40000000 ) ) {
1229  __kmp_monitor_stksize *= 2;
1230  goto retry;
1231  }
1232  __kmp_msg(
1233  kmp_ms_fatal,
1234  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1235  KMP_ERR( status ),
1236  KMP_HNT( IncreaseMonitorStackSize ),
1237  __kmp_msg_null
1238  );
1239  }; // if
1240  if ( status == ENOMEM ) {
1241  __kmp_msg(
1242  kmp_ms_fatal,
1243  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1244  KMP_ERR( status ),
1245  KMP_HNT( DecreaseMonitorStackSize ),
1246  __kmp_msg_null
1247  );
1248  }; // if
1249  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1250  if ( status == EAGAIN ) {
1251  __kmp_msg(
1252  kmp_ms_fatal,
1253  KMP_MSG( NoResourcesForMonitorThread ),
1254  KMP_ERR( status ),
1255  KMP_HNT( DecreaseNumberOfThreadsInUse ),
1256  __kmp_msg_null
1257  );
1258  }; // if
1259  KMP_SYSFAIL( "pthread_create", status );
1260  }; // if
1261 
1262  th->th.th_info.ds.ds_thread = handle;
1263 
1264  #if KMP_REAL_TIME_FIX
1265  // Wait for the monitor thread is really started and set its *priority*.
1266  KMP_DEBUG_ASSERT( sizeof( kmp_uint32 ) == sizeof( __kmp_global.g.g_time.dt.t_value ) );
1267  __kmp_wait_yield_4(
1268  (kmp_uint32 volatile *) & __kmp_global.g.g_time.dt.t_value, -1, & __kmp_neq_4, NULL
1269  );
1270  #endif // KMP_REAL_TIME_FIX
1271 
1272  #ifdef KMP_THREAD_ATTR
1273  status = pthread_attr_destroy( & thread_attr );
1274  if ( status != 0 ) {
1275  __kmp_msg( //
1276  kmp_ms_warning,
1277  KMP_MSG( CantDestroyThreadAttrs ),
1278  KMP_ERR( status ),
1279  __kmp_msg_null
1280  );
1281  }; // if
1282  #endif
1283 
1284  KMP_MB(); /* Flush all pending memory write invalidates. */
1285 
1286  KA_TRACE( 10, ( "__kmp_create_monitor: monitor created %#.8lx\n", th->th.th_info.ds.ds_thread ) );
1287 
1288 } // __kmp_create_monitor
1289 
1290 void
1291 __kmp_exit_thread(
1292  int exit_status
1293 ) {
1294  pthread_exit( (void *)(intptr_t) exit_status );
1295 } // __kmp_exit_thread
1296 
1297 void __kmp_resume_monitor();
1298 
1299 void
1300 __kmp_reap_monitor( kmp_info_t *th )
1301 {
1302  int status;
1303  void *exit_val;
1304 
1305  KA_TRACE( 10, ("__kmp_reap_monitor: try to reap monitor thread with handle %#.8lx\n",
1306  th->th.th_info.ds.ds_thread ) );
1307 
1308  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1309  // If both tid and gtid are 0, it means the monitor did not ever start.
1310  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1311  KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1312  if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1313  return;
1314  }; // if
1315 
1316  KMP_MB(); /* Flush all pending memory write invalidates. */
1317 
1318 
1319  /* First, check to see whether the monitor thread exists. This could prevent a hang,
1320  but if the monitor dies after the pthread_kill call and before the pthread_join
1321  call, it will still hang. */
1322 
1323  status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1324  if (status == ESRCH) {
1325 
1326  KA_TRACE( 10, ("__kmp_reap_monitor: monitor does not exist, returning\n") );
1327 
1328  } else
1329  {
1330  __kmp_resume_monitor(); // Wake up the monitor thread
1331  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1332  if (exit_val != th) {
1333  __kmp_msg(
1334  kmp_ms_fatal,
1335  KMP_MSG( ReapMonitorError ),
1336  KMP_ERR( status ),
1337  __kmp_msg_null
1338  );
1339  }
1340  }
1341 
1342  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1343  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1344 
1345  KA_TRACE( 10, ("__kmp_reap_monitor: done reaping monitor thread with handle %#.8lx\n",
1346  th->th.th_info.ds.ds_thread ) );
1347 
1348  KMP_MB(); /* Flush all pending memory write invalidates. */
1349 
1350 }
1351 
1352 void
1353 __kmp_reap_worker( kmp_info_t *th )
1354 {
1355  int status;
1356  void *exit_val;
1357 
1358  KMP_MB(); /* Flush all pending memory write invalidates. */
1359 
1360  KA_TRACE( 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid ) );
1361 
1362  /* First, check to see whether the worker thread exists. This could prevent a hang,
1363  but if the worker dies after the pthread_kill call and before the pthread_join
1364  call, it will still hang. */
1365 
1366  {
1367  status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1368  if (status == ESRCH) {
1369  KA_TRACE( 10, ("__kmp_reap_worker: worker T#%d does not exist, returning\n",
1370  th->th.th_info.ds.ds_gtid ) );
1371  }
1372  else {
1373  KA_TRACE( 10, ("__kmp_reap_worker: try to join with worker T#%d\n",
1374  th->th.th_info.ds.ds_gtid ) );
1375 
1376  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1377 #ifdef KMP_DEBUG
1378  /* Don't expose these to the user until we understand when they trigger */
1379  if ( status != 0 ) {
1380  __kmp_msg(
1381  kmp_ms_fatal,
1382  KMP_MSG( ReapWorkerError ),
1383  KMP_ERR( status ),
1384  __kmp_msg_null
1385  );
1386  }
1387  if ( exit_val != th ) {
1388  KA_TRACE( 10, ( "__kmp_reap_worker: worker T#%d did not reap properly, "
1389  "exit_val = %p\n",
1390  th->th.th_info.ds.ds_gtid, exit_val ) );
1391  }
1392 #endif /* KMP_DEBUG */
1393  }
1394  }
1395 
1396  KA_TRACE( 10, ("__kmp_reap_worker: done reaping T#%d\n", th->th.th_info.ds.ds_gtid ) );
1397 
1398  KMP_MB(); /* Flush all pending memory write invalidates. */
1399 }
1400 
1401 
1402 /* ------------------------------------------------------------------------ */
1403 /* ------------------------------------------------------------------------ */
1404 
1405 #if KMP_HANDLE_SIGNALS
1406 
1407 
1408 static void
1409 __kmp_null_handler( int signo )
1410 {
1411  // Do nothing, for doing SIG_IGN-type actions.
1412 } // __kmp_null_handler
1413 
1414 
1415 static void
1416 __kmp_team_handler( int signo )
1417 {
1418  if ( __kmp_global.g.g_abort == 0 ) {
1419  /* Stage 1 signal handler, let's shut down all of the threads */
1420  #ifdef KMP_DEBUG
1421  __kmp_debug_printf( "__kmp_team_handler: caught signal = %d\n", signo );
1422  #endif
1423  switch ( signo ) {
1424  case SIGHUP :
1425  case SIGINT :
1426  case SIGQUIT :
1427  case SIGILL :
1428  case SIGABRT :
1429  case SIGFPE :
1430  case SIGBUS :
1431  case SIGSEGV :
1432  #ifdef SIGSYS
1433  case SIGSYS :
1434  #endif
1435  case SIGTERM :
1436  if ( __kmp_debug_buf ) {
1437  __kmp_dump_debug_buffer( );
1438  }; // if
1439  KMP_MB(); // Flush all pending memory write invalidates.
1440  TCW_4( __kmp_global.g.g_abort, signo );
1441  KMP_MB(); // Flush all pending memory write invalidates.
1442  TCW_4( __kmp_global.g.g_done, TRUE );
1443  KMP_MB(); // Flush all pending memory write invalidates.
1444  break;
1445  default:
1446  #ifdef KMP_DEBUG
1447  __kmp_debug_printf( "__kmp_team_handler: unknown signal type" );
1448  #endif
1449  break;
1450  }; // switch
1451  }; // if
1452 } // __kmp_team_handler
1453 
1454 
1455 static
1456 void __kmp_sigaction( int signum, const struct sigaction * act, struct sigaction * oldact ) {
1457  int rc = sigaction( signum, act, oldact );
1458  KMP_CHECK_SYSFAIL_ERRNO( "sigaction", rc );
1459 }
1460 
1461 
1462 static void
1463 __kmp_install_one_handler( int sig, sig_func_t handler_func, int parallel_init )
1464 {
1465  KMP_MB(); // Flush all pending memory write invalidates.
1466  KB_TRACE( 60, ( "__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init ) );
1467  if ( parallel_init ) {
1468  struct sigaction new_action;
1469  struct sigaction old_action;
1470  new_action.sa_handler = handler_func;
1471  new_action.sa_flags = 0;
1472  sigfillset( & new_action.sa_mask );
1473  __kmp_sigaction( sig, & new_action, & old_action );
1474  if ( old_action.sa_handler == __kmp_sighldrs[ sig ].sa_handler ) {
1475  sigaddset( & __kmp_sigset, sig );
1476  } else {
1477  // Restore/keep user's handler if one previously installed.
1478  __kmp_sigaction( sig, & old_action, NULL );
1479  }; // if
1480  } else {
1481  // Save initial/system signal handlers to see if user handlers installed.
1482  __kmp_sigaction( sig, NULL, & __kmp_sighldrs[ sig ] );
1483  }; // if
1484  KMP_MB(); // Flush all pending memory write invalidates.
1485 } // __kmp_install_one_handler
1486 
1487 
1488 static void
1489 __kmp_remove_one_handler( int sig )
1490 {
1491  KB_TRACE( 60, ( "__kmp_remove_one_handler( %d )\n", sig ) );
1492  if ( sigismember( & __kmp_sigset, sig ) ) {
1493  struct sigaction old;
1494  KMP_MB(); // Flush all pending memory write invalidates.
1495  __kmp_sigaction( sig, & __kmp_sighldrs[ sig ], & old );
1496  if ( ( old.sa_handler != __kmp_team_handler ) && ( old.sa_handler != __kmp_null_handler ) ) {
1497  // Restore the users signal handler.
1498  KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1499  __kmp_sigaction( sig, & old, NULL );
1500  }; // if
1501  sigdelset( & __kmp_sigset, sig );
1502  KMP_MB(); // Flush all pending memory write invalidates.
1503  }; // if
1504 } // __kmp_remove_one_handler
1505 
1506 
1507 void
1508 __kmp_install_signals( int parallel_init )
1509 {
1510  KB_TRACE( 10, ( "__kmp_install_signals( %d )\n", parallel_init ) );
1511  if ( __kmp_handle_signals || ! parallel_init ) {
1512  // If ! parallel_init, we do not install handlers, just save original handlers.
1513  // Let us do it even __handle_signals is 0.
1514  sigemptyset( & __kmp_sigset );
1515  __kmp_install_one_handler( SIGHUP, __kmp_team_handler, parallel_init );
1516  __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1517  __kmp_install_one_handler( SIGQUIT, __kmp_team_handler, parallel_init );
1518  __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1519  __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1520  __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1521  __kmp_install_one_handler( SIGBUS, __kmp_team_handler, parallel_init );
1522  __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1523  #ifdef SIGSYS
1524  __kmp_install_one_handler( SIGSYS, __kmp_team_handler, parallel_init );
1525  #endif // SIGSYS
1526  __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1527  #ifdef SIGPIPE
1528  __kmp_install_one_handler( SIGPIPE, __kmp_team_handler, parallel_init );
1529  #endif // SIGPIPE
1530  }; // if
1531 } // __kmp_install_signals
1532 
1533 
1534 void
1535 __kmp_remove_signals( void )
1536 {
1537  int sig;
1538  KB_TRACE( 10, ( "__kmp_remove_signals()\n" ) );
1539  for ( sig = 1; sig < NSIG; ++ sig ) {
1540  __kmp_remove_one_handler( sig );
1541  }; // for sig
1542 } // __kmp_remove_signals
1543 
1544 
1545 #endif // KMP_HANDLE_SIGNALS
1546 
1547 /* ------------------------------------------------------------------------ */
1548 /* ------------------------------------------------------------------------ */
1549 
1550 void
1551 __kmp_enable( int new_state )
1552 {
1553  #ifdef KMP_CANCEL_THREADS
1554  int status, old_state;
1555  status = pthread_setcancelstate( new_state, & old_state );
1556  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1557  KMP_DEBUG_ASSERT( old_state == PTHREAD_CANCEL_DISABLE );
1558  #endif
1559 }
1560 
1561 void
1562 __kmp_disable( int * old_state )
1563 {
1564  #ifdef KMP_CANCEL_THREADS
1565  int status;
1566  status = pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, old_state );
1567  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1568  #endif
1569 }
1570 
1571 /* ------------------------------------------------------------------------ */
1572 /* ------------------------------------------------------------------------ */
1573 
1574 static void
1575 __kmp_atfork_prepare (void)
1576 {
1577  /* nothing to do */
1578 }
1579 
1580 static void
1581 __kmp_atfork_parent (void)
1582 {
1583  /* nothing to do */
1584 }
1585 
1586 /*
1587  Reset the library so execution in the child starts "all over again" with
1588  clean data structures in initial states. Don't worry about freeing memory
1589  allocated by parent, just abandon it to be safe.
1590 */
1591 static void
1592 __kmp_atfork_child (void)
1593 {
1594  /* TODO make sure this is done right for nested/sibling */
1595  // ATT: Memory leaks are here? TODO: Check it and fix.
1596  /* KMP_ASSERT( 0 ); */
1597 
1598  ++__kmp_fork_count;
1599 
1600  __kmp_init_runtime = FALSE;
1601  __kmp_init_monitor = 0;
1602  __kmp_init_parallel = FALSE;
1603  __kmp_init_middle = FALSE;
1604  __kmp_init_serial = FALSE;
1605  TCW_4(__kmp_init_gtid, FALSE);
1606  __kmp_init_common = FALSE;
1607 
1608  TCW_4(__kmp_init_user_locks, FALSE);
1609 #if ! KMP_USE_DYNAMIC_LOCK
1610  __kmp_user_lock_table.used = 1;
1611  __kmp_user_lock_table.allocated = 0;
1612  __kmp_user_lock_table.table = NULL;
1613  __kmp_lock_blocks = NULL;
1614 #endif
1615 
1616  __kmp_all_nth = 0;
1617  TCW_4(__kmp_nth, 0);
1618 
1619  /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate here
1620  so threadprivate doesn't use stale data */
1621  KA_TRACE( 10, ( "__kmp_atfork_child: checking cache address list %p\n",
1622  __kmp_threadpriv_cache_list ) );
1623 
1624  while ( __kmp_threadpriv_cache_list != NULL ) {
1625 
1626  if ( *__kmp_threadpriv_cache_list -> addr != NULL ) {
1627  KC_TRACE( 50, ( "__kmp_atfork_child: zeroing cache at address %p\n",
1628  &(*__kmp_threadpriv_cache_list -> addr) ) );
1629 
1630  *__kmp_threadpriv_cache_list -> addr = NULL;
1631  }
1632  __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list -> next;
1633  }
1634 
1635  __kmp_init_runtime = FALSE;
1636 
1637  /* reset statically initialized locks */
1638  __kmp_init_bootstrap_lock( &__kmp_initz_lock );
1639  __kmp_init_bootstrap_lock( &__kmp_stdio_lock );
1640  __kmp_init_bootstrap_lock( &__kmp_console_lock );
1641 
1642  /* This is necessary to make sure no stale data is left around */
1643  /* AC: customers complain that we use unsafe routines in the atfork
1644  handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1645  in dynamic_link when check the presence of shared tbbmalloc library.
1646  Suggestion is to make the library initialization lazier, similar
1647  to what done for __kmpc_begin(). */
1648  // TODO: synchronize all static initializations with regular library
1649  // startup; look at kmp_global.c and etc.
1650  //__kmp_internal_begin ();
1651 
1652 }
1653 
1654 void
1655 __kmp_register_atfork(void) {
1656  if ( __kmp_need_register_atfork ) {
1657  int status = pthread_atfork( __kmp_atfork_prepare, __kmp_atfork_parent, __kmp_atfork_child );
1658  KMP_CHECK_SYSFAIL( "pthread_atfork", status );
1659  __kmp_need_register_atfork = FALSE;
1660  }
1661 }
1662 
1663 void
1664 __kmp_suspend_initialize( void )
1665 {
1666  int status;
1667  status = pthread_mutexattr_init( &__kmp_suspend_mutex_attr );
1668  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
1669  status = pthread_condattr_init( &__kmp_suspend_cond_attr );
1670  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
1671 }
1672 
1673 static void
1674 __kmp_suspend_initialize_thread( kmp_info_t *th )
1675 {
1676  if ( th->th.th_suspend_init_count <= __kmp_fork_count ) {
1677  /* this means we haven't initialized the suspension pthread objects for this thread
1678  in this instance of the process */
1679  int status;
1680  status = pthread_cond_init( &th->th.th_suspend_cv.c_cond, &__kmp_suspend_cond_attr );
1681  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
1682  status = pthread_mutex_init( &th->th.th_suspend_mx.m_mutex, & __kmp_suspend_mutex_attr );
1683  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
1684  *(volatile int*)&th->th.th_suspend_init_count = __kmp_fork_count + 1;
1685  };
1686 }
1687 
1688 void
1689 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
1690 {
1691  if(th->th.th_suspend_init_count > __kmp_fork_count) {
1692  /* this means we have initialize the suspension pthread objects for this thread
1693  in this instance of the process */
1694  int status;
1695 
1696  status = pthread_cond_destroy( &th->th.th_suspend_cv.c_cond );
1697  if ( status != 0 && status != EBUSY ) {
1698  KMP_SYSFAIL( "pthread_cond_destroy", status );
1699  };
1700  status = pthread_mutex_destroy( &th->th.th_suspend_mx.m_mutex );
1701  if ( status != 0 && status != EBUSY ) {
1702  KMP_SYSFAIL( "pthread_mutex_destroy", status );
1703  };
1704  --th->th.th_suspend_init_count;
1705  KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
1706  }
1707 }
1708 
1709 /* This routine puts the calling thread to sleep after setting the
1710  * sleep bit for the indicated flag variable to true.
1711  */
1712 template <class C>
1713 static inline void __kmp_suspend_template( int th_gtid, C *flag )
1714 {
1715  KMP_TIME_BLOCK(USER_suspend);
1716  kmp_info_t *th = __kmp_threads[th_gtid];
1717  int status;
1718  typename C::flag_t old_spin;
1719 
1720  KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, flag->get() ) );
1721 
1722  __kmp_suspend_initialize_thread( th );
1723 
1724  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1725  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1726 
1727  KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1728  th_gtid, flag->get() ) );
1729 
1730  /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread
1731  gets called first?
1732  */
1733  old_spin = flag->set_sleeping();
1734 
1735  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%d\n",
1736  th_gtid, flag->get(), *(flag->get()) ) );
1737 
1738  if ( flag->done_check_val(old_spin) ) {
1739  old_spin = flag->unset_sleeping();
1740  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for spin(%p)\n",
1741  th_gtid, flag->get()) );
1742  } else {
1743  /* Encapsulate in a loop as the documentation states that this may
1744  * "with low probability" return when the condition variable has
1745  * not been signaled or broadcast
1746  */
1747  int deactivated = FALSE;
1748  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1749  while ( flag->is_sleeping() ) {
1750 #ifdef DEBUG_SUSPEND
1751  char buffer[128];
1752  __kmp_suspend_count++;
1753  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1754  __kmp_printf( "__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, buffer );
1755 #endif
1756  // Mark the thread as no longer active (only in the first iteration of the loop).
1757  if ( ! deactivated ) {
1758  th->th.th_active = FALSE;
1759  if ( th->th.th_active_in_pool ) {
1760  th->th.th_active_in_pool = FALSE;
1761  KMP_TEST_THEN_DEC32(
1762  (kmp_int32 *) &__kmp_thread_pool_active_nth );
1763  KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
1764  }
1765  deactivated = TRUE;
1766 
1767 
1768  }
1769 
1770 #if USE_SUSPEND_TIMEOUT
1771  struct timespec now;
1772  struct timeval tval;
1773  int msecs;
1774 
1775  status = gettimeofday( &tval, NULL );
1776  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
1777  TIMEVAL_TO_TIMESPEC( &tval, &now );
1778 
1779  msecs = (4*__kmp_dflt_blocktime) + 200;
1780  now.tv_sec += msecs / 1000;
1781  now.tv_nsec += (msecs % 1000)*1000;
1782 
1783  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_timedwait\n",
1784  th_gtid ) );
1785  status = pthread_cond_timedwait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex, & now );
1786 #else
1787  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_wait\n",
1788  th_gtid ) );
1789 
1790  status = pthread_cond_wait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex );
1791 #endif
1792 
1793  if ( (status != 0) && (status != EINTR) && (status != ETIMEDOUT) ) {
1794  KMP_SYSFAIL( "pthread_cond_wait", status );
1795  }
1796 #ifdef KMP_DEBUG
1797  if (status == ETIMEDOUT) {
1798  if ( flag->is_sleeping() ) {
1799  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid ) );
1800  } else {
1801  KF_TRACE( 2, ( "__kmp_suspend_template: T#%d timeout wakeup, sleep bit not set!\n",
1802  th_gtid ) );
1803  }
1804  } else if ( flag->is_sleeping() ) {
1805  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ) );
1806  }
1807 #endif
1808  } // while
1809 
1810  // Mark the thread as active again (if it was previous marked as inactive)
1811  if ( deactivated ) {
1812  th->th.th_active = TRUE;
1813  if ( TCR_4(th->th.th_in_pool) ) {
1814  KMP_TEST_THEN_INC32( (kmp_int32 *) &__kmp_thread_pool_active_nth );
1815  th->th.th_active_in_pool = TRUE;
1816  }
1817  }
1818  }
1819 
1820 #ifdef DEBUG_SUSPEND
1821  {
1822  char buffer[128];
1823  __kmp_print_cond( buffer, &th->th.th_suspend_cv);
1824  __kmp_printf( "__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, buffer );
1825  }
1826 #endif
1827 
1828 
1829  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1830  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1831 
1832  KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) );
1833 }
1834 
1835 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
1836  __kmp_suspend_template(th_gtid, flag);
1837 }
1838 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
1839  __kmp_suspend_template(th_gtid, flag);
1840 }
1841 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1842  __kmp_suspend_template(th_gtid, flag);
1843 }
1844 
1845 
1846 /* This routine signals the thread specified by target_gtid to wake up
1847  * after setting the sleep bit indicated by the flag argument to FALSE.
1848  * The target thread must already have called __kmp_suspend_template()
1849  */
1850 template <class C>
1851 static inline void __kmp_resume_template( int target_gtid, C *flag )
1852 {
1853  kmp_info_t *th = __kmp_threads[target_gtid];
1854  int status;
1855 
1856 #ifdef KMP_DEBUG
1857  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1858 #endif
1859 
1860  KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
1861  KMP_DEBUG_ASSERT( gtid != target_gtid );
1862 
1863  __kmp_suspend_initialize_thread( th );
1864 
1865  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1866  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1867 
1868  if (!flag) {
1869  flag = (C *)th->th.th_sleep_loc;
1870  }
1871 
1872  if (!flag) {
1873  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p)\n",
1874  gtid, target_gtid, NULL ) );
1875  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1876  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1877  return;
1878  }
1879  else {
1880  typename C::flag_t old_spin = flag->unset_sleeping();
1881  if ( ! flag->is_sleeping_val(old_spin) ) {
1882  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p): "
1883  "%u => %u\n",
1884  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1885 
1886  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1887  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1888  return;
1889  }
1890  KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p): "
1891  "%u => %u\n",
1892  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1893  }
1894  TCW_PTR(th->th.th_sleep_loc, NULL);
1895 
1896 
1897 #ifdef DEBUG_SUSPEND
1898  {
1899  char buffer[128];
1900  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1901  __kmp_printf( "__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, target_gtid, buffer );
1902  }
1903 #endif
1904 
1905 
1906  status = pthread_cond_signal( &th->th.th_suspend_cv.c_cond );
1907  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1908  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1909  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1910  KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
1911  gtid, target_gtid ) );
1912 }
1913 
1914 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
1915  __kmp_resume_template(target_gtid, flag);
1916 }
1917 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
1918  __kmp_resume_template(target_gtid, flag);
1919 }
1920 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1921  __kmp_resume_template(target_gtid, flag);
1922 }
1923 
1924 void
1925 __kmp_resume_monitor()
1926 {
1927  KMP_TIME_BLOCK(USER_resume);
1928  int status;
1929 #ifdef KMP_DEBUG
1930  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1931  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n",
1932  gtid, KMP_GTID_MONITOR ) );
1933  KMP_DEBUG_ASSERT( gtid != KMP_GTID_MONITOR );
1934 #endif
1935  status = pthread_mutex_lock( &__kmp_wait_mx.m_mutex );
1936  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1937 #ifdef DEBUG_SUSPEND
1938  {
1939  char buffer[128];
1940  __kmp_print_cond( buffer, &__kmp_wait_cv.c_cond );
1941  __kmp_printf( "__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, KMP_GTID_MONITOR, buffer );
1942  }
1943 #endif
1944  status = pthread_cond_signal( &__kmp_wait_cv.c_cond );
1945  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1946  status = pthread_mutex_unlock( &__kmp_wait_mx.m_mutex );
1947  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1948  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d exiting after signaling wake up for T#%d\n",
1949  gtid, KMP_GTID_MONITOR ) );
1950 }
1951 
1952 /* ------------------------------------------------------------------------ */
1953 /* ------------------------------------------------------------------------ */
1954 
1955 void
1956 __kmp_yield( int cond )
1957 {
1958  if (cond && __kmp_yielding_on) {
1959  sched_yield();
1960  }
1961 }
1962 
1963 /* ------------------------------------------------------------------------ */
1964 /* ------------------------------------------------------------------------ */
1965 
1966 void
1967 __kmp_gtid_set_specific( int gtid )
1968 {
1969  int status;
1970  KMP_ASSERT( __kmp_init_runtime );
1971  status = pthread_setspecific( __kmp_gtid_threadprivate_key, (void*)(intptr_t)(gtid+1) );
1972  KMP_CHECK_SYSFAIL( "pthread_setspecific", status );
1973 }
1974 
1975 int
1976 __kmp_gtid_get_specific()
1977 {
1978  int gtid;
1979  if ( !__kmp_init_runtime ) {
1980  KA_TRACE( 50, ("__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
1981  return KMP_GTID_SHUTDOWN;
1982  }
1983  gtid = (int)(size_t)pthread_getspecific( __kmp_gtid_threadprivate_key );
1984  if ( gtid == 0 ) {
1985  gtid = KMP_GTID_DNE;
1986  }
1987  else {
1988  gtid--;
1989  }
1990  KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1991  __kmp_gtid_threadprivate_key, gtid ));
1992  return gtid;
1993 }
1994 
1995 /* ------------------------------------------------------------------------ */
1996 /* ------------------------------------------------------------------------ */
1997 
1998 double
1999 __kmp_read_cpu_time( void )
2000 {
2001  /*clock_t t;*/
2002  struct tms buffer;
2003 
2004  /*t =*/ times( & buffer );
2005 
2006  return (buffer.tms_utime + buffer.tms_cutime) / (double) CLOCKS_PER_SEC;
2007 }
2008 
2009 int
2010 __kmp_read_system_info( struct kmp_sys_info *info )
2011 {
2012  int status;
2013  struct rusage r_usage;
2014 
2015  memset( info, 0, sizeof( *info ) );
2016 
2017  status = getrusage( RUSAGE_SELF, &r_usage);
2018  KMP_CHECK_SYSFAIL_ERRNO( "getrusage", status );
2019 
2020  info->maxrss = r_usage.ru_maxrss; /* the maximum resident set size utilized (in kilobytes) */
2021  info->minflt = r_usage.ru_minflt; /* the number of page faults serviced without any I/O */
2022  info->majflt = r_usage.ru_majflt; /* the number of page faults serviced that required I/O */
2023  info->nswap = r_usage.ru_nswap; /* the number of times a process was "swapped" out of memory */
2024  info->inblock = r_usage.ru_inblock; /* the number of times the file system had to perform input */
2025  info->oublock = r_usage.ru_oublock; /* the number of times the file system had to perform output */
2026  info->nvcsw = r_usage.ru_nvcsw; /* the number of times a context switch was voluntarily */
2027  info->nivcsw = r_usage.ru_nivcsw; /* the number of times a context switch was forced */
2028 
2029  return (status != 0);
2030 }
2031 
2032 /* ------------------------------------------------------------------------ */
2033 /* ------------------------------------------------------------------------ */
2034 
2035 
2036 void
2037 __kmp_read_system_time( double *delta )
2038 {
2039  double t_ns;
2040  struct timeval tval;
2041  struct timespec stop;
2042  int status;
2043 
2044  status = gettimeofday( &tval, NULL );
2045  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2046  TIMEVAL_TO_TIMESPEC( &tval, &stop );
2047  t_ns = TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start);
2048  *delta = (t_ns * 1e-9);
2049 }
2050 
2051 void
2052 __kmp_clear_system_time( void )
2053 {
2054  struct timeval tval;
2055  int status;
2056  status = gettimeofday( &tval, NULL );
2057  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2058  TIMEVAL_TO_TIMESPEC( &tval, &__kmp_sys_timer_data.start );
2059 }
2060 
2061 /* ------------------------------------------------------------------------ */
2062 /* ------------------------------------------------------------------------ */
2063 
2064 #ifdef BUILD_TV
2065 
2066 void
2067 __kmp_tv_threadprivate_store( kmp_info_t *th, void *global_addr, void *thread_addr )
2068 {
2069  struct tv_data *p;
2070 
2071  p = (struct tv_data *) __kmp_allocate( sizeof( *p ) );
2072 
2073  p->u.tp.global_addr = global_addr;
2074  p->u.tp.thread_addr = thread_addr;
2075 
2076  p->type = (void *) 1;
2077 
2078  p->next = th->th.th_local.tv_data;
2079  th->th.th_local.tv_data = p;
2080 
2081  if ( p->next == 0 ) {
2082  int rc = pthread_setspecific( __kmp_tv_key, p );
2083  KMP_CHECK_SYSFAIL( "pthread_setspecific", rc );
2084  }
2085 }
2086 
2087 #endif /* BUILD_TV */
2088 
2089 /* ------------------------------------------------------------------------ */
2090 /* ------------------------------------------------------------------------ */
2091 
2092 static int
2093 __kmp_get_xproc( void ) {
2094 
2095  int r = 0;
2096 
2097  #if KMP_OS_LINUX
2098 
2099  r = sysconf( _SC_NPROCESSORS_ONLN );
2100 
2101  #elif KMP_OS_DARWIN
2102 
2103  // Bug C77011 High "OpenMP Threads and number of active cores".
2104 
2105  // Find the number of available CPUs.
2106  kern_return_t rc;
2107  host_basic_info_data_t info;
2108  mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT;
2109  rc = host_info( mach_host_self(), HOST_BASIC_INFO, (host_info_t) & info, & num );
2110  if ( rc == 0 && num == HOST_BASIC_INFO_COUNT ) {
2111  // Cannot use KA_TRACE() here because this code works before trace support is
2112  // initialized.
2113  r = info.avail_cpus;
2114  } else {
2115  KMP_WARNING( CantGetNumAvailCPU );
2116  KMP_INFORM( AssumedNumCPU );
2117  }; // if
2118 
2119  #elif KMP_OS_FREEBSD
2120 
2121  int mib[] = { CTL_HW, HW_NCPU };
2122  size_t len = sizeof( r );
2123  if ( sysctl( mib, 2, &r, &len, NULL, 0 ) < 0 ) {
2124  r = 0;
2125  KMP_WARNING( CantGetNumAvailCPU );
2126  KMP_INFORM( AssumedNumCPU );
2127  }
2128 
2129  #else
2130 
2131  #error "Unknown or unsupported OS."
2132 
2133  #endif
2134 
2135  return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
2136 
2137 } // __kmp_get_xproc
2138 
2139 int
2140 __kmp_read_from_file( char const *path, char const *format, ... )
2141 {
2142  int result;
2143  va_list args;
2144 
2145  va_start(args, format);
2146  FILE *f = fopen(path, "rb");
2147  if ( f == NULL )
2148  return 0;
2149  result = vfscanf(f, format, args);
2150  fclose(f);
2151 
2152  return result;
2153 }
2154 
2155 void
2156 __kmp_runtime_initialize( void )
2157 {
2158  int status;
2159  pthread_mutexattr_t mutex_attr;
2160  pthread_condattr_t cond_attr;
2161 
2162  if ( __kmp_init_runtime ) {
2163  return;
2164  }; // if
2165 
2166  #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
2167  if ( ! __kmp_cpuinfo.initialized ) {
2168  __kmp_query_cpuid( &__kmp_cpuinfo );
2169  }; // if
2170  #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
2171 
2172  __kmp_xproc = __kmp_get_xproc();
2173 
2174  if ( sysconf( _SC_THREADS ) ) {
2175 
2176  /* Query the maximum number of threads */
2177  __kmp_sys_max_nth = sysconf( _SC_THREAD_THREADS_MAX );
2178  if ( __kmp_sys_max_nth == -1 ) {
2179  /* Unlimited threads for NPTL */
2180  __kmp_sys_max_nth = INT_MAX;
2181  }
2182  else if ( __kmp_sys_max_nth <= 1 ) {
2183  /* Can't tell, just use PTHREAD_THREADS_MAX */
2184  __kmp_sys_max_nth = KMP_MAX_NTH;
2185  }
2186 
2187  /* Query the minimum stack size */
2188  __kmp_sys_min_stksize = sysconf( _SC_THREAD_STACK_MIN );
2189  if ( __kmp_sys_min_stksize <= 1 ) {
2190  __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
2191  }
2192  }
2193 
2194  /* Set up minimum number of threads to switch to TLS gtid */
2195  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2196 
2197 
2198  #ifdef BUILD_TV
2199  {
2200  int rc = pthread_key_create( & __kmp_tv_key, 0 );
2201  KMP_CHECK_SYSFAIL( "pthread_key_create", rc );
2202  }
2203  #endif
2204 
2205  status = pthread_key_create( &__kmp_gtid_threadprivate_key, __kmp_internal_end_dest );
2206  KMP_CHECK_SYSFAIL( "pthread_key_create", status );
2207  status = pthread_mutexattr_init( & mutex_attr );
2208  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
2209  status = pthread_mutex_init( & __kmp_wait_mx.m_mutex, & mutex_attr );
2210  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
2211  status = pthread_condattr_init( & cond_attr );
2212  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
2213  status = pthread_cond_init( & __kmp_wait_cv.c_cond, & cond_attr );
2214  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
2215 #if USE_ITT_BUILD
2216  __kmp_itt_initialize();
2217 #endif /* USE_ITT_BUILD */
2218 
2219  __kmp_init_runtime = TRUE;
2220 }
2221 
2222 void
2223 __kmp_runtime_destroy( void )
2224 {
2225  int status;
2226 
2227  if ( ! __kmp_init_runtime ) {
2228  return; // Nothing to do.
2229  };
2230 
2231 #if USE_ITT_BUILD
2232  __kmp_itt_destroy();
2233 #endif /* USE_ITT_BUILD */
2234 
2235  status = pthread_key_delete( __kmp_gtid_threadprivate_key );
2236  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2237  #ifdef BUILD_TV
2238  status = pthread_key_delete( __kmp_tv_key );
2239  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2240  #endif
2241 
2242  status = pthread_mutex_destroy( & __kmp_wait_mx.m_mutex );
2243  if ( status != 0 && status != EBUSY ) {
2244  KMP_SYSFAIL( "pthread_mutex_destroy", status );
2245  }
2246  status = pthread_cond_destroy( & __kmp_wait_cv.c_cond );
2247  if ( status != 0 && status != EBUSY ) {
2248  KMP_SYSFAIL( "pthread_cond_destroy", status );
2249  }
2250  #if KMP_AFFINITY_SUPPORTED
2251  __kmp_affinity_uninitialize();
2252  #endif
2253 
2254  __kmp_init_runtime = FALSE;
2255 }
2256 
2257 
2258 /* Put the thread to sleep for a time period */
2259 /* NOTE: not currently used anywhere */
2260 void
2261 __kmp_thread_sleep( int millis )
2262 {
2263  sleep( ( millis + 500 ) / 1000 );
2264 }
2265 
2266 /* Calculate the elapsed wall clock time for the user */
2267 void
2268 __kmp_elapsed( double *t )
2269 {
2270  int status;
2271 # ifdef FIX_SGI_CLOCK
2272  struct timespec ts;
2273 
2274  status = clock_gettime( CLOCK_PROCESS_CPUTIME_ID, &ts );
2275  KMP_CHECK_SYSFAIL_ERRNO( "clock_gettime", status );
2276  *t = (double) ts.tv_nsec * (1.0 / (double) KMP_NSEC_PER_SEC) +
2277  (double) ts.tv_sec;
2278 # else
2279  struct timeval tv;
2280 
2281  status = gettimeofday( & tv, NULL );
2282  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2283  *t = (double) tv.tv_usec * (1.0 / (double) KMP_USEC_PER_SEC) +
2284  (double) tv.tv_sec;
2285 # endif
2286 }
2287 
2288 /* Calculate the elapsed wall clock tick for the user */
2289 void
2290 __kmp_elapsed_tick( double *t )
2291 {
2292  *t = 1 / (double) CLOCKS_PER_SEC;
2293 }
2294 
2295 /*
2296  Determine whether the given address is mapped into the current address space.
2297 */
2298 
2299 int
2300 __kmp_is_address_mapped( void * addr ) {
2301 
2302  int found = 0;
2303  int rc;
2304 
2305  #if KMP_OS_LINUX
2306 
2307  /*
2308  On Linux* OS, read the /proc/<pid>/maps pseudo-file to get all the address ranges mapped
2309  into the address space.
2310  */
2311 
2312  char * name = __kmp_str_format( "/proc/%d/maps", getpid() );
2313  FILE * file = NULL;
2314 
2315  file = fopen( name, "r" );
2316  KMP_ASSERT( file != NULL );
2317 
2318  for ( ; ; ) {
2319 
2320  void * beginning = NULL;
2321  void * ending = NULL;
2322  char perms[ 5 ];
2323 
2324  rc = fscanf( file, "%p-%p %4s %*[^\n]\n", & beginning, & ending, perms );
2325  if ( rc == EOF ) {
2326  break;
2327  }; // if
2328  KMP_ASSERT( rc == 3 && KMP_STRLEN( perms ) == 4 ); // Make sure all fields are read.
2329 
2330  // Ending address is not included in the region, but beginning is.
2331  if ( ( addr >= beginning ) && ( addr < ending ) ) {
2332  perms[ 2 ] = 0; // 3th and 4th character does not matter.
2333  if ( strcmp( perms, "rw" ) == 0 ) {
2334  // Memory we are looking for should be readable and writable.
2335  found = 1;
2336  }; // if
2337  break;
2338  }; // if
2339 
2340  }; // forever
2341 
2342  // Free resources.
2343  fclose( file );
2344  KMP_INTERNAL_FREE( name );
2345 
2346  #elif KMP_OS_DARWIN
2347 
2348  /*
2349  On OS X*, /proc pseudo filesystem is not available. Try to read memory using vm
2350  interface.
2351  */
2352 
2353  int buffer;
2354  vm_size_t count;
2355  rc =
2356  vm_read_overwrite(
2357  mach_task_self(), // Task to read memory of.
2358  (vm_address_t)( addr ), // Address to read from.
2359  1, // Number of bytes to be read.
2360  (vm_address_t)( & buffer ), // Address of buffer to save read bytes in.
2361  & count // Address of var to save number of read bytes in.
2362  );
2363  if ( rc == 0 ) {
2364  // Memory successfully read.
2365  found = 1;
2366  }; // if
2367 
2368  #elif KMP_OS_FREEBSD
2369 
2370  // FIXME(FreeBSD*): Implement this
2371  found = 1;
2372 
2373  #else
2374 
2375  #error "Unknown or unsupported OS"
2376 
2377  #endif
2378 
2379  return found;
2380 
2381 } // __kmp_is_address_mapped
2382 
2383 #ifdef USE_LOAD_BALANCE
2384 
2385 
2386 # if KMP_OS_DARWIN
2387 
2388 // The function returns the rounded value of the system load average
2389 // during given time interval which depends on the value of
2390 // __kmp_load_balance_interval variable (default is 60 sec, other values
2391 // may be 300 sec or 900 sec).
2392 // It returns -1 in case of error.
2393 int
2394 __kmp_get_load_balance( int max )
2395 {
2396  double averages[3];
2397  int ret_avg = 0;
2398 
2399  int res = getloadavg( averages, 3 );
2400 
2401  //Check __kmp_load_balance_interval to determine which of averages to use.
2402  // getloadavg() may return the number of samples less than requested that is
2403  // less than 3.
2404  if ( __kmp_load_balance_interval < 180 && ( res >= 1 ) ) {
2405  ret_avg = averages[0];// 1 min
2406  } else if ( ( __kmp_load_balance_interval >= 180
2407  && __kmp_load_balance_interval < 600 ) && ( res >= 2 ) ) {
2408  ret_avg = averages[1];// 5 min
2409  } else if ( ( __kmp_load_balance_interval >= 600 ) && ( res == 3 ) ) {
2410  ret_avg = averages[2];// 15 min
2411  } else {// Error occurred
2412  return -1;
2413  }
2414 
2415  return ret_avg;
2416 }
2417 
2418 # else // Linux* OS
2419 
2420 // The fuction returns number of running (not sleeping) threads, or -1 in case of error.
2421 // Error could be reported if Linux* OS kernel too old (without "/proc" support).
2422 // Counting running threads stops if max running threads encountered.
2423 int
2424 __kmp_get_load_balance( int max )
2425 {
2426  static int permanent_error = 0;
2427 
2428  static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */
2429  static double glb_call_time = 0; /* Thread balance algorithm call time */
2430 
2431  int running_threads = 0; // Number of running threads in the system.
2432 
2433  DIR * proc_dir = NULL; // Handle of "/proc/" directory.
2434  struct dirent * proc_entry = NULL;
2435 
2436  kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2437  DIR * task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2438  struct dirent * task_entry = NULL;
2439  int task_path_fixed_len;
2440 
2441  kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2442  int stat_file = -1;
2443  int stat_path_fixed_len;
2444 
2445  int total_processes = 0; // Total number of processes in system.
2446  int total_threads = 0; // Total number of threads in system.
2447 
2448  double call_time = 0.0;
2449 
2450  __kmp_str_buf_init( & task_path );
2451  __kmp_str_buf_init( & stat_path );
2452 
2453  __kmp_elapsed( & call_time );
2454 
2455  if ( glb_call_time &&
2456  ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
2457  running_threads = glb_running_threads;
2458  goto finish;
2459  }
2460 
2461  glb_call_time = call_time;
2462 
2463  // Do not spend time on scanning "/proc/" if we have a permanent error.
2464  if ( permanent_error ) {
2465  running_threads = -1;
2466  goto finish;
2467  }; // if
2468 
2469  if ( max <= 0 ) {
2470  max = INT_MAX;
2471  }; // if
2472 
2473  // Open "/proc/" directory.
2474  proc_dir = opendir( "/proc" );
2475  if ( proc_dir == NULL ) {
2476  // Cannot open "/prroc/". Probably the kernel does not support it. Return an error now and
2477  // in subsequent calls.
2478  running_threads = -1;
2479  permanent_error = 1;
2480  goto finish;
2481  }; // if
2482 
2483  // Initialize fixed part of task_path. This part will not change.
2484  __kmp_str_buf_cat( & task_path, "/proc/", 6 );
2485  task_path_fixed_len = task_path.used; // Remember number of used characters.
2486 
2487  proc_entry = readdir( proc_dir );
2488  while ( proc_entry != NULL ) {
2489  // Proc entry is a directory and name starts with a digit. Assume it is a process'
2490  // directory.
2491  if ( proc_entry->d_type == DT_DIR && isdigit( proc_entry->d_name[ 0 ] ) ) {
2492 
2493  ++ total_processes;
2494  // Make sure init process is the very first in "/proc", so we can replace
2495  // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes == 1.
2496  // We are going to check that total_processes == 1 => d_name == "1" is true (where
2497  // "=>" is implication). Since C++ does not have => operator, let us replace it with its
2498  // equivalent: a => b == ! a || b.
2499  KMP_DEBUG_ASSERT( total_processes != 1 || strcmp( proc_entry->d_name, "1" ) == 0 );
2500 
2501  // Construct task_path.
2502  task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2503  __kmp_str_buf_cat( & task_path, proc_entry->d_name, KMP_STRLEN( proc_entry->d_name ) );
2504  __kmp_str_buf_cat( & task_path, "/task", 5 );
2505 
2506  task_dir = opendir( task_path.str );
2507  if ( task_dir == NULL ) {
2508  // Process can finish between reading "/proc/" directory entry and opening process'
2509  // "task/" directory. So, in general case we should not complain, but have to skip
2510  // this process and read the next one.
2511  // But on systems with no "task/" support we will spend lot of time to scan "/proc/"
2512  // tree again and again without any benefit. "init" process (its pid is 1) should
2513  // exist always, so, if we cannot open "/proc/1/task/" directory, it means "task/"
2514  // is not supported by kernel. Report an error now and in the future.
2515  if ( strcmp( proc_entry->d_name, "1" ) == 0 ) {
2516  running_threads = -1;
2517  permanent_error = 1;
2518  goto finish;
2519  }; // if
2520  } else {
2521  // Construct fixed part of stat file path.
2522  __kmp_str_buf_clear( & stat_path );
2523  __kmp_str_buf_cat( & stat_path, task_path.str, task_path.used );
2524  __kmp_str_buf_cat( & stat_path, "/", 1 );
2525  stat_path_fixed_len = stat_path.used;
2526 
2527  task_entry = readdir( task_dir );
2528  while ( task_entry != NULL ) {
2529  // It is a directory and name starts with a digit.
2530  if ( proc_entry->d_type == DT_DIR && isdigit( task_entry->d_name[ 0 ] ) ) {
2531 
2532  ++ total_threads;
2533 
2534  // Consruct complete stat file path. Easiest way would be:
2535  // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str, task_entry->d_name );
2536  // but seriae of __kmp_str_buf_cat works a bit faster.
2537  stat_path.used = stat_path_fixed_len; // Reset stat path to its fixed part.
2538  __kmp_str_buf_cat( & stat_path, task_entry->d_name, KMP_STRLEN( task_entry->d_name ) );
2539  __kmp_str_buf_cat( & stat_path, "/stat", 5 );
2540 
2541  // Note: Low-level API (open/read/close) is used. High-level API
2542  // (fopen/fclose) works ~ 30 % slower.
2543  stat_file = open( stat_path.str, O_RDONLY );
2544  if ( stat_file == -1 ) {
2545  // We cannot report an error because task (thread) can terminate just
2546  // before reading this file.
2547  } else {
2548  /*
2549  Content of "stat" file looks like:
2550 
2551  24285 (program) S ...
2552 
2553  It is a single line (if program name does not include fanny
2554  symbols). First number is a thread id, then name of executable file
2555  name in paretheses, then state of the thread. We need just thread
2556  state.
2557 
2558  Good news: Length of program name is 15 characters max. Longer
2559  names are truncated.
2560 
2561  Thus, we need rather short buffer: 15 chars for program name +
2562  2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2563 
2564  Bad news: Program name may contain special symbols like space,
2565  closing parenthesis, or even new line. This makes parsing "stat"
2566  file not 100 % reliable. In case of fanny program names parsing
2567  may fail (report incorrect thread state).
2568 
2569  Parsing "status" file looks more promissing (due to different
2570  file structure and escaping special symbols) but reading and
2571  parsing of "status" file works slower.
2572 
2573  -- ln
2574  */
2575  char buffer[ 65 ];
2576  int len;
2577  len = read( stat_file, buffer, sizeof( buffer ) - 1 );
2578  if ( len >= 0 ) {
2579  buffer[ len ] = 0;
2580  // Using scanf:
2581  // sscanf( buffer, "%*d (%*s) %c ", & state );
2582  // looks very nice, but searching for a closing parenthesis works a
2583  // bit faster.
2584  char * close_parent = strstr( buffer, ") " );
2585  if ( close_parent != NULL ) {
2586  char state = * ( close_parent + 2 );
2587  if ( state == 'R' ) {
2588  ++ running_threads;
2589  if ( running_threads >= max ) {
2590  goto finish;
2591  }; // if
2592  }; // if
2593  }; // if
2594  }; // if
2595  close( stat_file );
2596  stat_file = -1;
2597  }; // if
2598  }; // if
2599  task_entry = readdir( task_dir );
2600  }; // while
2601  closedir( task_dir );
2602  task_dir = NULL;
2603  }; // if
2604  }; // if
2605  proc_entry = readdir( proc_dir );
2606  }; // while
2607 
2608  //
2609  // There _might_ be a timing hole where the thread executing this
2610  // code get skipped in the load balance, and running_threads is 0.
2611  // Assert in the debug builds only!!!
2612  //
2613  KMP_DEBUG_ASSERT( running_threads > 0 );
2614  if ( running_threads <= 0 ) {
2615  running_threads = 1;
2616  }
2617 
2618  finish: // Clean up and exit.
2619  if ( proc_dir != NULL ) {
2620  closedir( proc_dir );
2621  }; // if
2622  __kmp_str_buf_free( & task_path );
2623  if ( task_dir != NULL ) {
2624  closedir( task_dir );
2625  }; // if
2626  __kmp_str_buf_free( & stat_path );
2627  if ( stat_file != -1 ) {
2628  close( stat_file );
2629  }; // if
2630 
2631  glb_running_threads = running_threads;
2632 
2633  return running_threads;
2634 
2635 } // __kmp_get_load_balance
2636 
2637 # endif // KMP_OS_DARWIN
2638 
2639 #endif // USE_LOAD_BALANCE
2640 
2641 
2642 #if KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64)
2643 
2644 int __kmp_invoke_microtask( microtask_t pkfn, int gtid, int tid, int argc,
2645  void *p_argv[]
2646 #if OMPT_SUPPORT
2647  , void **exit_frame_ptr
2648 #endif
2649 )
2650 {
2651  int argc_full = argc + 2;
2652  int i;
2653  ffi_cif cif;
2654  ffi_type *types[argc_full];
2655  void *args[argc_full];
2656  void *idp[2];
2657 
2658 #if OMPT_SUPPORT
2659  *exit_frame_ptr = __builtin_frame_address(0);
2660 #endif
2661  /* We're only passing pointers to the target. */
2662  for (i = 0; i < argc_full; i++)
2663  types[i] = &ffi_type_pointer;
2664 
2665  /* Ugly double-indirection, but that's how it goes... */
2666  idp[0] = &gtid;
2667  idp[1] = &tid;
2668  args[0] = &idp[0];
2669  args[1] = &idp[1];
2670 
2671  for (i = 0; i < argc; i++)
2672  args[2 + i] = &p_argv[i];
2673 
2674  if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, argc_full,
2675  &ffi_type_void, types) != FFI_OK)
2676  abort();
2677 
2678  ffi_call(&cif, (void (*)(void))pkfn, NULL, args);
2679 
2680 #if OMPT_SUPPORT
2681  *exit_frame_ptr = 0;
2682 #endif
2683 
2684  return 1;
2685 }
2686 
2687 #endif // KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64)
2688 
2689 #if KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
2690 
2691 // we really only need the case with 1 argument, because CLANG always build
2692 // a struct of pointers to shared variables referenced in the outlined function
2693 int
2694 __kmp_invoke_microtask( microtask_t pkfn,
2695  int gtid, int tid,
2696  int argc, void *p_argv[]
2697 #if OMPT_SUPPORT
2698  , void **exit_frame_ptr
2699 #endif
2700 )
2701 {
2702 #if OMPT_SUPPORT
2703  *exit_frame_ptr = __builtin_frame_address(0);
2704 #endif
2705 
2706  switch (argc) {
2707  default:
2708  fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2709  fflush(stderr);
2710  exit(-1);
2711  case 0:
2712  (*pkfn)(&gtid, &tid);
2713  break;
2714  case 1:
2715  (*pkfn)(&gtid, &tid, p_argv[0]);
2716  break;
2717  case 2:
2718  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2719  break;
2720  case 3:
2721  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2722  break;
2723  case 4:
2724  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]);
2725  break;
2726  case 5:
2727  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]);
2728  break;
2729  case 6:
2730  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2731  p_argv[5]);
2732  break;
2733  case 7:
2734  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2735  p_argv[5], p_argv[6]);
2736  break;
2737  case 8:
2738  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2739  p_argv[5], p_argv[6], p_argv[7]);
2740  break;
2741  case 9:
2742  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2743  p_argv[5], p_argv[6], p_argv[7], p_argv[8]);
2744  break;
2745  case 10:
2746  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2747  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]);
2748  break;
2749  case 11:
2750  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2751  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2752  break;
2753  case 12:
2754  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2755  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2756  p_argv[11]);
2757  break;
2758  case 13:
2759  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2760  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2761  p_argv[11], p_argv[12]);
2762  break;
2763  case 14:
2764  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2765  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2766  p_argv[11], p_argv[12], p_argv[13]);
2767  break;
2768  case 15:
2769  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2770  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2771  p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2772  break;
2773  }
2774 
2775 #if OMPT_SUPPORT
2776  *exit_frame_ptr = 0;
2777 #endif
2778 
2779  return 1;
2780 }
2781 
2782 #endif
2783 
2784 // end of file //
2785 
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
Definition: kmp_stats.h:649
#define KMP_TIME_BLOCK(name)
Uses specified timer (name) to time code block.
Definition: kmp_stats.h:610