LLVM OpenMP* Runtime Library
z_Linux_util.cpp
1 /*
2  * z_Linux_util.cpp -- platform specific routines.
3  */
4 
5 
6 //===----------------------------------------------------------------------===//
7 //
8 // The LLVM Compiler Infrastructure
9 //
10 // This file is dual licensed under the MIT and the University of Illinois Open
11 // Source Licenses. See LICENSE.txt for details.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 
16 #include "kmp.h"
17 #include "kmp_wrapper_getpid.h"
18 #include "kmp_itt.h"
19 #include "kmp_str.h"
20 #include "kmp_i18n.h"
21 #include "kmp_lock.h"
22 #include "kmp_io.h"
23 #include "kmp_stats.h"
24 #include "kmp_wait_release.h"
25 #include "kmp_affinity.h"
26 
27 #if !KMP_OS_FREEBSD && !KMP_OS_NETBSD
28 # include <alloca.h>
29 #endif
30 #include <unistd.h>
31 #include <math.h> // HUGE_VAL.
32 #include <sys/time.h>
33 #include <sys/times.h>
34 #include <sys/resource.h>
35 #include <sys/syscall.h>
36 
37 #if KMP_OS_LINUX && !KMP_OS_CNK
38 # include <sys/sysinfo.h>
39 # if KMP_USE_FUTEX
40 // We should really include <futex.h>, but that causes compatibility problems on different
41 // Linux* OS distributions that either require that you include (or break when you try to include)
42 // <pci/types.h>.
43 // Since all we need is the two macros below (which are part of the kernel ABI, so can't change)
44 // we just define the constants here and don't include <futex.h>
45 # ifndef FUTEX_WAIT
46 # define FUTEX_WAIT 0
47 # endif
48 # ifndef FUTEX_WAKE
49 # define FUTEX_WAKE 1
50 # endif
51 # endif
52 #elif KMP_OS_DARWIN
53 # include <sys/sysctl.h>
54 # include <mach/mach.h>
55 #elif KMP_OS_FREEBSD
56 # include <pthread_np.h>
57 #endif
58 
59 #include <dirent.h>
60 #include <ctype.h>
61 #include <fcntl.h>
62 
63 #include "tsan_annotations.h"
64 
65 /* ------------------------------------------------------------------------ */
66 /* ------------------------------------------------------------------------ */
67 
68 struct kmp_sys_timer {
69  struct timespec start;
70 };
71 
72 // Convert timespec to nanoseconds.
73 #define TS2NS(timespec) (((timespec).tv_sec * 1e9) + (timespec).tv_nsec)
74 
75 static struct kmp_sys_timer __kmp_sys_timer_data;
76 
77 #if KMP_HANDLE_SIGNALS
78  typedef void (* sig_func_t )( int );
79  STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[ NSIG ];
80  static sigset_t __kmp_sigset;
81 #endif
82 
83 static int __kmp_init_runtime = FALSE;
84 
85 static int __kmp_fork_count = 0;
86 
87 static pthread_condattr_t __kmp_suspend_cond_attr;
88 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
89 
90 static kmp_cond_align_t __kmp_wait_cv;
91 static kmp_mutex_align_t __kmp_wait_mx;
92 
93 double __kmp_ticks_per_nsec;
94 
95 /* ------------------------------------------------------------------------ */
96 /* ------------------------------------------------------------------------ */
97 
98 #ifdef DEBUG_SUSPEND
99 static void
100 __kmp_print_cond( char *buffer, kmp_cond_align_t *cond )
101 {
102  KMP_SNPRINTF( buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
103  cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
104  cond->c_cond.__c_waiting );
105 }
106 #endif
107 
108 /* ------------------------------------------------------------------------ */
109 /* ------------------------------------------------------------------------ */
110 
111 #if ( KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED)
112 
113 /*
114  * Affinity support
115  */
116 
117 void
118 __kmp_affinity_bind_thread( int which )
119 {
120  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
121  "Illegal set affinity operation when not capable");
122 
123  kmp_affin_mask_t *mask;
124  KMP_CPU_ALLOC_ON_STACK(mask);
125  KMP_CPU_ZERO(mask);
126  KMP_CPU_SET(which, mask);
127  __kmp_set_system_affinity(mask, TRUE);
128  KMP_CPU_FREE_FROM_STACK(mask);
129 }
130 
131 /*
132  * Determine if we can access affinity functionality on this version of
133  * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
134  * __kmp_affin_mask_size to the appropriate value (0 means not capable).
135  */
136 void
137 __kmp_affinity_determine_capable(const char *env_var)
138 {
139  //
140  // Check and see if the OS supports thread affinity.
141  //
142 
143 # define KMP_CPU_SET_SIZE_LIMIT (1024*1024)
144 
145  int gCode;
146  int sCode;
147  unsigned char *buf;
148  buf = ( unsigned char * ) KMP_INTERNAL_MALLOC( KMP_CPU_SET_SIZE_LIMIT );
149 
150  // If Linux* OS:
151  // If the syscall fails or returns a suggestion for the size,
152  // then we don't have to search for an appropriate size.
153  gCode = syscall( __NR_sched_getaffinity, 0, KMP_CPU_SET_SIZE_LIMIT, buf );
154  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
155  "initial getaffinity call returned %d errno = %d\n",
156  gCode, errno));
157 
158  //if ((gCode < 0) && (errno == ENOSYS))
159  if (gCode < 0) {
160  //
161  // System call not supported
162  //
163  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
164  && (__kmp_affinity_type != affinity_none)
165  && (__kmp_affinity_type != affinity_default)
166  && (__kmp_affinity_type != affinity_disabled))) {
167  int error = errno;
168  kmp_msg_t err_code = KMP_ERR( error );
169  __kmp_msg(
170  kmp_ms_warning,
171  KMP_MSG( GetAffSysCallNotSupported, env_var ),
172  err_code,
173  __kmp_msg_null
174  );
175  if (__kmp_generate_warnings == kmp_warnings_off) {
176  __kmp_str_free(&err_code.str);
177  }
178  }
179  KMP_AFFINITY_DISABLE();
180  KMP_INTERNAL_FREE(buf);
181  return;
182  }
183  if (gCode > 0) { // Linux* OS only
184  // The optimal situation: the OS returns the size of the buffer
185  // it expects.
186  //
187  // A verification of correct behavior is that Isetaffinity on a NULL
188  // buffer with the same size fails with errno set to EFAULT.
189  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
190  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
191  "setaffinity for mask size %d returned %d errno = %d\n",
192  gCode, sCode, errno));
193  if (sCode < 0) {
194  if (errno == ENOSYS) {
195  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
196  && (__kmp_affinity_type != affinity_none)
197  && (__kmp_affinity_type != affinity_default)
198  && (__kmp_affinity_type != affinity_disabled))) {
199  int error = errno;
200  kmp_msg_t err_code = KMP_ERR( error );
201  __kmp_msg(
202  kmp_ms_warning,
203  KMP_MSG( SetAffSysCallNotSupported, env_var ),
204  err_code,
205  __kmp_msg_null
206  );
207  if (__kmp_generate_warnings == kmp_warnings_off) {
208  __kmp_str_free(&err_code.str);
209  }
210  }
211  KMP_AFFINITY_DISABLE();
212  KMP_INTERNAL_FREE(buf);
213  }
214  if (errno == EFAULT) {
215  KMP_AFFINITY_ENABLE(gCode);
216  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
217  "affinity supported (mask size %d)\n",
218  (int)__kmp_affin_mask_size));
219  KMP_INTERNAL_FREE(buf);
220  return;
221  }
222  }
223  }
224 
225  //
226  // Call the getaffinity system call repeatedly with increasing set sizes
227  // until we succeed, or reach an upper bound on the search.
228  //
229  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
230  "searching for proper set size\n"));
231  int size;
232  for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
233  gCode = syscall( __NR_sched_getaffinity, 0, size, buf );
234  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
235  "getaffinity for mask size %d returned %d errno = %d\n", size,
236  gCode, errno));
237 
238  if (gCode < 0) {
239  if ( errno == ENOSYS )
240  {
241  //
242  // We shouldn't get here
243  //
244  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
245  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
246  size));
247  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
248  && (__kmp_affinity_type != affinity_none)
249  && (__kmp_affinity_type != affinity_default)
250  && (__kmp_affinity_type != affinity_disabled))) {
251  int error = errno;
252  kmp_msg_t err_code = KMP_ERR( error );
253  __kmp_msg(
254  kmp_ms_warning,
255  KMP_MSG( GetAffSysCallNotSupported, env_var ),
256  err_code,
257  __kmp_msg_null
258  );
259  if (__kmp_generate_warnings == kmp_warnings_off) {
260  __kmp_str_free(&err_code.str);
261  }
262  }
263  KMP_AFFINITY_DISABLE();
264  KMP_INTERNAL_FREE(buf);
265  return;
266  }
267  continue;
268  }
269 
270  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
271  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
272  "setaffinity for mask size %d returned %d errno = %d\n",
273  gCode, sCode, errno));
274  if (sCode < 0) {
275  if (errno == ENOSYS) { // Linux* OS only
276  //
277  // We shouldn't get here
278  //
279  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
280  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
281  size));
282  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
283  && (__kmp_affinity_type != affinity_none)
284  && (__kmp_affinity_type != affinity_default)
285  && (__kmp_affinity_type != affinity_disabled))) {
286  int error = errno;
287  kmp_msg_t err_code = KMP_ERR( error );
288  __kmp_msg(
289  kmp_ms_warning,
290  KMP_MSG( SetAffSysCallNotSupported, env_var ),
291  err_code,
292  __kmp_msg_null
293  );
294  if (__kmp_generate_warnings == kmp_warnings_off) {
295  __kmp_str_free(&err_code.str);
296  }
297  }
298  KMP_AFFINITY_DISABLE();
299  KMP_INTERNAL_FREE(buf);
300  return;
301  }
302  if (errno == EFAULT) {
303  KMP_AFFINITY_ENABLE(gCode);
304  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
305  "affinity supported (mask size %d)\n",
306  (int)__kmp_affin_mask_size));
307  KMP_INTERNAL_FREE(buf);
308  return;
309  }
310  }
311  }
312  //int error = errno; // save uncaught error code
313  KMP_INTERNAL_FREE(buf);
314  // errno = error; // restore uncaught error code, will be printed at the next KMP_WARNING below
315 
316  //
317  // Affinity is not supported
318  //
319  KMP_AFFINITY_DISABLE();
320  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
321  "cannot determine mask size - affinity not supported\n"));
322  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
323  && (__kmp_affinity_type != affinity_none)
324  && (__kmp_affinity_type != affinity_default)
325  && (__kmp_affinity_type != affinity_disabled))) {
326  KMP_WARNING( AffCantGetMaskSize, env_var );
327  }
328 }
329 
330 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
331 
332 /* ------------------------------------------------------------------------ */
333 /* ------------------------------------------------------------------------ */
334 
335 #if KMP_USE_FUTEX
336 
337 int
338 __kmp_futex_determine_capable()
339 {
340  int loc = 0;
341  int rc = syscall( __NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0 );
342  int retval = ( rc == 0 ) || ( errno != ENOSYS );
343 
344  KA_TRACE(10, ( "__kmp_futex_determine_capable: rc = %d errno = %d\n", rc,
345  errno ) );
346  KA_TRACE(10, ( "__kmp_futex_determine_capable: futex syscall%s supported\n",
347  retval ? "" : " not" ) );
348 
349  return retval;
350 }
351 
352 #endif // KMP_USE_FUTEX
353 
354 /* ------------------------------------------------------------------------ */
355 /* ------------------------------------------------------------------------ */
356 
357 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS)
358 /*
359  * Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
360  * use compare_and_store for these routines
361  */
362 
363 kmp_int8
364 __kmp_test_then_or8( volatile kmp_int8 *p, kmp_int8 d )
365 {
366  kmp_int8 old_value, new_value;
367 
368  old_value = TCR_1( *p );
369  new_value = old_value | d;
370 
371  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
372  {
373  KMP_CPU_PAUSE();
374  old_value = TCR_1( *p );
375  new_value = old_value | d;
376  }
377  return old_value;
378 }
379 
380 kmp_int8
381 __kmp_test_then_and8( volatile kmp_int8 *p, kmp_int8 d )
382 {
383  kmp_int8 old_value, new_value;
384 
385  old_value = TCR_1( *p );
386  new_value = old_value & d;
387 
388  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
389  {
390  KMP_CPU_PAUSE();
391  old_value = TCR_1( *p );
392  new_value = old_value & d;
393  }
394  return old_value;
395 }
396 
397 kmp_int32
398 __kmp_test_then_or32( volatile kmp_int32 *p, kmp_int32 d )
399 {
400  kmp_int32 old_value, new_value;
401 
402  old_value = TCR_4( *p );
403  new_value = old_value | d;
404 
405  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
406  {
407  KMP_CPU_PAUSE();
408  old_value = TCR_4( *p );
409  new_value = old_value | d;
410  }
411  return old_value;
412 }
413 
414 kmp_int32
415 __kmp_test_then_and32( volatile kmp_int32 *p, kmp_int32 d )
416 {
417  kmp_int32 old_value, new_value;
418 
419  old_value = TCR_4( *p );
420  new_value = old_value & d;
421 
422  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
423  {
424  KMP_CPU_PAUSE();
425  old_value = TCR_4( *p );
426  new_value = old_value & d;
427  }
428  return old_value;
429 }
430 
431 # if KMP_ARCH_X86 || KMP_ARCH_PPC64 || (KMP_OS_LINUX && KMP_ARCH_AARCH64)
432 kmp_int8
433 __kmp_test_then_add8( volatile kmp_int8 *p, kmp_int8 d )
434 {
435  kmp_int8 old_value, new_value;
436 
437  old_value = TCR_1( *p );
438  new_value = old_value + d;
439 
440  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
441  {
442  KMP_CPU_PAUSE();
443  old_value = TCR_1( *p );
444  new_value = old_value + d;
445  }
446  return old_value;
447 }
448 
449 kmp_int64
450 __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 d )
451 {
452  kmp_int64 old_value, new_value;
453 
454  old_value = TCR_8( *p );
455  new_value = old_value + d;
456 
457  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
458  {
459  KMP_CPU_PAUSE();
460  old_value = TCR_8( *p );
461  new_value = old_value + d;
462  }
463  return old_value;
464 }
465 # endif /* KMP_ARCH_X86 || KMP_ARCH_PPC64 || (KMP_OS_LINUX && KMP_ARCH_AARCH64) */
466 
467 kmp_int64
468 __kmp_test_then_or64( volatile kmp_int64 *p, kmp_int64 d )
469 {
470  kmp_int64 old_value, new_value;
471 
472  old_value = TCR_8( *p );
473  new_value = old_value | d;
474  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
475  {
476  KMP_CPU_PAUSE();
477  old_value = TCR_8( *p );
478  new_value = old_value | d;
479  }
480  return old_value;
481 }
482 
483 kmp_int64
484 __kmp_test_then_and64( volatile kmp_int64 *p, kmp_int64 d )
485 {
486  kmp_int64 old_value, new_value;
487 
488  old_value = TCR_8( *p );
489  new_value = old_value & d;
490  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
491  {
492  KMP_CPU_PAUSE();
493  old_value = TCR_8( *p );
494  new_value = old_value & d;
495  }
496  return old_value;
497 }
498 
499 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
500 
501 void
502 __kmp_terminate_thread( int gtid )
503 {
504  int status;
505  kmp_info_t *th = __kmp_threads[ gtid ];
506 
507  if ( !th ) return;
508 
509  #ifdef KMP_CANCEL_THREADS
510  KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) );
511  status = pthread_cancel( th->th.th_info.ds.ds_thread );
512  if ( status != 0 && status != ESRCH ) {
513  __kmp_msg(
514  kmp_ms_fatal,
515  KMP_MSG( CantTerminateWorkerThread ),
516  KMP_ERR( status ),
517  __kmp_msg_null
518  );
519  }; // if
520  #endif
521  __kmp_yield( TRUE );
522 } //
523 
524 /* ------------------------------------------------------------------------ */
525 /* ------------------------------------------------------------------------ */
526 
527 /* ------------------------------------------------------------------------ */
528 /* ------------------------------------------------------------------------ */
529 
530 /*
531  * Set thread stack info according to values returned by
532  * pthread_getattr_np().
533  * If values are unreasonable, assume call failed and use
534  * incremental stack refinement method instead.
535  * Returns TRUE if the stack parameters could be determined exactly,
536  * FALSE if incremental refinement is necessary.
537  */
538 static kmp_int32
539 __kmp_set_stack_info( int gtid, kmp_info_t *th )
540 {
541  int stack_data;
542 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
543  /* Linux* OS only -- no pthread_getattr_np support on OS X* */
544  pthread_attr_t attr;
545  int status;
546  size_t size = 0;
547  void * addr = 0;
548 
549  /* Always do incremental stack refinement for ubermaster threads since the initial
550  thread stack range can be reduced by sibling thread creation so pthread_attr_getstack
551  may cause thread gtid aliasing */
552  if ( ! KMP_UBER_GTID(gtid) ) {
553 
554  /* Fetch the real thread attributes */
555  status = pthread_attr_init( &attr );
556  KMP_CHECK_SYSFAIL( "pthread_attr_init", status );
557 #if KMP_OS_FREEBSD || KMP_OS_NETBSD
558  status = pthread_attr_get_np( pthread_self(), &attr );
559  KMP_CHECK_SYSFAIL( "pthread_attr_get_np", status );
560 #else
561  status = pthread_getattr_np( pthread_self(), &attr );
562  KMP_CHECK_SYSFAIL( "pthread_getattr_np", status );
563 #endif
564  status = pthread_attr_getstack( &attr, &addr, &size );
565  KMP_CHECK_SYSFAIL( "pthread_attr_getstack", status );
566  KA_TRACE( 60, ( "__kmp_set_stack_info: T#%d pthread_attr_getstack returned size: %lu, "
567  "low addr: %p\n",
568  gtid, size, addr ));
569 
570  status = pthread_attr_destroy( &attr );
571  KMP_CHECK_SYSFAIL( "pthread_attr_destroy", status );
572  }
573 
574  if ( size != 0 && addr != 0 ) { /* was stack parameter determination successful? */
575  /* Store the correct base and size */
576  TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
577  TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
578  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
579  return TRUE;
580  }
581 #endif /* KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD */
582  /* Use incremental refinement starting from initial conservative estimate */
583  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
584  TCW_PTR(th -> th.th_info.ds.ds_stackbase, &stack_data);
585  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
586  return FALSE;
587 }
588 
589 static void*
590 __kmp_launch_worker( void *thr )
591 {
592  int status, old_type, old_state;
593 #ifdef KMP_BLOCK_SIGNALS
594  sigset_t new_set, old_set;
595 #endif /* KMP_BLOCK_SIGNALS */
596  void *exit_val;
597 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
598  void * volatile padding = 0;
599 #endif
600  int gtid;
601 
602  gtid = ((kmp_info_t*)thr) -> th.th_info.ds.ds_gtid;
603  __kmp_gtid_set_specific( gtid );
604 #ifdef KMP_TDATA_GTID
605  __kmp_gtid = gtid;
606 #endif
607 #if KMP_STATS_ENABLED
608  // set __thread local index to point to thread-specific stats
609  __kmp_stats_thread_ptr = ((kmp_info_t*)thr)->th.th_stats;
610  KMP_START_EXPLICIT_TIMER(OMP_worker_thread_life);
611  KMP_SET_THREAD_STATE(IDLE);
612  KMP_INIT_PARTITIONED_TIMERS(OMP_idle);
613 #endif
614 
615 #if USE_ITT_BUILD
616  __kmp_itt_thread_name( gtid );
617 #endif /* USE_ITT_BUILD */
618 
619 #if KMP_AFFINITY_SUPPORTED
620  __kmp_affinity_set_init_mask( gtid, FALSE );
621 #endif
622 
623 #ifdef KMP_CANCEL_THREADS
624  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
625  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
626  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
627  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
628  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
629 #endif
630 
631 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
632  //
633  // Set the FP control regs to be a copy of
634  // the parallel initialization thread's.
635  //
636  __kmp_clear_x87_fpu_status_word();
637  __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
638  __kmp_load_mxcsr( &__kmp_init_mxcsr );
639 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
640 
641 #ifdef KMP_BLOCK_SIGNALS
642  status = sigfillset( & new_set );
643  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
644  status = pthread_sigmask( SIG_BLOCK, & new_set, & old_set );
645  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
646 #endif /* KMP_BLOCK_SIGNALS */
647 
648 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
649  if ( __kmp_stkoffset > 0 && gtid > 0 ) {
650  padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
651  }
652 #endif
653 
654  KMP_MB();
655  __kmp_set_stack_info( gtid, (kmp_info_t*)thr );
656 
657  __kmp_check_stack_overlap( (kmp_info_t*)thr );
658 
659  exit_val = __kmp_launch_thread( (kmp_info_t *) thr );
660 
661 #ifdef KMP_BLOCK_SIGNALS
662  status = pthread_sigmask( SIG_SETMASK, & old_set, NULL );
663  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
664 #endif /* KMP_BLOCK_SIGNALS */
665 
666  return exit_val;
667 }
668 
669 #if KMP_USE_MONITOR
670 /* The monitor thread controls all of the threads in the complex */
671 
672 static void*
673 __kmp_launch_monitor( void *thr )
674 {
675  int status, old_type, old_state;
676 #ifdef KMP_BLOCK_SIGNALS
677  sigset_t new_set;
678 #endif /* KMP_BLOCK_SIGNALS */
679  struct timespec interval;
680  int yield_count;
681  int yield_cycles = 0;
682 
683  KMP_MB(); /* Flush all pending memory write invalidates. */
684 
685  KA_TRACE( 10, ("__kmp_launch_monitor: #1 launched\n" ) );
686 
687  /* register us as the monitor thread */
688  __kmp_gtid_set_specific( KMP_GTID_MONITOR );
689 #ifdef KMP_TDATA_GTID
690  __kmp_gtid = KMP_GTID_MONITOR;
691 #endif
692 
693  KMP_MB();
694 
695 #if USE_ITT_BUILD
696  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread.
697 #endif /* USE_ITT_BUILD */
698 
699  __kmp_set_stack_info( ((kmp_info_t*)thr)->th.th_info.ds.ds_gtid, (kmp_info_t*)thr );
700 
701  __kmp_check_stack_overlap( (kmp_info_t*)thr );
702 
703 #ifdef KMP_CANCEL_THREADS
704  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
705  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
706  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
707  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
708  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
709 #endif
710 
711  #if KMP_REAL_TIME_FIX
712  // This is a potential fix which allows application with real-time scheduling policy work.
713  // However, decision about the fix is not made yet, so it is disabled by default.
714  { // Are program started with real-time scheduling policy?
715  int sched = sched_getscheduler( 0 );
716  if ( sched == SCHED_FIFO || sched == SCHED_RR ) {
717  // Yes, we are a part of real-time application. Try to increase the priority of the
718  // monitor.
719  struct sched_param param;
720  int max_priority = sched_get_priority_max( sched );
721  int rc;
722  KMP_WARNING( RealTimeSchedNotSupported );
723  sched_getparam( 0, & param );
724  if ( param.sched_priority < max_priority ) {
725  param.sched_priority += 1;
726  rc = sched_setscheduler( 0, sched, & param );
727  if ( rc != 0 ) {
728  int error = errno;
729  kmp_msg_t err_code = KMP_ERR( error );
730  __kmp_msg(
731  kmp_ms_warning,
732  KMP_MSG( CantChangeMonitorPriority ),
733  err_code,
734  KMP_MSG( MonitorWillStarve ),
735  __kmp_msg_null
736  );
737  if (__kmp_generate_warnings == kmp_warnings_off) {
738  __kmp_str_free(&err_code.str);
739  }
740  }; // if
741  } else {
742  // We cannot abort here, because number of CPUs may be enough for all the threads,
743  // including the monitor thread, so application could potentially work...
744  __kmp_msg(
745  kmp_ms_warning,
746  KMP_MSG( RunningAtMaxPriority ),
747  KMP_MSG( MonitorWillStarve ),
748  KMP_HNT( RunningAtMaxPriority ),
749  __kmp_msg_null
750  );
751  }; // if
752  }; // if
753  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 ); // AC: free thread that waits for monitor started
754  }
755  #endif // KMP_REAL_TIME_FIX
756 
757  KMP_MB(); /* Flush all pending memory write invalidates. */
758 
759  if ( __kmp_monitor_wakeups == 1 ) {
760  interval.tv_sec = 1;
761  interval.tv_nsec = 0;
762  } else {
763  interval.tv_sec = 0;
764  interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
765  }
766 
767  KA_TRACE( 10, ("__kmp_launch_monitor: #2 monitor\n" ) );
768 
769  if (__kmp_yield_cycle) {
770  __kmp_yielding_on = 0; /* Start out with yielding shut off */
771  yield_count = __kmp_yield_off_count;
772  } else {
773  __kmp_yielding_on = 1; /* Yielding is on permanently */
774  }
775 
776  while( ! TCR_4( __kmp_global.g.g_done ) ) {
777  struct timespec now;
778  struct timeval tval;
779 
780  /* This thread monitors the state of the system */
781 
782  KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) );
783 
784  status = gettimeofday( &tval, NULL );
785  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
786  TIMEVAL_TO_TIMESPEC( &tval, &now );
787 
788  now.tv_sec += interval.tv_sec;
789  now.tv_nsec += interval.tv_nsec;
790 
791  if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
792  now.tv_sec += 1;
793  now.tv_nsec -= KMP_NSEC_PER_SEC;
794  }
795 
796  status = pthread_mutex_lock( & __kmp_wait_mx.m_mutex );
797  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
798  // AC: the monitor should not fall asleep if g_done has been set
799  if ( !TCR_4(__kmp_global.g.g_done) ) { // check once more under mutex
800  status = pthread_cond_timedwait( &__kmp_wait_cv.c_cond, &__kmp_wait_mx.m_mutex, &now );
801  if ( status != 0 ) {
802  if ( status != ETIMEDOUT && status != EINTR ) {
803  KMP_SYSFAIL( "pthread_cond_timedwait", status );
804  };
805  };
806  };
807  status = pthread_mutex_unlock( & __kmp_wait_mx.m_mutex );
808  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
809 
810  if (__kmp_yield_cycle) {
811  yield_cycles++;
812  if ( (yield_cycles % yield_count) == 0 ) {
813  if (__kmp_yielding_on) {
814  __kmp_yielding_on = 0; /* Turn it off now */
815  yield_count = __kmp_yield_off_count;
816  } else {
817  __kmp_yielding_on = 1; /* Turn it on now */
818  yield_count = __kmp_yield_on_count;
819  }
820  yield_cycles = 0;
821  }
822  } else {
823  __kmp_yielding_on = 1;
824  }
825 
826  TCW_4( __kmp_global.g.g_time.dt.t_value,
827  TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
828 
829  KMP_MB(); /* Flush all pending memory write invalidates. */
830  }
831 
832  KA_TRACE( 10, ("__kmp_launch_monitor: #3 cleanup\n" ) );
833 
834 #ifdef KMP_BLOCK_SIGNALS
835  status = sigfillset( & new_set );
836  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
837  status = pthread_sigmask( SIG_UNBLOCK, & new_set, NULL );
838  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
839 #endif /* KMP_BLOCK_SIGNALS */
840 
841  KA_TRACE( 10, ("__kmp_launch_monitor: #4 finished\n" ) );
842 
843  if( __kmp_global.g.g_abort != 0 ) {
844  /* now we need to terminate the worker threads */
845  /* the value of t_abort is the signal we caught */
846 
847  int gtid;
848 
849  KA_TRACE( 10, ("__kmp_launch_monitor: #5 terminate sig=%d\n", __kmp_global.g.g_abort ) );
850 
851  /* terminate the OpenMP worker threads */
852  /* TODO this is not valid for sibling threads!!
853  * the uber master might not be 0 anymore.. */
854  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
855  __kmp_terminate_thread( gtid );
856 
857  __kmp_cleanup();
858 
859  KA_TRACE( 10, ("__kmp_launch_monitor: #6 raise sig=%d\n", __kmp_global.g.g_abort ) );
860 
861  if (__kmp_global.g.g_abort > 0)
862  raise( __kmp_global.g.g_abort );
863 
864  }
865 
866  KA_TRACE( 10, ("__kmp_launch_monitor: #7 exit\n" ) );
867 
868  return thr;
869 }
870 #endif // KMP_USE_MONITOR
871 
872 void
873 __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
874 {
875  pthread_t handle;
876  pthread_attr_t thread_attr;
877  int status;
878 
879 
880  th->th.th_info.ds.ds_gtid = gtid;
881 
882 #if KMP_STATS_ENABLED
883  // sets up worker thread stats
884  __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
885 
886  // th->th.th_stats is used to transfer thread specific stats-pointer to __kmp_launch_worker
887  // So when thread is created (goes into __kmp_launch_worker) it will
888  // set it's __thread local pointer to th->th.th_stats
889  if(!KMP_UBER_GTID(gtid)) {
890  th->th.th_stats = __kmp_stats_list->push_back(gtid);
891  } else {
892  // For root threads, the __kmp_stats_thread_ptr is set in __kmp_register_root(), so
893  // set the th->th.th_stats field to it.
894  th->th.th_stats = __kmp_stats_thread_ptr;
895  }
896  __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
897 
898 #endif // KMP_STATS_ENABLED
899 
900  if ( KMP_UBER_GTID(gtid) ) {
901  KA_TRACE( 10, ("__kmp_create_worker: uber thread (%d)\n", gtid ) );
902  th -> th.th_info.ds.ds_thread = pthread_self();
903  __kmp_set_stack_info( gtid, th );
904  __kmp_check_stack_overlap( th );
905  return;
906  }; // if
907 
908  KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) );
909 
910  KMP_MB(); /* Flush all pending memory write invalidates. */
911 
912 #ifdef KMP_THREAD_ATTR
913  status = pthread_attr_init( &thread_attr );
914  if ( status != 0 ) {
915  __kmp_msg(kmp_ms_fatal, KMP_MSG( CantInitThreadAttrs ), KMP_ERR( status ), __kmp_msg_null);
916  }; // if
917  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
918  if ( status != 0 ) {
919  __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerState ), KMP_ERR( status ), __kmp_msg_null);
920  }; // if
921 
922  /* Set stack size for this thread now.
923  * The multiple of 2 is there because on some machines, requesting an unusual stacksize
924  * causes the thread to have an offset before the dummy alloca() takes place to create the
925  * offset. Since we want the user to have a sufficient stacksize AND support a stack offset, we
926  * alloca() twice the offset so that the upcoming alloca() does not eliminate any premade
927  * offset, and also gives the user the stack space they requested for all threads */
928  stack_size += gtid * __kmp_stkoffset * 2;
929 
930  KA_TRACE( 10, ( "__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
931  "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
932  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size ) );
933 
934 # ifdef _POSIX_THREAD_ATTR_STACKSIZE
935  status = pthread_attr_setstacksize( & thread_attr, stack_size );
936 # ifdef KMP_BACKUP_STKSIZE
937  if ( status != 0 ) {
938  if ( ! __kmp_env_stksize ) {
939  stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
940  __kmp_stksize = KMP_BACKUP_STKSIZE;
941  KA_TRACE( 10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
942  "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
943  "bytes\n",
944  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size )
945  );
946  status = pthread_attr_setstacksize( &thread_attr, stack_size );
947  }; // if
948  }; // if
949 # endif /* KMP_BACKUP_STKSIZE */
950  if ( status != 0 ) {
951  __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
952  KMP_HNT( ChangeWorkerStackSize ), __kmp_msg_null);
953  }; // if
954 # endif /* _POSIX_THREAD_ATTR_STACKSIZE */
955 
956 #endif /* KMP_THREAD_ATTR */
957 
958  status = pthread_create( & handle, & thread_attr, __kmp_launch_worker, (void *) th );
959  if ( status != 0 || ! handle ) { // ??? Why do we check handle??
960 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
961  if ( status == EINVAL ) {
962  __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
963  KMP_HNT( IncreaseWorkerStackSize ), __kmp_msg_null);
964  };
965  if ( status == ENOMEM ) {
966  __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
967  KMP_HNT( DecreaseWorkerStackSize ), __kmp_msg_null);
968  };
969 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
970  if ( status == EAGAIN ) {
971  __kmp_msg(kmp_ms_fatal, KMP_MSG( NoResourcesForWorkerThread ), KMP_ERR( status ),
972  KMP_HNT( Decrease_NUM_THREADS ), __kmp_msg_null);
973  }; // if
974  KMP_SYSFAIL( "pthread_create", status );
975  }; // if
976 
977  th->th.th_info.ds.ds_thread = handle;
978 
979 #ifdef KMP_THREAD_ATTR
980  status = pthread_attr_destroy( & thread_attr );
981  if ( status ) {
982  kmp_msg_t err_code = KMP_ERR( status );
983  __kmp_msg(kmp_ms_warning, KMP_MSG( CantDestroyThreadAttrs ), err_code, __kmp_msg_null);
984  if (__kmp_generate_warnings == kmp_warnings_off) {
985  __kmp_str_free(&err_code.str);
986  }
987  }; // if
988 #endif /* KMP_THREAD_ATTR */
989 
990  KMP_MB(); /* Flush all pending memory write invalidates. */
991 
992  KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) );
993 
994 } // __kmp_create_worker
995 
996 
997 #if KMP_USE_MONITOR
998 void
999 __kmp_create_monitor( kmp_info_t *th )
1000 {
1001  pthread_t handle;
1002  pthread_attr_t thread_attr;
1003  size_t size;
1004  int status;
1005  int auto_adj_size = FALSE;
1006 
1007  if( __kmp_dflt_blocktime == KMP_MAX_BLOCKTIME ) {
1008  // We don't need monitor thread in case of MAX_BLOCKTIME
1009  KA_TRACE( 10, ("__kmp_create_monitor: skipping monitor thread because of MAX blocktime\n" ) );
1010  th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1011  th->th.th_info.ds.ds_gtid = 0;
1012  return;
1013  }
1014  KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) );
1015 
1016  KMP_MB(); /* Flush all pending memory write invalidates. */
1017 
1018  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1019  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1020  #if KMP_REAL_TIME_FIX
1021  TCW_4( __kmp_global.g.g_time.dt.t_value, -1 ); // Will use it for synchronization a bit later.
1022  #else
1023  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1024  #endif // KMP_REAL_TIME_FIX
1025 
1026  #ifdef KMP_THREAD_ATTR
1027  if ( __kmp_monitor_stksize == 0 ) {
1028  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1029  auto_adj_size = TRUE;
1030  }
1031  status = pthread_attr_init( &thread_attr );
1032  if ( status != 0 ) {
1033  __kmp_msg(
1034  kmp_ms_fatal,
1035  KMP_MSG( CantInitThreadAttrs ),
1036  KMP_ERR( status ),
1037  __kmp_msg_null
1038  );
1039  }; // if
1040  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1041  if ( status != 0 ) {
1042  __kmp_msg(
1043  kmp_ms_fatal,
1044  KMP_MSG( CantSetMonitorState ),
1045  KMP_ERR( status ),
1046  __kmp_msg_null
1047  );
1048  }; // if
1049 
1050  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1051  status = pthread_attr_getstacksize( & thread_attr, & size );
1052  KMP_CHECK_SYSFAIL( "pthread_attr_getstacksize", status );
1053  #else
1054  size = __kmp_sys_min_stksize;
1055  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1056  #endif /* KMP_THREAD_ATTR */
1057 
1058  if ( __kmp_monitor_stksize == 0 ) {
1059  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1060  }
1061  if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1062  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1063  }
1064 
1065  KA_TRACE( 10, ( "__kmp_create_monitor: default stacksize = %lu bytes,"
1066  "requested stacksize = %lu bytes\n",
1067  size, __kmp_monitor_stksize ) );
1068 
1069  retry:
1070 
1071  /* Set stack size for this thread now. */
1072 
1073  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1074  KA_TRACE( 10, ( "__kmp_create_monitor: setting stacksize = %lu bytes,",
1075  __kmp_monitor_stksize ) );
1076  status = pthread_attr_setstacksize( & thread_attr, __kmp_monitor_stksize );
1077  if ( status != 0 ) {
1078  if ( auto_adj_size ) {
1079  __kmp_monitor_stksize *= 2;
1080  goto retry;
1081  }
1082  kmp_msg_t err_code = KMP_ERR( status );
1083  __kmp_msg(
1084  kmp_ms_warning, // should this be fatal? BB
1085  KMP_MSG( CantSetMonitorStackSize, (long int) __kmp_monitor_stksize ),
1086  err_code,
1087  KMP_HNT( ChangeMonitorStackSize ),
1088  __kmp_msg_null
1089  );
1090  if (__kmp_generate_warnings == kmp_warnings_off) {
1091  __kmp_str_free(&err_code.str);
1092  }
1093  }; // if
1094  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1095 
1096  status = pthread_create( &handle, & thread_attr, __kmp_launch_monitor, (void *) th );
1097 
1098  if ( status != 0 ) {
1099  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1100  if ( status == EINVAL ) {
1101  if ( auto_adj_size && ( __kmp_monitor_stksize < (size_t)0x40000000 ) ) {
1102  __kmp_monitor_stksize *= 2;
1103  goto retry;
1104  }
1105  __kmp_msg(
1106  kmp_ms_fatal,
1107  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1108  KMP_ERR( status ),
1109  KMP_HNT( IncreaseMonitorStackSize ),
1110  __kmp_msg_null
1111  );
1112  }; // if
1113  if ( status == ENOMEM ) {
1114  __kmp_msg(
1115  kmp_ms_fatal,
1116  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1117  KMP_ERR( status ),
1118  KMP_HNT( DecreaseMonitorStackSize ),
1119  __kmp_msg_null
1120  );
1121  }; // if
1122  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1123  if ( status == EAGAIN ) {
1124  __kmp_msg(
1125  kmp_ms_fatal,
1126  KMP_MSG( NoResourcesForMonitorThread ),
1127  KMP_ERR( status ),
1128  KMP_HNT( DecreaseNumberOfThreadsInUse ),
1129  __kmp_msg_null
1130  );
1131  }; // if
1132  KMP_SYSFAIL( "pthread_create", status );
1133  }; // if
1134 
1135  th->th.th_info.ds.ds_thread = handle;
1136 
1137  #if KMP_REAL_TIME_FIX
1138  // Wait for the monitor thread is really started and set its *priority*.
1139  KMP_DEBUG_ASSERT( sizeof( kmp_uint32 ) == sizeof( __kmp_global.g.g_time.dt.t_value ) );
1140  __kmp_wait_yield_4(
1141  (kmp_uint32 volatile *) & __kmp_global.g.g_time.dt.t_value, -1, & __kmp_neq_4, NULL
1142  );
1143  #endif // KMP_REAL_TIME_FIX
1144 
1145  #ifdef KMP_THREAD_ATTR
1146  status = pthread_attr_destroy( & thread_attr );
1147  if ( status != 0 ) {
1148  kmp_msg_t err_code = KMP_ERR( status );
1149  __kmp_msg(
1150  kmp_ms_warning,
1151  KMP_MSG( CantDestroyThreadAttrs ),
1152  err_code,
1153  __kmp_msg_null
1154  );
1155  if (__kmp_generate_warnings == kmp_warnings_off) {
1156  __kmp_str_free(&err_code.str);
1157  }
1158  }; // if
1159  #endif
1160 
1161  KMP_MB(); /* Flush all pending memory write invalidates. */
1162 
1163  KA_TRACE( 10, ( "__kmp_create_monitor: monitor created %#.8lx\n", th->th.th_info.ds.ds_thread ) );
1164 
1165 } // __kmp_create_monitor
1166 #endif // KMP_USE_MONITOR
1167 
1168 void
1169 __kmp_exit_thread(
1170  int exit_status
1171 ) {
1172  pthread_exit( (void *)(intptr_t) exit_status );
1173 } // __kmp_exit_thread
1174 
1175 #if KMP_USE_MONITOR
1176 void __kmp_resume_monitor();
1177 
1178 void
1179 __kmp_reap_monitor( kmp_info_t *th )
1180 {
1181  int status;
1182  void *exit_val;
1183 
1184  KA_TRACE( 10, ("__kmp_reap_monitor: try to reap monitor thread with handle %#.8lx\n",
1185  th->th.th_info.ds.ds_thread ) );
1186 
1187  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1188  // If both tid and gtid are 0, it means the monitor did not ever start.
1189  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1190  KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1191  if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1192  KA_TRACE( 10, ("__kmp_reap_monitor: monitor did not start, returning\n") );
1193  return;
1194  }; // if
1195 
1196  KMP_MB(); /* Flush all pending memory write invalidates. */
1197 
1198 
1199  /* First, check to see whether the monitor thread exists to wake it up. This is
1200  to avoid performance problem when the monitor sleeps during blocktime-size
1201  interval */
1202 
1203  status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1204  if (status != ESRCH) {
1205  __kmp_resume_monitor(); // Wake up the monitor thread
1206  }
1207  KA_TRACE( 10, ("__kmp_reap_monitor: try to join with monitor\n") );
1208  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1209  if (exit_val != th) {
1210  __kmp_msg(
1211  kmp_ms_fatal,
1212  KMP_MSG( ReapMonitorError ),
1213  KMP_ERR( status ),
1214  __kmp_msg_null
1215  );
1216  }
1217 
1218  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1219  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1220 
1221  KA_TRACE( 10, ("__kmp_reap_monitor: done reaping monitor thread with handle %#.8lx\n",
1222  th->th.th_info.ds.ds_thread ) );
1223 
1224  KMP_MB(); /* Flush all pending memory write invalidates. */
1225 
1226 }
1227 #endif // KMP_USE_MONITOR
1228 
1229 void
1230 __kmp_reap_worker( kmp_info_t *th )
1231 {
1232  int status;
1233  void *exit_val;
1234 
1235  KMP_MB(); /* Flush all pending memory write invalidates. */
1236 
1237  KA_TRACE( 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid ) );
1238 
1239  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1240 #ifdef KMP_DEBUG
1241  /* Don't expose these to the user until we understand when they trigger */
1242  if ( status != 0 ) {
1243  __kmp_msg(kmp_ms_fatal, KMP_MSG( ReapWorkerError ), KMP_ERR( status ), __kmp_msg_null);
1244  }
1245  if ( exit_val != th ) {
1246  KA_TRACE( 10, ( "__kmp_reap_worker: worker T#%d did not reap properly, exit_val = %p\n",
1247  th->th.th_info.ds.ds_gtid, exit_val ) );
1248  }
1249 #endif /* KMP_DEBUG */
1250 
1251  KA_TRACE( 10, ("__kmp_reap_worker: done reaping T#%d\n", th->th.th_info.ds.ds_gtid ) );
1252 
1253  KMP_MB(); /* Flush all pending memory write invalidates. */
1254 }
1255 
1256 
1257 /* ------------------------------------------------------------------------ */
1258 /* ------------------------------------------------------------------------ */
1259 
1260 #if KMP_HANDLE_SIGNALS
1261 
1262 
1263 static void
1264 __kmp_null_handler( int signo )
1265 {
1266  // Do nothing, for doing SIG_IGN-type actions.
1267 } // __kmp_null_handler
1268 
1269 
1270 static void
1271 __kmp_team_handler( int signo )
1272 {
1273  if ( __kmp_global.g.g_abort == 0 ) {
1274  /* Stage 1 signal handler, let's shut down all of the threads */
1275  #ifdef KMP_DEBUG
1276  __kmp_debug_printf( "__kmp_team_handler: caught signal = %d\n", signo );
1277  #endif
1278  switch ( signo ) {
1279  case SIGHUP :
1280  case SIGINT :
1281  case SIGQUIT :
1282  case SIGILL :
1283  case SIGABRT :
1284  case SIGFPE :
1285  case SIGBUS :
1286  case SIGSEGV :
1287  #ifdef SIGSYS
1288  case SIGSYS :
1289  #endif
1290  case SIGTERM :
1291  if ( __kmp_debug_buf ) {
1292  __kmp_dump_debug_buffer( );
1293  }; // if
1294  KMP_MB(); // Flush all pending memory write invalidates.
1295  TCW_4( __kmp_global.g.g_abort, signo );
1296  KMP_MB(); // Flush all pending memory write invalidates.
1297  TCW_4( __kmp_global.g.g_done, TRUE );
1298  KMP_MB(); // Flush all pending memory write invalidates.
1299  break;
1300  default:
1301  #ifdef KMP_DEBUG
1302  __kmp_debug_printf( "__kmp_team_handler: unknown signal type" );
1303  #endif
1304  break;
1305  }; // switch
1306  }; // if
1307 } // __kmp_team_handler
1308 
1309 
1310 static
1311 void __kmp_sigaction( int signum, const struct sigaction * act, struct sigaction * oldact ) {
1312  int rc = sigaction( signum, act, oldact );
1313  KMP_CHECK_SYSFAIL_ERRNO( "sigaction", rc );
1314 }
1315 
1316 
1317 static void
1318 __kmp_install_one_handler( int sig, sig_func_t handler_func, int parallel_init )
1319 {
1320  KMP_MB(); // Flush all pending memory write invalidates.
1321  KB_TRACE( 60, ( "__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init ) );
1322  if ( parallel_init ) {
1323  struct sigaction new_action;
1324  struct sigaction old_action;
1325  new_action.sa_handler = handler_func;
1326  new_action.sa_flags = 0;
1327  sigfillset( & new_action.sa_mask );
1328  __kmp_sigaction( sig, & new_action, & old_action );
1329  if ( old_action.sa_handler == __kmp_sighldrs[ sig ].sa_handler ) {
1330  sigaddset( & __kmp_sigset, sig );
1331  } else {
1332  // Restore/keep user's handler if one previously installed.
1333  __kmp_sigaction( sig, & old_action, NULL );
1334  }; // if
1335  } else {
1336  // Save initial/system signal handlers to see if user handlers installed.
1337  __kmp_sigaction( sig, NULL, & __kmp_sighldrs[ sig ] );
1338  }; // if
1339  KMP_MB(); // Flush all pending memory write invalidates.
1340 } // __kmp_install_one_handler
1341 
1342 
1343 static void
1344 __kmp_remove_one_handler( int sig )
1345 {
1346  KB_TRACE( 60, ( "__kmp_remove_one_handler( %d )\n", sig ) );
1347  if ( sigismember( & __kmp_sigset, sig ) ) {
1348  struct sigaction old;
1349  KMP_MB(); // Flush all pending memory write invalidates.
1350  __kmp_sigaction( sig, & __kmp_sighldrs[ sig ], & old );
1351  if ( ( old.sa_handler != __kmp_team_handler ) && ( old.sa_handler != __kmp_null_handler ) ) {
1352  // Restore the users signal handler.
1353  KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1354  __kmp_sigaction( sig, & old, NULL );
1355  }; // if
1356  sigdelset( & __kmp_sigset, sig );
1357  KMP_MB(); // Flush all pending memory write invalidates.
1358  }; // if
1359 } // __kmp_remove_one_handler
1360 
1361 
1362 void
1363 __kmp_install_signals( int parallel_init )
1364 {
1365  KB_TRACE( 10, ( "__kmp_install_signals( %d )\n", parallel_init ) );
1366  if ( __kmp_handle_signals || ! parallel_init ) {
1367  // If ! parallel_init, we do not install handlers, just save original handlers.
1368  // Let us do it even __handle_signals is 0.
1369  sigemptyset( & __kmp_sigset );
1370  __kmp_install_one_handler( SIGHUP, __kmp_team_handler, parallel_init );
1371  __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1372  __kmp_install_one_handler( SIGQUIT, __kmp_team_handler, parallel_init );
1373  __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1374  __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1375  __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1376  __kmp_install_one_handler( SIGBUS, __kmp_team_handler, parallel_init );
1377  __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1378  #ifdef SIGSYS
1379  __kmp_install_one_handler( SIGSYS, __kmp_team_handler, parallel_init );
1380  #endif // SIGSYS
1381  __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1382  #ifdef SIGPIPE
1383  __kmp_install_one_handler( SIGPIPE, __kmp_team_handler, parallel_init );
1384  #endif // SIGPIPE
1385  }; // if
1386 } // __kmp_install_signals
1387 
1388 
1389 void
1390 __kmp_remove_signals( void )
1391 {
1392  int sig;
1393  KB_TRACE( 10, ( "__kmp_remove_signals()\n" ) );
1394  for ( sig = 1; sig < NSIG; ++ sig ) {
1395  __kmp_remove_one_handler( sig );
1396  }; // for sig
1397 } // __kmp_remove_signals
1398 
1399 
1400 #endif // KMP_HANDLE_SIGNALS
1401 
1402 /* ------------------------------------------------------------------------ */
1403 /* ------------------------------------------------------------------------ */
1404 
1405 void
1406 __kmp_enable( int new_state )
1407 {
1408  #ifdef KMP_CANCEL_THREADS
1409  int status, old_state;
1410  status = pthread_setcancelstate( new_state, & old_state );
1411  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1412  KMP_DEBUG_ASSERT( old_state == PTHREAD_CANCEL_DISABLE );
1413  #endif
1414 }
1415 
1416 void
1417 __kmp_disable( int * old_state )
1418 {
1419  #ifdef KMP_CANCEL_THREADS
1420  int status;
1421  status = pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, old_state );
1422  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1423  #endif
1424 }
1425 
1426 /* ------------------------------------------------------------------------ */
1427 /* ------------------------------------------------------------------------ */
1428 
1429 static void
1430 __kmp_atfork_prepare (void)
1431 {
1432  /* nothing to do */
1433 }
1434 
1435 static void
1436 __kmp_atfork_parent (void)
1437 {
1438  /* nothing to do */
1439 }
1440 
1441 /*
1442  Reset the library so execution in the child starts "all over again" with
1443  clean data structures in initial states. Don't worry about freeing memory
1444  allocated by parent, just abandon it to be safe.
1445 */
1446 static void
1447 __kmp_atfork_child (void)
1448 {
1449  /* TODO make sure this is done right for nested/sibling */
1450  // ATT: Memory leaks are here? TODO: Check it and fix.
1451  /* KMP_ASSERT( 0 ); */
1452 
1453  ++__kmp_fork_count;
1454 
1455  __kmp_init_runtime = FALSE;
1456 #if KMP_USE_MONITOR
1457  __kmp_init_monitor = 0;
1458 #endif
1459  __kmp_init_parallel = FALSE;
1460  __kmp_init_middle = FALSE;
1461  __kmp_init_serial = FALSE;
1462  TCW_4(__kmp_init_gtid, FALSE);
1463  __kmp_init_common = FALSE;
1464 
1465  TCW_4(__kmp_init_user_locks, FALSE);
1466 #if ! KMP_USE_DYNAMIC_LOCK
1467  __kmp_user_lock_table.used = 1;
1468  __kmp_user_lock_table.allocated = 0;
1469  __kmp_user_lock_table.table = NULL;
1470  __kmp_lock_blocks = NULL;
1471 #endif
1472 
1473  __kmp_all_nth = 0;
1474  TCW_4(__kmp_nth, 0);
1475 
1476  /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate here
1477  so threadprivate doesn't use stale data */
1478  KA_TRACE( 10, ( "__kmp_atfork_child: checking cache address list %p\n",
1479  __kmp_threadpriv_cache_list ) );
1480 
1481  while ( __kmp_threadpriv_cache_list != NULL ) {
1482 
1483  if ( *__kmp_threadpriv_cache_list -> addr != NULL ) {
1484  KC_TRACE( 50, ( "__kmp_atfork_child: zeroing cache at address %p\n",
1485  &(*__kmp_threadpriv_cache_list -> addr) ) );
1486 
1487  *__kmp_threadpriv_cache_list -> addr = NULL;
1488  }
1489  __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list -> next;
1490  }
1491 
1492  __kmp_init_runtime = FALSE;
1493 
1494  /* reset statically initialized locks */
1495  __kmp_init_bootstrap_lock( &__kmp_initz_lock );
1496  __kmp_init_bootstrap_lock( &__kmp_stdio_lock );
1497  __kmp_init_bootstrap_lock( &__kmp_console_lock );
1498 
1499  /* This is necessary to make sure no stale data is left around */
1500  /* AC: customers complain that we use unsafe routines in the atfork
1501  handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1502  in dynamic_link when check the presence of shared tbbmalloc library.
1503  Suggestion is to make the library initialization lazier, similar
1504  to what done for __kmpc_begin(). */
1505  // TODO: synchronize all static initializations with regular library
1506  // startup; look at kmp_global.cpp and etc.
1507  //__kmp_internal_begin ();
1508 
1509 }
1510 
1511 void
1512 __kmp_register_atfork(void) {
1513  if ( __kmp_need_register_atfork ) {
1514  int status = pthread_atfork( __kmp_atfork_prepare, __kmp_atfork_parent, __kmp_atfork_child );
1515  KMP_CHECK_SYSFAIL( "pthread_atfork", status );
1516  __kmp_need_register_atfork = FALSE;
1517  }
1518 }
1519 
1520 void
1521 __kmp_suspend_initialize( void )
1522 {
1523  int status;
1524  status = pthread_mutexattr_init( &__kmp_suspend_mutex_attr );
1525  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
1526  status = pthread_condattr_init( &__kmp_suspend_cond_attr );
1527  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
1528 }
1529 
1530 static void
1531 __kmp_suspend_initialize_thread( kmp_info_t *th )
1532 {
1533  ANNOTATE_HAPPENS_AFTER(&th->th.th_suspend_init_count);
1534  if ( th->th.th_suspend_init_count <= __kmp_fork_count ) {
1535  /* this means we haven't initialized the suspension pthread objects for this thread
1536  in this instance of the process */
1537  int status;
1538  status = pthread_cond_init( &th->th.th_suspend_cv.c_cond, &__kmp_suspend_cond_attr );
1539  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
1540  status = pthread_mutex_init( &th->th.th_suspend_mx.m_mutex, & __kmp_suspend_mutex_attr );
1541  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
1542  *(volatile int*)&th->th.th_suspend_init_count = __kmp_fork_count + 1;
1543  ANNOTATE_HAPPENS_BEFORE(&th->th.th_suspend_init_count);
1544  };
1545 }
1546 
1547 void
1548 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
1549 {
1550  if(th->th.th_suspend_init_count > __kmp_fork_count) {
1551  /* this means we have initialize the suspension pthread objects for this thread
1552  in this instance of the process */
1553  int status;
1554 
1555  status = pthread_cond_destroy( &th->th.th_suspend_cv.c_cond );
1556  if ( status != 0 && status != EBUSY ) {
1557  KMP_SYSFAIL( "pthread_cond_destroy", status );
1558  };
1559  status = pthread_mutex_destroy( &th->th.th_suspend_mx.m_mutex );
1560  if ( status != 0 && status != EBUSY ) {
1561  KMP_SYSFAIL( "pthread_mutex_destroy", status );
1562  };
1563  --th->th.th_suspend_init_count;
1564  KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
1565  }
1566 }
1567 
1568 /* This routine puts the calling thread to sleep after setting the
1569  * sleep bit for the indicated flag variable to true.
1570  */
1571 template <class C>
1572 static inline void __kmp_suspend_template( int th_gtid, C *flag )
1573 {
1574  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
1575  kmp_info_t *th = __kmp_threads[th_gtid];
1576  int status;
1577  typename C::flag_t old_spin;
1578 
1579  KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, flag->get() ) );
1580 
1581  __kmp_suspend_initialize_thread( th );
1582 
1583  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1584  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1585 
1586  KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1587  th_gtid, flag->get() ) );
1588 
1589  /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread
1590  gets called first?
1591  */
1592  old_spin = flag->set_sleeping();
1593 
1594  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x, was %x\n",
1595  th_gtid, flag->get(), *(flag->get()), old_spin ) );
1596 
1597  if ( flag->done_check_val(old_spin) ) {
1598  old_spin = flag->unset_sleeping();
1599  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for spin(%p)\n",
1600  th_gtid, flag->get()) );
1601  } else {
1602  /* Encapsulate in a loop as the documentation states that this may
1603  * "with low probability" return when the condition variable has
1604  * not been signaled or broadcast
1605  */
1606  int deactivated = FALSE;
1607  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1608  while ( flag->is_sleeping() ) {
1609 #ifdef DEBUG_SUSPEND
1610  char buffer[128];
1611  __kmp_suspend_count++;
1612  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1613  __kmp_printf( "__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, buffer );
1614 #endif
1615  // Mark the thread as no longer active (only in the first iteration of the loop).
1616  if ( ! deactivated ) {
1617  th->th.th_active = FALSE;
1618  if ( th->th.th_active_in_pool ) {
1619  th->th.th_active_in_pool = FALSE;
1620  KMP_TEST_THEN_DEC32(
1621  (kmp_int32 *) &__kmp_thread_pool_active_nth );
1622  KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
1623  }
1624  deactivated = TRUE;
1625  }
1626 
1627 #if USE_SUSPEND_TIMEOUT
1628  struct timespec now;
1629  struct timeval tval;
1630  int msecs;
1631 
1632  status = gettimeofday( &tval, NULL );
1633  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
1634  TIMEVAL_TO_TIMESPEC( &tval, &now );
1635 
1636  msecs = (4*__kmp_dflt_blocktime) + 200;
1637  now.tv_sec += msecs / 1000;
1638  now.tv_nsec += (msecs % 1000)*1000;
1639 
1640  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_timedwait\n",
1641  th_gtid ) );
1642  status = pthread_cond_timedwait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex, & now );
1643 #else
1644  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_wait\n",
1645  th_gtid ) );
1646  status = pthread_cond_wait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex );
1647 #endif
1648 
1649  if ( (status != 0) && (status != EINTR) && (status != ETIMEDOUT) ) {
1650  KMP_SYSFAIL( "pthread_cond_wait", status );
1651  }
1652 #ifdef KMP_DEBUG
1653  if (status == ETIMEDOUT) {
1654  if ( flag->is_sleeping() ) {
1655  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid ) );
1656  } else {
1657  KF_TRACE( 2, ( "__kmp_suspend_template: T#%d timeout wakeup, sleep bit not set!\n",
1658  th_gtid ) );
1659  }
1660  } else if ( flag->is_sleeping() ) {
1661  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ) );
1662  }
1663 #endif
1664  } // while
1665 
1666  // Mark the thread as active again (if it was previous marked as inactive)
1667  if ( deactivated ) {
1668  th->th.th_active = TRUE;
1669  if ( TCR_4(th->th.th_in_pool) ) {
1670  KMP_TEST_THEN_INC32( (kmp_int32 *) &__kmp_thread_pool_active_nth );
1671  th->th.th_active_in_pool = TRUE;
1672  }
1673  }
1674  }
1675 
1676 #ifdef DEBUG_SUSPEND
1677  {
1678  char buffer[128];
1679  __kmp_print_cond( buffer, &th->th.th_suspend_cv);
1680  __kmp_printf( "__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, buffer );
1681  }
1682 #endif
1683 
1684  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1685  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1686 
1687  KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) );
1688 }
1689 
1690 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
1691  __kmp_suspend_template(th_gtid, flag);
1692 }
1693 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
1694  __kmp_suspend_template(th_gtid, flag);
1695 }
1696 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1697  __kmp_suspend_template(th_gtid, flag);
1698 }
1699 
1700 
1701 /* This routine signals the thread specified by target_gtid to wake up
1702  * after setting the sleep bit indicated by the flag argument to FALSE.
1703  * The target thread must already have called __kmp_suspend_template()
1704  */
1705 template <class C>
1706 static inline void __kmp_resume_template( int target_gtid, C *flag )
1707 {
1708  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1709  kmp_info_t *th = __kmp_threads[target_gtid];
1710  int status;
1711 
1712 #ifdef KMP_DEBUG
1713  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1714 #endif
1715 
1716  KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
1717  KMP_DEBUG_ASSERT( gtid != target_gtid );
1718 
1719  __kmp_suspend_initialize_thread( th );
1720 
1721  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1722  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1723 
1724  if (!flag) { // coming from __kmp_null_resume_wrapper
1725  flag = (C *)th->th.th_sleep_loc;
1726  }
1727 
1728  // First, check if the flag is null or its type has changed. If so, someone else woke it up.
1729  if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type simply shows what flag was cast to
1730  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p)\n",
1731  gtid, target_gtid, NULL ) );
1732  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1733  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1734  return;
1735  }
1736  else { // if multiple threads are sleeping, flag should be internally referring to a specific thread here
1737  typename C::flag_t old_spin = flag->unset_sleeping();
1738  if ( ! flag->is_sleeping_val(old_spin) ) {
1739  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p): "
1740  "%u => %u\n",
1741  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1742  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1743  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1744  return;
1745  }
1746  KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p): "
1747  "%u => %u\n",
1748  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1749  }
1750  TCW_PTR(th->th.th_sleep_loc, NULL);
1751 
1752 
1753 #ifdef DEBUG_SUSPEND
1754  {
1755  char buffer[128];
1756  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1757  __kmp_printf( "__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, target_gtid, buffer );
1758  }
1759 #endif
1760 
1761  status = pthread_cond_signal( &th->th.th_suspend_cv.c_cond );
1762  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1763  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1764  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1765  KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
1766  gtid, target_gtid ) );
1767 }
1768 
1769 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
1770  __kmp_resume_template(target_gtid, flag);
1771 }
1772 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
1773  __kmp_resume_template(target_gtid, flag);
1774 }
1775 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1776  __kmp_resume_template(target_gtid, flag);
1777 }
1778 
1779 #if KMP_USE_MONITOR
1780 void
1781 __kmp_resume_monitor()
1782 {
1783  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1784  int status;
1785 #ifdef KMP_DEBUG
1786  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1787  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n",
1788  gtid, KMP_GTID_MONITOR ) );
1789  KMP_DEBUG_ASSERT( gtid != KMP_GTID_MONITOR );
1790 #endif
1791  status = pthread_mutex_lock( &__kmp_wait_mx.m_mutex );
1792  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1793 #ifdef DEBUG_SUSPEND
1794  {
1795  char buffer[128];
1796  __kmp_print_cond( buffer, &__kmp_wait_cv.c_cond );
1797  __kmp_printf( "__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, KMP_GTID_MONITOR, buffer );
1798  }
1799 #endif
1800  status = pthread_cond_signal( &__kmp_wait_cv.c_cond );
1801  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1802  status = pthread_mutex_unlock( &__kmp_wait_mx.m_mutex );
1803  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1804  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d exiting after signaling wake up for T#%d\n",
1805  gtid, KMP_GTID_MONITOR ) );
1806 }
1807 #endif // KMP_USE_MONITOR
1808 
1809 /* ------------------------------------------------------------------------ */
1810 /* ------------------------------------------------------------------------ */
1811 
1812 void
1813 __kmp_yield( int cond )
1814 {
1815  if (cond
1816 #if KMP_USE_MONITOR
1817  && __kmp_yielding_on
1818 #endif
1819  ) {
1820  sched_yield();
1821  }
1822 }
1823 
1824 /* ------------------------------------------------------------------------ */
1825 /* ------------------------------------------------------------------------ */
1826 
1827 void
1828 __kmp_gtid_set_specific( int gtid )
1829 {
1830  if( __kmp_init_gtid ) {
1831  int status;
1832  status = pthread_setspecific( __kmp_gtid_threadprivate_key, (void*)(intptr_t)(gtid+1) );
1833  KMP_CHECK_SYSFAIL( "pthread_setspecific", status );
1834  } else {
1835  KA_TRACE( 50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n" ) );
1836  }
1837 }
1838 
1839 int
1840 __kmp_gtid_get_specific()
1841 {
1842  int gtid;
1843  if ( !__kmp_init_gtid ) {
1844  KA_TRACE( 50, ("__kmp_gtid_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
1845  return KMP_GTID_SHUTDOWN;
1846  }
1847  gtid = (int)(size_t)pthread_getspecific( __kmp_gtid_threadprivate_key );
1848  if ( gtid == 0 ) {
1849  gtid = KMP_GTID_DNE;
1850  }
1851  else {
1852  gtid--;
1853  }
1854  KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1855  __kmp_gtid_threadprivate_key, gtid ));
1856  return gtid;
1857 }
1858 
1859 /* ------------------------------------------------------------------------ */
1860 /* ------------------------------------------------------------------------ */
1861 
1862 double
1863 __kmp_read_cpu_time( void )
1864 {
1865  /*clock_t t;*/
1866  struct tms buffer;
1867 
1868  /*t =*/ times( & buffer );
1869 
1870  return (buffer.tms_utime + buffer.tms_cutime) / (double) CLOCKS_PER_SEC;
1871 }
1872 
1873 int
1874 __kmp_read_system_info( struct kmp_sys_info *info )
1875 {
1876  int status;
1877  struct rusage r_usage;
1878 
1879  memset( info, 0, sizeof( *info ) );
1880 
1881  status = getrusage( RUSAGE_SELF, &r_usage);
1882  KMP_CHECK_SYSFAIL_ERRNO( "getrusage", status );
1883 
1884  info->maxrss = r_usage.ru_maxrss; /* the maximum resident set size utilized (in kilobytes) */
1885  info->minflt = r_usage.ru_minflt; /* the number of page faults serviced without any I/O */
1886  info->majflt = r_usage.ru_majflt; /* the number of page faults serviced that required I/O */
1887  info->nswap = r_usage.ru_nswap; /* the number of times a process was "swapped" out of memory */
1888  info->inblock = r_usage.ru_inblock; /* the number of times the file system had to perform input */
1889  info->oublock = r_usage.ru_oublock; /* the number of times the file system had to perform output */
1890  info->nvcsw = r_usage.ru_nvcsw; /* the number of times a context switch was voluntarily */
1891  info->nivcsw = r_usage.ru_nivcsw; /* the number of times a context switch was forced */
1892 
1893  return (status != 0);
1894 }
1895 
1896 /* ------------------------------------------------------------------------ */
1897 /* ------------------------------------------------------------------------ */
1898 
1899 void
1900 __kmp_read_system_time( double *delta )
1901 {
1902  double t_ns;
1903  struct timeval tval;
1904  struct timespec stop;
1905  int status;
1906 
1907  status = gettimeofday( &tval, NULL );
1908  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
1909  TIMEVAL_TO_TIMESPEC( &tval, &stop );
1910  t_ns = TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start);
1911  *delta = (t_ns * 1e-9);
1912 }
1913 
1914 void
1915 __kmp_clear_system_time( void )
1916 {
1917  struct timeval tval;
1918  int status;
1919  status = gettimeofday( &tval, NULL );
1920  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
1921  TIMEVAL_TO_TIMESPEC( &tval, &__kmp_sys_timer_data.start );
1922 }
1923 
1924 /* ------------------------------------------------------------------------ */
1925 /* ------------------------------------------------------------------------ */
1926 
1927 #ifdef BUILD_TV
1928 
1929 void
1930 __kmp_tv_threadprivate_store( kmp_info_t *th, void *global_addr, void *thread_addr )
1931 {
1932  struct tv_data *p;
1933 
1934  p = (struct tv_data *) __kmp_allocate( sizeof( *p ) );
1935 
1936  p->u.tp.global_addr = global_addr;
1937  p->u.tp.thread_addr = thread_addr;
1938 
1939  p->type = (void *) 1;
1940 
1941  p->next = th->th.th_local.tv_data;
1942  th->th.th_local.tv_data = p;
1943 
1944  if ( p->next == 0 ) {
1945  int rc = pthread_setspecific( __kmp_tv_key, p );
1946  KMP_CHECK_SYSFAIL( "pthread_setspecific", rc );
1947  }
1948 }
1949 
1950 #endif /* BUILD_TV */
1951 
1952 /* ------------------------------------------------------------------------ */
1953 /* ------------------------------------------------------------------------ */
1954 
1955 static int
1956 __kmp_get_xproc( void ) {
1957 
1958  int r = 0;
1959 
1960  #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD
1961 
1962  r = sysconf( _SC_NPROCESSORS_ONLN );
1963 
1964  #elif KMP_OS_DARWIN
1965 
1966  // Bug C77011 High "OpenMP Threads and number of active cores".
1967 
1968  // Find the number of available CPUs.
1969  kern_return_t rc;
1970  host_basic_info_data_t info;
1971  mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT;
1972  rc = host_info( mach_host_self(), HOST_BASIC_INFO, (host_info_t) & info, & num );
1973  if ( rc == 0 && num == HOST_BASIC_INFO_COUNT ) {
1974  // Cannot use KA_TRACE() here because this code works before trace support is
1975  // initialized.
1976  r = info.avail_cpus;
1977  } else {
1978  KMP_WARNING( CantGetNumAvailCPU );
1979  KMP_INFORM( AssumedNumCPU );
1980  }; // if
1981 
1982  #else
1983 
1984  #error "Unknown or unsupported OS."
1985 
1986  #endif
1987 
1988  return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
1989 
1990 } // __kmp_get_xproc
1991 
1992 int
1993 __kmp_read_from_file( char const *path, char const *format, ... )
1994 {
1995  int result;
1996  va_list args;
1997 
1998  va_start(args, format);
1999  FILE *f = fopen(path, "rb");
2000  if ( f == NULL )
2001  return 0;
2002  result = vfscanf(f, format, args);
2003  fclose(f);
2004 
2005  return result;
2006 }
2007 
2008 void
2009 __kmp_runtime_initialize( void )
2010 {
2011  int status;
2012  pthread_mutexattr_t mutex_attr;
2013  pthread_condattr_t cond_attr;
2014 
2015  if ( __kmp_init_runtime ) {
2016  return;
2017  }; // if
2018 
2019  #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
2020  if ( ! __kmp_cpuinfo.initialized ) {
2021  __kmp_query_cpuid( &__kmp_cpuinfo );
2022  }; // if
2023  #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
2024 
2025  __kmp_xproc = __kmp_get_xproc();
2026 
2027  if ( sysconf( _SC_THREADS ) ) {
2028 
2029  /* Query the maximum number of threads */
2030  __kmp_sys_max_nth = sysconf( _SC_THREAD_THREADS_MAX );
2031  if ( __kmp_sys_max_nth == -1 ) {
2032  /* Unlimited threads for NPTL */
2033  __kmp_sys_max_nth = INT_MAX;
2034  }
2035  else if ( __kmp_sys_max_nth <= 1 ) {
2036  /* Can't tell, just use PTHREAD_THREADS_MAX */
2037  __kmp_sys_max_nth = KMP_MAX_NTH;
2038  }
2039 
2040  /* Query the minimum stack size */
2041  __kmp_sys_min_stksize = sysconf( _SC_THREAD_STACK_MIN );
2042  if ( __kmp_sys_min_stksize <= 1 ) {
2043  __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
2044  }
2045  }
2046 
2047  /* Set up minimum number of threads to switch to TLS gtid */
2048  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2049 
2050  #ifdef BUILD_TV
2051  {
2052  int rc = pthread_key_create( & __kmp_tv_key, 0 );
2053  KMP_CHECK_SYSFAIL( "pthread_key_create", rc );
2054  }
2055  #endif
2056 
2057  status = pthread_key_create( &__kmp_gtid_threadprivate_key, __kmp_internal_end_dest );
2058  KMP_CHECK_SYSFAIL( "pthread_key_create", status );
2059  status = pthread_mutexattr_init( & mutex_attr );
2060  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
2061  status = pthread_mutex_init( & __kmp_wait_mx.m_mutex, & mutex_attr );
2062  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
2063  status = pthread_condattr_init( & cond_attr );
2064  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
2065  status = pthread_cond_init( & __kmp_wait_cv.c_cond, & cond_attr );
2066  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
2067 #if USE_ITT_BUILD
2068  __kmp_itt_initialize();
2069 #endif /* USE_ITT_BUILD */
2070 
2071  __kmp_init_runtime = TRUE;
2072 }
2073 
2074 void
2075 __kmp_runtime_destroy( void )
2076 {
2077  int status;
2078 
2079  if ( ! __kmp_init_runtime ) {
2080  return; // Nothing to do.
2081  };
2082 
2083 #if USE_ITT_BUILD
2084  __kmp_itt_destroy();
2085 #endif /* USE_ITT_BUILD */
2086 
2087  status = pthread_key_delete( __kmp_gtid_threadprivate_key );
2088  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2089  #ifdef BUILD_TV
2090  status = pthread_key_delete( __kmp_tv_key );
2091  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2092  #endif
2093 
2094  status = pthread_mutex_destroy( & __kmp_wait_mx.m_mutex );
2095  if ( status != 0 && status != EBUSY ) {
2096  KMP_SYSFAIL( "pthread_mutex_destroy", status );
2097  }
2098  status = pthread_cond_destroy( & __kmp_wait_cv.c_cond );
2099  if ( status != 0 && status != EBUSY ) {
2100  KMP_SYSFAIL( "pthread_cond_destroy", status );
2101  }
2102  #if KMP_AFFINITY_SUPPORTED
2103  __kmp_affinity_uninitialize();
2104  #endif
2105 
2106  __kmp_init_runtime = FALSE;
2107 }
2108 
2109 
2110 /* Put the thread to sleep for a time period */
2111 /* NOTE: not currently used anywhere */
2112 void
2113 __kmp_thread_sleep( int millis )
2114 {
2115  sleep( ( millis + 500 ) / 1000 );
2116 }
2117 
2118 /* Calculate the elapsed wall clock time for the user */
2119 void
2120 __kmp_elapsed( double *t )
2121 {
2122  int status;
2123 # ifdef FIX_SGI_CLOCK
2124  struct timespec ts;
2125 
2126  status = clock_gettime( CLOCK_PROCESS_CPUTIME_ID, &ts );
2127  KMP_CHECK_SYSFAIL_ERRNO( "clock_gettime", status );
2128  *t = (double) ts.tv_nsec * (1.0 / (double) KMP_NSEC_PER_SEC) +
2129  (double) ts.tv_sec;
2130 # else
2131  struct timeval tv;
2132 
2133  status = gettimeofday( & tv, NULL );
2134  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2135  *t = (double) tv.tv_usec * (1.0 / (double) KMP_USEC_PER_SEC) +
2136  (double) tv.tv_sec;
2137 # endif
2138 }
2139 
2140 /* Calculate the elapsed wall clock tick for the user */
2141 void
2142 __kmp_elapsed_tick( double *t )
2143 {
2144  *t = 1 / (double) CLOCKS_PER_SEC;
2145 }
2146 
2147 /* Return the current time stamp in nsec */
2148 kmp_uint64
2149 __kmp_now_nsec()
2150 {
2151  struct timeval t;
2152  gettimeofday(&t, NULL);
2153  return KMP_NSEC_PER_SEC*t.tv_sec + 1000*t.tv_usec;
2154 }
2155 
2156 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
2157 /* Measure clock tick per nanosecond */
2158 void
2159 __kmp_initialize_system_tick()
2160 {
2161  kmp_uint64 delay = 100000; // 50~100 usec on most machines.
2162  kmp_uint64 nsec = __kmp_now_nsec();
2163  kmp_uint64 goal = __kmp_hardware_timestamp() + delay;
2164  kmp_uint64 now;
2165  while ((now = __kmp_hardware_timestamp()) < goal);
2166  __kmp_ticks_per_nsec = 1.0 * (delay + (now - goal)) / (__kmp_now_nsec() - nsec);
2167 }
2168 #endif
2169 
2170 /*
2171  Determine whether the given address is mapped into the current address space.
2172 */
2173 
2174 int
2175 __kmp_is_address_mapped( void * addr ) {
2176 
2177  int found = 0;
2178  int rc;
2179 
2180  #if KMP_OS_LINUX || KMP_OS_FREEBSD
2181 
2182  /*
2183  On Linux* OS, read the /proc/<pid>/maps pseudo-file to get all the address ranges mapped
2184  into the address space.
2185  */
2186 
2187  char * name = __kmp_str_format( "/proc/%d/maps", getpid() );
2188  FILE * file = NULL;
2189 
2190  file = fopen( name, "r" );
2191  KMP_ASSERT( file != NULL );
2192 
2193  for ( ; ; ) {
2194 
2195  void * beginning = NULL;
2196  void * ending = NULL;
2197  char perms[ 5 ];
2198 
2199  rc = fscanf( file, "%p-%p %4s %*[^\n]\n", & beginning, & ending, perms );
2200  if ( rc == EOF ) {
2201  break;
2202  }; // if
2203  KMP_ASSERT( rc == 3 && KMP_STRLEN( perms ) == 4 ); // Make sure all fields are read.
2204 
2205  // Ending address is not included in the region, but beginning is.
2206  if ( ( addr >= beginning ) && ( addr < ending ) ) {
2207  perms[ 2 ] = 0; // 3th and 4th character does not matter.
2208  if ( strcmp( perms, "rw" ) == 0 ) {
2209  // Memory we are looking for should be readable and writable.
2210  found = 1;
2211  }; // if
2212  break;
2213  }; // if
2214 
2215  }; // forever
2216 
2217  // Free resources.
2218  fclose( file );
2219  KMP_INTERNAL_FREE( name );
2220 
2221  #elif KMP_OS_DARWIN
2222 
2223  /*
2224  On OS X*, /proc pseudo filesystem is not available. Try to read memory using vm
2225  interface.
2226  */
2227 
2228  int buffer;
2229  vm_size_t count;
2230  rc =
2231  vm_read_overwrite(
2232  mach_task_self(), // Task to read memory of.
2233  (vm_address_t)( addr ), // Address to read from.
2234  1, // Number of bytes to be read.
2235  (vm_address_t)( & buffer ), // Address of buffer to save read bytes in.
2236  & count // Address of var to save number of read bytes in.
2237  );
2238  if ( rc == 0 ) {
2239  // Memory successfully read.
2240  found = 1;
2241  }; // if
2242 
2243  #elif KMP_OS_FREEBSD || KMP_OS_NETBSD
2244 
2245  // FIXME(FreeBSD, NetBSD): Implement this
2246  found = 1;
2247 
2248  #else
2249 
2250  #error "Unknown or unsupported OS"
2251 
2252  #endif
2253 
2254  return found;
2255 
2256 } // __kmp_is_address_mapped
2257 
2258 #ifdef USE_LOAD_BALANCE
2259 
2260 
2261 # if KMP_OS_DARWIN
2262 
2263 // The function returns the rounded value of the system load average
2264 // during given time interval which depends on the value of
2265 // __kmp_load_balance_interval variable (default is 60 sec, other values
2266 // may be 300 sec or 900 sec).
2267 // It returns -1 in case of error.
2268 int
2269 __kmp_get_load_balance( int max )
2270 {
2271  double averages[3];
2272  int ret_avg = 0;
2273 
2274  int res = getloadavg( averages, 3 );
2275 
2276  //Check __kmp_load_balance_interval to determine which of averages to use.
2277  // getloadavg() may return the number of samples less than requested that is
2278  // less than 3.
2279  if ( __kmp_load_balance_interval < 180 && ( res >= 1 ) ) {
2280  ret_avg = averages[0];// 1 min
2281  } else if ( ( __kmp_load_balance_interval >= 180
2282  && __kmp_load_balance_interval < 600 ) && ( res >= 2 ) ) {
2283  ret_avg = averages[1];// 5 min
2284  } else if ( ( __kmp_load_balance_interval >= 600 ) && ( res == 3 ) ) {
2285  ret_avg = averages[2];// 15 min
2286  } else {// Error occurred
2287  return -1;
2288  }
2289 
2290  return ret_avg;
2291 }
2292 
2293 # else // Linux* OS
2294 
2295 // The fuction returns number of running (not sleeping) threads, or -1 in case of error.
2296 // Error could be reported if Linux* OS kernel too old (without "/proc" support).
2297 // Counting running threads stops if max running threads encountered.
2298 int
2299 __kmp_get_load_balance( int max )
2300 {
2301  static int permanent_error = 0;
2302 
2303  static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */
2304  static double glb_call_time = 0; /* Thread balance algorithm call time */
2305 
2306  int running_threads = 0; // Number of running threads in the system.
2307 
2308  DIR * proc_dir = NULL; // Handle of "/proc/" directory.
2309  struct dirent * proc_entry = NULL;
2310 
2311  kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2312  DIR * task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2313  struct dirent * task_entry = NULL;
2314  int task_path_fixed_len;
2315 
2316  kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2317  int stat_file = -1;
2318  int stat_path_fixed_len;
2319 
2320  int total_processes = 0; // Total number of processes in system.
2321  int total_threads = 0; // Total number of threads in system.
2322 
2323  double call_time = 0.0;
2324 
2325  __kmp_str_buf_init( & task_path );
2326  __kmp_str_buf_init( & stat_path );
2327 
2328  __kmp_elapsed( & call_time );
2329 
2330  if ( glb_call_time &&
2331  ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
2332  running_threads = glb_running_threads;
2333  goto finish;
2334  }
2335 
2336  glb_call_time = call_time;
2337 
2338  // Do not spend time on scanning "/proc/" if we have a permanent error.
2339  if ( permanent_error ) {
2340  running_threads = -1;
2341  goto finish;
2342  }; // if
2343 
2344  if ( max <= 0 ) {
2345  max = INT_MAX;
2346  }; // if
2347 
2348  // Open "/proc/" directory.
2349  proc_dir = opendir( "/proc" );
2350  if ( proc_dir == NULL ) {
2351  // Cannot open "/prroc/". Probably the kernel does not support it. Return an error now and
2352  // in subsequent calls.
2353  running_threads = -1;
2354  permanent_error = 1;
2355  goto finish;
2356  }; // if
2357 
2358  // Initialize fixed part of task_path. This part will not change.
2359  __kmp_str_buf_cat( & task_path, "/proc/", 6 );
2360  task_path_fixed_len = task_path.used; // Remember number of used characters.
2361 
2362  proc_entry = readdir( proc_dir );
2363  while ( proc_entry != NULL ) {
2364  // Proc entry is a directory and name starts with a digit. Assume it is a process'
2365  // directory.
2366  if ( proc_entry->d_type == DT_DIR && isdigit( proc_entry->d_name[ 0 ] ) ) {
2367 
2368  ++ total_processes;
2369  // Make sure init process is the very first in "/proc", so we can replace
2370  // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes == 1.
2371  // We are going to check that total_processes == 1 => d_name == "1" is true (where
2372  // "=>" is implication). Since C++ does not have => operator, let us replace it with its
2373  // equivalent: a => b == ! a || b.
2374  KMP_DEBUG_ASSERT( total_processes != 1 || strcmp( proc_entry->d_name, "1" ) == 0 );
2375 
2376  // Construct task_path.
2377  task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2378  __kmp_str_buf_cat( & task_path, proc_entry->d_name, KMP_STRLEN( proc_entry->d_name ) );
2379  __kmp_str_buf_cat( & task_path, "/task", 5 );
2380 
2381  task_dir = opendir( task_path.str );
2382  if ( task_dir == NULL ) {
2383  // Process can finish between reading "/proc/" directory entry and opening process'
2384  // "task/" directory. So, in general case we should not complain, but have to skip
2385  // this process and read the next one.
2386  // But on systems with no "task/" support we will spend lot of time to scan "/proc/"
2387  // tree again and again without any benefit. "init" process (its pid is 1) should
2388  // exist always, so, if we cannot open "/proc/1/task/" directory, it means "task/"
2389  // is not supported by kernel. Report an error now and in the future.
2390  if ( strcmp( proc_entry->d_name, "1" ) == 0 ) {
2391  running_threads = -1;
2392  permanent_error = 1;
2393  goto finish;
2394  }; // if
2395  } else {
2396  // Construct fixed part of stat file path.
2397  __kmp_str_buf_clear( & stat_path );
2398  __kmp_str_buf_cat( & stat_path, task_path.str, task_path.used );
2399  __kmp_str_buf_cat( & stat_path, "/", 1 );
2400  stat_path_fixed_len = stat_path.used;
2401 
2402  task_entry = readdir( task_dir );
2403  while ( task_entry != NULL ) {
2404  // It is a directory and name starts with a digit.
2405  if ( proc_entry->d_type == DT_DIR && isdigit( task_entry->d_name[ 0 ] ) ) {
2406 
2407  ++ total_threads;
2408 
2409  // Consruct complete stat file path. Easiest way would be:
2410  // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str, task_entry->d_name );
2411  // but seriae of __kmp_str_buf_cat works a bit faster.
2412  stat_path.used = stat_path_fixed_len; // Reset stat path to its fixed part.
2413  __kmp_str_buf_cat( & stat_path, task_entry->d_name, KMP_STRLEN( task_entry->d_name ) );
2414  __kmp_str_buf_cat( & stat_path, "/stat", 5 );
2415 
2416  // Note: Low-level API (open/read/close) is used. High-level API
2417  // (fopen/fclose) works ~ 30 % slower.
2418  stat_file = open( stat_path.str, O_RDONLY );
2419  if ( stat_file == -1 ) {
2420  // We cannot report an error because task (thread) can terminate just
2421  // before reading this file.
2422  } else {
2423  /*
2424  Content of "stat" file looks like:
2425 
2426  24285 (program) S ...
2427 
2428  It is a single line (if program name does not include fanny
2429  symbols). First number is a thread id, then name of executable file
2430  name in paretheses, then state of the thread. We need just thread
2431  state.
2432 
2433  Good news: Length of program name is 15 characters max. Longer
2434  names are truncated.
2435 
2436  Thus, we need rather short buffer: 15 chars for program name +
2437  2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2438 
2439  Bad news: Program name may contain special symbols like space,
2440  closing parenthesis, or even new line. This makes parsing "stat"
2441  file not 100 % reliable. In case of fanny program names parsing
2442  may fail (report incorrect thread state).
2443 
2444  Parsing "status" file looks more promissing (due to different
2445  file structure and escaping special symbols) but reading and
2446  parsing of "status" file works slower.
2447 
2448  -- ln
2449  */
2450  char buffer[ 65 ];
2451  int len;
2452  len = read( stat_file, buffer, sizeof( buffer ) - 1 );
2453  if ( len >= 0 ) {
2454  buffer[ len ] = 0;
2455  // Using scanf:
2456  // sscanf( buffer, "%*d (%*s) %c ", & state );
2457  // looks very nice, but searching for a closing parenthesis works a
2458  // bit faster.
2459  char * close_parent = strstr( buffer, ") " );
2460  if ( close_parent != NULL ) {
2461  char state = * ( close_parent + 2 );
2462  if ( state == 'R' ) {
2463  ++ running_threads;
2464  if ( running_threads >= max ) {
2465  goto finish;
2466  }; // if
2467  }; // if
2468  }; // if
2469  }; // if
2470  close( stat_file );
2471  stat_file = -1;
2472  }; // if
2473  }; // if
2474  task_entry = readdir( task_dir );
2475  }; // while
2476  closedir( task_dir );
2477  task_dir = NULL;
2478  }; // if
2479  }; // if
2480  proc_entry = readdir( proc_dir );
2481  }; // while
2482 
2483  //
2484  // There _might_ be a timing hole where the thread executing this
2485  // code get skipped in the load balance, and running_threads is 0.
2486  // Assert in the debug builds only!!!
2487  //
2488  KMP_DEBUG_ASSERT( running_threads > 0 );
2489  if ( running_threads <= 0 ) {
2490  running_threads = 1;
2491  }
2492 
2493  finish: // Clean up and exit.
2494  if ( proc_dir != NULL ) {
2495  closedir( proc_dir );
2496  }; // if
2497  __kmp_str_buf_free( & task_path );
2498  if ( task_dir != NULL ) {
2499  closedir( task_dir );
2500  }; // if
2501  __kmp_str_buf_free( & stat_path );
2502  if ( stat_file != -1 ) {
2503  close( stat_file );
2504  }; // if
2505 
2506  glb_running_threads = running_threads;
2507 
2508  return running_threads;
2509 
2510 } // __kmp_get_load_balance
2511 
2512 # endif // KMP_OS_DARWIN
2513 
2514 #endif // USE_LOAD_BALANCE
2515 
2516 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || (KMP_OS_LINUX && KMP_ARCH_AARCH64) || KMP_ARCH_PPC64)
2517 
2518 // we really only need the case with 1 argument, because CLANG always build
2519 // a struct of pointers to shared variables referenced in the outlined function
2520 int
2521 __kmp_invoke_microtask( microtask_t pkfn,
2522  int gtid, int tid,
2523  int argc, void *p_argv[]
2524 #if OMPT_SUPPORT
2525  , void **exit_frame_ptr
2526 #endif
2527 )
2528 {
2529 #if OMPT_SUPPORT
2530  *exit_frame_ptr = __builtin_frame_address(0);
2531 #endif
2532 
2533  switch (argc) {
2534  default:
2535  fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2536  fflush(stderr);
2537  exit(-1);
2538  case 0:
2539  (*pkfn)(&gtid, &tid);
2540  break;
2541  case 1:
2542  (*pkfn)(&gtid, &tid, p_argv[0]);
2543  break;
2544  case 2:
2545  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2546  break;
2547  case 3:
2548  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2549  break;
2550  case 4:
2551  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]);
2552  break;
2553  case 5:
2554  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]);
2555  break;
2556  case 6:
2557  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2558  p_argv[5]);
2559  break;
2560  case 7:
2561  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2562  p_argv[5], p_argv[6]);
2563  break;
2564  case 8:
2565  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2566  p_argv[5], p_argv[6], p_argv[7]);
2567  break;
2568  case 9:
2569  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2570  p_argv[5], p_argv[6], p_argv[7], p_argv[8]);
2571  break;
2572  case 10:
2573  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2574  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]);
2575  break;
2576  case 11:
2577  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2578  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2579  break;
2580  case 12:
2581  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2582  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2583  p_argv[11]);
2584  break;
2585  case 13:
2586  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2587  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2588  p_argv[11], p_argv[12]);
2589  break;
2590  case 14:
2591  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2592  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2593  p_argv[11], p_argv[12], p_argv[13]);
2594  break;
2595  case 15:
2596  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2597  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2598  p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2599  break;
2600  }
2601 
2602 #if OMPT_SUPPORT
2603  *exit_frame_ptr = 0;
2604 #endif
2605 
2606  return 1;
2607 }
2608 
2609 #endif
2610 
2611 // end of file //
2612 
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
Definition: kmp_stats.h:761
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the paritioned timers to begin with name.
Definition: kmp_stats.h:802