LLVM OpenMP* Runtime Library
z_Windows_NT_util.c
1 /*
2  * z_Windows_NT_util.c -- platform specific routines.
3  */
4 
5 
6 //===----------------------------------------------------------------------===//
7 //
8 // The LLVM Compiler Infrastructure
9 //
10 // This file is dual licensed under the MIT and the University of Illinois Open
11 // Source Licenses. See LICENSE.txt for details.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 
16 #include "kmp.h"
17 #include "kmp_itt.h"
18 #include "kmp_i18n.h"
19 #include "kmp_io.h"
20 #include "kmp_wait_release.h"
21 
22 
23 
24 /* ----------------------------------------------------------------------------------- */
25 /* ----------------------------------------------------------------------------------- */
26 
27 /* This code is related to NtQuerySystemInformation() function. This function
28  is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
29  number of running threads in the system. */
30 
31 #include <ntstatus.h>
32 #include <ntsecapi.h> // UNICODE_STRING
33 
34 enum SYSTEM_INFORMATION_CLASS {
35  SystemProcessInformation = 5
36 }; // SYSTEM_INFORMATION_CLASS
37 
38 struct CLIENT_ID {
39  HANDLE UniqueProcess;
40  HANDLE UniqueThread;
41 }; // struct CLIENT_ID
42 
43 enum THREAD_STATE {
44  StateInitialized,
45  StateReady,
46  StateRunning,
47  StateStandby,
48  StateTerminated,
49  StateWait,
50  StateTransition,
51  StateUnknown
52 }; // enum THREAD_STATE
53 
54 struct VM_COUNTERS {
55  SIZE_T PeakVirtualSize;
56  SIZE_T VirtualSize;
57  ULONG PageFaultCount;
58  SIZE_T PeakWorkingSetSize;
59  SIZE_T WorkingSetSize;
60  SIZE_T QuotaPeakPagedPoolUsage;
61  SIZE_T QuotaPagedPoolUsage;
62  SIZE_T QuotaPeakNonPagedPoolUsage;
63  SIZE_T QuotaNonPagedPoolUsage;
64  SIZE_T PagefileUsage;
65  SIZE_T PeakPagefileUsage;
66  SIZE_T PrivatePageCount;
67 }; // struct VM_COUNTERS
68 
69 struct SYSTEM_THREAD {
70  LARGE_INTEGER KernelTime;
71  LARGE_INTEGER UserTime;
72  LARGE_INTEGER CreateTime;
73  ULONG WaitTime;
74  LPVOID StartAddress;
75  CLIENT_ID ClientId;
76  DWORD Priority;
77  LONG BasePriority;
78  ULONG ContextSwitchCount;
79  THREAD_STATE State;
80  ULONG WaitReason;
81 }; // SYSTEM_THREAD
82 
83 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, KernelTime ) == 0 );
84 #if KMP_ARCH_X86
85  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 28 );
86  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 52 );
87 #else
88  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 32 );
89  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 68 );
90 #endif
91 
92 struct SYSTEM_PROCESS_INFORMATION {
93  ULONG NextEntryOffset;
94  ULONG NumberOfThreads;
95  LARGE_INTEGER Reserved[ 3 ];
96  LARGE_INTEGER CreateTime;
97  LARGE_INTEGER UserTime;
98  LARGE_INTEGER KernelTime;
99  UNICODE_STRING ImageName;
100  DWORD BasePriority;
101  HANDLE ProcessId;
102  HANDLE ParentProcessId;
103  ULONG HandleCount;
104  ULONG Reserved2[ 2 ];
105  VM_COUNTERS VMCounters;
106  IO_COUNTERS IOCounters;
107  SYSTEM_THREAD Threads[ 1 ];
108 }; // SYSTEM_PROCESS_INFORMATION
109 typedef SYSTEM_PROCESS_INFORMATION * PSYSTEM_PROCESS_INFORMATION;
110 
111 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, NextEntryOffset ) == 0 );
112 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, CreateTime ) == 32 );
113 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ImageName ) == 56 );
114 #if KMP_ARCH_X86
115  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 68 );
116  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 76 );
117  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 88 );
118  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 136 );
119  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 184 );
120 #else
121  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 80 );
122  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 96 );
123  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 112 );
124  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 208 );
125  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 256 );
126 #endif
127 
128 typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG );
129 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
130 
131 HMODULE ntdll = NULL;
132 
133 /* End of NtQuerySystemInformation()-related code */
134 
135 #if KMP_GROUP_AFFINITY
136 static HMODULE kernel32 = NULL;
137 #endif /* KMP_GROUP_AFFINITY */
138 
139 /* ----------------------------------------------------------------------------------- */
140 /* ----------------------------------------------------------------------------------- */
141 
142 #if KMP_HANDLE_SIGNALS
143  typedef void (* sig_func_t )( int );
144  static sig_func_t __kmp_sighldrs[ NSIG ];
145  static int __kmp_siginstalled[ NSIG ];
146 #endif
147 
148 static HANDLE __kmp_monitor_ev;
149 static kmp_int64 __kmp_win32_time;
150 double __kmp_win32_tick;
151 
152 int __kmp_init_runtime = FALSE;
153 CRITICAL_SECTION __kmp_win32_section;
154 
155 void
156 __kmp_win32_mutex_init( kmp_win32_mutex_t *mx )
157 {
158  InitializeCriticalSection( & mx->cs );
159 #if USE_ITT_BUILD
160  __kmp_itt_system_object_created( & mx->cs, "Critical Section" );
161 #endif /* USE_ITT_BUILD */
162 }
163 
164 void
165 __kmp_win32_mutex_destroy( kmp_win32_mutex_t *mx )
166 {
167  DeleteCriticalSection( & mx->cs );
168 }
169 
170 void
171 __kmp_win32_mutex_lock( kmp_win32_mutex_t *mx )
172 {
173  EnterCriticalSection( & mx->cs );
174 }
175 
176 void
177 __kmp_win32_mutex_unlock( kmp_win32_mutex_t *mx )
178 {
179  LeaveCriticalSection( & mx->cs );
180 }
181 
182 void
183 __kmp_win32_cond_init( kmp_win32_cond_t *cv )
184 {
185  cv->waiters_count_ = 0;
186  cv->wait_generation_count_ = 0;
187  cv->release_count_ = 0;
188 
189  /* Initialize the critical section */
190  __kmp_win32_mutex_init( & cv->waiters_count_lock_ );
191 
192  /* Create a manual-reset event. */
193  cv->event_ = CreateEvent( NULL, // no security
194  TRUE, // manual-reset
195  FALSE, // non-signaled initially
196  NULL ); // unnamed
197 #if USE_ITT_BUILD
198  __kmp_itt_system_object_created( cv->event_, "Event" );
199 #endif /* USE_ITT_BUILD */
200 }
201 
202 void
203 __kmp_win32_cond_destroy( kmp_win32_cond_t *cv )
204 {
205  __kmp_win32_mutex_destroy( & cv->waiters_count_lock_ );
206  __kmp_free_handle( cv->event_ );
207  memset( cv, '\0', sizeof( *cv ) );
208 }
209 
210 /* TODO associate cv with a team instead of a thread so as to optimize
211  * the case where we wake up a whole team */
212 
213 void
214 __kmp_win32_cond_wait( kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx, kmp_info_t *th, int need_decrease_load )
215 {
216  int my_generation;
217  int last_waiter;
218 
219  /* Avoid race conditions */
220  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
221 
222  /* Increment count of waiters */
223  cv->waiters_count_++;
224 
225  /* Store current generation in our activation record. */
226  my_generation = cv->wait_generation_count_;
227 
228  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
229  __kmp_win32_mutex_unlock( mx );
230 
231 
232  for (;;) {
233  int wait_done;
234 
235  /* Wait until the event is signaled */
236  WaitForSingleObject( cv->event_, INFINITE );
237 
238  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
239 
240  /* Exit the loop when the <cv->event_> is signaled and
241  * there are still waiting threads from this <wait_generation>
242  * that haven't been released from this wait yet. */
243  wait_done = ( cv->release_count_ > 0 ) &&
244  ( cv->wait_generation_count_ != my_generation );
245 
246  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_);
247 
248  /* there used to be a semicolon after the if statement,
249  * it looked like a bug, so i removed it */
250  if( wait_done )
251  break;
252  }
253 
254  __kmp_win32_mutex_lock( mx );
255  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
256 
257  cv->waiters_count_--;
258  cv->release_count_--;
259 
260  last_waiter = ( cv->release_count_ == 0 );
261 
262  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
263 
264  if( last_waiter ) {
265  /* We're the last waiter to be notified, so reset the manual event. */
266  ResetEvent( cv->event_ );
267  }
268 }
269 
270 void
271 __kmp_win32_cond_broadcast( kmp_win32_cond_t *cv )
272 {
273  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
274 
275  if( cv->waiters_count_ > 0 ) {
276  SetEvent( cv->event_ );
277  /* Release all the threads in this generation. */
278 
279  cv->release_count_ = cv->waiters_count_;
280 
281  /* Start a new generation. */
282  cv->wait_generation_count_++;
283  }
284 
285  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
286 }
287 
288 void
289 __kmp_win32_cond_signal( kmp_win32_cond_t *cv )
290 {
291  __kmp_win32_cond_broadcast( cv );
292 }
293 
294 /* ------------------------------------------------------------------------ */
295 /* ------------------------------------------------------------------------ */
296 
297 void
298 __kmp_enable( int new_state )
299 {
300  if (__kmp_init_runtime)
301  LeaveCriticalSection( & __kmp_win32_section );
302 }
303 
304 void
305 __kmp_disable( int *old_state )
306 {
307  *old_state = 0;
308 
309  if (__kmp_init_runtime)
310  EnterCriticalSection( & __kmp_win32_section );
311 }
312 
313 void
314 __kmp_suspend_initialize( void )
315 {
316  /* do nothing */
317 }
318 
319 static void
320 __kmp_suspend_initialize_thread( kmp_info_t *th )
321 {
322  if ( ! TCR_4( th->th.th_suspend_init ) ) {
323  /* this means we haven't initialized the suspension pthread objects for this thread
324  in this instance of the process */
325  __kmp_win32_cond_init( &th->th.th_suspend_cv );
326  __kmp_win32_mutex_init( &th->th.th_suspend_mx );
327  TCW_4( th->th.th_suspend_init, TRUE );
328  }
329 }
330 
331 void
332 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
333 {
334  if ( TCR_4( th->th.th_suspend_init ) ) {
335  /* this means we have initialize the suspension pthread objects for this thread
336  in this instance of the process */
337  __kmp_win32_cond_destroy( & th->th.th_suspend_cv );
338  __kmp_win32_mutex_destroy( & th->th.th_suspend_mx );
339  TCW_4( th->th.th_suspend_init, FALSE );
340  }
341 }
342 
343 /* This routine puts the calling thread to sleep after setting the
344  * sleep bit for the indicated flag variable to true.
345  */
346 template <class C>
347 static inline void __kmp_suspend_template( int th_gtid, C *flag )
348 {
349  kmp_info_t *th = __kmp_threads[th_gtid];
350  int status;
351  typename C::flag_t old_spin;
352 
353  KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n", th_gtid, flag->get() ) );
354 
355  __kmp_suspend_initialize_thread( th );
356  __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
357 
358  KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for flag's loc(%p)\n",
359  th_gtid, flag->get() ) );
360 
361  /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread
362  gets called first?
363  */
364  old_spin = flag->set_sleeping();
365 
366  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for flag's loc(%p)==%d\n",
367  th_gtid, flag->get(), *(flag->get()) ) );
368 
369  if ( flag->done_check_val(old_spin) ) {
370  old_spin = flag->unset_sleeping();
371  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for flag's loc(%p)\n",
372  th_gtid, flag->get()) );
373  } else {
374 #ifdef DEBUG_SUSPEND
375  __kmp_suspend_count++;
376 #endif
377  /* Encapsulate in a loop as the documentation states that this may
378  * "with low probability" return when the condition variable has
379  * not been signaled or broadcast
380  */
381  int deactivated = FALSE;
382  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
383  while ( flag->is_sleeping() ) {
384  KF_TRACE( 15, ("__kmp_suspend_template: T#%d about to perform kmp_win32_cond_wait()\n",
385  th_gtid ) );
386  // Mark the thread as no longer active (only in the first iteration of the loop).
387  if ( ! deactivated ) {
388  th->th.th_active = FALSE;
389  if ( th->th.th_active_in_pool ) {
390  th->th.th_active_in_pool = FALSE;
391  KMP_TEST_THEN_DEC32(
392  (kmp_int32 *) &__kmp_thread_pool_active_nth );
393  KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
394  }
395  deactivated = TRUE;
396 
397 
398  __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
399  }
400  else {
401  __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
402  }
403 
404 #ifdef KMP_DEBUG
405  if( flag->is_sleeping() ) {
406  KF_TRACE( 100, ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ));
407  }
408 #endif /* KMP_DEBUG */
409 
410  } // while
411 
412  // Mark the thread as active again (if it was previous marked as inactive)
413  if ( deactivated ) {
414  th->th.th_active = TRUE;
415  if ( TCR_4(th->th.th_in_pool) ) {
416  KMP_TEST_THEN_INC32(
417  (kmp_int32 *) &__kmp_thread_pool_active_nth );
418  th->th.th_active_in_pool = TRUE;
419  }
420  }
421  }
422 
423 
424  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
425 
426  KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) );
427 }
428 
429 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
430  __kmp_suspend_template(th_gtid, flag);
431 }
432 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
433  __kmp_suspend_template(th_gtid, flag);
434 }
435 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
436  __kmp_suspend_template(th_gtid, flag);
437 }
438 
439 
440 /* This routine signals the thread specified by target_gtid to wake up
441  * after setting the sleep bit indicated by the flag argument to FALSE
442  */
443 template <class C>
444 static inline void __kmp_resume_template( int target_gtid, C *flag )
445 {
446  kmp_info_t *th = __kmp_threads[target_gtid];
447  int status;
448 
449 #ifdef KMP_DEBUG
450  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
451 #endif
452 
453  KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
454 
455  __kmp_suspend_initialize_thread( th );
456  __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
457 
458  if (!flag) {
459  flag = (C *)th->th.th_sleep_loc;
460  }
461 
462  if (!flag) {
463  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p)\n",
464  gtid, target_gtid, NULL ) );
465  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
466  return;
467  }
468  else {
469  typename C::flag_t old_spin = flag->unset_sleeping();
470  if ( !flag->is_sleeping_val(old_spin) ) {
471  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p): "
472  "%u => %u\n",
473  gtid, target_gtid, flag->get(), old_spin, *(flag->get()) ) );
474  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
475  return;
476  }
477  }
478  TCW_PTR(th->th.th_sleep_loc, NULL);
479 
480  KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p)\n",
481  gtid, target_gtid, flag->get() ) );
482 
483 
484  __kmp_win32_cond_signal( &th->th.th_suspend_cv );
485  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
486 
487  KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
488  gtid, target_gtid ) );
489 }
490 
491 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
492  __kmp_resume_template(target_gtid, flag);
493 }
494 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
495  __kmp_resume_template(target_gtid, flag);
496 }
497 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
498  __kmp_resume_template(target_gtid, flag);
499 }
500 
501 
502 /* ------------------------------------------------------------------------ */
503 /* ------------------------------------------------------------------------ */
504 
505 void
506 __kmp_yield( int cond )
507 {
508  if (cond)
509  Sleep(0);
510 }
511 
512 /* ------------------------------------------------------------------------ */
513 /* ------------------------------------------------------------------------ */
514 
515 void
516 __kmp_gtid_set_specific( int gtid )
517 {
518  KA_TRACE( 50, ("__kmp_gtid_set_specific: T#%d key:%d\n",
519  gtid, __kmp_gtid_threadprivate_key ));
520  KMP_ASSERT( __kmp_init_runtime );
521  if( ! TlsSetValue( __kmp_gtid_threadprivate_key, (LPVOID)(gtid+1)) )
522  KMP_FATAL( TLSSetValueFailed );
523 }
524 
525 int
526 __kmp_gtid_get_specific()
527 {
528  int gtid;
529  if( !__kmp_init_runtime ) {
530  KA_TRACE( 50, ("__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
531  return KMP_GTID_SHUTDOWN;
532  }
533  gtid = (int)(kmp_intptr_t)TlsGetValue( __kmp_gtid_threadprivate_key );
534  if ( gtid == 0 ) {
535  gtid = KMP_GTID_DNE;
536  }
537  else {
538  gtid--;
539  }
540  KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
541  __kmp_gtid_threadprivate_key, gtid ));
542  return gtid;
543 }
544 
545 /* ------------------------------------------------------------------------ */
546 /* ------------------------------------------------------------------------ */
547 
548 #if KMP_GROUP_AFFINITY
549 
550 //
551 // Only 1 DWORD in the mask should have any procs set.
552 // Return the appropriate index, or -1 for an invalid mask.
553 //
554 int
555 __kmp_get_proc_group( kmp_affin_mask_t const *mask )
556 {
557  int i;
558  int group = -1;
559  for (i = 0; i < __kmp_num_proc_groups; i++) {
560  if (mask[i] == 0) {
561  continue;
562  }
563  if (group >= 0) {
564  return -1;
565  }
566  group = i;
567  }
568  return group;
569 }
570 
571 #endif /* KMP_GROUP_AFFINITY */
572 
573 int
574 __kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
575 {
576 
577 #if KMP_GROUP_AFFINITY
578 
579  if (__kmp_num_proc_groups > 1) {
580  //
581  // Check for a valid mask.
582  //
583  GROUP_AFFINITY ga;
584  int group = __kmp_get_proc_group( mask );
585  if (group < 0) {
586  if (abort_on_error) {
587  KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity");
588  }
589  return -1;
590  }
591 
592  //
593  // Transform the bit vector into a GROUP_AFFINITY struct
594  // and make the system call to set affinity.
595  //
596  ga.Group = group;
597  ga.Mask = mask[group];
598  ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
599 
600  KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
601  if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
602  DWORD error = GetLastError();
603  if (abort_on_error) {
604  __kmp_msg(
605  kmp_ms_fatal,
606  KMP_MSG( CantSetThreadAffMask ),
607  KMP_ERR( error ),
608  __kmp_msg_null
609  );
610  }
611  return error;
612  }
613  }
614  else
615 
616 #endif /* KMP_GROUP_AFFINITY */
617 
618  {
619  if (!SetThreadAffinityMask( GetCurrentThread(), *mask )) {
620  DWORD error = GetLastError();
621  if (abort_on_error) {
622  __kmp_msg(
623  kmp_ms_fatal,
624  KMP_MSG( CantSetThreadAffMask ),
625  KMP_ERR( error ),
626  __kmp_msg_null
627  );
628  }
629  return error;
630  }
631  }
632  return 0;
633 }
634 
635 int
636 __kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
637 {
638 
639 #if KMP_GROUP_AFFINITY
640 
641  if (__kmp_num_proc_groups > 1) {
642  KMP_CPU_ZERO(mask);
643  GROUP_AFFINITY ga;
644  KMP_DEBUG_ASSERT(__kmp_GetThreadGroupAffinity != NULL);
645 
646  if (__kmp_GetThreadGroupAffinity(GetCurrentThread(), &ga) == 0) {
647  DWORD error = GetLastError();
648  if (abort_on_error) {
649  __kmp_msg(
650  kmp_ms_fatal,
651  KMP_MSG(FunctionError, "GetThreadGroupAffinity()"),
652  KMP_ERR(error),
653  __kmp_msg_null
654  );
655  }
656  return error;
657  }
658 
659  if ((ga.Group < 0) || (ga.Group > __kmp_num_proc_groups)
660  || (ga.Mask == 0)) {
661  return -1;
662  }
663 
664  mask[ga.Group] = ga.Mask;
665  }
666  else
667 
668 #endif /* KMP_GROUP_AFFINITY */
669 
670  {
671  kmp_affin_mask_t newMask, sysMask, retval;
672 
673  if (!GetProcessAffinityMask(GetCurrentProcess(), &newMask, &sysMask)) {
674  DWORD error = GetLastError();
675  if (abort_on_error) {
676  __kmp_msg(
677  kmp_ms_fatal,
678  KMP_MSG(FunctionError, "GetProcessAffinityMask()"),
679  KMP_ERR(error),
680  __kmp_msg_null
681  );
682  }
683  return error;
684  }
685  retval = SetThreadAffinityMask(GetCurrentThread(), newMask);
686  if (! retval) {
687  DWORD error = GetLastError();
688  if (abort_on_error) {
689  __kmp_msg(
690  kmp_ms_fatal,
691  KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
692  KMP_ERR(error),
693  __kmp_msg_null
694  );
695  }
696  return error;
697  }
698  newMask = SetThreadAffinityMask(GetCurrentThread(), retval);
699  if (! newMask) {
700  DWORD error = GetLastError();
701  if (abort_on_error) {
702  __kmp_msg(
703  kmp_ms_fatal,
704  KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
705  KMP_ERR(error),
706  __kmp_msg_null
707  );
708  }
709  }
710  *mask = retval;
711  }
712  return 0;
713 }
714 
715 void
716 __kmp_affinity_bind_thread( int proc )
717 {
718 
719 #if KMP_GROUP_AFFINITY
720 
721  if (__kmp_num_proc_groups > 1) {
722  //
723  // Form the GROUP_AFFINITY struct directly, rather than filling
724  // out a bit vector and calling __kmp_set_system_affinity().
725  //
726  GROUP_AFFINITY ga;
727  KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups
728  * CHAR_BIT * sizeof(DWORD_PTR))));
729  ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
730  ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
731  ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
732 
733  KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
734  if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
735  DWORD error = GetLastError();
736  if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
737  __kmp_msg(
738  kmp_ms_warning,
739  KMP_MSG( CantSetThreadAffMask ),
740  KMP_ERR( error ),
741  __kmp_msg_null
742  );
743  }
744  }
745  }
746  else
747 
748 #endif /* KMP_GROUP_AFFINITY */
749 
750  {
751  kmp_affin_mask_t mask;
752  KMP_CPU_ZERO(&mask);
753  KMP_CPU_SET(proc, &mask);
754  __kmp_set_system_affinity(&mask, TRUE);
755  }
756 }
757 
758 void
759 __kmp_affinity_determine_capable( const char *env_var )
760 {
761  //
762  // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
763  //
764 
765 #if KMP_GROUP_AFFINITY
766  KMP_AFFINITY_ENABLE(__kmp_num_proc_groups*sizeof(kmp_affin_mask_t));
767 #else
768  KMP_AFFINITY_ENABLE(sizeof(kmp_affin_mask_t));
769 #endif
770 
771  KA_TRACE( 10, (
772  "__kmp_affinity_determine_capable: "
773  "Windows* OS affinity interface functional (mask size = %" KMP_SIZE_T_SPEC ").\n",
774  __kmp_affin_mask_size
775  ) );
776 }
777 
778 double
779 __kmp_read_cpu_time( void )
780 {
781  FILETIME CreationTime, ExitTime, KernelTime, UserTime;
782  int status;
783  double cpu_time;
784 
785  cpu_time = 0;
786 
787  status = GetProcessTimes( GetCurrentProcess(), &CreationTime,
788  &ExitTime, &KernelTime, &UserTime );
789 
790  if (status) {
791  double sec = 0;
792 
793  sec += KernelTime.dwHighDateTime;
794  sec += UserTime.dwHighDateTime;
795 
796  /* Shift left by 32 bits */
797  sec *= (double) (1 << 16) * (double) (1 << 16);
798 
799  sec += KernelTime.dwLowDateTime;
800  sec += UserTime.dwLowDateTime;
801 
802  cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
803  }
804 
805  return cpu_time;
806 }
807 
808 int
809 __kmp_read_system_info( struct kmp_sys_info *info )
810 {
811  info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
812  info->minflt = 0; /* the number of page faults serviced without any I/O */
813  info->majflt = 0; /* the number of page faults serviced that required I/O */
814  info->nswap = 0; /* the number of times a process was "swapped" out of memory */
815  info->inblock = 0; /* the number of times the file system had to perform input */
816  info->oublock = 0; /* the number of times the file system had to perform output */
817  info->nvcsw = 0; /* the number of times a context switch was voluntarily */
818  info->nivcsw = 0; /* the number of times a context switch was forced */
819 
820  return 1;
821 }
822 
823 /* ------------------------------------------------------------------------ */
824 /* ------------------------------------------------------------------------ */
825 
826 
827 void
828 __kmp_runtime_initialize( void )
829 {
830  SYSTEM_INFO info;
831  kmp_str_buf_t path;
832  UINT path_size;
833 
834  if ( __kmp_init_runtime ) {
835  return;
836  };
837 
838 #if KMP_DYNAMIC_LIB
839  /* Pin dynamic library for the lifetime of application */
840  {
841  // First, turn off error message boxes
842  UINT err_mode = SetErrorMode (SEM_FAILCRITICALERRORS);
843  HMODULE h;
844  BOOL ret = GetModuleHandleEx( GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS
845  |GET_MODULE_HANDLE_EX_FLAG_PIN,
846  (LPCTSTR)&__kmp_serial_initialize, &h);
847  KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
848  SetErrorMode (err_mode); // Restore error mode
849  KA_TRACE( 10, ("__kmp_runtime_initialize: dynamic library pinned\n") );
850  }
851 #endif
852 
853  InitializeCriticalSection( & __kmp_win32_section );
854 #if USE_ITT_BUILD
855  __kmp_itt_system_object_created( & __kmp_win32_section, "Critical Section" );
856 #endif /* USE_ITT_BUILD */
857  __kmp_initialize_system_tick();
858 
859  #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
860  if ( ! __kmp_cpuinfo.initialized ) {
861  __kmp_query_cpuid( & __kmp_cpuinfo );
862  }; // if
863  #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
864 
865  /* Set up minimum number of threads to switch to TLS gtid */
866  #if KMP_OS_WINDOWS && ! defined KMP_DYNAMIC_LIB
867  // Windows* OS, static library.
868  /*
869  New thread may use stack space previously used by another thread, currently terminated.
870  On Windows* OS, in case of static linking, we do not know the moment of thread termination,
871  and our structures (__kmp_threads and __kmp_root arrays) are still keep info about dead
872  threads. This leads to problem in __kmp_get_global_thread_id() function: it wrongly
873  finds gtid (by searching through stack addresses of all known threads) for unregistered
874  foreign tread.
875 
876  Setting __kmp_tls_gtid_min to 0 workarounds this problem: __kmp_get_global_thread_id()
877  does not search through stacks, but get gtid from TLS immediately.
878 
879  --ln
880  */
881  __kmp_tls_gtid_min = 0;
882  #else
883  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
884  #endif
885 
886  /* for the static library */
887  if ( !__kmp_gtid_threadprivate_key ) {
888  __kmp_gtid_threadprivate_key = TlsAlloc();
889  if( __kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES ) {
890  KMP_FATAL( TLSOutOfIndexes );
891  }
892  }
893 
894 
895  //
896  // Load ntdll.dll.
897  //
898  /*
899  Simple
900  GetModuleHandle( "ntdll.dl" )
901  is not suitable due to security issue (see
902  http://www.microsoft.com/technet/security/advisory/2269637.mspx). We have to specify full
903  path to the library.
904  */
905  __kmp_str_buf_init( & path );
906  path_size = GetSystemDirectory( path.str, path.size );
907  KMP_DEBUG_ASSERT( path_size > 0 );
908  if ( path_size >= path.size ) {
909  //
910  // Buffer is too short. Expand the buffer and try again.
911  //
912  __kmp_str_buf_reserve( & path, path_size );
913  path_size = GetSystemDirectory( path.str, path.size );
914  KMP_DEBUG_ASSERT( path_size > 0 );
915  }; // if
916  if ( path_size > 0 && path_size < path.size ) {
917  //
918  // Now we have system directory name in the buffer.
919  // Append backslash and name of dll to form full path,
920  //
921  path.used = path_size;
922  __kmp_str_buf_print( & path, "\\%s", "ntdll.dll" );
923 
924  //
925  // Now load ntdll using full path.
926  //
927  ntdll = GetModuleHandle( path.str );
928  }
929 
930  KMP_DEBUG_ASSERT( ntdll != NULL );
931  if ( ntdll != NULL ) {
932  NtQuerySystemInformation = (NtQuerySystemInformation_t) GetProcAddress( ntdll, "NtQuerySystemInformation" );
933  }
934  KMP_DEBUG_ASSERT( NtQuerySystemInformation != NULL );
935 
936 #if KMP_GROUP_AFFINITY
937  //
938  // Load kernel32.dll.
939  // Same caveat - must use full system path name.
940  //
941  if ( path_size > 0 && path_size < path.size ) {
942  //
943  // Truncate the buffer back to just the system path length,
944  // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
945  //
946  path.used = path_size;
947  __kmp_str_buf_print( & path, "\\%s", "kernel32.dll" );
948 
949  //
950  // Load kernel32.dll using full path.
951  //
952  kernel32 = GetModuleHandle( path.str );
953  KA_TRACE( 10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str ) );
954 
955  //
956  // Load the function pointers to kernel32.dll routines
957  // that may or may not exist on this system.
958  //
959  if ( kernel32 != NULL ) {
960  __kmp_GetActiveProcessorCount = (kmp_GetActiveProcessorCount_t) GetProcAddress( kernel32, "GetActiveProcessorCount" );
961  __kmp_GetActiveProcessorGroupCount = (kmp_GetActiveProcessorGroupCount_t) GetProcAddress( kernel32, "GetActiveProcessorGroupCount" );
962  __kmp_GetThreadGroupAffinity = (kmp_GetThreadGroupAffinity_t) GetProcAddress( kernel32, "GetThreadGroupAffinity" );
963  __kmp_SetThreadGroupAffinity = (kmp_SetThreadGroupAffinity_t) GetProcAddress( kernel32, "SetThreadGroupAffinity" );
964 
965  KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount = %p\n", __kmp_GetActiveProcessorCount ) );
966  KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorGroupCount = %p\n", __kmp_GetActiveProcessorGroupCount ) );
967  KA_TRACE( 10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity = %p\n", __kmp_GetThreadGroupAffinity ) );
968  KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity = %p\n", __kmp_SetThreadGroupAffinity ) );
969  KA_TRACE( 10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n", sizeof(kmp_affin_mask_t) ) );
970 
971  //
972  // See if group affinity is supported on this system.
973  // If so, calculate the #groups and #procs.
974  //
975  // Group affinity was introduced with Windows* 7 OS and
976  // Windows* Server 2008 R2 OS.
977  //
978  if ( ( __kmp_GetActiveProcessorCount != NULL )
979  && ( __kmp_GetActiveProcessorGroupCount != NULL )
980  && ( __kmp_GetThreadGroupAffinity != NULL )
981  && ( __kmp_SetThreadGroupAffinity != NULL )
982  && ( ( __kmp_num_proc_groups
983  = __kmp_GetActiveProcessorGroupCount() ) > 1 ) ) {
984  //
985  // Calculate the total number of active OS procs.
986  //
987  int i;
988 
989  KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
990 
991  __kmp_xproc = 0;
992 
993  for ( i = 0; i < __kmp_num_proc_groups; i++ ) {
994  DWORD size = __kmp_GetActiveProcessorCount( i );
995  __kmp_xproc += size;
996  KA_TRACE( 10, ("__kmp_runtime_initialize: proc group %d size = %d\n", i, size ) );
997  }
998  }
999  else {
1000  KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
1001  }
1002  }
1003  }
1004  if ( __kmp_num_proc_groups <= 1 ) {
1005  GetSystemInfo( & info );
1006  __kmp_xproc = info.dwNumberOfProcessors;
1007  }
1008 #else
1009  GetSystemInfo( & info );
1010  __kmp_xproc = info.dwNumberOfProcessors;
1011 #endif /* KMP_GROUP_AFFINITY */
1012 
1013  //
1014  // If the OS said there were 0 procs, take a guess and use a value of 2.
1015  // This is done for Linux* OS, also. Do we need error / warning?
1016  //
1017  if ( __kmp_xproc <= 0 ) {
1018  __kmp_xproc = 2;
1019  }
1020 
1021  KA_TRACE( 5, ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc) );
1022 
1023  __kmp_str_buf_free( & path );
1024 
1025 #if USE_ITT_BUILD
1026  __kmp_itt_initialize();
1027 #endif /* USE_ITT_BUILD */
1028 
1029  __kmp_init_runtime = TRUE;
1030 } // __kmp_runtime_initialize
1031 
1032 void
1033 __kmp_runtime_destroy( void )
1034 {
1035  if ( ! __kmp_init_runtime ) {
1036  return;
1037  }
1038 
1039 #if USE_ITT_BUILD
1040  __kmp_itt_destroy();
1041 #endif /* USE_ITT_BUILD */
1042 
1043  /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
1044  /* due to the KX_TRACE() commands */
1045  KA_TRACE( 40, ("__kmp_runtime_destroy\n" ));
1046 
1047  if( __kmp_gtid_threadprivate_key ) {
1048  TlsFree( __kmp_gtid_threadprivate_key );
1049  __kmp_gtid_threadprivate_key = 0;
1050  }
1051 
1052  __kmp_affinity_uninitialize();
1053  DeleteCriticalSection( & __kmp_win32_section );
1054 
1055  ntdll = NULL;
1056  NtQuerySystemInformation = NULL;
1057 
1058 #if KMP_ARCH_X86_64
1059  kernel32 = NULL;
1060  __kmp_GetActiveProcessorCount = NULL;
1061  __kmp_GetActiveProcessorGroupCount = NULL;
1062  __kmp_GetThreadGroupAffinity = NULL;
1063  __kmp_SetThreadGroupAffinity = NULL;
1064 #endif // KMP_ARCH_X86_64
1065 
1066  __kmp_init_runtime = FALSE;
1067 }
1068 
1069 
1070 void
1071 __kmp_terminate_thread( int gtid )
1072 {
1073  kmp_info_t *th = __kmp_threads[ gtid ];
1074 
1075  if( !th ) return;
1076 
1077  KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) );
1078 
1079  if (TerminateThread( th->th.th_info.ds.ds_thread, (DWORD) -1) == FALSE) {
1080  /* It's OK, the thread may have exited already */
1081  }
1082  __kmp_free_handle( th->th.th_info.ds.ds_thread );
1083 }
1084 
1085 /* ------------------------------------------------------------------------ */
1086 /* ------------------------------------------------------------------------ */
1087 
1088 void
1089 __kmp_clear_system_time( void )
1090 {
1091  BOOL status;
1092  LARGE_INTEGER time;
1093  status = QueryPerformanceCounter( & time );
1094  __kmp_win32_time = (kmp_int64) time.QuadPart;
1095 }
1096 
1097 void
1098 __kmp_initialize_system_tick( void )
1099 {
1100  {
1101  BOOL status;
1102  LARGE_INTEGER freq;
1103 
1104  status = QueryPerformanceFrequency( & freq );
1105  if (! status) {
1106  DWORD error = GetLastError();
1107  __kmp_msg(
1108  kmp_ms_fatal,
1109  KMP_MSG( FunctionError, "QueryPerformanceFrequency()" ),
1110  KMP_ERR( error ),
1111  __kmp_msg_null
1112  );
1113 
1114  }
1115  else {
1116  __kmp_win32_tick = ((double) 1.0) / (double) freq.QuadPart;
1117  }
1118  }
1119 }
1120 
1121 /* Calculate the elapsed wall clock time for the user */
1122 
1123 void
1124 __kmp_elapsed( double *t )
1125 {
1126  BOOL status;
1127  LARGE_INTEGER now;
1128  status = QueryPerformanceCounter( & now );
1129  *t = ((double) now.QuadPart) * __kmp_win32_tick;
1130 }
1131 
1132 /* Calculate the elapsed wall clock tick for the user */
1133 
1134 void
1135 __kmp_elapsed_tick( double *t )
1136 {
1137  *t = __kmp_win32_tick;
1138 }
1139 
1140 void
1141 __kmp_read_system_time( double *delta )
1142 {
1143 
1144  if (delta != NULL) {
1145  BOOL status;
1146  LARGE_INTEGER now;
1147 
1148  status = QueryPerformanceCounter( & now );
1149 
1150  *delta = ((double) (((kmp_int64) now.QuadPart) - __kmp_win32_time))
1151  * __kmp_win32_tick;
1152  }
1153 }
1154 
1155 /* ------------------------------------------------------------------------ */
1156 /* ------------------------------------------------------------------------ */
1157 
1158 void * __stdcall
1159 __kmp_launch_worker( void *arg )
1160 {
1161  volatile void *stack_data;
1162  void *exit_val;
1163  void *padding = 0;
1164  kmp_info_t *this_thr = (kmp_info_t *) arg;
1165  int gtid;
1166 
1167  gtid = this_thr->th.th_info.ds.ds_gtid;
1168  __kmp_gtid_set_specific( gtid );
1169 #ifdef KMP_TDATA_GTID
1170  #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1171  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1172  "reference: http://support.microsoft.com/kb/118816"
1173  //__kmp_gtid = gtid;
1174 #endif
1175 
1176 #if USE_ITT_BUILD
1177  __kmp_itt_thread_name( gtid );
1178 #endif /* USE_ITT_BUILD */
1179 
1180  __kmp_affinity_set_init_mask( gtid, FALSE );
1181 
1182 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1183  //
1184  // Set the FP control regs to be a copy of
1185  // the parallel initialization thread's.
1186  //
1187  __kmp_clear_x87_fpu_status_word();
1188  __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
1189  __kmp_load_mxcsr( &__kmp_init_mxcsr );
1190 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1191 
1192  if ( __kmp_stkoffset > 0 && gtid > 0 ) {
1193  padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
1194  }
1195 
1196  KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1197  this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1198  TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1199 
1200  if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid
1201  TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
1202  KMP_ASSERT( this_thr -> th.th_info.ds.ds_stackgrow == FALSE );
1203  __kmp_check_stack_overlap( this_thr );
1204  }
1205  KMP_MB();
1206  exit_val = __kmp_launch_thread( this_thr );
1207  KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1208  TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1209  KMP_MB();
1210  return exit_val;
1211 }
1212 
1213 
1214 /* The monitor thread controls all of the threads in the complex */
1215 
1216 void * __stdcall
1217 __kmp_launch_monitor( void *arg )
1218 {
1219  DWORD wait_status;
1220  kmp_thread_t monitor;
1221  int status;
1222  int interval;
1223  kmp_info_t *this_thr = (kmp_info_t *) arg;
1224 
1225  KMP_DEBUG_ASSERT(__kmp_init_monitor);
1226  TCW_4( __kmp_init_monitor, 2 ); // AC: Signal the library that monitor has started
1227  // TODO: hide "2" in enum (like {true,false,started})
1228  this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1229  TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1230 
1231  KMP_MB(); /* Flush all pending memory write invalidates. */
1232  KA_TRACE( 10, ("__kmp_launch_monitor: launched\n" ) );
1233 
1234  monitor = GetCurrentThread();
1235 
1236  /* set thread priority */
1237  status = SetThreadPriority( monitor, THREAD_PRIORITY_HIGHEST );
1238  if (! status) {
1239  DWORD error = GetLastError();
1240  __kmp_msg(
1241  kmp_ms_fatal,
1242  KMP_MSG( CantSetThreadPriority ),
1243  KMP_ERR( error ),
1244  __kmp_msg_null
1245  );
1246  }
1247 
1248  /* register us as monitor */
1249  __kmp_gtid_set_specific( KMP_GTID_MONITOR );
1250 #ifdef KMP_TDATA_GTID
1251  #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1252  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1253  "reference: http://support.microsoft.com/kb/118816"
1254  //__kmp_gtid = KMP_GTID_MONITOR;
1255 #endif
1256 
1257 #if USE_ITT_BUILD
1258  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread.
1259 #endif /* USE_ITT_BUILD */
1260 
1261  KMP_MB(); /* Flush all pending memory write invalidates. */
1262 
1263  interval = ( 1000 / __kmp_monitor_wakeups ); /* in milliseconds */
1264 
1265  while (! TCR_4(__kmp_global.g.g_done)) {
1266  /* This thread monitors the state of the system */
1267 
1268  KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) );
1269 
1270  wait_status = WaitForSingleObject( __kmp_monitor_ev, interval );
1271 
1272  if (wait_status == WAIT_TIMEOUT) {
1273  TCW_4( __kmp_global.g.g_time.dt.t_value,
1274  TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
1275  }
1276 
1277  KMP_MB(); /* Flush all pending memory write invalidates. */
1278  }
1279 
1280  KA_TRACE( 10, ("__kmp_launch_monitor: finished\n" ) );
1281 
1282  status = SetThreadPriority( monitor, THREAD_PRIORITY_NORMAL );
1283  if (! status) {
1284  DWORD error = GetLastError();
1285  __kmp_msg(
1286  kmp_ms_fatal,
1287  KMP_MSG( CantSetThreadPriority ),
1288  KMP_ERR( error ),
1289  __kmp_msg_null
1290  );
1291  }
1292 
1293  if (__kmp_global.g.g_abort != 0) {
1294  /* now we need to terminate the worker threads */
1295  /* the value of t_abort is the signal we caught */
1296 
1297  int gtid;
1298 
1299  KA_TRACE( 10, ("__kmp_launch_monitor: terminate sig=%d\n", (__kmp_global.g.g_abort) ) );
1300 
1301  /* terminate the OpenMP worker threads */
1302  /* TODO this is not valid for sibling threads!!
1303  * the uber master might not be 0 anymore.. */
1304  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1305  __kmp_terminate_thread( gtid );
1306 
1307  __kmp_cleanup();
1308 
1309  Sleep( 0 );
1310 
1311  KA_TRACE( 10, ("__kmp_launch_monitor: raise sig=%d\n", (__kmp_global.g.g_abort) ) );
1312 
1313  if (__kmp_global.g.g_abort > 0) {
1314  raise( __kmp_global.g.g_abort );
1315  }
1316  }
1317 
1318  TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1319 
1320  KMP_MB();
1321  return arg;
1322 }
1323 
1324 void
1325 __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
1326 {
1327  kmp_thread_t handle;
1328  DWORD idThread;
1329 
1330  KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1331 
1332  th->th.th_info.ds.ds_gtid = gtid;
1333 
1334  if ( KMP_UBER_GTID(gtid) ) {
1335  int stack_data;
1336 
1337  /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for other threads to use.
1338  Is it appropriate to just use GetCurrentThread? When should we close this handle? When
1339  unregistering the root?
1340  */
1341  {
1342  BOOL rc;
1343  rc = DuplicateHandle(
1344  GetCurrentProcess(),
1345  GetCurrentThread(),
1346  GetCurrentProcess(),
1347  &th->th.th_info.ds.ds_thread,
1348  0,
1349  FALSE,
1350  DUPLICATE_SAME_ACCESS
1351  );
1352  KMP_ASSERT( rc );
1353  KA_TRACE( 10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, handle = %" KMP_UINTPTR_SPEC "\n",
1354  (LPVOID)th,
1355  th->th.th_info.ds.ds_thread ) );
1356  th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1357  }
1358  if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid
1359  /* we will dynamically update the stack range if gtid_mode == 1 */
1360  TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1361  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1362  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1363  __kmp_check_stack_overlap( th );
1364  }
1365  }
1366  else {
1367  KMP_MB(); /* Flush all pending memory write invalidates. */
1368 
1369  /* Set stack size for this thread now. */
1370  KA_TRACE( 10, ( "__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC
1371  " bytes\n", stack_size ) );
1372 
1373  stack_size += gtid * __kmp_stkoffset;
1374 
1375  TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1376  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1377 
1378  KA_TRACE( 10, ( "__kmp_create_worker: (before) stack_size = %"
1379  KMP_SIZE_T_SPEC
1380  " bytes, &__kmp_launch_worker = %p, th = %p, "
1381  "&idThread = %p\n",
1382  (SIZE_T) stack_size,
1383  (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1384  (LPVOID) th, &idThread ) );
1385 
1386  {
1387  handle = CreateThread( NULL, (SIZE_T) stack_size,
1388  (LPTHREAD_START_ROUTINE) __kmp_launch_worker,
1389  (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1390  }
1391 
1392  KA_TRACE( 10, ( "__kmp_create_worker: (after) stack_size = %"
1393  KMP_SIZE_T_SPEC
1394  " bytes, &__kmp_launch_worker = %p, th = %p, "
1395  "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
1396  (SIZE_T) stack_size,
1397  (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1398  (LPVOID) th, idThread, handle ) );
1399 
1400  {
1401  if ( handle == 0 ) {
1402  DWORD error = GetLastError();
1403  __kmp_msg(
1404  kmp_ms_fatal,
1405  KMP_MSG( CantCreateThread ),
1406  KMP_ERR( error ),
1407  __kmp_msg_null
1408  );
1409  } else {
1410  th->th.th_info.ds.ds_thread = handle;
1411  }
1412  }
1413  KMP_MB(); /* Flush all pending memory write invalidates. */
1414  }
1415 
1416  KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1417 }
1418 
1419 int
1420 __kmp_still_running(kmp_info_t *th) {
1421  return (WAIT_TIMEOUT == WaitForSingleObject( th->th.th_info.ds.ds_thread, 0));
1422 }
1423 
1424 void
1425 __kmp_create_monitor( kmp_info_t *th )
1426 {
1427  kmp_thread_t handle;
1428  DWORD idThread;
1429  int ideal, new_ideal;
1430  int caller_gtid = __kmp_get_gtid();
1431 
1432  KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) );
1433 
1434  KMP_MB(); /* Flush all pending memory write invalidates. */
1435 
1436  __kmp_monitor_ev = CreateEvent( NULL, TRUE, FALSE, NULL );
1437  if ( __kmp_monitor_ev == NULL ) {
1438  DWORD error = GetLastError();
1439  __kmp_msg(
1440  kmp_ms_fatal,
1441  KMP_MSG( CantCreateEvent ),
1442  KMP_ERR( error ),
1443  __kmp_msg_null
1444  );
1445  }; // if
1446 #if USE_ITT_BUILD
1447  __kmp_itt_system_object_created( __kmp_monitor_ev, "Event" );
1448 #endif /* USE_ITT_BUILD */
1449 
1450  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1451  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1452 
1453  // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
1454  // to automatically expand stacksize based on CreateThread error code.
1455  if ( __kmp_monitor_stksize == 0 ) {
1456  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1457  }
1458  if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1459  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1460  }
1461 
1462  KA_TRACE( 10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
1463  (int) __kmp_monitor_stksize ) );
1464 
1465  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1466 
1467  handle = CreateThread( NULL, (SIZE_T) __kmp_monitor_stksize,
1468  (LPTHREAD_START_ROUTINE) __kmp_launch_monitor,
1469  (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1470  if (handle == 0) {
1471  DWORD error = GetLastError();
1472  __kmp_msg(
1473  kmp_ms_fatal,
1474  KMP_MSG( CantCreateThread ),
1475  KMP_ERR( error ),
1476  __kmp_msg_null
1477  );
1478  }
1479  else
1480  th->th.th_info.ds.ds_thread = handle;
1481 
1482  KMP_MB(); /* Flush all pending memory write invalidates. */
1483 
1484  KA_TRACE( 10, ("__kmp_create_monitor: monitor created %p\n",
1485  (void *) th->th.th_info.ds.ds_thread ) );
1486 }
1487 
1488 /*
1489  Check to see if thread is still alive.
1490 
1491  NOTE: The ExitProcess(code) system call causes all threads to Terminate
1492  with a exit_val = code. Because of this we can not rely on
1493  exit_val having any particular value. So this routine may
1494  return STILL_ALIVE in exit_val even after the thread is dead.
1495 */
1496 
1497 int
1498 __kmp_is_thread_alive( kmp_info_t * th, DWORD *exit_val )
1499 {
1500  DWORD rc;
1501  rc = GetExitCodeThread( th->th.th_info.ds.ds_thread, exit_val );
1502  if ( rc == 0 ) {
1503  DWORD error = GetLastError();
1504  __kmp_msg(
1505  kmp_ms_fatal,
1506  KMP_MSG( FunctionError, "GetExitCodeThread()" ),
1507  KMP_ERR( error ),
1508  __kmp_msg_null
1509  );
1510  }; // if
1511  return ( *exit_val == STILL_ACTIVE );
1512 }
1513 
1514 
1515 void
1516 __kmp_exit_thread(
1517  int exit_status
1518 ) {
1519  ExitThread( exit_status );
1520 } // __kmp_exit_thread
1521 
1522 /*
1523  This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
1524 */
1525 static void
1526 __kmp_reap_common( kmp_info_t * th )
1527 {
1528  DWORD exit_val;
1529 
1530  KMP_MB(); /* Flush all pending memory write invalidates. */
1531 
1532  KA_TRACE( 10, ( "__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid ) );
1533 
1534  /*
1535  2006-10-19:
1536 
1537  There are two opposite situations:
1538 
1539  1. Windows* OS keep thread alive after it resets ds_alive flag and exits from thread
1540  function. (For example, see C70770/Q394281 "unloading of dll based on OMP is very
1541  slow".)
1542  2. Windows* OS may kill thread before it resets ds_alive flag.
1543 
1544  Right solution seems to be waiting for *either* thread termination *or* ds_alive resetting.
1545 
1546  */
1547 
1548  {
1549  // TODO: This code is very similar to KMP_WAIT_YIELD. Need to generalize KMP_WAIT_YIELD to
1550  // cover this usage also.
1551  void * obj = NULL;
1552  register kmp_uint32 spins;
1553 #if USE_ITT_BUILD
1554  KMP_FSYNC_SPIN_INIT( obj, (void*) & th->th.th_info.ds.ds_alive );
1555 #endif /* USE_ITT_BUILD */
1556  KMP_INIT_YIELD( spins );
1557  do {
1558 #if USE_ITT_BUILD
1559  KMP_FSYNC_SPIN_PREPARE( obj );
1560 #endif /* USE_ITT_BUILD */
1561  __kmp_is_thread_alive( th, &exit_val );
1562  KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc );
1563  KMP_YIELD_SPIN( spins );
1564  } while ( exit_val == STILL_ACTIVE && TCR_4( th->th.th_info.ds.ds_alive ) );
1565 #if USE_ITT_BUILD
1566  if ( exit_val == STILL_ACTIVE ) {
1567  KMP_FSYNC_CANCEL( obj );
1568  } else {
1569  KMP_FSYNC_SPIN_ACQUIRED( obj );
1570  }; // if
1571 #endif /* USE_ITT_BUILD */
1572  }
1573 
1574  __kmp_free_handle( th->th.th_info.ds.ds_thread );
1575 
1576  /*
1577  * NOTE: The ExitProcess(code) system call causes all threads to Terminate
1578  * with a exit_val = code. Because of this we can not rely on
1579  * exit_val having any particular value.
1580  */
1581  if ( exit_val == STILL_ACTIVE ) {
1582  KA_TRACE( 1, ( "__kmp_reap_common: thread still active.\n" ) );
1583  } else if ( (void *) exit_val != (void *) th) {
1584  KA_TRACE( 1, ( "__kmp_reap_common: ExitProcess / TerminateThread used?\n" ) );
1585  }; // if
1586 
1587  KA_TRACE( 10,
1588  (
1589  "__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC "\n",
1590  th->th.th_info.ds.ds_gtid,
1591  th->th.th_info.ds.ds_thread
1592  )
1593  );
1594 
1595  th->th.th_info.ds.ds_thread = 0;
1596  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1597  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1598  th->th.th_info.ds.ds_thread_id = 0;
1599 
1600  KMP_MB(); /* Flush all pending memory write invalidates. */
1601 }
1602 
1603 void
1604 __kmp_reap_monitor( kmp_info_t *th )
1605 {
1606  int status;
1607 
1608  KA_TRACE( 10, ("__kmp_reap_monitor: try to reap %p\n",
1609  (void *) th->th.th_info.ds.ds_thread ) );
1610 
1611  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1612  // If both tid and gtid are 0, it means the monitor did not ever start.
1613  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1614  KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1615  if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1616  return;
1617  }; // if
1618 
1619  KMP_MB(); /* Flush all pending memory write invalidates. */
1620 
1621  status = SetEvent( __kmp_monitor_ev );
1622  if ( status == FALSE ) {
1623  DWORD error = GetLastError();
1624  __kmp_msg(
1625  kmp_ms_fatal,
1626  KMP_MSG( CantSetEvent ),
1627  KMP_ERR( error ),
1628  __kmp_msg_null
1629  );
1630  }
1631  KA_TRACE( 10, ( "__kmp_reap_monitor: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1632  __kmp_reap_common( th );
1633 
1634  __kmp_free_handle( __kmp_monitor_ev );
1635 
1636  KMP_MB(); /* Flush all pending memory write invalidates. */
1637 }
1638 
1639 void
1640 __kmp_reap_worker( kmp_info_t * th )
1641 {
1642  KA_TRACE( 10, ( "__kmp_reap_worker: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1643  __kmp_reap_common( th );
1644 }
1645 
1646 /* ------------------------------------------------------------------------ */
1647 /* ------------------------------------------------------------------------ */
1648 
1649 #if KMP_HANDLE_SIGNALS
1650 
1651 
1652 static void
1653 __kmp_team_handler( int signo )
1654 {
1655  if ( __kmp_global.g.g_abort == 0 ) {
1656  // Stage 1 signal handler, let's shut down all of the threads.
1657  if ( __kmp_debug_buf ) {
1658  __kmp_dump_debug_buffer();
1659  }; // if
1660  KMP_MB(); // Flush all pending memory write invalidates.
1661  TCW_4( __kmp_global.g.g_abort, signo );
1662  KMP_MB(); // Flush all pending memory write invalidates.
1663  TCW_4( __kmp_global.g.g_done, TRUE );
1664  KMP_MB(); // Flush all pending memory write invalidates.
1665  }
1666 } // __kmp_team_handler
1667 
1668 
1669 
1670 static
1671 sig_func_t __kmp_signal( int signum, sig_func_t handler ) {
1672  sig_func_t old = signal( signum, handler );
1673  if ( old == SIG_ERR ) {
1674  int error = errno;
1675  __kmp_msg( kmp_ms_fatal, KMP_MSG( FunctionError, "signal" ), KMP_ERR( error ), __kmp_msg_null );
1676  }; // if
1677  return old;
1678 }
1679 
1680 static void
1681 __kmp_install_one_handler(
1682  int sig,
1683  sig_func_t handler,
1684  int parallel_init
1685 ) {
1686  sig_func_t old;
1687  KMP_MB(); /* Flush all pending memory write invalidates. */
1688  KB_TRACE( 60, ("__kmp_install_one_handler: called: sig=%d\n", sig ) );
1689  if ( parallel_init ) {
1690  old = __kmp_signal( sig, handler );
1691  // SIG_DFL on Windows* OS in NULL or 0.
1692  if ( old == __kmp_sighldrs[ sig ] ) {
1693  __kmp_siginstalled[ sig ] = 1;
1694  } else {
1695  // Restore/keep user's handler if one previously installed.
1696  old = __kmp_signal( sig, old );
1697  }; // if
1698  } else {
1699  // Save initial/system signal handlers to see if user handlers installed.
1700  // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals called once with
1701  // parallel_init == TRUE.
1702  old = __kmp_signal( sig, SIG_DFL );
1703  __kmp_sighldrs[ sig ] = old;
1704  __kmp_signal( sig, old );
1705  }; // if
1706  KMP_MB(); /* Flush all pending memory write invalidates. */
1707 } // __kmp_install_one_handler
1708 
1709 static void
1710 __kmp_remove_one_handler( int sig ) {
1711  if ( __kmp_siginstalled[ sig ] ) {
1712  sig_func_t old;
1713  KMP_MB(); // Flush all pending memory write invalidates.
1714  KB_TRACE( 60, ( "__kmp_remove_one_handler: called: sig=%d\n", sig ) );
1715  old = __kmp_signal( sig, __kmp_sighldrs[ sig ] );
1716  if ( old != __kmp_team_handler ) {
1717  KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1718  old = __kmp_signal( sig, old );
1719  }; // if
1720  __kmp_sighldrs[ sig ] = NULL;
1721  __kmp_siginstalled[ sig ] = 0;
1722  KMP_MB(); // Flush all pending memory write invalidates.
1723  }; // if
1724 } // __kmp_remove_one_handler
1725 
1726 
1727 void
1728 __kmp_install_signals( int parallel_init )
1729 {
1730  KB_TRACE( 10, ( "__kmp_install_signals: called\n" ) );
1731  if ( ! __kmp_handle_signals ) {
1732  KB_TRACE( 10, ( "__kmp_install_signals: KMP_HANDLE_SIGNALS is false - handlers not installed\n" ) );
1733  return;
1734  }; // if
1735  __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1736  __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1737  __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1738  __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1739  __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1740  __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1741 } // __kmp_install_signals
1742 
1743 
1744 void
1745 __kmp_remove_signals( void )
1746 {
1747  int sig;
1748  KB_TRACE( 10, ("__kmp_remove_signals: called\n" ) );
1749  for ( sig = 1; sig < NSIG; ++ sig ) {
1750  __kmp_remove_one_handler( sig );
1751  }; // for sig
1752 } // __kmp_remove_signals
1753 
1754 
1755 #endif // KMP_HANDLE_SIGNALS
1756 
1757 /* Put the thread to sleep for a time period */
1758 void
1759 __kmp_thread_sleep( int millis )
1760 {
1761  DWORD status;
1762 
1763  status = SleepEx( (DWORD) millis, FALSE );
1764  if ( status ) {
1765  DWORD error = GetLastError();
1766  __kmp_msg(
1767  kmp_ms_fatal,
1768  KMP_MSG( FunctionError, "SleepEx()" ),
1769  KMP_ERR( error ),
1770  __kmp_msg_null
1771  );
1772  }
1773 }
1774 
1775 /* Determine whether the given address is mapped into the current address space. */
1776 int
1777 __kmp_is_address_mapped( void * addr )
1778 {
1779  DWORD status;
1780  MEMORY_BASIC_INFORMATION lpBuffer;
1781  SIZE_T dwLength;
1782 
1783  dwLength = sizeof(MEMORY_BASIC_INFORMATION);
1784 
1785  status = VirtualQuery( addr, &lpBuffer, dwLength );
1786 
1787  return !((( lpBuffer.State == MEM_RESERVE) || ( lpBuffer.State == MEM_FREE )) ||
1788  (( lpBuffer.Protect == PAGE_NOACCESS ) || ( lpBuffer.Protect == PAGE_EXECUTE )));
1789 }
1790 
1791 kmp_uint64
1792 __kmp_hardware_timestamp(void)
1793 {
1794  kmp_uint64 r = 0;
1795 
1796  QueryPerformanceCounter((LARGE_INTEGER*) &r);
1797  return r;
1798 }
1799 
1800 /* Free handle and check the error code */
1801 void
1802 __kmp_free_handle( kmp_thread_t tHandle )
1803 {
1804 /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined as HANDLE */
1805  BOOL rc;
1806  rc = CloseHandle( tHandle );
1807  if ( !rc ) {
1808  DWORD error = GetLastError();
1809  __kmp_msg(
1810  kmp_ms_fatal,
1811  KMP_MSG( CantCloseHandle ),
1812  KMP_ERR( error ),
1813  __kmp_msg_null
1814  );
1815  }
1816 }
1817 
1818 int
1819 __kmp_get_load_balance( int max ) {
1820 
1821  static ULONG glb_buff_size = 100 * 1024;
1822 
1823  static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */
1824  static double glb_call_time = 0; /* Thread balance algorithm call time */
1825 
1826  int running_threads = 0; // Number of running threads in the system.
1827  NTSTATUS status = 0;
1828  ULONG buff_size = 0;
1829  ULONG info_size = 0;
1830  void * buffer = NULL;
1831  PSYSTEM_PROCESS_INFORMATION spi = NULL;
1832  int first_time = 1;
1833 
1834  double call_time = 0.0; //start, finish;
1835 
1836  __kmp_elapsed( & call_time );
1837 
1838  if ( glb_call_time &&
1839  ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
1840  running_threads = glb_running_threads;
1841  goto finish;
1842  }
1843  glb_call_time = call_time;
1844 
1845  // Do not spend time on running algorithm if we have a permanent error.
1846  if ( NtQuerySystemInformation == NULL ) {
1847  running_threads = -1;
1848  goto finish;
1849  }; // if
1850 
1851  if ( max <= 0 ) {
1852  max = INT_MAX;
1853  }; // if
1854 
1855  do {
1856 
1857  if ( first_time ) {
1858  buff_size = glb_buff_size;
1859  } else {
1860  buff_size = 2 * buff_size;
1861  }
1862 
1863  buffer = KMP_INTERNAL_REALLOC( buffer, buff_size );
1864  if ( buffer == NULL ) {
1865  running_threads = -1;
1866  goto finish;
1867  }; // if
1868  status = NtQuerySystemInformation( SystemProcessInformation, buffer, buff_size, & info_size );
1869  first_time = 0;
1870 
1871  } while ( status == STATUS_INFO_LENGTH_MISMATCH );
1872  glb_buff_size = buff_size;
1873 
1874  #define CHECK( cond ) \
1875  { \
1876  KMP_DEBUG_ASSERT( cond ); \
1877  if ( ! ( cond ) ) { \
1878  running_threads = -1; \
1879  goto finish; \
1880  } \
1881  }
1882 
1883  CHECK( buff_size >= info_size );
1884  spi = PSYSTEM_PROCESS_INFORMATION( buffer );
1885  for ( ; ; ) {
1886  ptrdiff_t offset = uintptr_t( spi ) - uintptr_t( buffer );
1887  CHECK( 0 <= offset && offset + sizeof( SYSTEM_PROCESS_INFORMATION ) < info_size );
1888  HANDLE pid = spi->ProcessId;
1889  ULONG num = spi->NumberOfThreads;
1890  CHECK( num >= 1 );
1891  size_t spi_size = sizeof( SYSTEM_PROCESS_INFORMATION ) + sizeof( SYSTEM_THREAD ) * ( num - 1 );
1892  CHECK( offset + spi_size < info_size ); // Make sure process info record fits the buffer.
1893  if ( spi->NextEntryOffset != 0 ) {
1894  CHECK( spi_size <= spi->NextEntryOffset ); // And do not overlap with the next record.
1895  }; // if
1896  // pid == 0 corresponds to the System Idle Process. It always has running threads
1897  // on all cores. So, we don't consider the running threads of this process.
1898  if ( pid != 0 ) {
1899  for ( int i = 0; i < num; ++ i ) {
1900  THREAD_STATE state = spi->Threads[ i ].State;
1901  // Count threads that have Ready or Running state.
1902  // !!! TODO: Why comment does not match the code???
1903  if ( state == StateRunning ) {
1904  ++ running_threads;
1905  // Stop counting running threads if the number is already greater than
1906  // the number of available cores
1907  if ( running_threads >= max ) {
1908  goto finish;
1909  }
1910  } // if
1911  }; // for i
1912  } // if
1913  if ( spi->NextEntryOffset == 0 ) {
1914  break;
1915  }; // if
1916  spi = PSYSTEM_PROCESS_INFORMATION( uintptr_t( spi ) + spi->NextEntryOffset );
1917  }; // forever
1918 
1919  #undef CHECK
1920 
1921  finish: // Clean up and exit.
1922 
1923  if ( buffer != NULL ) {
1924  KMP_INTERNAL_FREE( buffer );
1925  }; // if
1926 
1927  glb_running_threads = running_threads;
1928 
1929  return running_threads;
1930 
1931 } //__kmp_get_load_balance()
1932