LLVM OpenMP* Runtime Library
kmp_os.h
1 /*
2  * kmp_os.h -- KPTS runtime header file.
3  */
4 
5 
6 //===----------------------------------------------------------------------===//
7 //
8 // The LLVM Compiler Infrastructure
9 //
10 // This file is dual licensed under the MIT and the University of Illinois Open
11 // Source Licenses. See LICENSE.txt for details.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 
16 #ifndef KMP_OS_H
17 #define KMP_OS_H
18 
19 #include <stdlib.h>
20 
21 #define KMP_FTN_PLAIN 1
22 #define KMP_FTN_APPEND 2
23 #define KMP_FTN_UPPER 3
24 /*
25 #define KMP_FTN_PREPEND 4
26 #define KMP_FTN_UAPPEND 5
27 */
28 
29 #define KMP_PTR_SKIP (sizeof(void*))
30 
31 /* -------------------------- Compiler variations ------------------------ */
32 
33 #define KMP_OFF 0
34 #define KMP_ON 1
35 
36 #define KMP_MEM_CONS_VOLATILE 0
37 #define KMP_MEM_CONS_FENCE 1
38 
39 #ifndef KMP_MEM_CONS_MODEL
40 # define KMP_MEM_CONS_MODEL KMP_MEM_CONS_VOLATILE
41 #endif
42 
43 /* ------------------------- Compiler recognition ---------------------- */
44 #define KMP_COMPILER_ICC 0
45 #define KMP_COMPILER_GCC 0
46 #define KMP_COMPILER_CLANG 0
47 #define KMP_COMPILER_MSVC 0
48 
49 #if defined( __INTEL_COMPILER )
50 # undef KMP_COMPILER_ICC
51 # define KMP_COMPILER_ICC 1
52 #elif defined( __clang__ )
53 # undef KMP_COMPILER_CLANG
54 # define KMP_COMPILER_CLANG 1
55 #elif defined( __GNUC__ )
56 # undef KMP_COMPILER_GCC
57 # define KMP_COMPILER_GCC 1
58 #elif defined( _MSC_VER )
59 # undef KMP_COMPILER_MSVC
60 # define KMP_COMPILER_MSVC 1
61 #else
62 # error Unknown compiler
63 #endif
64 
65 #include "kmp_platform.h"
66 
67 #if (KMP_OS_LINUX || KMP_OS_WINDOWS) && !KMP_OS_CNK && !KMP_ARCH_PPC64
68 # define KMP_AFFINITY_SUPPORTED 1
69 # if KMP_OS_WINDOWS && KMP_ARCH_X86_64
70 # define KMP_GROUP_AFFINITY 1
71 # else
72 # define KMP_GROUP_AFFINITY 0
73 # endif
74 #else
75 # define KMP_AFFINITY_SUPPORTED 0
76 # define KMP_GROUP_AFFINITY 0
77 #endif
78 
79 /* Check for quad-precision extension. */
80 #define KMP_HAVE_QUAD 0
81 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
82 # if KMP_COMPILER_ICC
83  /* _Quad is already defined for icc */
84 # undef KMP_HAVE_QUAD
85 # define KMP_HAVE_QUAD 1
86 # elif KMP_COMPILER_CLANG
87  /* Clang doesn't support a software-implemented
88  128-bit extended precision type yet */
89  typedef long double _Quad;
90 # elif KMP_COMPILER_GCC
91  typedef __float128 _Quad;
92 # undef KMP_HAVE_QUAD
93 # define KMP_HAVE_QUAD 1
94 # elif KMP_COMPILER_MSVC
95  typedef long double _Quad;
96 # endif
97 #else
98 # if __LDBL_MAX_EXP__ >= 16384 && KMP_COMPILER_GCC
99  typedef long double _Quad;
100 # undef KMP_HAVE_QUAD
101 # define KMP_HAVE_QUAD 1
102 # endif
103 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
104 
105 #if KMP_OS_WINDOWS
106  typedef char kmp_int8;
107  typedef unsigned char kmp_uint8;
108  typedef short kmp_int16;
109  typedef unsigned short kmp_uint16;
110  typedef int kmp_int32;
111  typedef unsigned int kmp_uint32;
112 # define KMP_INT32_SPEC "d"
113 # define KMP_UINT32_SPEC "u"
114 # ifndef KMP_STRUCT64
115  typedef __int64 kmp_int64;
116  typedef unsigned __int64 kmp_uint64;
117  #define KMP_INT64_SPEC "I64d"
118  #define KMP_UINT64_SPEC "I64u"
119 # else
120  struct kmp_struct64 {
121  kmp_int32 a,b;
122  };
123  typedef struct kmp_struct64 kmp_int64;
124  typedef struct kmp_struct64 kmp_uint64;
125  /* Not sure what to use for KMP_[U]INT64_SPEC here */
126 # endif
127 # if KMP_ARCH_X86_64
128 # define KMP_INTPTR 1
129  typedef __int64 kmp_intptr_t;
130  typedef unsigned __int64 kmp_uintptr_t;
131 # define KMP_INTPTR_SPEC "I64d"
132 # define KMP_UINTPTR_SPEC "I64u"
133 # endif
134 #endif /* KMP_OS_WINDOWS */
135 
136 #if KMP_OS_UNIX
137  typedef char kmp_int8;
138  typedef unsigned char kmp_uint8;
139  typedef short kmp_int16;
140  typedef unsigned short kmp_uint16;
141  typedef int kmp_int32;
142  typedef unsigned int kmp_uint32;
143  typedef long long kmp_int64;
144  typedef unsigned long long kmp_uint64;
145 # define KMP_INT32_SPEC "d"
146 # define KMP_UINT32_SPEC "u"
147 # define KMP_INT64_SPEC "lld"
148 # define KMP_UINT64_SPEC "llu"
149 #endif /* KMP_OS_UNIX */
150 
151 #if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS
152 # define KMP_SIZE_T_SPEC KMP_UINT32_SPEC
153 #elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64
154 # define KMP_SIZE_T_SPEC KMP_UINT64_SPEC
155 #else
156 # error "Can't determine size_t printf format specifier."
157 #endif
158 
159 #if KMP_ARCH_X86
160 # define KMP_SIZE_T_MAX (0xFFFFFFFF)
161 #else
162 # define KMP_SIZE_T_MAX (0xFFFFFFFFFFFFFFFF)
163 #endif
164 
165 typedef size_t kmp_size_t;
166 typedef float kmp_real32;
167 typedef double kmp_real64;
168 
169 #ifndef KMP_INTPTR
170 # define KMP_INTPTR 1
171  typedef long kmp_intptr_t;
172  typedef unsigned long kmp_uintptr_t;
173 # define KMP_INTPTR_SPEC "ld"
174 # define KMP_UINTPTR_SPEC "lu"
175 #endif
176 
177 #ifdef KMP_I8
178  typedef kmp_int64 kmp_int;
179  typedef kmp_uint64 kmp_uint;
180 # define KMP_INT_SPEC KMP_INT64_SPEC
181 # define KMP_UINT_SPEC KMP_UINT64_SPEC
182 # define KMP_INT_MAX ((kmp_int64)0x7FFFFFFFFFFFFFFFLL)
183 # define KMP_INT_MIN ((kmp_int64)0x8000000000000000LL)
184 #else
185  typedef kmp_int32 kmp_int;
186  typedef kmp_uint32 kmp_uint;
187 # define KMP_INT_SPEC KMP_INT32_SPEC
188 # define KMP_UINT_SPEC KMP_UINT32_SPEC
189 # define KMP_INT_MAX ((kmp_int32)0x7FFFFFFF)
190 # define KMP_INT_MIN ((kmp_int32)0x80000000)
191 #endif /* KMP_I8 */
192 
193 #ifdef __cplusplus
194  //-------------------------------------------------------------------------
195  // template for debug prints specification ( d, u, lld, llu ), and to obtain
196  // signed/unsigned flavors of a type
197  template< typename T >
198  struct traits_t {
199  typedef T signed_t;
200  typedef T unsigned_t;
201  typedef T floating_t;
202  static char const * spec;
203  };
204  // int
205  template<>
206  struct traits_t< signed int > {
207  typedef signed int signed_t;
208  typedef unsigned int unsigned_t;
209  typedef double floating_t;
210  static char const * spec;
211  };
212  // unsigned int
213  template<>
214  struct traits_t< unsigned int > {
215  typedef signed int signed_t;
216  typedef unsigned int unsigned_t;
217  typedef double floating_t;
218  static char const * spec;
219  };
220  // long long
221  template<>
222  struct traits_t< signed long long > {
223  typedef signed long long signed_t;
224  typedef unsigned long long unsigned_t;
225  typedef long double floating_t;
226  static char const * spec;
227  };
228  // unsigned long long
229  template<>
230  struct traits_t< unsigned long long > {
231  typedef signed long long signed_t;
232  typedef unsigned long long unsigned_t;
233  typedef long double floating_t;
234  static char const * spec;
235  };
236  //-------------------------------------------------------------------------
237 #endif // __cplusplus
238 
239 #define KMP_EXPORT extern /* export declaration in guide libraries */
240 
241 #if __GNUC__ >= 4
242  #define __forceinline __inline
243 #endif
244 
245 #define PAGE_SIZE (0x4000)
246 #define PAGE_ALIGNED(_addr) ( ! ((size_t) _addr & \
247  (size_t)(PAGE_SIZE - 1)))
248 #define ALIGN_TO_PAGE(x) (void *)(((size_t)(x)) & ~((size_t)(PAGE_SIZE - 1)))
249 
250 /* ---------------------- Support for cache alignment, padding, etc. -----------------*/
251 
252 #ifdef __cplusplus
253 extern "C" {
254 #endif // __cplusplus
255 
256 #define INTERNODE_CACHE_LINE 4096 /* for multi-node systems */
257 
258 /* Define the default size of the cache line */
259 #ifndef CACHE_LINE
260  #define CACHE_LINE 128 /* cache line size in bytes */
261 #else
262  #if ( CACHE_LINE < 64 ) && ! defined( KMP_OS_DARWIN )
263  // 2006-02-13: This produces too many warnings on OS X*. Disable it for a while...
264  #warning CACHE_LINE is too small.
265  #endif
266 #endif /* CACHE_LINE */
267 
268 #define KMP_CACHE_PREFETCH(ADDR) /* nothing */
269 
270 /* Temporary note: if performance testing of this passes, we can remove
271  all references to KMP_DO_ALIGN and replace with KMP_ALIGN. */
272 #if KMP_OS_UNIX && defined(__GNUC__)
273 # define KMP_DO_ALIGN(bytes) __attribute__((aligned(bytes)))
274 # define KMP_ALIGN_CACHE __attribute__((aligned(CACHE_LINE)))
275 # define KMP_ALIGN_CACHE_INTERNODE __attribute__((aligned(INTERNODE_CACHE_LINE)))
276 # define KMP_ALIGN(bytes) __attribute__((aligned(bytes)))
277 #else
278 # define KMP_DO_ALIGN(bytes) __declspec( align(bytes) )
279 # define KMP_ALIGN_CACHE __declspec( align(CACHE_LINE) )
280 # define KMP_ALIGN_CACHE_INTERNODE __declspec( align(INTERNODE_CACHE_LINE) )
281 # define KMP_ALIGN(bytes) __declspec( align(bytes) )
282 #endif
283 
284 #if defined(__MIC__) || defined(__MIC2__)
285  #define KMP_MIC 1
286 // Intel(R) Composer XE (13.0) defines both __MIC__ and __MIC2__ !
287 # if __MIC2__ || __KNC__
288  #define KMP_MIC1 0
289  #define KMP_MIC2 1
290 # else
291  #define KMP_MIC1 1
292  #define KMP_MIC2 0
293 # endif
294 #else
295  #define KMP_MIC 0
296  #define KMP_MIC1 0
297  #define KMP_MIC2 0
298 #endif
299 
300 /* General purpose fence types for memory operations */
301 enum kmp_mem_fence_type {
302  kmp_no_fence, /* No memory fence */
303  kmp_acquire_fence, /* Acquire (read) memory fence */
304  kmp_release_fence, /* Release (write) memory fence */
305  kmp_full_fence /* Full (read+write) memory fence */
306 };
307 
308 
309 //
310 // Synchronization primitives
311 //
312 
313 #if KMP_ASM_INTRINS && KMP_OS_WINDOWS
314 
315 #include <Windows.h>
316 
317 #pragma intrinsic(InterlockedExchangeAdd)
318 #pragma intrinsic(InterlockedCompareExchange)
319 #pragma intrinsic(InterlockedExchange)
320 #pragma intrinsic(InterlockedExchange64)
321 
322 //
323 // Using InterlockedIncrement / InterlockedDecrement causes a library loading
324 // ordering problem, so we use InterlockedExchangeAdd instead.
325 //
326 # define KMP_TEST_THEN_INC32(p) InterlockedExchangeAdd( (volatile long *)(p), 1 )
327 # define KMP_TEST_THEN_INC_ACQ32(p) InterlockedExchangeAdd( (volatile long *)(p), 1 )
328 # define KMP_TEST_THEN_ADD4_32(p) InterlockedExchangeAdd( (volatile long *)(p), 4 )
329 # define KMP_TEST_THEN_ADD4_ACQ32(p) InterlockedExchangeAdd( (volatile long *)(p), 4 )
330 # define KMP_TEST_THEN_DEC32(p) InterlockedExchangeAdd( (volatile long *)(p), -1 )
331 # define KMP_TEST_THEN_DEC_ACQ32(p) InterlockedExchangeAdd( (volatile long *)(p), -1 )
332 # define KMP_TEST_THEN_ADD32(p, v) InterlockedExchangeAdd( (volatile long *)(p), (v) )
333 
334 extern kmp_int8 __kmp_test_then_add8( volatile kmp_int8 *p, kmp_int8 v );
335 extern kmp_int8 __kmp_test_then_or8( volatile kmp_int8 *p, kmp_int8 v );
336 extern kmp_int8 __kmp_test_then_and8( volatile kmp_int8 *p, kmp_int8 v );
337 # define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) InterlockedCompareExchange( (volatile long *)(p),(long)(sv),(long)(cv) )
338 
339 # define KMP_XCHG_FIXED32(p, v) InterlockedExchange( (volatile long *)(p), (long)(v) )
340 # define KMP_XCHG_FIXED64(p, v) InterlockedExchange64( (volatile kmp_int64 *)(p), (kmp_int64)(v) )
341 
342 inline kmp_real32 KMP_XCHG_REAL32( volatile kmp_real32 *p, kmp_real32 v)
343 {
344  kmp_int32 tmp = InterlockedExchange( (volatile long *)p, *(long *)&v);
345  return *(kmp_real32*)&tmp;
346 }
347 
348 //
349 // Routines that we still need to implement in assembly.
350 //
351 extern kmp_int32 __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 v );
352 extern kmp_int32 __kmp_test_then_or32( volatile kmp_int32 *p, kmp_int32 v );
353 extern kmp_int32 __kmp_test_then_and32( volatile kmp_int32 *p, kmp_int32 v );
354 extern kmp_int64 __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 v );
355 extern kmp_int64 __kmp_test_then_or64( volatile kmp_int64 *p, kmp_int64 v );
356 extern kmp_int64 __kmp_test_then_and64( volatile kmp_int64 *p, kmp_int64 v );
357 
358 extern kmp_int8 __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
359 extern kmp_int16 __kmp_compare_and_store16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
360 extern kmp_int32 __kmp_compare_and_store32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
361 extern kmp_int32 __kmp_compare_and_store64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
362 extern kmp_int8 __kmp_compare_and_store_ret8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
363 extern kmp_int16 __kmp_compare_and_store_ret16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
364 extern kmp_int32 __kmp_compare_and_store_ret32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
365 extern kmp_int64 __kmp_compare_and_store_ret64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
366 
367 extern kmp_int8 __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 v );
368 extern kmp_int16 __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 v );
369 extern kmp_int32 __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 v );
370 extern kmp_int64 __kmp_xchg_fixed64( volatile kmp_int64 *p, kmp_int64 v );
371 extern kmp_real32 __kmp_xchg_real32( volatile kmp_real32 *p, kmp_real32 v );
372 extern kmp_real64 __kmp_xchg_real64( volatile kmp_real64 *p, kmp_real64 v );
373 # define KMP_TEST_THEN_ADD8(p, v) __kmp_test_then_add8( (p), (v) )
374 
375 //# define KMP_TEST_THEN_INC32(p) __kmp_test_then_add32( (p), 1 )
376 # define KMP_TEST_THEN_OR8(p, v) __kmp_test_then_or8( (p), (v) )
377 # define KMP_TEST_THEN_AND8(p, v) __kmp_test_then_and8( (p), (v) )
378 //# define KMP_TEST_THEN_INC_ACQ32(p) __kmp_test_then_add32( (p), 1 )
379 # define KMP_TEST_THEN_INC64(p) __kmp_test_then_add64( (p), 1LL )
380 # define KMP_TEST_THEN_INC_ACQ64(p) __kmp_test_then_add64( (p), 1LL )
381 //# define KMP_TEST_THEN_ADD4_32(p) __kmp_test_then_add32( (p), 4 )
382 //# define KMP_TEST_THEN_ADD4_ACQ32(p) __kmp_test_then_add32( (p), 4 )
383 # define KMP_TEST_THEN_ADD4_64(p) __kmp_test_then_add64( (p), 4LL )
384 # define KMP_TEST_THEN_ADD4_ACQ64(p) __kmp_test_then_add64( (p), 4LL )
385 //# define KMP_TEST_THEN_DEC32(p) __kmp_test_then_add32( (p), -1 )
386 //# define KMP_TEST_THEN_DEC_ACQ32(p) __kmp_test_then_add32( (p), -1 )
387 # define KMP_TEST_THEN_DEC64(p) __kmp_test_then_add64( (p), -1LL )
388 # define KMP_TEST_THEN_DEC_ACQ64(p) __kmp_test_then_add64( (p), -1LL )
389 //# define KMP_TEST_THEN_ADD32(p, v) __kmp_test_then_add32( (p), (v) )
390 # define KMP_TEST_THEN_ADD64(p, v) __kmp_test_then_add64( (p), (v) )
391 
392 # define KMP_TEST_THEN_OR32(p, v) __kmp_test_then_or32( (p), (v) )
393 # define KMP_TEST_THEN_AND32(p, v) __kmp_test_then_and32( (p), (v) )
394 # define KMP_TEST_THEN_OR64(p, v) __kmp_test_then_or64( (p), (v) )
395 # define KMP_TEST_THEN_AND64(p, v) __kmp_test_then_and64( (p), (v) )
396 
397 # define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
398 # define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
399 # define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
400 # define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
401 # define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
402 # define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
403 # define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
404 # define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
405 
406 # if KMP_ARCH_X86
407 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store32( (volatile kmp_int32*)(p), (kmp_int32)(cv), (kmp_int32)(sv) )
408 # else /* 64 bit pointers */
409 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store64( (volatile kmp_int64*)(p), (kmp_int64)(cv), (kmp_int64)(sv) )
410 # endif /* KMP_ARCH_X86 */
411 
412 # define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) __kmp_compare_and_store_ret8( (p), (cv), (sv) )
413 # define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __kmp_compare_and_store_ret16( (p), (cv), (sv) )
414 //# define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) __kmp_compare_and_store_ret32( (p), (cv), (sv) )
415 # define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) __kmp_compare_and_store_ret64( (p), (cv), (sv) )
416 
417 # define KMP_XCHG_FIXED8(p, v) __kmp_xchg_fixed8( (p), (v) );
418 # define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16( (p), (v) );
419 //# define KMP_XCHG_FIXED32(p, v) __kmp_xchg_fixed32( (p), (v) );
420 //# define KMP_XCHG_FIXED64(p, v) __kmp_xchg_fixed64( (p), (v) );
421 //# define KMP_XCHG_REAL32(p, v) __kmp_xchg_real32( (p), (v) );
422 # define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64( (p), (v) );
423 
424 
425 #elif (KMP_ASM_INTRINS && KMP_OS_UNIX) || !(KMP_ARCH_X86 || KMP_ARCH_X86_64)
426 # define KMP_TEST_THEN_ADD8(p, v) __sync_fetch_and_add( (kmp_int8 *)(p), (v) )
427 
428 /* cast p to correct type so that proper intrinsic will be used */
429 # define KMP_TEST_THEN_INC32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 1 )
430 # define KMP_TEST_THEN_OR8(p, v) __sync_fetch_and_or( (kmp_int8 *)(p), (v) )
431 # define KMP_TEST_THEN_AND8(p, v) __sync_fetch_and_and( (kmp_int8 *)(p), (v) )
432 # define KMP_TEST_THEN_INC_ACQ32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 1 )
433 # define KMP_TEST_THEN_INC64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 1LL )
434 # define KMP_TEST_THEN_INC_ACQ64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 1LL )
435 # define KMP_TEST_THEN_ADD4_32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 4 )
436 # define KMP_TEST_THEN_ADD4_ACQ32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 4 )
437 # define KMP_TEST_THEN_ADD4_64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 4LL )
438 # define KMP_TEST_THEN_ADD4_ACQ64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 4LL )
439 # define KMP_TEST_THEN_DEC32(p) __sync_fetch_and_sub( (kmp_int32 *)(p), 1 )
440 # define KMP_TEST_THEN_DEC_ACQ32(p) __sync_fetch_and_sub( (kmp_int32 *)(p), 1 )
441 # define KMP_TEST_THEN_DEC64(p) __sync_fetch_and_sub( (kmp_int64 *)(p), 1LL )
442 # define KMP_TEST_THEN_DEC_ACQ64(p) __sync_fetch_and_sub( (kmp_int64 *)(p), 1LL )
443 # define KMP_TEST_THEN_ADD32(p, v) __sync_fetch_and_add( (kmp_int32 *)(p), (v) )
444 # define KMP_TEST_THEN_ADD64(p, v) __sync_fetch_and_add( (kmp_int64 *)(p), (v) )
445 
446 # define KMP_TEST_THEN_OR32(p, v) __sync_fetch_and_or( (kmp_int32 *)(p), (v) )
447 # define KMP_TEST_THEN_AND32(p, v) __sync_fetch_and_and( (kmp_int32 *)(p), (v) )
448 # define KMP_TEST_THEN_OR64(p, v) __sync_fetch_and_or( (kmp_int64 *)(p), (v) )
449 # define KMP_TEST_THEN_AND64(p, v) __sync_fetch_and_and( (kmp_int64 *)(p), (v) )
450 
451 # define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint8 *)(p),(kmp_uint8)(cv),(kmp_uint8)(sv) )
452 # define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint8 *)(p),(kmp_uint8)(cv),(kmp_uint8)(sv) )
453 # define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint16 *)(p),(kmp_uint16)(cv),(kmp_uint16)(sv) )
454 # define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint16 *)(p),(kmp_uint16)(cv),(kmp_uint16)(sv) )
455 # define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint32 *)(p),(kmp_uint32)(cv),(kmp_uint32)(sv) )
456 # define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint32 *)(p),(kmp_uint32)(cv),(kmp_uint32)(sv) )
457 # define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint64 *)(p),(kmp_uint64)(cv),(kmp_uint64)(sv) )
458 # define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint64 *)(p),(kmp_uint64)(cv),(kmp_uint64)(sv) )
459 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __sync_bool_compare_and_swap( (volatile void **)(p),(void *)(cv),(void *)(sv) )
460 
461 # define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint8 *)(p),(kmp_uint8)(cv),(kmp_uint8)(sv) )
462 # define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint16 *)(p),(kmp_uint16)(cv),(kmp_uint16)(sv) )
463 # define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint32 *)(p),(kmp_uint32)(cv),(kmp_uint32)(sv) )
464 # define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint64 *)(p),(kmp_uint64)(cv),(kmp_uint64)(sv) )
465 
466 #define KMP_XCHG_FIXED8(p, v) __sync_lock_test_and_set( (volatile kmp_uint8 *)(p), (kmp_uint8)(v) )
467 #define KMP_XCHG_FIXED16(p, v) __sync_lock_test_and_set( (volatile kmp_uint16 *)(p), (kmp_uint16)(v) )
468 #define KMP_XCHG_FIXED32(p, v) __sync_lock_test_and_set( (volatile kmp_uint32 *)(p), (kmp_uint32)(v) )
469 #define KMP_XCHG_FIXED64(p, v) __sync_lock_test_and_set( (volatile kmp_uint64 *)(p), (kmp_uint64)(v) )
470 
471 extern kmp_int8 __kmp_test_then_add8( volatile kmp_int8 *p, kmp_int8 v );
472 extern kmp_int8 __kmp_test_then_or8( volatile kmp_int8 *p, kmp_int8 v );
473 extern kmp_int8 __kmp_test_then_and8( volatile kmp_int8 *p, kmp_int8 v );
474 inline kmp_real32 KMP_XCHG_REAL32( volatile kmp_real32 *p, kmp_real32 v)
475 {
476  kmp_int32 tmp = __sync_lock_test_and_set( (kmp_int32*)p, *(kmp_int32*)&v);
477  return *(kmp_real32*)&tmp;
478 }
479 
480 inline kmp_real64 KMP_XCHG_REAL64( volatile kmp_real64 *p, kmp_real64 v)
481 {
482  kmp_int64 tmp = __sync_lock_test_and_set( (kmp_int64*)p, *(kmp_int64*)&v);
483  return *(kmp_real64*)&tmp;
484 }
485 
486 #else
487 
488 extern kmp_int32 __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 v );
489 extern kmp_int32 __kmp_test_then_or32( volatile kmp_int32 *p, kmp_int32 v );
490 extern kmp_int32 __kmp_test_then_and32( volatile kmp_int32 *p, kmp_int32 v );
491 extern kmp_int64 __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 v );
492 extern kmp_int64 __kmp_test_then_or64( volatile kmp_int64 *p, kmp_int64 v );
493 extern kmp_int64 __kmp_test_then_and64( volatile kmp_int64 *p, kmp_int64 v );
494 
495 extern kmp_int8 __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
496 extern kmp_int16 __kmp_compare_and_store16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
497 extern kmp_int32 __kmp_compare_and_store32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
498 extern kmp_int32 __kmp_compare_and_store64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
499 extern kmp_int8 __kmp_compare_and_store_ret8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
500 extern kmp_int16 __kmp_compare_and_store_ret16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
501 extern kmp_int32 __kmp_compare_and_store_ret32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
502 extern kmp_int64 __kmp_compare_and_store_ret64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
503 
504 extern kmp_int8 __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 v );
505 extern kmp_int16 __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 v );
506 extern kmp_int32 __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 v );
507 extern kmp_int64 __kmp_xchg_fixed64( volatile kmp_int64 *p, kmp_int64 v );
508 extern kmp_real32 __kmp_xchg_real32( volatile kmp_real32 *p, kmp_real32 v );
509 # define KMP_TEST_THEN_ADD8(p, v) __kmp_test_then_add8( (p), (v) )
510 extern kmp_real64 __kmp_xchg_real64( volatile kmp_real64 *p, kmp_real64 v );
511 
512 # define KMP_TEST_THEN_INC32(p) __kmp_test_then_add32( (p), 1 )
513 # define KMP_TEST_THEN_OR8(p, v) __kmp_test_then_or8( (p), (v) )
514 # define KMP_TEST_THEN_AND8(p, v) __kmp_test_then_and8( (p), (v) )
515 # define KMP_TEST_THEN_INC_ACQ32(p) __kmp_test_then_add32( (p), 1 )
516 # define KMP_TEST_THEN_INC64(p) __kmp_test_then_add64( (p), 1LL )
517 # define KMP_TEST_THEN_INC_ACQ64(p) __kmp_test_then_add64( (p), 1LL )
518 # define KMP_TEST_THEN_ADD4_32(p) __kmp_test_then_add32( (p), 4 )
519 # define KMP_TEST_THEN_ADD4_ACQ32(p) __kmp_test_then_add32( (p), 4 )
520 # define KMP_TEST_THEN_ADD4_64(p) __kmp_test_then_add64( (p), 4LL )
521 # define KMP_TEST_THEN_ADD4_ACQ64(p) __kmp_test_then_add64( (p), 4LL )
522 # define KMP_TEST_THEN_DEC32(p) __kmp_test_then_add32( (p), -1 )
523 # define KMP_TEST_THEN_DEC_ACQ32(p) __kmp_test_then_add32( (p), -1 )
524 # define KMP_TEST_THEN_DEC64(p) __kmp_test_then_add64( (p), -1LL )
525 # define KMP_TEST_THEN_DEC_ACQ64(p) __kmp_test_then_add64( (p), -1LL )
526 # define KMP_TEST_THEN_ADD32(p, v) __kmp_test_then_add32( (p), (v) )
527 # define KMP_TEST_THEN_ADD64(p, v) __kmp_test_then_add64( (p), (v) )
528 
529 # define KMP_TEST_THEN_OR32(p, v) __kmp_test_then_or32( (p), (v) )
530 # define KMP_TEST_THEN_AND32(p, v) __kmp_test_then_and32( (p), (v) )
531 # define KMP_TEST_THEN_OR64(p, v) __kmp_test_then_or64( (p), (v) )
532 # define KMP_TEST_THEN_AND64(p, v) __kmp_test_then_and64( (p), (v) )
533 
534 # define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
535 # define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
536 # define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
537 # define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
538 # define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
539 # define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
540 # define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
541 # define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
542 
543 # if KMP_ARCH_X86
544 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store32( (volatile kmp_int32*)(p), (kmp_int32)(cv), (kmp_int32)(sv) )
545 # else /* 64 bit pointers */
546 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store64( (volatile kmp_int64*)(p), (kmp_int64)(cv), (kmp_int64)(sv) )
547 # endif /* KMP_ARCH_X86 */
548 
549 # define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) __kmp_compare_and_store_ret8( (p), (cv), (sv) )
550 # define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __kmp_compare_and_store_ret16( (p), (cv), (sv) )
551 # define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) __kmp_compare_and_store_ret32( (p), (cv), (sv) )
552 # define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) __kmp_compare_and_store_ret64( (p), (cv), (sv) )
553 
554 # define KMP_XCHG_FIXED8(p, v) __kmp_xchg_fixed8( (p), (v) );
555 # define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16( (p), (v) );
556 # define KMP_XCHG_FIXED32(p, v) __kmp_xchg_fixed32( (p), (v) );
557 # define KMP_XCHG_FIXED64(p, v) __kmp_xchg_fixed64( (p), (v) );
558 # define KMP_XCHG_REAL32(p, v) __kmp_xchg_real32( (p), (v) );
559 # define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64( (p), (v) );
560 
561 #endif /* KMP_ASM_INTRINS */
562 
563 
564 /* ------------- relaxed consistency memory model stuff ------------------ */
565 
566 #if KMP_OS_WINDOWS
567 # ifdef __ABSOFT_WIN
568 # define KMP_MB() asm ("nop")
569 # define KMP_IMB() asm ("nop")
570 # else
571 # define KMP_MB() /* _asm{ nop } */
572 # define KMP_IMB() /* _asm{ nop } */
573 # endif
574 #endif /* KMP_OS_WINDOWS */
575 
576 #if KMP_ARCH_PPC64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64
577 # define KMP_MB() __sync_synchronize()
578 #endif
579 
580 #ifndef KMP_MB
581 # define KMP_MB() /* nothing to do */
582 #endif
583 
584 #ifndef KMP_IMB
585 # define KMP_IMB() /* nothing to do */
586 #endif
587 
588 #ifndef KMP_ST_REL32
589 # define KMP_ST_REL32(A,D) ( *(A) = (D) )
590 #endif
591 
592 #ifndef KMP_ST_REL64
593 # define KMP_ST_REL64(A,D) ( *(A) = (D) )
594 #endif
595 
596 #ifndef KMP_LD_ACQ32
597 # define KMP_LD_ACQ32(A) ( *(A) )
598 #endif
599 
600 #ifndef KMP_LD_ACQ64
601 # define KMP_LD_ACQ64(A) ( *(A) )
602 #endif
603 
604 #define TCR_1(a) (a)
605 #define TCW_1(a,b) (a) = (b)
606 /* ------------------------------------------------------------------------ */
607 //
608 // FIXME - maybe this should this be
609 //
610 // #define TCR_4(a) (*(volatile kmp_int32 *)(&a))
611 // #define TCW_4(a,b) (a) = (*(volatile kmp_int32 *)&(b))
612 //
613 // #define TCR_8(a) (*(volatile kmp_int64 *)(a))
614 // #define TCW_8(a,b) (a) = (*(volatile kmp_int64 *)(&b))
615 //
616 // I'm fairly certain this is the correct thing to do, but I'm afraid
617 // of performance regressions.
618 //
619 
620 #define TCR_4(a) (a)
621 #define TCW_4(a,b) (a) = (b)
622 #define TCR_8(a) (a)
623 #define TCW_8(a,b) (a) = (b)
624 #define TCR_SYNC_4(a) (a)
625 #define TCW_SYNC_4(a,b) (a) = (b)
626 #define TCX_SYNC_4(a,b,c) KMP_COMPARE_AND_STORE_REL32((volatile kmp_int32 *)(volatile void *)&(a), (kmp_int32)(b), (kmp_int32)(c))
627 #define TCR_SYNC_8(a) (a)
628 #define TCW_SYNC_8(a,b) (a) = (b)
629 #define TCX_SYNC_8(a,b,c) KMP_COMPARE_AND_STORE_REL64((volatile kmp_int64 *)(volatile void *)&(a), (kmp_int64)(b), (kmp_int64)(c))
630 
631 #if KMP_ARCH_X86 || KMP_ARCH_MIPS
632 // What about ARM?
633  #define TCR_PTR(a) ((void *)TCR_4(a))
634  #define TCW_PTR(a,b) TCW_4((a),(b))
635  #define TCR_SYNC_PTR(a) ((void *)TCR_SYNC_4(a))
636  #define TCW_SYNC_PTR(a,b) TCW_SYNC_4((a),(b))
637  #define TCX_SYNC_PTR(a,b,c) ((void *)TCX_SYNC_4((a),(b),(c)))
638 
639 #else /* 64 bit pointers */
640 
641  #define TCR_PTR(a) ((void *)TCR_8(a))
642  #define TCW_PTR(a,b) TCW_8((a),(b))
643  #define TCR_SYNC_PTR(a) ((void *)TCR_SYNC_8(a))
644  #define TCW_SYNC_PTR(a,b) TCW_SYNC_8((a),(b))
645  #define TCX_SYNC_PTR(a,b,c) ((void *)TCX_SYNC_8((a),(b),(c)))
646 
647 #endif /* KMP_ARCH_X86 */
648 
649 /*
650  * If these FTN_{TRUE,FALSE} values change, may need to
651  * change several places where they are used to check that
652  * language is Fortran, not C.
653  */
654 
655 #ifndef FTN_TRUE
656 # define FTN_TRUE TRUE
657 #endif
658 
659 #ifndef FTN_FALSE
660 # define FTN_FALSE FALSE
661 #endif
662 
663 typedef void (*microtask_t)( int *gtid, int *npr, ... );
664 
665 #ifdef USE_VOLATILE_CAST
666 # define VOLATILE_CAST(x) (volatile x)
667 #else
668 # define VOLATILE_CAST(x) (x)
669 #endif
670 
671 #ifdef KMP_I8
672 # define KMP_WAIT_YIELD __kmp_wait_yield_8
673 # define KMP_EQ __kmp_eq_8
674 # define KMP_NEQ __kmp_neq_8
675 # define KMP_LT __kmp_lt_8
676 # define KMP_GE __kmp_ge_8
677 # define KMP_LE __kmp_le_8
678 #else
679 # define KMP_WAIT_YIELD __kmp_wait_yield_4
680 # define KMP_EQ __kmp_eq_4
681 # define KMP_NEQ __kmp_neq_4
682 # define KMP_LT __kmp_lt_4
683 # define KMP_GE __kmp_ge_4
684 # define KMP_LE __kmp_le_4
685 #endif /* KMP_I8 */
686 
687 /* Workaround for Intel(R) 64 code gen bug when taking address of static array (Intel(R) 64 Tracker #138) */
688 #if (KMP_ARCH_X86_64 || KMP_ARCH_PPC64) && KMP_OS_LINUX
689 # define STATIC_EFI2_WORKAROUND
690 #else
691 # define STATIC_EFI2_WORKAROUND static
692 #endif
693 
694 // Support of BGET usage
695 #ifndef KMP_USE_BGET
696 #define KMP_USE_BGET 1
697 #endif
698 
699 
700 // Switches for OSS builds
701 #ifndef USE_SYSFS_INFO
702 # define USE_SYSFS_INFO 0
703 #endif
704 #ifndef USE_CMPXCHG_FIX
705 # define USE_CMPXCHG_FIX 1
706 #endif
707 
708 // Enable dynamic user lock
709 #ifndef KMP_USE_DYNAMIC_LOCK
710 # define KMP_USE_DYNAMIC_LOCK 0
711 #endif
712 
713 // Warning levels
714 enum kmp_warnings_level {
715  kmp_warnings_off = 0, /* No warnings */
716  kmp_warnings_low, /* Minimal warnings (default) */
717  kmp_warnings_explicit = 6, /* Explicitly set to ON - more warnings */
718  kmp_warnings_verbose /* reserved */
719 };
720 
721 #ifdef __cplusplus
722 } // extern "C"
723 #endif // __cplusplus
724 
725 #endif /* KMP_OS_H */
726 // Safe C API
727 #include "kmp_safe_c_api.h"
728