16 #include "kmp_atomic.h" 19 typedef unsigned char uchar;
20 typedef unsigned short ushort;
568 #ifndef KMP_GOMP_COMPAT 569 int __kmp_atomic_mode = 1;
571 int __kmp_atomic_mode = 2;
577 kmp_atomic_lock_t __kmp_atomic_lock;
579 kmp_atomic_lock_t __kmp_atomic_lock_1i;
581 kmp_atomic_lock_t __kmp_atomic_lock_2i;
583 kmp_atomic_lock_t __kmp_atomic_lock_4i;
585 kmp_atomic_lock_t __kmp_atomic_lock_4r;
587 kmp_atomic_lock_t __kmp_atomic_lock_8i;
589 kmp_atomic_lock_t __kmp_atomic_lock_8r;
591 kmp_atomic_lock_t __kmp_atomic_lock_8c;
593 kmp_atomic_lock_t __kmp_atomic_lock_10r;
595 kmp_atomic_lock_t __kmp_atomic_lock_16r;
597 kmp_atomic_lock_t __kmp_atomic_lock_16c;
599 kmp_atomic_lock_t __kmp_atomic_lock_20c;
601 kmp_atomic_lock_t __kmp_atomic_lock_32c;
608 #define KMP_ATOMIC_VOLATILE volatile 610 #if (KMP_ARCH_X86) && KMP_HAVE_QUAD 612 static inline void operator+=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
615 static inline void operator-=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
618 static inline void operator*=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
621 static inline void operator/=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
624 static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
625 return lhs.q < rhs.q;
627 static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
628 return lhs.q > rhs.q;
631 static inline void operator+=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
634 static inline void operator-=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
637 static inline void operator*=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
640 static inline void operator/=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
643 static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
644 return lhs.q < rhs.q;
646 static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
647 return lhs.q > rhs.q;
650 static inline void operator+=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
653 static inline void operator-=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
656 static inline void operator*=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
659 static inline void operator/=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
663 static inline void operator+=(kmp_cmplx128_a16_t &lhs,
664 kmp_cmplx128_a16_t &rhs) {
667 static inline void operator-=(kmp_cmplx128_a16_t &lhs,
668 kmp_cmplx128_a16_t &rhs) {
671 static inline void operator*=(kmp_cmplx128_a16_t &lhs,
672 kmp_cmplx128_a16_t &rhs) {
675 static inline void operator/=(kmp_cmplx128_a16_t &lhs,
676 kmp_cmplx128_a16_t &rhs) {
687 #define KMP_CHECK_GTID \ 688 if (gtid == KMP_GTID_UNKNOWN) { \ 689 gtid = __kmp_entry_gtid(); \ 690 } // check and get gtid when needed 697 #define ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 698 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 699 TYPE *lhs, TYPE rhs) { \ 700 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 701 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 705 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat 706 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char 707 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short 708 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int 709 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float 710 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int 711 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double 712 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex 713 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double 714 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad 715 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex 716 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex 717 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex 725 #define OP_CRITICAL(OP, LCK_ID) \ 726 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 730 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 754 #ifdef KMP_GOMP_COMPAT 755 #define OP_GOMP_CRITICAL(OP, FLAG) \ 756 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 758 OP_CRITICAL(OP, 0); \ 762 #define OP_GOMP_CRITICAL(OP, FLAG) 766 #define KMP_DO_PAUSE _mm_delay_32(1) 768 #define KMP_DO_PAUSE KMP_CPU_PAUSE() 776 #define OP_CMPXCHG(TYPE, BITS, OP) \ 778 TYPE old_value, new_value; \ 779 old_value = *(TYPE volatile *)lhs; \ 780 new_value = old_value OP rhs; \ 781 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 782 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 783 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 786 old_value = *(TYPE volatile *)lhs; \ 787 new_value = old_value OP rhs; \ 798 #define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 802 kmp_int##BITS *vvv; \ 804 struct _sss old_value, new_value; \ 805 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \ 806 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \ 807 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ 808 new_value.cmp = old_value.cmp OP rhs; \ 809 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 810 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \ 811 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \ 814 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \ 815 new_value.cmp = old_value.cmp OP rhs; \ 819 #endif // USE_CMPXCHG_FIX 821 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 825 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 827 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 828 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 830 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 833 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 835 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 836 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 837 OP_CMPXCHG(TYPE, BITS, OP) \ 842 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \ 844 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 845 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 846 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 854 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 856 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 857 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 858 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 860 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 863 OP_CRITICAL(OP## =, LCK_ID) \ 867 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 869 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 870 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 871 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 872 OP_CMPXCHG(TYPE, BITS, OP) \ 875 OP_CRITICAL(OP## =, LCK_ID) \ 881 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \ 883 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 884 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 885 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 886 OP_CMPXCHG(TYPE, BITS, OP) \ 889 OP_CRITICAL(OP## =, LCK_ID) \ 893 #endif // USE_CMPXCHG_FIX 897 ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3,
899 ATOMIC_FIXED_ADD(fixed4, sub, kmp_int32, 32, -, 4i, 3,
902 ATOMIC_CMPXCHG(float4, add, kmp_real32, 32, +, 4r, 3,
904 ATOMIC_CMPXCHG(float4, sub, kmp_real32, 32, -, 4r, 3,
908 ATOMIC_FIXED_ADD(fixed8, add, kmp_int64, 64, +, 8i, 7,
910 ATOMIC_FIXED_ADD(fixed8, sub, kmp_int64, 64, -, 8i, 7,
913 ATOMIC_CMPXCHG(float8, add, kmp_real64, 64, +, 8r, 7,
915 ATOMIC_CMPXCHG(float8, sub, kmp_real64, 64, -, 8r, 7,
933 ATOMIC_CMPXCHG(fixed1, add, kmp_int8, 8, +, 1i, 0,
935 ATOMIC_CMPXCHG(fixed1, andb, kmp_int8, 8, &, 1i, 0,
937 ATOMIC_CMPXCHG(fixed1, div, kmp_int8, 8, /, 1i, 0,
939 ATOMIC_CMPXCHG(fixed1u, div, kmp_uint8, 8, /, 1i, 0,
941 ATOMIC_CMPXCHG(fixed1, mul, kmp_int8, 8, *, 1i, 0,
943 ATOMIC_CMPXCHG(fixed1, orb, kmp_int8, 8, |, 1i, 0,
945 ATOMIC_CMPXCHG(fixed1, shl, kmp_int8, 8, <<, 1i, 0,
947 ATOMIC_CMPXCHG(fixed1, shr, kmp_int8, 8, >>, 1i, 0,
949 ATOMIC_CMPXCHG(fixed1u, shr, kmp_uint8, 8, >>, 1i, 0,
951 ATOMIC_CMPXCHG(fixed1, sub, kmp_int8, 8, -, 1i, 0,
953 ATOMIC_CMPXCHG(fixed1, xor, kmp_int8, 8, ^, 1i, 0,
955 ATOMIC_CMPXCHG(fixed2, add, kmp_int16, 16, +, 2i, 1,
957 ATOMIC_CMPXCHG(fixed2, andb, kmp_int16, 16, &, 2i, 1,
959 ATOMIC_CMPXCHG(fixed2, div, kmp_int16, 16, /, 2i, 1,
961 ATOMIC_CMPXCHG(fixed2u, div, kmp_uint16, 16, /, 2i, 1,
963 ATOMIC_CMPXCHG(fixed2, mul, kmp_int16, 16, *, 2i, 1,
965 ATOMIC_CMPXCHG(fixed2, orb, kmp_int16, 16, |, 2i, 1,
967 ATOMIC_CMPXCHG(fixed2, shl, kmp_int16, 16, <<, 2i, 1,
969 ATOMIC_CMPXCHG(fixed2, shr, kmp_int16, 16, >>, 2i, 1,
971 ATOMIC_CMPXCHG(fixed2u, shr, kmp_uint16, 16, >>, 2i, 1,
973 ATOMIC_CMPXCHG(fixed2, sub, kmp_int16, 16, -, 2i, 1,
975 ATOMIC_CMPXCHG(fixed2, xor, kmp_int16, 16, ^, 2i, 1,
977 ATOMIC_CMPXCHG(fixed4, andb, kmp_int32, 32, &, 4i, 3,
979 ATOMIC_CMPXCHG(fixed4, div, kmp_int32, 32, /, 4i, 3,
981 ATOMIC_CMPXCHG(fixed4u, div, kmp_uint32, 32, /, 4i, 3,
983 ATOMIC_CMPXCHG(fixed4, mul, kmp_int32, 32, *, 4i, 3,
985 ATOMIC_CMPXCHG(fixed4, orb, kmp_int32, 32, |, 4i, 3,
987 ATOMIC_CMPXCHG(fixed4, shl, kmp_int32, 32, <<, 4i, 3,
989 ATOMIC_CMPXCHG(fixed4, shr, kmp_int32, 32, >>, 4i, 3,
991 ATOMIC_CMPXCHG(fixed4u, shr, kmp_uint32, 32, >>, 4i, 3,
993 ATOMIC_CMPXCHG(fixed4, xor, kmp_int32, 32, ^, 4i, 3,
995 ATOMIC_CMPXCHG(fixed8, andb, kmp_int64, 64, &, 8i, 7,
997 ATOMIC_CMPXCHG(fixed8, div, kmp_int64, 64, /, 8i, 7,
999 ATOMIC_CMPXCHG(fixed8u, div, kmp_uint64, 64, /, 8i, 7,
1001 ATOMIC_CMPXCHG(fixed8, mul, kmp_int64, 64, *, 8i, 7,
1003 ATOMIC_CMPXCHG(fixed8, orb, kmp_int64, 64, |, 8i, 7,
1005 ATOMIC_CMPXCHG(fixed8, shl, kmp_int64, 64, <<, 8i, 7,
1007 ATOMIC_CMPXCHG(fixed8, shr, kmp_int64, 64, >>, 8i, 7,
1009 ATOMIC_CMPXCHG(fixed8u, shr, kmp_uint64, 64, >>, 8i, 7,
1011 ATOMIC_CMPXCHG(fixed8, xor, kmp_int64, 64, ^, 8i, 7,
1013 ATOMIC_CMPXCHG(float4, div, kmp_real32, 32, /, 4r, 3,
1015 ATOMIC_CMPXCHG(float4, mul, kmp_real32, 32, *, 4r, 3,
1017 ATOMIC_CMPXCHG(float8, div, kmp_real64, 64, /, 8r, 7,
1019 ATOMIC_CMPXCHG(float8, mul, kmp_real64, 64, *, 8r, 7,
1029 #define ATOMIC_CRIT_L(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1030 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1031 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1032 OP_CRITICAL(= *lhs OP, LCK_ID) \ 1035 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1039 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \ 1040 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1041 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1042 OP_CMPXCHG(TYPE, BITS, OP) \ 1048 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \ 1049 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1050 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \ 1051 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1052 OP_CMPXCHG(TYPE, BITS, OP) \ 1055 OP_CRITICAL(= *lhs OP, LCK_ID) \ 1060 ATOMIC_CMPX_L(fixed1, andl,
char, 8, &&, 1i, 0,
1062 ATOMIC_CMPX_L(fixed1, orl,
char, 8, ||, 1i, 0,
1064 ATOMIC_CMPX_L(fixed2, andl,
short, 16, &&, 2i, 1,
1066 ATOMIC_CMPX_L(fixed2, orl,
short, 16, ||, 2i, 1,
1068 ATOMIC_CMPX_L(fixed4, andl, kmp_int32, 32, &&, 4i, 3,
1070 ATOMIC_CMPX_L(fixed4, orl, kmp_int32, 32, ||, 4i, 3,
1072 ATOMIC_CMPX_L(fixed8, andl, kmp_int64, 64, &&, 8i, 7,
1074 ATOMIC_CMPX_L(fixed8, orl, kmp_int64, 64, ||, 8i, 7,
1086 #define MIN_MAX_CRITSECT(OP, LCK_ID) \ 1087 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1089 if (*lhs OP rhs) { \ 1092 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1095 #ifdef KMP_GOMP_COMPAT 1096 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) \ 1097 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1099 MIN_MAX_CRITSECT(OP, 0); \ 1103 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) 1107 #define MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1109 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1112 old_value = temp_val; \ 1113 while (old_value OP rhs && \ 1114 !KMP_COMPARE_AND_STORE_ACQ##BITS( \ 1115 (kmp_int##BITS *)lhs, \ 1116 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 1117 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \ 1120 old_value = temp_val; \ 1126 #define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1127 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1128 if (*lhs OP rhs) { \ 1129 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1130 MIN_MAX_CRITSECT(OP, LCK_ID) \ 1134 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1138 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1140 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1141 if (*lhs OP rhs) { \ 1142 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1143 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1150 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1152 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1153 if (*lhs OP rhs) { \ 1154 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \ 1155 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1156 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \ 1159 MIN_MAX_CRITSECT(OP, LCK_ID) \ 1165 MIN_MAX_COMPXCHG(fixed1, max,
char, 8, <, 1i, 0,
1167 MIN_MAX_COMPXCHG(fixed1, min,
char, 8, >, 1i, 0,
1169 MIN_MAX_COMPXCHG(fixed2, max,
short, 16, <, 2i, 1,
1171 MIN_MAX_COMPXCHG(fixed2, min,
short, 16, >, 2i, 1,
1173 MIN_MAX_COMPXCHG(fixed4, max, kmp_int32, 32, <, 4i, 3,
1175 MIN_MAX_COMPXCHG(fixed4, min, kmp_int32, 32, >, 4i, 3,
1177 MIN_MAX_COMPXCHG(fixed8, max, kmp_int64, 64, <, 8i, 7,
1179 MIN_MAX_COMPXCHG(fixed8, min, kmp_int64, 64, >, 8i, 7,
1181 MIN_MAX_COMPXCHG(float4, max, kmp_real32, 32, <, 4r, 3,
1183 MIN_MAX_COMPXCHG(float4, min, kmp_real32, 32, >, 4r, 3,
1185 MIN_MAX_COMPXCHG(float8, max, kmp_real64, 64, <, 8r, 7,
1187 MIN_MAX_COMPXCHG(float8, min, kmp_real64, 64, >, 8r, 7,
1190 MIN_MAX_CRITICAL(float16, max, QUAD_LEGACY, <, 16r,
1192 MIN_MAX_CRITICAL(float16, min, QUAD_LEGACY, >, 16r,
1195 MIN_MAX_CRITICAL(float16, max_a16, Quad_a16_t, <, 16r,
1197 MIN_MAX_CRITICAL(float16, min_a16, Quad_a16_t, >, 16r,
1204 #define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1205 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1206 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \ 1207 OP_CRITICAL(^= ~, LCK_ID) \ 1211 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1214 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1216 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1217 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \ 1218 OP_CMPXCHG(TYPE, BITS, OP) \ 1224 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \ 1226 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1227 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \ 1228 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1229 OP_CMPXCHG(TYPE, BITS, OP) \ 1232 OP_CRITICAL(^= ~, LCK_ID) \ 1237 ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0,
1239 ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1,
1241 ATOMIC_CMPXCHG(fixed4, neqv, kmp_int32, 32, ^, 4i, 3,
1243 ATOMIC_CMPXCHG(fixed8, neqv, kmp_int64, 64, ^, 8i, 7,
1245 ATOMIC_CMPX_EQV(fixed1, eqv, kmp_int8, 8, ^~, 1i, 0,
1247 ATOMIC_CMPX_EQV(fixed2, eqv, kmp_int16, 16, ^~, 2i, 1,
1249 ATOMIC_CMPX_EQV(fixed4, eqv, kmp_int32, 32, ^~, 4i, 3,
1251 ATOMIC_CMPX_EQV(fixed8, eqv, kmp_int64, 64, ^~, 8i, 7,
1260 #define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1261 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 1262 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1263 OP_CRITICAL(OP## =, LCK_ID) \ 1268 ATOMIC_CRITICAL(float10, add,
long double, +, 10r,
1270 ATOMIC_CRITICAL(float10, sub,
long double, -, 10r,
1272 ATOMIC_CRITICAL(float10, mul,
long double, *, 10r,
1274 ATOMIC_CRITICAL(float10, div,
long double, /, 10r,
1278 ATOMIC_CRITICAL(float16, add, QUAD_LEGACY, +, 16r,
1280 ATOMIC_CRITICAL(float16, sub, QUAD_LEGACY, -, 16r,
1282 ATOMIC_CRITICAL(float16, mul, QUAD_LEGACY, *, 16r,
1284 ATOMIC_CRITICAL(float16, div, QUAD_LEGACY, /, 16r,
1287 ATOMIC_CRITICAL(float16, add_a16, Quad_a16_t, +, 16r,
1289 ATOMIC_CRITICAL(float16, sub_a16, Quad_a16_t, -, 16r,
1291 ATOMIC_CRITICAL(float16, mul_a16, Quad_a16_t, *, 16r,
1293 ATOMIC_CRITICAL(float16, div_a16, Quad_a16_t, /, 16r,
1301 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, add, kmp_cmplx32, 64, +, 8c, 7,
1303 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7,
1305 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7,
1307 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, div, kmp_cmplx32, 64, /, 8c, 7,
1311 ATOMIC_CRITICAL(cmplx4, add, kmp_cmplx32, +, 8c, 1)
1312 ATOMIC_CRITICAL(cmplx4, sub, kmp_cmplx32, -, 8c, 1)
1313 ATOMIC_CRITICAL(cmplx4, mul, kmp_cmplx32, *, 8c, 1)
1314 ATOMIC_CRITICAL(cmplx4, div, kmp_cmplx32, /, 8c, 1)
1315 #endif // USE_CMPXCHG_FIX 1317 ATOMIC_CRITICAL(cmplx8, add, kmp_cmplx64, +, 16c, 1)
1318 ATOMIC_CRITICAL(cmplx8, sub, kmp_cmplx64, -, 16c, 1)
1319 ATOMIC_CRITICAL(cmplx8, mul, kmp_cmplx64, *, 16c, 1)
1320 ATOMIC_CRITICAL(cmplx8, div, kmp_cmplx64, /, 16c, 1)
1321 ATOMIC_CRITICAL(cmplx10, add, kmp_cmplx80, +, 20c,
1323 ATOMIC_CRITICAL(cmplx10, sub, kmp_cmplx80, -, 20c,
1325 ATOMIC_CRITICAL(cmplx10, mul, kmp_cmplx80, *, 20c,
1327 ATOMIC_CRITICAL(cmplx10, div, kmp_cmplx80, /, 20c,
1330 ATOMIC_CRITICAL(cmplx16, add, CPLX128_LEG, +, 32c,
1332 ATOMIC_CRITICAL(cmplx16, sub, CPLX128_LEG, -, 32c,
1334 ATOMIC_CRITICAL(cmplx16, mul, CPLX128_LEG, *, 32c,
1336 ATOMIC_CRITICAL(cmplx16, div, CPLX128_LEG, /, 32c,
1339 ATOMIC_CRITICAL(cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c,
1341 ATOMIC_CRITICAL(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1343 ATOMIC_CRITICAL(cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c,
1345 ATOMIC_CRITICAL(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1354 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1362 #define OP_CRITICAL_REV(OP, LCK_ID) \ 1363 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1365 (*lhs) = (rhs)OP(*lhs); \ 1367 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1369 #ifdef KMP_GOMP_COMPAT 1370 #define OP_GOMP_CRITICAL_REV(OP, FLAG) \ 1371 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1373 OP_CRITICAL_REV(OP, 0); \ 1377 #define OP_GOMP_CRITICAL_REV(OP, FLAG) 1385 #define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 1386 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev(ident_t *id_ref, int gtid, \ 1387 TYPE *lhs, TYPE rhs) { \ 1388 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1389 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid)); 1398 #define OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1400 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1401 TYPE old_value, new_value; \ 1403 old_value = temp_val; \ 1404 new_value = rhs OP old_value; \ 1405 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 1406 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 1407 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 1411 old_value = temp_val; \ 1412 new_value = rhs OP old_value; \ 1417 #define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \ 1418 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \ 1419 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ 1420 OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1437 ATOMIC_CMPXCHG_REV(fixed1, div, kmp_int8, 8, /, 1i,
1439 ATOMIC_CMPXCHG_REV(fixed1u, div, kmp_uint8, 8, /, 1i,
1441 ATOMIC_CMPXCHG_REV(fixed1, shl, kmp_int8, 8, <<, 1i,
1443 ATOMIC_CMPXCHG_REV(fixed1, shr, kmp_int8, 8, >>, 1i,
1445 ATOMIC_CMPXCHG_REV(fixed1u, shr, kmp_uint8, 8, >>, 1i,
1447 ATOMIC_CMPXCHG_REV(fixed1, sub, kmp_int8, 8, -, 1i,
1450 ATOMIC_CMPXCHG_REV(fixed2, div, kmp_int16, 16, /, 2i,
1452 ATOMIC_CMPXCHG_REV(fixed2u, div, kmp_uint16, 16, /, 2i,
1454 ATOMIC_CMPXCHG_REV(fixed2, shl, kmp_int16, 16, <<, 2i,
1456 ATOMIC_CMPXCHG_REV(fixed2, shr, kmp_int16, 16, >>, 2i,
1458 ATOMIC_CMPXCHG_REV(fixed2u, shr, kmp_uint16, 16, >>, 2i,
1460 ATOMIC_CMPXCHG_REV(fixed2, sub, kmp_int16, 16, -, 2i,
1463 ATOMIC_CMPXCHG_REV(fixed4, div, kmp_int32, 32, /, 4i,
1465 ATOMIC_CMPXCHG_REV(fixed4u, div, kmp_uint32, 32, /, 4i,
1467 ATOMIC_CMPXCHG_REV(fixed4, shl, kmp_int32, 32, <<, 4i,
1469 ATOMIC_CMPXCHG_REV(fixed4, shr, kmp_int32, 32, >>, 4i,
1471 ATOMIC_CMPXCHG_REV(fixed4u, shr, kmp_uint32, 32, >>, 4i,
1473 ATOMIC_CMPXCHG_REV(fixed4, sub, kmp_int32, 32, -, 4i,
1476 ATOMIC_CMPXCHG_REV(fixed8, div, kmp_int64, 64, /, 8i,
1478 ATOMIC_CMPXCHG_REV(fixed8u, div, kmp_uint64, 64, /, 8i,
1480 ATOMIC_CMPXCHG_REV(fixed8, shl, kmp_int64, 64, <<, 8i,
1482 ATOMIC_CMPXCHG_REV(fixed8, shr, kmp_int64, 64, >>, 8i,
1484 ATOMIC_CMPXCHG_REV(fixed8u, shr, kmp_uint64, 64, >>, 8i,
1486 ATOMIC_CMPXCHG_REV(fixed8, sub, kmp_int64, 64, -, 8i,
1489 ATOMIC_CMPXCHG_REV(float4, div, kmp_real32, 32, /, 4r,
1491 ATOMIC_CMPXCHG_REV(float4, sub, kmp_real32, 32, -, 4r,
1494 ATOMIC_CMPXCHG_REV(float8, div, kmp_real64, 64, /, 8r,
1496 ATOMIC_CMPXCHG_REV(float8, sub, kmp_real64, 64, -, 8r,
1506 #define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1507 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \ 1508 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ 1509 OP_CRITICAL_REV(OP, LCK_ID) \ 1514 ATOMIC_CRITICAL_REV(float10, sub,
long double, -, 10r,
1516 ATOMIC_CRITICAL_REV(float10, div,
long double, /, 10r,
1520 ATOMIC_CRITICAL_REV(float16, sub, QUAD_LEGACY, -, 16r,
1522 ATOMIC_CRITICAL_REV(float16, div, QUAD_LEGACY, /, 16r,
1525 ATOMIC_CRITICAL_REV(float16, sub_a16, Quad_a16_t, -, 16r,
1527 ATOMIC_CRITICAL_REV(float16, div_a16, Quad_a16_t, /, 16r,
1533 ATOMIC_CRITICAL_REV(cmplx4, sub, kmp_cmplx32, -, 8c,
1535 ATOMIC_CRITICAL_REV(cmplx4, div, kmp_cmplx32, /, 8c,
1537 ATOMIC_CRITICAL_REV(cmplx8, sub, kmp_cmplx64, -, 16c,
1539 ATOMIC_CRITICAL_REV(cmplx8, div, kmp_cmplx64, /, 16c,
1541 ATOMIC_CRITICAL_REV(cmplx10, sub, kmp_cmplx80, -, 20c,
1543 ATOMIC_CRITICAL_REV(cmplx10, div, kmp_cmplx80, /, 20c,
1546 ATOMIC_CRITICAL_REV(cmplx16, sub, CPLX128_LEG, -, 32c,
1548 ATOMIC_CRITICAL_REV(cmplx16, div, CPLX128_LEG, /, 32c,
1551 ATOMIC_CRITICAL_REV(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1553 ATOMIC_CRITICAL_REV(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1558 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 1561 #endif // OMP_40_ENABLED 1574 #define ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1575 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \ 1576 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs) { \ 1577 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1579 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \ 1583 #define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \ 1585 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1586 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1587 OP_CRITICAL(OP## =, LCK_ID) \ 1591 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1594 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1595 LCK_ID, MASK, GOMP_FLAG) \ 1596 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1597 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1598 OP_CMPXCHG(TYPE, BITS, OP) \ 1604 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1605 LCK_ID, MASK, GOMP_FLAG) \ 1606 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1607 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1608 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1609 OP_CMPXCHG(TYPE, BITS, OP) \ 1612 OP_CRITICAL(OP## =, LCK_ID) \ 1618 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1620 #define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 1621 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 1622 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1623 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ 1624 OP_CMPXCHG_REV(TYPE, BITS, OP) \ 1626 #define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 1627 LCK_ID, GOMP_FLAG) \ 1628 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1629 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \ 1630 OP_CRITICAL_REV(OP, LCK_ID) \ 1635 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0,
1637 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0,
1639 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1,
1641 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1,
1643 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3,
1645 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3,
1647 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7,
1649 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7,
1651 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3,
1653 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3,
1655 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3,
1657 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3,
1663 ATOMIC_CMPXCHG_MIX(fixed1,
char, add, 8, +, fp, _Quad, 1i, 0,
1665 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, add, 8, +, fp, _Quad, 1i, 0,
1667 ATOMIC_CMPXCHG_MIX(fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0,
1669 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, sub, 8, -, fp, _Quad, 1i, 0,
1671 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0,
1673 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, mul, 8, *, fp, _Quad, 1i, 0,
1675 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, fp, _Quad, 1i, 0,
1677 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0,
1680 ATOMIC_CMPXCHG_MIX(fixed2,
short, add, 16, +, fp, _Quad, 2i, 1,
1682 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, add, 16, +, fp, _Quad, 2i, 1,
1684 ATOMIC_CMPXCHG_MIX(fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1,
1686 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, sub, 16, -, fp, _Quad, 2i, 1,
1688 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1,
1690 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, mul, 16, *, fp, _Quad, 2i, 1,
1692 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, fp, _Quad, 2i, 1,
1694 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1,
1697 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3,
1699 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3,
1701 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3,
1703 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3,
1705 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3,
1707 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3,
1709 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3,
1711 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3,
1714 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7,
1716 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7,
1718 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7,
1720 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7,
1722 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7,
1724 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7,
1726 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7,
1728 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7,
1731 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3,
1733 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3,
1735 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3,
1737 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3,
1740 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7,
1742 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7,
1744 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7,
1746 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7,
1749 ATOMIC_CRITICAL_FP(float10,
long double, add, +, fp, _Quad, 10r,
1751 ATOMIC_CRITICAL_FP(float10,
long double, sub, -, fp, _Quad, 10r,
1753 ATOMIC_CRITICAL_FP(float10,
long double, mul, *, fp, _Quad, 10r,
1755 ATOMIC_CRITICAL_FP(float10,
long double, div, /, fp, _Quad, 10r,
1758 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1760 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, sub_rev, 8, -, fp, _Quad, 1i, 0,
1762 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, sub_rev, 8, -, fp, _Quad, 1i, 0,
1764 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, div_rev, 8, /, fp, _Quad, 1i, 0,
1766 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, div_rev, 8, /, fp, _Quad, 1i, 0,
1769 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, sub_rev, 16, -, fp, _Quad, 2i, 1,
1771 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, sub_rev, 16, -, fp, _Quad, 2i, 1,
1773 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, div_rev, 16, /, fp, _Quad, 2i, 1,
1775 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, div_rev, 16, /, fp, _Quad, 2i, 1,
1778 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1780 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1782 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, div_rev, 32, /, fp, _Quad, 4i, 3,
1784 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3,
1787 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1789 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1791 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, div_rev, 64, /, fp, _Quad, 8i, 7,
1793 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7,
1796 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3,
1798 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3,
1801 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7,
1803 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7,
1806 ATOMIC_CRITICAL_REV_FP(float10,
long double, sub_rev, -, fp, _Quad, 10r,
1808 ATOMIC_CRITICAL_REV_FP(float10,
long double, div_rev, /, fp, _Quad, 10r,
1814 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1819 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1820 LCK_ID, MASK, GOMP_FLAG) \ 1821 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1822 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1823 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \ 1827 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1828 LCK_ID, MASK, GOMP_FLAG) \ 1829 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1830 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1831 OP_CMPXCHG(TYPE, BITS, OP) \ 1833 #endif // USE_CMPXCHG_FIX 1837 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \ 1838 LCK_ID, MASK, GOMP_FLAG) \ 1839 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \ 1840 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \ 1841 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \ 1842 OP_CMPXCHG(TYPE, BITS, OP) \ 1845 OP_CRITICAL(OP## =, LCK_ID) \ 1850 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c,
1852 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c,
1854 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c,
1856 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c,
1860 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1871 #define ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 1872 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 1874 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1875 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 1886 #define OP_CMPXCHG_READ(TYPE, BITS, OP) \ 1888 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1891 kmp_int##BITS i_val; \ 1893 union f_i_union old_value; \ 1895 old_value.f_val = temp_val; \ 1896 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( \ 1897 (kmp_int##BITS *)loc, \ 1898 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val, \ 1899 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val); \ 1900 new_value = old_value.f_val; \ 1910 #define OP_CRITICAL_READ(OP, LCK_ID) \ 1911 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1913 new_value = (*loc); \ 1915 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1918 #ifdef KMP_GOMP_COMPAT 1919 #define OP_GOMP_CRITICAL_READ(OP, FLAG) \ 1920 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1922 OP_CRITICAL_READ(OP, 0); \ 1926 #define OP_GOMP_CRITICAL_READ(OP, FLAG) 1930 #define ATOMIC_FIXED_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 1931 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 1933 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 1934 new_value = KMP_TEST_THEN_ADD##BITS(loc, OP 0); \ 1938 #define ATOMIC_CMPXCHG_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 1939 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 1941 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 1942 OP_CMPXCHG_READ(TYPE, BITS, OP) \ 1950 #define ATOMIC_CRITICAL_READ(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1951 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \ 1953 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \ 1954 OP_CRITICAL_READ(OP, LCK_ID) \ 1962 #if (KMP_OS_WINDOWS) 1964 #define OP_CRITICAL_READ_WRK(OP, LCK_ID) \ 1965 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 1969 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 1971 #ifdef KMP_GOMP_COMPAT 1972 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) \ 1973 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 1975 OP_CRITICAL_READ_WRK(OP, 0); \ 1978 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) 1981 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \ 1982 void __kmpc_atomic_##TYPE_ID##_##OP_ID(TYPE *out, ident_t *id_ref, int gtid, \ 1984 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 1985 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 1988 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 1989 ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \ 1990 OP_GOMP_CRITICAL_READ_WRK(OP## =, GOMP_FLAG) \ 1991 OP_CRITICAL_READ_WRK(OP, LCK_ID) \ 1994 #endif // KMP_OS_WINDOWS 1998 ATOMIC_FIXED_READ(fixed4, rd, kmp_int32, 32, +, 0)
1999 ATOMIC_FIXED_READ(fixed8, rd, kmp_int64, 64, +,
2001 ATOMIC_CMPXCHG_READ(float4, rd, kmp_real32, 32, +,
2003 ATOMIC_CMPXCHG_READ(float8, rd, kmp_real64, 64, +,
2007 ATOMIC_CMPXCHG_READ(fixed1, rd, kmp_int8, 8, +,
2009 ATOMIC_CMPXCHG_READ(fixed2, rd, kmp_int16, 16, +,
2012 ATOMIC_CRITICAL_READ(float10, rd,
long double, +, 10r,
2015 ATOMIC_CRITICAL_READ(float16, rd, QUAD_LEGACY, +, 16r,
2017 #endif // KMP_HAVE_QUAD 2020 #if (KMP_OS_WINDOWS) 2021 ATOMIC_CRITICAL_READ_WRK(cmplx4, rd, kmp_cmplx32, +, 8c,
2024 ATOMIC_CRITICAL_READ(cmplx4, rd, kmp_cmplx32, +, 8c,
2027 ATOMIC_CRITICAL_READ(cmplx8, rd, kmp_cmplx64, +, 16c,
2029 ATOMIC_CRITICAL_READ(cmplx10, rd, kmp_cmplx80, +, 20c,
2032 ATOMIC_CRITICAL_READ(cmplx16, rd, CPLX128_LEG, +, 32c,
2035 ATOMIC_CRITICAL_READ(float16, a16_rd, Quad_a16_t, +, 16r,
2037 ATOMIC_CRITICAL_READ(cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c,
2045 #define ATOMIC_XCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2046 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2047 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2048 KMP_XCHG_FIXED##BITS(lhs, rhs); \ 2051 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2052 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2053 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2054 KMP_XCHG_REAL##BITS(lhs, rhs); \ 2064 #define OP_CMPXCHG_WR(TYPE, BITS, OP) \ 2066 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2067 TYPE old_value, new_value; \ 2069 old_value = temp_val; \ 2071 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2072 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2073 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2077 old_value = temp_val; \ 2083 #define ATOMIC_CMPXCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2084 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2085 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2086 OP_CMPXCHG_WR(TYPE, BITS, OP) \ 2095 #define ATOMIC_CRITICAL_WR(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2096 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \ 2097 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \ 2098 OP_CRITICAL(OP, LCK_ID) \ 2102 ATOMIC_XCHG_WR(fixed1, wr, kmp_int8, 8, =,
2104 ATOMIC_XCHG_WR(fixed2, wr, kmp_int16, 16, =,
2106 ATOMIC_XCHG_WR(fixed4, wr, kmp_int32, 32, =,
2109 ATOMIC_CMPXCHG_WR(fixed8, wr, kmp_int64, 64, =,
2112 ATOMIC_XCHG_WR(fixed8, wr, kmp_int64, 64, =,
2116 ATOMIC_XCHG_FLOAT_WR(float4, wr, kmp_real32, 32, =,
2119 ATOMIC_CMPXCHG_WR(float8, wr, kmp_real64, 64, =,
2122 ATOMIC_XCHG_FLOAT_WR(float8, wr, kmp_real64, 64, =,
2126 ATOMIC_CRITICAL_WR(float10, wr,
long double, =, 10r,
2129 ATOMIC_CRITICAL_WR(float16, wr, QUAD_LEGACY, =, 16r,
2132 ATOMIC_CRITICAL_WR(cmplx4, wr, kmp_cmplx32, =, 8c, 1)
2133 ATOMIC_CRITICAL_WR(cmplx8, wr, kmp_cmplx64, =, 16c,
2135 ATOMIC_CRITICAL_WR(cmplx10, wr, kmp_cmplx80, =, 20c,
2138 ATOMIC_CRITICAL_WR(cmplx16, wr, CPLX128_LEG, =, 32c,
2141 ATOMIC_CRITICAL_WR(float16, a16_wr, Quad_a16_t, =, 16r,
2143 ATOMIC_CRITICAL_WR(cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c,
2156 #define ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, RET_TYPE) \ 2157 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \ 2158 TYPE *lhs, TYPE rhs, int flag) { \ 2159 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2160 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 2168 #define OP_CRITICAL_CPT(OP, LCK_ID) \ 2169 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2173 new_value = (*lhs); \ 2175 new_value = (*lhs); \ 2179 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2183 #ifdef KMP_GOMP_COMPAT 2184 #define OP_GOMP_CRITICAL_CPT(OP, FLAG) \ 2185 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2187 OP_CRITICAL_CPT(OP## =, 0); \ 2190 #define OP_GOMP_CRITICAL_CPT(OP, FLAG) 2200 #define OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2202 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2203 TYPE old_value, new_value; \ 2205 old_value = temp_val; \ 2206 new_value = old_value OP rhs; \ 2207 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2208 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2209 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2213 old_value = temp_val; \ 2214 new_value = old_value OP rhs; \ 2223 #define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2224 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2226 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ 2227 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2231 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2232 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2233 TYPE old_value, new_value; \ 2234 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ 2236 old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \ 2238 return old_value OP rhs; \ 2244 ATOMIC_FIXED_ADD_CPT(fixed4, add_cpt, kmp_int32, 32, +,
2246 ATOMIC_FIXED_ADD_CPT(fixed4, sub_cpt, kmp_int32, 32, -,
2248 ATOMIC_FIXED_ADD_CPT(fixed8, add_cpt, kmp_int64, 64, +,
2250 ATOMIC_FIXED_ADD_CPT(fixed8, sub_cpt, kmp_int64, 64, -,
2253 ATOMIC_CMPXCHG_CPT(float4, add_cpt, kmp_real32, 32, +,
2255 ATOMIC_CMPXCHG_CPT(float4, sub_cpt, kmp_real32, 32, -,
2257 ATOMIC_CMPXCHG_CPT(float8, add_cpt, kmp_real64, 64, +,
2259 ATOMIC_CMPXCHG_CPT(float8, sub_cpt, kmp_real64, 64, -,
2274 ATOMIC_CMPXCHG_CPT(fixed1, add_cpt, kmp_int8, 8, +,
2276 ATOMIC_CMPXCHG_CPT(fixed1, andb_cpt, kmp_int8, 8, &,
2278 ATOMIC_CMPXCHG_CPT(fixed1, div_cpt, kmp_int8, 8, /,
2280 ATOMIC_CMPXCHG_CPT(fixed1u, div_cpt, kmp_uint8, 8, /,
2282 ATOMIC_CMPXCHG_CPT(fixed1, mul_cpt, kmp_int8, 8, *,
2284 ATOMIC_CMPXCHG_CPT(fixed1, orb_cpt, kmp_int8, 8, |,
2286 ATOMIC_CMPXCHG_CPT(fixed1, shl_cpt, kmp_int8, 8, <<,
2288 ATOMIC_CMPXCHG_CPT(fixed1, shr_cpt, kmp_int8, 8, >>,
2290 ATOMIC_CMPXCHG_CPT(fixed1u, shr_cpt, kmp_uint8, 8, >>,
2292 ATOMIC_CMPXCHG_CPT(fixed1, sub_cpt, kmp_int8, 8, -,
2294 ATOMIC_CMPXCHG_CPT(fixed1, xor_cpt, kmp_int8, 8, ^,
2296 ATOMIC_CMPXCHG_CPT(fixed2, add_cpt, kmp_int16, 16, +,
2298 ATOMIC_CMPXCHG_CPT(fixed2, andb_cpt, kmp_int16, 16, &,
2300 ATOMIC_CMPXCHG_CPT(fixed2, div_cpt, kmp_int16, 16, /,
2302 ATOMIC_CMPXCHG_CPT(fixed2u, div_cpt, kmp_uint16, 16, /,
2304 ATOMIC_CMPXCHG_CPT(fixed2, mul_cpt, kmp_int16, 16, *,
2306 ATOMIC_CMPXCHG_CPT(fixed2, orb_cpt, kmp_int16, 16, |,
2308 ATOMIC_CMPXCHG_CPT(fixed2, shl_cpt, kmp_int16, 16, <<,
2310 ATOMIC_CMPXCHG_CPT(fixed2, shr_cpt, kmp_int16, 16, >>,
2312 ATOMIC_CMPXCHG_CPT(fixed2u, shr_cpt, kmp_uint16, 16, >>,
2314 ATOMIC_CMPXCHG_CPT(fixed2, sub_cpt, kmp_int16, 16, -,
2316 ATOMIC_CMPXCHG_CPT(fixed2, xor_cpt, kmp_int16, 16, ^,
2318 ATOMIC_CMPXCHG_CPT(fixed4, andb_cpt, kmp_int32, 32, &,
2320 ATOMIC_CMPXCHG_CPT(fixed4, div_cpt, kmp_int32, 32, /,
2322 ATOMIC_CMPXCHG_CPT(fixed4u, div_cpt, kmp_uint32, 32, /,
2324 ATOMIC_CMPXCHG_CPT(fixed4, mul_cpt, kmp_int32, 32, *,
2326 ATOMIC_CMPXCHG_CPT(fixed4, orb_cpt, kmp_int32, 32, |,
2328 ATOMIC_CMPXCHG_CPT(fixed4, shl_cpt, kmp_int32, 32, <<,
2330 ATOMIC_CMPXCHG_CPT(fixed4, shr_cpt, kmp_int32, 32, >>,
2332 ATOMIC_CMPXCHG_CPT(fixed4u, shr_cpt, kmp_uint32, 32, >>,
2334 ATOMIC_CMPXCHG_CPT(fixed4, xor_cpt, kmp_int32, 32, ^,
2336 ATOMIC_CMPXCHG_CPT(fixed8, andb_cpt, kmp_int64, 64, &,
2338 ATOMIC_CMPXCHG_CPT(fixed8, div_cpt, kmp_int64, 64, /,
2340 ATOMIC_CMPXCHG_CPT(fixed8u, div_cpt, kmp_uint64, 64, /,
2342 ATOMIC_CMPXCHG_CPT(fixed8, mul_cpt, kmp_int64, 64, *,
2344 ATOMIC_CMPXCHG_CPT(fixed8, orb_cpt, kmp_int64, 64, |,
2346 ATOMIC_CMPXCHG_CPT(fixed8, shl_cpt, kmp_int64, 64, <<,
2348 ATOMIC_CMPXCHG_CPT(fixed8, shr_cpt, kmp_int64, 64, >>,
2350 ATOMIC_CMPXCHG_CPT(fixed8u, shr_cpt, kmp_uint64, 64, >>,
2352 ATOMIC_CMPXCHG_CPT(fixed8, xor_cpt, kmp_int64, 64, ^,
2354 ATOMIC_CMPXCHG_CPT(float4, div_cpt, kmp_real32, 32, /,
2356 ATOMIC_CMPXCHG_CPT(float4, mul_cpt, kmp_real32, 32, *,
2358 ATOMIC_CMPXCHG_CPT(float8, div_cpt, kmp_real64, 64, /,
2360 ATOMIC_CMPXCHG_CPT(float8, mul_cpt, kmp_real64, 64, *,
2372 #define ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2373 TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \ 2374 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs, int flag) { \ 2375 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2377 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \ 2381 #define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 2382 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 2383 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2385 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ 2386 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2390 #define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 2391 LCK_ID, GOMP_FLAG) \ 2392 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 2394 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ 2395 OP_CRITICAL_CPT(OP## =, LCK_ID) \ 2398 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, add_cpt, 8, +, fp, _Quad, 1i, 0,
2400 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0,
2402 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2404 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2406 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2408 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2410 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, div_cpt, 8, /, fp, _Quad, 1i, 0,
2412 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, div_cpt, 8, /, fp, _Quad, 1i, 0,
2415 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, add_cpt, 16, +, fp, _Quad, 2i, 1,
2417 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, add_cpt, 16, +, fp, _Quad, 2i, 1,
2419 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2421 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2423 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2425 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2427 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, div_cpt, 16, /, fp, _Quad, 2i, 1,
2429 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, div_cpt, 16, /, fp, _Quad, 2i, 1,
2432 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2434 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2436 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2438 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2440 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2442 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2444 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2446 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2449 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2451 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2453 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2455 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2457 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2459 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2461 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2463 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2466 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3,
2468 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3,
2470 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3,
2472 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3,
2475 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7,
2477 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7,
2479 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7,
2481 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7,
2484 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, add_cpt, +, fp, _Quad, 10r,
2486 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, sub_cpt, -, fp, _Quad, 10r,
2488 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, mul_cpt, *, fp, _Quad, 10r,
2490 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, div_cpt, /, fp, _Quad, 10r,
2493 #endif // KMP_HAVE_QUAD 2504 #define OP_CRITICAL_L_CPT(OP, LCK_ID) \ 2505 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2510 new_value = (*lhs); \ 2512 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); 2515 #ifdef KMP_GOMP_COMPAT 2516 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) \ 2517 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2519 OP_CRITICAL_L_CPT(OP, 0); \ 2523 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) 2528 #define ATOMIC_CMPX_L_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2529 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2531 OP_GOMP_CRITICAL_L_CPT(= *lhs OP, GOMP_FLAG) \ 2532 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2535 ATOMIC_CMPX_L_CPT(fixed1, andl_cpt,
char, 8, &&,
2537 ATOMIC_CMPX_L_CPT(fixed1, orl_cpt,
char, 8, ||,
2539 ATOMIC_CMPX_L_CPT(fixed2, andl_cpt,
short, 16, &&,
2541 ATOMIC_CMPX_L_CPT(fixed2, orl_cpt,
short, 16, ||,
2543 ATOMIC_CMPX_L_CPT(fixed4, andl_cpt, kmp_int32, 32, &&,
2545 ATOMIC_CMPX_L_CPT(fixed4, orl_cpt, kmp_int32, 32, ||,
2547 ATOMIC_CMPX_L_CPT(fixed8, andl_cpt, kmp_int64, 64, &&,
2549 ATOMIC_CMPX_L_CPT(fixed8, orl_cpt, kmp_int64, 64, ||,
2561 #define MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \ 2562 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2564 if (*lhs OP rhs) { \ 2570 new_value = old_value; \ 2572 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2576 #ifdef KMP_GOMP_COMPAT 2577 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) \ 2578 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2580 MIN_MAX_CRITSECT_CPT(OP, 0); \ 2583 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) 2587 #define MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \ 2589 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2592 old_value = temp_val; \ 2593 while (old_value OP rhs && \ 2594 !KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2595 (kmp_int##BITS *)lhs, \ 2596 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2597 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \ 2600 old_value = temp_val; \ 2610 #define MIN_MAX_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2611 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2612 TYPE new_value, old_value; \ 2613 if (*lhs OP rhs) { \ 2614 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \ 2615 MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \ 2620 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2621 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2622 TYPE new_value, old_value; \ 2623 if (*lhs OP rhs) { \ 2624 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \ 2625 MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \ 2630 MIN_MAX_COMPXCHG_CPT(fixed1, max_cpt,
char, 8, <,
2632 MIN_MAX_COMPXCHG_CPT(fixed1, min_cpt,
char, 8, >,
2634 MIN_MAX_COMPXCHG_CPT(fixed2, max_cpt,
short, 16, <,
2636 MIN_MAX_COMPXCHG_CPT(fixed2, min_cpt,
short, 16, >,
2638 MIN_MAX_COMPXCHG_CPT(fixed4, max_cpt, kmp_int32, 32, <,
2640 MIN_MAX_COMPXCHG_CPT(fixed4, min_cpt, kmp_int32, 32, >,
2642 MIN_MAX_COMPXCHG_CPT(fixed8, max_cpt, kmp_int64, 64, <,
2644 MIN_MAX_COMPXCHG_CPT(fixed8, min_cpt, kmp_int64, 64, >,
2646 MIN_MAX_COMPXCHG_CPT(float4, max_cpt, kmp_real32, 32, <,
2648 MIN_MAX_COMPXCHG_CPT(float4, min_cpt, kmp_real32, 32, >,
2650 MIN_MAX_COMPXCHG_CPT(float8, max_cpt, kmp_real64, 64, <,
2652 MIN_MAX_COMPXCHG_CPT(float8, min_cpt, kmp_real64, 64, >,
2655 MIN_MAX_CRITICAL_CPT(float16, max_cpt, QUAD_LEGACY, <, 16r,
2657 MIN_MAX_CRITICAL_CPT(float16, min_cpt, QUAD_LEGACY, >, 16r,
2660 MIN_MAX_CRITICAL_CPT(float16, max_a16_cpt, Quad_a16_t, <, 16r,
2662 MIN_MAX_CRITICAL_CPT(float16, min_a16_cpt, Quad_a16_t, >, 16r,
2668 #ifdef KMP_GOMP_COMPAT 2669 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) \ 2670 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2672 OP_CRITICAL_CPT(OP, 0); \ 2675 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) 2678 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2679 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2681 OP_GOMP_CRITICAL_EQV_CPT(^= ~, GOMP_FLAG) \ 2682 OP_CMPXCHG_CPT(TYPE, BITS, OP) \ 2687 ATOMIC_CMPXCHG_CPT(fixed1, neqv_cpt, kmp_int8, 8, ^,
2689 ATOMIC_CMPXCHG_CPT(fixed2, neqv_cpt, kmp_int16, 16, ^,
2691 ATOMIC_CMPXCHG_CPT(fixed4, neqv_cpt, kmp_int32, 32, ^,
2693 ATOMIC_CMPXCHG_CPT(fixed8, neqv_cpt, kmp_int64, 64, ^,
2695 ATOMIC_CMPX_EQV_CPT(fixed1, eqv_cpt, kmp_int8, 8, ^~,
2697 ATOMIC_CMPX_EQV_CPT(fixed2, eqv_cpt, kmp_int16, 16, ^~,
2699 ATOMIC_CMPX_EQV_CPT(fixed4, eqv_cpt, kmp_int32, 32, ^~,
2701 ATOMIC_CMPX_EQV_CPT(fixed8, eqv_cpt, kmp_int64, 64, ^~,
2710 #define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2711 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2713 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \ 2714 OP_CRITICAL_CPT(OP## =, LCK_ID) \ 2720 #define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \ 2721 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2731 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2735 #ifdef KMP_GOMP_COMPAT 2736 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) \ 2737 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2739 OP_CRITICAL_CPT_WRK(OP## =, 0); \ 2742 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) 2746 #define ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 2747 void __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, TYPE *lhs, \ 2748 TYPE rhs, TYPE *out, int flag) { \ 2749 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 2750 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid)); 2753 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2754 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 2755 OP_GOMP_CRITICAL_CPT_WRK(OP, GOMP_FLAG) \ 2756 OP_CRITICAL_CPT_WRK(OP## =, LCK_ID) \ 2762 ATOMIC_CRITICAL_CPT(float10, add_cpt,
long double, +, 10r,
2764 ATOMIC_CRITICAL_CPT(float10, sub_cpt,
long double, -, 10r,
2766 ATOMIC_CRITICAL_CPT(float10, mul_cpt,
long double, *, 10r,
2768 ATOMIC_CRITICAL_CPT(float10, div_cpt,
long double, /, 10r,
2772 ATOMIC_CRITICAL_CPT(float16, add_cpt, QUAD_LEGACY, +, 16r,
2774 ATOMIC_CRITICAL_CPT(float16, sub_cpt, QUAD_LEGACY, -, 16r,
2776 ATOMIC_CRITICAL_CPT(float16, mul_cpt, QUAD_LEGACY, *, 16r,
2778 ATOMIC_CRITICAL_CPT(float16, div_cpt, QUAD_LEGACY, /, 16r,
2781 ATOMIC_CRITICAL_CPT(float16, add_a16_cpt, Quad_a16_t, +, 16r,
2783 ATOMIC_CRITICAL_CPT(float16, sub_a16_cpt, Quad_a16_t, -, 16r,
2785 ATOMIC_CRITICAL_CPT(float16, mul_a16_cpt, Quad_a16_t, *, 16r,
2787 ATOMIC_CRITICAL_CPT(float16, div_a16_cpt, Quad_a16_t, /, 16r,
2795 ATOMIC_CRITICAL_CPT_WRK(cmplx4, add_cpt, kmp_cmplx32, +, 8c,
2797 ATOMIC_CRITICAL_CPT_WRK(cmplx4, sub_cpt, kmp_cmplx32, -, 8c,
2799 ATOMIC_CRITICAL_CPT_WRK(cmplx4, mul_cpt, kmp_cmplx32, *, 8c,
2801 ATOMIC_CRITICAL_CPT_WRK(cmplx4, div_cpt, kmp_cmplx32, /, 8c,
2804 ATOMIC_CRITICAL_CPT(cmplx8, add_cpt, kmp_cmplx64, +, 16c,
2806 ATOMIC_CRITICAL_CPT(cmplx8, sub_cpt, kmp_cmplx64, -, 16c,
2808 ATOMIC_CRITICAL_CPT(cmplx8, mul_cpt, kmp_cmplx64, *, 16c,
2810 ATOMIC_CRITICAL_CPT(cmplx8, div_cpt, kmp_cmplx64, /, 16c,
2812 ATOMIC_CRITICAL_CPT(cmplx10, add_cpt, kmp_cmplx80, +, 20c,
2814 ATOMIC_CRITICAL_CPT(cmplx10, sub_cpt, kmp_cmplx80, -, 20c,
2816 ATOMIC_CRITICAL_CPT(cmplx10, mul_cpt, kmp_cmplx80, *, 20c,
2818 ATOMIC_CRITICAL_CPT(cmplx10, div_cpt, kmp_cmplx80, /, 20c,
2821 ATOMIC_CRITICAL_CPT(cmplx16, add_cpt, CPLX128_LEG, +, 32c,
2823 ATOMIC_CRITICAL_CPT(cmplx16, sub_cpt, CPLX128_LEG, -, 32c,
2825 ATOMIC_CRITICAL_CPT(cmplx16, mul_cpt, CPLX128_LEG, *, 32c,
2827 ATOMIC_CRITICAL_CPT(cmplx16, div_cpt, CPLX128_LEG, /, 32c,
2830 ATOMIC_CRITICAL_CPT(cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c,
2832 ATOMIC_CRITICAL_CPT(cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c,
2834 ATOMIC_CRITICAL_CPT(cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c,
2836 ATOMIC_CRITICAL_CPT(cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c,
2853 #define OP_CRITICAL_CPT_REV(OP, LCK_ID) \ 2854 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2858 (*lhs) = (rhs)OP(*lhs); \ 2859 new_value = (*lhs); \ 2861 new_value = (*lhs); \ 2862 (*lhs) = (rhs)OP(*lhs); \ 2864 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 2868 #ifdef KMP_GOMP_COMPAT 2869 #define OP_GOMP_CRITICAL_CPT_REV(OP, FLAG) \ 2870 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 2872 OP_CRITICAL_CPT_REV(OP, 0); \ 2875 #define OP_GOMP_CRITICAL_CPT_REV(OP, FLAG) 2885 #define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 2887 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2888 TYPE old_value, new_value; \ 2890 old_value = temp_val; \ 2891 new_value = rhs OP old_value; \ 2892 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 2893 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 2894 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 2898 old_value = temp_val; \ 2899 new_value = rhs OP old_value; \ 2908 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \ 2909 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2911 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2912 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \ 2913 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 2916 ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev, kmp_int8, 8, /,
2918 ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev, kmp_uint8, 8, /,
2920 ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev, kmp_int8, 8, <<,
2922 ATOMIC_CMPXCHG_CPT_REV(fixed1, shr_cpt_rev, kmp_int8, 8, >>,
2924 ATOMIC_CMPXCHG_CPT_REV(fixed1u, shr_cpt_rev, kmp_uint8, 8, >>,
2926 ATOMIC_CMPXCHG_CPT_REV(fixed1, sub_cpt_rev, kmp_int8, 8, -,
2928 ATOMIC_CMPXCHG_CPT_REV(fixed2, div_cpt_rev, kmp_int16, 16, /,
2930 ATOMIC_CMPXCHG_CPT_REV(fixed2u, div_cpt_rev, kmp_uint16, 16, /,
2932 ATOMIC_CMPXCHG_CPT_REV(fixed2, shl_cpt_rev, kmp_int16, 16, <<,
2934 ATOMIC_CMPXCHG_CPT_REV(fixed2, shr_cpt_rev, kmp_int16, 16, >>,
2936 ATOMIC_CMPXCHG_CPT_REV(fixed2u, shr_cpt_rev, kmp_uint16, 16, >>,
2938 ATOMIC_CMPXCHG_CPT_REV(fixed2, sub_cpt_rev, kmp_int16, 16, -,
2940 ATOMIC_CMPXCHG_CPT_REV(fixed4, div_cpt_rev, kmp_int32, 32, /,
2942 ATOMIC_CMPXCHG_CPT_REV(fixed4u, div_cpt_rev, kmp_uint32, 32, /,
2944 ATOMIC_CMPXCHG_CPT_REV(fixed4, shl_cpt_rev, kmp_int32, 32, <<,
2946 ATOMIC_CMPXCHG_CPT_REV(fixed4, shr_cpt_rev, kmp_int32, 32, >>,
2948 ATOMIC_CMPXCHG_CPT_REV(fixed4u, shr_cpt_rev, kmp_uint32, 32, >>,
2950 ATOMIC_CMPXCHG_CPT_REV(fixed4, sub_cpt_rev, kmp_int32, 32, -,
2952 ATOMIC_CMPXCHG_CPT_REV(fixed8, div_cpt_rev, kmp_int64, 64, /,
2954 ATOMIC_CMPXCHG_CPT_REV(fixed8u, div_cpt_rev, kmp_uint64, 64, /,
2956 ATOMIC_CMPXCHG_CPT_REV(fixed8, shl_cpt_rev, kmp_int64, 64, <<,
2958 ATOMIC_CMPXCHG_CPT_REV(fixed8, shr_cpt_rev, kmp_int64, 64, >>,
2960 ATOMIC_CMPXCHG_CPT_REV(fixed8u, shr_cpt_rev, kmp_uint64, 64, >>,
2962 ATOMIC_CMPXCHG_CPT_REV(fixed8, sub_cpt_rev, kmp_int64, 64, -,
2964 ATOMIC_CMPXCHG_CPT_REV(float4, div_cpt_rev, kmp_real32, 32, /,
2966 ATOMIC_CMPXCHG_CPT_REV(float4, sub_cpt_rev, kmp_real32, 32, -,
2968 ATOMIC_CMPXCHG_CPT_REV(float8, div_cpt_rev, kmp_real64, 64, /,
2970 ATOMIC_CMPXCHG_CPT_REV(float8, sub_cpt_rev, kmp_real64, 64, -,
2980 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \ 2981 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \ 2983 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2985 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \ 2986 OP_CRITICAL_CPT_REV(OP, LCK_ID) \ 2991 ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev,
long double, -, 10r,
2993 ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev,
long double, /, 10r,
2997 ATOMIC_CRITICAL_CPT_REV(float16, sub_cpt_rev, QUAD_LEGACY, -, 16r,
2999 ATOMIC_CRITICAL_CPT_REV(float16, div_cpt_rev, QUAD_LEGACY, /, 16r,
3002 ATOMIC_CRITICAL_CPT_REV(float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r,
3004 ATOMIC_CRITICAL_CPT_REV(float16, div_a16_cpt_rev, Quad_a16_t, /, 16r,
3014 #define OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \ 3015 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3018 (*lhs) = (rhs)OP(*lhs); \ 3022 (*lhs) = (rhs)OP(*lhs); \ 3025 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3029 #ifdef KMP_GOMP_COMPAT 3030 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) \ 3031 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3033 OP_CRITICAL_CPT_REV_WRK(OP, 0); \ 3036 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) 3040 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, \ 3042 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \ 3043 OP_GOMP_CRITICAL_CPT_REV_WRK(OP, GOMP_FLAG) \ 3044 OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \ 3050 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c,
3052 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c,
3055 ATOMIC_CRITICAL_CPT_REV(cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c,
3057 ATOMIC_CRITICAL_CPT_REV(cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c,
3059 ATOMIC_CRITICAL_CPT_REV(cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c,
3061 ATOMIC_CRITICAL_CPT_REV(cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c,
3064 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c,
3066 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c,
3069 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c,
3071 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c,
3085 #define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \ 3086 RTYPE, LCK_ID, MASK, GOMP_FLAG) \ 3087 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 3089 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \ 3090 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \ 3094 #define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \ 3095 LCK_ID, GOMP_FLAG) \ 3096 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \ 3098 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \ 3099 OP_CRITICAL_CPT_REV(OP, LCK_ID) \ 3102 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3104 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3106 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3108 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3111 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1,
3113 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, sub_cpt_rev, 16, -, fp, _Quad, 2i,
3116 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, div_cpt_rev, 16, /, fp, _Quad, 2i, 1,
3118 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, div_cpt_rev, 16, /, fp, _Quad, 2i,
3122 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4i,
3124 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad,
3126 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4i,
3128 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad,
3131 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8i,
3134 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad,
3137 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8i,
3140 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad,
3144 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, sub_cpt_rev, 32, -, fp, _Quad,
3147 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, div_cpt_rev, 32, /, fp, _Quad,
3151 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, sub_cpt_rev, 64, -, fp, _Quad,
3154 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, div_cpt_rev, 64, /, fp, _Quad,
3158 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, sub_cpt_rev, -, fp, _Quad,
3160 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, div_cpt_rev, /, fp, _Quad,
3163 #endif // KMP_HAVE_QUAD 3167 #define ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3168 TYPE __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \ 3170 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 3171 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid)); 3173 #define CRITICAL_SWP(LCK_ID) \ 3174 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3176 old_value = (*lhs); \ 3179 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3183 #ifdef KMP_GOMP_COMPAT 3184 #define GOMP_CRITICAL_SWP(FLAG) \ 3185 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3190 #define GOMP_CRITICAL_SWP(FLAG) 3193 #define ATOMIC_XCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3194 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3196 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3197 old_value = KMP_XCHG_FIXED##BITS(lhs, rhs); \ 3201 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3202 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3204 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3205 old_value = KMP_XCHG_REAL##BITS(lhs, rhs); \ 3210 #define CMPXCHG_SWP(TYPE, BITS) \ 3212 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 3213 TYPE old_value, new_value; \ 3215 old_value = temp_val; \ 3217 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \ 3218 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \ 3219 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \ 3223 old_value = temp_val; \ 3230 #define ATOMIC_CMPXCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \ 3231 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3233 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3234 CMPXCHG_SWP(TYPE, BITS) \ 3237 ATOMIC_XCHG_SWP(fixed1, kmp_int8, 8, KMP_ARCH_X86)
3238 ATOMIC_XCHG_SWP(fixed2, kmp_int16, 16, KMP_ARCH_X86)
3239 ATOMIC_XCHG_SWP(fixed4, kmp_int32, 32, KMP_ARCH_X86)
3241 ATOMIC_XCHG_FLOAT_SWP(float4, kmp_real32, 32,
3245 ATOMIC_CMPXCHG_SWP(fixed8, kmp_int64, 64,
3247 ATOMIC_CMPXCHG_SWP(float8, kmp_real64, 64,
3250 ATOMIC_XCHG_SWP(fixed8, kmp_int64, 64, KMP_ARCH_X86)
3251 ATOMIC_XCHG_FLOAT_SWP(float8, kmp_real64, 64,
3258 #define ATOMIC_CRITICAL_SWP(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \ 3259 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \ 3261 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 3262 CRITICAL_SWP(LCK_ID) \ 3270 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \ 3271 void __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \ 3272 TYPE rhs, TYPE *out) { \ 3273 KMP_DEBUG_ASSERT(__kmp_init_serial); \ 3274 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid)); 3276 #define CRITICAL_SWP_WRK(LCK_ID) \ 3277 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3282 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \ 3286 #ifdef KMP_GOMP_COMPAT 3287 #define GOMP_CRITICAL_SWP_WRK(FLAG) \ 3288 if ((FLAG) && (__kmp_atomic_mode == 2)) { \ 3290 CRITICAL_SWP_WRK(0); \ 3293 #define GOMP_CRITICAL_SWP_WRK(FLAG) 3297 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \ 3298 ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \ 3300 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \ 3301 CRITICAL_SWP_WRK(LCK_ID) \ 3305 ATOMIC_CRITICAL_SWP(float10,
long double, 10r, 1)
3307 ATOMIC_CRITICAL_SWP(float16, QUAD_LEGACY, 16r, 1)
3310 ATOMIC_CRITICAL_SWP_WRK(cmplx4, kmp_cmplx32, 8c, 1)
3315 ATOMIC_CRITICAL_SWP(cmplx8, kmp_cmplx64, 16c, 1)
3316 ATOMIC_CRITICAL_SWP(cmplx10, kmp_cmplx80, 20c, 1)
3318 ATOMIC_CRITICAL_SWP(cmplx16, CPLX128_LEG, 32c, 1)
3320 ATOMIC_CRITICAL_SWP(float16_a16, Quad_a16_t, 16r,
3322 ATOMIC_CRITICAL_SWP(cmplx16_a16, kmp_cmplx128_a16_t, 32c,
3329 #endif // OMP_40_ENABLED 3331 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64 3338 void __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3339 void (*f)(
void *,
void *,
void *)) {
3340 KMP_DEBUG_ASSERT(__kmp_init_serial);
3343 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3349 kmp_int8 old_value, new_value;
3351 old_value = *(kmp_int8 *)lhs;
3352 (*f)(&new_value, &old_value, rhs);
3355 while (!KMP_COMPARE_AND_STORE_ACQ8((kmp_int8 *)lhs, *(kmp_int8 *)&old_value,
3356 *(kmp_int8 *)&new_value)) {
3359 old_value = *(kmp_int8 *)lhs;
3360 (*f)(&new_value, &old_value, rhs);
3367 #ifdef KMP_GOMP_COMPAT 3368 if (__kmp_atomic_mode == 2) {
3369 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3372 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3374 (*f)(lhs, lhs, rhs);
3376 #ifdef KMP_GOMP_COMPAT 3377 if (__kmp_atomic_mode == 2) {
3378 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3381 __kmp_release_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3385 void __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3386 void (*f)(
void *,
void *,
void *)) {
3388 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3390 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3393 !((kmp_uintptr_t)lhs & 0x1)
3396 kmp_int16 old_value, new_value;
3398 old_value = *(kmp_int16 *)lhs;
3399 (*f)(&new_value, &old_value, rhs);
3402 while (!KMP_COMPARE_AND_STORE_ACQ16(
3403 (kmp_int16 *)lhs, *(kmp_int16 *)&old_value, *(kmp_int16 *)&new_value)) {
3406 old_value = *(kmp_int16 *)lhs;
3407 (*f)(&new_value, &old_value, rhs);
3414 #ifdef KMP_GOMP_COMPAT 3415 if (__kmp_atomic_mode == 2) {
3416 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3419 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3421 (*f)(lhs, lhs, rhs);
3423 #ifdef KMP_GOMP_COMPAT 3424 if (__kmp_atomic_mode == 2) {
3425 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3428 __kmp_release_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3432 void __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3433 void (*f)(
void *,
void *,
void *)) {
3434 KMP_DEBUG_ASSERT(__kmp_init_serial);
3439 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
3442 !((kmp_uintptr_t)lhs & 0x3)
3445 kmp_int32 old_value, new_value;
3447 old_value = *(kmp_int32 *)lhs;
3448 (*f)(&new_value, &old_value, rhs);
3451 while (!KMP_COMPARE_AND_STORE_ACQ32(
3452 (kmp_int32 *)lhs, *(kmp_int32 *)&old_value, *(kmp_int32 *)&new_value)) {
3455 old_value = *(kmp_int32 *)lhs;
3456 (*f)(&new_value, &old_value, rhs);
3464 #ifdef KMP_GOMP_COMPAT 3465 if (__kmp_atomic_mode == 2) {
3466 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3469 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3471 (*f)(lhs, lhs, rhs);
3473 #ifdef KMP_GOMP_COMPAT 3474 if (__kmp_atomic_mode == 2) {
3475 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3478 __kmp_release_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3482 void __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3483 void (*f)(
void *,
void *,
void *)) {
3484 KMP_DEBUG_ASSERT(__kmp_init_serial);
3487 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3489 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3492 !((kmp_uintptr_t)lhs & 0x7)
3495 kmp_int64 old_value, new_value;
3497 old_value = *(kmp_int64 *)lhs;
3498 (*f)(&new_value, &old_value, rhs);
3500 while (!KMP_COMPARE_AND_STORE_ACQ64(
3501 (kmp_int64 *)lhs, *(kmp_int64 *)&old_value, *(kmp_int64 *)&new_value)) {
3504 old_value = *(kmp_int64 *)lhs;
3505 (*f)(&new_value, &old_value, rhs);
3513 #ifdef KMP_GOMP_COMPAT 3514 if (__kmp_atomic_mode == 2) {
3515 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3518 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3520 (*f)(lhs, lhs, rhs);
3522 #ifdef KMP_GOMP_COMPAT 3523 if (__kmp_atomic_mode == 2) {
3524 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3527 __kmp_release_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3531 void __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3532 void (*f)(
void *,
void *,
void *)) {
3533 KMP_DEBUG_ASSERT(__kmp_init_serial);
3535 #ifdef KMP_GOMP_COMPAT 3536 if (__kmp_atomic_mode == 2) {
3537 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3540 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3542 (*f)(lhs, lhs, rhs);
3544 #ifdef KMP_GOMP_COMPAT 3545 if (__kmp_atomic_mode == 2) {
3546 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3549 __kmp_release_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3552 void __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3553 void (*f)(
void *,
void *,
void *)) {
3554 KMP_DEBUG_ASSERT(__kmp_init_serial);
3556 #ifdef KMP_GOMP_COMPAT 3557 if (__kmp_atomic_mode == 2) {
3558 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3561 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3563 (*f)(lhs, lhs, rhs);
3565 #ifdef KMP_GOMP_COMPAT 3566 if (__kmp_atomic_mode == 2) {
3567 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3570 __kmp_release_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3573 void __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3574 void (*f)(
void *,
void *,
void *)) {
3575 KMP_DEBUG_ASSERT(__kmp_init_serial);
3577 #ifdef KMP_GOMP_COMPAT 3578 if (__kmp_atomic_mode == 2) {
3579 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3582 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3584 (*f)(lhs, lhs, rhs);
3586 #ifdef KMP_GOMP_COMPAT 3587 if (__kmp_atomic_mode == 2) {
3588 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3591 __kmp_release_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3594 void __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3595 void (*f)(
void *,
void *,
void *)) {
3596 KMP_DEBUG_ASSERT(__kmp_init_serial);
3598 #ifdef KMP_GOMP_COMPAT 3599 if (__kmp_atomic_mode == 2) {
3600 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3603 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3605 (*f)(lhs, lhs, rhs);
3607 #ifdef KMP_GOMP_COMPAT 3608 if (__kmp_atomic_mode == 2) {
3609 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3612 __kmp_release_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3618 void __kmpc_atomic_start(
void) {
3619 int gtid = __kmp_entry_gtid();
3620 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
3621 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3624 void __kmpc_atomic_end(
void) {
3625 int gtid = __kmp_get_gtid();
3626 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
3627 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);