20 #include "kmp_error.h"
21 #include "kmp_stats.h"
24 #include "ompt-internal.h"
25 #include "ompt-specific.h"
28 #define MAX_MESSAGE 512
49 if (__kmp_ignore_mppbeg() == FALSE) {
50 __kmp_internal_begin();
52 KC_TRACE( 10, (
"__kmpc_begin: called\n" ) );
70 if (__kmp_ignore_mppend() == FALSE) {
71 KC_TRACE( 10, (
"__kmpc_end: called\n" ) );
72 KA_TRACE( 30, (
"__kmpc_end\n" ));
74 __kmp_internal_end_thread( -1 );
100 kmp_int32 gtid = __kmp_entry_gtid();
102 KC_TRACE( 10, (
"__kmpc_global_thread_num: T#%d\n", gtid ) );
123 KC_TRACE( 10, (
"__kmpc_global_num_threads: num_threads = %d\n", __kmp_nth ) );
125 return TCR_4(__kmp_nth);
137 KC_TRACE( 10, (
"__kmpc_bound_thread_num: called\n" ) );
138 return __kmp_tid_from_gtid( __kmp_entry_gtid() );
149 KC_TRACE( 10, (
"__kmpc_bound_num_threads: called\n" ) );
151 return __kmp_entry_thread() -> th.th_team -> t.t_nproc;
173 if (__kmp_par_range == 0) {
180 semi2 = strchr(semi2,
';');
184 semi2 = strchr(semi2 + 1,
';');
188 if (__kmp_par_range_filename[0]) {
189 const char *name = semi2 - 1;
190 while ((name > loc->
psource) && (*name !=
'/') && (*name !=
';')) {
193 if ((*name ==
'/') || (*name ==
';')) {
196 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
197 return __kmp_par_range < 0;
200 semi3 = strchr(semi2 + 1,
';');
201 if (__kmp_par_range_routine[0]) {
202 if ((semi3 != NULL) && (semi3 > semi2)
203 && (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
204 return __kmp_par_range < 0;
207 if (KMP_SSCANF(semi3 + 1,
"%d", &line_no) == 1) {
208 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
209 return __kmp_par_range > 0;
211 return __kmp_par_range < 0;
227 return __kmp_entry_thread() -> th.th_root -> r.r_active;
242 KA_TRACE( 20, (
"__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
243 global_tid, num_threads ) );
245 __kmp_push_num_threads( loc, global_tid, num_threads );
249 __kmpc_pop_num_threads(
ident_t *loc, kmp_int32 global_tid )
251 KA_TRACE( 20, (
"__kmpc_pop_num_threads: enter\n" ) );
260 __kmpc_push_proc_bind(
ident_t *loc, kmp_int32 global_tid, kmp_int32 proc_bind )
262 KA_TRACE( 20, (
"__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n",
263 global_tid, proc_bind ) );
265 __kmp_push_proc_bind( loc, global_tid, (kmp_proc_bind_t)proc_bind );
285 int gtid = __kmp_entry_gtid();
289 va_start( ap, microtask );
292 kmp_info_t *master_th = __kmp_threads[ gtid ];
293 kmp_team_t *parent_team = master_th->th.th_team;
294 int tid = __kmp_tid_from_gtid( gtid );
295 parent_team->t.t_implicit_task_taskdata[tid].
296 ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0);
299 #if INCLUDE_SSC_MARKS
302 __kmp_fork_call( loc, gtid, fork_context_intel,
305 VOLATILE_CAST(
void *) microtask,
307 VOLATILE_CAST(microtask_t) microtask,
308 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
310 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
316 #if INCLUDE_SSC_MARKS
319 __kmp_join_call( loc, gtid );
324 if (ompt_status & ompt_status_track) {
325 parent_team->t.t_implicit_task_taskdata[tid].
326 ompt_task_info.frame.reenter_runtime_frame = 0;
348 KA_TRACE( 20, (
"__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
349 global_tid, num_teams, num_threads ) );
351 __kmp_push_num_teams( loc, global_tid, num_teams, num_threads );
366 int gtid = __kmp_entry_gtid();
367 kmp_info_t *this_thr = __kmp_threads[ gtid ];
369 va_start( ap, microtask );
372 this_thr->th.th_teams_microtask = microtask;
373 this_thr->th.th_teams_level = this_thr->th.th_team->t.t_level;
376 if ( this_thr->th.th_teams_size.nteams == 0 ) {
377 __kmp_push_num_teams( loc, gtid, 0, 0 );
379 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
380 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
381 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
383 __kmp_fork_call( loc, gtid, fork_context_intel,
386 VOLATILE_CAST(
void *) microtask,
388 VOLATILE_CAST(microtask_t) __kmp_teams_master,
389 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
390 #
if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
396 __kmp_join_call( loc, gtid );
397 this_thr->th.th_teams_microtask = NULL;
398 this_thr->th.th_teams_level = 0;
399 *(kmp_int64*)(&this_thr->th.th_teams_size) = 0L;
412 __kmpc_invoke_task_func(
int gtid )
414 return __kmp_invoke_task_func( gtid );
432 __kmp_serialized_parallel(loc, global_tid);
447 kmp_internal_control_t *top;
448 kmp_info_t *this_thr;
449 kmp_team_t *serial_team;
451 KC_TRACE( 10, (
"__kmpc_end_serialized_parallel: called by T#%d\n", global_tid ) );
459 if( ! TCR_4( __kmp_init_parallel ) )
460 __kmp_parallel_initialize();
462 this_thr = __kmp_threads[ global_tid ];
463 serial_team = this_thr->th.th_serial_team;
466 kmp_task_team_t * task_team = this_thr->th.th_task_team;
469 if ( task_team != NULL && task_team->tt.tt_found_proxy_tasks )
470 __kmp_task_team_wait(this_thr, serial_team, NULL );
474 KMP_DEBUG_ASSERT( serial_team );
475 KMP_ASSERT( serial_team -> t.t_serialized );
476 KMP_DEBUG_ASSERT( this_thr -> th.th_team == serial_team );
477 KMP_DEBUG_ASSERT( serial_team != this_thr->th.th_root->r.r_root_team );
478 KMP_DEBUG_ASSERT( serial_team -> t.t_threads );
479 KMP_DEBUG_ASSERT( serial_team -> t.t_threads[0] == this_thr );
482 top = serial_team -> t.t_control_stack_top;
483 if ( top && top -> serial_nesting_level == serial_team -> t.t_serialized ) {
484 copy_icvs( &serial_team -> t.t_threads[0] -> th.th_current_task -> td_icvs, top );
485 serial_team -> t.t_control_stack_top = top -> next;
490 serial_team -> t.t_level--;
493 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
495 dispatch_private_info_t * disp_buffer = serial_team->t.t_dispatch->th_disp_buffer;
496 serial_team->t.t_dispatch->th_disp_buffer =
497 serial_team->t.t_dispatch->th_disp_buffer->next;
498 __kmp_free( disp_buffer );
501 -- serial_team -> t.t_serialized;
502 if ( serial_team -> t.t_serialized == 0 ) {
506 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
507 if ( __kmp_inherit_fp_control && serial_team->t.t_fp_control_saved ) {
508 __kmp_clear_x87_fpu_status_word();
509 __kmp_load_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word );
510 __kmp_load_mxcsr( &serial_team->t.t_mxcsr );
514 this_thr -> th.th_team = serial_team -> t.t_parent;
515 this_thr -> th.th_info.ds.ds_tid = serial_team -> t.t_master_tid;
518 this_thr -> th.th_team_nproc = serial_team -> t.t_parent -> t.t_nproc;
519 this_thr -> th.th_team_master = serial_team -> t.t_parent -> t.t_threads[0];
520 this_thr -> th.th_team_serialized = this_thr -> th.th_team -> t.t_serialized;
523 this_thr -> th.th_dispatch = & this_thr -> th.th_team ->
524 t.t_dispatch[ serial_team -> t.t_master_tid ];
526 __kmp_pop_current_task_from_thread( this_thr );
528 KMP_ASSERT( this_thr -> th.th_current_task -> td_flags.executing == 0 );
529 this_thr -> th.th_current_task -> td_flags.executing = 1;
531 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
533 this_thr->th.th_task_team = this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
534 KA_TRACE( 20, (
"__kmpc_end_serialized_parallel: T#%d restoring task_team %p / team %p\n",
535 global_tid, this_thr -> th.th_task_team, this_thr -> th.th_team ) );
538 if ( __kmp_tasking_mode != tskm_immediate_exec ) {
539 KA_TRACE( 20, (
"__kmpc_end_serialized_parallel: T#%d decreasing nesting depth of serial team %p to %d\n",
540 global_tid, serial_team, serial_team -> t.t_serialized ) );
545 kmp_uint64 cur_time = 0;
547 if ( __itt_get_timestamp_ptr ) {
548 cur_time = __itt_get_timestamp();
551 if ( this_thr->th.th_team->t.t_level == 0
553 && this_thr->th.th_teams_microtask == NULL
557 this_thr->th.th_ident = loc;
558 if ( ( __itt_frame_submit_v3_ptr || KMP_ITT_DEBUG ) &&
559 ( __kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 1 ) )
561 __kmp_itt_frame_submit( global_tid, this_thr->th.th_frame_time_serialized,
562 cur_time, 0, loc, this_thr->th.th_team_nproc, 0 );
563 if ( __kmp_forkjoin_frames_mode == 3 )
565 __kmp_itt_frame_submit( global_tid, serial_team->t.t_region_time,
566 cur_time, 0, loc, this_thr->th.th_team_nproc, 2 );
567 }
else if ( ( __itt_frame_end_v3_ptr || KMP_ITT_DEBUG ) &&
568 ! __kmp_forkjoin_frames_mode && __kmp_forkjoin_frames )
570 __kmp_itt_region_joined( global_tid, 1 );
574 if ( __kmp_env_consistency_check )
575 __kmp_pop_parallel( global_tid, NULL );
589 KC_TRACE( 10, (
"__kmpc_flush: called\n" ) );
594 #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
607 if ( ! __kmp_cpuinfo.initialized ) {
608 __kmp_query_cpuid( & __kmp_cpuinfo );
610 if ( ! __kmp_cpuinfo.sse2 ) {
613 #if KMP_COMPILER_ICC || KMP_COMPILER_MSVC
616 __sync_synchronize();
617 #endif // KMP_COMPILER_ICC
620 #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64)
636 #error Unknown or unsupported architecture
657 KC_TRACE( 10, (
"__kmpc_barrier: called T#%d\n", global_tid ) );
659 if (! TCR_4(__kmp_init_parallel))
660 __kmp_parallel_initialize();
662 if ( __kmp_env_consistency_check ) {
664 KMP_WARNING( ConstructIdentInvalid );
667 __kmp_check_barrier( global_tid, ct_barrier, loc );
670 __kmp_threads[ global_tid ]->th.th_ident = loc;
678 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
694 KC_TRACE( 10, (
"__kmpc_master: called T#%d\n", global_tid ) );
696 if( ! TCR_4( __kmp_init_parallel ) )
697 __kmp_parallel_initialize();
699 if( KMP_MASTER_GTID( global_tid ))
702 #if OMPT_SUPPORT && OMPT_TRACE
704 if ((ompt_status == ompt_status_track_callback) &&
705 ompt_callbacks.ompt_callback(ompt_event_master_begin)) {
706 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
707 kmp_team_t *team = this_thr -> th.th_team;
709 int tid = __kmp_tid_from_gtid( global_tid );
710 ompt_callbacks.ompt_callback(ompt_event_master_begin)(
711 team->t.ompt_team_info.parallel_id,
712 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
717 if ( __kmp_env_consistency_check ) {
718 #if KMP_USE_DYNAMIC_LOCK
720 __kmp_push_sync( global_tid, ct_master, loc, NULL, 0 );
722 __kmp_check_sync( global_tid, ct_master, loc, NULL, 0 );
725 __kmp_push_sync( global_tid, ct_master, loc, NULL );
727 __kmp_check_sync( global_tid, ct_master, loc, NULL );
745 KC_TRACE( 10, (
"__kmpc_end_master: called T#%d\n", global_tid ) );
747 KMP_DEBUG_ASSERT( KMP_MASTER_GTID( global_tid ));
749 #if OMPT_SUPPORT && OMPT_TRACE
750 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
751 kmp_team_t *team = this_thr -> th.th_team;
752 if ((ompt_status == ompt_status_track_callback) &&
753 ompt_callbacks.ompt_callback(ompt_event_master_end)) {
754 int tid = __kmp_tid_from_gtid( global_tid );
755 ompt_callbacks.ompt_callback(ompt_event_master_end)(
756 team->t.ompt_team_info.parallel_id,
757 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
761 if ( __kmp_env_consistency_check ) {
763 KMP_WARNING( ThreadIdentInvalid );
765 if( KMP_MASTER_GTID( global_tid ))
766 __kmp_pop_sync( global_tid, ct_master, loc );
782 KMP_DEBUG_ASSERT( __kmp_init_serial );
784 KC_TRACE( 10, (
"__kmpc_ordered: called T#%d\n", gtid ));
786 if (! TCR_4(__kmp_init_parallel))
787 __kmp_parallel_initialize();
790 __kmp_itt_ordered_prep( gtid );
794 th = __kmp_threads[ gtid ];
796 #if OMPT_SUPPORT && OMPT_TRACE
797 if (ompt_status & ompt_status_track) {
799 th->th.ompt_thread_info.wait_id = (uint64_t) loc;
800 th->th.ompt_thread_info.state = ompt_state_wait_ordered;
803 if ((ompt_status == ompt_status_track_callback) &&
804 ompt_callbacks.ompt_callback(ompt_event_wait_ordered)) {
805 ompt_callbacks.ompt_callback(ompt_event_wait_ordered)(
806 th->th.ompt_thread_info.wait_id);
811 if ( th -> th.th_dispatch -> th_deo_fcn != 0 )
812 (*th->th.th_dispatch->th_deo_fcn)( & gtid, & cid, loc );
814 __kmp_parallel_deo( & gtid, & cid, loc );
816 #if OMPT_SUPPORT && OMPT_TRACE
817 if (ompt_status & ompt_status_track) {
819 th->th.ompt_thread_info.state = ompt_state_work_parallel;
820 th->th.ompt_thread_info.wait_id = 0;
823 if ((ompt_status == ompt_status_track_callback) &&
824 ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)) {
825 ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)(
826 th->th.ompt_thread_info.wait_id);
832 __kmp_itt_ordered_start( gtid );
849 KC_TRACE( 10, (
"__kmpc_end_ordered: called T#%d\n", gtid ) );
852 __kmp_itt_ordered_end( gtid );
856 th = __kmp_threads[ gtid ];
858 if ( th -> th.th_dispatch -> th_dxo_fcn != 0 )
859 (*th->th.th_dispatch->th_dxo_fcn)( & gtid, & cid, loc );
861 __kmp_parallel_dxo( & gtid, & cid, loc );
863 #if OMPT_SUPPORT && OMPT_BLAME
864 if ((ompt_status == ompt_status_track_callback) &&
865 ompt_callbacks.ompt_callback(ompt_event_release_ordered)) {
866 ompt_callbacks.ompt_callback(ompt_event_release_ordered)(
867 th->th.ompt_thread_info.wait_id);
872 #if KMP_USE_DYNAMIC_LOCK
874 static __forceinline kmp_indirect_lock_t *
875 __kmp_get_indirect_csptr(kmp_critical_name * crit,
ident_t const * loc, kmp_int32 gtid, kmp_dyna_lockseq_t seq)
879 kmp_indirect_lock_t **lck, *ret;
880 lck = (kmp_indirect_lock_t **)crit;
881 ret = (kmp_indirect_lock_t *)TCR_PTR(*lck);
884 kmp_indirect_locktag_t tag = DYNA_GET_I_TAG(seq);
885 kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
887 DYNA_I_LOCK_FUNC(ilk, init)(ilk->lock);
888 DYNA_SET_I_LOCK_LOCATION(ilk, loc);
889 DYNA_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
890 KA_TRACE(20, (
"__kmp_get_indirect_csptr: initialized indirect lock #%d\n", tag));
892 __kmp_itt_critical_creating(ilk->lock, loc);
894 int status = KMP_COMPARE_AND_STORE_PTR(lck, 0, ilk);
897 __kmp_itt_critical_destroyed(ilk->lock);
901 ret = (kmp_indirect_lock_t *)TCR_PTR(*lck);
902 KMP_DEBUG_ASSERT(ret != NULL);
909 #define DYNA_ACQUIRE_TAS_LOCK(lock, gtid) { \
910 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
911 if (l->lk.poll != DYNA_LOCK_FREE(tas) || \
912 ! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas))) { \
914 KMP_FSYNC_PREPARE(l); \
915 KMP_INIT_YIELD(spins); \
916 if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
919 KMP_YIELD_SPIN(spins); \
921 while (l->lk.poll != DYNA_LOCK_FREE(tas) || \
922 ! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas))) { \
923 if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
926 KMP_YIELD_SPIN(spins); \
930 KMP_FSYNC_ACQUIRED(l); \
934 #define DYNA_TEST_TAS_LOCK(lock, gtid, rc) { \
935 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
936 rc = l->lk.poll == DYNA_LOCK_FREE(tas) && \
937 KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas)); \
941 #define DYNA_RELEASE_TAS_LOCK(lock, gtid) { \
942 TCW_4(((kmp_tas_lock_t *)lock)->lk.poll, DYNA_LOCK_FREE(tas)); \
949 # include <sys/syscall.h>
951 # define FUTEX_WAIT 0
954 # define FUTEX_WAKE 1
958 #define DYNA_ACQUIRE_FUTEX_LOCK(lock, gtid) { \
959 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
960 kmp_int32 gtid_code = (gtid+1) << 1; \
962 KMP_FSYNC_PREPARE(ftx); \
963 kmp_int32 poll_val; \
964 while ((poll_val = KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), DYNA_LOCK_FREE(futex), \
965 DYNA_LOCK_BUSY(gtid_code, futex))) != DYNA_LOCK_FREE(futex)) { \
966 kmp_int32 cond = DYNA_LOCK_STRIP(poll_val) & 1; \
968 if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, poll_val | DYNA_LOCK_BUSY(1, futex))) { \
971 poll_val |= DYNA_LOCK_BUSY(1, futex); \
974 if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, NULL, NULL, 0)) != 0) { \
979 KMP_FSYNC_ACQUIRED(ftx); \
983 #define DYNA_TEST_FUTEX_LOCK(lock, gtid, rc) { \
984 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
985 if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), DYNA_LOCK_FREE(futex), DYNA_LOCK_BUSY(gtid+1, futex) << 1)) { \
986 KMP_FSYNC_ACQUIRED(ftx); \
994 #define DYNA_RELEASE_FUTEX_LOCK(lock, gtid) { \
995 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
997 KMP_FSYNC_RELEASING(ftx); \
998 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(ftx->lk.poll), DYNA_LOCK_FREE(futex)); \
999 if (DYNA_LOCK_STRIP(poll_val) & 1) { \
1000 syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, DYNA_LOCK_BUSY(1, futex), NULL, NULL, 0); \
1003 KMP_YIELD(TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \
1006 #endif // DYNA_HAS_FUTEX
1008 #else // KMP_USE_DYNAMIC_LOCK
1010 static kmp_user_lock_p
1011 __kmp_get_critical_section_ptr( kmp_critical_name * crit,
ident_t const * loc, kmp_int32 gtid )
1013 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1019 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
1021 if ( lck == NULL ) {
1026 lck = __kmp_user_lock_allocate( &idx, gtid, kmp_lf_critical_section );
1027 __kmp_init_user_lock_with_checks( lck );
1028 __kmp_set_user_lock_location( lck, loc );
1030 __kmp_itt_critical_creating( lck );
1043 int status = KMP_COMPARE_AND_STORE_PTR( lck_pp, 0, lck );
1045 if ( status == 0 ) {
1048 __kmp_itt_critical_destroyed( lck );
1052 __kmp_destroy_user_lock_with_checks( lck );
1053 __kmp_user_lock_free( &idx, gtid, lck );
1054 lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
1055 KMP_DEBUG_ASSERT( lck != NULL );
1061 #endif // KMP_USE_DYNAMIC_LOCK
1077 kmp_user_lock_p lck;
1079 KC_TRACE( 10, (
"__kmpc_critical: called T#%d\n", global_tid ) );
1081 #if KMP_USE_DYNAMIC_LOCK
1084 if (DYNA_IS_D_LOCK(__kmp_user_lock_seq)) {
1085 lck = (kmp_user_lock_p)crit;
1087 if (*((kmp_dyna_lock_t *)lck) == 0) {
1088 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)lck, 0, DYNA_GET_D_TAG(__kmp_user_lock_seq));
1090 if (__kmp_env_consistency_check) {
1091 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
1094 __kmp_itt_critical_acquiring(lck);
1096 # if DYNA_USE_FAST_TAS
1097 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1098 DYNA_ACQUIRE_TAS_LOCK(lck, global_tid);
1100 # elif DYNA_USE_FAST_FUTEX
1101 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1102 DYNA_ACQUIRE_FUTEX_LOCK(lck, global_tid);
1106 DYNA_D_LOCK_FUNC(lck, set)((kmp_dyna_lock_t *)lck, global_tid);
1109 kmp_indirect_lock_t *ilk = __kmp_get_indirect_csptr(crit, loc, global_tid, __kmp_user_lock_seq);
1111 if (__kmp_env_consistency_check) {
1112 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
1115 __kmp_itt_critical_acquiring(lck);
1117 DYNA_I_LOCK_FUNC(ilk, set)(lck, global_tid);
1120 #else // KMP_USE_DYNAMIC_LOCK
1124 KMP_CHECK_USER_LOCK_INIT();
1126 if ( ( __kmp_user_lock_kind == lk_tas )
1127 && (
sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1128 lck = (kmp_user_lock_p)crit;
1130 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1131 else if ( ( __kmp_user_lock_kind == lk_futex )
1132 && (
sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1133 lck = (kmp_user_lock_p)crit;
1137 lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
1140 if ( __kmp_env_consistency_check )
1141 __kmp_push_sync( global_tid, ct_critical, loc, lck );
1150 __kmp_itt_critical_acquiring( lck );
1153 __kmp_acquire_user_lock_with_checks( lck, global_tid );
1155 #endif // KMP_USE_DYNAMIC_LOCK
1158 __kmp_itt_critical_acquired( lck );
1161 KA_TRACE( 15, (
"__kmpc_critical: done T#%d\n", global_tid ));
1176 kmp_user_lock_p lck;
1178 KC_TRACE( 10, (
"__kmpc_end_critical: called T#%d\n", global_tid ));
1180 #if KMP_USE_DYNAMIC_LOCK
1181 if (DYNA_IS_D_LOCK(__kmp_user_lock_seq)) {
1182 lck = (kmp_user_lock_p)crit;
1183 KMP_ASSERT(lck != NULL);
1184 if (__kmp_env_consistency_check) {
1185 __kmp_pop_sync(global_tid, ct_critical, loc);
1188 __kmp_itt_critical_releasing( lck );
1190 # if DYNA_USE_FAST_TAS
1191 if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
1192 DYNA_RELEASE_TAS_LOCK(lck, global_tid);
1194 # elif DYNA_USE_FAST_FUTEX
1195 if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
1196 DYNA_RELEASE_FUTEX_LOCK(lck, global_tid);
1200 DYNA_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
1203 kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
1204 KMP_ASSERT(ilk != NULL);
1206 if (__kmp_env_consistency_check) {
1207 __kmp_pop_sync(global_tid, ct_critical, loc);
1210 __kmp_itt_critical_releasing( lck );
1212 DYNA_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
1215 #else // KMP_USE_DYNAMIC_LOCK
1217 if ( ( __kmp_user_lock_kind == lk_tas )
1218 && (
sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1219 lck = (kmp_user_lock_p)crit;
1221 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1222 else if ( ( __kmp_user_lock_kind == lk_futex )
1223 && (
sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
1224 lck = (kmp_user_lock_p)crit;
1228 lck = (kmp_user_lock_p) TCR_PTR(*((kmp_user_lock_p *)crit));
1231 KMP_ASSERT(lck != NULL);
1233 if ( __kmp_env_consistency_check )
1234 __kmp_pop_sync( global_tid, ct_critical, loc );
1237 __kmp_itt_critical_releasing( lck );
1240 __kmp_release_user_lock_with_checks( lck, global_tid );
1242 #if OMPT_SUPPORT && OMPT_BLAME
1243 if ((ompt_status == ompt_status_track_callback) &&
1244 ompt_callbacks.ompt_callback(ompt_event_release_critical)) {
1245 ompt_callbacks.ompt_callback(ompt_event_release_critical)(
1250 #endif // KMP_USE_DYNAMIC_LOCK
1252 KA_TRACE( 15, (
"__kmpc_end_critical: done T#%d\n", global_tid ));
1268 KC_TRACE( 10, (
"__kmpc_barrier_master: called T#%d\n", global_tid ) );
1270 if (! TCR_4(__kmp_init_parallel))
1271 __kmp_parallel_initialize();
1273 if ( __kmp_env_consistency_check )
1274 __kmp_check_barrier( global_tid, ct_barrier, loc );
1277 __kmp_threads[global_tid]->th.th_ident = loc;
1279 status = __kmp_barrier( bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL );
1281 return (status != 0) ? 0 : 1;
1296 KC_TRACE( 10, (
"__kmpc_end_barrier_master: called T#%d\n", global_tid ));
1298 __kmp_end_split_barrier ( bs_plain_barrier, global_tid );
1316 KC_TRACE( 10, (
"__kmpc_barrier_master_nowait: called T#%d\n", global_tid ));
1318 if (! TCR_4(__kmp_init_parallel))
1319 __kmp_parallel_initialize();
1321 if ( __kmp_env_consistency_check ) {
1323 KMP_WARNING( ConstructIdentInvalid );
1325 __kmp_check_barrier( global_tid, ct_barrier, loc );
1329 __kmp_threads[global_tid]->th.th_ident = loc;
1331 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
1335 if ( __kmp_env_consistency_check ) {
1339 if ( global_tid < 0 ) {
1340 KMP_WARNING( ThreadIdentInvalid );
1346 __kmp_pop_sync( global_tid, ct_master, loc );
1369 kmp_int32 rc = __kmp_enter_single( global_tid, loc, TRUE );
1371 #if OMPT_SUPPORT && OMPT_TRACE
1372 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
1373 kmp_team_t *team = this_thr -> th.th_team;
1374 int tid = __kmp_tid_from_gtid( global_tid );
1376 if ((ompt_status == ompt_status_track_callback)) {
1378 if (ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)) {
1379 ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)(
1380 team->t.ompt_team_info.parallel_id,
1381 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id,
1382 team->t.ompt_team_info.microtask);
1385 if (ompt_callbacks.ompt_callback(ompt_event_single_others_begin)) {
1386 ompt_callbacks.ompt_callback(ompt_event_single_others_begin)(
1387 team->t.ompt_team_info.parallel_id,
1388 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
1390 this_thr->th.ompt_thread_info.state = ompt_state_wait_single;
1410 __kmp_exit_single( global_tid );
1412 #if OMPT_SUPPORT && OMPT_TRACE
1413 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
1414 kmp_team_t *team = this_thr -> th.th_team;
1415 int tid = __kmp_tid_from_gtid( global_tid );
1417 if ((ompt_status == ompt_status_track_callback) &&
1418 ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)) {
1419 ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)(
1420 team->t.ompt_team_info.parallel_id,
1421 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
1436 KE_TRACE( 10, (
"__kmpc_for_static_fini called T#%d\n", global_tid));
1438 #if OMPT_SUPPORT && OMPT_TRACE
1439 kmp_info_t *this_thr = __kmp_threads[ global_tid ];
1440 kmp_team_t *team = this_thr -> th.th_team;
1441 int tid = __kmp_tid_from_gtid( global_tid );
1443 if ((ompt_status == ompt_status_track_callback) &&
1444 ompt_callbacks.ompt_callback(ompt_event_loop_end)) {
1445 ompt_callbacks.ompt_callback(ompt_event_loop_end)(
1446 team->t.ompt_team_info.parallel_id,
1447 team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
1451 if ( __kmp_env_consistency_check )
1452 __kmp_pop_workshare( global_tid, ct_pdo, loc );
1461 ompc_set_num_threads(
int arg )
1464 __kmp_set_num_threads( arg, __kmp_entry_gtid() );
1468 ompc_set_dynamic(
int flag )
1473 thread = __kmp_entry_thread();
1475 __kmp_save_internal_controls( thread );
1477 set__dynamic( thread, flag ? TRUE : FALSE );
1481 ompc_set_nested(
int flag )
1486 thread = __kmp_entry_thread();
1488 __kmp_save_internal_controls( thread );
1490 set__nested( thread, flag ? TRUE : FALSE );
1494 ompc_set_max_active_levels(
int max_active_levels )
1500 __kmp_set_max_active_levels( __kmp_entry_gtid(), max_active_levels );
1504 ompc_set_schedule( omp_sched_t kind,
int modifier )
1507 __kmp_set_schedule( __kmp_entry_gtid(), ( kmp_sched_t ) kind, modifier );
1511 ompc_get_ancestor_thread_num(
int level )
1513 return __kmp_get_ancestor_thread_num( __kmp_entry_gtid(), level );
1517 ompc_get_team_size(
int level )
1519 return __kmp_get_team_size( __kmp_entry_gtid(), level );
1523 kmpc_set_stacksize(
int arg )
1526 __kmp_aux_set_stacksize( arg );
1530 kmpc_set_stacksize_s(
size_t arg )
1533 __kmp_aux_set_stacksize( arg );
1537 kmpc_set_blocktime(
int arg )
1542 gtid = __kmp_entry_gtid();
1543 tid = __kmp_tid_from_gtid(gtid);
1544 thread = __kmp_thread_from_gtid(gtid);
1546 __kmp_aux_set_blocktime( arg, thread, tid );
1550 kmpc_set_library(
int arg )
1553 __kmp_user_set_library( (
enum library_type)arg );
1557 kmpc_set_defaults(
char const * str )
1560 __kmp_aux_set_defaults( str, KMP_STRLEN( str ) );
1564 kmpc_set_affinity_mask_proc(
int proc,
void **mask )
1566 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
1569 if ( ! TCR_4(__kmp_init_middle) ) {
1570 __kmp_middle_initialize();
1572 return __kmp_aux_set_affinity_mask_proc( proc, mask );
1577 kmpc_unset_affinity_mask_proc(
int proc,
void **mask )
1579 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
1582 if ( ! TCR_4(__kmp_init_middle) ) {
1583 __kmp_middle_initialize();
1585 return __kmp_aux_unset_affinity_mask_proc( proc, mask );
1590 kmpc_get_affinity_mask_proc(
int proc,
void **mask )
1592 #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
1595 if ( ! TCR_4(__kmp_init_middle) ) {
1596 __kmp_middle_initialize();
1598 return __kmp_aux_get_affinity_mask_proc( proc, mask );
1649 KC_TRACE( 10, (
"__kmpc_copyprivate: called T#%d\n", gtid ));
1653 data_ptr = & __kmp_team_from_gtid( gtid )->t.t_copypriv_data;
1655 if ( __kmp_env_consistency_check ) {
1657 KMP_WARNING( ConstructIdentInvalid );
1663 if (didit) *data_ptr = cpy_data;
1667 __kmp_threads[gtid]->th.th_ident = loc;
1669 __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
1671 if (! didit) (*cpy_func)( cpy_data, *data_ptr );
1677 __kmp_threads[gtid]->th.th_ident = loc;
1679 __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
1684 #define INIT_LOCK __kmp_init_user_lock_with_checks
1685 #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks
1686 #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks
1687 #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed
1688 #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks
1689 #define ACQUIRE_NESTED_LOCK_TIMED __kmp_acquire_nested_user_lock_with_checks_timed
1690 #define RELEASE_LOCK __kmp_release_user_lock_with_checks
1691 #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks
1692 #define TEST_LOCK __kmp_test_user_lock_with_checks
1693 #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks
1694 #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks
1695 #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks
1705 __kmpc_init_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1706 #if KMP_USE_DYNAMIC_LOCK
1707 KMP_DEBUG_ASSERT(__kmp_init_serial);
1708 if (__kmp_env_consistency_check && user_lock == NULL) {
1709 KMP_FATAL(LockIsUninitialized,
"omp_init_lock");
1711 if (DYNA_IS_D_LOCK(__kmp_user_lock_seq)) {
1712 DYNA_INIT_D_LOCK(user_lock, __kmp_user_lock_seq);
1714 __kmp_itt_lock_creating((kmp_user_lock_p)user_lock, NULL);
1717 DYNA_INIT_I_LOCK(user_lock, __kmp_user_lock_seq);
1718 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(user_lock);
1719 DYNA_SET_I_LOCK_LOCATION(ilk, loc);
1721 __kmp_itt_lock_creating(ilk->lock, loc);
1725 #else // KMP_USE_DYNAMIC_LOCK
1727 static char const *
const func =
"omp_init_lock";
1728 kmp_user_lock_p lck;
1729 KMP_DEBUG_ASSERT( __kmp_init_serial );
1731 if ( __kmp_env_consistency_check ) {
1732 if ( user_lock == NULL ) {
1733 KMP_FATAL( LockIsUninitialized, func );
1737 KMP_CHECK_USER_LOCK_INIT();
1739 if ( ( __kmp_user_lock_kind == lk_tas )
1740 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1741 lck = (kmp_user_lock_p)user_lock;
1743 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1744 else if ( ( __kmp_user_lock_kind == lk_futex )
1745 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1746 lck = (kmp_user_lock_p)user_lock;
1750 lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
1753 __kmp_set_user_lock_location( lck, loc );
1756 __kmp_itt_lock_creating( lck );
1759 #endif // KMP_USE_DYNAMIC_LOCK
1764 __kmpc_init_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1765 #if KMP_USE_DYNAMIC_LOCK
1767 KMP_DEBUG_ASSERT(__kmp_init_serial);
1768 if (__kmp_env_consistency_check && user_lock == NULL) {
1769 KMP_FATAL(LockIsUninitialized,
"omp_init_nest_lock");
1772 kmp_dyna_lockseq_t nested_seq;
1773 switch (__kmp_user_lock_seq) {
1774 case lockseq_tas: nested_seq = lockseq_nested_tas;
break;
1776 case lockseq_futex: nested_seq = lockseq_nested_futex;
break;
1778 case lockseq_ticket: nested_seq = lockseq_nested_ticket;
break;
1779 case lockseq_queuing: nested_seq = lockseq_nested_queuing;
break;
1780 case lockseq_drdpa: nested_seq = lockseq_nested_drdpa;
break;
1781 default: nested_seq = lockseq_nested_queuing;
break;
1784 DYNA_INIT_I_LOCK(user_lock, nested_seq);
1786 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(user_lock);
1787 DYNA_SET_I_LOCK_LOCATION(ilk, loc);
1789 __kmp_itt_lock_creating(ilk->lock, loc);
1792 #else // KMP_USE_DYNAMIC_LOCK
1794 static char const *
const func =
"omp_init_nest_lock";
1795 kmp_user_lock_p lck;
1796 KMP_DEBUG_ASSERT( __kmp_init_serial );
1798 if ( __kmp_env_consistency_check ) {
1799 if ( user_lock == NULL ) {
1800 KMP_FATAL( LockIsUninitialized, func );
1804 KMP_CHECK_USER_LOCK_INIT();
1806 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1807 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1808 lck = (kmp_user_lock_p)user_lock;
1810 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1811 else if ( ( __kmp_user_lock_kind == lk_futex )
1812 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1813 <= OMP_NEST_LOCK_T_SIZE ) ) {
1814 lck = (kmp_user_lock_p)user_lock;
1818 lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
1821 INIT_NESTED_LOCK( lck );
1822 __kmp_set_user_lock_location( lck, loc );
1825 __kmp_itt_lock_creating( lck );
1828 #endif // KMP_USE_DYNAMIC_LOCK
1832 __kmpc_destroy_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1833 #if KMP_USE_DYNAMIC_LOCK
1836 kmp_user_lock_p lck;
1837 if (DYNA_EXTRACT_D_TAG(user_lock) == 0) {
1838 lck = ((kmp_indirect_lock_t *)DYNA_LOOKUP_I_LOCK(user_lock))->lock;
1840 lck = (kmp_user_lock_p)user_lock;
1842 __kmp_itt_lock_destroyed(lck);
1844 DYNA_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
1846 kmp_user_lock_p lck;
1848 if ( ( __kmp_user_lock_kind == lk_tas )
1849 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1850 lck = (kmp_user_lock_p)user_lock;
1852 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1853 else if ( ( __kmp_user_lock_kind == lk_futex )
1854 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1855 lck = (kmp_user_lock_p)user_lock;
1859 lck = __kmp_lookup_user_lock( user_lock,
"omp_destroy_lock" );
1863 __kmp_itt_lock_destroyed( lck );
1865 DESTROY_LOCK( lck );
1867 if ( ( __kmp_user_lock_kind == lk_tas )
1868 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1871 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1872 else if ( ( __kmp_user_lock_kind == lk_futex )
1873 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1878 __kmp_user_lock_free( user_lock, gtid, lck );
1880 #endif // KMP_USE_DYNAMIC_LOCK
1885 __kmpc_destroy_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1886 #if KMP_USE_DYNAMIC_LOCK
1889 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(user_lock);
1890 __kmp_itt_lock_destroyed(ilk->lock);
1892 DYNA_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
1894 #else // KMP_USE_DYNAMIC_LOCK
1896 kmp_user_lock_p lck;
1898 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1899 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1900 lck = (kmp_user_lock_p)user_lock;
1902 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1903 else if ( ( __kmp_user_lock_kind == lk_futex )
1904 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1905 <= OMP_NEST_LOCK_T_SIZE ) ) {
1906 lck = (kmp_user_lock_p)user_lock;
1910 lck = __kmp_lookup_user_lock( user_lock,
"omp_destroy_nest_lock" );
1914 __kmp_itt_lock_destroyed( lck );
1917 DESTROY_NESTED_LOCK( lck );
1919 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
1920 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
1923 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1924 else if ( ( __kmp_user_lock_kind == lk_futex )
1925 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
1926 <= OMP_NEST_LOCK_T_SIZE ) ) {
1931 __kmp_user_lock_free( user_lock, gtid, lck );
1933 #endif // KMP_USE_DYNAMIC_LOCK
1937 __kmpc_set_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1939 #if KMP_USE_DYNAMIC_LOCK
1940 int tag = DYNA_EXTRACT_D_TAG(user_lock);
1942 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
1944 # if DYNA_USE_FAST_TAS
1945 if (tag == locktag_tas && !__kmp_env_consistency_check) {
1946 DYNA_ACQUIRE_TAS_LOCK(user_lock, gtid);
1948 # elif DYNA_USE_FAST_FUTEX
1949 if (tag == locktag_futex && !__kmp_env_consistency_check) {
1950 DYNA_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
1954 __kmp_direct_set_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
1957 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
1960 #else // KMP_USE_DYNAMIC_LOCK
1962 kmp_user_lock_p lck;
1964 if ( ( __kmp_user_lock_kind == lk_tas )
1965 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1966 lck = (kmp_user_lock_p)user_lock;
1968 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1969 else if ( ( __kmp_user_lock_kind == lk_futex )
1970 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
1971 lck = (kmp_user_lock_p)user_lock;
1975 lck = __kmp_lookup_user_lock( user_lock,
"omp_set_lock" );
1979 __kmp_itt_lock_acquiring( lck );
1982 ACQUIRE_LOCK( lck, gtid );
1985 __kmp_itt_lock_acquired( lck );
1988 #endif // KMP_USE_DYNAMIC_LOCK
1992 __kmpc_set_nest_lock(
ident_t * loc, kmp_int32 gtid,
void ** user_lock ) {
1993 #if KMP_USE_DYNAMIC_LOCK
1996 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
1998 DYNA_D_LOCK_FUNC(user_lock, set)((kmp_dyna_lock_t *)user_lock, gtid);
2000 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2003 #else // KMP_USE_DYNAMIC_LOCK
2004 kmp_user_lock_p lck;
2006 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
2007 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
2008 lck = (kmp_user_lock_p)user_lock;
2010 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2011 else if ( ( __kmp_user_lock_kind == lk_futex )
2012 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
2013 <= OMP_NEST_LOCK_T_SIZE ) ) {
2014 lck = (kmp_user_lock_p)user_lock;
2018 lck = __kmp_lookup_user_lock( user_lock,
"omp_set_nest_lock" );
2022 __kmp_itt_lock_acquiring( lck );
2025 ACQUIRE_NESTED_LOCK( lck, gtid );
2028 __kmp_itt_lock_acquired( lck );
2030 #endif // KMP_USE_DYNAMIC_LOCK
2034 __kmpc_unset_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2036 #if KMP_USE_DYNAMIC_LOCK
2038 int tag = DYNA_EXTRACT_D_TAG(user_lock);
2040 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2042 # if DYNA_USE_FAST_TAS
2043 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2044 DYNA_RELEASE_TAS_LOCK(user_lock, gtid);
2046 # elif DYNA_USE_FAST_FUTEX
2047 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2048 DYNA_RELEASE_FUTEX_LOCK(user_lock, gtid);
2052 __kmp_direct_unset_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2055 #else // KMP_USE_DYNAMIC_LOCK
2057 kmp_user_lock_p lck;
2062 if ( ( __kmp_user_lock_kind == lk_tas )
2063 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2064 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2067 __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
2069 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
2073 lck = (kmp_user_lock_p)user_lock;
2076 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2077 else if ( ( __kmp_user_lock_kind == lk_futex )
2078 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2079 lck = (kmp_user_lock_p)user_lock;
2083 lck = __kmp_lookup_user_lock( user_lock,
"omp_unset_lock" );
2087 __kmp_itt_lock_releasing( lck );
2090 RELEASE_LOCK( lck, gtid );
2092 #if OMPT_SUPPORT && OMPT_BLAME
2093 if ((ompt_status == ompt_status_track_callback) &&
2094 ompt_callbacks.ompt_callback(ompt_event_release_lock)) {
2095 ompt_callbacks.ompt_callback(ompt_event_release_lock)((uint64_t) lck);
2099 #endif // KMP_USE_DYNAMIC_LOCK
2104 __kmpc_unset_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2106 #if KMP_USE_DYNAMIC_LOCK
2109 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
2111 DYNA_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
2113 #else // KMP_USE_DYNAMIC_LOCK
2115 kmp_user_lock_p lck;
2119 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
2120 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
2121 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2123 kmp_tas_lock_t *tl = (kmp_tas_lock_t*)user_lock;
2125 __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
2127 if ( --(tl->lk.depth_locked) == 0 ) {
2128 TCW_4(tl->lk.poll, 0);
2133 lck = (kmp_user_lock_p)user_lock;
2136 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2137 else if ( ( __kmp_user_lock_kind == lk_futex )
2138 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
2139 <= OMP_NEST_LOCK_T_SIZE ) ) {
2140 lck = (kmp_user_lock_p)user_lock;
2144 lck = __kmp_lookup_user_lock( user_lock,
"omp_unset_nest_lock" );
2148 __kmp_itt_lock_releasing( lck );
2152 release_status = RELEASE_NESTED_LOCK( lck, gtid );
2153 #if OMPT_SUPPORT && OMPT_BLAME
2154 if (ompt_status == ompt_status_track_callback) {
2155 if (release_status == KMP_LOCK_RELEASED) {
2156 if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)) {
2157 ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)(
2160 }
else if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)) {
2161 ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)(
2167 #endif // KMP_USE_DYNAMIC_LOCK
2172 __kmpc_test_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2177 #if KMP_USE_DYNAMIC_LOCK
2179 int tag = DYNA_EXTRACT_D_TAG(user_lock);
2181 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2183 # if DYNA_USE_FAST_TAS
2184 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2185 DYNA_TEST_TAS_LOCK(user_lock, gtid, rc);
2187 # elif DYNA_USE_FAST_FUTEX
2188 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2189 DYNA_TEST_FUTEX_LOCK(user_lock, gtid, rc);
2193 rc = __kmp_direct_test_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2197 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2202 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
2207 #else // KMP_USE_DYNAMIC_LOCK
2209 kmp_user_lock_p lck;
2212 if ( ( __kmp_user_lock_kind == lk_tas )
2213 && (
sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2214 lck = (kmp_user_lock_p)user_lock;
2216 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2217 else if ( ( __kmp_user_lock_kind == lk_futex )
2218 && (
sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
2219 lck = (kmp_user_lock_p)user_lock;
2223 lck = __kmp_lookup_user_lock( user_lock,
"omp_test_lock" );
2227 __kmp_itt_lock_acquiring( lck );
2230 rc = TEST_LOCK( lck, gtid );
2233 __kmp_itt_lock_acquired( lck );
2235 __kmp_itt_lock_cancelled( lck );
2238 return ( rc ? FTN_TRUE : FTN_FALSE );
2242 #endif // KMP_USE_DYNAMIC_LOCK
2247 __kmpc_test_nest_lock(
ident_t *loc, kmp_int32 gtid,
void **user_lock )
2249 #if KMP_USE_DYNAMIC_LOCK
2252 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2254 rc = DYNA_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
2257 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2259 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
2264 #else // KMP_USE_DYNAMIC_LOCK
2266 kmp_user_lock_p lck;
2269 if ( ( __kmp_user_lock_kind == lk_tas ) && (
sizeof( lck->tas.lk.poll )
2270 +
sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
2271 lck = (kmp_user_lock_p)user_lock;
2273 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
2274 else if ( ( __kmp_user_lock_kind == lk_futex )
2275 && (
sizeof( lck->futex.lk.poll ) +
sizeof( lck->futex.lk.depth_locked )
2276 <= OMP_NEST_LOCK_T_SIZE ) ) {
2277 lck = (kmp_user_lock_p)user_lock;
2281 lck = __kmp_lookup_user_lock( user_lock,
"omp_test_nest_lock" );
2285 __kmp_itt_lock_acquiring( lck );
2288 rc = TEST_NESTED_LOCK( lck, gtid );
2291 __kmp_itt_lock_acquired( lck );
2293 __kmp_itt_lock_cancelled( lck );
2300 #endif // KMP_USE_DYNAMIC_LOCK
2313 #define __KMP_SET_REDUCTION_METHOD(gtid,rmethod) \
2314 ( ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method ) = ( rmethod ) )
2316 #define __KMP_GET_REDUCTION_METHOD(gtid) \
2317 ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method )
2323 static __forceinline
void
2324 __kmp_enter_critical_section_reduce_block(
ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
2330 kmp_user_lock_p lck;
2332 #if KMP_USE_DYNAMIC_LOCK
2334 if (DYNA_IS_D_LOCK(__kmp_user_lock_seq)) {
2335 lck = (kmp_user_lock_p)crit;
2336 if (*((kmp_dyna_lock_t *)lck) == 0) {
2337 KMP_COMPARE_AND_STORE_ACQ32((
volatile kmp_int32 *)lck, 0, DYNA_GET_D_TAG(__kmp_user_lock_seq));
2339 KMP_DEBUG_ASSERT(lck != NULL);
2340 if (__kmp_env_consistency_check) {
2341 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
2343 DYNA_D_LOCK_FUNC(lck, set)((kmp_dyna_lock_t *)lck, global_tid);
2345 kmp_indirect_lock_t *ilk = __kmp_get_indirect_csptr(crit, loc, global_tid, __kmp_user_lock_seq);
2346 KMP_DEBUG_ASSERT(ilk != NULL);
2347 if (__kmp_env_consistency_check) {
2348 __kmp_push_sync(global_tid, ct_critical, loc, ilk->lock, __kmp_user_lock_seq);
2350 DYNA_I_LOCK_FUNC(ilk, set)(ilk->lock, global_tid);
2353 #else // KMP_USE_DYNAMIC_LOCK
2358 if ( __kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE ) {
2359 lck = (kmp_user_lock_p)crit;
2362 lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
2364 KMP_DEBUG_ASSERT( lck != NULL );
2366 if ( __kmp_env_consistency_check )
2367 __kmp_push_sync( global_tid, ct_critical, loc, lck );
2369 __kmp_acquire_user_lock_with_checks( lck, global_tid );
2371 #endif // KMP_USE_DYNAMIC_LOCK
2375 static __forceinline
void
2376 __kmp_end_critical_section_reduce_block(
ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
2378 kmp_user_lock_p lck;
2380 #if KMP_USE_DYNAMIC_LOCK
2382 if (DYNA_IS_D_LOCK(__kmp_user_lock_seq)) {
2383 lck = (kmp_user_lock_p)crit;
2384 if (__kmp_env_consistency_check)
2385 __kmp_pop_sync(global_tid, ct_critical, loc);
2386 DYNA_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
2388 kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
2389 if (__kmp_env_consistency_check)
2390 __kmp_pop_sync(global_tid, ct_critical, loc);
2391 DYNA_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
2394 #else // KMP_USE_DYNAMIC_LOCK
2398 if ( __kmp_base_user_lock_size > 32 ) {
2399 lck = *( (kmp_user_lock_p *) crit );
2400 KMP_ASSERT( lck != NULL );
2402 lck = (kmp_user_lock_p) crit;
2405 if ( __kmp_env_consistency_check )
2406 __kmp_pop_sync( global_tid, ct_critical, loc );
2408 __kmp_release_user_lock_with_checks( lck, global_tid );
2410 #endif // KMP_USE_DYNAMIC_LOCK
2430 ident_t *loc, kmp_int32 global_tid,
2431 kmp_int32 num_vars,
size_t reduce_size,
void *reduce_data,
void (*reduce_func)(
void *lhs_data,
void *rhs_data),
2432 kmp_critical_name *lck ) {
2436 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2440 int teams_swapped = 0, task_state;
2442 KA_TRACE( 10, (
"__kmpc_reduce_nowait() enter: called T#%d\n", global_tid ) );
2449 if( ! TCR_4( __kmp_init_parallel ) )
2450 __kmp_parallel_initialize();
2453 #if KMP_USE_DYNAMIC_LOCK
2454 if ( __kmp_env_consistency_check )
2455 __kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 );
2457 if ( __kmp_env_consistency_check )
2458 __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
2462 th = __kmp_thread_from_gtid(global_tid);
2463 if( th->th.th_teams_microtask ) {
2464 team = th->th.th_team;
2465 if( team->t.t_level == th->th.th_teams_level ) {
2467 KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid);
2470 th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2471 th->th.th_team = team->t.t_parent;
2472 th->th.th_team_nproc = th->th.th_team->t.t_nproc;
2473 th->th.th_task_team = th->th.th_team->t.t_task_team[0];
2474 task_state = th->th.th_task_state;
2475 th->th.th_task_state = 0;
2478 #endif // OMP_40_ENABLED
2489 packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
2490 __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
2492 if( packed_reduction_method == critical_reduce_block ) {
2494 __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
2497 }
else if( packed_reduction_method == empty_reduce_block ) {
2502 }
else if( packed_reduction_method == atomic_reduce_block ) {
2509 if ( __kmp_env_consistency_check )
2510 __kmp_pop_sync( global_tid, ct_reduce, loc );
2512 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2525 __kmp_threads[global_tid]->th.th_ident = loc;
2527 retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, FALSE, reduce_size, reduce_data, reduce_func );
2528 retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
2532 if ( __kmp_env_consistency_check ) {
2534 __kmp_pop_sync( global_tid, ct_reduce, loc );
2545 if( teams_swapped ) {
2547 th->th.th_info.ds.ds_tid = 0;
2548 th->th.th_team = team;
2549 th->th.th_team_nproc = team->t.t_nproc;
2550 th->th.th_task_team = team->t.t_task_team[task_state];
2551 th->th.th_task_state = task_state;
2554 KA_TRACE( 10, (
"__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
2570 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2572 KA_TRACE( 10, (
"__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid ) );
2574 packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
2576 if( packed_reduction_method == critical_reduce_block ) {
2578 __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
2580 }
else if( packed_reduction_method == empty_reduce_block ) {
2584 }
else if( packed_reduction_method == atomic_reduce_block ) {
2591 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2602 if ( __kmp_env_consistency_check )
2603 __kmp_pop_sync( global_tid, ct_reduce, loc );
2605 KA_TRACE( 10, (
"__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
2627 ident_t *loc, kmp_int32 global_tid,
2628 kmp_int32 num_vars,
size_t reduce_size,
void *reduce_data,
2629 void (*reduce_func)(
void *lhs_data,
void *rhs_data),
2630 kmp_critical_name *lck )
2634 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2636 KA_TRACE( 10, (
"__kmpc_reduce() enter: called T#%d\n", global_tid ) );
2643 if( ! TCR_4( __kmp_init_parallel ) )
2644 __kmp_parallel_initialize();
2647 #if KMP_USE_DYNAMIC_LOCK
2648 if ( __kmp_env_consistency_check )
2649 __kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 );
2651 if ( __kmp_env_consistency_check )
2652 __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
2655 packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
2656 __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
2658 if( packed_reduction_method == critical_reduce_block ) {
2660 __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
2663 }
else if( packed_reduction_method == empty_reduce_block ) {
2668 }
else if( packed_reduction_method == atomic_reduce_block ) {
2672 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2678 __kmp_threads[global_tid]->th.th_ident = loc;
2680 retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, TRUE, reduce_size, reduce_data, reduce_func );
2681 retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
2685 if ( __kmp_env_consistency_check ) {
2687 __kmp_pop_sync( global_tid, ct_reduce, loc );
2698 KA_TRACE( 10, (
"__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
2715 PACKED_REDUCTION_METHOD_T packed_reduction_method;
2717 KA_TRACE( 10, (
"__kmpc_end_reduce() enter: called T#%d\n", global_tid ) );
2719 packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
2724 if( packed_reduction_method == critical_reduce_block ) {
2726 __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
2730 __kmp_threads[global_tid]->th.th_ident = loc;
2732 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2734 }
else if( packed_reduction_method == empty_reduce_block ) {
2740 __kmp_threads[global_tid]->th.th_ident = loc;
2742 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2744 }
else if( packed_reduction_method == atomic_reduce_block ) {
2748 __kmp_threads[global_tid]->th.th_ident = loc;
2750 __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
2752 }
else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
2755 __kmp_end_split_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid );
2764 if ( __kmp_env_consistency_check )
2765 __kmp_pop_sync( global_tid, ct_reduce, loc );
2767 KA_TRACE( 10, (
"__kmpc_end_reduce() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
2772 #undef __KMP_GET_REDUCTION_METHOD
2773 #undef __KMP_SET_REDUCTION_METHOD
2778 __kmpc_get_taskid() {
2781 kmp_info_t * thread;
2783 gtid = __kmp_get_gtid();
2787 thread = __kmp_thread_from_gtid( gtid );
2788 return thread->th.th_current_task->td_task_id;
2794 __kmpc_get_parent_taskid() {
2797 kmp_info_t * thread;
2798 kmp_taskdata_t * parent_task;
2800 gtid = __kmp_get_gtid();
2804 thread = __kmp_thread_from_gtid( gtid );
2805 parent_task = thread->th.th_current_task->td_parent;
2806 return ( parent_task == NULL ? 0 : parent_task->td_task_id );
2810 void __kmpc_place_threads(
int nC,
int nT,
int nO)
2812 if ( ! __kmp_init_serial ) {
2813 __kmp_serial_initialize();
2815 __kmp_place_num_cores = nC;
2816 __kmp_place_num_threads_per_core = nT;
2817 __kmp_place_core_offset = nO;
kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
#define KMP_STOP_EXPLICIT_TIMER(name)
"Stops" an explicit timer.
#define KMP_TIME_BLOCK(name)
Uses specified timer (name) to time code block.
void __kmpc_flush(ident_t *loc)
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end(ident_t *loc)
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
#define KMP_IDENT_AUTOPAR
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
kmp_int32 __kmpc_in_parallel(ident_t *loc)
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)