19 #include "kmp_error.h"
21 #define MAX_MESSAGE 512
30 #define KMP_DEBUG_REF_CTS(x) KF_TRACE(1, x);
32 #define THREAD_ALLOC_FOR_TASKQ
35 in_parallel_context( kmp_team_t *team )
37 return ! team -> t.t_serialized;
41 __kmp_taskq_eo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref )
44 int tid = __kmp_tid_from_gtid( gtid );
46 kmpc_task_queue_t *taskq;
47 kmp_taskq_t *tq = & __kmp_threads[gtid] -> th.th_team -> t.t_taskq;
49 if ( __kmp_env_consistency_check )
50 #if KMP_USE_DYNAMIC_LOCK
51 __kmp_push_sync( gtid, ct_ordered_in_taskq, loc_ref, NULL, 0 );
53 __kmp_push_sync( gtid, ct_ordered_in_taskq, loc_ref, NULL );
56 if ( ! __kmp_threads[ gtid ]-> th.th_team -> t.t_serialized ) {
62 my_token =tq->tq_curr_thunk[ tid ]-> th_tasknum;
64 taskq = tq->tq_curr_thunk[ tid ]-> th.th_shareds -> sv_queue;
66 KMP_WAIT_YIELD(&taskq->tq_tasknum_serving, my_token, KMP_EQ, NULL);
72 __kmp_taskq_xo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref )
75 int tid = __kmp_tid_from_gtid( gtid );
77 kmp_taskq_t *tq = & __kmp_threads[gtid] -> th.th_team -> t.t_taskq;
79 if ( __kmp_env_consistency_check )
80 __kmp_pop_sync( gtid, ct_ordered_in_taskq, loc_ref );
82 if ( ! __kmp_threads[ gtid ]-> th.th_team -> t.t_serialized ) {
88 my_token = tq->tq_curr_thunk[ tid ]->th_tasknum;
92 tq->tq_curr_thunk[ tid ]-> th.th_shareds -> sv_queue -> tq_tasknum_serving = my_token + 1;
99 __kmp_taskq_check_ordered( kmp_int32 gtid, kmpc_thunk_t *thunk )
102 kmpc_task_queue_t *taskq;
108 my_token = thunk -> th_tasknum;
110 taskq = thunk -> th.th_shareds -> sv_queue;
112 if(taskq->tq_tasknum_serving <= my_token) {
113 KMP_WAIT_YIELD(&taskq->tq_tasknum_serving, my_token, KMP_GE, NULL);
115 taskq->tq_tasknum_serving = my_token +1;
123 __kmp_dump_TQF(kmp_int32 flags)
125 if (flags & TQF_IS_ORDERED)
126 __kmp_printf(
"ORDERED ");
127 if (flags & TQF_IS_LASTPRIVATE)
128 __kmp_printf(
"LAST_PRIV ");
129 if (flags & TQF_IS_NOWAIT)
130 __kmp_printf(
"NOWAIT ");
131 if (flags & TQF_HEURISTICS)
132 __kmp_printf(
"HEURIST ");
133 if (flags & TQF_INTERFACE_RESERVED1)
134 __kmp_printf(
"RESERV1 ");
135 if (flags & TQF_INTERFACE_RESERVED2)
136 __kmp_printf(
"RESERV2 ");
137 if (flags & TQF_INTERFACE_RESERVED3)
138 __kmp_printf(
"RESERV3 ");
139 if (flags & TQF_INTERFACE_RESERVED4)
140 __kmp_printf(
"RESERV4 ");
141 if (flags & TQF_IS_LAST_TASK)
142 __kmp_printf(
"LAST_TASK ");
143 if (flags & TQF_TASKQ_TASK)
144 __kmp_printf(
"TASKQ_TASK ");
145 if (flags & TQF_RELEASE_WORKERS)
146 __kmp_printf(
"RELEASE ");
147 if (flags & TQF_ALL_TASKS_QUEUED)
148 __kmp_printf(
"ALL_QUEUED ");
149 if (flags & TQF_PARALLEL_CONTEXT)
150 __kmp_printf(
"PARALLEL ");
151 if (flags & TQF_DEALLOCATED)
152 __kmp_printf(
"DEALLOC ");
153 if (!(flags & (TQF_INTERNAL_FLAGS|TQF_INTERFACE_FLAGS)))
154 __kmp_printf(
"(NONE)");
158 __kmp_dump_thunk( kmp_taskq_t *tq, kmpc_thunk_t *thunk, kmp_int32 global_tid )
161 int nproc = __kmp_threads[global_tid] -> th.th_team -> t.t_nproc;
163 __kmp_printf(
"\tThunk at %p on (%d): ", thunk, global_tid);
166 for (i = 0; i < nproc; i++) {
167 if( tq->tq_curr_thunk[i] == thunk ) {
168 __kmp_printf(
"[%i] ", i);
171 __kmp_printf(
"th_shareds=%p, ", thunk->th.th_shareds);
172 __kmp_printf(
"th_task=%p, ", thunk->th_task);
173 __kmp_printf(
"th_encl_thunk=%p, ", thunk->th_encl_thunk);
174 __kmp_printf(
"th_status=%d, ", thunk->th_status);
175 __kmp_printf(
"th_tasknum=%u, ", thunk->th_tasknum);
176 __kmp_printf(
"th_flags="); __kmp_dump_TQF(thunk->th_flags);
183 __kmp_dump_thunk_stack(kmpc_thunk_t *thunk, kmp_int32 thread_num)
187 __kmp_printf(
" Thunk stack for T#%d: ", thread_num);
189 for (th = thunk; th != NULL; th = th->th_encl_thunk )
190 __kmp_printf(
"%p ", th);
196 __kmp_dump_task_queue( kmp_taskq_t *tq, kmpc_task_queue_t *queue, kmp_int32 global_tid )
200 kmpc_task_queue_t *taskq;
202 __kmp_printf(
"Task Queue at %p on (%d):\n", queue, global_tid);
205 int in_parallel = queue->tq_flags & TQF_PARALLEL_CONTEXT;
207 if ( __kmp_env_consistency_check ) {
208 __kmp_printf(
" tq_loc : ");
220 __kmp_printf(
" tq_parent : %p\n", queue->tq.tq_parent);
221 __kmp_printf(
" tq_first_child : %p\n", queue->tq_first_child);
222 __kmp_printf(
" tq_next_child : %p\n", queue->tq_next_child);
223 __kmp_printf(
" tq_prev_child : %p\n", queue->tq_prev_child);
224 __kmp_printf(
" tq_ref_count : %d\n", queue->tq_ref_count);
238 __kmp_printf(
" tq_shareds : ");
239 for (i=0; i<((queue == tq->tq_root) ? queue->tq_nproc : 1); i++)
240 __kmp_printf(
"%p ", queue->tq_shareds[i].ai_data);
244 __kmp_printf(
" tq_tasknum_queuing : %u\n", queue->tq_tasknum_queuing);
245 __kmp_printf(
" tq_tasknum_serving : %u\n", queue->tq_tasknum_serving);
248 __kmp_printf(
" tq_queue : %p\n", queue->tq_queue);
249 __kmp_printf(
" tq_thunk_space : %p\n", queue->tq_thunk_space);
250 __kmp_printf(
" tq_taskq_slot : %p\n", queue->tq_taskq_slot);
252 __kmp_printf(
" tq_free_thunks : ");
253 for (thunk = queue->tq_free_thunks; thunk != NULL; thunk = thunk->th.th_next_free )
254 __kmp_printf(
"%p ", thunk);
257 __kmp_printf(
" tq_nslots : %d\n", queue->tq_nslots);
258 __kmp_printf(
" tq_head : %d\n", queue->tq_head);
259 __kmp_printf(
" tq_tail : %d\n", queue->tq_tail);
260 __kmp_printf(
" tq_nfull : %d\n", queue->tq_nfull);
261 __kmp_printf(
" tq_hiwat : %d\n", queue->tq_hiwat);
262 __kmp_printf(
" tq_flags : "); __kmp_dump_TQF(queue->tq_flags);
266 __kmp_printf(
" tq_th_thunks : ");
267 for (i = 0; i < queue->tq_nproc; i++) {
268 __kmp_printf(
"%d ", queue->tq_th_thunks[i].ai_data);
274 __kmp_printf(
" Queue slots:\n");
278 for ( count = 0; count < queue->tq_nfull; ++count ) {
279 __kmp_printf(
"(%d)", qs);
280 __kmp_dump_thunk( tq, queue->tq_queue[qs].qs_thunk, global_tid );
281 qs = (qs+1) % queue->tq_nslots;
287 if (queue->tq_taskq_slot != NULL) {
288 __kmp_printf(
" TaskQ slot:\n");
289 __kmp_dump_thunk( tq, (kmpc_thunk_t *) queue->tq_taskq_slot, global_tid );
297 __kmp_printf(
" Taskq freelist: ");
304 for( taskq = tq->tq_freelist; taskq != NULL; taskq = taskq->tq.tq_next_free )
305 __kmp_printf(
"%p ", taskq);
309 __kmp_printf(
"\n\n");
313 __kmp_aux_dump_task_queue_tree( kmp_taskq_t *tq, kmpc_task_queue_t *curr_queue, kmp_int32 level, kmp_int32 global_tid )
316 int nproc = __kmp_threads[global_tid] -> th.th_team -> t.t_nproc;
317 kmpc_task_queue_t *queue = curr_queue;
319 if (curr_queue == NULL)
324 for (i=0; i<level; i++)
327 __kmp_printf(
"%p", curr_queue);
329 for (i = 0; i < nproc; i++) {
330 if( tq->tq_curr_thunk[i] && tq->tq_curr_thunk[i]->th.th_shareds->sv_queue == curr_queue ) {
331 __kmp_printf(
" [%i]", i);
342 qs = curr_queue->tq_tail;
344 for ( count = 0; count < curr_queue->tq_nfull; ++count ) {
345 __kmp_printf(
"%p ", curr_queue->tq_queue[qs].qs_thunk);
346 qs = (qs+1) % curr_queue->tq_nslots;
353 if (curr_queue->tq_first_child) {
359 if (curr_queue->tq_first_child) {
360 for(queue = (kmpc_task_queue_t *)curr_queue->tq_first_child;
362 queue = queue->tq_next_child) {
363 __kmp_aux_dump_task_queue_tree( tq, queue, level+1, global_tid );
372 __kmp_dump_task_queue_tree( kmp_taskq_t *tq, kmpc_task_queue_t *tqroot, kmp_int32 global_tid)
374 __kmp_printf(
"TaskQ Tree at root %p on (%d):\n", tqroot, global_tid);
376 __kmp_aux_dump_task_queue_tree( tq, tqroot, 0, global_tid );
391 __kmp_taskq_allocate(
size_t size, kmp_int32 global_tid)
393 void *addr, *orig_addr;
396 KB_TRACE( 5, (
"__kmp_taskq_allocate: called size=%d, gtid=%d\n", (
int) size, global_tid ) );
398 bytes =
sizeof(
void *) + CACHE_LINE + size;
400 #ifdef THREAD_ALLOC_FOR_TASKQ
401 orig_addr = (
void *) __kmp_thread_malloc( __kmp_thread_from_gtid(global_tid), bytes );
403 KE_TRACE( 10, (
"%%%%%% MALLOC( %d )\n", bytes ) );
404 orig_addr = (
void *) KMP_INTERNAL_MALLOC( bytes );
408 KMP_FATAL( OutOfHeapMemory );
412 if (((kmp_uintptr_t) addr & ( CACHE_LINE - 1 )) != 0) {
413 KB_TRACE( 50, (
"__kmp_taskq_allocate: adjust for cache alignment\n" ) );
414 addr = (
void *) (((kmp_uintptr_t) addr + CACHE_LINE) & ~( CACHE_LINE - 1 ));
417 (* (
void **) addr) = orig_addr;
419 KB_TRACE( 10, (
"__kmp_taskq_allocate: allocate: %p, use: %p - %p, size: %d, gtid: %d\n",
420 orig_addr, ((
void **) addr) + 1, ((
char *)(((
void **) addr) + 1)) + size-1,
421 (
int) size, global_tid ));
423 return ( ((
void **) addr) + 1 );
427 __kmpc_taskq_free(
void *p, kmp_int32 global_tid)
429 KB_TRACE( 5, (
"__kmpc_taskq_free: called addr=%p, gtid=%d\n", p, global_tid ) );
431 KB_TRACE(10, (
"__kmpc_taskq_free: freeing: %p, gtid: %d\n", (*( ((
void **) p)-1)), global_tid ));
433 #ifdef THREAD_ALLOC_FOR_TASKQ
434 __kmp_thread_free( __kmp_thread_from_gtid(global_tid), *( ((
void **) p)-1) );
436 KMP_INTERNAL_FREE( *( ((
void **) p)-1) );
447 static kmpc_task_queue_t *
448 __kmp_alloc_taskq ( kmp_taskq_t *tq,
int in_parallel, kmp_int32 nslots, kmp_int32 nthunks,
449 kmp_int32 nshareds, kmp_int32 nproc,
size_t sizeof_thunk,
450 size_t sizeof_shareds, kmpc_thunk_t **new_taskq_thunk, kmp_int32 global_tid )
454 kmpc_task_queue_t *new_queue;
455 kmpc_aligned_shared_vars_t *shared_var_array;
456 char *shared_var_storage;
459 __kmp_acquire_lock( & tq->tq_freelist_lck, global_tid );
464 if( tq->tq_freelist ) {
465 new_queue = tq -> tq_freelist;
466 tq -> tq_freelist = tq -> tq_freelist -> tq.tq_next_free;
468 KMP_DEBUG_ASSERT(new_queue->tq_flags & TQF_DEALLOCATED);
470 new_queue->tq_flags = 0;
472 __kmp_release_lock( & tq->tq_freelist_lck, global_tid );
475 __kmp_release_lock( & tq->tq_freelist_lck, global_tid );
477 new_queue = (kmpc_task_queue_t *) __kmp_taskq_allocate (
sizeof (kmpc_task_queue_t), global_tid);
478 new_queue->tq_flags = 0;
484 sizeof_thunk += (CACHE_LINE - (sizeof_thunk % CACHE_LINE));
485 pt = (
char *) __kmp_taskq_allocate (nthunks * sizeof_thunk, global_tid);
486 new_queue->tq_thunk_space = (kmpc_thunk_t *)pt;
487 *new_taskq_thunk = (kmpc_thunk_t *)(pt + (nthunks - 1) * sizeof_thunk);
491 new_queue->tq_free_thunks = (kmpc_thunk_t *)pt;
493 for (i = 0; i < (nthunks - 2); i++) {
494 ((kmpc_thunk_t *)(pt+i*sizeof_thunk))->th.th_next_free = (kmpc_thunk_t *)(pt + (i+1)*sizeof_thunk);
496 ((kmpc_thunk_t *)(pt+i*sizeof_thunk))->th_flags = TQF_DEALLOCATED;
500 ((kmpc_thunk_t *)(pt+(nthunks-2)*sizeof_thunk))->th.th_next_free = NULL;
502 ((kmpc_thunk_t *)(pt+(nthunks-2)*sizeof_thunk))->th_flags = TQF_DEALLOCATED;
508 __kmp_init_lock( & new_queue->tq_link_lck );
509 __kmp_init_lock( & new_queue->tq_free_thunks_lck );
510 __kmp_init_lock( & new_queue->tq_queue_lck );
515 bytes = nslots *
sizeof (kmpc_aligned_queue_slot_t);
516 new_queue->tq_queue = (kmpc_aligned_queue_slot_t *) __kmp_taskq_allocate( bytes, global_tid );
519 sizeof_shareds +=
sizeof(kmpc_task_queue_t *);
520 sizeof_shareds += (CACHE_LINE - (sizeof_shareds % CACHE_LINE));
522 bytes = nshareds *
sizeof (kmpc_aligned_shared_vars_t);
523 shared_var_array = (kmpc_aligned_shared_vars_t *) __kmp_taskq_allocate ( bytes, global_tid);
525 bytes = nshareds * sizeof_shareds;
526 shared_var_storage = (
char *) __kmp_taskq_allocate ( bytes, global_tid);
528 for (i=0; i<nshareds; i++) {
529 shared_var_array[i].ai_data = (kmpc_shared_vars_t *) (shared_var_storage + i*sizeof_shareds);
530 shared_var_array[i].ai_data->sv_queue = new_queue;
532 new_queue->tq_shareds = shared_var_array;
538 bytes = nproc *
sizeof(kmpc_aligned_int32_t);
539 new_queue->tq_th_thunks = (kmpc_aligned_int32_t *) __kmp_taskq_allocate ( bytes, global_tid);
540 new_queue->tq_nproc = nproc;
542 for (i=0; i<nproc; i++)
543 new_queue->tq_th_thunks[i].ai_data = 0;
550 __kmp_free_taskq (kmp_taskq_t *tq, kmpc_task_queue_t *p,
int in_parallel, kmp_int32 global_tid)
552 __kmpc_taskq_free(p->tq_thunk_space, global_tid);
553 __kmpc_taskq_free(p->tq_queue, global_tid);
556 __kmpc_taskq_free((
void *) p->tq_shareds[0].ai_data, global_tid);
559 __kmpc_taskq_free(p->tq_shareds, global_tid);
562 p->tq_first_child = NULL;
563 p->tq_next_child = NULL;
564 p->tq_prev_child = NULL;
565 p->tq_ref_count = -10;
566 p->tq_shareds = NULL;
567 p->tq_tasknum_queuing = 0;
568 p->tq_tasknum_serving = 0;
570 p->tq_thunk_space = NULL;
571 p->tq_taskq_slot = NULL;
572 p->tq_free_thunks = NULL;
582 for (i=0; i<p->tq_nproc; i++)
583 p->tq_th_thunks[i].ai_data = 0;
585 if ( __kmp_env_consistency_check )
587 KMP_DEBUG_ASSERT( p->tq_flags & TQF_DEALLOCATED );
588 p->tq_flags = TQF_DEALLOCATED;
592 __kmpc_taskq_free(p->tq_th_thunks, global_tid);
593 __kmp_destroy_lock(& p->tq_link_lck);
594 __kmp_destroy_lock(& p->tq_queue_lck);
595 __kmp_destroy_lock(& p->tq_free_thunks_lck);
598 p->tq_th_thunks = NULL;
604 __kmp_acquire_lock( & tq->tq_freelist_lck, global_tid );
605 p->tq.tq_next_free = tq->tq_freelist;
608 __kmp_release_lock( & tq->tq_freelist_lck, global_tid );
617 static kmpc_thunk_t *
618 __kmp_alloc_thunk (kmpc_task_queue_t *queue,
int in_parallel, kmp_int32 global_tid)
623 __kmp_acquire_lock(& queue->tq_free_thunks_lck, global_tid);
629 fl = queue->tq_free_thunks;
631 KMP_DEBUG_ASSERT (fl != NULL);
633 queue->tq_free_thunks = fl->th.th_next_free;
637 __kmp_release_lock(& queue->tq_free_thunks_lck, global_tid);
643 __kmp_free_thunk (kmpc_task_queue_t *queue, kmpc_thunk_t *p,
int in_parallel, kmp_int32 global_tid)
647 p->th_encl_thunk = 0;
654 __kmp_acquire_lock(& queue->tq_free_thunks_lck, global_tid);
660 p->th.th_next_free = queue->tq_free_thunks;
661 queue->tq_free_thunks = p;
664 p->th_flags = TQF_DEALLOCATED;
668 __kmp_release_lock(& queue->tq_free_thunks_lck, global_tid);
676 __kmp_enqueue_task ( kmp_taskq_t *tq, kmp_int32 global_tid, kmpc_task_queue_t *queue, kmpc_thunk_t *thunk,
int in_parallel )
682 __kmp_acquire_lock(& queue->tq_queue_lck, global_tid);
688 KMP_DEBUG_ASSERT (queue->tq_nfull < queue->tq_nslots);
690 queue->tq_queue[(queue->tq_head)++].qs_thunk = thunk;
692 if (queue->tq_head >= queue->tq_nslots)
699 ret = (in_parallel) ? (queue->tq_nfull == queue->tq_nslots) : FALSE;
703 __kmp_release_lock(& queue->tq_queue_lck, global_tid);
705 if( tq->tq_global_flags & TQF_RELEASE_WORKERS ) {
712 tq->tq_global_flags &= ~TQF_RELEASE_WORKERS;
723 static kmpc_thunk_t *
724 __kmp_dequeue_task (kmp_int32 global_tid, kmpc_task_queue_t *queue,
int in_parallel)
727 int tid = __kmp_tid_from_gtid( global_tid );
729 KMP_DEBUG_ASSERT (queue->tq_nfull > 0);
731 if (queue->tq.tq_parent != NULL && in_parallel) {
733 __kmp_acquire_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
734 ct = ++(queue->tq_ref_count);
735 __kmp_release_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
736 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p inc %d\n",
737 __LINE__, global_tid, queue, ct));
740 pt = queue->tq_queue[(queue->tq_tail)++].qs_thunk;
742 if (queue->tq_tail >= queue->tq_nslots)
746 queue->tq_th_thunks[tid].ai_data++;
750 KF_TRACE(200, (
"__kmp_dequeue_task: T#%d(:%d) now has %d outstanding thunks from queue %p\n",
751 global_tid, tid, queue->tq_th_thunks[tid].ai_data, queue));
761 KMP_DEBUG_ASSERT(queue->tq_nfull >= 0);
764 KMP_DEBUG_ASSERT(queue->tq_th_thunks[tid].ai_data <= __KMP_TASKQ_THUNKS_PER_TH);
795 static kmpc_thunk_t *
796 __kmp_find_task_in_queue (kmp_int32 global_tid, kmpc_task_queue_t *queue)
798 kmpc_thunk_t *pt = NULL;
799 int tid = __kmp_tid_from_gtid( global_tid );
802 if ( !(queue->tq_flags & TQF_DEALLOCATED) ) {
804 __kmp_acquire_lock(& queue->tq_queue_lck, global_tid);
807 if ( !(queue->tq_flags & TQF_DEALLOCATED) ) {
812 if ((queue->tq_taskq_slot != NULL) && (queue->tq_nfull <= queue->tq_hiwat)) {
815 pt = (kmpc_thunk_t *) queue->tq_taskq_slot;
816 queue->tq_taskq_slot = NULL;
818 else if (queue->tq_nfull == 0 ||
819 queue->tq_th_thunks[tid].ai_data >= __KMP_TASKQ_THUNKS_PER_TH) {
825 else if (queue->tq_nfull > 1) {
828 pt = __kmp_dequeue_task (global_tid, queue, TRUE);
830 else if (!(queue->tq_flags & TQF_IS_LASTPRIVATE)) {
833 pt = __kmp_dequeue_task (global_tid, queue, TRUE);
835 else if (queue->tq_flags & TQF_IS_LAST_TASK) {
840 pt = __kmp_dequeue_task (global_tid, queue, TRUE);
841 pt->th_flags |= TQF_IS_LAST_TASK;
846 __kmp_release_lock(& queue->tq_queue_lck, global_tid);
858 static kmpc_thunk_t *
859 __kmp_find_task_in_descendant_queue (kmp_int32 global_tid, kmpc_task_queue_t *curr_queue)
861 kmpc_thunk_t *pt = NULL;
862 kmpc_task_queue_t *queue = curr_queue;
864 if (curr_queue->tq_first_child != NULL) {
865 __kmp_acquire_lock(& curr_queue->tq_link_lck, global_tid);
870 queue = (kmpc_task_queue_t *) curr_queue->tq_first_child;
872 __kmp_release_lock(& curr_queue->tq_link_lck, global_tid);
876 while (queue != NULL) {
878 kmpc_task_queue_t *next;
880 ct= ++(queue->tq_ref_count);
881 __kmp_release_lock(& curr_queue->tq_link_lck, global_tid);
882 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p inc %d\n",
883 __LINE__, global_tid, queue, ct));
885 pt = __kmp_find_task_in_queue (global_tid, queue);
890 __kmp_acquire_lock(& curr_queue->tq_link_lck, global_tid);
895 ct = --(queue->tq_ref_count);
896 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p dec %d\n",
897 __LINE__, global_tid, queue, ct));
898 KMP_DEBUG_ASSERT( queue->tq_ref_count >= 0 );
900 __kmp_release_lock(& curr_queue->tq_link_lck, global_tid);
908 pt = __kmp_find_task_in_descendant_queue (global_tid, queue);
913 __kmp_acquire_lock(& curr_queue->tq_link_lck, global_tid);
918 ct = --(queue->tq_ref_count);
919 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p dec %d\n",
920 __LINE__, global_tid, queue, ct));
921 KMP_DEBUG_ASSERT( ct >= 0 );
923 __kmp_release_lock(& curr_queue->tq_link_lck, global_tid);
928 __kmp_acquire_lock(& curr_queue->tq_link_lck, global_tid);
933 next = queue->tq_next_child;
935 ct = --(queue->tq_ref_count);
936 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p dec %d\n",
937 __LINE__, global_tid, queue, ct));
938 KMP_DEBUG_ASSERT( ct >= 0 );
943 __kmp_release_lock(& curr_queue->tq_link_lck, global_tid);
955 static kmpc_thunk_t *
956 __kmp_find_task_in_ancestor_queue (kmp_taskq_t *tq, kmp_int32 global_tid, kmpc_task_queue_t *curr_queue)
958 kmpc_task_queue_t *queue;
963 if (curr_queue->tq.tq_parent != NULL) {
964 queue = curr_queue->tq.tq_parent;
966 while (queue != NULL) {
967 if (queue->tq.tq_parent != NULL) {
969 __kmp_acquire_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
974 ct = ++(queue->tq_ref_count);
975 __kmp_release_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
976 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p inc %d\n",
977 __LINE__, global_tid, queue, ct));
980 pt = __kmp_find_task_in_queue (global_tid, queue);
982 if (queue->tq.tq_parent != NULL) {
984 __kmp_acquire_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
989 ct = --(queue->tq_ref_count);
990 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p dec %d\n",
991 __LINE__, global_tid, queue, ct));
992 KMP_DEBUG_ASSERT( ct >= 0 );
994 __kmp_release_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
1000 if (queue->tq.tq_parent != NULL) {
1002 __kmp_acquire_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
1007 ct = --(queue->tq_ref_count);
1008 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p dec %d\n",
1009 __LINE__, global_tid, queue, ct));
1010 KMP_DEBUG_ASSERT( ct >= 0 );
1012 queue = queue->tq.tq_parent;
1015 __kmp_release_lock(& queue->tq_link_lck, global_tid);
1020 pt = __kmp_find_task_in_descendant_queue( global_tid, tq->tq_root );
1026 __kmp_taskq_tasks_finished (kmpc_task_queue_t *queue)
1032 for (i=0; i<queue->tq_nproc; i++) {
1033 if (queue->tq_th_thunks[i].ai_data != 0)
1041 __kmp_taskq_has_any_children (kmpc_task_queue_t *queue)
1043 return (queue->tq_first_child != NULL);
1047 __kmp_remove_queue_from_tree( kmp_taskq_t *tq, kmp_int32 global_tid, kmpc_task_queue_t *queue,
int in_parallel )
1051 kmpc_thunk_t *thunk;
1054 KF_TRACE(50, (
"Before Deletion of TaskQ at %p on (%d):\n", queue, global_tid));
1055 KF_DUMP(50, __kmp_dump_task_queue( tq, queue, global_tid ));
1058 KMP_DEBUG_ASSERT (queue->tq.tq_parent != NULL);
1061 __kmp_acquire_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
1067 KMP_DEBUG_ASSERT (queue->tq_first_child == NULL);
1070 if (queue->tq_prev_child != NULL)
1071 queue->tq_prev_child->tq_next_child = queue->tq_next_child;
1072 if (queue->tq_next_child != NULL)
1073 queue->tq_next_child->tq_prev_child = queue->tq_prev_child;
1074 if (queue->tq.tq_parent->tq_first_child == queue)
1075 queue->tq.tq_parent->tq_first_child = queue->tq_next_child;
1077 queue->tq_prev_child = NULL;
1078 queue->tq_next_child = NULL;
1081 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p waiting for ref_count of %d to reach 1\n",
1082 __LINE__, global_tid, queue, queue->tq_ref_count));
1085 while (queue->tq_ref_count > 1) {
1086 __kmp_release_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
1088 KMP_WAIT_YIELD((
volatile kmp_uint32*)&queue->tq_ref_count, 1, KMP_LE, NULL);
1090 __kmp_acquire_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
1096 __kmp_release_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
1099 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p freeing queue\n",
1100 __LINE__, global_tid, queue));
1103 KMP_DEBUG_ASSERT(queue->tq_flags & TQF_ALL_TASKS_QUEUED);
1104 KMP_DEBUG_ASSERT(queue->tq_nfull == 0);
1106 for (i=0; i<queue->tq_nproc; i++) {
1107 KMP_DEBUG_ASSERT(queue->tq_th_thunks[i].ai_data == 0);
1111 for (thunk=queue->tq_free_thunks; thunk != NULL; thunk=thunk->th.th_next_free)
1114 KMP_ASSERT (i == queue->tq_nslots + (queue->tq_nproc * __KMP_TASKQ_THUNKS_PER_TH));
1118 __kmp_free_taskq ( tq, queue, TRUE, global_tid );
1120 KF_TRACE(50, (
"After Deletion of TaskQ at %p on (%d):\n", queue, global_tid));
1121 KF_DUMP(50, __kmp_dump_task_queue_tree( tq, tq->tq_root, global_tid ));
1132 __kmp_find_and_remove_finished_child_taskq( kmp_taskq_t *tq, kmp_int32 global_tid, kmpc_task_queue_t *curr_queue )
1134 kmpc_task_queue_t *queue = curr_queue;
1136 if (curr_queue->tq_first_child != NULL) {
1137 __kmp_acquire_lock(& curr_queue->tq_link_lck, global_tid);
1142 queue = (kmpc_task_queue_t *) curr_queue->tq_first_child;
1143 if (queue != NULL) {
1144 __kmp_release_lock(& curr_queue->tq_link_lck, global_tid);
1148 while (queue != NULL) {
1149 kmpc_task_queue_t *next;
1150 int ct = ++(queue->tq_ref_count);
1151 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p inc %d\n",
1152 __LINE__, global_tid, queue, ct));
1159 if (queue->tq_flags & TQF_IS_NOWAIT) {
1160 __kmp_find_and_remove_finished_child_taskq ( tq, global_tid, queue );
1162 if ((queue->tq_flags & TQF_ALL_TASKS_QUEUED) && (queue->tq_nfull == 0) &&
1163 __kmp_taskq_tasks_finished(queue) && ! __kmp_taskq_has_any_children(queue)) {
1170 if ( __kmp_test_lock(& queue->tq_queue_lck, global_tid) ) {
1171 if ( !(queue->tq_flags & TQF_DEALLOCATED) ) {
1172 queue->tq_flags |= TQF_DEALLOCATED;
1173 __kmp_release_lock(& queue->tq_queue_lck, global_tid);
1175 __kmp_remove_queue_from_tree( tq, global_tid, queue, TRUE );
1181 __kmp_release_lock(& queue->tq_queue_lck, global_tid);
1188 __kmp_acquire_lock(& curr_queue->tq_link_lck, global_tid);
1193 next = queue->tq_next_child;
1195 ct = --(queue->tq_ref_count);
1196 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p dec %d\n",
1197 __LINE__, global_tid, queue, ct));
1198 KMP_DEBUG_ASSERT( ct >= 0 );
1203 __kmp_release_lock(& curr_queue->tq_link_lck, global_tid);
1214 __kmp_remove_all_child_taskq( kmp_taskq_t *tq, kmp_int32 global_tid, kmpc_task_queue_t *queue )
1216 kmpc_task_queue_t *next_child;
1218 queue = (kmpc_task_queue_t *) queue->tq_first_child;
1220 while (queue != NULL) {
1221 __kmp_remove_all_child_taskq ( tq, global_tid, queue );
1223 next_child = queue->tq_next_child;
1224 queue->tq_flags |= TQF_DEALLOCATED;
1225 __kmp_remove_queue_from_tree ( tq, global_tid, queue, FALSE );
1231 __kmp_execute_task_from_queue( kmp_taskq_t *tq,
ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *thunk,
int in_parallel )
1233 kmpc_task_queue_t *queue = thunk->th.th_shareds->sv_queue;
1234 kmp_int32 tid = __kmp_tid_from_gtid( global_tid );
1236 KF_TRACE(100, (
"After dequeueing this Task on (%d):\n", global_tid));
1237 KF_DUMP(100, __kmp_dump_thunk( tq, thunk, global_tid ));
1238 KF_TRACE(100, (
"Task Queue: %p looks like this (%d):\n", queue, global_tid));
1239 KF_DUMP(100, __kmp_dump_task_queue( tq, queue, global_tid ));
1255 if (!(thunk->th_flags & TQF_TASKQ_TASK)) {
1256 kmp_int32 index = (queue == tq->tq_root) ? tid : 0;
1257 thunk->th.th_shareds = (kmpc_shared_vars_t *) queue->tq_shareds[index].ai_data;
1259 if ( __kmp_env_consistency_check ) {
1260 __kmp_push_workshare( global_tid,
1261 (queue->tq_flags & TQF_IS_ORDERED) ? ct_task_ordered : ct_task,
1266 if ( __kmp_env_consistency_check )
1267 __kmp_push_workshare( global_tid, ct_taskq, queue->tq_loc );
1271 thunk->th_encl_thunk = tq->tq_curr_thunk[tid];
1272 tq->tq_curr_thunk[tid] = thunk;
1274 KF_DUMP( 200, __kmp_dump_thunk_stack( tq->tq_curr_thunk[tid], global_tid ));
1277 KF_TRACE( 50, (
"Begin Executing Thunk %p from queue %p on (%d)\n", thunk, queue, global_tid));
1278 thunk->th_task (global_tid, thunk);
1279 KF_TRACE( 50, (
"End Executing Thunk %p from queue %p on (%d)\n", thunk, queue, global_tid));
1281 if (!(thunk->th_flags & TQF_TASKQ_TASK)) {
1282 if ( __kmp_env_consistency_check )
1283 __kmp_pop_workshare( global_tid, (queue->tq_flags & TQF_IS_ORDERED) ? ct_task_ordered : ct_task,
1287 tq->tq_curr_thunk[tid] = thunk->th_encl_thunk;
1288 thunk->th_encl_thunk = NULL;
1289 KF_DUMP( 200, __kmp_dump_thunk_stack( tq->tq_curr_thunk[tid], global_tid ));
1292 if ((thunk->th_flags & TQF_IS_ORDERED) && in_parallel) {
1293 __kmp_taskq_check_ordered(global_tid, thunk);
1296 __kmp_free_thunk (queue, thunk, in_parallel, global_tid);
1298 KF_TRACE(100, (
"T#%d After freeing thunk: %p, TaskQ looks like this:\n", global_tid, thunk));
1299 KF_DUMP(100, __kmp_dump_task_queue( tq, queue, global_tid ));
1304 KMP_DEBUG_ASSERT(queue->tq_th_thunks[tid].ai_data >= 1);
1306 KF_TRACE( 200, (
"__kmp_execute_task_from_queue: T#%d has %d thunks in queue %p\n",
1307 global_tid, queue->tq_th_thunks[tid].ai_data-1, queue));
1309 queue->tq_th_thunks[tid].ai_data--;
1314 if (queue->tq.tq_parent != NULL && in_parallel) {
1316 __kmp_acquire_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
1317 ct = --(queue->tq_ref_count);
1318 __kmp_release_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
1319 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p dec %d\n",
1320 __LINE__, global_tid, queue, ct));
1321 KMP_DEBUG_ASSERT( ct >= 0 );
1332 __kmpc_taskq(
ident_t *loc, kmp_int32 global_tid, kmpc_task_t taskq_task,
1333 size_t sizeof_thunk,
size_t sizeof_shareds,
1334 kmp_int32 flags, kmpc_shared_vars_t **shareds )
1337 kmp_int32 nslots, nthunks, nshareds, nproc;
1338 kmpc_task_queue_t *new_queue, *curr_queue;
1339 kmpc_thunk_t *new_taskq_thunk;
1345 KE_TRACE( 10, (
"__kmpc_taskq called (%d)\n", global_tid));
1347 th = __kmp_threads[ global_tid ];
1348 team = th -> th.th_team;
1349 tq = & team -> t.t_taskq;
1350 nproc = team -> t.t_nproc;
1351 tid = __kmp_tid_from_gtid( global_tid );
1354 in_parallel = in_parallel_context( team );
1356 if( ! tq->tq_root ) {
1359 th->th.th_dispatch->th_deo_fcn = __kmp_taskq_eo;
1362 th->th.th_dispatch->th_dxo_fcn = __kmp_taskq_xo;
1368 if ( __kmp_barrier( bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL )) {
1375 *shareds = (kmpc_shared_vars_t *) tq->tq_root->tq_shareds[tid].ai_data;
1377 KE_TRACE( 10, (
"__kmpc_taskq return (%d)\n", global_tid));
1385 if( tq->tq_curr_thunk_capacity < nproc ) {
1386 if(tq->tq_curr_thunk)
1387 __kmp_free(tq->tq_curr_thunk);
1390 __kmp_init_lock( & tq->tq_freelist_lck );
1393 tq->tq_curr_thunk = (kmpc_thunk_t **) __kmp_allocate( nproc *
sizeof(kmpc_thunk_t *) );
1394 tq -> tq_curr_thunk_capacity = nproc;
1398 tq->tq_global_flags = TQF_RELEASE_WORKERS;
1404 nslots = (in_parallel) ? (2 * nproc) : 1;
1409 nthunks = (in_parallel) ? (nslots + (nproc * __KMP_TASKQ_THUNKS_PER_TH) + 1) : nslots + 2;
1414 nshareds = ( !tq->tq_root && in_parallel) ? nproc : 1;
1418 new_queue = __kmp_alloc_taskq ( tq, in_parallel, nslots, nthunks, nshareds, nproc,
1419 sizeof_thunk, sizeof_shareds, &new_taskq_thunk, global_tid );
1423 new_queue->tq_flags = flags & TQF_INTERFACE_FLAGS;
1426 new_queue->tq_tasknum_queuing = 0;
1427 new_queue->tq_tasknum_serving = 0;
1428 new_queue->tq_flags |= TQF_PARALLEL_CONTEXT;
1431 new_queue->tq_taskq_slot = NULL;
1432 new_queue->tq_nslots = nslots;
1433 new_queue->tq_hiwat = HIGH_WATER_MARK (nslots);
1434 new_queue->tq_nfull = 0;
1435 new_queue->tq_head = 0;
1436 new_queue->tq_tail = 0;
1437 new_queue->tq_loc = loc;
1439 if ((new_queue->tq_flags & TQF_IS_ORDERED) && in_parallel) {
1441 new_queue->tq_tasknum_serving = 1;
1444 th->th.th_dispatch->th_deo_fcn = __kmp_taskq_eo;
1447 th->th.th_dispatch->th_dxo_fcn = __kmp_taskq_xo;
1451 *shareds = (kmpc_shared_vars_t *) new_queue->tq_shareds[0].ai_data;
1453 new_taskq_thunk->th.th_shareds = *shareds;
1454 new_taskq_thunk->th_task = taskq_task;
1455 new_taskq_thunk->th_flags = new_queue->tq_flags | TQF_TASKQ_TASK;
1456 new_taskq_thunk->th_status = 0;
1458 KMP_DEBUG_ASSERT (new_taskq_thunk->th_flags & TQF_TASKQ_TASK);
1465 if( ! tq->tq_root ) {
1466 new_queue->tq.tq_parent = NULL;
1467 new_queue->tq_first_child = NULL;
1468 new_queue->tq_next_child = NULL;
1469 new_queue->tq_prev_child = NULL;
1470 new_queue->tq_ref_count = 1;
1471 tq->tq_root = new_queue;
1474 curr_queue = tq->tq_curr_thunk[tid]->th.th_shareds->sv_queue;
1475 new_queue->tq.tq_parent = curr_queue;
1476 new_queue->tq_first_child = NULL;
1477 new_queue->tq_prev_child = NULL;
1478 new_queue->tq_ref_count = 1;
1480 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p alloc %d\n",
1481 __LINE__, global_tid, new_queue, new_queue->tq_ref_count));
1483 __kmp_acquire_lock(& curr_queue->tq_link_lck, global_tid);
1488 new_queue->tq_next_child = (
struct kmpc_task_queue_t *) curr_queue->tq_first_child;
1490 if (curr_queue->tq_first_child != NULL)
1491 curr_queue->tq_first_child->tq_prev_child = new_queue;
1493 curr_queue->tq_first_child = new_queue;
1495 __kmp_release_lock(& curr_queue->tq_link_lck, global_tid);
1499 new_taskq_thunk->th_encl_thunk = tq->tq_curr_thunk[tid];
1500 tq->tq_curr_thunk[tid] = new_taskq_thunk;
1502 KF_DUMP( 200, __kmp_dump_thunk_stack( tq->tq_curr_thunk[tid], global_tid ));
1505 new_taskq_thunk->th_encl_thunk = 0;
1506 new_queue->tq.tq_parent = NULL;
1507 new_queue->tq_first_child = NULL;
1508 new_queue->tq_next_child = NULL;
1509 new_queue->tq_prev_child = NULL;
1510 new_queue->tq_ref_count = 1;
1514 KF_TRACE(150, (
"Creating TaskQ Task on (%d):\n", global_tid));
1515 KF_DUMP(150, __kmp_dump_thunk( tq, new_taskq_thunk, global_tid ));
1518 KF_TRACE(25, (
"After TaskQ at %p Creation on (%d):\n", new_queue, global_tid));
1520 KF_TRACE(25, (
"After Serial TaskQ at %p Creation on (%d):\n", new_queue, global_tid));
1523 KF_DUMP(25, __kmp_dump_task_queue( tq, new_queue, global_tid ));
1526 KF_DUMP(50, __kmp_dump_task_queue_tree( tq, tq->tq_root, global_tid ));
1530 if ( __kmp_env_consistency_check )
1531 __kmp_push_workshare( global_tid, ct_taskq, new_queue->tq_loc );
1533 KE_TRACE( 10, (
"__kmpc_taskq return (%d)\n", global_tid));
1535 return new_taskq_thunk;
1542 __kmpc_end_taskq(
ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *taskq_thunk)
1550 kmp_int32 is_outermost;
1551 kmpc_task_queue_t *queue;
1552 kmpc_thunk_t *thunk;
1555 KE_TRACE( 10, (
"__kmpc_end_taskq called (%d)\n", global_tid));
1557 tq = & __kmp_threads[global_tid] -> th.th_team -> t.t_taskq;
1558 nproc = __kmp_threads[global_tid] -> th.th_team -> t.t_nproc;
1561 queue = (taskq_thunk == NULL) ? tq->tq_root : taskq_thunk->th.th_shareds->sv_queue;
1563 KE_TRACE( 50, (
"__kmpc_end_taskq queue=%p (%d) \n", queue, global_tid));
1564 is_outermost = (queue == tq->tq_root);
1565 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT);
1573 if (is_outermost && (KMP_MASTER_GTID( global_tid ))) {
1574 if( tq->tq_global_flags & TQF_RELEASE_WORKERS ) {
1576 tq->tq_global_flags &= ~TQF_RELEASE_WORKERS;
1578 __kmp_end_split_barrier( bs_plain_barrier, global_tid );
1586 KMP_INIT_YIELD(spins);
1588 while ( (queue->tq_nfull == 0)
1589 && (queue->tq_taskq_slot == NULL)
1590 && (! __kmp_taskq_has_any_children(queue) )
1591 && (! (queue->tq_flags & TQF_ALL_TASKS_QUEUED) )
1593 KMP_YIELD_WHEN( TRUE, spins );
1597 while ( ( (queue->tq_nfull != 0) || (queue->tq_taskq_slot != NULL) )
1598 && (thunk = __kmp_find_task_in_queue(global_tid, queue)) != NULL
1600 KF_TRACE(50, (
"Found thunk: %p in primary queue %p (%d)\n", thunk, queue, global_tid));
1601 __kmp_execute_task_from_queue( tq, loc, global_tid, thunk, in_parallel );
1605 if ( (__kmp_taskq_has_any_children(queue))
1606 && (thunk = __kmp_find_task_in_descendant_queue(global_tid, queue)) != NULL
1609 KF_TRACE(50, (
"Stole thunk: %p in descendant queue: %p while waiting in queue: %p (%d)\n",
1610 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid ));
1612 __kmp_execute_task_from_queue( tq, loc, global_tid, thunk, in_parallel );
1615 }
while ( (! (queue->tq_flags & TQF_ALL_TASKS_QUEUED))
1616 || (queue->tq_nfull != 0)
1619 KF_TRACE(50, (
"All tasks queued and dequeued in queue: %p (%d)\n", queue, global_tid));
1624 while ( (!__kmp_taskq_tasks_finished(queue))
1625 && (thunk = __kmp_find_task_in_descendant_queue(global_tid, queue)) != NULL
1628 KF_TRACE(50, (
"Stole thunk: %p in descendant queue: %p while waiting in queue: %p (%d)\n",
1629 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid));
1631 __kmp_execute_task_from_queue( tq, loc, global_tid, thunk, in_parallel );
1634 KF_TRACE(50, (
"No work found in descendent queues or all work finished in queue: %p (%d)\n", queue, global_tid));
1636 if (!is_outermost) {
1639 if (queue->tq_flags & TQF_IS_NOWAIT) {
1640 __kmp_acquire_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
1641 queue->tq_ref_count--;
1642 KMP_DEBUG_ASSERT( queue->tq_ref_count >= 0 );
1643 __kmp_release_lock(& queue->tq.tq_parent->tq_link_lck, global_tid);
1645 KE_TRACE( 10, (
"__kmpc_end_taskq return for nowait case (%d)\n", global_tid));
1650 __kmp_find_and_remove_finished_child_taskq( tq, global_tid, queue );
1653 KMP_INIT_YIELD(spins);
1655 while (!__kmp_taskq_tasks_finished(queue) || __kmp_taskq_has_any_children(queue)) {
1656 thunk = __kmp_find_task_in_ancestor_queue( tq, global_tid, queue );
1658 if (thunk != NULL) {
1659 KF_TRACE(50, (
"Stole thunk: %p in ancestor queue: %p while waiting in queue: %p (%d)\n",
1660 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid));
1661 __kmp_execute_task_from_queue( tq, loc, global_tid, thunk, in_parallel );
1664 KMP_YIELD_WHEN( thunk == NULL, spins );
1666 __kmp_find_and_remove_finished_child_taskq( tq, global_tid, queue );
1669 __kmp_acquire_lock(& queue->tq_queue_lck, global_tid);
1670 if ( !(queue->tq_flags & TQF_DEALLOCATED) ) {
1671 queue->tq_flags |= TQF_DEALLOCATED;
1673 __kmp_release_lock(& queue->tq_queue_lck, global_tid);
1676 if (taskq_thunk != NULL) {
1677 __kmp_remove_queue_from_tree( tq, global_tid, queue, TRUE );
1680 KE_TRACE( 10, (
"__kmpc_end_taskq return for non_outermost queue, wait case (%d)\n", global_tid));
1687 KMP_INIT_YIELD(spins);
1689 while (!__kmp_taskq_tasks_finished(queue)) {
1690 thunk = __kmp_find_task_in_descendant_queue(global_tid, queue);
1692 if (thunk != NULL) {
1693 KF_TRACE(50, (
"Stole thunk: %p in descendant queue: %p while waiting in queue: %p (%d)\n",
1694 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid));
1696 __kmp_execute_task_from_queue( tq, loc, global_tid, thunk, in_parallel );
1699 KMP_YIELD_WHEN( thunk == NULL, spins );
1705 if ( !__kmp_barrier( bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL )) {
1710 __kmp_remove_all_child_taskq( tq, global_tid, queue );
1713 KF_TRACE(100, (
"T#%d Before Deletion of top-level TaskQ at %p:\n", global_tid, queue ));
1714 KF_DUMP(100, __kmp_dump_task_queue( tq, queue, global_tid ));
1718 KMP_DEBUG_ASSERT ((queue->tq.tq_parent == NULL) && (queue->tq_next_child == NULL));
1721 KMP_DEBUG_ASSERT (queue->tq_first_child == NULL);
1723 for (i=0; i<nproc; i++) {
1724 KMP_DEBUG_ASSERT(queue->tq_th_thunks[i].ai_data == 0);
1727 for (i=0, thunk=queue->tq_free_thunks; thunk != NULL; i++, thunk=thunk->th.th_next_free);
1729 KMP_DEBUG_ASSERT (i == queue->tq_nslots + (nproc * __KMP_TASKQ_THUNKS_PER_TH));
1731 for (i = 0; i < nproc; i++) {
1732 KMP_DEBUG_ASSERT( ! tq->tq_curr_thunk[i] );
1736 tq -> tq_root = NULL;
1739 KF_TRACE(50, (
"After Deletion of top-level TaskQ at %p on (%d):\n", queue, global_tid));
1741 queue->tq_flags |= TQF_DEALLOCATED;
1742 __kmp_free_taskq ( tq, queue, in_parallel, global_tid );
1744 KF_DUMP(50, __kmp_dump_task_queue_tree( tq, tq->tq_root, global_tid ));
1747 __kmp_end_split_barrier( bs_plain_barrier, global_tid );
1750 th = __kmp_threads[ global_tid ];
1753 th->th.th_dispatch->th_deo_fcn = 0;
1756 th->th.th_dispatch->th_dxo_fcn = 0;
1762 if (queue->tq_nfull > 0) {
1763 KMP_DEBUG_ASSERT(queue->tq_nfull == 1);
1765 thunk = __kmp_dequeue_task(global_tid, queue, in_parallel);
1767 if (queue->tq_flags & TQF_IS_LAST_TASK) {
1773 thunk->th_flags |= TQF_IS_LAST_TASK;
1776 KF_TRACE(50, (
"T#%d found thunk: %p in serial queue: %p\n", global_tid, thunk, queue));
1778 __kmp_execute_task_from_queue( tq, loc, global_tid, thunk, in_parallel );
1782 KF_TRACE(100, (
"Before Deletion of Serialized TaskQ at %p on (%d):\n", queue, global_tid));
1783 KF_DUMP(100, __kmp_dump_task_queue( tq, queue, global_tid ));
1787 for (thunk=queue->tq_free_thunks; thunk != NULL; thunk=thunk->th.th_next_free)
1789 KMP_DEBUG_ASSERT (i == queue->tq_nslots + 1);
1792 KF_TRACE(50, (
"Serialized TaskQ at %p deleted on (%d).\n", queue, global_tid));
1794 queue->tq_flags |= TQF_DEALLOCATED;
1795 __kmp_free_taskq ( tq, queue, in_parallel, global_tid );
1798 KE_TRACE( 10, (
"__kmpc_end_taskq return (%d)\n", global_tid));
1805 __kmpc_task(
ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *thunk)
1808 kmpc_task_queue_t *queue;
1812 KE_TRACE( 10, (
"__kmpc_task called (%d)\n", global_tid));
1814 KMP_DEBUG_ASSERT (!(thunk->th_flags & TQF_TASKQ_TASK));
1816 tq = &__kmp_threads[global_tid] -> th.th_team -> t.t_taskq;
1817 queue = thunk->th.th_shareds->sv_queue;
1818 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT);
1820 if (in_parallel && (thunk->th_flags & TQF_IS_ORDERED))
1821 thunk->th_tasknum = ++queue->tq_tasknum_queuing;
1826 if (!in_parallel && queue->tq_nfull > 0) {
1827 kmpc_thunk_t *prev_thunk;
1829 KMP_DEBUG_ASSERT(queue->tq_nfull == 1);
1831 prev_thunk = __kmp_dequeue_task(global_tid, queue, in_parallel);
1833 KF_TRACE(50, (
"T#%d found thunk: %p in serial queue: %p\n", global_tid, prev_thunk, queue));
1835 __kmp_execute_task_from_queue( tq, loc, global_tid, prev_thunk, in_parallel );
1843 KF_TRACE(100, (
"After enqueueing this Task on (%d):\n", global_tid));
1844 KF_DUMP(100, __kmp_dump_thunk( tq, thunk, global_tid ));
1846 ret = __kmp_enqueue_task ( tq, global_tid, queue, thunk, in_parallel );
1848 KF_TRACE(100, (
"Task Queue looks like this on (%d):\n", global_tid));
1849 KF_DUMP(100, __kmp_dump_task_queue( tq, queue, global_tid ));
1851 KE_TRACE( 10, (
"__kmpc_task return (%d)\n", global_tid));
1860 __kmpc_taskq_task(
ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *thunk, kmp_int32 status)
1862 kmpc_task_queue_t *queue;
1863 kmp_taskq_t *tq = &__kmp_threads[global_tid] -> th.th_team -> t.t_taskq;
1864 int tid = __kmp_tid_from_gtid( global_tid );
1866 KE_TRACE( 10, (
"__kmpc_taskq_task called (%d)\n", global_tid));
1867 KF_TRACE(100, (
"TaskQ Task argument thunk on (%d):\n", global_tid));
1868 KF_DUMP(100, __kmp_dump_thunk( tq, thunk, global_tid ));
1870 queue = thunk->th.th_shareds->sv_queue;
1872 if ( __kmp_env_consistency_check )
1873 __kmp_pop_workshare( global_tid, ct_taskq, loc );
1876 KMP_DEBUG_ASSERT (thunk->th_flags & TQF_TASKQ_TASK);
1879 KMP_DEBUG_ASSERT (queue->tq_taskq_slot == NULL);
1882 tq->tq_curr_thunk[tid] = thunk->th_encl_thunk;
1883 thunk->th_encl_thunk = NULL;
1885 KF_DUMP( 200, __kmp_dump_thunk_stack( tq->tq_curr_thunk[tid], global_tid ));
1887 thunk->th_status = status;
1895 queue->tq_taskq_slot = thunk;
1897 KE_TRACE( 10, (
"__kmpc_taskq_task return (%d)\n", global_tid));
1903 __kmpc_end_taskq_task(
ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *thunk)
1906 kmpc_task_queue_t *queue;
1910 KE_TRACE( 10, (
"__kmpc_end_taskq_task called (%d)\n", global_tid));
1912 tq = &__kmp_threads[global_tid] -> th.th_team -> t.t_taskq;
1913 queue = thunk->th.th_shareds->sv_queue;
1914 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT);
1915 tid = __kmp_tid_from_gtid( global_tid );
1917 if ( __kmp_env_consistency_check )
1918 __kmp_pop_workshare( global_tid, ct_taskq, loc );
1921 #if KMP_ARCH_X86 || \
1924 KMP_TEST_THEN_OR32( &queue->tq_flags, (kmp_int32) TQF_ALL_TASKS_QUEUED );
1927 __kmp_acquire_lock(& queue->tq_queue_lck, global_tid);
1932 queue->tq_flags |= TQF_ALL_TASKS_QUEUED;
1934 __kmp_release_lock(& queue->tq_queue_lck, global_tid);
1939 if (thunk->th_flags & TQF_IS_LASTPRIVATE) {
1949 if (! in_parallel) {
1951 queue->tq_flags |= TQF_IS_LAST_TASK;
1954 #if KMP_ARCH_X86 || \
1957 KMP_TEST_THEN_OR32( &queue->tq_flags, (kmp_int32) TQF_IS_LAST_TASK );
1960 __kmp_acquire_lock(& queue->tq_queue_lck, global_tid);
1965 queue->tq_flags |= TQF_IS_LAST_TASK;
1967 __kmp_release_lock(& queue->tq_queue_lck, global_tid);
1978 tq->tq_curr_thunk[tid] = thunk->th_encl_thunk;
1979 thunk->th_encl_thunk = NULL;
1981 KF_DUMP( 200, __kmp_dump_thunk_stack( tq->tq_curr_thunk[tid], global_tid ));
1984 KE_TRACE( 10, (
"__kmpc_end_taskq_task return (%d)\n", global_tid));
1991 __kmpc_task_buffer(
ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *taskq_thunk, kmpc_task_t task)
1994 kmpc_task_queue_t *queue;
1995 kmpc_thunk_t *new_thunk;
1998 KE_TRACE( 10, (
"__kmpc_task_buffer called (%d)\n", global_tid));
2000 KMP_DEBUG_ASSERT (taskq_thunk->th_flags & TQF_TASKQ_TASK);
2002 tq = &__kmp_threads[global_tid] -> th.th_team -> t.t_taskq;
2003 queue = taskq_thunk->th.th_shareds->sv_queue;
2004 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT);
2012 new_thunk = __kmp_alloc_thunk (queue, in_parallel, global_tid);
2013 new_thunk->th.th_shareds = (kmpc_shared_vars_t *) queue->tq_shareds[0].ai_data;
2014 new_thunk->th_encl_thunk = NULL;
2015 new_thunk->th_task = task;
2018 new_thunk->th_flags = queue->tq_flags & TQF_INTERFACE_FLAGS;
2020 new_thunk->th_status = 0;
2022 KMP_DEBUG_ASSERT (!(new_thunk->th_flags & TQF_TASKQ_TASK));
2024 KF_TRACE(100, (
"Creating Regular Task on (%d):\n", global_tid));
2025 KF_DUMP(100, __kmp_dump_thunk( tq, new_thunk, global_tid ));
2027 KE_TRACE( 10, (
"__kmpc_task_buffer return (%d)\n", global_tid));
KMP_EXPORT void __kmpc_end_barrier_master(ident_t *, kmp_int32 global_tid)