27 # include <sys/syscall.h> 45 __kmp_validate_locks(
void )
51 x = ~((kmp_uint32) 0) - 2;
54 for (i = 0; i < 8; ++i, ++x, ++y) {
55 kmp_uint32 z = (x - y);
59 KMP_ASSERT( offsetof( kmp_base_queuing_lock, tail_id ) % 8 == 0 );
77 __kmp_get_tas_lock_owner( kmp_tas_lock_t *lck )
79 return KMP_LOCK_STRIP(TCR_4( lck->lk.poll )) - 1;
83 __kmp_is_tas_lock_nestable( kmp_tas_lock_t *lck )
85 return lck->lk.depth_locked != -1;
88 __forceinline
static int 89 __kmp_acquire_tas_lock_timed_template( kmp_tas_lock_t *lck, kmp_int32 gtid )
93 #ifdef USE_LOCK_PROFILE 94 kmp_uint32 curr = KMP_LOCK_STRIP( TCR_4( lck->lk.poll ) );
95 if ( ( curr != 0 ) && ( curr != gtid + 1 ) )
96 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
100 if ( ( lck->lk.poll == KMP_LOCK_FREE(tas) )
101 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas) ) ) {
102 KMP_FSYNC_ACQUIRED(lck);
103 return KMP_LOCK_ACQUIRED_FIRST;
107 KMP_FSYNC_PREPARE( lck );
108 KMP_INIT_YIELD( spins );
109 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
114 KMP_YIELD_SPIN( spins );
117 kmp_backoff_t backoff = __kmp_spin_backoff_params;
118 while ( ( lck->lk.poll != KMP_LOCK_FREE(tas) ) ||
119 ( ! KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas) ) ) ) {
121 __kmp_spin_backoff(&backoff);
122 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
127 KMP_YIELD_SPIN( spins );
130 KMP_FSYNC_ACQUIRED( lck );
131 return KMP_LOCK_ACQUIRED_FIRST;
135 __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
137 return __kmp_acquire_tas_lock_timed_template( lck, gtid );
141 __kmp_acquire_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
143 char const *
const func =
"omp_set_lock";
144 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
145 && __kmp_is_tas_lock_nestable( lck ) ) {
146 KMP_FATAL( LockNestableUsedAsSimple, func );
148 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) == gtid ) ) {
149 KMP_FATAL( LockIsAlreadyOwned, func );
151 return __kmp_acquire_tas_lock( lck, gtid );
155 __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
157 if ( ( lck->lk.poll == KMP_LOCK_FREE(tas) )
158 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas) ) ) {
159 KMP_FSYNC_ACQUIRED( lck );
166 __kmp_test_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
168 char const *
const func =
"omp_test_lock";
169 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
170 && __kmp_is_tas_lock_nestable( lck ) ) {
171 KMP_FATAL( LockNestableUsedAsSimple, func );
173 return __kmp_test_tas_lock( lck, gtid );
177 __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
181 KMP_FSYNC_RELEASING(lck);
182 KMP_ST_REL32( &(lck->lk.poll), KMP_LOCK_FREE(tas) );
185 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
187 return KMP_LOCK_RELEASED;
191 __kmp_release_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
193 char const *
const func =
"omp_unset_lock";
195 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
196 && __kmp_is_tas_lock_nestable( lck ) ) {
197 KMP_FATAL( LockNestableUsedAsSimple, func );
199 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
200 KMP_FATAL( LockUnsettingFree, func );
202 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) >= 0 )
203 && ( __kmp_get_tas_lock_owner( lck ) != gtid ) ) {
204 KMP_FATAL( LockUnsettingSetByAnother, func );
206 return __kmp_release_tas_lock( lck, gtid );
210 __kmp_init_tas_lock( kmp_tas_lock_t * lck )
212 TCW_4( lck->lk.poll, KMP_LOCK_FREE(tas) );
216 __kmp_init_tas_lock_with_checks( kmp_tas_lock_t * lck )
218 __kmp_init_tas_lock( lck );
222 __kmp_destroy_tas_lock( kmp_tas_lock_t *lck )
228 __kmp_destroy_tas_lock_with_checks( kmp_tas_lock_t *lck )
230 char const *
const func =
"omp_destroy_lock";
231 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
232 && __kmp_is_tas_lock_nestable( lck ) ) {
233 KMP_FATAL( LockNestableUsedAsSimple, func );
235 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
236 KMP_FATAL( LockStillOwned, func );
238 __kmp_destroy_tas_lock( lck );
247 __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
249 KMP_DEBUG_ASSERT( gtid >= 0 );
251 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
252 lck->lk.depth_locked += 1;
253 return KMP_LOCK_ACQUIRED_NEXT;
256 __kmp_acquire_tas_lock_timed_template( lck, gtid );
257 lck->lk.depth_locked = 1;
258 return KMP_LOCK_ACQUIRED_FIRST;
263 __kmp_acquire_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
265 char const *
const func =
"omp_set_nest_lock";
266 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
267 KMP_FATAL( LockSimpleUsedAsNestable, func );
269 return __kmp_acquire_nested_tas_lock( lck, gtid );
273 __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
277 KMP_DEBUG_ASSERT( gtid >= 0 );
279 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
280 retval = ++lck->lk.depth_locked;
282 else if ( !__kmp_test_tas_lock( lck, gtid ) ) {
287 retval = lck->lk.depth_locked = 1;
293 __kmp_test_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
295 char const *
const func =
"omp_test_nest_lock";
296 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
297 KMP_FATAL( LockSimpleUsedAsNestable, func );
299 return __kmp_test_nested_tas_lock( lck, gtid );
303 __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
305 KMP_DEBUG_ASSERT( gtid >= 0 );
308 if ( --(lck->lk.depth_locked) == 0 ) {
309 __kmp_release_tas_lock( lck, gtid );
310 return KMP_LOCK_RELEASED;
312 return KMP_LOCK_STILL_HELD;
316 __kmp_release_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
318 char const *
const func =
"omp_unset_nest_lock";
320 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
321 KMP_FATAL( LockSimpleUsedAsNestable, func );
323 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
324 KMP_FATAL( LockUnsettingFree, func );
326 if ( __kmp_get_tas_lock_owner( lck ) != gtid ) {
327 KMP_FATAL( LockUnsettingSetByAnother, func );
329 return __kmp_release_nested_tas_lock( lck, gtid );
333 __kmp_init_nested_tas_lock( kmp_tas_lock_t * lck )
335 __kmp_init_tas_lock( lck );
336 lck->lk.depth_locked = 0;
340 __kmp_init_nested_tas_lock_with_checks( kmp_tas_lock_t * lck )
342 __kmp_init_nested_tas_lock( lck );
346 __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck )
348 __kmp_destroy_tas_lock( lck );
349 lck->lk.depth_locked = 0;
353 __kmp_destroy_nested_tas_lock_with_checks( kmp_tas_lock_t *lck )
355 char const *
const func =
"omp_destroy_nest_lock";
356 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
357 KMP_FATAL( LockSimpleUsedAsNestable, func );
359 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
360 KMP_FATAL( LockStillOwned, func );
362 __kmp_destroy_nested_tas_lock( lck );
377 __kmp_get_futex_lock_owner( kmp_futex_lock_t *lck )
379 return KMP_LOCK_STRIP(( TCR_4( lck->lk.poll ) >> 1 )) - 1;
383 __kmp_is_futex_lock_nestable( kmp_futex_lock_t *lck )
385 return lck->lk.depth_locked != -1;
388 __forceinline
static int 389 __kmp_acquire_futex_lock_timed_template( kmp_futex_lock_t *lck, kmp_int32 gtid )
391 kmp_int32 gtid_code = ( gtid + 1 ) << 1;
395 #ifdef USE_LOCK_PROFILE 396 kmp_uint32 curr = KMP_LOCK_STRIP( TCR_4( lck->lk.poll ) );
397 if ( ( curr != 0 ) && ( curr != gtid_code ) )
398 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
402 KMP_FSYNC_PREPARE( lck );
403 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
404 lck, lck->lk.poll, gtid ) );
408 while ( ( poll_val = KMP_COMPARE_AND_STORE_RET32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex),
409 KMP_LOCK_BUSY(gtid_code, futex) ) ) != KMP_LOCK_FREE(futex) ) {
411 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1;
412 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
413 lck, gtid, poll_val, cond ) );
428 if ( ! KMP_COMPARE_AND_STORE_REL32( & ( lck->lk.poll ), poll_val, poll_val | KMP_LOCK_BUSY(1, futex) ) ) {
429 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
430 lck, lck->lk.poll, gtid ) );
433 poll_val |= KMP_LOCK_BUSY(1, futex);
435 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n",
436 lck, lck->lk.poll, gtid ) );
439 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
440 lck, gtid, poll_val ) );
443 if ( ( rc = syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAIT,
444 poll_val, NULL, NULL, 0 ) ) != 0 ) {
445 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) failed (rc=%d errno=%d)\n",
446 lck, gtid, poll_val, rc, errno ) );
450 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
451 lck, gtid, poll_val ) );
461 KMP_FSYNC_ACQUIRED( lck );
462 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n",
463 lck, lck->lk.poll, gtid ) );
464 return KMP_LOCK_ACQUIRED_FIRST;
468 __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
470 return __kmp_acquire_futex_lock_timed_template( lck, gtid );
474 __kmp_acquire_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
476 char const *
const func =
"omp_set_lock";
477 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
478 && __kmp_is_futex_lock_nestable( lck ) ) {
479 KMP_FATAL( LockNestableUsedAsSimple, func );
481 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) == gtid ) ) {
482 KMP_FATAL( LockIsAlreadyOwned, func );
484 return __kmp_acquire_futex_lock( lck, gtid );
488 __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
490 if ( KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex), KMP_LOCK_BUSY((gtid+1) << 1, futex) ) ) {
491 KMP_FSYNC_ACQUIRED( lck );
498 __kmp_test_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
500 char const *
const func =
"omp_test_lock";
501 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
502 && __kmp_is_futex_lock_nestable( lck ) ) {
503 KMP_FATAL( LockNestableUsedAsSimple, func );
505 return __kmp_test_futex_lock( lck, gtid );
509 __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
513 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
514 lck, lck->lk.poll, gtid ) );
516 KMP_FSYNC_RELEASING(lck);
518 kmp_int32 poll_val = KMP_XCHG_FIXED32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex) );
520 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
521 lck, gtid, poll_val ) );
523 if ( KMP_LOCK_STRIP(poll_val) & 1 ) {
524 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
526 syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex), NULL, NULL, 0 );
531 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n",
532 lck, lck->lk.poll, gtid ) );
534 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
536 return KMP_LOCK_RELEASED;
540 __kmp_release_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
542 char const *
const func =
"omp_unset_lock";
544 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
545 && __kmp_is_futex_lock_nestable( lck ) ) {
546 KMP_FATAL( LockNestableUsedAsSimple, func );
548 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
549 KMP_FATAL( LockUnsettingFree, func );
551 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) >= 0 )
552 && ( __kmp_get_futex_lock_owner( lck ) != gtid ) ) {
553 KMP_FATAL( LockUnsettingSetByAnother, func );
555 return __kmp_release_futex_lock( lck, gtid );
559 __kmp_init_futex_lock( kmp_futex_lock_t * lck )
561 TCW_4( lck->lk.poll, KMP_LOCK_FREE(futex) );
565 __kmp_init_futex_lock_with_checks( kmp_futex_lock_t * lck )
567 __kmp_init_futex_lock( lck );
571 __kmp_destroy_futex_lock( kmp_futex_lock_t *lck )
577 __kmp_destroy_futex_lock_with_checks( kmp_futex_lock_t *lck )
579 char const *
const func =
"omp_destroy_lock";
580 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
581 && __kmp_is_futex_lock_nestable( lck ) ) {
582 KMP_FATAL( LockNestableUsedAsSimple, func );
584 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
585 KMP_FATAL( LockStillOwned, func );
587 __kmp_destroy_futex_lock( lck );
596 __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
598 KMP_DEBUG_ASSERT( gtid >= 0 );
600 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
601 lck->lk.depth_locked += 1;
602 return KMP_LOCK_ACQUIRED_NEXT;
605 __kmp_acquire_futex_lock_timed_template( lck, gtid );
606 lck->lk.depth_locked = 1;
607 return KMP_LOCK_ACQUIRED_FIRST;
612 __kmp_acquire_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
614 char const *
const func =
"omp_set_nest_lock";
615 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
616 KMP_FATAL( LockSimpleUsedAsNestable, func );
618 return __kmp_acquire_nested_futex_lock( lck, gtid );
622 __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
626 KMP_DEBUG_ASSERT( gtid >= 0 );
628 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
629 retval = ++lck->lk.depth_locked;
631 else if ( !__kmp_test_futex_lock( lck, gtid ) ) {
636 retval = lck->lk.depth_locked = 1;
642 __kmp_test_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
644 char const *
const func =
"omp_test_nest_lock";
645 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
646 KMP_FATAL( LockSimpleUsedAsNestable, func );
648 return __kmp_test_nested_futex_lock( lck, gtid );
652 __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
654 KMP_DEBUG_ASSERT( gtid >= 0 );
657 if ( --(lck->lk.depth_locked) == 0 ) {
658 __kmp_release_futex_lock( lck, gtid );
659 return KMP_LOCK_RELEASED;
661 return KMP_LOCK_STILL_HELD;
665 __kmp_release_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
667 char const *
const func =
"omp_unset_nest_lock";
669 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
670 KMP_FATAL( LockSimpleUsedAsNestable, func );
672 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
673 KMP_FATAL( LockUnsettingFree, func );
675 if ( __kmp_get_futex_lock_owner( lck ) != gtid ) {
676 KMP_FATAL( LockUnsettingSetByAnother, func );
678 return __kmp_release_nested_futex_lock( lck, gtid );
682 __kmp_init_nested_futex_lock( kmp_futex_lock_t * lck )
684 __kmp_init_futex_lock( lck );
685 lck->lk.depth_locked = 0;
689 __kmp_init_nested_futex_lock_with_checks( kmp_futex_lock_t * lck )
691 __kmp_init_nested_futex_lock( lck );
695 __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck )
697 __kmp_destroy_futex_lock( lck );
698 lck->lk.depth_locked = 0;
702 __kmp_destroy_nested_futex_lock_with_checks( kmp_futex_lock_t *lck )
704 char const *
const func =
"omp_destroy_nest_lock";
705 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
706 KMP_FATAL( LockSimpleUsedAsNestable, func );
708 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
709 KMP_FATAL( LockStillOwned, func );
711 __kmp_destroy_nested_futex_lock( lck );
714 #endif // KMP_USE_FUTEX 721 __kmp_get_ticket_lock_owner( kmp_ticket_lock_t *lck )
723 return std::atomic_load_explicit( &lck->lk.owner_id, std::memory_order_relaxed ) - 1;
727 __kmp_is_ticket_lock_nestable( kmp_ticket_lock_t *lck )
729 return std::atomic_load_explicit( &lck->lk.depth_locked, std::memory_order_relaxed ) != -1;
733 __kmp_bakery_check(
void *now_serving, kmp_uint32 my_ticket )
735 return std::atomic_load_explicit( (std::atomic<unsigned> *)now_serving, std::memory_order_acquire ) == my_ticket;
738 __forceinline
static int 739 __kmp_acquire_ticket_lock_timed_template( kmp_ticket_lock_t *lck, kmp_int32 gtid )
741 kmp_uint32 my_ticket = std::atomic_fetch_add_explicit( &lck->lk.next_ticket, 1U, std::memory_order_relaxed );
743 #ifdef USE_LOCK_PROFILE 744 if ( std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_relaxed ) != my_ticket )
745 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
749 if ( std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_acquire ) == my_ticket ) {
750 return KMP_LOCK_ACQUIRED_FIRST;
752 KMP_WAIT_YIELD_PTR( &lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck );
753 return KMP_LOCK_ACQUIRED_FIRST;
757 __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
759 return __kmp_acquire_ticket_lock_timed_template( lck, gtid );
763 __kmp_acquire_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
765 char const *
const func =
"omp_set_lock";
767 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
768 KMP_FATAL( LockIsUninitialized, func );
770 if ( lck->lk.self != lck ) {
771 KMP_FATAL( LockIsUninitialized, func );
773 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
774 KMP_FATAL( LockNestableUsedAsSimple, func );
776 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) == gtid ) ) {
777 KMP_FATAL( LockIsAlreadyOwned, func );
780 __kmp_acquire_ticket_lock( lck, gtid );
782 std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
783 return KMP_LOCK_ACQUIRED_FIRST;
787 __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
789 kmp_uint32 my_ticket = std::atomic_load_explicit( &lck->lk.next_ticket, std::memory_order_relaxed );
791 if ( std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_relaxed ) == my_ticket ) {
792 kmp_uint32 next_ticket = my_ticket + 1;
793 if ( std::atomic_compare_exchange_strong_explicit( &lck->lk.next_ticket,
794 &my_ticket, next_ticket, std::memory_order_acquire, std::memory_order_acquire )) {
802 __kmp_test_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
804 char const *
const func =
"omp_test_lock";
806 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
807 KMP_FATAL( LockIsUninitialized, func );
809 if ( lck->lk.self != lck ) {
810 KMP_FATAL( LockIsUninitialized, func );
812 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
813 KMP_FATAL( LockNestableUsedAsSimple, func );
816 int retval = __kmp_test_ticket_lock( lck, gtid );
819 std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
825 __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
827 kmp_uint32 distance = std::atomic_load_explicit( &lck->lk.next_ticket, std::memory_order_relaxed ) - std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_relaxed );
829 std::atomic_fetch_add_explicit( &lck->lk.now_serving, 1U, std::memory_order_release );
832 > (kmp_uint32) (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) );
833 return KMP_LOCK_RELEASED;
837 __kmp_release_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
839 char const *
const func =
"omp_unset_lock";
841 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
842 KMP_FATAL( LockIsUninitialized, func );
844 if ( lck->lk.self != lck ) {
845 KMP_FATAL( LockIsUninitialized, func );
847 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
848 KMP_FATAL( LockNestableUsedAsSimple, func );
850 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
851 KMP_FATAL( LockUnsettingFree, func );
853 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) >= 0 )
854 && ( __kmp_get_ticket_lock_owner( lck ) != gtid ) ) {
855 KMP_FATAL( LockUnsettingSetByAnother, func );
857 std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed );
858 return __kmp_release_ticket_lock( lck, gtid );
862 __kmp_init_ticket_lock( kmp_ticket_lock_t * lck )
864 lck->lk.location = NULL;
866 std::atomic_store_explicit( &lck->lk.next_ticket, 0U, std::memory_order_relaxed );
867 std::atomic_store_explicit( &lck->lk.now_serving, 0U, std::memory_order_relaxed );
868 std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed );
869 std::atomic_store_explicit( &lck->lk.depth_locked, -1, std::memory_order_relaxed );
870 std::atomic_store_explicit( &lck->lk.initialized,
true, std::memory_order_release );
874 __kmp_init_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
876 __kmp_init_ticket_lock( lck );
880 __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck )
882 std::atomic_store_explicit( &lck->lk.initialized,
false, std::memory_order_release );
884 lck->lk.location = NULL;
885 std::atomic_store_explicit( &lck->lk.next_ticket, 0U, std::memory_order_relaxed );
886 std::atomic_store_explicit( &lck->lk.now_serving, 0U, std::memory_order_relaxed );
887 std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed );
888 std::atomic_store_explicit( &lck->lk.depth_locked, -1, std::memory_order_relaxed );
892 __kmp_destroy_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
894 char const *
const func =
"omp_destroy_lock";
896 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
897 KMP_FATAL( LockIsUninitialized, func );
899 if ( lck->lk.self != lck ) {
900 KMP_FATAL( LockIsUninitialized, func );
902 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
903 KMP_FATAL( LockNestableUsedAsSimple, func );
905 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
906 KMP_FATAL( LockStillOwned, func );
908 __kmp_destroy_ticket_lock( lck );
917 __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
919 KMP_DEBUG_ASSERT( gtid >= 0 );
921 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
922 std::atomic_fetch_add_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed );
923 return KMP_LOCK_ACQUIRED_NEXT;
926 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
927 std::atomic_store_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed );
928 std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
929 return KMP_LOCK_ACQUIRED_FIRST;
934 __kmp_acquire_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
936 char const *
const func =
"omp_set_nest_lock";
938 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
939 KMP_FATAL( LockIsUninitialized, func );
941 if ( lck->lk.self != lck ) {
942 KMP_FATAL( LockIsUninitialized, func );
944 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
945 KMP_FATAL( LockSimpleUsedAsNestable, func );
947 return __kmp_acquire_nested_ticket_lock( lck, gtid );
951 __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
955 KMP_DEBUG_ASSERT( gtid >= 0 );
957 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
958 retval = std::atomic_fetch_add_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed ) + 1;
960 else if ( !__kmp_test_ticket_lock( lck, gtid ) ) {
964 std::atomic_store_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed );
965 std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
972 __kmp_test_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck,
975 char const *
const func =
"omp_test_nest_lock";
977 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
978 KMP_FATAL( LockIsUninitialized, func );
980 if ( lck->lk.self != lck ) {
981 KMP_FATAL( LockIsUninitialized, func );
983 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
984 KMP_FATAL( LockSimpleUsedAsNestable, func );
986 return __kmp_test_nested_ticket_lock( lck, gtid );
990 __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
992 KMP_DEBUG_ASSERT( gtid >= 0 );
994 if ( ( std::atomic_fetch_add_explicit( &lck->lk.depth_locked, -1, std::memory_order_relaxed ) - 1 ) == 0 ) {
995 std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed );
996 __kmp_release_ticket_lock( lck, gtid );
997 return KMP_LOCK_RELEASED;
999 return KMP_LOCK_STILL_HELD;
1003 __kmp_release_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
1005 char const *
const func =
"omp_unset_nest_lock";
1007 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
1008 KMP_FATAL( LockIsUninitialized, func );
1010 if ( lck->lk.self != lck ) {
1011 KMP_FATAL( LockIsUninitialized, func );
1013 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1014 KMP_FATAL( LockSimpleUsedAsNestable, func );
1016 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
1017 KMP_FATAL( LockUnsettingFree, func );
1019 if ( __kmp_get_ticket_lock_owner( lck ) != gtid ) {
1020 KMP_FATAL( LockUnsettingSetByAnother, func );
1022 return __kmp_release_nested_ticket_lock( lck, gtid );
1026 __kmp_init_nested_ticket_lock( kmp_ticket_lock_t * lck )
1028 __kmp_init_ticket_lock( lck );
1029 std::atomic_store_explicit( &lck->lk.depth_locked, 0, std::memory_order_relaxed );
1033 __kmp_init_nested_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
1035 __kmp_init_nested_ticket_lock( lck );
1039 __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck )
1041 __kmp_destroy_ticket_lock( lck );
1042 std::atomic_store_explicit( &lck->lk.depth_locked, 0, std::memory_order_relaxed );
1046 __kmp_destroy_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
1048 char const *
const func =
"omp_destroy_nest_lock";
1050 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
1051 KMP_FATAL( LockIsUninitialized, func );
1053 if ( lck->lk.self != lck ) {
1054 KMP_FATAL( LockIsUninitialized, func );
1056 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1057 KMP_FATAL( LockSimpleUsedAsNestable, func );
1059 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
1060 KMP_FATAL( LockStillOwned, func );
1062 __kmp_destroy_nested_ticket_lock( lck );
1071 __kmp_is_ticket_lock_initialized( kmp_ticket_lock_t *lck )
1073 return std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) && ( lck->lk.self == lck);
1077 __kmp_get_ticket_lock_location( kmp_ticket_lock_t *lck )
1079 return lck->lk.location;
1083 __kmp_set_ticket_lock_location( kmp_ticket_lock_t *lck,
const ident_t *loc )
1085 lck->lk.location = loc;
1088 static kmp_lock_flags_t
1089 __kmp_get_ticket_lock_flags( kmp_ticket_lock_t *lck )
1091 return lck->lk.flags;
1095 __kmp_set_ticket_lock_flags( kmp_ticket_lock_t *lck, kmp_lock_flags_t flags )
1097 lck->lk.flags = flags;
1156 #ifdef DEBUG_QUEUING_LOCKS 1159 #define TRACE_BUF_ELE 1024 1160 static char traces[TRACE_BUF_ELE][128] = { 0 }
1162 #define TRACE_LOCK(X,Y) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y ); 1163 #define TRACE_LOCK_T(X,Y,Z) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X,Y,Z ); 1164 #define TRACE_LOCK_HT(X,Y,Z,Q) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, Z, Q ); 1167 __kmp_dump_queuing_lock( kmp_info_t *this_thr, kmp_int32 gtid,
1168 kmp_queuing_lock_t *lck, kmp_int32 head_id, kmp_int32 tail_id )
1172 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n" );
1174 i = tc % TRACE_BUF_ELE;
1175 __kmp_printf_no_lock(
"%s\n", traces[i] );
1176 i = (i+1) % TRACE_BUF_ELE;
1177 while ( i != (tc % TRACE_BUF_ELE) ) {
1178 __kmp_printf_no_lock(
"%s", traces[i] );
1179 i = (i+1) % TRACE_BUF_ELE;
1181 __kmp_printf_no_lock(
"\n" );
1183 __kmp_printf_no_lock(
1184 "\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, next_wait:%d, head_id:%d, tail_id:%d\n",
1185 gtid+1, this_thr->th.th_spin_here, this_thr->th.th_next_waiting,
1188 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id );
1190 if ( lck->lk.head_id >= 1 ) {
1191 t = __kmp_threads[lck->lk.head_id-1]->th.th_next_waiting;
1193 __kmp_printf_no_lock(
"-> %d ", t );
1194 t = __kmp_threads[t-1]->th.th_next_waiting;
1197 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id );
1198 __kmp_printf_no_lock(
"\n\n" );
1204 __kmp_get_queuing_lock_owner( kmp_queuing_lock_t *lck )
1206 return TCR_4( lck->lk.owner_id ) - 1;
1210 __kmp_is_queuing_lock_nestable( kmp_queuing_lock_t *lck )
1212 return lck->lk.depth_locked != -1;
1216 template <
bool takeTime>
1219 __forceinline
static int 1220 __kmp_acquire_queuing_lock_timed_template( kmp_queuing_lock_t *lck,
1223 register kmp_info_t *this_thr = __kmp_thread_from_gtid( gtid );
1224 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1225 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1226 volatile kmp_uint32 *spin_here_p;
1227 kmp_int32 need_mf = 1;
1230 ompt_state_t prev_state = ompt_state_undefined;
1233 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1235 KMP_FSYNC_PREPARE( lck );
1236 KMP_DEBUG_ASSERT( this_thr != NULL );
1237 spin_here_p = & this_thr->th.th_spin_here;
1239 #ifdef DEBUG_QUEUING_LOCKS 1240 TRACE_LOCK( gtid+1,
"acq ent" );
1242 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1243 if ( this_thr->th.th_next_waiting != 0 )
1244 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1246 KMP_DEBUG_ASSERT( !*spin_here_p );
1247 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1257 *spin_here_p = TRUE;
1270 #ifdef DEBUG_QUEUING_LOCKS 1272 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1281 enqueued = KMP_COMPARE_AND_STORE_ACQ64( (
volatile kmp_int64 *) tail_id_p,
1282 KMP_PACK_64( -1, 0 ),
1283 KMP_PACK_64( gtid+1, gtid+1 ) );
1284 #ifdef DEBUG_QUEUING_LOCKS 1285 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (-1,0)->(tid,tid)" );
1293 KMP_DEBUG_ASSERT( tail != gtid + 1 );
1295 #ifdef DEBUG_QUEUING_LOCKS 1296 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1305 enqueued = KMP_COMPARE_AND_STORE_ACQ32( tail_id_p, tail, gtid+1 );
1307 #ifdef DEBUG_QUEUING_LOCKS 1308 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (h,t)->(h,tid)" );
1316 kmp_int32 grabbed_lock;
1318 #ifdef DEBUG_QUEUING_LOCKS 1320 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1325 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 );
1327 if ( grabbed_lock ) {
1329 *spin_here_p = FALSE;
1331 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1333 #ifdef DEBUG_QUEUING_LOCKS 1334 TRACE_LOCK_HT( gtid+1,
"acq exit: ", head, 0 );
1338 if (ompt_enabled && prev_state != ompt_state_undefined) {
1340 this_thr->th.ompt_thread_info.state = prev_state;
1341 this_thr->th.ompt_thread_info.wait_id = 0;
1345 KMP_FSYNC_ACQUIRED( lck );
1346 return KMP_LOCK_ACQUIRED_FIRST;
1354 if (ompt_enabled && prev_state == ompt_state_undefined) {
1356 prev_state = this_thr->th.ompt_thread_info.state;
1357 this_thr->th.ompt_thread_info.wait_id = (uint64_t) lck;
1358 this_thr->th.ompt_thread_info.state = ompt_state_wait_lock;
1364 kmp_info_t *tail_thr = __kmp_thread_from_gtid( tail - 1 );
1365 KMP_ASSERT( tail_thr != NULL );
1366 tail_thr->th.th_next_waiting = gtid+1;
1369 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n", lck, gtid ));
1376 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1378 #ifdef DEBUG_QUEUING_LOCKS 1379 TRACE_LOCK( gtid+1,
"acq spin" );
1381 if ( this_thr->th.th_next_waiting != 0 )
1382 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1384 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1385 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after waiting on queue\n",
1388 #ifdef DEBUG_QUEUING_LOCKS 1389 TRACE_LOCK( gtid+1,
"acq exit 2" );
1394 this_thr->th.ompt_thread_info.state = prev_state;
1395 this_thr->th.ompt_thread_info.wait_id = 0;
1399 return KMP_LOCK_ACQUIRED_FIRST;
1405 KMP_YIELD( TCR_4( __kmp_nth ) > (__kmp_avail_proc ? __kmp_avail_proc :
1407 #ifdef DEBUG_QUEUING_LOCKS 1408 TRACE_LOCK( gtid+1,
"acq retry" );
1412 KMP_ASSERT2( 0,
"should not get here" );
1413 return KMP_LOCK_ACQUIRED_FIRST;
1417 __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1419 KMP_DEBUG_ASSERT( gtid >= 0 );
1421 return __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1425 __kmp_acquire_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1428 char const *
const func =
"omp_set_lock";
1429 if ( lck->lk.initialized != lck ) {
1430 KMP_FATAL( LockIsUninitialized, func );
1432 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1433 KMP_FATAL( LockNestableUsedAsSimple, func );
1435 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1436 KMP_FATAL( LockIsAlreadyOwned, func );
1439 __kmp_acquire_queuing_lock( lck, gtid );
1441 lck->lk.owner_id = gtid + 1;
1442 return KMP_LOCK_ACQUIRED_FIRST;
1446 __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1448 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1451 kmp_info_t *this_thr;
1454 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid ));
1455 KMP_DEBUG_ASSERT( gtid >= 0 );
1457 this_thr = __kmp_thread_from_gtid( gtid );
1458 KMP_DEBUG_ASSERT( this_thr != NULL );
1459 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1468 if ( KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 ) ) {
1469 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid ));
1470 KMP_FSYNC_ACQUIRED(lck);
1475 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid ));
1480 __kmp_test_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1482 char const *
const func =
"omp_test_lock";
1483 if ( lck->lk.initialized != lck ) {
1484 KMP_FATAL( LockIsUninitialized, func );
1486 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1487 KMP_FATAL( LockNestableUsedAsSimple, func );
1490 int retval = __kmp_test_queuing_lock( lck, gtid );
1493 lck->lk.owner_id = gtid + 1;
1499 __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1501 register kmp_info_t *this_thr;
1502 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1503 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1505 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1506 KMP_DEBUG_ASSERT( gtid >= 0 );
1507 this_thr = __kmp_thread_from_gtid( gtid );
1508 KMP_DEBUG_ASSERT( this_thr != NULL );
1509 #ifdef DEBUG_QUEUING_LOCKS 1510 TRACE_LOCK( gtid+1,
"rel ent" );
1512 if ( this_thr->th.th_spin_here )
1513 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1514 if ( this_thr->th.th_next_waiting != 0 )
1515 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1517 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1518 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1520 KMP_FSYNC_RELEASING(lck);
1529 #ifdef DEBUG_QUEUING_LOCKS 1531 TRACE_LOCK_HT( gtid+1,
"rel read: ", head, tail );
1532 if ( head == 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1534 KMP_DEBUG_ASSERT( head != 0 );
1539 if ( KMP_COMPARE_AND_STORE_REL32( head_id_p, -1, 0 ) ) {
1540 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1542 #ifdef DEBUG_QUEUING_LOCKS 1543 TRACE_LOCK_HT( gtid+1,
"rel exit: ", 0, 0 );
1550 return KMP_LOCK_RELEASED;
1558 if ( head == tail ) {
1560 #ifdef DEBUG_QUEUING_LOCKS 1561 if ( head <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1563 KMP_DEBUG_ASSERT( head > 0 );
1566 dequeued = KMP_COMPARE_AND_STORE_REL64( (kmp_int64 *) tail_id_p,
1567 KMP_PACK_64( head, head ), KMP_PACK_64( -1, 0 ) );
1568 #ifdef DEBUG_QUEUING_LOCKS 1569 TRACE_LOCK( gtid+1,
"rel deq: (h,h)->(-1,0)" );
1574 volatile kmp_int32 *waiting_id_p;
1575 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1576 KMP_DEBUG_ASSERT( head_thr != NULL );
1577 waiting_id_p = & head_thr->th.th_next_waiting;
1580 #ifdef DEBUG_QUEUING_LOCKS 1581 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1583 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1589 *head_id_p = KMP_WAIT_YIELD((
volatile kmp_uint32*)waiting_id_p, 0, KMP_NEQ, NULL);
1590 #ifdef DEBUG_QUEUING_LOCKS 1591 TRACE_LOCK( gtid+1,
"rel deq: (h,t)->(h',t)" );
1598 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1599 KMP_DEBUG_ASSERT( head_thr != NULL );
1602 #ifdef DEBUG_QUEUING_LOCKS 1603 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1605 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1610 head_thr->th.th_next_waiting = 0;
1611 #ifdef DEBUG_QUEUING_LOCKS 1612 TRACE_LOCK_T( gtid+1,
"rel nw=0 for t=", head );
1617 head_thr->th.th_spin_here = FALSE;
1619 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after dequeuing\n",
1621 #ifdef DEBUG_QUEUING_LOCKS 1622 TRACE_LOCK( gtid+1,
"rel exit 2" );
1624 return KMP_LOCK_RELEASED;
1628 #ifdef DEBUG_QUEUING_LOCKS 1629 TRACE_LOCK( gtid+1,
"rel retry" );
1633 KMP_ASSERT2( 0,
"should not get here" );
1634 return KMP_LOCK_RELEASED;
1638 __kmp_release_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1641 char const *
const func =
"omp_unset_lock";
1643 if ( lck->lk.initialized != lck ) {
1644 KMP_FATAL( LockIsUninitialized, func );
1646 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1647 KMP_FATAL( LockNestableUsedAsSimple, func );
1649 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1650 KMP_FATAL( LockUnsettingFree, func );
1652 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1653 KMP_FATAL( LockUnsettingSetByAnother, func );
1655 lck->lk.owner_id = 0;
1656 return __kmp_release_queuing_lock( lck, gtid );
1660 __kmp_init_queuing_lock( kmp_queuing_lock_t *lck )
1662 lck->lk.location = NULL;
1663 lck->lk.head_id = 0;
1664 lck->lk.tail_id = 0;
1665 lck->lk.next_ticket = 0;
1666 lck->lk.now_serving = 0;
1667 lck->lk.owner_id = 0;
1668 lck->lk.depth_locked = -1;
1669 lck->lk.initialized = lck;
1671 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1675 __kmp_init_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1677 __kmp_init_queuing_lock( lck );
1681 __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck )
1683 lck->lk.initialized = NULL;
1684 lck->lk.location = NULL;
1685 lck->lk.head_id = 0;
1686 lck->lk.tail_id = 0;
1687 lck->lk.next_ticket = 0;
1688 lck->lk.now_serving = 0;
1689 lck->lk.owner_id = 0;
1690 lck->lk.depth_locked = -1;
1694 __kmp_destroy_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1696 char const *
const func =
"omp_destroy_lock";
1697 if ( lck->lk.initialized != lck ) {
1698 KMP_FATAL( LockIsUninitialized, func );
1700 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1701 KMP_FATAL( LockNestableUsedAsSimple, func );
1703 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1704 KMP_FATAL( LockStillOwned, func );
1706 __kmp_destroy_queuing_lock( lck );
1715 __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1717 KMP_DEBUG_ASSERT( gtid >= 0 );
1719 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1720 lck->lk.depth_locked += 1;
1721 return KMP_LOCK_ACQUIRED_NEXT;
1724 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1726 lck->lk.depth_locked = 1;
1728 lck->lk.owner_id = gtid + 1;
1729 return KMP_LOCK_ACQUIRED_FIRST;
1734 __kmp_acquire_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1736 char const *
const func =
"omp_set_nest_lock";
1737 if ( lck->lk.initialized != lck ) {
1738 KMP_FATAL( LockIsUninitialized, func );
1740 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1741 KMP_FATAL( LockSimpleUsedAsNestable, func );
1743 return __kmp_acquire_nested_queuing_lock( lck, gtid );
1747 __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1751 KMP_DEBUG_ASSERT( gtid >= 0 );
1753 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1754 retval = ++lck->lk.depth_locked;
1756 else if ( !__kmp_test_queuing_lock( lck, gtid ) ) {
1761 retval = lck->lk.depth_locked = 1;
1763 lck->lk.owner_id = gtid + 1;
1769 __kmp_test_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1772 char const *
const func =
"omp_test_nest_lock";
1773 if ( lck->lk.initialized != lck ) {
1774 KMP_FATAL( LockIsUninitialized, func );
1776 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1777 KMP_FATAL( LockSimpleUsedAsNestable, func );
1779 return __kmp_test_nested_queuing_lock( lck, gtid );
1783 __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1785 KMP_DEBUG_ASSERT( gtid >= 0 );
1788 if ( --(lck->lk.depth_locked) == 0 ) {
1790 lck->lk.owner_id = 0;
1791 __kmp_release_queuing_lock( lck, gtid );
1792 return KMP_LOCK_RELEASED;
1794 return KMP_LOCK_STILL_HELD;
1798 __kmp_release_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1800 char const *
const func =
"omp_unset_nest_lock";
1802 if ( lck->lk.initialized != lck ) {
1803 KMP_FATAL( LockIsUninitialized, func );
1805 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1806 KMP_FATAL( LockSimpleUsedAsNestable, func );
1808 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1809 KMP_FATAL( LockUnsettingFree, func );
1811 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1812 KMP_FATAL( LockUnsettingSetByAnother, func );
1814 return __kmp_release_nested_queuing_lock( lck, gtid );
1818 __kmp_init_nested_queuing_lock( kmp_queuing_lock_t * lck )
1820 __kmp_init_queuing_lock( lck );
1821 lck->lk.depth_locked = 0;
1825 __kmp_init_nested_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1827 __kmp_init_nested_queuing_lock( lck );
1831 __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck )
1833 __kmp_destroy_queuing_lock( lck );
1834 lck->lk.depth_locked = 0;
1838 __kmp_destroy_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1840 char const *
const func =
"omp_destroy_nest_lock";
1841 if ( lck->lk.initialized != lck ) {
1842 KMP_FATAL( LockIsUninitialized, func );
1844 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1845 KMP_FATAL( LockSimpleUsedAsNestable, func );
1847 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1848 KMP_FATAL( LockStillOwned, func );
1850 __kmp_destroy_nested_queuing_lock( lck );
1859 __kmp_is_queuing_lock_initialized( kmp_queuing_lock_t *lck )
1861 return lck == lck->lk.initialized;
1865 __kmp_get_queuing_lock_location( kmp_queuing_lock_t *lck )
1867 return lck->lk.location;
1871 __kmp_set_queuing_lock_location( kmp_queuing_lock_t *lck,
const ident_t *loc )
1873 lck->lk.location = loc;
1876 static kmp_lock_flags_t
1877 __kmp_get_queuing_lock_flags( kmp_queuing_lock_t *lck )
1879 return lck->lk.flags;
1883 __kmp_set_queuing_lock_flags( kmp_queuing_lock_t *lck, kmp_lock_flags_t flags )
1885 lck->lk.flags = flags;
1888 #if KMP_USE_ADAPTIVE_LOCKS 1894 #if KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 1896 #include <immintrin.h> 1897 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1902 #define _XBEGIN_STARTED (~0u) 1903 #define _XABORT_EXPLICIT (1 << 0) 1904 #define _XABORT_RETRY (1 << 1) 1905 #define _XABORT_CONFLICT (1 << 2) 1906 #define _XABORT_CAPACITY (1 << 3) 1907 #define _XABORT_DEBUG (1 << 4) 1908 #define _XABORT_NESTED (1 << 5) 1909 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF)) 1912 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1914 #define STRINGIZE_INTERNAL(arg) #arg 1915 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg) 1923 static __inline
int _xbegin()
1952 #endif // KMP_ARCH_X86_64 1962 __asm__
volatile (
"1: .byte 0xC7; .byte 0xF8;\n" 1965 "1: movl %%eax,%0\n" 1967 :
"+r"(res)::
"memory",
"%eax");
1968 #endif // KMP_OS_WINDOWS 1975 static __inline
void _xend()
1984 __asm__
volatile (
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1994 #define _xabort(ARG) \ 1999 #define _xabort(ARG) \ 2000 __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG) :::"memory"); 2003 #endif // KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 2008 #if KMP_DEBUG_ADAPTIVE_LOCKS 2013 static kmp_adaptive_lock_statistics_t destroyedStats;
2016 static kmp_adaptive_lock_info_t liveLocks;
2019 static kmp_bootstrap_lock_t chain_lock;
2023 __kmp_init_speculative_stats()
2025 kmp_adaptive_lock_info_t *lck = &liveLocks;
2027 memset( (
void * ) & ( lck->stats ), 0,
sizeof( lck->stats ) );
2028 lck->stats.next = lck;
2029 lck->stats.prev = lck;
2031 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2032 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2034 __kmp_init_bootstrap_lock( &chain_lock );
2040 __kmp_remember_lock( kmp_adaptive_lock_info_t * lck )
2042 __kmp_acquire_bootstrap_lock( &chain_lock );
2044 lck->stats.next = liveLocks.stats.next;
2045 lck->stats.prev = &liveLocks;
2047 liveLocks.stats.next = lck;
2048 lck->stats.next->stats.prev = lck;
2050 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2051 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2053 __kmp_release_bootstrap_lock( &chain_lock );
2057 __kmp_forget_lock( kmp_adaptive_lock_info_t * lck )
2059 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2060 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2062 kmp_adaptive_lock_info_t * n = lck->stats.next;
2063 kmp_adaptive_lock_info_t * p = lck->stats.prev;
2070 __kmp_zero_speculative_stats( kmp_adaptive_lock_info_t * lck )
2072 memset( (
void * )&lck->stats, 0,
sizeof( lck->stats ) );
2073 __kmp_remember_lock( lck );
2077 __kmp_add_stats( kmp_adaptive_lock_statistics_t * t, kmp_adaptive_lock_info_t * lck )
2079 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
2081 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
2082 t->successfulSpeculations += s->successfulSpeculations;
2083 t->hardFailedSpeculations += s->hardFailedSpeculations;
2084 t->softFailedSpeculations += s->softFailedSpeculations;
2085 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
2086 t->lemmingYields += s->lemmingYields;
2090 __kmp_accumulate_speculative_stats( kmp_adaptive_lock_info_t * lck)
2092 kmp_adaptive_lock_statistics_t *t = &destroyedStats;
2094 __kmp_acquire_bootstrap_lock( &chain_lock );
2096 __kmp_add_stats( &destroyedStats, lck );
2097 __kmp_forget_lock( lck );
2099 __kmp_release_bootstrap_lock( &chain_lock );
2103 percent (kmp_uint32 count, kmp_uint32 total)
2105 return (total == 0) ? 0.0: (100.0 * count)/total;
2109 FILE * __kmp_open_stats_file()
2111 if (strcmp (__kmp_speculative_statsfile,
"-") == 0)
2114 size_t buffLen = KMP_STRLEN( __kmp_speculative_statsfile ) + 20;
2115 char buffer[buffLen];
2116 KMP_SNPRINTF (&buffer[0], buffLen, __kmp_speculative_statsfile,
2117 (kmp_int32)getpid());
2118 FILE * result = fopen(&buffer[0],
"w");
2121 return result ? result : stdout;
2125 __kmp_print_speculative_stats()
2127 if (__kmp_user_lock_kind != lk_adaptive)
2130 FILE * statsFile = __kmp_open_stats_file();
2132 kmp_adaptive_lock_statistics_t total = destroyedStats;
2133 kmp_adaptive_lock_info_t *lck;
2135 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
2136 __kmp_add_stats( &total, lck );
2138 kmp_adaptive_lock_statistics_t *t = &total;
2139 kmp_uint32 totalSections = t->nonSpeculativeAcquires + t->successfulSpeculations;
2140 kmp_uint32 totalSpeculations = t->successfulSpeculations + t->hardFailedSpeculations +
2141 t->softFailedSpeculations;
2143 fprintf ( statsFile,
"Speculative lock statistics (all approximate!)\n");
2144 fprintf ( statsFile,
" Lock parameters: \n" 2145 " max_soft_retries : %10d\n" 2146 " max_badness : %10d\n",
2147 __kmp_adaptive_backoff_params.max_soft_retries,
2148 __kmp_adaptive_backoff_params.max_badness);
2149 fprintf( statsFile,
" Non-speculative acquire attempts : %10d\n", t->nonSpeculativeAcquireAttempts );
2150 fprintf( statsFile,
" Total critical sections : %10d\n", totalSections );
2151 fprintf( statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
2152 t->successfulSpeculations, percent( t->successfulSpeculations, totalSections ) );
2153 fprintf( statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
2154 t->nonSpeculativeAcquires, percent( t->nonSpeculativeAcquires, totalSections ) );
2155 fprintf( statsFile,
" Lemming yields : %10d\n\n", t->lemmingYields );
2157 fprintf( statsFile,
" Speculative acquire attempts : %10d\n", totalSpeculations );
2158 fprintf( statsFile,
" Successes : %10d (%5.1f%%)\n",
2159 t->successfulSpeculations, percent( t->successfulSpeculations, totalSpeculations ) );
2160 fprintf( statsFile,
" Soft failures : %10d (%5.1f%%)\n",
2161 t->softFailedSpeculations, percent( t->softFailedSpeculations, totalSpeculations ) );
2162 fprintf( statsFile,
" Hard failures : %10d (%5.1f%%)\n",
2163 t->hardFailedSpeculations, percent( t->hardFailedSpeculations, totalSpeculations ) );
2165 if (statsFile != stdout)
2166 fclose( statsFile );
2169 # define KMP_INC_STAT(lck,stat) ( lck->lk.adaptive.stats.stat++ ) 2171 # define KMP_INC_STAT(lck,stat) 2173 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 2176 __kmp_is_unlocked_queuing_lock( kmp_queuing_lock_t *lck )
2180 bool res = lck->lk.head_id == 0;
2184 #if KMP_COMPILER_ICC 2187 __sync_synchronize();
2194 static __inline
void 2195 __kmp_update_badness_after_success( kmp_adaptive_lock_t *lck )
2198 lck->lk.adaptive.badness = 0;
2199 KMP_INC_STAT(lck,successfulSpeculations);
2203 static __inline
void 2204 __kmp_step_badness( kmp_adaptive_lock_t *lck )
2206 kmp_uint32 newBadness = ( lck->lk.adaptive.badness << 1 ) | 1;
2207 if ( newBadness > lck->lk.adaptive.max_badness) {
2210 lck->lk.adaptive.badness = newBadness;
2216 __kmp_should_speculate( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2218 kmp_uint32 badness = lck->lk.adaptive.badness;
2219 kmp_uint32 attempts= lck->lk.adaptive.acquire_attempts;
2220 int res = (attempts & badness) == 0;
2228 __kmp_test_adaptive_lock_only( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2230 int retries = lck->lk.adaptive.max_soft_retries;
2238 kmp_uint32 status = _xbegin();
2243 if (status == _XBEGIN_STARTED )
2249 if (! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2255 KMP_ASSERT2( 0,
"should not get here" );
2260 if ( status & SOFT_ABORT_MASK)
2262 KMP_INC_STAT(lck,softFailedSpeculations);
2267 KMP_INC_STAT(lck,hardFailedSpeculations);
2272 }
while( retries-- );
2276 __kmp_step_badness( lck );
2284 __kmp_test_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2287 if ( __kmp_should_speculate( lck, gtid ) && __kmp_test_adaptive_lock_only( lck, gtid ) )
2292 lck->lk.adaptive.acquire_attempts++;
2295 if ( __kmp_test_queuing_lock( GET_QLK_PTR(lck), gtid ) )
2297 KMP_INC_STAT(lck,nonSpeculativeAcquires);
2307 __kmp_test_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2309 char const *
const func =
"omp_test_lock";
2310 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2311 KMP_FATAL( LockIsUninitialized, func );
2314 int retval = __kmp_test_adaptive_lock( lck, gtid );
2317 lck->lk.qlk.owner_id = gtid + 1;
2337 __kmp_acquire_adaptive_lock( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2339 if ( __kmp_should_speculate( lck, gtid ) )
2341 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2343 if ( __kmp_test_adaptive_lock_only( lck , gtid ) )
2355 while ( ! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2357 KMP_INC_STAT(lck,lemmingYields);
2361 if ( __kmp_test_adaptive_lock_only( lck, gtid ) )
2368 lck->lk.adaptive.acquire_attempts++;
2370 __kmp_acquire_queuing_lock_timed_template<FALSE>( GET_QLK_PTR(lck), gtid );
2372 KMP_INC_STAT(lck,nonSpeculativeAcquires );
2376 __kmp_acquire_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2378 char const *
const func =
"omp_set_lock";
2379 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2380 KMP_FATAL( LockIsUninitialized, func );
2382 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == gtid ) {
2383 KMP_FATAL( LockIsAlreadyOwned, func );
2386 __kmp_acquire_adaptive_lock( lck, gtid );
2388 lck->lk.qlk.owner_id = gtid + 1;
2392 __kmp_release_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2394 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2399 __kmp_update_badness_after_success( lck );
2404 __kmp_release_queuing_lock( GET_QLK_PTR(lck), gtid );
2406 return KMP_LOCK_RELEASED;
2410 __kmp_release_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2412 char const *
const func =
"omp_unset_lock";
2414 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2415 KMP_FATAL( LockIsUninitialized, func );
2417 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == -1 ) {
2418 KMP_FATAL( LockUnsettingFree, func );
2420 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != gtid ) {
2421 KMP_FATAL( LockUnsettingSetByAnother, func );
2423 lck->lk.qlk.owner_id = 0;
2424 __kmp_release_adaptive_lock( lck, gtid );
2425 return KMP_LOCK_RELEASED;
2429 __kmp_init_adaptive_lock( kmp_adaptive_lock_t *lck )
2431 __kmp_init_queuing_lock( GET_QLK_PTR(lck) );
2432 lck->lk.adaptive.badness = 0;
2433 lck->lk.adaptive.acquire_attempts = 0;
2434 lck->lk.adaptive.max_soft_retries = __kmp_adaptive_backoff_params.max_soft_retries;
2435 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2436 #if KMP_DEBUG_ADAPTIVE_LOCKS 2437 __kmp_zero_speculative_stats( &lck->lk.adaptive );
2439 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2443 __kmp_init_adaptive_lock_with_checks( kmp_adaptive_lock_t * lck )
2445 __kmp_init_adaptive_lock( lck );
2449 __kmp_destroy_adaptive_lock( kmp_adaptive_lock_t *lck )
2451 #if KMP_DEBUG_ADAPTIVE_LOCKS 2452 __kmp_accumulate_speculative_stats( &lck->lk.adaptive );
2454 __kmp_destroy_queuing_lock (GET_QLK_PTR(lck));
2459 __kmp_destroy_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck )
2461 char const *
const func =
"omp_destroy_lock";
2462 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2463 KMP_FATAL( LockIsUninitialized, func );
2465 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != -1 ) {
2466 KMP_FATAL( LockStillOwned, func );
2468 __kmp_destroy_adaptive_lock( lck );
2472 #endif // KMP_USE_ADAPTIVE_LOCKS 2480 __kmp_get_drdpa_lock_owner( kmp_drdpa_lock_t *lck )
2482 return TCR_4( lck->lk.owner_id ) - 1;
2486 __kmp_is_drdpa_lock_nestable( kmp_drdpa_lock_t *lck )
2488 return lck->lk.depth_locked != -1;
2491 __forceinline
static int 2492 __kmp_acquire_drdpa_lock_timed_template( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2494 kmp_uint64 ticket = KMP_TEST_THEN_INC64((kmp_int64 *)&lck->lk.next_ticket);
2495 kmp_uint64 mask = TCR_8(lck->lk.mask);
2496 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2497 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2498 TCR_PTR(lck->lk.polls);
2500 #ifdef USE_LOCK_PROFILE 2501 if (TCR_8(polls[ticket & mask].poll) != ticket)
2502 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2518 KMP_FSYNC_PREPARE(lck);
2519 KMP_INIT_YIELD(spins);
2520 while (TCR_8(polls[ticket & mask].poll) < ticket) {
2525 KMP_YIELD(TCR_4(__kmp_nth)
2526 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2527 KMP_YIELD_SPIN(spins);
2538 mask = TCR_8(lck->lk.mask);
2539 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2540 TCR_PTR(lck->lk.polls);
2546 KMP_FSYNC_ACQUIRED(lck);
2547 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2549 lck->lk.now_serving = ticket;
2558 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2559 __kmp_free((
void *)lck->lk.old_polls);
2560 lck->lk.old_polls = NULL;
2561 lck->lk.cleanup_ticket = 0;
2569 if (lck->lk.old_polls == NULL) {
2570 bool reconfigure =
false;
2571 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *old_polls = polls;
2572 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2574 if (TCR_4(__kmp_nth)
2575 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2580 if (num_polls > 1) {
2582 num_polls = TCR_4(lck->lk.num_polls);
2585 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2586 __kmp_allocate(num_polls *
sizeof(*polls));
2587 polls[0].poll = ticket;
2596 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2597 if (num_waiting > num_polls) {
2598 kmp_uint32 old_num_polls = num_polls;
2601 mask = (mask << 1) | 1;
2603 }
while (num_polls <= num_waiting);
2611 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2612 __kmp_allocate(num_polls *
sizeof(*polls));
2614 for (i = 0; i < old_num_polls; i++) {
2615 polls[i].poll = old_polls[i].poll;
2632 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring lock %p to %d polls\n",
2633 ticket, lck, num_polls));
2635 lck->lk.old_polls = old_polls;
2636 lck->lk.polls = polls;
2640 lck->lk.num_polls = num_polls;
2641 lck->lk.mask = mask;
2651 lck->lk.cleanup_ticket = TCR_8(lck->lk.next_ticket);
2654 return KMP_LOCK_ACQUIRED_FIRST;
2658 __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2660 return __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2664 __kmp_acquire_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2666 char const *
const func =
"omp_set_lock";
2667 if ( lck->lk.initialized != lck ) {
2668 KMP_FATAL( LockIsUninitialized, func );
2670 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2671 KMP_FATAL( LockNestableUsedAsSimple, func );
2673 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) ) {
2674 KMP_FATAL( LockIsAlreadyOwned, func );
2677 __kmp_acquire_drdpa_lock( lck, gtid );
2679 lck->lk.owner_id = gtid + 1;
2680 return KMP_LOCK_ACQUIRED_FIRST;
2684 __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2690 kmp_uint64 ticket = TCR_8(lck->lk.next_ticket);
2691 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2692 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2693 TCR_PTR(lck->lk.polls);
2694 kmp_uint64 mask = TCR_8(lck->lk.mask);
2695 if (TCR_8(polls[ticket & mask].poll) == ticket) {
2696 kmp_uint64 next_ticket = ticket + 1;
2697 if (KMP_COMPARE_AND_STORE_ACQ64((kmp_int64 *)&lck->lk.next_ticket,
2698 ticket, next_ticket)) {
2699 KMP_FSYNC_ACQUIRED(lck);
2700 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2702 lck->lk.now_serving = ticket;
2720 __kmp_test_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2722 char const *
const func =
"omp_test_lock";
2723 if ( lck->lk.initialized != lck ) {
2724 KMP_FATAL( LockIsUninitialized, func );
2726 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2727 KMP_FATAL( LockNestableUsedAsSimple, func );
2730 int retval = __kmp_test_drdpa_lock( lck, gtid );
2733 lck->lk.owner_id = gtid + 1;
2739 __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2746 kmp_uint64 ticket = lck->lk.now_serving + 1;
2747 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2748 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2749 TCR_PTR(lck->lk.polls);
2750 kmp_uint64 mask = TCR_8(lck->lk.mask);
2751 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2753 KMP_FSYNC_RELEASING(lck);
2754 KMP_ST_REL64(&(polls[ticket & mask].poll), ticket);
2755 return KMP_LOCK_RELEASED;
2759 __kmp_release_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2761 char const *
const func =
"omp_unset_lock";
2763 if ( lck->lk.initialized != lck ) {
2764 KMP_FATAL( LockIsUninitialized, func );
2766 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2767 KMP_FATAL( LockNestableUsedAsSimple, func );
2769 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2770 KMP_FATAL( LockUnsettingFree, func );
2772 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) >= 0 )
2773 && ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) ) {
2774 KMP_FATAL( LockUnsettingSetByAnother, func );
2776 lck->lk.owner_id = 0;
2777 return __kmp_release_drdpa_lock( lck, gtid );
2781 __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck )
2783 lck->lk.location = NULL;
2785 lck->lk.num_polls = 1;
2786 lck->lk.polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2787 __kmp_allocate(lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2788 lck->lk.cleanup_ticket = 0;
2789 lck->lk.old_polls = NULL;
2790 lck->lk.next_ticket = 0;
2791 lck->lk.now_serving = 0;
2792 lck->lk.owner_id = 0;
2793 lck->lk.depth_locked = -1;
2794 lck->lk.initialized = lck;
2796 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2800 __kmp_init_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2802 __kmp_init_drdpa_lock( lck );
2806 __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck )
2808 lck->lk.initialized = NULL;
2809 lck->lk.location = NULL;
2810 if (lck->lk.polls != NULL) {
2811 __kmp_free((
void *)lck->lk.polls);
2812 lck->lk.polls = NULL;
2814 if (lck->lk.old_polls != NULL) {
2815 __kmp_free((
void *)lck->lk.old_polls);
2816 lck->lk.old_polls = NULL;
2819 lck->lk.num_polls = 0;
2820 lck->lk.cleanup_ticket = 0;
2821 lck->lk.next_ticket = 0;
2822 lck->lk.now_serving = 0;
2823 lck->lk.owner_id = 0;
2824 lck->lk.depth_locked = -1;
2828 __kmp_destroy_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2830 char const *
const func =
"omp_destroy_lock";
2831 if ( lck->lk.initialized != lck ) {
2832 KMP_FATAL( LockIsUninitialized, func );
2834 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2835 KMP_FATAL( LockNestableUsedAsSimple, func );
2837 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2838 KMP_FATAL( LockStillOwned, func );
2840 __kmp_destroy_drdpa_lock( lck );
2849 __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2851 KMP_DEBUG_ASSERT( gtid >= 0 );
2853 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2854 lck->lk.depth_locked += 1;
2855 return KMP_LOCK_ACQUIRED_NEXT;
2858 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2860 lck->lk.depth_locked = 1;
2862 lck->lk.owner_id = gtid + 1;
2863 return KMP_LOCK_ACQUIRED_FIRST;
2868 __kmp_acquire_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2870 char const *
const func =
"omp_set_nest_lock";
2871 if ( lck->lk.initialized != lck ) {
2872 KMP_FATAL( LockIsUninitialized, func );
2874 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2875 KMP_FATAL( LockSimpleUsedAsNestable, func );
2877 __kmp_acquire_nested_drdpa_lock( lck, gtid );
2881 __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2885 KMP_DEBUG_ASSERT( gtid >= 0 );
2887 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2888 retval = ++lck->lk.depth_locked;
2890 else if ( !__kmp_test_drdpa_lock( lck, gtid ) ) {
2895 retval = lck->lk.depth_locked = 1;
2897 lck->lk.owner_id = gtid + 1;
2903 __kmp_test_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2905 char const *
const func =
"omp_test_nest_lock";
2906 if ( lck->lk.initialized != lck ) {
2907 KMP_FATAL( LockIsUninitialized, func );
2909 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2910 KMP_FATAL( LockSimpleUsedAsNestable, func );
2912 return __kmp_test_nested_drdpa_lock( lck, gtid );
2916 __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2918 KMP_DEBUG_ASSERT( gtid >= 0 );
2921 if ( --(lck->lk.depth_locked) == 0 ) {
2923 lck->lk.owner_id = 0;
2924 __kmp_release_drdpa_lock( lck, gtid );
2925 return KMP_LOCK_RELEASED;
2927 return KMP_LOCK_STILL_HELD;
2931 __kmp_release_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2933 char const *
const func =
"omp_unset_nest_lock";
2935 if ( lck->lk.initialized != lck ) {
2936 KMP_FATAL( LockIsUninitialized, func );
2938 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2939 KMP_FATAL( LockSimpleUsedAsNestable, func );
2941 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2942 KMP_FATAL( LockUnsettingFree, func );
2944 if ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) {
2945 KMP_FATAL( LockUnsettingSetByAnother, func );
2947 return __kmp_release_nested_drdpa_lock( lck, gtid );
2951 __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t * lck )
2953 __kmp_init_drdpa_lock( lck );
2954 lck->lk.depth_locked = 0;
2958 __kmp_init_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2960 __kmp_init_nested_drdpa_lock( lck );
2964 __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck )
2966 __kmp_destroy_drdpa_lock( lck );
2967 lck->lk.depth_locked = 0;
2971 __kmp_destroy_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2973 char const *
const func =
"omp_destroy_nest_lock";
2974 if ( lck->lk.initialized != lck ) {
2975 KMP_FATAL( LockIsUninitialized, func );
2977 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2978 KMP_FATAL( LockSimpleUsedAsNestable, func );
2980 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2981 KMP_FATAL( LockStillOwned, func );
2983 __kmp_destroy_nested_drdpa_lock( lck );
2992 __kmp_is_drdpa_lock_initialized( kmp_drdpa_lock_t *lck )
2994 return lck == lck->lk.initialized;
2998 __kmp_get_drdpa_lock_location( kmp_drdpa_lock_t *lck )
3000 return lck->lk.location;
3004 __kmp_set_drdpa_lock_location( kmp_drdpa_lock_t *lck,
const ident_t *loc )
3006 lck->lk.location = loc;
3009 static kmp_lock_flags_t
3010 __kmp_get_drdpa_lock_flags( kmp_drdpa_lock_t *lck )
3012 return lck->lk.flags;
3016 __kmp_set_drdpa_lock_flags( kmp_drdpa_lock_t *lck, kmp_lock_flags_t flags )
3018 lck->lk.flags = flags;
3022 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 3023 # define __kmp_tsc() __kmp_hardware_timestamp() 3025 kmp_backoff_t __kmp_spin_backoff_params = { 1, 4096, 100 };
3028 extern kmp_uint64 __kmp_now_nsec();
3029 kmp_backoff_t __kmp_spin_backoff_params = { 1, 256, 100 };
3030 # define __kmp_tsc() __kmp_now_nsec() 3041 static inline bool before(kmp_uint64 a, kmp_uint64 b)
3043 return ((kmp_int64)b - (kmp_int64)a) > 0;
3048 __kmp_spin_backoff(kmp_backoff_t *boff)
3052 for (i = boff->step; i > 0; i--) {
3053 kmp_uint64 goal = __kmp_tsc() + boff->min_tick;
3056 }
while (before(__kmp_tsc(), goal));
3058 boff->step = (boff->step<<1 | 1) & (boff->max_backoff-1);
3061 #if KMP_USE_DYNAMIC_LOCK 3064 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck, kmp_dyna_lockseq_t seq)
3066 TCW_4(*lck, KMP_GET_D_TAG(seq));
3067 KA_TRACE(20, (
"__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
3073 #define HLE_ACQUIRE ".byte 0xf2;" 3074 #define HLE_RELEASE ".byte 0xf3;" 3076 static inline kmp_uint32
3077 swap4(kmp_uint32
volatile *p, kmp_uint32 v)
3079 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0" 3087 __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck)
3093 __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3096 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
3099 while (*(kmp_uint32
volatile *)lck != KMP_LOCK_FREE(hle)) {
3100 for (
int i = delay; i != 0; --i)
3102 delay = ((delay << 1) | 1) & 7;
3104 }
while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
3109 __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3111 __kmp_acquire_hle_lock(lck, gtid);
3115 __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3117 __asm__
volatile(HLE_RELEASE
"movl %1,%0" 3119 :
"r"(KMP_LOCK_FREE(hle))
3121 return KMP_LOCK_RELEASED;
3125 __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3127 return __kmp_release_hle_lock(lck, gtid);
3131 __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3133 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
3137 __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3139 return __kmp_test_hle_lock(lck, gtid);
3143 __kmp_init_rtm_lock(kmp_queuing_lock_t *lck)
3145 __kmp_init_queuing_lock(lck);
3149 __kmp_destroy_rtm_lock(kmp_queuing_lock_t *lck)
3151 __kmp_destroy_queuing_lock(lck);
3155 __kmp_acquire_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3157 unsigned retries=3, status;
3160 if (status == _XBEGIN_STARTED) {
3161 if (__kmp_is_unlocked_queuing_lock(lck))
3165 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
3167 while (! __kmp_is_unlocked_queuing_lock(lck))
3170 else if (!(status & _XABORT_RETRY))
3172 }
while (retries--);
3175 __kmp_acquire_queuing_lock(lck, gtid);
3179 __kmp_acquire_rtm_lock_with_checks(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3181 __kmp_acquire_rtm_lock(lck, gtid);
3185 __kmp_release_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3187 if (__kmp_is_unlocked_queuing_lock(lck)) {
3193 __kmp_release_queuing_lock(lck, gtid);
3195 return KMP_LOCK_RELEASED;
3199 __kmp_release_rtm_lock_with_checks(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3201 return __kmp_release_rtm_lock(lck, gtid);
3205 __kmp_test_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3207 unsigned retries=3, status;
3210 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) {
3213 if (!(status & _XABORT_RETRY))
3215 }
while (retries--);
3217 return (__kmp_is_unlocked_queuing_lock(lck))? 1: 0;
3221 __kmp_test_rtm_lock_with_checks(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3223 return __kmp_test_rtm_lock(lck, gtid);
3226 #endif // KMP_USE_TSX 3229 static void __kmp_init_indirect_lock(kmp_dyna_lock_t * l, kmp_dyna_lockseq_t tag);
3230 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock);
3231 static void __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3232 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3233 static int __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3234 static void __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3235 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3236 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3244 #define expand(l, op) 0,__kmp_init_direct_lock, 3245 void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t)
3246 = { __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init) };
3250 #define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock, 3251 void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *)
3252 = { __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy) };
3256 #define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 3257 static void (*direct_set[])(kmp_dyna_lock_t *, kmp_int32)
3258 = { __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire) };
3260 #define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks, 3261 static void (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32)
3262 = { __kmp_set_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, acquire) };
3266 #define expand(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 3267 static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32)
3268 = { __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release) };
3269 static int (*direct_test[])(kmp_dyna_lock_t *, kmp_int32)
3270 = { __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test) };
3272 #define expand(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks, 3273 static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32)
3274 = { __kmp_unset_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, release) };
3275 static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32)
3276 = { __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test) };
3280 void (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32) = 0;
3281 int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32) = 0;
3282 int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32) = 0;
3287 #define expand(l, op) (void (*)(kmp_user_lock_p))__kmp_##op##_##l##_##lock, 3288 void (*__kmp_indirect_init[])(kmp_user_lock_p) = { KMP_FOREACH_I_LOCK(expand, init) };
3289 void (*__kmp_indirect_destroy[])(kmp_user_lock_p) = { KMP_FOREACH_I_LOCK(expand, destroy) };
3293 #define expand(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock, 3294 static void (*indirect_set[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, acquire) };
3296 #define expand(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks, 3297 static void (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, acquire) };
3301 #define expand(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock, 3302 static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, release) };
3303 static int (*indirect_test[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, test) };
3305 #define expand(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks, 3306 static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, release) };
3307 static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, test) };
3311 void (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32) = 0;
3312 int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32) = 0;
3313 int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32) = 0;
3316 kmp_indirect_lock_table_t __kmp_i_lock_table;
3319 static kmp_uint32 __kmp_indirect_lock_size[KMP_NUM_I_LOCKS] = { 0 };
3322 void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
const ident_t *) = { 0 };
3323 void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_lock_flags_t) = { 0 };
3324 const ident_t * (*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3325 kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3328 static kmp_indirect_lock_t * __kmp_indirect_lock_pool[KMP_NUM_I_LOCKS] = { 0 };
3334 kmp_indirect_lock_t *
3335 __kmp_allocate_indirect_lock(
void **user_lock, kmp_int32 gtid, kmp_indirect_locktag_t tag)
3337 kmp_indirect_lock_t *lck;
3338 kmp_lock_index_t idx;
3340 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3342 if (__kmp_indirect_lock_pool[tag] != NULL) {
3344 lck = __kmp_indirect_lock_pool[tag];
3345 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3346 idx = lck->lock->pool.index;
3347 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3348 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: reusing an existing lock %p\n", lck));
3350 idx = __kmp_i_lock_table.next;
3352 if (idx == __kmp_i_lock_table.size) {
3354 int row = __kmp_i_lock_table.size/KMP_I_LOCK_CHUNK;
3355 kmp_indirect_lock_t **old_table = __kmp_i_lock_table.table;
3356 __kmp_i_lock_table.table = (kmp_indirect_lock_t **)__kmp_allocate(2*row*
sizeof(kmp_indirect_lock_t *));
3357 KMP_MEMCPY(__kmp_i_lock_table.table, old_table, row*
sizeof(kmp_indirect_lock_t *));
3358 __kmp_free(old_table);
3360 for (
int i = row; i < 2*row; ++i)
3361 *(__kmp_i_lock_table.table + i) = (kmp_indirect_lock_t *)
3362 __kmp_allocate(KMP_I_LOCK_CHUNK*
sizeof(kmp_indirect_lock_t));
3363 __kmp_i_lock_table.size = 2*idx;
3365 __kmp_i_lock_table.next++;
3366 lck = KMP_GET_I_LOCK(idx);
3368 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3369 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck));
3372 __kmp_release_lock(&__kmp_global_lock, gtid);
3376 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3377 *((kmp_lock_index_t *)user_lock) = idx << 1;
3379 *((kmp_indirect_lock_t **)user_lock) = lck;
3386 static __forceinline
3387 kmp_indirect_lock_t *
3388 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func)
3390 if (__kmp_env_consistency_check) {
3391 kmp_indirect_lock_t *lck = NULL;
3392 if (user_lock == NULL) {
3393 KMP_FATAL(LockIsUninitialized, func);
3395 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3396 kmp_lock_index_t idx = KMP_EXTRACT_I_INDEX(user_lock);
3397 if (idx >= __kmp_i_lock_table.size) {
3398 KMP_FATAL(LockIsUninitialized, func);
3400 lck = KMP_GET_I_LOCK(idx);
3402 lck = *((kmp_indirect_lock_t **)user_lock);
3405 KMP_FATAL(LockIsUninitialized, func);
3409 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3410 return KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(user_lock));
3412 return *((kmp_indirect_lock_t **)user_lock);
3418 __kmp_init_indirect_lock(kmp_dyna_lock_t * lock, kmp_dyna_lockseq_t seq)
3420 #if KMP_USE_ADAPTIVE_LOCKS 3421 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3422 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3423 seq = lockseq_queuing;
3427 if (seq == lockseq_rtm && !__kmp_cpuinfo.rtm) {
3428 seq = lockseq_queuing;
3431 kmp_indirect_locktag_t tag = KMP_GET_I_TAG(seq);
3432 kmp_indirect_lock_t *l = __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3433 KMP_I_LOCK_FUNC(l, init)(l->lock);
3434 KA_TRACE(20, (
"__kmp_init_indirect_lock: initialized indirect lock with type#%d\n", seq));
3438 __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock)
3440 kmp_uint32 gtid = __kmp_entry_gtid();
3441 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3442 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3443 kmp_indirect_locktag_t tag = l->type;
3445 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3448 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3449 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3450 l->lock->pool.index = KMP_EXTRACT_I_INDEX(lock);
3452 __kmp_indirect_lock_pool[tag] = l;
3454 __kmp_release_lock(&__kmp_global_lock, gtid);
3458 __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3460 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3461 KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3465 __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3467 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3468 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3472 __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3474 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3475 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3479 __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3481 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3482 KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3486 __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3488 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3489 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3493 __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3495 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3496 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3499 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3503 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq)
3507 case lockseq_nested_tas:
3508 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3511 case lockseq_nested_futex:
3512 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3514 case lockseq_ticket:
3515 case lockseq_nested_ticket:
3516 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3517 case lockseq_queuing:
3518 case lockseq_nested_queuing:
3519 #if KMP_USE_ADAPTIVE_LOCKS 3520 case lockseq_adaptive:
3522 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3524 case lockseq_nested_drdpa:
3525 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3533 __kmp_init_dynamic_user_locks()
3536 if (__kmp_env_consistency_check) {
3537 __kmp_direct_set = direct_set_check;
3538 __kmp_direct_unset = direct_unset_check;
3539 __kmp_direct_test = direct_test_check;
3540 __kmp_indirect_set = indirect_set_check;
3541 __kmp_indirect_unset = indirect_unset_check;
3542 __kmp_indirect_test = indirect_test_check;
3545 __kmp_direct_set = direct_set;
3546 __kmp_direct_unset = direct_unset;
3547 __kmp_direct_test = direct_test;
3548 __kmp_indirect_set = indirect_set;
3549 __kmp_indirect_unset = indirect_unset;
3550 __kmp_indirect_test = indirect_test;
3554 __kmp_i_lock_table.size = KMP_I_LOCK_CHUNK;
3555 __kmp_i_lock_table.table = (kmp_indirect_lock_t **)__kmp_allocate(
sizeof(kmp_indirect_lock_t *));
3556 *(__kmp_i_lock_table.table) = (kmp_indirect_lock_t *)
3557 __kmp_allocate(KMP_I_LOCK_CHUNK*
sizeof(kmp_indirect_lock_t));
3558 __kmp_i_lock_table.next = 0;
3561 __kmp_indirect_lock_size[locktag_ticket] =
sizeof(kmp_ticket_lock_t);
3562 __kmp_indirect_lock_size[locktag_queuing] =
sizeof(kmp_queuing_lock_t);
3563 #if KMP_USE_ADAPTIVE_LOCKS 3564 __kmp_indirect_lock_size[locktag_adaptive] =
sizeof(kmp_adaptive_lock_t);
3566 __kmp_indirect_lock_size[locktag_drdpa] =
sizeof(kmp_drdpa_lock_t);
3568 __kmp_indirect_lock_size[locktag_rtm] =
sizeof(kmp_queuing_lock_t);
3570 __kmp_indirect_lock_size[locktag_nested_tas] =
sizeof(kmp_tas_lock_t);
3572 __kmp_indirect_lock_size[locktag_nested_futex] =
sizeof(kmp_futex_lock_t);
3574 __kmp_indirect_lock_size[locktag_nested_ticket] =
sizeof(kmp_ticket_lock_t);
3575 __kmp_indirect_lock_size[locktag_nested_queuing] =
sizeof(kmp_queuing_lock_t);
3576 __kmp_indirect_lock_size[locktag_nested_drdpa] =
sizeof(kmp_drdpa_lock_t);
3579 #define fill_jumps(table, expand, sep) { \ 3580 table[locktag##sep##ticket] = expand(ticket); \ 3581 table[locktag##sep##queuing] = expand(queuing); \ 3582 table[locktag##sep##drdpa] = expand(drdpa); \ 3585 #if KMP_USE_ADAPTIVE_LOCKS 3586 # define fill_table(table, expand) { \ 3587 fill_jumps(table, expand, _); \ 3588 table[locktag_adaptive] = expand(queuing); \ 3589 fill_jumps(table, expand, _nested_); \ 3592 # define fill_table(table, expand) { \ 3593 fill_jumps(table, expand, _); \ 3594 fill_jumps(table, expand, _nested_); \ 3596 #endif // KMP_USE_ADAPTIVE_LOCKS 3598 #define expand(l) (void (*)(kmp_user_lock_p, const ident_t *))__kmp_set_##l##_lock_location 3599 fill_table(__kmp_indirect_set_location, expand);
3601 #define expand(l) (void (*)(kmp_user_lock_p, kmp_lock_flags_t))__kmp_set_##l##_lock_flags 3602 fill_table(__kmp_indirect_set_flags, expand);
3604 #define expand(l) (const ident_t * (*)(kmp_user_lock_p))__kmp_get_##l##_lock_location 3605 fill_table(__kmp_indirect_get_location, expand);
3607 #define expand(l) (kmp_lock_flags_t (*)(kmp_user_lock_p))__kmp_get_##l##_lock_flags 3608 fill_table(__kmp_indirect_get_flags, expand);
3611 __kmp_init_user_locks = TRUE;
3616 __kmp_cleanup_indirect_user_locks()
3622 for (k = 0; k < KMP_NUM_I_LOCKS; ++k) {
3623 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3625 kmp_indirect_lock_t *ll = l;
3626 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3627 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: freeing %p from pool\n", ll));
3628 __kmp_free(ll->lock);
3631 __kmp_indirect_lock_pool[k] = NULL;
3634 for (i = 0; i < __kmp_i_lock_table.next; i++) {
3635 kmp_indirect_lock_t *l = KMP_GET_I_LOCK(i);
3636 if (l->lock != NULL) {
3638 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3639 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: destroy/freeing %p from table\n", l));
3640 __kmp_free(l->lock);
3644 for (i = 0; i < __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK; i++)
3645 __kmp_free(__kmp_i_lock_table.table[i]);
3646 __kmp_free(__kmp_i_lock_table.table);
3648 __kmp_init_user_locks = FALSE;
3651 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3652 int __kmp_num_locks_in_block = 1;
3654 #else // KMP_USE_DYNAMIC_LOCK 3663 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3665 size_t __kmp_base_user_lock_size = 0;
3666 size_t __kmp_user_lock_size = 0;
3668 kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck ) = NULL;
3669 int ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3671 int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3672 int ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3673 void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3674 void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck ) = NULL;
3675 void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3676 int ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3678 int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3679 int ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3680 void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3681 void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3683 int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck ) = NULL;
3684 const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck ) = NULL;
3685 void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc ) = NULL;
3686 kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck ) = NULL;
3687 void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags ) = NULL;
3689 void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind )
3691 switch ( user_lock_kind ) {
3697 __kmp_base_user_lock_size =
sizeof( kmp_base_tas_lock_t );
3698 __kmp_user_lock_size =
sizeof( kmp_tas_lock_t );
3700 __kmp_get_user_lock_owner_ =
3701 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3702 ( &__kmp_get_tas_lock_owner );
3704 if ( __kmp_env_consistency_check ) {
3705 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3706 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3709 KMP_BIND_USER_LOCK(tas);
3710 KMP_BIND_NESTED_USER_LOCK(tas);
3713 __kmp_destroy_user_lock_ =
3714 ( void ( * )( kmp_user_lock_p ) )
3715 ( &__kmp_destroy_tas_lock );
3717 __kmp_is_user_lock_initialized_ =
3718 ( int ( * )( kmp_user_lock_p ) ) NULL;
3720 __kmp_get_user_lock_location_ =
3721 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3723 __kmp_set_user_lock_location_ =
3724 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3726 __kmp_get_user_lock_flags_ =
3727 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3729 __kmp_set_user_lock_flags_ =
3730 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3737 __kmp_base_user_lock_size =
sizeof( kmp_base_futex_lock_t );
3738 __kmp_user_lock_size =
sizeof( kmp_futex_lock_t );
3740 __kmp_get_user_lock_owner_ =
3741 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3742 ( &__kmp_get_futex_lock_owner );
3744 if ( __kmp_env_consistency_check ) {
3745 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3746 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3749 KMP_BIND_USER_LOCK(futex);
3750 KMP_BIND_NESTED_USER_LOCK(futex);
3753 __kmp_destroy_user_lock_ =
3754 ( void ( * )( kmp_user_lock_p ) )
3755 ( &__kmp_destroy_futex_lock );
3757 __kmp_is_user_lock_initialized_ =
3758 ( int ( * )( kmp_user_lock_p ) ) NULL;
3760 __kmp_get_user_lock_location_ =
3761 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3763 __kmp_set_user_lock_location_ =
3764 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3766 __kmp_get_user_lock_flags_ =
3767 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3769 __kmp_set_user_lock_flags_ =
3770 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3774 #endif // KMP_USE_FUTEX 3777 __kmp_base_user_lock_size =
sizeof( kmp_base_ticket_lock_t );
3778 __kmp_user_lock_size =
sizeof( kmp_ticket_lock_t );
3780 __kmp_get_user_lock_owner_ =
3781 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3782 ( &__kmp_get_ticket_lock_owner );
3784 if ( __kmp_env_consistency_check ) {
3785 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3786 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3789 KMP_BIND_USER_LOCK(ticket);
3790 KMP_BIND_NESTED_USER_LOCK(ticket);
3793 __kmp_destroy_user_lock_ =
3794 ( void ( * )( kmp_user_lock_p ) )
3795 ( &__kmp_destroy_ticket_lock );
3797 __kmp_is_user_lock_initialized_ =
3798 ( int ( * )( kmp_user_lock_p ) )
3799 ( &__kmp_is_ticket_lock_initialized );
3801 __kmp_get_user_lock_location_ =
3802 (
const ident_t * ( * )( kmp_user_lock_p ) )
3803 ( &__kmp_get_ticket_lock_location );
3805 __kmp_set_user_lock_location_ =
3806 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3807 ( &__kmp_set_ticket_lock_location );
3809 __kmp_get_user_lock_flags_ =
3810 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3811 ( &__kmp_get_ticket_lock_flags );
3813 __kmp_set_user_lock_flags_ =
3814 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3815 ( &__kmp_set_ticket_lock_flags );
3820 __kmp_base_user_lock_size =
sizeof( kmp_base_queuing_lock_t );
3821 __kmp_user_lock_size =
sizeof( kmp_queuing_lock_t );
3823 __kmp_get_user_lock_owner_ =
3824 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3825 ( &__kmp_get_queuing_lock_owner );
3827 if ( __kmp_env_consistency_check ) {
3828 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3829 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3832 KMP_BIND_USER_LOCK(queuing);
3833 KMP_BIND_NESTED_USER_LOCK(queuing);
3836 __kmp_destroy_user_lock_ =
3837 ( void ( * )( kmp_user_lock_p ) )
3838 ( &__kmp_destroy_queuing_lock );
3840 __kmp_is_user_lock_initialized_ =
3841 ( int ( * )( kmp_user_lock_p ) )
3842 ( &__kmp_is_queuing_lock_initialized );
3844 __kmp_get_user_lock_location_ =
3845 (
const ident_t * ( * )( kmp_user_lock_p ) )
3846 ( &__kmp_get_queuing_lock_location );
3848 __kmp_set_user_lock_location_ =
3849 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3850 ( &__kmp_set_queuing_lock_location );
3852 __kmp_get_user_lock_flags_ =
3853 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3854 ( &__kmp_get_queuing_lock_flags );
3856 __kmp_set_user_lock_flags_ =
3857 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3858 ( &__kmp_set_queuing_lock_flags );
3862 #if KMP_USE_ADAPTIVE_LOCKS 3864 __kmp_base_user_lock_size =
sizeof( kmp_base_adaptive_lock_t );
3865 __kmp_user_lock_size =
sizeof( kmp_adaptive_lock_t );
3867 __kmp_get_user_lock_owner_ =
3868 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3869 ( &__kmp_get_queuing_lock_owner );
3871 if ( __kmp_env_consistency_check ) {
3872 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3875 KMP_BIND_USER_LOCK(adaptive);
3878 __kmp_destroy_user_lock_ =
3879 ( void ( * )( kmp_user_lock_p ) )
3880 ( &__kmp_destroy_adaptive_lock );
3882 __kmp_is_user_lock_initialized_ =
3883 ( int ( * )( kmp_user_lock_p ) )
3884 ( &__kmp_is_queuing_lock_initialized );
3886 __kmp_get_user_lock_location_ =
3887 (
const ident_t * ( * )( kmp_user_lock_p ) )
3888 ( &__kmp_get_queuing_lock_location );
3890 __kmp_set_user_lock_location_ =
3891 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3892 ( &__kmp_set_queuing_lock_location );
3894 __kmp_get_user_lock_flags_ =
3895 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3896 ( &__kmp_get_queuing_lock_flags );
3898 __kmp_set_user_lock_flags_ =
3899 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3900 ( &__kmp_set_queuing_lock_flags );
3904 #endif // KMP_USE_ADAPTIVE_LOCKS 3907 __kmp_base_user_lock_size =
sizeof( kmp_base_drdpa_lock_t );
3908 __kmp_user_lock_size =
sizeof( kmp_drdpa_lock_t );
3910 __kmp_get_user_lock_owner_ =
3911 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3912 ( &__kmp_get_drdpa_lock_owner );
3914 if ( __kmp_env_consistency_check ) {
3915 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3916 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3919 KMP_BIND_USER_LOCK(drdpa);
3920 KMP_BIND_NESTED_USER_LOCK(drdpa);
3923 __kmp_destroy_user_lock_ =
3924 ( void ( * )( kmp_user_lock_p ) )
3925 ( &__kmp_destroy_drdpa_lock );
3927 __kmp_is_user_lock_initialized_ =
3928 ( int ( * )( kmp_user_lock_p ) )
3929 ( &__kmp_is_drdpa_lock_initialized );
3931 __kmp_get_user_lock_location_ =
3932 (
const ident_t * ( * )( kmp_user_lock_p ) )
3933 ( &__kmp_get_drdpa_lock_location );
3935 __kmp_set_user_lock_location_ =
3936 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3937 ( &__kmp_set_drdpa_lock_location );
3939 __kmp_get_user_lock_flags_ =
3940 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3941 ( &__kmp_get_drdpa_lock_flags );
3943 __kmp_set_user_lock_flags_ =
3944 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3945 ( &__kmp_set_drdpa_lock_flags );
3955 kmp_lock_table_t __kmp_user_lock_table = { 1, 0, NULL };
3956 kmp_user_lock_p __kmp_lock_pool = NULL;
3959 kmp_block_of_locks* __kmp_lock_blocks = NULL;
3960 int __kmp_num_locks_in_block = 1;
3962 static kmp_lock_index_t
3963 __kmp_lock_table_insert( kmp_user_lock_p lck )
3966 kmp_lock_index_t index;
3967 if ( __kmp_user_lock_table.used >= __kmp_user_lock_table.allocated ) {
3968 kmp_lock_index_t size;
3969 kmp_user_lock_p *table;
3971 if ( __kmp_user_lock_table.allocated == 0 ) {
3975 size = __kmp_user_lock_table.allocated * 2;
3977 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof( kmp_user_lock_p ) * size );
3978 KMP_MEMCPY( table + 1, __kmp_user_lock_table.table + 1,
sizeof( kmp_user_lock_p ) * ( __kmp_user_lock_table.used - 1 ) );
3979 table[ 0 ] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3984 __kmp_user_lock_table.table = table;
3985 __kmp_user_lock_table.allocated = size;
3987 KMP_DEBUG_ASSERT( __kmp_user_lock_table.used < __kmp_user_lock_table.allocated );
3988 index = __kmp_user_lock_table.used;
3989 __kmp_user_lock_table.table[ index ] = lck;
3990 ++ __kmp_user_lock_table.used;
3994 static kmp_user_lock_p
3995 __kmp_lock_block_allocate()
3998 static int last_index = 0;
3999 if ( ( last_index >= __kmp_num_locks_in_block )
4000 || ( __kmp_lock_blocks == NULL ) ) {
4004 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
4005 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
4006 char* buffer = (
char*)__kmp_allocate( space_for_locks +
sizeof( kmp_block_of_locks ) );
4008 kmp_block_of_locks *new_block = (kmp_block_of_locks *)(& buffer[space_for_locks]);
4009 new_block->next_block = __kmp_lock_blocks;
4010 new_block->locks = (
void *)buffer;
4013 __kmp_lock_blocks = new_block;
4015 kmp_user_lock_p ret = (kmp_user_lock_p)(& ( ( (
char *)( __kmp_lock_blocks->locks ) )
4016 [ last_index * __kmp_user_lock_size ] ) );
4026 __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
4027 kmp_lock_flags_t flags )
4029 kmp_user_lock_p lck;
4030 kmp_lock_index_t index;
4031 KMP_DEBUG_ASSERT( user_lock );
4033 __kmp_acquire_lock( &__kmp_global_lock, gtid );
4035 if ( __kmp_lock_pool == NULL ) {
4037 if ( __kmp_num_locks_in_block <= 1 ) {
4038 lck = (kmp_user_lock_p) __kmp_allocate( __kmp_user_lock_size );
4041 lck = __kmp_lock_block_allocate();
4046 index = __kmp_lock_table_insert( lck );
4050 lck = __kmp_lock_pool;
4051 index = __kmp_lock_pool->pool.index;
4052 __kmp_lock_pool = __kmp_lock_pool->pool.next;
4059 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4060 * ( (kmp_lock_index_t *) user_lock ) = index;
4063 * ( (kmp_user_lock_p *) user_lock ) = lck;
4067 __kmp_set_user_lock_flags( lck, flags );
4069 __kmp_release_lock( & __kmp_global_lock, gtid );
4076 __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck )
4078 KMP_DEBUG_ASSERT( user_lock != NULL );
4079 KMP_DEBUG_ASSERT( lck != NULL );
4081 __kmp_acquire_lock( & __kmp_global_lock, gtid );
4083 lck->pool.next = __kmp_lock_pool;
4084 __kmp_lock_pool = lck;
4085 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4086 kmp_lock_index_t index = * ( (kmp_lock_index_t *) user_lock );
4087 KMP_DEBUG_ASSERT( 0 < index && index <= __kmp_user_lock_table.used );
4088 lck->pool.index = index;
4091 __kmp_release_lock( & __kmp_global_lock, gtid );
4095 __kmp_lookup_user_lock(
void **user_lock,
char const *func )
4097 kmp_user_lock_p lck = NULL;
4099 if ( __kmp_env_consistency_check ) {
4100 if ( user_lock == NULL ) {
4101 KMP_FATAL( LockIsUninitialized, func );
4105 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4106 kmp_lock_index_t index = *( (kmp_lock_index_t *)user_lock );
4107 if ( __kmp_env_consistency_check ) {
4108 if ( ! ( 0 < index && index < __kmp_user_lock_table.used ) ) {
4109 KMP_FATAL( LockIsUninitialized, func );
4112 KMP_DEBUG_ASSERT( 0 < index && index < __kmp_user_lock_table.used );
4113 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
4114 lck = __kmp_user_lock_table.table[index];
4117 lck = *( (kmp_user_lock_p *)user_lock );
4120 if ( __kmp_env_consistency_check ) {
4121 if ( lck == NULL ) {
4122 KMP_FATAL( LockIsUninitialized, func );
4130 __kmp_cleanup_user_locks(
void )
4137 __kmp_lock_pool = NULL;
4139 #define IS_CRITICAL(lck) \ 4140 ( ( __kmp_get_user_lock_flags_ != NULL ) && \ 4141 ( ( *__kmp_get_user_lock_flags_ )( lck ) & kmp_lf_critical_section ) ) 4171 while ( __kmp_user_lock_table.used > 1 ) {
4178 kmp_user_lock_p lck = __kmp_user_lock_table.table[
4179 --__kmp_user_lock_table.used ];
4181 if ( ( __kmp_is_user_lock_initialized_ != NULL ) &&
4182 ( *__kmp_is_user_lock_initialized_ )( lck ) ) {
4189 if ( __kmp_env_consistency_check && ( ! IS_CRITICAL( lck ) ) &&
4190 ( ( loc = __kmp_get_user_lock_location( lck ) ) != NULL ) &&
4192 kmp_str_loc_t str_loc = __kmp_str_loc_init( loc->
psource, 0 );
4193 KMP_WARNING( CnsLockNotDestroyed, str_loc.file, str_loc.line );
4194 __kmp_str_loc_free( &str_loc);
4198 if ( IS_CRITICAL( lck ) ) {
4199 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n", lck, *(
void**)lck ) );
4202 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck, *(
void**)lck ) );
4210 __kmp_destroy_user_lock( lck );
4216 if ( __kmp_lock_blocks == NULL ) {
4226 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
4227 __kmp_user_lock_table.table = NULL;
4228 __kmp_user_lock_table.allocated = 0;
4230 while ( table_ptr != NULL ) {
4235 kmp_user_lock_p *next = (kmp_user_lock_p *)( table_ptr[ 0 ] );
4236 __kmp_free( table_ptr );
4243 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
4244 __kmp_lock_blocks = NULL;
4246 while ( block_ptr != NULL ) {
4247 kmp_block_of_locks_t *next = block_ptr->next_block;
4248 __kmp_free( block_ptr->locks );
4255 TCW_4(__kmp_init_user_locks, FALSE);
4258 #endif // KMP_USE_DYNAMIC_LOCK