1349cc55cSDimitry Andric //===----------------------------------------------------------------------===// 25ffd83dbSDimitry Andric // 35ffd83dbSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 45ffd83dbSDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 55ffd83dbSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 65ffd83dbSDimitry Andric // 75ffd83dbSDimitry Andric //===----------------------------------------------------------------------===// 85ffd83dbSDimitry Andric 95ffd83dbSDimitry Andric #include <barrier> 105ffd83dbSDimitry Andric #include <thread> 115ffd83dbSDimitry Andric 125ffd83dbSDimitry Andric _LIBCPP_BEGIN_NAMESPACE_STD 135ffd83dbSDimitry Andric 140eae32dcSDimitry Andric #if !defined(_LIBCPP_HAS_NO_TREE_BARRIER) 155ffd83dbSDimitry Andric 165ffd83dbSDimitry Andric class __barrier_algorithm_base { 175ffd83dbSDimitry Andric public: 18cb14a3feSDimitry Andric struct alignas(64) /* naturally-align the heap state */ __state_t { 195ffd83dbSDimitry Andric struct { 200eae32dcSDimitry Andric __atomic_base<__barrier_phase_t> __phase{0}; 215ffd83dbSDimitry Andric } __tickets[64]; 225ffd83dbSDimitry Andric }; 235ffd83dbSDimitry Andric 24*0fca6ea1SDimitry Andric ptrdiff_t& __expected_; 25*0fca6ea1SDimitry Andric unique_ptr<__state_t[]> __state_; 265ffd83dbSDimitry Andric 27*0fca6ea1SDimitry Andric _LIBCPP_HIDDEN __barrier_algorithm_base(ptrdiff_t& __expected) : __expected_(__expected) { 285ffd83dbSDimitry Andric size_t const __count = (__expected + 1) >> 1; 29*0fca6ea1SDimitry Andric __state_ = unique_ptr<__state_t[]>(new __state_t[__count]); 305ffd83dbSDimitry Andric } 31cb14a3feSDimitry Andric _LIBCPP_HIDDEN bool __arrive(__barrier_phase_t __old_phase) { 32cb14a3feSDimitry Andric __barrier_phase_t const __half_step = __old_phase + 1, __full_step = __old_phase + 2; 33*0fca6ea1SDimitry Andric size_t __current_expected = __expected_, 34*0fca6ea1SDimitry Andric __current = hash<thread::id>()(this_thread::get_id()) % ((__expected_ + 1) >> 1); 355ffd83dbSDimitry Andric for (int __round = 0;; ++__round) { 365ffd83dbSDimitry Andric if (__current_expected <= 1) 375ffd83dbSDimitry Andric return true; 38cb14a3feSDimitry Andric size_t const __end_node = ((__current_expected + 1) >> 1), __last_node = __end_node - 1; 395ffd83dbSDimitry Andric for (;; ++__current) { 405ffd83dbSDimitry Andric if (__current == __end_node) 415ffd83dbSDimitry Andric __current = 0; 425ffd83dbSDimitry Andric __barrier_phase_t expect = __old_phase; 43cb14a3feSDimitry Andric if (__current == __last_node && (__current_expected & 1)) { 44*0fca6ea1SDimitry Andric if (__state_[__current].__tickets[__round].__phase.compare_exchange_strong( 45cb14a3feSDimitry Andric expect, __full_step, memory_order_acq_rel)) 465ffd83dbSDimitry Andric break; // I'm 1 in 1, go to next __round 47*0fca6ea1SDimitry Andric } else if (__state_[__current].__tickets[__round].__phase.compare_exchange_strong( 48cb14a3feSDimitry Andric expect, __half_step, memory_order_acq_rel)) { 495ffd83dbSDimitry Andric return false; // I'm 1 in 2, done with arrival 50cb14a3feSDimitry Andric } else if (expect == __half_step) { 51*0fca6ea1SDimitry Andric if (__state_[__current].__tickets[__round].__phase.compare_exchange_strong( 52cb14a3feSDimitry Andric expect, __full_step, memory_order_acq_rel)) 535ffd83dbSDimitry Andric break; // I'm 2 in 2, go to next __round 545ffd83dbSDimitry Andric } 555ffd83dbSDimitry Andric } 565ffd83dbSDimitry Andric __current_expected = __last_node + 1; 575ffd83dbSDimitry Andric __current >>= 1; 585ffd83dbSDimitry Andric } 595ffd83dbSDimitry Andric } 605ffd83dbSDimitry Andric }; 615ffd83dbSDimitry Andric 62cb14a3feSDimitry Andric _LIBCPP_EXPORTED_FROM_ABI __barrier_algorithm_base* __construct_barrier_algorithm_base(ptrdiff_t& __expected) { 635ffd83dbSDimitry Andric return new __barrier_algorithm_base(__expected); 645ffd83dbSDimitry Andric } 65cb14a3feSDimitry Andric _LIBCPP_EXPORTED_FROM_ABI bool 66*0fca6ea1SDimitry Andric __arrive_barrier_algorithm_base(__barrier_algorithm_base* __barrier, __barrier_phase_t __old_phase) noexcept { 675ffd83dbSDimitry Andric return __barrier->__arrive(__old_phase); 685ffd83dbSDimitry Andric } 69*0fca6ea1SDimitry Andric _LIBCPP_EXPORTED_FROM_ABI void __destroy_barrier_algorithm_base(__barrier_algorithm_base* __barrier) noexcept { 705ffd83dbSDimitry Andric delete __barrier; 715ffd83dbSDimitry Andric } 725ffd83dbSDimitry Andric 730eae32dcSDimitry Andric #endif // !defined(_LIBCPP_HAS_NO_TREE_BARRIER) 745ffd83dbSDimitry Andric 755ffd83dbSDimitry Andric _LIBCPP_END_NAMESPACE_STD 76