30 #ifndef _GLIBCXX_ATOMIC_WAIT_H
31 #define _GLIBCXX_ATOMIC_WAIT_H 1
33 #pragma GCC system_header
36 #if defined _GLIBCXX_HAS_GTHREADS || defined _GLIBCXX_HAVE_LINUX_FUTEX
38 #include <bits/gthr.h>
41 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
48 # define _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE 1
53 #define __cpp_lib_atomic_wait 201907L
55 namespace std _GLIBCXX_VISIBILITY(default)
57 _GLIBCXX_BEGIN_NAMESPACE_VERSION
60 using __platform_wait_t = int;
62 constexpr
auto __atomic_spin_count_1 = 16;
63 constexpr
auto __atomic_spin_count_2 = 12;
65 template<
typename _Tp>
66 inline constexpr
bool __platform_wait_uses_type
67 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
68 = is_same_v<remove_cv_t<_Tp>, __platform_wait_t>;
73 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
74 enum class __futex_wait_flags : int
76 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE
85 __wait_private = __wait | __private_flag,
86 __wake_private = __wake | __private_flag,
87 __wait_bitset_private = __wait_bitset | __private_flag,
88 __wake_bitset_private = __wake_bitset | __private_flag,
89 __bitset_match_any = -1
92 template<
typename _Tp>
94 __platform_wait(
const _Tp* __addr, __platform_wait_t __val) noexcept
98 auto __e = syscall (SYS_futex,
static_cast<const void*
>(__addr),
99 static_cast<int>(__futex_wait_flags::__wait_private),
101 if (!__e || errno == EAGAIN)
103 else if (errno != EINTR)
104 __throw_system_error(__e);
108 template<
typename _Tp>
110 __platform_notify(
const _Tp* __addr,
bool __all) noexcept
112 syscall (SYS_futex,
static_cast<const void*
>(__addr),
113 static_cast<int>(__futex_wait_flags::__wake_private),
114 __all ? INT_MAX : 1);
120 alignas(64) __platform_wait_t _M_ver = 0;
121 alignas(64) __platform_wait_t _M_wait = 0;
123 #ifndef _GLIBCXX_HAVE_LINUX_FUTEX
124 using __lock_t = lock_guard<mutex>;
128 __waiters() noexcept = default;
132 _M_enter_wait() noexcept
134 __platform_wait_t __res;
135 __atomic_load(&_M_ver, &__res, __ATOMIC_ACQUIRE);
136 __atomic_fetch_add(&_M_wait, 1, __ATOMIC_ACQ_REL);
141 _M_leave_wait() noexcept
143 __atomic_fetch_sub(&_M_wait, 1, __ATOMIC_ACQ_REL);
147 _M_do_wait(__platform_wait_t __version) noexcept
149 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
150 __platform_wait(&_M_ver, __version);
152 __platform_wait_t __cur = 0;
153 while (__cur <= __version)
155 __waiters::__lock_t __l(_M_mtx);
157 __platform_wait_t __last = __cur;
158 __atomic_load(&_M_ver, &__cur, __ATOMIC_ACQUIRE);
166 _M_waiting() const noexcept
168 __platform_wait_t __res;
169 __atomic_load(&_M_wait, &__res, __ATOMIC_ACQUIRE);
174 _M_notify(
bool __all) noexcept
176 __atomic_fetch_add(&_M_ver, 1, __ATOMIC_ACQ_REL);
177 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
178 __platform_notify(&_M_ver, __all);
188 _S_for(
const void* __t)
190 const unsigned char __mask = 0xf;
191 static __waiters __w[__mask + 1];
193 auto __key = _Hash_impl::hash(__t) & __mask;
201 __platform_wait_t _M_version;
203 template<
typename _Tp>
204 __waiter(
const _Tp* __addr) noexcept
205 : _M_w(__waiters::_S_for(
static_cast<const void*
>(__addr)))
206 , _M_version(_M_w._M_enter_wait())
210 { _M_w._M_leave_wait(); }
212 void _M_do_wait() noexcept
213 { _M_w._M_do_wait(_M_version); }
217 __thread_yield() noexcept
219 #if defined _GLIBCXX_HAS_GTHREADS && defined _GLIBCXX_USE_SCHED_YIELD
225 __thread_relax() noexcept
227 #if defined __i386__ || defined __x86_64__
228 __builtin_ia32_pause();
235 template<
typename _Pred>
237 __atomic_spin(_Pred& __pred) noexcept
239 for (
auto __i = 0; __i < __detail::__atomic_spin_count_1; ++__i)
244 if (__i < __detail::__atomic_spin_count_2)
245 __detail::__thread_relax();
247 __detail::__thread_yield();
252 template<
typename _Tp,
typename _Pred>
254 __atomic_wait(
const _Tp* __addr, _Tp __old, _Pred __pred) noexcept
256 using namespace __detail;
257 if (std::__atomic_spin(__pred))
260 __waiter __w(__addr);
263 if constexpr (__platform_wait_uses_type<_Tp>)
265 __platform_wait(__addr, __old);
275 template<
typename _Tp>
277 __atomic_notify(
const _Tp* __addr,
bool __all) noexcept
279 using namespace __detail;
280 auto& __w = __waiters::_S_for((
void*)__addr);
281 if (!__w._M_waiting())
284 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
285 if constexpr (__platform_wait_uses_type<_Tp>)
287 __platform_notify((__platform_wait_t*)(
void*) __addr, __all);
292 __w._M_notify(__all);
295 _GLIBCXX_END_NAMESPACE_VERSION
ISO C++ entities toplevel namespace is std.