1// -*- C++ -*- header.
2
3// Copyright (C) 2020-2026 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25/** @file bits/atomic_wait.h
26 * This is an internal header file, included by other library headers.
27 * Do not attempt to use it directly. @headername{atomic}
28 */
29
30#ifndef _GLIBCXX_ATOMIC_WAIT_H
31#define _GLIBCXX_ATOMIC_WAIT_H 1
32
33#ifdef _GLIBCXX_SYSHDR
34#pragma GCC system_header
35#endif
36
37#include <bits/version.h>
38
39#if __glibcxx_atomic_wait
40#include <bits/gthr.h>
41#include <ext/numeric_traits.h>
42
43#include <bits/stl_pair.h>
44
45namespace std _GLIBCXX_VISIBILITY(default)
46{
47_GLIBCXX_BEGIN_NAMESPACE_VERSION
48
49 namespace __detail
50 {
51 // TODO: this needs to be false for types with padding, e.g. __int20.
52 // TODO: should this be true only for integral, enum, and pointer types?
53 template<typename _Tp>
54 concept __waitable
55 = is_scalar_v<_Tp> && __builtin_popcountg(sizeof(_Tp)) == 1
56 && (sizeof(_Tp) <= sizeof(__UINT64_TYPE__));
57 }
58
59#if defined _GLIBCXX_HAVE_LINUX_FUTEX
60 namespace __detail
61 {
62 // Use futex syscall on int objects.
63 using __platform_wait_t = int;
64 inline constexpr size_t __platform_wait_alignment = 4;
65 }
66 // Defined to true for a subset of __waitable types which are statically
67 // known to definitely be able to use futex, not a proxy wait.
68 template<typename _Tp>
69 inline constexpr bool __platform_wait_uses_type
70 = __detail::__waitable<_Tp>
71 && sizeof(_Tp) == sizeof(int) && alignof(_Tp) >= 4;
72#elif defined __APPLE__ \
73 && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 101200
74 namespace __detail
75 {
76 using __platform_wait_t = __INT32_TYPE__;
77 inline constexpr size_t __platform_wait_alignment = 4;
78 }
79 // Defined to true for a subset of __waitable types which are statically
80 // known to definitely be able to use __ulock_wait, not a proxy wait.
81 // We know that OS Versions later than 10.15 support 64b wait types even
82 // though we must make the __platform_wait_t 32b for compatibility with
83 // earlier versions of __ulock_xxxx.
84 template<typename _Tp>
85 inline constexpr bool __platform_wait_uses_type
86 = __detail::__waitable<_Tp>
87# if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101500
88 && sizeof(_Tp) == 4 && alignof(_Tp) >= 4;
89# else
90 && ((sizeof(_Tp) == 4 && alignof(_Tp) >= 4)
91 || (sizeof(_Tp) == 8 && alignof(_Tp) >= 8));
92# endif
93#elif defined __FreeBSD__ && __SIZEOF_LONG__ == 8
94 namespace __detail
95 {
96 using __platform_wait_t = __UINT64_TYPE__;
97 inline constexpr size_t __platform_wait_alignment = 8;
98 }
99 template<typename _Tp>
100 inline constexpr bool __platform_wait_uses_type
101 = __detail::__waitable<_Tp>
102 && ((sizeof(_Tp) == 4 && alignof(_Tp) >= 4)
103 || (sizeof(_Tp) == 8 && alignof(_Tp) >= 8));
104#else
105// define _GLIBCX_HAVE_PLATFORM_WAIT and implement __platform_wait()
106// and __platform_notify() if there is a more efficient primitive supported
107// by the platform (e.g. __ulock_wait()/__ulock_wake()) which is better than
108// a mutex/condvar based wait.
109 namespace __detail
110 {
111# if defined __OpenBSD__ || defined __DragonFly__
112 // These targets provide 32-bit futex-like syscalls.
113 // We don't currently make use of them, but we want to in future.
114 using __platform_wait_t = unsigned int;
115# elif ATOMIC_LONG_LOCK_FREE == 2
116 using __platform_wait_t = unsigned long;
117# else
118 using __platform_wait_t = unsigned int;
119# endif
120 inline constexpr size_t __platform_wait_alignment
121 = sizeof(__platform_wait_t) < __alignof__(__platform_wait_t)
122 ? __alignof__(__platform_wait_t) : sizeof(__platform_wait_t);
123 } // namespace __detail
124
125 // This must be false for the general case where we don't know of any
126 // futex-like syscall.
127 template<typename>
128 inline constexpr bool __platform_wait_uses_type = false;
129#endif
130
131 namespace __detail
132 {
133 inline void
134 __thread_yield() noexcept
135 {
136#if defined _GLIBCXX_HAS_GTHREADS && defined _GLIBCXX_USE_SCHED_YIELD
137 __gthread_yield();
138#endif
139 }
140
141 inline void
142 __thread_relax() noexcept
143 {
144#if defined __i386__ || defined __x86_64__
145 __builtin_ia32_pause();
146#else
147 __thread_yield();
148#endif
149 }
150
151 // return true if equal
152 template<typename _Tp>
153 inline bool
154 __atomic_eq(const _Tp& __a, const _Tp& __b)
155 {
156 // TODO make this do the correct padding bit ignoring comparison
157 return __builtin_memcmp(std::addressof(__a), std::addressof(__b),
158 sizeof(_Tp)) == 0;
159 }
160
161 // Storage for up to 64 bits of value, should be considered opaque bits.
162 using __wait_value_type = __UINT64_TYPE__;
163
164 // lightweight std::optional<__wait_value_type>
165 struct __wait_result_type
166 {
167 __wait_value_type _M_val;
168 unsigned char _M_has_val : 1; // _M_val value was loaded before return.
169 unsigned char _M_timeout : 1; // Waiting function ended with timeout.
170 unsigned char _M_unused : 6; // padding
171 };
172
173 enum class __wait_flags : __UINT_LEAST32_TYPE__
174 {
175 __abi_version = 0x00000000,
176 // currently unused = 1,
177 __track_contention = 2,
178 __do_spin = 4,
179 __spin_only = 8, // Ignored unless __do_spin is also set.
180 // __abi_version_mask = 0xff000000,
181 };
182
183 [[__gnu__::__always_inline__]]
184 constexpr __wait_flags
185 operator|(__wait_flags __l, __wait_flags __r) noexcept
186 {
187 using _Ut = underlying_type_t<__wait_flags>;
188 return static_cast<__wait_flags>(static_cast<_Ut>(__l)
189 | static_cast<_Ut>(__r));
190 }
191
192 [[__gnu__::__always_inline__]]
193 constexpr __wait_flags&
194 operator|=(__wait_flags& __l, __wait_flags __r) noexcept
195 { return __l = __l | __r; }
196
197 // Simple aggregate containing arguments used by implementation details.
198 struct __wait_args_base
199 {
200 __wait_flags _M_flags;
201 int _M_order = __ATOMIC_ACQUIRE; // Memory order for loads from _M_obj.
202 __wait_value_type _M_old = 0; // Previous value of *_M_obj.
203 void* _M_wait_state = nullptr; // For proxy wait and tracking contention.
204 const void* _M_obj = nullptr; // The address of the object to wait on.
205 unsigned char _M_obj_size = 0; // The size of that object.
206
207 // Test whether _M_flags & __flags is non-zero.
208 bool
209 operator&(__wait_flags __flags) const noexcept
210 {
211 using _Ut = underlying_type_t<__wait_flags>;
212 return static_cast<_Ut>(_M_flags) & static_cast<_Ut>(__flags);
213 }
214 };
215
216 // Utility for populating a __wait_args_base structure.
217 struct __wait_args : __wait_args_base
218 {
219 template<typename _Tp> requires (!is_same_v<_Tp, __wait_args>)
220 explicit
221 __wait_args(const _Tp* __addr, bool __bare_wait = false) noexcept
222 : __wait_args_base{ _S_flags_for(__addr, __bare_wait) }
223 {
224 _M_obj = __addr; // Might be replaced by _M_setup_wait
225 if constexpr (__waitable<_Tp>)
226 // __wait_impl might be able to wait directly on __addr
227 // instead of using a proxy, depending on its size.
228 _M_obj_size = sizeof(_Tp);
229 }
230
231 __wait_args(const __platform_wait_t* __addr, __platform_wait_t __old,
232 int __order, bool __bare_wait = false) noexcept
233 : __wait_args(__addr, __bare_wait)
234 {
235 _M_order = __order;
236 _M_old = __old;
237 }
238
239 __wait_args(const __wait_args&) noexcept = default;
240 __wait_args& operator=(const __wait_args&) noexcept = default;
241
242 template<typename _Tp, typename _ValFn>
243 _Tp
244 _M_setup_wait(const _Tp* __addr, _ValFn __vfn)
245 {
246 static_assert(is_same_v<_Tp, decay_t<decltype(__vfn())>>);
247
248 if constexpr (!__platform_wait_uses_type<_Tp>)
249 if (_M_setup_proxy_wait(__addr))
250 {
251 // We will use a proxy wait for this object.
252 // The library has set _M_wait_state, _M_obj, _M_obj_size,
253 // and _M_old.
254 // Call __vfn to load the current value from *__addr
255 // (which must happen after the call to _M_setup_proxy_wait).
256 return __vfn();
257 }
258
259 // We will use a futex-like operation to wait on this object,
260 // so just load the value, store it into _M_old, and return it.
261 return _M_store(__vfn());
262 }
263
264 // Called after a wait returns, to prepare to wait again.
265 template<typename _Tp, typename _ValFn>
266 _Tp
267 _M_on_wake(const _Tp* __addr, _ValFn __vfn, __wait_result_type __res)
268 {
269 if constexpr (!__platform_wait_uses_type<_Tp>) // maybe a proxy wait
270 if (_M_obj != __addr) // definitely a proxy wait
271 {
272 if (__res._M_has_val)
273 // Previous wait loaded a recent value from the proxy.
274 _M_old = __res._M_val;
275 else // Load a new value from the proxy and store in _M_old.
276 (void) _M_setup_proxy_wait(addr: nullptr);
277 // Read the current value of *__addr
278 return __vfn();
279 }
280
281 if (__res._M_has_val) // The previous wait loaded a recent value.
282 {
283 _M_old = __res._M_val;
284
285 // Not a proxy wait, so the value in __res._M_val was loaded
286 // from *__addr and we don't need to call __vfn().
287 if constexpr (sizeof(_Tp) == sizeof(__UINT64_TYPE__))
288 return __builtin_bit_cast(_Tp, (__UINT64_TYPE__)_M_old);
289 else if constexpr (sizeof(_Tp) == sizeof(__UINT32_TYPE__))
290 return __builtin_bit_cast(_Tp, (__UINT32_TYPE__)_M_old);
291 else if constexpr (sizeof(_Tp) == sizeof(__UINT16_TYPE__))
292 return __builtin_bit_cast(_Tp, (__UINT16_TYPE__)_M_old);
293 else if constexpr (sizeof(_Tp) == sizeof(__UINT8_TYPE__))
294 return __builtin_bit_cast(_Tp, (__UINT8_TYPE__)_M_old);
295 else // Should be a proxy wait for this size!
296 __glibcxx_assert(false);
297 }
298 else
299 return _M_store(__vfn());
300 }
301
302 private:
303 // Store __val in _M_old.
304 // pre: This must be a non-proxy wait.
305 template<typename _Tp>
306 [[__gnu__::__always_inline__]]
307 _Tp
308 _M_store(_Tp __val)
309 {
310 // We have to consider various sizes, because a future libstdc++.so
311 // might enable non-proxy waits for additional sizes.
312 if constexpr (sizeof(_Tp) == sizeof(__UINT64_TYPE__))
313 _M_old = __builtin_bit_cast(__UINT64_TYPE__, __val);
314 else if constexpr (sizeof(_Tp) == sizeof(__UINT32_TYPE__))
315 _M_old = __builtin_bit_cast(__UINT32_TYPE__, __val);
316 else if constexpr (sizeof(_Tp) == sizeof(__UINT16_TYPE__))
317 _M_old = __builtin_bit_cast(__UINT16_TYPE__, __val);
318 else if constexpr (sizeof(_Tp) == sizeof(__UINT8_TYPE__))
319 _M_old = __builtin_bit_cast(__UINT8_TYPE__, __val);
320 else // Should be a proxy wait for this size!
321 __glibcxx_assert(false);
322 return __val;
323 }
324
325 // Prepare `*this` for a call to `__wait_impl` or `__wait_until_impl`.
326 // See comments in src/c++20/atomic.cc for more details.
327 bool
328 _M_setup_proxy_wait(const void* __addr);
329
330 template<typename _Tp>
331 static constexpr __wait_flags
332 _S_flags_for(const _Tp*, bool __bare_wait) noexcept
333 {
334 using enum __wait_flags;
335 __wait_flags __res = __abi_version | __do_spin;
336 if (!__bare_wait)
337 __res |= __track_contention;
338 return __res;
339 }
340 };
341
342 __wait_result_type
343 __wait_impl(const void* __addr, __wait_args_base&);
344
345 void
346 __notify_impl(const void* __addr, bool __all, const __wait_args_base&);
347 } // namespace __detail
348
349 // Wait on __addr while __pred(__vfn()) is false.
350 // If __bare_wait is false, increment a counter while waiting.
351 // For callers that keep their own count of waiters, use __bare_wait=true.
352 // The effect of __vfn() must be an atomic load from __addr and nothing else.
353 template<typename _Tp, typename _Pred, typename _ValFn>
354 void
355 __atomic_wait_address(const _Tp* __addr, _Pred&& __pred, _ValFn&& __vfn,
356 bool __bare_wait = false) noexcept
357 {
358 __detail::__wait_args __args{ __addr, __bare_wait };
359 _Tp __val = __args._M_setup_wait(__addr, __vfn);
360 while (!__pred(__val))
361 {
362 auto __res = __detail::__wait_impl(__addr, __args);
363 __val = __args._M_on_wake(__addr, __vfn, __res);
364 }
365 // C++26 will return __val
366 }
367
368 // Wait on __addr while *__addr == __old is true.
369 inline void
370 __atomic_wait_address_v(const __detail::__platform_wait_t* __addr,
371 __detail::__platform_wait_t __old,
372 int __order, bool __bare_wait = false)
373 {
374 // This function must not be used if __wait_impl might use a proxy wait:
375 __glibcxx_assert(__platform_wait_uses_type<__detail::__platform_wait_t>);
376
377 __detail::__wait_args __args{ __addr, __old, __order, __bare_wait };
378 // C++26 will not ignore the return value here
379 __detail::__wait_impl(__addr, __args);
380 }
381
382 // Wait on __addr while __vfn() == __old is true.
383 template<typename _Tp, typename _ValFn>
384 void
385 __atomic_wait_address_v(const _Tp* __addr, _Tp __old,
386 _ValFn __vfn) noexcept
387 {
388 auto __pfn = [&](const _Tp& __val)
389 { return !__detail::__atomic_eq(__old, __val); };
390 std::__atomic_wait_address(__addr, __pfn, forward<_ValFn>(__vfn));
391 }
392
393 template<typename _Tp>
394 void
395 __atomic_notify_address(const _Tp* __addr, bool __all,
396 bool __bare_wait = false) noexcept
397 {
398 __detail::__wait_args __args{ __addr, __bare_wait };
399 __detail::__notify_impl(__addr, __all, __args);
400 }
401
402_GLIBCXX_END_NAMESPACE_VERSION
403} // namespace std
404#endif // __glibcxx_atomic_wait
405#endif // _GLIBCXX_ATOMIC_WAIT_H
406