1// -*- C++ -*- header.
2
3// Copyright (C) 2008-2025 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25/** @file bits/atomic_base.h
26 * This is an internal header file, included by other library headers.
27 * Do not attempt to use it directly. @headername{atomic}
28 */
29
30#ifndef _GLIBCXX_ATOMIC_BASE_H
31#define _GLIBCXX_ATOMIC_BASE_H 1
32
33#ifdef _GLIBCXX_SYSHDR
34#pragma GCC system_header
35#endif
36
37#include <bits/c++config.h>
38#include <new> // For placement new
39#include <bits/atomic_lockfree_defines.h>
40#include <bits/move.h>
41
42#if __cplusplus > 201703L && _GLIBCXX_HOSTED
43#include <bits/atomic_wait.h>
44#endif
45
46#ifndef _GLIBCXX_ALWAYS_INLINE
47#define _GLIBCXX_ALWAYS_INLINE inline __attribute__((__always_inline__))
48#endif
49
50#include <bits/version.h>
51
52namespace std _GLIBCXX_VISIBILITY(default)
53{
54_GLIBCXX_BEGIN_NAMESPACE_VERSION
55
56 /**
57 * @defgroup atomics Atomics
58 *
59 * Components for performing atomic operations.
60 * @{
61 */
62
63 /// Enumeration for memory_order
64#if __cplusplus > 201703L
65 enum class memory_order : int
66 {
67 relaxed,
68 consume,
69 acquire,
70 release,
71 acq_rel,
72 seq_cst
73 };
74
75 inline constexpr memory_order memory_order_relaxed = memory_order::relaxed;
76 inline constexpr memory_order memory_order_consume = memory_order::consume;
77 inline constexpr memory_order memory_order_acquire = memory_order::acquire;
78 inline constexpr memory_order memory_order_release = memory_order::release;
79 inline constexpr memory_order memory_order_acq_rel = memory_order::acq_rel;
80 inline constexpr memory_order memory_order_seq_cst = memory_order::seq_cst;
81#else
82 enum memory_order : int
83 {
84 memory_order_relaxed,
85 memory_order_consume,
86 memory_order_acquire,
87 memory_order_release,
88 memory_order_acq_rel,
89 memory_order_seq_cst
90 };
91#endif
92
93 /// @cond undocumented
94 enum __memory_order_modifier
95 {
96 __memory_order_mask = 0x0ffff,
97 __memory_order_modifier_mask = 0xffff0000,
98 __memory_order_hle_acquire = 0x10000,
99 __memory_order_hle_release = 0x20000
100 };
101 /// @endcond
102
103 constexpr memory_order
104 operator|(memory_order __m, __memory_order_modifier __mod) noexcept
105 {
106 return memory_order(int(__m) | int(__mod));
107 }
108
109 constexpr memory_order
110 operator&(memory_order __m, __memory_order_modifier __mod) noexcept
111 {
112 return memory_order(int(__m) & int(__mod));
113 }
114
115 /// @cond undocumented
116
117 // Drop release ordering as per [atomics.types.operations.req]/21
118 constexpr memory_order
119 __cmpexch_failure_order2(memory_order __m) noexcept
120 {
121 return __m == memory_order_acq_rel ? memory_order_acquire
122 : __m == memory_order_release ? memory_order_relaxed : __m;
123 }
124
125 constexpr memory_order
126 __cmpexch_failure_order(memory_order __m) noexcept
127 {
128 return memory_order(__cmpexch_failure_order2(m: __m & __memory_order_mask)
129 | __memory_order_modifier(__m & __memory_order_modifier_mask));
130 }
131
132 constexpr bool
133 __is_valid_cmpexch_failure_order(memory_order __m) noexcept
134 {
135 return (__m & __memory_order_mask) != memory_order_release
136 && (__m & __memory_order_mask) != memory_order_acq_rel;
137 }
138
139 // Base types for atomics.
140 template<typename _IntTp>
141 struct __atomic_base;
142
143 /// @endcond
144
145 _GLIBCXX_ALWAYS_INLINE void
146 atomic_thread_fence(memory_order __m) noexcept
147 { __atomic_thread_fence(int(__m)); }
148
149 _GLIBCXX_ALWAYS_INLINE void
150 atomic_signal_fence(memory_order __m) noexcept
151 { __atomic_signal_fence(int(__m)); }
152
153 /// kill_dependency
154 template<typename _Tp>
155 inline _Tp
156 kill_dependency(_Tp __y) noexcept
157 {
158 _Tp __ret(__y);
159 return __ret;
160 }
161
162/// @cond undocumented
163#if __glibcxx_atomic_value_initialization
164# define _GLIBCXX20_INIT(I) = I
165#else
166# define _GLIBCXX20_INIT(I)
167#endif
168/// @endcond
169
170#define ATOMIC_VAR_INIT(_VI) { _VI }
171
172 template<typename _Tp>
173 struct atomic;
174
175 template<typename _Tp>
176 struct atomic<_Tp*>;
177
178 /* The target's "set" value for test-and-set may not be exactly 1. */
179#if __GCC_ATOMIC_TEST_AND_SET_TRUEVAL == 1
180 typedef bool __atomic_flag_data_type;
181#else
182 typedef unsigned char __atomic_flag_data_type;
183#endif
184
185 /// @cond undocumented
186
187 /*
188 * Base type for atomic_flag.
189 *
190 * Base type is POD with data, allowing atomic_flag to derive from
191 * it and meet the standard layout type requirement. In addition to
192 * compatibility with a C interface, this allows different
193 * implementations of atomic_flag to use the same atomic operation
194 * functions, via a standard conversion to the __atomic_flag_base
195 * argument.
196 */
197 _GLIBCXX_BEGIN_EXTERN_C
198
199 struct __atomic_flag_base
200 {
201 __atomic_flag_data_type _M_i _GLIBCXX20_INIT({});
202 };
203
204 _GLIBCXX_END_EXTERN_C
205
206 /// @endcond
207
208#define ATOMIC_FLAG_INIT { 0 }
209
210 /// atomic_flag
211 struct atomic_flag : public __atomic_flag_base
212 {
213 atomic_flag() noexcept = default;
214 ~atomic_flag() noexcept = default;
215 atomic_flag(const atomic_flag&) = delete;
216 atomic_flag& operator=(const atomic_flag&) = delete;
217 atomic_flag& operator=(const atomic_flag&) volatile = delete;
218
219 // Conversion to ATOMIC_FLAG_INIT.
220 constexpr atomic_flag(bool __i) noexcept
221 : __atomic_flag_base{ ._M_i: _S_init(__i) }
222 { }
223
224 _GLIBCXX_ALWAYS_INLINE bool
225 test_and_set(memory_order __m = memory_order_seq_cst) noexcept
226 {
227 return __atomic_test_and_set (&_M_i, int(__m));
228 }
229
230 _GLIBCXX_ALWAYS_INLINE bool
231 test_and_set(memory_order __m = memory_order_seq_cst) volatile noexcept
232 {
233 return __atomic_test_and_set (&_M_i, int(__m));
234 }
235
236#ifdef __glibcxx_atomic_flag_test // C++ >= 20
237 _GLIBCXX_ALWAYS_INLINE bool
238 test(memory_order __m = memory_order_seq_cst) const noexcept
239 {
240 __atomic_flag_data_type __v;
241 __atomic_load(&_M_i, &__v, int(__m));
242 return __v == __GCC_ATOMIC_TEST_AND_SET_TRUEVAL;
243 }
244
245 _GLIBCXX_ALWAYS_INLINE bool
246 test(memory_order __m = memory_order_seq_cst) const volatile noexcept
247 {
248 __atomic_flag_data_type __v;
249 __atomic_load(&_M_i, &__v, int(__m));
250 return __v == __GCC_ATOMIC_TEST_AND_SET_TRUEVAL;
251 }
252#endif
253
254#if __glibcxx_atomic_wait // C++ >= 20 && (linux_futex || gthread)
255 _GLIBCXX_ALWAYS_INLINE void
256 wait(bool __old,
257 memory_order __m = memory_order_seq_cst) const noexcept
258 {
259 const __atomic_flag_data_type __v
260 = __old ? __GCC_ATOMIC_TEST_AND_SET_TRUEVAL : 0;
261
262 std::__atomic_wait_address_v(addr: &_M_i, old: __v,
263 vfn: [__m, this] { return __atomic_load_n(&_M_i, int(__m)); });
264 }
265
266 // TODO add const volatile overload
267
268 _GLIBCXX_ALWAYS_INLINE void
269 notify_one() noexcept
270 { std::__atomic_notify_address(addr: &_M_i, all: false); }
271
272 // TODO add const volatile overload
273
274 _GLIBCXX_ALWAYS_INLINE void
275 notify_all() noexcept
276 { std::__atomic_notify_address(addr: &_M_i, all: true); }
277
278 // TODO add const volatile overload
279#endif // __glibcxx_atomic_wait
280
281 _GLIBCXX_ALWAYS_INLINE void
282 clear(memory_order __m = memory_order_seq_cst) noexcept
283 {
284 memory_order __b __attribute__ ((__unused__))
285 = __m & __memory_order_mask;
286 __glibcxx_assert(__b != memory_order_consume);
287 __glibcxx_assert(__b != memory_order_acquire);
288 __glibcxx_assert(__b != memory_order_acq_rel);
289
290 __atomic_clear (&_M_i, int(__m));
291 }
292
293 _GLIBCXX_ALWAYS_INLINE void
294 clear(memory_order __m = memory_order_seq_cst) volatile noexcept
295 {
296 memory_order __b __attribute__ ((__unused__))
297 = __m & __memory_order_mask;
298 __glibcxx_assert(__b != memory_order_consume);
299 __glibcxx_assert(__b != memory_order_acquire);
300 __glibcxx_assert(__b != memory_order_acq_rel);
301
302 __atomic_clear (&_M_i, int(__m));
303 }
304
305 private:
306 static constexpr __atomic_flag_data_type
307 _S_init(bool __i)
308 { return __i ? __GCC_ATOMIC_TEST_AND_SET_TRUEVAL : 0; }
309 };
310
311 /// @cond undocumented
312
313 /// Base class for atomic integrals.
314 //
315 // For each of the integral types, define atomic_[integral type] struct
316 //
317 // atomic_bool bool
318 // atomic_char char
319 // atomic_schar signed char
320 // atomic_uchar unsigned char
321 // atomic_short short
322 // atomic_ushort unsigned short
323 // atomic_int int
324 // atomic_uint unsigned int
325 // atomic_long long
326 // atomic_ulong unsigned long
327 // atomic_llong long long
328 // atomic_ullong unsigned long long
329 // atomic_char8_t char8_t
330 // atomic_char16_t char16_t
331 // atomic_char32_t char32_t
332 // atomic_wchar_t wchar_t
333 //
334 // NB: Assuming _ITp is an integral scalar type that is 1, 2, 4, or
335 // 8 bytes, since that is what GCC built-in functions for atomic
336 // memory access expect.
337 template<typename _ITp>
338 struct __atomic_base
339 {
340 using value_type = _ITp;
341 using difference_type = value_type;
342
343 private:
344 typedef _ITp __int_type;
345
346 static constexpr int _S_alignment =
347 sizeof(_ITp) > alignof(_ITp) ? sizeof(_ITp) : alignof(_ITp);
348
349 alignas(_S_alignment) __int_type _M_i _GLIBCXX20_INIT(0);
350
351 public:
352 __atomic_base() noexcept = default;
353 ~__atomic_base() noexcept = default;
354 __atomic_base(const __atomic_base&) = delete;
355 __atomic_base& operator=(const __atomic_base&) = delete;
356 __atomic_base& operator=(const __atomic_base&) volatile = delete;
357
358 constexpr __atomic_base(__int_type __i) noexcept : _M_i (__i) { }
359
360 operator __int_type() const noexcept
361 { return load(); }
362
363 operator __int_type() const volatile noexcept
364 { return load(); }
365
366 __int_type
367 operator=(__int_type __i) noexcept
368 {
369 store(__i);
370 return __i;
371 }
372
373 __int_type
374 operator=(__int_type __i) volatile noexcept
375 {
376 store(__i);
377 return __i;
378 }
379
380 __int_type
381 operator++(int) noexcept
382 { return fetch_add(1); }
383
384 __int_type
385 operator++(int) volatile noexcept
386 { return fetch_add(1); }
387
388 __int_type
389 operator--(int) noexcept
390 { return fetch_sub(1); }
391
392 __int_type
393 operator--(int) volatile noexcept
394 { return fetch_sub(1); }
395
396 __int_type
397 operator++() noexcept
398 { return __atomic_add_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
399
400 __int_type
401 operator++() volatile noexcept
402 { return __atomic_add_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
403
404 __int_type
405 operator--() noexcept
406 { return __atomic_sub_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
407
408 __int_type
409 operator--() volatile noexcept
410 { return __atomic_sub_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
411
412 __int_type
413 operator+=(__int_type __i) noexcept
414 { return __atomic_add_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
415
416 __int_type
417 operator+=(__int_type __i) volatile noexcept
418 { return __atomic_add_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
419
420 __int_type
421 operator-=(__int_type __i) noexcept
422 { return __atomic_sub_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
423
424 __int_type
425 operator-=(__int_type __i) volatile noexcept
426 { return __atomic_sub_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
427
428 __int_type
429 operator&=(__int_type __i) noexcept
430 { return __atomic_and_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
431
432 __int_type
433 operator&=(__int_type __i) volatile noexcept
434 { return __atomic_and_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
435
436 __int_type
437 operator|=(__int_type __i) noexcept
438 { return __atomic_or_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
439
440 __int_type
441 operator|=(__int_type __i) volatile noexcept
442 { return __atomic_or_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
443
444 __int_type
445 operator^=(__int_type __i) noexcept
446 { return __atomic_xor_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
447
448 __int_type
449 operator^=(__int_type __i) volatile noexcept
450 { return __atomic_xor_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
451
452 bool
453 is_lock_free() const noexcept
454 {
455 // Use a fake, minimally aligned pointer.
456 return __atomic_is_lock_free(sizeof(_M_i),
457 reinterpret_cast<void *>(-_S_alignment));
458 }
459
460 bool
461 is_lock_free() const volatile noexcept
462 {
463 // Use a fake, minimally aligned pointer.
464 return __atomic_is_lock_free(sizeof(_M_i),
465 reinterpret_cast<void *>(-_S_alignment));
466 }
467
468 _GLIBCXX_ALWAYS_INLINE void
469 store(__int_type __i, memory_order __m = memory_order_seq_cst) noexcept
470 {
471 memory_order __b __attribute__ ((__unused__))
472 = __m & __memory_order_mask;
473 __glibcxx_assert(__b != memory_order_acquire);
474 __glibcxx_assert(__b != memory_order_acq_rel);
475 __glibcxx_assert(__b != memory_order_consume);
476
477 __atomic_store_n(&_M_i, __i, int(__m));
478 }
479
480 _GLIBCXX_ALWAYS_INLINE void
481 store(__int_type __i,
482 memory_order __m = memory_order_seq_cst) volatile noexcept
483 {
484 memory_order __b __attribute__ ((__unused__))
485 = __m & __memory_order_mask;
486 __glibcxx_assert(__b != memory_order_acquire);
487 __glibcxx_assert(__b != memory_order_acq_rel);
488 __glibcxx_assert(__b != memory_order_consume);
489
490 __atomic_store_n(&_M_i, __i, int(__m));
491 }
492
493 _GLIBCXX_ALWAYS_INLINE __int_type
494 load(memory_order __m = memory_order_seq_cst) const noexcept
495 {
496 memory_order __b __attribute__ ((__unused__))
497 = __m & __memory_order_mask;
498 __glibcxx_assert(__b != memory_order_release);
499 __glibcxx_assert(__b != memory_order_acq_rel);
500
501 return __atomic_load_n(&_M_i, int(__m));
502 }
503
504 _GLIBCXX_ALWAYS_INLINE __int_type
505 load(memory_order __m = memory_order_seq_cst) const volatile noexcept
506 {
507 memory_order __b __attribute__ ((__unused__))
508 = __m & __memory_order_mask;
509 __glibcxx_assert(__b != memory_order_release);
510 __glibcxx_assert(__b != memory_order_acq_rel);
511
512 return __atomic_load_n(&_M_i, int(__m));
513 }
514
515 _GLIBCXX_ALWAYS_INLINE __int_type
516 exchange(__int_type __i,
517 memory_order __m = memory_order_seq_cst) noexcept
518 {
519 return __atomic_exchange_n(&_M_i, __i, int(__m));
520 }
521
522
523 _GLIBCXX_ALWAYS_INLINE __int_type
524 exchange(__int_type __i,
525 memory_order __m = memory_order_seq_cst) volatile noexcept
526 {
527 return __atomic_exchange_n(&_M_i, __i, int(__m));
528 }
529
530 _GLIBCXX_ALWAYS_INLINE bool
531 compare_exchange_weak(__int_type& __i1, __int_type __i2,
532 memory_order __m1, memory_order __m2) noexcept
533 {
534 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
535
536 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1,
537 int(__m1), int(__m2));
538 }
539
540 _GLIBCXX_ALWAYS_INLINE bool
541 compare_exchange_weak(__int_type& __i1, __int_type __i2,
542 memory_order __m1,
543 memory_order __m2) volatile noexcept
544 {
545 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
546
547 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1,
548 int(__m1), int(__m2));
549 }
550
551 _GLIBCXX_ALWAYS_INLINE bool
552 compare_exchange_weak(__int_type& __i1, __int_type __i2,
553 memory_order __m = memory_order_seq_cst) noexcept
554 {
555 return compare_exchange_weak(__i1, __i2, __m,
556 __cmpexch_failure_order(__m));
557 }
558
559 _GLIBCXX_ALWAYS_INLINE bool
560 compare_exchange_weak(__int_type& __i1, __int_type __i2,
561 memory_order __m = memory_order_seq_cst) volatile noexcept
562 {
563 return compare_exchange_weak(__i1, __i2, __m,
564 __cmpexch_failure_order(__m));
565 }
566
567 _GLIBCXX_ALWAYS_INLINE bool
568 compare_exchange_strong(__int_type& __i1, __int_type __i2,
569 memory_order __m1, memory_order __m2) noexcept
570 {
571 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
572
573 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0,
574 int(__m1), int(__m2));
575 }
576
577 _GLIBCXX_ALWAYS_INLINE bool
578 compare_exchange_strong(__int_type& __i1, __int_type __i2,
579 memory_order __m1,
580 memory_order __m2) volatile noexcept
581 {
582 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
583
584 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0,
585 int(__m1), int(__m2));
586 }
587
588 _GLIBCXX_ALWAYS_INLINE bool
589 compare_exchange_strong(__int_type& __i1, __int_type __i2,
590 memory_order __m = memory_order_seq_cst) noexcept
591 {
592 return compare_exchange_strong(__i1, __i2, __m,
593 __cmpexch_failure_order(__m));
594 }
595
596 _GLIBCXX_ALWAYS_INLINE bool
597 compare_exchange_strong(__int_type& __i1, __int_type __i2,
598 memory_order __m = memory_order_seq_cst) volatile noexcept
599 {
600 return compare_exchange_strong(__i1, __i2, __m,
601 __cmpexch_failure_order(__m));
602 }
603
604#if __glibcxx_atomic_wait
605 _GLIBCXX_ALWAYS_INLINE void
606 wait(__int_type __old,
607 memory_order __m = memory_order_seq_cst) const noexcept
608 {
609 std::__atomic_wait_address_v(&_M_i, __old,
610 [__m, this] { return this->load(__m); });
611 }
612
613 // TODO add const volatile overload
614
615 _GLIBCXX_ALWAYS_INLINE void
616 notify_one() noexcept
617 { std::__atomic_notify_address(&_M_i, false); }
618
619 // TODO add const volatile overload
620
621 _GLIBCXX_ALWAYS_INLINE void
622 notify_all() noexcept
623 { std::__atomic_notify_address(&_M_i, true); }
624
625 // TODO add const volatile overload
626#endif // __glibcxx_atomic_wait
627
628 _GLIBCXX_ALWAYS_INLINE __int_type
629 fetch_add(__int_type __i,
630 memory_order __m = memory_order_seq_cst) noexcept
631 { return __atomic_fetch_add(&_M_i, __i, int(__m)); }
632
633 _GLIBCXX_ALWAYS_INLINE __int_type
634 fetch_add(__int_type __i,
635 memory_order __m = memory_order_seq_cst) volatile noexcept
636 { return __atomic_fetch_add(&_M_i, __i, int(__m)); }
637
638 _GLIBCXX_ALWAYS_INLINE __int_type
639 fetch_sub(__int_type __i,
640 memory_order __m = memory_order_seq_cst) noexcept
641 { return __atomic_fetch_sub(&_M_i, __i, int(__m)); }
642
643 _GLIBCXX_ALWAYS_INLINE __int_type
644 fetch_sub(__int_type __i,
645 memory_order __m = memory_order_seq_cst) volatile noexcept
646 { return __atomic_fetch_sub(&_M_i, __i, int(__m)); }
647
648 _GLIBCXX_ALWAYS_INLINE __int_type
649 fetch_and(__int_type __i,
650 memory_order __m = memory_order_seq_cst) noexcept
651 { return __atomic_fetch_and(&_M_i, __i, int(__m)); }
652
653 _GLIBCXX_ALWAYS_INLINE __int_type
654 fetch_and(__int_type __i,
655 memory_order __m = memory_order_seq_cst) volatile noexcept
656 { return __atomic_fetch_and(&_M_i, __i, int(__m)); }
657
658 _GLIBCXX_ALWAYS_INLINE __int_type
659 fetch_or(__int_type __i,
660 memory_order __m = memory_order_seq_cst) noexcept
661 { return __atomic_fetch_or(&_M_i, __i, int(__m)); }
662
663 _GLIBCXX_ALWAYS_INLINE __int_type
664 fetch_or(__int_type __i,
665 memory_order __m = memory_order_seq_cst) volatile noexcept
666 { return __atomic_fetch_or(&_M_i, __i, int(__m)); }
667
668 _GLIBCXX_ALWAYS_INLINE __int_type
669 fetch_xor(__int_type __i,
670 memory_order __m = memory_order_seq_cst) noexcept
671 { return __atomic_fetch_xor(&_M_i, __i, int(__m)); }
672
673 _GLIBCXX_ALWAYS_INLINE __int_type
674 fetch_xor(__int_type __i,
675 memory_order __m = memory_order_seq_cst) volatile noexcept
676 { return __atomic_fetch_xor(&_M_i, __i, int(__m)); }
677 };
678
679
680 /// Partial specialization for pointer types.
681 template<typename _PTp>
682 struct __atomic_base<_PTp*>
683 {
684 private:
685 typedef _PTp* __pointer_type;
686
687 __pointer_type _M_p _GLIBCXX20_INIT(nullptr);
688
689 static constexpr ptrdiff_t
690 _S_type_size(ptrdiff_t __d)
691 { return __d * sizeof(_PTp); }
692
693 public:
694 __atomic_base() noexcept = default;
695 ~__atomic_base() noexcept = default;
696 __atomic_base(const __atomic_base&) = delete;
697 __atomic_base& operator=(const __atomic_base&) = delete;
698 __atomic_base& operator=(const __atomic_base&) volatile = delete;
699
700 // Requires __pointer_type convertible to _M_p.
701 constexpr __atomic_base(__pointer_type __p) noexcept : _M_p (__p) { }
702
703 operator __pointer_type() const noexcept
704 { return load(); }
705
706 operator __pointer_type() const volatile noexcept
707 { return load(); }
708
709 __pointer_type
710 operator=(__pointer_type __p) noexcept
711 {
712 store(__p);
713 return __p;
714 }
715
716 __pointer_type
717 operator=(__pointer_type __p) volatile noexcept
718 {
719 store(__p);
720 return __p;
721 }
722
723 __pointer_type
724 operator++(int) noexcept
725 { return fetch_add(1); }
726
727 __pointer_type
728 operator++(int) volatile noexcept
729 { return fetch_add(1); }
730
731 __pointer_type
732 operator--(int) noexcept
733 { return fetch_sub(1); }
734
735 __pointer_type
736 operator--(int) volatile noexcept
737 { return fetch_sub(1); }
738
739 __pointer_type
740 operator++() noexcept
741 { return __atomic_add_fetch(&_M_p, _S_type_size(d: 1),
742 int(memory_order_seq_cst)); }
743
744 __pointer_type
745 operator++() volatile noexcept
746 { return __atomic_add_fetch(&_M_p, _S_type_size(d: 1),
747 int(memory_order_seq_cst)); }
748
749 __pointer_type
750 operator--() noexcept
751 { return __atomic_sub_fetch(&_M_p, _S_type_size(d: 1),
752 int(memory_order_seq_cst)); }
753
754 __pointer_type
755 operator--() volatile noexcept
756 { return __atomic_sub_fetch(&_M_p, _S_type_size(d: 1),
757 int(memory_order_seq_cst)); }
758
759 __pointer_type
760 operator+=(ptrdiff_t __d) noexcept
761 { return __atomic_add_fetch(&_M_p, _S_type_size(__d),
762 int(memory_order_seq_cst)); }
763
764 __pointer_type
765 operator+=(ptrdiff_t __d) volatile noexcept
766 { return __atomic_add_fetch(&_M_p, _S_type_size(__d),
767 int(memory_order_seq_cst)); }
768
769 __pointer_type
770 operator-=(ptrdiff_t __d) noexcept
771 { return __atomic_sub_fetch(&_M_p, _S_type_size(__d),
772 int(memory_order_seq_cst)); }
773
774 __pointer_type
775 operator-=(ptrdiff_t __d) volatile noexcept
776 { return __atomic_sub_fetch(&_M_p, _S_type_size(__d),
777 int(memory_order_seq_cst)); }
778
779 bool
780 is_lock_free() const noexcept
781 {
782 // Produce a fake, minimally aligned pointer.
783 return __atomic_is_lock_free(sizeof(_M_p),
784 reinterpret_cast<void *>(-__alignof(_M_p)));
785 }
786
787 bool
788 is_lock_free() const volatile noexcept
789 {
790 // Produce a fake, minimally aligned pointer.
791 return __atomic_is_lock_free(sizeof(_M_p),
792 reinterpret_cast<void *>(-__alignof(_M_p)));
793 }
794
795 _GLIBCXX_ALWAYS_INLINE void
796 store(__pointer_type __p,
797 memory_order __m = memory_order_seq_cst) noexcept
798 {
799 memory_order __b __attribute__ ((__unused__))
800 = __m & __memory_order_mask;
801
802 __glibcxx_assert(__b != memory_order_acquire);
803 __glibcxx_assert(__b != memory_order_acq_rel);
804 __glibcxx_assert(__b != memory_order_consume);
805
806 __atomic_store_n(&_M_p, __p, int(__m));
807 }
808
809 _GLIBCXX_ALWAYS_INLINE void
810 store(__pointer_type __p,
811 memory_order __m = memory_order_seq_cst) volatile noexcept
812 {
813 memory_order __b __attribute__ ((__unused__))
814 = __m & __memory_order_mask;
815 __glibcxx_assert(__b != memory_order_acquire);
816 __glibcxx_assert(__b != memory_order_acq_rel);
817 __glibcxx_assert(__b != memory_order_consume);
818
819 __atomic_store_n(&_M_p, __p, int(__m));
820 }
821
822 _GLIBCXX_ALWAYS_INLINE __pointer_type
823 load(memory_order __m = memory_order_seq_cst) const noexcept
824 {
825 memory_order __b __attribute__ ((__unused__))
826 = __m & __memory_order_mask;
827 __glibcxx_assert(__b != memory_order_release);
828 __glibcxx_assert(__b != memory_order_acq_rel);
829
830 return __atomic_load_n(&_M_p, int(__m));
831 }
832
833 _GLIBCXX_ALWAYS_INLINE __pointer_type
834 load(memory_order __m = memory_order_seq_cst) const volatile noexcept
835 {
836 memory_order __b __attribute__ ((__unused__))
837 = __m & __memory_order_mask;
838 __glibcxx_assert(__b != memory_order_release);
839 __glibcxx_assert(__b != memory_order_acq_rel);
840
841 return __atomic_load_n(&_M_p, int(__m));
842 }
843
844 _GLIBCXX_ALWAYS_INLINE __pointer_type
845 exchange(__pointer_type __p,
846 memory_order __m = memory_order_seq_cst) noexcept
847 {
848 return __atomic_exchange_n(&_M_p, __p, int(__m));
849 }
850
851
852 _GLIBCXX_ALWAYS_INLINE __pointer_type
853 exchange(__pointer_type __p,
854 memory_order __m = memory_order_seq_cst) volatile noexcept
855 {
856 return __atomic_exchange_n(&_M_p, __p, int(__m));
857 }
858
859 _GLIBCXX_ALWAYS_INLINE bool
860 compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
861 memory_order __m1,
862 memory_order __m2) noexcept
863 {
864 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
865
866 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 1,
867 int(__m1), int(__m2));
868 }
869
870 _GLIBCXX_ALWAYS_INLINE bool
871 compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
872 memory_order __m1,
873 memory_order __m2) volatile noexcept
874 {
875 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
876
877 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 1,
878 int(__m1), int(__m2));
879 }
880
881 _GLIBCXX_ALWAYS_INLINE bool
882 compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
883 memory_order __m1,
884 memory_order __m2) noexcept
885 {
886 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
887
888 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0,
889 int(__m1), int(__m2));
890 }
891
892 _GLIBCXX_ALWAYS_INLINE bool
893 compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
894 memory_order __m1,
895 memory_order __m2) volatile noexcept
896 {
897 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
898
899 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0,
900 int(__m1), int(__m2));
901 }
902
903#if __glibcxx_atomic_wait
904 _GLIBCXX_ALWAYS_INLINE void
905 wait(__pointer_type __old,
906 memory_order __m = memory_order_seq_cst) const noexcept
907 {
908 std::__atomic_wait_address_v(&_M_p, __old,
909 [__m, this]
910 { return this->load(__m); });
911 }
912
913 // TODO add const volatile overload
914
915 _GLIBCXX_ALWAYS_INLINE void
916 notify_one() const noexcept
917 { std::__atomic_notify_address(&_M_p, false); }
918
919 // TODO add const volatile overload
920
921 _GLIBCXX_ALWAYS_INLINE void
922 notify_all() const noexcept
923 { std::__atomic_notify_address(&_M_p, true); }
924
925 // TODO add const volatile overload
926#endif // __glibcxx_atomic_wait
927
928 _GLIBCXX_ALWAYS_INLINE __pointer_type
929 fetch_add(ptrdiff_t __d,
930 memory_order __m = memory_order_seq_cst) noexcept
931 { return __atomic_fetch_add(&_M_p, _S_type_size(__d), int(__m)); }
932
933 _GLIBCXX_ALWAYS_INLINE __pointer_type
934 fetch_add(ptrdiff_t __d,
935 memory_order __m = memory_order_seq_cst) volatile noexcept
936 { return __atomic_fetch_add(&_M_p, _S_type_size(__d), int(__m)); }
937
938 _GLIBCXX_ALWAYS_INLINE __pointer_type
939 fetch_sub(ptrdiff_t __d,
940 memory_order __m = memory_order_seq_cst) noexcept
941 { return __atomic_fetch_sub(&_M_p, _S_type_size(__d), int(__m)); }
942
943 _GLIBCXX_ALWAYS_INLINE __pointer_type
944 fetch_sub(ptrdiff_t __d,
945 memory_order __m = memory_order_seq_cst) volatile noexcept
946 { return __atomic_fetch_sub(&_M_p, _S_type_size(__d), int(__m)); }
947 };
948
949 namespace __atomic_impl
950 {
951 // Implementation details of atomic padding handling
952
953 template<typename _Tp>
954 constexpr bool
955 __maybe_has_padding()
956 {
957#if ! __has_builtin(__builtin_clear_padding)
958 return false;
959#elif __has_builtin(__has_unique_object_representations)
960 return !__has_unique_object_representations(_Tp)
961 && !is_same<_Tp, float>::value && !is_same<_Tp, double>::value;
962#else
963 return true;
964#endif
965 }
966
967 template<typename _Tp>
968 _GLIBCXX_ALWAYS_INLINE _GLIBCXX14_CONSTEXPR _Tp*
969 __clear_padding(_Tp& __val) noexcept
970 {
971 auto* __ptr = std::__addressof(__val);
972#if __has_builtin(__builtin_clear_padding)
973 if _GLIBCXX17_CONSTEXPR (__atomic_impl::__maybe_has_padding<_Tp>())
974 __builtin_clear_padding(__ptr);
975#endif
976 return __ptr;
977 }
978
979 // Remove volatile and create a non-deduced context for value arguments.
980 template<typename _Tp>
981 using _Val = typename remove_volatile<_Tp>::type;
982
983#pragma GCC diagnostic push
984#pragma GCC diagnostic ignored "-Wc++17-extensions"
985
986 template<bool _AtomicRef = false, typename _Tp>
987 _GLIBCXX_ALWAYS_INLINE bool
988 __compare_exchange(_Tp& __val, _Val<_Tp>& __e, _Val<_Tp>& __i,
989 bool __is_weak,
990 memory_order __s, memory_order __f) noexcept
991 {
992 __glibcxx_assert(__is_valid_cmpexch_failure_order(__f));
993
994 using _Vp = _Val<_Tp>;
995 _Tp* const __pval = std::__addressof(__val);
996
997 if constexpr (!__atomic_impl::__maybe_has_padding<_Vp>())
998 {
999 return __atomic_compare_exchange(__pval, std::__addressof(__e),
1000 std::__addressof(__i), __is_weak,
1001 int(__s), int(__f));
1002 }
1003 else if constexpr (!_AtomicRef) // std::atomic<T>
1004 {
1005 // Clear padding of the value we want to set:
1006 _Vp* const __pi = __atomic_impl::__clear_padding(__i);
1007 // Only allowed to modify __e on failure, so make a copy:
1008 _Vp __exp = __e;
1009 // Clear padding of the expected value:
1010 _Vp* const __pexp = __atomic_impl::__clear_padding(__exp);
1011
1012 // For std::atomic<T> we know that the contained value will already
1013 // have zeroed padding, so trivial memcmp semantics are OK.
1014 if (__atomic_compare_exchange(__pval, __pexp, __pi,
1015 __is_weak, int(__s), int(__f)))
1016 return true;
1017 // Value bits must be different, copy from __exp back to __e:
1018 __builtin_memcpy(std::__addressof(__e), __pexp, sizeof(_Vp));
1019 return false;
1020 }
1021 else // std::atomic_ref<T> where T has padding bits.
1022 {
1023 // Clear padding of the value we want to set:
1024 _Vp* const __pi = __atomic_impl::__clear_padding(__i);
1025
1026 // Only allowed to modify __e on failure, so make a copy:
1027 _Vp __exp = __e;
1028 // Optimistically assume that a previous store had zeroed padding
1029 // so that zeroing it in the expected value will match first time.
1030 _Vp* const __pexp = __atomic_impl::__clear_padding(__exp);
1031
1032 // compare_exchange is specified to compare value representations.
1033 // Need to check whether a failure is 'real' or just due to
1034 // differences in padding bits. This loop should run no more than
1035 // three times, because the worst case scenario is:
1036 // First CAS fails because the actual value has non-zero padding.
1037 // Second CAS fails because another thread stored the same value,
1038 // but now with padding cleared. Third CAS succeeds.
1039 // We will never need to loop a fourth time, because any value
1040 // written by another thread (whether via store, exchange or
1041 // compare_exchange) will have had its padding cleared.
1042 while (true)
1043 {
1044 // Copy of the expected value so we can clear its padding.
1045 _Vp __orig = __exp;
1046
1047 if (__atomic_compare_exchange(__pval, __pexp, __pi,
1048 __is_weak, int(__s), int(__f)))
1049 return true;
1050
1051 // Copy of the actual value so we can clear its padding.
1052 _Vp __curr = __exp;
1053
1054 // Compare value representations (i.e. ignoring padding).
1055 if (__builtin_memcmp(__atomic_impl::__clear_padding(__orig),
1056 __atomic_impl::__clear_padding(__curr),
1057 sizeof(_Vp)))
1058 {
1059 // Value representations compare unequal, real failure.
1060 __builtin_memcpy(std::__addressof(__e), __pexp,
1061 sizeof(_Vp));
1062 return false;
1063 }
1064 }
1065 }
1066 }
1067#pragma GCC diagnostic pop
1068 } // namespace __atomic_impl
1069
1070#if __cplusplus > 201703L
1071 // Implementation details of atomic_ref and atomic<floating-point>.
1072 namespace __atomic_impl
1073 {
1074 // Like _Val<T> above, but for difference_type arguments.
1075 template<typename _Tp>
1076 using _Diff = __conditional_t<is_pointer_v<_Tp>, ptrdiff_t, _Val<_Tp>>;
1077
1078 template<size_t _Size, size_t _Align>
1079 _GLIBCXX_ALWAYS_INLINE bool
1080 is_lock_free() noexcept
1081 {
1082 // Produce a fake, minimally aligned pointer.
1083 return __atomic_is_lock_free(_Size, reinterpret_cast<void *>(-_Align));
1084 }
1085
1086 template<typename _Tp>
1087 _GLIBCXX_ALWAYS_INLINE void
1088 store(_Tp* __ptr, _Val<_Tp> __t, memory_order __m) noexcept
1089 {
1090 __atomic_store(__ptr, __atomic_impl::__clear_padding(__t), int(__m));
1091 }
1092
1093 template<typename _Tp>
1094 _GLIBCXX_ALWAYS_INLINE _Val<_Tp>
1095 load(const _Tp* __ptr, memory_order __m) noexcept
1096 {
1097 alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
1098 auto* __dest = reinterpret_cast<_Val<_Tp>*>(__buf);
1099 __atomic_load(__ptr, __dest, int(__m));
1100 return *__dest;
1101 }
1102
1103 template<typename _Tp>
1104 _GLIBCXX_ALWAYS_INLINE _Val<_Tp>
1105 exchange(_Tp* __ptr, _Val<_Tp> __desired, memory_order __m) noexcept
1106 {
1107 alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
1108 auto* __dest = reinterpret_cast<_Val<_Tp>*>(__buf);
1109 __atomic_exchange(__ptr, __atomic_impl::__clear_padding(__desired),
1110 __dest, int(__m));
1111 return *__dest;
1112 }
1113
1114 template<bool _AtomicRef = false, typename _Tp>
1115 _GLIBCXX_ALWAYS_INLINE bool
1116 compare_exchange_weak(_Tp* __ptr, _Val<_Tp>& __expected,
1117 _Val<_Tp> __desired, memory_order __success,
1118 memory_order __failure) noexcept
1119 {
1120 return __atomic_impl::__compare_exchange<_AtomicRef>(
1121 *__ptr, __expected, __desired, true, __success, __failure);
1122 }
1123
1124 template<bool _AtomicRef = false, typename _Tp>
1125 _GLIBCXX_ALWAYS_INLINE bool
1126 compare_exchange_strong(_Tp* __ptr, _Val<_Tp>& __expected,
1127 _Val<_Tp> __desired, memory_order __success,
1128 memory_order __failure) noexcept
1129 {
1130 return __atomic_impl::__compare_exchange<_AtomicRef>(
1131 *__ptr, __expected, __desired, false, __success, __failure);
1132 }
1133
1134#if __glibcxx_atomic_wait
1135 template<typename _Tp>
1136 _GLIBCXX_ALWAYS_INLINE void
1137 wait(const _Tp* __ptr, _Val<_Tp> __old,
1138 memory_order __m = memory_order_seq_cst) noexcept
1139 {
1140 std::__atomic_wait_address_v(__ptr, __old,
1141 [__ptr, __m]() { return __atomic_impl::load(__ptr, __m); });
1142 }
1143
1144 // TODO add const volatile overload
1145
1146 template<typename _Tp>
1147 _GLIBCXX_ALWAYS_INLINE void
1148 notify_one(const _Tp* __ptr) noexcept
1149 { std::__atomic_notify_address(__ptr, false); }
1150
1151 // TODO add const volatile overload
1152
1153 template<typename _Tp>
1154 _GLIBCXX_ALWAYS_INLINE void
1155 notify_all(const _Tp* __ptr) noexcept
1156 { std::__atomic_notify_address(__ptr, true); }
1157
1158 // TODO add const volatile overload
1159#endif // __glibcxx_atomic_wait
1160
1161 template<typename _Tp>
1162 _GLIBCXX_ALWAYS_INLINE _Tp
1163 fetch_add(_Tp* __ptr, _Diff<_Tp> __i, memory_order __m) noexcept
1164 { return __atomic_fetch_add(__ptr, __i, int(__m)); }
1165
1166 template<typename _Tp>
1167 _GLIBCXX_ALWAYS_INLINE _Tp
1168 fetch_sub(_Tp* __ptr, _Diff<_Tp> __i, memory_order __m) noexcept
1169 { return __atomic_fetch_sub(__ptr, __i, int(__m)); }
1170
1171 template<typename _Tp>
1172 _GLIBCXX_ALWAYS_INLINE _Tp
1173 fetch_and(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1174 { return __atomic_fetch_and(__ptr, __i, int(__m)); }
1175
1176 template<typename _Tp>
1177 _GLIBCXX_ALWAYS_INLINE _Tp
1178 fetch_or(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1179 { return __atomic_fetch_or(__ptr, __i, int(__m)); }
1180
1181 template<typename _Tp>
1182 _GLIBCXX_ALWAYS_INLINE _Tp
1183 fetch_xor(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1184 { return __atomic_fetch_xor(__ptr, __i, int(__m)); }
1185
1186 template<typename _Tp>
1187 _GLIBCXX_ALWAYS_INLINE _Tp
1188 __add_fetch(_Tp* __ptr, _Diff<_Tp> __i) noexcept
1189 { return __atomic_add_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1190
1191 template<typename _Tp>
1192 _GLIBCXX_ALWAYS_INLINE _Tp
1193 __sub_fetch(_Tp* __ptr, _Diff<_Tp> __i) noexcept
1194 { return __atomic_sub_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1195
1196 template<typename _Tp>
1197 _GLIBCXX_ALWAYS_INLINE _Tp
1198 __and_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
1199 { return __atomic_and_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1200
1201 template<typename _Tp>
1202 _GLIBCXX_ALWAYS_INLINE _Tp
1203 __or_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
1204 { return __atomic_or_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1205
1206 template<typename _Tp>
1207 _GLIBCXX_ALWAYS_INLINE _Tp
1208 __xor_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
1209 { return __atomic_xor_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1210
1211 template<typename _Tp>
1212 concept __atomic_fetch_addable
1213 = requires (_Tp __t) { __atomic_fetch_add(&__t, __t, 0); };
1214
1215 template<typename _Tp>
1216 _Tp
1217 __fetch_add_flt(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1218 {
1219 if constexpr (__atomic_fetch_addable<_Tp>)
1220 return __atomic_fetch_add(__ptr, __i, int(__m));
1221 else
1222 {
1223 _Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
1224 _Val<_Tp> __newval = __oldval + __i;
1225 while (!compare_exchange_weak (__ptr, __oldval, __newval, __m,
1226 memory_order_relaxed))
1227 __newval = __oldval + __i;
1228 return __oldval;
1229 }
1230 }
1231
1232 template<typename _Tp>
1233 concept __atomic_fetch_subtractable
1234 = requires (_Tp __t) { __atomic_fetch_sub(&__t, __t, 0); };
1235
1236 template<typename _Tp>
1237 _Tp
1238 __fetch_sub_flt(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1239 {
1240 if constexpr (__atomic_fetch_subtractable<_Tp>)
1241 return __atomic_fetch_sub(__ptr, __i, int(__m));
1242 else
1243 {
1244 _Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
1245 _Val<_Tp> __newval = __oldval - __i;
1246 while (!compare_exchange_weak (__ptr, __oldval, __newval, __m,
1247 memory_order_relaxed))
1248 __newval = __oldval - __i;
1249 return __oldval;
1250 }
1251 }
1252
1253 template<typename _Tp>
1254 concept __atomic_add_fetchable
1255 = requires (_Tp __t) { __atomic_add_fetch(&__t, __t, 0); };
1256
1257 template<typename _Tp>
1258 _Tp
1259 __add_fetch_flt(_Tp* __ptr, _Val<_Tp> __i) noexcept
1260 {
1261 if constexpr (__atomic_add_fetchable<_Tp>)
1262 return __atomic_add_fetch(__ptr, __i, __ATOMIC_SEQ_CST);
1263 else
1264 {
1265 _Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
1266 _Val<_Tp> __newval = __oldval + __i;
1267 while (!compare_exchange_weak (__ptr, __oldval, __newval,
1268 memory_order_seq_cst,
1269 memory_order_relaxed))
1270 __newval = __oldval + __i;
1271 return __newval;
1272 }
1273 }
1274
1275 template<typename _Tp>
1276 concept __atomic_sub_fetchable
1277 = requires (_Tp __t) { __atomic_sub_fetch(&__t, __t, 0); };
1278
1279 template<typename _Tp>
1280 _Tp
1281 __sub_fetch_flt(_Tp* __ptr, _Val<_Tp> __i) noexcept
1282 {
1283 if constexpr (__atomic_sub_fetchable<_Tp>)
1284 return __atomic_sub_fetch(__ptr, __i, __ATOMIC_SEQ_CST);
1285 else
1286 {
1287 _Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
1288 _Val<_Tp> __newval = __oldval - __i;
1289 while (!compare_exchange_weak (__ptr, __oldval, __newval,
1290 memory_order_seq_cst,
1291 memory_order_relaxed))
1292 __newval = __oldval - __i;
1293 return __newval;
1294 }
1295 }
1296 } // namespace __atomic_impl
1297
1298 // base class for atomic<floating-point-type>
1299 template<typename _Fp>
1300 struct __atomic_float
1301 {
1302 static_assert(is_floating_point_v<_Fp>);
1303
1304 static constexpr size_t _S_alignment = __alignof__(_Fp);
1305
1306 public:
1307 using value_type = _Fp;
1308 using difference_type = value_type;
1309
1310 static constexpr bool is_always_lock_free
1311 = __atomic_always_lock_free(sizeof(_Fp), 0);
1312
1313 __atomic_float() = default;
1314
1315 constexpr
1316 __atomic_float(_Fp __t) : _M_fp(__t)
1317 { __atomic_impl::__clear_padding(_M_fp); }
1318
1319 __atomic_float(const __atomic_float&) = delete;
1320 __atomic_float& operator=(const __atomic_float&) = delete;
1321 __atomic_float& operator=(const __atomic_float&) volatile = delete;
1322
1323 _Fp
1324 operator=(_Fp __t) volatile noexcept
1325 {
1326 this->store(__t);
1327 return __t;
1328 }
1329
1330 _Fp
1331 operator=(_Fp __t) noexcept
1332 {
1333 this->store(__t);
1334 return __t;
1335 }
1336
1337 bool
1338 is_lock_free() const volatile noexcept
1339 { return __atomic_impl::is_lock_free<sizeof(_Fp), _S_alignment>(); }
1340
1341 bool
1342 is_lock_free() const noexcept
1343 { return __atomic_impl::is_lock_free<sizeof(_Fp), _S_alignment>(); }
1344
1345 void
1346 store(_Fp __t, memory_order __m = memory_order_seq_cst) volatile noexcept
1347 { __atomic_impl::store(&_M_fp, __t, __m); }
1348
1349 void
1350 store(_Fp __t, memory_order __m = memory_order_seq_cst) noexcept
1351 { __atomic_impl::store(&_M_fp, __t, __m); }
1352
1353 _Fp
1354 load(memory_order __m = memory_order_seq_cst) const volatile noexcept
1355 { return __atomic_impl::load(&_M_fp, __m); }
1356
1357 _Fp
1358 load(memory_order __m = memory_order_seq_cst) const noexcept
1359 { return __atomic_impl::load(&_M_fp, __m); }
1360
1361 operator _Fp() const volatile noexcept { return this->load(); }
1362 operator _Fp() const noexcept { return this->load(); }
1363
1364 _Fp
1365 exchange(_Fp __desired,
1366 memory_order __m = memory_order_seq_cst) volatile noexcept
1367 { return __atomic_impl::exchange(&_M_fp, __desired, __m); }
1368
1369 _Fp
1370 exchange(_Fp __desired,
1371 memory_order __m = memory_order_seq_cst) noexcept
1372 { return __atomic_impl::exchange(&_M_fp, __desired, __m); }
1373
1374 bool
1375 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1376 memory_order __success,
1377 memory_order __failure) noexcept
1378 {
1379 return __atomic_impl::compare_exchange_weak(&_M_fp,
1380 __expected, __desired,
1381 __success, __failure);
1382 }
1383
1384 bool
1385 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1386 memory_order __success,
1387 memory_order __failure) volatile noexcept
1388 {
1389 return __atomic_impl::compare_exchange_weak(&_M_fp,
1390 __expected, __desired,
1391 __success, __failure);
1392 }
1393
1394 bool
1395 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1396 memory_order __success,
1397 memory_order __failure) noexcept
1398 {
1399 return __atomic_impl::compare_exchange_strong(&_M_fp,
1400 __expected, __desired,
1401 __success, __failure);
1402 }
1403
1404 bool
1405 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1406 memory_order __success,
1407 memory_order __failure) volatile noexcept
1408 {
1409 return __atomic_impl::compare_exchange_strong(&_M_fp,
1410 __expected, __desired,
1411 __success, __failure);
1412 }
1413
1414 bool
1415 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1416 memory_order __order = memory_order_seq_cst)
1417 noexcept
1418 {
1419 return compare_exchange_weak(__expected, __desired, __order,
1420 __cmpexch_failure_order(m: __order));
1421 }
1422
1423 bool
1424 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1425 memory_order __order = memory_order_seq_cst)
1426 volatile noexcept
1427 {
1428 return compare_exchange_weak(__expected, __desired, __order,
1429 __cmpexch_failure_order(m: __order));
1430 }
1431
1432 bool
1433 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1434 memory_order __order = memory_order_seq_cst)
1435 noexcept
1436 {
1437 return compare_exchange_strong(__expected, __desired, __order,
1438 __cmpexch_failure_order(m: __order));
1439 }
1440
1441 bool
1442 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1443 memory_order __order = memory_order_seq_cst)
1444 volatile noexcept
1445 {
1446 return compare_exchange_strong(__expected, __desired, __order,
1447 __cmpexch_failure_order(m: __order));
1448 }
1449
1450#if __glibcxx_atomic_wait
1451 _GLIBCXX_ALWAYS_INLINE void
1452 wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept
1453 { __atomic_impl::wait(&_M_fp, __old, __m); }
1454
1455 // TODO add const volatile overload
1456
1457 _GLIBCXX_ALWAYS_INLINE void
1458 notify_one() const noexcept
1459 { __atomic_impl::notify_one(&_M_fp); }
1460
1461 // TODO add const volatile overload
1462
1463 _GLIBCXX_ALWAYS_INLINE void
1464 notify_all() const noexcept
1465 { __atomic_impl::notify_all(&_M_fp); }
1466
1467 // TODO add const volatile overload
1468#endif // __glibcxx_atomic_wait
1469
1470 value_type
1471 fetch_add(value_type __i,
1472 memory_order __m = memory_order_seq_cst) noexcept
1473 { return __atomic_impl::__fetch_add_flt(&_M_fp, __i, __m); }
1474
1475 value_type
1476 fetch_add(value_type __i,
1477 memory_order __m = memory_order_seq_cst) volatile noexcept
1478 { return __atomic_impl::__fetch_add_flt(&_M_fp, __i, __m); }
1479
1480 value_type
1481 fetch_sub(value_type __i,
1482 memory_order __m = memory_order_seq_cst) noexcept
1483 { return __atomic_impl::__fetch_sub_flt(&_M_fp, __i, __m); }
1484
1485 value_type
1486 fetch_sub(value_type __i,
1487 memory_order __m = memory_order_seq_cst) volatile noexcept
1488 { return __atomic_impl::__fetch_sub_flt(&_M_fp, __i, __m); }
1489
1490 value_type
1491 operator+=(value_type __i) noexcept
1492 { return __atomic_impl::__add_fetch_flt(&_M_fp, __i); }
1493
1494 value_type
1495 operator+=(value_type __i) volatile noexcept
1496 { return __atomic_impl::__add_fetch_flt(&_M_fp, __i); }
1497
1498 value_type
1499 operator-=(value_type __i) noexcept
1500 { return __atomic_impl::__sub_fetch_flt(&_M_fp, __i); }
1501
1502 value_type
1503 operator-=(value_type __i) volatile noexcept
1504 { return __atomic_impl::__sub_fetch_flt(&_M_fp, __i); }
1505
1506 private:
1507 alignas(_S_alignment) _Fp _M_fp _GLIBCXX20_INIT(0);
1508 };
1509#undef _GLIBCXX20_INIT
1510
1511 template<typename _Tp,
1512 bool = is_integral_v<_Tp> && !is_same_v<_Tp, bool>,
1513 bool = is_floating_point_v<_Tp>>
1514 struct __atomic_ref;
1515
1516 // base class for non-integral, non-floating-point, non-pointer types
1517 template<typename _Tp>
1518 struct __atomic_ref<_Tp, false, false>
1519 {
1520 static_assert(is_trivially_copyable_v<_Tp>);
1521
1522 // 1/2/4/8/16-byte types must be aligned to at least their size.
1523 static constexpr int _S_min_alignment
1524 = (sizeof(_Tp) & (sizeof(_Tp) - 1)) || sizeof(_Tp) > 16
1525 ? 0 : sizeof(_Tp);
1526
1527 public:
1528 using value_type = _Tp;
1529
1530 static constexpr bool is_always_lock_free
1531 = __atomic_always_lock_free(sizeof(_Tp), 0);
1532
1533 static constexpr size_t required_alignment
1534 = _S_min_alignment > alignof(_Tp) ? _S_min_alignment : alignof(_Tp);
1535
1536 __atomic_ref& operator=(const __atomic_ref&) = delete;
1537
1538 explicit
1539 __atomic_ref(_Tp& __t) : _M_ptr(std::__addressof(__t))
1540 {
1541 __glibcxx_assert(((__UINTPTR_TYPE__)_M_ptr % required_alignment) == 0);
1542 }
1543
1544 __atomic_ref(const __atomic_ref&) noexcept = default;
1545
1546 _Tp
1547 operator=(_Tp __t) const noexcept
1548 {
1549 this->store(__t);
1550 return __t;
1551 }
1552
1553 operator _Tp() const noexcept { return this->load(); }
1554
1555 bool
1556 is_lock_free() const noexcept
1557 { return __atomic_impl::is_lock_free<sizeof(_Tp), required_alignment>(); }
1558
1559 void
1560 store(_Tp __t, memory_order __m = memory_order_seq_cst) const noexcept
1561 { __atomic_impl::store(_M_ptr, __t, __m); }
1562
1563 _Tp
1564 load(memory_order __m = memory_order_seq_cst) const noexcept
1565 { return __atomic_impl::load(_M_ptr, __m); }
1566
1567 _Tp
1568 exchange(_Tp __desired, memory_order __m = memory_order_seq_cst)
1569 const noexcept
1570 { return __atomic_impl::exchange(_M_ptr, __desired, __m); }
1571
1572 bool
1573 compare_exchange_weak(_Tp& __expected, _Tp __desired,
1574 memory_order __success,
1575 memory_order __failure) const noexcept
1576 {
1577 return __atomic_impl::compare_exchange_weak<true>(
1578 _M_ptr, __expected, __desired, __success, __failure);
1579 }
1580
1581 bool
1582 compare_exchange_strong(_Tp& __expected, _Tp __desired,
1583 memory_order __success,
1584 memory_order __failure) const noexcept
1585 {
1586 return __atomic_impl::compare_exchange_strong<true>(
1587 _M_ptr, __expected, __desired, __success, __failure);
1588 }
1589
1590 bool
1591 compare_exchange_weak(_Tp& __expected, _Tp __desired,
1592 memory_order __order = memory_order_seq_cst)
1593 const noexcept
1594 {
1595 return compare_exchange_weak(__expected, __desired, __order,
1596 __cmpexch_failure_order(m: __order));
1597 }
1598
1599 bool
1600 compare_exchange_strong(_Tp& __expected, _Tp __desired,
1601 memory_order __order = memory_order_seq_cst)
1602 const noexcept
1603 {
1604 return compare_exchange_strong(__expected, __desired, __order,
1605 __cmpexch_failure_order(m: __order));
1606 }
1607
1608#if __glibcxx_atomic_wait
1609 _GLIBCXX_ALWAYS_INLINE void
1610 wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept
1611 { __atomic_impl::wait(_M_ptr, __old, __m); }
1612
1613 // TODO add const volatile overload
1614
1615 _GLIBCXX_ALWAYS_INLINE void
1616 notify_one() const noexcept
1617 { __atomic_impl::notify_one(_M_ptr); }
1618
1619 // TODO add const volatile overload
1620
1621 _GLIBCXX_ALWAYS_INLINE void
1622 notify_all() const noexcept
1623 { __atomic_impl::notify_all(_M_ptr); }
1624
1625 // TODO add const volatile overload
1626#endif // __glibcxx_atomic_wait
1627
1628 private:
1629 _Tp* _M_ptr;
1630 };
1631
1632 // base class for atomic_ref<integral-type>
1633 template<typename _Tp>
1634 struct __atomic_ref<_Tp, true, false>
1635 {
1636 static_assert(is_integral_v<_Tp>);
1637
1638 public:
1639 using value_type = _Tp;
1640 using difference_type = value_type;
1641
1642 static constexpr bool is_always_lock_free
1643 = __atomic_always_lock_free(sizeof(_Tp), 0);
1644
1645 static constexpr size_t required_alignment
1646 = sizeof(_Tp) > alignof(_Tp) ? sizeof(_Tp) : alignof(_Tp);
1647
1648 __atomic_ref() = delete;
1649 __atomic_ref& operator=(const __atomic_ref&) = delete;
1650
1651 explicit
1652 __atomic_ref(_Tp& __t) : _M_ptr(&__t)
1653 {
1654 __glibcxx_assert(((__UINTPTR_TYPE__)_M_ptr % required_alignment) == 0);
1655 }
1656
1657 __atomic_ref(const __atomic_ref&) noexcept = default;
1658
1659 _Tp
1660 operator=(_Tp __t) const noexcept
1661 {
1662 this->store(__t);
1663 return __t;
1664 }
1665
1666 operator _Tp() const noexcept { return this->load(); }
1667
1668 bool
1669 is_lock_free() const noexcept
1670 {
1671 return __atomic_impl::is_lock_free<sizeof(_Tp), required_alignment>();
1672 }
1673
1674 void
1675 store(_Tp __t, memory_order __m = memory_order_seq_cst) const noexcept
1676 { __atomic_impl::store(_M_ptr, __t, __m); }
1677
1678 _Tp
1679 load(memory_order __m = memory_order_seq_cst) const noexcept
1680 { return __atomic_impl::load(_M_ptr, __m); }
1681
1682 _Tp
1683 exchange(_Tp __desired,
1684 memory_order __m = memory_order_seq_cst) const noexcept
1685 { return __atomic_impl::exchange(_M_ptr, __desired, __m); }
1686
1687 bool
1688 compare_exchange_weak(_Tp& __expected, _Tp __desired,
1689 memory_order __success,
1690 memory_order __failure) const noexcept
1691 {
1692 return __atomic_impl::compare_exchange_weak<true>(
1693 _M_ptr, __expected, __desired, __success, __failure);
1694 }
1695
1696 bool
1697 compare_exchange_strong(_Tp& __expected, _Tp __desired,
1698 memory_order __success,
1699 memory_order __failure) const noexcept
1700 {
1701 return __atomic_impl::compare_exchange_strong<true>(
1702 _M_ptr, __expected, __desired, __success, __failure);
1703 }
1704
1705 bool
1706 compare_exchange_weak(_Tp& __expected, _Tp __desired,
1707 memory_order __order = memory_order_seq_cst)
1708 const noexcept
1709 {
1710 return compare_exchange_weak(__expected, __desired, __order,
1711 __cmpexch_failure_order(m: __order));
1712 }
1713
1714 bool
1715 compare_exchange_strong(_Tp& __expected, _Tp __desired,
1716 memory_order __order = memory_order_seq_cst)
1717 const noexcept
1718 {
1719 return compare_exchange_strong(__expected, __desired, __order,
1720 __cmpexch_failure_order(m: __order));
1721 }
1722
1723#if __glibcxx_atomic_wait
1724 _GLIBCXX_ALWAYS_INLINE void
1725 wait(_Tp __old, memory_order __m = memory_order_seq_cst) const noexcept
1726 { __atomic_impl::wait(_M_ptr, __old, __m); }
1727
1728 // TODO add const volatile overload
1729
1730 _GLIBCXX_ALWAYS_INLINE void
1731 notify_one() const noexcept
1732 { __atomic_impl::notify_one(_M_ptr); }
1733
1734 // TODO add const volatile overload
1735
1736 _GLIBCXX_ALWAYS_INLINE void
1737 notify_all() const noexcept
1738 { __atomic_impl::notify_all(_M_ptr); }
1739
1740 // TODO add const volatile overload
1741#endif // __glibcxx_atomic_wait
1742
1743 value_type
1744 fetch_add(value_type __i,
1745 memory_order __m = memory_order_seq_cst) const noexcept
1746 { return __atomic_impl::fetch_add(_M_ptr, __i, __m); }
1747
1748 value_type
1749 fetch_sub(value_type __i,
1750 memory_order __m = memory_order_seq_cst) const noexcept
1751 { return __atomic_impl::fetch_sub(_M_ptr, __i, __m); }
1752
1753 value_type
1754 fetch_and(value_type __i,
1755 memory_order __m = memory_order_seq_cst) const noexcept
1756 { return __atomic_impl::fetch_and(_M_ptr, __i, __m); }
1757
1758 value_type
1759 fetch_or(value_type __i,
1760 memory_order __m = memory_order_seq_cst) const noexcept
1761 { return __atomic_impl::fetch_or(_M_ptr, __i, __m); }
1762
1763 value_type
1764 fetch_xor(value_type __i,
1765 memory_order __m = memory_order_seq_cst) const noexcept
1766 { return __atomic_impl::fetch_xor(_M_ptr, __i, __m); }
1767
1768 _GLIBCXX_ALWAYS_INLINE value_type
1769 operator++(int) const noexcept
1770 { return fetch_add(i: 1); }
1771
1772 _GLIBCXX_ALWAYS_INLINE value_type
1773 operator--(int) const noexcept
1774 { return fetch_sub(i: 1); }
1775
1776 value_type
1777 operator++() const noexcept
1778 { return __atomic_impl::__add_fetch(_M_ptr, value_type(1)); }
1779
1780 value_type
1781 operator--() const noexcept
1782 { return __atomic_impl::__sub_fetch(_M_ptr, value_type(1)); }
1783
1784 value_type
1785 operator+=(value_type __i) const noexcept
1786 { return __atomic_impl::__add_fetch(_M_ptr, __i); }
1787
1788 value_type
1789 operator-=(value_type __i) const noexcept
1790 { return __atomic_impl::__sub_fetch(_M_ptr, __i); }
1791
1792 value_type
1793 operator&=(value_type __i) const noexcept
1794 { return __atomic_impl::__and_fetch(_M_ptr, __i); }
1795
1796 value_type
1797 operator|=(value_type __i) const noexcept
1798 { return __atomic_impl::__or_fetch(_M_ptr, __i); }
1799
1800 value_type
1801 operator^=(value_type __i) const noexcept
1802 { return __atomic_impl::__xor_fetch(_M_ptr, __i); }
1803
1804 private:
1805 _Tp* _M_ptr;
1806 };
1807
1808 // base class for atomic_ref<floating-point-type>
1809 template<typename _Fp>
1810 struct __atomic_ref<_Fp, false, true>
1811 {
1812 static_assert(is_floating_point_v<_Fp>);
1813
1814 public:
1815 using value_type = _Fp;
1816 using difference_type = value_type;
1817
1818 static constexpr bool is_always_lock_free
1819 = __atomic_always_lock_free(sizeof(_Fp), 0);
1820
1821 static constexpr size_t required_alignment = __alignof__(_Fp);
1822
1823 __atomic_ref() = delete;
1824 __atomic_ref& operator=(const __atomic_ref&) = delete;
1825
1826 explicit
1827 __atomic_ref(_Fp& __t) : _M_ptr(&__t)
1828 {
1829 __glibcxx_assert(((__UINTPTR_TYPE__)_M_ptr % required_alignment) == 0);
1830 }
1831
1832 __atomic_ref(const __atomic_ref&) noexcept = default;
1833
1834 _Fp
1835 operator=(_Fp __t) const noexcept
1836 {
1837 this->store(__t);
1838 return __t;
1839 }
1840
1841 operator _Fp() const noexcept { return this->load(); }
1842
1843 bool
1844 is_lock_free() const noexcept
1845 {
1846 return __atomic_impl::is_lock_free<sizeof(_Fp), required_alignment>();
1847 }
1848
1849 void
1850 store(_Fp __t, memory_order __m = memory_order_seq_cst) const noexcept
1851 { __atomic_impl::store(_M_ptr, __t, __m); }
1852
1853 _Fp
1854 load(memory_order __m = memory_order_seq_cst) const noexcept
1855 { return __atomic_impl::load(_M_ptr, __m); }
1856
1857 _Fp
1858 exchange(_Fp __desired,
1859 memory_order __m = memory_order_seq_cst) const noexcept
1860 { return __atomic_impl::exchange(_M_ptr, __desired, __m); }
1861
1862 bool
1863 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1864 memory_order __success,
1865 memory_order __failure) const noexcept
1866 {
1867 return __atomic_impl::compare_exchange_weak<true>(
1868 _M_ptr, __expected, __desired, __success, __failure);
1869 }
1870
1871 bool
1872 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1873 memory_order __success,
1874 memory_order __failure) const noexcept
1875 {
1876 return __atomic_impl::compare_exchange_strong<true>(
1877 _M_ptr, __expected, __desired, __success, __failure);
1878 }
1879
1880 bool
1881 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1882 memory_order __order = memory_order_seq_cst)
1883 const noexcept
1884 {
1885 return compare_exchange_weak(__expected, __desired, __order,
1886 __cmpexch_failure_order(m: __order));
1887 }
1888
1889 bool
1890 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1891 memory_order __order = memory_order_seq_cst)
1892 const noexcept
1893 {
1894 return compare_exchange_strong(__expected, __desired, __order,
1895 __cmpexch_failure_order(m: __order));
1896 }
1897
1898#if __glibcxx_atomic_wait
1899 _GLIBCXX_ALWAYS_INLINE void
1900 wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept
1901 { __atomic_impl::wait(_M_ptr, __old, __m); }
1902
1903 // TODO add const volatile overload
1904
1905 _GLIBCXX_ALWAYS_INLINE void
1906 notify_one() const noexcept
1907 { __atomic_impl::notify_one(_M_ptr); }
1908
1909 // TODO add const volatile overload
1910
1911 _GLIBCXX_ALWAYS_INLINE void
1912 notify_all() const noexcept
1913 { __atomic_impl::notify_all(_M_ptr); }
1914
1915 // TODO add const volatile overload
1916#endif // __glibcxx_atomic_wait
1917
1918 value_type
1919 fetch_add(value_type __i,
1920 memory_order __m = memory_order_seq_cst) const noexcept
1921 { return __atomic_impl::__fetch_add_flt(_M_ptr, __i, __m); }
1922
1923 value_type
1924 fetch_sub(value_type __i,
1925 memory_order __m = memory_order_seq_cst) const noexcept
1926 { return __atomic_impl::__fetch_sub_flt(_M_ptr, __i, __m); }
1927
1928 value_type
1929 operator+=(value_type __i) const noexcept
1930 { return __atomic_impl::__add_fetch_flt(_M_ptr, __i); }
1931
1932 value_type
1933 operator-=(value_type __i) const noexcept
1934 { return __atomic_impl::__sub_fetch_flt(_M_ptr, __i); }
1935
1936 private:
1937 _Fp* _M_ptr;
1938 };
1939
1940 // base class for atomic_ref<pointer-type>
1941 template<typename _Tp>
1942 struct __atomic_ref<_Tp*, false, false>
1943 {
1944 public:
1945 using value_type = _Tp*;
1946 using difference_type = ptrdiff_t;
1947
1948 static constexpr bool is_always_lock_free = ATOMIC_POINTER_LOCK_FREE == 2;
1949
1950 static constexpr size_t required_alignment = __alignof__(_Tp*);
1951
1952 __atomic_ref() = delete;
1953 __atomic_ref& operator=(const __atomic_ref&) = delete;
1954
1955 explicit
1956 __atomic_ref(_Tp*& __t) : _M_ptr(std::__addressof(__t))
1957 {
1958 __glibcxx_assert(((__UINTPTR_TYPE__)_M_ptr % required_alignment) == 0);
1959 }
1960
1961 __atomic_ref(const __atomic_ref&) noexcept = default;
1962
1963 _Tp*
1964 operator=(_Tp* __t) const noexcept
1965 {
1966 this->store(__t);
1967 return __t;
1968 }
1969
1970 operator _Tp*() const noexcept { return this->load(); }
1971
1972 bool
1973 is_lock_free() const noexcept
1974 {
1975 return __atomic_impl::is_lock_free<sizeof(_Tp*), required_alignment>();
1976 }
1977
1978 void
1979 store(_Tp* __t, memory_order __m = memory_order_seq_cst) const noexcept
1980 { __atomic_impl::store(_M_ptr, __t, __m); }
1981
1982 _Tp*
1983 load(memory_order __m = memory_order_seq_cst) const noexcept
1984 { return __atomic_impl::load(_M_ptr, __m); }
1985
1986 _Tp*
1987 exchange(_Tp* __desired,
1988 memory_order __m = memory_order_seq_cst) const noexcept
1989 { return __atomic_impl::exchange(_M_ptr, __desired, __m); }
1990
1991 bool
1992 compare_exchange_weak(_Tp*& __expected, _Tp* __desired,
1993 memory_order __success,
1994 memory_order __failure) const noexcept
1995 {
1996 return __atomic_impl::compare_exchange_weak<true>(
1997 _M_ptr, __expected, __desired, __success, __failure);
1998 }
1999
2000 bool
2001 compare_exchange_strong(_Tp*& __expected, _Tp* __desired,
2002 memory_order __success,
2003 memory_order __failure) const noexcept
2004 {
2005 return __atomic_impl::compare_exchange_strong<true>(
2006 _M_ptr, __expected, __desired, __success, __failure);
2007 }
2008
2009 bool
2010 compare_exchange_weak(_Tp*& __expected, _Tp* __desired,
2011 memory_order __order = memory_order_seq_cst)
2012 const noexcept
2013 {
2014 return compare_exchange_weak(__expected, __desired, __order,
2015 __cmpexch_failure_order(m: __order));
2016 }
2017
2018 bool
2019 compare_exchange_strong(_Tp*& __expected, _Tp* __desired,
2020 memory_order __order = memory_order_seq_cst)
2021 const noexcept
2022 {
2023 return compare_exchange_strong(__expected, __desired, __order,
2024 __cmpexch_failure_order(m: __order));
2025 }
2026
2027#if __glibcxx_atomic_wait
2028 _GLIBCXX_ALWAYS_INLINE void
2029 wait(_Tp* __old, memory_order __m = memory_order_seq_cst) const noexcept
2030 { __atomic_impl::wait(_M_ptr, __old, __m); }
2031
2032 // TODO add const volatile overload
2033
2034 _GLIBCXX_ALWAYS_INLINE void
2035 notify_one() const noexcept
2036 { __atomic_impl::notify_one(_M_ptr); }
2037
2038 // TODO add const volatile overload
2039
2040 _GLIBCXX_ALWAYS_INLINE void
2041 notify_all() const noexcept
2042 { __atomic_impl::notify_all(_M_ptr); }
2043
2044 // TODO add const volatile overload
2045#endif // __glibcxx_atomic_wait
2046
2047 _GLIBCXX_ALWAYS_INLINE value_type
2048 fetch_add(difference_type __d,
2049 memory_order __m = memory_order_seq_cst) const noexcept
2050 { return __atomic_impl::fetch_add(_M_ptr, _S_type_size(__d), __m); }
2051
2052 _GLIBCXX_ALWAYS_INLINE value_type
2053 fetch_sub(difference_type __d,
2054 memory_order __m = memory_order_seq_cst) const noexcept
2055 { return __atomic_impl::fetch_sub(_M_ptr, _S_type_size(__d), __m); }
2056
2057 value_type
2058 operator++(int) const noexcept
2059 { return fetch_add(d: 1); }
2060
2061 value_type
2062 operator--(int) const noexcept
2063 { return fetch_sub(d: 1); }
2064
2065 value_type
2066 operator++() const noexcept
2067 {
2068 return __atomic_impl::__add_fetch(_M_ptr, _S_type_size(d: 1));
2069 }
2070
2071 value_type
2072 operator--() const noexcept
2073 {
2074 return __atomic_impl::__sub_fetch(_M_ptr, _S_type_size(d: 1));
2075 }
2076
2077 value_type
2078 operator+=(difference_type __d) const noexcept
2079 {
2080 return __atomic_impl::__add_fetch(_M_ptr, _S_type_size(__d));
2081 }
2082
2083 value_type
2084 operator-=(difference_type __d) const noexcept
2085 {
2086 return __atomic_impl::__sub_fetch(_M_ptr, _S_type_size(__d));
2087 }
2088
2089 private:
2090 static constexpr ptrdiff_t
2091 _S_type_size(ptrdiff_t __d) noexcept
2092 {
2093 static_assert(is_object_v<_Tp>);
2094 return __d * sizeof(_Tp);
2095 }
2096
2097 _Tp** _M_ptr;
2098 };
2099#endif // C++2a
2100
2101 /// @endcond
2102
2103 /// @} group atomics
2104
2105_GLIBCXX_END_NAMESPACE_VERSION
2106} // namespace std
2107
2108#endif
2109