1// -*- C++ -*- header.
2
3// Copyright (C) 2008-2026 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25/** @file bits/atomic_base.h
26 * This is an internal header file, included by other library headers.
27 * Do not attempt to use it directly. @headername{atomic}
28 */
29
30#ifndef _GLIBCXX_ATOMIC_BASE_H
31#define _GLIBCXX_ATOMIC_BASE_H 1
32
33#ifdef _GLIBCXX_SYSHDR
34#pragma GCC system_header
35#endif
36
37#include <bits/c++config.h>
38#include <new> // For placement new
39#include <bits/atomic_lockfree_defines.h>
40#include <bits/move.h>
41
42#if __cplusplus > 201703L && _GLIBCXX_HOSTED
43#include <bits/atomic_wait.h>
44#endif
45
46#ifndef _GLIBCXX_ALWAYS_INLINE
47#define _GLIBCXX_ALWAYS_INLINE inline __attribute__((__always_inline__))
48#endif
49
50#include <bits/version.h>
51
52namespace std _GLIBCXX_VISIBILITY(default)
53{
54_GLIBCXX_BEGIN_NAMESPACE_VERSION
55
56 /**
57 * @defgroup atomics Atomics
58 *
59 * Components for performing atomic operations.
60 * @{
61 */
62
63 /// Enumeration for memory_order
64#if __cplusplus > 201703L
65 enum class memory_order : int
66 {
67 relaxed,
68 consume,
69 acquire,
70 release,
71 acq_rel,
72 seq_cst
73 };
74
75 inline constexpr memory_order memory_order_relaxed = memory_order::relaxed;
76 inline constexpr memory_order memory_order_consume = memory_order::consume;
77 inline constexpr memory_order memory_order_acquire = memory_order::acquire;
78 inline constexpr memory_order memory_order_release = memory_order::release;
79 inline constexpr memory_order memory_order_acq_rel = memory_order::acq_rel;
80 inline constexpr memory_order memory_order_seq_cst = memory_order::seq_cst;
81#else
82 enum memory_order : int
83 {
84 memory_order_relaxed,
85 memory_order_consume,
86 memory_order_acquire,
87 memory_order_release,
88 memory_order_acq_rel,
89 memory_order_seq_cst
90 };
91#endif
92
93 /// @cond undocumented
94 enum __memory_order_modifier
95 {
96 __memory_order_mask = 0x0ffff,
97 __memory_order_modifier_mask = 0xffff0000,
98 __memory_order_hle_acquire = 0x10000,
99 __memory_order_hle_release = 0x20000
100 };
101 /// @endcond
102
103 constexpr memory_order
104 operator|(memory_order __m, __memory_order_modifier __mod) noexcept
105 {
106 return memory_order(int(__m) | int(__mod));
107 }
108
109 constexpr memory_order
110 operator&(memory_order __m, __memory_order_modifier __mod) noexcept
111 {
112 return memory_order(int(__m) & int(__mod));
113 }
114
115 /// @cond undocumented
116
117 // Drop release ordering as per [atomics.types.operations.req]/21
118 constexpr memory_order
119 __cmpexch_failure_order2(memory_order __m) noexcept
120 {
121 return __m == memory_order_acq_rel ? memory_order_acquire
122 : __m == memory_order_release ? memory_order_relaxed : __m;
123 }
124
125 constexpr memory_order
126 __cmpexch_failure_order(memory_order __m) noexcept
127 {
128 return memory_order(__cmpexch_failure_order2(m: __m & __memory_order_mask)
129 | __memory_order_modifier(__m & __memory_order_modifier_mask));
130 }
131
132 constexpr bool
133 __is_valid_cmpexch_failure_order(memory_order __m) noexcept
134 {
135 return (__m & __memory_order_mask) != memory_order_release
136 && (__m & __memory_order_mask) != memory_order_acq_rel;
137 }
138
139 // Base types for atomics.
140 template<typename _IntTp>
141 struct __atomic_base;
142
143 /// @endcond
144
145 _GLIBCXX_ALWAYS_INLINE void
146 atomic_thread_fence(memory_order __m) noexcept
147 { __atomic_thread_fence(int(__m)); }
148
149 _GLIBCXX_ALWAYS_INLINE void
150 atomic_signal_fence(memory_order __m) noexcept
151 { __atomic_signal_fence(int(__m)); }
152
153 /// kill_dependency
154 template<typename _Tp>
155 inline _Tp
156 kill_dependency(_Tp __y) noexcept
157 {
158 _Tp __ret(__y);
159 return __ret;
160 }
161
162/// @cond undocumented
163#if __glibcxx_atomic_value_initialization
164# define _GLIBCXX20_INIT(I) = I
165#else
166# define _GLIBCXX20_INIT(I)
167#endif
168/// @endcond
169
170#define ATOMIC_VAR_INIT(_VI) { _VI }
171
172 template<typename _Tp>
173 struct atomic;
174
175 template<typename _Tp>
176 struct atomic<_Tp*>;
177
178 /* The target's "set" value for test-and-set may not be exactly 1. */
179#if __GCC_ATOMIC_TEST_AND_SET_TRUEVAL == 1
180 typedef bool __atomic_flag_data_type;
181#else
182 typedef unsigned char __atomic_flag_data_type;
183#endif
184
185 /// @cond undocumented
186
187 /*
188 * Base type for atomic_flag.
189 *
190 * Base type is POD with data, allowing atomic_flag to derive from
191 * it and meet the standard layout type requirement. In addition to
192 * compatibility with a C interface, this allows different
193 * implementations of atomic_flag to use the same atomic operation
194 * functions, via a standard conversion to the __atomic_flag_base
195 * argument.
196 */
197 _GLIBCXX_BEGIN_EXTERN_C
198
199 struct __atomic_flag_base
200 {
201 __atomic_flag_data_type _M_i _GLIBCXX20_INIT({});
202 };
203
204 _GLIBCXX_END_EXTERN_C
205
206 /// @endcond
207
208#define ATOMIC_FLAG_INIT { 0 }
209
210 /// atomic_flag
211 struct atomic_flag : public __atomic_flag_base
212 {
213 atomic_flag() noexcept = default;
214 ~atomic_flag() noexcept = default;
215 atomic_flag(const atomic_flag&) = delete;
216 atomic_flag& operator=(const atomic_flag&) = delete;
217 atomic_flag& operator=(const atomic_flag&) volatile = delete;
218
219 // Conversion to ATOMIC_FLAG_INIT.
220 constexpr atomic_flag(bool __i) noexcept
221 : __atomic_flag_base{ ._M_i: _S_init(__i) }
222 { }
223
224 _GLIBCXX_ALWAYS_INLINE bool
225 test_and_set(memory_order __m = memory_order_seq_cst) noexcept
226 {
227 return __atomic_test_and_set (&_M_i, int(__m));
228 }
229
230 _GLIBCXX_ALWAYS_INLINE bool
231 test_and_set(memory_order __m = memory_order_seq_cst) volatile noexcept
232 {
233 return __atomic_test_and_set (&_M_i, int(__m));
234 }
235
236#ifdef __glibcxx_atomic_flag_test // C++ >= 20
237 _GLIBCXX_ALWAYS_INLINE bool
238 test(memory_order __m = memory_order_seq_cst) const noexcept
239 {
240 __atomic_flag_data_type __v;
241 __atomic_load(&_M_i, &__v, int(__m));
242 return __v == __GCC_ATOMIC_TEST_AND_SET_TRUEVAL;
243 }
244
245 _GLIBCXX_ALWAYS_INLINE bool
246 test(memory_order __m = memory_order_seq_cst) const volatile noexcept
247 {
248 __atomic_flag_data_type __v;
249 __atomic_load(&_M_i, &__v, int(__m));
250 return __v == __GCC_ATOMIC_TEST_AND_SET_TRUEVAL;
251 }
252#endif
253
254#if __glibcxx_atomic_wait // C++ >= 20 && (linux_futex || gthread)
255 _GLIBCXX_ALWAYS_INLINE void
256 wait(bool __old,
257 memory_order __m = memory_order_seq_cst) const noexcept
258 {
259 const __atomic_flag_data_type __v
260 = __old ? __GCC_ATOMIC_TEST_AND_SET_TRUEVAL : 0;
261
262 std::__atomic_wait_address_v(addr: &_M_i, old: __v,
263 vfn: [__m, this] { return __atomic_load_n(&_M_i, int(__m)); });
264 }
265
266 // TODO add const volatile overload
267
268 _GLIBCXX_ALWAYS_INLINE void
269 notify_one() noexcept
270 { std::__atomic_notify_address(addr: &_M_i, all: false); }
271
272 // TODO add const volatile overload
273
274 _GLIBCXX_ALWAYS_INLINE void
275 notify_all() noexcept
276 { std::__atomic_notify_address(addr: &_M_i, all: true); }
277
278 // TODO add const volatile overload
279#endif // __glibcxx_atomic_wait
280
281 _GLIBCXX_ALWAYS_INLINE void
282 clear(memory_order __m = memory_order_seq_cst) noexcept
283 {
284 memory_order __b __attribute__ ((__unused__))
285 = __m & __memory_order_mask;
286 __glibcxx_assert(__b != memory_order_consume);
287 __glibcxx_assert(__b != memory_order_acquire);
288 __glibcxx_assert(__b != memory_order_acq_rel);
289
290 __atomic_clear (&_M_i, int(__m));
291 }
292
293 _GLIBCXX_ALWAYS_INLINE void
294 clear(memory_order __m = memory_order_seq_cst) volatile noexcept
295 {
296 memory_order __b __attribute__ ((__unused__))
297 = __m & __memory_order_mask;
298 __glibcxx_assert(__b != memory_order_consume);
299 __glibcxx_assert(__b != memory_order_acquire);
300 __glibcxx_assert(__b != memory_order_acq_rel);
301
302 __atomic_clear (&_M_i, int(__m));
303 }
304
305 private:
306 static constexpr __atomic_flag_data_type
307 _S_init(bool __i)
308 { return __i ? __GCC_ATOMIC_TEST_AND_SET_TRUEVAL : 0; }
309 };
310
311 /// @cond undocumented
312
313 /// Base class for atomic integrals.
314 //
315 // For each of the integral types, define atomic_[integral type] struct
316 //
317 // atomic_bool bool
318 // atomic_char char
319 // atomic_schar signed char
320 // atomic_uchar unsigned char
321 // atomic_short short
322 // atomic_ushort unsigned short
323 // atomic_int int
324 // atomic_uint unsigned int
325 // atomic_long long
326 // atomic_ulong unsigned long
327 // atomic_llong long long
328 // atomic_ullong unsigned long long
329 // atomic_char8_t char8_t
330 // atomic_char16_t char16_t
331 // atomic_char32_t char32_t
332 // atomic_wchar_t wchar_t
333 //
334 // NB: Assuming _ITp is an integral scalar type that is 1, 2, 4, or
335 // 8 bytes, since that is what GCC built-in functions for atomic
336 // memory access expect.
337
338 namespace __atomic_impl
339 {
340 template<typename _Tp>
341 using _Val = typename remove_volatile<_Tp>::type;
342
343#if __glibcxx_atomic_min_max
344 template<typename _Tp>
345 _Tp
346 __fetch_min(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept;
347
348 template<typename _Tp>
349 _Tp
350 __fetch_max(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept;
351#endif
352 }
353
354 template<typename _ITp>
355 struct __atomic_base
356 {
357 using value_type = _ITp;
358 using difference_type = value_type;
359
360 private:
361 typedef _ITp __int_type;
362
363 static constexpr int _S_alignment =
364 sizeof(_ITp) > alignof(_ITp) ? sizeof(_ITp) : alignof(_ITp);
365
366 alignas(_S_alignment) __int_type _M_i _GLIBCXX20_INIT(0);
367
368 public:
369 __atomic_base() noexcept = default;
370 ~__atomic_base() noexcept = default;
371 __atomic_base(const __atomic_base&) = delete;
372 __atomic_base& operator=(const __atomic_base&) = delete;
373 __atomic_base& operator=(const __atomic_base&) volatile = delete;
374
375 constexpr __atomic_base(__int_type __i) noexcept : _M_i (__i) { }
376
377 operator __int_type() const noexcept
378 { return load(); }
379
380 operator __int_type() const volatile noexcept
381 { return load(); }
382
383 __int_type
384 operator=(__int_type __i) noexcept
385 {
386 store(__i);
387 return __i;
388 }
389
390 __int_type
391 operator=(__int_type __i) volatile noexcept
392 {
393 store(__i);
394 return __i;
395 }
396
397 __int_type
398 operator++(int) noexcept
399 { return fetch_add(1); }
400
401 __int_type
402 operator++(int) volatile noexcept
403 { return fetch_add(1); }
404
405 __int_type
406 operator--(int) noexcept
407 { return fetch_sub(1); }
408
409 __int_type
410 operator--(int) volatile noexcept
411 { return fetch_sub(1); }
412
413 __int_type
414 operator++() noexcept
415 { return __atomic_add_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
416
417 __int_type
418 operator++() volatile noexcept
419 { return __atomic_add_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
420
421 __int_type
422 operator--() noexcept
423 { return __atomic_sub_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
424
425 __int_type
426 operator--() volatile noexcept
427 { return __atomic_sub_fetch(&_M_i, 1, int(memory_order_seq_cst)); }
428
429 __int_type
430 operator+=(__int_type __i) noexcept
431 { return __atomic_add_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
432
433 __int_type
434 operator+=(__int_type __i) volatile noexcept
435 { return __atomic_add_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
436
437 __int_type
438 operator-=(__int_type __i) noexcept
439 { return __atomic_sub_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
440
441 __int_type
442 operator-=(__int_type __i) volatile noexcept
443 { return __atomic_sub_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
444
445 __int_type
446 operator&=(__int_type __i) noexcept
447 { return __atomic_and_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
448
449 __int_type
450 operator&=(__int_type __i) volatile noexcept
451 { return __atomic_and_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
452
453 __int_type
454 operator|=(__int_type __i) noexcept
455 { return __atomic_or_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
456
457 __int_type
458 operator|=(__int_type __i) volatile noexcept
459 { return __atomic_or_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
460
461 __int_type
462 operator^=(__int_type __i) noexcept
463 { return __atomic_xor_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
464
465 __int_type
466 operator^=(__int_type __i) volatile noexcept
467 { return __atomic_xor_fetch(&_M_i, __i, int(memory_order_seq_cst)); }
468
469 bool
470 is_lock_free() const noexcept
471 {
472 // Use a fake, minimally aligned pointer.
473 return __atomic_is_lock_free(sizeof(_M_i),
474 reinterpret_cast<void *>(-_S_alignment));
475 }
476
477 bool
478 is_lock_free() const volatile noexcept
479 {
480 // Use a fake, minimally aligned pointer.
481 return __atomic_is_lock_free(sizeof(_M_i),
482 reinterpret_cast<void *>(-_S_alignment));
483 }
484
485 _GLIBCXX_ALWAYS_INLINE void
486 store(__int_type __i, memory_order __m = memory_order_seq_cst) noexcept
487 {
488 memory_order __b __attribute__ ((__unused__))
489 = __m & __memory_order_mask;
490 __glibcxx_assert(__b != memory_order_acquire);
491 __glibcxx_assert(__b != memory_order_acq_rel);
492 __glibcxx_assert(__b != memory_order_consume);
493
494 __atomic_store_n(&_M_i, __i, int(__m));
495 }
496
497 _GLIBCXX_ALWAYS_INLINE void
498 store(__int_type __i,
499 memory_order __m = memory_order_seq_cst) volatile noexcept
500 {
501 memory_order __b __attribute__ ((__unused__))
502 = __m & __memory_order_mask;
503 __glibcxx_assert(__b != memory_order_acquire);
504 __glibcxx_assert(__b != memory_order_acq_rel);
505 __glibcxx_assert(__b != memory_order_consume);
506
507 __atomic_store_n(&_M_i, __i, int(__m));
508 }
509
510 _GLIBCXX_ALWAYS_INLINE __int_type
511 load(memory_order __m = memory_order_seq_cst) const noexcept
512 {
513 memory_order __b __attribute__ ((__unused__))
514 = __m & __memory_order_mask;
515 __glibcxx_assert(__b != memory_order_release);
516 __glibcxx_assert(__b != memory_order_acq_rel);
517
518 return __atomic_load_n(&_M_i, int(__m));
519 }
520
521 _GLIBCXX_ALWAYS_INLINE __int_type
522 load(memory_order __m = memory_order_seq_cst) const volatile noexcept
523 {
524 memory_order __b __attribute__ ((__unused__))
525 = __m & __memory_order_mask;
526 __glibcxx_assert(__b != memory_order_release);
527 __glibcxx_assert(__b != memory_order_acq_rel);
528
529 return __atomic_load_n(&_M_i, int(__m));
530 }
531
532 _GLIBCXX_ALWAYS_INLINE __int_type
533 exchange(__int_type __i,
534 memory_order __m = memory_order_seq_cst) noexcept
535 {
536 return __atomic_exchange_n(&_M_i, __i, int(__m));
537 }
538
539
540 _GLIBCXX_ALWAYS_INLINE __int_type
541 exchange(__int_type __i,
542 memory_order __m = memory_order_seq_cst) volatile noexcept
543 {
544 return __atomic_exchange_n(&_M_i, __i, int(__m));
545 }
546
547 _GLIBCXX_ALWAYS_INLINE bool
548 compare_exchange_weak(__int_type& __i1, __int_type __i2,
549 memory_order __m1, memory_order __m2) noexcept
550 {
551 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
552
553 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1,
554 int(__m1), int(__m2));
555 }
556
557 _GLIBCXX_ALWAYS_INLINE bool
558 compare_exchange_weak(__int_type& __i1, __int_type __i2,
559 memory_order __m1,
560 memory_order __m2) volatile noexcept
561 {
562 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
563
564 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1,
565 int(__m1), int(__m2));
566 }
567
568 _GLIBCXX_ALWAYS_INLINE bool
569 compare_exchange_weak(__int_type& __i1, __int_type __i2,
570 memory_order __m = memory_order_seq_cst) noexcept
571 {
572 return compare_exchange_weak(__i1, __i2, __m,
573 __cmpexch_failure_order(__m));
574 }
575
576 _GLIBCXX_ALWAYS_INLINE bool
577 compare_exchange_weak(__int_type& __i1, __int_type __i2,
578 memory_order __m = memory_order_seq_cst) volatile noexcept
579 {
580 return compare_exchange_weak(__i1, __i2, __m,
581 __cmpexch_failure_order(__m));
582 }
583
584 _GLIBCXX_ALWAYS_INLINE bool
585 compare_exchange_strong(__int_type& __i1, __int_type __i2,
586 memory_order __m1, memory_order __m2) noexcept
587 {
588 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
589
590 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0,
591 int(__m1), int(__m2));
592 }
593
594 _GLIBCXX_ALWAYS_INLINE bool
595 compare_exchange_strong(__int_type& __i1, __int_type __i2,
596 memory_order __m1,
597 memory_order __m2) volatile noexcept
598 {
599 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
600
601 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0,
602 int(__m1), int(__m2));
603 }
604
605 _GLIBCXX_ALWAYS_INLINE bool
606 compare_exchange_strong(__int_type& __i1, __int_type __i2,
607 memory_order __m = memory_order_seq_cst) noexcept
608 {
609 return compare_exchange_strong(__i1, __i2, __m,
610 __cmpexch_failure_order(__m));
611 }
612
613 _GLIBCXX_ALWAYS_INLINE bool
614 compare_exchange_strong(__int_type& __i1, __int_type __i2,
615 memory_order __m = memory_order_seq_cst) volatile noexcept
616 {
617 return compare_exchange_strong(__i1, __i2, __m,
618 __cmpexch_failure_order(__m));
619 }
620
621#if __glibcxx_atomic_wait
622 _GLIBCXX_ALWAYS_INLINE void
623 wait(__int_type __old,
624 memory_order __m = memory_order_seq_cst) const noexcept
625 {
626 std::__atomic_wait_address_v(&_M_i, __old,
627 [__m, this] { return this->load(__m); });
628 }
629
630 // TODO add const volatile overload
631
632 _GLIBCXX_ALWAYS_INLINE void
633 notify_one() noexcept
634 { std::__atomic_notify_address(&_M_i, false); }
635
636 // TODO add const volatile overload
637
638 _GLIBCXX_ALWAYS_INLINE void
639 notify_all() noexcept
640 { std::__atomic_notify_address(&_M_i, true); }
641
642 // TODO add const volatile overload
643#endif // __glibcxx_atomic_wait
644
645 _GLIBCXX_ALWAYS_INLINE __int_type
646 fetch_add(__int_type __i,
647 memory_order __m = memory_order_seq_cst) noexcept
648 { return __atomic_fetch_add(&_M_i, __i, int(__m)); }
649
650 _GLIBCXX_ALWAYS_INLINE __int_type
651 fetch_add(__int_type __i,
652 memory_order __m = memory_order_seq_cst) volatile noexcept
653 { return __atomic_fetch_add(&_M_i, __i, int(__m)); }
654
655 _GLIBCXX_ALWAYS_INLINE __int_type
656 fetch_sub(__int_type __i,
657 memory_order __m = memory_order_seq_cst) noexcept
658 { return __atomic_fetch_sub(&_M_i, __i, int(__m)); }
659
660 _GLIBCXX_ALWAYS_INLINE __int_type
661 fetch_sub(__int_type __i,
662 memory_order __m = memory_order_seq_cst) volatile noexcept
663 { return __atomic_fetch_sub(&_M_i, __i, int(__m)); }
664
665 _GLIBCXX_ALWAYS_INLINE __int_type
666 fetch_and(__int_type __i,
667 memory_order __m = memory_order_seq_cst) noexcept
668 { return __atomic_fetch_and(&_M_i, __i, int(__m)); }
669
670 _GLIBCXX_ALWAYS_INLINE __int_type
671 fetch_and(__int_type __i,
672 memory_order __m = memory_order_seq_cst) volatile noexcept
673 { return __atomic_fetch_and(&_M_i, __i, int(__m)); }
674
675 _GLIBCXX_ALWAYS_INLINE __int_type
676 fetch_or(__int_type __i,
677 memory_order __m = memory_order_seq_cst) noexcept
678 { return __atomic_fetch_or(&_M_i, __i, int(__m)); }
679
680 _GLIBCXX_ALWAYS_INLINE __int_type
681 fetch_or(__int_type __i,
682 memory_order __m = memory_order_seq_cst) volatile noexcept
683 { return __atomic_fetch_or(&_M_i, __i, int(__m)); }
684
685 _GLIBCXX_ALWAYS_INLINE __int_type
686 fetch_xor(__int_type __i,
687 memory_order __m = memory_order_seq_cst) noexcept
688 { return __atomic_fetch_xor(&_M_i, __i, int(__m)); }
689
690 _GLIBCXX_ALWAYS_INLINE __int_type
691 fetch_xor(__int_type __i,
692 memory_order __m = memory_order_seq_cst) volatile noexcept
693 { return __atomic_fetch_xor(&_M_i, __i, int(__m)); }
694
695#if __glibcxx_atomic_min_max
696 _GLIBCXX_ALWAYS_INLINE __int_type
697 fetch_min(__int_type __i,
698 memory_order __m = memory_order_seq_cst) noexcept
699 { return __atomic_impl::__fetch_min(&_M_i, __i, __m); }
700
701 _GLIBCXX_ALWAYS_INLINE __int_type
702 fetch_min(__int_type __i,
703 memory_order __m = memory_order_seq_cst) volatile noexcept
704 { return __atomic_impl::__fetch_min(&_M_i, __i, __m); }
705
706 _GLIBCXX_ALWAYS_INLINE __int_type
707 fetch_max(__int_type __i,
708 memory_order __m = memory_order_seq_cst) noexcept
709 { return __atomic_impl::__fetch_max(&_M_i, __i, __m); }
710
711 _GLIBCXX_ALWAYS_INLINE __int_type
712 fetch_max(__int_type __i,
713 memory_order __m = memory_order_seq_cst) volatile noexcept
714 { return __atomic_impl::__fetch_max(&_M_i, __i, __m); }
715#endif
716 };
717
718
719 /// Partial specialization for pointer types.
720 template<typename _PTp>
721 struct __atomic_base<_PTp*>
722 {
723 private:
724 typedef _PTp* __pointer_type;
725
726 __pointer_type _M_p _GLIBCXX20_INIT(nullptr);
727
728 static constexpr ptrdiff_t
729 _S_type_size(ptrdiff_t __d)
730 { return __d * sizeof(_PTp); }
731
732 public:
733 __atomic_base() noexcept = default;
734 ~__atomic_base() noexcept = default;
735 __atomic_base(const __atomic_base&) = delete;
736 __atomic_base& operator=(const __atomic_base&) = delete;
737 __atomic_base& operator=(const __atomic_base&) volatile = delete;
738
739 // Requires __pointer_type convertible to _M_p.
740 constexpr __atomic_base(__pointer_type __p) noexcept : _M_p (__p) { }
741
742 operator __pointer_type() const noexcept
743 { return load(); }
744
745 operator __pointer_type() const volatile noexcept
746 { return load(); }
747
748 __pointer_type
749 operator=(__pointer_type __p) noexcept
750 {
751 store(__p);
752 return __p;
753 }
754
755 __pointer_type
756 operator=(__pointer_type __p) volatile noexcept
757 {
758 store(__p);
759 return __p;
760 }
761
762 __pointer_type
763 operator++(int) noexcept
764 { return fetch_add(1); }
765
766 __pointer_type
767 operator++(int) volatile noexcept
768 { return fetch_add(1); }
769
770 __pointer_type
771 operator--(int) noexcept
772 { return fetch_sub(1); }
773
774 __pointer_type
775 operator--(int) volatile noexcept
776 { return fetch_sub(1); }
777
778 __pointer_type
779 operator++() noexcept
780 { return __atomic_add_fetch(&_M_p, _S_type_size(d: 1),
781 int(memory_order_seq_cst)); }
782
783 __pointer_type
784 operator++() volatile noexcept
785 { return __atomic_add_fetch(&_M_p, _S_type_size(d: 1),
786 int(memory_order_seq_cst)); }
787
788 __pointer_type
789 operator--() noexcept
790 { return __atomic_sub_fetch(&_M_p, _S_type_size(d: 1),
791 int(memory_order_seq_cst)); }
792
793 __pointer_type
794 operator--() volatile noexcept
795 { return __atomic_sub_fetch(&_M_p, _S_type_size(d: 1),
796 int(memory_order_seq_cst)); }
797
798 __pointer_type
799 operator+=(ptrdiff_t __d) noexcept
800 { return __atomic_add_fetch(&_M_p, _S_type_size(__d),
801 int(memory_order_seq_cst)); }
802
803 __pointer_type
804 operator+=(ptrdiff_t __d) volatile noexcept
805 { return __atomic_add_fetch(&_M_p, _S_type_size(__d),
806 int(memory_order_seq_cst)); }
807
808 __pointer_type
809 operator-=(ptrdiff_t __d) noexcept
810 { return __atomic_sub_fetch(&_M_p, _S_type_size(__d),
811 int(memory_order_seq_cst)); }
812
813 __pointer_type
814 operator-=(ptrdiff_t __d) volatile noexcept
815 { return __atomic_sub_fetch(&_M_p, _S_type_size(__d),
816 int(memory_order_seq_cst)); }
817
818 bool
819 is_lock_free() const noexcept
820 {
821 // Produce a fake, minimally aligned pointer.
822 return __atomic_is_lock_free(sizeof(_M_p),
823 reinterpret_cast<void *>(-__alignof(_M_p)));
824 }
825
826 bool
827 is_lock_free() const volatile noexcept
828 {
829 // Produce a fake, minimally aligned pointer.
830 return __atomic_is_lock_free(sizeof(_M_p),
831 reinterpret_cast<void *>(-__alignof(_M_p)));
832 }
833
834 _GLIBCXX_ALWAYS_INLINE void
835 store(__pointer_type __p,
836 memory_order __m = memory_order_seq_cst) noexcept
837 {
838 memory_order __b __attribute__ ((__unused__))
839 = __m & __memory_order_mask;
840
841 __glibcxx_assert(__b != memory_order_acquire);
842 __glibcxx_assert(__b != memory_order_acq_rel);
843 __glibcxx_assert(__b != memory_order_consume);
844
845 __atomic_store_n(&_M_p, __p, int(__m));
846 }
847
848 _GLIBCXX_ALWAYS_INLINE void
849 store(__pointer_type __p,
850 memory_order __m = memory_order_seq_cst) volatile noexcept
851 {
852 memory_order __b __attribute__ ((__unused__))
853 = __m & __memory_order_mask;
854 __glibcxx_assert(__b != memory_order_acquire);
855 __glibcxx_assert(__b != memory_order_acq_rel);
856 __glibcxx_assert(__b != memory_order_consume);
857
858 __atomic_store_n(&_M_p, __p, int(__m));
859 }
860
861 _GLIBCXX_ALWAYS_INLINE __pointer_type
862 load(memory_order __m = memory_order_seq_cst) const noexcept
863 {
864 memory_order __b __attribute__ ((__unused__))
865 = __m & __memory_order_mask;
866 __glibcxx_assert(__b != memory_order_release);
867 __glibcxx_assert(__b != memory_order_acq_rel);
868
869 return __atomic_load_n(&_M_p, int(__m));
870 }
871
872 _GLIBCXX_ALWAYS_INLINE __pointer_type
873 load(memory_order __m = memory_order_seq_cst) const volatile noexcept
874 {
875 memory_order __b __attribute__ ((__unused__))
876 = __m & __memory_order_mask;
877 __glibcxx_assert(__b != memory_order_release);
878 __glibcxx_assert(__b != memory_order_acq_rel);
879
880 return __atomic_load_n(&_M_p, int(__m));
881 }
882
883 _GLIBCXX_ALWAYS_INLINE __pointer_type
884 exchange(__pointer_type __p,
885 memory_order __m = memory_order_seq_cst) noexcept
886 {
887 return __atomic_exchange_n(&_M_p, __p, int(__m));
888 }
889
890
891 _GLIBCXX_ALWAYS_INLINE __pointer_type
892 exchange(__pointer_type __p,
893 memory_order __m = memory_order_seq_cst) volatile noexcept
894 {
895 return __atomic_exchange_n(&_M_p, __p, int(__m));
896 }
897
898 _GLIBCXX_ALWAYS_INLINE bool
899 compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
900 memory_order __m1,
901 memory_order __m2) noexcept
902 {
903 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
904
905 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 1,
906 int(__m1), int(__m2));
907 }
908
909 _GLIBCXX_ALWAYS_INLINE bool
910 compare_exchange_weak(__pointer_type& __p1, __pointer_type __p2,
911 memory_order __m1,
912 memory_order __m2) volatile noexcept
913 {
914 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
915
916 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 1,
917 int(__m1), int(__m2));
918 }
919
920 _GLIBCXX_ALWAYS_INLINE bool
921 compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
922 memory_order __m1,
923 memory_order __m2) noexcept
924 {
925 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
926
927 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0,
928 int(__m1), int(__m2));
929 }
930
931 _GLIBCXX_ALWAYS_INLINE bool
932 compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
933 memory_order __m1,
934 memory_order __m2) volatile noexcept
935 {
936 __glibcxx_assert(__is_valid_cmpexch_failure_order(__m2));
937
938 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0,
939 int(__m1), int(__m2));
940 }
941
942#if __glibcxx_atomic_wait
943 _GLIBCXX_ALWAYS_INLINE void
944 wait(__pointer_type __old,
945 memory_order __m = memory_order_seq_cst) const noexcept
946 {
947 std::__atomic_wait_address_v(&_M_p, __old,
948 [__m, this]
949 { return this->load(__m); });
950 }
951
952 // TODO add const volatile overload
953
954 _GLIBCXX_ALWAYS_INLINE void
955 notify_one() const noexcept
956 { std::__atomic_notify_address(&_M_p, false); }
957
958 // TODO add const volatile overload
959
960 _GLIBCXX_ALWAYS_INLINE void
961 notify_all() const noexcept
962 { std::__atomic_notify_address(&_M_p, true); }
963
964 // TODO add const volatile overload
965#endif // __glibcxx_atomic_wait
966
967 _GLIBCXX_ALWAYS_INLINE __pointer_type
968 fetch_add(ptrdiff_t __d,
969 memory_order __m = memory_order_seq_cst) noexcept
970 { return __atomic_fetch_add(&_M_p, _S_type_size(__d), int(__m)); }
971
972 _GLIBCXX_ALWAYS_INLINE __pointer_type
973 fetch_add(ptrdiff_t __d,
974 memory_order __m = memory_order_seq_cst) volatile noexcept
975 { return __atomic_fetch_add(&_M_p, _S_type_size(__d), int(__m)); }
976
977 _GLIBCXX_ALWAYS_INLINE __pointer_type
978 fetch_sub(ptrdiff_t __d,
979 memory_order __m = memory_order_seq_cst) noexcept
980 { return __atomic_fetch_sub(&_M_p, _S_type_size(__d), int(__m)); }
981
982 _GLIBCXX_ALWAYS_INLINE __pointer_type
983 fetch_sub(ptrdiff_t __d,
984 memory_order __m = memory_order_seq_cst) volatile noexcept
985 { return __atomic_fetch_sub(&_M_p, _S_type_size(__d), int(__m)); }
986
987#if __glibcxx_atomic_min_max
988 _GLIBCXX_ALWAYS_INLINE __pointer_type
989 fetch_min(__pointer_type __p,
990 memory_order __m = memory_order_seq_cst) noexcept
991 { return __atomic_impl::__fetch_min(&_M_p, __p, __m); }
992
993 _GLIBCXX_ALWAYS_INLINE __pointer_type
994 fetch_min(__pointer_type __p,
995 memory_order __m = memory_order_seq_cst) volatile noexcept
996 { return __atomic_impl::__fetch_min(&_M_p, __p, __m); }
997
998 _GLIBCXX_ALWAYS_INLINE __pointer_type
999 fetch_max(__pointer_type __p,
1000 memory_order __m = memory_order_seq_cst) noexcept
1001 { return __atomic_impl::__fetch_max(&_M_p, __p, __m); }
1002
1003 _GLIBCXX_ALWAYS_INLINE __pointer_type
1004 fetch_max(__pointer_type __p,
1005 memory_order __m = memory_order_seq_cst) volatile noexcept
1006 { return __atomic_impl::__fetch_max(&_M_p, __p, __m); }
1007#endif
1008 };
1009
1010 namespace __atomic_impl
1011 {
1012 // Implementation details of atomic padding handling
1013
1014 template<typename _Tp>
1015 constexpr bool
1016 __maybe_has_padding()
1017 {
1018#if ! __has_builtin(__builtin_clear_padding)
1019 return false;
1020#elif __has_builtin(__has_unique_object_representations)
1021 return !__has_unique_object_representations(_Tp)
1022 && !is_same<_Tp, float>::value && !is_same<_Tp, double>::value;
1023#else
1024 return true;
1025#endif
1026 }
1027
1028#pragma GCC diagnostic push
1029#pragma GCC diagnostic ignored "-Wc++17-extensions"
1030
1031 template<typename _Tp>
1032 _GLIBCXX_ALWAYS_INLINE _GLIBCXX14_CONSTEXPR _Tp*
1033 __clear_padding(_Tp& __val) noexcept
1034 {
1035 auto* __ptr = std::__addressof(__val);
1036#if __has_builtin(__builtin_clear_padding)
1037 if constexpr (__atomic_impl::__maybe_has_padding<_Tp>())
1038 __builtin_clear_padding(__ptr);
1039#endif
1040 return __ptr;
1041 }
1042
1043 template<bool _AtomicRef = false, typename _Tp>
1044 _GLIBCXX_ALWAYS_INLINE bool
1045 __compare_exchange(_Tp& __val, _Val<_Tp>& __e, _Val<_Tp>& __i,
1046 bool __is_weak,
1047 memory_order __s, memory_order __f) noexcept
1048 {
1049 __glibcxx_assert(__is_valid_cmpexch_failure_order(__f));
1050
1051 using _Vp = _Val<_Tp>;
1052 _Tp* const __pval = std::__addressof(__val);
1053
1054 if constexpr (!__atomic_impl::__maybe_has_padding<_Vp>())
1055 {
1056 return __atomic_compare_exchange(__pval, std::__addressof(__e),
1057 std::__addressof(__i), __is_weak,
1058 int(__s), int(__f));
1059 }
1060 else if constexpr (!_AtomicRef) // std::atomic<T>
1061 {
1062 // Clear padding of the value we want to set:
1063 _Vp* const __pi = __atomic_impl::__clear_padding(__i);
1064 // Only allowed to modify __e on failure, so make a copy:
1065 _Vp __exp = __e;
1066 // Clear padding of the expected value:
1067 _Vp* const __pexp = __atomic_impl::__clear_padding(__exp);
1068
1069 // For std::atomic<T> we know that the contained value will already
1070 // have zeroed padding, so trivial memcmp semantics are OK.
1071 if (__atomic_compare_exchange(__pval, __pexp, __pi,
1072 __is_weak, int(__s), int(__f)))
1073 return true;
1074 // Value bits must be different, copy from __exp back to __e:
1075 __builtin_memcpy(std::__addressof(__e), __pexp, sizeof(_Vp));
1076 return false;
1077 }
1078 else // std::atomic_ref<T> where T has padding bits.
1079 {
1080 // Clear padding of the value we want to set:
1081 _Vp* const __pi = __atomic_impl::__clear_padding(__i);
1082
1083 // Only allowed to modify __e on failure, so make a copy:
1084 _Vp __exp = __e;
1085 // Optimistically assume that a previous store had zeroed padding
1086 // so that zeroing it in the expected value will match first time.
1087 _Vp* const __pexp = __atomic_impl::__clear_padding(__exp);
1088
1089 // compare_exchange is specified to compare value representations.
1090 // Need to check whether a failure is 'real' or just due to
1091 // differences in padding bits. This loop should run no more than
1092 // three times, because the worst case scenario is:
1093 // First CAS fails because the actual value has non-zero padding.
1094 // Second CAS fails because another thread stored the same value,
1095 // but now with padding cleared. Third CAS succeeds.
1096 // We will never need to loop a fourth time, because any value
1097 // written by another thread (whether via store, exchange or
1098 // compare_exchange) will have had its padding cleared.
1099 while (true)
1100 {
1101 // Copy of the expected value so we can clear its padding.
1102 _Vp __orig = __exp;
1103
1104 if (__atomic_compare_exchange(__pval, __pexp, __pi,
1105 __is_weak, int(__s), int(__f)))
1106 return true;
1107
1108 // Copy of the actual value so we can clear its padding.
1109 _Vp __curr = __exp;
1110
1111 // Compare value representations (i.e. ignoring padding).
1112 if (__builtin_memcmp(__atomic_impl::__clear_padding(__orig),
1113 __atomic_impl::__clear_padding(__curr),
1114 sizeof(_Vp)))
1115 {
1116 // Value representations compare unequal, real failure.
1117 __builtin_memcpy(std::__addressof(__e), __pexp,
1118 sizeof(_Vp));
1119 return false;
1120 }
1121 }
1122 }
1123 }
1124#pragma GCC diagnostic pop
1125 } // namespace __atomic_impl
1126
1127#if __cplusplus > 201703L
1128 // Implementation details of atomic_ref and atomic<floating-point>.
1129 namespace __atomic_impl
1130 {
1131 // Like _Val<T> above, but for difference_type arguments.
1132 template<typename _Tp>
1133 using _Diff = __conditional_t<is_pointer_v<_Tp>, ptrdiff_t, _Val<_Tp>>;
1134
1135 template<size_t _Size, size_t _Align>
1136 _GLIBCXX_ALWAYS_INLINE bool
1137 is_lock_free() noexcept
1138 {
1139 // Produce a fake, minimally aligned pointer.
1140 return __atomic_is_lock_free(_Size, reinterpret_cast<void *>(-_Align));
1141 }
1142
1143 template<typename _Tp>
1144 _GLIBCXX_ALWAYS_INLINE void
1145 store(_Tp* __ptr, _Val<_Tp> __t, memory_order __m) noexcept
1146 {
1147 __atomic_store(__ptr, __atomic_impl::__clear_padding(__t), int(__m));
1148 }
1149
1150 template<typename _Tp>
1151 _GLIBCXX_ALWAYS_INLINE _Val<_Tp>
1152 load(const _Tp* __ptr, memory_order __m) noexcept
1153 {
1154 alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
1155 auto* __dest = reinterpret_cast<_Val<_Tp>*>(__buf);
1156 __atomic_load(__ptr, __dest, int(__m));
1157 return *__dest;
1158 }
1159
1160 template<typename _Tp>
1161 _GLIBCXX_ALWAYS_INLINE _Val<_Tp>
1162 exchange(_Tp* __ptr, _Val<_Tp> __desired, memory_order __m) noexcept
1163 {
1164 alignas(_Tp) unsigned char __buf[sizeof(_Tp)];
1165 auto* __dest = reinterpret_cast<_Val<_Tp>*>(__buf);
1166 __atomic_exchange(__ptr, __atomic_impl::__clear_padding(__desired),
1167 __dest, int(__m));
1168 return *__dest;
1169 }
1170
1171 template<bool _AtomicRef = false, typename _Tp>
1172 _GLIBCXX_ALWAYS_INLINE bool
1173 compare_exchange_weak(_Tp* __ptr, _Val<_Tp>& __expected,
1174 _Val<_Tp> __desired, memory_order __success,
1175 memory_order __failure) noexcept
1176 {
1177 return __atomic_impl::__compare_exchange<_AtomicRef>(
1178 *__ptr, __expected, __desired, true, __success, __failure);
1179 }
1180
1181 template<bool _AtomicRef = false, typename _Tp>
1182 _GLIBCXX_ALWAYS_INLINE bool
1183 compare_exchange_strong(_Tp* __ptr, _Val<_Tp>& __expected,
1184 _Val<_Tp> __desired, memory_order __success,
1185 memory_order __failure) noexcept
1186 {
1187 return __atomic_impl::__compare_exchange<_AtomicRef>(
1188 *__ptr, __expected, __desired, false, __success, __failure);
1189 }
1190
1191#if __glibcxx_atomic_wait
1192 template<typename _Tp>
1193 _GLIBCXX_ALWAYS_INLINE void
1194 wait(const _Tp* __ptr, _Val<_Tp> __old,
1195 memory_order __m = memory_order_seq_cst) noexcept
1196 {
1197 std::__atomic_wait_address_v(__ptr, __old,
1198 [__ptr, __m]() { return __atomic_impl::load(__ptr, __m); });
1199 }
1200
1201 // TODO add const volatile overload
1202
1203 template<typename _Tp>
1204 _GLIBCXX_ALWAYS_INLINE void
1205 notify_one(const _Tp* __ptr) noexcept
1206 { std::__atomic_notify_address(__ptr, false); }
1207
1208 // TODO add const volatile overload
1209
1210 template<typename _Tp>
1211 _GLIBCXX_ALWAYS_INLINE void
1212 notify_all(const _Tp* __ptr) noexcept
1213 { std::__atomic_notify_address(__ptr, true); }
1214
1215 // TODO add const volatile overload
1216#endif // __glibcxx_atomic_wait
1217
1218 template<typename _Tp>
1219 _GLIBCXX_ALWAYS_INLINE _Tp
1220 fetch_add(_Tp* __ptr, _Diff<_Tp> __i, memory_order __m) noexcept
1221 { return __atomic_fetch_add(__ptr, __i, int(__m)); }
1222
1223 template<typename _Tp>
1224 _GLIBCXX_ALWAYS_INLINE _Tp
1225 fetch_sub(_Tp* __ptr, _Diff<_Tp> __i, memory_order __m) noexcept
1226 { return __atomic_fetch_sub(__ptr, __i, int(__m)); }
1227
1228 template<typename _Tp>
1229 _GLIBCXX_ALWAYS_INLINE _Tp
1230 fetch_and(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1231 { return __atomic_fetch_and(__ptr, __i, int(__m)); }
1232
1233 template<typename _Tp>
1234 _GLIBCXX_ALWAYS_INLINE _Tp
1235 fetch_or(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1236 { return __atomic_fetch_or(__ptr, __i, int(__m)); }
1237
1238 template<typename _Tp>
1239 _GLIBCXX_ALWAYS_INLINE _Tp
1240 fetch_xor(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1241 { return __atomic_fetch_xor(__ptr, __i, int(__m)); }
1242
1243 template<typename _Tp>
1244 _GLIBCXX_ALWAYS_INLINE _Tp
1245 __add_fetch(_Tp* __ptr, _Diff<_Tp> __i) noexcept
1246 { return __atomic_add_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1247
1248 template<typename _Tp>
1249 _GLIBCXX_ALWAYS_INLINE _Tp
1250 __sub_fetch(_Tp* __ptr, _Diff<_Tp> __i) noexcept
1251 { return __atomic_sub_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1252
1253 template<typename _Tp>
1254 _GLIBCXX_ALWAYS_INLINE _Tp
1255 __and_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
1256 { return __atomic_and_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1257
1258 template<typename _Tp>
1259 _GLIBCXX_ALWAYS_INLINE _Tp
1260 __or_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
1261 { return __atomic_or_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1262
1263 template<typename _Tp>
1264 _GLIBCXX_ALWAYS_INLINE _Tp
1265 __xor_fetch(_Tp* __ptr, _Val<_Tp> __i) noexcept
1266 { return __atomic_xor_fetch(__ptr, __i, __ATOMIC_SEQ_CST); }
1267
1268 template<typename _Tp>
1269 concept __atomic_fetch_addable
1270 = requires (_Tp __t) { __atomic_fetch_add(&__t, __t, 0); };
1271
1272 template<typename _Tp>
1273 _Tp
1274 __fetch_add_flt(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1275 {
1276 if constexpr (__atomic_fetch_addable<_Tp>)
1277 return __atomic_fetch_add(__ptr, __i, int(__m));
1278 else
1279 {
1280 _Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
1281 _Val<_Tp> __newval = __oldval + __i;
1282 while (!compare_exchange_weak (__ptr, __oldval, __newval, __m,
1283 memory_order_relaxed))
1284 __newval = __oldval + __i;
1285 return __oldval;
1286 }
1287 }
1288
1289 template<typename _Tp>
1290 concept __atomic_fetch_subtractable
1291 = requires (_Tp __t) { __atomic_fetch_sub(&__t, __t, 0); };
1292
1293 template<typename _Tp>
1294 _Tp
1295 __fetch_sub_flt(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1296 {
1297 if constexpr (__atomic_fetch_subtractable<_Tp>)
1298 return __atomic_fetch_sub(__ptr, __i, int(__m));
1299 else
1300 {
1301 _Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
1302 _Val<_Tp> __newval = __oldval - __i;
1303 while (!compare_exchange_weak (__ptr, __oldval, __newval, __m,
1304 memory_order_relaxed))
1305 __newval = __oldval - __i;
1306 return __oldval;
1307 }
1308 }
1309
1310 template<typename _Tp>
1311 concept __atomic_add_fetchable
1312 = requires (_Tp __t) { __atomic_add_fetch(&__t, __t, 0); };
1313
1314 template<typename _Tp>
1315 _Tp
1316 __add_fetch_flt(_Tp* __ptr, _Val<_Tp> __i) noexcept
1317 {
1318 if constexpr (__atomic_add_fetchable<_Tp>)
1319 return __atomic_add_fetch(__ptr, __i, __ATOMIC_SEQ_CST);
1320 else
1321 {
1322 _Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
1323 _Val<_Tp> __newval = __oldval + __i;
1324 while (!compare_exchange_weak (__ptr, __oldval, __newval,
1325 memory_order_seq_cst,
1326 memory_order_relaxed))
1327 __newval = __oldval + __i;
1328 return __newval;
1329 }
1330 }
1331
1332 template<typename _Tp>
1333 concept __atomic_sub_fetchable
1334 = requires (_Tp __t) { __atomic_sub_fetch(&__t, __t, 0); };
1335
1336 template<typename _Tp>
1337 _Tp
1338 __sub_fetch_flt(_Tp* __ptr, _Val<_Tp> __i) noexcept
1339 {
1340 if constexpr (__atomic_sub_fetchable<_Tp>)
1341 return __atomic_sub_fetch(__ptr, __i, __ATOMIC_SEQ_CST);
1342 else
1343 {
1344 _Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
1345 _Val<_Tp> __newval = __oldval - __i;
1346 while (!compare_exchange_weak (__ptr, __oldval, __newval,
1347 memory_order_seq_cst,
1348 memory_order_relaxed))
1349 __newval = __oldval - __i;
1350 return __newval;
1351 }
1352 }
1353
1354#if __glibcxx_atomic_min_max
1355 template<typename _Tp>
1356 concept __atomic_fetch_minmaxable
1357 = requires (_Tp __t) {
1358 __atomic_fetch_min(&__t, __t, 0);
1359 __atomic_fetch_max(&__t, __t, 0);
1360 };
1361
1362 template<typename _Tp>
1363 _Tp
1364 __fetch_min(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1365 {
1366 if constexpr (__atomic_fetch_minmaxable<_Tp>)
1367 return __atomic_fetch_min(__ptr, __i, int(__m));
1368 else
1369 {
1370 _Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
1371 _Val<_Tp> __newval = __oldval < __i ? __oldval : __i;
1372 while (!compare_exchange_weak (__ptr, __oldval, __newval, __m,
1373 memory_order_relaxed))
1374 __newval = __oldval < __i ? __oldval : __i;
1375 return __oldval;
1376 }
1377 }
1378
1379 template<typename _Tp>
1380 _Tp
1381 __fetch_max(_Tp* __ptr, _Val<_Tp> __i, memory_order __m) noexcept
1382 {
1383 if constexpr (__atomic_fetch_minmaxable<_Tp>)
1384 return __atomic_fetch_max(__ptr, __i, int(__m));
1385 else
1386 {
1387 _Val<_Tp> __oldval = load (__ptr, memory_order_relaxed);
1388 _Val<_Tp> __newval = __oldval > __i ? __oldval : __i;
1389 while (!compare_exchange_weak (__ptr, __oldval, __newval, __m,
1390 memory_order_relaxed))
1391 __newval = __oldval > __i ? __oldval : __i;
1392 return __oldval;
1393 }
1394 }
1395#endif
1396 } // namespace __atomic_impl
1397
1398 // base class for atomic<floating-point-type>
1399 template<typename _Fp>
1400 struct __atomic_float
1401 {
1402 static_assert(is_floating_point_v<_Fp>);
1403
1404 static constexpr size_t _S_alignment = __alignof__(_Fp);
1405
1406 public:
1407 using value_type = _Fp;
1408 using difference_type = value_type;
1409
1410 static constexpr bool is_always_lock_free
1411 = __atomic_always_lock_free(sizeof(_Fp), 0);
1412
1413 __atomic_float() = default;
1414
1415 constexpr
1416 __atomic_float(_Fp __t) : _M_fp(__t)
1417 {
1418 if (!std::__is_constant_evaluated())
1419 __atomic_impl::__clear_padding(_M_fp);
1420 }
1421
1422 __atomic_float(const __atomic_float&) = delete;
1423 __atomic_float& operator=(const __atomic_float&) = delete;
1424 __atomic_float& operator=(const __atomic_float&) volatile = delete;
1425
1426 _Fp
1427 operator=(_Fp __t) volatile noexcept
1428 {
1429 this->store(__t);
1430 return __t;
1431 }
1432
1433 _Fp
1434 operator=(_Fp __t) noexcept
1435 {
1436 this->store(__t);
1437 return __t;
1438 }
1439
1440 bool
1441 is_lock_free() const volatile noexcept
1442 { return __atomic_impl::is_lock_free<sizeof(_Fp), _S_alignment>(); }
1443
1444 bool
1445 is_lock_free() const noexcept
1446 { return __atomic_impl::is_lock_free<sizeof(_Fp), _S_alignment>(); }
1447
1448 void
1449 store(_Fp __t, memory_order __m = memory_order_seq_cst) volatile noexcept
1450 { __atomic_impl::store(&_M_fp, __t, __m); }
1451
1452 void
1453 store(_Fp __t, memory_order __m = memory_order_seq_cst) noexcept
1454 { __atomic_impl::store(&_M_fp, __t, __m); }
1455
1456 _Fp
1457 load(memory_order __m = memory_order_seq_cst) const volatile noexcept
1458 { return __atomic_impl::load(&_M_fp, __m); }
1459
1460 _Fp
1461 load(memory_order __m = memory_order_seq_cst) const noexcept
1462 { return __atomic_impl::load(&_M_fp, __m); }
1463
1464 operator _Fp() const volatile noexcept { return this->load(); }
1465 operator _Fp() const noexcept { return this->load(); }
1466
1467 _Fp
1468 exchange(_Fp __desired,
1469 memory_order __m = memory_order_seq_cst) volatile noexcept
1470 { return __atomic_impl::exchange(&_M_fp, __desired, __m); }
1471
1472 _Fp
1473 exchange(_Fp __desired,
1474 memory_order __m = memory_order_seq_cst) noexcept
1475 { return __atomic_impl::exchange(&_M_fp, __desired, __m); }
1476
1477 bool
1478 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1479 memory_order __success,
1480 memory_order __failure) noexcept
1481 {
1482 return __atomic_impl::compare_exchange_weak(&_M_fp,
1483 __expected, __desired,
1484 __success, __failure);
1485 }
1486
1487 bool
1488 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1489 memory_order __success,
1490 memory_order __failure) volatile noexcept
1491 {
1492 return __atomic_impl::compare_exchange_weak(&_M_fp,
1493 __expected, __desired,
1494 __success, __failure);
1495 }
1496
1497 bool
1498 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1499 memory_order __success,
1500 memory_order __failure) noexcept
1501 {
1502 return __atomic_impl::compare_exchange_strong(&_M_fp,
1503 __expected, __desired,
1504 __success, __failure);
1505 }
1506
1507 bool
1508 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1509 memory_order __success,
1510 memory_order __failure) volatile noexcept
1511 {
1512 return __atomic_impl::compare_exchange_strong(&_M_fp,
1513 __expected, __desired,
1514 __success, __failure);
1515 }
1516
1517 bool
1518 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1519 memory_order __order = memory_order_seq_cst)
1520 noexcept
1521 {
1522 return compare_exchange_weak(__expected, __desired, __order,
1523 __cmpexch_failure_order(m: __order));
1524 }
1525
1526 bool
1527 compare_exchange_weak(_Fp& __expected, _Fp __desired,
1528 memory_order __order = memory_order_seq_cst)
1529 volatile noexcept
1530 {
1531 return compare_exchange_weak(__expected, __desired, __order,
1532 __cmpexch_failure_order(m: __order));
1533 }
1534
1535 bool
1536 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1537 memory_order __order = memory_order_seq_cst)
1538 noexcept
1539 {
1540 return compare_exchange_strong(__expected, __desired, __order,
1541 __cmpexch_failure_order(m: __order));
1542 }
1543
1544 bool
1545 compare_exchange_strong(_Fp& __expected, _Fp __desired,
1546 memory_order __order = memory_order_seq_cst)
1547 volatile noexcept
1548 {
1549 return compare_exchange_strong(__expected, __desired, __order,
1550 __cmpexch_failure_order(m: __order));
1551 }
1552
1553#if __glibcxx_atomic_wait
1554 _GLIBCXX_ALWAYS_INLINE void
1555 wait(_Fp __old, memory_order __m = memory_order_seq_cst) const noexcept
1556 { __atomic_impl::wait(&_M_fp, __old, __m); }
1557
1558 // TODO add const volatile overload
1559
1560 _GLIBCXX_ALWAYS_INLINE void
1561 notify_one() const noexcept
1562 { __atomic_impl::notify_one(&_M_fp); }
1563
1564 // TODO add const volatile overload
1565
1566 _GLIBCXX_ALWAYS_INLINE void
1567 notify_all() const noexcept
1568 { __atomic_impl::notify_all(&_M_fp); }
1569
1570 // TODO add const volatile overload
1571#endif // __glibcxx_atomic_wait
1572
1573 value_type
1574 fetch_add(value_type __i,
1575 memory_order __m = memory_order_seq_cst) noexcept
1576 { return __atomic_impl::__fetch_add_flt(&_M_fp, __i, __m); }
1577
1578 value_type
1579 fetch_add(value_type __i,
1580 memory_order __m = memory_order_seq_cst) volatile noexcept
1581 { return __atomic_impl::__fetch_add_flt(&_M_fp, __i, __m); }
1582
1583 value_type
1584 fetch_sub(value_type __i,
1585 memory_order __m = memory_order_seq_cst) noexcept
1586 { return __atomic_impl::__fetch_sub_flt(&_M_fp, __i, __m); }
1587
1588 value_type
1589 fetch_sub(value_type __i,
1590 memory_order __m = memory_order_seq_cst) volatile noexcept
1591 { return __atomic_impl::__fetch_sub_flt(&_M_fp, __i, __m); }
1592
1593#if __glibcxx_atomic_min_max
1594 value_type
1595 fetch_min(value_type __i,
1596 memory_order __m = memory_order_seq_cst) noexcept
1597 { return __atomic_impl::__fetch_min(&_M_fp, __i, __m); }
1598
1599 value_type
1600 fetch_min(value_type __i,
1601 memory_order __m = memory_order_seq_cst) volatile noexcept
1602 { return __atomic_impl::__fetch_min(&_M_fp, __i, __m); }
1603
1604 value_type
1605 fetch_max(value_type __i,
1606 memory_order __m = memory_order_seq_cst) noexcept
1607 { return __atomic_impl::__fetch_max(&_M_fp, __i, __m); }
1608
1609 value_type
1610 fetch_max(value_type __i,
1611 memory_order __m = memory_order_seq_cst) volatile noexcept
1612 { return __atomic_impl::__fetch_max(&_M_fp, __i, __m); }
1613#endif
1614
1615 value_type
1616 operator+=(value_type __i) noexcept
1617 { return __atomic_impl::__add_fetch_flt(&_M_fp, __i); }
1618
1619 value_type
1620 operator+=(value_type __i) volatile noexcept
1621 { return __atomic_impl::__add_fetch_flt(&_M_fp, __i); }
1622
1623 value_type
1624 operator-=(value_type __i) noexcept
1625 { return __atomic_impl::__sub_fetch_flt(&_M_fp, __i); }
1626
1627 value_type
1628 operator-=(value_type __i) volatile noexcept
1629 { return __atomic_impl::__sub_fetch_flt(&_M_fp, __i); }
1630
1631 private:
1632 alignas(_S_alignment) _Fp _M_fp _GLIBCXX20_INIT(0);
1633 };
1634#undef _GLIBCXX20_INIT
1635
1636 // __atomic_ref_base<const _Tp> provides the common APIs for const and
1637 // non-const types,
1638 // __atomic_ref_base<_Tp> inherits from __atomic_ref_base<const _Tp>,
1639 // and provides the common APIs implementing constraints in [atomic.ref].
1640 // __atomic_ref<_Tp> inherits from __atomic_ref_base<_Tp> (const or non-const)
1641 // adds type-specific mutating APIs.
1642 // atomic_ref inherits from __atomic_ref;
1643
1644 template<typename _Tp>
1645 struct __atomic_ref_base;
1646
1647 template<typename _Tp>
1648 struct __atomic_ref_base<const _Tp>
1649 {
1650 private:
1651 using _Vt = remove_cv_t<_Tp>;
1652 using _Address_return_t = __conditional_t<is_volatile_v<_Tp>,
1653 const volatile void*, const void*>;
1654
1655 static consteval bool
1656 _S_is_always_lock_free()
1657 {
1658 if constexpr (is_pointer_v<_Vt>)
1659 return ATOMIC_POINTER_LOCK_FREE == 2;
1660 else
1661 return __atomic_always_lock_free(sizeof(_Vt), 0);
1662 }
1663
1664 static consteval int
1665 _S_required_alignment()
1666 {
1667 if constexpr (is_floating_point_v<_Vt> || is_pointer_v<_Vt>)
1668 return __alignof__(_Vt);
1669 else if constexpr ((sizeof(_Vt) & (sizeof(_Vt) - 1)) || sizeof(_Vt) > 16)
1670 return alignof(_Vt);
1671 else
1672 // 1/2/4/8/16-byte types, including integral types,
1673 // must be aligned to at least their size.
1674 return (sizeof(_Vt) > alignof(_Vt)) ? sizeof(_Vt) : alignof(_Vt);
1675 }
1676
1677 public:
1678 using value_type = _Vt;
1679 static_assert(is_trivially_copyable_v<value_type>);
1680
1681 static constexpr bool is_always_lock_free = _S_is_always_lock_free();
1682 static_assert(is_always_lock_free || !is_volatile_v<_Tp>,
1683 "atomic operations on volatile T must be lock-free");
1684
1685 static constexpr size_t required_alignment = _S_required_alignment();
1686
1687 __atomic_ref_base() = delete;
1688 __atomic_ref_base& operator=(const __atomic_ref_base&) = delete;
1689
1690 explicit
1691 __atomic_ref_base(const _Tp* __ptr) noexcept
1692 : _M_ptr(const_cast<_Tp*>(__ptr))
1693 { }
1694
1695 __atomic_ref_base(const __atomic_ref_base&) noexcept = default;
1696
1697 operator value_type() const noexcept { return this->load(); }
1698
1699 bool
1700 is_lock_free() const noexcept
1701 { return __atomic_impl::is_lock_free<sizeof(_Tp), required_alignment>(); }
1702
1703 value_type
1704 load(memory_order __m = memory_order_seq_cst) const noexcept
1705 { return __atomic_impl::load(_M_ptr, __m); }
1706
1707#if __glibcxx_atomic_wait
1708 _GLIBCXX_ALWAYS_INLINE void
1709 wait(value_type __old, memory_order __m = memory_order_seq_cst) const noexcept
1710 {
1711 // TODO remove when volatile is supported
1712 static_assert(!is_volatile_v<_Tp>,
1713 "atomic waits on volatile are not supported");
1714 __atomic_impl::wait(_M_ptr, __old, __m);
1715 }
1716#endif // __glibcxx_atomic_wait
1717
1718#if __glibcxx_atomic_ref >= 202603L
1719 _GLIBCXX_ALWAYS_INLINE constexpr _Address_return_t
1720 address() const noexcept
1721 { return _M_ptr; }
1722#endif // __glibcxx_atomic_ref >= 202411L
1723
1724 protected:
1725 _Tp* _M_ptr;
1726 };
1727
1728 template<typename _Tp>
1729 struct __atomic_ref_base
1730 : __atomic_ref_base<const _Tp>
1731 {
1732 using _Address_return_t = __conditional_t<is_volatile_v<_Tp>,
1733 volatile void*, void*>;
1734
1735 public:
1736 using value_type = typename __atomic_ref_base<const _Tp>::value_type;
1737
1738 explicit
1739 __atomic_ref_base(_Tp* __ptr) noexcept
1740 : __atomic_ref_base<const _Tp>(__ptr)
1741 { }
1742
1743 value_type
1744 operator=(value_type __t) const noexcept
1745 {
1746 this->store(__t);
1747 return __t;
1748 }
1749
1750 void
1751 store(value_type __t, memory_order __m = memory_order_seq_cst) const noexcept
1752 { __atomic_impl::store(this->_M_ptr, __t, __m); }
1753
1754 value_type
1755 exchange(value_type __desired, memory_order __m = memory_order_seq_cst)
1756 const noexcept
1757 { return __atomic_impl::exchange(this->_M_ptr, __desired, __m); }
1758
1759 bool
1760 compare_exchange_weak(value_type& __expected, value_type __desired,
1761 memory_order __success,
1762 memory_order __failure) const noexcept
1763 {
1764 return __atomic_impl::compare_exchange_weak<true>(
1765 this->_M_ptr, __expected, __desired, __success, __failure);
1766 }
1767
1768 bool
1769 compare_exchange_strong(value_type& __expected, value_type __desired,
1770 memory_order __success,
1771 memory_order __failure) const noexcept
1772 {
1773 return __atomic_impl::compare_exchange_strong<true>(
1774 this->_M_ptr, __expected, __desired, __success, __failure);
1775 }
1776
1777 bool
1778 compare_exchange_weak(value_type& __expected, value_type __desired,
1779 memory_order __order = memory_order_seq_cst)
1780 const noexcept
1781 {
1782 return compare_exchange_weak(__expected, __desired, __order,
1783 __cmpexch_failure_order(m: __order));
1784 }
1785
1786 bool
1787 compare_exchange_strong(value_type& __expected, value_type __desired,
1788 memory_order __order = memory_order_seq_cst)
1789 const noexcept
1790 {
1791 return compare_exchange_strong(__expected, __desired, __order,
1792 __cmpexch_failure_order(m: __order));
1793 }
1794
1795#if __glibcxx_atomic_wait
1796 _GLIBCXX_ALWAYS_INLINE void
1797 notify_one() const noexcept
1798 {
1799 // TODO remove when volatile is supported
1800 static_assert(!is_volatile_v<_Tp>,
1801 "atomic waits on volatile are not supported");
1802 __atomic_impl::notify_one(this->_M_ptr);
1803 }
1804
1805 _GLIBCXX_ALWAYS_INLINE void
1806 notify_all() const noexcept
1807 {
1808 // TODO remove when volatile is supported
1809 static_assert(!is_volatile_v<_Tp>,
1810 "atomic waits on volatile are not supported");
1811 __atomic_impl::notify_all(this->_M_ptr);
1812 }
1813#endif // __glibcxx_atomic_wait
1814
1815#if __glibcxx_atomic_ref >= 202603L
1816 _GLIBCXX_ALWAYS_INLINE constexpr _Address_return_t
1817 address() const noexcept
1818 { return this->_M_ptr; }
1819#endif // __glibcxx_atomic_ref >= 202411L
1820 };
1821
1822 template<typename _Tp,
1823 bool = is_integral_v<_Tp> && !is_same_v<remove_cv_t<_Tp>, bool>,
1824 bool = is_floating_point_v<_Tp>,
1825 bool = is_pointer_v<_Tp>>
1826 struct __atomic_ref;
1827
1828 // base class for non-integral, non-floating-point, non-pointer types
1829 template<typename _Tp>
1830 struct __atomic_ref<_Tp, false, false, false>
1831 : __atomic_ref_base<_Tp>
1832 {
1833 using __atomic_ref_base<_Tp>::__atomic_ref_base;
1834 using __atomic_ref_base<_Tp>::operator=;
1835 };
1836
1837 template<typename _Tp>
1838 struct __atomic_ref<const _Tp, false, false, false>
1839 : __atomic_ref_base<const _Tp>
1840 {
1841 using __atomic_ref_base<const _Tp>::__atomic_ref_base;
1842 };
1843
1844 // base class for atomic_ref<integral-type>
1845 template<typename _Tp>
1846 struct __atomic_ref<_Tp, true, false, false>
1847 : __atomic_ref_base<_Tp>
1848 {
1849 using value_type = typename __atomic_ref_base<_Tp>::value_type;
1850 using difference_type = value_type;
1851
1852 using __atomic_ref_base<_Tp>::__atomic_ref_base;
1853 using __atomic_ref_base<_Tp>::operator=;
1854
1855 value_type
1856 fetch_add(value_type __i,
1857 memory_order __m = memory_order_seq_cst) const noexcept
1858 { return __atomic_impl::fetch_add(this->_M_ptr, __i, __m); }
1859
1860 value_type
1861 fetch_sub(value_type __i,
1862 memory_order __m = memory_order_seq_cst) const noexcept
1863 { return __atomic_impl::fetch_sub(this->_M_ptr, __i, __m); }
1864
1865 value_type
1866 fetch_and(value_type __i,
1867 memory_order __m = memory_order_seq_cst) const noexcept
1868 { return __atomic_impl::fetch_and(this->_M_ptr, __i, __m); }
1869
1870 value_type
1871 fetch_or(value_type __i,
1872 memory_order __m = memory_order_seq_cst) const noexcept
1873 { return __atomic_impl::fetch_or(this->_M_ptr, __i, __m); }
1874
1875 value_type
1876 fetch_xor(value_type __i,
1877 memory_order __m = memory_order_seq_cst) const noexcept
1878 { return __atomic_impl::fetch_xor(this->_M_ptr, __i, __m); }
1879
1880#if __glibcxx_atomic_min_max
1881 value_type
1882 fetch_min(value_type __i,
1883 memory_order __m = memory_order_seq_cst) const noexcept
1884 { return __atomic_impl::__fetch_min(this->_M_ptr, __i, __m); }
1885
1886 value_type
1887 fetch_max(value_type __i,
1888 memory_order __m = memory_order_seq_cst) const noexcept
1889 { return __atomic_impl::__fetch_max(this->_M_ptr, __i, __m); }
1890#endif
1891
1892 _GLIBCXX_ALWAYS_INLINE value_type
1893 operator++(int) const noexcept
1894 { return fetch_add(i: 1); }
1895
1896 _GLIBCXX_ALWAYS_INLINE value_type
1897 operator--(int) const noexcept
1898 { return fetch_sub(i: 1); }
1899
1900 value_type
1901 operator++() const noexcept
1902 { return __atomic_impl::__add_fetch(this->_M_ptr, value_type(1)); }
1903
1904 value_type
1905 operator--() const noexcept
1906 { return __atomic_impl::__sub_fetch(this->_M_ptr, value_type(1)); }
1907
1908 value_type
1909 operator+=(value_type __i) const noexcept
1910 { return __atomic_impl::__add_fetch(this->_M_ptr, __i); }
1911
1912 value_type
1913 operator-=(value_type __i) const noexcept
1914 { return __atomic_impl::__sub_fetch(this->_M_ptr, __i); }
1915
1916 value_type
1917 operator&=(value_type __i) const noexcept
1918 { return __atomic_impl::__and_fetch(this->_M_ptr, __i); }
1919
1920 value_type
1921 operator|=(value_type __i) const noexcept
1922 { return __atomic_impl::__or_fetch(this->_M_ptr, __i); }
1923
1924 value_type
1925 operator^=(value_type __i) const noexcept
1926 { return __atomic_impl::__xor_fetch(this->_M_ptr, __i); }
1927 };
1928
1929 template<typename _Tp>
1930 struct __atomic_ref<const _Tp, true, false, false>
1931 : __atomic_ref_base<const _Tp>
1932 {
1933 using difference_type = typename __atomic_ref_base<const _Tp>::value_type;
1934 using __atomic_ref_base<const _Tp>::__atomic_ref_base;
1935 };
1936
1937 // base class for atomic_ref<floating-point-type>
1938 template<typename _Fp>
1939 struct __atomic_ref<_Fp, false, true, false>
1940 : __atomic_ref_base<_Fp>
1941 {
1942 using value_type = typename __atomic_ref_base<_Fp>::value_type;
1943 using difference_type = value_type;
1944
1945 using __atomic_ref_base<_Fp>::__atomic_ref_base;
1946 using __atomic_ref_base<_Fp>::operator=;
1947
1948 value_type
1949 fetch_add(value_type __i,
1950 memory_order __m = memory_order_seq_cst) const noexcept
1951 { return __atomic_impl::__fetch_add_flt(this->_M_ptr, __i, __m); }
1952
1953 value_type
1954 fetch_sub(value_type __i,
1955 memory_order __m = memory_order_seq_cst) const noexcept
1956 { return __atomic_impl::__fetch_sub_flt(this->_M_ptr, __i, __m); }
1957
1958#if __glibcxx_atomic_min_max
1959 value_type
1960 fetch_min(value_type __i,
1961 memory_order __m = memory_order_seq_cst) const noexcept
1962 { return __atomic_impl::__fetch_min(this->_M_ptr, __i, __m); }
1963
1964 value_type
1965 fetch_max(value_type __i,
1966 memory_order __m = memory_order_seq_cst) const noexcept
1967 { return __atomic_impl::__fetch_max(this->_M_ptr, __i, __m); }
1968#endif
1969
1970 value_type
1971 operator+=(value_type __i) const noexcept
1972 { return __atomic_impl::__add_fetch_flt(this->_M_ptr, __i); }
1973
1974 value_type
1975 operator-=(value_type __i) const noexcept
1976 { return __atomic_impl::__sub_fetch_flt(this->_M_ptr, __i); }
1977 };
1978
1979 template<typename _Fp>
1980 struct __atomic_ref<const _Fp, false, true, false>
1981 : __atomic_ref_base<const _Fp>
1982 {
1983 using difference_type = typename __atomic_ref_base<const _Fp>::value_type;
1984 using __atomic_ref_base<const _Fp>::__atomic_ref_base;
1985 };
1986
1987 // base class for atomic_ref<pointer-type>
1988 template<typename _Pt>
1989 struct __atomic_ref<_Pt, false, false, true>
1990 : __atomic_ref_base<_Pt>
1991 {
1992 using value_type = typename __atomic_ref_base<_Pt>::value_type;
1993 using difference_type = ptrdiff_t;
1994
1995 using __atomic_ref_base<_Pt>::__atomic_ref_base;
1996 using __atomic_ref_base<_Pt>::operator=;
1997 _GLIBCXX_ALWAYS_INLINE value_type
1998 fetch_add(difference_type __d,
1999 memory_order __m = memory_order_seq_cst) const noexcept
2000 { return __atomic_impl::fetch_add(this->_M_ptr, _S_type_size(__d), __m); }
2001
2002 _GLIBCXX_ALWAYS_INLINE value_type
2003 fetch_sub(difference_type __d,
2004 memory_order __m = memory_order_seq_cst) const noexcept
2005 { return __atomic_impl::fetch_sub(this->_M_ptr, _S_type_size(__d), __m); }
2006
2007#if __glibcxx_atomic_min_max
2008 _GLIBCXX_ALWAYS_INLINE value_type
2009 fetch_min(value_type __i,
2010 memory_order __m = memory_order_seq_cst) const noexcept
2011 { return __atomic_impl::__fetch_min(this->_M_ptr, __i, __m); }
2012
2013 _GLIBCXX_ALWAYS_INLINE value_type
2014 fetch_max(value_type __i,
2015 memory_order __m = memory_order_seq_cst) const noexcept
2016 { return __atomic_impl::__fetch_max(this->_M_ptr, __i, __m); }
2017#endif
2018
2019 value_type
2020 operator++(int) const noexcept
2021 { return fetch_add(d: 1); }
2022
2023 value_type
2024 operator--(int) const noexcept
2025 { return fetch_sub(d: 1); }
2026
2027 value_type
2028 operator++() const noexcept
2029 {
2030 return __atomic_impl::__add_fetch(this->_M_ptr, _S_type_size(d: 1));
2031 }
2032
2033 value_type
2034 operator--() const noexcept
2035 {
2036 return __atomic_impl::__sub_fetch(this->_M_ptr, _S_type_size(d: 1));
2037 }
2038
2039 value_type
2040 operator+=(difference_type __d) const noexcept
2041 {
2042 return __atomic_impl::__add_fetch(this->_M_ptr, _S_type_size(__d));
2043 }
2044
2045 value_type
2046 operator-=(difference_type __d) const noexcept
2047 {
2048 return __atomic_impl::__sub_fetch(this->_M_ptr, _S_type_size(__d));
2049 }
2050
2051 private:
2052 static constexpr ptrdiff_t
2053 _S_type_size(ptrdiff_t __d) noexcept
2054 {
2055 using _Et = remove_pointer_t<value_type>;
2056 static_assert(is_object_v<_Et>);
2057 return __d * sizeof(_Et);
2058 }
2059 };
2060
2061 template<typename _Pt>
2062 struct __atomic_ref<const _Pt, false, false, true>
2063 : __atomic_ref_base<const _Pt>
2064 {
2065 using difference_type = ptrdiff_t;
2066 using __atomic_ref_base<const _Pt>::__atomic_ref_base;
2067 };
2068#endif // C++2a
2069
2070 /// @endcond
2071
2072 /// @} group atomics
2073
2074_GLIBCXX_END_NAMESPACE_VERSION
2075} // namespace std
2076
2077#endif
2078