libstdc++: barrier Source File

libstdc++

barrier
Go to the documentation of this file.
1// <barrier> -*- C++ -*-
2
3// Copyright (C) 2020-2026 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25// This implementation is based on libcxx/include/barrier
26//===-- barrier.h --------------------------------------------------===//
27//
28// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
29// See https://llvm.org/LICENSE.txt for license information.
30// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
31//
32//===---------------------------------------------------------------===//
33
34/** @file include/barrier
35 * This is a Standard C++ Library header.
36 */
37
38#ifndef _GLIBCXX_BARRIER
39#define _GLIBCXX_BARRIER 1
40
41#ifdef _GLIBCXX_SYSHDR
42#pragma GCC system_header
43#endif
44
45#include <bits/requires_hosted.h> // threading primitive
46
47#define __glibcxx_want_barrier
48#include <bits/version.h>
49
50#ifdef __cpp_lib_barrier // C++ >= 20 && __cpp_aligned_new && lib_atomic_wait
51#include <bits/atomic_base.h>
52#include <bits/std_thread.h>
53#include <bits/unique_ptr.h>
54
55#include <array>
56
57namespace std _GLIBCXX_VISIBILITY(default)
58{
59_GLIBCXX_BEGIN_NAMESPACE_VERSION
60
61 struct __empty_completion
62 {
63 _GLIBCXX_ALWAYS_INLINE void
64 operator()() noexcept
65 { }
66 };
67
68/*
69
70The default implementation of __tree_barrier is a classic tree barrier.
71
72It looks different from literature pseudocode for two main reasons:
73 1. Threads that call into std::barrier functions do not provide indices,
74 so a numbering step is added before the actual barrier algorithm,
75 appearing as an N+1 round to the N rounds of the tree barrier.
76 2. A great deal of attention has been paid to avoid cache line thrashing
77 by flattening the tree structure into cache-line sized arrays, that
78 are indexed in an efficient way.
79
80*/
81
82 enum class __barrier_phase_t : unsigned char { };
83
84 struct __tree_barrier_base
85 {
86 static constexpr ptrdiff_t
87 max() noexcept
88 { return __PTRDIFF_MAX__ - 1; }
89
90 protected:
91 using __atomic_phase_ref_t = std::__atomic_ref<__barrier_phase_t>;
92 using __atomic_phase_const_ref_t = std::__atomic_ref<const __barrier_phase_t>;
93 static constexpr auto __phase_alignment =
94 __atomic_phase_ref_t::required_alignment;
95
96 using __tickets_t = std::array<__barrier_phase_t, 64>;
97 struct alignas(64) /* naturally-align the heap state */ __state_t
98 {
99 alignas(__phase_alignment) __tickets_t __tickets;
100 };
101
102 ptrdiff_t _M_expected;
103 __atomic_base<__state_t*> _M_state{nullptr};
104 __atomic_base<ptrdiff_t> _M_expected_adjustment{0};
105 alignas(__phase_alignment) __barrier_phase_t _M_phase{};
106
107 explicit constexpr
108 __tree_barrier_base(ptrdiff_t __expected)
109 : _M_expected(__expected)
110 {
111 __glibcxx_assert(__expected >= 0 && __expected <= max());
112
113 if (!std::is_constant_evaluated())
114 _M_state.store(_M_alloc_state().release(), memory_order_release);
115 }
116
117 ~__tree_barrier_base()
118 { delete[] _M_state.load(memory_order_relaxed); }
119
120 __tree_barrier_base(const __tree_barrier_base&&) = delete;
121 __tree_barrier_base& operator=(const __tree_barrier_base&&) = delete;
122
123 unique_ptr<__state_t[]>
124 _M_alloc_state()
125 {
126 size_t const __count = (_M_expected + 1) >> 1;
127 return std::make_unique<__state_t[]>(__count);
128 }
129
130 bool
131 _M_arrive(__barrier_phase_t __old_phase, size_t __current)
132 {
133 const auto __old_phase_val = static_cast<unsigned char>(__old_phase);
134 const auto __half_step =
135 static_cast<__barrier_phase_t>(__old_phase_val + 1);
136 const auto __full_step =
137 static_cast<__barrier_phase_t>(__old_phase_val + 2);
138
139 size_t __current_expected = _M_expected;
140 __current %= ((_M_expected + 1) >> 1);
141
142 __state_t* const __state = _M_state.load(memory_order_relaxed);
143
144 for (int __round = 0; ; ++__round)
145 {
146 if (__current_expected <= 1)
147 return true;
148 size_t const __end_node = ((__current_expected + 1) >> 1),
149 __last_node = __end_node - 1;
150 for ( ; ; ++__current)
151 {
152 if (__current == __end_node)
153 __current = 0;
154 auto __expect = __old_phase;
155 __atomic_phase_ref_t __phase(&__state[__current]
156 .__tickets[__round]);
157 if (__current == __last_node && (__current_expected & 1))
158 {
159 if (__phase.compare_exchange_strong(__expect, __full_step,
160 memory_order_acq_rel))
161 break; // I'm 1 in 1, go to next __round
162 }
163 else if (__phase.compare_exchange_strong(__expect, __half_step,
164 memory_order_acq_rel))
165 {
166 return false; // I'm 1 in 2, done with arrival
167 }
168 else if (__expect == __half_step)
169 {
170 if (__phase.compare_exchange_strong(__expect, __full_step,
171 memory_order_acq_rel))
172 break; // I'm 2 in 2, go to next __round
173 }
174 }
175 __current_expected = __last_node + 1;
176 __current >>= 1;
177 }
178 }
179 };
180
181 template<typename _CompletionF>
182 class __tree_barrier : public __tree_barrier_base
183 {
184 [[no_unique_address]] _CompletionF _M_completion;
185
186 // _GLIBCXX_RESOLVE_LIB_DEFECTS
187 // 3898. Possibly unintended preconditions for completion functions
188 void _M_invoke_completion() noexcept { _M_completion(); }
189
190 public:
191 using arrival_token = __barrier_phase_t;
192
193 constexpr
194 __tree_barrier(ptrdiff_t __expected, _CompletionF __completion)
195 : __tree_barrier_base(__expected), _M_completion(std::move(__completion))
196 { }
197
198 [[nodiscard]] arrival_token
199 arrive(ptrdiff_t __update)
200 {
201 __glibcxx_assert(__update > 0);
202 // FIXME: Check that update is less than or equal to the expected count
203 // for the current barrier phase.
204
205 std::hash<std::thread::id> __hasher;
206 size_t __current = __hasher(std::this_thread::get_id());
207 __atomic_phase_ref_t __phase(&_M_phase);
208 const auto __old_phase = __phase.load(memory_order_relaxed);
209 const auto __cur = static_cast<unsigned char>(__old_phase);
210
211 if (__cur == 0 && !_M_state.load(memory_order_relaxed)) [[unlikely]]
212 {
213 auto __p = _M_alloc_state();
214 __state_t* __val = nullptr;
215 if (_M_state.compare_exchange_strong(__val, __p.get(),
216 memory_order_seq_cst,
217 memory_order_acquire))
218 __p.release();
219 }
220
221 for (; __update; --__update)
222 {
223 if (_M_arrive(__old_phase, __current))
224 {
225 _M_invoke_completion();
226 _M_expected += _M_expected_adjustment.load(memory_order_relaxed);
227 _M_expected_adjustment.store(0, memory_order_relaxed);
228 auto __new_phase = static_cast<__barrier_phase_t>(__cur + 2);
229 __phase.store(__new_phase, memory_order_release);
230 __phase.notify_all();
231 }
232 }
233 return __old_phase;
234 }
235
236 void
237 wait(arrival_token&& __old_phase) const
238 {
239 __atomic_phase_const_ref_t __phase(&_M_phase);
240 __phase.wait(__old_phase, memory_order_acquire);
241 }
242
243 void
244 arrive_and_drop()
245 {
246 _M_expected_adjustment.fetch_sub(1, memory_order_relaxed);
247 (void)arrive(1);
248 }
249 };
250
251 template<typename _CompletionF = __empty_completion>
252 class barrier
253 {
254 static_assert(is_invocable_v<_CompletionF&>);
255
256 // Note, we may introduce a "central" barrier algorithm at some point
257 // for more space constrained targets
258 using __algorithm_t = __tree_barrier<_CompletionF>;
259 __algorithm_t _M_b;
260
261 public:
262 class arrival_token final
263 {
264 public:
265 arrival_token(arrival_token&&) = default;
266 arrival_token& operator=(arrival_token&&) = default;
267 ~arrival_token() = default;
268
269 private:
270 friend class barrier;
271 using __token = typename __algorithm_t::arrival_token;
272 explicit arrival_token(__token __tok) noexcept : _M_tok(__tok) { }
273 __token _M_tok;
274 };
275
276 static constexpr ptrdiff_t
277 max() noexcept
278 { return __algorithm_t::max(); }
279
280 constexpr explicit
281 barrier(ptrdiff_t __count, _CompletionF __completion = _CompletionF())
282 : _M_b(__count, std::move(__completion))
283 { }
284
285 barrier(barrier const&) = delete;
286 barrier& operator=(barrier const&) = delete;
287
288 [[nodiscard]] arrival_token
289 arrive(ptrdiff_t __update = 1)
290 { return arrival_token{_M_b.arrive(__update)}; }
291
292 void
293 wait(arrival_token&& __phase) const
294 { _M_b.wait(std::move(__phase._M_tok)); }
295
296 void
297 arrive_and_wait()
298 { wait(arrive()); }
299
300 void
301 arrive_and_drop()
302 { _M_b.arrive_and_drop(); }
303 };
304
305_GLIBCXX_END_NAMESPACE_VERSION
306} // namespace
307#endif // __cpp_lib_barrier
308#endif // _GLIBCXX_BARRIER
array
atomic_base.h
version.h
unique_ptr.h
requires_hosted.h
std_thread.h
std::move
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
Definition move.h:138
std
ISO C++ entities toplevel namespace is std.
std::this_thread::get_id
thread::id get_id() noexcept
The unique identifier of the current thread.
Definition std_thread.h:361