libstdc++
simd_loadstore.h
1// Implementation of <simd> -*- C++ -*-
2
3// Copyright The GNU Toolchain Authors.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_SIMD_LOADSTORE_H
26#define _GLIBCXX_SIMD_LOADSTORE_H 1
27
28#ifdef _GLIBCXX_SYSHDR
29#pragma GCC system_header
30#endif
31
32#if __cplusplus >= 202400L
33
34#include "simd_vec.h"
35
36// psabi warnings are bogus because the ABI of the internal types never leaks into user code
37#pragma GCC diagnostic push
38#pragma GCC diagnostic ignored "-Wpsabi"
39
40// [simd.reductions] ----------------------------------------------------------
41namespace std _GLIBCXX_VISIBILITY(default)
42{
43_GLIBCXX_BEGIN_NAMESPACE_VERSION
44namespace simd
45{
46 template <typename _Vp, typename _Tp>
47 struct __vec_load_return
48 { using type = _Vp; };
49
50 template <typename _Tp>
51 struct __vec_load_return<void, _Tp>
52 { using type = basic_vec<_Tp>; };
53
54 template <typename _Vp, typename _Tp>
55 using __vec_load_return_t = typename __vec_load_return<_Vp, _Tp>::type;
56
57 template <typename _Vp, typename _Tp>
58 using __load_mask_type_t = typename __vec_load_return_t<_Vp, _Tp>::mask_type;
59
60 template <typename _Tp>
61 concept __sized_contiguous_range
62 = ranges::contiguous_range<_Tp> && ranges::sized_range<_Tp>;
63
64 template <typename _Vp = void, __sized_contiguous_range _Rg, typename... _Flags>
65 [[__gnu__::__always_inline__]]
66 constexpr __vec_load_return_t<_Vp, ranges::range_value_t<_Rg>>
67 unchecked_load(_Rg&& __r, flags<_Flags...> __f = {})
68 {
69 using _Tp = ranges::range_value_t<_Rg>;
70 using _RV = __vec_load_return_t<_Vp, _Tp>;
71 using _Rp = typename _RV::value_type;
72 static_assert(__loadstore_convertible_to<ranges::range_value_t<_Rg>, _Rp, _Flags...>,
73 "'flag_convert' must be used for conversions that are not value-preserving");
74
75 constexpr bool __allow_out_of_bounds = __f._S_test(__allow_partial_loadstore);
76 constexpr size_t __static_size = __static_range_size(__r);
77
78 if constexpr (!__allow_out_of_bounds && ranges::__static_sized_range<_Rg>)
79 static_assert(ranges::size(__r) >= _RV::size(), "given range must have sufficient size");
80
81 const auto* __ptr = __f.template _S_adjust_pointer<_RV>(ranges::data(__r));
82 const auto __rg_size = std::ranges::size(__r);
83 if constexpr (!__allow_out_of_bounds)
84 __glibcxx_simd_precondition(
85 std::ranges::size(__r) >= _RV::size(),
86 "Input range is too small. Did you mean to use 'partial_load'?");
87
88 if consteval
89 {
90 return _RV([&](size_t __i) -> _Rp {
91 if (__i >= __rg_size)
92 return _Rp();
93 else if constexpr (__complex_like<_Rp> && !__complex_like<_Tp>)
94 return static_cast<typename _Rp::value_type>(__r[__i]);
95 else
96 return static_cast<_Rp>(__r[__i]);
97 });
98 }
99 else
100 {
101 if constexpr ((__static_size != dynamic_extent && __static_size >= size_t(_RV::size()))
102 || !__allow_out_of_bounds)
103 return _RV(_LoadCtorTag(), __ptr);
104 else
105 return _RV::_S_partial_load(__ptr, __rg_size);
106 }
107 }
108
109 template <typename _Vp = void, __sized_contiguous_range _Rg, typename... _Flags>
110 [[__gnu__::__always_inline__]]
111 constexpr __vec_load_return_t<_Vp, ranges::range_value_t<_Rg>>
112 unchecked_load(_Rg&& __r, const __load_mask_type_t<_Vp, ranges::range_value_t<_Rg>>& __mask,
113 flags<_Flags...> __f = {})
114 {
115 using _Tp = ranges::range_value_t<_Rg>;
116 using _RV = __vec_load_return_t<_Vp, _Tp>;
117 using _Rp = typename _RV::value_type;
118 static_assert(__vectorizable<_Tp>);
119 static_assert(__explicitly_convertible_to<_Tp, _Rp>);
120 static_assert(__loadstore_convertible_to<_Tp, _Rp, _Flags...>,
121 "'flag_convert' must be used for conversions that are not value-preserving");
122
123 constexpr bool __allow_out_of_bounds = __f._S_test(__allow_partial_loadstore);
124 constexpr auto __static_size = __static_range_size(__r);
125
126 if constexpr (!__allow_out_of_bounds && ranges::__static_sized_range<_Rg>)
127 static_assert(ranges::size(__r) >= _RV::size(), "given range must have sufficient size");
128
129 const auto* __ptr = __f.template _S_adjust_pointer<_RV>(ranges::data(__r));
130
131 if constexpr (!__allow_out_of_bounds)
132 __glibcxx_simd_precondition(
133 ranges::size(__r) >= size_t(_RV::size()),
134 "Input range is too small. Did you mean to use 'partial_load'?");
135
136 const size_t __rg_size = ranges::size(__r);
137 if consteval
138 {
139 return _RV([&](size_t __i) -> _Rp {
140 if (__i >= __rg_size || !__mask[int(__i)])
141 return _Rp();
142 else if constexpr (__complex_like<_Rp> && !__complex_like<_Tp>)
143 return static_cast<typename _Rp::value_type>(__r[__i]);
144 else
145 return static_cast<_Rp>(__r[__i]);
146 });
147 }
148 else
149 {
150 constexpr bool __no_size_check
151 = !__allow_out_of_bounds
152 || (__static_size != dynamic_extent
153 && __static_size >= size_t(_RV::size.value));
154 if constexpr (_RV::size() == 1)
155 return __mask[0] && (__no_size_check || __rg_size > 0) ? _RV(_LoadCtorTag(), __ptr)
156 : _RV();
157 else if constexpr (__no_size_check)
158 return _RV::_S_masked_load(__ptr, __mask);
159 else if (__rg_size >= size_t(_RV::size()))
160 return _RV::_S_masked_load(__ptr, __mask);
161 else if (__rg_size > 0)
162 return _RV::_S_masked_load(
163 __ptr, __mask && _RV::mask_type::_S_partial_mask_of_n(int(__rg_size)));
164 else
165 return _RV();
166 }
167 }
168
169 template <typename _Vp = void, contiguous_iterator _It, typename... _Flags>
170 [[__gnu__::__always_inline__]]
171 constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
172 unchecked_load(_It __first, iter_difference_t<_It> __n, flags<_Flags...> __f = {})
173 { return simd::unchecked_load<_Vp>(span<const iter_value_t<_It>>(__first, __n), __f); }
174
175 template <typename _Vp = void, contiguous_iterator _It, typename... _Flags>
176 [[__gnu__::__always_inline__]]
177 constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
178 unchecked_load(_It __first, iter_difference_t<_It> __n,
179 const __load_mask_type_t<_Vp, iter_value_t<_It>>& __mask,
180 flags<_Flags...> __f = {})
181 { return simd::unchecked_load<_Vp>(span<const iter_value_t<_It>>(__first, __n), __mask, __f); }
182
183 template <typename _Vp = void, contiguous_iterator _It, sized_sentinel_for<_It> _Sp,
184 typename... _Flags>
185 [[__gnu__::__always_inline__]]
186 constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
187 unchecked_load(_It __first, _Sp __last, flags<_Flags...> __f = {})
188 { return simd::unchecked_load<_Vp>(span<const iter_value_t<_It>>(__first, __last), __f); }
189
190 template <typename _Vp = void, contiguous_iterator _It, sized_sentinel_for<_It> _Sp,
191 typename... _Flags>
192 [[__gnu__::__always_inline__]]
193 constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
194 unchecked_load(_It __first, _Sp __last,
195 const __load_mask_type_t<_Vp, iter_value_t<_It>>& __mask,
196 flags<_Flags...> __f = {})
197 {
198 return simd::unchecked_load<_Vp>(span<const iter_value_t<_It>>(__first, __last), __mask, __f);
199 }
200
201 template <typename _Vp = void, __sized_contiguous_range _Rg, typename... _Flags>
202 [[__gnu__::__always_inline__]]
203 constexpr __vec_load_return_t<_Vp, ranges::range_value_t<_Rg>>
204 partial_load(_Rg&& __r, flags<_Flags...> __f = {})
205 { return simd::unchecked_load<_Vp>(__r, __f | __allow_partial_loadstore); }
206
207 template <typename _Vp = void, __sized_contiguous_range _Rg, typename... _Flags>
208 [[__gnu__::__always_inline__]]
209 constexpr __vec_load_return_t<_Vp, ranges::range_value_t<_Rg>>
210 partial_load(_Rg&& __r, const __load_mask_type_t<_Vp, ranges::range_value_t<_Rg>>& __mask,
211 flags<_Flags...> __f = {})
212 { return simd::unchecked_load<_Vp>(__r, __mask, __f | __allow_partial_loadstore); }
213
214 template <typename _Vp = void, contiguous_iterator _It, typename... _Flags>
215 [[__gnu__::__always_inline__]]
216 constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
217 partial_load(_It __first, iter_difference_t<_It> __n, flags<_Flags...> __f = {})
218 { return partial_load<_Vp>(span<const iter_value_t<_It>>(__first, __n), __f); }
219
220 template <typename _Vp = void, contiguous_iterator _It, typename... _Flags>
221 [[__gnu__::__always_inline__]]
222 constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
223 partial_load(_It __first, iter_difference_t<_It> __n,
224 const __load_mask_type_t<_Vp, iter_value_t<_It>>& __mask,
225 flags<_Flags...> __f = {})
226 { return partial_load<_Vp>(span<const iter_value_t<_It>>(__first, __n), __mask, __f); }
227
228 template <typename _Vp = void, contiguous_iterator _It, sized_sentinel_for<_It> _Sp,
229 typename... _Flags>
230 [[__gnu__::__always_inline__]]
231 constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
232 partial_load(_It __first, _Sp __last, flags<_Flags...> __f = {})
233 { return partial_load<_Vp>(span<const iter_value_t<_It>>(__first, __last), __f); }
234
235 template <typename _Vp = void, contiguous_iterator _It, sized_sentinel_for<_It> _Sp,
236 typename... _Flags>
237 [[__gnu__::__always_inline__]]
238 constexpr __vec_load_return_t<_Vp, iter_value_t<_It>>
239 partial_load(_It __first, _Sp __last, const __load_mask_type_t<_Vp, iter_value_t<_It>>& __mask,
240 flags<_Flags...> __f = {})
241 { return partial_load<_Vp>(span<const iter_value_t<_It>>(__first, __last), __mask, __f); }
242
243 template <typename _Tp, typename _Ap, __sized_contiguous_range _Rg, typename... _Flags>
244 requires indirectly_writable<ranges::iterator_t<_Rg>, _Tp>
245 [[__gnu__::__always_inline__]]
246 constexpr void
247 unchecked_store(const basic_vec<_Tp, _Ap>& __v, _Rg&& __r, flags<_Flags...> __f = {})
248 {
249 using _TV = basic_vec<_Tp, _Ap>;
250 static_assert(destructible<_TV>);
251 static_assert(__loadstore_convertible_to<_Tp, ranges::range_value_t<_Rg>, _Flags...>,
252 "'flag_convert' must be used for conversions that are not value-preserving");
253
254 constexpr bool __allow_out_of_bounds = __f._S_test(__allow_partial_loadstore);
255 if constexpr (!__allow_out_of_bounds && ranges::__static_sized_range<_Rg>)
256 static_assert(ranges::size(__r) >= _TV::size(), "given range must have sufficient size");
257
258 auto* __ptr = __f.template _S_adjust_pointer<_TV>(ranges::data(__r));
259 const auto __rg_size = ranges::size(__r);
260 if constexpr (!__allow_out_of_bounds)
261 __glibcxx_simd_precondition(
262 ranges::size(__r) >= _TV::size(),
263 "output range is too small. Did you mean to use 'partial_store'?");
264
265 if consteval
266 {
267 for (unsigned __i = 0; __i < __rg_size && __i < _TV::size(); ++__i)
268 __ptr[__i] = static_cast<ranges::range_value_t<_Rg>>(__v[__i]);
269 }
270 else
271 {
272 if constexpr (!__allow_out_of_bounds)
273 __v._M_store(__ptr);
274 else
275 _TV::_S_partial_store(__v, __ptr, __rg_size);
276 }
277 }
278
279 template <typename _Tp, typename _Ap, __sized_contiguous_range _Rg, typename... _Flags>
280 requires indirectly_writable<ranges::iterator_t<_Rg>, _Tp>
281 [[__gnu__::__always_inline__]]
282 constexpr void
283 unchecked_store(const basic_vec<_Tp, _Ap>& __v, _Rg&& __r,
284 const typename basic_vec<_Tp, _Ap>::mask_type& __mask,
285 flags<_Flags...> __f = {})
286 {
287 using _TV = basic_vec<_Tp, _Ap>;
288 static_assert(__loadstore_convertible_to<_Tp, ranges::range_value_t<_Rg>, _Flags...>,
289 "'flag_convert' must be used for conversions that are not value-preserving");
290
291 constexpr bool __allow_out_of_bounds = __f._S_test(__allow_partial_loadstore);
292 if constexpr (!__allow_out_of_bounds && ranges::__static_sized_range<_Rg>)
293 static_assert(ranges::size(__r) >= _TV::size(), "given range must have sufficient size");
294
295 auto* __ptr = __f.template _S_adjust_pointer<_TV>(ranges::data(__r));
296
297 if constexpr (!__allow_out_of_bounds)
298 __glibcxx_simd_precondition(
299 ranges::size(__r) >= size_t(_TV::size()),
300 "output range is too small. Did you mean to use 'partial_store'?");
301
302 const size_t __rg_size = ranges::size(__r);
303 if consteval
304 {
305 for (int __i = 0; __i < _TV::size(); ++__i)
306 {
307 if (__mask[__i] && (!__allow_out_of_bounds || size_t(__i) < __rg_size))
308 __ptr[__i] = static_cast<ranges::range_value_t<_Rg>>(__v[__i]);
309 }
310 }
311 else
312 {
313 if (__allow_out_of_bounds && __rg_size < size_t(_TV::size()))
314 _TV::_S_masked_store(__v, __ptr,
315 __mask && _TV::mask_type::_S_partial_mask_of_n(int(__rg_size)));
316 else
317 _TV::_S_masked_store(__v, __ptr, __mask);
318 }
319 }
320
321 template <typename _Tp, typename _Ap, contiguous_iterator _It, typename... _Flags>
322 requires indirectly_writable<_It, _Tp>
323 [[__gnu__::__always_inline__]]
324 constexpr void
325 unchecked_store(const basic_vec<_Tp, _Ap>& __v, _It __first,
326 iter_difference_t<_It> __n, flags<_Flags...> __f = {})
327 { simd::unchecked_store(__v, std::span<iter_value_t<_It>>(__first, __n), __f); }
328
329 template <typename _Tp, typename _Ap, contiguous_iterator _It, typename... _Flags>
330 requires indirectly_writable<_It, _Tp>
331 [[__gnu__::__always_inline__]]
332 constexpr void
333 unchecked_store(const basic_vec<_Tp, _Ap>& __v, _It __first, iter_difference_t<_It> __n,
334 const typename basic_vec<_Tp, _Ap>::mask_type& __mask,
335 flags<_Flags...> __f = {})
336 { simd::unchecked_store(__v, std::span<iter_value_t<_It>>(__first, __n), __mask, __f); }
337
338 template <typename _Tp, typename _Ap, contiguous_iterator _It, sized_sentinel_for<_It> _Sp,
339 typename... _Flags>
340 requires indirectly_writable<_It, _Tp>
341 [[__gnu__::__always_inline__]]
342 constexpr void
343 unchecked_store(const basic_vec<_Tp, _Ap>& __v, _It __first, _Sp __last,
344 flags<_Flags...> __f = {})
345 { simd::unchecked_store(__v, std::span<iter_value_t<_It>>(__first, __last), __f); }
346
347 template <typename _Tp, typename _Ap, contiguous_iterator _It, sized_sentinel_for<_It> _Sp,
348 typename... _Flags>
349 requires indirectly_writable<_It, _Tp>
350 [[__gnu__::__always_inline__]]
351 constexpr void
352 unchecked_store(const basic_vec<_Tp, _Ap>& __v, _It __first, _Sp __last,
353 const typename basic_vec<_Tp, _Ap>::mask_type& __mask,
354 flags<_Flags...> __f = {})
355 { simd::unchecked_store(__v, std::span<iter_value_t<_It>>(__first, __last), __mask, __f); }
356
357 template <typename _Tp, typename _Ap, __sized_contiguous_range _Rg, typename... _Flags>
358 requires indirectly_writable<ranges::iterator_t<_Rg>, _Tp>
359 [[__gnu__::__always_inline__]]
360 constexpr void
361 partial_store(const basic_vec<_Tp, _Ap>& __v, _Rg&& __r, flags<_Flags...> __f = {})
362 { simd::unchecked_store(__v, __r, __f | __allow_partial_loadstore); }
363
364 template <typename _Tp, typename _Ap, __sized_contiguous_range _Rg, typename... _Flags>
365 requires indirectly_writable<ranges::iterator_t<_Rg>, _Tp>
366 [[__gnu__::__always_inline__]]
367 constexpr void
368 partial_store(const basic_vec<_Tp, _Ap>& __v, _Rg&& __r,
369 const typename basic_vec<_Tp, _Ap>::mask_type& __mask,
370 flags<_Flags...> __f = {})
371 { simd::unchecked_store(__v, __r, __mask, __f | __allow_partial_loadstore); }
372
373 template <typename _Tp, typename _Ap, contiguous_iterator _It, typename... _Flags>
374 requires indirectly_writable<_It, _Tp>
375 [[__gnu__::__always_inline__]]
376 constexpr void
377 partial_store(const basic_vec<_Tp, _Ap>& __v, _It __first, iter_difference_t<_It> __n,
378 flags<_Flags...> __f = {})
379 { partial_store(__v, span(__first, __n), __f); }
380
381 template <typename _Tp, typename _Ap, contiguous_iterator _It, typename... _Flags>
382 requires indirectly_writable<_It, _Tp>
383 [[__gnu__::__always_inline__]]
384 constexpr void
385 partial_store(const basic_vec<_Tp, _Ap>& __v, _It __first, iter_difference_t<_It> __n,
386 const typename basic_vec<_Tp, _Ap>::mask_type& __mask, flags<_Flags...> __f = {})
387 { partial_store(__v, span(__first, __n), __mask, __f); }
388
389 template <typename _Tp, typename _Ap, contiguous_iterator _It, sized_sentinel_for<_It> _Sp,
390 typename... _Flags>
391 requires indirectly_writable<_It, _Tp>
392 [[__gnu__::__always_inline__]]
393 constexpr void
394 partial_store(const basic_vec<_Tp, _Ap>& __v, _It __first, _Sp __last,
395 flags<_Flags...> __f = {})
396 { partial_store(__v, span(__first, __last), __f); }
397
398 template <typename _Tp, typename _Ap, contiguous_iterator _It, sized_sentinel_for<_It> _Sp,
399 typename... _Flags>
400 requires indirectly_writable<_It, _Tp>
401 [[__gnu__::__always_inline__]]
402 constexpr void
403 partial_store(const basic_vec<_Tp, _Ap>& __v, _It __first, _Sp __last,
404 const typename basic_vec<_Tp, _Ap>::mask_type& __mask, flags<_Flags...> __f = {})
405 { partial_store(__v, span(__first, __last), __mask, __f); }
406} // namespace simd
407_GLIBCXX_END_NAMESPACE_VERSION
408} // namespace std
409
410#pragma GCC diagnostic pop
411#endif // C++26
412#endif // _GLIBCXX_SIMD_LOADSTORE_H
std
ISO C++ entities toplevel namespace is std.