libstdc++: simd_details.h Source File

libstdc++

simd_details.h
1// Implementation of <simd> -*- C++ -*-
2
3// Copyright The GNU Toolchain Authors.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_SIMD_DETAILS_H
26#define _GLIBCXX_SIMD_DETAILS_H 1
27
28#ifdef _GLIBCXX_SYSHDR
29#pragma GCC system_header
30#endif
31
32#if __cplusplus >= 202400L
33
34#include <bit>
35#include <bits/c++config.h> // _GLIBCXX_FLOAT_IS_IEEE_BINARY32
36#include <bits/stl_function.h> // plus, minus, multiplies, ...
37#include <bits/utility.h> // integer_sequence, etc.
38#include <cmath> // for math_errhandling :(
39#include <concepts>
40#include <cstdint>
41#include <limits>
42#include <span> // for dynamic_extent
43
44#if __CHAR_BIT__ != 8
45// There are simply too many constants and bit operators that currently depend on CHAR_BIT == 8.
46// Generalization to CHAR_BIT != 8 does not make sense without testability (i.e. a test target).
47#error "<simd> is not supported for CHAR_BIT != 8"
48#endif
49
50// psabi warnings are bogus because the ABI of the internal types never leaks into user code
51#pragma GCC diagnostic push
52#pragma GCC diagnostic ignored "-Wpsabi"
53
54#if defined __x86_64__ || defined __i386__
55#define _GLIBCXX_X86 1
56#else
57#define _GLIBCXX_X86 0
58#endif
59
60#ifndef _GLIBCXX_SIMD_NOEXCEPT
61/** @internal
62 * For unit-testing preconditions, use this macro to remove noexcept.
63 */
64#define _GLIBCXX_SIMD_NOEXCEPT noexcept
65#endif
66
67#define _GLIBCXX_SIMD_TOSTRING_IMPL(x) #x
68#define _GLIBCXX_SIMD_TOSTRING(x) _GLIBCXX_SIMD_TOSTRING_IMPL(x)
69
70// This is used for unit-testing precondition checking
71#define __glibcxx_simd_precondition(expr, msg, ...) \
72 __glibcxx_assert(expr)
73
74namespace std _GLIBCXX_VISIBILITY(default)
75{
76_GLIBCXX_BEGIN_NAMESPACE_VERSION
77
78 template<typename> class complex;
79
80namespace simd
81{
82 template <typename _Tp>
83 inline constexpr _Tp
84 __iota = [] { static_assert(false, "invalid __iota specialization"); }();
85
86 // [simd.general] vectorizable types
87 template <typename _Tp>
88 concept __complex_like_impl
89 = same_as<_Tp, complex<typename _Tp::value_type>>;
90
91 /** @internal
92 * Satisfied if @p _Tp implements the std::complex interface.
93 */
94 template <typename _Tp>
95 concept __complex_like = __complex_like_impl<remove_cvref_t<_Tp>>;
96
97 template <typename _Tp>
98 concept __vectorizable_scalar
99 = same_as<remove_cv_t<_Tp>, _Tp>
100#ifdef __STDCPP_BFLOAT16_T__
101 && !same_as<_Tp, __gnu_cxx::__bfloat16_t>
102#endif
103 && ((integral<_Tp> && sizeof(_Tp) <= sizeof(0ULL) && !same_as<_Tp, bool>)
104 || (floating_point<_Tp> && sizeof(_Tp) <= sizeof(double)));
105
106 // [simd.general] p2
107 template <typename _Tp>
108 concept __vectorizable
109 = __vectorizable_scalar<_Tp>
110 || (__complex_like_impl<_Tp> && __vectorizable_scalar<typename _Tp::value_type>
111 && floating_point<typename _Tp::value_type>);
112
113 /** @internal
114 * Describes variants of _Abi.
115 */
116 enum class _AbiVariant : unsigned long long
117 {
118 _BitMask = 0x01, // AVX512 bit-masks
119 _MaskVariants = 0x0f, // vector masks if bits [0:3] are 0
120 _CxIleav = 0x10, // store complex components interleaved (ririri...)
121 // mask elements are stored for both (001122...)
122 _CxCtgus = 0x20, // ... or store complex components contiguously (rrrr iiii)
123 // mask elements are store for one component (0123)
124 _CxVariants = _CxIleav | _CxCtgus,
125 };
126
127 /** @internal
128 * Return @p __in with only bits set that are set in any of @p __to_keep.
129 */
130 consteval _AbiVariant
131 __filter_abi_variant(_AbiVariant __in, same_as<_AbiVariant> auto... __to_keep)
132 {
133 using _Up = underlying_type_t<_AbiVariant>;
134 return static_cast<_AbiVariant>(static_cast<_Up>(__in) & (static_cast<_Up>(__to_keep) | ...));
135 }
136
137 /** @internal
138 * Type used whenever no valid integer/value type exists.
139 */
140 struct _InvalidInteger
141 {};
142
143 /** @internal
144 * Alias for a signed integer type T such that sizeof(T) equals _Bytes.
145 *
146 * C++26 [simd.expos.defn]
147 */
148 template <size_t _Bytes>
149 using __integer_from
150 = decltype([] consteval {
151 if constexpr (sizeof(signed char) == _Bytes)
152 return static_cast<signed char>(0);
153 else if constexpr (sizeof(signed short) == _Bytes)
154 return static_cast<signed short>(0);
155 else if constexpr (sizeof(signed int) == _Bytes)
156 return static_cast<signed int>(0);
157 else if constexpr (sizeof(signed long long) == _Bytes)
158 return static_cast<signed long long>(0);
159 else
160 return _InvalidInteger();
161 }());
162
163 template <size_t _Bytes>
164 using __float_from = decltype([] consteval {
165 if constexpr (sizeof(double) == _Bytes)
166 return double();
167 else if constexpr (sizeof(float) == _Bytes)
168 return float();
169 else if constexpr (sizeof(_Float16) == _Bytes)
170 return _Float16();
171 }());
172
173 /** @internal
174 * Alias for an unsigned integer type T such that sizeof(T) equals _Bytes.
175 */
176 template <size_t _Bytes>
177 using _UInt = make_unsigned_t<__integer_from<_Bytes>>;
178
179 /** @internal
180 * Divide @p __x by @p __y while rounding up instead of down.
181 *
182 * Preconditions: __x >= 0 && __y > 0.
183 */
184 template <typename _Tp>
185 consteval _Tp
186 __div_ceil(_Tp __x, _Tp __y)
187 { return (__x + __y - 1) / __y; }
188
189 /** @internal
190 * Alias for an unsigned integer type that can store at least @p _NBits bits.
191 */
192 template <int _NBits>
193 requires (_NBits > 0 && _NBits <= numeric_limits<unsigned long long>::digits)
194 using _Bitmask = _UInt<__div_ceil(__bit_ceil(unsigned(_NBits)), unsigned(__CHAR_BIT__))>;
195
196 /** @internal
197 * Map a given type @p _Tp to an equivalent type.
198 *
199 * This helps with reducing the necessary branches && casts in the implementation as well as
200 * reducing the number of template instantiations.
201 */
202 template <typename _Tp>
203 struct __canonical_vec_type
204 { using type = _Tp; };
205
206 template <typename _Tp>
207 using __canonical_vec_type_t = typename __canonical_vec_type<_Tp>::type;
208
209#if __SIZEOF_INT__ == __SIZEOF_LONG__
210 template <>
211 struct __canonical_vec_type<long>
212 { using type = int; };
213
214 template <>
215 struct __canonical_vec_type<unsigned long>
216 { using type = unsigned int; };
217#elif __SIZEOF_LONG_LONG__ == __SIZEOF_LONG__
218 template <>
219 struct __canonical_vec_type<long>
220 { using type = long long; };
221
222 template <>
223 struct __canonical_vec_type<unsigned long>
224 { using type = unsigned long long; };
225#endif
226
227 template <typename _Tp>
228 requires std::is_enum_v<_Tp>
229 struct __canonical_vec_type<_Tp>
230 { using type = __canonical_vec_type<std::underlying_type_t<_Tp>>::type; };
231
232 template <>
233 struct __canonical_vec_type<char>
234#if __CHAR_UNSIGNED__
235 { using type = unsigned char; };
236#else
237 { using type = signed char; };
238#endif
239
240 template <>
241 struct __canonical_vec_type<char8_t>
242 { using type = unsigned char; };
243
244 template <>
245 struct __canonical_vec_type<char16_t>
246 { using type = uint_least16_t; };
247
248 template <>
249 struct __canonical_vec_type<char32_t>
250 { using type = uint_least32_t; };
251
252 template <>
253 struct __canonical_vec_type<wchar_t>
254 {
255 using type = std::__conditional_t<std::is_signed_v<wchar_t>,
256 simd::__integer_from<sizeof(wchar_t)>,
257 simd::_UInt<sizeof(wchar_t)>>;
258 };
259
260#if defined(__FLT64_DIG__) && defined(_GLIBCXX_DOUBLE_IS_IEEE_BINARY64)
261 template <>
262 struct __canonical_vec_type<_Float64>
263 { using type = double; };
264#endif
265
266#if defined(__FLT32_DIG__) && defined(_GLIBCXX_FLOAT_IS_IEEE_BINARY32)
267 template <>
268 struct __canonical_vec_type<_Float32>
269 { using type = float; };
270#endif
271
272 /** @internal
273 * @brief This ABI tag determines the data member(s) of basic_vec and basic_mask.
274 *
275 * `_Nreg` determines the number of recursive basic_vec/basic_mask data members where `_Nreg` is
276 * equal to 1. With `_Nreg` equal to 1, the basic_vec/basic_mask holds one vector builtin ( `_Np`
277 * greater than 1) or a scalar (`_Np` equal to 1).
278 * @f$\lceil\frac{\mathtt{Np}}{\mathtt{Nreg}}\rceil@f$ therefore determines the number of elements
279 * in a register (except for a remainder where it can be smaller). If `_Np` equals `_Nreg`, (the
280 * aforementioned quotient is 1), then basic_vec (recursively) holds non-vector data members and
281 * basic_mask holds bools.
282 *
283 * The `_Var` parameter determines details about the data member in the one register case. Masks
284 * can be represented as vector masks (the default comparison result of GNU vector builtins),
285 * bit-masks as used by AVX-512, bit-masks as used by ARM SVE (not yet implemented), or a single
286 * bool (for the `_Np` equals 1 case). For basic_mask it determines the actual data layout and
287 * for basic_mask it determines the result of compares.
288 *
289 * @tparam _Np The number of elements.
290 * @tparam _Nreg The number of registers needed to store `_Np` elements.
291 * @tparam _Var Determines how complex value-types are laid out and whether mask types use
292 * bit-masks or vector-masks.
293 */
294 template <int _Np, int _Nreg, underlying_type_t<_AbiVariant> _Var>
295 struct _Abi
296 {
297 static constexpr int _S_size = _Np;
298
299 /** @internal
300 * The number of registers needed to represent one basic_vec for the element type that was
301 * used on ABI deduction.
302 *
303 * For _CxCtgus the value applies twice, once per reals and once per imags.
304 *
305 * Examples:
306 * - '_Abi< 8, 2>' for 'int' is 2x 128-bit
307 * - '_Abi< 9, 3>' for 'int' is 2x 128-bit and 1x 32-bit
308 * - '_Abi<10, 3>' for 'int' is 2x 128-bit and 1x 64-bit
309 * - '_Abi<10, 1>' for 'int' is 1x 512-bit
310 * - '_Abi<10, 2>' for 'int' is 1x 256-bit and 1x 64-bit
311 * - '_Abi< 8, 2, _CxIleav>' for 'complex<float>' is 2x 256-bit
312 * - '_Abi< 9, 2, _CxIleav>' for 'complex<float>' is 1x 512-bit and 1x 64-bit
313 * - '_Abi< 8, 1, _CxCtgus>' for 'complex<float>' is 2x 256-bit
314 */
315 static constexpr int _S_nreg = _Nreg;
316
317 static_assert(_S_size > 0);
318 static_assert(_S_nreg > 0);
319
320 static constexpr _AbiVariant _S_variant = static_cast<_AbiVariant>(_Var);
321
322 static constexpr bool _S_is_cx_ileav
323 = __filter_abi_variant(_S_variant, _AbiVariant::_CxIleav) == _AbiVariant::_CxIleav;
324
325 static constexpr bool _S_is_cx_ctgus
326 = __filter_abi_variant(_S_variant, _AbiVariant::_CxCtgus) == _AbiVariant::_CxCtgus;
327
328 static_assert(!(_S_is_cx_ileav && _S_is_cx_ctgus)); // can't be both
329
330 static_assert(_S_size >= _S_nreg || (_S_is_cx_ileav && _S_size * 2 >= _S_nreg));
331
332 static constexpr bool _S_is_bitmask
333 = __filter_abi_variant(_S_variant, _AbiVariant::_BitMask) == _AbiVariant::_BitMask;
334
335 static constexpr bool _S_is_vecmask = !_S_is_bitmask;
336
337 template <typename _Tp>
338 using _DataType = decltype([] {
339 static_assert(_S_nreg == 1);
340 if constexpr (_S_size == 1)
341 return __canonical_vec_type_t<_Tp>();
342 else
343 {
344 constexpr int __n = __bit_ceil(unsigned(_S_size));
345 using _Vp [[__gnu__::__vector_size__(sizeof(_Tp) * __n)]]
346 = __canonical_vec_type_t<_Tp>;
347 return _Vp();
348 }
349 }());
350
351 template <size_t _Bytes>
352 using _MaskDataType
353 = decltype([] {
354 static_assert(_S_nreg == 1);
355 if constexpr (_S_size == 1)
356 return bool();
357 else if constexpr (_S_is_vecmask)
358 {
359 constexpr unsigned __vbytes = _Bytes * __bit_ceil(unsigned(_S_size));
360 using _Vp [[__gnu__::__vector_size__(__vbytes)]] = __integer_from<_Bytes>;
361 return _Vp();
362 }
363 else if constexpr (_Nreg > 1)
364 return _InvalidInteger();
365 else
366 return _Bitmask<_S_size>();
367 }());
368
369 template <int _N2, int _Nreg2 = __div_ceil(_N2, _S_size)>
370 static consteval auto
371 _S_resize()
372 {
373 if constexpr (_N2 == 1)
374 return _Abi<1, 1, _Var>();
375 else
376 return _Abi<_N2, _Nreg2, _Var>();
377 }
378 };
379
380 /** @internal
381 * Alias for an _Abi specialization where the _AbiVariant bits are combined into a single integer
382 * value.
383 *
384 * Rationale: Consider diagnostic output and mangling of e.g. vec<int, 4> with AVX512. That's an
385 * alias for std::simd::basic_vec<int, std::simd::_Abi<4, 1, 1ull>>. If _AbiVariant were the
386 * template argument type of _Abi, the diagnostic output would be 'std::simd::basic_vec<int,
387 * std::simd::_Abi<4, 1, (std::simd::_AbiVariant)std::simd::_AbiVariant::_BitMask>>'. That's a lot
388 * longer, requires longer mangled names, and bakes the names of the enumerators into the ABI. As
389 * soon as bits of multiple _AbiVariants are combined, this becomes hard to parse for humans
390 * anyway.
391 */
392 template <int _Np, int _Nreg, _AbiVariant... _Vs>
393 using _Abi_t = _Abi<_Np, _Nreg, (static_cast<underlying_type_t<_AbiVariant>>(_Vs) | ... | 0)>;
394
395 /** @internal
396 * This type is used whenever ABI tag deduction can't give a useful answer.
397 */
398 struct _InvalidAbi
399 { static constexpr int _S_size = 0; };
400
401 /** @internal
402 * Satisfied if @p _Tp is a valid simd ABI tag. This is a necessary but not sufficient condition
403 * for an enabled basic_vec/basic_mask specialization.
404 */
405 template <typename _Tp>
406 concept __abi_tag
407 = same_as<decltype(_Tp::_S_variant), const _AbiVariant>
408 && (_Tp::_S_size >= _Tp::_S_nreg) && (_Tp::_S_nreg >= 1)
409 && requires(_Tp __x) {
410 { __x.template _S_resize<_Tp::_S_size, _Tp::_S_nreg>() } -> same_as<_Tp>;
411 };
412
413 /** @internal
414 * Satisfied if `_Tp` is a valid simd ABI tag and one element is stored per register (number of
415 * registers equals size).
416 */
417 template <typename _Tp>
418 concept __scalar_abi_tag
419 = same_as<_Tp, _Abi_t<_Tp::_S_size, _Tp::_S_size, _Tp::_S_variant>> && __abi_tag<_Tp>;
420
421 // Determine if math functions must *raise* floating-point exceptions.
422 // math_errhandling may expand to an extern symbol, in which case we must assume fp exceptions
423 // need to be considered. A conforming C library must define math_errhandling, but in case it
424 // isn't defined we simply use the fallback.
425#ifdef math_errhandling
426 template <int = 0>
427 requires requires { typename bool_constant<0 != (math_errhandling & MATH_ERREXCEPT)>; }
428 consteval bool
429 __handle_fpexcept_impl(int)
430 { return 0 != (math_errhandling & MATH_ERREXCEPT); }
431#endif
432
433 // Fallback if math_errhandling doesn't work: implement correct exception behavior.
434 consteval bool
435 __handle_fpexcept_impl(float)
436 { return true; }
437
438 /** @internal
439 * This type can be used as a template parameter for avoiding ODR violations, where code needs to
440 * differ depending on optimization flags (mostly fp-math related).
441 */
442 struct _OptTraits
443 {
444 consteval bool
445 _M_test(int __bit) const
446 { return ((_M_build_flags >> __bit) & 1) == 1; }
447
448 // true iff floating-point operations can signal an exception (allow non-default handler)
449 consteval bool
450 _M_fp_may_signal() const
451 { return _M_test(0); }
452
453 // true iff floating-point operations can raise an exception flag
454 consteval bool
455 _M_fp_may_raise() const
456 { return _M_test(12); }
457
458 consteval bool
459 _M_fast_math() const
460 { return _M_test(1); }
461
462 consteval bool
463 _M_finite_math_only() const
464 { return _M_test(2); }
465
466 consteval bool
467 _M_no_signed_zeros() const
468 { return _M_test(3); }
469
470 consteval bool
471 _M_signed_zeros() const
472 { return !_M_test(3); }
473
474 consteval bool
475 _M_reciprocal_math() const
476 { return _M_test(4); }
477
478 consteval bool
479 _M_no_math_errno() const
480 { return _M_test(5); }
481
482 consteval bool
483 _M_math_errno() const
484 { return !_M_test(5); }
485
486 consteval bool
487 _M_associative_math() const
488 { return _M_test(6); }
489
490 consteval bool
491 _M_conforming_to_STDC_annex_G() const
492 { return _M_test(10) && !_M_finite_math_only(); }
493
494 consteval bool
495 _M_support_snan() const
496 { return _M_test(11); }
497
498 __UINT64_TYPE__ _M_build_flags
499 = 0
500#if !__NO_TRAPPING_MATH__
501 + (1 << 0)
502#endif
503 + (__handle_fpexcept_impl(0) << 12)
504#if __FAST_MATH__
505 + (1 << 1)
506#endif
507#if __FINITE_MATH_ONLY__
508 + (1 << 2)
509#endif
510#if __NO_SIGNED_ZEROS__
511 + (1 << 3)
512#endif
513#if __RECIPROCAL_MATH__
514 + (1 << 4)
515#endif
516#if __NO_MATH_ERRNO__
517 + (1 << 5)
518#endif
519#if __ASSOCIATIVE_MATH__
520 + (1 << 6)
521#endif
522 // bits 7, 8, and 9 reserved for __FLT_EVAL_METHOD__
523#if __FLT_EVAL_METHOD__ == 1
524 + (1 << 7)
525#elif __FLT_EVAL_METHOD__ == 2
526 + (2 << 7)
527#elif __FLT_EVAL_METHOD__ != 0
528 + (3 << 7)
529#endif
530
531 // C Annex G defines the behavior of complex<T> where T is IEC60559 floating-point. If
532 // __STDC_IEC_60559_COMPLEX__ is defined then Annex G is implemented - and simd<complex>
533 // will do so as well. However, Clang never defines the macro.
534#if defined __STDC_IEC_60559_COMPLEX__ || defined __STDC_IEC_559_COMPLEX__ || defined _GLIBCXX_CLANG
535 + (1 << 10)
536#endif
537#if __SUPPORT_SNAN__
538 + (1 << 11)
539#endif
540 ;
541 };
542
543 /** @internal
544 * Return true iff @p __s equals "1".
545 */
546 consteval bool
547 __streq_to_1(const char* __s)
548 { return __s != nullptr && __s[0] == '1' && __s[1] == '\0'; }
549
550 /** @internal
551 * If the macro given as @p feat is defined to 1, expands to a bit set at position @p off.
552 * Otherwise, expand to zero.
553 */
554#define _GLIBCXX_SIMD_ARCH_FLAG(off, feat) \
555 (static_cast<__UINT64_TYPE__>(std::simd::__streq_to_1(_GLIBCXX_SIMD_TOSTRING_IMPL(feat))) << off)
556
557#if _GLIBCXX_X86
558
559#define _GLIBCXX_SIMD_ARCH_TRAITS_INIT { \
560 _GLIBCXX_SIMD_ARCH_FLAG(0, __MMX__) \
561 | _GLIBCXX_SIMD_ARCH_FLAG( 1, __SSE__) \
562 | _GLIBCXX_SIMD_ARCH_FLAG( 2, __SSE2__) \
563 | _GLIBCXX_SIMD_ARCH_FLAG( 3, __SSE3__) \
564 | _GLIBCXX_SIMD_ARCH_FLAG( 4, __SSSE3__) \
565 | _GLIBCXX_SIMD_ARCH_FLAG( 5, __SSE4_1__) \
566 | _GLIBCXX_SIMD_ARCH_FLAG( 6, __SSE4_2__) \
567 | _GLIBCXX_SIMD_ARCH_FLAG( 7, __POPCNT__) \
568 | _GLIBCXX_SIMD_ARCH_FLAG( 8, __AVX__) \
569 | _GLIBCXX_SIMD_ARCH_FLAG( 9, __F16C__) \
570 | _GLIBCXX_SIMD_ARCH_FLAG(10, __BMI__) \
571 | _GLIBCXX_SIMD_ARCH_FLAG(11, __BMI2__) \
572 | _GLIBCXX_SIMD_ARCH_FLAG(12, __LZCNT__) \
573 | _GLIBCXX_SIMD_ARCH_FLAG(13, __AVX2__) \
574 | _GLIBCXX_SIMD_ARCH_FLAG(14, __FMA__) \
575 | _GLIBCXX_SIMD_ARCH_FLAG(15, __AVX512F__) \
576 | _GLIBCXX_SIMD_ARCH_FLAG(16, __AVX512CD__) \
577 | _GLIBCXX_SIMD_ARCH_FLAG(17, __AVX512DQ__) \
578 | _GLIBCXX_SIMD_ARCH_FLAG(18, __AVX512BW__) \
579 | _GLIBCXX_SIMD_ARCH_FLAG(19, __AVX512VL__) \
580 | _GLIBCXX_SIMD_ARCH_FLAG(20, __AVX512BITALG__) \
581 | _GLIBCXX_SIMD_ARCH_FLAG(21, __AVX512VBMI__) \
582 | _GLIBCXX_SIMD_ARCH_FLAG(22, __AVX512VBMI2__) \
583 | _GLIBCXX_SIMD_ARCH_FLAG(23, __AVX512IFMA__) \
584 | _GLIBCXX_SIMD_ARCH_FLAG(24, __AVX512VNNI__) \
585 | _GLIBCXX_SIMD_ARCH_FLAG(25, __AVX512VPOPCNTDQ__) \
586 | _GLIBCXX_SIMD_ARCH_FLAG(26, __AVX512FP16__) \
587 | _GLIBCXX_SIMD_ARCH_FLAG(27, __AVX512BF16__) \
588 | _GLIBCXX_SIMD_ARCH_FLAG(28, __AVXIFMA__) \
589 | _GLIBCXX_SIMD_ARCH_FLAG(29, __AVXNECONVERT__) \
590 | _GLIBCXX_SIMD_ARCH_FLAG(30, __AVXVNNI__) \
591 | _GLIBCXX_SIMD_ARCH_FLAG(31, __AVXVNNIINT8__) \
592 | _GLIBCXX_SIMD_ARCH_FLAG(32, __AVXVNNIINT16__) \
593 | _GLIBCXX_SIMD_ARCH_FLAG(33, __AVX10_1__) \
594 | _GLIBCXX_SIMD_ARCH_FLAG(34, __AVX10_2__) \
595 | _GLIBCXX_SIMD_ARCH_FLAG(35, __AVX512VP2INTERSECT__) \
596 | _GLIBCXX_SIMD_ARCH_FLAG(36, __SSE4A__) \
597 | _GLIBCXX_SIMD_ARCH_FLAG(37, __FMA4__) \
598 | _GLIBCXX_SIMD_ARCH_FLAG(38, __XOP__) \
599 }
600 // Should this include __APX_F__? I don't think it's relevant for use in constexpr-if branches =>
601 // no ODR issue? The same could be said about several other flags above that are not checked
602 // anywhere.
603
604 struct _ArchTraits
605 {
606 __UINT64_TYPE__ _M_flags = _GLIBCXX_SIMD_ARCH_TRAITS_INIT;
607
608 consteval bool
609 _M_test(int __bit) const
610 { return ((_M_flags >> __bit) & 1) == 1; }
611
612 consteval bool
613 _M_have_mmx() const
614 { return _M_test(0); }
615
616 consteval bool
617 _M_have_sse() const
618 { return _M_test(1); }
619
620 consteval bool
621 _M_have_sse2() const
622 { return _M_test(2); }
623
624 consteval bool
625 _M_have_sse3() const
626 { return _M_test(3); }
627
628 consteval bool
629 _M_have_ssse3() const
630 { return _M_test(4); }
631
632 consteval bool
633 _M_have_sse4_1() const
634 { return _M_test(5); }
635
636 consteval bool
637 _M_have_sse4_2() const
638 { return _M_test(6); }
639
640 consteval bool
641 _M_have_popcnt() const
642 { return _M_test(7); }
643
644 consteval bool
645 _M_have_avx() const
646 { return _M_test(8); }
647
648 consteval bool
649 _M_have_f16c() const
650 { return _M_test(9); }
651
652 consteval bool
653 _M_have_bmi() const
654 { return _M_test(10); }
655
656 consteval bool
657 _M_have_bmi2() const
658 { return _M_test(11); }
659
660 consteval bool
661 _M_have_lzcnt() const
662 { return _M_test(12); }
663
664 consteval bool
665 _M_have_avx2() const
666 { return _M_test(13); }
667
668 consteval bool
669 _M_have_fma() const
670 { return _M_test(14); }
671
672 consteval bool
673 _M_have_avx512f() const
674 { return _M_test(15); }
675
676 consteval bool
677 _M_have_avx512cd() const
678 { return _M_test(16); }
679
680 consteval bool
681 _M_have_avx512dq() const
682 { return _M_test(17); }
683
684 consteval bool
685 _M_have_avx512bw() const
686 { return _M_test(18); }
687
688 consteval bool
689 _M_have_avx512vl() const
690 { return _M_test(19); }
691
692 consteval bool
693 _M_have_avx512bitalg() const
694 { return _M_test(20); }
695
696 consteval bool
697 _M_have_avx512vbmi() const
698 { return _M_test(21); }
699
700 consteval bool
701 _M_have_avx512vbmi2() const
702 { return _M_test(22); }
703
704 consteval bool
705 _M_have_avx512ifma() const
706 { return _M_test(23); }
707
708 consteval bool
709 _M_have_avx512vnni() const
710 { return _M_test(24); }
711
712 consteval bool
713 _M_have_avx512vpopcntdq() const
714 { return _M_test(25); }
715
716 consteval bool
717 _M_have_avx512fp16() const
718 { return _M_test(26); }
719
720 consteval bool
721 _M_have_avx512bf16() const
722 { return _M_test(27); }
723
724 consteval bool
725 _M_have_avxifma() const
726 { return _M_test(28); }
727
728 consteval bool
729 _M_have_avxneconvert() const
730 { return _M_test(29); }
731
732 consteval bool
733 _M_have_avxvnni() const
734 { return _M_test(30); }
735
736 consteval bool
737 _M_have_avxvnniint8() const
738 { return _M_test(31); }
739
740 consteval bool
741 _M_have_avxvnniint16() const
742 { return _M_test(32); }
743
744 consteval bool
745 _M_have_avx10_1() const
746 { return _M_test(33); }
747
748 consteval bool
749 _M_have_avx10_2() const
750 { return _M_test(34); }
751
752 consteval bool
753 _M_have_avx512vp2intersect() const
754 { return _M_test(35); }
755
756 consteval bool
757 _M_have_sse4a() const
758 { return _M_test(36); }
759
760 consteval bool
761 _M_have_fma4() const
762 { return _M_test(37); }
763
764 consteval bool
765 _M_have_xop() const
766 { return _M_test(38); }
767
768 template <typename _Tp>
769 consteval bool
770 _M_eval_as_f32() const
771 { return is_same_v<_Tp, _Float16> && !_M_have_avx512fp16(); }
772
773 consteval bool
774 _M_have_addsub() const
775 { return _M_have_sse3(); }
776 };
777
778 template <typename _Tp, _ArchTraits _Traits = {}>
779 consteval auto
780 __native_abi()
781 {
782 constexpr int __adj_sizeof = sizeof(_Tp) * (1 + is_same_v<_Tp, _Float16>);
783 if constexpr (!__vectorizable<_Tp>)
784 return _InvalidAbi();
785 else if constexpr (__complex_like<_Tp>)
786 {
787 constexpr auto __underlying = std::simd::__native_abi<typename _Tp::value_type>();
788 constexpr int __cx_size = __underlying._S_size / (__underlying._S_size == 1 ? 1 : 2);
789 return _Abi_t<__cx_size, 1, __underlying._S_variant, _AbiVariant::_CxIleav>();
790 }
791 else if constexpr (_Traits._M_have_avx512fp16())
792 return _Abi_t<64 / sizeof(_Tp), 1, _AbiVariant::_BitMask>();
793 else if constexpr (_Traits._M_have_avx512f())
794 return _Abi_t<64 / __adj_sizeof, 1, _AbiVariant::_BitMask>();
795 else if constexpr (is_same_v<_Tp, _Float16> && !_Traits._M_have_f16c())
796 return _Abi_t<1, 1>();
797 else if constexpr (_Traits._M_have_avx2())
798 return _Abi_t<32 / __adj_sizeof, 1>();
799 else if constexpr (_Traits._M_have_avx() && is_floating_point_v<_Tp>)
800 return _Abi_t<32 / __adj_sizeof, 1>();
801 else if constexpr (_Traits._M_have_sse2())
802 return _Abi_t<16 / __adj_sizeof, 1>();
803 else if constexpr (_Traits._M_have_sse() && is_floating_point_v<_Tp>
804 && sizeof(_Tp) == sizeof(float))
805 return _Abi_t<16 / __adj_sizeof, 1>();
806 // no MMX: we can't emit EMMS where it would be necessary
807 else
808 return _Abi_t<1, 1>();
809 }
810
811#else
812
813 // scalar fallback
814 struct _ArchTraits
815 {
816 __UINT64_TYPE__ _M_flags = 0;
817
818 constexpr bool
819 _M_test(int __bit) const
820 { return ((_M_flags >> __bit) & 1) == 1; }
821 };
822
823 template <typename _Tp>
824 consteval auto
825 __native_abi()
826 {
827 if constexpr (!__vectorizable<_Tp>)
828 return _InvalidAbi();
829 else if constexpr (__complex_like<_Tp>)
830 return _Abi_t<1, 1, _AbiVariant::_CxIleav>();
831 else
832 return _Abi_t<1, 1>();
833 }
834
835#endif
836
837 /** @internal
838 * You must use this type as template argument to function templates that are not declared
839 * always_inline (to avoid issues when linking code compiled with different compiler flags).
840 */
841 struct _TargetTraits
842 : _ArchTraits, _OptTraits
843 {};
844
845 /** @internal
846 * Alias for an ABI tag such that basic_vec<_Tp, __native_abi_t_<_Tp>> stores one SIMD register of
847 * optimal width.
848 *
849 * @tparam _Tp A vectorizable type.
850 *
851 * C++26 [simd.expos.abi]
852 */
853 template <typename _Tp>
854 using __native_abi_t = decltype(std::simd::__native_abi<_Tp>());
855
856 template <typename _Tp, int _Np, _TargetTraits _Target = {}>
857 consteval auto
858 __deduce_abi()
859 {
860 constexpr auto __native = std::simd::__native_abi<_Tp>();
861 if constexpr (0 == __native._S_size || _Np <= 0)
862 return _InvalidAbi();
863 else if constexpr (_Np == __native._S_size)
864 return __native;
865 else
866 return __native.template _S_resize<_Np>();
867 }
868
869 /** @internal
870 * Alias for an ABI tag @c A such that `basic_vec<_Tp, A>` stores @p _Np elements.
871 *
872 * C++26 [simd.expos.abi]
873 */
874 template <typename _Tp, int _Np>
875 using __deduce_abi_t = decltype(std::simd::__deduce_abi<_Tp, _Np>());
876
877 /** @internal
878 * \c rebind implementation detail for basic_vec, and basic_mask where we know the destination
879 * value-type
880 */
881 template <typename _Tp, int _Np, __abi_tag _A0, _ArchTraits = {}>
882 consteval auto
883 __abi_rebind()
884 {
885 if constexpr (_Np <= 0 || !__vectorizable<_Tp>)
886 return _InvalidAbi();
887
888 else
889 {
890 using _Native = remove_const_t<decltype(std::simd::__native_abi<_Tp>())>;
891 static_assert(0 != _Native::_S_size);
892 constexpr int __nreg = __div_ceil(_Np, _Native::_S_size);
893
894 // __scalar_abi_tag is sticky (unless we reach size 1, where we can't know whether it was
895 // an explicit __scalar_abi_tag before some resize_t)
896 if constexpr (__scalar_abi_tag<_Native> || (__scalar_abi_tag<_A0> && _A0::_S_size >= 2))
897 {
898 constexpr bool __remove_cx
899 = __filter_abi_variant(_A0::_S_variant, _AbiVariant::_CxVariants) != _AbiVariant()
900 && !__complex_like<_Tp>;
901 constexpr bool __add_cx
902 = __filter_abi_variant(_A0::_S_variant, _AbiVariant::_CxVariants) == _AbiVariant()
903 && __complex_like<_Tp>;
904
905 if constexpr (__remove_cx)
906 return _Abi_t<_Np, _Np,
907 __filter_abi_variant(_A0::_S_variant, _AbiVariant::_MaskVariants)>();
908 else if constexpr (__add_cx)
909 return _Abi_t<_Np, _Np, _A0::_S_variant,
910 __filter_abi_variant(_Native::_S_variant, _AbiVariant::_CxVariants)>();
911 else
912 return _A0::template _S_resize<_Np, _Np>();
913 }
914
915 else if constexpr (__complex_like<_Tp> && _A0::_S_is_cx_ctgus && _Native::_S_is_cx_ileav)
916 // we need half the number of registers since the number applies twice, to reals and
917 // imaginaries.
918 return _A0::template _S_resize<_Np, __div_ceil(__nreg, 2)>();
919
920 else if constexpr (__complex_like<_Tp> && _A0::_S_is_cx_ileav && _Native::_S_is_cx_ctgus)
921 return _A0::template _S_resize<_Np, __nreg * 2>();
922
923 else if constexpr (__complex_like<_Tp> && (_A0::_S_is_cx_ctgus || _A0::_S_is_cx_ileav))
924 return _A0::template _S_resize<_Np, __nreg>();
925
926 else if constexpr (__complex_like<_Tp>)
927 // Bit vs. Vec Mask determined by _A0, _CxVariant determined by _Native
928 return _Abi_t<_Native::_S_size, 1, _A0::_S_variant,
929 __filter_abi_variant(_Native::_S_variant, _AbiVariant::_CxVariants)>
930 ::template _S_resize<_Np, __nreg>();
931
932 else
933 return _Abi_t<_Native::_S_size, 1, __filter_abi_variant(_A0::_S_variant,
934 _AbiVariant::_MaskVariants)
935 >::template _S_resize<_Np, __nreg>();
936 }
937 }
938
939 /** @internal
940 * @c rebind implementation detail for basic_mask.
941 *
942 * The important difference here is that we have no information about the actual value-type other
943 * than its @c sizeof. So `_Bytes == 8` could mean `complex<float>`, @c double, or @c int64_t.
944 * E.g. `_Np == 4` with AVX w/o AVX2 that's `vector(4) int`, `vector(4) long long`, or `2x
945 * vector(2) long long`.
946 * That's why this overload has the additional @p _IsOnlyResize parameter, which tells us that the
947 * value-type doesn't change.
948 */
949 template <size_t _Bytes, int _Np, __abi_tag _A0, bool _IsOnlyResize, _ArchTraits _Traits = {}>
950 consteval auto
951 __abi_rebind()
952 {
953 constexpr bool __from_cx = _A0::_S_is_cx_ctgus || _A0::_S_is_cx_ileav;
954
955 if constexpr (_Bytes == 0 || _Np <= 0)
956 return _InvalidAbi();
957
958 // If the source ABI is complex, _Bytes == sizeof(complex<float>) or
959 // sizeof(complex<float16_t>), and _IsOnlyResize is true, then it's a mask<complex<float>,
960 // _Np>
961 else if constexpr (__from_cx && _IsOnlyResize && _Bytes == 2 * sizeof(double))
962 return __abi_rebind<complex<double>, _Np, _A0>();
963 else if constexpr (__from_cx && _IsOnlyResize && _Bytes == 2 * sizeof(float))
964 return __abi_rebind<complex<float>, _Np, _A0>();
965 else if constexpr (__from_cx && _IsOnlyResize && _Bytes == 2 * sizeof(_Float16))
966 return __abi_rebind<complex<_Float16>, _Np, _A0>();
967
968#if _GLIBCXX_X86
969 // AVX w/o AVX2:
970 // e.g. resize_t<8, mask<float, Whatever>> needs to be _Abi<8, 1> not _Abi<8, 2>
971 // We determine whether _A0 identifies an AVX vector by looking at the size of a native
972 // register. If it's 32, it's a YMM register, otherwise it's 16 or less.
973 else if constexpr (_IsOnlyResize
974 && _Traits._M_have_avx() && !_Traits._M_have_avx2()
975 && __bit_ceil(__div_ceil<unsigned>(
976 _A0::_S_size, _A0::_S_nreg)) * _Bytes == 32)
977 {
978 if constexpr (_Bytes == sizeof(double))
979 return __abi_rebind<double, _Np, _A0>();
980 else if constexpr (_Bytes == sizeof(float))
981 return __abi_rebind<float, _Np, _A0>();
982 else if constexpr (_Traits._M_have_f16c() && _Bytes == sizeof(_Float16))
983 return __abi_rebind<_Float16, _Np, _A0>();
984 else // impossible
985 static_assert(false);
986 }
987#endif
988
989 else
990 return __abi_rebind<__integer_from<_Bytes>, _Np, _A0>();
991 }
992
993 /** @internal
994 * Returns true unless _GLIBCXX_SIMD_COND_EXPLICIT_MASK_CONVERSION is defined.
995 *
996 * On IvyBridge, (vec<float> == 0.f) == (rebind_t<int, vec<float>> == 0) does not compile. It does
997 * compile on basically every other target, though. This is due to the difference in ABI tag:
998 * _Abi<8, 1, [...]> vs. _Abi<8, 2, [...]> (8 elements, 1 vs. 2 registers).
999 * I know how to define this function for libstdc++ to avoid interconvertible masks. The question
1000 * is whether we can specify this in general for C++29.
1001 *
1002 * Idea: Is rebind_t<integer-from<...>, mask>::abi_type the same type as
1003 * deduce-t<integer-from<...>, mask::size()>? If yes, it's the "better" ABI tag. However, this
1004 * makes the conversion behavior dependent on compiler flags. Probably not what we want.
1005 */
1006 template <typename _To, typename _From>
1007 consteval bool
1008 __is_mask_conversion_explicit([[maybe_unused]] size_t __b0, [[maybe_unused]] size_t __b1)
1009 {
1010 constexpr int __n = _To::_S_size;
1011 static_assert(__n == _From::_S_size);
1012#ifndef _GLIBCXX_SIMD_COND_EXPLICIT_MASK_CONVERSION
1013 /// C++26 [simd.mask.ctor] uses unconditional explicit
1014 return true;
1015#else
1016 if (__b0 != __b1)
1017 return true;
1018
1019 // converting to a bit-mask is better
1020 else if constexpr (_To::_S_is_vecmask != _From::_S_is_vecmask)
1021 return _To::_S_is_vecmask; // to vector-mask is explicit
1022
1023 // with vec-masks, fewer registers is better
1024 else if constexpr (_From::_S_nreg != _To::_S_nreg)
1025 return _From::_S_nreg < _To::_S_nreg;
1026
1027 // differ only on _Cx flags
1028 // interleaved complex is worse
1029 else if constexpr (_To::_S_is_cx_ileav)
1030 return true;
1031 else if constexpr (_From::_S_is_cx_ileav)
1032 return false;
1033
1034 // prefer non-_Cx over _CxCtgus
1035 else if constexpr (_To::_S_is_cx_ctgus)
1036 return true;
1037 else if constexpr (_From::_S_is_cx_ctgus)
1038 return false;
1039 else
1040 __builtin_unreachable();
1041#endif
1042 }
1043
1044 /** @internal
1045 * An alias for a signed integer type.
1046 *
1047 * libstdc++ unconditionally uses @c int here, since it matches the return type of
1048 * 'Bit Operation Builtins' in GCC.
1049 *
1050 * C++26 [simd.expos.defn]
1051 */
1052 using __simd_size_type = int;
1053
1054 // integral_constant shortcut
1055 template <__simd_size_type _Xp>
1056 inline constexpr integral_constant<__simd_size_type, _Xp> __simd_size_c = {};
1057
1058 // [simd.syn]
1059 template <typename _Tp, typename _Ap = __native_abi_t<_Tp>>
1060 class basic_vec;
1061
1062 template <typename _Tp, __simd_size_type _Np = __native_abi_t<_Tp>::_S_size>
1063 using vec = basic_vec<_Tp, __deduce_abi_t<_Tp, _Np>>;
1064
1065 template <size_t _Bytes, typename _Ap = __native_abi_t<__integer_from<_Bytes>>>
1066 class basic_mask;
1067
1068 template <typename _Tp, __simd_size_type _Np = __native_abi_t<_Tp>::_S_size>
1069 using mask = basic_mask<sizeof(_Tp), __deduce_abi_t<_Tp, _Np>>;
1070
1071 // [simd.ctor] load constructor constraints
1072 template <typename _Rg>
1073 consteval size_t
1074 __static_range_size(_Rg& __r)
1075 {
1076 if constexpr (ranges::__static_sized_range<_Rg>)
1077 return ranges::size(__r);
1078 else
1079 return dynamic_extent;
1080 }
1081
1082 // [simd.general] value-preserving
1083 template <typename _From, typename _To>
1084 concept __arithmetic_only_value_preserving_convertible_to
1085 = convertible_to<_From, _To> && is_arithmetic_v<_From> && is_arithmetic_v<_To>
1086 && !(is_signed_v<_From> && is_unsigned_v<_To>)
1087 && numeric_limits<_From>::digits <= numeric_limits<_To>::digits
1088 && numeric_limits<_From>::max() <= numeric_limits<_To>::max()
1089 && numeric_limits<_From>::lowest() >= numeric_limits<_To>::lowest();
1090
1091 /** @internal
1092 * Satisfied if the conversion from @p _From to @p _To is a value-preserving conversion.
1093 *
1094 * C++26 [simd.general]
1095 */
1096 template <typename _From, typename _To>
1097 concept __value_preserving_convertible_to
1098 = __arithmetic_only_value_preserving_convertible_to<_From, _To>
1099 || (__complex_like<_To> && __arithmetic_only_value_preserving_convertible_to<
1100 _From, typename _To::value_type>);
1101
1102 // LWG4420
1103 template <typename _From, typename _To>
1104 concept __explicitly_convertible_to = requires {
1105 static_cast<_To>(declval<_From>());
1106 };
1107
1108 /** @internal
1109 * C++26 [simd.expos]
1110 */
1111 // [simd.ctor] explicit(...) of broadcast ctor
1112 template <auto _From, typename _To>
1113 concept __non_narrowing_constexpr_conversion
1114 = is_arithmetic_v<decltype(_From)>
1115 && static_cast<decltype(_From)>(static_cast<_To>(_From)) == _From
1116 && !(unsigned_integral<_To> && _From < decltype(_From)())
1117 && _From <= std::numeric_limits<_To>::max()
1118 && _From >= std::numeric_limits<_To>::lowest();
1119
1120 // [simd.ctor] p4
1121 // This implements LWG4436 (submitted on 2025-10-28)
1122 template <typename _From, typename _To>
1123 concept __broadcast_constructible
1124 = ((convertible_to<_From, _To> && !is_arithmetic_v<remove_cvref_t<_From>>
1125 && !__constexpr_wrapper_like<remove_cvref_t<_From>>) // 4.1
1126 || __value_preserving_convertible_to<remove_cvref_t<_From>, _To> // 4.2
1127 || (__constexpr_wrapper_like<remove_cvref_t<_From>> // 4.3
1128 && __non_narrowing_constexpr_conversion<auto(remove_cvref_t<_From>::value),
1129 _To>));
1130
1131 // __higher_floating_point_rank_than<_Tp, U> (_Tp has higher or equal floating point rank than U)
1132 template <typename _From, typename _To>
1133 consteval bool
1134 __higher_floating_point_rank_than()
1135 {
1136 return floating_point<_From> && floating_point<_To>
1137 && is_same_v<common_type_t<_From, _To>, _From> && !is_same_v<_From, _To>;
1138 }
1139
1140 // __higher_integer_rank_than<_Tp, U> (_Tp has higher or equal integer rank than U)
1141 template <typename _From, typename _To>
1142 consteval bool
1143 __higher_integer_rank_than()
1144 {
1145 return integral<_From> && integral<_To>
1146 && (sizeof(_From) > sizeof(_To) || is_same_v<common_type_t<_From, _To>, _From>)
1147 && !is_same_v<_From, _To>;
1148 }
1149
1150 template <typename _From, typename _To>
1151 concept __higher_rank_than
1152 = __higher_floating_point_rank_than<_From, _To>() || __higher_integer_rank_than<_From, _To>();
1153
1154 struct __convert_flag;
1155
1156 template <typename _From, typename _To, typename... _Flags>
1157 concept __loadstore_convertible_to
1158 = same_as<_From, _To>
1159 || (__vectorizable<_From> && __vectorizable<_To>
1160 && (__value_preserving_convertible_to<_From, _To>
1161 || (__explicitly_convertible_to<_From, _To>
1162 && (std::is_same_v<_Flags, __convert_flag> || ...))));
1163
1164 template <typename _From, typename _To>
1165 concept __simd_generator_convertible_to
1166 = std::convertible_to<_From, _To>
1167 && (!is_arithmetic_v<_From> || __value_preserving_convertible_to<_From, _To>);
1168
1169 template <typename _Fp, typename _Tp, __simd_size_type... _Is>
1170 requires (__simd_generator_convertible_to<
1171 decltype(declval<_Fp>()(__simd_size_c<_Is>)), _Tp> && ...)
1172 constexpr void
1173 __simd_generator_invokable_impl(integer_sequence<__simd_size_type, _Is...>);
1174
1175 template <typename _Fp, typename _Tp, __simd_size_type _Np>
1176 concept __simd_generator_invokable = requires {
1177 __simd_generator_invokable_impl<_Fp, _Tp>(make_integer_sequence<__simd_size_type, _Np>());
1178 };
1179
1180 template <typename _Fp>
1181 concept __index_permutation_function_sized = requires(_Fp const& __f)
1182 {
1183 { __f(0, 0) } -> std::integral;
1184 };
1185
1186 template <typename _Fp, typename _Simd>
1187 concept __index_permutation_function
1188 = __index_permutation_function_sized<_Fp> || requires(_Fp const& __f) {
1189 { __f(0) } -> std::integral;
1190 };
1191
1192 /** @internal
1193 * The value of the @c _Bytes template argument to a @c basic_mask specialization.
1194 *
1195 * C++26 [simd.expos.defn]
1196 */
1197 template <typename _Tp>
1198 constexpr size_t __mask_element_size = 0;
1199
1200 template <size_t _Bytes, __abi_tag _Ap>
1201 constexpr size_t __mask_element_size<basic_mask<_Bytes, _Ap>> = _Bytes;
1202
1203 // [simd.expos]
1204 template <typename _Vp>
1205 concept __simd_vec_type
1206 = same_as<_Vp, basic_vec<typename _Vp::value_type, typename _Vp::abi_type>>
1207 && is_default_constructible_v<_Vp>;
1208
1209 template <typename _Vp>
1210 concept __simd_mask_type
1211 = same_as<_Vp, basic_mask<__mask_element_size<_Vp>, typename _Vp::abi_type>>
1212 && is_default_constructible_v<_Vp>;
1213
1214 /** @internal
1215 * Satisfied if @p _Tp is a data-parallel type.
1216 */
1217 template <typename _Vp>
1218 concept __simd_vec_or_mask_type = __simd_vec_type<_Vp> || __simd_mask_type<_Vp>;
1219
1220 template <typename _Vp>
1221 concept __simd_floating_point
1222 = __simd_vec_type<_Vp> && floating_point<typename _Vp::value_type>;
1223
1224 template <typename _Vp>
1225 concept __simd_integral
1226 = __simd_vec_type<_Vp> && integral<typename _Vp::value_type>;
1227
1228 template <typename _Vp>
1229 concept __simd_unsigned_integer
1230 = __simd_vec_type<_Vp> && __unsigned_integer<typename _Vp::value_type>;
1231
1232 template <typename _Vp>
1233 using __simd_complex_value_type = typename _Vp::value_type::value_type;
1234
1235 template <typename _Vp>
1236 concept __simd_complex
1237 = __simd_vec_type<_Vp> && __complex_like_impl<typename _Vp::value_type>;
1238
1239 template <typename _Tp>
1240 concept __converts_to_vec
1241 = __simd_vec_type<decltype(declval<const _Tp&>() + declval<const _Tp&>())>;
1242
1243 template <__converts_to_vec _Tp>
1244 using __deduced_vec_t = decltype(declval<const _Tp&>() + declval<const _Tp&>());
1245
1246 template <typename _Vp, typename _Tp>
1247 using __make_compatible_simd_t
1248 = decltype([] {
1249 using _Up = decltype(declval<const _Tp&>() + declval<const _Tp&>());
1250 if constexpr (__simd_vec_type<_Up>)
1251 return _Up();
1252 else
1253 return vec<_Up, _Vp::size()>();
1254 }());
1255
1256 template <typename _Tp>
1257 concept __math_floating_point = __simd_floating_point<__deduced_vec_t<_Tp>>;
1258
1259 template <typename _BinaryOperation, typename _Tp>
1260 concept __reduction_binary_operation
1261 = requires (const _BinaryOperation __binary_op, const vec<_Tp, 1> __v) {
1262 { __binary_op(__v, __v) } -> same_as<vec<_Tp, 1>>;
1263 };
1264
1265 /** @internal
1266 * Returns the highest index @c i where `(__bits >> i) & 1` equals @c 1.
1267 */
1268 [[__gnu__::__always_inline__]]
1269 constexpr __simd_size_type
1270 __highest_bit(std::unsigned_integral auto __bits)
1271 {
1272 using __gnu_cxx::__int_traits;
1273 constexpr auto _Nd = __int_traits<decltype(__bits)>::__digits;
1274 return _Nd - 1 - __countl_zero(__bits);
1275 }
1276
1277 template <__vectorizable _Tp, __simd_size_type _Np, __abi_tag _Ap>
1278 using __similar_mask = basic_mask<sizeof(_Tp), decltype(__abi_rebind<_Tp, _Np, _Ap>())>;
1279
1280 template <size_t _Bytes, __abi_tag _Ap>
1281 using __component_mask_for_ileav
1282 = basic_mask<_Bytes / 2,
1283 decltype(__abi_rebind<__float_from<_Bytes / 2>, _Ap::_S_size * 2, _Ap>())>;
1284
1285 template <size_t _Bytes, __abi_tag _Ap>
1286 using __component_mask_for_ctgus
1287 = basic_mask<_Bytes / 2,
1288 decltype(__abi_rebind<__float_from<_Bytes / 2>, _Ap::_S_size, _Ap>())>;
1289
1290 // Allow _Tp to be _InvalidInteger for __integer_from<16>
1291 template <typename _Tp, __simd_size_type _Np, __abi_tag _Ap>
1292 using __similar_vec = basic_vec<_Tp, decltype(__abi_rebind<_Tp, _Np, _Ap>())>;
1293
1294 // LWG4470 [simd.expos]
1295 template <size_t _Bytes, typename _Ap>
1296 using __simd_vec_from_mask_t = __similar_vec<__integer_from<_Bytes>, _Ap::_S_size, _Ap>;
1297
1298#if _GLIBCXX_SIMD_THROW_ON_BAD_VALUE // used for unit tests (also see P3844)
1299 class __bad_value_preserving_cast
1300 {};
1301
1302#define __glibcxx_on_bad_value_preserving_cast throw __bad_value_preserving_cast
1303#else
1304 void __bad_value_preserving_cast(); // not defined
1305
1306#define __glibcxx_on_bad_value_preserving_cast __bad_value_preserving_cast
1307#endif
1308
1309 template <typename _To, typename _From>
1310#if _GLIBCXX_SIMD_THROW_ON_BAD_VALUE // see P3844
1311 [[__gnu__::__optimize__("exceptions")]] // work around potential -fno-exceptions
1312#endif
1313 consteval _To
1314 __value_preserving_cast(const _From& __x)
1315 {
1316 static_assert(is_arithmetic_v<_From>);
1317 if constexpr (!__value_preserving_convertible_to<_From, _To>)
1318 {
1319 using _Up = typename __make_unsigned<_From>::__type;
1320 if (static_cast<_Up>(static_cast<_To>(__x)) != static_cast<_Up>(__x))
1321 __glibcxx_on_bad_value_preserving_cast();
1322 else if constexpr (is_signed_v<_From> && is_unsigned_v<_To>)
1323 {
1324 if (__x < _From())
1325 __glibcxx_on_bad_value_preserving_cast();
1326 }
1327 else if constexpr (unsigned_integral<_From> && signed_integral<_To>)
1328 {
1329 if (__x > numeric_limits<_To>::max())
1330 __glibcxx_on_bad_value_preserving_cast();
1331 }
1332 }
1333 return static_cast<_To>(__x);
1334 }
1335
1336 /** @internal
1337 * std::pair is not trivially copyable, this one is
1338 */
1339 template <typename _T0, typename _T1>
1340 struct __trivial_pair
1341 {
1342 _T0 _M_first;
1343 _T1 _M_second;
1344 };
1345
1346 template <typename _From, typename _To>
1347 concept __converts_trivially = convertible_to<_From, _To>
1348 && sizeof(_From) == sizeof(_To)
1349 && is_integral_v<_From> == is_integral_v<_To>
1350 && is_floating_point_v<_From> == is_floating_point_v<_To>;
1351
1352 [[__gnu__::__always_inline__]]
1353 constexpr void
1354 __bit_foreach(unsigned_integral auto __bits, auto&& __fun)
1355 {
1356 static_assert(sizeof(__bits) >= sizeof(int)); // avoid promotion to int
1357 while (__bits)
1358 {
1359 __fun(__countr_zero(__bits));
1360 __bits &= (__bits - 1);
1361 }
1362 }
1363
1364 /** @internal
1365 * Optimized @c memcpy for use in partial loads and stores.
1366 *
1367 * The implementation uses at most two fixed-size power-of-2 @c memcpy calls and reduces the
1368 * number of branches to a minimum. The variable size is achieved by overlapping two @c memcpy
1369 * calls.
1370 *
1371 * @tparam _Chunk Copies @p __n times @p _Chunk bytes.
1372 * @tparam _Max Copy no more than @p _Max bytes.
1373 *
1374 * @param __dst The destination pointer.
1375 * @param __src The source pointer.
1376 * @param __n Thu number of chunks that need to be copied.
1377 */
1378 template <size_t _Chunk, size_t _Max>
1379 inline void
1380 __memcpy_chunks(byte* __restrict__ __dst, const byte* __restrict__ __src,
1381 size_t __n)
1382 {
1383 static_assert(_Max <= 64);
1384 static_assert(__has_single_bit(_Chunk) && _Chunk <= 8);
1385 size_t __bytes = _Chunk * __n;
1386 if (__builtin_constant_p(__bytes))
1387 { // If __n is known via constant propagation use a single memcpy call. Since this is still
1388 // a fixed-size memcpy to the compiler, this leaves more room for optimization.
1389 __builtin_memcpy(__dst, __src, __bytes);
1390 }
1391 else if (__bytes > 32 && _Max > 32)
1392 {
1393 __builtin_memcpy(__dst, __src, 32);
1394 __bytes -= 32;
1395 __builtin_memcpy(__dst + __bytes, __src + __bytes, 32);
1396 }
1397 else if (__bytes > 16 && _Max > 16)
1398 {
1399 __builtin_memcpy(__dst, __src, 16);
1400 if constexpr (_Chunk == 8)
1401 {
1402 __bytes -= 8;
1403 __builtin_memcpy(__dst + __bytes, __src + __bytes, 8);
1404 }
1405 else
1406 {
1407 __bytes -= 16;
1408 __builtin_memcpy(__dst + __bytes, __src + __bytes, 16);
1409 }
1410 }
1411 else if (__bytes > 8 && _Max > 8)
1412 {
1413 __builtin_memcpy(__dst, __src, 8);
1414 if constexpr (_Chunk == 4)
1415 {
1416 __bytes -= 4;
1417 __builtin_memcpy(__dst + __bytes, __src + __bytes, 4);
1418 }
1419 else if constexpr (_Chunk < 4)
1420 {
1421 __bytes -= 8;
1422 __builtin_memcpy(__dst + __bytes, __src + __bytes, 8);
1423 }
1424 }
1425 else if (__bytes > 4 && _Max > 4)
1426 {
1427 __builtin_memcpy(__dst, __src, 4);
1428 if constexpr (_Chunk == 2)
1429 {
1430 __bytes -= 2;
1431 __builtin_memcpy(__dst + __bytes, __src + __bytes, 2);
1432 }
1433 else if constexpr (_Chunk == 1)
1434 {
1435 __bytes -= 4;
1436 __builtin_memcpy(__dst + __bytes, __src + __bytes, 4);
1437 }
1438 }
1439 else if (__bytes >= 2)
1440 {
1441 __builtin_memcpy(__dst, __src, 2);
1442 if constexpr (_Chunk == 2)
1443 {
1444 __bytes -= 2;
1445 __builtin_memcpy(__dst + __bytes, __src + __bytes, 2);
1446 }
1447 else if constexpr (_Chunk == 1)
1448 {
1449 __bytes -= 1;
1450 __builtin_memcpy(__dst + __bytes, __src + __bytes, 1);
1451 }
1452 }
1453 else if (__bytes == 1)
1454 __builtin_memcpy(__dst, __src, 1);
1455 }
1456
1457 // [simd.reductions] identity_element = *see below*
1458 template <typename _Tp, typename _BinaryOperation>
1459 requires __is_one_of<_BinaryOperation,
1460 plus<>, multiplies<>, bit_and<>, bit_or<>, bit_xor<>>::value
1461 consteval _Tp
1462 __default_identity_element()
1463 {
1464 if constexpr (same_as<_BinaryOperation, multiplies<>>)
1465 return _Tp(1);
1466 else if constexpr (same_as<_BinaryOperation, bit_and<>>)
1467 return _Tp(~_Tp());
1468 else
1469 return _Tp(0);
1470 }
1471} // namespace simd
1472_GLIBCXX_END_NAMESPACE_VERSION
1473} // namespace std
1474
1475#pragma GCC diagnostic pop
1476#endif // C++26
1477#endif // _GLIBCXX_SIMD_DETAILS_H
bit
concepts
limits
span
cmath
cstdint
stl_function.h
utility.h
c++config.h
std::underlying_type_t
typename underlying_type< _Tp >::type underlying_type_t
Alias template for underlying_type.
Definition type_traits:2979
std::make_unsigned_t
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition type_traits:2273
std::declval
auto declval() noexcept -> decltype(__declval< _Tp >(0))
Definition type_traits:2741
std
ISO C++ entities toplevel namespace is std.
std::make_integer_sequence
__make_integer_seq< integer_sequence, _Tp, _Num > make_integer_sequence
Alias template make_integer_sequence.
Definition utility.h:516
__gnu_cxx::__int_traits
__numeric_traits_integer< _Tp > __int_traits
Convenience alias for __numeric_traits<integer-type>.
Definition ext/numeric_traits.h:134
std::complex
Definition complex:157
std::__numeric_limits_base::digits
static constexpr int digits
Definition limits:224
std::numeric_limits::max
static constexpr _Tp max() noexcept
Definition limits:336
std::numeric_limits::lowest
static constexpr _Tp lowest() noexcept
Definition limits:342