libstdc++: simd.h Source File

libstdc++

simd.h
1// Definition of the public simd interfaces -*- C++ -*-
2
3// Copyright (C) 2020-2026 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26#define _GLIBCXX_EXPERIMENTAL_SIMD_H
27
28#if __cplusplus >= 201703L
29
30#include "simd_detail.h"
31#include "numeric_traits.h"
32#include <bit>
33#include <bitset>
34#ifdef _GLIBCXX_DEBUG_UB
35#include <cstdio> // for stderr
36#endif
37#include <cstring>
38#include <cmath>
39#include <functional>
40#include <iosfwd>
41#include <utility>
42#include <algorithm>
43
44#if _GLIBCXX_SIMD_X86INTRIN
45#include <x86intrin.h>
46#elif _GLIBCXX_SIMD_HAVE_NEON
47#pragma GCC diagnostic push
48// narrowing conversion of '__a' from 'uint64_t' {aka 'long long unsigned int'} to
49// 'int64x1_t' {aka 'long long int'} [-Wnarrowing]
50#pragma GCC diagnostic ignored "-Wnarrowing"
51#include <arm_neon.h>
52#pragma GCC diagnostic pop
53#endif
54#if _GLIBCXX_SIMD_HAVE_SVE
55#include <arm_sve.h>
56#endif
57
58/** @ingroup ts_simd
59 * @{
60 */
61/* There are several closely related types, with the following naming
62 * convention:
63 * _Tp: vectorizable (arithmetic) type (or any type)
64 * _TV: __vector_type_t<_Tp, _Np>
65 * _TW: _SimdWrapper<_Tp, _Np>
66 * _TI: __intrinsic_type_t<_Tp, _Np>
67 * _TVT: _VectorTraits<_TV> or _VectorTraits<_TW>
68 * If one additional type is needed use _U instead of _T.
69 * Otherwise use _T\d, _TV\d, _TW\d, TI\d, _TVT\d.
70 *
71 * More naming conventions:
72 * _Ap or _Abi: An ABI tag from the simd_abi namespace
73 * _Ip: often used for integer types with sizeof(_Ip) == sizeof(_Tp),
74 * _IV, _IW as for _TV, _TW
75 * _Np: number of elements (not bytes)
76 * _Bytes: number of bytes
77 *
78 * Variable names:
79 * __k: mask object (vector- or bitmask)
80 */
81_GLIBCXX_SIMD_BEGIN_NAMESPACE
82
83#if !_GLIBCXX_SIMD_X86INTRIN
84using __m128 [[__gnu__::__vector_size__(16)]] = float;
85using __m128d [[__gnu__::__vector_size__(16)]] = double;
86using __m128i [[__gnu__::__vector_size__(16)]] = long long;
87using __m256 [[__gnu__::__vector_size__(32)]] = float;
88using __m256d [[__gnu__::__vector_size__(32)]] = double;
89using __m256i [[__gnu__::__vector_size__(32)]] = long long;
90using __m512 [[__gnu__::__vector_size__(64)]] = float;
91using __m512d [[__gnu__::__vector_size__(64)]] = double;
92using __m512i [[__gnu__::__vector_size__(64)]] = long long;
93#endif
94
95#if _GLIBCXX_SIMD_HAVE_SVE
96constexpr inline int __sve_vectorized_size_bytes = __ARM_FEATURE_SVE_BITS / 8;
97#else
98constexpr inline int __sve_vectorized_size_bytes = 0;
99#endif
100
101namespace simd_abi {
102// simd_abi forward declarations {{{
103// implementation details:
104struct _Scalar;
105
106template <int _Np>
107 struct _Fixed;
108
109// There are two major ABIs that appear on different architectures.
110// Both have non-boolean values packed into an N Byte register
111// -> #elements = N / sizeof(T)
112// Masks differ:
113// 1. Use value vector registers for masks (all 0 or all 1)
114// 2. Use bitmasks (mask registers) with one bit per value in the corresponding
115// value vector
116//
117// Both can be partially used, masking off the rest when doing horizontal
118// operations or operations that can trap (e.g. FP_INVALID or integer division
119// by 0). This is encoded as the number of used bytes.
120template <int _UsedBytes>
121 struct _VecBuiltin;
122
123template <int _UsedBytes>
124 struct _VecBltnBtmsk;
125
126template <int _UsedBytes, int _TotalBytes = __sve_vectorized_size_bytes>
127 struct _SveAbi;
128
129template <typename _Tp, int _Np>
130 using _VecN = _VecBuiltin<sizeof(_Tp) * _Np>;
131
132template <int _UsedBytes = 16>
133 using _Sse = _VecBuiltin<_UsedBytes>;
134
135template <int _UsedBytes = 32>
136 using _Avx = _VecBuiltin<_UsedBytes>;
137
138template <int _UsedBytes = 64>
139 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
140
141template <int _UsedBytes = 16>
142 using _Neon = _VecBuiltin<_UsedBytes>;
143
144template <int _UsedBytes = __sve_vectorized_size_bytes>
145 using _Sve = _SveAbi<_UsedBytes, __sve_vectorized_size_bytes>;
146
147// implementation-defined:
148using __sse = _Sse<>;
149using __avx = _Avx<>;
150using __avx512 = _Avx512<>;
151using __neon = _Neon<>;
152using __neon128 = _Neon<16>;
153using __neon64 = _Neon<8>;
154using __sve = _Sve<>;
155
156// standard:
157template <typename _Tp, size_t _Np, typename...>
158 struct deduce;
159
160template <int _Np>
161 using fixed_size = _Fixed<_Np>;
162
163using scalar = _Scalar;
164
165// }}}
166} // namespace simd_abi
167// forward declarations is_simd(_mask), simd(_mask), simd_size {{{
168template <typename _Tp>
169 struct is_simd;
170
171template <typename _Tp>
172 struct is_simd_mask;
173
174template <typename _Tp, typename _Abi>
175 class simd;
176
177template <typename _Tp, typename _Abi>
178 class simd_mask;
179
180template <typename _Tp, typename _Abi>
181 struct simd_size;
182
183// }}}
184// load/store flags {{{
185struct element_aligned_tag
186{
187 template <typename _Tp, typename _Up = typename _Tp::value_type>
188 static constexpr size_t _S_alignment = alignof(_Up);
189
190 template <typename _Tp, typename _Up>
191 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
192 _S_apply(_Up* __ptr)
193 { return __ptr; }
194};
195
196struct vector_aligned_tag
197{
198 template <typename _Tp, typename _Up = typename _Tp::value_type>
199 static constexpr size_t _S_alignment
200 = std::__bit_ceil(sizeof(_Up) * _Tp::size());
201
202 template <typename _Tp, typename _Up>
203 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
204 _S_apply(_Up* __ptr)
205 { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>)); }
206};
207
208template <size_t _Np> struct overaligned_tag
209{
210 template <typename _Tp, typename _Up = typename _Tp::value_type>
211 static constexpr size_t _S_alignment = _Np;
212
213 template <typename _Tp, typename _Up>
214 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
215 _S_apply(_Up* __ptr)
216 { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); }
217};
218
219inline constexpr element_aligned_tag element_aligned = {};
220
221inline constexpr vector_aligned_tag vector_aligned = {};
222
223template <size_t _Np>
224 inline constexpr overaligned_tag<_Np> overaligned = {};
225
226// }}}
227template <size_t _Xp>
228 using _SizeConstant = integral_constant<size_t, _Xp>;
229// constexpr feature detection{{{
230constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
231constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
232constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
233constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
234constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
235constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
236constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
237constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
238constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
239constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
240constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
241constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
242constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
243constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
244constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
245constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
246constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
247constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
248constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
249constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
250constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
251constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
252constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
253constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
254constexpr inline bool __have_avx512bitalg = _GLIBCXX_SIMD_HAVE_AVX512BITALG;
255constexpr inline bool __have_avx512vbmi2 = _GLIBCXX_SIMD_HAVE_AVX512VBMI2;
256constexpr inline bool __have_avx512vbmi = _GLIBCXX_SIMD_HAVE_AVX512VBMI;
257constexpr inline bool __have_avx512ifma = _GLIBCXX_SIMD_HAVE_AVX512IFMA;
258constexpr inline bool __have_avx512cd = _GLIBCXX_SIMD_HAVE_AVX512CD;
259constexpr inline bool __have_avx512vnni = _GLIBCXX_SIMD_HAVE_AVX512VNNI;
260constexpr inline bool __have_avx512vpopcntdq = _GLIBCXX_SIMD_HAVE_AVX512VPOPCNTDQ;
261constexpr inline bool __have_avx512vp2intersect = _GLIBCXX_SIMD_HAVE_AVX512VP2INTERSECT;
262
263constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
264constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
265constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
266constexpr inline bool __support_neon_float =
267#if defined __GCC_IEC_559
268 __GCC_IEC_559 == 0;
269#elif defined __FAST_MATH__
270 true;
271#else
272 false;
273#endif
274
275constexpr inline bool __have_sve = _GLIBCXX_SIMD_HAVE_SVE;
276constexpr inline bool __have_sve2 = _GLIBCXX_SIMD_HAVE_SVE2;
277
278#ifdef _ARCH_PWR10
279constexpr inline bool __have_power10vec = true;
280#else
281constexpr inline bool __have_power10vec = false;
282#endif
283#ifdef __POWER9_VECTOR__
284constexpr inline bool __have_power9vec = true;
285#else
286constexpr inline bool __have_power9vec = false;
287#endif
288#if defined __POWER8_VECTOR__
289constexpr inline bool __have_power8vec = true;
290#else
291constexpr inline bool __have_power8vec = __have_power9vec;
292#endif
293#if defined __VSX__
294constexpr inline bool __have_power_vsx = true;
295#else
296constexpr inline bool __have_power_vsx = __have_power8vec;
297#endif
298#if defined __ALTIVEC__
299constexpr inline bool __have_power_vmx = true;
300#else
301constexpr inline bool __have_power_vmx = __have_power_vsx;
302#endif
303
304// }}}
305
306namespace __detail
307{
308#ifdef math_errhandling
309 // Determines _S_handle_fpexcept from math_errhandling if it is defined and expands to a constant
310 // expression. math_errhandling may expand to an extern symbol, in which case a constexpr value
311 // must be guessed.
312 template <int = math_errhandling>
313 constexpr bool
314 __handle_fpexcept_impl(int)
315 { return math_errhandling & MATH_ERREXCEPT; }
316#endif
317
318 // Fallback if math_errhandling doesn't work: with fast-math assume floating-point exceptions are
319 // ignored, otherwise implement correct exception behavior.
320 constexpr bool
321 __handle_fpexcept_impl(float)
322 {
323#if defined __FAST_MATH__
324 return false;
325#else
326 return true;
327#endif
328 }
329
330 /// True if math functions must raise floating-point exceptions as specified by C17.
331 static constexpr bool _S_handle_fpexcept = __handle_fpexcept_impl(0);
332
333 constexpr std::uint_least64_t
334 __floating_point_flags()
335 {
336 std::uint_least64_t __flags = 0;
337 if constexpr (_S_handle_fpexcept)
338 __flags |= 1;
339#ifdef __FAST_MATH__
340 __flags |= 1 << 1;
341#elif __FINITE_MATH_ONLY__
342 __flags |= 2 << 1;
343#elif __GCC_IEC_559 < 2
344 __flags |= 3 << 1;
345#endif
346 __flags |= (__FLT_EVAL_METHOD__ + 1) << 3;
347 return __flags;
348 }
349
350 constexpr std::uint_least64_t
351 __machine_flags()
352 {
353 if constexpr (__have_mmx || __have_sse)
354 return __have_mmx
355 | (__have_sse << 1)
356 | (__have_sse2 << 2)
357 | (__have_sse3 << 3)
358 | (__have_ssse3 << 4)
359 | (__have_sse4_1 << 5)
360 | (__have_sse4_2 << 6)
361 | (__have_xop << 7)
362 | (__have_avx << 8)
363 | (__have_avx2 << 9)
364 | (__have_bmi << 10)
365 | (__have_bmi2 << 11)
366 | (__have_lzcnt << 12)
367 | (__have_sse4a << 13)
368 | (__have_fma << 14)
369 | (__have_fma4 << 15)
370 | (__have_f16c << 16)
371 | (__have_popcnt << 17)
372 | (__have_avx512f << 18)
373 | (__have_avx512dq << 19)
374 | (__have_avx512vl << 20)
375 | (__have_avx512bw << 21)
376 | (__have_avx512bitalg << 22)
377 | (__have_avx512vbmi2 << 23)
378 | (__have_avx512vbmi << 24)
379 | (__have_avx512ifma << 25)
380 | (__have_avx512cd << 26)
381 | (__have_avx512vnni << 27)
382 | (__have_avx512vpopcntdq << 28)
383 | (__have_avx512vp2intersect << 29);
384 else if constexpr (__have_neon || __have_sve)
385 return __have_neon
386 | (__have_neon_a32 << 1)
387 | (__have_neon_a64 << 2)
388 | (__have_neon_a64 << 2)
389 | (__support_neon_float << 3)
390 | (__have_sve << 4)
391 | (__have_sve2 << 5);
392 else if constexpr (__have_power_vmx)
393 return __have_power_vmx
394 | (__have_power_vsx << 1)
395 | (__have_power8vec << 2)
396 | (__have_power9vec << 3)
397 | (__have_power10vec << 4);
398 else
399 return 0;
400 }
401
402 namespace
403 {
404 struct _OdrEnforcer {};
405 }
406
407 template <std::uint_least64_t...>
408 struct _MachineFlagsTemplate {};
409
410 /**@internal
411 * Use this type as default template argument to all function templates that
412 * are not declared always_inline. It ensures, that a function
413 * specialization, which the compiler decides not to inline, has a unique symbol
414 * (_OdrEnforcer) or a symbol matching the machine/architecture flags
415 * (_MachineFlagsTemplate). This helps to avoid ODR violations in cases where
416 * users link TUs compiled with different flags. This is especially important
417 * for using simd in libraries.
418 */
419 using __odr_helper
420 = conditional_t<__machine_flags() == 0, _OdrEnforcer,
421 _MachineFlagsTemplate<__machine_flags(), __floating_point_flags()>>;
422
423 struct _Minimum
424 {
425 template <typename _Tp>
426 _GLIBCXX_SIMD_INTRINSIC constexpr
427 _Tp
428 operator()(_Tp __a, _Tp __b) const
429 {
430 using std::min;
431 return min(__a, __b);
432 }
433 };
434
435 struct _Maximum
436 {
437 template <typename _Tp>
438 _GLIBCXX_SIMD_INTRINSIC constexpr
439 _Tp
440 operator()(_Tp __a, _Tp __b) const
441 {
442 using std::max;
443 return max(__a, __b);
444 }
445 };
446} // namespace __detail
447
448// unrolled/pack execution helpers
449// __execute_n_times{{{
450template <typename _Fp, size_t... _I>
451 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
452 void
453 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
454 { ((void)__f(_SizeConstant<_I>()), ...); }
455
456template <typename _Fp>
457 _GLIBCXX_SIMD_INTRINSIC constexpr void
458 __execute_on_index_sequence(_Fp&&, index_sequence<>)
459 { }
460
461template <size_t _Np, typename _Fp>
462 _GLIBCXX_SIMD_INTRINSIC constexpr void
463 __execute_n_times(_Fp&& __f)
464 {
465 __execute_on_index_sequence(static_cast<_Fp&&>(__f),
466 make_index_sequence<_Np>{});
467 }
468
469// }}}
470// __generate_from_n_evaluations{{{
471template <typename _R, typename _Fp, size_t... _I>
472 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
473 _R
474 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
475 { return _R{__f(_SizeConstant<_I>())...}; }
476
477template <size_t _Np, typename _R, typename _Fp>
478 _GLIBCXX_SIMD_INTRINSIC constexpr _R
479 __generate_from_n_evaluations(_Fp&& __f)
480 {
481 return __execute_on_index_sequence_with_return<_R>(
482 static_cast<_Fp&&>(__f), make_index_sequence<_Np>{});
483 }
484
485// }}}
486// __call_with_n_evaluations{{{
487template <size_t... _I, typename _F0, typename _FArgs>
488 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
489 auto
490 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
491 { return __f0(__fargs(_SizeConstant<_I>())...); }
492
493template <size_t _Np, typename _F0, typename _FArgs>
494 _GLIBCXX_SIMD_INTRINSIC constexpr auto
495 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
496 {
497 return __call_with_n_evaluations(make_index_sequence<_Np>{},
498 static_cast<_F0&&>(__f0),
499 static_cast<_FArgs&&>(__fargs));
500 }
501
502// }}}
503// __call_with_subscripts{{{
504template <size_t _First = 0, size_t... _It, typename _Tp, typename _Fp>
505 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
506 auto
507 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
508 { return __fun(__x[_First + _It]...); }
509
510template <size_t _Np, size_t _First = 0, typename _Tp, typename _Fp>
511 _GLIBCXX_SIMD_INTRINSIC constexpr auto
512 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
513 {
514 return __call_with_subscripts<_First>(static_cast<_Tp&&>(__x),
515 make_index_sequence<_Np>(),
516 static_cast<_Fp&&>(__fun));
517 }
518
519// }}}
520
521// vvv ---- type traits ---- vvv
522// integer type aliases{{{
523using _UChar = unsigned char;
524using _SChar = signed char;
525using _UShort = unsigned short;
526using _UInt = unsigned int;
527using _ULong = unsigned long;
528using _ULLong = unsigned long long;
529using _LLong = long long;
530
531//}}}
532// __first_of_pack{{{
533template <typename _T0, typename...>
534 struct __first_of_pack
535 { using type = _T0; };
536
537template <typename... _Ts>
538 using __first_of_pack_t = typename __first_of_pack<_Ts...>::type;
539
540//}}}
541// __value_type_or_identity_t {{{
542template <typename _Tp>
543 typename _Tp::value_type
544 __value_type_or_identity_impl(int);
545
546template <typename _Tp>
547 _Tp
548 __value_type_or_identity_impl(float);
549
550template <typename _Tp>
551 using __value_type_or_identity_t
552 = decltype(__value_type_or_identity_impl<_Tp>(int()));
553
554// }}}
555// __is_vectorizable {{{
556template <typename _Tp>
557 struct __is_vectorizable : public is_arithmetic<_Tp> {};
558
559template <>
560 struct __is_vectorizable<bool> : public false_type {};
561
562template <typename _Tp>
563 inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
564
565// Deduces to a vectorizable type
566template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
567 using _Vectorizable = _Tp;
568
569// }}}
570// _LoadStorePtr / __is_possible_loadstore_conversion {{{
571template <typename _Ptr, typename _ValueType>
572 struct __is_possible_loadstore_conversion
573 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
574
575template <>
576 struct __is_possible_loadstore_conversion<bool, bool> : true_type {};
577
578// Deduces to a type allowed for load/store with the given value type.
579template <typename _Ptr, typename _ValueType,
580 typename = enable_if_t<
581 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
582 using _LoadStorePtr = _Ptr;
583
584// }}}
585// __is_bitmask{{{
586template <typename _Tp, typename = void_t<>>
587 struct __is_bitmask : false_type {};
588
589template <typename _Tp>
590 inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
591
592// the __mmaskXX case:
593template <typename _Tp>
594 struct __is_bitmask<_Tp,
595 void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>>
596 : true_type {};
597
598// }}}
599// __int_for_sizeof{{{
600#pragma GCC diagnostic push
601#pragma GCC diagnostic ignored "-Wpedantic"
602template <size_t _Bytes>
603 constexpr auto
604 __int_for_sizeof()
605 {
606 static_assert(_Bytes > 0);
607 if constexpr (_Bytes == sizeof(int))
608 return int();
609 else if constexpr (_Bytes == sizeof(_SChar))
610 return _SChar();
611 else if constexpr (_Bytes == sizeof(short))
612 return short();
613 else if constexpr (_Bytes == sizeof(long))
614 return long();
615 else if constexpr (_Bytes == sizeof(_LLong))
616 return _LLong();
617 #ifdef __SIZEOF_INT128__
618 else if constexpr (_Bytes == sizeof(__int128))
619 return __int128();
620 #endif // __SIZEOF_INT128__
621 else if constexpr (_Bytes % sizeof(int) == 0)
622 {
623 constexpr size_t _Np = _Bytes / sizeof(int);
624 struct _Ip
625 {
626 int _M_data[_Np];
627
628 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
629 operator&(_Ip __rhs) const
630 {
631 return __generate_from_n_evaluations<_Np, _Ip>(
632 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
633 return __rhs._M_data[__i] & _M_data[__i];
634 });
635 }
636
637 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
638 operator|(_Ip __rhs) const
639 {
640 return __generate_from_n_evaluations<_Np, _Ip>(
641 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
642 return __rhs._M_data[__i] | _M_data[__i];
643 });
644 }
645
646 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
647 operator^(_Ip __rhs) const
648 {
649 return __generate_from_n_evaluations<_Np, _Ip>(
650 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
651 return __rhs._M_data[__i] ^ _M_data[__i];
652 });
653 }
654
655 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
656 operator~() const
657 {
658 return __generate_from_n_evaluations<_Np, _Ip>(
659 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return ~_M_data[__i]; });
660 }
661 };
662 return _Ip{};
663 }
664 else
665 static_assert(_Bytes == 0, "this should be unreachable");
666 }
667#pragma GCC diagnostic pop
668
669template <typename _Tp>
670 using __int_for_sizeof_t = decltype(__int_for_sizeof<sizeof(_Tp)>());
671
672template <size_t _Np>
673 using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
674
675// }}}
676// __is_fixed_size_abi{{{
677template <typename _Tp>
678 struct __is_fixed_size_abi : false_type {};
679
680template <int _Np>
681 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> : true_type {};
682
683template <typename _Tp>
684 inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
685
686// }}}
687// __is_scalar_abi {{{
688template <typename _Abi>
689 constexpr bool
690 __is_scalar_abi()
691 { return is_same_v<simd_abi::scalar, _Abi>; }
692
693// }}}
694// __abi_bytes_v {{{
695template <template <int> class _Abi, int _Bytes>
696 constexpr int
697 __abi_bytes_impl(_Abi<_Bytes>*)
698 { return _Bytes; }
699
700template <typename _Tp>
701 constexpr int
702 __abi_bytes_impl(_Tp*)
703 { return -1; }
704
705template <typename _Abi>
706 inline constexpr int __abi_bytes_v
707 = __abi_bytes_impl(static_cast<_Abi*>(nullptr));
708
709// }}}
710// __is_builtin_bitmask_abi {{{
711template <typename _Abi>
712 constexpr bool
713 __is_builtin_bitmask_abi()
714 { return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
715
716// }}}
717// __is_sse_abi {{{
718template <typename _Abi>
719 constexpr bool
720 __is_sse_abi()
721 {
722 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
723 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
724 }
725
726// }}}
727// __is_avx_abi {{{
728template <typename _Abi>
729 constexpr bool
730 __is_avx_abi()
731 {
732 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
733 return _Bytes > 16 && _Bytes <= 32
734 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
735 }
736
737// }}}
738// __is_avx512_abi {{{
739template <typename _Abi>
740 constexpr bool
741 __is_avx512_abi()
742 {
743 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
744 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
745 }
746
747// }}}
748// __is_neon_abi {{{
749template <typename _Abi>
750 constexpr bool
751 __is_neon_abi()
752 {
753 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
754 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
755 }
756
757// }}}
758// __is_sve_abi {{{
759template <typename _Abi>
760 constexpr bool
761 __is_sve_abi()
762 {
763 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
764 return _Bytes <= __sve_vectorized_size_bytes && is_same_v<simd_abi::_Sve<_Bytes>, _Abi>;
765 }
766
767// }}}
768// __make_dependent_t {{{
769template <typename, typename _Up>
770 struct __make_dependent
771 { using type = _Up; };
772
773template <typename _Tp, typename _Up>
774 using __make_dependent_t = typename __make_dependent<_Tp, _Up>::type;
775
776// }}}
777// ^^^ ---- type traits ---- ^^^
778
779// __invoke_ub{{{
780template <typename... _Args>
781 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE void
782 __invoke_ub([[maybe_unused]] const char* __msg, [[maybe_unused]] const _Args&... __args)
783 {
784#ifdef _GLIBCXX_DEBUG_UB
785 __builtin_fprintf(stderr, __msg, __args...);
786 __builtin_trap();
787#else
788 __builtin_unreachable();
789#endif
790 }
791
792// }}}
793// __assert_unreachable{{{
794template <typename _Tp>
795 struct __assert_unreachable
796 { static_assert(!is_same_v<_Tp, _Tp>, "this should be unreachable"); };
797
798// }}}
799// __size_or_zero_v {{{
800template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
801 constexpr size_t
802 __size_or_zero_dispatch(int)
803 { return _Np; }
804
805template <typename _Tp, typename _Ap>
806 constexpr size_t
807 __size_or_zero_dispatch(float)
808 { return 0; }
809
810template <typename _Tp, typename _Ap>
811 inline constexpr size_t __size_or_zero_v
812 = __size_or_zero_dispatch<_Tp, _Ap>(0);
813
814// }}}
815// __div_roundup {{{
816inline constexpr size_t
817__div_roundup(size_t __a, size_t __b)
818{ return (__a + __b - 1) / __b; }
819
820// }}}
821// _ExactBool{{{
822class _ExactBool
823{
824 const bool _M_data;
825
826public:
827 _GLIBCXX_SIMD_INTRINSIC constexpr
828 _ExactBool(bool __b) : _M_data(__b) {}
829
830 _ExactBool(int) = delete;
831
832 _GLIBCXX_SIMD_INTRINSIC constexpr
833 operator bool() const
834 { return _M_data; }
835};
836
837// }}}
838// __may_alias{{{
839/**@internal
840 * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an
841 * aliasing pointer. This adds the __may_alias attribute to _Tp (with compilers
842 * that support it).
843 */
844template <typename _Tp>
845 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
846
847// }}}
848// _UnsupportedBase {{{
849// simd and simd_mask base for unsupported <_Tp, _Abi>
850struct _UnsupportedBase
851{
852 _UnsupportedBase() = delete;
853 _UnsupportedBase(const _UnsupportedBase&) = delete;
854 _UnsupportedBase& operator=(const _UnsupportedBase&) = delete;
855 ~_UnsupportedBase() = delete;
856};
857
858// }}}
859// _InvalidTraits {{{
860/**
861 * @internal
862 * Defines the implementation of __a given <_Tp, _Abi>.
863 *
864 * Implementations must ensure that only valid <_Tp, _Abi> instantiations are
865 * possible. Static assertions in the type definition do not suffice. It is
866 * important that SFINAE works.
867 */
868struct _InvalidTraits
869{
870 using _IsValid = false_type;
871 using _SimdBase = _UnsupportedBase;
872 using _MaskBase = _UnsupportedBase;
873
874 static constexpr size_t _S_full_size = 0;
875 static constexpr bool _S_is_partial = false;
876
877 static constexpr size_t _S_simd_align = 1;
878 struct _SimdImpl;
879 struct _SimdMember {};
880 struct _SimdCastType;
881
882 static constexpr size_t _S_mask_align = 1;
883 struct _MaskImpl;
884 struct _MaskMember {};
885 struct _MaskCastType;
886};
887
888// }}}
889// _SimdTraits {{{
890template <typename _Tp, typename _Abi, typename = void_t<>>
891 struct _SimdTraits : _InvalidTraits {};
892
893// }}}
894// __private_init, __bitset_init{{{
895/**
896 * @internal
897 * Tag used for private init constructor of simd and simd_mask
898 */
899inline constexpr struct _PrivateInit {} __private_init = {};
900
901inline constexpr struct _BitsetInit {} __bitset_init = {};
902
903// }}}
904// __is_narrowing_conversion<_From, _To>{{{
905template <typename _From, typename _To, bool = is_arithmetic_v<_From>,
906 bool = is_arithmetic_v<_To>>
907 struct __is_narrowing_conversion;
908
909// ignore "signed/unsigned mismatch" in the following trait.
910// The implicit conversions will do the right thing here.
911template <typename _From, typename _To>
912 struct __is_narrowing_conversion<_From, _To, true, true>
913 : public __bool_constant<(
914 __digits_v<_From> > __digits_v<_To>
915 || __finite_max_v<_From> > __finite_max_v<_To>
916 || __finite_min_v<_From> < __finite_min_v<_To>
917 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
918
919template <typename _Tp>
920 struct __is_narrowing_conversion<_Tp, bool, true, true>
921 : public true_type {};
922
923template <>
924 struct __is_narrowing_conversion<bool, bool, true, true>
925 : public false_type {};
926
927template <typename _Tp>
928 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
929 : public false_type {};
930
931template <typename _From, typename _To>
932 struct __is_narrowing_conversion<_From, _To, false, true>
933 : public negation<is_convertible<_From, _To>> {};
934
935// }}}
936// __converts_to_higher_integer_rank{{{
937template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))>
938 struct __converts_to_higher_integer_rank : public true_type {};
939
940// this may fail for char -> short if sizeof(char) == sizeof(short)
941template <typename _From, typename _To>
942 struct __converts_to_higher_integer_rank<_From, _To, false>
943 : public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
944
945// }}}
946// __data(simd/simd_mask) {{{
947template <typename _Tp, typename _Ap>
948 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
949 __data(const simd<_Tp, _Ap>& __x);
950
951template <typename _Tp, typename _Ap>
952 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
953 __data(simd<_Tp, _Ap>& __x);
954
955template <typename _Tp, typename _Ap>
956 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
957 __data(const simd_mask<_Tp, _Ap>& __x);
958
959template <typename _Tp, typename _Ap>
960 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
961 __data(simd_mask<_Tp, _Ap>& __x);
962
963// }}}
964// _SimdConverter {{{
965template <typename _FromT, typename _FromA, typename _ToT, typename _ToA,
966 typename = void>
967 struct _SimdConverter;
968
969template <typename _Tp, typename _Ap>
970 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
971 {
972 template <typename _Up>
973 _GLIBCXX_SIMD_INTRINSIC const _Up&
974 operator()(const _Up& __x)
975 { return __x; }
976 };
977
978// }}}
979// __to_value_type_or_member_type {{{
980template <typename _V>
981 _GLIBCXX_SIMD_INTRINSIC constexpr auto
982 __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x))
983 { return __data(__x); }
984
985template <typename _V>
986 _GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type&
987 __to_value_type_or_member_type(const typename _V::value_type& __x)
988 { return __x; }
989
990// }}}
991// __bool_storage_member_type{{{
992template <size_t _Size>
993 struct __bool_storage_member_type;
994
995template <size_t _Size>
996 using __bool_storage_member_type_t =
997 typename __bool_storage_member_type<_Size>::type;
998
999// }}}
1000// _SimdTuple {{{
1001// why not tuple?
1002// 1. tuple gives no guarantee about the storage order, but I require
1003// storage
1004// equivalent to array<_Tp, _Np>
1005// 2. direct access to the element type (first template argument)
1006// 3. enforces equal element type, only different _Abi types are allowed
1007template <typename _Tp, typename... _Abis>
1008 struct _SimdTuple;
1009
1010//}}}
1011// __fixed_size_storage_t {{{
1012template <typename _Tp, int _Np>
1013 struct __fixed_size_storage;
1014
1015template <typename _Tp, int _Np>
1016 using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _Np>::type;
1017
1018// }}}
1019// _SimdWrapper fwd decl{{{
1020template <typename _Tp, size_t _Size, typename = void_t<>>
1021 struct _SimdWrapper;
1022
1023template <typename _Tp>
1024 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>;
1025template <typename _Tp>
1026 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>;
1027template <typename _Tp>
1028 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>;
1029template <typename _Tp>
1030 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>;
1031
1032template <typename _Tp, size_t _Width>
1033 struct _SveSimdWrapper;
1034
1035// }}}
1036// __is_simd_wrapper {{{
1037template <typename _Tp>
1038 struct __is_simd_wrapper : false_type {};
1039
1040template <typename _Tp, size_t _Np>
1041 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> : true_type {};
1042
1043template <typename _Tp>
1044 inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
1045
1046// }}}
1047// _BitOps {{{
1048struct _BitOps
1049{
1050 // _S_bit_iteration {{{
1051 template <typename _Tp, typename _Fp>
1052 static void
1053 _S_bit_iteration(_Tp __mask, _Fp&& __f)
1054 {
1055 static_assert(sizeof(_ULLong) >= sizeof(_Tp));
1056 conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k;
1057 if constexpr (is_convertible_v<_Tp, decltype(__k)>)
1058 __k = __mask;
1059 else
1060 __k = __mask.to_ullong();
1061 while(__k)
1062 {
1063 __f(std::__countr_zero(__k));
1064 __k &= (__k - 1);
1065 }
1066 }
1067
1068 //}}}
1069};
1070
1071//}}}
1072// __increment, __decrement {{{
1073template <typename _Tp = void>
1074 struct __increment
1075 { constexpr _Tp operator()(_Tp __a) const { return ++__a; } };
1076
1077template <>
1078 struct __increment<void>
1079 {
1080 template <typename _Tp>
1081 constexpr _Tp
1082 operator()(_Tp __a) const
1083 { return ++__a; }
1084 };
1085
1086template <typename _Tp = void>
1087 struct __decrement
1088 { constexpr _Tp operator()(_Tp __a) const { return --__a; } };
1089
1090template <>
1091 struct __decrement<void>
1092 {
1093 template <typename _Tp>
1094 constexpr _Tp
1095 operator()(_Tp __a) const
1096 { return --__a; }
1097 };
1098
1099// }}}
1100// _ValuePreserving(OrInt) {{{
1101template <typename _From, typename _To,
1102 typename = enable_if_t<negation<
1103 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
1104 using _ValuePreserving = _From;
1105
1106template <typename _From, typename _To,
1107 typename _DecayedFrom = __remove_cvref_t<_From>,
1108 typename = enable_if_t<conjunction<
1109 is_convertible<_From, _To>,
1110 disjunction<
1111 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
1112 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
1113 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
1114 using _ValuePreservingOrInt = _From;
1115
1116// }}}
1117// __intrinsic_type {{{
1118template <typename _Tp, size_t _Bytes, typename = void_t<>>
1119 struct __intrinsic_type;
1120
1121template <typename _Tp, size_t _Size>
1122 using __intrinsic_type_t =
1123 typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type;
1124
1125template <typename _Tp>
1126 using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type;
1127template <typename _Tp>
1128 using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type;
1129template <typename _Tp>
1130 using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type;
1131template <typename _Tp>
1132 using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type;
1133template <typename _Tp>
1134 using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type;
1135template <typename _Tp>
1136 using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type;
1137
1138// }}}
1139// _BitMask {{{
1140template <size_t _Np, bool _Sanitized = false>
1141 struct _BitMask;
1142
1143template <size_t _Np, bool _Sanitized>
1144 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> : true_type {};
1145
1146template <size_t _Np>
1147 using _SanitizedBitMask = _BitMask<_Np, true>;
1148
1149template <size_t _Np, bool _Sanitized>
1150 struct _BitMask
1151 {
1152 static_assert(_Np > 0);
1153
1154 static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
1155
1156 using _Tp = conditional_t<_Np == 1, bool,
1157 make_unsigned_t<__int_with_sizeof_t<std::min(
1158 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
1159
1160 static constexpr int _S_array_size = __div_roundup(_NBytes, sizeof(_Tp));
1161
1162 _Tp _M_bits[_S_array_size];
1163
1164 static constexpr int _S_unused_bits
1165 = _Np == 1 ? 0 : _S_array_size * sizeof(_Tp) * __CHAR_BIT__ - _Np;
1166
1167 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1168
1169 constexpr _BitMask() noexcept = default;
1170
1171 constexpr _BitMask(unsigned long long __x) noexcept
1172 : _M_bits{static_cast<_Tp>(__x)} {}
1173
1174 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1175
1176 constexpr _BitMask(const _BitMask&) noexcept = default;
1177
1178 template <bool _RhsSanitized, typename = enable_if_t<_RhsSanitized == false
1179 && _Sanitized == true>>
1180 constexpr _BitMask(const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1181 : _BitMask(__rhs._M_sanitized()) {}
1182
1183 constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1184 {
1185 static_assert(_S_array_size == 1);
1186 return _M_bits[0];
1187 }
1188
1189 // precondition: is sanitized
1190 constexpr _Tp
1191 _M_to_bits() const noexcept
1192 {
1193 static_assert(_S_array_size == 1);
1194 return _M_bits[0];
1195 }
1196
1197 // precondition: is sanitized
1198 constexpr unsigned long long
1199 to_ullong() const noexcept
1200 {
1201 static_assert(_S_array_size == 1);
1202 return _M_bits[0];
1203 }
1204
1205 // precondition: is sanitized
1206 constexpr unsigned long
1207 to_ulong() const noexcept
1208 {
1209 static_assert(_S_array_size == 1);
1210 return _M_bits[0];
1211 }
1212
1213 constexpr bitset<_Np>
1214 _M_to_bitset() const noexcept
1215 {
1216 static_assert(_S_array_size == 1);
1217 return _M_bits[0];
1218 }
1219
1220 constexpr decltype(auto)
1221 _M_sanitized() const noexcept
1222 {
1223 if constexpr (_Sanitized)
1224 return *this;
1225 else if constexpr (_Np == 1)
1226 return _SanitizedBitMask<_Np>(_M_bits[0]);
1227 else
1228 {
1229 _SanitizedBitMask<_Np> __r = {};
1230 for (int __i = 0; __i < _S_array_size; ++__i)
1231 __r._M_bits[__i] = _M_bits[__i];
1232 if constexpr (_S_unused_bits > 0)
1233 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1234 return __r;
1235 }
1236 }
1237
1238 template <size_t _Mp, bool _LSanitized>
1239 constexpr _BitMask<_Np + _Mp, _Sanitized>
1240 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb) const noexcept
1241 {
1242 constexpr size_t _RN = _Np + _Mp;
1243 using _Rp = _BitMask<_RN, _Sanitized>;
1244 if constexpr (_Rp::_S_array_size == 1)
1245 {
1246 _Rp __r{{_M_bits[0]}};
1247 __r._M_bits[0] <<= _Mp;
1248 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1249 return __r;
1250 }
1251 else
1252 __assert_unreachable<_Rp>();
1253 }
1254
1255 // Return a new _BitMask with size _NewSize while dropping _DropLsb least
1256 // significant bits. If the operation implicitly produces a sanitized bitmask,
1257 // the result type will have _Sanitized set.
1258 template <size_t _DropLsb, size_t _NewSize = _Np - _DropLsb>
1259 constexpr auto
1260 _M_extract() const noexcept
1261 {
1262 static_assert(_Np > _DropLsb);
1263 static_assert(_DropLsb + _NewSize <= sizeof(_ULLong) * __CHAR_BIT__,
1264 "not implemented for bitmasks larger than one ullong");
1265 if constexpr (_NewSize == 1)
1266 // must sanitize because the return _Tp is bool
1267 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1268 else
1269 return _BitMask<_NewSize,
1270 ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1271 && _NewSize + _DropLsb <= _Np)
1272 || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1273 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1274 >> _DropLsb);
1275 }
1276
1277 // True if all bits are set. Implicitly sanitizes if _Sanitized == false.
1278 constexpr bool
1279 all() const noexcept
1280 {
1281 if constexpr (_Np == 1)
1282 return _M_bits[0];
1283 else if constexpr (!_Sanitized)
1284 return _M_sanitized().all();
1285 else
1286 {
1287 constexpr _Tp __allbits = ~_Tp();
1288 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1289 if (_M_bits[__i] != __allbits)
1290 return false;
1291 return _M_bits[_S_array_size - 1] == _S_bitmask;
1292 }
1293 }
1294
1295 // True if at least one bit is set. Implicitly sanitizes if _Sanitized ==
1296 // false.
1297 constexpr bool
1298 any() const noexcept
1299 {
1300 if constexpr (_Np == 1)
1301 return _M_bits[0];
1302 else if constexpr (!_Sanitized)
1303 return _M_sanitized().any();
1304 else
1305 {
1306 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1307 if (_M_bits[__i] != 0)
1308 return true;
1309 return _M_bits[_S_array_size - 1] != 0;
1310 }
1311 }
1312
1313 // True if no bit is set. Implicitly sanitizes if _Sanitized == false.
1314 constexpr bool
1315 none() const noexcept
1316 {
1317 if constexpr (_Np == 1)
1318 return !_M_bits[0];
1319 else if constexpr (!_Sanitized)
1320 return _M_sanitized().none();
1321 else
1322 {
1323 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1324 if (_M_bits[__i] != 0)
1325 return false;
1326 return _M_bits[_S_array_size - 1] == 0;
1327 }
1328 }
1329
1330 // Returns the number of set bits. Implicitly sanitizes if _Sanitized ==
1331 // false.
1332 constexpr int
1333 count() const noexcept
1334 {
1335 if constexpr (_Np == 1)
1336 return _M_bits[0];
1337 else if constexpr (!_Sanitized)
1338 return _M_sanitized().none();
1339 else
1340 {
1341 int __result = __builtin_popcountll(_M_bits[0]);
1342 for (int __i = 1; __i < _S_array_size; ++__i)
1343 __result += __builtin_popcountll(_M_bits[__i]);
1344 return __result;
1345 }
1346 }
1347
1348 // Returns the bit at offset __i as bool.
1349 constexpr bool
1350 operator[](size_t __i) const noexcept
1351 {
1352 if constexpr (_Np == 1)
1353 return _M_bits[0];
1354 else if constexpr (_S_array_size == 1)
1355 return (_M_bits[0] >> __i) & 1;
1356 else
1357 {
1358 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1359 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1360 return (_M_bits[__j] >> __shift) & 1;
1361 }
1362 }
1363
1364 template <size_t __i>
1365 constexpr bool
1366 operator[](_SizeConstant<__i>) const noexcept
1367 {
1368 static_assert(__i < _Np);
1369 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1370 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1371 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1372 }
1373
1374 // Set the bit at offset __i to __x.
1375 constexpr void
1376 set(size_t __i, bool __x) noexcept
1377 {
1378 if constexpr (_Np == 1)
1379 _M_bits[0] = __x;
1380 else if constexpr (_S_array_size == 1)
1381 {
1382 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1383 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1384 }
1385 else
1386 {
1387 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1388 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1389 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1390 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1391 }
1392 }
1393
1394 template <size_t __i>
1395 constexpr void
1396 set(_SizeConstant<__i>, bool __x) noexcept
1397 {
1398 static_assert(__i < _Np);
1399 if constexpr (_Np == 1)
1400 _M_bits[0] = __x;
1401 else
1402 {
1403 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1404 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1405 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1406 _M_bits[__j] &= __mask;
1407 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1408 }
1409 }
1410
1411 // Inverts all bits. Sanitized input leads to sanitized output.
1412 constexpr _BitMask
1413 operator~() const noexcept
1414 {
1415 if constexpr (_Np == 1)
1416 return !_M_bits[0];
1417 else
1418 {
1419 _BitMask __result{};
1420 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1421 __result._M_bits[__i] = ~_M_bits[__i];
1422 if constexpr (_Sanitized)
1423 __result._M_bits[_S_array_size - 1]
1424 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1425 else
1426 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1427 return __result;
1428 }
1429 }
1430
1431 constexpr _BitMask&
1432 operator^=(const _BitMask& __b) & noexcept
1433 {
1434 __execute_n_times<_S_array_size>(
1435 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] ^= __b._M_bits[__i]; });
1436 return *this;
1437 }
1438
1439 constexpr _BitMask&
1440 operator|=(const _BitMask& __b) & noexcept
1441 {
1442 __execute_n_times<_S_array_size>(
1443 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] |= __b._M_bits[__i]; });
1444 return *this;
1445 }
1446
1447 constexpr _BitMask&
1448 operator&=(const _BitMask& __b) & noexcept
1449 {
1450 __execute_n_times<_S_array_size>(
1451 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] &= __b._M_bits[__i]; });
1452 return *this;
1453 }
1454
1455 friend constexpr _BitMask
1456 operator^(const _BitMask& __a, const _BitMask& __b) noexcept
1457 {
1458 _BitMask __r = __a;
1459 __r ^= __b;
1460 return __r;
1461 }
1462
1463 friend constexpr _BitMask
1464 operator|(const _BitMask& __a, const _BitMask& __b) noexcept
1465 {
1466 _BitMask __r = __a;
1467 __r |= __b;
1468 return __r;
1469 }
1470
1471 friend constexpr _BitMask
1472 operator&(const _BitMask& __a, const _BitMask& __b) noexcept
1473 {
1474 _BitMask __r = __a;
1475 __r &= __b;
1476 return __r;
1477 }
1478
1479 _GLIBCXX_SIMD_INTRINSIC
1480 constexpr bool
1481 _M_is_constprop() const
1482 {
1483 if constexpr (_S_array_size == 0)
1484 return __builtin_constant_p(_M_bits[0]);
1485 else
1486 {
1487 for (int __i = 0; __i < _S_array_size; ++__i)
1488 if (!__builtin_constant_p(_M_bits[__i]))
1489 return false;
1490 return true;
1491 }
1492 }
1493 };
1494
1495// }}}
1496
1497// vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv
1498// __min_vector_size {{{
1499template <typename _Tp = void>
1500 static inline constexpr int __min_vector_size = 2 * sizeof(_Tp);
1501
1502#if _GLIBCXX_SIMD_HAVE_NEON
1503template <>
1504 inline constexpr int __min_vector_size<void> = 8;
1505#else
1506template <>
1507 inline constexpr int __min_vector_size<void> = 16;
1508#endif
1509
1510// }}}
1511// __vector_type {{{
1512template <typename _Tp, size_t _Np, typename = void>
1513 struct __vector_type_n {};
1514
1515// substition failure for 0-element case
1516template <typename _Tp>
1517 struct __vector_type_n<_Tp, 0, void> {};
1518
1519// special case 1-element to be _Tp itself
1520template <typename _Tp>
1521 struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>>
1522 { using type = _Tp; };
1523
1524// else, use GNU-style builtin vector types
1525template <typename _Tp, size_t _Np>
1526 struct __vector_type_n<_Tp, _Np, enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1527 {
1528 static constexpr size_t _S_Np2 = std::__bit_ceil(_Np * sizeof(_Tp));
1529
1530 static constexpr size_t _S_Bytes =
1531#ifdef __i386__
1532 // Using [[gnu::vector_size(8)]] would wreak havoc on the FPU because
1533 // those objects are passed via MMX registers and nothing ever calls EMMS.
1534 _S_Np2 == 8 ? 16 :
1535#endif
1536 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1537 : _S_Np2;
1538
1539 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1540 };
1541
1542template <typename _Tp, size_t _Bytes, size_t = _Bytes % sizeof(_Tp)>
1543 struct __vector_type;
1544
1545template <typename _Tp, size_t _Bytes>
1546 struct __vector_type<_Tp, _Bytes, 0>
1547 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1548
1549template <typename _Tp, size_t _Size>
1550 using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type;
1551
1552template <typename _Tp>
1553 using __vector_type2_t = typename __vector_type<_Tp, 2>::type;
1554template <typename _Tp>
1555 using __vector_type4_t = typename __vector_type<_Tp, 4>::type;
1556template <typename _Tp>
1557 using __vector_type8_t = typename __vector_type<_Tp, 8>::type;
1558template <typename _Tp>
1559 using __vector_type16_t = typename __vector_type<_Tp, 16>::type;
1560template <typename _Tp>
1561 using __vector_type32_t = typename __vector_type<_Tp, 32>::type;
1562template <typename _Tp>
1563 using __vector_type64_t = typename __vector_type<_Tp, 64>::type;
1564
1565// }}}
1566// __is_vector_type {{{
1567template <typename _Tp, typename = void_t<>>
1568 struct __is_vector_type : false_type {};
1569
1570template <typename _Tp>
1571 struct __is_vector_type<
1572 _Tp, void_t<typename __vector_type<
1573 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1574 : is_same<_Tp, typename __vector_type<
1575 remove_reference_t<decltype(declval<_Tp>()[0])>,
1576 sizeof(_Tp)>::type> {};
1577
1578template <typename _Tp>
1579 inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1580
1581// }}}
1582// __is_intrinsic_type {{{
1583#if _GLIBCXX_SIMD_HAVE_SSE_ABI
1584template <typename _Tp>
1585 using __is_intrinsic_type = __is_vector_type<_Tp>;
1586#else // not SSE (x86)
1587template <typename _Tp, typename = void_t<>>
1588 struct __is_intrinsic_type : false_type {};
1589
1590template <typename _Tp>
1591 struct __is_intrinsic_type<
1592 _Tp, void_t<typename __intrinsic_type<
1593 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1594 : is_same<_Tp, typename __intrinsic_type<
1595 remove_reference_t<decltype(declval<_Tp>()[0])>,
1596 sizeof(_Tp)>::type> {};
1597#endif
1598
1599template <typename _Tp>
1600 inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1601
1602// }}}
1603// _VectorTraits{{{
1604template <typename _Tp, typename = void_t<>>
1605 struct _VectorTraitsImpl;
1606
1607template <typename _Tp>
1608 struct _VectorTraitsImpl<_Tp, enable_if_t<__is_vector_type_v<_Tp>
1609 || __is_intrinsic_type_v<_Tp>>>
1610 {
1611 using type = _Tp;
1612 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1613 static constexpr int _S_full_size = sizeof(_Tp) / sizeof(value_type);
1614 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1615 template <typename _Up, int _W = _S_full_size>
1616 static constexpr bool _S_is
1617 = is_same_v<value_type, _Up> && _W == _S_full_size;
1618 };
1619
1620template <typename _Tp, size_t _Np>
1621 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1622 void_t<__vector_type_t<_Tp, _Np>>>
1623 {
1624 using type = __vector_type_t<_Tp, _Np>;
1625 using value_type = _Tp;
1626 static constexpr int _S_full_size = sizeof(type) / sizeof(value_type);
1627 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1628 static constexpr bool _S_is_partial = (_Np == _S_full_size);
1629 static constexpr int _S_partial_width = _Np;
1630 template <typename _Up, int _W = _S_full_size>
1631 static constexpr bool _S_is
1632 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1633 };
1634
1635template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1636 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1637
1638// }}}
1639// __as_vector{{{
1640template <typename _V>
1641 _GLIBCXX_SIMD_INTRINSIC constexpr auto
1642 __as_vector(_V __x)
1643 {
1644 if constexpr (__is_vector_type_v<_V>)
1645 return __x;
1646 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1647 {
1648 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
1649 {
1650 static_assert(is_simd<_V>::value);
1651 static_assert(_V::abi_type::template __traits<
1652 typename _V::value_type>::_SimdMember::_S_tuple_size == 1);
1653 return __as_vector(__data(__x).first);
1654 }
1655 else if constexpr (_V::size() > 1)
1656 return __data(__x)._M_data;
1657 else
1658 {
1659 static_assert(is_simd<_V>::value);
1660 using _Tp = typename _V::value_type;
1661#ifdef __i386__
1662 constexpr auto __bytes = sizeof(_Tp) == 8 ? 16 : sizeof(_Tp);
1663 using _RV [[__gnu__::__vector_size__(__bytes)]] = _Tp;
1664#else
1665 using _RV [[__gnu__::__vector_size__(sizeof(_Tp))]] = _Tp;
1666#endif
1667 return _RV{__data(__x)};
1668 }
1669 }
1670 else if constexpr (__is_vectorizable_v<_V>)
1671 return __vector_type_t<_V, 2>{__x};
1672 else
1673 return __x._M_data;
1674 }
1675
1676// }}}
1677// __as_wrapper{{{
1678template <size_t _Np = 0, typename _V>
1679 _GLIBCXX_SIMD_INTRINSIC constexpr auto
1680 __as_wrapper(_V __x)
1681 {
1682 if constexpr (__is_vector_type_v<_V>)
1683 return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1684 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1685 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1686 {
1687 static_assert(_V::size() == _Np);
1688 return __data(__x);
1689 }
1690 else
1691 {
1692 static_assert(_V::_S_size == _Np);
1693 return __x;
1694 }
1695 }
1696
1697// }}}
1698// __intrin_bitcast{{{
1699template <typename _To, typename _From>
1700 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1701 __intrin_bitcast(_From __v)
1702 {
1703 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1704 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1705 if constexpr (sizeof(_To) == sizeof(_From))
1706 return reinterpret_cast<_To>(__v);
1707 else if constexpr (sizeof(_From) > sizeof(_To))
1708 if constexpr (sizeof(_To) >= 16)
1709 return reinterpret_cast<const __may_alias<_To>&>(__v);
1710 else
1711 {
1712 _To __r;
1713 __builtin_memcpy(&__r, &__v, sizeof(_To));
1714 return __r;
1715 }
1716#if _GLIBCXX_SIMD_X86INTRIN && !defined _GLIBCXX_CLANG
1717 else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32)
1718 return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1719 reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1720 else if constexpr (__have_avx512f && sizeof(_From) == 16
1721 && sizeof(_To) == 64)
1722 return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1723 reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1724 else if constexpr (__have_avx512f && sizeof(_From) == 32
1725 && sizeof(_To) == 64)
1726 return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1727 reinterpret_cast<__vector_type_t<float, 8>>(__v)));
1728#endif // _GLIBCXX_SIMD_X86INTRIN
1729 else if constexpr (sizeof(__v) <= 8)
1730 return reinterpret_cast<_To>(
1731 __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1732 reinterpret_cast<__int_for_sizeof_t<_From>>(__v)});
1733 else
1734 {
1735 static_assert(sizeof(_To) > sizeof(_From));
1736 _To __r = {};
1737 __builtin_memcpy(&__r, &__v, sizeof(_From));
1738 return __r;
1739 }
1740 }
1741
1742// }}}
1743// __vector_bitcast{{{
1744template <typename _To, size_t _NN = 0, typename _From,
1745 typename _FromVT = _VectorTraits<_From>,
1746 size_t _Np = _NN == 0 ? sizeof(_From) / sizeof(_To) : _NN>
1747 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1748 __vector_bitcast(_From __x)
1749 {
1750 using _R = __vector_type_t<_To, _Np>;
1751 return __intrin_bitcast<_R>(__x);
1752 }
1753
1754template <typename _To, size_t _NN = 0, typename _Tp, size_t _Nx,
1755 size_t _Np
1756 = _NN == 0 ? sizeof(_SimdWrapper<_Tp, _Nx>) / sizeof(_To) : _NN>
1757 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1758 __vector_bitcast(const _SimdWrapper<_Tp, _Nx>& __x)
1759 {
1760 static_assert(_Np > 1);
1761 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1762 }
1763
1764// }}}
1765// __convert_x86 declarations {{{
1766#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1767template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1768 _To __convert_x86(_Tp);
1769
1770template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1771 _To __convert_x86(_Tp, _Tp);
1772
1773template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1774 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1775
1776template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1777 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1778
1779template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1780 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1781 _Tp, _Tp, _Tp, _Tp);
1782#endif // _GLIBCXX_SIMD_WORKAROUND_PR85048
1783
1784//}}}
1785// __bit_cast {{{
1786template <typename _To, typename _From>
1787 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1788 __bit_cast(const _From __x)
1789 {
1790#if __has_builtin(__builtin_bit_cast)
1791 return __builtin_bit_cast(_To, __x);
1792#else
1793 static_assert(sizeof(_To) == sizeof(_From));
1794 constexpr bool __to_is_vectorizable
1795 = is_arithmetic_v<_To> || is_enum_v<_To>;
1796 constexpr bool __from_is_vectorizable
1797 = is_arithmetic_v<_From> || is_enum_v<_From>;
1798 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1799 return reinterpret_cast<_To>(__x);
1800 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1801 {
1802 using _FV [[__gnu__::__vector_size__(sizeof(_From))]] = _From;
1803 return reinterpret_cast<_To>(_FV{__x});
1804 }
1805 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1806 {
1807 using _TV [[__gnu__::__vector_size__(sizeof(_To))]] = _To;
1808 using _FV [[__gnu__::__vector_size__(sizeof(_From))]] = _From;
1809 return reinterpret_cast<_TV>(_FV{__x})[0];
1810 }
1811 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1812 {
1813 using _TV [[__gnu__::__vector_size__(sizeof(_To))]] = _To;
1814 return reinterpret_cast<_TV>(__x)[0];
1815 }
1816 else
1817 {
1818 _To __r;
1819 __builtin_memcpy(reinterpret_cast<char*>(&__r),
1820 reinterpret_cast<const char*>(&__x), sizeof(_To));
1821 return __r;
1822 }
1823#endif
1824 }
1825
1826// }}}
1827// __to_intrin {{{
1828template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1829 typename _R = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_full_size>>
1830 _GLIBCXX_SIMD_INTRINSIC constexpr _R
1831 __to_intrin(_Tp __x)
1832 {
1833 static_assert(sizeof(__x) <= sizeof(_R),
1834 "__to_intrin may never drop values off the end");
1835 if constexpr (sizeof(__x) == sizeof(_R))
1836 return reinterpret_cast<_R>(__as_vector(__x));
1837 else
1838 {
1839 using _Up = __int_for_sizeof_t<_Tp>;
1840 return reinterpret_cast<_R>(
1841 __vector_type_t<_Up, sizeof(_R) / sizeof(_Up)>{__bit_cast<_Up>(__x)});
1842 }
1843 }
1844
1845// }}}
1846// __make_vector{{{
1847template <typename _Tp, typename... _Args>
1848 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)>
1849 __make_vector(const _Args&... __args)
1850 { return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(__args)...}; }
1851
1852// }}}
1853// __vector_broadcast{{{
1854template <size_t _Np, typename _Tp, size_t... _I>
1855 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1856 __vector_broadcast_impl(_Tp __x, index_sequence<_I...>)
1857 { return __vector_type_t<_Tp, _Np>{((void)_I, __x)...}; }
1858
1859template <size_t _Np, typename _Tp>
1860 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1861 __vector_broadcast(_Tp __x)
1862 { return __vector_broadcast_impl<_Np, _Tp>(__x, make_index_sequence<_Np>()); }
1863
1864// }}}
1865// __generate_vector{{{
1866 template <typename _Tp, size_t _Np, typename _Gp, size_t... _I>
1867 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1868 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1869 { return __vector_type_t<_Tp, _Np>{ static_cast<_Tp>(__gen(_SizeConstant<_I>()))...}; }
1870
1871template <typename _V, typename _VVT = _VectorTraits<_V>, typename _Gp>
1872 _GLIBCXX_SIMD_INTRINSIC constexpr _V
1873 __generate_vector(_Gp&& __gen)
1874 {
1875 if constexpr (__is_vector_type_v<_V>)
1876 return __generate_vector_impl<typename _VVT::value_type,
1877 _VVT::_S_full_size>(
1878 static_cast<_Gp&&>(__gen), make_index_sequence<_VVT::_S_full_size>());
1879 else
1880 return __generate_vector_impl<typename _VVT::value_type,
1881 _VVT::_S_partial_width>(
1882 static_cast<_Gp&&>(__gen),
1883 make_index_sequence<_VVT::_S_partial_width>());
1884 }
1885
1886template <typename _Tp, size_t _Np, typename _Gp>
1887 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1888 __generate_vector(_Gp&& __gen)
1889 {
1890 return __generate_vector_impl<_Tp, _Np>(static_cast<_Gp&&>(__gen),
1891 make_index_sequence<_Np>());
1892 }
1893
1894// }}}
1895// __xor{{{
1896template <typename _TW>
1897 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1898 __xor(_TW __a, _TW __b) noexcept
1899 {
1900 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1901 {
1902 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1903 _VectorTraitsImpl<_TW>>::value_type;
1904 if constexpr (is_floating_point_v<_Tp>)
1905 {
1906 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1907 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1908 ^ __vector_bitcast<_Ip>(__b));
1909 }
1910 else if constexpr (__is_vector_type_v<_TW>)
1911 return __a ^ __b;
1912 else
1913 return __a._M_data ^ __b._M_data;
1914 }
1915 else
1916 return __a ^ __b;
1917 }
1918
1919// }}}
1920// __or{{{
1921template <typename _TW>
1922 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1923 __or(_TW __a, _TW __b) noexcept
1924 {
1925 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1926 {
1927 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1928 _VectorTraitsImpl<_TW>>::value_type;
1929 if constexpr (is_floating_point_v<_Tp>)
1930 {
1931 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1932 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1933 | __vector_bitcast<_Ip>(__b));
1934 }
1935 else if constexpr (__is_vector_type_v<_TW>)
1936 return __a | __b;
1937 else
1938 return __a._M_data | __b._M_data;
1939 }
1940 else
1941 return __a | __b;
1942 }
1943
1944// }}}
1945// __and{{{
1946template <typename _TW>
1947 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1948 __and(_TW __a, _TW __b) noexcept
1949 {
1950 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1951 {
1952 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1953 _VectorTraitsImpl<_TW>>::value_type;
1954 if constexpr (is_floating_point_v<_Tp>)
1955 {
1956 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1957 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1958 & __vector_bitcast<_Ip>(__b));
1959 }
1960 else if constexpr (__is_vector_type_v<_TW>)
1961 return __a & __b;
1962 else
1963 return __a._M_data & __b._M_data;
1964 }
1965 else
1966 return __a & __b;
1967 }
1968
1969// }}}
1970// __andnot{{{
1971#if _GLIBCXX_SIMD_X86INTRIN && !defined _GLIBCXX_CLANG
1972static constexpr struct
1973{
1974 _GLIBCXX_SIMD_INTRINSIC __v4sf
1975 operator()(__v4sf __a, __v4sf __b) const noexcept
1976 { return __builtin_ia32_andnps(__a, __b); }
1977
1978 _GLIBCXX_SIMD_INTRINSIC __v2df
1979 operator()(__v2df __a, __v2df __b) const noexcept
1980 { return __builtin_ia32_andnpd(__a, __b); }
1981
1982 _GLIBCXX_SIMD_INTRINSIC __v2di
1983 operator()(__v2di __a, __v2di __b) const noexcept
1984 { return __builtin_ia32_pandn128(__a, __b); }
1985
1986 _GLIBCXX_SIMD_INTRINSIC __v8sf
1987 operator()(__v8sf __a, __v8sf __b) const noexcept
1988 { return __builtin_ia32_andnps256(__a, __b); }
1989
1990 _GLIBCXX_SIMD_INTRINSIC __v4df
1991 operator()(__v4df __a, __v4df __b) const noexcept
1992 { return __builtin_ia32_andnpd256(__a, __b); }
1993
1994 _GLIBCXX_SIMD_INTRINSIC __v4di
1995 operator()(__v4di __a, __v4di __b) const noexcept
1996 {
1997 if constexpr (__have_avx2)
1998 return __builtin_ia32_andnotsi256(__a, __b);
1999 else
2000 return reinterpret_cast<__v4di>(
2001 __builtin_ia32_andnpd256(reinterpret_cast<__v4df>(__a),
2002 reinterpret_cast<__v4df>(__b)));
2003 }
2004
2005 _GLIBCXX_SIMD_INTRINSIC __v16sf
2006 operator()(__v16sf __a, __v16sf __b) const noexcept
2007 {
2008 if constexpr (__have_avx512dq)
2009 return _mm512_andnot_ps(__a, __b);
2010 else
2011 return reinterpret_cast<__v16sf>(
2012 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
2013 reinterpret_cast<__v8di>(__b)));
2014 }
2015
2016 _GLIBCXX_SIMD_INTRINSIC __v8df
2017 operator()(__v8df __a, __v8df __b) const noexcept
2018 {
2019 if constexpr (__have_avx512dq)
2020 return _mm512_andnot_pd(__a, __b);
2021 else
2022 return reinterpret_cast<__v8df>(
2023 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
2024 reinterpret_cast<__v8di>(__b)));
2025 }
2026
2027 _GLIBCXX_SIMD_INTRINSIC __v8di
2028 operator()(__v8di __a, __v8di __b) const noexcept
2029 { return _mm512_andnot_si512(__a, __b); }
2030} _S_x86_andnot;
2031#endif // _GLIBCXX_SIMD_X86INTRIN && !_GLIBCXX_CLANG
2032
2033template <typename _TW>
2034 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
2035 __andnot(_TW __a, _TW __b) noexcept
2036 {
2037 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
2038 {
2039 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
2040 _VectorTraitsImpl<_TW>>;
2041 using _Tp = typename _TVT::value_type;
2042#if _GLIBCXX_SIMD_X86INTRIN && !defined _GLIBCXX_CLANG
2043 if constexpr (sizeof(_TW) >= 16)
2044 {
2045 const auto __ai = __to_intrin(__a);
2046 const auto __bi = __to_intrin(__b);
2047 if (!__builtin_is_constant_evaluated()
2048 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
2049 {
2050 const auto __r = _S_x86_andnot(__ai, __bi);
2051 if constexpr (is_convertible_v<decltype(__r), _TW>)
2052 return __r;
2053 else
2054 return reinterpret_cast<typename _TVT::type>(__r);
2055 }
2056 }
2057#endif // _GLIBCXX_SIMD_X86INTRIN
2058 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
2059 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
2060 & __vector_bitcast<_Ip>(__b));
2061 }
2062 else
2063 return ~__a & __b;
2064 }
2065
2066// }}}
2067// __not{{{
2068template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2069 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2070 __not(_Tp __a) noexcept
2071 {
2072 if constexpr (is_floating_point_v<typename _TVT::value_type>)
2073 return reinterpret_cast<typename _TVT::type>(
2074 ~__vector_bitcast<unsigned>(__a));
2075 else
2076 return ~__a;
2077 }
2078
2079// }}}
2080// __vec_shuffle{{{
2081template <typename _T0, typename _T1, typename _Fun, size_t... _Is>
2082 _GLIBCXX_SIMD_INTRINSIC constexpr
2083 __vector_type_t<remove_reference_t<decltype(declval<_T0>()[0])>, sizeof...(_Is)>
2084 __vec_shuffle(_T0 __x, _T1 __y, index_sequence<_Is...> __seq, _Fun __idx_perm)
2085 {
2086 constexpr int _N0 = sizeof(__x) / sizeof(__x[0]);
2087 constexpr int _N1 = sizeof(__y) / sizeof(__y[0]);
2088 using _Tp = remove_reference_t<decltype(declval<_T0>()[0])>;
2089 using _RV [[maybe_unused]] = __vector_type_t<_Tp, sizeof...(_Is)>;
2090#if __has_builtin(__builtin_shufflevector)
2091#ifdef _GLIBCXX_CLANG
2092 // Clang requires _T0 == _T1
2093 if constexpr (sizeof(__x) > sizeof(__y) and _N1 == 1)
2094 return __vec_shuffle(__x, _T0{__y[0]}, __seq, __idx_perm);
2095 else if constexpr (sizeof(__x) > sizeof(__y))
2096 return __vec_shuffle(__x, __intrin_bitcast<_T0>(__y), __seq, __idx_perm);
2097 else if constexpr (sizeof(__x) < sizeof(__y) and _N0 == 1)
2098 return __vec_shuffle(_T1{__x[0]}, __y, __seq, [=](int __i) {
2099 __i = __idx_perm(__i);
2100 return __i < _N0 ? __i : __i - _N0 + _N1;
2101 });
2102 else if constexpr (sizeof(__x) < sizeof(__y))
2103 return __vec_shuffle(__intrin_bitcast<_T1>(__x), __y, __seq, [=](int __i) {
2104 __i = __idx_perm(__i);
2105 return __i < _N0 ? __i : __i - _N0 + _N1;
2106 });
2107 else
2108#endif
2109 {
2110 const auto __r = __builtin_shufflevector(__x, __y, [=] {
2111 constexpr int __j = __idx_perm(_Is);
2112 static_assert(__j < _N0 + _N1);
2113 return __j;
2114 }()...);
2115#ifdef __i386__
2116 if constexpr (sizeof(__r) == sizeof(_RV))
2117 return __r;
2118 else
2119 return _RV {__r[_Is]...};
2120#else
2121 return __r;
2122#endif
2123 }
2124#else
2125 return _RV {
2126 [=]() -> _Tp {
2127 constexpr int __j = __idx_perm(_Is);
2128 static_assert(__j < _N0 + _N1);
2129 if constexpr (__j < 0)
2130 return 0;
2131 else if constexpr (__j < _N0)
2132 return __x[__j];
2133 else
2134 return __y[__j - _N0];
2135 }()...
2136 };
2137#endif
2138 }
2139
2140template <typename _T0, typename _Fun, typename _Seq>
2141 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2142 __vec_shuffle(_T0 __x, _Seq __seq, _Fun __idx_perm)
2143 { return __vec_shuffle(__x, _T0(), __seq, __idx_perm); }
2144
2145// }}}
2146// __concat{{{
2147template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
2148 typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size * 2>>
2149 constexpr _R
2150 __concat(_Tp a_, _Tp b_)
2151 {
2152#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2153 using _W
2154 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
2155 conditional_t<(sizeof(_Tp) >= 2 * sizeof(long long)),
2156 long long, typename _TVT::value_type>>;
2157 constexpr int input_width = sizeof(_Tp) / sizeof(_W);
2158 const auto __a = __vector_bitcast<_W>(a_);
2159 const auto __b = __vector_bitcast<_W>(b_);
2160 using _Up = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>;
2161#else
2162 constexpr int input_width = _TVT::_S_full_size;
2163 const _Tp& __a = a_;
2164 const _Tp& __b = b_;
2165 using _Up = _R;
2166#endif
2167 if constexpr (input_width == 2)
2168 return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
2169 else if constexpr (input_width == 4)
2170 return reinterpret_cast<_R>(
2171 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
2172 else if constexpr (input_width == 8)
2173 return reinterpret_cast<_R>(
2174 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
2175 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
2176 else if constexpr (input_width == 16)
2177 return reinterpret_cast<_R>(
2178 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2179 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2180 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
2181 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
2182 __b[12], __b[13], __b[14], __b[15]});
2183 else if constexpr (input_width == 32)
2184 return reinterpret_cast<_R>(
2185 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2186 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2187 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
2188 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
2189 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
2190 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
2191 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
2192 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
2193 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
2194 __b[31]});
2195 }
2196
2197// }}}
2198// __zero_extend {{{
2199template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2200 struct _ZeroExtendProxy
2201 {
2202 using value_type = typename _TVT::value_type;
2203 static constexpr size_t _Np = _TVT::_S_full_size;
2204 const _Tp __x;
2205
2206 template <typename _To, typename _ToVT = _VectorTraits<_To>,
2207 typename
2208 = enable_if_t<is_same_v<typename _ToVT::value_type, value_type>>>
2209 _GLIBCXX_SIMD_INTRINSIC operator _To() const
2210 {
2211 constexpr size_t _ToN = _ToVT::_S_full_size;
2212 if constexpr (_ToN == _Np)
2213 return __x;
2214 else if constexpr (_ToN == 2 * _Np)
2215 {
2216#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2217 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
2218 return __vector_bitcast<value_type>(
2219 _mm256_insertf128_ps(__m256(), __x, 0));
2220 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
2221 return __vector_bitcast<value_type>(
2222 _mm256_insertf128_pd(__m256d(), __x, 0));
2223 else if constexpr (__have_avx2 && _Np * sizeof(value_type) == 16)
2224 return __vector_bitcast<value_type>(
2225 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
2226 else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
2227 {
2228 if constexpr (__have_avx512dq)
2229 return __vector_bitcast<value_type>(
2230 _mm512_insertf32x8(__m512(), __x, 0));
2231 else
2232 return reinterpret_cast<__m512>(
2233 _mm512_insertf64x4(__m512d(),
2234 reinterpret_cast<__m256d>(__x), 0));
2235 }
2236 else if constexpr (__have_avx512f
2237 && _TVT::template _S_is<double, 4>)
2238 return __vector_bitcast<value_type>(
2239 _mm512_insertf64x4(__m512d(), __x, 0));
2240 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 32)
2241 return __vector_bitcast<value_type>(
2242 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
2243#endif
2244 return __concat(__x, _Tp());
2245 }
2246 else if constexpr (_ToN == 4 * _Np)
2247 {
2248#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2249 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2250 {
2251 return __vector_bitcast<value_type>(
2252 _mm512_insertf64x2(__m512d(), __x, 0));
2253 }
2254 else if constexpr (__have_avx512f
2255 && is_floating_point_v<value_type>)
2256 {
2257 return __vector_bitcast<value_type>(
2258 _mm512_insertf32x4(__m512(), reinterpret_cast<__m128>(__x),
2259 0));
2260 }
2261 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 16)
2262 {
2263 return __vector_bitcast<value_type>(
2264 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2265 }
2266#endif
2267 return __concat(__concat(__x, _Tp()),
2268 __vector_type_t<value_type, _Np * 2>());
2269 }
2270 else if constexpr (_ToN == 8 * _Np)
2271 return __concat(operator __vector_type_t<value_type, _Np * 4>(),
2272 __vector_type_t<value_type, _Np * 4>());
2273 else if constexpr (_ToN == 16 * _Np)
2274 return __concat(operator __vector_type_t<value_type, _Np * 8>(),
2275 __vector_type_t<value_type, _Np * 8>());
2276 else
2277 __assert_unreachable<_Tp>();
2278 }
2279 };
2280
2281template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2282 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2283 __zero_extend(_Tp __x)
2284 { return {__x}; }
2285
2286// }}}
2287// __extract<_Np, By>{{{
2288template <int _Offset,
2289 int _SplitBy,
2290 typename _Tp,
2291 typename _TVT = _VectorTraits<_Tp>,
2292 typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size / _SplitBy>>
2293 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2294 __extract(_Tp __in)
2295 {
2296 using value_type = typename _TVT::value_type;
2297#if _GLIBCXX_SIMD_X86INTRIN // {{{
2298 if constexpr (sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2299 {
2300 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2301 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2302 else if constexpr (is_floating_point_v<value_type>)
2303 return __vector_bitcast<value_type>(
2304 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2305 else
2306 return reinterpret_cast<_R>(
2307 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2308 _Offset));
2309 }
2310 else
2311#endif // _GLIBCXX_SIMD_X86INTRIN }}}
2312 {
2313#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2314 using _W = conditional_t<
2315 is_floating_point_v<value_type>, double,
2316 conditional_t<(sizeof(_R) >= 16), long long, value_type>>;
2317 static_assert(sizeof(_R) % sizeof(_W) == 0);
2318 constexpr int __return_width = sizeof(_R) / sizeof(_W);
2319 using _Up = __vector_type_t<_W, __return_width>;
2320 const auto __x = __vector_bitcast<_W>(__in);
2321#else
2322 constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2323 using _Up = _R;
2324 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2325 = __in; // only needed for _Tp = _SimdWrapper<value_type, _Np>
2326#endif
2327 constexpr int _O = _Offset * __return_width;
2328 return __call_with_subscripts<__return_width, _O>(
2329 __x, [](auto... __entries) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2330 return reinterpret_cast<_R>(_Up{__entries...});
2331 });
2332 }
2333 }
2334
2335// }}}
2336// __lo/__hi64[z]{{{
2337template <typename _Tp,
2338 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2339 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2340 __lo64(_Tp __x)
2341 {
2342 _R __r{};
2343 __builtin_memcpy(&__r, &__x, 8);
2344 return __r;
2345 }
2346
2347template <typename _Tp,
2348 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2349 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2350 __hi64(_Tp __x)
2351 {
2352 static_assert(sizeof(_Tp) == 16, "use __hi64z if you meant it");
2353 _R __r{};
2354 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2355 return __r;
2356 }
2357
2358template <typename _Tp,
2359 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2360 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2361 __hi64z([[maybe_unused]] _Tp __x)
2362 {
2363 _R __r{};
2364 if constexpr (sizeof(_Tp) == 16)
2365 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2366 return __r;
2367 }
2368
2369// }}}
2370// __lo/__hi128{{{
2371template <typename _Tp>
2372 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2373 __lo128(_Tp __x)
2374 { return __extract<0, sizeof(_Tp) / 16>(__x); }
2375
2376template <typename _Tp>
2377 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2378 __hi128(_Tp __x)
2379 {
2380 static_assert(sizeof(__x) == 32);
2381 return __extract<1, 2>(__x);
2382 }
2383
2384// }}}
2385// __lo/__hi256{{{
2386template <typename _Tp>
2387 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2388 __lo256(_Tp __x)
2389 {
2390 static_assert(sizeof(__x) == 64);
2391 return __extract<0, 2>(__x);
2392 }
2393
2394template <typename _Tp>
2395 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2396 __hi256(_Tp __x)
2397 {
2398 static_assert(sizeof(__x) == 64);
2399 return __extract<1, 2>(__x);
2400 }
2401
2402// }}}
2403// __auto_bitcast{{{
2404template <typename _Tp>
2405 struct _AutoCast
2406 {
2407 static_assert(__is_vector_type_v<_Tp>);
2408
2409 const _Tp __x;
2410
2411 template <typename _Up, typename _UVT = _VectorTraits<_Up>>
2412 _GLIBCXX_SIMD_INTRINSIC constexpr operator _Up() const
2413 { return __intrin_bitcast<typename _UVT::type>(__x); }
2414 };
2415
2416template <typename _Tp>
2417 _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2418 __auto_bitcast(const _Tp& __x)
2419 { return {__x}; }
2420
2421template <typename _Tp, size_t _Np>
2422 _GLIBCXX_SIMD_INTRINSIC constexpr
2423 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2424 __auto_bitcast(const _SimdWrapper<_Tp, _Np>& __x)
2425 { return {__x._M_data}; }
2426
2427// }}}
2428// ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^
2429
2430#if _GLIBCXX_SIMD_HAVE_SSE_ABI
2431// __bool_storage_member_type{{{
2432#if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2433template <size_t _Size>
2434 struct __bool_storage_member_type
2435 {
2436 static_assert((_Size & (_Size - 1)) != 0,
2437 "This trait may only be used for non-power-of-2 sizes. "
2438 "Power-of-2 sizes must be specialized.");
2439 using type =
2440 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2441 };
2442
2443template <>
2444 struct __bool_storage_member_type<1> { using type = bool; };
2445
2446template <>
2447 struct __bool_storage_member_type<2> { using type = __mmask8; };
2448
2449template <>
2450 struct __bool_storage_member_type<4> { using type = __mmask8; };
2451
2452template <>
2453 struct __bool_storage_member_type<8> { using type = __mmask8; };
2454
2455template <>
2456 struct __bool_storage_member_type<16> { using type = __mmask16; };
2457
2458template <>
2459 struct __bool_storage_member_type<32> { using type = __mmask32; };
2460
2461template <>
2462 struct __bool_storage_member_type<64> { using type = __mmask64; };
2463#endif // _GLIBCXX_SIMD_HAVE_AVX512F
2464
2465// }}}
2466// __intrinsic_type (x86){{{
2467// the following excludes bool via __is_vectorizable
2468#if _GLIBCXX_SIMD_HAVE_SSE
2469template <typename _Tp, size_t _Bytes>
2470 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2471 {
2472 // allow _Tp == long double with -mlong-double-64
2473 static_assert(!(is_same_v<_Tp, long double>
2474 && sizeof(long double) > sizeof(double)),
2475 "no __intrinsic_type support for long double on x86");
2476
2477 static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16 : _Bytes <= 32 ? 32 : 64;
2478
2479 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2480 = conditional_t<is_integral_v<_Tp>, long long int,
2481 conditional_t<is_same_v<_Tp, long double>, double, _Tp> >;
2482 };
2483#endif // _GLIBCXX_SIMD_HAVE_SSE
2484
2485// }}}
2486#endif // _GLIBCXX_SIMD_HAVE_SSE_ABI
2487// __intrinsic_type (ARM){{{
2488#if _GLIBCXX_SIMD_HAVE_NEON
2489template <>
2490 struct __intrinsic_type<float, 8, void>
2491 { using type = float32x2_t; };
2492
2493template <>
2494 struct __intrinsic_type<float, 16, void>
2495 { using type = float32x4_t; };
2496
2497template <>
2498 struct __intrinsic_type<double, 8, void>
2499 {
2500#if _GLIBCXX_SIMD_HAVE_NEON_A64
2501 using type = float64x1_t;
2502#endif
2503 };
2504
2505template <>
2506 struct __intrinsic_type<double, 16, void>
2507 {
2508#if _GLIBCXX_SIMD_HAVE_NEON_A64
2509 using type = float64x2_t;
2510#endif
2511 };
2512
2513#define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2514template <> \
2515 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2516 _Np * _Bits / 8, void> \
2517 { using type = int##_Bits##x##_Np##_t; }; \
2518template <> \
2519 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2520 _Np * _Bits / 8, void> \
2521 { using type = uint##_Bits##x##_Np##_t; }
2522_GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2523_GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2524_GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2525_GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2526_GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2527_GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2528_GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2529_GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2530#undef _GLIBCXX_SIMD_ARM_INTRIN
2531
2532template <typename _Tp, size_t _Bytes>
2533 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2534 {
2535 static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2536
2537 using _Ip = __int_for_sizeof_t<_Tp>;
2538
2539 using _Up = conditional_t<
2540 is_floating_point_v<_Tp>, _Tp,
2541 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2542
2543 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2544 "should use explicit specialization above");
2545
2546 using type = typename __intrinsic_type<_Up, _SVecBytes>::type;
2547 };
2548#endif // _GLIBCXX_SIMD_HAVE_NEON
2549
2550// }}}
2551// __intrinsic_type (PPC){{{
2552#ifdef __ALTIVEC__
2553template <typename _Tp>
2554 struct __intrinsic_type_impl;
2555
2556#define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2557 template <> \
2558 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2559_GLIBCXX_SIMD_PPC_INTRIN(float);
2560#ifdef __VSX__
2561_GLIBCXX_SIMD_PPC_INTRIN(double);
2562#endif
2563_GLIBCXX_SIMD_PPC_INTRIN(signed char);
2564_GLIBCXX_SIMD_PPC_INTRIN(unsigned char);
2565_GLIBCXX_SIMD_PPC_INTRIN(signed short);
2566_GLIBCXX_SIMD_PPC_INTRIN(unsigned short);
2567_GLIBCXX_SIMD_PPC_INTRIN(signed int);
2568_GLIBCXX_SIMD_PPC_INTRIN(unsigned int);
2569#if defined __VSX__ || __SIZEOF_LONG__ == 4
2570_GLIBCXX_SIMD_PPC_INTRIN(signed long);
2571_GLIBCXX_SIMD_PPC_INTRIN(unsigned long);
2572#endif
2573#ifdef __VSX__
2574_GLIBCXX_SIMD_PPC_INTRIN(signed long long);
2575_GLIBCXX_SIMD_PPC_INTRIN(unsigned long long);
2576#endif
2577#undef _GLIBCXX_SIMD_PPC_INTRIN
2578
2579template <typename _Tp, size_t _Bytes>
2580 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2581 {
2582 static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2583
2584 // allow _Tp == long double with -mlong-double-64
2585 static_assert(!(_S_is_ldouble && sizeof(long double) > sizeof(double)),
2586 "no __intrinsic_type support for 128-bit floating point on PowerPC");
2587
2588#ifndef __VSX__
2589 static_assert(!(is_same_v<_Tp, double>
2590 || (_S_is_ldouble && sizeof(long double) == sizeof(double))),
2591 "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX");
2592#endif
2593
2594 static constexpr auto __element_type()
2595 {
2596 if constexpr (is_floating_point_v<_Tp>)
2597 {
2598 if constexpr (_S_is_ldouble)
2599 return double {};
2600 else
2601 return _Tp {};
2602 }
2603 else if constexpr (is_signed_v<_Tp>)
2604 {
2605 if constexpr (sizeof(_Tp) == sizeof(_SChar))
2606 return _SChar {};
2607 else if constexpr (sizeof(_Tp) == sizeof(short))
2608 return short {};
2609 else if constexpr (sizeof(_Tp) == sizeof(int))
2610 return int {};
2611 else if constexpr (sizeof(_Tp) == sizeof(_LLong))
2612 return _LLong {};
2613 }
2614 else
2615 {
2616 if constexpr (sizeof(_Tp) == sizeof(_UChar))
2617 return _UChar {};
2618 else if constexpr (sizeof(_Tp) == sizeof(_UShort))
2619 return _UShort {};
2620 else if constexpr (sizeof(_Tp) == sizeof(_UInt))
2621 return _UInt {};
2622 else if constexpr (sizeof(_Tp) == sizeof(_ULLong))
2623 return _ULLong {};
2624 }
2625 }
2626
2627 using type = typename __intrinsic_type_impl<decltype(__element_type())>::type;
2628 };
2629#endif // __ALTIVEC__
2630
2631// }}}
2632// _SimdWrapper<bool>{{{1
2633template <size_t _Width>
2634 struct _SimdWrapper<bool, _Width,
2635 void_t<typename __bool_storage_member_type<_Width>::type>>
2636 {
2637 using _BuiltinType = typename __bool_storage_member_type<_Width>::type;
2638 using value_type = bool;
2639
2640 static constexpr size_t _S_full_size = sizeof(_BuiltinType) * __CHAR_BIT__;
2641
2642 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2643 __as_full_vector() const
2644 { return _M_data; }
2645
2646 _GLIBCXX_SIMD_INTRINSIC constexpr
2647 _SimdWrapper() = default;
2648
2649 _GLIBCXX_SIMD_INTRINSIC constexpr
2650 _SimdWrapper(_BuiltinType __k) : _M_data(__k) {};
2651
2652 _GLIBCXX_SIMD_INTRINSIC
2653 operator const _BuiltinType&() const
2654 { return _M_data; }
2655
2656 _GLIBCXX_SIMD_INTRINSIC
2657 operator _BuiltinType&()
2658 { return _M_data; }
2659
2660 _GLIBCXX_SIMD_INTRINSIC _BuiltinType
2661 __intrin() const
2662 { return _M_data; }
2663
2664 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2665 operator[](size_t __i) const
2666 { return _M_data & (_BuiltinType(1) << __i); }
2667
2668 template <size_t __i>
2669 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2670 operator[](_SizeConstant<__i>) const
2671 { return _M_data & (_BuiltinType(1) << __i); }
2672
2673 _GLIBCXX_SIMD_INTRINSIC constexpr void
2674 _M_set(size_t __i, value_type __x)
2675 {
2676 if (__x)
2677 _M_data |= (_BuiltinType(1) << __i);
2678 else
2679 _M_data &= ~(_BuiltinType(1) << __i);
2680 }
2681
2682 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2683 _M_is_constprop() const
2684 { return __builtin_constant_p(_M_data); }
2685
2686 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2687 _M_is_constprop_none_of() const
2688 {
2689 if (__builtin_constant_p(_M_data))
2690 {
2691 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2692 constexpr _BuiltinType __active_mask
2693 = ~_BuiltinType() >> (__nbits - _Width);
2694 return (_M_data & __active_mask) == 0;
2695 }
2696 return false;
2697 }
2698
2699 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2700 _M_is_constprop_all_of() const
2701 {
2702 if (__builtin_constant_p(_M_data))
2703 {
2704 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2705 constexpr _BuiltinType __active_mask
2706 = ~_BuiltinType() >> (__nbits - _Width);
2707 return (_M_data & __active_mask) == __active_mask;
2708 }
2709 return false;
2710 }
2711
2712 _BuiltinType _M_data;
2713 };
2714
2715// _SimdWrapperBase{{{1
2716template <bool _MustZeroInitPadding, typename _BuiltinType>
2717 struct _SimdWrapperBase;
2718
2719template <typename _BuiltinType>
2720 struct _SimdWrapperBase<false, _BuiltinType> // no padding or no SNaNs
2721 {
2722 _GLIBCXX_SIMD_INTRINSIC constexpr
2723 _SimdWrapperBase() = default;
2724
2725 _GLIBCXX_SIMD_INTRINSIC constexpr
2726 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2727
2728 _BuiltinType _M_data;
2729 };
2730
2731template <typename _BuiltinType>
2732 struct _SimdWrapperBase<true, _BuiltinType> // with padding that needs to
2733 // never become SNaN
2734 {
2735 _GLIBCXX_SIMD_INTRINSIC constexpr
2736 _SimdWrapperBase() : _M_data() {}
2737
2738 _GLIBCXX_SIMD_INTRINSIC constexpr
2739 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2740
2741 _BuiltinType _M_data;
2742 };
2743
2744// }}}
2745// _SimdWrapper{{{
2746struct _DisabledSimdWrapper;
2747
2748template <typename _Tp, size_t _Width>
2749 struct _SimdWrapper<
2750 _Tp, _Width,
2751 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2752 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2753 && sizeof(_Tp) * _Width
2754 == sizeof(__vector_type_t<_Tp, _Width>),
2755 __vector_type_t<_Tp, _Width>>
2756 {
2757 static constexpr bool _S_need_default_init
2758 = __has_iec559_behavior<__signaling_NaN, _Tp>::value
2759 and sizeof(_Tp) * _Width == sizeof(__vector_type_t<_Tp, _Width>);
2760
2761 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2762
2763 using _Base = _SimdWrapperBase<_S_need_default_init, _BuiltinType>;
2764
2765 static_assert(__is_vectorizable_v<_Tp>);
2766 static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then
2767
2768 using value_type = _Tp;
2769
2770 static inline constexpr size_t _S_full_size
2771 = sizeof(_BuiltinType) / sizeof(value_type);
2772 static inline constexpr int _S_size = _Width;
2773 static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2774
2775 using _Base::_M_data;
2776
2777 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2778 __as_full_vector() const
2779 { return _M_data; }
2780
2781 _GLIBCXX_SIMD_INTRINSIC constexpr
2782 _SimdWrapper(initializer_list<_Tp> __init)
2783 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2784 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2785 return __init.begin()[__i.value];
2786 })) {}
2787
2788 _GLIBCXX_SIMD_INTRINSIC constexpr
2789 _SimdWrapper() = default;
2790
2791 _GLIBCXX_SIMD_INTRINSIC constexpr
2792 _SimdWrapper(const _SimdWrapper&) = default;
2793
2794 _GLIBCXX_SIMD_INTRINSIC constexpr
2795 _SimdWrapper(_SimdWrapper&&) = default;
2796
2797 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2798 operator=(const _SimdWrapper&) = default;
2799
2800 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2801 operator=(_SimdWrapper&&) = default;
2802
2803 // Convert from exactly matching __vector_type_t
2804 using _SimdWrapperBase<_S_need_default_init, _BuiltinType>::_SimdWrapperBase;
2805
2806 // Convert from __intrinsic_type_t if __intrinsic_type_t and __vector_type_t differ, otherwise
2807 // this ctor should not exist. Making the argument type unusable is our next best solution.
2808 _GLIBCXX_SIMD_INTRINSIC constexpr
2809 _SimdWrapper(conditional_t<is_same_v<_BuiltinType, __intrinsic_type_t<_Tp, _Width>>,
2810 _DisabledSimdWrapper, __intrinsic_type_t<_Tp, _Width>> __x)
2811 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2812
2813 // Convert from different __vector_type_t, but only if bit reinterpretation is a correct
2814 // conversion of the value_type
2815 template <typename _V, typename _TVT = _VectorTraits<_V>,
2816 typename = enable_if_t<sizeof(typename _TVT::value_type) == sizeof(_Tp)
2817 and sizeof(_V) == sizeof(_BuiltinType)
2818 and is_integral_v<_Tp>
2819 and is_integral_v<typename _TVT::value_type>>>
2820 _GLIBCXX_SIMD_INTRINSIC constexpr
2821 _SimdWrapper(_V __x)
2822 : _Base(reinterpret_cast<_BuiltinType>(__x)) {}
2823
2824 template <typename... _As,
2825 typename = enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2826 && sizeof...(_As) <= _Width)>>
2827 _GLIBCXX_SIMD_INTRINSIC constexpr
2828 operator _SimdTuple<_Tp, _As...>() const
2829 {
2830 return __generate_from_n_evaluations<sizeof...(_As), _SimdTuple<_Tp, _As...>>(
2831 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2832 { return _M_data[int(__i)]; });
2833 }
2834
2835 _GLIBCXX_SIMD_INTRINSIC constexpr
2836 operator const _BuiltinType&() const
2837 { return _M_data; }
2838
2839 _GLIBCXX_SIMD_INTRINSIC constexpr
2840 operator _BuiltinType&()
2841 { return _M_data; }
2842
2843 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2844 operator[](size_t __i) const
2845 { return _M_data[__i]; }
2846
2847 template <size_t __i>
2848 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2849 operator[](_SizeConstant<__i>) const
2850 { return _M_data[__i]; }
2851
2852 _GLIBCXX_SIMD_INTRINSIC constexpr void
2853 _M_set(size_t __i, _Tp __x)
2854 {
2855 if (__builtin_is_constant_evaluated())
2856 _M_data = __generate_from_n_evaluations<_Width, _BuiltinType>([&](auto __j) {
2857 return __j == __i ? __x : _M_data[__j()];
2858 });
2859 else
2860 _M_data[__i] = __x;
2861 }
2862
2863 _GLIBCXX_SIMD_INTRINSIC
2864 constexpr bool
2865 _M_is_constprop() const
2866 { return __builtin_constant_p(_M_data); }
2867
2868 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2869 _M_is_constprop_none_of() const
2870 {
2871 if (__builtin_constant_p(_M_data))
2872 {
2873 bool __r = true;
2874 if constexpr (is_floating_point_v<_Tp>)
2875 {
2876 using _Ip = __int_for_sizeof_t<_Tp>;
2877 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2878 __execute_n_times<_Width>(
2879 [&](auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2880 }
2881 else
2882 __execute_n_times<_Width>(
2883 [&](auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2884 if (__builtin_constant_p(__r))
2885 return __r;
2886 }
2887 return false;
2888 }
2889
2890 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2891 _M_is_constprop_all_of() const
2892 {
2893 if (__builtin_constant_p(_M_data))
2894 {
2895 bool __r = true;
2896 if constexpr (is_floating_point_v<_Tp>)
2897 {
2898 using _Ip = __int_for_sizeof_t<_Tp>;
2899 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2900 __execute_n_times<_Width>(
2901 [&](auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2902 }
2903 else
2904 __execute_n_times<_Width>(
2905 [&](auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2906 if (__builtin_constant_p(__r))
2907 return __r;
2908 }
2909 return false;
2910 }
2911 };
2912
2913// }}}
2914
2915// __vectorized_sizeof {{{
2916template <typename _Tp>
2917 constexpr size_t
2918 __vectorized_sizeof()
2919 {
2920 if constexpr (!__is_vectorizable_v<_Tp>)
2921 return 0;
2922
2923 if constexpr (sizeof(_Tp) <= 8)
2924 {
2925 // X86:
2926 if constexpr (__have_avx512bw)
2927 return 64;
2928 if constexpr (__have_avx512f && sizeof(_Tp) >= 4)
2929 return 64;
2930 if constexpr (__have_avx2)
2931 return 32;
2932 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2933 return 32;
2934 if constexpr (__have_sse2)
2935 return 16;
2936 if constexpr (__have_sse && is_same_v<_Tp, float>)
2937 return 16;
2938 /* The following is too much trouble because of mixed MMX and x87 code.
2939 * While nothing here explicitly calls MMX instructions of registers,
2940 * they are still emitted but no EMMS cleanup is done.
2941 if constexpr (__have_mmx && sizeof(_Tp) <= 4 && is_integral_v<_Tp>)
2942 return 8;
2943 */
2944
2945 // PowerPC:
2946 if constexpr (__have_power8vec
2947 || (__have_power_vmx && (sizeof(_Tp) < 8))
2948 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2949 return 16;
2950
2951 // ARM:
2952 if constexpr (__have_neon_a64)
2953 return 16;
2954 if constexpr (__have_neon_a32 and (not is_floating_point_v<_Tp>
2955 or is_same_v<_Tp, float>))
2956 return 16;
2957 if constexpr (__have_neon
2958 && sizeof(_Tp) < 8
2959 // Only allow fp if the user allows non-ICE559 fp (e.g.
2960 // via -ffast-math). ARMv7 NEON fp is not conforming to
2961 // IEC559.
2962 && (__support_neon_float || !is_floating_point_v<_Tp>))
2963 return 16;
2964 }
2965
2966 return sizeof(_Tp);
2967 }
2968
2969// }}}
2970namespace simd_abi {
2971// most of simd_abi is defined in simd_detail.h
2972template <typename _Tp>
2973 inline constexpr int max_fixed_size
2974 = ((__have_avx512bw && sizeof(_Tp) == 1)
2975 || (__have_sve && __sve_vectorized_size_bytes/sizeof(_Tp) >= 64)) ? 64 : 32;
2976
2977// compatible {{{
2978#if defined __x86_64__ || defined __aarch64__
2979template <typename _Tp>
2980 using compatible = conditional_t<(sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2981#elif defined __ARM_NEON
2982// FIXME: not sure, probably needs to be scalar (or dependent on the hard-float
2983// ABI?)
2984template <typename _Tp>
2985 using compatible
2986 = conditional_t<(sizeof(_Tp) < 8
2987 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2988 _VecBuiltin<16>, scalar>;
2989#else
2990template <typename>
2991 using compatible = scalar;
2992#endif
2993
2994// }}}
2995// native {{{
2996template <typename _Tp>
2997 constexpr auto
2998 __determine_native_abi()
2999 {
3000 constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
3001 if constexpr (__bytes == sizeof(_Tp))
3002 return static_cast<scalar*>(nullptr);
3003 else if constexpr (__have_sve)
3004 return static_cast<_SveAbi<__sve_vectorized_size_bytes>*>(nullptr);
3005 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
3006 return static_cast<_VecBltnBtmsk<__bytes>*>(nullptr);
3007 else
3008 return static_cast<_VecBuiltin<__bytes>*>(nullptr);
3009 }
3010
3011template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
3012 using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
3013
3014// }}}
3015// __default_abi {{{
3016#if defined _GLIBCXX_SIMD_DEFAULT_ABI
3017template <typename _Tp>
3018 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
3019#else
3020template <typename _Tp>
3021 using __default_abi = compatible<_Tp>;
3022#endif
3023
3024// }}}
3025} // namespace simd_abi
3026
3027// traits {{{1
3028template <typename _Tp>
3029 struct is_simd_flag_type
3030 : false_type
3031 {};
3032
3033template <>
3034 struct is_simd_flag_type<element_aligned_tag>
3035 : true_type
3036 {};
3037
3038template <>
3039 struct is_simd_flag_type<vector_aligned_tag>
3040 : true_type
3041 {};
3042
3043template <size_t _Np>
3044 struct is_simd_flag_type<overaligned_tag<_Np>>
3045 : __bool_constant<(_Np > 0) and __has_single_bit(_Np)>
3046 {};
3047
3048template <typename _Tp>
3049 inline constexpr bool is_simd_flag_type_v = is_simd_flag_type<_Tp>::value;
3050
3051template <typename _Tp, typename = enable_if_t<is_simd_flag_type_v<_Tp>>>
3052 using _IsSimdFlagType = _Tp;
3053
3054// is_abi_tag {{{2
3055template <typename _Tp, typename = void_t<>>
3056 struct is_abi_tag : false_type {};
3057
3058template <typename _Tp>
3059 struct is_abi_tag<_Tp, void_t<typename _Tp::_IsValidAbiTag>>
3060 : public _Tp::_IsValidAbiTag {};
3061
3062template <typename _Tp>
3063 inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
3064
3065// is_simd(_mask) {{{2
3066template <typename _Tp>
3067 struct is_simd : public false_type {};
3068
3069template <typename _Tp>
3070 inline constexpr bool is_simd_v = is_simd<_Tp>::value;
3071
3072template <typename _Tp>
3073 struct is_simd_mask : public false_type {};
3074
3075template <typename _Tp>
3076inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
3077
3078// simd_size {{{2
3079template <typename _Tp, typename _Abi, typename = void>
3080 struct __simd_size_impl {};
3081
3082template <typename _Tp, typename _Abi>
3083 struct __simd_size_impl<
3084 _Tp, _Abi,
3085 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
3086 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
3087
3088template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
3089 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
3090
3091template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
3092 inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
3093
3094// simd_abi::deduce {{{2
3095template <typename _Tp, size_t _Np, typename = void>
3096 struct __deduce_impl;
3097
3098template <typename _Tp, size_t _Np, typename = void>
3099 struct __no_sve_deduce_impl;
3100
3101namespace simd_abi {
3102/**
3103 * @tparam _Tp The requested `value_type` for the elements.
3104 * @tparam _Np The requested number of elements.
3105 * @tparam _Abis This parameter is ignored, since this implementation cannot
3106 * make any use of it. Either __a good native ABI is matched and used as `type`
3107 * alias, or the `fixed_size<_Np>` ABI is used, which internally is built from
3108 * the best matching native ABIs.
3109 */
3110 template <typename _Tp, size_t _Np, typename...>
3111 struct deduce : __deduce_impl<_Tp, _Np> {};
3112
3113template <typename _Tp, size_t _Np, typename... _Abis>
3114 using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type;
3115
3116template <typename _Tp, size_t _Np, typename...>
3117 struct __no_sve_deduce : __no_sve_deduce_impl<_Tp, _Np> {};
3118
3119template <typename _Tp, size_t _Np, typename... _Abis>
3120 using __no_sve_deduce_t = typename __no_sve_deduce<_Tp, _Np, _Abis...>::type;
3121} // namespace simd_abi
3122
3123// }}}2
3124// rebind_simd {{{2
3125template <typename _Tp, typename _V, typename = void>
3126 struct rebind_simd;
3127
3128template <typename _Tp, typename _Up, typename _Abi>
3129 struct rebind_simd<_Tp, simd<_Up, _Abi>,
3130 void_t<std::conditional_t<!__is_sve_abi<_Abi>(),
3131 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3132 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>>
3133 {
3134 using type = simd<_Tp, std::conditional_t<
3135 !__is_sve_abi<_Abi>(),
3136 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3137 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>;
3138 };
3139
3140template <typename _Tp, typename _Up, typename _Abi>
3141 struct rebind_simd<_Tp, simd_mask<_Up, _Abi>,
3142 void_t<std::conditional_t<!__is_sve_abi<_Abi>(),
3143 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3144 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>>
3145 {
3146 using type = simd_mask<_Tp, std::conditional_t<
3147 !__is_sve_abi<_Abi>(),
3148 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3149 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>;
3150 };
3151
3152template <typename _Tp, typename _V>
3153 using rebind_simd_t = typename rebind_simd<_Tp, _V>::type;
3154
3155// resize_simd {{{2
3156template <int _Np, typename _V, typename = void>
3157 struct resize_simd;
3158
3159template <int _Np, typename _Tp, typename _Abi>
3160 struct resize_simd<_Np, simd<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
3161 { using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
3162
3163template <int _Np, typename _Tp, typename _Abi>
3164 struct resize_simd<_Np, simd_mask<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
3165 { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
3166
3167template <int _Np, typename _V>
3168 using resize_simd_t = typename resize_simd<_Np, _V>::type;
3169
3170// }}}2
3171// memory_alignment {{{2
3172template <typename _Tp, typename _Up = typename _Tp::value_type>
3173 struct memory_alignment
3174 : public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
3175
3176template <typename _Tp, typename _Up = typename _Tp::value_type>
3177 inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
3178
3179// class template simd [simd] {{{1
3180template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
3181 class simd;
3182
3183template <typename _Tp, typename _Abi>
3184 struct is_simd<simd<_Tp, _Abi>> : public true_type {};
3185
3186template <typename _Tp>
3187 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
3188
3189template <typename _Tp, int _Np>
3190 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
3191
3192template <typename _Tp, size_t _Np>
3193 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
3194
3195// class template simd_mask [simd_mask] {{{1
3196template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
3197 class simd_mask;
3198
3199template <typename _Tp, typename _Abi>
3200 struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {};
3201
3202template <typename _Tp>
3203 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
3204
3205template <typename _Tp, int _Np>
3206 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
3207
3208template <typename _Tp, size_t _Np>
3209 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
3210
3211// casts [simd.casts] {{{1
3212// static_simd_cast {{{2
3213template <typename _Tp, typename _Up, typename _Ap, bool = is_simd_v<_Tp>, typename = void>
3214 struct __static_simd_cast_return_type;
3215
3216template <typename _Tp, typename _A0, typename _Up, typename _Ap>
3217 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false, void>
3218 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
3219
3220template <typename _Tp, typename _Up, typename _Ap>
3221 struct __static_simd_cast_return_type<
3222 _Tp, _Up, _Ap, true, enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
3223 { using type = _Tp; };
3224
3225template <typename _Tp, typename _Ap>
3226 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
3227#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
3228 enable_if_t<__is_vectorizable_v<_Tp>>
3229#else
3230 void
3231#endif
3232 >
3233 { using type = simd<_Tp, _Ap>; };
3234
3235template <typename _Tp, typename = void>
3236 struct __safe_make_signed { using type = _Tp;};
3237
3238template <typename _Tp>
3239 struct __safe_make_signed<_Tp, enable_if_t<is_integral_v<_Tp>>>
3240 {
3241 // the extra make_unsigned_t is because of PR85951
3242 using type = make_signed_t<make_unsigned_t<_Tp>>;
3243 };
3244
3245template <typename _Tp>
3246 using safe_make_signed_t = typename __safe_make_signed<_Tp>::type;
3247
3248template <typename _Tp, typename _Up, typename _Ap>
3249 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
3250#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
3251 enable_if_t<__is_vectorizable_v<_Tp>>
3252#else
3253 void
3254#endif
3255 >
3256 {
3257 using type = conditional_t<
3258 (is_integral_v<_Up> && is_integral_v<_Tp> &&
3259#ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
3260 is_signed_v<_Up> != is_signed_v<_Tp> &&
3261#endif
3262 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
3263 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
3264 };
3265
3266template <typename _Tp, typename _Up, typename _Ap,
3267 typename _R
3268 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3269 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
3270 static_simd_cast(const simd<_Up, _Ap>& __x)
3271 {
3272 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
3273 return __x;
3274 else
3275 {
3276 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
3277 __c;
3278 return _R(__private_init, __c(__data(__x)));
3279 }
3280 }
3281
3282namespace __proposed {
3283template <typename _Tp, typename _Up, typename _Ap,
3284 typename _R
3285 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3286 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR typename _R::mask_type
3287 static_simd_cast(const simd_mask<_Up, _Ap>& __x)
3288 {
3289 using _RM = typename _R::mask_type;
3290 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
3291 typename _RM::simd_type::value_type>(__x)};
3292 }
3293
3294template <typename _To, typename _Up, typename _Abi>
3295 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3296 _To
3297 simd_bit_cast(const simd<_Up, _Abi>& __x)
3298 {
3299 using _Tp = typename _To::value_type;
3300 using _ToMember = typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3301 using _From = simd<_Up, _Abi>;
3302 using _FromMember = typename _SimdTraits<_Up, _Abi>::_SimdMember;
3303 // with concepts, the following should be constraints
3304 static_assert(sizeof(_To) == sizeof(_From));
3305 static_assert(is_trivially_copyable_v<_Tp> && is_trivially_copyable_v<_Up>);
3306 static_assert(is_trivially_copyable_v<_ToMember> && is_trivially_copyable_v<_FromMember>);
3307#if __has_builtin(__builtin_bit_cast)
3308 return {__private_init, __builtin_bit_cast(_ToMember, __data(__x))};
3309#else
3310 return {__private_init, __bit_cast<_ToMember>(__data(__x))};
3311#endif
3312 }
3313
3314template <typename _To, typename _Up, typename _Abi>
3315 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3316 _To
3317 simd_bit_cast(const simd_mask<_Up, _Abi>& __x)
3318 {
3319 using _From = simd_mask<_Up, _Abi>;
3320 static_assert(sizeof(_To) == sizeof(_From));
3321 static_assert(is_trivially_copyable_v<_From>);
3322 // _To can be simd<T, A>, specifically simd<T, fixed_size<N>> in which case _To is not trivially
3323 // copyable.
3324 if constexpr (is_simd_v<_To>)
3325 {
3326 using _Tp = typename _To::value_type;
3327 using _ToMember = typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3328 static_assert(is_trivially_copyable_v<_ToMember>);
3329#if __has_builtin(__builtin_bit_cast)
3330 return {__private_init, __builtin_bit_cast(_ToMember, __x)};
3331#else
3332 return {__private_init, __bit_cast<_ToMember>(__x)};
3333#endif
3334 }
3335 else
3336 {
3337 static_assert(is_trivially_copyable_v<_To>);
3338#if __has_builtin(__builtin_bit_cast)
3339 return __builtin_bit_cast(_To, __x);
3340#else
3341 return __bit_cast<_To>(__x);
3342#endif
3343 }
3344 }
3345} // namespace __proposed
3346
3347// simd_cast {{{2
3348template <typename _Tp, typename _Up, typename _Ap,
3349 typename _To = __value_type_or_identity_t<_Tp>>
3350 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3351 simd_cast(const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
3352 -> decltype(static_simd_cast<_Tp>(__x))
3353 { return static_simd_cast<_Tp>(__x); }
3354
3355namespace __proposed {
3356template <typename _Tp, typename _Up, typename _Ap,
3357 typename _To = __value_type_or_identity_t<_Tp>>
3358 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3359 simd_cast(const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
3360 -> decltype(static_simd_cast<_Tp>(__x))
3361 { return static_simd_cast<_Tp>(__x); }
3362} // namespace __proposed
3363
3364// }}}2
3365// resizing_simd_cast {{{
3366namespace __proposed {
3367/* Proposed spec:
3368
3369template <class T, class U, class Abi>
3370T resizing_simd_cast(const simd<U, Abi>& x)
3371
3372p1 Constraints:
3373 - is_simd_v<T> is true and
3374 - T::value_type is the same type as U
3375
3376p2 Returns:
3377 A simd object with the i^th element initialized to x[i] for all i in the
3378 range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3379 than simd_size_v<U, Abi>, the remaining elements are value-initialized.
3380
3381template <class T, class U, class Abi>
3382T resizing_simd_cast(const simd_mask<U, Abi>& x)
3383
3384p1 Constraints: is_simd_mask_v<T> is true
3385
3386p2 Returns:
3387 A simd_mask object with the i^th element initialized to x[i] for all i in
3388the range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3389 than simd_size_v<U, Abi>, the remaining elements are initialized to false.
3390
3391 */
3392
3393template <typename _Tp, typename _Up, typename _Ap>
3394 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR enable_if_t<
3395 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3396 resizing_simd_cast(const simd<_Up, _Ap>& __x)
3397 {
3398 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3399 return __x;
3400 else if (__builtin_is_constant_evaluated())
3401 return _Tp([&](auto __i) constexpr {
3402 return __i < simd_size_v<_Up, _Ap> ? __x[__i] : _Up();
3403 });
3404 else if constexpr (simd_size_v<_Up, _Ap> == 1)
3405 {
3406 _Tp __r{};
3407 __r[0] = __x[0];
3408 return __r;
3409 }
3410 else if constexpr (_Tp::size() == 1)
3411 return __x[0];
3412 else if constexpr (sizeof(_Tp) == sizeof(__x)
3413 && !__is_fixed_size_abi_v<_Ap> && !__is_sve_abi<_Ap>())
3414 return {__private_init,
3415 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3416 _Ap::_S_masked(__data(__x))._M_data)};
3417 else
3418 {
3419 _Tp __r{};
3420 __builtin_memcpy(&__data(__r), &__data(__x),
3421 sizeof(_Up)
3422 * std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3423 return __r;
3424 }
3425 }
3426
3427template <typename _Tp, typename _Up, typename _Ap>
3428 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3429 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3430 resizing_simd_cast(const simd_mask<_Up, _Ap>& __x)
3431 {
3432 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3433 typename _Tp::simd_type::value_type>(__x)};
3434 }
3435} // namespace __proposed
3436
3437// }}}
3438// to_fixed_size {{{2
3439template <typename _Tp, int _Np>
3440 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3441 to_fixed_size(const fixed_size_simd<_Tp, _Np>& __x)
3442 { return __x; }
3443
3444template <typename _Tp, int _Np>
3445 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3446 to_fixed_size(const fixed_size_simd_mask<_Tp, _Np>& __x)
3447 { return __x; }
3448
3449template <typename _Tp, typename _Ap>
3450 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>
3451 to_fixed_size(const simd<_Tp, _Ap>& __x)
3452 {
3453 using _Rp = fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>;
3454 return _Rp([&__x](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3455 }
3456
3457template <typename _Tp, typename _Ap>
3458 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, simd_size_v<_Tp, _Ap>>
3459 to_fixed_size(const simd_mask<_Tp, _Ap>& __x)
3460 {
3461 return {__private_init,
3462 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }};
3463 }
3464
3465// to_native {{{2
3466template <typename _Tp, int _Np>
3467 _GLIBCXX_SIMD_INTRINSIC
3468 enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3469 to_native(const fixed_size_simd<_Tp, _Np>& __x)
3470 {
3471 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3472 __x.copy_to(__mem, vector_aligned);
3473 return {__mem, vector_aligned};
3474 }
3475
3476template <typename _Tp, int _Np>
3477 _GLIBCXX_SIMD_INTRINSIC
3478 enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3479 to_native(const fixed_size_simd_mask<_Tp, _Np>& __x)
3480 {
3481 return native_simd_mask<_Tp>(
3482 __private_init,
3483 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3484 }
3485
3486// to_compatible {{{2
3487template <typename _Tp, int _Np>
3488 _GLIBCXX_SIMD_INTRINSIC enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3489 to_compatible(const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3490 {
3491 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3492 __x.copy_to(__mem, vector_aligned);
3493 return {__mem, vector_aligned};
3494 }
3495
3496template <typename _Tp, int _Np>
3497 _GLIBCXX_SIMD_INTRINSIC
3498 enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3499 to_compatible(const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3500 {
3501 return simd_mask<_Tp>(
3502 __private_init,
3503 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3504 }
3505
3506// masked assignment [simd_mask.where] {{{1
3507
3508// where_expression {{{1
3509// const_where_expression<M, T> {{{2
3510template <typename _M, typename _Tp>
3511 class const_where_expression
3512 {
3513 using _V = _Tp;
3514 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3515
3516 struct _Wrapper { using value_type = _V; };
3517
3518 protected:
3519 using _Impl = typename _V::_Impl;
3520
3521 using value_type =
3522 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3523
3524 _GLIBCXX_SIMD_INTRINSIC friend const _M&
3525 __get_mask(const const_where_expression& __x)
3526 { return __x._M_k; }
3527
3528 _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3529 __get_lvalue(const const_where_expression& __x)
3530 { return __x._M_value; }
3531
3532 const _M& _M_k;
3533 _Tp& _M_value;
3534
3535 public:
3536 const_where_expression(const const_where_expression&) = delete;
3537
3538 const_where_expression& operator=(const const_where_expression&) = delete;
3539
3540 _GLIBCXX_SIMD_INTRINSIC constexpr
3541 const_where_expression(const _M& __kk, const _Tp& dd)
3542 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3543
3544 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3545 operator-() const&&
3546 {
3547 return {__private_init,
3548 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3549 __data(_M_value))};
3550 }
3551
3552 template <typename _Up, typename _Flags>
3553 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3554 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3555 {
3556 return {__private_init,
3557 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3558 _Flags::template _S_apply<_V>(__mem))};
3559 }
3560
3561 template <typename _Up, typename _Flags>
3562 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3563 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3564 {
3565 _Impl::_S_masked_store(__data(_M_value),
3566 _Flags::template _S_apply<_V>(__mem),
3567 __data(_M_k));
3568 }
3569 };
3570
3571// const_where_expression<bool, T> {{{2
3572template <typename _Tp>
3573 class const_where_expression<bool, _Tp>
3574 {
3575 using _M = bool;
3576 using _V = _Tp;
3577
3578 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3579
3580 struct _Wrapper { using value_type = _V; };
3581
3582 protected:
3583 using value_type
3584 = typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3585
3586 _GLIBCXX_SIMD_INTRINSIC friend const _M&
3587 __get_mask(const const_where_expression& __x)
3588 { return __x._M_k; }
3589
3590 _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3591 __get_lvalue(const const_where_expression& __x)
3592 { return __x._M_value; }
3593
3594 const bool _M_k;
3595 _Tp& _M_value;
3596
3597 public:
3598 const_where_expression(const const_where_expression&) = delete;
3599 const_where_expression& operator=(const const_where_expression&) = delete;
3600
3601 _GLIBCXX_SIMD_INTRINSIC constexpr
3602 const_where_expression(const bool __kk, const _Tp& dd)
3603 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3604
3605 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3606 operator-() const&&
3607 { return _M_k ? -_M_value : _M_value; }
3608
3609 template <typename _Up, typename _Flags>
3610 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3611 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3612 { return _M_k ? static_cast<_V>(__mem[0]) : _M_value; }
3613
3614 template <typename _Up, typename _Flags>
3615 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3616 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3617 {
3618 if (_M_k)
3619 __mem[0] = _M_value;
3620 }
3621 };
3622
3623// where_expression<M, T> {{{2
3624template <typename _M, typename _Tp>
3625 class where_expression : public const_where_expression<_M, _Tp>
3626 {
3627 using _Impl = typename const_where_expression<_M, _Tp>::_Impl;
3628
3629 static_assert(!is_const<_Tp>::value,
3630 "where_expression may only be instantiated with __a non-const "
3631 "_Tp parameter");
3632
3633 using typename const_where_expression<_M, _Tp>::value_type;
3634 using const_where_expression<_M, _Tp>::_M_k;
3635 using const_where_expression<_M, _Tp>::_M_value;
3636
3637 static_assert(
3638 is_same<typename _M::abi_type, typename _Tp::abi_type>::value, "");
3639 static_assert(_M::size() == _Tp::size(), "");
3640
3641 _GLIBCXX_SIMD_INTRINSIC friend constexpr _Tp&
3642 __get_lvalue(where_expression& __x)
3643 { return __x._M_value; }
3644
3645 public:
3646 where_expression(const where_expression&) = delete;
3647 where_expression& operator=(const where_expression&) = delete;
3648
3649 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3650 where_expression(const _M& __kk, _Tp& dd)
3651 : const_where_expression<_M, _Tp>(__kk, dd) {}
3652
3653 template <typename _Up>
3654 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3655 operator=(_Up&& __x) &&
3656 {
3657 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3658 __to_value_type_or_member_type<_Tp>(
3659 static_cast<_Up&&>(__x)));
3660 }
3661
3662#define _GLIBCXX_SIMD_OP_(__op, __name) \
3663 template <typename _Up> \
3664 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3665 operator __op##=(_Up&& __x)&& \
3666 { \
3667 _Impl::template _S_masked_cassign( \
3668 __data(_M_k), __data(_M_value), \
3669 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3670 [](auto __impl, auto __lhs, auto __rhs) \
3671 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
3672 { return __impl.__name(__lhs, __rhs); }); \
3673 } \
3674 static_assert(true)
3675 _GLIBCXX_SIMD_OP_(+, _S_plus);
3676 _GLIBCXX_SIMD_OP_(-, _S_minus);
3677 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3678 _GLIBCXX_SIMD_OP_(/, _S_divides);
3679 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3680 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3681 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3682 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3683 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3684 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3685#undef _GLIBCXX_SIMD_OP_
3686
3687 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3688 operator++() &&
3689 {
3690 __data(_M_value)
3691 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3692 }
3693
3694 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3695 operator++(int) &&
3696 {
3697 __data(_M_value)
3698 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3699 }
3700
3701 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3702 operator--() &&
3703 {
3704 __data(_M_value)
3705 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3706 }
3707
3708 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3709 operator--(int) &&
3710 {
3711 __data(_M_value)
3712 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3713 }
3714
3715 // intentionally hides const_where_expression::copy_from
3716 template <typename _Up, typename _Flags>
3717 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3718 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3719 {
3720 __data(_M_value) = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3721 _Flags::template _S_apply<_Tp>(__mem));
3722 }
3723 };
3724
3725// where_expression<bool, T> {{{2
3726template <typename _Tp>
3727 class where_expression<bool, _Tp>
3728 : public const_where_expression<bool, _Tp>
3729 {
3730 using _M = bool;
3731 using typename const_where_expression<_M, _Tp>::value_type;
3732 using const_where_expression<_M, _Tp>::_M_k;
3733 using const_where_expression<_M, _Tp>::_M_value;
3734
3735 public:
3736 where_expression(const where_expression&) = delete;
3737 where_expression& operator=(const where_expression&) = delete;
3738
3739 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3740 where_expression(const _M& __kk, _Tp& dd)
3741 : const_where_expression<_M, _Tp>(__kk, dd) {}
3742
3743#define _GLIBCXX_SIMD_OP_(__op) \
3744 template <typename _Up> \
3745 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3746 operator __op(_Up&& __x)&& \
3747 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3748
3749 _GLIBCXX_SIMD_OP_(=)
3750 _GLIBCXX_SIMD_OP_(+=)
3751 _GLIBCXX_SIMD_OP_(-=)
3752 _GLIBCXX_SIMD_OP_(*=)
3753 _GLIBCXX_SIMD_OP_(/=)
3754 _GLIBCXX_SIMD_OP_(%=)
3755 _GLIBCXX_SIMD_OP_(&=)
3756 _GLIBCXX_SIMD_OP_(|=)
3757 _GLIBCXX_SIMD_OP_(^=)
3758 _GLIBCXX_SIMD_OP_(<<=)
3759 _GLIBCXX_SIMD_OP_(>>=)
3760 #undef _GLIBCXX_SIMD_OP_
3761
3762 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3763 operator++() &&
3764 { if (_M_k) ++_M_value; }
3765
3766 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3767 operator++(int) &&
3768 { if (_M_k) ++_M_value; }
3769
3770 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3771 operator--() &&
3772 { if (_M_k) --_M_value; }
3773
3774 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3775 operator--(int) &&
3776 { if (_M_k) --_M_value; }
3777
3778 // intentionally hides const_where_expression::copy_from
3779 template <typename _Up, typename _Flags>
3780 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3781 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3782 { if (_M_k) _M_value = __mem[0]; }
3783 };
3784
3785// where {{{1
3786template <typename _Tp, typename _Ap>
3787 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3788 where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3789 where(const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3790 { return {__k, __value}; }
3791
3792template <typename _Tp, typename _Ap>
3793 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3794 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3795 where(const typename simd<_Tp, _Ap>::mask_type& __k, const simd<_Tp, _Ap>& __value)
3796 { return {__k, __value}; }
3797
3798template <typename _Tp, typename _Ap>
3799 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3800 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3801 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, simd_mask<_Tp, _Ap>& __value)
3802 { return {__k, __value}; }
3803
3804template <typename _Tp, typename _Ap>
3805 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3806 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3807 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, const simd_mask<_Tp, _Ap>& __value)
3808 { return {__k, __value}; }
3809
3810template <typename _Tp>
3811 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR where_expression<bool, _Tp>
3812 where(_ExactBool __k, _Tp& __value)
3813 { return {__k, __value}; }
3814
3815template <typename _Tp>
3816 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR const_where_expression<bool, _Tp>
3817 where(_ExactBool __k, const _Tp& __value)
3818 { return {__k, __value}; }
3819
3820template <typename _Tp, typename _Ap>
3821 _GLIBCXX_SIMD_CONSTEXPR void
3822 where(bool __k, simd<_Tp, _Ap>& __value) = delete;
3823
3824template <typename _Tp, typename _Ap>
3825 _GLIBCXX_SIMD_CONSTEXPR void
3826 where(bool __k, const simd<_Tp, _Ap>& __value) = delete;
3827
3828// proposed mask iterations {{{1
3829namespace __proposed {
3830template <size_t _Np>
3831 class where_range
3832 {
3833 const bitset<_Np> __bits;
3834
3835 public:
3836 where_range(bitset<_Np> __b) : __bits(__b) {}
3837
3838 class iterator
3839 {
3840 size_t __mask;
3841 size_t __bit;
3842
3843 _GLIBCXX_SIMD_INTRINSIC void
3844 __next_bit()
3845 { __bit = __builtin_ctzl(__mask); }
3846
3847 _GLIBCXX_SIMD_INTRINSIC void
3848 __reset_lsb()
3849 {
3850 // 01100100 - 1 = 01100011
3851 __mask &= (__mask - 1);
3852 // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit));
3853 }
3854
3855 public:
3856 iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3857 iterator(const iterator&) = default;
3858 iterator(iterator&&) = default;
3859
3860 _GLIBCXX_SIMD_ALWAYS_INLINE size_t
3861 operator->() const
3862 { return __bit; }
3863
3864 _GLIBCXX_SIMD_ALWAYS_INLINE size_t
3865 operator*() const
3866 { return __bit; }
3867
3868 _GLIBCXX_SIMD_ALWAYS_INLINE iterator&
3869 operator++()
3870 {
3871 __reset_lsb();
3872 __next_bit();
3873 return *this;
3874 }
3875
3876 _GLIBCXX_SIMD_ALWAYS_INLINE iterator
3877 operator++(int)
3878 {
3879 iterator __tmp = *this;
3880 __reset_lsb();
3881 __next_bit();
3882 return __tmp;
3883 }
3884
3885 _GLIBCXX_SIMD_ALWAYS_INLINE bool
3886 operator==(const iterator& __rhs) const
3887 { return __mask == __rhs.__mask; }
3888
3889 _GLIBCXX_SIMD_ALWAYS_INLINE bool
3890 operator!=(const iterator& __rhs) const
3891 { return __mask != __rhs.__mask; }
3892 };
3893
3894 iterator
3895 begin() const
3896 { return __bits.to_ullong(); }
3897
3898 iterator
3899 end() const
3900 { return 0; }
3901 };
3902
3903template <typename _Tp, typename _Ap>
3904 where_range<simd_size_v<_Tp, _Ap>>
3905 where(const simd_mask<_Tp, _Ap>& __k)
3906 { return __k.__to_bitset(); }
3907
3908} // namespace __proposed
3909
3910// }}}1
3911// reductions [simd.reductions] {{{1
3912template <typename _Tp, typename _Abi, typename _BinaryOperation = plus<>>
3913 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3914 reduce(const simd<_Tp, _Abi>& __v, _BinaryOperation __binary_op = _BinaryOperation())
3915 { return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3916
3917template <typename _M, typename _V, typename _BinaryOperation = plus<>>
3918 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3919 reduce(const const_where_expression<_M, _V>& __x,
3920 typename _V::value_type __identity_element, _BinaryOperation __binary_op)
3921 {
3922 if (__builtin_expect(none_of(__get_mask(__x)), false))
3923 return __identity_element;
3924
3925 _V __tmp = __identity_element;
3926 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3927 __data(__get_lvalue(__x)));
3928 return reduce(__tmp, __binary_op);
3929 }
3930
3931template <typename _M, typename _V>
3932 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3933 reduce(const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3934 { return reduce(__x, 0, __binary_op); }
3935
3936template <typename _M, typename _V>
3937 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3938 reduce(const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3939 { return reduce(__x, 1, __binary_op); }
3940
3941template <typename _M, typename _V>
3942 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3943 reduce(const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3944 { return reduce(__x, ~typename _V::value_type(), __binary_op); }
3945
3946template <typename _M, typename _V>
3947 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3948 reduce(const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3949 { return reduce(__x, 0, __binary_op); }
3950
3951template <typename _M, typename _V>
3952 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3953 reduce(const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3954 { return reduce(__x, 0, __binary_op); }
3955
3956template <typename _Tp, typename _Abi>
3957 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3958 hmin(const simd<_Tp, _Abi>& __v) noexcept
3959 { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum()); }
3960
3961template <typename _Tp, typename _Abi>
3962 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3963 hmax(const simd<_Tp, _Abi>& __v) noexcept
3964 { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum()); }
3965
3966template <typename _M, typename _V>
3967 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3968 typename _V::value_type
3969 hmin(const const_where_expression<_M, _V>& __x) noexcept
3970 {
3971 using _Tp = typename _V::value_type;
3972 constexpr _Tp __id_elem =
3973#ifdef __FINITE_MATH_ONLY__
3974 __finite_max_v<_Tp>;
3975#else
3976 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3977#endif
3978 _V __tmp = __id_elem;
3979 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3980 __data(__get_lvalue(__x)));
3981 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3982 }
3983
3984template <typename _M, typename _V>
3985 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3986 typename _V::value_type
3987 hmax(const const_where_expression<_M, _V>& __x) noexcept
3988 {
3989 using _Tp = typename _V::value_type;
3990 constexpr _Tp __id_elem =
3991#ifdef __FINITE_MATH_ONLY__
3992 __finite_min_v<_Tp>;
3993#else
3994 [] {
3995 if constexpr (__value_exists_v<__infinity, _Tp>)
3996 return -__infinity_v<_Tp>;
3997 else
3998 return __finite_min_v<_Tp>;
3999 }();
4000#endif
4001 _V __tmp = __id_elem;
4002 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
4003 __data(__get_lvalue(__x)));
4004 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
4005 }
4006
4007// }}}1
4008// algorithms [simd.alg] {{{
4009template <typename _Tp, typename _Ap>
4010 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
4011 min(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
4012 { return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
4013
4014template <typename _Tp, typename _Ap>
4015 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
4016 max(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
4017 { return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
4018
4019template <typename _Tp, typename _Ap>
4020 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4021 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
4022 minmax(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
4023 {
4024 const auto pair_of_members
4025 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
4026 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
4027 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
4028 }
4029
4030template <typename _Tp, typename _Ap>
4031 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
4032 clamp(const simd<_Tp, _Ap>& __v, const simd<_Tp, _Ap>& __lo, const simd<_Tp, _Ap>& __hi)
4033 {
4034 using _Impl = typename _Ap::_SimdImpl;
4035 return {__private_init,
4036 _Impl::_S_min(__data(__hi),
4037 _Impl::_S_max(__data(__lo), __data(__v)))};
4038 }
4039
4040// }}}
4041
4042template <size_t... _Sizes, typename _Tp, typename _Ap,
4043 typename = enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
4044 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
4045 split(const simd<_Tp, _Ap>&);
4046
4047// __extract_part {{{
4048template <int _Index, int _Total, int _Combine = 1, typename _Tp, size_t _Np>
4049 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr
4050 conditional_t<_Np == _Total and _Combine == 1, _Tp, _SimdWrapper<_Tp, _Np / _Total * _Combine>>
4051 __extract_part(const _SimdWrapper<_Tp, _Np> __x);
4052
4053template <int _Index, int _Parts, int _Combine = 1, typename _Tp, typename _A0, typename... _As>
4054 _GLIBCXX_SIMD_INTRINSIC constexpr auto
4055 __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x);
4056
4057// }}}
4058// _SizeList {{{
4059template <size_t _V0, size_t... _Values>
4060 struct _SizeList
4061 {
4062 template <size_t _I>
4063 static constexpr size_t
4064 _S_at(_SizeConstant<_I> = {})
4065 {
4066 if constexpr (_I == 0)
4067 return _V0;
4068 else
4069 return _SizeList<_Values...>::template _S_at<_I - 1>();
4070 }
4071
4072 template <size_t _I>
4073 static constexpr auto
4074 _S_before(_SizeConstant<_I> = {})
4075 {
4076 if constexpr (_I == 0)
4077 return _SizeConstant<0>();
4078 else
4079 return _SizeConstant<
4080 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
4081 }
4082
4083 template <size_t _Np>
4084 static constexpr auto
4085 _S_pop_front(_SizeConstant<_Np> = {})
4086 {
4087 if constexpr (_Np == 0)
4088 return _SizeList();
4089 else
4090 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
4091 }
4092 };
4093
4094// }}}
4095// __extract_center {{{
4096template <typename _Tp, size_t _Np>
4097 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
4098 __extract_center(_SimdWrapper<_Tp, _Np> __x)
4099 {
4100 static_assert(_Np >= 4);
4101 static_assert(_Np % 4 == 0); // x0 - x1 - x2 - x3 -> return {x1, x2}
4102#if _GLIBCXX_SIMD_X86INTRIN // {{{
4103 if constexpr (__have_avx512f && sizeof(_Tp) * _Np == 64)
4104 {
4105 const auto __intrin = __to_intrin(__x);
4106 if constexpr (is_integral_v<_Tp>)
4107 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
4108 _mm512_shuffle_i32x4(__intrin, __intrin,
4109 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4110 else if constexpr (sizeof(_Tp) == 4)
4111 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
4112 _mm512_shuffle_f32x4(__intrin, __intrin,
4113 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4114 else if constexpr (sizeof(_Tp) == 8)
4115 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
4116 _mm512_shuffle_f64x2(__intrin, __intrin,
4117 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4118 else
4119 __assert_unreachable<_Tp>();
4120 }
4121 else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
4122 return __vector_bitcast<_Tp>(
4123 _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
4124 __hi128(__vector_bitcast<double>(__x)), 1));
4125 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
4126 return __vector_bitcast<_Tp>(
4127 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
4128 __lo128(__vector_bitcast<_LLong>(__x)),
4129 sizeof(_Tp) * _Np / 4));
4130 else
4131#endif // _GLIBCXX_SIMD_X86INTRIN }}}
4132 {
4133 __vector_type_t<_Tp, _Np / 2> __r;
4134 __builtin_memcpy(&__r,
4135 reinterpret_cast<const char*>(&__x)
4136 + sizeof(_Tp) * _Np / 4,
4137 sizeof(_Tp) * _Np / 2);
4138 return __r;
4139 }
4140 }
4141
4142template <typename _Tp, typename _A0, typename... _As>
4143 _GLIBCXX_SIMD_INTRINSIC
4144 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
4145 __extract_center(const _SimdTuple<_Tp, _A0, _As...>& __x)
4146 {
4147 if constexpr (sizeof...(_As) == 0)
4148 return __extract_center(__x.first);
4149 else
4150 return __extract_part<1, 4, 2>(__x);
4151 }
4152
4153// }}}
4154// __split_wrapper {{{
4155template <size_t... _Sizes, typename _Tp, typename... _As>
4156 auto
4157 __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
4158 {
4159 return split<_Sizes...>(
4160 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
4161 __x));
4162 }
4163
4164// }}}
4165
4166// split<simd>(simd) {{{
4167template <typename _V, typename _Ap,
4168 size_t _Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
4169 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == _Parts * _V::size()
4170 && is_simd_v<_V>, array<_V, _Parts>>
4171 split(const simd<typename _V::value_type, _Ap>& __x)
4172 {
4173 using _Tp = typename _V::value_type;
4174
4175 auto __gen_fallback = [&]() constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4176 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4177 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4178 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4179 { return __x[__i * _V::size() + __j]; });
4180 });
4181 };
4182
4183 if constexpr (_Parts == 1)
4184 {
4185 return {simd_cast<_V>(__x)};
4186 }
4187 else if (__x._M_is_constprop())
4188 {
4189 return __gen_fallback();
4190 }
4191#if _GLIBCXX_SIMD_HAVE_SVE
4192 else if constexpr(__is_sve_abi<_Ap>)
4193 {
4194 return __gen_fallback();
4195 }
4196#endif
4197 else if constexpr (
4198 __is_fixed_size_abi_v<_Ap>
4199 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
4200 || (__is_fixed_size_abi_v<typename _V::abi_type>
4201 && sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding
4202 )))
4203 {
4204 // fixed_size -> fixed_size (w/o padding) or scalar
4205#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4206 const __may_alias<_Tp>* const __element_ptr
4207 = reinterpret_cast<const __may_alias<_Tp>*>(&__data(__x));
4208 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4209 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4210 { return _V(__element_ptr + __i * _V::size(), vector_aligned); });
4211#else
4212 const auto& __xx = __data(__x);
4213 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4214 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4215 [[maybe_unused]] constexpr size_t __offset
4216 = decltype(__i)::value * _V::size();
4217 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4218 constexpr _SizeConstant<__j + __offset> __k;
4219 return __xx[__k];
4220 });
4221 });
4222#endif
4223 }
4224 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
4225 {
4226 // normally memcpy should work here as well
4227 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4228 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
4229 }
4230 else
4231 {
4232 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4233 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4234 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
4235 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4236 return __x[__i * _V::size() + __j];
4237 });
4238 else
4239 return _V(__private_init,
4240 __extract_part<decltype(__i)::value, _Parts>(__data(__x)));
4241 });
4242 }
4243 }
4244
4245// }}}
4246// split<simd_mask>(simd_mask) {{{
4247template <typename _V, typename _Ap,
4248 size_t _Parts = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
4249 enable_if_t<is_simd_mask_v<_V> && simd_size_v<typename
4250 _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
4251 split(const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
4252 {
4253 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
4254 return {__x};
4255 else if constexpr (_Parts == 1)
4256 return {__proposed::static_simd_cast<_V>(__x)};
4257 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
4258 && __is_avx_abi<_Ap>())
4259 return {_V(__private_init, __lo128(__data(__x))),
4260 _V(__private_init, __hi128(__data(__x)))};
4261 else if constexpr (_V::size() <= __CHAR_BIT__ * sizeof(_ULLong))
4262 {
4263 const bitset __bits = __x.__to_bitset();
4264 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4265 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4266 constexpr size_t __offset = __i * _V::size();
4267 return _V(__bitset_init, (__bits >> __offset).to_ullong());
4268 });
4269 }
4270 else
4271 {
4272 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4273 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4274 constexpr size_t __offset = __i * _V::size();
4275 return _V(__private_init,
4276 [&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4277 return __x[__j + __offset];
4278 });
4279 });
4280 }
4281 }
4282
4283// }}}
4284// split<_Sizes...>(simd) {{{
4285template <size_t... _Sizes, typename _Tp, typename _Ap, typename>
4286 _GLIBCXX_SIMD_ALWAYS_INLINE
4287 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
4288 split(const simd<_Tp, _Ap>& __x)
4289 {
4290 using _SL = _SizeList<_Sizes...>;
4291 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
4292 constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
4293 constexpr size_t _N0 = _SL::template _S_at<0>();
4294 using _V = __deduced_simd<_Tp, _N0>;
4295
4296 auto __gen_fallback = [&]() constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4297 {
4298 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4299 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4300 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4301 constexpr size_t __offset = _SL::_S_before(__i);
4302 return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4303 return __x[__offset + __j];
4304 });
4305 });
4306 };
4307
4308 if (__x._M_is_constprop())
4309 __gen_fallback();
4310#if _GLIBCXX_SIMD_HAVE_SVE
4311 else if constexpr (__have_sve)
4312 __gen_fallback();
4313#endif
4314 else if constexpr (_Np == _N0)
4315 {
4316 static_assert(sizeof...(_Sizes) == 1);
4317 return {simd_cast<_V>(__x)};
4318 }
4319 else if constexpr // split from fixed_size, such that __x::first.size == _N0
4320 (__is_fixed_size_abi_v<
4321 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
4322 {
4323 static_assert(
4324 !__is_fixed_size_abi_v<typename _V::abi_type>,
4325 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
4326 "fixed_size_simd "
4327 "when deduced?");
4328 // extract first and recurse (__split_wrapper is needed to deduce a new
4329 // _Sizes pack)
4330 return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
4331 __split_wrapper(_SL::template _S_pop_front<1>(),
4332 __data(__x).second));
4333 }
4334 else if constexpr ((!__is_fixed_size_abi_v<simd_abi::deduce_t<_Tp, _Sizes>> && ...))
4335 {
4336 constexpr array<size_t, sizeof...(_Sizes)> __size = {_Sizes...};
4337 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4338 [&](auto __i) constexpr {
4339 constexpr size_t __offset = [&]() {
4340 size_t __r = 0;
4341 for (unsigned __j = 0; __j < __i; ++__j)
4342 __r += __size[__j];
4343 return __r;
4344 }();
4345 return __deduced_simd<_Tp, __size[__i]>(
4346 __private_init,
4347 __extract_part<__offset, _Np, __size[__i]>(__data(__x)));
4348 });
4349 }
4350#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4351 const __may_alias<_Tp>* const __element_ptr
4352 = reinterpret_cast<const __may_alias<_Tp>*>(&__x);
4353 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4354 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4355 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4356 constexpr size_t __offset = _SL::_S_before(__i);
4357 constexpr size_t __base_align = alignof(simd<_Tp, _Ap>);
4358 constexpr size_t __a
4359 = __base_align - ((__offset * sizeof(_Tp)) % __base_align);
4360 constexpr size_t __b = ((__a - 1) & __a) ^ __a;
4361 constexpr size_t __alignment = __b == 0 ? __a : __b;
4362 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
4363 });
4364#else
4365 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4366 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4367 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4368 const auto& __xx = __data(__x);
4369 using _Offset = decltype(_SL::_S_before(__i));
4370 return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4371 constexpr _SizeConstant<_Offset::value + __j> __k;
4372 return __xx[__k];
4373 });
4374 });
4375#endif
4376 }
4377
4378// }}}
4379
4380// __subscript_in_pack {{{
4381template <size_t _I, typename _Tp, typename _Ap, typename... _As>
4382 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
4383 __subscript_in_pack(const simd<_Tp, _Ap>& __x, const simd<_Tp, _As>&... __xs)
4384 {
4385 if constexpr (_I < simd_size_v<_Tp, _Ap>)
4386 return __x[_I];
4387 else
4388 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
4389 }
4390
4391// }}}
4392// __store_pack_of_simd {{{
4393template <typename _Tp, typename _A0, typename... _As>
4394 _GLIBCXX_SIMD_INTRINSIC void
4395 __store_pack_of_simd(char* __mem, const simd<_Tp, _A0>& __x0, const simd<_Tp, _As>&... __xs)
4396 {
4397 constexpr size_t __n_bytes = sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4398 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4399 if constexpr (sizeof...(__xs) > 0)
4400 __store_pack_of_simd(__mem + __n_bytes, __xs...);
4401 }
4402
4403// }}}
4404// concat(simd...) {{{
4405template <typename _Tp, typename... _As, typename = __detail::__odr_helper>
4406 inline _GLIBCXX_SIMD_CONSTEXPR
4407 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4408 concat(const simd<_Tp, _As>&... __xs)
4409 {
4410 constexpr int _Np = (simd_size_v<_Tp, _As> + ...);
4411 using _Abi = simd_abi::deduce_t<_Tp, _Np>;
4412 using _Rp = simd<_Tp, _Abi>;
4413 using _RW = typename _SimdTraits<_Tp, _Abi>::_SimdMember;
4414 if constexpr (sizeof...(__xs) == 1)
4415 return simd_cast<_Rp>(__xs...);
4416 else if ((... && __xs._M_is_constprop()))
4417 return _Rp([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4418 { return __subscript_in_pack<__i>(__xs...); });
4419 else if constexpr (__is_simd_wrapper_v<_RW> and sizeof...(__xs) == 2)
4420 {
4421 return {__private_init,
4422 __vec_shuffle(__as_vector(__xs)..., std::make_index_sequence<_RW::_S_full_size>(),
4423 [](int __i) {
4424 constexpr int __sizes[2] = {int(simd_size_v<_Tp, _As>)...};
4425 constexpr int __vsizes[2]
4426 = {int(sizeof(__as_vector(__xs)) / sizeof(_Tp))...};
4427 constexpr int __padding0 = __vsizes[0] - __sizes[0];
4428 return __i >= _Np ? -1 : __i < __sizes[0] ? __i : __i + __padding0;
4429 })};
4430 }
4431 else if constexpr (__is_simd_wrapper_v<_RW> and sizeof...(__xs) == 3)
4432 return [](const auto& __x0, const auto& __x1, const auto& __x2)
4433 _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4434 return concat(concat(__x0, __x1), __x2);
4435 }(__xs...);
4436 else if constexpr (__is_simd_wrapper_v<_RW> and sizeof...(__xs) > 3)
4437 return [](const auto& __x0, const auto& __x1, const auto&... __rest)
4438 _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4439 return concat(concat(__x0, __x1), concat(__rest...));
4440 }(__xs...);
4441 else
4442 {
4443 _Rp __r{};
4444 __store_pack_of_simd(reinterpret_cast<char*>(&__data(__r)), __xs...);
4445 return __r;
4446 }
4447 }
4448
4449// }}}
4450// concat(array<simd>) {{{
4451template <typename _Tp, typename _Abi, size_t _Np>
4452 _GLIBCXX_SIMD_ALWAYS_INLINE
4453 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4454 concat(const array<simd<_Tp, _Abi>, _Np>& __x)
4455 {
4456 return __call_with_subscripts<_Np>(
4457 __x, [](const auto&... __xs) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4458 return concat(__xs...);
4459 });
4460 }
4461
4462// }}}
4463
4464/// @cond undocumented
4465// _SmartReference {{{
4466template <typename _Up, typename _Accessor = _Up,
4467 typename _ValueType = typename _Up::value_type>
4468 class _SmartReference
4469 {
4470 friend _Accessor;
4471 int _M_index;
4472 _Up& _M_obj;
4473
4474 _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType
4475 _M_read() const noexcept
4476 {
4477 if constexpr (is_arithmetic_v<_Up>)
4478 return _M_obj;
4479 else
4480 return _M_obj[_M_index];
4481 }
4482
4483 template <typename _Tp>
4484 _GLIBCXX_SIMD_INTRINSIC constexpr void
4485 _M_write(_Tp&& __x) const
4486 { _Accessor::_S_set(_M_obj, _M_index, static_cast<_Tp&&>(__x)); }
4487
4488 public:
4489 _GLIBCXX_SIMD_INTRINSIC constexpr
4490 _SmartReference(_Up& __o, int __i) noexcept
4491 : _M_index(__i), _M_obj(__o) {}
4492
4493 using value_type = _ValueType;
4494
4495 _GLIBCXX_SIMD_INTRINSIC
4496 _SmartReference(const _SmartReference&) = delete;
4497
4498 _GLIBCXX_SIMD_INTRINSIC constexpr
4499 operator value_type() const noexcept
4500 { return _M_read(); }
4501
4502 template <typename _Tp, typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4503 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4504 operator=(_Tp&& __x) &&
4505 {
4506 _M_write(static_cast<_Tp&&>(__x));
4507 return {_M_obj, _M_index};
4508 }
4509
4510#define _GLIBCXX_SIMD_OP_(__op) \
4511 template <typename _Tp, \
4512 typename _TT = decltype(declval<value_type>() __op declval<_Tp>()), \
4513 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4514 typename = _ValuePreservingOrInt<_TT, value_type>> \
4515 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4516 operator __op##=(_Tp&& __x) && \
4517 { \
4518 const value_type& __lhs = _M_read(); \
4519 _M_write(__lhs __op __x); \
4520 return {_M_obj, _M_index}; \
4521 }
4522 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4523 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4524 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4525#undef _GLIBCXX_SIMD_OP_
4526
4527 template <typename _Tp = void,
4528 typename = decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4529 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4530 operator++() &&
4531 {
4532 value_type __x = _M_read();
4533 _M_write(++__x);
4534 return {_M_obj, _M_index};
4535 }
4536
4537 template <typename _Tp = void,
4538 typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4539 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
4540 operator++(int) &&
4541 {
4542 const value_type __r = _M_read();
4543 value_type __x = __r;
4544 _M_write(++__x);
4545 return __r;
4546 }
4547
4548 template <typename _Tp = void,
4549 typename = decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4550 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4551 operator--() &&
4552 {
4553 value_type __x = _M_read();
4554 _M_write(--__x);
4555 return {_M_obj, _M_index};
4556 }
4557
4558 template <typename _Tp = void,
4559 typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4560 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
4561 operator--(int) &&
4562 {
4563 const value_type __r = _M_read();
4564 value_type __x = __r;
4565 _M_write(--__x);
4566 return __r;
4567 }
4568
4569 _GLIBCXX_SIMD_INTRINSIC friend void
4570 swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4571 conjunction<
4572 is_nothrow_constructible<value_type, _SmartReference&&>,
4573 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4574 {
4575 value_type __tmp = static_cast<_SmartReference&&>(__a);
4576 static_cast<_SmartReference&&>(__a) = static_cast<value_type>(__b);
4577 static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4578 }
4579
4580 _GLIBCXX_SIMD_INTRINSIC friend void
4581 swap(value_type& __a, _SmartReference&& __b) noexcept(
4582 conjunction<
4583 is_nothrow_constructible<value_type, value_type&&>,
4584 is_nothrow_assignable<value_type&, value_type&&>,
4585 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4586 {
4587 value_type __tmp(std::move(__a));
4588 __a = static_cast<value_type>(__b);
4589 static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4590 }
4591
4592 _GLIBCXX_SIMD_INTRINSIC friend void
4593 swap(_SmartReference&& __a, value_type& __b) noexcept(
4594 conjunction<
4595 is_nothrow_constructible<value_type, _SmartReference&&>,
4596 is_nothrow_assignable<value_type&, value_type&&>,
4597 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4598 {
4599 value_type __tmp(__a);
4600 static_cast<_SmartReference&&>(__a) = std::move(__b);
4601 __b = std::move(__tmp);
4602 }
4603 };
4604
4605// }}}
4606// __scalar_abi_wrapper {{{
4607template <int _Bytes>
4608 struct __scalar_abi_wrapper
4609 {
4610 template <typename _Tp> static constexpr size_t _S_full_size = 1;
4611 template <typename _Tp> static constexpr size_t _S_size = 1;
4612 template <typename _Tp> static constexpr size_t _S_is_partial = false;
4613
4614 template <typename _Tp, typename _Abi = simd_abi::scalar>
4615 static constexpr bool _S_is_valid_v
4616 = _Abi::template _IsValid<_Tp>::value && sizeof(_Tp) == _Bytes;
4617 };
4618
4619// }}}
4620// __decay_abi metafunction {{{
4621template <typename _Tp>
4622 struct __decay_abi { using type = _Tp; };
4623
4624template <int _Bytes>
4625 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4626 { using type = simd_abi::scalar; };
4627
4628// }}}
4629// __find_next_valid_abi metafunction {{{1
4630// Given an ABI tag A<N>, find an N2 < N such that A<N2>::_S_is_valid_v<_Tp> ==
4631// true, N2 is a power-of-2, and A<N2>::_S_is_partial<_Tp> is false. Break
4632// recursion at 2 elements in the resulting ABI tag. In this case
4633// type::_S_is_valid_v<_Tp> may be false.
4634template <template <int> class _Abi, int _Bytes, typename _Tp>
4635 struct __find_next_valid_abi
4636 {
4637 static constexpr auto
4638 _S_choose()
4639 {
4640 constexpr int _NextBytes = std::__bit_ceil((unsigned)_Bytes) / 2;
4641 using _NextAbi = _Abi<_NextBytes>;
4642 if constexpr (_NextBytes < sizeof(_Tp) * 2) // break recursion
4643 return _Abi<_Bytes>();
4644 else if constexpr (_NextAbi::template _S_is_partial<_Tp> == false
4645 && _NextAbi::template _S_is_valid_v<_Tp>)
4646 return _NextAbi();
4647 else
4648 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4649 }
4650
4651 using type = decltype(_S_choose());
4652 };
4653
4654template <int _Bytes, typename _Tp>
4655 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4656 { using type = simd_abi::scalar; };
4657
4658// _AbiList {{{1
4659template <template <int> class...>
4660 struct _AbiList
4661 {
4662 template <typename, int> static constexpr bool _S_has_valid_abi = false;
4663 template <typename, int> using _FirstValidAbi = void;
4664 template <typename, int> using _BestAbi = void;
4665 };
4666
4667template <template <int> class _A0, template <int> class... _Rest>
4668 struct _AbiList<_A0, _Rest...>
4669 {
4670 template <typename _Tp, int _Np>
4671 static constexpr bool _S_has_valid_abi
4672 = _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<
4673 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4674
4675 template <typename _Tp, int _Np>
4676 using _FirstValidAbi = conditional_t<
4677 _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4678 typename __decay_abi<_A0<sizeof(_Tp) * _Np>>::type,
4679 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4680
4681 template <typename _Tp, int _Np>
4682 static constexpr auto
4683 _S_determine_best_abi()
4684 {
4685 static_assert(_Np >= 1);
4686 constexpr int _Bytes = sizeof(_Tp) * _Np;
4687 if constexpr (_Np == 1)
4688 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4689 else
4690 {
4691 constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4692 // _A0<_Bytes> is good if:
4693 // 1. The ABI tag is valid for _Tp
4694 // 2. The storage overhead is no more than padding to fill the next
4695 // power-of-2 number of bytes
4696 if constexpr (_A0<_Bytes>::template _S_is_valid_v<_Tp>
4697 && ((__is_sve_abi<_A0<_Bytes>>() && __have_sve
4698 && (_Np <= __sve_vectorized_size_bytes/sizeof(_Tp)))
4699 || (__fullsize / 2 < _Np))
4700 )
4701 return typename __decay_abi<_A0<_Bytes>>::type{};
4702 else
4703 {
4704 using _Bp =
4705 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4706 if constexpr (_Bp::template _S_is_valid_v<
4707 _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4708 return _Bp{};
4709 else
4710 return
4711 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4712 }
4713 }
4714 }
4715
4716 template <typename _Tp, int _Np>
4717 using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4718 };
4719
4720// }}}1
4721
4722// the following lists all native ABIs, which makes them accessible to
4723// simd_abi::deduce and select_best_vector_type_t (for fixed_size). Order
4724// matters: Whatever comes first has higher priority.
4725using _AllNativeAbis = _AbiList<
4726#if _GLIBCXX_SIMD_HAVE_SVE
4727 simd_abi::_SveAbi,
4728#endif
4729 simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin, __scalar_abi_wrapper>;
4730
4731using _NoSveAllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4732 __scalar_abi_wrapper>;
4733
4734// valid _SimdTraits specialization {{{1
4735template <typename _Tp, typename _Abi>
4736 struct _SimdTraits<_Tp, _Abi, void_t<typename _Abi::template _IsValid<_Tp>>>
4737 : _Abi::template __traits<_Tp> {};
4738
4739// __deduce_impl specializations {{{1
4740// try all native ABIs (including scalar) first
4741template <typename _Tp, size_t _Np>
4742 struct __deduce_impl<
4743 _Tp, _Np, enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4744 { using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4745
4746template <typename _Tp, size_t _Np>
4747 struct __no_sve_deduce_impl<
4748 _Tp, _Np, enable_if_t<_NoSveAllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4749 { using type = _NoSveAllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4750
4751// fall back to fixed_size only if scalar and native ABIs don't match
4752template <typename _Tp, size_t _Np, typename = void>
4753 struct __deduce_fixed_size_fallback {};
4754
4755template <typename _Tp, size_t _Np>
4756 struct __deduce_fixed_size_fallback<_Tp, _Np,
4757 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4758 { using type = simd_abi::fixed_size<_Np>; };
4759
4760template <typename _Tp, size_t _Np, typename>
4761 struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _Np> {};
4762
4763template <typename _Tp, size_t _Np, typename>
4764 struct __no_sve_deduce_impl
4765 : public __deduce_fixed_size_fallback<_Tp, _Np>
4766 {};
4767
4768
4769//}}}1
4770/// @endcond
4771
4772// simd_mask {{{
4773template <typename _Tp, typename _Abi>
4774 class simd_mask : public _SimdTraits<_Tp, _Abi>::_MaskBase
4775 {
4776 // types, tags, and friends {{{
4777 using _Traits = _SimdTraits<_Tp, _Abi>;
4778 using _MemberType = typename _Traits::_MaskMember;
4779
4780 // We map all masks with equal element sizeof to a single integer type, the
4781 // one given by __int_for_sizeof_t<_Tp>. This is the approach
4782 // [[gnu::vector_size(N)]] types take as well and it reduces the number of
4783 // template specializations in the implementation classes.
4784 using _Ip = __int_for_sizeof_t<_Tp>;
4785 static constexpr _Ip* _S_type_tag = nullptr;
4786
4787 friend typename _Traits::_MaskBase;
4788 friend class simd<_Tp, _Abi>; // to construct masks on return
4789 friend typename _Traits::_SimdImpl; // to construct masks on return and
4790 // inspect data on masked operations
4791 public:
4792 using _Impl = typename _Traits::_MaskImpl;
4793 friend _Impl;
4794
4795 // }}}
4796 // member types {{{
4797 using value_type = bool;
4798 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4799 using simd_type = simd<_Tp, _Abi>;
4800 using abi_type = _Abi;
4801
4802 // }}}
4803 static constexpr size_t size() // {{{
4804 { return __size_or_zero_v<_Tp, _Abi>; }
4805
4806 // }}}
4807 // constructors & assignment {{{
4808 simd_mask() = default;
4809 simd_mask(const simd_mask&) = default;
4810 simd_mask(simd_mask&&) = default;
4811 simd_mask& operator=(const simd_mask&) = default;
4812 simd_mask& operator=(simd_mask&&) = default;
4813
4814 // }}}
4815 // access to internal representation (optional feature) {{{
4816 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR explicit
4817 simd_mask(typename _Traits::_MaskCastType __init)
4818 : _M_data{__init} {}
4819 // conversions to internal type is done in _MaskBase
4820
4821 // }}}
4822 // bitset interface (extension to be proposed) {{{
4823 // TS_FEEDBACK:
4824 // Conversion of simd_mask to and from bitset makes it much easier to
4825 // interface with other facilities. I suggest adding `static
4826 // simd_mask::from_bitset` and `simd_mask::to_bitset`.
4827 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR static simd_mask
4828 __from_bitset(bitset<size()> bs)
4829 { return {__bitset_init, bs}; }
4830
4831 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bitset<size()>
4832 __to_bitset() const
4833 { return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4834
4835 // }}}
4836 // explicit broadcast constructor {{{
4837 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4838 simd_mask(value_type __x)
4839 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4840
4841 // }}}
4842 // implicit type conversion constructor {{{
4843 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4844 // proposed improvement
4845 template <typename _Up, typename _A2,
4846 typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4847 _GLIBCXX_SIMD_ALWAYS_INLINE explicit(sizeof(_MemberType)
4848 != sizeof(typename _SimdTraits<_Up, _A2>::_MaskMember))
4849 simd_mask(const simd_mask<_Up, _A2>& __x)
4850 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4851 #else
4852 // conforming to ISO/IEC 19570:2018
4853 template <typename _Up, typename = enable_if_t<conjunction<
4854 is_same<abi_type, simd_abi::fixed_size<size()>>,
4855 is_same<_Up, _Up>>::value>>
4856 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4857 simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4858 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4859 #endif
4860
4861 // }}}
4862 // load constructor {{{
4863 template <typename _Flags>
4864 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4865 simd_mask(const value_type* __mem, _IsSimdFlagType<_Flags>)
4866 : _M_data(_Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem))) {}
4867
4868 template <typename _Flags>
4869 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4870 simd_mask(const value_type* __mem, simd_mask __k, _IsSimdFlagType<_Flags>)
4871 : _M_data{}
4872 {
4873 _M_data = _Impl::_S_masked_load(_M_data, __k._M_data,
4874 _Flags::template _S_apply<simd_mask>(__mem));
4875 }
4876
4877 // }}}
4878 // loads [simd_mask.load] {{{
4879 template <typename _Flags>
4880 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
4881 copy_from(const value_type* __mem, _IsSimdFlagType<_Flags>)
4882 { _M_data = _Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem)); }
4883
4884 // }}}
4885 // stores [simd_mask.store] {{{
4886 template <typename _Flags>
4887 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
4888 copy_to(value_type* __mem, _IsSimdFlagType<_Flags>) const
4889 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4890
4891 // }}}
4892 // scalar access {{{
4893 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
4894 operator[](size_t __i)
4895 {
4896 if (__i >= size())
4897 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4898 return {_M_data, int(__i)};
4899 }
4900
4901 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
4902 operator[](size_t __i) const
4903 {
4904 if (__i >= size())
4905 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4906 if constexpr (__is_scalar_abi<_Abi>())
4907 return _M_data;
4908 else
4909 return static_cast<bool>(_M_data[__i]);
4910 }
4911
4912 // }}}
4913 // negation {{{
4914 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd_mask
4915 operator!() const
4916 { return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4917
4918 // }}}
4919 // simd_mask binary operators [simd_mask.binary] {{{
4920 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4921 // simd_mask<int> && simd_mask<uint> needs disambiguation
4922 template <typename _Up, typename _A2,
4923 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4924 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4925 operator&&(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4926 {
4927 return {__private_init,
4928 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4929 }
4930
4931 template <typename _Up, typename _A2,
4932 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4933 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4934 operator||(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4935 {
4936 return {__private_init,
4937 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4938 }
4939 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4940
4941 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4942 operator&&(const simd_mask& __x, const simd_mask& __y)
4943 { return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)}; }
4944
4945 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4946 operator||(const simd_mask& __x, const simd_mask& __y)
4947 { return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)}; }
4948
4949 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4950 operator&(const simd_mask& __x, const simd_mask& __y)
4951 { return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4952
4953 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4954 operator|(const simd_mask& __x, const simd_mask& __y)
4955 { return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4956
4957 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4958 operator^(const simd_mask& __x, const simd_mask& __y)
4959 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4960
4961 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4962 operator&=(simd_mask& __x, const simd_mask& __y)
4963 {
4964 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4965 return __x;
4966 }
4967
4968 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4969 operator|=(simd_mask& __x, const simd_mask& __y)
4970 {
4971 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4972 return __x;
4973 }
4974
4975 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4976 operator^=(simd_mask& __x, const simd_mask& __y)
4977 {
4978 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4979 return __x;
4980 }
4981
4982 // }}}
4983 // simd_mask compares [simd_mask.comparison] {{{
4984 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4985 operator==(const simd_mask& __x, const simd_mask& __y)
4986 { return !operator!=(__x, __y); }
4987
4988 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4989 operator!=(const simd_mask& __x, const simd_mask& __y)
4990 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4991
4992 // }}}
4993 // private_init ctor {{{
4994 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4995 simd_mask(_PrivateInit, typename _Traits::_MaskMember __init)
4996 : _M_data(__init) {}
4997
4998 // }}}
4999 // private_init generator ctor {{{
5000 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
5001 _GLIBCXX_SIMD_INTRINSIC constexpr
5002 simd_mask(_PrivateInit, _Fp&& __gen)
5003 : _M_data()
5004 {
5005 __execute_n_times<size()>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5006 _Impl::_S_set(_M_data, __i, __gen(__i));
5007 });
5008 }
5009
5010 // }}}
5011 // bitset_init ctor {{{
5012 _GLIBCXX_SIMD_INTRINSIC constexpr
5013 simd_mask(_BitsetInit, bitset<size()> __init)
5014 : _M_data(_Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
5015 {}
5016
5017 // }}}
5018 // __cvt {{{
5019 // TS_FEEDBACK:
5020 // The conversion operator this implements should be a ctor on simd_mask.
5021 // Once you call .__cvt() on a simd_mask it converts conveniently.
5022 // A useful variation: add `explicit(sizeof(_Tp) != sizeof(_Up))`
5023 struct _CvtProxy
5024 {
5025 template <typename _Up, typename _A2,
5026 typename = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
5027 _GLIBCXX_SIMD_ALWAYS_INLINE
5028 operator simd_mask<_Up, _A2>() &&
5029 {
5030 using namespace std::experimental::__proposed;
5031 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
5032 }
5033
5034 const simd_mask<_Tp, _Abi>& _M_data;
5035 };
5036
5037 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
5038 __cvt() const
5039 { return {*this}; }
5040
5041 // }}}
5042 // operator?: overloads (suggested extension) {{{
5043 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5044 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
5045 operator?:(const simd_mask& __k, const simd_mask& __where_true,
5046 const simd_mask& __where_false)
5047 {
5048 auto __ret = __where_false;
5049 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
5050 return __ret;
5051 }
5052
5053 template <typename _U1, typename _U2,
5054 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
5055 typename = enable_if_t<conjunction_v<
5056 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
5057 is_convertible<simd_mask, typename _Rp::mask_type>>>>
5058 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend _Rp
5059 operator?:(const simd_mask& __k, const _U1& __where_true,
5060 const _U2& __where_false)
5061 {
5062 _Rp __ret = __where_false;
5063 _Rp::_Impl::_S_masked_assign(
5064 __data(static_cast<typename _Rp::mask_type>(__k)), __data(__ret),
5065 __data(static_cast<_Rp>(__where_true)));
5066 return __ret;
5067 }
5068
5069 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
5070 template <typename _Kp, typename _Ak, typename _Up, typename _Au,
5071 typename = enable_if_t<
5072 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
5073 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
5074 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
5075 operator?:(const simd_mask<_Kp, _Ak>& __k, const simd_mask& __where_true,
5076 const simd_mask<_Up, _Au>& __where_false)
5077 {
5078 simd_mask __ret = __where_false;
5079 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
5080 __where_true._M_data);
5081 return __ret;
5082 }
5083 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
5084 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5085
5086 // }}}
5087 // _M_is_constprop {{{
5088 _GLIBCXX_SIMD_INTRINSIC constexpr bool
5089 _M_is_constprop() const
5090 {
5091 if constexpr (__is_scalar_abi<_Abi>())
5092 return __builtin_constant_p(_M_data);
5093 else
5094 return _M_data._M_is_constprop();
5095 }
5096
5097 // }}}
5098
5099 private:
5100 friend const auto& __data<_Tp, abi_type>(const simd_mask&);
5101 friend auto& __data<_Tp, abi_type>(simd_mask&);
5102 alignas(_Traits::_S_mask_align) _MemberType _M_data;
5103 };
5104
5105// }}}
5106
5107/// @cond undocumented
5108// __data(simd_mask) {{{
5109template <typename _Tp, typename _Ap>
5110 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5111 __data(const simd_mask<_Tp, _Ap>& __x)
5112 { return __x._M_data; }
5113
5114template <typename _Tp, typename _Ap>
5115 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5116 __data(simd_mask<_Tp, _Ap>& __x)
5117 { return __x._M_data; }
5118
5119// }}}
5120/// @endcond
5121
5122// simd_mask reductions [simd_mask.reductions] {{{
5123template <typename _Tp, typename _Abi>
5124 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5125 all_of(const simd_mask<_Tp, _Abi>& __k) noexcept
5126 {
5127 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5128 {
5129 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5130 if (!__k[__i])
5131 return false;
5132 return true;
5133 }
5134 else
5135 return _Abi::_MaskImpl::_S_all_of(__k);
5136 }
5137
5138template <typename _Tp, typename _Abi>
5139 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5140 any_of(const simd_mask<_Tp, _Abi>& __k) noexcept
5141 {
5142 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5143 {
5144 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5145 if (__k[__i])
5146 return true;
5147 return false;
5148 }
5149 else
5150 return _Abi::_MaskImpl::_S_any_of(__k);
5151 }
5152
5153template <typename _Tp, typename _Abi>
5154 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5155 none_of(const simd_mask<_Tp, _Abi>& __k) noexcept
5156 {
5157 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5158 {
5159 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5160 if (__k[__i])
5161 return false;
5162 return true;
5163 }
5164 else
5165 return _Abi::_MaskImpl::_S_none_of(__k);
5166 }
5167
5168template <typename _Tp, typename _Abi>
5169 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5170 some_of(const simd_mask<_Tp, _Abi>& __k) noexcept
5171 {
5172 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5173 {
5174 for (size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
5175 if (__k[__i] != __k[__i - 1])
5176 return true;
5177 return false;
5178 }
5179 else
5180 return _Abi::_MaskImpl::_S_some_of(__k);
5181 }
5182
5183template <typename _Tp, typename _Abi>
5184 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5185 popcount(const simd_mask<_Tp, _Abi>& __k) noexcept
5186 {
5187 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5188 {
5189 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
5190 __k, [](auto... __elements) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5191 return ((__elements != 0) + ...);
5192 });
5193 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
5194 return __r;
5195 }
5196 return _Abi::_MaskImpl::_S_popcount(__k);
5197 }
5198
5199template <typename _Tp, typename _Abi>
5200 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5201 find_first_set(const simd_mask<_Tp, _Abi>& __k)
5202 {
5203 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5204 {
5205 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5206 const size_t _Idx = __call_with_n_evaluations<_Np>(
5207 [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5208 return std::min({__indexes...});
5209 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5210 return __k[__i] ? +__i : _Np;
5211 });
5212 if (_Idx >= _Np)
5213 __invoke_ub("find_first_set(empty mask) is UB");
5214 if (__builtin_constant_p(_Idx))
5215 return _Idx;
5216 }
5217 return _Abi::_MaskImpl::_S_find_first_set(__k);
5218 }
5219
5220template <typename _Tp, typename _Abi>
5221 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5222 find_last_set(const simd_mask<_Tp, _Abi>& __k)
5223 {
5224 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5225 {
5226 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5227 const int _Idx = __call_with_n_evaluations<_Np>(
5228 [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5229 return std::max({__indexes...});
5230 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5231 return __k[__i] ? int(__i) : -1;
5232 });
5233 if (_Idx < 0)
5234 __invoke_ub("find_first_set(empty mask) is UB");
5235 if (__builtin_constant_p(_Idx))
5236 return _Idx;
5237 }
5238 return _Abi::_MaskImpl::_S_find_last_set(__k);
5239 }
5240
5241_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5242all_of(_ExactBool __x) noexcept
5243{ return __x; }
5244
5245_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5246any_of(_ExactBool __x) noexcept
5247{ return __x; }
5248
5249_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5250none_of(_ExactBool __x) noexcept
5251{ return !__x; }
5252
5253_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5254some_of(_ExactBool) noexcept
5255{ return false; }
5256
5257_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5258popcount(_ExactBool __x) noexcept
5259{ return __x; }
5260
5261_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5262find_first_set(_ExactBool)
5263{ return 0; }
5264
5265_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5266find_last_set(_ExactBool)
5267{ return 0; }
5268
5269// }}}
5270
5271/// @cond undocumented
5272// _SimdIntOperators{{{1
5273template <typename _V, typename _Tp, typename _Abi, bool>
5274 class _SimdIntOperators {};
5275
5276template <typename _V, typename _Tp, typename _Abi>
5277 class _SimdIntOperators<_V, _Tp, _Abi, true>
5278 {
5279 using _Impl = typename _SimdTraits<_Tp, _Abi>::_SimdImpl;
5280
5281 _GLIBCXX_SIMD_INTRINSIC constexpr const _V&
5282 __derived() const
5283 { return *static_cast<const _V*>(this); }
5284
5285 template <typename _Up>
5286 _GLIBCXX_SIMD_INTRINSIC static _GLIBCXX_SIMD_CONSTEXPR _V
5287 _S_make_derived(_Up&& __d)
5288 { return {__private_init, static_cast<_Up&&>(__d)}; }
5289
5290 public:
5291 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5292 _V&
5293 operator%=(_V& __lhs, const _V& __x)
5294 { return __lhs = __lhs % __x; }
5295
5296 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5297 _V&
5298 operator&=(_V& __lhs, const _V& __x)
5299 { return __lhs = __lhs & __x; }
5300
5301 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5302 _V&
5303 operator|=(_V& __lhs, const _V& __x)
5304 { return __lhs = __lhs | __x; }
5305
5306 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5307 _V&
5308 operator^=(_V& __lhs, const _V& __x)
5309 { return __lhs = __lhs ^ __x; }
5310
5311 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5312 _V&
5313 operator<<=(_V& __lhs, const _V& __x)
5314 { return __lhs = __lhs << __x; }
5315
5316 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5317 _V&
5318 operator>>=(_V& __lhs, const _V& __x)
5319 { return __lhs = __lhs >> __x; }
5320
5321 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5322 _V&
5323 operator<<=(_V& __lhs, int __x)
5324 { return __lhs = __lhs << __x; }
5325
5326 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5327 _V&
5328 operator>>=(_V& __lhs, int __x)
5329 { return __lhs = __lhs >> __x; }
5330
5331 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5332 _V
5333 operator%(const _V& __x, const _V& __y)
5334 {
5335 return _SimdIntOperators::_S_make_derived(
5336 _Impl::_S_modulus(__data(__x), __data(__y)));
5337 }
5338
5339 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5340 _V
5341 operator&(const _V& __x, const _V& __y)
5342 {
5343 return _SimdIntOperators::_S_make_derived(
5344 _Impl::_S_bit_and(__data(__x), __data(__y)));
5345 }
5346
5347 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5348 _V
5349 operator|(const _V& __x, const _V& __y)
5350 {
5351 return _SimdIntOperators::_S_make_derived(
5352 _Impl::_S_bit_or(__data(__x), __data(__y)));
5353 }
5354
5355 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5356 _V
5357 operator^(const _V& __x, const _V& __y)
5358 {
5359 return _SimdIntOperators::_S_make_derived(
5360 _Impl::_S_bit_xor(__data(__x), __data(__y)));
5361 }
5362
5363 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5364 _V
5365 operator<<(const _V& __x, const _V& __y)
5366 {
5367 return _SimdIntOperators::_S_make_derived(
5368 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
5369 }
5370
5371 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5372 _V
5373 operator>>(const _V& __x, const _V& __y)
5374 {
5375 return _SimdIntOperators::_S_make_derived(
5376 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
5377 }
5378
5379 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5380 _V
5381 operator<<(const _V& __x, int __y)
5382 {
5383 if (__y < 0)
5384 __invoke_ub("The behavior is undefined if the right operand of a "
5385 "shift operation is negative. [expr.shift]\nA shift by "
5386 "%d was requested",
5387 __y);
5388 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5389 __invoke_ub(
5390 "The behavior is undefined if the right operand of a "
5391 "shift operation is greater than or equal to the width of the "
5392 "promoted left operand. [expr.shift]\nA shift by %d was requested",
5393 __y);
5394 return _SimdIntOperators::_S_make_derived(
5395 _Impl::_S_bit_shift_left(__data(__x), __y));
5396 }
5397
5398 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5399 _V
5400 operator>>(const _V& __x, int __y)
5401 {
5402 if (__y < 0)
5403 __invoke_ub(
5404 "The behavior is undefined if the right operand of a shift "
5405 "operation is negative. [expr.shift]\nA shift by %d was requested",
5406 __y);
5407 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5408 __invoke_ub(
5409 "The behavior is undefined if the right operand of a shift "
5410 "operation is greater than or equal to the width of the promoted "
5411 "left operand. [expr.shift]\nA shift by %d was requested",
5412 __y);
5413 return _SimdIntOperators::_S_make_derived(
5414 _Impl::_S_bit_shift_right(__data(__x), __y));
5415 }
5416
5417 // unary operators (for integral _Tp)
5418 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5419 _V
5420 operator~() const
5421 { return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
5422 };
5423
5424//}}}1
5425/// @endcond
5426
5427// simd {{{
5428template <typename _Tp, typename _Abi>
5429 class simd : public _SimdIntOperators<
5430 simd<_Tp, _Abi>, _Tp, _Abi,
5431 conjunction<is_integral<_Tp>,
5432 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
5433 public _SimdTraits<_Tp, _Abi>::_SimdBase
5434 {
5435 using _Traits = _SimdTraits<_Tp, _Abi>;
5436 using _MemberType = typename _Traits::_SimdMember;
5437 using _CastType = typename _Traits::_SimdCastType;
5438 static constexpr _Tp* _S_type_tag = nullptr;
5439 friend typename _Traits::_SimdBase;
5440
5441 public:
5442 using _Impl = typename _Traits::_SimdImpl;
5443 friend _Impl;
5444 friend _SimdIntOperators<simd, _Tp, _Abi, true>;
5445
5446 using value_type = _Tp;
5447 using reference = _SmartReference<_MemberType, _Impl, value_type>;
5448 using mask_type = simd_mask<_Tp, _Abi>;
5449 using abi_type = _Abi;
5450
5451 static constexpr size_t size()
5452 { return __size_or_zero_v<_Tp, _Abi>; }
5453
5454 _GLIBCXX_SIMD_CONSTEXPR simd() = default;
5455 _GLIBCXX_SIMD_CONSTEXPR simd(const simd&) = default;
5456 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
5457 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
5458 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
5459
5460 // implicit broadcast constructor
5461 template <typename _Up,
5462 typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>, bool>>>
5463 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5464 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
5465 : _M_data(
5466 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
5467 {}
5468
5469 // implicit type conversion constructor (convert from fixed_size to
5470 // fixed_size)
5471 template <typename _Up>
5472 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5473 simd(const simd<_Up, simd_abi::fixed_size<size()>>& __x,
5474 enable_if_t<
5475 conjunction<
5476 is_same<simd_abi::fixed_size<size()>, abi_type>,
5477 negation<__is_narrowing_conversion<_Up, value_type>>,
5478 __converts_to_higher_integer_rank<_Up, value_type>>::value,
5479 void*> = nullptr)
5480 : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5481
5482 // explicit type conversion constructor
5483#ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5484 template <typename _Up, typename _A2,
5485 typename = decltype(static_simd_cast<simd>(
5486 declval<const simd<_Up, _A2>&>()))>
5487 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5488 simd(const simd<_Up, _A2>& __x)
5489 : simd(static_simd_cast<simd>(__x)) {}
5490#endif // _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5491
5492 // generator constructor
5493 template <typename _Fp>
5494 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5495 simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
5496 declval<_SizeConstant<0>&>())),
5497 value_type>* = nullptr)
5498 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5499
5500 // load constructor
5501 template <typename _Up, typename _Flags>
5502 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5503 simd(const _Up* __mem, _IsSimdFlagType<_Flags>)
5504 : _M_data(
5505 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5506 {}
5507
5508 // loads [simd.load]
5509 template <typename _Up, typename _Flags>
5510 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
5511 copy_from(const _Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>)
5512 {
5513 _M_data = static_cast<decltype(_M_data)>(
5514 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5515 }
5516
5517 // stores [simd.store]
5518 template <typename _Up, typename _Flags>
5519 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
5520 copy_to(_Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>) const
5521 {
5522 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5523 _S_type_tag);
5524 }
5525
5526 // scalar access
5527 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5528 operator[](size_t __i)
5529 { return {_M_data, int(__i)}; }
5530
5531 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5532 operator[]([[maybe_unused]] size_t __i) const
5533 {
5534 if constexpr (__is_scalar_abi<_Abi>())
5535 {
5536 _GLIBCXX_DEBUG_ASSERT(__i == 0);
5537 return _M_data;
5538 }
5539 else
5540 return _M_data[__i];
5541 }
5542
5543 // increment and decrement:
5544 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5545 operator++()
5546 {
5547 _Impl::_S_increment(_M_data);
5548 return *this;
5549 }
5550
5551 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5552 operator++(int)
5553 {
5554 simd __r = *this;
5555 _Impl::_S_increment(_M_data);
5556 return __r;
5557 }
5558
5559 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5560 operator--()
5561 {
5562 _Impl::_S_decrement(_M_data);
5563 return *this;
5564 }
5565
5566 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5567 operator--(int)
5568 {
5569 simd __r = *this;
5570 _Impl::_S_decrement(_M_data);
5571 return __r;
5572 }
5573
5574 // unary operators (for any _Tp)
5575 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5576 operator!() const
5577 { return {__private_init, _Impl::_S_negate(_M_data)}; }
5578
5579 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5580 operator+() const
5581 { return *this; }
5582
5583 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5584 operator-() const
5585 { return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5586
5587 // access to internal representation (suggested extension)
5588 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5589 simd(_CastType __init) : _M_data(__init) {}
5590
5591 // compound assignment [simd.cassign]
5592 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5593 operator+=(simd& __lhs, const simd& __x)
5594 { return __lhs = __lhs + __x; }
5595
5596 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5597 operator-=(simd& __lhs, const simd& __x)
5598 { return __lhs = __lhs - __x; }
5599
5600 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5601 operator*=(simd& __lhs, const simd& __x)
5602 { return __lhs = __lhs * __x; }
5603
5604 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5605 operator/=(simd& __lhs, const simd& __x)
5606 { return __lhs = __lhs / __x; }
5607
5608 // binary operators [simd.binary]
5609 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5610 operator+(const simd& __x, const simd& __y)
5611 { return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5612
5613 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5614 operator-(const simd& __x, const simd& __y)
5615 { return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5616
5617 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5618 operator*(const simd& __x, const simd& __y)
5619 { return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5620
5621 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5622 operator/(const simd& __x, const simd& __y)
5623 { return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5624
5625 // compares [simd.comparison]
5626 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5627 operator==(const simd& __x, const simd& __y)
5628 { return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5629
5630 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5631 operator!=(const simd& __x, const simd& __y)
5632 {
5633 return simd::_S_make_mask(
5634 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5635 }
5636
5637 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5638 operator<(const simd& __x, const simd& __y)
5639 { return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5640
5641 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5642 operator<=(const simd& __x, const simd& __y)
5643 {
5644 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5645 }
5646
5647 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5648 operator>(const simd& __x, const simd& __y)
5649 { return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5650
5651 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5652 operator>=(const simd& __x, const simd& __y)
5653 {
5654 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5655 }
5656
5657 // operator?: overloads (suggested extension) {{{
5658#ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5659 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5660 operator?:(const mask_type& __k, const simd& __where_true,
5661 const simd& __where_false)
5662 {
5663 auto __ret = __where_false;
5664 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5665 return __ret;
5666 }
5667
5668#endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5669 // }}}
5670
5671 // "private" because of the first arguments's namespace
5672 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5673 simd(_PrivateInit, const _MemberType& __init)
5674 : _M_data(__init) {}
5675
5676 // "private" because of the first arguments's namespace
5677 _GLIBCXX_SIMD_INTRINSIC
5678 simd(_BitsetInit, bitset<size()> __init) : _M_data()
5679 { where(mask_type(__bitset_init, __init), *this) = ~*this; }
5680
5681 _GLIBCXX_SIMD_INTRINSIC constexpr bool
5682 _M_is_constprop() const
5683 {
5684 if constexpr (__is_scalar_abi<_Abi>())
5685 return __builtin_constant_p(_M_data);
5686 else
5687 return _M_data._M_is_constprop();
5688 }
5689
5690 private:
5691 _GLIBCXX_SIMD_INTRINSIC static constexpr mask_type
5692 _S_make_mask(typename mask_type::_MemberType __k)
5693 { return {__private_init, __k}; }
5694
5695 friend const auto& __data<value_type, abi_type>(const simd&);
5696 friend auto& __data<value_type, abi_type>(simd&);
5697 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5698 };
5699
5700// }}}
5701/// @cond undocumented
5702// __data {{{
5703template <typename _Tp, typename _Ap>
5704 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5705 __data(const simd<_Tp, _Ap>& __x)
5706 { return __x._M_data; }
5707
5708template <typename _Tp, typename _Ap>
5709 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5710 __data(simd<_Tp, _Ap>& __x)
5711 { return __x._M_data; }
5712
5713// }}}
5714namespace __float_bitwise_operators { //{{{
5715template <typename _Tp, typename _Ap>
5716 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5717 operator^(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5718 { return {__private_init, _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))}; }
5719
5720template <typename _Tp, typename _Ap>
5721 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5722 operator|(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5723 { return {__private_init, _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))}; }
5724
5725template <typename _Tp, typename _Ap>
5726 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5727 operator&(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5728 { return {__private_init, _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))}; }
5729
5730template <typename _Tp, typename _Ap>
5731 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5732 enable_if_t<is_floating_point_v<_Tp>, simd<_Tp, _Ap>>
5733 operator~(const simd<_Tp, _Ap>& __a)
5734 { return {__private_init, _Ap::_SimdImpl::_S_complement(__data(__a))}; }
5735} // namespace __float_bitwise_operators }}}
5736/// @endcond
5737
5738/// @}
5739_GLIBCXX_SIMD_END_NAMESPACE
5740
5741#endif // __cplusplus >= 201703L
5742#endif // _GLIBCXX_EXPERIMENTAL_SIMD_H
5743
5744// vim: foldmethod=marker foldmarker={{{,}}}
algorithm
bit
bitset
functional
iosfwd
utility
cmath
cstdio
cstring
std::chrono::operator%
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator%(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition chrono.h:787
std::conditional_t
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition type_traits:2948
std::remove_reference_t
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
Definition type_traits:1890
std::make_unsigned_t
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition type_traits:2250
std::enable_if_t
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
Definition type_traits:2944
std::false_type
__bool_constant< false > false_type
The type used as a compile-time boolean with false value.
Definition type_traits:122
std::tuple_cat
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
Create a tuple containing all elements from multiple tuple-like objects.
Definition tuple:2859
std::declval
auto declval() noexcept -> decltype(__declval< _Tp >(0))
Definition type_traits:2718
std::make_tuple
constexpr tuple< typename __decay_and_strip< _Elements >::__type... > make_tuple(_Elements &&... __args)
Create a tuple containing copies of the arguments.
Definition tuple:2723
std::move
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
Definition move.h:138
std::max
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
Definition stl_algobase.h:256
std::min
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition stl_algobase.h:232
std::experimental::fundamentals_v2::void_t
void void_t
A metafunction that always yields void, used for detecting valid types.
Definition experimental/type_traits:262
std
ISO C++ entities toplevel namespace is std.
std::make_index_sequence
make_integer_sequence< size_t, _Num > make_index_sequence
Alias template make_index_sequence.
Definition utility.h:559
std::operator^
constexpr bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1638
std::operator>>
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition bitset:1658
std::operator<<
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition bitset:1754
std::operator|
constexpr bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1628
std::operator&
constexpr bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1618
simd_abi::deduce
Definition simd.h:3110