libstdc++: simd.h Source File

libstdc++

simd.h
1// Definition of the public simd interfaces -*- C++ -*-
2
3// Copyright (C) 2020-2026 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26#define _GLIBCXX_EXPERIMENTAL_SIMD_H
27
28#if __cplusplus >= 201703L
29
30#include "simd_detail.h"
31#include "numeric_traits.h"
32#include <bit>
33#include <bitset>
34#ifdef _GLIBCXX_DEBUG_UB
35#include <cstdio> // for stderr
36#endif
37#include <cstring>
38#include <cmath>
39#include <functional>
40#include <iosfwd>
41#include <utility>
42#include <algorithm>
43
44#if _GLIBCXX_SIMD_X86INTRIN
45#include <x86intrin.h>
46#elif _GLIBCXX_SIMD_HAVE_NEON
47#pragma GCC diagnostic push
48// narrowing conversion of '__a' from 'uint64_t' {aka 'long long unsigned int'} to
49// 'int64x1_t' {aka 'long long int'} [-Wnarrowing]
50#pragma GCC diagnostic ignored "-Wnarrowing"
51#include <arm_neon.h>
52#pragma GCC diagnostic pop
53#endif
54#if _GLIBCXX_SIMD_HAVE_SVE
55#include <arm_sve.h>
56#endif
57
58/** @namespace std::experimental::parallelism_v2
59 * @ingroup ts_simd
60 */
61_GLIBCXX_SIMD_BEGIN_NAMESPACE
62
63/** @ingroup ts_simd
64 * @{
65 */
66/* There are several closely related types, with the following naming
67 * convention:
68 * _Tp: vectorizable (arithmetic) type (or any type)
69 * _TV: __vector_type_t<_Tp, _Np>
70 * _TW: _SimdWrapper<_Tp, _Np>
71 * _TI: __intrinsic_type_t<_Tp, _Np>
72 * _TVT: _VectorTraits<_TV> or _VectorTraits<_TW>
73 * If one additional type is needed use _U instead of _T.
74 * Otherwise use _T\d, _TV\d, _TW\d, TI\d, _TVT\d.
75 *
76 * More naming conventions:
77 * _Ap or _Abi: An ABI tag from the simd_abi namespace
78 * _Ip: often used for integer types with sizeof(_Ip) == sizeof(_Tp),
79 * _IV, _IW as for _TV, _TW
80 * _Np: number of elements (not bytes)
81 * _Bytes: number of bytes
82 *
83 * Variable names:
84 * __k: mask object (vector- or bitmask)
85 */
86
87#if !_GLIBCXX_SIMD_X86INTRIN
88using __m128 [[__gnu__::__vector_size__(16)]] = float;
89using __m128d [[__gnu__::__vector_size__(16)]] = double;
90using __m128i [[__gnu__::__vector_size__(16)]] = long long;
91using __m256 [[__gnu__::__vector_size__(32)]] = float;
92using __m256d [[__gnu__::__vector_size__(32)]] = double;
93using __m256i [[__gnu__::__vector_size__(32)]] = long long;
94using __m512 [[__gnu__::__vector_size__(64)]] = float;
95using __m512d [[__gnu__::__vector_size__(64)]] = double;
96using __m512i [[__gnu__::__vector_size__(64)]] = long long;
97#endif
98
99#if _GLIBCXX_SIMD_HAVE_SVE
100constexpr inline int __sve_vectorized_size_bytes = __ARM_FEATURE_SVE_BITS / 8;
101#else
102constexpr inline int __sve_vectorized_size_bytes = 0;
103#endif
104
105namespace simd_abi {
106// simd_abi forward declarations {{{
107// implementation details:
108struct _Scalar;
109
110template <int _Np>
111 struct _Fixed;
112
113// There are two major ABIs that appear on different architectures.
114// Both have non-boolean values packed into an N Byte register
115// -> #elements = N / sizeof(T)
116// Masks differ:
117// 1. Use value vector registers for masks (all 0 or all 1)
118// 2. Use bitmasks (mask registers) with one bit per value in the corresponding
119// value vector
120//
121// Both can be partially used, masking off the rest when doing horizontal
122// operations or operations that can trap (e.g. FP_INVALID or integer division
123// by 0). This is encoded as the number of used bytes.
124template <int _UsedBytes>
125 struct _VecBuiltin;
126
127template <int _UsedBytes>
128 struct _VecBltnBtmsk;
129
130template <int _UsedBytes, int _TotalBytes = __sve_vectorized_size_bytes>
131 struct _SveAbi;
132
133template <typename _Tp, int _Np>
134 using _VecN = _VecBuiltin<sizeof(_Tp) * _Np>;
135
136template <int _UsedBytes = 16>
137 using _Sse = _VecBuiltin<_UsedBytes>;
138
139template <int _UsedBytes = 32>
140 using _Avx = _VecBuiltin<_UsedBytes>;
141
142template <int _UsedBytes = 64>
143 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
144
145template <int _UsedBytes = 16>
146 using _Neon = _VecBuiltin<_UsedBytes>;
147
148template <int _UsedBytes = __sve_vectorized_size_bytes>
149 using _Sve = _SveAbi<_UsedBytes, __sve_vectorized_size_bytes>;
150
151// implementation-defined:
152using __sse = _Sse<>;
153using __avx = _Avx<>;
154using __avx512 = _Avx512<>;
155using __neon = _Neon<>;
156using __neon128 = _Neon<16>;
157using __neon64 = _Neon<8>;
158using __sve = _Sve<>;
159
160// standard:
161template <typename _Tp, size_t _Np, typename...>
162 struct deduce;
163
164template <int _Np>
165 using fixed_size = _Fixed<_Np>;
166
167using scalar = _Scalar;
168
169// }}}
170} // namespace simd_abi
171// forward declarations is_simd(_mask), simd(_mask), simd_size {{{
172template <typename _Tp>
173 struct is_simd;
174
175template <typename _Tp>
176 struct is_simd_mask;
177
178template <typename _Tp, typename _Abi>
179 class simd;
180
181template <typename _Tp, typename _Abi>
182 class simd_mask;
183
184template <typename _Tp, typename _Abi>
185 struct simd_size;
186
187// }}}
188// load/store flags {{{
189struct element_aligned_tag
190{
191 template <typename _Tp, typename _Up = typename _Tp::value_type>
192 static constexpr size_t _S_alignment = alignof(_Up);
193
194 template <typename _Tp, typename _Up>
195 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
196 _S_apply(_Up* __ptr)
197 { return __ptr; }
198};
199
200struct vector_aligned_tag
201{
202 template <typename _Tp, typename _Up = typename _Tp::value_type>
203 static constexpr size_t _S_alignment
204 = std::__bit_ceil(sizeof(_Up) * _Tp::size());
205
206 template <typename _Tp, typename _Up>
207 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
208 _S_apply(_Up* __ptr)
209 { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>)); }
210};
211
212template <size_t _Np> struct overaligned_tag
213{
214 template <typename _Tp, typename _Up = typename _Tp::value_type>
215 static constexpr size_t _S_alignment = _Np;
216
217 template <typename _Tp, typename _Up>
218 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
219 _S_apply(_Up* __ptr)
220 { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); }
221};
222
223inline constexpr element_aligned_tag element_aligned = {};
224
225inline constexpr vector_aligned_tag vector_aligned = {};
226
227template <size_t _Np>
228 inline constexpr overaligned_tag<_Np> overaligned = {};
229
230// }}}
231template <size_t _Xp>
232 using _SizeConstant = integral_constant<size_t, _Xp>;
233// constexpr feature detection{{{
234constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
235constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
236constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
237constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
238constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
239constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
240constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
241constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
242constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
243constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
244constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
245constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
246constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
247constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
248constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
249constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
250constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
251constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
252constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
253constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
254constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
255constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
256constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
257constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
258constexpr inline bool __have_avx512bitalg = _GLIBCXX_SIMD_HAVE_AVX512BITALG;
259constexpr inline bool __have_avx512vbmi2 = _GLIBCXX_SIMD_HAVE_AVX512VBMI2;
260constexpr inline bool __have_avx512vbmi = _GLIBCXX_SIMD_HAVE_AVX512VBMI;
261constexpr inline bool __have_avx512ifma = _GLIBCXX_SIMD_HAVE_AVX512IFMA;
262constexpr inline bool __have_avx512cd = _GLIBCXX_SIMD_HAVE_AVX512CD;
263constexpr inline bool __have_avx512vnni = _GLIBCXX_SIMD_HAVE_AVX512VNNI;
264constexpr inline bool __have_avx512vpopcntdq = _GLIBCXX_SIMD_HAVE_AVX512VPOPCNTDQ;
265constexpr inline bool __have_avx512vp2intersect = _GLIBCXX_SIMD_HAVE_AVX512VP2INTERSECT;
266
267constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
268constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
269constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
270constexpr inline bool __support_neon_float =
271#if defined __GCC_IEC_559
272 __GCC_IEC_559 == 0;
273#elif defined __FAST_MATH__
274 true;
275#else
276 false;
277#endif
278
279constexpr inline bool __have_sve = _GLIBCXX_SIMD_HAVE_SVE;
280constexpr inline bool __have_sve2 = _GLIBCXX_SIMD_HAVE_SVE2;
281
282#ifdef _ARCH_PWR10
283constexpr inline bool __have_power10vec = true;
284#else
285constexpr inline bool __have_power10vec = false;
286#endif
287#ifdef __POWER9_VECTOR__
288constexpr inline bool __have_power9vec = true;
289#else
290constexpr inline bool __have_power9vec = false;
291#endif
292#if defined __POWER8_VECTOR__
293constexpr inline bool __have_power8vec = true;
294#else
295constexpr inline bool __have_power8vec = __have_power9vec;
296#endif
297#if defined __VSX__
298constexpr inline bool __have_power_vsx = true;
299#else
300constexpr inline bool __have_power_vsx = __have_power8vec;
301#endif
302#if defined __ALTIVEC__
303constexpr inline bool __have_power_vmx = true;
304#else
305constexpr inline bool __have_power_vmx = __have_power_vsx;
306#endif
307
308// }}}
309
310namespace __detail
311{
312#ifdef math_errhandling
313 // Determines _S_handle_fpexcept from math_errhandling if it is defined and expands to a constant
314 // expression. math_errhandling may expand to an extern symbol, in which case a constexpr value
315 // must be guessed.
316 template <int = math_errhandling>
317 constexpr bool
318 __handle_fpexcept_impl(int)
319 { return math_errhandling & MATH_ERREXCEPT; }
320#endif
321
322 // Fallback if math_errhandling doesn't work: with fast-math assume floating-point exceptions are
323 // ignored, otherwise implement correct exception behavior.
324 constexpr bool
325 __handle_fpexcept_impl(float)
326 {
327#if defined __FAST_MATH__
328 return false;
329#else
330 return true;
331#endif
332 }
333
334 /// True if math functions must raise floating-point exceptions as specified by C17.
335 static constexpr bool _S_handle_fpexcept = __handle_fpexcept_impl(0);
336
337 constexpr std::uint_least64_t
338 __floating_point_flags()
339 {
340 std::uint_least64_t __flags = 0;
341 if constexpr (_S_handle_fpexcept)
342 __flags |= 1;
343#ifdef __FAST_MATH__
344 __flags |= 1 << 1;
345#elif __FINITE_MATH_ONLY__
346 __flags |= 2 << 1;
347#elif __GCC_IEC_559 < 2
348 __flags |= 3 << 1;
349#endif
350 __flags |= (__FLT_EVAL_METHOD__ + 1) << 3;
351 return __flags;
352 }
353
354 constexpr std::uint_least64_t
355 __machine_flags()
356 {
357 if constexpr (__have_mmx || __have_sse)
358 return __have_mmx
359 | (__have_sse << 1)
360 | (__have_sse2 << 2)
361 | (__have_sse3 << 3)
362 | (__have_ssse3 << 4)
363 | (__have_sse4_1 << 5)
364 | (__have_sse4_2 << 6)
365 | (__have_xop << 7)
366 | (__have_avx << 8)
367 | (__have_avx2 << 9)
368 | (__have_bmi << 10)
369 | (__have_bmi2 << 11)
370 | (__have_lzcnt << 12)
371 | (__have_sse4a << 13)
372 | (__have_fma << 14)
373 | (__have_fma4 << 15)
374 | (__have_f16c << 16)
375 | (__have_popcnt << 17)
376 | (__have_avx512f << 18)
377 | (__have_avx512dq << 19)
378 | (__have_avx512vl << 20)
379 | (__have_avx512bw << 21)
380 | (__have_avx512bitalg << 22)
381 | (__have_avx512vbmi2 << 23)
382 | (__have_avx512vbmi << 24)
383 | (__have_avx512ifma << 25)
384 | (__have_avx512cd << 26)
385 | (__have_avx512vnni << 27)
386 | (__have_avx512vpopcntdq << 28)
387 | (__have_avx512vp2intersect << 29);
388 else if constexpr (__have_neon || __have_sve)
389 return __have_neon
390 | (__have_neon_a32 << 1)
391 | (__have_neon_a64 << 2)
392 | (__have_neon_a64 << 2)
393 | (__support_neon_float << 3)
394 | (__have_sve << 4)
395 | (__have_sve2 << 5);
396 else if constexpr (__have_power_vmx)
397 return __have_power_vmx
398 | (__have_power_vsx << 1)
399 | (__have_power8vec << 2)
400 | (__have_power9vec << 3)
401 | (__have_power10vec << 4);
402 else
403 return 0;
404 }
405
406 namespace
407 {
408 struct _OdrEnforcer {};
409 }
410
411 template <std::uint_least64_t...>
412 struct _MachineFlagsTemplate {};
413
414 /**@internal
415 * Use this type as default template argument to all function templates that
416 * are not declared always_inline. It ensures, that a function
417 * specialization, which the compiler decides not to inline, has a unique symbol
418 * (_OdrEnforcer) or a symbol matching the machine/architecture flags
419 * (_MachineFlagsTemplate). This helps to avoid ODR violations in cases where
420 * users link TUs compiled with different flags. This is especially important
421 * for using simd in libraries.
422 */
423 using __odr_helper
424 = conditional_t<__machine_flags() == 0, _OdrEnforcer,
425 _MachineFlagsTemplate<__machine_flags(), __floating_point_flags()>>;
426
427 struct _Minimum
428 {
429 template <typename _Tp>
430 _GLIBCXX_SIMD_INTRINSIC constexpr
431 _Tp
432 operator()(_Tp __a, _Tp __b) const
433 {
434 using std::min;
435 return min(__a, __b);
436 }
437 };
438
439 struct _Maximum
440 {
441 template <typename _Tp>
442 _GLIBCXX_SIMD_INTRINSIC constexpr
443 _Tp
444 operator()(_Tp __a, _Tp __b) const
445 {
446 using std::max;
447 return max(__a, __b);
448 }
449 };
450} // namespace __detail
451
452// unrolled/pack execution helpers
453// __execute_n_times{{{
454template <typename _Fp, size_t... _I>
455 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
456 void
457 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
458 { ((void)__f(_SizeConstant<_I>()), ...); }
459
460template <typename _Fp>
461 _GLIBCXX_SIMD_INTRINSIC constexpr void
462 __execute_on_index_sequence(_Fp&&, index_sequence<>)
463 { }
464
465template <size_t _Np, typename _Fp>
466 _GLIBCXX_SIMD_INTRINSIC constexpr void
467 __execute_n_times(_Fp&& __f)
468 {
469 __execute_on_index_sequence(static_cast<_Fp&&>(__f),
470 make_index_sequence<_Np>{});
471 }
472
473// }}}
474// __generate_from_n_evaluations{{{
475template <typename _R, typename _Fp, size_t... _I>
476 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
477 _R
478 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
479 { return _R{__f(_SizeConstant<_I>())...}; }
480
481template <size_t _Np, typename _R, typename _Fp>
482 _GLIBCXX_SIMD_INTRINSIC constexpr _R
483 __generate_from_n_evaluations(_Fp&& __f)
484 {
485 return __execute_on_index_sequence_with_return<_R>(
486 static_cast<_Fp&&>(__f), make_index_sequence<_Np>{});
487 }
488
489// }}}
490// __call_with_n_evaluations{{{
491template <size_t... _I, typename _F0, typename _FArgs>
492 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
493 auto
494 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
495 { return __f0(__fargs(_SizeConstant<_I>())...); }
496
497template <size_t _Np, typename _F0, typename _FArgs>
498 _GLIBCXX_SIMD_INTRINSIC constexpr auto
499 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
500 {
501 return __call_with_n_evaluations(make_index_sequence<_Np>{},
502 static_cast<_F0&&>(__f0),
503 static_cast<_FArgs&&>(__fargs));
504 }
505
506// }}}
507// __call_with_subscripts{{{
508template <size_t _First = 0, size_t... _It, typename _Tp, typename _Fp>
509 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
510 auto
511 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
512 { return __fun(__x[_First + _It]...); }
513
514template <size_t _Np, size_t _First = 0, typename _Tp, typename _Fp>
515 _GLIBCXX_SIMD_INTRINSIC constexpr auto
516 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
517 {
518 return __call_with_subscripts<_First>(static_cast<_Tp&&>(__x),
519 make_index_sequence<_Np>(),
520 static_cast<_Fp&&>(__fun));
521 }
522
523// }}}
524
525// vvv ---- type traits ---- vvv
526// integer type aliases{{{
527using _UChar = unsigned char;
528using _SChar = signed char;
529using _UShort = unsigned short;
530using _UInt = unsigned int;
531using _ULong = unsigned long;
532using _ULLong = unsigned long long;
533using _LLong = long long;
534
535//}}}
536// __first_of_pack{{{
537template <typename _T0, typename...>
538 struct __first_of_pack
539 { using type = _T0; };
540
541template <typename... _Ts>
542 using __first_of_pack_t = typename __first_of_pack<_Ts...>::type;
543
544//}}}
545// __value_type_or_identity_t {{{
546template <typename _Tp>
547 typename _Tp::value_type
548 __value_type_or_identity_impl(int);
549
550template <typename _Tp>
551 _Tp
552 __value_type_or_identity_impl(float);
553
554template <typename _Tp>
555 using __value_type_or_identity_t
556 = decltype(__value_type_or_identity_impl<_Tp>(int()));
557
558// }}}
559// __is_vectorizable {{{
560template <typename _Tp>
561 struct __is_vectorizable : public is_arithmetic<_Tp> {};
562
563template <>
564 struct __is_vectorizable<bool> : public false_type {};
565
566template <typename _Tp>
567 inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
568
569// Deduces to a vectorizable type
570template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
571 using _Vectorizable = _Tp;
572
573// }}}
574// _LoadStorePtr / __is_possible_loadstore_conversion {{{
575template <typename _Ptr, typename _ValueType>
576 struct __is_possible_loadstore_conversion
577 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
578
579template <>
580 struct __is_possible_loadstore_conversion<bool, bool> : true_type {};
581
582// Deduces to a type allowed for load/store with the given value type.
583template <typename _Ptr, typename _ValueType,
584 typename = enable_if_t<
585 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
586 using _LoadStorePtr = _Ptr;
587
588// }}}
589// __is_bitmask{{{
590template <typename _Tp, typename = void_t<>>
591 struct __is_bitmask : false_type {};
592
593template <typename _Tp>
594 inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
595
596// the __mmaskXX case:
597template <typename _Tp>
598 struct __is_bitmask<_Tp,
599 void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>>
600 : true_type {};
601
602// }}}
603// __int_for_sizeof{{{
604#pragma GCC diagnostic push
605#pragma GCC diagnostic ignored "-Wpedantic"
606template <size_t _Bytes>
607 constexpr auto
608 __int_for_sizeof()
609 {
610 static_assert(_Bytes > 0);
611 if constexpr (_Bytes == sizeof(int))
612 return int();
613 else if constexpr (_Bytes == sizeof(_SChar))
614 return _SChar();
615 else if constexpr (_Bytes == sizeof(short))
616 return short();
617 else if constexpr (_Bytes == sizeof(long))
618 return long();
619 else if constexpr (_Bytes == sizeof(_LLong))
620 return _LLong();
621 #ifdef __SIZEOF_INT128__
622 else if constexpr (_Bytes == sizeof(__int128))
623 return __int128();
624 #endif // __SIZEOF_INT128__
625 else if constexpr (_Bytes % sizeof(int) == 0)
626 {
627 constexpr size_t _Np = _Bytes / sizeof(int);
628 struct _Ip
629 {
630 int _M_data[_Np];
631
632 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
633 operator&(_Ip __rhs) const
634 {
635 return __generate_from_n_evaluations<_Np, _Ip>(
636 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
637 return __rhs._M_data[__i] & _M_data[__i];
638 });
639 }
640
641 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
642 operator|(_Ip __rhs) const
643 {
644 return __generate_from_n_evaluations<_Np, _Ip>(
645 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
646 return __rhs._M_data[__i] | _M_data[__i];
647 });
648 }
649
650 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
651 operator^(_Ip __rhs) const
652 {
653 return __generate_from_n_evaluations<_Np, _Ip>(
654 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
655 return __rhs._M_data[__i] ^ _M_data[__i];
656 });
657 }
658
659 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
660 operator~() const
661 {
662 return __generate_from_n_evaluations<_Np, _Ip>(
663 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return ~_M_data[__i]; });
664 }
665 };
666 return _Ip{};
667 }
668 else
669 static_assert(_Bytes == 0, "this should be unreachable");
670 }
671#pragma GCC diagnostic pop
672
673template <typename _Tp>
674 using __int_for_sizeof_t = decltype(__int_for_sizeof<sizeof(_Tp)>());
675
676template <size_t _Np>
677 using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
678
679// }}}
680// __is_fixed_size_abi{{{
681template <typename _Tp>
682 struct __is_fixed_size_abi : false_type {};
683
684template <int _Np>
685 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> : true_type {};
686
687template <typename _Tp>
688 inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
689
690// }}}
691// __is_scalar_abi {{{
692template <typename _Abi>
693 constexpr bool
694 __is_scalar_abi()
695 { return is_same_v<simd_abi::scalar, _Abi>; }
696
697// }}}
698// __abi_bytes_v {{{
699template <template <int> class _Abi, int _Bytes>
700 constexpr int
701 __abi_bytes_impl(_Abi<_Bytes>*)
702 { return _Bytes; }
703
704template <typename _Tp>
705 constexpr int
706 __abi_bytes_impl(_Tp*)
707 { return -1; }
708
709template <typename _Abi>
710 inline constexpr int __abi_bytes_v
711 = __abi_bytes_impl(static_cast<_Abi*>(nullptr));
712
713// }}}
714// __is_builtin_bitmask_abi {{{
715template <typename _Abi>
716 constexpr bool
717 __is_builtin_bitmask_abi()
718 { return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
719
720// }}}
721// __is_sse_abi {{{
722template <typename _Abi>
723 constexpr bool
724 __is_sse_abi()
725 {
726 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
727 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
728 }
729
730// }}}
731// __is_avx_abi {{{
732template <typename _Abi>
733 constexpr bool
734 __is_avx_abi()
735 {
736 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
737 return _Bytes > 16 && _Bytes <= 32
738 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
739 }
740
741// }}}
742// __is_avx512_abi {{{
743template <typename _Abi>
744 constexpr bool
745 __is_avx512_abi()
746 {
747 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
748 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
749 }
750
751// }}}
752// __is_neon_abi {{{
753template <typename _Abi>
754 constexpr bool
755 __is_neon_abi()
756 {
757 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
758 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
759 }
760
761// }}}
762// __is_sve_abi {{{
763template <typename _Abi>
764 constexpr bool
765 __is_sve_abi()
766 {
767 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
768 return _Bytes <= __sve_vectorized_size_bytes && is_same_v<simd_abi::_Sve<_Bytes>, _Abi>;
769 }
770
771// }}}
772// __make_dependent_t {{{
773template <typename, typename _Up>
774 struct __make_dependent
775 { using type = _Up; };
776
777template <typename _Tp, typename _Up>
778 using __make_dependent_t = typename __make_dependent<_Tp, _Up>::type;
779
780// }}}
781// ^^^ ---- type traits ---- ^^^
782
783// __invoke_ub{{{
784template <typename... _Args>
785 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE void
786 __invoke_ub([[maybe_unused]] const char* __msg, [[maybe_unused]] const _Args&... __args)
787 {
788#ifdef _GLIBCXX_DEBUG_UB
789 __builtin_fprintf(stderr, __msg, __args...);
790 __builtin_trap();
791#else
792 __builtin_unreachable();
793#endif
794 }
795
796// }}}
797// __assert_unreachable{{{
798template <typename _Tp>
799 struct __assert_unreachable
800 { static_assert(!is_same_v<_Tp, _Tp>, "this should be unreachable"); };
801
802// }}}
803// __size_or_zero_v {{{
804template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
805 constexpr size_t
806 __size_or_zero_dispatch(int)
807 { return _Np; }
808
809template <typename _Tp, typename _Ap>
810 constexpr size_t
811 __size_or_zero_dispatch(float)
812 { return 0; }
813
814template <typename _Tp, typename _Ap>
815 inline constexpr size_t __size_or_zero_v
816 = __size_or_zero_dispatch<_Tp, _Ap>(0);
817
818// }}}
819// __div_roundup {{{
820inline constexpr size_t
821__div_roundup(size_t __a, size_t __b)
822{ return (__a + __b - 1) / __b; }
823
824// }}}
825// _ExactBool{{{
826class _ExactBool
827{
828 const bool _M_data;
829
830public:
831 _GLIBCXX_SIMD_INTRINSIC constexpr
832 _ExactBool(bool __b) : _M_data(__b) {}
833
834 _ExactBool(int) = delete;
835
836 _GLIBCXX_SIMD_INTRINSIC constexpr
837 operator bool() const
838 { return _M_data; }
839};
840
841// }}}
842// __may_alias{{{
843/**@internal
844 * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an
845 * aliasing pointer. This adds the __may_alias attribute to _Tp (with compilers
846 * that support it).
847 */
848template <typename _Tp>
849 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
850
851// }}}
852// _UnsupportedBase {{{
853// simd and simd_mask base for unsupported <_Tp, _Abi>
854struct _UnsupportedBase
855{
856 _UnsupportedBase() = delete;
857 _UnsupportedBase(const _UnsupportedBase&) = delete;
858 _UnsupportedBase& operator=(const _UnsupportedBase&) = delete;
859 ~_UnsupportedBase() = delete;
860};
861
862// }}}
863// _InvalidTraits {{{
864/**
865 * @internal
866 * Defines the implementation of __a given <_Tp, _Abi>.
867 *
868 * Implementations must ensure that only valid <_Tp, _Abi> instantiations are
869 * possible. Static assertions in the type definition do not suffice. It is
870 * important that SFINAE works.
871 */
872struct _InvalidTraits
873{
874 using _IsValid = false_type;
875 using _SimdBase = _UnsupportedBase;
876 using _MaskBase = _UnsupportedBase;
877
878 static constexpr size_t _S_full_size = 0;
879 static constexpr bool _S_is_partial = false;
880
881 static constexpr size_t _S_simd_align = 1;
882 struct _SimdImpl;
883 struct _SimdMember {};
884 struct _SimdCastType;
885
886 static constexpr size_t _S_mask_align = 1;
887 struct _MaskImpl;
888 struct _MaskMember {};
889 struct _MaskCastType;
890};
891
892// }}}
893// _SimdTraits {{{
894template <typename _Tp, typename _Abi, typename = void_t<>>
895 struct _SimdTraits : _InvalidTraits {};
896
897// }}}
898// __private_init, __bitset_init{{{
899/**
900 * @internal
901 * Tag used for private init constructor of simd and simd_mask
902 */
903inline constexpr struct _PrivateInit {} __private_init = {};
904
905inline constexpr struct _BitsetInit {} __bitset_init = {};
906
907// }}}
908// __is_narrowing_conversion<_From, _To>{{{
909template <typename _From, typename _To, bool = is_arithmetic_v<_From>,
910 bool = is_arithmetic_v<_To>>
911 struct __is_narrowing_conversion;
912
913// ignore "signed/unsigned mismatch" in the following trait.
914// The implicit conversions will do the right thing here.
915template <typename _From, typename _To>
916 struct __is_narrowing_conversion<_From, _To, true, true>
917 : public __bool_constant<(
918 __digits_v<_From> > __digits_v<_To>
919 || __finite_max_v<_From> > __finite_max_v<_To>
920 || __finite_min_v<_From> < __finite_min_v<_To>
921 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
922
923template <typename _Tp>
924 struct __is_narrowing_conversion<_Tp, bool, true, true>
925 : public true_type {};
926
927template <>
928 struct __is_narrowing_conversion<bool, bool, true, true>
929 : public false_type {};
930
931template <typename _Tp>
932 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
933 : public false_type {};
934
935template <typename _From, typename _To>
936 struct __is_narrowing_conversion<_From, _To, false, true>
937 : public negation<is_convertible<_From, _To>> {};
938
939// }}}
940// __converts_to_higher_integer_rank{{{
941template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))>
942 struct __converts_to_higher_integer_rank : public true_type {};
943
944// this may fail for char -> short if sizeof(char) == sizeof(short)
945template <typename _From, typename _To>
946 struct __converts_to_higher_integer_rank<_From, _To, false>
947 : public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
948
949// }}}
950// __data(simd/simd_mask) {{{
951template <typename _Tp, typename _Ap>
952 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
953 __data(const simd<_Tp, _Ap>& __x);
954
955template <typename _Tp, typename _Ap>
956 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
957 __data(simd<_Tp, _Ap>& __x);
958
959template <typename _Tp, typename _Ap>
960 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
961 __data(const simd_mask<_Tp, _Ap>& __x);
962
963template <typename _Tp, typename _Ap>
964 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
965 __data(simd_mask<_Tp, _Ap>& __x);
966
967// }}}
968// _SimdConverter {{{
969template <typename _FromT, typename _FromA, typename _ToT, typename _ToA,
970 typename = void>
971 struct _SimdConverter;
972
973template <typename _Tp, typename _Ap>
974 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
975 {
976 template <typename _Up>
977 _GLIBCXX_SIMD_INTRINSIC const _Up&
978 operator()(const _Up& __x)
979 { return __x; }
980 };
981
982// }}}
983// __to_value_type_or_member_type {{{
984template <typename _V>
985 _GLIBCXX_SIMD_INTRINSIC constexpr auto
986 __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x))
987 { return __data(__x); }
988
989template <typename _V>
990 _GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type&
991 __to_value_type_or_member_type(const typename _V::value_type& __x)
992 { return __x; }
993
994// }}}
995// __bool_storage_member_type{{{
996template <size_t _Size>
997 struct __bool_storage_member_type;
998
999template <size_t _Size>
1000 using __bool_storage_member_type_t =
1001 typename __bool_storage_member_type<_Size>::type;
1002
1003// }}}
1004// _SimdTuple {{{
1005// why not tuple?
1006// 1. tuple gives no guarantee about the storage order, but I require
1007// storage
1008// equivalent to array<_Tp, _Np>
1009// 2. direct access to the element type (first template argument)
1010// 3. enforces equal element type, only different _Abi types are allowed
1011template <typename _Tp, typename... _Abis>
1012 struct _SimdTuple;
1013
1014//}}}
1015// __fixed_size_storage_t {{{
1016template <typename _Tp, int _Np>
1017 struct __fixed_size_storage;
1018
1019template <typename _Tp, int _Np>
1020 using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _Np>::type;
1021
1022// }}}
1023// _SimdWrapper fwd decl{{{
1024template <typename _Tp, size_t _Size, typename = void_t<>>
1025 struct _SimdWrapper;
1026
1027template <typename _Tp>
1028 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>;
1029template <typename _Tp>
1030 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>;
1031template <typename _Tp>
1032 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>;
1033template <typename _Tp>
1034 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>;
1035
1036template <typename _Tp, size_t _Width>
1037 struct _SveSimdWrapper;
1038
1039// }}}
1040// __is_simd_wrapper {{{
1041template <typename _Tp>
1042 struct __is_simd_wrapper : false_type {};
1043
1044template <typename _Tp, size_t _Np>
1045 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> : true_type {};
1046
1047template <typename _Tp>
1048 inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
1049
1050// }}}
1051// _BitOps {{{
1052struct _BitOps
1053{
1054 // _S_bit_iteration {{{
1055 template <typename _Tp, typename _Fp>
1056 static void
1057 _S_bit_iteration(_Tp __mask, _Fp&& __f)
1058 {
1059 static_assert(sizeof(_ULLong) >= sizeof(_Tp));
1060 conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k;
1061 if constexpr (is_convertible_v<_Tp, decltype(__k)>)
1062 __k = __mask;
1063 else
1064 __k = __mask.to_ullong();
1065 while(__k)
1066 {
1067 __f(std::__countr_zero(__k));
1068 __k &= (__k - 1);
1069 }
1070 }
1071
1072 //}}}
1073};
1074
1075//}}}
1076// __increment, __decrement {{{
1077template <typename _Tp = void>
1078 struct __increment
1079 { constexpr _Tp operator()(_Tp __a) const { return ++__a; } };
1080
1081template <>
1082 struct __increment<void>
1083 {
1084 template <typename _Tp>
1085 constexpr _Tp
1086 operator()(_Tp __a) const
1087 { return ++__a; }
1088 };
1089
1090template <typename _Tp = void>
1091 struct __decrement
1092 { constexpr _Tp operator()(_Tp __a) const { return --__a; } };
1093
1094template <>
1095 struct __decrement<void>
1096 {
1097 template <typename _Tp>
1098 constexpr _Tp
1099 operator()(_Tp __a) const
1100 { return --__a; }
1101 };
1102
1103// }}}
1104// _ValuePreserving(OrInt) {{{
1105template <typename _From, typename _To,
1106 typename = enable_if_t<negation<
1107 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
1108 using _ValuePreserving = _From;
1109
1110template <typename _From, typename _To,
1111 typename _DecayedFrom = __remove_cvref_t<_From>,
1112 typename = enable_if_t<conjunction<
1113 is_convertible<_From, _To>,
1114 disjunction<
1115 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
1116 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
1117 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
1118 using _ValuePreservingOrInt = _From;
1119
1120// }}}
1121// __intrinsic_type {{{
1122template <typename _Tp, size_t _Bytes, typename = void_t<>>
1123 struct __intrinsic_type;
1124
1125template <typename _Tp, size_t _Size>
1126 using __intrinsic_type_t =
1127 typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type;
1128
1129template <typename _Tp>
1130 using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type;
1131template <typename _Tp>
1132 using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type;
1133template <typename _Tp>
1134 using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type;
1135template <typename _Tp>
1136 using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type;
1137template <typename _Tp>
1138 using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type;
1139template <typename _Tp>
1140 using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type;
1141
1142// }}}
1143// _BitMask {{{
1144template <size_t _Np, bool _Sanitized = false>
1145 struct _BitMask;
1146
1147template <size_t _Np, bool _Sanitized>
1148 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> : true_type {};
1149
1150template <size_t _Np>
1151 using _SanitizedBitMask = _BitMask<_Np, true>;
1152
1153template <size_t _Np, bool _Sanitized>
1154 struct _BitMask
1155 {
1156 static_assert(_Np > 0);
1157
1158 static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
1159
1160 using _Tp = conditional_t<_Np == 1, bool,
1161 make_unsigned_t<__int_with_sizeof_t<std::min(
1162 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
1163
1164 static constexpr int _S_array_size = __div_roundup(_NBytes, sizeof(_Tp));
1165
1166 _Tp _M_bits[_S_array_size];
1167
1168 static constexpr int _S_unused_bits
1169 = _Np == 1 ? 0 : _S_array_size * sizeof(_Tp) * __CHAR_BIT__ - _Np;
1170
1171 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1172
1173 constexpr _BitMask() noexcept = default;
1174
1175 constexpr _BitMask(unsigned long long __x) noexcept
1176 : _M_bits{static_cast<_Tp>(__x)} {}
1177
1178 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1179
1180 constexpr _BitMask(const _BitMask&) noexcept = default;
1181
1182 template <bool _RhsSanitized, typename = enable_if_t<_RhsSanitized == false
1183 && _Sanitized == true>>
1184 constexpr _BitMask(const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1185 : _BitMask(__rhs._M_sanitized()) {}
1186
1187 constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1188 {
1189 static_assert(_S_array_size == 1);
1190 return _M_bits[0];
1191 }
1192
1193 // precondition: is sanitized
1194 constexpr _Tp
1195 _M_to_bits() const noexcept
1196 {
1197 static_assert(_S_array_size == 1);
1198 return _M_bits[0];
1199 }
1200
1201 // precondition: is sanitized
1202 constexpr unsigned long long
1203 to_ullong() const noexcept
1204 {
1205 static_assert(_S_array_size == 1);
1206 return _M_bits[0];
1207 }
1208
1209 // precondition: is sanitized
1210 constexpr unsigned long
1211 to_ulong() const noexcept
1212 {
1213 static_assert(_S_array_size == 1);
1214 return _M_bits[0];
1215 }
1216
1217 constexpr bitset<_Np>
1218 _M_to_bitset() const noexcept
1219 {
1220 static_assert(_S_array_size == 1);
1221 return _M_bits[0];
1222 }
1223
1224 constexpr decltype(auto)
1225 _M_sanitized() const noexcept
1226 {
1227 if constexpr (_Sanitized)
1228 return *this;
1229 else if constexpr (_Np == 1)
1230 return _SanitizedBitMask<_Np>(_M_bits[0]);
1231 else
1232 {
1233 _SanitizedBitMask<_Np> __r = {};
1234 for (int __i = 0; __i < _S_array_size; ++__i)
1235 __r._M_bits[__i] = _M_bits[__i];
1236 if constexpr (_S_unused_bits > 0)
1237 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1238 return __r;
1239 }
1240 }
1241
1242 template <size_t _Mp, bool _LSanitized>
1243 constexpr _BitMask<_Np + _Mp, _Sanitized>
1244 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb) const noexcept
1245 {
1246 constexpr size_t _RN = _Np + _Mp;
1247 using _Rp = _BitMask<_RN, _Sanitized>;
1248 if constexpr (_Rp::_S_array_size == 1)
1249 {
1250 _Rp __r{{_M_bits[0]}};
1251 __r._M_bits[0] <<= _Mp;
1252 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1253 return __r;
1254 }
1255 else
1256 __assert_unreachable<_Rp>();
1257 }
1258
1259 // Return a new _BitMask with size _NewSize while dropping _DropLsb least
1260 // significant bits. If the operation implicitly produces a sanitized bitmask,
1261 // the result type will have _Sanitized set.
1262 template <size_t _DropLsb, size_t _NewSize = _Np - _DropLsb>
1263 constexpr auto
1264 _M_extract() const noexcept
1265 {
1266 static_assert(_Np > _DropLsb);
1267 static_assert(_DropLsb + _NewSize <= sizeof(_ULLong) * __CHAR_BIT__,
1268 "not implemented for bitmasks larger than one ullong");
1269 if constexpr (_NewSize == 1)
1270 // must sanitize because the return _Tp is bool
1271 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1272 else
1273 return _BitMask<_NewSize,
1274 ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1275 && _NewSize + _DropLsb <= _Np)
1276 || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1277 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1278 >> _DropLsb);
1279 }
1280
1281 // True if all bits are set. Implicitly sanitizes if _Sanitized == false.
1282 constexpr bool
1283 all() const noexcept
1284 {
1285 if constexpr (_Np == 1)
1286 return _M_bits[0];
1287 else if constexpr (!_Sanitized)
1288 return _M_sanitized().all();
1289 else
1290 {
1291 constexpr _Tp __allbits = ~_Tp();
1292 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1293 if (_M_bits[__i] != __allbits)
1294 return false;
1295 return _M_bits[_S_array_size - 1] == _S_bitmask;
1296 }
1297 }
1298
1299 // True if at least one bit is set. Implicitly sanitizes if _Sanitized ==
1300 // false.
1301 constexpr bool
1302 any() const noexcept
1303 {
1304 if constexpr (_Np == 1)
1305 return _M_bits[0];
1306 else if constexpr (!_Sanitized)
1307 return _M_sanitized().any();
1308 else
1309 {
1310 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1311 if (_M_bits[__i] != 0)
1312 return true;
1313 return _M_bits[_S_array_size - 1] != 0;
1314 }
1315 }
1316
1317 // True if no bit is set. Implicitly sanitizes if _Sanitized == false.
1318 constexpr bool
1319 none() const noexcept
1320 {
1321 if constexpr (_Np == 1)
1322 return !_M_bits[0];
1323 else if constexpr (!_Sanitized)
1324 return _M_sanitized().none();
1325 else
1326 {
1327 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1328 if (_M_bits[__i] != 0)
1329 return false;
1330 return _M_bits[_S_array_size - 1] == 0;
1331 }
1332 }
1333
1334 // Returns the number of set bits. Implicitly sanitizes if _Sanitized ==
1335 // false.
1336 constexpr int
1337 count() const noexcept
1338 {
1339 if constexpr (_Np == 1)
1340 return _M_bits[0];
1341 else if constexpr (!_Sanitized)
1342 return _M_sanitized().none();
1343 else
1344 {
1345 int __result = __builtin_popcountll(_M_bits[0]);
1346 for (int __i = 1; __i < _S_array_size; ++__i)
1347 __result += __builtin_popcountll(_M_bits[__i]);
1348 return __result;
1349 }
1350 }
1351
1352 // Returns the bit at offset __i as bool.
1353 constexpr bool
1354 operator[](size_t __i) const noexcept
1355 {
1356 if constexpr (_Np == 1)
1357 return _M_bits[0];
1358 else if constexpr (_S_array_size == 1)
1359 return (_M_bits[0] >> __i) & 1;
1360 else
1361 {
1362 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1363 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1364 return (_M_bits[__j] >> __shift) & 1;
1365 }
1366 }
1367
1368 template <size_t __i>
1369 constexpr bool
1370 operator[](_SizeConstant<__i>) const noexcept
1371 {
1372 static_assert(__i < _Np);
1373 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1374 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1375 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1376 }
1377
1378 // Set the bit at offset __i to __x.
1379 constexpr void
1380 set(size_t __i, bool __x) noexcept
1381 {
1382 if constexpr (_Np == 1)
1383 _M_bits[0] = __x;
1384 else if constexpr (_S_array_size == 1)
1385 {
1386 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1387 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1388 }
1389 else
1390 {
1391 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1392 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1393 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1394 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1395 }
1396 }
1397
1398 template <size_t __i>
1399 constexpr void
1400 set(_SizeConstant<__i>, bool __x) noexcept
1401 {
1402 static_assert(__i < _Np);
1403 if constexpr (_Np == 1)
1404 _M_bits[0] = __x;
1405 else
1406 {
1407 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1408 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1409 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1410 _M_bits[__j] &= __mask;
1411 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1412 }
1413 }
1414
1415 // Inverts all bits. Sanitized input leads to sanitized output.
1416 constexpr _BitMask
1417 operator~() const noexcept
1418 {
1419 if constexpr (_Np == 1)
1420 return !_M_bits[0];
1421 else
1422 {
1423 _BitMask __result{};
1424 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1425 __result._M_bits[__i] = ~_M_bits[__i];
1426 if constexpr (_Sanitized)
1427 __result._M_bits[_S_array_size - 1]
1428 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1429 else
1430 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1431 return __result;
1432 }
1433 }
1434
1435 constexpr _BitMask&
1436 operator^=(const _BitMask& __b) & noexcept
1437 {
1438 __execute_n_times<_S_array_size>(
1439 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] ^= __b._M_bits[__i]; });
1440 return *this;
1441 }
1442
1443 constexpr _BitMask&
1444 operator|=(const _BitMask& __b) & noexcept
1445 {
1446 __execute_n_times<_S_array_size>(
1447 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] |= __b._M_bits[__i]; });
1448 return *this;
1449 }
1450
1451 constexpr _BitMask&
1452 operator&=(const _BitMask& __b) & noexcept
1453 {
1454 __execute_n_times<_S_array_size>(
1455 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] &= __b._M_bits[__i]; });
1456 return *this;
1457 }
1458
1459 friend constexpr _BitMask
1460 operator^(const _BitMask& __a, const _BitMask& __b) noexcept
1461 {
1462 _BitMask __r = __a;
1463 __r ^= __b;
1464 return __r;
1465 }
1466
1467 friend constexpr _BitMask
1468 operator|(const _BitMask& __a, const _BitMask& __b) noexcept
1469 {
1470 _BitMask __r = __a;
1471 __r |= __b;
1472 return __r;
1473 }
1474
1475 friend constexpr _BitMask
1476 operator&(const _BitMask& __a, const _BitMask& __b) noexcept
1477 {
1478 _BitMask __r = __a;
1479 __r &= __b;
1480 return __r;
1481 }
1482
1483 _GLIBCXX_SIMD_INTRINSIC
1484 constexpr bool
1485 _M_is_constprop() const
1486 {
1487 if constexpr (_S_array_size == 0)
1488 return __builtin_constant_p(_M_bits[0]);
1489 else
1490 {
1491 for (int __i = 0; __i < _S_array_size; ++__i)
1492 if (!__builtin_constant_p(_M_bits[__i]))
1493 return false;
1494 return true;
1495 }
1496 }
1497 };
1498
1499// }}}
1500
1501// vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv
1502// __min_vector_size {{{
1503template <typename _Tp = void>
1504 static inline constexpr int __min_vector_size = 2 * sizeof(_Tp);
1505
1506#if _GLIBCXX_SIMD_HAVE_NEON
1507template <>
1508 inline constexpr int __min_vector_size<void> = 8;
1509#else
1510template <>
1511 inline constexpr int __min_vector_size<void> = 16;
1512#endif
1513
1514// }}}
1515// __vector_type {{{
1516template <typename _Tp, size_t _Np, typename = void>
1517 struct __vector_type_n {};
1518
1519// substitution failure for 0-element case
1520template <typename _Tp>
1521 struct __vector_type_n<_Tp, 0, void> {};
1522
1523// special case 1-element to be _Tp itself
1524template <typename _Tp>
1525 struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>>
1526 { using type = _Tp; };
1527
1528// else, use GNU-style builtin vector types
1529template <typename _Tp, size_t _Np>
1530 struct __vector_type_n<_Tp, _Np, enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1531 {
1532 static constexpr size_t _S_Np2 = std::__bit_ceil(_Np * sizeof(_Tp));
1533
1534 static constexpr size_t _S_Bytes =
1535#ifdef __i386__
1536 // Using [[gnu::vector_size(8)]] would wreak havoc on the FPU because
1537 // those objects are passed via MMX registers and nothing ever calls EMMS.
1538 _S_Np2 == 8 ? 16 :
1539#endif
1540 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1541 : _S_Np2;
1542
1543 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1544 };
1545
1546template <typename _Tp, size_t _Bytes, size_t = _Bytes % sizeof(_Tp)>
1547 struct __vector_type;
1548
1549template <typename _Tp, size_t _Bytes>
1550 struct __vector_type<_Tp, _Bytes, 0>
1551 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1552
1553template <typename _Tp, size_t _Size>
1554 using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type;
1555
1556template <typename _Tp>
1557 using __vector_type2_t = typename __vector_type<_Tp, 2>::type;
1558template <typename _Tp>
1559 using __vector_type4_t = typename __vector_type<_Tp, 4>::type;
1560template <typename _Tp>
1561 using __vector_type8_t = typename __vector_type<_Tp, 8>::type;
1562template <typename _Tp>
1563 using __vector_type16_t = typename __vector_type<_Tp, 16>::type;
1564template <typename _Tp>
1565 using __vector_type32_t = typename __vector_type<_Tp, 32>::type;
1566template <typename _Tp>
1567 using __vector_type64_t = typename __vector_type<_Tp, 64>::type;
1568
1569// }}}
1570// __is_vector_type {{{
1571template <typename _Tp, typename = void_t<>>
1572 struct __is_vector_type : false_type {};
1573
1574template <typename _Tp>
1575 struct __is_vector_type<
1576 _Tp, void_t<typename __vector_type<
1577 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1578 : is_same<_Tp, typename __vector_type<
1579 remove_reference_t<decltype(declval<_Tp>()[0])>,
1580 sizeof(_Tp)>::type> {};
1581
1582template <typename _Tp>
1583 inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1584
1585// }}}
1586// __is_intrinsic_type {{{
1587#if _GLIBCXX_SIMD_HAVE_SSE_ABI
1588template <typename _Tp>
1589 using __is_intrinsic_type = __is_vector_type<_Tp>;
1590#else // not SSE (x86)
1591template <typename _Tp, typename = void_t<>>
1592 struct __is_intrinsic_type : false_type {};
1593
1594template <typename _Tp>
1595 struct __is_intrinsic_type<
1596 _Tp, void_t<typename __intrinsic_type<
1597 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1598 : is_same<_Tp, typename __intrinsic_type<
1599 remove_reference_t<decltype(declval<_Tp>()[0])>,
1600 sizeof(_Tp)>::type> {};
1601#endif
1602
1603template <typename _Tp>
1604 inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1605
1606// }}}
1607// _VectorTraits{{{
1608template <typename _Tp, typename = void_t<>>
1609 struct _VectorTraitsImpl;
1610
1611template <typename _Tp>
1612 struct _VectorTraitsImpl<_Tp, enable_if_t<__is_vector_type_v<_Tp>
1613 || __is_intrinsic_type_v<_Tp>>>
1614 {
1615 using type = _Tp;
1616 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1617 static constexpr int _S_full_size = sizeof(_Tp) / sizeof(value_type);
1618 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1619 template <typename _Up, int _W = _S_full_size>
1620 static constexpr bool _S_is
1621 = is_same_v<value_type, _Up> && _W == _S_full_size;
1622 };
1623
1624template <typename _Tp, size_t _Np>
1625 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1626 void_t<__vector_type_t<_Tp, _Np>>>
1627 {
1628 using type = __vector_type_t<_Tp, _Np>;
1629 using value_type = _Tp;
1630 static constexpr int _S_full_size = sizeof(type) / sizeof(value_type);
1631 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1632 static constexpr bool _S_is_partial = (_Np == _S_full_size);
1633 static constexpr int _S_partial_width = _Np;
1634 template <typename _Up, int _W = _S_full_size>
1635 static constexpr bool _S_is
1636 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1637 };
1638
1639template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1640 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1641
1642// }}}
1643// __as_vector{{{
1644template <typename _V>
1645 _GLIBCXX_SIMD_INTRINSIC constexpr auto
1646 __as_vector(_V __x)
1647 {
1648 if constexpr (__is_vector_type_v<_V>)
1649 return __x;
1650 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1651 {
1652 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
1653 {
1654 static_assert(is_simd<_V>::value);
1655 static_assert(_V::abi_type::template __traits<
1656 typename _V::value_type>::_SimdMember::_S_tuple_size == 1);
1657 return __as_vector(__data(__x).first);
1658 }
1659 else if constexpr (_V::size() > 1)
1660 return __data(__x)._M_data;
1661 else
1662 {
1663 static_assert(is_simd<_V>::value);
1664 using _Tp = typename _V::value_type;
1665#ifdef __i386__
1666 constexpr auto __bytes = sizeof(_Tp) == 8 ? 16 : sizeof(_Tp);
1667 using _RV [[__gnu__::__vector_size__(__bytes)]] = _Tp;
1668#else
1669 using _RV [[__gnu__::__vector_size__(sizeof(_Tp))]] = _Tp;
1670#endif
1671 return _RV{__data(__x)};
1672 }
1673 }
1674 else if constexpr (__is_vectorizable_v<_V>)
1675 return __vector_type_t<_V, 2>{__x};
1676 else
1677 return __x._M_data;
1678 }
1679
1680// }}}
1681// __as_wrapper{{{
1682template <size_t _Np = 0, typename _V>
1683 _GLIBCXX_SIMD_INTRINSIC constexpr auto
1684 __as_wrapper(_V __x)
1685 {
1686 if constexpr (__is_vector_type_v<_V>)
1687 return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1688 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1689 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1690 {
1691 static_assert(_V::size() == _Np);
1692 return __data(__x);
1693 }
1694 else
1695 {
1696 static_assert(_V::_S_size == _Np);
1697 return __x;
1698 }
1699 }
1700
1701// }}}
1702// __intrin_bitcast{{{
1703template <typename _To, typename _From>
1704 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1705 __intrin_bitcast(_From __v)
1706 {
1707 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1708 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1709 if constexpr (sizeof(_To) == sizeof(_From))
1710 return reinterpret_cast<_To>(__v);
1711 else if constexpr (sizeof(_From) > sizeof(_To))
1712 if constexpr (sizeof(_To) >= 16)
1713 return reinterpret_cast<const __may_alias<_To>&>(__v);
1714 else
1715 {
1716 _To __r;
1717 __builtin_memcpy(&__r, &__v, sizeof(_To));
1718 return __r;
1719 }
1720#if _GLIBCXX_SIMD_X86INTRIN && !defined _GLIBCXX_CLANG
1721 else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32)
1722 return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1723 reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1724 else if constexpr (__have_avx512f && sizeof(_From) == 16
1725 && sizeof(_To) == 64)
1726 return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1727 reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1728 else if constexpr (__have_avx512f && sizeof(_From) == 32
1729 && sizeof(_To) == 64)
1730 return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1731 reinterpret_cast<__vector_type_t<float, 8>>(__v)));
1732#endif // _GLIBCXX_SIMD_X86INTRIN
1733 else if constexpr (sizeof(__v) <= 8)
1734 return reinterpret_cast<_To>(
1735 __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1736 reinterpret_cast<__int_for_sizeof_t<_From>>(__v)});
1737 else
1738 {
1739 static_assert(sizeof(_To) > sizeof(_From));
1740 _To __r = {};
1741 __builtin_memcpy(&__r, &__v, sizeof(_From));
1742 return __r;
1743 }
1744 }
1745
1746// }}}
1747// __vector_bitcast{{{
1748template <typename _To, size_t _NN = 0, typename _From,
1749 typename _FromVT = _VectorTraits<_From>,
1750 size_t _Np = _NN == 0 ? sizeof(_From) / sizeof(_To) : _NN>
1751 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1752 __vector_bitcast(_From __x)
1753 {
1754 using _R = __vector_type_t<_To, _Np>;
1755 return __intrin_bitcast<_R>(__x);
1756 }
1757
1758template <typename _To, size_t _NN = 0, typename _Tp, size_t _Nx,
1759 size_t _Np
1760 = _NN == 0 ? sizeof(_SimdWrapper<_Tp, _Nx>) / sizeof(_To) : _NN>
1761 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1762 __vector_bitcast(const _SimdWrapper<_Tp, _Nx>& __x)
1763 {
1764 static_assert(_Np > 1);
1765 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1766 }
1767
1768// }}}
1769// __convert_x86 declarations {{{
1770#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1771template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1772 _To __convert_x86(_Tp);
1773
1774template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1775 _To __convert_x86(_Tp, _Tp);
1776
1777template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1778 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1779
1780template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1781 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1782
1783template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1784 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1785 _Tp, _Tp, _Tp, _Tp);
1786#endif // _GLIBCXX_SIMD_WORKAROUND_PR85048
1787
1788//}}}
1789// __bit_cast {{{
1790template <typename _To, typename _From>
1791 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1792 __bit_cast(const _From __x)
1793 {
1794#if __has_builtin(__builtin_bit_cast)
1795 return __builtin_bit_cast(_To, __x);
1796#else
1797 static_assert(sizeof(_To) == sizeof(_From));
1798 constexpr bool __to_is_vectorizable
1799 = is_arithmetic_v<_To> || is_enum_v<_To>;
1800 constexpr bool __from_is_vectorizable
1801 = is_arithmetic_v<_From> || is_enum_v<_From>;
1802 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1803 return reinterpret_cast<_To>(__x);
1804 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1805 {
1806 using _FV [[__gnu__::__vector_size__(sizeof(_From))]] = _From;
1807 return reinterpret_cast<_To>(_FV{__x});
1808 }
1809 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1810 {
1811 using _TV [[__gnu__::__vector_size__(sizeof(_To))]] = _To;
1812 using _FV [[__gnu__::__vector_size__(sizeof(_From))]] = _From;
1813 return reinterpret_cast<_TV>(_FV{__x})[0];
1814 }
1815 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1816 {
1817 using _TV [[__gnu__::__vector_size__(sizeof(_To))]] = _To;
1818 return reinterpret_cast<_TV>(__x)[0];
1819 }
1820 else
1821 {
1822 _To __r;
1823 __builtin_memcpy(reinterpret_cast<char*>(&__r),
1824 reinterpret_cast<const char*>(&__x), sizeof(_To));
1825 return __r;
1826 }
1827#endif
1828 }
1829
1830// }}}
1831// __to_intrin {{{
1832template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1833 typename _R = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_full_size>>
1834 _GLIBCXX_SIMD_INTRINSIC constexpr _R
1835 __to_intrin(_Tp __x)
1836 {
1837 static_assert(sizeof(__x) <= sizeof(_R),
1838 "__to_intrin may never drop values off the end");
1839 if constexpr (sizeof(__x) == sizeof(_R))
1840 return reinterpret_cast<_R>(__as_vector(__x));
1841 else
1842 {
1843 using _Up = __int_for_sizeof_t<_Tp>;
1844 return reinterpret_cast<_R>(
1845 __vector_type_t<_Up, sizeof(_R) / sizeof(_Up)>{__bit_cast<_Up>(__x)});
1846 }
1847 }
1848
1849// }}}
1850// __make_vector{{{
1851template <typename _Tp, typename... _Args>
1852 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)>
1853 __make_vector(const _Args&... __args)
1854 { return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(__args)...}; }
1855
1856// }}}
1857// __vector_broadcast{{{
1858template <size_t _Np, typename _Tp, size_t... _I>
1859 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1860 __vector_broadcast_impl(_Tp __x, index_sequence<_I...>)
1861 { return __vector_type_t<_Tp, _Np>{((void)_I, __x)...}; }
1862
1863template <size_t _Np, typename _Tp>
1864 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1865 __vector_broadcast(_Tp __x)
1866 { return __vector_broadcast_impl<_Np, _Tp>(__x, make_index_sequence<_Np>()); }
1867
1868// }}}
1869// __generate_vector{{{
1870 template <typename _Tp, size_t _Np, typename _Gp, size_t... _I>
1871 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1872 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1873 { return __vector_type_t<_Tp, _Np>{ static_cast<_Tp>(__gen(_SizeConstant<_I>()))...}; }
1874
1875template <typename _V, typename _VVT = _VectorTraits<_V>, typename _Gp>
1876 _GLIBCXX_SIMD_INTRINSIC constexpr _V
1877 __generate_vector(_Gp&& __gen)
1878 {
1879 if constexpr (__is_vector_type_v<_V>)
1880 return __generate_vector_impl<typename _VVT::value_type,
1881 _VVT::_S_full_size>(
1882 static_cast<_Gp&&>(__gen), make_index_sequence<_VVT::_S_full_size>());
1883 else
1884 return __generate_vector_impl<typename _VVT::value_type,
1885 _VVT::_S_partial_width>(
1886 static_cast<_Gp&&>(__gen),
1887 make_index_sequence<_VVT::_S_partial_width>());
1888 }
1889
1890template <typename _Tp, size_t _Np, typename _Gp>
1891 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1892 __generate_vector(_Gp&& __gen)
1893 {
1894 return __generate_vector_impl<_Tp, _Np>(static_cast<_Gp&&>(__gen),
1895 make_index_sequence<_Np>());
1896 }
1897
1898// }}}
1899// __xor{{{
1900template <typename _TW>
1901 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1902 __xor(_TW __a, _TW __b) noexcept
1903 {
1904 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1905 {
1906 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1907 _VectorTraitsImpl<_TW>>::value_type;
1908 if constexpr (is_floating_point_v<_Tp>)
1909 {
1910 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1911 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1912 ^ __vector_bitcast<_Ip>(__b));
1913 }
1914 else if constexpr (__is_vector_type_v<_TW>)
1915 return __a ^ __b;
1916 else
1917 return __a._M_data ^ __b._M_data;
1918 }
1919 else
1920 return __a ^ __b;
1921 }
1922
1923// }}}
1924// __or{{{
1925template <typename _TW>
1926 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1927 __or(_TW __a, _TW __b) noexcept
1928 {
1929 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1930 {
1931 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1932 _VectorTraitsImpl<_TW>>::value_type;
1933 if constexpr (is_floating_point_v<_Tp>)
1934 {
1935 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1936 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1937 | __vector_bitcast<_Ip>(__b));
1938 }
1939 else if constexpr (__is_vector_type_v<_TW>)
1940 return __a | __b;
1941 else
1942 return __a._M_data | __b._M_data;
1943 }
1944 else
1945 return __a | __b;
1946 }
1947
1948// }}}
1949// __and{{{
1950template <typename _TW>
1951 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1952 __and(_TW __a, _TW __b) noexcept
1953 {
1954 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1955 {
1956 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1957 _VectorTraitsImpl<_TW>>::value_type;
1958 if constexpr (is_floating_point_v<_Tp>)
1959 {
1960 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1961 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1962 & __vector_bitcast<_Ip>(__b));
1963 }
1964 else if constexpr (__is_vector_type_v<_TW>)
1965 return __a & __b;
1966 else
1967 return __a._M_data & __b._M_data;
1968 }
1969 else
1970 return __a & __b;
1971 }
1972
1973// }}}
1974// __andnot{{{
1975#if _GLIBCXX_SIMD_X86INTRIN && !defined _GLIBCXX_CLANG
1976static constexpr struct
1977{
1978 _GLIBCXX_SIMD_INTRINSIC __v4sf
1979 operator()(__v4sf __a, __v4sf __b) const noexcept
1980 { return __builtin_ia32_andnps(__a, __b); }
1981
1982 _GLIBCXX_SIMD_INTRINSIC __v2df
1983 operator()(__v2df __a, __v2df __b) const noexcept
1984 { return __builtin_ia32_andnpd(__a, __b); }
1985
1986 _GLIBCXX_SIMD_INTRINSIC __v2di
1987 operator()(__v2di __a, __v2di __b) const noexcept
1988 { return __builtin_ia32_pandn128(__a, __b); }
1989
1990 _GLIBCXX_SIMD_INTRINSIC __v8sf
1991 operator()(__v8sf __a, __v8sf __b) const noexcept
1992 { return __builtin_ia32_andnps256(__a, __b); }
1993
1994 _GLIBCXX_SIMD_INTRINSIC __v4df
1995 operator()(__v4df __a, __v4df __b) const noexcept
1996 { return __builtin_ia32_andnpd256(__a, __b); }
1997
1998 _GLIBCXX_SIMD_INTRINSIC __v4di
1999 operator()(__v4di __a, __v4di __b) const noexcept
2000 {
2001 if constexpr (__have_avx2)
2002 return __builtin_ia32_andnotsi256(__a, __b);
2003 else
2004 return reinterpret_cast<__v4di>(
2005 __builtin_ia32_andnpd256(reinterpret_cast<__v4df>(__a),
2006 reinterpret_cast<__v4df>(__b)));
2007 }
2008
2009 _GLIBCXX_SIMD_INTRINSIC __v16sf
2010 operator()(__v16sf __a, __v16sf __b) const noexcept
2011 {
2012 if constexpr (__have_avx512dq)
2013 return _mm512_andnot_ps(__a, __b);
2014 else
2015 return reinterpret_cast<__v16sf>(
2016 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
2017 reinterpret_cast<__v8di>(__b)));
2018 }
2019
2020 _GLIBCXX_SIMD_INTRINSIC __v8df
2021 operator()(__v8df __a, __v8df __b) const noexcept
2022 {
2023 if constexpr (__have_avx512dq)
2024 return _mm512_andnot_pd(__a, __b);
2025 else
2026 return reinterpret_cast<__v8df>(
2027 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
2028 reinterpret_cast<__v8di>(__b)));
2029 }
2030
2031 _GLIBCXX_SIMD_INTRINSIC __v8di
2032 operator()(__v8di __a, __v8di __b) const noexcept
2033 { return _mm512_andnot_si512(__a, __b); }
2034} _S_x86_andnot;
2035#endif // _GLIBCXX_SIMD_X86INTRIN && !_GLIBCXX_CLANG
2036
2037template <typename _TW>
2038 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
2039 __andnot(_TW __a, _TW __b) noexcept
2040 {
2041 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
2042 {
2043 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
2044 _VectorTraitsImpl<_TW>>;
2045 using _Tp = typename _TVT::value_type;
2046#if _GLIBCXX_SIMD_X86INTRIN && !defined _GLIBCXX_CLANG
2047 if constexpr (sizeof(_TW) >= 16)
2048 {
2049 const auto __ai = __to_intrin(__a);
2050 const auto __bi = __to_intrin(__b);
2051 if (!__builtin_is_constant_evaluated()
2052 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
2053 {
2054 const auto __r = _S_x86_andnot(__ai, __bi);
2055 if constexpr (is_convertible_v<decltype(__r), _TW>)
2056 return __r;
2057 else
2058 return reinterpret_cast<typename _TVT::type>(__r);
2059 }
2060 }
2061#endif // _GLIBCXX_SIMD_X86INTRIN
2062 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
2063 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
2064 & __vector_bitcast<_Ip>(__b));
2065 }
2066 else
2067 return ~__a & __b;
2068 }
2069
2070// }}}
2071// __not{{{
2072template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2073 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2074 __not(_Tp __a) noexcept
2075 {
2076 if constexpr (is_floating_point_v<typename _TVT::value_type>)
2077 return reinterpret_cast<typename _TVT::type>(
2078 ~__vector_bitcast<unsigned>(__a));
2079 else
2080 return ~__a;
2081 }
2082
2083// }}}
2084// __vec_shuffle{{{
2085template <typename _T0, typename _T1, typename _Fun, size_t... _Is>
2086 _GLIBCXX_SIMD_INTRINSIC constexpr
2087 __vector_type_t<remove_reference_t<decltype(declval<_T0>()[0])>, sizeof...(_Is)>
2088 __vec_shuffle(_T0 __x, _T1 __y, index_sequence<_Is...> __seq, _Fun __idx_perm)
2089 {
2090 constexpr int _N0 = sizeof(__x) / sizeof(__x[0]);
2091 constexpr int _N1 = sizeof(__y) / sizeof(__y[0]);
2092 using _Tp = remove_reference_t<decltype(declval<_T0>()[0])>;
2093 using _RV [[maybe_unused]] = __vector_type_t<_Tp, sizeof...(_Is)>;
2094#if __has_builtin(__builtin_shufflevector)
2095#ifdef _GLIBCXX_CLANG
2096 // Clang requires _T0 == _T1
2097 if constexpr (sizeof(__x) > sizeof(__y) and _N1 == 1)
2098 return __vec_shuffle(__x, _T0{__y[0]}, __seq, __idx_perm);
2099 else if constexpr (sizeof(__x) > sizeof(__y))
2100 return __vec_shuffle(__x, __intrin_bitcast<_T0>(__y), __seq, __idx_perm);
2101 else if constexpr (sizeof(__x) < sizeof(__y) and _N0 == 1)
2102 return __vec_shuffle(_T1{__x[0]}, __y, __seq, [=](int __i) {
2103 __i = __idx_perm(__i);
2104 return __i < _N0 ? __i : __i - _N0 + _N1;
2105 });
2106 else if constexpr (sizeof(__x) < sizeof(__y))
2107 return __vec_shuffle(__intrin_bitcast<_T1>(__x), __y, __seq, [=](int __i) {
2108 __i = __idx_perm(__i);
2109 return __i < _N0 ? __i : __i - _N0 + _N1;
2110 });
2111 else
2112#endif
2113 {
2114 const auto __r = __builtin_shufflevector(__x, __y, [=] {
2115 constexpr int __j = __idx_perm(_Is);
2116 static_assert(__j < _N0 + _N1);
2117 return __j;
2118 }()...);
2119#ifdef __i386__
2120 if constexpr (sizeof(__r) == sizeof(_RV))
2121 return __r;
2122 else
2123 return _RV {__r[_Is]...};
2124#else
2125 return __r;
2126#endif
2127 }
2128#else
2129 return _RV {
2130 [=]() -> _Tp {
2131 constexpr int __j = __idx_perm(_Is);
2132 static_assert(__j < _N0 + _N1);
2133 if constexpr (__j < 0)
2134 return 0;
2135 else if constexpr (__j < _N0)
2136 return __x[__j];
2137 else
2138 return __y[__j - _N0];
2139 }()...
2140 };
2141#endif
2142 }
2143
2144template <typename _T0, typename _Fun, typename _Seq>
2145 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2146 __vec_shuffle(_T0 __x, _Seq __seq, _Fun __idx_perm)
2147 { return __vec_shuffle(__x, _T0(), __seq, __idx_perm); }
2148
2149// }}}
2150// __concat{{{
2151template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
2152 typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size * 2>>
2153 constexpr _R
2154 __concat(_Tp a_, _Tp b_)
2155 {
2156#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2157 using _W
2158 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
2159 conditional_t<(sizeof(_Tp) >= 2 * sizeof(long long)),
2160 long long, typename _TVT::value_type>>;
2161 constexpr int input_width = sizeof(_Tp) / sizeof(_W);
2162 const auto __a = __vector_bitcast<_W>(a_);
2163 const auto __b = __vector_bitcast<_W>(b_);
2164 using _Up = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>;
2165#else
2166 constexpr int input_width = _TVT::_S_full_size;
2167 const _Tp& __a = a_;
2168 const _Tp& __b = b_;
2169 using _Up = _R;
2170#endif
2171 if constexpr (input_width == 2)
2172 return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
2173 else if constexpr (input_width == 4)
2174 return reinterpret_cast<_R>(
2175 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
2176 else if constexpr (input_width == 8)
2177 return reinterpret_cast<_R>(
2178 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
2179 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
2180 else if constexpr (input_width == 16)
2181 return reinterpret_cast<_R>(
2182 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2183 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2184 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
2185 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
2186 __b[12], __b[13], __b[14], __b[15]});
2187 else if constexpr (input_width == 32)
2188 return reinterpret_cast<_R>(
2189 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2190 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2191 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
2192 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
2193 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
2194 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
2195 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
2196 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
2197 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
2198 __b[31]});
2199 }
2200
2201// }}}
2202// __zero_extend {{{
2203template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2204 struct _ZeroExtendProxy
2205 {
2206 using value_type = typename _TVT::value_type;
2207 static constexpr size_t _Np = _TVT::_S_full_size;
2208 const _Tp __x;
2209
2210 template <typename _To, typename _ToVT = _VectorTraits<_To>,
2211 typename
2212 = enable_if_t<is_same_v<typename _ToVT::value_type, value_type>>>
2213 _GLIBCXX_SIMD_INTRINSIC operator _To() const
2214 {
2215 constexpr size_t _ToN = _ToVT::_S_full_size;
2216 if constexpr (_ToN == _Np)
2217 return __x;
2218 else if constexpr (_ToN == 2 * _Np)
2219 {
2220#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2221 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
2222 return __vector_bitcast<value_type>(
2223 _mm256_insertf128_ps(__m256(), __x, 0));
2224 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
2225 return __vector_bitcast<value_type>(
2226 _mm256_insertf128_pd(__m256d(), __x, 0));
2227 else if constexpr (__have_avx2 && _Np * sizeof(value_type) == 16)
2228 return __vector_bitcast<value_type>(
2229 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
2230 else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
2231 {
2232 if constexpr (__have_avx512dq)
2233 return __vector_bitcast<value_type>(
2234 _mm512_insertf32x8(__m512(), __x, 0));
2235 else
2236 return reinterpret_cast<__m512>(
2237 _mm512_insertf64x4(__m512d(),
2238 reinterpret_cast<__m256d>(__x), 0));
2239 }
2240 else if constexpr (__have_avx512f
2241 && _TVT::template _S_is<double, 4>)
2242 return __vector_bitcast<value_type>(
2243 _mm512_insertf64x4(__m512d(), __x, 0));
2244 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 32)
2245 return __vector_bitcast<value_type>(
2246 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
2247#endif
2248 return __concat(__x, _Tp());
2249 }
2250 else if constexpr (_ToN == 4 * _Np)
2251 {
2252#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2253 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2254 {
2255 return __vector_bitcast<value_type>(
2256 _mm512_insertf64x2(__m512d(), __x, 0));
2257 }
2258 else if constexpr (__have_avx512f
2259 && is_floating_point_v<value_type>)
2260 {
2261 return __vector_bitcast<value_type>(
2262 _mm512_insertf32x4(__m512(), reinterpret_cast<__m128>(__x),
2263 0));
2264 }
2265 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 16)
2266 {
2267 return __vector_bitcast<value_type>(
2268 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2269 }
2270#endif
2271 return __concat(__concat(__x, _Tp()),
2272 __vector_type_t<value_type, _Np * 2>());
2273 }
2274 else if constexpr (_ToN == 8 * _Np)
2275 return __concat(operator __vector_type_t<value_type, _Np * 4>(),
2276 __vector_type_t<value_type, _Np * 4>());
2277 else if constexpr (_ToN == 16 * _Np)
2278 return __concat(operator __vector_type_t<value_type, _Np * 8>(),
2279 __vector_type_t<value_type, _Np * 8>());
2280 else
2281 __assert_unreachable<_Tp>();
2282 }
2283 };
2284
2285template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2286 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2287 __zero_extend(_Tp __x)
2288 { return {__x}; }
2289
2290// }}}
2291// __extract<_Np, By>{{{
2292template <int _Offset,
2293 int _SplitBy,
2294 typename _Tp,
2295 typename _TVT = _VectorTraits<_Tp>,
2296 typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size / _SplitBy>>
2297 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2298 __extract(_Tp __in)
2299 {
2300 using value_type = typename _TVT::value_type;
2301#if _GLIBCXX_SIMD_X86INTRIN // {{{
2302 if constexpr (sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2303 {
2304 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2305 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2306 else if constexpr (is_floating_point_v<value_type>)
2307 return __vector_bitcast<value_type>(
2308 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2309 else
2310 return reinterpret_cast<_R>(
2311 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2312 _Offset));
2313 }
2314 else
2315#endif // _GLIBCXX_SIMD_X86INTRIN }}}
2316 {
2317#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2318 using _W = conditional_t<
2319 is_floating_point_v<value_type>, double,
2320 conditional_t<(sizeof(_R) >= 16), long long, value_type>>;
2321 static_assert(sizeof(_R) % sizeof(_W) == 0);
2322 constexpr int __return_width = sizeof(_R) / sizeof(_W);
2323 using _Up = __vector_type_t<_W, __return_width>;
2324 const auto __x = __vector_bitcast<_W>(__in);
2325#else
2326 constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2327 using _Up = _R;
2328 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2329 = __in; // only needed for _Tp = _SimdWrapper<value_type, _Np>
2330#endif
2331 constexpr int _O = _Offset * __return_width;
2332 return __call_with_subscripts<__return_width, _O>(
2333 __x, [](auto... __entries) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2334 return reinterpret_cast<_R>(_Up{__entries...});
2335 });
2336 }
2337 }
2338
2339// }}}
2340// __lo/__hi64[z]{{{
2341template <typename _Tp,
2342 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2343 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2344 __lo64(_Tp __x)
2345 {
2346 _R __r{};
2347 __builtin_memcpy(&__r, &__x, 8);
2348 return __r;
2349 }
2350
2351template <typename _Tp,
2352 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2353 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2354 __hi64(_Tp __x)
2355 {
2356 static_assert(sizeof(_Tp) == 16, "use __hi64z if you meant it");
2357 _R __r{};
2358 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2359 return __r;
2360 }
2361
2362template <typename _Tp,
2363 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2364 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2365 __hi64z([[maybe_unused]] _Tp __x)
2366 {
2367 _R __r{};
2368 if constexpr (sizeof(_Tp) == 16)
2369 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2370 return __r;
2371 }
2372
2373// }}}
2374// __lo/__hi128{{{
2375template <typename _Tp>
2376 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2377 __lo128(_Tp __x)
2378 { return __extract<0, sizeof(_Tp) / 16>(__x); }
2379
2380template <typename _Tp>
2381 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2382 __hi128(_Tp __x)
2383 {
2384 static_assert(sizeof(__x) == 32);
2385 return __extract<1, 2>(__x);
2386 }
2387
2388// }}}
2389// __lo/__hi256{{{
2390template <typename _Tp>
2391 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2392 __lo256(_Tp __x)
2393 {
2394 static_assert(sizeof(__x) == 64);
2395 return __extract<0, 2>(__x);
2396 }
2397
2398template <typename _Tp>
2399 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2400 __hi256(_Tp __x)
2401 {
2402 static_assert(sizeof(__x) == 64);
2403 return __extract<1, 2>(__x);
2404 }
2405
2406// }}}
2407// __auto_bitcast{{{
2408template <typename _Tp>
2409 struct _AutoCast
2410 {
2411 static_assert(__is_vector_type_v<_Tp>);
2412
2413 const _Tp __x;
2414
2415 template <typename _Up, typename _UVT = _VectorTraits<_Up>>
2416 _GLIBCXX_SIMD_INTRINSIC constexpr operator _Up() const
2417 { return __intrin_bitcast<typename _UVT::type>(__x); }
2418 };
2419
2420template <typename _Tp>
2421 _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2422 __auto_bitcast(const _Tp& __x)
2423 { return {__x}; }
2424
2425template <typename _Tp, size_t _Np>
2426 _GLIBCXX_SIMD_INTRINSIC constexpr
2427 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2428 __auto_bitcast(const _SimdWrapper<_Tp, _Np>& __x)
2429 { return {__x._M_data}; }
2430
2431// }}}
2432// ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^
2433
2434#if _GLIBCXX_SIMD_HAVE_SSE_ABI
2435// __bool_storage_member_type{{{
2436#if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2437template <size_t _Size>
2438 struct __bool_storage_member_type
2439 {
2440 static_assert((_Size & (_Size - 1)) != 0,
2441 "This trait may only be used for non-power-of-2 sizes. "
2442 "Power-of-2 sizes must be specialized.");
2443 using type =
2444 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2445 };
2446
2447template <>
2448 struct __bool_storage_member_type<1> { using type = bool; };
2449
2450template <>
2451 struct __bool_storage_member_type<2> { using type = __mmask8; };
2452
2453template <>
2454 struct __bool_storage_member_type<4> { using type = __mmask8; };
2455
2456template <>
2457 struct __bool_storage_member_type<8> { using type = __mmask8; };
2458
2459template <>
2460 struct __bool_storage_member_type<16> { using type = __mmask16; };
2461
2462template <>
2463 struct __bool_storage_member_type<32> { using type = __mmask32; };
2464
2465template <>
2466 struct __bool_storage_member_type<64> { using type = __mmask64; };
2467#endif // _GLIBCXX_SIMD_HAVE_AVX512F
2468
2469// }}}
2470// __intrinsic_type (x86){{{
2471// the following excludes bool via __is_vectorizable
2472#if _GLIBCXX_SIMD_HAVE_SSE
2473template <typename _Tp, size_t _Bytes>
2474 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2475 {
2476 // allow _Tp == long double with -mlong-double-64
2477 static_assert(!(is_same_v<_Tp, long double>
2478 && sizeof(long double) > sizeof(double)),
2479 "no __intrinsic_type support for long double on x86");
2480
2481 static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16 : _Bytes <= 32 ? 32 : 64;
2482
2483 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2484 = conditional_t<is_integral_v<_Tp>, long long int,
2485 conditional_t<is_same_v<_Tp, long double>, double, _Tp> >;
2486 };
2487#endif // _GLIBCXX_SIMD_HAVE_SSE
2488
2489// }}}
2490#endif // _GLIBCXX_SIMD_HAVE_SSE_ABI
2491// __intrinsic_type (ARM){{{
2492#if _GLIBCXX_SIMD_HAVE_NEON
2493template <>
2494 struct __intrinsic_type<float, 8, void>
2495 { using type = float32x2_t; };
2496
2497template <>
2498 struct __intrinsic_type<float, 16, void>
2499 { using type = float32x4_t; };
2500
2501template <>
2502 struct __intrinsic_type<double, 8, void>
2503 {
2504#if _GLIBCXX_SIMD_HAVE_NEON_A64
2505 using type = float64x1_t;
2506#endif
2507 };
2508
2509template <>
2510 struct __intrinsic_type<double, 16, void>
2511 {
2512#if _GLIBCXX_SIMD_HAVE_NEON_A64
2513 using type = float64x2_t;
2514#endif
2515 };
2516
2517#define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2518template <> \
2519 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2520 _Np * _Bits / 8, void> \
2521 { using type = int##_Bits##x##_Np##_t; }; \
2522template <> \
2523 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2524 _Np * _Bits / 8, void> \
2525 { using type = uint##_Bits##x##_Np##_t; }
2526_GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2527_GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2528_GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2529_GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2530_GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2531_GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2532_GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2533_GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2534#undef _GLIBCXX_SIMD_ARM_INTRIN
2535
2536template <typename _Tp, size_t _Bytes>
2537 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2538 {
2539 static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2540
2541 using _Ip = __int_for_sizeof_t<_Tp>;
2542
2543 using _Up = conditional_t<
2544 is_floating_point_v<_Tp>, _Tp,
2545 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2546
2547 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2548 "should use explicit specialization above");
2549
2550 using type = typename __intrinsic_type<_Up, _SVecBytes>::type;
2551 };
2552#endif // _GLIBCXX_SIMD_HAVE_NEON
2553
2554// }}}
2555// __intrinsic_type (PPC){{{
2556#ifdef __ALTIVEC__
2557template <typename _Tp>
2558 struct __intrinsic_type_impl;
2559
2560#define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2561 template <> \
2562 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2563_GLIBCXX_SIMD_PPC_INTRIN(float);
2564#ifdef __VSX__
2565_GLIBCXX_SIMD_PPC_INTRIN(double);
2566#endif
2567_GLIBCXX_SIMD_PPC_INTRIN(signed char);
2568_GLIBCXX_SIMD_PPC_INTRIN(unsigned char);
2569_GLIBCXX_SIMD_PPC_INTRIN(signed short);
2570_GLIBCXX_SIMD_PPC_INTRIN(unsigned short);
2571_GLIBCXX_SIMD_PPC_INTRIN(signed int);
2572_GLIBCXX_SIMD_PPC_INTRIN(unsigned int);
2573#if defined __VSX__ || __SIZEOF_LONG__ == 4
2574_GLIBCXX_SIMD_PPC_INTRIN(signed long);
2575_GLIBCXX_SIMD_PPC_INTRIN(unsigned long);
2576#endif
2577#ifdef __VSX__
2578_GLIBCXX_SIMD_PPC_INTRIN(signed long long);
2579_GLIBCXX_SIMD_PPC_INTRIN(unsigned long long);
2580#endif
2581#undef _GLIBCXX_SIMD_PPC_INTRIN
2582
2583template <typename _Tp, size_t _Bytes>
2584 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2585 {
2586 static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2587
2588 // allow _Tp == long double with -mlong-double-64
2589 static_assert(!(_S_is_ldouble && sizeof(long double) > sizeof(double)),
2590 "no __intrinsic_type support for 128-bit floating point on PowerPC");
2591
2592#ifndef __VSX__
2593 static_assert(!(is_same_v<_Tp, double>
2594 || (_S_is_ldouble && sizeof(long double) == sizeof(double))),
2595 "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX");
2596#endif
2597
2598 static constexpr auto __element_type()
2599 {
2600 if constexpr (is_floating_point_v<_Tp>)
2601 {
2602 if constexpr (_S_is_ldouble)
2603 return double {};
2604 else
2605 return _Tp {};
2606 }
2607 else if constexpr (is_signed_v<_Tp>)
2608 {
2609 if constexpr (sizeof(_Tp) == sizeof(_SChar))
2610 return _SChar {};
2611 else if constexpr (sizeof(_Tp) == sizeof(short))
2612 return short {};
2613 else if constexpr (sizeof(_Tp) == sizeof(int))
2614 return int {};
2615 else if constexpr (sizeof(_Tp) == sizeof(_LLong))
2616 return _LLong {};
2617 }
2618 else
2619 {
2620 if constexpr (sizeof(_Tp) == sizeof(_UChar))
2621 return _UChar {};
2622 else if constexpr (sizeof(_Tp) == sizeof(_UShort))
2623 return _UShort {};
2624 else if constexpr (sizeof(_Tp) == sizeof(_UInt))
2625 return _UInt {};
2626 else if constexpr (sizeof(_Tp) == sizeof(_ULLong))
2627 return _ULLong {};
2628 }
2629 }
2630
2631 using type = typename __intrinsic_type_impl<decltype(__element_type())>::type;
2632 };
2633#endif // __ALTIVEC__
2634
2635// }}}
2636// _SimdWrapper<bool>{{{1
2637template <size_t _Width>
2638 struct _SimdWrapper<bool, _Width,
2639 void_t<typename __bool_storage_member_type<_Width>::type>>
2640 {
2641 using _BuiltinType = typename __bool_storage_member_type<_Width>::type;
2642 using value_type = bool;
2643
2644 static constexpr size_t _S_full_size = sizeof(_BuiltinType) * __CHAR_BIT__;
2645
2646 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2647 __as_full_vector() const
2648 { return _M_data; }
2649
2650 _GLIBCXX_SIMD_INTRINSIC constexpr
2651 _SimdWrapper() = default;
2652
2653 _GLIBCXX_SIMD_INTRINSIC constexpr
2654 _SimdWrapper(_BuiltinType __k) : _M_data(__k) {};
2655
2656 _GLIBCXX_SIMD_INTRINSIC
2657 operator const _BuiltinType&() const
2658 { return _M_data; }
2659
2660 _GLIBCXX_SIMD_INTRINSIC
2661 operator _BuiltinType&()
2662 { return _M_data; }
2663
2664 _GLIBCXX_SIMD_INTRINSIC _BuiltinType
2665 __intrin() const
2666 { return _M_data; }
2667
2668 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2669 operator[](size_t __i) const
2670 { return _M_data & (_BuiltinType(1) << __i); }
2671
2672 template <size_t __i>
2673 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2674 operator[](_SizeConstant<__i>) const
2675 { return _M_data & (_BuiltinType(1) << __i); }
2676
2677 _GLIBCXX_SIMD_INTRINSIC constexpr void
2678 _M_set(size_t __i, value_type __x)
2679 {
2680 if (__x)
2681 _M_data |= (_BuiltinType(1) << __i);
2682 else
2683 _M_data &= ~(_BuiltinType(1) << __i);
2684 }
2685
2686 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2687 _M_is_constprop() const
2688 { return __builtin_constant_p(_M_data); }
2689
2690 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2691 _M_is_constprop_none_of() const
2692 {
2693 if (__builtin_constant_p(_M_data))
2694 {
2695 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2696 constexpr _BuiltinType __active_mask
2697 = ~_BuiltinType() >> (__nbits - _Width);
2698 return (_M_data & __active_mask) == 0;
2699 }
2700 return false;
2701 }
2702
2703 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2704 _M_is_constprop_all_of() const
2705 {
2706 if (__builtin_constant_p(_M_data))
2707 {
2708 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2709 constexpr _BuiltinType __active_mask
2710 = ~_BuiltinType() >> (__nbits - _Width);
2711 return (_M_data & __active_mask) == __active_mask;
2712 }
2713 return false;
2714 }
2715
2716 _BuiltinType _M_data;
2717 };
2718
2719// _SimdWrapperBase{{{1
2720template <bool _MustZeroInitPadding, typename _BuiltinType>
2721 struct _SimdWrapperBase;
2722
2723template <typename _BuiltinType>
2724 struct _SimdWrapperBase<false, _BuiltinType> // no padding or no SNaNs
2725 {
2726 _GLIBCXX_SIMD_INTRINSIC constexpr
2727 _SimdWrapperBase() = default;
2728
2729 _GLIBCXX_SIMD_INTRINSIC constexpr
2730 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2731
2732 _BuiltinType _M_data;
2733 };
2734
2735template <typename _BuiltinType>
2736 struct _SimdWrapperBase<true, _BuiltinType> // with padding that needs to
2737 // never become SNaN
2738 {
2739 _GLIBCXX_SIMD_INTRINSIC constexpr
2740 _SimdWrapperBase() : _M_data() {}
2741
2742 _GLIBCXX_SIMD_INTRINSIC constexpr
2743 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2744
2745 _BuiltinType _M_data;
2746 };
2747
2748// }}}
2749// _SimdWrapper{{{
2750struct _DisabledSimdWrapper;
2751
2752template <typename _Tp, size_t _Width>
2753 struct _SimdWrapper<
2754 _Tp, _Width,
2755 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2756 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2757 && sizeof(_Tp) * _Width
2758 == sizeof(__vector_type_t<_Tp, _Width>),
2759 __vector_type_t<_Tp, _Width>>
2760 {
2761 static constexpr bool _S_need_default_init
2762 = __has_iec559_behavior<__signaling_NaN, _Tp>::value
2763 and sizeof(_Tp) * _Width == sizeof(__vector_type_t<_Tp, _Width>);
2764
2765 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2766
2767 using _Base = _SimdWrapperBase<_S_need_default_init, _BuiltinType>;
2768
2769 static_assert(__is_vectorizable_v<_Tp>);
2770 static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then
2771
2772 using value_type = _Tp;
2773
2774 static inline constexpr size_t _S_full_size
2775 = sizeof(_BuiltinType) / sizeof(value_type);
2776 static inline constexpr int _S_size = _Width;
2777 static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2778
2779 using _Base::_M_data;
2780
2781 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2782 __as_full_vector() const
2783 { return _M_data; }
2784
2785 _GLIBCXX_SIMD_INTRINSIC constexpr
2786 _SimdWrapper(initializer_list<_Tp> __init)
2787 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2788 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2789 return __init.begin()[__i.value];
2790 })) {}
2791
2792 _GLIBCXX_SIMD_INTRINSIC constexpr
2793 _SimdWrapper() = default;
2794
2795 _GLIBCXX_SIMD_INTRINSIC constexpr
2796 _SimdWrapper(const _SimdWrapper&) = default;
2797
2798 _GLIBCXX_SIMD_INTRINSIC constexpr
2799 _SimdWrapper(_SimdWrapper&&) = default;
2800
2801 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2802 operator=(const _SimdWrapper&) = default;
2803
2804 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2805 operator=(_SimdWrapper&&) = default;
2806
2807 // Convert from exactly matching __vector_type_t
2808 using _SimdWrapperBase<_S_need_default_init, _BuiltinType>::_SimdWrapperBase;
2809
2810 // Convert from __intrinsic_type_t if __intrinsic_type_t and __vector_type_t differ, otherwise
2811 // this ctor should not exist. Making the argument type unusable is our next best solution.
2812 _GLIBCXX_SIMD_INTRINSIC constexpr
2813 _SimdWrapper(conditional_t<is_same_v<_BuiltinType, __intrinsic_type_t<_Tp, _Width>>,
2814 _DisabledSimdWrapper, __intrinsic_type_t<_Tp, _Width>> __x)
2815 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2816
2817 // Convert from different __vector_type_t, but only if bit reinterpretation is a correct
2818 // conversion of the value_type
2819 template <typename _V, typename _TVT = _VectorTraits<_V>,
2820 typename = enable_if_t<sizeof(typename _TVT::value_type) == sizeof(_Tp)
2821 and sizeof(_V) == sizeof(_BuiltinType)
2822 and is_integral_v<_Tp>
2823 and is_integral_v<typename _TVT::value_type>>>
2824 _GLIBCXX_SIMD_INTRINSIC constexpr
2825 _SimdWrapper(_V __x)
2826 : _Base(reinterpret_cast<_BuiltinType>(__x)) {}
2827
2828 template <typename... _As,
2829 typename = enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2830 && sizeof...(_As) <= _Width)>>
2831 _GLIBCXX_SIMD_INTRINSIC constexpr
2832 operator _SimdTuple<_Tp, _As...>() const
2833 {
2834 return __generate_from_n_evaluations<sizeof...(_As), _SimdTuple<_Tp, _As...>>(
2835 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2836 { return _M_data[int(__i)]; });
2837 }
2838
2839 _GLIBCXX_SIMD_INTRINSIC constexpr
2840 operator const _BuiltinType&() const
2841 { return _M_data; }
2842
2843 _GLIBCXX_SIMD_INTRINSIC constexpr
2844 operator _BuiltinType&()
2845 { return _M_data; }
2846
2847 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2848 operator[](size_t __i) const
2849 { return _M_data[__i]; }
2850
2851 template <size_t __i>
2852 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2853 operator[](_SizeConstant<__i>) const
2854 { return _M_data[__i]; }
2855
2856 _GLIBCXX_SIMD_INTRINSIC constexpr void
2857 _M_set(size_t __i, _Tp __x)
2858 {
2859 if (__builtin_is_constant_evaluated())
2860 _M_data = __generate_from_n_evaluations<_Width, _BuiltinType>([&](auto __j) {
2861 return __j == __i ? __x : _M_data[__j()];
2862 });
2863 else
2864 _M_data[__i] = __x;
2865 }
2866
2867 _GLIBCXX_SIMD_INTRINSIC
2868 constexpr bool
2869 _M_is_constprop() const
2870 { return __builtin_constant_p(_M_data); }
2871
2872 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2873 _M_is_constprop_none_of() const
2874 {
2875 if (__builtin_constant_p(_M_data))
2876 {
2877 bool __r = true;
2878 if constexpr (is_floating_point_v<_Tp>)
2879 {
2880 using _Ip = __int_for_sizeof_t<_Tp>;
2881 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2882 __execute_n_times<_Width>(
2883 [&](auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2884 }
2885 else
2886 __execute_n_times<_Width>(
2887 [&](auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2888 if (__builtin_constant_p(__r))
2889 return __r;
2890 }
2891 return false;
2892 }
2893
2894 _GLIBCXX_SIMD_INTRINSIC constexpr bool
2895 _M_is_constprop_all_of() const
2896 {
2897 if (__builtin_constant_p(_M_data))
2898 {
2899 bool __r = true;
2900 if constexpr (is_floating_point_v<_Tp>)
2901 {
2902 using _Ip = __int_for_sizeof_t<_Tp>;
2903 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2904 __execute_n_times<_Width>(
2905 [&](auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2906 }
2907 else
2908 __execute_n_times<_Width>(
2909 [&](auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2910 if (__builtin_constant_p(__r))
2911 return __r;
2912 }
2913 return false;
2914 }
2915 };
2916
2917// }}}
2918
2919// __vectorized_sizeof {{{
2920template <typename _Tp>
2921 constexpr size_t
2922 __vectorized_sizeof()
2923 {
2924 if constexpr (!__is_vectorizable_v<_Tp>)
2925 return 0;
2926
2927 if constexpr (sizeof(_Tp) <= 8)
2928 {
2929 // X86:
2930 if constexpr (__have_avx512bw)
2931 return 64;
2932 if constexpr (__have_avx512f && sizeof(_Tp) >= 4)
2933 return 64;
2934 if constexpr (__have_avx2)
2935 return 32;
2936 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2937 return 32;
2938 if constexpr (__have_sse2)
2939 return 16;
2940 if constexpr (__have_sse && is_same_v<_Tp, float>)
2941 return 16;
2942 /* The following is too much trouble because of mixed MMX and x87 code.
2943 * While nothing here explicitly calls MMX instructions of registers,
2944 * they are still emitted but no EMMS cleanup is done.
2945 if constexpr (__have_mmx && sizeof(_Tp) <= 4 && is_integral_v<_Tp>)
2946 return 8;
2947 */
2948
2949 // PowerPC:
2950 if constexpr (__have_power8vec
2951 || (__have_power_vmx && (sizeof(_Tp) < 8))
2952 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2953 return 16;
2954
2955 // ARM:
2956 if constexpr (__have_neon_a64)
2957 return 16;
2958 if constexpr (__have_neon_a32 and (not is_floating_point_v<_Tp>
2959 or is_same_v<_Tp, float>))
2960 return 16;
2961 if constexpr (__have_neon
2962 && sizeof(_Tp) < 8
2963 // Only allow fp if the user allows non-ICE559 fp (e.g.
2964 // via -ffast-math). ARMv7 NEON fp is not conforming to
2965 // IEC559.
2966 && (__support_neon_float || !is_floating_point_v<_Tp>))
2967 return 16;
2968 }
2969
2970 return sizeof(_Tp);
2971 }
2972
2973// }}}
2974namespace simd_abi {
2975// most of simd_abi is defined in simd_detail.h
2976template <typename _Tp>
2977 inline constexpr int max_fixed_size
2978 = ((__have_avx512bw && sizeof(_Tp) == 1)
2979 || (__have_sve && __sve_vectorized_size_bytes/sizeof(_Tp) >= 64)) ? 64 : 32;
2980
2981// compatible {{{
2982#if defined __x86_64__ || defined __aarch64__
2983template <typename _Tp>
2984 using compatible = conditional_t<(sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2985#elif defined __ARM_NEON
2986// FIXME: not sure, probably needs to be scalar (or dependent on the hard-float
2987// ABI?)
2988template <typename _Tp>
2989 using compatible
2990 = conditional_t<(sizeof(_Tp) < 8
2991 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2992 _VecBuiltin<16>, scalar>;
2993#else
2994template <typename>
2995 using compatible = scalar;
2996#endif
2997
2998// }}}
2999// native {{{
3000template <typename _Tp>
3001 constexpr auto
3002 __determine_native_abi()
3003 {
3004 constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
3005 if constexpr (__bytes == sizeof(_Tp))
3006 return static_cast<scalar*>(nullptr);
3007 else if constexpr (__have_sve)
3008 return static_cast<_SveAbi<__sve_vectorized_size_bytes>*>(nullptr);
3009 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
3010 return static_cast<_VecBltnBtmsk<__bytes>*>(nullptr);
3011 else
3012 return static_cast<_VecBuiltin<__bytes>*>(nullptr);
3013 }
3014
3015template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
3016 using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
3017
3018// }}}
3019// __default_abi {{{
3020#if defined _GLIBCXX_SIMD_DEFAULT_ABI
3021template <typename _Tp>
3022 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
3023#else
3024template <typename _Tp>
3025 using __default_abi = compatible<_Tp>;
3026#endif
3027
3028// }}}
3029} // namespace simd_abi
3030
3031// traits {{{1
3032template <typename _Tp>
3033 struct is_simd_flag_type
3034 : false_type
3035 {};
3036
3037template <>
3038 struct is_simd_flag_type<element_aligned_tag>
3039 : true_type
3040 {};
3041
3042template <>
3043 struct is_simd_flag_type<vector_aligned_tag>
3044 : true_type
3045 {};
3046
3047template <size_t _Np>
3048 struct is_simd_flag_type<overaligned_tag<_Np>>
3049 : __bool_constant<(_Np > 0) and __has_single_bit(_Np)>
3050 {};
3051
3052template <typename _Tp>
3053 inline constexpr bool is_simd_flag_type_v = is_simd_flag_type<_Tp>::value;
3054
3055template <typename _Tp, typename = enable_if_t<is_simd_flag_type_v<_Tp>>>
3056 using _IsSimdFlagType = _Tp;
3057
3058// is_abi_tag {{{2
3059template <typename _Tp, typename = void_t<>>
3060 struct is_abi_tag : false_type {};
3061
3062template <typename _Tp>
3063 struct is_abi_tag<_Tp, void_t<typename _Tp::_IsValidAbiTag>>
3064 : public _Tp::_IsValidAbiTag {};
3065
3066template <typename _Tp>
3067 inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
3068
3069// is_simd(_mask) {{{2
3070template <typename _Tp>
3071 struct is_simd : public false_type {};
3072
3073template <typename _Tp>
3074 inline constexpr bool is_simd_v = is_simd<_Tp>::value;
3075
3076template <typename _Tp>
3077 struct is_simd_mask : public false_type {};
3078
3079template <typename _Tp>
3080inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
3081
3082// simd_size {{{2
3083template <typename _Tp, typename _Abi, typename = void>
3084 struct __simd_size_impl {};
3085
3086template <typename _Tp, typename _Abi>
3087 struct __simd_size_impl<
3088 _Tp, _Abi,
3089 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
3090 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
3091
3092template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
3093 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
3094
3095template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
3096 inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
3097
3098// simd_abi::deduce {{{2
3099template <typename _Tp, size_t _Np, typename = void>
3100 struct __deduce_impl;
3101
3102template <typename _Tp, size_t _Np, typename = void>
3103 struct __no_sve_deduce_impl;
3104
3105namespace simd_abi {
3106/**
3107 * @tparam _Tp The requested `value_type` for the elements.
3108 * @tparam _Np The requested number of elements.
3109 * @tparam _Abis This parameter is ignored, since this implementation cannot
3110 * make any use of it. Either __a good native ABI is matched and used as `type`
3111 * alias, or the `fixed_size<_Np>` ABI is used, which internally is built from
3112 * the best matching native ABIs.
3113 */
3114 template <typename _Tp, size_t _Np, typename...>
3115 struct deduce : __deduce_impl<_Tp, _Np> {};
3116
3117template <typename _Tp, size_t _Np, typename... _Abis>
3118 using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type;
3119
3120template <typename _Tp, size_t _Np, typename...>
3121 struct __no_sve_deduce : __no_sve_deduce_impl<_Tp, _Np> {};
3122
3123template <typename _Tp, size_t _Np, typename... _Abis>
3124 using __no_sve_deduce_t = typename __no_sve_deduce<_Tp, _Np, _Abis...>::type;
3125} // namespace simd_abi
3126
3127// }}}2
3128// rebind_simd {{{2
3129template <typename _Tp, typename _V, typename = void>
3130 struct rebind_simd;
3131
3132template <typename _Tp, typename _Up, typename _Abi>
3133 struct rebind_simd<_Tp, simd<_Up, _Abi>,
3134 void_t<std::conditional_t<!__is_sve_abi<_Abi>(),
3135 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3136 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>>
3137 {
3138 using type = simd<_Tp, std::conditional_t<
3139 !__is_sve_abi<_Abi>(),
3140 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3141 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>;
3142 };
3143
3144template <typename _Tp, typename _Up, typename _Abi>
3145 struct rebind_simd<_Tp, simd_mask<_Up, _Abi>,
3146 void_t<std::conditional_t<!__is_sve_abi<_Abi>(),
3147 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3148 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>>
3149 {
3150 using type = simd_mask<_Tp, std::conditional_t<
3151 !__is_sve_abi<_Abi>(),
3152 simd_abi::__no_sve_deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>,
3153 simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>;
3154 };
3155
3156template <typename _Tp, typename _V>
3157 using rebind_simd_t = typename rebind_simd<_Tp, _V>::type;
3158
3159// resize_simd {{{2
3160template <int _Np, typename _V, typename = void>
3161 struct resize_simd;
3162
3163template <int _Np, typename _Tp, typename _Abi>
3164 struct resize_simd<_Np, simd<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
3165 { using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
3166
3167template <int _Np, typename _Tp, typename _Abi>
3168 struct resize_simd<_Np, simd_mask<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
3169 { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
3170
3171template <int _Np, typename _V>
3172 using resize_simd_t = typename resize_simd<_Np, _V>::type;
3173
3174// }}}2
3175// memory_alignment {{{2
3176template <typename _Tp, typename _Up = typename _Tp::value_type>
3177 struct memory_alignment
3178 : public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
3179
3180template <typename _Tp, typename _Up = typename _Tp::value_type>
3181 inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
3182
3183// class template simd [simd] {{{1
3184template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
3185 class simd;
3186
3187template <typename _Tp, typename _Abi>
3188 struct is_simd<simd<_Tp, _Abi>> : public true_type {};
3189
3190template <typename _Tp>
3191 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
3192
3193template <typename _Tp, int _Np>
3194 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
3195
3196template <typename _Tp, size_t _Np>
3197 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
3198
3199// class template simd_mask [simd_mask] {{{1
3200template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
3201 class simd_mask;
3202
3203template <typename _Tp, typename _Abi>
3204 struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {};
3205
3206template <typename _Tp>
3207 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
3208
3209template <typename _Tp, int _Np>
3210 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
3211
3212template <typename _Tp, size_t _Np>
3213 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
3214
3215// casts [simd.casts] {{{1
3216// static_simd_cast {{{2
3217template <typename _Tp, typename _Up, typename _Ap, bool = is_simd_v<_Tp>, typename = void>
3218 struct __static_simd_cast_return_type;
3219
3220template <typename _Tp, typename _A0, typename _Up, typename _Ap>
3221 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false, void>
3222 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
3223
3224template <typename _Tp, typename _Up, typename _Ap>
3225 struct __static_simd_cast_return_type<
3226 _Tp, _Up, _Ap, true, enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
3227 { using type = _Tp; };
3228
3229template <typename _Tp, typename _Ap>
3230 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
3231#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
3232 enable_if_t<__is_vectorizable_v<_Tp>>
3233#else
3234 void
3235#endif
3236 >
3237 { using type = simd<_Tp, _Ap>; };
3238
3239template <typename _Tp, typename = void>
3240 struct __safe_make_signed { using type = _Tp;};
3241
3242template <typename _Tp>
3243 struct __safe_make_signed<_Tp, enable_if_t<is_integral_v<_Tp>>>
3244 {
3245 // the extra make_unsigned_t is because of PR85951
3246 using type = make_signed_t<make_unsigned_t<_Tp>>;
3247 };
3248
3249template <typename _Tp>
3250 using safe_make_signed_t = typename __safe_make_signed<_Tp>::type;
3251
3252template <typename _Tp, typename _Up, typename _Ap>
3253 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
3254#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
3255 enable_if_t<__is_vectorizable_v<_Tp>>
3256#else
3257 void
3258#endif
3259 >
3260 {
3261 using type = conditional_t<
3262 (is_integral_v<_Up> && is_integral_v<_Tp> &&
3263#ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
3264 is_signed_v<_Up> != is_signed_v<_Tp> &&
3265#endif
3266 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
3267 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
3268 };
3269
3270template <typename _Tp, typename _Up, typename _Ap,
3271 typename _R
3272 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3273 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
3274 static_simd_cast(const simd<_Up, _Ap>& __x)
3275 {
3276 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
3277 return __x;
3278 else
3279 {
3280 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
3281 __c;
3282 return _R(__private_init, __c(__data(__x)));
3283 }
3284 }
3285
3286namespace __proposed {
3287template <typename _Tp, typename _Up, typename _Ap,
3288 typename _R
3289 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3290 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR typename _R::mask_type
3291 static_simd_cast(const simd_mask<_Up, _Ap>& __x)
3292 {
3293 using _RM = typename _R::mask_type;
3294 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
3295 typename _RM::simd_type::value_type>(__x)};
3296 }
3297
3298template <typename _To, typename _Up, typename _Abi>
3299 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3300 _To
3301 simd_bit_cast(const simd<_Up, _Abi>& __x)
3302 {
3303 using _Tp = typename _To::value_type;
3304 using _ToMember = typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3305 using _From = simd<_Up, _Abi>;
3306 using _FromMember = typename _SimdTraits<_Up, _Abi>::_SimdMember;
3307 // with concepts, the following should be constraints
3308 static_assert(sizeof(_To) == sizeof(_From));
3309 static_assert(is_trivially_copyable_v<_Tp> && is_trivially_copyable_v<_Up>);
3310 static_assert(is_trivially_copyable_v<_ToMember> && is_trivially_copyable_v<_FromMember>);
3311#if __has_builtin(__builtin_bit_cast)
3312 return {__private_init, __builtin_bit_cast(_ToMember, __data(__x))};
3313#else
3314 return {__private_init, __bit_cast<_ToMember>(__data(__x))};
3315#endif
3316 }
3317
3318template <typename _To, typename _Up, typename _Abi>
3319 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3320 _To
3321 simd_bit_cast(const simd_mask<_Up, _Abi>& __x)
3322 {
3323 using _From = simd_mask<_Up, _Abi>;
3324 static_assert(sizeof(_To) == sizeof(_From));
3325 static_assert(is_trivially_copyable_v<_From>);
3326 // _To can be simd<T, A>, specifically simd<T, fixed_size<N>> in which case _To is not trivially
3327 // copyable.
3328 if constexpr (is_simd_v<_To>)
3329 {
3330 using _Tp = typename _To::value_type;
3331 using _ToMember = typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3332 static_assert(is_trivially_copyable_v<_ToMember>);
3333#if __has_builtin(__builtin_bit_cast)
3334 return {__private_init, __builtin_bit_cast(_ToMember, __x)};
3335#else
3336 return {__private_init, __bit_cast<_ToMember>(__x)};
3337#endif
3338 }
3339 else
3340 {
3341 static_assert(is_trivially_copyable_v<_To>);
3342#if __has_builtin(__builtin_bit_cast)
3343 return __builtin_bit_cast(_To, __x);
3344#else
3345 return __bit_cast<_To>(__x);
3346#endif
3347 }
3348 }
3349} // namespace __proposed
3350
3351// simd_cast {{{2
3352template <typename _Tp, typename _Up, typename _Ap,
3353 typename _To = __value_type_or_identity_t<_Tp>>
3354 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3355 simd_cast(const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
3356 -> decltype(static_simd_cast<_Tp>(__x))
3357 { return static_simd_cast<_Tp>(__x); }
3358
3359namespace __proposed {
3360template <typename _Tp, typename _Up, typename _Ap,
3361 typename _To = __value_type_or_identity_t<_Tp>>
3362 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3363 simd_cast(const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
3364 -> decltype(static_simd_cast<_Tp>(__x))
3365 { return static_simd_cast<_Tp>(__x); }
3366} // namespace __proposed
3367
3368// }}}2
3369// resizing_simd_cast {{{
3370namespace __proposed {
3371/* Proposed spec:
3372
3373template <class T, class U, class Abi>
3374T resizing_simd_cast(const simd<U, Abi>& x)
3375
3376p1 Constraints:
3377 - is_simd_v<T> is true and
3378 - T::value_type is the same type as U
3379
3380p2 Returns:
3381 A simd object with the i^th element initialized to x[i] for all i in the
3382 range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3383 than simd_size_v<U, Abi>, the remaining elements are value-initialized.
3384
3385template <class T, class U, class Abi>
3386T resizing_simd_cast(const simd_mask<U, Abi>& x)
3387
3388p1 Constraints: is_simd_mask_v<T> is true
3389
3390p2 Returns:
3391 A simd_mask object with the i^th element initialized to x[i] for all i in
3392the range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3393 than simd_size_v<U, Abi>, the remaining elements are initialized to false.
3394
3395 */
3396
3397template <typename _Tp, typename _Up, typename _Ap>
3398 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR enable_if_t<
3399 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3400 resizing_simd_cast(const simd<_Up, _Ap>& __x)
3401 {
3402 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3403 return __x;
3404 else if (__builtin_is_constant_evaluated())
3405 return _Tp([&](auto __i) constexpr {
3406 return __i < simd_size_v<_Up, _Ap> ? __x[__i] : _Up();
3407 });
3408 else if constexpr (simd_size_v<_Up, _Ap> == 1)
3409 {
3410 _Tp __r{};
3411 __r[0] = __x[0];
3412 return __r;
3413 }
3414 else if constexpr (_Tp::size() == 1)
3415 return __x[0];
3416 else if constexpr (sizeof(_Tp) == sizeof(__x)
3417 && !__is_fixed_size_abi_v<_Ap> && !__is_sve_abi<_Ap>())
3418 return {__private_init,
3419 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3420 _Ap::_S_masked(__data(__x))._M_data)};
3421 else
3422 {
3423 _Tp __r{};
3424 __builtin_memcpy(&__data(__r), &__data(__x),
3425 sizeof(_Up)
3426 * std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3427 return __r;
3428 }
3429 }
3430
3431template <typename _Tp, typename _Up, typename _Ap>
3432 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3433 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3434 resizing_simd_cast(const simd_mask<_Up, _Ap>& __x)
3435 {
3436 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3437 typename _Tp::simd_type::value_type>(__x)};
3438 }
3439} // namespace __proposed
3440
3441// }}}
3442// to_fixed_size {{{2
3443template <typename _Tp, int _Np>
3444 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3445 to_fixed_size(const fixed_size_simd<_Tp, _Np>& __x)
3446 { return __x; }
3447
3448template <typename _Tp, int _Np>
3449 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3450 to_fixed_size(const fixed_size_simd_mask<_Tp, _Np>& __x)
3451 { return __x; }
3452
3453template <typename _Tp, typename _Ap>
3454 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>
3455 to_fixed_size(const simd<_Tp, _Ap>& __x)
3456 {
3457 using _Rp = fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>;
3458 return _Rp([&__x](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3459 }
3460
3461template <typename _Tp, typename _Ap>
3462 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, simd_size_v<_Tp, _Ap>>
3463 to_fixed_size(const simd_mask<_Tp, _Ap>& __x)
3464 {
3465 return {__private_init,
3466 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }};
3467 }
3468
3469// to_native {{{2
3470template <typename _Tp, int _Np>
3471 _GLIBCXX_SIMD_INTRINSIC
3472 enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3473 to_native(const fixed_size_simd<_Tp, _Np>& __x)
3474 {
3475 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3476 __x.copy_to(__mem, vector_aligned);
3477 return {__mem, vector_aligned};
3478 }
3479
3480template <typename _Tp, int _Np>
3481 _GLIBCXX_SIMD_INTRINSIC
3482 enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3483 to_native(const fixed_size_simd_mask<_Tp, _Np>& __x)
3484 {
3485 return native_simd_mask<_Tp>(
3486 __private_init,
3487 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3488 }
3489
3490// to_compatible {{{2
3491template <typename _Tp, int _Np>
3492 _GLIBCXX_SIMD_INTRINSIC enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3493 to_compatible(const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3494 {
3495 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3496 __x.copy_to(__mem, vector_aligned);
3497 return {__mem, vector_aligned};
3498 }
3499
3500template <typename _Tp, int _Np>
3501 _GLIBCXX_SIMD_INTRINSIC
3502 enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3503 to_compatible(const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3504 {
3505 return simd_mask<_Tp>(
3506 __private_init,
3507 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3508 }
3509
3510// masked assignment [simd_mask.where] {{{1
3511
3512// where_expression {{{1
3513// const_where_expression<M, T> {{{2
3514template <typename _M, typename _Tp>
3515 class const_where_expression
3516 {
3517 using _V = _Tp;
3518 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3519
3520 struct _Wrapper { using value_type = _V; };
3521
3522 protected:
3523 using _Impl = typename _V::_Impl;
3524
3525 using value_type =
3526 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3527
3528 _GLIBCXX_SIMD_INTRINSIC friend const _M&
3529 __get_mask(const const_where_expression& __x)
3530 { return __x._M_k; }
3531
3532 _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3533 __get_lvalue(const const_where_expression& __x)
3534 { return __x._M_value; }
3535
3536 const _M& _M_k;
3537 _Tp& _M_value;
3538
3539 public:
3540 const_where_expression(const const_where_expression&) = delete;
3541
3542 const_where_expression& operator=(const const_where_expression&) = delete;
3543
3544 _GLIBCXX_SIMD_INTRINSIC constexpr
3545 const_where_expression(const _M& __kk, const _Tp& dd)
3546 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3547
3548 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3549 operator-() const&&
3550 {
3551 return {__private_init,
3552 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3553 __data(_M_value))};
3554 }
3555
3556 template <typename _Up, typename _Flags>
3557 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3558 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3559 {
3560 return {__private_init,
3561 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3562 _Flags::template _S_apply<_V>(__mem))};
3563 }
3564
3565 template <typename _Up, typename _Flags>
3566 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3567 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3568 {
3569 _Impl::_S_masked_store(__data(_M_value),
3570 _Flags::template _S_apply<_V>(__mem),
3571 __data(_M_k));
3572 }
3573 };
3574
3575// const_where_expression<bool, T> {{{2
3576template <typename _Tp>
3577 class const_where_expression<bool, _Tp>
3578 {
3579 using _M = bool;
3580 using _V = _Tp;
3581
3582 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3583
3584 struct _Wrapper { using value_type = _V; };
3585
3586 protected:
3587 using value_type
3588 = typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3589
3590 _GLIBCXX_SIMD_INTRINSIC friend const _M&
3591 __get_mask(const const_where_expression& __x)
3592 { return __x._M_k; }
3593
3594 _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3595 __get_lvalue(const const_where_expression& __x)
3596 { return __x._M_value; }
3597
3598 const bool _M_k;
3599 _Tp& _M_value;
3600
3601 public:
3602 const_where_expression(const const_where_expression&) = delete;
3603 const_where_expression& operator=(const const_where_expression&) = delete;
3604
3605 _GLIBCXX_SIMD_INTRINSIC constexpr
3606 const_where_expression(const bool __kk, const _Tp& dd)
3607 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3608
3609 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3610 operator-() const&&
3611 { return _M_k ? -_M_value : _M_value; }
3612
3613 template <typename _Up, typename _Flags>
3614 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V
3615 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3616 { return _M_k ? static_cast<_V>(__mem[0]) : _M_value; }
3617
3618 template <typename _Up, typename _Flags>
3619 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3620 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&&
3621 {
3622 if (_M_k)
3623 __mem[0] = _M_value;
3624 }
3625 };
3626
3627// where_expression<M, T> {{{2
3628template <typename _M, typename _Tp>
3629 class where_expression : public const_where_expression<_M, _Tp>
3630 {
3631 using _Impl = typename const_where_expression<_M, _Tp>::_Impl;
3632
3633 static_assert(!is_const<_Tp>::value,
3634 "where_expression may only be instantiated with __a non-const "
3635 "_Tp parameter");
3636
3637 using typename const_where_expression<_M, _Tp>::value_type;
3638 using const_where_expression<_M, _Tp>::_M_k;
3639 using const_where_expression<_M, _Tp>::_M_value;
3640
3641 static_assert(
3642 is_same<typename _M::abi_type, typename _Tp::abi_type>::value, "");
3643 static_assert(_M::size() == _Tp::size(), "");
3644
3645 _GLIBCXX_SIMD_INTRINSIC friend constexpr _Tp&
3646 __get_lvalue(where_expression& __x)
3647 { return __x._M_value; }
3648
3649 public:
3650 where_expression(const where_expression&) = delete;
3651 where_expression& operator=(const where_expression&) = delete;
3652
3653 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3654 where_expression(const _M& __kk, _Tp& dd)
3655 : const_where_expression<_M, _Tp>(__kk, dd) {}
3656
3657 template <typename _Up>
3658 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3659 operator=(_Up&& __x) &&
3660 {
3661 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3662 __to_value_type_or_member_type<_Tp>(
3663 static_cast<_Up&&>(__x)));
3664 }
3665
3666#define _GLIBCXX_SIMD_OP_(__op, __name) \
3667 template <typename _Up> \
3668 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3669 operator __op##=(_Up&& __x)&& \
3670 { \
3671 _Impl::template _S_masked_cassign( \
3672 __data(_M_k), __data(_M_value), \
3673 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3674 [](auto __impl, auto __lhs, auto __rhs) \
3675 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
3676 { return __impl.__name(__lhs, __rhs); }); \
3677 } \
3678 static_assert(true)
3679 _GLIBCXX_SIMD_OP_(+, _S_plus);
3680 _GLIBCXX_SIMD_OP_(-, _S_minus);
3681 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3682 _GLIBCXX_SIMD_OP_(/, _S_divides);
3683 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3684 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3685 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3686 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3687 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3688 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3689#undef _GLIBCXX_SIMD_OP_
3690
3691 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3692 operator++() &&
3693 {
3694 __data(_M_value)
3695 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3696 }
3697
3698 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3699 operator++(int) &&
3700 {
3701 __data(_M_value)
3702 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3703 }
3704
3705 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3706 operator--() &&
3707 {
3708 __data(_M_value)
3709 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3710 }
3711
3712 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3713 operator--(int) &&
3714 {
3715 __data(_M_value)
3716 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3717 }
3718
3719 // intentionally hides const_where_expression::copy_from
3720 template <typename _Up, typename _Flags>
3721 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3722 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3723 {
3724 __data(_M_value) = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3725 _Flags::template _S_apply<_Tp>(__mem));
3726 }
3727 };
3728
3729// where_expression<bool, T> {{{2
3730template <typename _Tp>
3731 class where_expression<bool, _Tp>
3732 : public const_where_expression<bool, _Tp>
3733 {
3734 using _M = bool;
3735 using typename const_where_expression<_M, _Tp>::value_type;
3736 using const_where_expression<_M, _Tp>::_M_k;
3737 using const_where_expression<_M, _Tp>::_M_value;
3738
3739 public:
3740 where_expression(const where_expression&) = delete;
3741 where_expression& operator=(const where_expression&) = delete;
3742
3743 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3744 where_expression(const _M& __kk, _Tp& dd)
3745 : const_where_expression<_M, _Tp>(__kk, dd) {}
3746
3747#define _GLIBCXX_SIMD_OP_(__op) \
3748 template <typename _Up> \
3749 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \
3750 operator __op(_Up&& __x)&& \
3751 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3752
3753 _GLIBCXX_SIMD_OP_(=)
3754 _GLIBCXX_SIMD_OP_(+=)
3755 _GLIBCXX_SIMD_OP_(-=)
3756 _GLIBCXX_SIMD_OP_(*=)
3757 _GLIBCXX_SIMD_OP_(/=)
3758 _GLIBCXX_SIMD_OP_(%=)
3759 _GLIBCXX_SIMD_OP_(&=)
3760 _GLIBCXX_SIMD_OP_(|=)
3761 _GLIBCXX_SIMD_OP_(^=)
3762 _GLIBCXX_SIMD_OP_(<<=)
3763 _GLIBCXX_SIMD_OP_(>>=)
3764 #undef _GLIBCXX_SIMD_OP_
3765
3766 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3767 operator++() &&
3768 { if (_M_k) ++_M_value; }
3769
3770 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3771 operator++(int) &&
3772 { if (_M_k) ++_M_value; }
3773
3774 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3775 operator--() &&
3776 { if (_M_k) --_M_value; }
3777
3778 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3779 operator--(int) &&
3780 { if (_M_k) --_M_value; }
3781
3782 // intentionally hides const_where_expression::copy_from
3783 template <typename _Up, typename _Flags>
3784 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void
3785 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) &&
3786 { if (_M_k) _M_value = __mem[0]; }
3787 };
3788
3789// where {{{1
3790template <typename _Tp, typename _Ap>
3791 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3792 where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3793 where(const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3794 { return {__k, __value}; }
3795
3796template <typename _Tp, typename _Ap>
3797 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3798 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3799 where(const typename simd<_Tp, _Ap>::mask_type& __k, const simd<_Tp, _Ap>& __value)
3800 { return {__k, __value}; }
3801
3802template <typename _Tp, typename _Ap>
3803 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3804 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3805 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, simd_mask<_Tp, _Ap>& __value)
3806 { return {__k, __value}; }
3807
3808template <typename _Tp, typename _Ap>
3809 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3810 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3811 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, const simd_mask<_Tp, _Ap>& __value)
3812 { return {__k, __value}; }
3813
3814template <typename _Tp>
3815 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR where_expression<bool, _Tp>
3816 where(_ExactBool __k, _Tp& __value)
3817 { return {__k, __value}; }
3818
3819template <typename _Tp>
3820 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR const_where_expression<bool, _Tp>
3821 where(_ExactBool __k, const _Tp& __value)
3822 { return {__k, __value}; }
3823
3824template <typename _Tp, typename _Ap>
3825 _GLIBCXX_SIMD_CONSTEXPR void
3826 where(bool __k, simd<_Tp, _Ap>& __value) = delete;
3827
3828template <typename _Tp, typename _Ap>
3829 _GLIBCXX_SIMD_CONSTEXPR void
3830 where(bool __k, const simd<_Tp, _Ap>& __value) = delete;
3831
3832// proposed mask iterations {{{1
3833namespace __proposed {
3834template <size_t _Np>
3835 class where_range
3836 {
3837 const bitset<_Np> __bits;
3838
3839 public:
3840 where_range(bitset<_Np> __b) : __bits(__b) {}
3841
3842 class iterator
3843 {
3844 size_t __mask;
3845 size_t __bit;
3846
3847 _GLIBCXX_SIMD_INTRINSIC void
3848 __next_bit()
3849 { __bit = __builtin_ctzl(__mask); }
3850
3851 _GLIBCXX_SIMD_INTRINSIC void
3852 __reset_lsb()
3853 {
3854 // 01100100 - 1 = 01100011
3855 __mask &= (__mask - 1);
3856 // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit));
3857 }
3858
3859 public:
3860 iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3861 iterator(const iterator&) = default;
3862 iterator(iterator&&) = default;
3863
3864 _GLIBCXX_SIMD_ALWAYS_INLINE size_t
3865 operator->() const
3866 { return __bit; }
3867
3868 _GLIBCXX_SIMD_ALWAYS_INLINE size_t
3869 operator*() const
3870 { return __bit; }
3871
3872 _GLIBCXX_SIMD_ALWAYS_INLINE iterator&
3873 operator++()
3874 {
3875 __reset_lsb();
3876 __next_bit();
3877 return *this;
3878 }
3879
3880 _GLIBCXX_SIMD_ALWAYS_INLINE iterator
3881 operator++(int)
3882 {
3883 iterator __tmp = *this;
3884 __reset_lsb();
3885 __next_bit();
3886 return __tmp;
3887 }
3888
3889 _GLIBCXX_SIMD_ALWAYS_INLINE bool
3890 operator==(const iterator& __rhs) const
3891 { return __mask == __rhs.__mask; }
3892
3893 _GLIBCXX_SIMD_ALWAYS_INLINE bool
3894 operator!=(const iterator& __rhs) const
3895 { return __mask != __rhs.__mask; }
3896 };
3897
3898 iterator
3899 begin() const
3900 { return __bits.to_ullong(); }
3901
3902 iterator
3903 end() const
3904 { return 0; }
3905 };
3906
3907template <typename _Tp, typename _Ap>
3908 where_range<simd_size_v<_Tp, _Ap>>
3909 where(const simd_mask<_Tp, _Ap>& __k)
3910 { return __k.__to_bitset(); }
3911
3912} // namespace __proposed
3913
3914// }}}1
3915// reductions [simd.reductions] {{{1
3916template <typename _Tp, typename _Abi, typename _BinaryOperation = plus<>>
3917 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3918 reduce(const simd<_Tp, _Abi>& __v, _BinaryOperation __binary_op = _BinaryOperation())
3919 { return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3920
3921template <typename _M, typename _V, typename _BinaryOperation = plus<>>
3922 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3923 reduce(const const_where_expression<_M, _V>& __x,
3924 typename _V::value_type __identity_element, _BinaryOperation __binary_op)
3925 {
3926 if (__builtin_expect(none_of(__get_mask(__x)), false))
3927 return __identity_element;
3928
3929 _V __tmp = __identity_element;
3930 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3931 __data(__get_lvalue(__x)));
3932 return reduce(__tmp, __binary_op);
3933 }
3934
3935template <typename _M, typename _V>
3936 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3937 reduce(const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3938 { return reduce(__x, 0, __binary_op); }
3939
3940template <typename _M, typename _V>
3941 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3942 reduce(const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3943 { return reduce(__x, 1, __binary_op); }
3944
3945template <typename _M, typename _V>
3946 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3947 reduce(const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3948 { return reduce(__x, ~typename _V::value_type(), __binary_op); }
3949
3950template <typename _M, typename _V>
3951 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3952 reduce(const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3953 { return reduce(__x, 0, __binary_op); }
3954
3955template <typename _M, typename _V>
3956 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3957 reduce(const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3958 { return reduce(__x, 0, __binary_op); }
3959
3960template <typename _Tp, typename _Abi>
3961 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3962 hmin(const simd<_Tp, _Abi>& __v) noexcept
3963 { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum()); }
3964
3965template <typename _Tp, typename _Abi>
3966 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3967 hmax(const simd<_Tp, _Abi>& __v) noexcept
3968 { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum()); }
3969
3970template <typename _M, typename _V>
3971 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3972 typename _V::value_type
3973 hmin(const const_where_expression<_M, _V>& __x) noexcept
3974 {
3975 using _Tp = typename _V::value_type;
3976 constexpr _Tp __id_elem =
3977#ifdef __FINITE_MATH_ONLY__
3978 __finite_max_v<_Tp>;
3979#else
3980 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3981#endif
3982 _V __tmp = __id_elem;
3983 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3984 __data(__get_lvalue(__x)));
3985 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3986 }
3987
3988template <typename _M, typename _V>
3989 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3990 typename _V::value_type
3991 hmax(const const_where_expression<_M, _V>& __x) noexcept
3992 {
3993 using _Tp = typename _V::value_type;
3994 constexpr _Tp __id_elem =
3995#ifdef __FINITE_MATH_ONLY__
3996 __finite_min_v<_Tp>;
3997#else
3998 [] {
3999 if constexpr (__value_exists_v<__infinity, _Tp>)
4000 return -__infinity_v<_Tp>;
4001 else
4002 return __finite_min_v<_Tp>;
4003 }();
4004#endif
4005 _V __tmp = __id_elem;
4006 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
4007 __data(__get_lvalue(__x)));
4008 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
4009 }
4010
4011// }}}1
4012// algorithms [simd.alg] {{{
4013template <typename _Tp, typename _Ap>
4014 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
4015 min(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
4016 { return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
4017
4018template <typename _Tp, typename _Ap>
4019 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
4020 max(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
4021 { return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
4022
4023template <typename _Tp, typename _Ap>
4024 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4025 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
4026 minmax(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
4027 {
4028 const auto pair_of_members
4029 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
4030 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
4031 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
4032 }
4033
4034template <typename _Tp, typename _Ap>
4035 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
4036 clamp(const simd<_Tp, _Ap>& __v, const simd<_Tp, _Ap>& __lo, const simd<_Tp, _Ap>& __hi)
4037 {
4038 using _Impl = typename _Ap::_SimdImpl;
4039 return {__private_init,
4040 _Impl::_S_min(__data(__hi),
4041 _Impl::_S_max(__data(__lo), __data(__v)))};
4042 }
4043
4044// }}}
4045
4046template <size_t... _Sizes, typename _Tp, typename _Ap,
4047 typename = enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
4048 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
4049 split(const simd<_Tp, _Ap>&);
4050
4051// __extract_part {{{
4052template <int _Index, int _Total, int _Combine = 1, typename _Tp, size_t _Np>
4053 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr
4054 conditional_t<_Np == _Total and _Combine == 1, _Tp, _SimdWrapper<_Tp, _Np / _Total * _Combine>>
4055 __extract_part(const _SimdWrapper<_Tp, _Np> __x);
4056
4057template <int _Index, int _Parts, int _Combine = 1, typename _Tp, typename _A0, typename... _As>
4058 _GLIBCXX_SIMD_INTRINSIC constexpr auto
4059 __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x);
4060
4061// }}}
4062// _SizeList {{{
4063template <size_t _V0, size_t... _Values>
4064 struct _SizeList
4065 {
4066 template <size_t _I>
4067 static constexpr size_t
4068 _S_at(_SizeConstant<_I> = {})
4069 {
4070 if constexpr (_I == 0)
4071 return _V0;
4072 else
4073 return _SizeList<_Values...>::template _S_at<_I - 1>();
4074 }
4075
4076 template <size_t _I>
4077 static constexpr auto
4078 _S_before(_SizeConstant<_I> = {})
4079 {
4080 if constexpr (_I == 0)
4081 return _SizeConstant<0>();
4082 else
4083 return _SizeConstant<
4084 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
4085 }
4086
4087 template <size_t _Np>
4088 static constexpr auto
4089 _S_pop_front(_SizeConstant<_Np> = {})
4090 {
4091 if constexpr (_Np == 0)
4092 return _SizeList();
4093 else
4094 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
4095 }
4096 };
4097
4098// }}}
4099// __extract_center {{{
4100template <typename _Tp, size_t _Np>
4101 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
4102 __extract_center(_SimdWrapper<_Tp, _Np> __x)
4103 {
4104 static_assert(_Np >= 4);
4105 static_assert(_Np % 4 == 0); // x0 - x1 - x2 - x3 -> return {x1, x2}
4106#if _GLIBCXX_SIMD_X86INTRIN // {{{
4107 if constexpr (__have_avx512f && sizeof(_Tp) * _Np == 64)
4108 {
4109 const auto __intrin = __to_intrin(__x);
4110 if constexpr (is_integral_v<_Tp>)
4111 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
4112 _mm512_shuffle_i32x4(__intrin, __intrin,
4113 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4114 else if constexpr (sizeof(_Tp) == 4)
4115 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
4116 _mm512_shuffle_f32x4(__intrin, __intrin,
4117 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4118 else if constexpr (sizeof(_Tp) == 8)
4119 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
4120 _mm512_shuffle_f64x2(__intrin, __intrin,
4121 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
4122 else
4123 __assert_unreachable<_Tp>();
4124 }
4125 else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
4126 return __vector_bitcast<_Tp>(
4127 _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
4128 __hi128(__vector_bitcast<double>(__x)), 1));
4129 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
4130 return __vector_bitcast<_Tp>(
4131 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
4132 __lo128(__vector_bitcast<_LLong>(__x)),
4133 sizeof(_Tp) * _Np / 4));
4134 else
4135#endif // _GLIBCXX_SIMD_X86INTRIN }}}
4136 {
4137 __vector_type_t<_Tp, _Np / 2> __r;
4138 __builtin_memcpy(&__r,
4139 reinterpret_cast<const char*>(&__x)
4140 + sizeof(_Tp) * _Np / 4,
4141 sizeof(_Tp) * _Np / 2);
4142 return __r;
4143 }
4144 }
4145
4146template <typename _Tp, typename _A0, typename... _As>
4147 _GLIBCXX_SIMD_INTRINSIC
4148 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
4149 __extract_center(const _SimdTuple<_Tp, _A0, _As...>& __x)
4150 {
4151 if constexpr (sizeof...(_As) == 0)
4152 return __extract_center(__x.first);
4153 else
4154 return __extract_part<1, 4, 2>(__x);
4155 }
4156
4157// }}}
4158// __split_wrapper {{{
4159template <size_t... _Sizes, typename _Tp, typename... _As>
4160 auto
4161 __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
4162 {
4163 return split<_Sizes...>(
4164 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
4165 __x));
4166 }
4167
4168// }}}
4169
4170// split<simd>(simd) {{{
4171template <typename _V, typename _Ap,
4172 size_t _Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
4173 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == _Parts * _V::size()
4174 && is_simd_v<_V>, array<_V, _Parts>>
4175 split(const simd<typename _V::value_type, _Ap>& __x)
4176 {
4177 using _Tp = typename _V::value_type;
4178
4179 auto __gen_fallback = [&]() constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4180 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4181 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4182 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4183 { return __x[__i * _V::size() + __j]; });
4184 });
4185 };
4186
4187 if constexpr (_Parts == 1)
4188 {
4189 return {simd_cast<_V>(__x)};
4190 }
4191 else if (__x._M_is_constprop())
4192 {
4193 return __gen_fallback();
4194 }
4195#if _GLIBCXX_SIMD_HAVE_SVE
4196 else if constexpr(__is_sve_abi<_Ap>)
4197 {
4198 return __gen_fallback();
4199 }
4200#endif
4201 else if constexpr (
4202 __is_fixed_size_abi_v<_Ap>
4203 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
4204 || (__is_fixed_size_abi_v<typename _V::abi_type>
4205 && sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding
4206 )))
4207 {
4208 // fixed_size -> fixed_size (w/o padding) or scalar
4209#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4210 const __may_alias<_Tp>* const __element_ptr
4211 = reinterpret_cast<const __may_alias<_Tp>*>(&__data(__x));
4212 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4213 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4214 { return _V(__element_ptr + __i * _V::size(), vector_aligned); });
4215#else
4216 const auto& __xx = __data(__x);
4217 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4218 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4219 [[maybe_unused]] constexpr size_t __offset
4220 = decltype(__i)::value * _V::size();
4221 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4222 constexpr _SizeConstant<__j + __offset> __k;
4223 return __xx[__k];
4224 });
4225 });
4226#endif
4227 }
4228 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
4229 {
4230 // normally memcpy should work here as well
4231 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4232 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
4233 }
4234 else
4235 {
4236 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4237 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4238 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
4239 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4240 return __x[__i * _V::size() + __j];
4241 });
4242 else
4243 return _V(__private_init,
4244 __extract_part<decltype(__i)::value, _Parts>(__data(__x)));
4245 });
4246 }
4247 }
4248
4249// }}}
4250// split<simd_mask>(simd_mask) {{{
4251template <typename _V, typename _Ap,
4252 size_t _Parts = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
4253 enable_if_t<is_simd_mask_v<_V> && simd_size_v<typename
4254 _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
4255 split(const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
4256 {
4257 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
4258 return {__x};
4259 else if constexpr (_Parts == 1)
4260 return {__proposed::static_simd_cast<_V>(__x)};
4261 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
4262 && __is_avx_abi<_Ap>())
4263 return {_V(__private_init, __lo128(__data(__x))),
4264 _V(__private_init, __hi128(__data(__x)))};
4265 else if constexpr (_V::size() <= __CHAR_BIT__ * sizeof(_ULLong))
4266 {
4267 const bitset __bits = __x.__to_bitset();
4268 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4269 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4270 constexpr size_t __offset = __i * _V::size();
4271 return _V(__bitset_init, (__bits >> __offset).to_ullong());
4272 });
4273 }
4274 else
4275 {
4276 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4277 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4278 constexpr size_t __offset = __i * _V::size();
4279 return _V(__private_init,
4280 [&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4281 return __x[__j + __offset];
4282 });
4283 });
4284 }
4285 }
4286
4287// }}}
4288// split<_Sizes...>(simd) {{{
4289template <size_t... _Sizes, typename _Tp, typename _Ap, typename>
4290 _GLIBCXX_SIMD_ALWAYS_INLINE
4291 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
4292 split(const simd<_Tp, _Ap>& __x)
4293 {
4294 using _SL = _SizeList<_Sizes...>;
4295 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
4296 constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
4297 constexpr size_t _N0 = _SL::template _S_at<0>();
4298 using _V = __deduced_simd<_Tp, _N0>;
4299
4300 auto __gen_fallback = [&]() constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4301 {
4302 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4303 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4304 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4305 constexpr size_t __offset = _SL::_S_before(__i);
4306 return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4307 return __x[__offset + __j];
4308 });
4309 });
4310 };
4311
4312 if (__x._M_is_constprop())
4313 __gen_fallback();
4314#if _GLIBCXX_SIMD_HAVE_SVE
4315 else if constexpr (__have_sve)
4316 __gen_fallback();
4317#endif
4318 else if constexpr (_Np == _N0)
4319 {
4320 static_assert(sizeof...(_Sizes) == 1);
4321 return {simd_cast<_V>(__x)};
4322 }
4323 else if constexpr // split from fixed_size, such that __x::first.size == _N0
4324 (__is_fixed_size_abi_v<
4325 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
4326 {
4327 static_assert(
4328 !__is_fixed_size_abi_v<typename _V::abi_type>,
4329 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
4330 "fixed_size_simd "
4331 "when deduced?");
4332 // extract first and recurse (__split_wrapper is needed to deduce a new
4333 // _Sizes pack)
4334 return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
4335 __split_wrapper(_SL::template _S_pop_front<1>(),
4336 __data(__x).second));
4337 }
4338 else if constexpr ((!__is_fixed_size_abi_v<simd_abi::deduce_t<_Tp, _Sizes>> && ...))
4339 {
4340 constexpr array<size_t, sizeof...(_Sizes)> __size = {_Sizes...};
4341 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4342 [&](auto __i) constexpr {
4343 constexpr size_t __offset = [&]() {
4344 size_t __r = 0;
4345 for (unsigned __j = 0; __j < __i; ++__j)
4346 __r += __size[__j];
4347 return __r;
4348 }();
4349 return __deduced_simd<_Tp, __size[__i]>(
4350 __private_init,
4351 __extract_part<__offset, _Np, __size[__i]>(__data(__x)));
4352 });
4353 }
4354#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4355 const __may_alias<_Tp>* const __element_ptr
4356 = reinterpret_cast<const __may_alias<_Tp>*>(&__x);
4357 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4358 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4359 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4360 constexpr size_t __offset = _SL::_S_before(__i);
4361 constexpr size_t __base_align = alignof(simd<_Tp, _Ap>);
4362 constexpr size_t __a
4363 = __base_align - ((__offset * sizeof(_Tp)) % __base_align);
4364 constexpr size_t __b = ((__a - 1) & __a) ^ __a;
4365 constexpr size_t __alignment = __b == 0 ? __a : __b;
4366 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
4367 });
4368#else
4369 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4370 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4371 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4372 const auto& __xx = __data(__x);
4373 using _Offset = decltype(_SL::_S_before(__i));
4374 return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4375 constexpr _SizeConstant<_Offset::value + __j> __k;
4376 return __xx[__k];
4377 });
4378 });
4379#endif
4380 }
4381
4382// }}}
4383
4384// __subscript_in_pack {{{
4385template <size_t _I, typename _Tp, typename _Ap, typename... _As>
4386 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
4387 __subscript_in_pack(const simd<_Tp, _Ap>& __x, const simd<_Tp, _As>&... __xs)
4388 {
4389 if constexpr (_I < simd_size_v<_Tp, _Ap>)
4390 return __x[_I];
4391 else
4392 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
4393 }
4394
4395// }}}
4396// __store_pack_of_simd {{{
4397template <typename _Tp, typename _A0, typename... _As>
4398 _GLIBCXX_SIMD_INTRINSIC void
4399 __store_pack_of_simd(char* __mem, const simd<_Tp, _A0>& __x0, const simd<_Tp, _As>&... __xs)
4400 {
4401 constexpr size_t __n_bytes = sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4402 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4403 if constexpr (sizeof...(__xs) > 0)
4404 __store_pack_of_simd(__mem + __n_bytes, __xs...);
4405 }
4406
4407// }}}
4408// concat(simd...) {{{
4409template <typename _Tp, typename... _As, typename = __detail::__odr_helper>
4410 inline _GLIBCXX_SIMD_CONSTEXPR
4411 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4412 concat(const simd<_Tp, _As>&... __xs)
4413 {
4414 constexpr int _Np = (simd_size_v<_Tp, _As> + ...);
4415 using _Abi = simd_abi::deduce_t<_Tp, _Np>;
4416 using _Rp = simd<_Tp, _Abi>;
4417 using _RW = typename _SimdTraits<_Tp, _Abi>::_SimdMember;
4418 if constexpr (sizeof...(__xs) == 1)
4419 return simd_cast<_Rp>(__xs...);
4420 else if ((... && __xs._M_is_constprop()))
4421 return _Rp([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4422 { return __subscript_in_pack<__i>(__xs...); });
4423 else if constexpr (__is_simd_wrapper_v<_RW> and sizeof...(__xs) == 2)
4424 {
4425 return {__private_init,
4426 __vec_shuffle(__as_vector(__xs)..., std::make_index_sequence<_RW::_S_full_size>(),
4427 [](int __i) {
4428 constexpr int __sizes[2] = {int(simd_size_v<_Tp, _As>)...};
4429 constexpr int __vsizes[2]
4430 = {int(sizeof(__as_vector(__xs)) / sizeof(_Tp))...};
4431 constexpr int __padding0 = __vsizes[0] - __sizes[0];
4432 return __i >= _Np ? -1 : __i < __sizes[0] ? __i : __i + __padding0;
4433 })};
4434 }
4435 else if constexpr (__is_simd_wrapper_v<_RW> and sizeof...(__xs) == 3)
4436 return [](const auto& __x0, const auto& __x1, const auto& __x2)
4437 _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4438 return concat(concat(__x0, __x1), __x2);
4439 }(__xs...);
4440 else if constexpr (__is_simd_wrapper_v<_RW> and sizeof...(__xs) > 3)
4441 return [](const auto& __x0, const auto& __x1, const auto&... __rest)
4442 _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4443 return concat(concat(__x0, __x1), concat(__rest...));
4444 }(__xs...);
4445 else
4446 {
4447 _Rp __r{};
4448 __store_pack_of_simd(reinterpret_cast<char*>(&__data(__r)), __xs...);
4449 return __r;
4450 }
4451 }
4452
4453// }}}
4454// concat(array<simd>) {{{
4455template <typename _Tp, typename _Abi, size_t _Np>
4456 _GLIBCXX_SIMD_ALWAYS_INLINE
4457 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4458 concat(const array<simd<_Tp, _Abi>, _Np>& __x)
4459 {
4460 return __call_with_subscripts<_Np>(
4461 __x, [](const auto&... __xs) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4462 return concat(__xs...);
4463 });
4464 }
4465
4466// }}}
4467
4468/// @cond undocumented
4469// _SmartReference {{{
4470template <typename _Up, typename _Accessor = _Up,
4471 typename _ValueType = typename _Up::value_type>
4472 class _SmartReference
4473 {
4474 friend _Accessor;
4475 int _M_index;
4476 _Up& _M_obj;
4477
4478 _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType
4479 _M_read() const noexcept
4480 {
4481 if constexpr (is_arithmetic_v<_Up>)
4482 return _M_obj;
4483 else
4484 return _M_obj[_M_index];
4485 }
4486
4487 template <typename _Tp>
4488 _GLIBCXX_SIMD_INTRINSIC constexpr void
4489 _M_write(_Tp&& __x) const
4490 { _Accessor::_S_set(_M_obj, _M_index, static_cast<_Tp&&>(__x)); }
4491
4492 public:
4493 _GLIBCXX_SIMD_INTRINSIC constexpr
4494 _SmartReference(_Up& __o, int __i) noexcept
4495 : _M_index(__i), _M_obj(__o) {}
4496
4497 using value_type = _ValueType;
4498
4499 _GLIBCXX_SIMD_INTRINSIC
4500 _SmartReference(const _SmartReference&) = delete;
4501
4502 _GLIBCXX_SIMD_INTRINSIC constexpr
4503 operator value_type() const noexcept
4504 { return _M_read(); }
4505
4506 template <typename _Tp, typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4507 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4508 operator=(_Tp&& __x) &&
4509 {
4510 _M_write(static_cast<_Tp&&>(__x));
4511 return {_M_obj, _M_index};
4512 }
4513
4514#define _GLIBCXX_SIMD_OP_(__op) \
4515 template <typename _Tp, \
4516 typename _TT = decltype(declval<value_type>() __op declval<_Tp>()), \
4517 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4518 typename = _ValuePreservingOrInt<_TT, value_type>> \
4519 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4520 operator __op##=(_Tp&& __x) && \
4521 { \
4522 const value_type& __lhs = _M_read(); \
4523 _M_write(__lhs __op __x); \
4524 return {_M_obj, _M_index}; \
4525 }
4526 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4527 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4528 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4529#undef _GLIBCXX_SIMD_OP_
4530
4531 template <typename _Tp = void,
4532 typename = decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4533 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4534 operator++() &&
4535 {
4536 value_type __x = _M_read();
4537 _M_write(++__x);
4538 return {_M_obj, _M_index};
4539 }
4540
4541 template <typename _Tp = void,
4542 typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4543 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
4544 operator++(int) &&
4545 {
4546 const value_type __r = _M_read();
4547 value_type __x = __r;
4548 _M_write(++__x);
4549 return __r;
4550 }
4551
4552 template <typename _Tp = void,
4553 typename = decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4554 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference
4555 operator--() &&
4556 {
4557 value_type __x = _M_read();
4558 _M_write(--__x);
4559 return {_M_obj, _M_index};
4560 }
4561
4562 template <typename _Tp = void,
4563 typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4564 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
4565 operator--(int) &&
4566 {
4567 const value_type __r = _M_read();
4568 value_type __x = __r;
4569 _M_write(--__x);
4570 return __r;
4571 }
4572
4573 _GLIBCXX_SIMD_INTRINSIC friend void
4574 swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4575 conjunction<
4576 is_nothrow_constructible<value_type, _SmartReference&&>,
4577 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4578 {
4579 value_type __tmp = static_cast<_SmartReference&&>(__a);
4580 static_cast<_SmartReference&&>(__a) = static_cast<value_type>(__b);
4581 static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4582 }
4583
4584 _GLIBCXX_SIMD_INTRINSIC friend void
4585 swap(value_type& __a, _SmartReference&& __b) noexcept(
4586 conjunction<
4587 is_nothrow_constructible<value_type, value_type&&>,
4588 is_nothrow_assignable<value_type&, value_type&&>,
4589 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4590 {
4591 value_type __tmp(std::move(__a));
4592 __a = static_cast<value_type>(__b);
4593 static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4594 }
4595
4596 _GLIBCXX_SIMD_INTRINSIC friend void
4597 swap(_SmartReference&& __a, value_type& __b) noexcept(
4598 conjunction<
4599 is_nothrow_constructible<value_type, _SmartReference&&>,
4600 is_nothrow_assignable<value_type&, value_type&&>,
4601 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4602 {
4603 value_type __tmp(__a);
4604 static_cast<_SmartReference&&>(__a) = std::move(__b);
4605 __b = std::move(__tmp);
4606 }
4607 };
4608
4609// }}}
4610// __scalar_abi_wrapper {{{
4611template <int _Bytes>
4612 struct __scalar_abi_wrapper
4613 {
4614 template <typename _Tp> static constexpr size_t _S_full_size = 1;
4615 template <typename _Tp> static constexpr size_t _S_size = 1;
4616 template <typename _Tp> static constexpr size_t _S_is_partial = false;
4617
4618 template <typename _Tp, typename _Abi = simd_abi::scalar>
4619 static constexpr bool _S_is_valid_v
4620 = _Abi::template _IsValid<_Tp>::value && sizeof(_Tp) == _Bytes;
4621 };
4622
4623// }}}
4624// __decay_abi metafunction {{{
4625template <typename _Tp>
4626 struct __decay_abi { using type = _Tp; };
4627
4628template <int _Bytes>
4629 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4630 { using type = simd_abi::scalar; };
4631
4632// }}}
4633// __find_next_valid_abi metafunction {{{1
4634// Given an ABI tag A<N>, find an N2 < N such that A<N2>::_S_is_valid_v<_Tp> ==
4635// true, N2 is a power-of-2, and A<N2>::_S_is_partial<_Tp> is false. Break
4636// recursion at 2 elements in the resulting ABI tag. In this case
4637// type::_S_is_valid_v<_Tp> may be false.
4638template <template <int> class _Abi, int _Bytes, typename _Tp>
4639 struct __find_next_valid_abi
4640 {
4641 static constexpr auto
4642 _S_choose()
4643 {
4644 constexpr int _NextBytes = std::__bit_ceil((unsigned)_Bytes) / 2;
4645 using _NextAbi = _Abi<_NextBytes>;
4646 if constexpr (_NextBytes < sizeof(_Tp) * 2) // break recursion
4647 return _Abi<_Bytes>();
4648 else if constexpr (_NextAbi::template _S_is_partial<_Tp> == false
4649 && _NextAbi::template _S_is_valid_v<_Tp>)
4650 return _NextAbi();
4651 else
4652 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4653 }
4654
4655 using type = decltype(_S_choose());
4656 };
4657
4658template <int _Bytes, typename _Tp>
4659 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4660 { using type = simd_abi::scalar; };
4661
4662// _AbiList {{{1
4663template <template <int> class...>
4664 struct _AbiList
4665 {
4666 template <typename, int> static constexpr bool _S_has_valid_abi = false;
4667 template <typename, int> using _FirstValidAbi = void;
4668 template <typename, int> using _BestAbi = void;
4669 };
4670
4671template <template <int> class _A0, template <int> class... _Rest>
4672 struct _AbiList<_A0, _Rest...>
4673 {
4674 template <typename _Tp, int _Np>
4675 static constexpr bool _S_has_valid_abi
4676 = _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<
4677 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4678
4679 template <typename _Tp, int _Np>
4680 using _FirstValidAbi = conditional_t<
4681 _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4682 typename __decay_abi<_A0<sizeof(_Tp) * _Np>>::type,
4683 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4684
4685 template <typename _Tp, int _Np>
4686 static constexpr auto
4687 _S_determine_best_abi()
4688 {
4689 static_assert(_Np >= 1);
4690 constexpr int _Bytes = sizeof(_Tp) * _Np;
4691 if constexpr (_Np == 1)
4692 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4693 else
4694 {
4695 constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4696 // _A0<_Bytes> is good if:
4697 // 1. The ABI tag is valid for _Tp
4698 // 2. The storage overhead is no more than padding to fill the next
4699 // power-of-2 number of bytes
4700 if constexpr (_A0<_Bytes>::template _S_is_valid_v<_Tp>
4701 && ((__is_sve_abi<_A0<_Bytes>>() && __have_sve
4702 && (_Np <= __sve_vectorized_size_bytes/sizeof(_Tp)))
4703 || (__fullsize / 2 < _Np))
4704 )
4705 return typename __decay_abi<_A0<_Bytes>>::type{};
4706 else
4707 {
4708 using _Bp =
4709 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4710 if constexpr (_Bp::template _S_is_valid_v<
4711 _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4712 return _Bp{};
4713 else
4714 return
4715 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4716 }
4717 }
4718 }
4719
4720 template <typename _Tp, int _Np>
4721 using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4722 };
4723
4724// }}}1
4725
4726// the following lists all native ABIs, which makes them accessible to
4727// simd_abi::deduce and select_best_vector_type_t (for fixed_size). Order
4728// matters: Whatever comes first has higher priority.
4729using _AllNativeAbis = _AbiList<
4730#if _GLIBCXX_SIMD_HAVE_SVE
4731 simd_abi::_SveAbi,
4732#endif
4733 simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin, __scalar_abi_wrapper>;
4734
4735using _NoSveAllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4736 __scalar_abi_wrapper>;
4737
4738// valid _SimdTraits specialization {{{1
4739template <typename _Tp, typename _Abi>
4740 struct _SimdTraits<_Tp, _Abi, void_t<typename _Abi::template _IsValid<_Tp>>>
4741 : _Abi::template __traits<_Tp> {};
4742
4743// __deduce_impl specializations {{{1
4744// try all native ABIs (including scalar) first
4745template <typename _Tp, size_t _Np>
4746 struct __deduce_impl<
4747 _Tp, _Np, enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4748 { using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4749
4750template <typename _Tp, size_t _Np>
4751 struct __no_sve_deduce_impl<
4752 _Tp, _Np, enable_if_t<_NoSveAllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4753 { using type = _NoSveAllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4754
4755// fall back to fixed_size only if scalar and native ABIs don't match
4756template <typename _Tp, size_t _Np, typename = void>
4757 struct __deduce_fixed_size_fallback {};
4758
4759template <typename _Tp, size_t _Np>
4760 struct __deduce_fixed_size_fallback<_Tp, _Np,
4761 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4762 { using type = simd_abi::fixed_size<_Np>; };
4763
4764template <typename _Tp, size_t _Np, typename>
4765 struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _Np> {};
4766
4767template <typename _Tp, size_t _Np, typename>
4768 struct __no_sve_deduce_impl
4769 : public __deduce_fixed_size_fallback<_Tp, _Np>
4770 {};
4771
4772
4773//}}}1
4774/// @endcond
4775
4776// simd_mask {{{
4777template <typename _Tp, typename _Abi>
4778 class simd_mask : public _SimdTraits<_Tp, _Abi>::_MaskBase
4779 {
4780 // types, tags, and friends {{{
4781 using _Traits = _SimdTraits<_Tp, _Abi>;
4782 using _MemberType = typename _Traits::_MaskMember;
4783
4784 // We map all masks with equal element sizeof to a single integer type, the
4785 // one given by __int_for_sizeof_t<_Tp>. This is the approach
4786 // [[gnu::vector_size(N)]] types take as well and it reduces the number of
4787 // template specializations in the implementation classes.
4788 using _Ip = __int_for_sizeof_t<_Tp>;
4789 static constexpr _Ip* _S_type_tag = nullptr;
4790
4791 friend typename _Traits::_MaskBase;
4792 friend class simd<_Tp, _Abi>; // to construct masks on return
4793 friend typename _Traits::_SimdImpl; // to construct masks on return and
4794 // inspect data on masked operations
4795 public:
4796 using _Impl = typename _Traits::_MaskImpl;
4797 friend _Impl;
4798
4799 // }}}
4800 // member types {{{
4801 using value_type = bool;
4802 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4803 using simd_type = simd<_Tp, _Abi>;
4804 using abi_type = _Abi;
4805
4806 // }}}
4807 static constexpr size_t size() // {{{
4808 { return __size_or_zero_v<_Tp, _Abi>; }
4809
4810 // }}}
4811 // constructors & assignment {{{
4812 simd_mask() = default;
4813 simd_mask(const simd_mask&) = default;
4814 simd_mask(simd_mask&&) = default;
4815 simd_mask& operator=(const simd_mask&) = default;
4816 simd_mask& operator=(simd_mask&&) = default;
4817
4818 // }}}
4819 // access to internal representation (optional feature) {{{
4820 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR explicit
4821 simd_mask(typename _Traits::_MaskCastType __init)
4822 : _M_data{__init} {}
4823 // conversions to internal type is done in _MaskBase
4824
4825 // }}}
4826 // bitset interface (extension to be proposed) {{{
4827 // TS_FEEDBACK:
4828 // Conversion of simd_mask to and from bitset makes it much easier to
4829 // interface with other facilities. I suggest adding `static
4830 // simd_mask::from_bitset` and `simd_mask::to_bitset`.
4831 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR static simd_mask
4832 __from_bitset(bitset<size()> bs)
4833 { return {__bitset_init, bs}; }
4834
4835 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bitset<size()>
4836 __to_bitset() const
4837 { return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4838
4839 // }}}
4840 // explicit broadcast constructor {{{
4841 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4842 simd_mask(value_type __x)
4843 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4844
4845 // }}}
4846 // implicit type conversion constructor {{{
4847 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4848 // proposed improvement
4849 template <typename _Up, typename _A2,
4850 typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4851 _GLIBCXX_SIMD_ALWAYS_INLINE explicit(sizeof(_MemberType)
4852 != sizeof(typename _SimdTraits<_Up, _A2>::_MaskMember))
4853 simd_mask(const simd_mask<_Up, _A2>& __x)
4854 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4855 #else
4856 // conforming to ISO/IEC 19570:2018
4857 template <typename _Up, typename = enable_if_t<conjunction<
4858 is_same<abi_type, simd_abi::fixed_size<size()>>,
4859 is_same<_Up, _Up>>::value>>
4860 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4861 simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4862 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4863 #endif
4864
4865 // }}}
4866 // load constructor {{{
4867 template <typename _Flags>
4868 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4869 simd_mask(const value_type* __mem, _IsSimdFlagType<_Flags>)
4870 : _M_data(_Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem))) {}
4871
4872 template <typename _Flags>
4873 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
4874 simd_mask(const value_type* __mem, simd_mask __k, _IsSimdFlagType<_Flags>)
4875 : _M_data{}
4876 {
4877 _M_data = _Impl::_S_masked_load(_M_data, __k._M_data,
4878 _Flags::template _S_apply<simd_mask>(__mem));
4879 }
4880
4881 // }}}
4882 // loads [simd_mask.load] {{{
4883 template <typename _Flags>
4884 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
4885 copy_from(const value_type* __mem, _IsSimdFlagType<_Flags>)
4886 { _M_data = _Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem)); }
4887
4888 // }}}
4889 // stores [simd_mask.store] {{{
4890 template <typename _Flags>
4891 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
4892 copy_to(value_type* __mem, _IsSimdFlagType<_Flags>) const
4893 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4894
4895 // }}}
4896 // scalar access {{{
4897 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
4898 operator[](size_t __i)
4899 {
4900 if (__i >= size())
4901 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4902 return {_M_data, int(__i)};
4903 }
4904
4905 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
4906 operator[](size_t __i) const
4907 {
4908 if (__i >= size())
4909 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4910 if constexpr (__is_scalar_abi<_Abi>())
4911 return _M_data;
4912 else
4913 return static_cast<bool>(_M_data[__i]);
4914 }
4915
4916 // }}}
4917 // negation {{{
4918 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd_mask
4919 operator!() const
4920 { return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4921
4922 // }}}
4923 // simd_mask binary operators [simd_mask.binary] {{{
4924 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4925 // simd_mask<int> && simd_mask<uint> needs disambiguation
4926 template <typename _Up, typename _A2,
4927 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4928 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4929 operator&&(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4930 {
4931 return {__private_init,
4932 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4933 }
4934
4935 template <typename _Up, typename _A2,
4936 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4937 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4938 operator||(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4939 {
4940 return {__private_init,
4941 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4942 }
4943 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4944
4945 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4946 operator&&(const simd_mask& __x, const simd_mask& __y)
4947 { return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)}; }
4948
4949 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4950 operator||(const simd_mask& __x, const simd_mask& __y)
4951 { return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)}; }
4952
4953 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4954 operator&(const simd_mask& __x, const simd_mask& __y)
4955 { return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4956
4957 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4958 operator|(const simd_mask& __x, const simd_mask& __y)
4959 { return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4960
4961 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4962 operator^(const simd_mask& __x, const simd_mask& __y)
4963 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4964
4965 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4966 operator&=(simd_mask& __x, const simd_mask& __y)
4967 {
4968 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4969 return __x;
4970 }
4971
4972 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4973 operator|=(simd_mask& __x, const simd_mask& __y)
4974 {
4975 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4976 return __x;
4977 }
4978
4979 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask&
4980 operator^=(simd_mask& __x, const simd_mask& __y)
4981 {
4982 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4983 return __x;
4984 }
4985
4986 // }}}
4987 // simd_mask compares [simd_mask.comparison] {{{
4988 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4989 operator==(const simd_mask& __x, const simd_mask& __y)
4990 { return !operator!=(__x, __y); }
4991
4992 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4993 operator!=(const simd_mask& __x, const simd_mask& __y)
4994 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4995
4996 // }}}
4997 // private_init ctor {{{
4998 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4999 simd_mask(_PrivateInit, typename _Traits::_MaskMember __init)
5000 : _M_data(__init) {}
5001
5002 // }}}
5003 // private_init generator ctor {{{
5004 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
5005 _GLIBCXX_SIMD_INTRINSIC constexpr
5006 simd_mask(_PrivateInit, _Fp&& __gen)
5007 : _M_data()
5008 {
5009 __execute_n_times<size()>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5010 _Impl::_S_set(_M_data, __i, __gen(__i));
5011 });
5012 }
5013
5014 // }}}
5015 // bitset_init ctor {{{
5016 _GLIBCXX_SIMD_INTRINSIC constexpr
5017 simd_mask(_BitsetInit, bitset<size()> __init)
5018 : _M_data(_Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
5019 {}
5020
5021 // }}}
5022 // __cvt {{{
5023 // TS_FEEDBACK:
5024 // The conversion operator this implements should be a ctor on simd_mask.
5025 // Once you call .__cvt() on a simd_mask it converts conveniently.
5026 // A useful variation: add `explicit(sizeof(_Tp) != sizeof(_Up))`
5027 struct _CvtProxy
5028 {
5029 template <typename _Up, typename _A2,
5030 typename = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
5031 _GLIBCXX_SIMD_ALWAYS_INLINE
5032 operator simd_mask<_Up, _A2>() &&
5033 {
5034 using namespace std::experimental::__proposed;
5035 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
5036 }
5037
5038 const simd_mask<_Tp, _Abi>& _M_data;
5039 };
5040
5041 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
5042 __cvt() const
5043 { return {*this}; }
5044
5045 // }}}
5046 // operator?: overloads (suggested extension) {{{
5047 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5048 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
5049 operator?:(const simd_mask& __k, const simd_mask& __where_true,
5050 const simd_mask& __where_false)
5051 {
5052 auto __ret = __where_false;
5053 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
5054 return __ret;
5055 }
5056
5057 template <typename _U1, typename _U2,
5058 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
5059 typename = enable_if_t<conjunction_v<
5060 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
5061 is_convertible<simd_mask, typename _Rp::mask_type>>>>
5062 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend _Rp
5063 operator?:(const simd_mask& __k, const _U1& __where_true,
5064 const _U2& __where_false)
5065 {
5066 _Rp __ret = __where_false;
5067 _Rp::_Impl::_S_masked_assign(
5068 __data(static_cast<typename _Rp::mask_type>(__k)), __data(__ret),
5069 __data(static_cast<_Rp>(__where_true)));
5070 return __ret;
5071 }
5072
5073 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
5074 template <typename _Kp, typename _Ak, typename _Up, typename _Au,
5075 typename = enable_if_t<
5076 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
5077 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
5078 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
5079 operator?:(const simd_mask<_Kp, _Ak>& __k, const simd_mask& __where_true,
5080 const simd_mask<_Up, _Au>& __where_false)
5081 {
5082 simd_mask __ret = __where_false;
5083 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
5084 __where_true._M_data);
5085 return __ret;
5086 }
5087 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
5088 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5089
5090 // }}}
5091 // _M_is_constprop {{{
5092 _GLIBCXX_SIMD_INTRINSIC constexpr bool
5093 _M_is_constprop() const
5094 {
5095 if constexpr (__is_scalar_abi<_Abi>())
5096 return __builtin_constant_p(_M_data);
5097 else
5098 return _M_data._M_is_constprop();
5099 }
5100
5101 // }}}
5102
5103 private:
5104 friend const auto& __data<_Tp, abi_type>(const simd_mask&);
5105 friend auto& __data<_Tp, abi_type>(simd_mask&);
5106 alignas(_Traits::_S_mask_align) _MemberType _M_data;
5107 };
5108
5109// }}}
5110
5111/// @cond undocumented
5112// __data(simd_mask) {{{
5113template <typename _Tp, typename _Ap>
5114 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5115 __data(const simd_mask<_Tp, _Ap>& __x)
5116 { return __x._M_data; }
5117
5118template <typename _Tp, typename _Ap>
5119 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5120 __data(simd_mask<_Tp, _Ap>& __x)
5121 { return __x._M_data; }
5122
5123// }}}
5124/// @endcond
5125
5126// simd_mask reductions [simd_mask.reductions] {{{
5127template <typename _Tp, typename _Abi>
5128 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5129 all_of(const simd_mask<_Tp, _Abi>& __k) noexcept
5130 {
5131 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5132 {
5133 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5134 if (!__k[__i])
5135 return false;
5136 return true;
5137 }
5138 else
5139 return _Abi::_MaskImpl::_S_all_of(__k);
5140 }
5141
5142template <typename _Tp, typename _Abi>
5143 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5144 any_of(const simd_mask<_Tp, _Abi>& __k) noexcept
5145 {
5146 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5147 {
5148 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5149 if (__k[__i])
5150 return true;
5151 return false;
5152 }
5153 else
5154 return _Abi::_MaskImpl::_S_any_of(__k);
5155 }
5156
5157template <typename _Tp, typename _Abi>
5158 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5159 none_of(const simd_mask<_Tp, _Abi>& __k) noexcept
5160 {
5161 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5162 {
5163 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
5164 if (__k[__i])
5165 return false;
5166 return true;
5167 }
5168 else
5169 return _Abi::_MaskImpl::_S_none_of(__k);
5170 }
5171
5172template <typename _Tp, typename _Abi>
5173 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5174 some_of(const simd_mask<_Tp, _Abi>& __k) noexcept
5175 {
5176 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5177 {
5178 for (size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
5179 if (__k[__i] != __k[__i - 1])
5180 return true;
5181 return false;
5182 }
5183 else
5184 return _Abi::_MaskImpl::_S_some_of(__k);
5185 }
5186
5187template <typename _Tp, typename _Abi>
5188 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5189 popcount(const simd_mask<_Tp, _Abi>& __k) noexcept
5190 {
5191 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5192 {
5193 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
5194 __k, [](auto... __elements) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5195 return ((__elements != 0) + ...);
5196 });
5197 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
5198 return __r;
5199 }
5200 return _Abi::_MaskImpl::_S_popcount(__k);
5201 }
5202
5203template <typename _Tp, typename _Abi>
5204 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5205 find_first_set(const simd_mask<_Tp, _Abi>& __k)
5206 {
5207 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5208 {
5209 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5210 const size_t _Idx = __call_with_n_evaluations<_Np>(
5211 [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5212 return std::min({__indexes...});
5213 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5214 return __k[__i] ? +__i : _Np;
5215 });
5216 if (_Idx >= _Np)
5217 __invoke_ub("find_first_set(empty mask) is UB");
5218 if (__builtin_constant_p(_Idx))
5219 return _Idx;
5220 }
5221 return _Abi::_MaskImpl::_S_find_first_set(__k);
5222 }
5223
5224template <typename _Tp, typename _Abi>
5225 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5226 find_last_set(const simd_mask<_Tp, _Abi>& __k)
5227 {
5228 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
5229 {
5230 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
5231 const int _Idx = __call_with_n_evaluations<_Np>(
5232 [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5233 return std::max({__indexes...});
5234 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
5235 return __k[__i] ? int(__i) : -1;
5236 });
5237 if (_Idx < 0)
5238 __invoke_ub("find_first_set(empty mask) is UB");
5239 if (__builtin_constant_p(_Idx))
5240 return _Idx;
5241 }
5242 return _Abi::_MaskImpl::_S_find_last_set(__k);
5243 }
5244
5245_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5246all_of(_ExactBool __x) noexcept
5247{ return __x; }
5248
5249_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5250any_of(_ExactBool __x) noexcept
5251{ return __x; }
5252
5253_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5254none_of(_ExactBool __x) noexcept
5255{ return !__x; }
5256
5257_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
5258some_of(_ExactBool) noexcept
5259{ return false; }
5260
5261_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5262popcount(_ExactBool __x) noexcept
5263{ return __x; }
5264
5265_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5266find_first_set(_ExactBool)
5267{ return 0; }
5268
5269_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
5270find_last_set(_ExactBool)
5271{ return 0; }
5272
5273// }}}
5274
5275/// @cond undocumented
5276// _SimdIntOperators{{{1
5277template <typename _V, typename _Tp, typename _Abi, bool>
5278 class _SimdIntOperators {};
5279
5280template <typename _V, typename _Tp, typename _Abi>
5281 class _SimdIntOperators<_V, _Tp, _Abi, true>
5282 {
5283 using _Impl = typename _SimdTraits<_Tp, _Abi>::_SimdImpl;
5284
5285 _GLIBCXX_SIMD_INTRINSIC constexpr const _V&
5286 __derived() const
5287 { return *static_cast<const _V*>(this); }
5288
5289 template <typename _Up>
5290 _GLIBCXX_SIMD_INTRINSIC static _GLIBCXX_SIMD_CONSTEXPR _V
5291 _S_make_derived(_Up&& __d)
5292 { return {__private_init, static_cast<_Up&&>(__d)}; }
5293
5294 public:
5295 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5296 _V&
5297 operator%=(_V& __lhs, const _V& __x)
5298 { return __lhs = __lhs % __x; }
5299
5300 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5301 _V&
5302 operator&=(_V& __lhs, const _V& __x)
5303 { return __lhs = __lhs & __x; }
5304
5305 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5306 _V&
5307 operator|=(_V& __lhs, const _V& __x)
5308 { return __lhs = __lhs | __x; }
5309
5310 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5311 _V&
5312 operator^=(_V& __lhs, const _V& __x)
5313 { return __lhs = __lhs ^ __x; }
5314
5315 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5316 _V&
5317 operator<<=(_V& __lhs, const _V& __x)
5318 { return __lhs = __lhs << __x; }
5319
5320 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5321 _V&
5322 operator>>=(_V& __lhs, const _V& __x)
5323 { return __lhs = __lhs >> __x; }
5324
5325 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5326 _V&
5327 operator<<=(_V& __lhs, int __x)
5328 { return __lhs = __lhs << __x; }
5329
5330 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5331 _V&
5332 operator>>=(_V& __lhs, int __x)
5333 { return __lhs = __lhs >> __x; }
5334
5335 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5336 _V
5337 operator%(const _V& __x, const _V& __y)
5338 {
5339 return _SimdIntOperators::_S_make_derived(
5340 _Impl::_S_modulus(__data(__x), __data(__y)));
5341 }
5342
5343 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5344 _V
5345 operator&(const _V& __x, const _V& __y)
5346 {
5347 return _SimdIntOperators::_S_make_derived(
5348 _Impl::_S_bit_and(__data(__x), __data(__y)));
5349 }
5350
5351 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5352 _V
5353 operator|(const _V& __x, const _V& __y)
5354 {
5355 return _SimdIntOperators::_S_make_derived(
5356 _Impl::_S_bit_or(__data(__x), __data(__y)));
5357 }
5358
5359 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5360 _V
5361 operator^(const _V& __x, const _V& __y)
5362 {
5363 return _SimdIntOperators::_S_make_derived(
5364 _Impl::_S_bit_xor(__data(__x), __data(__y)));
5365 }
5366
5367 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5368 _V
5369 operator<<(const _V& __x, const _V& __y)
5370 {
5371 return _SimdIntOperators::_S_make_derived(
5372 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
5373 }
5374
5375 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5376 _V
5377 operator>>(const _V& __x, const _V& __y)
5378 {
5379 return _SimdIntOperators::_S_make_derived(
5380 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
5381 }
5382
5383 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5384 _V
5385 operator<<(const _V& __x, int __y)
5386 {
5387 if (__y < 0)
5388 __invoke_ub("The behavior is undefined if the right operand of a "
5389 "shift operation is negative. [expr.shift]\nA shift by "
5390 "%d was requested",
5391 __y);
5392 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5393 __invoke_ub(
5394 "The behavior is undefined if the right operand of a "
5395 "shift operation is greater than or equal to the width of the "
5396 "promoted left operand. [expr.shift]\nA shift by %d was requested",
5397 __y);
5398 return _SimdIntOperators::_S_make_derived(
5399 _Impl::_S_bit_shift_left(__data(__x), __y));
5400 }
5401
5402 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5403 _V
5404 operator>>(const _V& __x, int __y)
5405 {
5406 if (__y < 0)
5407 __invoke_ub(
5408 "The behavior is undefined if the right operand of a shift "
5409 "operation is negative. [expr.shift]\nA shift by %d was requested",
5410 __y);
5411 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5412 __invoke_ub(
5413 "The behavior is undefined if the right operand of a shift "
5414 "operation is greater than or equal to the width of the promoted "
5415 "left operand. [expr.shift]\nA shift by %d was requested",
5416 __y);
5417 return _SimdIntOperators::_S_make_derived(
5418 _Impl::_S_bit_shift_right(__data(__x), __y));
5419 }
5420
5421 // unary operators (for integral _Tp)
5422 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5423 _V
5424 operator~() const
5425 { return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
5426 };
5427
5428//}}}1
5429/// @endcond
5430
5431// simd {{{
5432template <typename _Tp, typename _Abi>
5433 class simd : public _SimdIntOperators<
5434 simd<_Tp, _Abi>, _Tp, _Abi,
5435 conjunction<is_integral<_Tp>,
5436 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
5437 public _SimdTraits<_Tp, _Abi>::_SimdBase
5438 {
5439 using _Traits = _SimdTraits<_Tp, _Abi>;
5440 using _MemberType = typename _Traits::_SimdMember;
5441 using _CastType = typename _Traits::_SimdCastType;
5442 static constexpr _Tp* _S_type_tag = nullptr;
5443 friend typename _Traits::_SimdBase;
5444
5445 public:
5446 using _Impl = typename _Traits::_SimdImpl;
5447 friend _Impl;
5448 friend _SimdIntOperators<simd, _Tp, _Abi, true>;
5449
5450 using value_type = _Tp;
5451 using reference = _SmartReference<_MemberType, _Impl, value_type>;
5452 using mask_type = simd_mask<_Tp, _Abi>;
5453 using abi_type = _Abi;
5454
5455 static constexpr size_t size()
5456 { return __size_or_zero_v<_Tp, _Abi>; }
5457
5458 _GLIBCXX_SIMD_CONSTEXPR simd() = default;
5459 _GLIBCXX_SIMD_CONSTEXPR simd(const simd&) = default;
5460 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
5461 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
5462 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
5463
5464 // implicit broadcast constructor
5465 template <typename _Up,
5466 typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>, bool>>>
5467 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5468 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
5469 : _M_data(
5470 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
5471 {}
5472
5473 // implicit type conversion constructor (convert from fixed_size to
5474 // fixed_size)
5475 template <typename _Up>
5476 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5477 simd(const simd<_Up, simd_abi::fixed_size<size()>>& __x,
5478 enable_if_t<
5479 conjunction<
5480 is_same<simd_abi::fixed_size<size()>, abi_type>,
5481 negation<__is_narrowing_conversion<_Up, value_type>>,
5482 __converts_to_higher_integer_rank<_Up, value_type>>::value,
5483 void*> = nullptr)
5484 : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5485
5486 // explicit type conversion constructor
5487#ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5488 template <typename _Up, typename _A2,
5489 typename = decltype(static_simd_cast<simd>(
5490 declval<const simd<_Up, _A2>&>()))>
5491 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5492 simd(const simd<_Up, _A2>& __x)
5493 : simd(static_simd_cast<simd>(__x)) {}
5494#endif // _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5495
5496 // generator constructor
5497 template <typename _Fp>
5498 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5499 simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
5500 declval<_SizeConstant<0>&>())),
5501 value_type>* = nullptr)
5502 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5503
5504 // load constructor
5505 template <typename _Up, typename _Flags>
5506 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5507 simd(const _Up* __mem, _IsSimdFlagType<_Flags>)
5508 : _M_data(
5509 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5510 {}
5511
5512 // loads [simd.load]
5513 template <typename _Up, typename _Flags>
5514 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
5515 copy_from(const _Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>)
5516 {
5517 _M_data = static_cast<decltype(_M_data)>(
5518 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5519 }
5520
5521 // stores [simd.store]
5522 template <typename _Up, typename _Flags>
5523 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void
5524 copy_to(_Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>) const
5525 {
5526 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5527 _S_type_tag);
5528 }
5529
5530 // scalar access
5531 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5532 operator[](size_t __i)
5533 { return {_M_data, int(__i)}; }
5534
5535 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5536 operator[]([[maybe_unused]] size_t __i) const
5537 {
5538 if constexpr (__is_scalar_abi<_Abi>())
5539 {
5540 _GLIBCXX_DEBUG_ASSERT(__i == 0);
5541 return _M_data;
5542 }
5543 else
5544 return _M_data[__i];
5545 }
5546
5547 // increment and decrement:
5548 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5549 operator++()
5550 {
5551 _Impl::_S_increment(_M_data);
5552 return *this;
5553 }
5554
5555 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5556 operator++(int)
5557 {
5558 simd __r = *this;
5559 _Impl::_S_increment(_M_data);
5560 return __r;
5561 }
5562
5563 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5564 operator--()
5565 {
5566 _Impl::_S_decrement(_M_data);
5567 return *this;
5568 }
5569
5570 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5571 operator--(int)
5572 {
5573 simd __r = *this;
5574 _Impl::_S_decrement(_M_data);
5575 return __r;
5576 }
5577
5578 // unary operators (for any _Tp)
5579 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5580 operator!() const
5581 { return {__private_init, _Impl::_S_negate(_M_data)}; }
5582
5583 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5584 operator+() const
5585 { return *this; }
5586
5587 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5588 operator-() const
5589 { return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5590
5591 // access to internal representation (suggested extension)
5592 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5593 simd(_CastType __init) : _M_data(__init) {}
5594
5595 // compound assignment [simd.cassign]
5596 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5597 operator+=(simd& __lhs, const simd& __x)
5598 { return __lhs = __lhs + __x; }
5599
5600 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5601 operator-=(simd& __lhs, const simd& __x)
5602 { return __lhs = __lhs - __x; }
5603
5604 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5605 operator*=(simd& __lhs, const simd& __x)
5606 { return __lhs = __lhs * __x; }
5607
5608 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5609 operator/=(simd& __lhs, const simd& __x)
5610 { return __lhs = __lhs / __x; }
5611
5612 // binary operators [simd.binary]
5613 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5614 operator+(const simd& __x, const simd& __y)
5615 { return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5616
5617 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5618 operator-(const simd& __x, const simd& __y)
5619 { return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5620
5621 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5622 operator*(const simd& __x, const simd& __y)
5623 { return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5624
5625 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5626 operator/(const simd& __x, const simd& __y)
5627 { return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5628
5629 // compares [simd.comparison]
5630 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5631 operator==(const simd& __x, const simd& __y)
5632 { return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5633
5634 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5635 operator!=(const simd& __x, const simd& __y)
5636 {
5637 return simd::_S_make_mask(
5638 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5639 }
5640
5641 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5642 operator<(const simd& __x, const simd& __y)
5643 { return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5644
5645 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5646 operator<=(const simd& __x, const simd& __y)
5647 {
5648 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5649 }
5650
5651 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5652 operator>(const simd& __x, const simd& __y)
5653 { return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5654
5655 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5656 operator>=(const simd& __x, const simd& __y)
5657 {
5658 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5659 }
5660
5661 // operator?: overloads (suggested extension) {{{
5662#ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5663 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5664 operator?:(const mask_type& __k, const simd& __where_true,
5665 const simd& __where_false)
5666 {
5667 auto __ret = __where_false;
5668 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5669 return __ret;
5670 }
5671
5672#endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5673 // }}}
5674
5675 // "private" because of the first arguments's namespace
5676 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5677 simd(_PrivateInit, const _MemberType& __init)
5678 : _M_data(__init) {}
5679
5680 // "private" because of the first arguments's namespace
5681 _GLIBCXX_SIMD_INTRINSIC
5682 simd(_BitsetInit, bitset<size()> __init) : _M_data()
5683 { where(mask_type(__bitset_init, __init), *this) = ~*this; }
5684
5685 _GLIBCXX_SIMD_INTRINSIC constexpr bool
5686 _M_is_constprop() const
5687 {
5688 if constexpr (__is_scalar_abi<_Abi>())
5689 return __builtin_constant_p(_M_data);
5690 else
5691 return _M_data._M_is_constprop();
5692 }
5693
5694 private:
5695 _GLIBCXX_SIMD_INTRINSIC static constexpr mask_type
5696 _S_make_mask(typename mask_type::_MemberType __k)
5697 { return {__private_init, __k}; }
5698
5699 friend const auto& __data<value_type, abi_type>(const simd&);
5700 friend auto& __data<value_type, abi_type>(simd&);
5701 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5702 };
5703
5704// }}}
5705/// @cond undocumented
5706// __data {{{
5707template <typename _Tp, typename _Ap>
5708 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5709 __data(const simd<_Tp, _Ap>& __x)
5710 { return __x._M_data; }
5711
5712template <typename _Tp, typename _Ap>
5713 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5714 __data(simd<_Tp, _Ap>& __x)
5715 { return __x._M_data; }
5716
5717// }}}
5718namespace __float_bitwise_operators { //{{{
5719template <typename _Tp, typename _Ap>
5720 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5721 operator^(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5722 { return {__private_init, _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))}; }
5723
5724template <typename _Tp, typename _Ap>
5725 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5726 operator|(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5727 { return {__private_init, _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))}; }
5728
5729template <typename _Tp, typename _Ap>
5730 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5731 operator&(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5732 { return {__private_init, _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))}; }
5733
5734template <typename _Tp, typename _Ap>
5735 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5736 enable_if_t<is_floating_point_v<_Tp>, simd<_Tp, _Ap>>
5737 operator~(const simd<_Tp, _Ap>& __a)
5738 { return {__private_init, _Ap::_SimdImpl::_S_complement(__data(__a))}; }
5739} // namespace __float_bitwise_operators }}}
5740/// @endcond
5741
5742/// @}
5743_GLIBCXX_SIMD_END_NAMESPACE
5744
5745#endif // __cplusplus >= 201703L
5746#endif // _GLIBCXX_EXPERIMENTAL_SIMD_H
5747
5748// vim: foldmethod=marker foldmarker={{{,}}}
algorithm
bit
bitset
functional
iosfwd
utility
cmath
cstdio
cstring
std::chrono::operator<=
constexpr bool operator<=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:863
std::chrono::operator>=
constexpr bool operator>=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:877
std::chrono::operator%
constexpr duration< __common_rep_t< _Rep1, __disable_if_is_duration< _Rep2 > >, _Period > operator%(const duration< _Rep1, _Period > &__d, const _Rep2 &__s)
Definition chrono.h:787
std::chrono::operator<
constexpr bool operator<(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:830
std::chrono::operator>
constexpr bool operator>(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:870
std::operator*
constexpr complex< _Tp > operator*(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x times y.
Definition complex:434
std::operator-
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
Definition complex:404
std::operator+
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
Definition complex:374
std::operator/
constexpr complex< _Tp > operator/(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x divided by y.
Definition complex:464
std::conditional_t
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition type_traits:2971
std::remove_reference_t
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
Definition type_traits:1913
std::make_unsigned_t
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition type_traits:2273
std::enable_if_t
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
Definition type_traits:2967
std::false_type
__bool_constant< false > false_type
The type used as a compile-time boolean with false value.
Definition type_traits:122
std::tuple_cat
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
Create a tuple containing all elements from multiple tuple-like objects.
Definition tuple:2822
std::declval
auto declval() noexcept -> decltype(__declval< _Tp >(0))
Definition type_traits:2741
std::make_tuple
constexpr tuple< typename __decay_and_strip< _Elements >::__type... > make_tuple(_Elements &&... __args)
Create a tuple containing copies of the arguments.
Definition tuple:2686
std::move
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
Definition move.h:138
std::max
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
Definition stl_algobase.h:256
std::min
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition stl_algobase.h:232
std::reduce
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
Definition numeric:294
std::experimental::fundamentals_v2::void_t
void void_t
A metafunction that always yields void, used for detecting valid types.
Definition experimental/type_traits:262
std
ISO C++ entities toplevel namespace is std.
std::make_index_sequence
make_integer_sequence< size_t, _Num > make_index_sequence
Alias template make_index_sequence.
Definition utility.h:559
std::operator^
constexpr bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1639
std::operator>>
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition bitset:1659
std::operator<<
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition bitset:1755
std::operator|
constexpr bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1629
std::operator&
constexpr bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1619
parallelism_v2::simd_abi::deduce
Definition simd.h:3114