libstdc++
bits/simd_x86.h
1// Implementation of <simd> -*- C++ -*-
2
3// Copyright The GNU Toolchain Authors.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_SIMD_X86_H
26#define _GLIBCXX_SIMD_X86_H 1
27
28#ifdef _GLIBCXX_SYSHDR
29#pragma GCC system_header
30#endif
31
32#if __cplusplus >= 202400L
33
34#include "vec_ops.h"
35
36#if !_GLIBCXX_X86
37#error "wrong include for this target"
38#endif
39
40#pragma GCC push_options
41// ensure GCC knows about the __builtin_ia32_* calls
42#pragma GCC target("avx2,bmi,bmi2,avx512vl,avx512bw,avx512dq,avx10.2")
43#pragma GCC pop_options
44
45// psabi warnings are bogus because the ABI of the internal types never leaks into user code
46#pragma GCC diagnostic push
47#pragma GCC diagnostic ignored "-Wpsabi"
48
49namespace std _GLIBCXX_VISIBILITY(default)
50{
51_GLIBCXX_BEGIN_NAMESPACE_VERSION
52namespace simd
53{
54 static constexpr size_t __x86_max_general_register_size
55#ifdef __x86_64__
56 = 8;
57#else
58 = 4;
59#endif
60
61 /** @internal
62 * Return a bit-mask for the given vector-mask.
63 *
64 * Caveats:
65 * 1. The bit-mask of 2-Byte vector-masks has duplicated entries (because of missing instruction)
66 * 2. The return type internally is 'int', but that fails on conversion to uint64 if the MSB of a
67 * YMM 1/2-Byte vector-mask is set (sign extension). Therefore these helper functions return
68 * unsigned instead.
69 * 3. ZMM inputs are not supported.
70 */
71 [[__gnu__::__always_inline__]]
72 inline unsigned
73 __x86_movmsk(__vec_builtin_type_bytes<__integer_from<8>, 16> __x)
74 { return __builtin_ia32_movmskpd(__vec_bit_cast<double>(__x)); }
75
76 [[__gnu__::__always_inline__]]
77 inline unsigned
78 __x86_movmsk(__vec_builtin_type_bytes<__integer_from<8>, 32> __x)
79 { return __builtin_ia32_movmskpd256(__vec_bit_cast<double>(__x)); }
80
81 [[__gnu__::__always_inline__]]
82 inline unsigned
83 __x86_movmsk(__vec_builtin_type_bytes<__integer_from<4>, 16> __x)
84 { return __builtin_ia32_movmskps(__vec_bit_cast<float>(__x)); }
85
86 template <_ArchTraits _Traits = {}>
87 [[__gnu__::__always_inline__]]
88 inline _Bitmask<8>
89 __x86_movmsk(__vec_builtin_type_bytes<__integer_from<4>, 8> __x)
90 {
91#if __has_builtin(__builtin_ia32_pext_di)
92 if constexpr (_Traits._M_have_bmi2())
93 return _Bitmask<8>(__builtin_ia32_pext_di(
94 __builtin_bit_cast(unsigned long long, __x),
95 0x80000000'80000000ULL));
96#endif
97 return _Bitmask<8>(__x86_movmsk(__vec_zero_pad_to_16(__x)));
98 }
99
100 [[__gnu__::__always_inline__]]
101 inline unsigned
102 __x86_movmsk(__vec_builtin_type_bytes<__integer_from<4>, 32> __x)
103 { return __builtin_ia32_movmskps256(__vec_bit_cast<float>(__x)); }
104
105 template <__vec_builtin _TV, auto _Traits = _ArchTraits()>
106 requires (sizeof(__vec_value_type<_TV>) <= 2)
107 [[__gnu__::__always_inline__]]
108 inline unsigned
109 __x86_movmsk(_TV __x)
110 {
111 static_assert(__width_of<_TV> > 1);
112 if constexpr (sizeof(__x) == 32)
113 return __builtin_ia32_pmovmskb256(__vec_bit_cast<char>(__x));
114 else if constexpr (sizeof(__x) == 16)
115 return __builtin_ia32_pmovmskb128(__vec_bit_cast<char>(__x));
116 else if constexpr (sizeof(__x) == 8)
117 {
118#if __has_builtin(__builtin_ia32_pext_di)
119 if constexpr (_Traits._M_have_bmi2())
120 return __builtin_ia32_pext_di(__builtin_bit_cast(unsigned long long, __x),
121 0x8080'8080'8080'8080ULL);
122#endif
123 return __x86_movmsk(__vec_zero_pad_to_16(__x));
124 }
125 else if constexpr (sizeof(__x) == 4)
126 {
127#if __has_builtin(__builtin_ia32_pext_si)
128 if constexpr (_Traits._M_have_bmi2())
129 return __builtin_ia32_pext_si(__builtin_bit_cast(unsigned int, __x), 0x80808080u);
130#endif
131 return __x86_movmsk(__vec_zero_pad_to_16(__x));
132 }
133 else if constexpr (sizeof(__x) == 2)
134 {
135 auto __bits = __builtin_bit_cast(unsigned short, __x);
136#if __has_builtin(__builtin_ia32_pext_si)
137 if constexpr (_Traits._M_have_bmi2())
138 return __builtin_ia32_pext_si(__bits, 0x00008080u);
139#endif
140 return ((__bits >> 7) & 1) | ((__bits & 0x8000) >> 14);
141 }
142 else
143 static_assert(false);
144 }
145
146 template <__vec_builtin _TV, _ArchTraits _Traits = {}>
147 [[__gnu__::__always_inline__]]
148 inline bool
149 __x86_vec_is_zero(_TV __a)
150 {
151 using _Tp = __vec_value_type<_TV>;
152 static_assert(is_integral_v<_Tp>);
153 if constexpr (sizeof(_TV) <= __x86_max_general_register_size)
154 return __builtin_bit_cast(__integer_from<sizeof(_TV)>, __a) == 0;
155 else if constexpr (_Traits._M_have_avx())
156 {
157 if constexpr (sizeof(_TV) == 32)
158 return __builtin_ia32_ptestz256(__vec_bit_cast<long long>(__a),
159 __vec_bit_cast<long long>(__a));
160 else if constexpr (sizeof(_TV) == 16)
161 return __builtin_ia32_ptestz128(__vec_bit_cast<long long>(__a),
162 __vec_bit_cast<long long>(__a));
163 else if constexpr (sizeof(_TV) < 16)
164 return __x86_vec_is_zero(__vec_zero_pad_to_16(__a));
165 else
166 static_assert(false);
167 }
168 else if constexpr (_Traits._M_have_sse4_1())
169 {
170 if constexpr (sizeof(_TV) == 16)
171 return __builtin_ia32_ptestz128(__vec_bit_cast<long long>(__a),
172 __vec_bit_cast<long long>(__a));
173 else if constexpr (sizeof(_TV) < 16)
174 return __x86_vec_is_zero(__vec_zero_pad_to_16(__a));
175 else
176 static_assert(false);
177 }
178 else
179 return __x86_movmsk(__a) == 0;
180 }
181
182 template <__vec_builtin _TV, _ArchTraits _Traits = {}>
183 [[__gnu__::__always_inline__]]
184 inline int
185 __x86_vec_testz(_TV __a, _TV __b)
186 {
187 static_assert(sizeof(_TV) == 16 || sizeof(_TV) == 32);
188 static_assert(_Traits._M_have_sse4_1());
189 if constexpr (sizeof(_TV) == 32)
190 return __builtin_ia32_ptestz256(__vec_bit_cast<long long>(__a),
191 __vec_bit_cast<long long>(__b));
192 else
193 return __builtin_ia32_ptestz128(__vec_bit_cast<long long>(__a),
194 __vec_bit_cast<long long>(__b));
195 }
196
197 template <__vec_builtin _TV, _ArchTraits _Traits = {}>
198 [[__gnu__::__always_inline__]]
199 inline int
200 __x86_vec_testc(_TV __a, _TV __b)
201 {
202 static_assert(sizeof(_TV) == 16 || sizeof(_TV) == 32);
203 static_assert(_Traits._M_have_sse4_1());
204 if constexpr (sizeof(_TV) == 32)
205 return __builtin_ia32_ptestc256(__vec_bit_cast<long long>(__a),
206 __vec_bit_cast<long long>(__b));
207 else
208 return __builtin_ia32_ptestc128(__vec_bit_cast<long long>(__a),
209 __vec_bit_cast<long long>(__b));
210 }
211
212 template <int _Np, __vec_builtin _TV, _ArchTraits _Traits = {}>
213 [[__gnu__::__always_inline__]]
214 inline bool
215 __x86_vecmask_all(_TV __k)
216 {
217 using _Tp = __vec_value_type<_TV>;
218 static_assert(is_integral_v<_Tp> && is_signed_v<_Tp>);
219 constexpr int __width = __width_of<_TV>;
220 static_assert(sizeof(__k) <= 32);
221 if constexpr (_Np == __width)
222 {
223 if constexpr (sizeof(__k) <= __x86_max_general_register_size)
224 {
225 using _Ip = __integer_from<sizeof(__k)>;
226 return __builtin_bit_cast(_Ip, __k) == ~_Ip();
227 }
228 else if constexpr (!_Traits._M_have_sse4_1())
229 {
230 constexpr unsigned __valid_bits = (1u << (sizeof(_Tp) == 2 ? _Np * 2 : _Np)) - 1;
231 return __x86_movmsk(__k) == __valid_bits;
232 }
233 else if constexpr (sizeof(__k) < 16)
234 return __x86_vecmask_all<_Np>(__vec_zero_pad_to_16(__k));
235 else
236 return 0 != __x86_vec_testc(__k, ~_TV());
237 }
238 else if constexpr (sizeof(__k) <= __x86_max_general_register_size)
239 {
240 using _Ip = __integer_from<sizeof(__k)>;
241 constexpr _Ip __valid_bits = (_Ip(1) << (_Np * sizeof(_Tp) * __CHAR_BIT__)) - 1;
242 return (__builtin_bit_cast(_Ip, __k) & __valid_bits) == __valid_bits;
243 }
244 else if constexpr (!_Traits._M_have_sse4_1())
245 {
246 constexpr unsigned __valid_bits = (1u << (sizeof(_Tp) == 2 ? _Np * 2 : _Np)) - 1;
247 return (__x86_movmsk(__k) & __valid_bits) == __valid_bits;
248 }
249 else if constexpr (sizeof(__k) < 16)
250 return __x86_vecmask_all<_Np>(__vec_zero_pad_to_16(__k));
251 else
252 return 0 != __x86_vec_testc(__k, _S_vec_implicit_mask<_Np, _TV>);
253 }
254
255 template <int _Np, __vec_builtin _TV, _ArchTraits _Traits = {}>
256 [[__gnu__::__always_inline__]]
257 inline bool
258 __x86_vecmask_any(_TV __k)
259 {
260 using _Tp = __vec_value_type<_TV>;
261 static_assert(is_integral_v<_Tp> && is_signed_v<_Tp>);
262 constexpr int __width = __width_of<_TV>;
263 static_assert(sizeof(__k) <= 32);
264 if constexpr (_Np == __width)
265 return !__x86_vec_is_zero(__k);
266 else if constexpr (sizeof(__k) <= __x86_max_general_register_size)
267 {
268 using _Ip = __integer_from<sizeof(__k)>;
269 constexpr _Ip __valid_bits = (_Ip(1) << (_Np * sizeof(_Tp) * __CHAR_BIT__)) - 1;
270 return (__builtin_bit_cast(_Ip, __k) & __valid_bits) != _Ip();
271 }
272 else if constexpr (!_Traits._M_have_sse4_1())
273 {
274 constexpr unsigned __valid_bits = (1u << (sizeof(_Tp) == 2 ? _Np * 2 : _Np)) - 1;
275 return (__x86_movmsk(__k) & __valid_bits) != 0;
276 }
277 else if constexpr (sizeof(__k) < 16)
278 return __x86_vecmask_any<_Np>(__vec_zero_pad_to_16(__k));
279 else
280 return 0 == __x86_vec_testz(__k, _S_vec_implicit_mask<_Np, _TV>);
281 }
282
283 template <int _Np, __vec_builtin _TV, _ArchTraits _Traits = {}>
284 [[__gnu__::__always_inline__]]
285 inline bool
286 __x86_vecmask_none(_TV __k)
287 {
288 using _Tp = __vec_value_type<_TV>;
289 static_assert(is_integral_v<_Tp> && is_signed_v<_Tp>);
290 constexpr int __width = __width_of<_TV>;
291 static_assert(sizeof(__k) <= 32);
292 if constexpr (_Np == __width)
293 return __x86_vec_is_zero(__k);
294 else if constexpr (sizeof(__k) <= __x86_max_general_register_size)
295 {
296 using _Ip = __integer_from<sizeof(__k)>;
297 constexpr _Ip __valid_bits = (_Ip(1) << (_Np * sizeof(_Tp) * __CHAR_BIT__)) - 1;
298 return (__builtin_bit_cast(_Ip, __k) & __valid_bits) == _Ip();
299 }
300 else if constexpr (!_Traits._M_have_sse4_1())
301 {
302 constexpr unsigned __valid_bits = (1u << (sizeof(_Tp) == 2 ? _Np * 2 : _Np)) - 1;
303 return (__x86_movmsk(__k) & __valid_bits) == 0;
304 }
305 else if constexpr (sizeof(__k) < 16)
306 return __x86_vecmask_none<_Np>(__vec_zero_pad_to_16(__k));
307 else
308 return 0 != __x86_vec_testz(__k, _S_vec_implicit_mask<_Np, _TV>);
309 }
310
311 enum class _X86Cmp
312 {
313 _Eq = 0,
314 _Lt = 1,
315 _Le = 2,
316 _Unord = 3,
317 _Neq = 4,
318 _Nlt = 5,
319 _Nle = 6,
320 };
321
322 template <_X86Cmp _Cmp, __vec_builtin _TV, _ArchTraits _Traits = {}>
323 requires is_floating_point_v<__vec_value_type<_TV>>
324 [[__gnu__::__always_inline__]]
325 inline auto
326 __x86_bitmask_cmp(_TV __x, _TV __y)
327 {
328 constexpr int __c = int(_Cmp);
329 using _Tp = __vec_value_type<_TV>;
330 if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 8)
331 return __builtin_ia32_cmppd512_mask(__x, __y, __c, -1, 4);
332 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 4)
333 return __builtin_ia32_cmpps512_mask(__x, __y, __c, -1, 4);
334 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 8)
335 return __builtin_ia32_cmppd256_mask(__x, __y, __c, -1);
336 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 4)
337 return __builtin_ia32_cmpps256_mask(__x, __y, __c, -1);
338 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 8)
339 return __builtin_ia32_cmppd128_mask(__x, __y, __c, -1);
340 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 4)
341 return __builtin_ia32_cmpps128_mask(__x, __y, __c, -1);
342 else if constexpr (is_same_v<_Tp, _Float16>)
343 {
344 if constexpr (sizeof(_TV) == 64 && _Traits._M_have_avx512fp16())
345 return __builtin_ia32_cmpph512_mask(__x, __y, __c, -1);
346 else if constexpr (sizeof(_TV) == 32 && _Traits._M_have_avx512fp16())
347 return __builtin_ia32_cmpph256_mask(__x, __y, __c, -1);
348 else if constexpr (sizeof(_TV) == 16 && _Traits._M_have_avx512fp16())
349 return __builtin_ia32_cmpph128_mask(__x, __y, __c, -1);
350 else if constexpr (sizeof(_TV) < 16 && _Traits._M_have_avx512fp16())
351 return __x86_bitmask_cmp<_Cmp>(__vec_zero_pad_to_16(__x), __vec_zero_pad_to_16(__y));
352 else
353 {
354 // without AVX512_FP16, float16_t size needs to match float32_t size
355 // (cf. __native_abi())
356 static_assert(sizeof(_TV) <= 32);
357 return __x86_bitmask_cmp<_Cmp>(__vec_cast<float>(__x), __vec_cast<float>(__y));
358 }
359 }
360 else if constexpr (sizeof(_TV) < 16)
361 return __x86_bitmask_cmp<_Cmp>(__vec_zero_pad_to_16(__x), __vec_zero_pad_to_16(__y));
362 else
363 static_assert(false);
364 }
365
366 template <typename _Tp>
367 using __x86_intrin_int
368 = decltype([] {
369 if constexpr (sizeof(_Tp) == 1)
370 return char();
371 else
372 return __integer_from<sizeof(_Tp)>();
373 }());
374
375 template <typename _Tp>
376 using __x86_intrin_type
377 = decltype([] {
378 if constexpr (is_integral_v<_Tp> || sizeof(_Tp) <= 2)
379 return __x86_intrin_int<_Tp>();
380 else
381 return __canonical_vec_type_t<_Tp>();
382 }());
383
384 template <typename _Tp>
385 using __x86_intel_intrin_value_type
386 = decltype([] {
387 if constexpr (is_integral_v<_Tp>)
388 return 0ll;
389 else if constexpr (sizeof(_Tp) == 8)
390 return 0.;
391 else if constexpr (sizeof(_Tp) == 4)
392 return 0.f;
393 else if constexpr (sizeof(_Tp) == 2)
394 return 0.f16;
395 }());
396
397#if !_GLIBCXX_CLANG
398 // overload __vec_andnot from simd_detail.h
399 template <__vec_builtin _TV>
400 requires (sizeof(_TV) >= 16)
401 [[__gnu__::__always_inline__]]
402 constexpr _TV
403 __vec_andnot(_TV __a, _TV __b)
404 {
405 constexpr _TargetTraits _Traits = {};
406 using _Tp = __vec_value_type<_TV>;
407 using _UV = __vec_builtin_type<_UInt<sizeof(_Tp)>, __width_of<_TV>>;
408 if (__builtin_is_constant_evaluated()
409 || (__builtin_constant_p(__a) && __builtin_constant_p(__b)))
410 return reinterpret_cast<_TV>(~reinterpret_cast<_UV>(__a) & reinterpret_cast<_UV>(__b));
411 else if constexpr (is_same_v<_Tp, _Float16>)
412 return reinterpret_cast<_TV>(__vec_andnot(__vec_bit_cast<float>(__a),
413 __vec_bit_cast<float>(__b)));
414 else if constexpr (sizeof(_TV) == 16 && is_same_v<_Tp, float>)
415 return __builtin_ia32_andnps(__a, __b);
416 else if constexpr (sizeof(_TV) == 16 && is_same_v<_Tp, double>)
417 return __builtin_ia32_andnpd(__a, __b);
418 else if constexpr (sizeof(_TV) == 32 && is_same_v<_Tp, float>)
419 return __builtin_ia32_andnps256(__a, __b);
420 else if constexpr (sizeof(_TV) == 32 && is_same_v<_Tp, double>)
421 return __builtin_ia32_andnpd256(__a, __b);
422 else if constexpr (sizeof(_TV) == 64 && is_same_v<_Tp, float> && _Traits._M_have_avx512dq())
423 return __builtin_ia32_andnps512_mask(__a, __b, _TV{}, -1);
424 else if constexpr (sizeof(_TV) == 64 && is_same_v<_Tp, double> && _Traits._M_have_avx512dq())
425 return __builtin_ia32_andnpd512_mask(__a, __b, _TV{}, -1);
426 else
427 {
428 auto __all = __vec_bit_cast<long long>(__a);
429 auto __bll = __vec_bit_cast<long long>(__b);
430 if constexpr (sizeof(_TV) == 16 && is_integral_v<_Tp>)
431 return reinterpret_cast<_TV>(__builtin_ia32_pandn128(__all, __bll));
432 else if constexpr (sizeof(_TV) == 32 && is_integral_v<_Tp> && _Traits._M_have_avx2())
433 return reinterpret_cast<_TV>(__builtin_ia32_andnotsi256(__all, __bll));
434 else if constexpr (sizeof(_TV) == 32 && is_integral_v<_Tp>)
435 return reinterpret_cast<_TV>(__builtin_ia32_andnpd256(__vec_bit_cast<double>(__a),
436 __vec_bit_cast<double>(__b)));
437 else if constexpr (sizeof(_TV) == 64)
438 {
439 auto __ai = __vec_bit_cast<int>(__a);
440 auto __bi = __vec_bit_cast<int>(__b);
441 return reinterpret_cast<_TV>(
442 __builtin_ia32_pandnd512_mask(__ai, __bi, decltype(__ai)(), -1));
443 }
444 }
445 }
446#endif // not Clang
447
448 template <_X86Cmp _Cmp, __vec_builtin _TV, _ArchTraits _Traits = {}>
449 requires is_integral_v<__vec_value_type<_TV>>
450 [[__gnu__::__always_inline__]]
451 inline auto
452 __x86_bitmask_cmp(_TV __x, _TV __y)
453 {
454 constexpr int __c = int(_Cmp);
455 using _Tp = __vec_value_type<_TV>;
456 if constexpr (sizeof(_TV) < 16)
457 return __x86_bitmask_cmp<_Cmp>(__vec_zero_pad_to_16(__x), __vec_zero_pad_to_16(__y));
458 else if constexpr (is_signed_v<_Tp>)
459 {
460 const auto __xi = __vec_bit_cast<__x86_intrin_int<_Tp>>(__x);
461 const auto __yi = __vec_bit_cast<__x86_intrin_int<_Tp>>(__y);
462 if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 8)
463 return __builtin_ia32_cmpq512_mask(__xi, __yi, __c, -1);
464 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 4)
465 return __builtin_ia32_cmpd512_mask(__xi, __yi, __c, -1);
466 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 2)
467 return __builtin_ia32_cmpw512_mask(__xi, __yi, __c, -1);
468 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 1)
469 return __builtin_ia32_cmpb512_mask(__xi, __yi, __c, -1);
470 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 8)
471 return __builtin_ia32_cmpq256_mask(__xi, __yi, __c, -1);
472 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 4)
473 return __builtin_ia32_cmpd256_mask(__xi, __yi, __c, -1);
474 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 2)
475 return __builtin_ia32_cmpw256_mask(__xi, __yi, __c, -1);
476 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 1)
477 return __builtin_ia32_cmpb256_mask(__xi, __yi, __c, -1);
478 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 8)
479 return __builtin_ia32_cmpq128_mask(__xi, __yi, __c, -1);
480 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 4)
481 return __builtin_ia32_cmpd128_mask(__xi, __yi, __c, -1);
482 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 2)
483 return __builtin_ia32_cmpw128_mask(__xi, __yi, __c, -1);
484 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 1)
485 return __builtin_ia32_cmpb128_mask(__xi, __yi, __c, -1);
486 else
487 static_assert(false);
488 }
489 else
490 {
491 const auto __xi = __vec_bit_cast<__x86_intrin_int<_Tp>>(__x);
492 const auto __yi = __vec_bit_cast<__x86_intrin_int<_Tp>>(__y);
493 if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 8)
494 return __builtin_ia32_ucmpq512_mask(__xi, __yi, __c, -1);
495 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 4)
496 return __builtin_ia32_ucmpd512_mask(__xi, __yi, __c, -1);
497 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 2)
498 return __builtin_ia32_ucmpw512_mask(__xi, __yi, __c, -1);
499 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 1)
500 return __builtin_ia32_ucmpb512_mask(__xi, __yi, __c, -1);
501 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 8)
502 return __builtin_ia32_ucmpq256_mask(__xi, __yi, __c, -1);
503 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 4)
504 return __builtin_ia32_ucmpd256_mask(__xi, __yi, __c, -1);
505 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 2)
506 return __builtin_ia32_ucmpw256_mask(__xi, __yi, __c, -1);
507 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 1)
508 return __builtin_ia32_ucmpb256_mask(__xi, __yi, __c, -1);
509 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 8)
510 return __builtin_ia32_ucmpq128_mask(__xi, __yi, __c, -1);
511 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 4)
512 return __builtin_ia32_ucmpd128_mask(__xi, __yi, __c, -1);
513 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 2)
514 return __builtin_ia32_ucmpw128_mask(__xi, __yi, __c, -1);
515 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 1)
516 return __builtin_ia32_ucmpb128_mask(__xi, __yi, __c, -1);
517 else
518 static_assert(false);
519 }
520 }
521
522 template <__vec_builtin _TV, _ArchTraits _Traits = {}>
523 [[__gnu__::__always_inline__]]
524 inline auto
525 __x86_bitmask_isinf(_TV __x)
526 {
527 static_assert(_Traits._M_have_avx512dq());
528 using _Tp = __vec_value_type<_TV>;
529 static_assert(is_floating_point_v<_Tp>);
530 if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 8)
531 return __builtin_ia32_fpclasspd512_mask(__x, 0x18, -1);
532 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 8)
533 return __builtin_ia32_fpclasspd256_mask(__x, 0x18, -1);
534 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 8)
535 return __builtin_ia32_fpclasspd128_mask(__x, 0x18, -1);
536 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 4)
537 return __builtin_ia32_fpclassps512_mask(__x, 0x18, -1);
538 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 4)
539 return __builtin_ia32_fpclassps256_mask(__x, 0x18, -1);
540 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 4)
541 return __builtin_ia32_fpclassps128_mask(__x, 0x18, -1);
542 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 2 && _Traits._M_have_avx512fp16())
543 return __builtin_ia32_fpclassph512_mask(__x, 0x18, -1);
544 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 2 && _Traits._M_have_avx512fp16())
545 return __builtin_ia32_fpclassph256_mask(__x, 0x18, -1);
546 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 2 && _Traits._M_have_avx512fp16())
547 return __builtin_ia32_fpclassph128_mask(__x, 0x18, -1);
548 else if constexpr (sizeof(_Tp) == 2 && !_Traits._M_have_avx512fp16())
549 return __x86_bitmask_isinf(__vec_cast<float>(__x));
550 else if constexpr (sizeof(_TV) < 16)
551 return __x86_bitmask_isinf(__vec_zero_pad_to_16(__x));
552 else
553 static_assert(false);
554 }
555
556 template <__vec_builtin _KV, _ArchTraits _Traits = {}>
557 [[__gnu__::__always_inline__]]
558 inline _KV
559 __x86_bit_to_vecmask(std::integral auto __bits)
560 {
561 using _Kp = __vec_value_type<_KV>;
562 static_assert((sizeof(__bits) * __CHAR_BIT__ == __width_of<_KV>)
563 || (sizeof(__bits) == 1 && __CHAR_BIT__ > __width_of<_KV>));
564
565 if constexpr (sizeof(_Kp) == 1 && sizeof(_KV) == 64)
566 return __builtin_ia32_cvtmask2b512(__bits);
567 else if constexpr (sizeof(_Kp) == 1 && sizeof(_KV) == 32)
568 return __builtin_ia32_cvtmask2b256(__bits);
569 else if constexpr (sizeof(_Kp) == 1 && sizeof(_KV) == 16)
570 return __builtin_ia32_cvtmask2b128(__bits);
571 else if constexpr (sizeof(_Kp) == 1 && sizeof(_KV) <= 8)
572 return _VecOps<_KV>::_S_extract(__builtin_ia32_cvtmask2b128(__bits));
573
574 else if constexpr (sizeof(_Kp) == 2 && sizeof(_KV) == 64)
575 return __builtin_ia32_cvtmask2w512(__bits);
576 else if constexpr (sizeof(_Kp) == 2 && sizeof(_KV) == 32)
577 return __builtin_ia32_cvtmask2w256(__bits);
578 else if constexpr (sizeof(_Kp) == 2 && sizeof(_KV) == 16)
579 return __builtin_ia32_cvtmask2w128(__bits);
580 else if constexpr (sizeof(_Kp) == 2 && sizeof(_KV) <= 8)
581 return _VecOps<_KV>::_S_extract(__builtin_ia32_cvtmask2w128(__bits));
582
583 else if constexpr (sizeof(_Kp) == 4 && sizeof(_KV) == 64)
584 return __builtin_ia32_cvtmask2d512(__bits);
585 else if constexpr (sizeof(_Kp) == 4 && sizeof(_KV) == 32)
586 return __builtin_ia32_cvtmask2d256(__bits);
587 else if constexpr (sizeof(_Kp) == 4 && sizeof(_KV) <= 16)
588 return _VecOps<_KV>::_S_extract(__builtin_ia32_cvtmask2d128(__bits));
589
590 else if constexpr (sizeof(_Kp) == 8 && sizeof(_KV) == 64)
591 return __builtin_ia32_cvtmask2q512(__bits);
592 else if constexpr (sizeof(_Kp) == 8 && sizeof(_KV) == 32)
593 return __builtin_ia32_cvtmask2q256(__bits);
594 else if constexpr (sizeof(_Kp) == 8 && sizeof(_KV) == 16)
595 return __builtin_ia32_cvtmask2q128(__bits);
596
597 else
598 static_assert(false);
599 }
600
601 template <unsigned_integral _Kp, __vec_builtin _TV, _ArchTraits _Traits = {}>
602 requires is_integral_v<__vec_value_type<_TV>>
603 [[__gnu__::__always_inline__]]
604 constexpr inline _TV
605 __x86_bitmask_blend(_Kp __k, _TV __t, _TV __f)
606 {
607 using _Tp = __vec_value_type<_TV>;
608 using _Ip = __x86_intrin_int<_Tp>;
609 if constexpr (!is_same_v<_Ip, _Tp>)
610 return reinterpret_cast<_TV>(__x86_bitmask_blend(__k, __vec_bit_cast<_Ip>(__t),
611 __vec_bit_cast<_Ip>(__f)));
612 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 8)
613 return __builtin_ia32_blendmq_512_mask (__f, __t, __k);
614 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 4)
615 return __builtin_ia32_blendmd_512_mask (__f, __t, __k);
616 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 2)
617 return __builtin_ia32_blendmw_512_mask (__f, __t, __k);
618 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 1)
619 return __builtin_ia32_blendmb_512_mask (__f, __t, __k);
620 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 8)
621 return __builtin_ia32_blendmq_256_mask (__f, __t, __k);
622 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 4)
623 return __builtin_ia32_blendmd_256_mask (__f, __t, __k);
624 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 2)
625 return __builtin_ia32_blendmw_256_mask (__f, __t, __k);
626 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 1)
627 return __builtin_ia32_blendmb_256_mask (__f, __t, __k);
628 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 8)
629 return __builtin_ia32_blendmq_128_mask (__f, __t, __k);
630 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 4)
631 return __builtin_ia32_blendmd_128_mask (__f, __t, __k);
632 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 2)
633 return __builtin_ia32_blendmw_128_mask (__f, __t, __k);
634 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 1)
635 return __builtin_ia32_blendmb_128_mask (__f, __t, __k);
636 else if constexpr (sizeof(_TV) < 16)
637 return _VecOps<_TV>::_S_extract(__x86_bitmask_blend(__k, __vec_zero_pad_to_16(__t),
638 __vec_zero_pad_to_16(__f)));
639 else
640 static_assert(false);
641 }
642
643 template <unsigned_integral _Kp, __vec_builtin _TV, _ArchTraits _Traits = {}>
644 requires is_floating_point_v<__vec_value_type<_TV>>
645 [[__gnu__::__always_inline__]]
646 constexpr inline _TV
647 __x86_bitmask_blend(_Kp __k, _TV __t, _TV __f)
648 {
649 using _Tp = __vec_value_type<_TV>;
650 if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 8)
651 return __builtin_ia32_blendmpd_512_mask (__f, __t, __k);
652 else if constexpr (sizeof(_TV) == 64 && sizeof(_Tp) == 4)
653 return __builtin_ia32_blendmps_512_mask (__f, __t, __k);
654 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 8)
655 return __builtin_ia32_blendmpd_256_mask (__f, __t, __k);
656 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 4)
657 return __builtin_ia32_blendmps_256_mask (__f, __t, __k);
658 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 8)
659 return __builtin_ia32_blendmpd_128_mask (__f, __t, __k);
660 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 4)
661 return __builtin_ia32_blendmps_128_mask (__f, __t, __k);
662 else if constexpr (is_same_v<_Tp, _Float16>)
663 {
664 using _Up = __integer_from<sizeof(_Tp)>;
665 return __vec_bit_cast<_Float16>(__x86_bitmask_blend(__k, __vec_bit_cast<_Up>(__t),
666 __vec_bit_cast<_Up>(__f)));
667 }
668 else if constexpr (sizeof(_TV) < 16)
669 return _VecOps<_TV>::_S_extract(__x86_bitmask_blend(__k, __vec_zero_pad_to_16(__t),
670 __vec_zero_pad_to_16(__f)));
671 else
672 static_assert(false);
673 }
674
675 template <int _OutputBits = 4, _ArchTraits _Traits = {}>
676 constexpr _Bitmask<1>
677 __bit_extract_even(_UInt<1> __x)
678 {
679 static_assert(_OutputBits <= 4);
680 constexpr _UInt<1> __mask = 0x55u >> ((4 - _OutputBits) * 2);
681#if __has_builtin(__builtin_ia32_pext_si)
682 if constexpr (_Traits._M_have_bmi2())
683 return __builtin_ia32_pext_si(__x, __mask);
684#endif
685 __x &= __mask;
686 __x |= __x >> 1;
687 __x &= 0x33u;
688 __x |= __x >> 2;
689 __x &= 0x0Fu;
690 return __x;
691 }
692
693 template <int _OutputBits = 8, _ArchTraits _Traits = {}>
694 constexpr _Bitmask<1>
695 __bit_extract_even(_UInt<2> __x)
696 {
697 if constexpr (_OutputBits <= 4)
698 return __bit_extract_even<_OutputBits>(_UInt<1>(__x));
699 else
700 {
701 static_assert(_OutputBits <= 8);
702 constexpr _UInt<2> __mask = 0x5555u >> ((8 - _OutputBits) * 2);
703#if __has_builtin(__builtin_ia32_pext_si)
704 if constexpr (_Traits._M_have_bmi2())
705 return __builtin_ia32_pext_si(__x, __mask);
706#endif
707 __x &= __mask;
708 __x |= __x >> 1;
709 __x &= 0x3333u;
710 __x |= __x >> 2;
711 __x &= 0x0F0Fu;
712 __x |= __x >> 4;
713 return __x;
714 }
715 }
716
717 template <int _OutputBits = 16, _ArchTraits _Traits = {}>
718 constexpr _Bitmask<_OutputBits>
719 __bit_extract_even(_UInt<4> __x)
720 {
721 if constexpr (_OutputBits <= 4)
722 return __bit_extract_even<_OutputBits>(_UInt<1>(__x));
723 else if constexpr (_OutputBits <= 8)
724 return __bit_extract_even<_OutputBits>(_UInt<2>(__x));
725 else
726 {
727 static_assert(_OutputBits <= 16);
728 constexpr _UInt<4> __mask = 0x5555'5555u >> ((16 - _OutputBits) * 2);
729#if __has_builtin(__builtin_ia32_pext_si)
730 if constexpr (_Traits._M_have_bmi2())
731 return __builtin_ia32_pext_si(__x, __mask);
732#endif
733 __x &= __mask;
734 __x |= __x >> 1;
735 __x &= 0x3333'3333u;
736 __x |= __x >> 2;
737 __x &= 0x0F0F'0F0Fu;
738 __x |= __x >> 4;
739 __x &= 0x00FF'00FFu;
740 __x |= __x >> 8;
741 return __x;
742 }
743 }
744
745 template <int _OutputBits = 32, _ArchTraits _Traits = {}>
746 constexpr _Bitmask<_OutputBits>
747 __bit_extract_even(_UInt<8> __x)
748 {
749 if constexpr (_OutputBits <= 4)
750 return __bit_extract_even<_OutputBits>(_UInt<1>(__x));
751 else if constexpr (_OutputBits <= 8)
752 return __bit_extract_even<_OutputBits>(_UInt<2>(__x));
753 else if constexpr (_OutputBits <= 16)
754 return __bit_extract_even<_OutputBits>(_UInt<4>(__x));
755 else
756 {
757 static_assert(_OutputBits <= 32);
758 constexpr _UInt<8> __mask = 0x5555'5555'5555'5555ull >> ((32 - _OutputBits) * 2);
759#if __has_builtin(__builtin_ia32_pext_si)
760 if constexpr (_Traits._M_have_bmi2())
761 {
762#if __has_builtin(__builtin_ia32_pext_di)
763 return __builtin_ia32_pext_di(__x, __mask);
764#else
765 return __builtin_ia32_pext_si(__x, static_cast<unsigned>(__mask))
766 | (__builtin_ia32_pext_si(__x >> 32, __mask >> 32) << 16);
767#endif
768 }
769#endif
770 __x &= __mask;
771 __x |= __x >> 1;
772 __x &= 0x3333'3333'3333'3333ull;
773 __x |= __x >> 2;
774 __x &= 0x0F0F'0F0F'0F0F'0F0Full;
775 __x |= __x >> 4;
776 __x &= 0x00FF'00FF'00FF'00FFull;
777 __x |= __x >> 8;
778 __x &= 0x0000'FFFF'0000'FFFFull;
779 __x |= __x >> 16;
780 return __x;
781 }
782 }
783
784 // input bits must be 0 for all bits > _InputBits
785 template <int _InputBits = -1, _ArchTraits _Traits = {}>
786 constexpr auto
787 __duplicate_each_bit(unsigned_integral auto __x)
788 {
789 constexpr int __input_bits = _InputBits == -1 ? sizeof(__x) * __CHAR_BIT__ : _InputBits;
790 static_assert(__input_bits >= 1);
791 static_assert(sizeof(__x) * __CHAR_BIT__ >= __input_bits);
792 if constexpr (__input_bits <= 8)
793 {
794 constexpr _UInt<2> __mask = 0x5555u >> ((8 - __input_bits) * 2);
795 if constexpr (__input_bits == 1)
796 return _UInt<1>(__x * 3u);
797#if __has_builtin(__builtin_ia32_pdep_si)
798 else if constexpr (_Traits._M_have_bmi2())
799 return _Bitmask<__input_bits * 2>(3u * __builtin_ia32_pdep_si(__x, __mask));
800#endif
801 else if constexpr (__input_bits == 2) // 0000'00BA
802 return _UInt<1>(((__x + 0b0010u) & 0b0101u) * 3u); // 0B?A -> 0B0A -> BBAA
803 else if constexpr (__input_bits <= 4) // 0000'DCBA
804 {
805 __x = ((__x << 2) | __x ) & 0b0011'0011u; // 00DC'??BA -> 00DC'00BA
806 return _UInt<1>(((__x + 0b0010'0010u) & __mask) * 3u); // -> DDCC'BBAA
807 }
808 else
809 { // HGFE'DCBA
810 _UInt<2> __y = ((__x << 4) | __x) & 0x0F0Fu; // HGFE'0000'DCBA
811 __y |= __y << 2; // 00HG'??FE'00DC'??BA
812 __y &= 0x3333u; // 00HG'00FE'00DC'00BA
813 __y += 0x2222u; // 0H?G'0F?E'0D?C'0B?A
814 return _UInt<2>((__y & __mask) * 3u); // HHGG'FFEE'DDCC'BBAA
815 }
816 }
817 else if constexpr (__input_bits <= 16)
818 {
819 constexpr _UInt<4> __mask = 0x5555'5555u >> ((16 - __input_bits) * 2);
820#if __has_builtin(__builtin_ia32_pdep_si)
821 if constexpr (_Traits._M_have_bmi2())
822 return 3u * __builtin_ia32_pdep_si(__x, __mask);
823#endif
824 _UInt<4> __y = ((__x << 8) | __x) & 0x00FF00FFu;
825 __y |= __y << 4;
826 __y &= 0x0F0F'0F0Fu;
827 __y |= __y << 2;
828 __y &= 0x3333'3333u;
829 return ((__y + 0x2222'2222u) & __mask) * 3;
830 }
831 else if constexpr (__input_bits <= 32)
832 {
833 constexpr _UInt<8> __mask = 0x5555'5555'5555'5555u >> ((32 - __input_bits) * 2);
834#if __has_builtin(__builtin_ia32_pdep_si)
835 if constexpr (_Traits._M_have_bmi2())
836 {
837#if __has_builtin(__builtin_ia32_pdep_di)
838 return 3ull * __builtin_ia32_pdep_di(__x, __mask);
839#else
840 const _UInt<8> __hi = 3 * __builtin_ia32_pdep_si(__x >> 16, __mask >> 32);
841 return (3u * __builtin_ia32_pdep_si(__x, static_cast<unsigned>(__mask))) | __hi << 32;
842#endif
843 }
844#endif
845 _UInt<8> __y = ((__x & 0xFFFF'0000ull) << 16) | (__x & 0x0000'FFFFu);
846 __y |= __y << 8;
847 __y &= 0x00FF'00FF'00FF'00FFull;
848 __y |= __y << 4;
849 __y &= 0x0F0F'0F0F'0F0F'0F0Full;
850 __y |= __y << 2;
851 __y &= 0x3333'3333'3333'3333ull;
852 return ((__y + 0x2222'2222'2222'2222ull) & __mask) * 3;
853 }
854 else
855 return __trivial_pair { __duplicate_each_bit(_UInt<4>(__x)),
856 __duplicate_each_bit<__input_bits - 32>(
857 _Bitmask<__input_bits - 32>(__x >> 32)) };
858 }
859
860 template <int _InputBits = -1, typename _U0, typename _U1>
861 constexpr auto
862 __duplicate_each_bit(const __trivial_pair<_U0, _U1>& __x)
863 {
864 static_assert(_InputBits != -1 || is_unsigned_v<_U1>);
865 constexpr int __input_bits = _InputBits == -1 ? (sizeof(_U0) + sizeof(_U1)) * __CHAR_BIT__
866 : _InputBits;
867 constexpr int __in0 = min(int(sizeof(_U0)) * __CHAR_BIT__, __input_bits);
868 constexpr int __in1 = __input_bits - __in0;
869 if constexpr (__in1 == 0)
870 return __duplicate_each_bit<__in0>(__x._M_first);
871 else
872 return __trivial_pair { __duplicate_each_bit<__in0>(__x._M_first),
873 __duplicate_each_bit<__in1>(__x._M_second) };
874 }
875
876 template <__vec_builtin _TV, _ArchTraits _Traits = {}>
877 [[__gnu__::__always_inline__]]
878 inline _TV
879 __x86_complex_multiplies(_TV __x, _TV __y)
880 {
881 using _Tp = __vec_value_type<_TV>;
882 using _VO = _VecOps<_TV>;
883
884 static_assert(_Traits._M_have_fma());
885 static_assert(is_floating_point_v<_Tp>);
886
887 if constexpr (!_Traits._M_have_avx512fp16() && sizeof(_Tp) == 2)
888 return __vec_cast<_Tp>(__x86_complex_multiplies(__vec_cast<float>(__x),
889 __vec_cast<float>(__y)));
890 else if constexpr (sizeof(_TV) < 16)
891 return _VO::_S_extract(__x86_complex_multiplies(__vec_zero_pad_to_16(__x),
892 __vec_zero_pad_to_16(__y)));
893
894 else
895 {
896 _TV __x_real = _VO::_S_dup_even(__x);
897 _TV __x_imag = _VO::_S_dup_odd(__x);
898 _TV __y_swapped = _VO::_S_swap_neighbors(__y);
899
900 if constexpr (sizeof(__x) == 16 && sizeof(_Tp) == 2)
901 return __builtin_ia32_vfmaddsubph128_mask(__x_real, __y, __x_imag * __y_swapped, -1);
902 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) == 2)
903 return __builtin_ia32_vfmaddsubph256_mask(__x_real, __y, __x_imag * __y_swapped, -1);
904 else if constexpr (sizeof(__x) == 64 && sizeof(_Tp) == 2)
905 return __builtin_ia32_vfmaddsubph512_mask(
906 __x_real, __y, __x_imag * __y_swapped, -1, 0x04);
907
908 else if constexpr (sizeof(__x) == 16 && sizeof(_Tp) == 4)
909 return __builtin_ia32_vfmaddsubps(__x_real, __y, __x_imag * __y_swapped);
910 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) == 4)
911 return __builtin_ia32_vfmaddsubps256(__x_real, __y, __x_imag * __y_swapped);
912 else if constexpr (sizeof(__x) == 64 && sizeof(_Tp) == 4)
913 return __builtin_ia32_vfmaddsubps512_mask(
914 __x_real, __y, __x_imag * __y_swapped, -1, 0x04);
915
916 else if constexpr (sizeof(__x) == 16 && sizeof(_Tp) == 8)
917 return __builtin_ia32_vfmaddsubpd(__x_real, __y, __x_imag * __y_swapped);
918 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) == 8)
919 return __builtin_ia32_vfmaddsubpd256(__x_real, __y, __x_imag * __y_swapped);
920 else if constexpr (sizeof(__x) == 64 && sizeof(_Tp) == 8)
921 return __builtin_ia32_vfmaddsubpd512_mask(
922 __x_real, __y, __x_imag * __y_swapped, -1, 0x04);
923
924 else
925 static_assert(false);
926 }
927 }
928
929 // FIXME: Work around PR121688
930 template <__vec_builtin _UV, __vec_builtin _TV>
931 [[__gnu__::__always_inline__]]
932 inline _UV
933 __x86_cvt_f16c(_TV __v)
934 {
935 constexpr bool __from_f16 = is_same_v<__vec_value_type<_TV>, _Float16>;
936 constexpr bool __to_f16 = !__from_f16;
937 if constexpr (__to_f16 && !is_same_v<__vec_value_type<_TV>, float>)
938 return __x86_cvt_f16c<_UV>(__vec_cast<float>(__v));
939 else if constexpr (__from_f16 && !is_same_v<__vec_value_type<_UV>, float>)
940 return __vec_cast<_UV>(__x86_cvt_f16c<__vec_builtin_type<float, __width_of<_TV>>>(__v));
941 else if constexpr (__from_f16)
942 {
943 const auto __vi = __vec_bit_cast<__x86_intrin_int<_Float16>>(__v);
944 if constexpr (sizeof(_TV) == 4)
945 return __vec_split_lo(__builtin_ia32_vcvtph2ps(__vec_zero_pad_to_16(__vi)));
946 else if constexpr (sizeof(_TV) == 8)
947 return __builtin_ia32_vcvtph2ps(__vec_zero_pad_to_16(__vi));
948 else if constexpr (sizeof(_TV) == 16)
949 return __builtin_ia32_vcvtph2ps256(__vi);
950 else if constexpr (sizeof(_TV) == 32)
951 return __builtin_ia32_vcvtph2ps512_mask(__vi, __vec_builtin_type<float, 16>(), -1, 4);
952 else if constexpr (sizeof(_TV) >= 64)
953 return __vec_concat(__x86_cvt_f16c<__half_vec_builtin_t<_UV>>(__vec_split_lo(__v)),
954 __x86_cvt_f16c<__half_vec_builtin_t<_UV>>(__vec_split_hi(__v)));
955 else
956 static_assert(false);
957 }
958 else if constexpr (sizeof(_TV) == 8)
959 return reinterpret_cast<_UV>(
960 __vec_split_lo(__vec_split_lo(__builtin_ia32_vcvtps2ph(
961 __vec_zero_pad_to_16(__v), 4))));
962 else if constexpr (sizeof(_TV) == 16)
963 return reinterpret_cast<_UV>(__vec_split_lo(__builtin_ia32_vcvtps2ph(__v, 4)));
964 else if constexpr (sizeof(_TV) == 32)
965 return reinterpret_cast<_UV>(__builtin_ia32_vcvtps2ph256(__v, 4));
966 else if constexpr (sizeof(_TV) == 64)
967 return reinterpret_cast<_UV>(__builtin_ia32_vcvtps2ph512_mask(
968 __v, 4, __vec_builtin_type<short, 16>(), -1));
969 else if constexpr (sizeof(_TV) >= 128)
970 return __vec_concat(__x86_cvt_f16c<__half_vec_builtin_t<_UV>>(__vec_split_lo(__v)),
971 __x86_cvt_f16c<__half_vec_builtin_t<_UV>>(__vec_split_hi(__v)));
972 else
973 static_assert(false);
974 }
975
976 /** @internal
977 * AVX instructions typically work per 128-bit chunk. Horizontal operations thus produce vectors
978 * where the two 128-bit chunks in the center are swapped. This function works as a fix-up step.
979 */
980 template <__vec_builtin _TV>
981 [[__gnu__::__always_inline__]]
982 inline _TV
983 __x86_swizzle4x64_acbd(_TV __x)
984 {
985 static_assert(sizeof(_TV) == 32);
986 using _UV = __vec_builtin_type_bytes<long long, 32>;
987 return reinterpret_cast<_TV>(__builtin_shufflevector(reinterpret_cast<_UV>(__x), _UV(),
988 0, 2, 1, 3));
989 }
990
991 /** @internal
992 * Like __builtin_convertvector but with a precondition that input values are either 0 or -1.
993 */
994 template <__vec_builtin _To, __vec_builtin _From>
995 [[__gnu__::__always_inline__]]
996 inline _To
997 __x86_cvt_vecmask(_From __k)
998 {
999 using _T0 = __vec_value_type<_From>;
1000 using _T1 = __vec_value_type<_To>;
1001 if constexpr (sizeof(_From) > sizeof(_To) && sizeof(_From) < 16)
1002 {
1003 using _ToPadded = __vec_builtin_type_bytes<_T1, sizeof(_To) * 16 / sizeof(_From)>;
1004 return _VecOps<_To>::_S_extract(__x86_cvt_vecmask<_ToPadded>(__vec_zero_pad_to_16(__k)));
1005 }
1006 else if constexpr (sizeof(_T0) == 2 && sizeof(_T1) == 1) // -> packsswb
1007 {
1008 if constexpr (sizeof(__k) == 16)
1009 return reinterpret_cast<_To>(__vec_split_lo(__builtin_ia32_packsswb128(__k, __k)));
1010 else if constexpr (sizeof(__k) == 32)
1011 return reinterpret_cast<_To>(
1012 __vec_split_lo(__x86_swizzle4x64_acbd(
1013 __builtin_ia32_packsswb256(__k, __k))));
1014 else
1015 static_assert(false);
1016 }
1017 else
1018 static_assert(false, "TODO");
1019 }
1020
1021 /** @internal
1022 * Overload that concatenates @p __k0 and @p __k1 while converting.
1023 */
1024 template <__vec_builtin _To, __vec_builtin _From>
1025 [[__gnu__::__always_inline__]]
1026 inline _To
1027 __x86_cvt_vecmask(_From __k0, _From __k1)
1028 {
1029 using _T0 = __vec_value_type<_From>;
1030 using _T1 = __vec_value_type<_To>;
1031 static_assert(sizeof(_From) >= 16);
1032 if constexpr (sizeof(_T0) == 2 && sizeof(_T1) == 1) // -> packsswb
1033 {
1034 if constexpr (sizeof(__k0) == 16)
1035 return reinterpret_cast<_To>(__builtin_ia32_packsswb128(__k0, __k1));
1036 else if constexpr (sizeof(__k0) == 32)
1037 return reinterpret_cast<_To>(__x86_swizzle4x64_acbd(
1038 __builtin_ia32_packsswb256(__k0, __k1)));
1039 else
1040 static_assert(false);
1041 }
1042 else
1043 static_assert(false, "TODO");
1044 }
1045
1046 template <_ArchTraits _Traits = {}, __vec_builtin _TV>
1047 [[__gnu__::__always_inline__]]
1048 inline _Bitmask<__width_of<_TV>>
1049 __x86_cvt_vecmask_to_bitmask(const _TV __k)
1050 {
1051 using _Tp = __vec_value_type<_TV>;
1052 constexpr int _Bytes = sizeof(_Tp);
1053 constexpr int _Np = __width_of<_TV>;
1054 constexpr bool __vl = _Traits._M_have_avx512vl();
1055 constexpr bool __bw = _Traits._M_have_avx512bw();
1056 constexpr bool __dq = _Traits._M_have_avx512dq();
1057 if constexpr (__vl && __bw && _Bytes == 1 && sizeof(__k) == 16)
1058 return __builtin_ia32_cvtb2mask128(__k);
1059 else if constexpr (__vl && __bw && _Bytes == 1 && sizeof(__k) == 32)
1060 return __builtin_ia32_cvtb2mask256(__k);
1061 else if constexpr (__bw && _Bytes == 1 && sizeof(__k) == 64)
1062 return __builtin_ia32_cvtb2mask512(__k);
1063 else if constexpr (__vl && __bw && _Bytes == 2 && sizeof(__k) == 16)
1064 return __builtin_ia32_cvtw2mask128(__k);
1065 else if constexpr (__vl && __bw && _Bytes == 2 && sizeof(__k) == 32)
1066 return __builtin_ia32_cvtw2mask256(__k);
1067 else if constexpr (__bw && _Bytes == 2 && sizeof(__k) == 64)
1068 return __builtin_ia32_cvtw2mask512(__k);
1069 else if constexpr (__vl && __dq && _Bytes == 4 && sizeof(__k) == 16)
1070 return __builtin_ia32_cvtd2mask128(__k);
1071 else if constexpr (__vl && __dq && _Bytes == 4 && sizeof(__k) == 32)
1072 return __builtin_ia32_cvtd2mask256(__k);
1073 else if constexpr (__dq && _Bytes == 4 && sizeof(__k) == 64)
1074 return __builtin_ia32_cvtd2mask512(__k);
1075 else if constexpr (__vl && __dq && _Bytes == 8 && sizeof(__k) == 16)
1076 return __builtin_ia32_cvtq2mask128(__k);
1077 else if constexpr (__vl && __dq && _Bytes == 8 && sizeof(__k) == 32)
1078 return __builtin_ia32_cvtq2mask256(__k);
1079 else if constexpr (__dq && _Bytes == 8 && sizeof(__k) == 64)
1080 return __builtin_ia32_cvtq2mask512(__k);
1081 else if constexpr (__vl && __dq && __bw && sizeof(__k) < 16)
1082 return __x86_cvt_vecmask_to_bitmask(__vec_zero_pad_to_16(__k));
1083 else if constexpr (_Bytes != 2) // movmskb would duplicate each bit
1084 return __x86_movmsk(__k);
1085 else if constexpr (_Traits._M_have_bmi2())
1086 return __bit_extract_even<_Np>(__x86_movmsk(__k));
1087 else
1088 return __x86_cvt_vecmask_to_bitmask(
1089 __x86_cvt_vecmask<__vec_builtin_type<char, _Np>>(__k));
1090 }
1091
1092 /** @internal
1093 * AVX512 masked (converting) loads
1094 *
1095 * @note AVX512VL and AVX512BW is required
1096 */
1097 template <__vec_builtin _TV, typename _Up, _ArchTraits _Traits = {}>
1098 [[__gnu__::__always_inline__]]
1099 inline _TV
1100 __x86_masked_load(const _Up* __mem, unsigned_integral auto __k)
1101 {
1102 static_assert(_Traits._M_have_avx512vl() && _Traits._M_have_avx512bw());
1103 using _Tp = __vec_value_type<_TV>;
1104 constexpr int __n = __width_of<_TV>;
1105 if constexpr (!__converts_trivially<_Up, _Tp>)
1106 {
1107 const auto __uvec
1108 = __x86_masked_load<__vec_builtin_type<__canonical_vec_type_t<_Up>, __n>>(__mem, __k);
1109 return __vec_cast<_TV>(__uvec);
1110 }
1111 else if constexpr (sizeof(_TV) < 16)
1112 {
1113 return _VecOps<_TV>::_S_extract(
1114 __x86_masked_load<__vec_builtin_type_bytes<_Tp, 16>>(__mem, __k));
1115 }
1116 else if constexpr (sizeof(_TV) > 64)
1117 {
1118 return __vec_concat(
1119 __x86_masked_load<__vec_builtin_type<_Tp, __n / 2>>(__mem, __k),
1120 __x86_masked_load<__vec_builtin_type<_Tp, __n / 2>>(__mem + __n / 2, __k >> __n / 2)
1121 );
1122 }
1123 else if constexpr (sizeof(_TV) == 64)
1124 {
1125 const auto* __src = reinterpret_cast<const __x86_intrin_type<_Up>*>(__mem);
1126 const __vec_builtin_type_bytes<__x86_intrin_type<_Up>, 64> __z = {};
1127 if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
1128 return __builtin_ia32_loadups512_mask(__src, __z, __k);
1129 else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
1130 return __builtin_ia32_loadupd512_mask(__src, __z, __k);
1131 else if constexpr (sizeof(_Tp) == 1)
1132 return reinterpret_cast<_TV>(__builtin_ia32_loaddquqi512_mask(__src, __z, __k));
1133 else if constexpr (sizeof(_Tp) == 2)
1134 return reinterpret_cast<_TV>(__builtin_ia32_loaddquhi512_mask(__src, __z, __k));
1135 else if constexpr (sizeof(_Tp) == 4)
1136 return reinterpret_cast<_TV>(__builtin_ia32_loaddqusi512_mask(__src, __z, __k));
1137 else if constexpr (sizeof(_Tp) == 8)
1138 return reinterpret_cast<_TV>(__builtin_ia32_loaddqudi512_mask(__src, __z, __k));
1139 else
1140 static_assert(false);
1141 }
1142 else if constexpr (sizeof(_TV) == 32)
1143 {
1144 const auto* __src = reinterpret_cast<const __x86_intrin_type<_Up>*>(__mem);
1145 const __vec_builtin_type_bytes<__x86_intrin_type<_Up>, 32> __z = {};
1146 if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
1147 return __builtin_ia32_loadups256_mask(__src, __z, __k);
1148 else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
1149 return __builtin_ia32_loadupd256_mask(__src, __z, __k);
1150 else if constexpr (sizeof(_Tp) == 1)
1151 return reinterpret_cast<_TV>(__builtin_ia32_loaddquqi256_mask(__src, __z, __k));
1152 else if constexpr (sizeof(_Tp) == 2)
1153 return reinterpret_cast<_TV>(__builtin_ia32_loaddquhi256_mask(__src, __z, __k));
1154 else if constexpr (sizeof(_Tp) == 4)
1155 return reinterpret_cast<_TV>(__builtin_ia32_loaddqusi256_mask(__src, __z, __k));
1156 else if constexpr (sizeof(_Tp) == 8)
1157 return reinterpret_cast<_TV>(__builtin_ia32_loaddqudi256_mask(__src, __z, __k));
1158 else
1159 static_assert(false);
1160 }
1161 else if constexpr (sizeof(_TV) == 16)
1162 {
1163 const auto* __src = reinterpret_cast<const __x86_intrin_type<_Up>*>(__mem);
1164 const __vec_builtin_type_bytes<__x86_intrin_type<_Up>, 16> __z = {};
1165 if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
1166 return __builtin_ia32_loadups128_mask(__src, __z, __k);
1167 else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
1168 return __builtin_ia32_loadupd128_mask(__src, __z, __k);
1169 else if constexpr (sizeof(_Tp) == 1)
1170 return reinterpret_cast<_TV>(__builtin_ia32_loaddquqi128_mask(__src, __z, __k));
1171 else if constexpr (sizeof(_Tp) == 2)
1172 return reinterpret_cast<_TV>(__builtin_ia32_loaddquhi128_mask(__src, __z, __k));
1173 else if constexpr (sizeof(_Tp) == 4)
1174 return reinterpret_cast<_TV>(__builtin_ia32_loaddqusi128_mask(__src, __z, __k));
1175 else if constexpr (sizeof(_Tp) == 8)
1176 return reinterpret_cast<_TV>(__builtin_ia32_loaddqudi128_mask(__src, __z, __k));
1177 else
1178 static_assert(false);
1179 }
1180 else
1181 static_assert(false);
1182 }
1183
1184 /** @internal
1185 * AVX(2) masked loads (only trivial conversions)
1186 */
1187 template <__vec_builtin _TV, typename _Up, __vec_builtin _KV, _ArchTraits _Traits = {}>
1188 [[__gnu__::__always_inline__]]
1189 inline _TV
1190 __x86_masked_load(const _Up* __mem, const _KV __k)
1191 {
1192 using _Tp = __vec_value_type<_TV>;
1193 static_assert(_Traits._M_have_avx() && __converts_trivially<_Up, _Tp> && sizeof(_Up) >= 4);
1194 constexpr int __n = __width_of<_TV>;
1195 using _IV = __vec_builtin_type<__x86_intrin_int<_Tp>, __n>;
1196 const auto __vk = reinterpret_cast<_IV>(__k);
1197 if constexpr (sizeof(_TV) < 16)
1198 return _VecOps<_TV>::_S_extract(__x86_masked_load<__vec_builtin_type_bytes<_Tp, 16>>(
1199 __mem, __vec_zero_pad_to_16(__k)));
1200 else if constexpr (_Traits._M_have_avx2() && is_integral_v<_Up>)
1201 {
1202 const auto* __src
1203 = reinterpret_cast<const __vec_builtin_type<__x86_intrin_int<_Up>, __n>*>(__mem);
1204 if constexpr (sizeof(_Up) == 4 && sizeof(_TV) == 32)
1205 return reinterpret_cast<_TV>(__builtin_ia32_maskloadd256(__src, __vk));
1206 else if constexpr (sizeof(_Up) == 4 && sizeof(_TV) == 16)
1207 return reinterpret_cast<_TV>(__builtin_ia32_maskloadd(__src, __vk));
1208 else if constexpr (sizeof(_Up) == 8 && sizeof(_TV) == 32)
1209 return reinterpret_cast<_TV>(__builtin_ia32_maskloadq256(__src, __vk));
1210 else if constexpr (sizeof(_Up) == 8 && sizeof(_TV) == 16)
1211 return reinterpret_cast<_TV>(__builtin_ia32_maskloadq(__src, __vk));
1212 else
1213 static_assert(false);
1214 }
1215 else if constexpr (sizeof(_Up) == 4)
1216 {
1217 const auto* __src = reinterpret_cast<const __vec_builtin_type<float, __n>*>(__mem);
1218 if constexpr (sizeof(_TV) == 32)
1219 return reinterpret_cast<_TV>(__builtin_ia32_maskloadps256(__src, __vk));
1220 else if constexpr (sizeof(_TV) == 16)
1221 return reinterpret_cast<_TV>(__builtin_ia32_maskloadps(__src, __vk));
1222 else
1223 static_assert(false);
1224 }
1225 else
1226 {
1227 const auto* __src = reinterpret_cast<const __vec_builtin_type<double, __n>*>(__mem);
1228 if constexpr (sizeof(_TV) == 32)
1229 return reinterpret_cast<_TV>(__builtin_ia32_maskloadpd256(__src, __vk));
1230 else if constexpr (sizeof(_TV) == 16)
1231 return reinterpret_cast<_TV>(__builtin_ia32_maskloadpd(__src, __vk));
1232 else
1233 static_assert(false);
1234 }
1235 }
1236
1237 /** @internal
1238 * AVX512 masked stores
1239 *
1240 * @note AVX512VL is required
1241 */
1242 template <__vec_builtin _TV, typename _Up>
1243 [[__gnu__::__always_inline__]]
1244 inline void
1245 __x86_masked_store(const _TV __v, _Up* __mem, unsigned_integral auto __k)
1246 {
1247 using _Tp = __vec_value_type<_TV>;
1248 constexpr int __n = __width_of<_TV>;
1249 [[maybe_unused]] const auto __w = __vec_bit_cast<__x86_intrin_type<_Tp>>(__v);
1250 if constexpr (sizeof(_TV) == 64)
1251 {
1252 if constexpr (sizeof(_Tp) > sizeof(_Up) && is_integral_v<_Tp> && is_integral_v<_Up>)
1253 {
1254 auto* __dst = reinterpret_cast<
1255 __vec_builtin_type<__x86_intrin_int<_Up>, __n>*>(__mem);
1256 if constexpr (sizeof(_Tp) == 2)
1257 __builtin_ia32_pmovwb512mem_mask(__dst, __w, __k);
1258 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 1)
1259 __builtin_ia32_pmovdb512mem_mask(__dst, __w, __k);
1260 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 2)
1261 __builtin_ia32_pmovdw512mem_mask(__dst, __w, __k);
1262 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 1)
1263 __builtin_ia32_pmovqb512mem_mask(__dst, __w, __k);
1264 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 2)
1265 __builtin_ia32_pmovqw512mem_mask(__dst, __w, __k);
1266 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 4)
1267 __builtin_ia32_pmovqd512mem_mask(__dst, __w, __k);
1268 else
1269 static_assert(false);
1270 }
1271 else if constexpr (__converts_trivially<_Tp, _Up>)
1272 {
1273 auto* __dst = reinterpret_cast<__x86_intrin_type<_Up>*>(__mem);
1274 if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
1275 __builtin_ia32_storeups512_mask(__dst, __w, __k);
1276 else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
1277 __builtin_ia32_storeupd512_mask(__dst, __w, __k);
1278 else if constexpr (sizeof(_Tp) == 1)
1279 __builtin_ia32_storedquqi512_mask(__dst, __w, __k);
1280 else if constexpr (sizeof(_Tp) == 2)
1281 __builtin_ia32_storedquhi512_mask(__dst, __w, __k);
1282 else if constexpr (sizeof(_Tp) == 4)
1283 __builtin_ia32_storedqusi512_mask(__dst, __w, __k);
1284 else if constexpr (sizeof(_Tp) == 8)
1285 __builtin_ia32_storedqudi512_mask(__dst, __w, __k);
1286 else
1287 static_assert(false);
1288 }
1289 else if constexpr (sizeof(_Tp) >= sizeof(_Up))
1290 {
1291 if constexpr (is_floating_point_v<_Tp> && is_integral_v<_Up>
1292 && sizeof(_Tp) > sizeof(_Up))
1293 __x86_masked_store(__vec_cast<__integer_from<sizeof(_Tp)>>(__v), __mem, __k);
1294 else
1295 __x86_masked_store(__vec_cast<_Up>(__v), __mem, __k);
1296 }
1297 else
1298 {
1299 __x86_masked_store(__vec_split_lo(__v), __mem, _Bitmask<__n / 2>(__k));
1300 __x86_masked_store(__vec_split_hi(__v), __mem + __n / 2,
1301 _Bitmask<__n / 2>(__k >> (__n / 2)));
1302 }
1303 }
1304 else if constexpr (sizeof(_TV) == 32)
1305 {
1306 if constexpr (sizeof(_Tp) > sizeof(_Up) && is_integral_v<_Tp> && is_integral_v<_Up>)
1307 {
1308 auto* __dst = reinterpret_cast<
1309 __vec_builtin_type<__x86_intrin_int<_Up>, __n>*>(__mem);
1310 if constexpr (sizeof(_Tp) == 2)
1311 __builtin_ia32_pmovwb256mem_mask(__dst, __w, __k);
1312 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 1)
1313 __builtin_ia32_pmovdb256mem_mask(__dst, __w, __k);
1314 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 2)
1315 __builtin_ia32_pmovdw256mem_mask(__dst, __w, __k);
1316 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 1)
1317 __builtin_ia32_pmovqb256mem_mask(__dst, __w, __k);
1318 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 2)
1319 __builtin_ia32_pmovqw256mem_mask(__dst, __w, __k);
1320 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 4)
1321 __builtin_ia32_pmovqd256mem_mask(__dst, __w, __k);
1322 else
1323 static_assert(false);
1324 }
1325 else if constexpr (__converts_trivially<_Tp, _Up>)
1326 {
1327 auto* __dst = reinterpret_cast<__x86_intrin_type<_Up>*>(__mem);
1328 if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
1329 __builtin_ia32_storeups256_mask(__dst, __w, __k);
1330 else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
1331 __builtin_ia32_storeupd256_mask(__dst, __w, __k);
1332 else if constexpr (sizeof(_Tp) == 1)
1333 __builtin_ia32_storedquqi256_mask(__dst, __w, __k);
1334 else if constexpr (sizeof(_Tp) == 2)
1335 __builtin_ia32_storedquhi256_mask(__dst, __w, __k);
1336 else if constexpr (sizeof(_Tp) == 4)
1337 __builtin_ia32_storedqusi256_mask(__dst, __w, __k);
1338 else if constexpr (sizeof(_Tp) == 8)
1339 __builtin_ia32_storedqudi256_mask(__dst, __w, __k);
1340 else
1341 static_assert(false);
1342 }
1343 else if constexpr (2 * sizeof(_Tp) >= sizeof(_Up))
1344 {
1345 __x86_masked_store(__vec_cast<_Up>(__v), __mem, __k);
1346 }
1347 else
1348 {
1349 __x86_masked_store(__vec_split_lo(__v), __mem, _Bitmask<__n / 2>(__k));
1350 __x86_masked_store(__vec_split_hi(__v), __mem + __n / 2,
1351 _Bitmask<__n / 2>(__k >> (__n / 2)));
1352 }
1353 }
1354 else if constexpr (sizeof(_TV) == 16)
1355 {
1356 if constexpr (sizeof(_Tp) > sizeof(_Up) && is_integral_v<_Tp> && is_integral_v<_Up>)
1357 {
1358 auto* __dst = reinterpret_cast<
1359 __vec_builtin_type<__x86_intrin_int<_Up>, __n>*>(__mem);
1360 if constexpr (sizeof(_Tp) == 2)
1361 __builtin_ia32_pmovwb128mem_mask(__dst, __w, __k);
1362 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 1)
1363 __builtin_ia32_pmovdb128mem_mask(__dst, __w, __k);
1364 else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 2)
1365 __builtin_ia32_pmovdw128mem_mask(__dst, __w, __k);
1366 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 1)
1367 __builtin_ia32_pmovqb128mem_mask(__dst, __w, __k);
1368 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 2)
1369 __builtin_ia32_pmovqw128mem_mask(__dst, __w, __k);
1370 else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 4)
1371 __builtin_ia32_pmovqd128mem_mask(reinterpret_cast<unsigned long long*>(__mem),
1372 __w, __k);
1373 else
1374 static_assert(false);
1375 }
1376 else if constexpr (__converts_trivially<_Tp, _Up>)
1377 {
1378 auto* __dst = reinterpret_cast<__x86_intrin_type<_Up>*>(__mem);
1379 if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 4)
1380 __builtin_ia32_storeups128_mask(__dst, __w, __k);
1381 else if constexpr (is_floating_point_v<_Tp> && sizeof(_Tp) == 8)
1382 __builtin_ia32_storeupd128_mask(__dst, __w, __k);
1383 else if constexpr (sizeof(_Tp) == 1)
1384 __builtin_ia32_storedquqi128_mask(__dst, __w, __k);
1385 else if constexpr (sizeof(_Tp) == 2)
1386 __builtin_ia32_storedquhi128_mask(__dst, __w, __k);
1387 else if constexpr (sizeof(_Tp) == 4)
1388 __builtin_ia32_storedqusi128_mask(__dst, __w, __k);
1389 else if constexpr (sizeof(_Tp) == 8)
1390 __builtin_ia32_storedqudi128_mask(__dst, __w, __k);
1391 else
1392 static_assert(false);
1393 }
1394 else if constexpr (4 * sizeof(_Tp) >= sizeof(_Up))
1395 {
1396 __x86_masked_store(__vec_cast<_Up>(__v), __mem, __k);
1397 }
1398 else
1399 {
1400 __x86_masked_store(__vec_cast<_Up>(__vec_split_lo(__v)), __mem,
1401 _Bitmask<__n / 2>(__k));
1402 __x86_masked_store(__vec_cast<_Up>(__vec_split_hi(__v)), __mem + __n / 2,
1403 _Bitmask<__n / 2>(__k >> (__n / 2)));
1404 }
1405 }
1406 else
1407 __x86_masked_store(__vec_zero_pad_to_16(__v), __mem, __k);
1408 }
1409
1410 /** @internal
1411 * AVX(2) masked stores
1412 */
1413 template <__vec_builtin _TV, typename _Up, __vec_builtin _KV, _ArchTraits _Traits = {}>
1414 [[__gnu__::__always_inline__]]
1415 inline void
1416 __x86_masked_store(const _TV __v, _Up* __mem, const _KV __k)
1417 {
1418 using _Tp = __vec_value_type<_TV>;
1419 constexpr int __n = __width_of<_TV>;
1420 static_assert(sizeof(_Tp) == 4 || sizeof(_Tp) == 8);
1421 auto* __dst = reinterpret_cast<
1422 __vec_builtin_type<__x86_intrin_type<_Up>, __n>*>(__mem);
1423 [[maybe_unused]] const auto __w = __vec_bit_cast<__x86_intrin_type<_Tp>>(__v);
1424 if constexpr (sizeof(_TV) < 16)
1425 __x86_masked_store(__vec_zero_pad_to_16(__v), __mem, __vec_zero_pad_to_16(__k));
1426 else if constexpr (_Traits._M_have_avx2() && is_integral_v<_Tp>)
1427 {
1428 if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 4)
1429 __builtin_ia32_maskstored256(__dst, __k, __w);
1430 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 4)
1431 __builtin_ia32_maskstored(__dst, __k, __w);
1432 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 8)
1433 __builtin_ia32_maskstoreq256(__dst, __k, __w);
1434 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 8)
1435 __builtin_ia32_maskstoreq(__dst, __k, __w);
1436 else
1437 static_assert(false);
1438 }
1439 else
1440 {
1441 if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 4)
1442 __builtin_ia32_maskstoreps256(__dst, __k, __w);
1443 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 4)
1444 __builtin_ia32_maskstoreps(__dst, __k, __w);
1445 else if constexpr (sizeof(_TV) == 32 && sizeof(_Tp) == 8)
1446 __builtin_ia32_maskstorepd256(__dst, __k, __w);
1447 else if constexpr (sizeof(_TV) == 16 && sizeof(_Tp) == 8)
1448 __builtin_ia32_maskstorepd(__dst, __k, __w);
1449 else
1450 static_assert(false);
1451 }
1452 }
1453} // namespace simd
1454_GLIBCXX_END_NAMESPACE_VERSION
1455} // namespace std
1456
1457#pragma GCC diagnostic pop
1458#endif // C++26
1459#endif // _GLIBCXX_SIMD_X86_H
std
ISO C++ entities toplevel namespace is std.