libstdc++: vec_ops.h Source File

libstdc++

vec_ops.h
1// Implementation of <simd> -*- C++ -*-
2
3// Copyright The GNU Toolchain Authors.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_VEC_OPS_H
26#define _GLIBCXX_VEC_OPS_H 1
27
28#ifdef _GLIBCXX_SYSHDR
29#pragma GCC system_header
30#endif
31
32#if __cplusplus >= 202400L
33
34#include "simd_details.h"
35
36#include <bit>
37#include <bits/utility.h>
38
39// psabi warnings are bogus because the ABI of the internal types never leaks into user code
40#pragma GCC diagnostic push
41#pragma GCC diagnostic ignored "-Wpsabi"
42
43namespace std _GLIBCXX_VISIBILITY(default)
44{
45_GLIBCXX_BEGIN_NAMESPACE_VERSION
46namespace simd
47{
48 template <std::signed_integral _Tp>
49 constexpr bool
50 __signed_has_single_bit(_Tp __x)
51 { return __has_single_bit(make_unsigned_t<_Tp>(__x)); }
52
53 /**
54 * Alias for a vector builtin with given value type and total sizeof.
55 */
56 template <__vectorizable _Tp, size_t _Bytes>
57 requires (__has_single_bit(_Bytes))
58 using __vec_builtin_type_bytes [[__gnu__::__vector_size__(_Bytes)]] = _Tp;
59
60 /**
61 * Alias for a vector builtin with given value type @p _Tp and @p _Width.
62 */
63 template <__vectorizable _Tp, __simd_size_type _Width>
64 requires (__signed_has_single_bit(_Width))
65 using __vec_builtin_type = __vec_builtin_type_bytes<_Tp, sizeof(_Tp) * _Width>;
66
67 /**
68 * Constrain to any vector builtin with given value type and optional width.
69 */
70 template <typename _Tp, typename _ValueType,
71 __simd_size_type _Width = sizeof(_Tp) / sizeof(_ValueType)>
72 concept __vec_builtin_of
73 = !is_class_v<_Tp> && !is_pointer_v<_Tp> && !is_arithmetic_v<_Tp>
74 && __vectorizable<_ValueType>
75 && _Width >= 1 && sizeof(_Tp) / sizeof(_ValueType) == _Width
76 && same_as<__vec_builtin_type_bytes<_ValueType, sizeof(_Tp)>, _Tp>
77 && requires(_Tp& __v, _ValueType __x) { __v[0] = __x; };
78
79 /**
80 * Constrain to any vector builtin.
81 */
82 template <typename _Tp>
83 concept __vec_builtin
84 = __vec_builtin_of<_Tp, remove_cvref_t<decltype(declval<const _Tp>()[0])>>;
85
86 /**
87 * Alias for the value type of the given __vec_builtin type @p _Tp.
88 */
89 template <__vec_builtin _Tp>
90 using __vec_value_type = remove_cvref_t<decltype(declval<const _Tp>()[0])>;
91
92 /**
93 * The width (number of value_type elements) of the given vector builtin or arithmetic type.
94 */
95 template <typename _Tp>
96 inline constexpr __simd_size_type __width_of = 1;
97
98 template <typename _Tp>
99 requires __vec_builtin<_Tp>
100 inline constexpr __simd_size_type __width_of<_Tp> = sizeof(_Tp) / sizeof(__vec_value_type<_Tp>);
101
102 /**
103 * Alias for a vector builtin with equal value type and new width @p _Np.
104 */
105 template <__simd_size_type _Np, __vec_builtin _TV>
106 using __resize_vec_builtin_t = __vec_builtin_type<__vec_value_type<_TV>, _Np>;
107
108 template <__vec_builtin _TV>
109 requires (__width_of<_TV> > 1)
110 using __half_vec_builtin_t = __resize_vec_builtin_t<__width_of<_TV> / 2, _TV>;
111
112 template <__vec_builtin _TV>
113 using __double_vec_builtin_t = __resize_vec_builtin_t<__width_of<_TV> * 2, _TV>;
114
115 template <typename _Up, __vec_builtin _TV>
116 [[__gnu__::__always_inline__]]
117 constexpr __vec_builtin_type_bytes<_Up, sizeof(_TV)>
118 __vec_bit_cast(_TV __v)
119 { return reinterpret_cast<__vec_builtin_type_bytes<_Up, sizeof(_TV)>>(__v); }
120
121 template <int _Np, __vec_builtin _TV>
122 requires signed_integral<__vec_value_type<_TV>>
123 static constexpr _TV _S_vec_implicit_mask = []<int... _Is> (integer_sequence<int, _Is...>) {
124 return _TV{ (_Is < _Np ? -1 : 0)... };
125 } (make_integer_sequence<int, __width_of<_TV>>());
126
127 /**
128 * Helper function to work around Clang not allowing v[i] in constant expressions.
129 */
130 template <__vec_builtin _TV>
131 [[__gnu__::__always_inline__]]
132 constexpr __vec_value_type<_TV>
133 __vec_get(_TV __v, int __i)
134 {
135#ifdef _GLIBCXX_CLANG
136 if consteval
137 {
138 return __builtin_bit_cast(array<__vec_value_type<_TV>, __width_of<_TV>>, __v)[__i];
139 }
140 else
141#endif
142 {
143 return __v[__i];
144 }
145 }
146
147 /**
148 * Helper function to work around Clang and GCC not allowing assignment to v[i] in constant
149 * expressions.
150 */
151 template <__vec_builtin _TV>
152 [[__gnu__::__always_inline__]]
153 constexpr void
154 __vec_set(_TV& __v, int __i, __vec_value_type<_TV> __x)
155 {
156 if consteval
157 {
158#ifdef _GLIBCXX_CLANG
159 auto __arr = __builtin_bit_cast(array<__vec_value_type<_TV>, __width_of<_TV>>, __v);
160 __arr[__i] = __x;
161 __v = __builtin_bit_cast(_TV, __arr);
162#else
163 constexpr auto [...__j] = _IotaArray<__width_of<_TV>>;
164 __v = _TV{(__i == __j ? __x : __v[__j])...};
165#endif
166 }
167 else
168 {
169 __v[__i] = __x;
170 }
171 }
172
173 /** @internal
174 * Return vector builtin with all values from @p __a and @p __b.
175 */
176 template <__vec_builtin _TV>
177 [[__gnu__::__always_inline__]]
178 constexpr __vec_builtin_type<__vec_value_type<_TV>, __width_of<_TV> * 2>
179 __vec_concat(_TV __a, _TV __b)
180 {
181 constexpr auto [...__is] = _IotaArray<__width_of<_TV> * 2>;
182 return __builtin_shufflevector(__a, __b, __is...);
183 }
184
185 /** @internal
186 * Concatenate the first @p _N0 elements from @p __a with the first @p _N1 elements from @p __b
187 * with the elements from applying this function recursively to @p __rest.
188 *
189 * @pre _N0 <= __width_of<_TV0> && _N1 <= __width_of<_TV1> && _Ns <= __width_of<_TVs> && ...
190 *
191 * Strategy: Aim for a power-of-2 tree concat. E.g.
192 * - cat(2, 2, 2, 2) -> cat(4, 2, 2) -> cat(4, 4)
193 * - cat(2, 2, 2, 2, 8) -> cat(4, 2, 2, 8) -> cat(4, 4, 8) -> cat(8, 8)
194 */
195 template <int _N0, int _N1, int... _Ns, __vec_builtin _TV0, __vec_builtin _TV1,
196 __vec_builtin... _TVs>
197 [[__gnu__::__always_inline__]]
198 constexpr __vec_builtin_type<__vec_value_type<_TV0>,
199 __bit_ceil(unsigned(_N0 + (_N1 + ... + _Ns)))>
200 __vec_concat_sized(const _TV0& __a, const _TV1& __b, const _TVs&... __rest);
201
202 template <int _N0, int _N1, int _N2, int... _Ns, __vec_builtin _TV0, __vec_builtin _TV1,
203 __vec_builtin _TV2, __vec_builtin... _TVs>
204 requires (__has_single_bit(unsigned(_N0))) && (_N0 >= (_N1 + _N2))
205 [[__gnu__::__always_inline__]]
206 constexpr __vec_builtin_type<__vec_value_type<_TV0>,
207 __bit_ceil(unsigned(_N0 + _N1 + (_N2 + ... + _Ns)))>
208 __vec_concat_sized(const _TV0& __a, const _TV1& __b, const _TV2& __c, const _TVs&... __rest)
209 {
210 return __vec_concat_sized<_N0, _N1 + _N2, _Ns...>(
211 __a, __vec_concat_sized<_N1, _N2>(__b, __c), __rest...);
212 }
213
214 template <int _N0, int _N1, int... _Ns, __vec_builtin _TV0, __vec_builtin _TV1,
215 __vec_builtin... _TVs>
216 [[__gnu__::__always_inline__]]
217 constexpr __vec_builtin_type<__vec_value_type<_TV0>,
218 __bit_ceil(unsigned(_N0 + (_N1 + ... + _Ns)))>
219 __vec_concat_sized(const _TV0& __a, const _TV1& __b, const _TVs&... __rest)
220 {
221 // __is is rounded up because we need to generate a power-of-2 vector:
222 constexpr auto [...__is] = _IotaArray<__bit_ceil(unsigned(_N0 + _N1)), int>;
223 const auto __ab = __builtin_shufflevector(__a, __b, [](int __i) consteval {
224 if (__i < _N0) // copy from __a
225 return __i;
226 else if (__i < _N0 + _N1) // copy from __b
227 return __i - _N0 + __width_of<_TV0>; // _N0 <= __width_of<_TV0>
228 else // can't index into __rest
229 return -1; // don't care
230 }(__is)...);
231 if constexpr (sizeof...(__rest) == 0)
232 return __ab;
233 else
234 return __vec_concat_sized<_N0 + _N1, _Ns...>(__ab, __rest...);
235 }
236
237 template <__vec_builtin _TV>
238 [[__gnu__::__always_inline__]]
239 constexpr __half_vec_builtin_t<_TV>
240 __vec_split_lo(_TV __v)
241 {
242 constexpr int __n = __width_of<_TV> / 2;
243 constexpr auto [...__is] = _IotaArray<__n>;
244 return __builtin_shufflevector(__v, __v, __is...);
245 }
246
247 template <__vec_builtin _TV>
248 [[__gnu__::__always_inline__]]
249 constexpr __half_vec_builtin_t<_TV>
250 __vec_split_hi(_TV __v)
251 {
252 constexpr int __n = __width_of<_TV> / 2;
253 constexpr auto [...__is] = _IotaArray<__n>;
254 return __builtin_shufflevector(__v, __v, (__n + __is)...);
255 }
256
257 /** @internal
258 * Return @p __x zero-padded to @p _Bytes bytes.
259 *
260 * Use this function when you need two objects of the same size (e.g. for __vec_concat).
261 */
262 template <size_t _Bytes, __vec_builtin _TV>
263 [[__gnu__::__always_inline__]]
264 constexpr auto
265 __vec_zero_pad_to(_TV __x)
266 {
267 if constexpr (sizeof(_TV) == _Bytes)
268 return __x;
269 else if constexpr (sizeof(_TV) <= sizeof(0ull))
270 {
271 using _Up = _UInt<sizeof(_TV)>;
272 __vec_builtin_type_bytes<_Up, _Bytes> __tmp = {__builtin_bit_cast(_Up, __x)};
273 return __builtin_bit_cast(__vec_builtin_type_bytes<__vec_value_type<_TV>, _Bytes>, __tmp);
274 }
275 else if constexpr (sizeof(_TV) < _Bytes)
276 return __vec_zero_pad_to<_Bytes>(__vec_concat(__x, _TV()));
277 else
278 static_assert(false);
279 }
280
281 /** @internal
282 * Return a type with sizeof 16, add zero-padding to @p __x. The input must be smaller.
283 *
284 * Use this function instead of the above when you need to pad an argument for a SIMD builtin.
285 */
286 template <__vec_builtin _TV>
287 [[__gnu__::__always_inline__]]
288 constexpr auto
289 __vec_zero_pad_to_16(_TV __x)
290 {
291 static_assert(sizeof(_TV) < 16);
292 return __vec_zero_pad_to<16>(__x);
293 }
294
295 // work around __builtin_constant_p returning false unless passed a variable
296 // (__builtin_constant_p(x[0]) is false while __is_const_known(x[0]) is true)
297 template <typename _Tp>
298 [[__gnu__::__always_inline__]]
299 constexpr bool
300 __is_const_known(const _Tp& __x)
301 {
302 if constexpr (__complex_like<_Tp>)
303 return __is_const_known(__x.real()) && __is_const_known(__x.imag());
304 else
305 return __builtin_constant_p(__x);
306 }
307
308 [[__gnu__::__always_inline__]]
309 constexpr bool
310 __is_const_known(const auto&... __xs) requires(sizeof...(__xs) >= 2)
311 {
312 if consteval
313 {
314 return true;
315 }
316 else
317 {
318 return (__is_const_known(__xs) && ...);
319 }
320 }
321
322 [[__gnu__::__always_inline__]]
323 constexpr bool
324 __is_const_known_equal_to(const auto& __x, const auto& __expect)
325 { return __is_const_known(__x == __expect) && __x == __expect; }
326
327#if _GLIBCXX_X86
328 template <__vec_builtin _UV, __vec_builtin _TV>
329 inline _UV
330 __x86_cvt_f16c(_TV __v);
331#endif
332
333
334 /** @internal
335 * Simple wrapper around __builtin_convertvector to provide static_cast-like syntax.
336 *
337 * Works around GCC failing to use the F16C/AVX512F cvtps2ph/cvtph2ps instructions.
338 */
339 template <__vec_builtin _UV, __vec_builtin _TV, _ArchTraits _Traits = {}>
340 [[__gnu__::__always_inline__]]
341 constexpr _UV
342 __vec_cast(_TV __v)
343 {
344 static_assert(__width_of<_UV> == __width_of<_TV>);
345#if _GLIBCXX_X86
346 using _Up = __vec_value_type<_UV>;
347 using _Tp = __vec_value_type<_TV>;
348 constexpr bool __to_f16 = is_same_v<_Up, _Float16>;
349 constexpr bool __from_f16 = is_same_v<_Tp, _Float16>;
350 constexpr bool __needs_f16c = _Traits._M_have_f16c() && !_Traits._M_have_avx512fp16()
351 && (__to_f16 || __from_f16);
352 if (__needs_f16c && !__is_const_known(__v))
353 { // Work around PR121688
354 if constexpr (__needs_f16c)
355 return __x86_cvt_f16c<_UV>(__v);
356 }
357 if constexpr (is_floating_point_v<_Tp> && is_integral_v<_Up>
358 && sizeof(_UV) < sizeof(_TV) && sizeof(_Up) < sizeof(int))
359 {
360 using _Ip = __integer_from<std::min(sizeof(int), sizeof(_Tp))>;
361 using _IV = __vec_builtin_type<_Ip, __width_of<_TV>>;
362 return __vec_cast<_UV>(__vec_cast<_IV>(__v));
363 }
364#endif
365 return __builtin_convertvector(__v, _UV);
366 }
367
368 /** @internal
369 * Overload of the above cast function that determines the destination vector type from a given
370 * element type @p _Up and the `__width_of` the argument type.
371 *
372 * Calls the above overload.
373 */
374 template <__vectorizable _Up, __vec_builtin _TV>
375 [[__gnu__::__always_inline__]]
376 constexpr __vec_builtin_type<_Up, __width_of<_TV>>
377 __vec_cast(_TV __v)
378 { return __vec_cast<__vec_builtin_type<_Up, __width_of<_TV>>>(__v); }
379
380 /** @internal
381 * As above, but with additional precondition on possible values of the argument.
382 *
383 * Precondition: __k[i] is either 0 or -1 for all i.
384 */
385 template <__vec_builtin _UV, __vec_builtin _TV>
386 [[__gnu__::__always_inline__]]
387 constexpr _UV
388 __vec_mask_cast(_TV __k)
389 {
390 static_assert(signed_integral<__vec_value_type<_UV>>);
391 static_assert(signed_integral<__vec_value_type<_TV>>);
392 // TODO: __builtin_convertvector cannot be optimal because it doesn't consider input and
393 // output can only be 0 or -1.
394 return __builtin_convertvector(__k, _UV);
395 }
396
397 template <__vec_builtin _TV>
398 [[__gnu__::__always_inline__]]
399 constexpr _TV
400 __vec_xor(_TV __a, _TV __b)
401 {
402 using _Tp = __vec_value_type<_TV>;
403 if constexpr (is_floating_point_v<_Tp>)
404 {
405 using _UV = __vec_builtin_type<__integer_from<sizeof(_Tp)>, __width_of<_TV>>;
406 return __builtin_bit_cast(
407 _TV, __builtin_bit_cast(_UV, __a) ^ __builtin_bit_cast(_UV, __b));
408 }
409 else
410 return __a ^ __b;
411 }
412
413 template <__vec_builtin _TV>
414 [[__gnu__::__always_inline__]]
415 constexpr _TV
416 __vec_or(_TV __a, _TV __b)
417 {
418 using _Tp = __vec_value_type<_TV>;
419 if constexpr (is_floating_point_v<_Tp>)
420 {
421 using _UV = __vec_builtin_type<__integer_from<sizeof(_Tp)>, __width_of<_TV>>;
422 return __builtin_bit_cast(
423 _TV, __builtin_bit_cast(_UV, __a) | __builtin_bit_cast(_UV, __b));
424 }
425 else
426 return __a | __b;
427 }
428
429 template <__vec_builtin _TV>
430 [[__gnu__::__always_inline__]]
431 constexpr _TV
432 __vec_and(_TV __a, _TV __b)
433 {
434 using _Tp = __vec_value_type<_TV>;
435 if constexpr (is_floating_point_v<_Tp>)
436 {
437 using _UV = __vec_builtin_type<__integer_from<sizeof(_Tp)>, __width_of<_TV>>;
438 return __builtin_bit_cast(
439 _TV, __builtin_bit_cast(_UV, __a) & __builtin_bit_cast(_UV, __b));
440 }
441 else
442 return __a & __b;
443 }
444
445 /** @internal
446 * Returns the bit-wise and of not @p __a and @p __b.
447 *
448 * Use __vec_and(__vec_not(__a), __b) unless an andnot instruction is necessary for optimization.
449 *
450 * @see __vec_andnot in simd_x86.h
451 */
452 template <__vec_builtin _TV>
453 [[__gnu__::__always_inline__]]
454 constexpr _TV
455 __vec_andnot(_TV __a, _TV __b)
456 {
457 using _Tp = __vec_value_type<_TV>;
458 using _UV = __vec_builtin_type<__integer_from<sizeof(_Tp)>, __width_of<_TV>>;
459 return __builtin_bit_cast(
460 _TV, ~__builtin_bit_cast(_UV, __a) & __builtin_bit_cast(_UV, __b));
461 }
462
463 template <__vec_builtin _TV>
464 [[__gnu__::__always_inline__]]
465 constexpr _TV
466 __vec_not(_TV __a)
467 {
468 using _Tp = __vec_value_type<_TV>;
469 using _UV = __vec_builtin_type_bytes<__integer_from<sizeof(_Tp)>, sizeof(_TV)>;
470 if constexpr (is_floating_point_v<__vec_value_type<_TV>>)
471 return __builtin_bit_cast(_TV, ~__builtin_bit_cast(_UV, __a));
472 else
473 return ~__a;
474 }
475
476 /**
477 * An object of given type where only the sign bits are 1.
478 */
479 template <__vec_builtin _V>
480 requires std::floating_point<__vec_value_type<_V>>
481 constexpr _V _S_signmask = __vec_xor(_V() + 1, _V() - 1);
482
483 template <__vec_builtin _TV, int _Np = __width_of<_TV>,
484 typename = make_integer_sequence<int, _Np>>
485 struct _VecOps;
486
487 template <__vec_builtin _TV, int _Np, int... _Is>
488 struct _VecOps<_TV, _Np, integer_sequence<int, _Is...>>
489 {
490 static_assert(_Np <= __width_of<_TV>);
491
492 using _Tp = __vec_value_type<_TV>;
493
494 using _HV = __half_vec_builtin_t<__conditional_t<_Np >= 2, _TV, __double_vec_builtin_t<_TV>>>;
495
496 [[__gnu__::__always_inline__]]
497 static constexpr _TV
498 _S_broadcast_to_even(_Tp __init)
499 { return _TV {((_Is & 1) == 0 ? __init : _Tp())...}; }
500
501 [[__gnu__::__always_inline__]]
502 static constexpr _TV
503 _S_broadcast_to_odd(_Tp __init)
504 { return _TV {((_Is & 1) == 1 ? __init : _Tp())...}; }
505
506 [[__gnu__::__always_inline__]]
507 static constexpr bool
508 _S_all_of(_TV __k) noexcept
509 { return (... && (__k[_Is] != 0)); }
510
511 [[__gnu__::__always_inline__]]
512 static constexpr bool
513 _S_any_of(_TV __k) noexcept
514 { return (... || (__k[_Is] != 0)); }
515
516 [[__gnu__::__always_inline__]]
517 static constexpr bool
518 _S_none_of(_TV __k) noexcept
519 { return (... && (__k[_Is] == 0)); }
520
521 template <typename _Offset = integral_constant<int, 0>>
522 [[__gnu__::__always_inline__]]
523 static constexpr _TV
524 _S_extract(__vec_builtin auto __x, _Offset = {})
525 {
526 static_assert(is_same_v<__vec_value_type<_TV>, __vec_value_type<decltype(__x)>>);
527 return __builtin_shufflevector(__x, decltype(__x)(), (_Is + _Offset::value)...);
528 }
529
530 // swap neighboring elements
531 [[__gnu__::__always_inline__]]
532 static constexpr _TV
533 _S_swap_neighbors(_TV __x)
534 { return __builtin_shufflevector(__x, __x, (_Is ^ 1)...); }
535
536 // duplicate even indexed elements, dropping the odd ones
537 [[__gnu__::__always_inline__]]
538 static constexpr _TV
539 _S_dup_even(_TV __x)
540 { return __builtin_shufflevector(__x, __x, (_Is & ~1)...); }
541
542 // duplicate odd indexed elements, dropping the even ones
543 [[__gnu__::__always_inline__]]
544 static constexpr _TV
545 _S_dup_odd(_TV __x)
546 { return __builtin_shufflevector(__x, __x, (_Is | 1)...); }
547
548 [[__gnu__::__always_inline__]]
549 static constexpr void
550 _S_overwrite_even_elements(_TV& __x, _HV __y) requires (_Np > 1)
551 {
552 constexpr __simd_size_type __n = __width_of<_TV>;
553 __x = __builtin_shufflevector(__x,
554#ifdef _GLIBCXX_CLANG
555 __vec_concat(__y, __y),
556#else
557 __y,
558#endif
559 ((_Is & 1) == 0 ? __n + _Is / 2 : _Is)...);
560 }
561
562 [[__gnu__::__always_inline__]]
563 static constexpr void
564 _S_overwrite_even_elements(_TV& __xl, _TV& __xh, _TV __y)
565 {
566 constexpr __simd_size_type __nl = __width_of<_TV>;
567 constexpr __simd_size_type __nh = __nl * 3 / 2;
568 __xl = __builtin_shufflevector(__xl, __y, ((_Is & 1) == 0 ? __nl + _Is / 2 : _Is)...);
569 __xh = __builtin_shufflevector(__xh, __y, ((_Is & 1) == 0 ? __nh + _Is / 2 : _Is)...);
570 }
571
572 [[__gnu__::__always_inline__]]
573 static constexpr void
574 _S_overwrite_odd_elements(_TV& __x, _HV __y) requires (_Np > 1)
575 {
576 constexpr __simd_size_type __n = __width_of<_TV>;
577 __x = __builtin_shufflevector(__x,
578#ifdef _GLIBCXX_CLANG
579 __vec_concat(__y, __y),
580#else
581 __y,
582#endif
583 ((_Is & 1) == 1 ? __n + _Is / 2 : _Is)...);
584 }
585
586 [[__gnu__::__always_inline__]]
587 static constexpr void
588 _S_overwrite_odd_elements(_TV& __xl, _TV& __xh, _TV __y)
589 {
590 constexpr __simd_size_type __nl = __width_of<_TV>;
591 constexpr __simd_size_type __nh = __nl * 3 / 2;
592 __xl = __builtin_shufflevector(__xl, __y, ((_Is & 1) == 1 ? __nl + _Is / 2 : _Is)...);
593 __xh = __builtin_shufflevector(__xh, __y, ((_Is & 1) == 1 ? __nh + _Is / 2 : _Is)...);
594 }
595
596 // negate every even element (real part of interleaved complex)
597 [[__gnu__::__always_inline__]]
598 static constexpr _TV
599 _S_complex_negate_real(_TV __x)
600 { return __vec_xor(_S_broadcast_to_even(_S_signmask<_TV>[0]), __x); }
601
602 // negate every odd element (imaginary part of interleaved complex)
603 [[__gnu__::__always_inline__]]
604 static constexpr _TV
605 _S_complex_negate_imag(_TV __x)
606 { return __vec_xor(_S_broadcast_to_odd(_S_signmask<_TV>[0]), __x); }
607
608 // Subtract elements with even index, add elements with odd index.
609 template <_ArchTraits _Traits = {}>
610 [[__gnu__::__always_inline__]]
611 static constexpr _TV
612 _S_addsub(_TV __x, _TV __y)
613 {
614 if constexpr (_Traits._M_have_addsub())
615 // GCC recognizes this pattern as addsub
616 return __builtin_shufflevector(__x - __y, __x + __y,
617 (_Is + (_Is & 1) * __width_of<_TV>)...);
618 else
619 return __x + _S_complex_negate_real(__y);
620 }
621
622 // true if all elements are know to be equal to __ref at compile time
623 [[__gnu__::__always_inline__]]
624 static constexpr bool
625 _S_is_const_known_equal_to(_TV __x, _Tp __ref)
626 { return (__is_const_known_equal_to(__x[_Is], __ref) && ...); }
627
628 // True iff all elements at even indexes are zero. This includes signed zeros only when
629 // -fno-signed-zeros is in effect.
630 template <_OptTraits _Traits = {}>
631 [[__gnu__::__always_inline__]]
632 static constexpr bool
633 _S_complex_real_is_const_known_zero(_TV __x)
634 {
635 if constexpr (_Traits._M_conforming_to_STDC_annex_G())
636 {
637 using _Up = _UInt<sizeof(_Tp)>;
638 return (((_Is & 1) == 1 || __is_const_known_equal_to(
639 __builtin_bit_cast(_Up, __x[_Is]), _Up())) && ...);
640 }
641 else
642 return (((_Is & 1) == 1 || __is_const_known_equal_to(__x[_Is], _Tp())) && ...);
643 }
644
645 // True iff all elements at odd indexes are zero. This includes signed zeros only when
646 // -fno-signed-zeros is in effect.
647 template <_OptTraits _Traits = {}>
648 [[__gnu__::__always_inline__]]
649 static constexpr bool
650 _S_complex_imag_is_const_known_zero(_TV __x)
651 {
652 if constexpr (_Traits._M_conforming_to_STDC_annex_G())
653 {
654 using _Up = _UInt<sizeof(_Tp)>;
655 return (((_Is & 1) == 0 || __is_const_known_equal_to(
656 __builtin_bit_cast(_Up, __x[_Is]), _Up())) && ...);
657 }
658 else
659 return (((_Is & 1) == 0 || __is_const_known_equal_to(__x[_Is], _Tp())) && ...);
660 }
661 };
662} // namespace simd
663_GLIBCXX_END_NAMESPACE_VERSION
664} // namespace std
665
666#pragma GCC diagnostic pop
667#endif // C++26
668#endif // _GLIBCXX_VEC_OPS_H
bit
utility.h
std::make_unsigned_t
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition type_traits:2273
std::min
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition stl_algobase.h:232
std
ISO C++ entities toplevel namespace is std.
std::make_integer_sequence
__make_integer_seq< integer_sequence, _Tp, _Num > make_integer_sequence
Alias template make_integer_sequence.
Definition utility.h:516
std::integer_sequence
Class template integer_sequence.
Definition utility.h:470
std::same_as
[concept.same], concept same_as
Definition concepts:65
std::simd::__vec_builtin_of
Definition vec_ops.h:73
std::simd::__vec_builtin
Definition vec_ops.h:84