libstdc++: vec_ops.h Source File

libstdc++

vec_ops.h
1// Implementation of <simd> -*- C++ -*-
2
3// Copyright The GNU Toolchain Authors.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_VEC_OPS_H
26#define _GLIBCXX_VEC_OPS_H 1
27
28#ifdef _GLIBCXX_SYSHDR
29#pragma GCC system_header
30#endif
31
32#if __cplusplus >= 202400L
33
34#include "simd_details.h"
35
36#include <bit>
37#include <bits/utility.h>
38
39// psabi warnings are bogus because the ABI of the internal types never leaks into user code
40#pragma GCC diagnostic push
41#pragma GCC diagnostic ignored "-Wpsabi"
42
43namespace std _GLIBCXX_VISIBILITY(default)
44{
45_GLIBCXX_BEGIN_NAMESPACE_VERSION
46namespace simd
47{
48 template <std::signed_integral _Tp>
49 constexpr bool
50 __signed_has_single_bit(_Tp __x)
51 { return __has_single_bit(make_unsigned_t<_Tp>(__x)); }
52
53 /**
54 * Alias for a vector builtin with given value type and total sizeof.
55 */
56 template <__vectorizable _Tp, size_t _Bytes>
57 requires (__has_single_bit(_Bytes))
58 using __vec_builtin_type_bytes [[__gnu__::__vector_size__(_Bytes)]] = _Tp;
59
60 /**
61 * Alias for a vector builtin with given value type @p _Tp and @p _Width.
62 */
63 template <__vectorizable _Tp, __simd_size_type _Width>
64 requires (__signed_has_single_bit(_Width))
65 using __vec_builtin_type = __vec_builtin_type_bytes<_Tp, sizeof(_Tp) * _Width>;
66
67 /**
68 * Constrain to any vector builtin with given value type and optional width.
69 */
70 template <typename _Tp, typename _ValueType,
71 __simd_size_type _Width = sizeof(_Tp) / sizeof(_ValueType)>
72 concept __vec_builtin_of
73 = !is_class_v<_Tp> && !is_pointer_v<_Tp> && !is_arithmetic_v<_Tp>
74 && __vectorizable<_ValueType>
75 && _Width >= 1 && sizeof(_Tp) / sizeof(_ValueType) == _Width
76 && same_as<__vec_builtin_type_bytes<_ValueType, sizeof(_Tp)>, _Tp>
77 && requires(_Tp& __v, _ValueType __x) { __v[0] = __x; };
78
79 /**
80 * Constrain to any vector builtin.
81 */
82 template <typename _Tp>
83 concept __vec_builtin
84 = __vec_builtin_of<_Tp, remove_cvref_t<decltype(declval<const _Tp>()[0])>>;
85
86 /**
87 * Alias for the value type of the given __vec_builtin type @p _Tp.
88 */
89 template <__vec_builtin _Tp>
90 using __vec_value_type = remove_cvref_t<decltype(declval<const _Tp>()[0])>;
91
92 /**
93 * The width (number of value_type elements) of the given vector builtin or arithmetic type.
94 */
95 template <typename _Tp>
96 inline constexpr __simd_size_type __width_of = 1;
97
98 template <typename _Tp>
99 requires __vec_builtin<_Tp>
100 inline constexpr __simd_size_type __width_of<_Tp> = sizeof(_Tp) / sizeof(__vec_value_type<_Tp>);
101
102 /**
103 * Alias for a vector builtin with equal value type and new width @p _Np.
104 */
105 template <__simd_size_type _Np, __vec_builtin _TV>
106 using __resize_vec_builtin_t = __vec_builtin_type<__vec_value_type<_TV>, _Np>;
107
108 template <__vec_builtin _TV>
109 requires (__width_of<_TV> > 1)
110 using __half_vec_builtin_t = __resize_vec_builtin_t<__width_of<_TV> / 2, _TV>;
111
112 template <__vec_builtin _TV>
113 using __double_vec_builtin_t = __resize_vec_builtin_t<__width_of<_TV> * 2, _TV>;
114
115 template <typename _Up, __vec_builtin _TV>
116 [[__gnu__::__always_inline__]]
117 constexpr __vec_builtin_type_bytes<_Up, sizeof(_TV)>
118 __vec_bit_cast(_TV __v)
119 { return reinterpret_cast<__vec_builtin_type_bytes<_Up, sizeof(_TV)>>(__v); }
120
121 template <int _Np, __vec_builtin _TV>
122 requires signed_integral<__vec_value_type<_TV>>
123 static constexpr _TV _S_vec_implicit_mask = []<int... _Is> (integer_sequence<int, _Is...>) {
124 return _TV{ (_Is < _Np ? -1 : 0)... };
125 } (make_integer_sequence<int, __width_of<_TV>>());
126
127 /**
128 * Helper function to work around Clang not allowing v[i] in constant expressions.
129 */
130 template <__vec_builtin _TV>
131 [[__gnu__::__always_inline__]]
132 constexpr __vec_value_type<_TV>
133 __vec_get(_TV __v, int __i)
134 {
135#ifdef _GLIBCXX_CLANG
136 if consteval
137 {
138 return __builtin_bit_cast(array<__vec_value_type<_TV>, __width_of<_TV>>, __v)[__i];
139 }
140 else
141#endif
142 {
143 return __v[__i];
144 }
145 }
146
147 /**
148 * Helper function to work around Clang and GCC not allowing assignment to v[i] in constant
149 * expressions.
150 */
151 template <__vec_builtin _TV>
152 [[__gnu__::__always_inline__]]
153 constexpr void
154 __vec_set(_TV& __v, int __i, __vec_value_type<_TV> __x)
155 {
156 if consteval
157 {
158#ifdef _GLIBCXX_CLANG
159 auto __arr = __builtin_bit_cast(array<__vec_value_type<_TV>, __width_of<_TV>>, __v);
160 __arr[__i] = __x;
161 __v = __builtin_bit_cast(_TV, __arr);
162#else
163 constexpr auto [...__j] = _IotaArray<__width_of<_TV>>;
164 __v = _TV{(__i == __j ? __x : __v[__j])...};
165#endif
166 }
167 else
168 {
169 __v[__i] = __x;
170 }
171 }
172
173 /** @internal
174 * Return vector builtin with all values from @p __a and @p __b.
175 */
176 template <__vec_builtin _TV>
177 [[__gnu__::__always_inline__]]
178 constexpr __vec_builtin_type<__vec_value_type<_TV>, __width_of<_TV> * 2>
179 __vec_concat(_TV __a, _TV __b)
180 {
181 constexpr auto [...__is] = _IotaArray<__width_of<_TV> * 2>;
182 return __builtin_shufflevector(__a, __b, __is...);
183 }
184
185 /** @internal
186 * Concatenate the first @p _N0 elements from @p __a with the first @p _N1 elements from @p __b
187 * with the elements from applying this function recursively to @p __rest.
188 *
189 * @pre _N0 <= __width_of<_TV0> && _N1 <= __width_of<_TV1> && _Ns <= __width_of<_TVs> && ...
190 *
191 * Strategy: Aim for a power-of-2 tree concat. E.g.
192 * - cat(2, 2, 2, 2) -> cat(4, 2, 2) -> cat(4, 4)
193 * - cat(2, 2, 2, 2, 8) -> cat(4, 2, 2, 8) -> cat(4, 4, 8) -> cat(8, 8)
194 */
195 template <int _N0, int _N1, int... _Ns, __vec_builtin _TV0, __vec_builtin _TV1,
196 __vec_builtin... _TVs>
197 [[__gnu__::__always_inline__]]
198 constexpr __vec_builtin_type<__vec_value_type<_TV0>,
199 __bit_ceil(unsigned(_N0 + (_N1 + ... + _Ns)))>
200 __vec_concat_sized(const _TV0& __a, const _TV1& __b, const _TVs&... __rest);
201
202 template <int _N0, int _N1, int _N2, int... _Ns, __vec_builtin _TV0, __vec_builtin _TV1,
203 __vec_builtin _TV2, __vec_builtin... _TVs>
204 requires (__has_single_bit(unsigned(_N0))) && (_N0 >= (_N1 + _N2))
205 [[__gnu__::__always_inline__]]
206 constexpr __vec_builtin_type<__vec_value_type<_TV0>,
207 __bit_ceil(unsigned(_N0 + _N1 + (_N2 + ... + _Ns)))>
208 __vec_concat_sized(const _TV0& __a, const _TV1& __b, const _TV2& __c, const _TVs&... __rest)
209 {
210 return __vec_concat_sized<_N0, _N1 + _N2, _Ns...>(
211 __a, __vec_concat_sized<_N1, _N2>(__b, __c), __rest...);
212 }
213
214 template <int _N0, int _N1, int... _Ns, __vec_builtin _TV0, __vec_builtin _TV1,
215 __vec_builtin... _TVs>
216 [[__gnu__::__always_inline__]]
217 constexpr __vec_builtin_type<__vec_value_type<_TV0>,
218 __bit_ceil(unsigned(_N0 + (_N1 + ... + _Ns)))>
219 __vec_concat_sized(const _TV0& __a, const _TV1& __b, const _TVs&... __rest)
220 {
221 // __is is rounded up because we need to generate a power-of-2 vector:
222 constexpr auto [...__is] = _IotaArray<__bit_ceil(unsigned(_N0 + _N1)), int>;
223 const auto __ab = __builtin_shufflevector(__a, __b, [](int __i) consteval {
224 if (__i < _N0) // copy from __a
225 return __i;
226 else if (__i < _N0 + _N1) // copy from __b
227 return __i - _N0 + __width_of<_TV0>; // _N0 <= __width_of<_TV0>
228 else // can't index into __rest
229 return -1; // don't care
230 }(__is)...);
231 if constexpr (sizeof...(__rest) == 0)
232 return __ab;
233 else
234 return __vec_concat_sized<_N0 + _N1, _Ns...>(__ab, __rest...);
235 }
236
237 template <__vec_builtin _TV>
238 [[__gnu__::__always_inline__]]
239 constexpr __half_vec_builtin_t<_TV>
240 __vec_split_lo(_TV __v)
241 {
242 constexpr int __n = __width_of<_TV> / 2;
243 constexpr auto [...__is] = _IotaArray<__n>;
244 return __builtin_shufflevector(__v, __v, __is...);
245 }
246
247 template <__vec_builtin _TV>
248 [[__gnu__::__always_inline__]]
249 constexpr __half_vec_builtin_t<_TV>
250 __vec_split_hi(_TV __v)
251 {
252 constexpr int __n = __width_of<_TV> / 2;
253 constexpr auto [...__is] = _IotaArray<__n>;
254 return __builtin_shufflevector(__v, __v, (__n + __is)...);
255 }
256
257 /** @internal
258 * Return @p __x zero-padded to @p _Bytes bytes.
259 *
260 * Use this function when you need two objects of the same size (e.g. for __vec_concat).
261 */
262 template <size_t _Bytes, __vec_builtin _TV>
263 [[__gnu__::__always_inline__]]
264 constexpr auto
265 __vec_zero_pad_to(_TV __x)
266 {
267 if constexpr (sizeof(_TV) == _Bytes)
268 return __x;
269 else if constexpr (sizeof(_TV) <= sizeof(0ull))
270 {
271 using _Up = _UInt<sizeof(_TV)>;
272 __vec_builtin_type_bytes<_Up, _Bytes> __tmp = {__builtin_bit_cast(_Up, __x)};
273 return __builtin_bit_cast(__vec_builtin_type_bytes<__vec_value_type<_TV>, _Bytes>, __tmp);
274 }
275 else if constexpr (sizeof(_TV) < _Bytes)
276 return __vec_zero_pad_to<_Bytes>(__vec_concat(__x, _TV()));
277 else
278 static_assert(false);
279 }
280
281 /** @internal
282 * Return a type with sizeof 16, add zero-padding to @p __x. The input must be smaller.
283 *
284 * Use this function instead of the above when you need to pad an argument for a SIMD builtin.
285 */
286 template <__vec_builtin _TV>
287 [[__gnu__::__always_inline__]]
288 constexpr auto
289 __vec_zero_pad_to_16(_TV __x)
290 {
291 static_assert(sizeof(_TV) < 16);
292 return __vec_zero_pad_to<16>(__x);
293 }
294
295 // work around __builtin_constant_p returning false unless passed a variable
296 // (__builtin_constant_p(x[0]) is false while __is_const_known(x[0]) is true)
297 template <typename _Tp>
298 [[__gnu__::__always_inline__]]
299 constexpr bool
300 __is_const_known(const _Tp& __x)
301 {
302 return __builtin_constant_p(__x);
303 }
304
305 [[__gnu__::__always_inline__]]
306 constexpr bool
307 __is_const_known(const auto&... __xs) requires(sizeof...(__xs) >= 2)
308 {
309 if consteval
310 {
311 return true;
312 }
313 else
314 {
315 return (__is_const_known(__xs) && ...);
316 }
317 }
318
319 [[__gnu__::__always_inline__]]
320 constexpr bool
321 __is_const_known_equal_to(const auto& __x, const auto& __expect)
322 { return __is_const_known(__x == __expect) && __x == __expect; }
323
324#if _GLIBCXX_X86
325 template <__vec_builtin _UV, __vec_builtin _TV>
326 inline _UV
327 __x86_cvt_f16c(_TV __v);
328#endif
329
330
331 /** @internal
332 * Simple wrapper around __builtin_convertvector to provide static_cast-like syntax.
333 *
334 * Works around GCC failing to use the F16C/AVX512F cvtps2ph/cvtph2ps instructions.
335 */
336 template <__vec_builtin _UV, __vec_builtin _TV, _ArchTraits _Traits = {}>
337 [[__gnu__::__always_inline__]]
338 constexpr _UV
339 __vec_cast(_TV __v)
340 {
341 static_assert(__width_of<_UV> == __width_of<_TV>);
342#if _GLIBCXX_X86
343 using _Up = __vec_value_type<_UV>;
344 using _Tp = __vec_value_type<_TV>;
345 constexpr bool __to_f16 = is_same_v<_Up, _Float16>;
346 constexpr bool __from_f16 = is_same_v<_Tp, _Float16>;
347 constexpr bool __needs_f16c = _Traits._M_have_f16c() && !_Traits._M_have_avx512fp16()
348 && (__to_f16 || __from_f16);
349 if (__needs_f16c && !__is_const_known(__v))
350 { // Work around PR121688
351 if constexpr (__needs_f16c)
352 return __x86_cvt_f16c<_UV>(__v);
353 }
354 if constexpr (is_floating_point_v<_Tp> && is_integral_v<_Up>
355 && sizeof(_UV) < sizeof(_TV) && sizeof(_Up) < sizeof(int))
356 {
357 using _Ip = __integer_from<std::min(sizeof(int), sizeof(_Tp))>;
358 using _IV = __vec_builtin_type<_Ip, __width_of<_TV>>;
359 return __vec_cast<_UV>(__vec_cast<_IV>(__v));
360 }
361#endif
362 return __builtin_convertvector(__v, _UV);
363 }
364
365 /** @internal
366 * Overload of the above cast function that determines the destination vector type from a given
367 * element type @p _Up and the `__width_of` the argument type.
368 *
369 * Calls the above overload.
370 */
371 template <__vectorizable _Up, __vec_builtin _TV>
372 [[__gnu__::__always_inline__]]
373 constexpr __vec_builtin_type<_Up, __width_of<_TV>>
374 __vec_cast(_TV __v)
375 { return __vec_cast<__vec_builtin_type<_Up, __width_of<_TV>>>(__v); }
376
377 /** @internal
378 * As above, but with additional precondition on possible values of the argument.
379 *
380 * Precondition: __k[i] is either 0 or -1 for all i.
381 */
382 template <__vec_builtin _UV, __vec_builtin _TV>
383 [[__gnu__::__always_inline__]]
384 constexpr _UV
385 __vec_mask_cast(_TV __k)
386 {
387 static_assert(signed_integral<__vec_value_type<_UV>>);
388 static_assert(signed_integral<__vec_value_type<_TV>>);
389 // TODO: __builtin_convertvector cannot be optimal because it doesn't consider input and
390 // output can only be 0 or -1.
391 return __builtin_convertvector(__k, _UV);
392 }
393
394 template <__vec_builtin _TV>
395 [[__gnu__::__always_inline__]]
396 constexpr _TV
397 __vec_xor(_TV __a, _TV __b)
398 {
399 using _Tp = __vec_value_type<_TV>;
400 if constexpr (is_floating_point_v<_Tp>)
401 {
402 using _UV = __vec_builtin_type<__integer_from<sizeof(_Tp)>, __width_of<_TV>>;
403 return __builtin_bit_cast(
404 _TV, __builtin_bit_cast(_UV, __a) ^ __builtin_bit_cast(_UV, __b));
405 }
406 else
407 return __a ^ __b;
408 }
409
410 template <__vec_builtin _TV>
411 [[__gnu__::__always_inline__]]
412 constexpr _TV
413 __vec_or(_TV __a, _TV __b)
414 {
415 using _Tp = __vec_value_type<_TV>;
416 if constexpr (is_floating_point_v<_Tp>)
417 {
418 using _UV = __vec_builtin_type<__integer_from<sizeof(_Tp)>, __width_of<_TV>>;
419 return __builtin_bit_cast(
420 _TV, __builtin_bit_cast(_UV, __a) | __builtin_bit_cast(_UV, __b));
421 }
422 else
423 return __a | __b;
424 }
425
426 template <__vec_builtin _TV>
427 [[__gnu__::__always_inline__]]
428 constexpr _TV
429 __vec_and(_TV __a, _TV __b)
430 {
431 using _Tp = __vec_value_type<_TV>;
432 if constexpr (is_floating_point_v<_Tp>)
433 {
434 using _UV = __vec_builtin_type<__integer_from<sizeof(_Tp)>, __width_of<_TV>>;
435 return __builtin_bit_cast(
436 _TV, __builtin_bit_cast(_UV, __a) & __builtin_bit_cast(_UV, __b));
437 }
438 else
439 return __a & __b;
440 }
441
442 /** @internal
443 * Returns the bit-wise and of not @p __a and @p __b.
444 *
445 * Use __vec_and(__vec_not(__a), __b) unless an andnot instruction is necessary for optimization.
446 *
447 * @see __vec_andnot in simd_x86.h
448 */
449 template <__vec_builtin _TV>
450 [[__gnu__::__always_inline__]]
451 constexpr _TV
452 __vec_andnot(_TV __a, _TV __b)
453 {
454 using _Tp = __vec_value_type<_TV>;
455 using _UV = __vec_builtin_type<__integer_from<sizeof(_Tp)>, __width_of<_TV>>;
456 return __builtin_bit_cast(
457 _TV, ~__builtin_bit_cast(_UV, __a) & __builtin_bit_cast(_UV, __b));
458 }
459
460 template <__vec_builtin _TV>
461 [[__gnu__::__always_inline__]]
462 constexpr _TV
463 __vec_not(_TV __a)
464 {
465 using _Tp = __vec_value_type<_TV>;
466 using _UV = __vec_builtin_type_bytes<__integer_from<sizeof(_Tp)>, sizeof(_TV)>;
467 if constexpr (is_floating_point_v<__vec_value_type<_TV>>)
468 return __builtin_bit_cast(_TV, ~__builtin_bit_cast(_UV, __a));
469 else
470 return ~__a;
471 }
472
473 /**
474 * An object of given type where only the sign bits are 1.
475 */
476 template <__vec_builtin _V>
477 requires std::floating_point<__vec_value_type<_V>>
478 constexpr _V _S_signmask = __vec_xor(_V() + 1, _V() - 1);
479
480 template <__vec_builtin _TV, int _Np = __width_of<_TV>,
481 typename = make_integer_sequence<int, _Np>>
482 struct _VecOps;
483
484 template <__vec_builtin _TV, int _Np, int... _Is>
485 struct _VecOps<_TV, _Np, integer_sequence<int, _Is...>>
486 {
487 static_assert(_Np <= __width_of<_TV>);
488
489 using _Tp = __vec_value_type<_TV>;
490
491 using _HV = __half_vec_builtin_t<__conditional_t<_Np >= 2, _TV, __double_vec_builtin_t<_TV>>>;
492
493 [[__gnu__::__always_inline__]]
494 static constexpr _TV
495 _S_broadcast_to_even(_Tp __init)
496 { return _TV {((_Is & 1) == 0 ? __init : _Tp())...}; }
497
498 [[__gnu__::__always_inline__]]
499 static constexpr _TV
500 _S_broadcast_to_odd(_Tp __init)
501 { return _TV {((_Is & 1) == 1 ? __init : _Tp())...}; }
502
503 [[__gnu__::__always_inline__]]
504 static constexpr bool
505 _S_all_of(_TV __k) noexcept
506 { return (... && (__k[_Is] != 0)); }
507
508 [[__gnu__::__always_inline__]]
509 static constexpr bool
510 _S_any_of(_TV __k) noexcept
511 { return (... || (__k[_Is] != 0)); }
512
513 [[__gnu__::__always_inline__]]
514 static constexpr bool
515 _S_none_of(_TV __k) noexcept
516 { return (... && (__k[_Is] == 0)); }
517
518 template <typename _Offset = integral_constant<int, 0>>
519 [[__gnu__::__always_inline__]]
520 static constexpr _TV
521 _S_extract(__vec_builtin auto __x, _Offset = {})
522 {
523 static_assert(is_same_v<__vec_value_type<_TV>, __vec_value_type<decltype(__x)>>);
524 return __builtin_shufflevector(__x, decltype(__x)(), (_Is + _Offset::value)...);
525 }
526
527 // swap neighboring elements
528 [[__gnu__::__always_inline__]]
529 static constexpr _TV
530 _S_swap_neighbors(_TV __x)
531 { return __builtin_shufflevector(__x, __x, (_Is ^ 1)...); }
532
533 // duplicate even indexed elements, dropping the odd ones
534 [[__gnu__::__always_inline__]]
535 static constexpr _TV
536 _S_dup_even(_TV __x)
537 { return __builtin_shufflevector(__x, __x, (_Is & ~1)...); }
538
539 // duplicate odd indexed elements, dropping the even ones
540 [[__gnu__::__always_inline__]]
541 static constexpr _TV
542 _S_dup_odd(_TV __x)
543 { return __builtin_shufflevector(__x, __x, (_Is | 1)...); }
544
545 [[__gnu__::__always_inline__]]
546 static constexpr void
547 _S_overwrite_even_elements(_TV& __x, _HV __y) requires (_Np > 1)
548 {
549 constexpr __simd_size_type __n = __width_of<_TV>;
550 __x = __builtin_shufflevector(__x,
551#ifdef _GLIBCXX_CLANG
552 __vec_concat(__y, __y),
553#else
554 __y,
555#endif
556 ((_Is & 1) == 0 ? __n + _Is / 2 : _Is)...);
557 }
558
559 [[__gnu__::__always_inline__]]
560 static constexpr void
561 _S_overwrite_even_elements(_TV& __xl, _TV& __xh, _TV __y)
562 {
563 constexpr __simd_size_type __nl = __width_of<_TV>;
564 constexpr __simd_size_type __nh = __nl * 3 / 2;
565 __xl = __builtin_shufflevector(__xl, __y, ((_Is & 1) == 0 ? __nl + _Is / 2 : _Is)...);
566 __xh = __builtin_shufflevector(__xh, __y, ((_Is & 1) == 0 ? __nh + _Is / 2 : _Is)...);
567 }
568
569 [[__gnu__::__always_inline__]]
570 static constexpr void
571 _S_overwrite_odd_elements(_TV& __x, _HV __y) requires (_Np > 1)
572 {
573 constexpr __simd_size_type __n = __width_of<_TV>;
574 __x = __builtin_shufflevector(__x,
575#ifdef _GLIBCXX_CLANG
576 __vec_concat(__y, __y),
577#else
578 __y,
579#endif
580 ((_Is & 1) == 1 ? __n + _Is / 2 : _Is)...);
581 }
582
583 [[__gnu__::__always_inline__]]
584 static constexpr void
585 _S_overwrite_odd_elements(_TV& __xl, _TV& __xh, _TV __y)
586 {
587 constexpr __simd_size_type __nl = __width_of<_TV>;
588 constexpr __simd_size_type __nh = __nl * 3 / 2;
589 __xl = __builtin_shufflevector(__xl, __y, ((_Is & 1) == 1 ? __nl + _Is / 2 : _Is)...);
590 __xh = __builtin_shufflevector(__xh, __y, ((_Is & 1) == 1 ? __nh + _Is / 2 : _Is)...);
591 }
592
593 // true if all elements are know to be equal to __ref at compile time
594 [[__gnu__::__always_inline__]]
595 static constexpr bool
596 _S_is_const_known_equal_to(_TV __x, _Tp __ref)
597 { return (__is_const_known_equal_to(__x[_Is], __ref) && ...); }
598
599 };
600} // namespace simd
601_GLIBCXX_END_NAMESPACE_VERSION
602} // namespace std
603
604#pragma GCC diagnostic pop
605#endif // C++26
606#endif // _GLIBCXX_VEC_OPS_H
bit
utility.h
std::make_unsigned_t
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition type_traits:2246
std::min
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition stl_algobase.h:232
std
ISO C++ entities toplevel namespace is std.
std::make_integer_sequence
__make_integer_seq< integer_sequence, _Tp, _Num > make_integer_sequence
Alias template make_integer_sequence.
Definition utility.h:548
std::integer_sequence
Class template integer_sequence.
Definition utility.h:502
std::same_as
[concept.same], concept same_as
Definition concepts:65
std::simd::__vec_builtin_of
Definition vec_ops.h:73
std::simd::__vec_builtin
Definition vec_ops.h:84