libstdc++
simd_vec.h
1// Implementation of <simd> -*- C++ -*-
2
3// Copyright The GNU Toolchain Authors.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_SIMD_VEC_H
26#define _GLIBCXX_SIMD_VEC_H 1
27
28#ifdef _GLIBCXX_SYSHDR
29#pragma GCC system_header
30#endif
31
32#if __cplusplus >= 202400L
33
34#include "simd_mask.h"
35#include "simd_flags.h"
36
37#include <bits/utility.h>
38#include <bits/stl_function.h>
39#include <cmath>
40
41// psabi warnings are bogus because the ABI of the internal types never leaks into user code
42#pragma GCC diagnostic push
43#pragma GCC diagnostic ignored "-Wpsabi"
44
45namespace std _GLIBCXX_VISIBILITY(default)
46{
47_GLIBCXX_BEGIN_NAMESPACE_VERSION
48namespace simd
49{
50 // disabled basic_vec
51 template <typename _Tp, typename _Ap>
52 class basic_vec
53 {
54 public:
55 using value_type = _Tp;
56
57 using abi_type = _Ap;
58
59 using mask_type = basic_mask<0, void>; // disabled
60
61#define _GLIBCXX_DELETE_SIMD "This specialization is disabled because of an invalid combination " \
62 "of template arguments to basic_vec."
63
64 basic_vec() = delete(_GLIBCXX_DELETE_SIMD);
65
66 ~basic_vec() = delete(_GLIBCXX_DELETE_SIMD);
67
68 basic_vec(const basic_vec&) = delete(_GLIBCXX_DELETE_SIMD);
69
70 basic_vec& operator=(const basic_vec&) = delete(_GLIBCXX_DELETE_SIMD);
71
72#undef _GLIBCXX_DELETE_SIMD
73 };
74
75 template <typename _Tp, typename _Ap>
76 class _VecBase
77 {
78 using _Vp = basic_vec<_Tp, _Ap>;
79
80 public:
81 using value_type = _Tp;
82
83 using abi_type = _Ap;
84
85 using mask_type = basic_mask<sizeof(_Tp), abi_type>;
86
87 using iterator = __iterator<_Vp>;
88
89 using const_iterator = __iterator<const _Vp>;
90
91 constexpr iterator
92 begin() noexcept
93 { return {static_cast<_Vp&>(*this), 0}; }
94
95 constexpr const_iterator
96 begin() const noexcept
97 { return cbegin(); }
98
99 constexpr const_iterator
100 cbegin() const noexcept
101 { return {static_cast<const _Vp&>(*this), 0}; }
102
103 constexpr default_sentinel_t
104 end() const noexcept
105 { return {}; }
106
107 constexpr default_sentinel_t
108 cend() const noexcept
109 { return {}; }
110
111 static constexpr auto size = __simd_size_c<_Ap::_S_size>;
112
113 _VecBase() = default;
114
115 // LWG issue from 2026-03-04 / P4042R0
116 template <typename _Up, typename _UAbi>
117 requires (_Ap::_S_size != _UAbi::_S_size)
118 _VecBase(const basic_vec<_Up, _UAbi>&) = delete("size mismatch");
119
120 template <typename _Up, typename _UAbi>
121 requires (_Ap::_S_size == _UAbi::_S_size) && (!__explicitly_convertible_to<_Up, _Tp>)
122 explicit
123 _VecBase(const basic_vec<_Up, _UAbi>&)
124 = delete("the value types are not convertible");
125
126 [[__gnu__::__always_inline__]]
127 friend constexpr _Vp
128 operator+(const _Vp& __x, const _Vp& __y) noexcept
129 {
130 _Vp __r = __x;
131 __r += __y;
132 return __r;
133 }
134
135 [[__gnu__::__always_inline__]]
136 friend constexpr _Vp
137 operator-(const _Vp& __x, const _Vp& __y) noexcept
138 {
139 _Vp __r = __x;
140 __r -= __y;
141 return __r;
142 }
143
144 [[__gnu__::__always_inline__]]
145 friend constexpr _Vp
146 operator*(const _Vp& __x, const _Vp& __y) noexcept
147 {
148 _Vp __r = __x;
149 __r *= __y;
150 return __r;
151 }
152
153 [[__gnu__::__always_inline__]]
154 friend constexpr _Vp
155 operator/(const _Vp& __x, const _Vp& __y) noexcept
156 {
157 _Vp __r = __x;
158 __r /= __y;
159 return __r;
160 }
161
162 [[__gnu__::__always_inline__]]
163 friend constexpr _Vp
164 operator%(const _Vp& __x, const _Vp& __y) noexcept
165 requires requires (_Tp __a) { __a % __a; }
166 {
167 _Vp __r = __x;
168 __r %= __y;
169 return __r;
170 }
171
172 [[__gnu__::__always_inline__]]
173 friend constexpr _Vp
174 operator&(const _Vp& __x, const _Vp& __y) noexcept
175 requires requires (_Tp __a) { __a & __a; }
176 {
177 _Vp __r = __x;
178 __r &= __y;
179 return __r;
180 }
181
182 [[__gnu__::__always_inline__]]
183 friend constexpr _Vp
184 operator|(const _Vp& __x, const _Vp& __y) noexcept
185 requires requires (_Tp __a) { __a | __a; }
186 {
187 _Vp __r = __x;
188 __r |= __y;
189 return __r;
190 }
191
192 [[__gnu__::__always_inline__]]
193 friend constexpr _Vp
194 operator^(const _Vp& __x, const _Vp& __y) noexcept
195 requires requires (_Tp __a) { __a ^ __a; }
196 {
197 _Vp __r = __x;
198 __r ^= __y;
199 return __r;
200 }
201
202 [[__gnu__::__always_inline__]]
203 friend constexpr _Vp
204 operator<<(const _Vp& __x, const _Vp& __y) _GLIBCXX_SIMD_NOEXCEPT
205 requires requires (_Tp __a) { __a << __a; }
206 {
207 _Vp __r = __x;
208 __r <<= __y;
209 return __r;
210 }
211
212 [[__gnu__::__always_inline__]]
213 friend constexpr _Vp
214 operator<<(const _Vp& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
215 requires requires (_Tp __a, __simd_size_type __b) { __a << __b; }
216 {
217 _Vp __r = __x;
218 __r <<= __y;
219 return __r;
220 }
221
222 [[__gnu__::__always_inline__]]
223 friend constexpr _Vp
224 operator>>(const _Vp& __x, const _Vp& __y) _GLIBCXX_SIMD_NOEXCEPT
225 requires requires (_Tp __a) { __a >> __a; }
226 {
227 _Vp __r = __x;
228 __r >>= __y;
229 return __r;
230 }
231
232 [[__gnu__::__always_inline__]]
233 friend constexpr _Vp
234 operator>>(const _Vp& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
235 requires requires (_Tp __a, __simd_size_type __b) { __a >> __b; }
236 {
237 _Vp __r = __x;
238 __r >>= __y;
239 return __r;
240 }
241 };
242
243 struct _LoadCtorTag
244 {};
245
246 template <integral _Tp>
247 inline constexpr _Tp __max_shift
248 = (sizeof(_Tp) < sizeof(int) ? sizeof(int) : sizeof(_Tp)) * __CHAR_BIT__;
249
250 template <__vectorizable _Tp, __abi_tag _Ap>
251 requires (_Ap::_S_nreg == 1)
252 && (!__complex_like<_Tp>)
253 class basic_vec<_Tp, _Ap>
254 : public _VecBase<_Tp, _Ap>
255 {
256 template <typename, typename>
257 friend class basic_vec;
258
259 template <size_t, typename>
260 friend class basic_mask;
261
262 static constexpr int _S_size = _Ap::_S_size;
263
264 static constexpr int _S_full_size = __bit_ceil(unsigned(_S_size));
265
266 static constexpr bool _S_is_scalar = _S_size == 1;
267
268 static constexpr bool _S_use_bitmask = _Ap::_S_is_bitmask && !_S_is_scalar;
269
270 using _DataType = typename _Ap::template _DataType<_Tp>;
271
272 /** @internal
273 * @brief Underlying vector data storage.
274 *
275 * This member holds the vector object using a GNU vector type or a platform-specific vector
276 * type determined by the ABI tag. For size 1 vectors, this is a single value (_Tp).
277 */
278 _DataType _M_data;
279
280 static constexpr bool _S_is_partial = sizeof(_M_data) > sizeof(_Tp) * _S_size;
281
282 using __canon_value_type = __canonical_vec_type_t<_Tp>;
283
284 public:
285 using value_type = _Tp;
286
287 using mask_type = _VecBase<_Tp, _Ap>::mask_type;
288
289 // internal but public API ----------------------------------------------
290 [[__gnu__::__always_inline__]]
291 static constexpr basic_vec
292 _S_init(_DataType __x)
293 {
294 basic_vec __r;
295 __r._M_data = __x;
296 return __r;
297 }
298
299 [[__gnu__::__always_inline__]]
300 constexpr _DataType&
301 _M_get() noexcept
302 { return _M_data; }
303
304 [[__gnu__::__always_inline__]]
305 constexpr const _DataType&
306 _M_get() const noexcept
307 { return _M_data; }
308
309 [[__gnu__::__always_inline__]]
310 friend constexpr bool
311 __is_const_known(const basic_vec& __x)
312 { return __builtin_constant_p(__x._M_data); }
313
314 [[__gnu__::__always_inline__]]
315 constexpr auto
316 _M_concat_data([[maybe_unused]] bool __do_sanitize = false) const
317 {
318 if constexpr (_S_is_scalar)
319 return __vec_builtin_type<__canon_value_type, 1>{_M_data};
320 else
321 return _M_data;
322 }
323
324 template <int _Size = _S_size, int _Offset = 0, typename _A0, typename _Fp>
325 [[__gnu__::__always_inline__]]
326 static constexpr basic_vec
327 _S_static_permute(const basic_vec<value_type, _A0>& __x, _Fp&& __idxmap)
328 {
329 using _Xp = basic_vec<value_type, _A0>;
330 basic_vec __r;
331 if constexpr (_S_is_scalar)
332 {
333 constexpr __simd_size_type __j = [&] consteval {
334 if constexpr (__index_permutation_function_sized<_Fp>)
335 return __idxmap(_Offset, _Size);
336 else
337 return __idxmap(_Offset);
338 }();
339 if constexpr (__j == simd::zero_element || __j == simd::uninit_element)
340 return basic_vec();
341 else
342 static_assert(__j >= 0 && __j < _Xp::_S_size);
343 __r._M_data = __x[__j];
344 }
345 else
346 {
347 auto __idxmap2 = [=](auto __i) consteval {
348 if constexpr (int(__i + _Offset) >= _Size) // _S_full_size > _Size
349 return __simd_size_c<simd::uninit_element>;
350 else if constexpr (__index_permutation_function_sized<_Fp>)
351 return __simd_size_c<__idxmap(__i + _Offset, _Size)>;
352 else
353 return __simd_size_c<__idxmap(__i + _Offset)>;
354 };
355 constexpr auto __adj_idx = [](auto __i) {
356 constexpr int __j = __i;
357 if constexpr (__j == simd::zero_element)
358 return __simd_size_c<__bit_ceil(unsigned(_Xp::_S_size))>;
359 else if constexpr (__j == simd::uninit_element)
360 return __simd_size_c<-1>;
361 else
362 {
363 static_assert(__j >= 0 && __j < _Xp::_S_size);
364 return __simd_size_c<__j>;
365 }
366 };
367 constexpr auto [...__is0] = _IotaArray<_S_size>;
368 constexpr bool __needs_zero_element
369 = ((__idxmap2(__simd_size_c<__is0>).value == simd::zero_element) || ...);
370 constexpr auto [...__is_full] = _IotaArray<_S_full_size>;
371 if constexpr (_A0::_S_nreg == 2 && !__needs_zero_element)
372 {
373 __r._M_data = __builtin_shufflevector(
374 __x._M_data0._M_data, __x._M_data1._M_data,
375 __adj_idx(__idxmap2(__simd_size_c<__is_full>)).value...);
376 }
377 else
378 {
379 __r._M_data = __builtin_shufflevector(
380 __x._M_concat_data(), decltype(__x._M_concat_data())(),
381 __adj_idx(__idxmap2(__simd_size_c<__is_full>)).value...);
382 }
383 }
384 return __r;
385 }
386
387 template <typename _Vp>
388 [[__gnu__::__always_inline__]]
389 constexpr auto
390 _M_chunk() const noexcept
391 {
392 constexpr int __n = _S_size / _Vp::_S_size;
393 constexpr int __rem = _S_size % _Vp::_S_size;
394 constexpr auto [...__is] = _IotaArray<__n>;
395 if constexpr (__rem == 0)
396 return array<_Vp, __n> {__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>, *this)...};
397 else
398 {
399 using _Rest = resize_t<__rem, _Vp>;
400 return tuple(__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>, *this)...,
401 __extract_simd_at<_Rest>(cw<_Vp::_S_size * __n>, *this));
402 }
403 }
404
405 [[__gnu__::__always_inline__]]
406 static constexpr basic_vec
407 _S_concat(const basic_vec& __x0) noexcept
408 { return __x0; }
409
410 template <typename... _As>
411 requires (sizeof...(_As) > 1)
412 [[__gnu__::__always_inline__]]
413 static constexpr basic_vec
414 _S_concat(const basic_vec<value_type, _As>&... __xs) noexcept
415 {
416 static_assert(_S_size == (_As::_S_size + ...));
417 return __extract_simd_at<basic_vec>(cw<0>, __xs...);
418 }
419
420 /** @internal
421 * Shifts elements to the front by @p _Shift positions (or to the back for negative @p
422 * _Shift).
423 *
424 * This function moves elements towards lower indices (front of the vector).
425 * Elements that would shift beyond the vector bounds are replaced with zero. Negative shift
426 * values shift in the opposite direction.
427 *
428 * @warning The naming can be confusing due to little-endian byte order:
429 * - Despite the name "shifted_to_front", the underlying hardware instruction
430 * shifts bits to the right (psrl...)
431 * - The function name refers to element indices, not bit positions
432 *
433 * @tparam _Shift Number of positions to shift elements towards the front.
434 * Must be -size() < _Shift < size().
435 *
436 * @return A new vector with elements shifted to front or back.
437 *
438 * Example:
439 * @code
440 * __iota<vec<int, 4>>._M_elements_shifted_to_front<2>(); // {2, 3, 0, 0}
441 * __iota<vec<int, 4>>._M_elements_shifted_to_front<-2>(); // {0, 0, 0, 1}
442 * @endcode
443 */
444 template <int _Shift, _ArchTraits _Traits = {}>
445 [[__gnu__::__always_inline__]]
446 constexpr basic_vec
447 _M_elements_shifted_to_front() const
448 {
449 static_assert(_Shift < _S_size && -_Shift < _S_size);
450 if constexpr (_Shift == 0)
451 return *this;
452 else
453 return _S_static_permute(*this, [](int __i) consteval {
454 int __off = __i + _Shift;
455 return __off >= _S_size || __off < 0 ? zero_element : __off;
456 });
457 }
458
459 /** @internal
460 * @brief Set padding elements to @p __id; add more padding elements if necessary.
461 *
462 * @note This function can rearrange the element order since the result is only used for
463 * reductions.
464 */
465 template <typename _Vp, __canon_value_type __id>
466 [[__gnu__::__always_inline__]]
467 constexpr _Vp
468 _M_pad_to_T_with_value() const noexcept
469 {
470 static_assert(!_Vp::_S_is_partial);
471 static_assert(_Ap::_S_nreg == 1);
472 if constexpr (sizeof(_Vp) == 32)
473 { // when we need to reduce from a 512-bit register
474 static_assert(sizeof(_M_data) == 32);
475 constexpr auto __k = _Vp::mask_type::_S_partial_mask_of_n(_S_size);
476 return __select_impl(__k, _Vp::_S_init(_M_data), __id);
477 }
478 else
479 {
480 static_assert(sizeof(_Vp) <= 16); // => max. 7 Bytes need to be zeroed
481 static_assert(sizeof(_M_data) <= sizeof(_Vp));
482 _Vp __v1 = __vec_zero_pad_to<sizeof(_Vp)>(_M_data);
483 if constexpr (__id == 0 && _S_is_partial)
484 // cheapest solution: shift values to the back while shifting in zeros
485 // This is valid because we shift out padding elements and use all elements in a
486 // subsequent reduction.
487 __v1 = __v1.template _M_elements_shifted_to_front<-(_Vp::_S_size - _S_size)>();
488 else if constexpr (_Vp::_S_size - _S_size == 1)
489 // if a single element needs to be changed, use an insert instruction
490 __vec_set(__v1._M_data, _Vp::_S_size - 1, __id);
491 else if constexpr (__has_single_bit(unsigned(_Vp::_S_size - _S_size)))
492 { // if 2^n elements need to be changed, use a single insert instruction
493 constexpr int __n = _Vp::_S_size - _S_size;
494 using _Ip = __integer_from<__n * sizeof(__canon_value_type)>;
495 constexpr auto [...__is] = _IotaArray<__n>;
496 constexpr __canon_value_type __idn[__n] = {((void)__is, __id)...};
497 auto __vn = __vec_bit_cast<_Ip>(__v1._M_data);
498 __vec_set(__vn, _Vp::_S_size / __n - 1, __builtin_bit_cast(_Ip, __idn));
499 __v1._M_data = reinterpret_cast<typename _Vp::_DataType>(__vn);
500 }
501 else if constexpr (__id != 0 && !_S_is_partial)
502 { // if __vec_zero_pad_to added zeros in all the places where we need __id, a
503 // bitwise or is sufficient (needs a vector constant for the __id vector, which
504 // isn't optimal)
505 constexpr _Vp __idn([](int __i) {
506 return __i >= _S_size ? __id : __canon_value_type();
507 });
508 __v1._M_data = __vec_or(__v1._M_data, __idn._M_data);
509 }
510 else if constexpr (__id != 0 || _S_is_partial)
511 { // fallback
512 constexpr auto __k = _Vp::mask_type::_S_partial_mask_of_n(_S_size);
513 __v1 = __select_impl(__k, __v1, __id);
514 }
515 return __v1;
516 }
517 }
518
519 [[__gnu__::__always_inline__]]
520 constexpr auto
521 _M_reduce_to_half(auto __binary_op) const
522 {
523 static_assert(__has_single_bit(unsigned(_S_size)));
524 auto [__a, __b] = chunk<_S_size / 2>(*this);
525 return __binary_op(__a, __b);
526 }
527
528 template <typename _Rest, typename _BinaryOp>
529 [[__gnu__::__always_inline__]]
530 constexpr value_type
531 _M_reduce_tail(const _Rest& __rest, _BinaryOp __binary_op) const
532 {
533 if constexpr (_S_is_scalar)
534 return __binary_op(*this, __rest)._M_data;
535 else if constexpr (_Rest::_S_size == _S_size)
536 return __binary_op(*this, __rest)._M_reduce(__binary_op);
537 else if constexpr (_Rest::_S_size > _S_size)
538 {
539 auto [__a, __b] = __rest.template _M_chunk<basic_vec>();
540 return __binary_op(*this, __a)._M_reduce_tail(__b, __binary_op);
541 }
542 else if constexpr (_Rest::_S_size == 1)
543 return __binary_op(_Rest(_M_reduce(__binary_op)), __rest)[0];
544 else if constexpr (sizeof(_M_data) <= 16
545 && requires { __default_identity_element<__canon_value_type, _BinaryOp>(); })
546 { // extend __rest with identity element for more parallelism
547 constexpr __canon_value_type __id
548 = __default_identity_element<__canon_value_type, _BinaryOp>();
549 return __binary_op(_M_data, __rest.template _M_pad_to_T_with_value<basic_vec, __id>())
550 ._M_reduce(__binary_op);
551 }
552 else
553 return _M_reduce_to_half(__binary_op)._M_reduce_tail(__rest, __binary_op);
554 }
555
556 /** @internal
557 * @brief Reduction over @p __binary_op of all (non-padding) elements.
558 *
559 * @note The implementation assumes it is most efficient to first reduce to one 128-bit SIMD
560 * register and then shuffle elements while sticking to 128-bit registers.
561 */
562 template <typename _BinaryOp, _ArchTraits _Traits = {}>
563 [[__gnu__::__always_inline__]]
564 constexpr value_type
565 _M_reduce(_BinaryOp __binary_op) const
566 {
567 constexpr bool __have_id_elem
568 = requires { __default_identity_element<__canon_value_type, _BinaryOp>(); };
569 if constexpr (_S_size == 1)
570 return operator[](0);
571 else if constexpr (_Traits.template _M_eval_as_f32<value_type>()
572 && (is_same_v<_BinaryOp, plus<>>
573 || is_same_v<_BinaryOp, multiplies<>>))
574 return value_type(rebind_t<float, basic_vec>(*this)._M_reduce(__binary_op));
575#ifdef __SSE2__
576 else if constexpr (is_integral_v<value_type> && sizeof(value_type) == 1
577 && is_same_v<decltype(__binary_op), multiplies<>>)
578 {
579 // convert to unsigned short because of missing 8-bit mul instruction
580 // we don't need to preserve the order of elements
581 //
582 // The left columns under Latency and Throughput show bit-cast to ushort with shift by
583 // 8. The right column uses the alternative in the else branch.
584 // Benchmark on Intel Ultra 7 165U (AVX2)
585 // TYPE Latency Throughput
586 // [cycles/call] [cycles/call]
587 //schar, 2 9.11 7.73 3.17 3.21
588 //schar, 4 31.6 34.9 5.11 6.97
589 //schar, 8 35.7 41.5 7.77 7.17
590 //schar, 16 36.7 44.1 6.66 8.96
591 //schar, 32 42.2 61.1 8.82 10.1
592 if constexpr (!_S_is_partial)
593 { // If all elements participate in the reduction we can take this shortcut
594 using _V16 = resize_t<_S_size / 2, rebind_t<unsigned short, basic_vec>>;
595 auto __a = __builtin_bit_cast(_V16, *this);
596 return __binary_op(__a, __a >> 8)._M_reduce(__binary_op);
597 }
598 else
599 {
600 using _V16 = rebind_t<unsigned short, basic_vec>;
601 return _V16(*this)._M_reduce(__binary_op);
602 }
603 }
604#endif
605 else if constexpr (__has_single_bit(unsigned(_S_size)))
606 {
607 if constexpr (sizeof(_M_data) > 16)
608 return _M_reduce_to_half(__binary_op)._M_reduce(__binary_op);
609 else if constexpr (_S_size == 2)
610 return _M_reduce_to_half(__binary_op)[0];
611 else
612 {
613 static_assert(_S_size <= 16);
614 auto __x = *this;
615#ifdef __SSE2__
616 if constexpr (sizeof(_M_data) <= 16 && is_integral_v<value_type>)
617 {
618 if constexpr (_S_size > 8)
619 __x = __binary_op(__x, __x.template _M_elements_shifted_to_front<8>());
620 if constexpr (_S_size > 4)
621 __x = __binary_op(__x, __x.template _M_elements_shifted_to_front<4>());
622 if constexpr (_S_size > 2)
623 __x = __binary_op(__x, __x.template _M_elements_shifted_to_front<2>());
624 // We could also call __binary_op with vec<T, 1> arguments. However,
625 // micro-benchmarking on Intel Ultra 7 165U showed this to be more efficient:
626 return __binary_op(__x, __x.template _M_elements_shifted_to_front<1>())[0];
627 }
628#endif
629 if constexpr (_S_size > 8)
630 __x = __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<8>()));
631 if constexpr (_S_size > 4)
632 __x = __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<4>()));
633#ifdef __SSE2__
634 // avoid pshufb by "promoting" to int
635 if constexpr (is_integral_v<value_type> && sizeof(value_type) <= 1)
636 return value_type(resize_t<4, rebind_t<int, basic_vec>>(chunk<4>(__x)[0])
637 ._M_reduce(__binary_op));
638#endif
639 if constexpr (_S_size > 2)
640 __x = __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<2>()));
641 if constexpr (is_integral_v<value_type> && sizeof(value_type) == 2)
642 return __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<1>()))[0];
643 else
644 return __binary_op(vec<value_type, 1>(__x[0]), vec<value_type, 1>(__x[1]))[0];
645 }
646 }
647 else if constexpr (sizeof(_M_data) == 32)
648 {
649 const auto [__lo, __hi] = chunk<__bit_floor(unsigned(_S_size))>(*this);
650 return __lo._M_reduce_tail(__hi, __binary_op);
651 }
652 else if constexpr (sizeof(_M_data) == 64)
653 {
654 // e.g. _S_size = 16 + 16 + 15 (vec<char, 47>)
655 // -> 8 + 8 + 7 -> 4 + 4 + 3 -> 2 + 2 + 1 -> 1
656 auto __chunked = chunk<__bit_floor(unsigned(_S_size)) / 2>(*this);
657 using _Cp = decltype(__chunked);
658 if constexpr (tuple_size_v<_Cp> == 4)
659 {
660 const auto& [__a, __b, __c, __rest] = __chunked;
661 constexpr bool __amd_cpu = _Traits._M_have_sse4a();
662 if constexpr (__have_id_elem && __rest._S_size > 1 && __amd_cpu)
663 { // do one 256-bit op -> one 128-bit op
664 // 4 cycles on Zen4/5 until _M_reduce (short, 26, plus<>)
665 // 9 cycles on Skylake-AVX512 until _M_reduce
666 // 9 cycles on Zen4/5 until _M_reduce (short, 27, multiplies<>)
667 // 17 cycles on Skylake-AVX512 until _M_reduce (short, 27, multiplies<>)
668 const auto& [__a, __rest] = chunk<__bit_floor(unsigned(_S_size))>(*this);
669 using _Vp = remove_cvref_t<decltype(__a)>;
670 constexpr __canon_value_type __id
671 = __default_identity_element<__canon_value_type, _BinaryOp>();
672 const _Vp __b = __rest.template _M_pad_to_T_with_value<_Vp, __id>();
673 return __binary_op(__a, __b)._M_reduce(__binary_op);
674 }
675 else if constexpr (__have_id_elem && __rest._S_size > 1)
676 { // do two 128-bit ops -> one 128-bit op
677 // 5 cycles on Zen4/5 until _M_reduce (short, 26, plus<>)
678 // 7 cycles on Skylake-AVX512 until _M_reduce (short, 26, plus<>)
679 // 9 cycles on Zen4/5 until _M_reduce (short, 27, multiplies<>)
680 // 16 cycles on Skylake-AVX512 until _M_reduce (short, 27, multiplies<>)
681 using _Vp = remove_cvref_t<decltype(__a)>;
682 constexpr __canon_value_type __id
683 = __default_identity_element<__canon_value_type, _BinaryOp>();
684 const _Vp __d = __rest.template _M_pad_to_T_with_value<_Vp, __id>();
685 return __binary_op(__binary_op(__a, __b), __binary_op(__c, __d))
686 ._M_reduce(__binary_op);
687 }
688 else
689 return __binary_op(__binary_op(__a, __b), __c)
690 ._M_reduce_tail(__rest, __binary_op);
691 }
692 else if constexpr (tuple_size_v<_Cp> == 3)
693 {
694 const auto& [__a, __b, __rest] = __chunked;
695 return __binary_op(__a, __b)._M_reduce_tail(__rest, __binary_op);
696 }
697 else
698 static_assert(false);
699 }
700 else if constexpr (__have_id_elem)
701 {
702 constexpr __canon_value_type __id
703 = __default_identity_element<__canon_value_type, _BinaryOp>();
704 using _Vp = resize_t<__bit_ceil(unsigned(_S_size)), basic_vec>;
705 return _M_pad_to_T_with_value<_Vp, __id>()._M_reduce(__binary_op);
706 }
707 else
708 {
709 const auto& [__a, __rest] = chunk<__bit_floor(unsigned(_S_size))>(*this);
710 return __a._M_reduce_tail(__rest, __binary_op);
711 }
712 }
713
714 // [simd.math] ----------------------------------------------------------
715 //
716 // ISO/IEC 60559 on the classification operations (5.7.2 General Operations):
717 // "They are never exceptional, even for signaling NaNs."
718 //
719 template <_OptTraits _Traits = {}>
720 [[__gnu__::__always_inline__]]
721 constexpr mask_type
722 _M_isnan() const requires is_floating_point_v<value_type>
723 {
724 if constexpr (_Traits._M_finite_math_only())
725 return mask_type(false);
726 else if constexpr (_S_is_scalar)
727 return mask_type(std::isnan(_M_data));
728 else if constexpr (_S_use_bitmask)
729 return _M_isunordered(*this);
730 else if constexpr (!_Traits._M_support_snan())
731 return !(*this == *this);
732 else if (__is_const_known(_M_data))
733 return mask_type([&](int __i) { return std::isnan(_M_data[__i]); });
734 else
735 {
736 // 60559: NaN is represented as Inf + non-zero mantissa bits
737 using _Ip = __integer_from<sizeof(value_type)>;
738 return __builtin_bit_cast(_Ip, numeric_limits<value_type>::infinity())
739 < __builtin_bit_cast(rebind_t<_Ip, basic_vec>, _M_fabs());
740 }
741 }
742
743 template <_TargetTraits _Traits = {}>
744 [[__gnu__::__always_inline__]]
745 constexpr mask_type
746 _M_isinf() const requires is_floating_point_v<value_type>
747 {
748 if constexpr (_Traits._M_finite_math_only())
749 return mask_type(false);
750 else if constexpr (_S_is_scalar)
751 return mask_type(std::isinf(_M_data));
752 else if (__is_const_known(_M_data))
753 return mask_type([&](int __i) { return std::isinf(_M_data[__i]); });
754#ifdef _GLIBCXX_X86
755 else if constexpr (_S_use_bitmask)
756 return mask_type::_S_init(__x86_bitmask_isinf(_M_data));
757 else if constexpr (_Traits._M_have_avx512dq())
758 return __x86_bit_to_vecmask<typename mask_type::_DataType>(
759 __x86_bitmask_isinf(_M_data));
760#endif
761 else
762 {
763 using _Ip = __integer_from<sizeof(value_type)>;
764 return __vec_bit_cast<_Ip>(_M_fabs()._M_data)
765 == __builtin_bit_cast(_Ip, numeric_limits<value_type>::infinity());
766 }
767 }
768
769 [[__gnu__::__always_inline__]]
770 constexpr basic_vec
771 _M_abs() const requires signed_integral<value_type>
772 { return _M_data < 0 ? -_M_data : _M_data; }
773
774 [[__gnu__::__always_inline__]]
775 constexpr basic_vec
776 _M_fabs() const requires floating_point<value_type>
777 {
778 if constexpr (_S_is_scalar)
779 return std::fabs(_M_data);
780 else
781 return __vec_and(__vec_not(_S_signmask<_DataType>), _M_data);
782 }
783
784 template <_TargetTraits _Traits = {}>
785 [[__gnu__::__always_inline__]]
786 constexpr mask_type
787 _M_isunordered(basic_vec __y) const requires is_floating_point_v<value_type>
788 {
789 if constexpr (_Traits._M_finite_math_only())
790 return mask_type(false);
791 else if constexpr (_S_is_scalar)
792 return mask_type(std::isunordered(_M_data, __y._M_data));
793#ifdef _GLIBCXX_X86
794 else if constexpr (_S_use_bitmask)
795 return _M_bitmask_cmp<_X86Cmp::_Unord>(__y._M_data);
796#endif
797 else
798 return mask_type([&](int __i) {
799 return std::isunordered(_M_data[__i], __y._M_data[__i]);
800 });
801 }
802
803 /** @internal
804 * Implementation of @ref partial_load.
805 *
806 * @param __mem A pointer to an array of @p __n values. Can be complex or real.
807 * @param __n Read no more than @p __n values from memory. However, depending on @p __mem
808 * alignment, out of bounds reads are benign.
809 */
810 template <typename _Up, _ArchTraits _Traits = {}>
811 static inline basic_vec
812 _S_partial_load(const _Up* __mem, size_t __n)
813 {
814 if constexpr (_S_is_scalar)
815 return __n == 0 ? basic_vec() : basic_vec(static_cast<value_type>(*__mem));
816 else if (__is_const_known_equal_to(__n >= size_t(_S_size), true))
817 return basic_vec(_LoadCtorTag(), __mem);
818 else if constexpr (!__converts_trivially<_Up, value_type>)
819 return static_cast<basic_vec>(rebind_t<_Up, basic_vec>::_S_partial_load(__mem, __n));
820 else
821 {
822#if _GLIBCXX_X86
823 if constexpr (_Traits._M_have_avx512f()
824 || (_Traits._M_have_avx() && sizeof(_Up) >= 4))
825 {
826 const auto __k = __n < _S_size ? mask_type::_S_partial_mask_of_n(int(__n))
827 : mask_type(true);
828 return _S_masked_load(__mem, mask_type::_S_partial_mask_of_n(int(__n)));
829 }
830#endif
831 if (__n >= size_t(_S_size)) [[unlikely]]
832 return basic_vec(_LoadCtorTag(), __mem);
833#if _GLIBCXX_X86 // TODO: where else is this "safe"?
834 // allow out-of-bounds read when it cannot lead to a #GP
835 else if (__is_const_known_equal_to(
836 is_sufficiently_aligned<sizeof(_Up) * _S_full_size>(__mem), true))
837 return __select_impl(mask_type::_S_partial_mask_of_n(int(__n)),
838 basic_vec(_LoadCtorTag(), __mem), basic_vec());
839#endif
840 else if constexpr (_S_size > 4)
841 {
842 alignas(_DataType) byte __dst[sizeof(_DataType)] = {};
843 const byte* __src = reinterpret_cast<const byte*>(__mem);
844 __memcpy_chunks<sizeof(_Up), sizeof(_DataType)>(__dst, __src, __n);
845 return __builtin_bit_cast(_DataType, __dst);
846 }
847 else if (__n == 0) [[unlikely]]
848 return basic_vec();
849 else if constexpr (_S_size == 2)
850 return _DataType {static_cast<value_type>(__mem[0]), 0};
851 else
852 {
853 constexpr auto [...__is] = _IotaArray<_S_size - 2>;
854 return _DataType{
855 static_cast<value_type>(__mem[0]),
856 static_cast<value_type>(__is + 1 < __n ? __mem[__is + 1] : 0)...
857 };
858 }
859 }
860 }
861
862 /** @internal
863 * Loads elements from @p __mem according to mask @p __k.
864 *
865 * @param __mem Pointer (in)to array.
866 * @param __k Mask controlling which elements to load. For each bit i in the mask:
867 * - If bit i is 1: copy __mem[i] into result[i]
868 * - If bit i is 0: result[i] is default initialized
869 *
870 * @note This function assumes it's called after determining that no other method
871 * (like full load) is more appropriate. Calling with all mask bits set to 1
872 * is suboptimal for performance but still correct.
873 */
874 template <typename _Up, _ArchTraits _Traits = {}>
875 static inline basic_vec
876 _S_masked_load(const _Up* __mem, mask_type __k)
877 {
878 if constexpr (_S_size == 1)
879 return __k[0] ? static_cast<value_type>(__mem[0]) : value_type();
880#if _GLIBCXX_X86
881 else if constexpr (_Traits._M_have_avx512f())
882 return __x86_masked_load<_DataType>(__mem, __k._M_data);
883 else if constexpr (_Traits._M_have_avx() && (sizeof(_Up) == 4 || sizeof(_Up) == 8))
884 {
885 if constexpr (__converts_trivially<_Up, value_type>)
886 return __x86_masked_load<_DataType>(__mem, __k._M_data);
887 else
888 {
889 using _UV = rebind_t<_Up, basic_vec>;
890 return basic_vec(_UV::_S_masked_load(__mem, typename _UV::mask_type(__k)));
891 }
892 }
893#endif
894 else if (__k._M_none_of()) [[unlikely]]
895 return basic_vec();
896 else if constexpr (_S_is_scalar)
897 return basic_vec(static_cast<value_type>(*__mem));
898 else
899 {
900 // Use at least 4-byte __bits in __bit_foreach for better code-gen
901 _Bitmask<_S_size < 32 ? 32 : _S_size> __bits = __k._M_to_uint();
902 [[assume(__bits != 0)]]; // because of '__k._M_none_of()' branch above
903 if constexpr (__converts_trivially<_Up, value_type>)
904 {
905 _DataType __r = {};
906 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
907 __r[__i] = __mem[__i];
908 });
909 return __r;
910 }
911 else
912 {
913 using _UV = rebind_t<_Up, basic_vec>;
914 alignas(_UV) _Up __tmp[sizeof(_UV) / sizeof(_Up)] = {};
915 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
916 __tmp[__i] = __mem[__i];
917 });
918 return basic_vec(__builtin_bit_cast(_UV, __tmp));
919 }
920 }
921 }
922
923 template <typename _Up>
924 [[__gnu__::__always_inline__]]
925 inline void
926 _M_store(_Up* __mem) const
927 {
928 if constexpr (__converts_trivially<value_type, _Up>)
929 __builtin_memcpy(__mem, &_M_data, sizeof(_Up) * _S_size);
930 else
931 rebind_t<_Up, basic_vec>(*this)._M_store(__mem);
932 }
933
934 /** @internal
935 * Implementation of @ref partial_store.
936 *
937 * @note This is a static function to allow passing @p __v via register in case the function
938 * is not inlined.
939 *
940 * @note The function is not marked @c __always_inline__ since code-gen can become fairly
941 * long.
942 */
943 template <typename _Up, _ArchTraits _Traits = {}>
944 static inline void
945 _S_partial_store(const basic_vec __v, _Up* __mem, size_t __n)
946 {
947 if (__is_const_known_equal_to(__n >= _S_size, true))
948 __v._M_store(__mem);
949#if _GLIBCXX_X86
950 else if constexpr (_Traits._M_have_avx512f() && !_S_is_scalar)
951 {
952 const auto __k = __n < _S_size ? mask_type::_S_partial_mask_of_n(int(__n))
953 : mask_type(true);
954 return _S_masked_store(__v, __mem, __k);
955 }
956#endif
957 else if (__n >= _S_size) [[unlikely]]
958 __v._M_store(__mem);
959 else if (__n == 0) [[unlikely]]
960 return;
961 else if constexpr (__converts_trivially<value_type, _Up>)
962 {
963 byte* __dst = reinterpret_cast<byte*>(__mem);
964 const byte* __src = reinterpret_cast<const byte*>(&__v._M_data);
965 __memcpy_chunks<sizeof(_Up), sizeof(_M_data)>(__dst, __src, __n);
966 }
967 else
968 {
969 using _UV = rebind_t<_Up, basic_vec>;
970 _UV::_S_partial_store(_UV(__v), __mem, __n);
971 }
972 }
973
974 /** @internal
975 * Stores elements of @p __v to @p __mem according to mask @p __k.
976 *
977 * @param __v Values to store to @p __mem.
978 * @param __mem Pointer (in)to array.
979 * @param __k Mask controlling which elements to store. For each bit i in the mask:
980 * - If bit i is 1: store __v[i] to __mem[i]
981 * - If bit i is 0: __mem[i] is left unchanged
982 *
983 * @note This function assumes it's called after determining that no other method
984 * (like full store) is more appropriate. Calling with all mask bits set to 1
985 * is suboptimal for performance but still correct.
986 */
987 template <typename _Up, _ArchTraits _Traits = {}>
988 //[[__gnu__::__always_inline__]]
989 static inline void
990 _S_masked_store(const basic_vec __v, _Up* __mem, const mask_type __k)
991 {
992#if _GLIBCXX_X86
993 if constexpr (_Traits._M_have_avx512f())
994 {
995 __x86_masked_store(__v._M_data, __mem, __k._M_data);
996 return;
997 }
998 else if constexpr (_Traits._M_have_avx() && (sizeof(_Up) == 4 || sizeof(_Up) == 8))
999 {
1000 if constexpr (__converts_trivially<value_type, _Up>)
1001 __x86_masked_store(__v._M_data, __mem, __k._M_data);
1002 else
1003 {
1004 using _UV = rebind_t<_Up, basic_vec>;
1005 _UV::_S_masked_store(_UV(__v), __mem, typename _UV::mask_type(__k));
1006 }
1007 return;
1008 }
1009#endif
1010 if (__k._M_none_of()) [[unlikely]]
1011 return;
1012 else if constexpr (_S_is_scalar)
1013 __mem[0] = __v._M_data;
1014 else
1015 {
1016 // Use at least 4-byte __bits in __bit_foreach for better code-gen
1017 _Bitmask<_S_size < 32 ? 32 : _S_size> __bits = __k._M_to_uint();
1018 [[assume(__bits != 0)]]; // because of '__k._M_none_of()' branch above
1019 if constexpr (__converts_trivially<value_type, _Up>)
1020 {
1021 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
1022 __mem[__i] = __v[__i];
1023 });
1024 }
1025 else
1026 {
1027 const rebind_t<_Up, basic_vec> __cvted(__v);
1028 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
1029 __mem[__i] = __cvted[__i];
1030 });
1031 }
1032 }
1033 }
1034
1035 // [simd.overview] default constructor ----------------------------------
1036 basic_vec() = default;
1037
1038 // [simd.overview] p2 impl-def conversions ------------------------------
1039 using _NativeVecType = decltype([] {
1040 if constexpr (_S_is_scalar)
1041 return __vec_builtin_type<__canon_value_type, 1>();
1042 else
1043 return _DataType();
1044 }());
1045 /**
1046 * @brief Converting constructor from GCC vector builtins.
1047 *
1048 * This constructor enables direct construction from GCC vector builtins
1049 * (`[[gnu::vector_size(N)]]`).
1050 *
1051 * @param __x GCC vector builtin to convert from.
1052 *
1053 * @note This constructor is not available when size() equals 1.
1054 *
1055 * @see operator _NativeVecType() for the reverse conversion.
1056 */
1057 constexpr
1058 basic_vec(_NativeVecType __x)
1059 : _M_data([&] [[__gnu__::__always_inline__]] {
1060 if constexpr (_S_is_scalar)
1061 return __x[0];
1062 else
1063 return __x;
1064 }())
1065 {}
1066
1067 /**
1068 * @brief Conversion operator to GCC vector builtins.
1069 *
1070 * This operator enables implicit conversion from basic_vec to GCC vector builtins.
1071 *
1072 * @note This operator is not available when size() equals 1.
1073 *
1074 * @see basic_vec(_NativeVecType) for the reverse conversion.
1075 */
1076 constexpr
1077 operator _NativeVecType() const
1078 {
1079 if constexpr (_S_is_scalar)
1080 return _NativeVecType{_M_data};
1081 else
1082 return _M_data;
1083 }
1084
1085#if _GLIBCXX_X86
1086 /**
1087 * @brief Converting constructor from Intel Intrinsics (__m128, __m128i, ...).
1088 */
1089 template <__vec_builtin _IV>
1090 requires same_as<__x86_intel_intrin_value_type<value_type>, __vec_value_type<_IV>>
1091 && (sizeof(_IV) == sizeof(_DataType) && sizeof(_IV) >= 16
1092 && !is_same_v<_IV, _DataType>)
1093 constexpr
1094 basic_vec(_IV __x)
1095 : _M_data(reinterpret_cast<_DataType>(__x))
1096 {}
1097
1098 /**
1099 * @brief Conversion operator to Intel Intrinsics (__m128, __m128i, ...).
1100 */
1101 template <__vec_builtin _IV>
1102 requires same_as<__x86_intel_intrin_value_type<value_type>, __vec_value_type<_IV>>
1103 && (sizeof(_IV) == sizeof(_DataType) && sizeof(_IV) >= 16
1104 && !is_same_v<_IV, _DataType>)
1105 constexpr
1106 operator _IV() const
1107 { return reinterpret_cast<_IV>(_M_data); }
1108#endif
1109
1110 // [simd.ctor] broadcast constructor ------------------------------------
1111 /**
1112 * @brief Broadcast constructor from scalar value.
1113 *
1114 * Constructs a vector where all elements are initialized to the same scalar value.
1115 * The scalar value is converted to the vector's element type.
1116 *
1117 * @param __x Scalar value to broadcast to all vector elements.
1118 * @tparam _Up Type of scalar value (must be explicitly convertible to value_type).
1119 *
1120 * @note The constructor is implicit if the conversion (if any) is value-preserving.
1121 */
1122 template <__broadcast_constructible<value_type> _Up>
1123 [[__gnu__::__always_inline__]]
1124 constexpr
1125 basic_vec(_Up&& __x) noexcept
1126 : _M_data(_DataType() == _DataType() ? static_cast<value_type>(__x) : value_type())
1127 {}
1128
1129 // [simd.ctor] conversion constructor -----------------------------------
1130 template <typename _Up, typename _UAbi, _TargetTraits _Traits = {}>
1131 requires (_S_size == _UAbi::_S_size)
1132 && __explicitly_convertible_to<_Up, value_type>
1133 [[__gnu__::__always_inline__]]
1134 constexpr
1135 explicit(!__value_preserving_convertible_to<_Up, value_type>
1136 || __higher_rank_than<_Up, value_type>)
1137 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
1138 : _M_data([&] [[__gnu__::__always_inline__]] {
1139 if constexpr (_S_is_scalar)
1140 return static_cast<value_type>(__x[0]);
1141 else if constexpr (_UAbi::_S_nreg >= 2)
1142 // __builtin_convertvector (__vec_cast) is inefficient for over-sized inputs.
1143 // Also e.g. vec<float, 12> -> vec<char, 12> (with SSE2) would otherwise emit 4
1144 // vcvttps2dq instructions, where only 3 are needed
1145 return _S_concat(resize_t<__x._N0, basic_vec>(__x._M_data0),
1146 resize_t<__x._N1, basic_vec>(__x._M_data1))._M_data;
1147 else
1148 return __vec_cast<_DataType>(__x._M_concat_data());
1149 }())
1150 {}
1151
1152 using _VecBase<_Tp, _Ap>::_VecBase;
1153
1154 // [simd.ctor] generator constructor ------------------------------------
1155 template <__simd_generator_invokable<value_type, _S_size> _Fp>
1156 [[__gnu__::__always_inline__]]
1157 constexpr explicit
1158 basic_vec(_Fp&& __gen)
1159 : _M_data([&] [[__gnu__::__always_inline__]] {
1160 constexpr auto [...__is] = _IotaArray<_S_size>;
1161 return _DataType{static_cast<value_type>(__gen(__simd_size_c<__is>))...};
1162 }())
1163 {}
1164
1165 // [simd.ctor] load constructor -----------------------------------------
1166 template <typename _Up>
1167 [[__gnu__::__always_inline__]]
1168 constexpr
1169 basic_vec(_LoadCtorTag, const _Up* __ptr)
1170 : _M_data()
1171 {
1172 if constexpr (_S_is_scalar)
1173 _M_data = static_cast<value_type>(__ptr[0]);
1174 else if consteval
1175 {
1176 constexpr auto [...__is] = _IotaArray<_S_size>;
1177 _M_data = _DataType{static_cast<value_type>(__ptr[__is])...};
1178 }
1179 else
1180 {
1181 if constexpr (__converts_trivially<_Up, value_type>)
1182 // This assumes std::floatN_t to be bitwise equal to float/double
1183 __builtin_memcpy(&_M_data, __ptr, sizeof(value_type) * _S_size);
1184 else
1185 {
1186 __vec_builtin_type<_Up, _S_full_size> __tmp = {};
1187 __builtin_memcpy(&__tmp, __ptr, sizeof(_Up) * _S_size);
1188 _M_data = __vec_cast<_DataType>(__tmp);
1189 }
1190 }
1191 }
1192
1193 template <ranges::contiguous_range _Rg, typename... _Flags>
1194 requires ranges::__static_sized_range<_Rg>
1195 && __vectorizable<ranges::range_value_t<_Rg>>
1196 && __explicitly_convertible_to<ranges::range_value_t<_Rg>, value_type>
1197 [[__gnu__::__always_inline__]]
1198 constexpr
1199 basic_vec(_Rg&& __range, flags<_Flags...> __flags = {})
1200 requires (ranges::size(__range) == _S_size)
1201 : basic_vec(_LoadCtorTag(), __flags.template _S_adjust_pointer<basic_vec>(
1202 ranges::data(__range)))
1203 {
1204 static_assert(__loadstore_convertible_to<ranges::range_value_t<_Rg>, value_type,
1205 _Flags...>);
1206 }
1207
1208 // [simd.subscr] --------------------------------------------------------
1209 /**
1210 * @brief Return the value of the element at index @p __i.
1211 *
1212 * @pre __i >= 0 && __i < size().
1213 */
1214 [[__gnu__::__always_inline__]]
1215 constexpr value_type
1216 operator[](__simd_size_type __i) const
1217 {
1218 __glibcxx_simd_precondition(__i >= 0 && __i < _S_size, "subscript is out of bounds");
1219 if constexpr (_S_is_scalar)
1220 return _M_data;
1221 else
1222 return _M_data[__i];
1223 }
1224
1225 // [simd.unary] unary operators -----------------------------------------
1226 // increment and decrement are implemented in terms of operator+=/-= which avoids UB on
1227 // padding elements while not breaking UBsan
1228 [[__gnu__::__always_inline__]]
1229 constexpr basic_vec&
1230 operator++() noexcept requires requires(value_type __a) { ++__a; }
1231 { return *this += value_type(1); }
1232
1233 [[__gnu__::__always_inline__]]
1234 constexpr basic_vec
1235 operator++(int) noexcept requires requires(value_type __a) { __a++; }
1236 {
1237 basic_vec __r = *this;
1238 *this += value_type(1);
1239 return __r;
1240 }
1241
1242 [[__gnu__::__always_inline__]]
1243 constexpr basic_vec&
1244 operator--() noexcept requires requires(value_type __a) { --__a; }
1245 { return *this -= value_type(1); }
1246
1247 [[__gnu__::__always_inline__]]
1248 constexpr basic_vec
1249 operator--(int) noexcept requires requires(value_type __a) { __a--; }
1250 {
1251 basic_vec __r = *this;
1252 *this -= value_type(1);
1253 return __r;
1254 }
1255
1256 [[__gnu__::__always_inline__]]
1257 constexpr mask_type
1258 operator!() const noexcept requires requires(value_type __a) { !__a; }
1259 { return *this == value_type(); }
1260
1261 /**
1262 * @brief Unary plus operator (no-op).
1263 *
1264 * Returns an unchanged copy of the object.
1265 */
1266 [[__gnu__::__always_inline__]]
1267 constexpr basic_vec
1268 operator+() const noexcept requires requires(value_type __a) { +__a; }
1269 { return *this; }
1270
1271 /**
1272 * @brief Unary negation operator.
1273 *
1274 * Returns a new SIMD vector after element-wise negation.
1275 */
1276 [[__gnu__::__always_inline__]]
1277 constexpr basic_vec
1278 operator-() const noexcept requires requires(value_type __a) { -__a; }
1279 { return _S_init(-_M_data); }
1280
1281 /**
1282 * @brief Bitwise NOT / complement operator.
1283 *
1284 * Returns a new SIMD vector after element-wise complement.
1285 */
1286 [[__gnu__::__always_inline__]]
1287 constexpr basic_vec
1288 operator~() const noexcept requires requires(value_type __a) { ~__a; }
1289 { return _S_init(~_M_data); }
1290
1291 // [simd.cassign] binary operators
1292 /**
1293 * @brief Bitwise AND operator.
1294 *
1295 * Returns a new SIMD vector after element-wise AND.
1296 */
1297 [[__gnu__::__always_inline__]]
1298 friend constexpr basic_vec&
1299 operator&=(basic_vec& __x, const basic_vec& __y) noexcept
1300 requires requires(value_type __a) { __a & __a; }
1301 {
1302 __x._M_data &= __y._M_data;
1303 return __x;
1304 }
1305
1306 /**
1307 * @brief Bitwise OR operator.
1308 *
1309 * Returns a new SIMD vector after element-wise OR.
1310 */
1311 [[__gnu__::__always_inline__]]
1312 friend constexpr basic_vec&
1313 operator|=(basic_vec& __x, const basic_vec& __y) noexcept
1314 requires requires(value_type __a) { __a | __a; }
1315 {
1316 __x._M_data |= __y._M_data;
1317 return __x;
1318 }
1319
1320 /**
1321 * @brief Bitwise XOR operator.
1322 *
1323 * Returns a new SIMD vector after element-wise XOR.
1324 */
1325 [[__gnu__::__always_inline__]]
1326 friend constexpr basic_vec&
1327 operator^=(basic_vec& __x, const basic_vec& __y) noexcept
1328 requires requires(value_type __a) { __a ^ __a; }
1329 {
1330 __x._M_data ^= __y._M_data;
1331 return __x;
1332 }
1333
1334 /**
1335 * @brief Applies the compound assignment operator element-wise.
1336 *
1337 * @pre If @c value_type is a signed integral type, the result is representable by @c
1338 * value_type. (This does not apply to padding elements the implementation might add for
1339 * non-power-of-2 widths.) UBsan will only see a call to @c unreachable() on overflow.
1340 *
1341 * @note The overflow detection code is discarded unless UBsan is active.
1342 */
1343 [[__gnu__::__always_inline__]]
1344 friend constexpr basic_vec&
1345 operator+=(basic_vec& __x, const basic_vec& __y) noexcept
1346 requires requires(value_type __a) { __a + __a; }
1347 {
1348 if constexpr (_S_is_partial && is_integral_v<value_type> && is_signed_v<value_type>)
1349 { // avoid spurious UB on signed integer overflow of the padding element(s). But don't
1350 // remove UB of the active elements (so that UBsan can still do its job).
1351 //
1352 // This check is essentially free (at runtime) because DCE removes everything except
1353 // the final change to _M_data. The overflow check is only emitted if UBsan is active.
1354 //
1355 // The alternative would be to always zero padding elements after operations that can
1356 // produce non-zero values. However, right now:
1357 // - auto f(simd::mask<int, 3> k) { return +k; } is a single VPABSD and would have to
1358 // sanitize
1359 // - bit_cast to basic_vec with non-zero padding elements is fine
1360 // - conversion from intrinsics can create non-zero padding elements
1361 // - shuffles are allowed to put whatever they want into padding elements for
1362 // optimization purposes (e.g. for better instruction selection)
1363 using _UV = typename _Ap::template _DataType<make_unsigned_t<value_type>>;
1364 const _DataType __result
1365 = reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
1366 + reinterpret_cast<_UV>(__y._M_data));
1367 const auto __positive = __y > value_type();
1368 const auto __overflow = __positive != (__result > __x);
1369 if (__overflow._M_any_of())
1370 __builtin_unreachable(); // trigger UBsan
1371 __x._M_data = __result;
1372 }
1373 else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
1374 __x = basic_vec(rebind_t<float, basic_vec>(__x) + __y);
1375 else
1376 __x._M_data += __y._M_data;
1377 return __x;
1378 }
1379
1380 /** @copydoc operator+=
1381 */
1382 [[__gnu__::__always_inline__]]
1383 friend constexpr basic_vec&
1384 operator-=(basic_vec& __x, const basic_vec& __y) noexcept
1385 requires requires(value_type __a) { __a - __a; }
1386 {
1387 if constexpr (_S_is_partial && is_integral_v<value_type> && is_signed_v<value_type>)
1388 { // see comment on operator+=
1389 using _UV = typename _Ap::template _DataType<make_unsigned_t<value_type>>;
1390 const _DataType __result
1391 = reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
1392 - reinterpret_cast<_UV>(__y._M_data));
1393 const auto __positive = __y > value_type();
1394 const auto __overflow = __positive != (__result < __x);
1395 if (__overflow._M_any_of())
1396 __builtin_unreachable(); // trigger UBsan
1397 __x._M_data = __result;
1398 }
1399 else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
1400 __x = basic_vec(rebind_t<float, basic_vec>(__x) - __y);
1401 else
1402 __x._M_data -= __y._M_data;
1403 return __x;
1404 }
1405
1406 /** @copydoc operator+=
1407 */
1408 [[__gnu__::__always_inline__]]
1409 friend constexpr basic_vec&
1410 operator*=(basic_vec& __x, const basic_vec& __y) noexcept
1411 requires requires(value_type __a) { __a * __a; }
1412 {
1413 if constexpr (_S_is_partial && is_integral_v<value_type> && is_signed_v<value_type>)
1414 { // see comment on operator+=
1415 for (int __i = 0; __i < _S_size; ++__i)
1416 {
1417 if (__builtin_mul_overflow_p(__x._M_data[__i], __y._M_data[__i], value_type()))
1418 __builtin_unreachable();
1419 }
1420 using _UV = typename _Ap::template _DataType<make_unsigned_t<value_type>>;
1421 __x._M_data = reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
1422 * reinterpret_cast<_UV>(__y._M_data));
1423 }
1424
1425 // 'uint16 * uint16' promotes to int and can therefore lead to UB. The standard does not
1426 // require to avoid the undefined behavior. It's unnecessary and easy to avoid. It's also
1427 // unexpected because there's no UB on the vector types (which don't promote).
1428 else if constexpr (_S_is_scalar && is_unsigned_v<value_type>
1429 && is_signed_v<decltype(value_type() * value_type())>)
1430 __x._M_data = unsigned(__x._M_data) * unsigned(__y._M_data);
1431
1432 else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
1433 __x = basic_vec(rebind_t<float, basic_vec>(__x) * __y);
1434
1435 else
1436 __x._M_data *= __y._M_data;
1437 return __x;
1438 }
1439
1440 template <_TargetTraits _Traits = {}>
1441 [[__gnu__::__always_inline__]]
1442 friend constexpr basic_vec&
1443 operator/=(basic_vec& __x, const basic_vec& __y) noexcept
1444 requires requires(value_type __a) { __a / __a; }
1445 {
1446 const basic_vec __result([&](int __i) -> value_type { return __x[__i] / __y[__i]; });
1447 if (__is_const_known(__result))
1448 // the optimizer already knows the values of the result
1449 return __x = __result;
1450
1451#ifdef __SSE2__
1452 // x86 doesn't have integral SIMD division instructions
1453 // While division is faster, the required conversions are still a problem:
1454 // see PR121274, PR121284, and PR121296 for missed optimizations wrt. conversions
1455 //
1456 // With only 1 or 2 divisions, the conversion to and from fp is too expensive.
1457 if constexpr (is_integral_v<value_type> && _S_size > 2
1458 && __value_preserving_convertible_to<value_type, double>)
1459 {
1460 // If the denominator (y) is known to the optimizer, don't convert to fp because the
1461 // integral division can be translated into shifts/multiplications.
1462 if (!__is_const_known(__y))
1463 {
1464 // With AVX512FP16 use vdivph for 8-bit integers
1465 if constexpr (_Traits._M_have_avx512fp16()
1466 && __value_preserving_convertible_to<value_type, _Float16>)
1467 return __x = basic_vec(rebind_t<_Float16, basic_vec>(__x) / __y);
1468 else if constexpr (__value_preserving_convertible_to<value_type, float>)
1469 return __x = basic_vec(rebind_t<float, basic_vec>(__x) / __y);
1470 else
1471 return __x = basic_vec(rebind_t<double, basic_vec>(__x) / __y);
1472 }
1473 }
1474#endif
1475 if constexpr (_Traits._M_eval_as_f32<value_type>())
1476 return __x = basic_vec(rebind_t<float, basic_vec>(__x) / __y);
1477
1478 basic_vec __y1 = __y;
1479 if constexpr (_S_is_partial)
1480 {
1481 if constexpr (is_integral_v<value_type>)
1482 {
1483 // Assume integral division doesn't have SIMD instructions and must be done per
1484 // element anyway. Partial vectors should skip their padding elements.
1485 for (int __i = 0; __i < _S_size; ++__i)
1486 __x._M_data[__i] /= __y._M_data[__i];
1487 return __x;
1488 }
1489 else
1490 __y1 = __select_impl(mask_type::_S_init(mask_type::_S_implicit_mask),
1491 __y, basic_vec(value_type(1)));
1492 }
1493 __x._M_data /= __y1._M_data;
1494 return __x;
1495 }
1496
1497 [[__gnu__::__always_inline__]]
1498 friend constexpr basic_vec&
1499 operator%=(basic_vec& __x, const basic_vec& __y) noexcept
1500 requires requires(value_type __a) { __a % __a; }
1501 {
1502 static_assert(is_integral_v<value_type>);
1503 if constexpr (_S_is_partial)
1504 {
1505 const basic_vec __y1 = __select_impl(mask_type::_S_init(mask_type::_S_implicit_mask),
1506 __y, basic_vec(value_type(1)));
1507 if (__is_const_known(__y1))
1508 __x._M_data %= __y1._M_data;
1509 else
1510 {
1511 // Assume integral division doesn't have SIMD instructions and must be done per
1512 // element anyway. Partial vectors should skip their padding elements.
1513 for (int __i = 0; __i < _S_size; ++__i)
1514 __x._M_data[__i] %= __y._M_data[__i];
1515 }
1516 }
1517 else
1518 __x._M_data %= __y._M_data;
1519 return __x;
1520 }
1521
1522 [[__gnu__::__always_inline__]]
1523 friend constexpr basic_vec&
1524 operator<<=(basic_vec& __x, const basic_vec& __y) _GLIBCXX_SIMD_NOEXCEPT
1525 requires requires(value_type __a) { __a << __a; }
1526 {
1527 __glibcxx_simd_precondition(is_unsigned_v<value_type> || all_of(__y >= value_type()),
1528 "negative shift is undefined behavior");
1529 __glibcxx_simd_precondition(all_of(__y < __max_shift<value_type>),
1530 "too large shift invokes undefined behavior");
1531 __x._M_data <<= __y._M_data;
1532 return __x;
1533 }
1534
1535 [[__gnu__::__always_inline__]]
1536 friend constexpr basic_vec&
1537 operator>>=(basic_vec& __x, const basic_vec& __y) _GLIBCXX_SIMD_NOEXCEPT
1538 requires requires(value_type __a) { __a >> __a; }
1539 {
1540 __glibcxx_simd_precondition(is_unsigned_v<value_type> || all_of(__y >= value_type()),
1541 "negative shift is undefined behavior");
1542 __glibcxx_simd_precondition(all_of(__y < __max_shift<value_type>),
1543 "too large shift invokes undefined behavior");
1544 __x._M_data >>= __y._M_data;
1545 return __x;
1546 }
1547
1548 [[__gnu__::__always_inline__]]
1549 friend constexpr basic_vec&
1550 operator<<=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
1551 requires requires(value_type __a, __simd_size_type __b) { __a << __b; }
1552 {
1553 __glibcxx_simd_precondition(__y >= 0, "negative shift is undefined behavior");
1554 __glibcxx_simd_precondition(__y < int(__max_shift<value_type>),
1555 "too large shift invokes undefined behavior");
1556 __x._M_data <<= __y;
1557 return __x;
1558 }
1559
1560 [[__gnu__::__always_inline__]]
1561 friend constexpr basic_vec&
1562 operator>>=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
1563 requires requires(value_type __a, __simd_size_type __b) { __a >> __b; }
1564 {
1565 __glibcxx_simd_precondition(__y >= 0, "negative shift is undefined behavior");
1566 __glibcxx_simd_precondition(__y < int(__max_shift<value_type>),
1567 "too large shift invokes undefined behavior");
1568 __x._M_data >>= __y;
1569 return __x;
1570 }
1571
1572 // [simd.comparison] ----------------------------------------------------
1573#if _GLIBCXX_X86
1574 template <_X86Cmp _Cmp>
1575 [[__gnu__::__always_inline__]]
1576 constexpr mask_type
1577 _M_bitmask_cmp(_DataType __y) const
1578 {
1579 static_assert(_S_use_bitmask);
1580 if (__is_const_known(_M_data, __y))
1581 {
1582 constexpr auto [...__is] = _IotaArray<_S_size>;
1583 constexpr auto __cmp_op = [] [[__gnu__::__always_inline__]]
1584 (value_type __a, value_type __b) {
1585 if constexpr (_Cmp == _X86Cmp::_Eq)
1586 return __a == __b;
1587 else if constexpr (_Cmp == _X86Cmp::_Lt)
1588 return __a < __b;
1589 else if constexpr (_Cmp == _X86Cmp::_Le)
1590 return __a <= __b;
1591 else if constexpr (_Cmp == _X86Cmp::_Unord)
1592 return std::isunordered(__a, __b);
1593 else if constexpr (_Cmp == _X86Cmp::_Neq)
1594 return __a != __b;
1595 else if constexpr (_Cmp == _X86Cmp::_Nlt)
1596 return !(__a < __b);
1597 else if constexpr (_Cmp == _X86Cmp::_Nle)
1598 return !(__a <= __b);
1599 else
1600 static_assert(false);
1601 };
1602 const _Bitmask<_S_size> __bits
1603 = ((__cmp_op(__vec_get(_M_data, __is), __vec_get(__y, __is))
1604 ? (1ULL << __is) : 0) | ...);
1605 return mask_type::_S_init(__bits);
1606 }
1607 else
1608 return mask_type::_S_init(__x86_bitmask_cmp<_Cmp>(_M_data, __y));
1609 }
1610#endif
1611
1612 [[__gnu__::__always_inline__]]
1613 friend constexpr mask_type
1614 operator==(const basic_vec& __x, const basic_vec& __y) noexcept
1615 {
1616#if _GLIBCXX_X86
1617 if constexpr (_S_use_bitmask)
1618 return __x._M_bitmask_cmp<_X86Cmp::_Eq>(__y._M_data);
1619 else
1620#endif
1621 return mask_type::_S_init(__x._M_data == __y._M_data);
1622 }
1623
1624 [[__gnu__::__always_inline__]]
1625 friend constexpr mask_type
1626 operator!=(const basic_vec& __x, const basic_vec& __y) noexcept
1627 {
1628#if _GLIBCXX_X86
1629 if constexpr (_S_use_bitmask)
1630 return __x._M_bitmask_cmp<_X86Cmp::_Neq>(__y._M_data);
1631 else
1632#endif
1633 return mask_type::_S_init(__x._M_data != __y._M_data);
1634 }
1635
1636 [[__gnu__::__always_inline__]]
1637 friend constexpr mask_type
1638 operator<(const basic_vec& __x, const basic_vec& __y) noexcept
1639 {
1640#if _GLIBCXX_X86
1641 if constexpr (_S_use_bitmask)
1642 return __x._M_bitmask_cmp<_X86Cmp::_Lt>(__y._M_data);
1643 else
1644#endif
1645 return mask_type::_S_init(__x._M_data < __y._M_data);
1646 }
1647
1648 [[__gnu__::__always_inline__]]
1649 friend constexpr mask_type
1650 operator<=(const basic_vec& __x, const basic_vec& __y) noexcept
1651 {
1652#if _GLIBCXX_X86
1653 if constexpr (_S_use_bitmask)
1654 return __x._M_bitmask_cmp<_X86Cmp::_Le>(__y._M_data);
1655 else
1656#endif
1657 return mask_type::_S_init(__x._M_data <= __y._M_data);
1658 }
1659
1660 [[__gnu__::__always_inline__]]
1661 friend constexpr mask_type
1662 operator>(const basic_vec& __x, const basic_vec& __y) noexcept
1663 { return __y < __x; }
1664
1665 [[__gnu__::__always_inline__]]
1666 friend constexpr mask_type
1667 operator>=(const basic_vec& __x, const basic_vec& __y) noexcept
1668 { return __y <= __x; }
1669
1670 // [simd.cond] ---------------------------------------------------------
1671 template <_TargetTraits _Traits = {}>
1672 [[__gnu__::__always_inline__]]
1673 friend constexpr basic_vec
1674 __select_impl(const mask_type& __k, const basic_vec& __t, const basic_vec& __f) noexcept
1675 {
1676 if constexpr (_S_size == 1)
1677 return __k[0] ? __t : __f;
1678 else if constexpr (_S_use_bitmask)
1679 {
1680#if _GLIBCXX_X86
1681 if (__is_const_known(__k, __t, __f))
1682 return basic_vec([&](int __i) { return __k[__i] ? __t[__i] : __f[__i]; });
1683 else
1684 return __x86_bitmask_blend(__k._M_data, __t._M_data, __f._M_data);
1685#else
1686 static_assert(false, "TODO");
1687#endif
1688 }
1689 else if consteval
1690 {
1691 return __k._M_data ? __t._M_data : __f._M_data;
1692 }
1693 else
1694 {
1695 constexpr bool __uses_simd_register = sizeof(_M_data) >= 8;
1696 using _VO = _VecOps<_DataType>;
1697 if (_VO::_S_is_const_known_equal_to(__f._M_data, 0))
1698 {
1699 if (is_integral_v<value_type> && __uses_simd_register
1700 && _VO::_S_is_const_known_equal_to(__t._M_data, 1))
1701 // This is equivalent to converting the mask into a vec of 0s and 1s. So +__k.
1702 // However, basic_mask::operator+ arrives here; returning +__k would be
1703 // recursive. Instead we use -__k (which is a no-op for vector-masks) and then
1704 // flip all -1 elements to +1 by taking the absolute value.
1705 return basic_vec((-__k)._M_abs());
1706 else
1707 return __vec_and(reinterpret_cast<_DataType>(__k._M_data), __t._M_data);
1708 }
1709 else if (_VecOps<_DataType>::_S_is_const_known_equal_to(__t._M_data, 0))
1710 {
1711 if (is_integral_v<value_type> && __uses_simd_register
1712 && _VO::_S_is_const_known_equal_to(__f._M_data, 1))
1713 return value_type(1) + basic_vec(-__k);
1714 else
1715 return __vec_and(reinterpret_cast<_DataType>(__vec_not(__k._M_data)), __f._M_data);
1716 }
1717 else
1718 {
1719#if _GLIBCXX_X86
1720 // this works around bad code-gen when the compiler can't see that __k is a vector-mask.
1721 // This pattern, is recognized to match the x86 blend instructions, which only consider
1722 // the sign bit of the mask register. Also, without SSE4, if the compiler knows that __k
1723 // is a vector-mask, then the '< 0' is elided.
1724 return __k._M_data < 0 ? __t._M_data : __f._M_data;
1725#endif
1726 return __k._M_data ? __t._M_data : __f._M_data;
1727 }
1728 }
1729 }
1730 };
1731
1732 template <__vectorizable _Tp, __abi_tag _Ap>
1733 requires (_Ap::_S_nreg > 1)
1734 && (!__complex_like<_Tp>)
1735 class basic_vec<_Tp, _Ap>
1736 : public _VecBase<_Tp, _Ap>
1737 {
1738 template <typename, typename>
1739 friend class basic_vec;
1740
1741 template <size_t, typename>
1742 friend class basic_mask;
1743
1744 static constexpr int _S_size = _Ap::_S_size;
1745
1746 static constexpr int _N0 = __bit_ceil(unsigned(_S_size)) / 2;
1747
1748 static constexpr int _N1 = _S_size - _N0;
1749
1750 using _DataType0 = __similar_vec<_Tp, _N0, _Ap>;
1751
1752 // the implementation (and users) depend on elements being contiguous in memory
1753 static_assert(_N0 * sizeof(_Tp) == sizeof(_DataType0));
1754
1755 using _DataType1 = __similar_vec<_Tp, _N1, _Ap>;
1756
1757 static_assert(_DataType0::abi_type::_S_nreg + _DataType1::abi_type::_S_nreg == _Ap::_S_nreg);
1758
1759 static constexpr bool _S_is_scalar = _DataType0::_S_is_scalar;
1760
1761 _DataType0 _M_data0;
1762
1763 _DataType1 _M_data1;
1764
1765 static constexpr bool _S_use_bitmask = _DataType0::_S_use_bitmask;
1766
1767 static constexpr bool _S_is_partial = _DataType1::_S_is_partial;
1768
1769 public:
1770 using value_type = _Tp;
1771
1772 using mask_type = _VecBase<_Tp, _Ap>::mask_type;
1773
1774 [[__gnu__::__always_inline__]]
1775 static constexpr basic_vec
1776 _S_init(const _DataType0& __x, const _DataType1& __y)
1777 {
1778 basic_vec __r;
1779 __r._M_data0 = __x;
1780 __r._M_data1 = __y;
1781 return __r;
1782 }
1783
1784 [[__gnu__::__always_inline__]]
1785 constexpr _DataType0&
1786 _M_get_low() noexcept
1787 { return _M_data0; }
1788
1789 [[__gnu__::__always_inline__]]
1790 constexpr const _DataType0&
1791 _M_get_low() const noexcept
1792 { return _M_data0; }
1793
1794 [[__gnu__::__always_inline__]]
1795 constexpr _DataType1&
1796 _M_get_high() noexcept
1797 { return _M_data1; }
1798
1799 [[__gnu__::__always_inline__]]
1800 constexpr const _DataType1&
1801 _M_get_high() const noexcept
1802 { return _M_data1; }
1803
1804 [[__gnu__::__always_inline__]]
1805 friend constexpr bool
1806 __is_const_known(const basic_vec& __x)
1807 { return __is_const_known(__x._M_data0) && __is_const_known(__x._M_data1); }
1808
1809 [[__gnu__::__always_inline__]]
1810 constexpr auto
1811 _M_concat_data([[maybe_unused]] bool __do_sanitize = false) const
1812 {
1813 return __vec_concat(_M_data0._M_concat_data(false),
1814 __vec_zero_pad_to<sizeof(_M_data0)>(
1815 _M_data1._M_concat_data(__do_sanitize)));
1816 }
1817
1818 template <int _Size = _S_size, int _Offset = 0, typename _A0, typename _Fp>
1819 [[__gnu__::__always_inline__]]
1820 static constexpr basic_vec
1821 _S_static_permute(const basic_vec<value_type, _A0>& __x, _Fp&& __idxmap)
1822 {
1823 return _S_init(
1824 _DataType0::template _S_static_permute<_Size, _Offset>(__x, __idxmap),
1825 _DataType1::template _S_static_permute<_Size, _Offset + _N0>(__x, __idxmap));
1826 }
1827
1828 template <typename _Vp>
1829 [[__gnu__::__always_inline__]]
1830 constexpr auto
1831 _M_chunk() const noexcept
1832 {
1833 constexpr int __n = _S_size / _Vp::_S_size;
1834 constexpr int __rem = _S_size % _Vp::_S_size;
1835 constexpr auto [...__is] = _IotaArray<__n>;
1836 if constexpr (__rem == 0)
1837 return array<_Vp, __n>{__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>,
1838 _M_data0, _M_data1)...};
1839 else
1840 {
1841 using _Rest = resize_t<__rem, _Vp>;
1842 return tuple(__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>, _M_data0, _M_data1)...,
1843 __extract_simd_at<_Rest>(cw<_Vp::_S_size * __n>, _M_data0, _M_data1));
1844 }
1845 }
1846
1847 [[__gnu__::__always_inline__]]
1848 static constexpr const basic_vec&
1849 _S_concat(const basic_vec& __x0) noexcept
1850 { return __x0; }
1851
1852 template <typename... _As>
1853 requires (sizeof...(_As) >= 2)
1854 [[__gnu__::__always_inline__]]
1855 static constexpr basic_vec
1856 _S_concat(const basic_vec<value_type, _As>&... __xs) noexcept
1857 {
1858 static_assert(_S_size == (_As::_S_size + ...));
1859 return _S_init(__extract_simd_at<_DataType0>(cw<0>, __xs...),
1860 __extract_simd_at<_DataType1>(cw<_N0>, __xs...));
1861 }
1862
1863 [[__gnu__::__always_inline__]]
1864 constexpr auto
1865 _M_reduce_to_half(auto __binary_op) const requires (_N0 == _N1)
1866 { return __binary_op(_M_data0, _M_data1); }
1867
1868 [[__gnu__::__always_inline__]]
1869 constexpr value_type
1870 _M_reduce_tail(const auto& __rest, auto __binary_op) const
1871 {
1872 if constexpr (__rest.size() > _S_size)
1873 {
1874 auto [__a, __b] = __rest.template _M_chunk<basic_vec>();
1875 return __binary_op(*this, __a)._M_reduce_tail(__b, __binary_op);
1876 }
1877 else if constexpr (__rest.size() == _S_size)
1878 return __binary_op(*this, __rest)._M_reduce(__binary_op);
1879 else
1880 return _M_reduce_to_half(__binary_op)._M_reduce_tail(__rest, __binary_op);
1881 }
1882
1883 template <typename _BinaryOp, _TargetTraits _Traits = {}>
1884 [[__gnu__::__always_inline__]]
1885 constexpr value_type
1886 _M_reduce(_BinaryOp __binary_op) const
1887 {
1888 if constexpr (_Traits.template _M_eval_as_f32<value_type>()
1889 && (is_same_v<_BinaryOp, plus<>>
1890 || is_same_v<_BinaryOp, multiplies<>>))
1891 return value_type(rebind_t<float, basic_vec>(*this)._M_reduce(__binary_op));
1892#ifdef __SSE2__
1893 else if constexpr (is_integral_v<value_type> && sizeof(value_type) == 1
1894 && is_same_v<decltype(__binary_op), multiplies<>>)
1895 {
1896 // convert to unsigned short because of missing 8-bit mul instruction
1897 // we don't need to preserve the order of elements
1898 //
1899 // The left columns under Latency and Throughput show bit-cast to ushort with shift by
1900 // 8. The right column uses the alternative in the else branch.
1901 // Benchmark on Intel Ultra 7 165U (AVX2)
1902 // TYPE Latency Throughput
1903 // [cycles/call] [cycles/call]
1904 //schar, 64 59.9 70.7 10.5 13.3
1905 //schar, 128 81.4 97.2 12.2 21
1906 //schar, 256 92.4 129 17.2 35.2
1907 if constexpr (_DataType1::_S_is_scalar)
1908 return __binary_op(_DataType1(_M_data0._M_reduce(__binary_op)), _M_data1)[0];
1909 // TODO: optimize trailing scalar (e.g. (8+8)+(8+1))
1910 else if constexpr (_S_size % 2 == 0)
1911 { // If all elements participate in the reduction we can take this shortcut
1912 using _V16 = resize_t<_S_size / 2, rebind_t<unsigned short, basic_vec>>;
1913 auto __a = __builtin_bit_cast(_V16, *this);
1914 return __binary_op(__a, __a >> __CHAR_BIT__)._M_reduce(__binary_op);
1915 }
1916 else
1917 {
1918 using _V16 = rebind_t<unsigned short, basic_vec>;
1919 return _V16(*this)._M_reduce(__binary_op);
1920 }
1921 }
1922#endif
1923 else
1924 return _M_data0._M_reduce_tail(_M_data1, __binary_op);
1925 }
1926
1927 [[__gnu__::__always_inline__]]
1928 constexpr mask_type
1929 _M_isnan() const requires is_floating_point_v<value_type>
1930 { return mask_type::_S_init(_M_data0._M_isnan(), _M_data1._M_isnan()); }
1931
1932 [[__gnu__::__always_inline__]]
1933 constexpr mask_type
1934 _M_isinf() const requires is_floating_point_v<value_type>
1935 { return mask_type::_S_init(_M_data0._M_isinf(), _M_data1._M_isinf()); }
1936
1937 [[__gnu__::__always_inline__]]
1938 constexpr mask_type
1939 _M_isunordered(basic_vec __y) const requires is_floating_point_v<value_type>
1940 {
1941 return mask_type::_S_init(_M_data0._M_isunordered(__y._M_data0),
1942 _M_data1._M_isunordered(__y._M_data1));
1943 }
1944
1945 [[__gnu__::__always_inline__]]
1946 constexpr basic_vec
1947 _M_abs() const requires signed_integral<value_type>
1948 { return _S_init(_M_data0._M_abs(), _M_data1._M_abs()); }
1949
1950 [[__gnu__::__always_inline__]]
1951 constexpr basic_vec
1952 _M_fabs() const requires floating_point<value_type>
1953 { return _S_init(_M_data0._M_fabs(), _M_data1._M_fabs()); }
1954
1955 template <typename _Up>
1956 [[__gnu__::__always_inline__]]
1957 static inline basic_vec
1958 _S_partial_load(const _Up* __mem, size_t __n)
1959 {
1960 if (__n >= _N0)
1961 return _S_init(_DataType0(_LoadCtorTag(), __mem),
1962 _DataType1::_S_partial_load(__mem + _N0, __n - _N0));
1963 else
1964 return _S_init(_DataType0::_S_partial_load(__mem, __n),
1965 _DataType1());
1966 }
1967
1968 template <typename _Up, _ArchTraits _Traits = {}>
1969 static inline basic_vec
1970 _S_masked_load(const _Up* __mem, mask_type __k)
1971 {
1972 return _S_init(_DataType0::_S_masked_load(__mem, __k._M_data0),
1973 _DataType1::_S_masked_load(__mem + _N0, __k._M_data1));
1974 }
1975
1976 template <typename _Up>
1977 [[__gnu__::__always_inline__]]
1978 inline void
1979 _M_store(_Up* __mem) const
1980 {
1981 _M_data0._M_store(__mem);
1982 _M_data1._M_store(__mem + _N0);
1983 }
1984
1985 template <typename _Up>
1986 [[__gnu__::__always_inline__]]
1987 static inline void
1988 _S_partial_store(const basic_vec& __v, _Up* __mem, size_t __n)
1989 {
1990 if (__n >= _N0)
1991 {
1992 __v._M_data0._M_store(__mem);
1993 _DataType1::_S_partial_store(__v._M_data1, __mem + _N0, __n - _N0);
1994 }
1995 else
1996 {
1997 _DataType0::_S_partial_store(__v._M_data0, __mem, __n);
1998 }
1999 }
2000
2001 template <typename _Up>
2002 [[__gnu__::__always_inline__]]
2003 static inline void
2004 _S_masked_store(const basic_vec& __v, _Up* __mem, const mask_type& __k)
2005 {
2006 _DataType0::_S_masked_store(__v._M_data0, __mem, __k._M_data0);
2007 _DataType1::_S_masked_store(__v._M_data1, __mem + _N0, __k._M_data1);
2008 }
2009
2010 basic_vec() = default;
2011
2012 // [simd.overview] p2 impl-def conversions ------------------------------
2013 using _NativeVecType = __vec_builtin_type<value_type, __bit_ceil(unsigned(_S_size))>;
2014
2015 [[__gnu__::__always_inline__]]
2016 constexpr
2017 basic_vec(const _NativeVecType& __x)
2018 : _M_data0(_VecOps<__vec_builtin_type<value_type, _N0>>::_S_extract(__x)),
2019 _M_data1(_VecOps<__vec_builtin_type<value_type, __bit_ceil(unsigned(_N1))>>
2020 ::_S_extract(__x, integral_constant<int, _N0>()))
2021 {}
2022
2023 [[__gnu__::__always_inline__]]
2024 constexpr
2025 operator _NativeVecType() const
2026 { return _M_concat_data(); }
2027
2028 // [simd.ctor] broadcast constructor ------------------------------------
2029 template <__broadcast_constructible<value_type> _Up>
2030 [[__gnu__::__always_inline__]]
2031 constexpr
2032 basic_vec(_Up&& __x) noexcept
2033 : _M_data0(static_cast<value_type>(__x)), _M_data1(static_cast<value_type>(__x))
2034 {}
2035
2036 // [simd.ctor] conversion constructor -----------------------------------
2037 template <typename _Up, typename _UAbi>
2038 requires (_S_size == _UAbi::_S_size)
2039 && __explicitly_convertible_to<_Up, value_type>
2040 [[__gnu__::__always_inline__]]
2041 constexpr
2042 explicit(!__value_preserving_convertible_to<_Up, value_type>
2043 || __higher_rank_than<_Up, value_type>)
2044 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
2045 : _M_data0(get<0>(chunk<_N0>(__x))),
2046 _M_data1(get<1>(chunk<_N0>(__x)))
2047 {}
2048
2049 using _VecBase<_Tp, _Ap>::_VecBase;
2050
2051 // [simd.ctor] generator constructor ------------------------------------
2052 template <__simd_generator_invokable<value_type, _S_size> _Fp>
2053 [[__gnu__::__always_inline__]]
2054 constexpr explicit
2055 basic_vec(_Fp&& __gen)
2056 : _M_data0(__gen), _M_data1([&] [[__gnu__::__always_inline__]] (auto __i) {
2057 return __gen(__simd_size_c<__i + _N0>);
2058 })
2059 {}
2060
2061 // [simd.ctor] load constructor -----------------------------------------
2062 template <typename _Up>
2063 [[__gnu__::__always_inline__]]
2064 constexpr
2065 basic_vec(_LoadCtorTag, const _Up* __ptr)
2066 : _M_data0(_LoadCtorTag(), __ptr),
2067 _M_data1(_LoadCtorTag(), __ptr + _N0)
2068 {}
2069
2070 template <ranges::contiguous_range _Rg, typename... _Flags>
2071 requires ranges::__static_sized_range<_Rg>
2072 && __vectorizable<ranges::range_value_t<_Rg>>
2073 && __explicitly_convertible_to<ranges::range_value_t<_Rg>, value_type>
2074 constexpr
2075 basic_vec(_Rg&& __range, flags<_Flags...> __flags = {})
2076 requires (ranges::size(__range) == _S_size)
2077 : basic_vec(_LoadCtorTag(),
2078 __flags.template _S_adjust_pointer<basic_vec>(ranges::data(__range)))
2079 {
2080 static_assert(__loadstore_convertible_to<ranges::range_value_t<_Rg>, value_type,
2081 _Flags...>);
2082 }
2083
2084 // [simd.subscr] --------------------------------------------------------
2085 [[__gnu__::__always_inline__]]
2086 constexpr value_type
2087 operator[](__simd_size_type __i) const
2088 {
2089 __glibcxx_simd_precondition(__i >= 0 && __i < _S_size, "subscript is out of bounds");
2090 if (__is_const_known(__i))
2091 return __i < _N0 ? _M_data0[__i] : _M_data1[__i - _N0];
2092 else
2093 {
2094 using _AliasingT [[__gnu__::__may_alias__]] = value_type;
2095 return reinterpret_cast<const _AliasingT*>(this)[__i];
2096 }
2097 }
2098
2099 // [simd.unary] unary operators -----------------------------------------
2100 [[__gnu__::__always_inline__]]
2101 constexpr basic_vec&
2102 operator++() noexcept requires requires(value_type __a) { ++__a; }
2103 {
2104 ++_M_data0;
2105 ++_M_data1;
2106 return *this;
2107 }
2108
2109 [[__gnu__::__always_inline__]]
2110 constexpr basic_vec
2111 operator++(int) noexcept requires requires(value_type __a) { __a++; }
2112 {
2113 basic_vec __r = *this;
2114 ++_M_data0;
2115 ++_M_data1;
2116 return __r;
2117 }
2118
2119 [[__gnu__::__always_inline__]]
2120 constexpr basic_vec&
2121 operator--() noexcept requires requires(value_type __a) { --__a; }
2122 {
2123 --_M_data0;
2124 --_M_data1;
2125 return *this;
2126 }
2127
2128 [[__gnu__::__always_inline__]]
2129 constexpr basic_vec
2130 operator--(int) noexcept requires requires(value_type __a) { __a--; }
2131 {
2132 basic_vec __r = *this;
2133 --_M_data0;
2134 --_M_data1;
2135 return __r;
2136 }
2137
2138 [[__gnu__::__always_inline__]]
2139 constexpr mask_type
2140 operator!() const noexcept requires requires(value_type __a) { !__a; }
2141 { return mask_type::_S_init(!_M_data0, !_M_data1); }
2142
2143 [[__gnu__::__always_inline__]]
2144 constexpr basic_vec
2145 operator+() const noexcept requires requires(value_type __a) { +__a; }
2146 { return *this; }
2147
2148 [[__gnu__::__always_inline__]]
2149 constexpr basic_vec
2150 operator-() const noexcept requires requires(value_type __a) { -__a; }
2151 { return _S_init(-_M_data0, -_M_data1); }
2152
2153 [[__gnu__::__always_inline__]]
2154 constexpr basic_vec
2155 operator~() const noexcept requires requires(value_type __a) { ~__a; }
2156 { return _S_init(~_M_data0, ~_M_data1); }
2157
2158 // [simd.cassign] -------------------------------------------------------
2159#define _GLIBCXX_SIMD_DEFINE_OP(sym) \
2160 [[__gnu__::__always_inline__]] \
2161 friend constexpr basic_vec& \
2162 operator sym##=(basic_vec& __x, const basic_vec& __y) _GLIBCXX_SIMD_NOEXCEPT \
2163 { \
2164 __x._M_data0 sym##= __y._M_data0; \
2165 __x._M_data1 sym##= __y._M_data1; \
2166 return __x; \
2167 }
2168
2169 _GLIBCXX_SIMD_DEFINE_OP(+)
2170 _GLIBCXX_SIMD_DEFINE_OP(-)
2171 _GLIBCXX_SIMD_DEFINE_OP(*)
2172 _GLIBCXX_SIMD_DEFINE_OP(/)
2173 _GLIBCXX_SIMD_DEFINE_OP(%)
2174 _GLIBCXX_SIMD_DEFINE_OP(&)
2175 _GLIBCXX_SIMD_DEFINE_OP(|)
2176 _GLIBCXX_SIMD_DEFINE_OP(^)
2177 _GLIBCXX_SIMD_DEFINE_OP(<<)
2178 _GLIBCXX_SIMD_DEFINE_OP(>>)
2179
2180#undef _GLIBCXX_SIMD_DEFINE_OP
2181
2182 [[__gnu__::__always_inline__]]
2183 friend constexpr basic_vec&
2184 operator<<=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
2185 requires requires(value_type __a, __simd_size_type __b) { __a << __b; }
2186 {
2187 __x._M_data0 <<= __y;
2188 __x._M_data1 <<= __y;
2189 return __x;
2190 }
2191
2192 [[__gnu__::__always_inline__]]
2193 friend constexpr basic_vec&
2194 operator>>=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
2195 requires requires(value_type __a, __simd_size_type __b) { __a >> __b; }
2196 {
2197 __x._M_data0 >>= __y;
2198 __x._M_data1 >>= __y;
2199 return __x;
2200 }
2201
2202 // [simd.comparison] ----------------------------------------------------
2203 [[__gnu__::__always_inline__]]
2204 friend constexpr mask_type
2205 operator==(const basic_vec& __x, const basic_vec& __y) noexcept
2206 { return mask_type::_S_init(__x._M_data0 == __y._M_data0, __x._M_data1 == __y._M_data1); }
2207
2208 [[__gnu__::__always_inline__]]
2209 friend constexpr mask_type
2210 operator!=(const basic_vec& __x, const basic_vec& __y) noexcept
2211 { return mask_type::_S_init(__x._M_data0 != __y._M_data0, __x._M_data1 != __y._M_data1); }
2212
2213 [[__gnu__::__always_inline__]]
2214 friend constexpr mask_type
2215 operator<(const basic_vec& __x, const basic_vec& __y) noexcept
2216 { return mask_type::_S_init(__x._M_data0 < __y._M_data0, __x._M_data1 < __y._M_data1); }
2217
2218 [[__gnu__::__always_inline__]]
2219 friend constexpr mask_type
2220 operator<=(const basic_vec& __x, const basic_vec& __y) noexcept
2221 { return mask_type::_S_init(__x._M_data0 <= __y._M_data0, __x._M_data1 <= __y._M_data1); }
2222
2223 [[__gnu__::__always_inline__]]
2224 friend constexpr mask_type
2225 operator>(const basic_vec& __x, const basic_vec& __y) noexcept
2226 { return mask_type::_S_init(__x._M_data0 > __y._M_data0, __x._M_data1 > __y._M_data1); }
2227
2228 [[__gnu__::__always_inline__]]
2229 friend constexpr mask_type
2230 operator>=(const basic_vec& __x, const basic_vec& __y) noexcept
2231 { return mask_type::_S_init(__x._M_data0 >= __y._M_data0, __x._M_data1 >= __y._M_data1); }
2232
2233 // [simd.cond] ---------------------------------------------------------
2234 [[__gnu__::__always_inline__]]
2235 friend constexpr basic_vec
2236 __select_impl(const mask_type& __k, const basic_vec& __t, const basic_vec& __f) noexcept
2237 {
2238 return _S_init(__select_impl(__k._M_data0, __t._M_data0, __f._M_data0),
2239 __select_impl(__k._M_data1, __t._M_data1, __f._M_data1));
2240 }
2241 };
2242
2243 // [simd.overview] deduction guide ------------------------------------------
2244 template <ranges::contiguous_range _Rg, typename... _Ts>
2245 requires ranges::__static_sized_range<_Rg>
2246 basic_vec(_Rg&& __r, _Ts...)
2247 -> basic_vec<ranges::range_value_t<_Rg>,
2248 __deduce_abi_t<ranges::range_value_t<_Rg>,
2249#if 0 // PR117849
2250 static_cast<__simd_size_type>(ranges::size(__r))>>;
2251#else
2252 static_cast<__simd_size_type>(decltype(std::span(__r))::extent)>>;
2253#endif
2254
2255 template <size_t _Bytes, typename _Ap>
2256 basic_vec(basic_mask<_Bytes, _Ap>)
2257 -> basic_vec<__integer_from<_Bytes>,
2258 decltype(__abi_rebind<__integer_from<_Bytes>, basic_mask<_Bytes, _Ap>::size.value,
2259 _Ap>())>;
2260
2261 // [P3319R5] ----------------------------------------------------------------
2262 template <__vectorizable _Tp>
2263 requires is_arithmetic_v<_Tp>
2264 inline constexpr _Tp
2265 __iota<_Tp> = _Tp();
2266
2267 template <typename _Tp, typename _Ap>
2268 inline constexpr basic_vec<_Tp, _Ap>
2269 __iota<basic_vec<_Tp, _Ap>> = basic_vec<_Tp, _Ap>([](_Tp __i) -> _Tp {
2270 static_assert(_Ap::_S_size - 1 <= numeric_limits<_Tp>::max(),
2271 "iota object would overflow");
2272 return __i;
2273 });
2274} // namespace simd
2275_GLIBCXX_END_NAMESPACE_VERSION
2276} // namespace std
2277
2278#pragma GCC diagnostic pop
2279#endif // C++26
2280#endif // _GLIBCXX_SIMD_VEC_H
std::chrono::operator<=
constexpr bool operator<=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:863
std::chrono::operator>=
constexpr bool operator>=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:877
std::chrono::operator<
constexpr bool operator<(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:830
std::chrono::operator>
constexpr bool operator>(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:870
std::operator-
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
Definition complex:404
std::operator+
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
Definition complex:374
std::is_sufficiently_aligned
bool is_sufficiently_aligned(_Tp *__ptr)
Is __ptr aligned to an _Align byte boundary?
Definition align.h:118
std
ISO C++ entities toplevel namespace is std.
std::fabs
_Tp fabs(const std::complex< _Tp > &__z)
fabs(__z) TR1 8.1.8 [tr.c99.cmplx.fabs]
Definition complex:2525
std::data
constexpr auto data(_Container &__cont) noexcept(noexcept(__cont.data())) -> decltype(__cont.data())
Return the data pointer of a container.
Definition range_access.h:354
std::numeric_limits::max
static constexpr _Tp max() noexcept
Definition limits:336
std::numeric_limits::infinity
static constexpr _Tp infinity() noexcept
Definition limits:356