libstdc++
simd_vec.h
1// Implementation of <simd> -*- C++ -*-
2
3// Copyright The GNU Toolchain Authors.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_SIMD_VEC_H
26#define _GLIBCXX_SIMD_VEC_H 1
27
28#ifdef _GLIBCXX_SYSHDR
29#pragma GCC system_header
30#endif
31
32#if __cplusplus >= 202400L
33
34#include "simd_mask.h"
35#include "simd_flags.h"
36
37#include <bits/utility.h>
38#include <bits/stl_function.h>
39#include <cmath>
40
41// psabi warnings are bogus because the ABI of the internal types never leaks into user code
42#pragma GCC diagnostic push
43#pragma GCC diagnostic ignored "-Wpsabi"
44
45namespace std _GLIBCXX_VISIBILITY(default)
46{
47_GLIBCXX_BEGIN_NAMESPACE_VERSION
48namespace simd
49{
50 // disabled basic_vec
51 template <typename _Tp, typename _Ap>
52 class basic_vec
53 {
54 public:
55 using value_type = _Tp;
56
57 using abi_type = _Ap;
58
59 using mask_type = basic_mask<0, void>; // disabled
60
61#define _GLIBCXX_DELETE_SIMD "This specialization is disabled because of an invalid combination " \
62 "of template arguments to basic_vec."
63
64 basic_vec() = delete(_GLIBCXX_DELETE_SIMD);
65
66 ~basic_vec() = delete(_GLIBCXX_DELETE_SIMD);
67
68 basic_vec(const basic_vec&) = delete(_GLIBCXX_DELETE_SIMD);
69
70 basic_vec& operator=(const basic_vec&) = delete(_GLIBCXX_DELETE_SIMD);
71
72#undef _GLIBCXX_DELETE_SIMD
73 };
74
75 template <typename _Tp, typename _Ap>
76 class _VecBase
77 {
78 using _Vp = basic_vec<_Tp, _Ap>;
79
80 public:
81 using value_type = _Tp;
82
83 using abi_type = _Ap;
84
85 using mask_type = basic_mask<sizeof(_Tp), abi_type>;
86
87 using iterator = __iterator<_Vp>;
88
89 using const_iterator = __iterator<const _Vp>;
90
91 constexpr iterator
92 begin() noexcept
93 { return {static_cast<_Vp&>(*this), 0}; }
94
95 constexpr const_iterator
96 begin() const noexcept
97 { return cbegin(); }
98
99 constexpr const_iterator
100 cbegin() const noexcept
101 { return {static_cast<const _Vp&>(*this), 0}; }
102
103 constexpr default_sentinel_t
104 end() const noexcept
105 { return {}; }
106
107 constexpr default_sentinel_t
108 cend() const noexcept
109 { return {}; }
110
111 static constexpr auto size = __simd_size_c<_Ap::_S_size>;
112
113 _VecBase() = default;
114
115 // LWG issue from 2026-03-04 / P4042R0
116 template <typename _Up, typename _UAbi>
117 requires (_Ap::_S_size != _UAbi::_S_size)
118 _VecBase(const basic_vec<_Up, _UAbi>&) = delete("size mismatch");
119
120 template <typename _Up, typename _UAbi>
121 requires (_Ap::_S_size == _UAbi::_S_size) && (!__explicitly_convertible_to<_Up, _Tp>)
122 explicit
123 _VecBase(const basic_vec<_Up, _UAbi>&)
124 = delete("the value types are not convertible");
125
126 [[__gnu__::__always_inline__]]
127 friend constexpr _Vp
128 operator+(const _Vp& __x, const _Vp& __y) noexcept
129 {
130 _Vp __r = __x;
131 __r += __y;
132 return __r;
133 }
134
135 [[__gnu__::__always_inline__]]
136 friend constexpr _Vp
137 operator-(const _Vp& __x, const _Vp& __y) noexcept
138 {
139 _Vp __r = __x;
140 __r -= __y;
141 return __r;
142 }
143
144 [[__gnu__::__always_inline__]]
145 friend constexpr _Vp
146 operator*(const _Vp& __x, const _Vp& __y) noexcept
147 {
148 _Vp __r = __x;
149 __r *= __y;
150 return __r;
151 }
152
153 [[__gnu__::__always_inline__]]
154 friend constexpr _Vp
155 operator/(const _Vp& __x, const _Vp& __y) noexcept
156 {
157 _Vp __r = __x;
158 __r /= __y;
159 return __r;
160 }
161
162 [[__gnu__::__always_inline__]]
163 friend constexpr _Vp
164 operator%(const _Vp& __x, const _Vp& __y) noexcept
165 requires requires (_Tp __a) { __a % __a; }
166 {
167 _Vp __r = __x;
168 __r %= __y;
169 return __r;
170 }
171
172 [[__gnu__::__always_inline__]]
173 friend constexpr _Vp
174 operator&(const _Vp& __x, const _Vp& __y) noexcept
175 requires requires (_Tp __a) { __a & __a; }
176 {
177 _Vp __r = __x;
178 __r &= __y;
179 return __r;
180 }
181
182 [[__gnu__::__always_inline__]]
183 friend constexpr _Vp
184 operator|(const _Vp& __x, const _Vp& __y) noexcept
185 requires requires (_Tp __a) { __a | __a; }
186 {
187 _Vp __r = __x;
188 __r |= __y;
189 return __r;
190 }
191
192 [[__gnu__::__always_inline__]]
193 friend constexpr _Vp
194 operator^(const _Vp& __x, const _Vp& __y) noexcept
195 requires requires (_Tp __a) { __a ^ __a; }
196 {
197 _Vp __r = __x;
198 __r ^= __y;
199 return __r;
200 }
201
202 [[__gnu__::__always_inline__]]
203 friend constexpr _Vp
204 operator<<(const _Vp& __x, const _Vp& __y) _GLIBCXX_SIMD_NOEXCEPT
205 requires requires (_Tp __a) { __a << __a; }
206 {
207 _Vp __r = __x;
208 __r <<= __y;
209 return __r;
210 }
211
212 [[__gnu__::__always_inline__]]
213 friend constexpr _Vp
214 operator<<(const _Vp& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
215 requires requires (_Tp __a, __simd_size_type __b) { __a << __b; }
216 {
217 _Vp __r = __x;
218 __r <<= __y;
219 return __r;
220 }
221
222 [[__gnu__::__always_inline__]]
223 friend constexpr _Vp
224 operator>>(const _Vp& __x, const _Vp& __y) _GLIBCXX_SIMD_NOEXCEPT
225 requires requires (_Tp __a) { __a >> __a; }
226 {
227 _Vp __r = __x;
228 __r >>= __y;
229 return __r;
230 }
231
232 [[__gnu__::__always_inline__]]
233 friend constexpr _Vp
234 operator>>(const _Vp& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
235 requires requires (_Tp __a, __simd_size_type __b) { __a >> __b; }
236 {
237 _Vp __r = __x;
238 __r >>= __y;
239 return __r;
240 }
241 };
242
243 struct _LoadCtorTag
244 {};
245
246 template <integral _Tp>
247 inline constexpr _Tp __max_shift
248 = (sizeof(_Tp) < sizeof(int) ? sizeof(int) : sizeof(_Tp)) * __CHAR_BIT__;
249
250 template <__vectorizable _Tp, __abi_tag _Ap>
251 requires (_Ap::_S_nreg == 1)
252 && (!__complex_like<_Tp>)
253 class basic_vec<_Tp, _Ap>
254 : public _VecBase<_Tp, _Ap>
255 {
256 template <typename, typename>
257 friend class basic_vec;
258
259 template <size_t, typename>
260 friend class basic_mask;
261
262 static constexpr int _S_size = _Ap::_S_size;
263
264 static constexpr int _S_full_size = __bit_ceil(unsigned(_S_size));
265
266 static constexpr bool _S_is_scalar = _S_size == 1;
267
268 static constexpr bool _S_use_bitmask = _Ap::_S_is_bitmask && !_S_is_scalar;
269
270 using _DataType = typename _Ap::template _DataType<_Tp>;
271
272 /** @internal
273 * @brief Underlying vector data storage.
274 *
275 * This member holds the vector object using a GNU vector type or a platform-specific vector
276 * type determined by the ABI tag. For size 1 vectors, this is a single value (_Tp).
277 */
278 _DataType _M_data;
279
280 static constexpr bool _S_is_partial = sizeof(_M_data) > sizeof(_Tp) * _S_size;
281
282 using __canon_value_type = __canonical_vec_type_t<_Tp>;
283
284 public:
285 using value_type = _Tp;
286
287 using mask_type = _VecBase<_Tp, _Ap>::mask_type;
288
289 // internal but public API ----------------------------------------------
290 [[__gnu__::__always_inline__]]
291 static constexpr basic_vec
292 _S_init(_DataType __x)
293 {
294 basic_vec __r;
295 __r._M_data = __x;
296 return __r;
297 }
298
299 [[__gnu__::__always_inline__]]
300 constexpr _DataType&
301 _M_get() noexcept
302 { return _M_data; }
303
304 [[__gnu__::__always_inline__]]
305 constexpr const _DataType&
306 _M_get() const noexcept
307 { return _M_data; }
308
309 [[__gnu__::__always_inline__]]
310 friend constexpr bool
311 __is_const_known(const basic_vec& __x)
312 { return __builtin_constant_p(__x._M_data); }
313
314 [[__gnu__::__always_inline__]]
315 constexpr auto
316 _M_concat_data([[maybe_unused]] bool __do_sanitize = false) const
317 {
318 if constexpr (_S_is_scalar)
319 return __vec_builtin_type<__canon_value_type, 1>{_M_data};
320 else
321 return _M_data;
322 }
323
324 template <int _Size = _S_size, int _Offset = 0, typename _A0, typename _Fp>
325 [[__gnu__::__always_inline__]]
326 static constexpr basic_vec
327 _S_static_permute(const basic_vec<value_type, _A0>& __x, _Fp&& __idxmap)
328 {
329 using _Xp = basic_vec<value_type, _A0>;
330 basic_vec __r;
331 if constexpr (_S_is_scalar)
332 {
333 constexpr __simd_size_type __j = [&] consteval {
334 if constexpr (__index_permutation_function_sized<_Fp>)
335 return __idxmap(_Offset, _Size);
336 else
337 return __idxmap(_Offset);
338 }();
339 if constexpr (__j == simd::zero_element || __j == simd::uninit_element)
340 return basic_vec();
341 else
342 static_assert(__j >= 0 && __j < _Xp::_S_size);
343 __r._M_data = __x[__j];
344 }
345 else
346 {
347 auto __idxmap2 = [=](auto __i) consteval {
348 if constexpr (int(__i + _Offset) >= _Size) // _S_full_size > _Size
349 return __simd_size_c<simd::uninit_element>;
350 else if constexpr (__index_permutation_function_sized<_Fp>)
351 return __simd_size_c<__idxmap(__i + _Offset, _Size)>;
352 else
353 return __simd_size_c<__idxmap(__i + _Offset)>;
354 };
355 constexpr auto __adj_idx = [](auto __i) {
356 constexpr int __j = __i;
357 if constexpr (__j == simd::zero_element)
358 return __simd_size_c<__bit_ceil(unsigned(_Xp::_S_size))>;
359 else if constexpr (__j == simd::uninit_element)
360 return __simd_size_c<-1>;
361 else
362 {
363 static_assert(__j >= 0 && __j < _Xp::_S_size);
364 return __simd_size_c<__j>;
365 }
366 };
367 constexpr auto [...__is0] = _IotaArray<_S_size>;
368 constexpr bool __needs_zero_element
369 = ((__idxmap2(__simd_size_c<__is0>).value == simd::zero_element) || ...);
370 constexpr auto [...__is_full] = _IotaArray<_S_full_size>;
371 if constexpr (_A0::_S_nreg == 2 && !__needs_zero_element)
372 {
373 __r._M_data = __builtin_shufflevector(
374 __x._M_data0._M_data, __x._M_data1._M_data,
375 __adj_idx(__idxmap2(__simd_size_c<__is_full>)).value...);
376 }
377 else
378 {
379 __r._M_data = __builtin_shufflevector(
380 __x._M_concat_data(), decltype(__x._M_concat_data())(),
381 __adj_idx(__idxmap2(__simd_size_c<__is_full>)).value...);
382 }
383 }
384 return __r;
385 }
386
387 template <typename _Vp>
388 [[__gnu__::__always_inline__]]
389 constexpr auto
390 _M_chunk() const noexcept
391 {
392 constexpr int __n = _S_size / _Vp::_S_size;
393 constexpr int __rem = _S_size % _Vp::_S_size;
394 constexpr auto [...__is] = _IotaArray<__n>;
395 if constexpr (__rem == 0)
396 return array<_Vp, __n> {__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>, *this)...};
397 else
398 {
399 using _Rest = resize_t<__rem, _Vp>;
400 return tuple(__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>, *this)...,
401 __extract_simd_at<_Rest>(cw<_Vp::_S_size * __n>, *this));
402 }
403 }
404
405 [[__gnu__::__always_inline__]]
406 static constexpr basic_vec
407 _S_concat(const basic_vec& __x0) noexcept
408 { return __x0; }
409
410 template <typename... _As>
411 requires (sizeof...(_As) > 1)
412 [[__gnu__::__always_inline__]]
413 static constexpr basic_vec
414 _S_concat(const basic_vec<value_type, _As>&... __xs) noexcept
415 {
416 static_assert(_S_size == (_As::_S_size + ...));
417 return __extract_simd_at<basic_vec>(cw<0>, __xs...);
418 }
419
420 /** @internal
421 * Shifts elements to the front by @p _Shift positions (or to the back for negative @p
422 * _Shift).
423 *
424 * This function moves elements towards lower indices (front of the vector).
425 * Elements that would shift beyond the vector bounds are replaced with zero. Negative shift
426 * values shift in the opposite direction.
427 *
428 * @warning The naming can be confusing due to little-endian byte order:
429 * - Despite the name "shifted_to_front", the underlying hardware instruction
430 * shifts bits to the right (psrl...)
431 * - The function name refers to element indices, not bit positions
432 *
433 * @tparam _Shift Number of positions to shift elements towards the front.
434 * Must be -size() < _Shift < size().
435 *
436 * @return A new vector with elements shifted to front or back.
437 *
438 * Example:
439 * @code
440 * __iota<vec<int, 4>>._M_elements_shifted_to_front<2>(); // {2, 3, 0, 0}
441 * __iota<vec<int, 4>>._M_elements_shifted_to_front<-2>(); // {0, 0, 0, 1}
442 * @endcode
443 */
444 template <int _Shift, _ArchTraits _Traits = {}>
445 [[__gnu__::__always_inline__]]
446 constexpr basic_vec
447 _M_elements_shifted_to_front() const
448 {
449 static_assert(_Shift < _S_size && -_Shift < _S_size);
450 if constexpr (_Shift == 0)
451 return *this;
452#ifdef __SSE2__
453 else if (!__is_const_known(*this))
454 {
455 if constexpr (sizeof(_M_data) == 16 && _Shift > 0)
456 return reinterpret_cast<_DataType>(
457 __builtin_ia32_psrldqi128(__vec_bit_cast<long long>(_M_data),
458 _Shift * sizeof(value_type) * 8));
459 else if constexpr (sizeof(_M_data) == 16 && _Shift < 0)
460 return reinterpret_cast<_DataType>(
461 __builtin_ia32_pslldqi128(__vec_bit_cast<long long>(_M_data),
462 -_Shift * sizeof(value_type) * 8));
463 else if constexpr (sizeof(_M_data) < 16)
464 {
465 auto __x = reinterpret_cast<__vec_builtin_type_bytes<long long, 16>>(
466 __vec_zero_pad_to_16(_M_data));
467 if constexpr (_Shift > 0)
468 __x = __builtin_ia32_psrldqi128(__x, _Shift * sizeof(value_type) * 8);
469 else
470 __x = __builtin_ia32_pslldqi128(__x, -_Shift * sizeof(value_type) * 8);
471 return _VecOps<_DataType>::_S_extract(__vec_bit_cast<__canon_value_type>(__x));
472 }
473 }
474#endif
475 return _S_static_permute(*this, [](int __i) consteval {
476 int __off = __i + _Shift;
477 return __off >= _S_size || __off < 0 ? zero_element : __off;
478 });
479 }
480
481 /** @internal
482 * @brief Set padding elements to @p __id; add more padding elements if necessary.
483 *
484 * @note This function can rearrange the element order since the result is only used for
485 * reductions.
486 */
487 template <typename _Vp, __canon_value_type __id>
488 [[__gnu__::__always_inline__]]
489 constexpr _Vp
490 _M_pad_to_T_with_value() const noexcept
491 {
492 static_assert(!_Vp::_S_is_partial);
493 static_assert(_Ap::_S_nreg == 1);
494 if constexpr (sizeof(_Vp) == 32)
495 { // when we need to reduce from a 512-bit register
496 static_assert(sizeof(_M_data) == 32);
497 constexpr auto __k = _Vp::mask_type::_S_partial_mask_of_n(_S_size);
498 return __select_impl(__k, _Vp::_S_init(_M_data), __id);
499 }
500 else
501 {
502 static_assert(sizeof(_Vp) <= 16); // => max. 7 Bytes need to be zeroed
503 static_assert(sizeof(_M_data) <= sizeof(_Vp));
504 _Vp __v1 = __vec_zero_pad_to<sizeof(_Vp)>(_M_data);
505 if constexpr (__id == 0 && _S_is_partial)
506 // cheapest solution: shift values to the back while shifting in zeros
507 // This is valid because we shift out padding elements and use all elements in a
508 // subsequent reduction.
509 __v1 = __v1.template _M_elements_shifted_to_front<-(_Vp::_S_size - _S_size)>();
510 else if constexpr (_Vp::_S_size - _S_size == 1)
511 // if a single element needs to be changed, use an insert instruction
512 __vec_set(__v1._M_data, _Vp::_S_size - 1, __id);
513 else if constexpr (__has_single_bit(unsigned(_Vp::_S_size - _S_size)))
514 { // if 2^n elements need to be changed, use a single insert instruction
515 constexpr int __n = _Vp::_S_size - _S_size;
516 using _Ip = __integer_from<__n * sizeof(__canon_value_type)>;
517 constexpr auto [...__is] = _IotaArray<__n>;
518 constexpr __canon_value_type __idn[__n] = {((void)__is, __id)...};
519 auto __vn = __vec_bit_cast<_Ip>(__v1._M_data);
520 __vec_set(__vn, _Vp::_S_size / __n - 1, __builtin_bit_cast(_Ip, __idn));
521 __v1._M_data = reinterpret_cast<typename _Vp::_DataType>(__vn);
522 }
523 else if constexpr (__id != 0 && !_S_is_partial)
524 { // if __vec_zero_pad_to added zeros in all the places where we need __id, a
525 // bitwise or is sufficient (needs a vector constant for the __id vector, which
526 // isn't optimal)
527 constexpr _Vp __idn([](int __i) {
528 return __i >= _S_size ? __id : __canon_value_type();
529 });
530 __v1._M_data = __vec_or(__v1._M_data, __idn._M_data);
531 }
532 else if constexpr (__id != 0 || _S_is_partial)
533 { // fallback
534 constexpr auto __k = _Vp::mask_type::_S_partial_mask_of_n(_S_size);
535 __v1 = __select_impl(__k, __v1, __id);
536 }
537 return __v1;
538 }
539 }
540
541 [[__gnu__::__always_inline__]]
542 constexpr auto
543 _M_reduce_to_half(auto __binary_op) const
544 {
545 static_assert(__has_single_bit(unsigned(_S_size)));
546 auto [__a, __b] = chunk<_S_size / 2>(*this);
547 return __binary_op(__a, __b);
548 }
549
550 template <typename _Rest, typename _BinaryOp>
551 [[__gnu__::__always_inline__]]
552 constexpr value_type
553 _M_reduce_tail(const _Rest& __rest, _BinaryOp __binary_op) const
554 {
555 if constexpr (_S_is_scalar)
556 return __binary_op(*this, __rest)._M_data;
557 else if constexpr (_Rest::_S_size == _S_size)
558 return __binary_op(*this, __rest)._M_reduce(__binary_op);
559 else if constexpr (_Rest::_S_size > _S_size)
560 {
561 auto [__a, __b] = __rest.template _M_chunk<basic_vec>();
562 return __binary_op(*this, __a)._M_reduce_tail(__b, __binary_op);
563 }
564 else if constexpr (_Rest::_S_size == 1)
565 return __binary_op(_Rest(_M_reduce(__binary_op)), __rest)[0];
566 else if constexpr (sizeof(_M_data) <= 16
567 && requires { __default_identity_element<__canon_value_type, _BinaryOp>(); })
568 { // extend __rest with identity element for more parallelism
569 constexpr __canon_value_type __id
570 = __default_identity_element<__canon_value_type, _BinaryOp>();
571 return __binary_op(_M_data, __rest.template _M_pad_to_T_with_value<basic_vec, __id>())
572 ._M_reduce(__binary_op);
573 }
574 else
575 return _M_reduce_to_half(__binary_op)._M_reduce_tail(__rest, __binary_op);
576 }
577
578 /** @internal
579 * @brief Reduction over @p __binary_op of all (non-padding) elements.
580 *
581 * @note The implementation assumes it is most efficient to first reduce to one 128-bit SIMD
582 * register and then shuffle elements while sticking to 128-bit registers.
583 */
584 template <typename _BinaryOp, _ArchTraits _Traits = {}>
585 [[__gnu__::__always_inline__]]
586 constexpr value_type
587 _M_reduce(_BinaryOp __binary_op) const
588 {
589 constexpr bool __have_id_elem
590 = requires { __default_identity_element<__canon_value_type, _BinaryOp>(); };
591 if constexpr (_S_size == 1)
592 return operator[](0);
593 else if constexpr (_Traits.template _M_eval_as_f32<value_type>()
594 && (is_same_v<_BinaryOp, plus<>>
595 || is_same_v<_BinaryOp, multiplies<>>))
596 return value_type(rebind_t<float, basic_vec>(*this)._M_reduce(__binary_op));
597#ifdef __SSE2__
598 else if constexpr (is_integral_v<value_type> && sizeof(value_type) == 1
599 && is_same_v<decltype(__binary_op), multiplies<>>)
600 {
601 // convert to unsigned short because of missing 8-bit mul instruction
602 // we don't need to preserve the order of elements
603 //
604 // The left columns under Latency and Throughput show bit-cast to ushort with shift by
605 // 8. The right column uses the alternative in the else branch.
606 // Benchmark on Intel Ultra 7 165U (AVX2)
607 // TYPE Latency Throughput
608 // [cycles/call] [cycles/call]
609 //schar, 2 9.11 7.73 3.17 3.21
610 //schar, 4 31.6 34.9 5.11 6.97
611 //schar, 8 35.7 41.5 7.77 7.17
612 //schar, 16 36.7 44.1 6.66 8.96
613 //schar, 32 42.2 61.1 8.82 10.1
614 if constexpr (!_S_is_partial)
615 { // If all elements participate in the reduction we can take this shortcut
616 using _V16 = resize_t<_S_size / 2, rebind_t<unsigned short, basic_vec>>;
617 auto __a = __builtin_bit_cast(_V16, *this);
618 return __binary_op(__a, __a >> 8)._M_reduce(__binary_op);
619 }
620 else
621 {
622 using _V16 = rebind_t<unsigned short, basic_vec>;
623 return _V16(*this)._M_reduce(__binary_op);
624 }
625 }
626#endif
627 else if constexpr (__has_single_bit(unsigned(_S_size)))
628 {
629 if constexpr (sizeof(_M_data) > 16)
630 return _M_reduce_to_half(__binary_op)._M_reduce(__binary_op);
631 else if constexpr (_S_size == 2)
632 return _M_reduce_to_half(__binary_op)[0];
633 else
634 {
635 static_assert(_S_size <= 16);
636 auto __x = *this;
637#ifdef __SSE2__
638 if constexpr (sizeof(_M_data) <= 16 && is_integral_v<value_type>)
639 {
640 if constexpr (_S_size > 8)
641 __x = __binary_op(__x, __x.template _M_elements_shifted_to_front<8>());
642 if constexpr (_S_size > 4)
643 __x = __binary_op(__x, __x.template _M_elements_shifted_to_front<4>());
644 if constexpr (_S_size > 2)
645 __x = __binary_op(__x, __x.template _M_elements_shifted_to_front<2>());
646 // We could also call __binary_op with vec<T, 1> arguments. However,
647 // micro-benchmarking on Intel Ultra 7 165U showed this to be more efficient:
648 return __binary_op(__x, __x.template _M_elements_shifted_to_front<1>())[0];
649 }
650#endif
651 if constexpr (_S_size > 8)
652 __x = __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<8>()));
653 if constexpr (_S_size > 4)
654 __x = __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<4>()));
655#ifdef __SSE2__
656 // avoid pshufb by "promoting" to int
657 if constexpr (is_integral_v<value_type> && sizeof(value_type) <= 1)
658 return value_type(resize_t<4, rebind_t<int, basic_vec>>(chunk<4>(__x)[0])
659 ._M_reduce(__binary_op));
660#endif
661 if constexpr (_S_size > 2)
662 __x = __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<2>()));
663 if constexpr (is_integral_v<value_type> && sizeof(value_type) == 2)
664 return __binary_op(__x, _S_static_permute(__x, _SwapNeighbors<1>()))[0];
665 else
666 return __binary_op(vec<value_type, 1>(__x[0]), vec<value_type, 1>(__x[1]))[0];
667 }
668 }
669 else if constexpr (sizeof(_M_data) == 32)
670 {
671 const auto [__lo, __hi] = chunk<__bit_floor(unsigned(_S_size))>(*this);
672 return __lo._M_reduce_tail(__hi, __binary_op);
673 }
674 else if constexpr (sizeof(_M_data) == 64)
675 {
676 // e.g. _S_size = 16 + 16 + 15 (vec<char, 47>)
677 // -> 8 + 8 + 7 -> 4 + 4 + 3 -> 2 + 2 + 1 -> 1
678 auto __chunked = chunk<__bit_floor(unsigned(_S_size)) / 2>(*this);
679 using _Cp = decltype(__chunked);
680 if constexpr (tuple_size_v<_Cp> == 4)
681 {
682 const auto& [__a, __b, __c, __rest] = __chunked;
683 constexpr bool __amd_cpu = _Traits._M_have_sse4a();
684 if constexpr (__have_id_elem && __rest._S_size > 1 && __amd_cpu)
685 { // do one 256-bit op -> one 128-bit op
686 // 4 cycles on Zen4/5 until _M_reduce (short, 26, plus<>)
687 // 9 cycles on Skylake-AVX512 until _M_reduce
688 // 9 cycles on Zen4/5 until _M_reduce (short, 27, multiplies<>)
689 // 17 cycles on Skylake-AVX512 until _M_reduce (short, 27, multiplies<>)
690 const auto& [__a, __rest] = chunk<__bit_floor(unsigned(_S_size))>(*this);
691 using _Vp = remove_cvref_t<decltype(__a)>;
692 constexpr __canon_value_type __id
693 = __default_identity_element<__canon_value_type, _BinaryOp>();
694 const _Vp __b = __rest.template _M_pad_to_T_with_value<_Vp, __id>();
695 return __binary_op(__a, __b)._M_reduce(__binary_op);
696 }
697 else if constexpr (__have_id_elem && __rest._S_size > 1)
698 { // do two 128-bit ops -> one 128-bit op
699 // 5 cycles on Zen4/5 until _M_reduce (short, 26, plus<>)
700 // 7 cycles on Skylake-AVX512 until _M_reduce (short, 26, plus<>)
701 // 9 cycles on Zen4/5 until _M_reduce (short, 27, multiplies<>)
702 // 16 cycles on Skylake-AVX512 until _M_reduce (short, 27, multiplies<>)
703 using _Vp = remove_cvref_t<decltype(__a)>;
704 constexpr __canon_value_type __id
705 = __default_identity_element<__canon_value_type, _BinaryOp>();
706 const _Vp __d = __rest.template _M_pad_to_T_with_value<_Vp, __id>();
707 return __binary_op(__binary_op(__a, __b), __binary_op(__c, __d))
708 ._M_reduce(__binary_op);
709 }
710 else
711 return __binary_op(__binary_op(__a, __b), __c)
712 ._M_reduce_tail(__rest, __binary_op);
713 }
714 else if constexpr (tuple_size_v<_Cp> == 3)
715 {
716 const auto& [__a, __b, __rest] = __chunked;
717 return __binary_op(__a, __b)._M_reduce_tail(__rest, __binary_op);
718 }
719 else
720 static_assert(false);
721 }
722 else if constexpr (__have_id_elem)
723 {
724 constexpr __canon_value_type __id
725 = __default_identity_element<__canon_value_type, _BinaryOp>();
726 using _Vp = resize_t<__bit_ceil(unsigned(_S_size)), basic_vec>;
727 return _M_pad_to_T_with_value<_Vp, __id>()._M_reduce(__binary_op);
728 }
729 else
730 {
731 const auto& [__a, __rest] = chunk<__bit_floor(unsigned(_S_size))>(*this);
732 return __a._M_reduce_tail(__rest, __binary_op);
733 }
734 }
735
736 // [simd.math] ----------------------------------------------------------
737 //
738 // ISO/IEC 60559 on the classification operations (5.7.2 General Operations):
739 // "They are never exceptional, even for signaling NaNs."
740 //
741 template <_OptTraits _Traits = {}>
742 [[__gnu__::__always_inline__]]
743 constexpr mask_type
744 _M_isnan() const requires is_floating_point_v<value_type>
745 {
746 if constexpr (_Traits._M_finite_math_only())
747 return mask_type(false);
748 else if constexpr (_S_is_scalar)
749 return mask_type(std::isnan(_M_data));
750 else if constexpr (_S_use_bitmask)
751 return _M_isunordered(*this);
752 else if constexpr (!_Traits._M_support_snan())
753 return !(*this == *this);
754 else if (__is_const_known(_M_data))
755 return mask_type([&](int __i) { return std::isnan(_M_data[__i]); });
756 else
757 {
758 // 60559: NaN is represented as Inf + non-zero mantissa bits
759 using _Ip = __integer_from<sizeof(value_type)>;
760 return __builtin_bit_cast(_Ip, numeric_limits<value_type>::infinity())
761 < __builtin_bit_cast(rebind_t<_Ip, basic_vec>, _M_fabs());
762 }
763 }
764
765 template <_TargetTraits _Traits = {}>
766 [[__gnu__::__always_inline__]]
767 constexpr mask_type
768 _M_isinf() const requires is_floating_point_v<value_type>
769 {
770 if constexpr (_Traits._M_finite_math_only())
771 return mask_type(false);
772 else if constexpr (_S_is_scalar)
773 return mask_type(std::isinf(_M_data));
774 else if (__is_const_known(_M_data))
775 return mask_type([&](int __i) { return std::isinf(_M_data[__i]); });
776#ifdef _GLIBCXX_X86
777 else if constexpr (_S_use_bitmask)
778 return mask_type::_S_init(__x86_bitmask_isinf(_M_data));
779 else if constexpr (_Traits._M_have_avx512dq())
780 return __x86_bit_to_vecmask<typename mask_type::_DataType>(
781 __x86_bitmask_isinf(_M_data));
782#endif
783 else
784 {
785 using _Ip = __integer_from<sizeof(value_type)>;
786 return __vec_bit_cast<_Ip>(_M_fabs()._M_data)
787 == __builtin_bit_cast(_Ip, numeric_limits<value_type>::infinity());
788 }
789 }
790
791 [[__gnu__::__always_inline__]]
792 constexpr basic_vec
793 _M_abs() const requires signed_integral<value_type>
794 { return _M_data < 0 ? -_M_data : _M_data; }
795
796 [[__gnu__::__always_inline__]]
797 constexpr basic_vec
798 _M_fabs() const requires floating_point<value_type>
799 {
800 if constexpr (_S_is_scalar)
801 return std::fabs(_M_data);
802 else
803 return __vec_and(__vec_not(_S_signmask<_DataType>), _M_data);
804 }
805
806 template <_TargetTraits _Traits = {}>
807 [[__gnu__::__always_inline__]]
808 constexpr mask_type
809 _M_isunordered(basic_vec __y) const requires is_floating_point_v<value_type>
810 {
811 if constexpr (_Traits._M_finite_math_only())
812 return mask_type(false);
813 else if constexpr (_S_is_scalar)
814 return mask_type(std::isunordered(_M_data, __y._M_data));
815#ifdef _GLIBCXX_X86
816 else if constexpr (_S_use_bitmask)
817 return _M_bitmask_cmp<_X86Cmp::_Unord>(__y._M_data);
818#endif
819 else
820 return mask_type([&](int __i) {
821 return std::isunordered(_M_data[__i], __y._M_data[__i]);
822 });
823 }
824
825 /** @internal
826 * Implementation of @ref partial_load.
827 *
828 * @param __mem A pointer to an array of @p __n values. Can be complex or real.
829 * @param __n Read no more than @p __n values from memory. However, depending on @p __mem
830 * alignment, out of bounds reads are benign.
831 */
832 template <typename _Up, _ArchTraits _Traits = {}>
833 static inline basic_vec
834 _S_partial_load(const _Up* __mem, size_t __n)
835 {
836 if constexpr (_S_is_scalar)
837 return __n == 0 ? basic_vec() : basic_vec(static_cast<value_type>(*__mem));
838 else if (__is_const_known_equal_to(__n >= size_t(_S_size), true))
839 return basic_vec(_LoadCtorTag(), __mem);
840 else if constexpr (!__converts_trivially<_Up, value_type>)
841 return static_cast<basic_vec>(rebind_t<_Up, basic_vec>::_S_partial_load(__mem, __n));
842 else
843 {
844#if _GLIBCXX_X86
845 if constexpr (_Traits._M_have_avx512f()
846 || (_Traits._M_have_avx() && sizeof(_Up) >= 4))
847 {
848 const auto __k = __n < _S_size ? mask_type::_S_partial_mask_of_n(int(__n))
849 : mask_type(true);
850 return _S_masked_load(__mem, mask_type::_S_partial_mask_of_n(int(__n)));
851 }
852#endif
853 if (__n >= size_t(_S_size)) [[unlikely]]
854 return basic_vec(_LoadCtorTag(), __mem);
855#if _GLIBCXX_X86 // TODO: where else is this "safe"?
856 // allow out-of-bounds read when it cannot lead to a #GP
857 else if (__is_const_known_equal_to(
858 is_sufficiently_aligned<sizeof(_Up) * _S_full_size>(__mem), true))
859 return __select_impl(mask_type::_S_partial_mask_of_n(int(__n)),
860 basic_vec(_LoadCtorTag(), __mem), basic_vec());
861#endif
862 else if constexpr (_S_size > 4)
863 {
864 alignas(_DataType) byte __dst[sizeof(_DataType)] = {};
865 const byte* __src = reinterpret_cast<const byte*>(__mem);
866 __memcpy_chunks<sizeof(_Up), sizeof(_DataType)>(__dst, __src, __n);
867 return __builtin_bit_cast(_DataType, __dst);
868 }
869 else if (__n == 0) [[unlikely]]
870 return basic_vec();
871 else if constexpr (_S_size == 2)
872 return _DataType {static_cast<value_type>(__mem[0]), 0};
873 else
874 {
875 constexpr auto [...__is] = _IotaArray<_S_size - 2>;
876 return _DataType{
877 static_cast<value_type>(__mem[0]),
878 static_cast<value_type>(__is + 1 < __n ? __mem[__is + 1] : 0)...
879 };
880 }
881 }
882 }
883
884 /** @internal
885 * Loads elements from @p __mem according to mask @p __k.
886 *
887 * @param __mem Pointer (in)to array.
888 * @param __k Mask controlling which elements to load. For each bit i in the mask:
889 * - If bit i is 1: copy __mem[i] into result[i]
890 * - If bit i is 0: result[i] is default initialized
891 *
892 * @note This function assumes it's called after determining that no other method
893 * (like full load) is more appropriate. Calling with all mask bits set to 1
894 * is suboptimal for performance but still correct.
895 */
896 template <typename _Up, _ArchTraits _Traits = {}>
897 static inline basic_vec
898 _S_masked_load(const _Up* __mem, mask_type __k)
899 {
900 if constexpr (_S_size == 1)
901 return __k[0] ? static_cast<value_type>(__mem[0]) : value_type();
902#if _GLIBCXX_X86
903 else if constexpr (_Traits._M_have_avx512f())
904 return __x86_masked_load<_DataType>(__mem, __k._M_data);
905 else if constexpr (_Traits._M_have_avx() && (sizeof(_Up) == 4 || sizeof(_Up) == 8))
906 {
907 if constexpr (__converts_trivially<_Up, value_type>)
908 return __x86_masked_load<_DataType>(__mem, __k._M_data);
909 else
910 {
911 using _UV = rebind_t<_Up, basic_vec>;
912 return basic_vec(_UV::_S_masked_load(__mem, typename _UV::mask_type(__k)));
913 }
914 }
915#endif
916 else if (__k._M_none_of()) [[unlikely]]
917 return basic_vec();
918 else if constexpr (_S_is_scalar)
919 return basic_vec(static_cast<value_type>(*__mem));
920 else
921 {
922 // Use at least 4-byte __bits in __bit_foreach for better code-gen
923 _Bitmask<_S_size < 32 ? 32 : _S_size> __bits = __k._M_to_uint();
924 [[assume(__bits != 0)]]; // because of '__k._M_none_of()' branch above
925 if constexpr (__converts_trivially<_Up, value_type>)
926 {
927 _DataType __r = {};
928 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
929 __r[__i] = __mem[__i];
930 });
931 return __r;
932 }
933 else
934 {
935 using _UV = rebind_t<_Up, basic_vec>;
936 alignas(_UV) _Up __tmp[sizeof(_UV) / sizeof(_Up)] = {};
937 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
938 __tmp[__i] = __mem[__i];
939 });
940 return basic_vec(__builtin_bit_cast(_UV, __tmp));
941 }
942 }
943 }
944
945 template <typename _Up>
946 [[__gnu__::__always_inline__]]
947 inline void
948 _M_store(_Up* __mem) const
949 {
950 if constexpr (__converts_trivially<value_type, _Up>)
951 __builtin_memcpy(__mem, &_M_data, sizeof(_Up) * _S_size);
952 else
953 rebind_t<_Up, basic_vec>(*this)._M_store(__mem);
954 }
955
956 /** @internal
957 * Implementation of @ref partial_store.
958 *
959 * @note This is a static function to allow passing @p __v via register in case the function
960 * is not inlined.
961 *
962 * @note The function is not marked @c __always_inline__ since code-gen can become fairly
963 * long.
964 */
965 template <typename _Up, _ArchTraits _Traits = {}>
966 static inline void
967 _S_partial_store(const basic_vec __v, _Up* __mem, size_t __n)
968 {
969 if (__is_const_known_equal_to(__n >= _S_size, true))
970 __v._M_store(__mem);
971#if _GLIBCXX_X86
972 else if constexpr (_Traits._M_have_avx512f() && !_S_is_scalar)
973 {
974 const auto __k = __n < _S_size ? mask_type::_S_partial_mask_of_n(int(__n))
975 : mask_type(true);
976 return _S_masked_store(__v, __mem, __k);
977 }
978#endif
979 else if (__n >= _S_size) [[unlikely]]
980 __v._M_store(__mem);
981 else if (__n == 0) [[unlikely]]
982 return;
983 else if constexpr (__converts_trivially<value_type, _Up>)
984 {
985 byte* __dst = reinterpret_cast<byte*>(__mem);
986 const byte* __src = reinterpret_cast<const byte*>(&__v._M_data);
987 __memcpy_chunks<sizeof(_Up), sizeof(_M_data)>(__dst, __src, __n);
988 }
989 else
990 {
991 using _UV = rebind_t<_Up, basic_vec>;
992 _UV::_S_partial_store(_UV(__v), __mem, __n);
993 }
994 }
995
996 /** @internal
997 * Stores elements of @p __v to @p __mem according to mask @p __k.
998 *
999 * @param __v Values to store to @p __mem.
1000 * @param __mem Pointer (in)to array.
1001 * @param __k Mask controlling which elements to store. For each bit i in the mask:
1002 * - If bit i is 1: store __v[i] to __mem[i]
1003 * - If bit i is 0: __mem[i] is left unchanged
1004 *
1005 * @note This function assumes it's called after determining that no other method
1006 * (like full store) is more appropriate. Calling with all mask bits set to 1
1007 * is suboptimal for performance but still correct.
1008 */
1009 template <typename _Up, _ArchTraits _Traits = {}>
1010 //[[__gnu__::__always_inline__]]
1011 static inline void
1012 _S_masked_store(const basic_vec __v, _Up* __mem, const mask_type __k)
1013 {
1014#if _GLIBCXX_X86
1015 if constexpr (_Traits._M_have_avx512f())
1016 {
1017 __x86_masked_store(__v._M_data, __mem, __k._M_data);
1018 return;
1019 }
1020 else if constexpr (_Traits._M_have_avx() && (sizeof(_Up) == 4 || sizeof(_Up) == 8))
1021 {
1022 if constexpr (__converts_trivially<value_type, _Up>)
1023 __x86_masked_store(__v._M_data, __mem, __k._M_data);
1024 else
1025 {
1026 using _UV = rebind_t<_Up, basic_vec>;
1027 _UV::_S_masked_store(_UV(__v), __mem, typename _UV::mask_type(__k));
1028 }
1029 return;
1030 }
1031#endif
1032 if (__k._M_none_of()) [[unlikely]]
1033 return;
1034 else if constexpr (_S_is_scalar)
1035 __mem[0] = __v._M_data;
1036 else
1037 {
1038 // Use at least 4-byte __bits in __bit_foreach for better code-gen
1039 _Bitmask<_S_size < 32 ? 32 : _S_size> __bits = __k._M_to_uint();
1040 [[assume(__bits != 0)]]; // because of '__k._M_none_of()' branch above
1041 if constexpr (__converts_trivially<value_type, _Up>)
1042 {
1043 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
1044 __mem[__i] = __v[__i];
1045 });
1046 }
1047 else
1048 {
1049 const rebind_t<_Up, basic_vec> __cvted(__v);
1050 __bit_foreach(__bits, [&] [[__gnu__::__always_inline__]] (int __i) {
1051 __mem[__i] = __cvted[__i];
1052 });
1053 }
1054 }
1055 }
1056
1057 // [simd.overview] default constructor ----------------------------------
1058 basic_vec() = default;
1059
1060 // [simd.overview] p2 impl-def conversions ------------------------------
1061 using _NativeVecType = decltype([] {
1062 if constexpr (_S_is_scalar)
1063 return __vec_builtin_type<__canon_value_type, 1>();
1064 else
1065 return _DataType();
1066 }());
1067 /**
1068 * @brief Converting constructor from GCC vector builtins.
1069 *
1070 * This constructor enables direct construction from GCC vector builtins
1071 * (`[[gnu::vector_size(N)]]`).
1072 *
1073 * @param __x GCC vector builtin to convert from.
1074 *
1075 * @note This constructor is not available when size() equals 1.
1076 *
1077 * @see operator _NativeVecType() for the reverse conversion.
1078 */
1079 constexpr
1080 basic_vec(_NativeVecType __x)
1081 : _M_data([&] [[__gnu__::__always_inline__]] {
1082 if constexpr (_S_is_scalar)
1083 return __x[0];
1084 else
1085 return __x;
1086 }())
1087 {}
1088
1089 /**
1090 * @brief Conversion operator to GCC vector builtins.
1091 *
1092 * This operator enables implicit conversion from basic_vec to GCC vector builtins.
1093 *
1094 * @note This operator is not available when size() equals 1.
1095 *
1096 * @see basic_vec(_NativeVecType) for the reverse conversion.
1097 */
1098 constexpr
1099 operator _NativeVecType() const
1100 {
1101 if constexpr (_S_is_scalar)
1102 return _NativeVecType{_M_data};
1103 else
1104 return _M_data;
1105 }
1106
1107#if _GLIBCXX_X86
1108 /**
1109 * @brief Converting constructor from Intel Intrinsics (__m128, __m128i, ...).
1110 */
1111 template <__vec_builtin _IV>
1112 requires same_as<__x86_intel_intrin_value_type<value_type>, __vec_value_type<_IV>>
1113 && (sizeof(_IV) == sizeof(_DataType) && sizeof(_IV) >= 16
1114 && !is_same_v<_IV, _DataType>)
1115 constexpr
1116 basic_vec(_IV __x)
1117 : _M_data(reinterpret_cast<_DataType>(__x))
1118 {}
1119
1120 /**
1121 * @brief Conversion operator to Intel Intrinsics (__m128, __m128i, ...).
1122 */
1123 template <__vec_builtin _IV>
1124 requires same_as<__x86_intel_intrin_value_type<value_type>, __vec_value_type<_IV>>
1125 && (sizeof(_IV) == sizeof(_DataType) && sizeof(_IV) >= 16
1126 && !is_same_v<_IV, _DataType>)
1127 constexpr
1128 operator _IV() const
1129 { return reinterpret_cast<_IV>(_M_data); }
1130#endif
1131
1132 // [simd.ctor] broadcast constructor ------------------------------------
1133 /**
1134 * @brief Broadcast constructor from scalar value.
1135 *
1136 * Constructs a vector where all elements are initialized to the same scalar value.
1137 * The scalar value is converted to the vector's element type.
1138 *
1139 * @param __x Scalar value to broadcast to all vector elements.
1140 * @tparam _Up Type of scalar value (must be explicitly convertible to value_type).
1141 *
1142 * @note The constructor is implicit if the conversion (if any) is value-preserving.
1143 */
1144 template <__broadcast_constructible<value_type> _Up>
1145 [[__gnu__::__always_inline__]]
1146 constexpr
1147 basic_vec(_Up&& __x) noexcept
1148 : _M_data(_DataType() == _DataType() ? static_cast<value_type>(__x) : value_type())
1149 {}
1150
1151 // [simd.ctor] conversion constructor -----------------------------------
1152 template <typename _Up, typename _UAbi, _TargetTraits _Traits = {}>
1153 requires (_S_size == _UAbi::_S_size)
1154 && __explicitly_convertible_to<_Up, value_type>
1155 [[__gnu__::__always_inline__]]
1156 constexpr
1157 explicit(!__value_preserving_convertible_to<_Up, value_type>
1158 || __higher_rank_than<_Up, value_type>)
1159 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
1160 : _M_data([&] [[__gnu__::__always_inline__]] {
1161 if constexpr (_S_is_scalar)
1162 return static_cast<value_type>(__x[0]);
1163 else if constexpr (_UAbi::_S_nreg >= 2)
1164 // __builtin_convertvector (__vec_cast) is inefficient for over-sized inputs.
1165 // Also e.g. vec<float, 12> -> vec<char, 12> (with SSE2) would otherwise emit 4
1166 // vcvttps2dq instructions, where only 3 are needed
1167 return _S_concat(resize_t<__x._N0, basic_vec>(__x._M_data0),
1168 resize_t<__x._N1, basic_vec>(__x._M_data1))._M_data;
1169 else
1170 return __vec_cast<_DataType>(__x._M_concat_data());
1171 }())
1172 {}
1173
1174 using _VecBase<_Tp, _Ap>::_VecBase;
1175
1176 // [simd.ctor] generator constructor ------------------------------------
1177 template <__simd_generator_invokable<value_type, _S_size> _Fp>
1178 [[__gnu__::__always_inline__]]
1179 constexpr explicit
1180 basic_vec(_Fp&& __gen)
1181 : _M_data([&] [[__gnu__::__always_inline__]] {
1182 constexpr auto [...__is] = _IotaArray<_S_size>;
1183 return _DataType{static_cast<value_type>(__gen(__simd_size_c<__is>))...};
1184 }())
1185 {}
1186
1187 // [simd.ctor] load constructor -----------------------------------------
1188 template <typename _Up>
1189 [[__gnu__::__always_inline__]]
1190 constexpr
1191 basic_vec(_LoadCtorTag, const _Up* __ptr)
1192 : _M_data()
1193 {
1194 if constexpr (_S_is_scalar)
1195 _M_data = static_cast<value_type>(__ptr[0]);
1196 else if consteval
1197 {
1198 constexpr auto [...__is] = _IotaArray<_S_size>;
1199 _M_data = _DataType{static_cast<value_type>(__ptr[__is])...};
1200 }
1201 else
1202 {
1203 if constexpr (__converts_trivially<_Up, value_type>)
1204 // This assumes std::floatN_t to be bitwise equal to float/double
1205 __builtin_memcpy(&_M_data, __ptr, sizeof(value_type) * _S_size);
1206 else
1207 {
1208 __vec_builtin_type<_Up, _S_full_size> __tmp = {};
1209 __builtin_memcpy(&__tmp, __ptr, sizeof(_Up) * _S_size);
1210 _M_data = __vec_cast<_DataType>(__tmp);
1211 }
1212 }
1213 }
1214
1215 template <ranges::contiguous_range _Rg, typename... _Flags>
1216 requires ranges::__static_sized_range<_Rg>
1217 && __vectorizable<ranges::range_value_t<_Rg>>
1218 && __explicitly_convertible_to<ranges::range_value_t<_Rg>, value_type>
1219 [[__gnu__::__always_inline__]]
1220 constexpr
1221 basic_vec(_Rg&& __range, flags<_Flags...> __flags = {})
1222 requires (ranges::size(__range) == _S_size)
1223 : basic_vec(_LoadCtorTag(), __flags.template _S_adjust_pointer<basic_vec>(
1224 ranges::data(__range)))
1225 {
1226 static_assert(__loadstore_convertible_to<ranges::range_value_t<_Rg>, value_type,
1227 _Flags...>);
1228 }
1229
1230 // [simd.subscr] --------------------------------------------------------
1231 /**
1232 * @brief Return the value of the element at index @p __i.
1233 *
1234 * @pre __i >= 0 && __i < size().
1235 */
1236 [[__gnu__::__always_inline__]]
1237 constexpr value_type
1238 operator[](__simd_size_type __i) const
1239 {
1240 __glibcxx_simd_precondition(__i >= 0 && __i < _S_size, "subscript is out of bounds");
1241 if constexpr (_S_is_scalar)
1242 return _M_data;
1243 else
1244 return _M_data[__i];
1245 }
1246
1247 // [simd.unary] unary operators -----------------------------------------
1248 // increment and decrement are implemented in terms of operator+=/-= which avoids UB on
1249 // padding elements while not breaking UBsan
1250 [[__gnu__::__always_inline__]]
1251 constexpr basic_vec&
1252 operator++() noexcept requires requires(value_type __a) { ++__a; }
1253 { return *this += value_type(1); }
1254
1255 [[__gnu__::__always_inline__]]
1256 constexpr basic_vec
1257 operator++(int) noexcept requires requires(value_type __a) { __a++; }
1258 {
1259 basic_vec __r = *this;
1260 *this += value_type(1);
1261 return __r;
1262 }
1263
1264 [[__gnu__::__always_inline__]]
1265 constexpr basic_vec&
1266 operator--() noexcept requires requires(value_type __a) { --__a; }
1267 { return *this -= value_type(1); }
1268
1269 [[__gnu__::__always_inline__]]
1270 constexpr basic_vec
1271 operator--(int) noexcept requires requires(value_type __a) { __a--; }
1272 {
1273 basic_vec __r = *this;
1274 *this -= value_type(1);
1275 return __r;
1276 }
1277
1278 [[__gnu__::__always_inline__]]
1279 constexpr mask_type
1280 operator!() const noexcept requires requires(value_type __a) { !__a; }
1281 { return *this == value_type(); }
1282
1283 /**
1284 * @brief Unary plus operator (no-op).
1285 *
1286 * Returns an unchanged copy of the object.
1287 */
1288 [[__gnu__::__always_inline__]]
1289 constexpr basic_vec
1290 operator+() const noexcept requires requires(value_type __a) { +__a; }
1291 { return *this; }
1292
1293 /**
1294 * @brief Unary negation operator.
1295 *
1296 * Returns a new SIMD vector after element-wise negation.
1297 */
1298 [[__gnu__::__always_inline__]]
1299 constexpr basic_vec
1300 operator-() const noexcept requires requires(value_type __a) { -__a; }
1301 { return _S_init(-_M_data); }
1302
1303 /**
1304 * @brief Bitwise NOT / complement operator.
1305 *
1306 * Returns a new SIMD vector after element-wise complement.
1307 */
1308 [[__gnu__::__always_inline__]]
1309 constexpr basic_vec
1310 operator~() const noexcept requires requires(value_type __a) { ~__a; }
1311 { return _S_init(~_M_data); }
1312
1313 // [simd.cassign] binary operators
1314 /**
1315 * @brief Bitwise AND operator.
1316 *
1317 * Returns a new SIMD vector after element-wise AND.
1318 */
1319 [[__gnu__::__always_inline__]]
1320 friend constexpr basic_vec&
1321 operator&=(basic_vec& __x, const basic_vec& __y) noexcept
1322 requires requires(value_type __a) { __a & __a; }
1323 {
1324 __x._M_data &= __y._M_data;
1325 return __x;
1326 }
1327
1328 /**
1329 * @brief Bitwise OR operator.
1330 *
1331 * Returns a new SIMD vector after element-wise OR.
1332 */
1333 [[__gnu__::__always_inline__]]
1334 friend constexpr basic_vec&
1335 operator|=(basic_vec& __x, const basic_vec& __y) noexcept
1336 requires requires(value_type __a) { __a | __a; }
1337 {
1338 __x._M_data |= __y._M_data;
1339 return __x;
1340 }
1341
1342 /**
1343 * @brief Bitwise XOR operator.
1344 *
1345 * Returns a new SIMD vector after element-wise XOR.
1346 */
1347 [[__gnu__::__always_inline__]]
1348 friend constexpr basic_vec&
1349 operator^=(basic_vec& __x, const basic_vec& __y) noexcept
1350 requires requires(value_type __a) { __a ^ __a; }
1351 {
1352 __x._M_data ^= __y._M_data;
1353 return __x;
1354 }
1355
1356 /**
1357 * @brief Applies the compound assignment operator element-wise.
1358 *
1359 * @pre If @c value_type is a signed integral type, the result is representable by @c
1360 * value_type. (This does not apply to padding elements the implementation might add for
1361 * non-power-of-2 widths.) UBsan will only see a call to @c unreachable() on overflow.
1362 *
1363 * @note The overflow detection code is discarded unless UBsan is active.
1364 */
1365 [[__gnu__::__always_inline__]]
1366 friend constexpr basic_vec&
1367 operator+=(basic_vec& __x, const basic_vec& __y) noexcept
1368 requires requires(value_type __a) { __a + __a; }
1369 {
1370 if constexpr (_S_is_partial && is_integral_v<value_type> && is_signed_v<value_type>)
1371 { // avoid spurious UB on signed integer overflow of the padding element(s). But don't
1372 // remove UB of the active elements (so that UBsan can still do its job).
1373 //
1374 // This check is essentially free (at runtime) because DCE removes everything except
1375 // the final change to _M_data. The overflow check is only emitted if UBsan is active.
1376 //
1377 // The alternative would be to always zero padding elements after operations that can
1378 // produce non-zero values. However, right now:
1379 // - auto f(simd::mask<int, 3> k) { return +k; } is a single VPABSD and would have to
1380 // sanitize
1381 // - bit_cast to basic_vec with non-zero padding elements is fine
1382 // - conversion from intrinsics can create non-zero padding elements
1383 // - shuffles are allowed to put whatever they want into padding elements for
1384 // optimization purposes (e.g. for better instruction selection)
1385 using _UV = typename _Ap::template _DataType<make_unsigned_t<value_type>>;
1386 const _DataType __result
1387 = reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
1388 + reinterpret_cast<_UV>(__y._M_data));
1389 const auto __positive = __y > value_type();
1390 const auto __overflow = __positive != (__result > __x);
1391 if (__overflow._M_any_of())
1392 __builtin_unreachable(); // trigger UBsan
1393 __x._M_data = __result;
1394 }
1395 else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
1396 __x = basic_vec(rebind_t<float, basic_vec>(__x) + __y);
1397 else
1398 __x._M_data += __y._M_data;
1399 return __x;
1400 }
1401
1402 /** @copydoc operator+=
1403 */
1404 [[__gnu__::__always_inline__]]
1405 friend constexpr basic_vec&
1406 operator-=(basic_vec& __x, const basic_vec& __y) noexcept
1407 requires requires(value_type __a) { __a - __a; }
1408 {
1409 if constexpr (_S_is_partial && is_integral_v<value_type> && is_signed_v<value_type>)
1410 { // see comment on operator+=
1411 using _UV = typename _Ap::template _DataType<make_unsigned_t<value_type>>;
1412 const _DataType __result
1413 = reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
1414 - reinterpret_cast<_UV>(__y._M_data));
1415 const auto __positive = __y > value_type();
1416 const auto __overflow = __positive != (__result < __x);
1417 if (__overflow._M_any_of())
1418 __builtin_unreachable(); // trigger UBsan
1419 __x._M_data = __result;
1420 }
1421 else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
1422 __x = basic_vec(rebind_t<float, basic_vec>(__x) - __y);
1423 else
1424 __x._M_data -= __y._M_data;
1425 return __x;
1426 }
1427
1428 /** @copydoc operator+=
1429 */
1430 [[__gnu__::__always_inline__]]
1431 friend constexpr basic_vec&
1432 operator*=(basic_vec& __x, const basic_vec& __y) noexcept
1433 requires requires(value_type __a) { __a * __a; }
1434 {
1435 if constexpr (_S_is_partial && is_integral_v<value_type> && is_signed_v<value_type>)
1436 { // see comment on operator+=
1437 for (int __i = 0; __i < _S_size; ++__i)
1438 {
1439 if (__builtin_mul_overflow_p(__x._M_data[__i], __y._M_data[__i], value_type()))
1440 __builtin_unreachable();
1441 }
1442 using _UV = typename _Ap::template _DataType<make_unsigned_t<value_type>>;
1443 __x._M_data = reinterpret_cast<_DataType>(reinterpret_cast<_UV>(__x._M_data)
1444 * reinterpret_cast<_UV>(__y._M_data));
1445 }
1446
1447 // 'uint16 * uint16' promotes to int and can therefore lead to UB. The standard does not
1448 // require to avoid the undefined behavior. It's unnecessary and easy to avoid. It's also
1449 // unexpected because there's no UB on the vector types (which don't promote).
1450 else if constexpr (_S_is_scalar && is_unsigned_v<value_type>
1451 && is_signed_v<decltype(value_type() * value_type())>)
1452 __x._M_data = unsigned(__x._M_data) * unsigned(__y._M_data);
1453
1454 else if constexpr (_TargetTraits()._M_eval_as_f32<value_type>())
1455 __x = basic_vec(rebind_t<float, basic_vec>(__x) * __y);
1456
1457 else
1458 __x._M_data *= __y._M_data;
1459 return __x;
1460 }
1461
1462 template <_TargetTraits _Traits = {}>
1463 [[__gnu__::__always_inline__]]
1464 friend constexpr basic_vec&
1465 operator/=(basic_vec& __x, const basic_vec& __y) noexcept
1466 requires requires(value_type __a) { __a / __a; }
1467 {
1468 const basic_vec __result([&](int __i) -> value_type { return __x[__i] / __y[__i]; });
1469 if (__is_const_known(__result))
1470 // the optimizer already knows the values of the result
1471 return __x = __result;
1472
1473#ifdef __SSE2__
1474 // x86 doesn't have integral SIMD division instructions
1475 // While division is faster, the required conversions are still a problem:
1476 // see PR121274, PR121284, and PR121296 for missed optimizations wrt. conversions
1477 //
1478 // With only 1 or 2 divisions, the conversion to and from fp is too expensive.
1479 if constexpr (is_integral_v<value_type> && _S_size > 2
1480 && __value_preserving_convertible_to<value_type, double>)
1481 {
1482 // If the denominator (y) is known to the optimizer, don't convert to fp because the
1483 // integral division can be translated into shifts/multiplications.
1484 if (!__is_const_known(__y))
1485 {
1486 // With AVX512FP16 use vdivph for 8-bit integers
1487 if constexpr (_Traits._M_have_avx512fp16()
1488 && __value_preserving_convertible_to<value_type, _Float16>)
1489 return __x = basic_vec(rebind_t<_Float16, basic_vec>(__x) / __y);
1490 else if constexpr (__value_preserving_convertible_to<value_type, float>)
1491 return __x = basic_vec(rebind_t<float, basic_vec>(__x) / __y);
1492 else
1493 return __x = basic_vec(rebind_t<double, basic_vec>(__x) / __y);
1494 }
1495 }
1496#endif
1497 if constexpr (_Traits._M_eval_as_f32<value_type>())
1498 return __x = basic_vec(rebind_t<float, basic_vec>(__x) / __y);
1499
1500 basic_vec __y1 = __y;
1501 if constexpr (_S_is_partial)
1502 {
1503 if constexpr (is_integral_v<value_type>)
1504 {
1505 // Assume integral division doesn't have SIMD instructions and must be done per
1506 // element anyway. Partial vectors should skip their padding elements.
1507 for (int __i = 0; __i < _S_size; ++__i)
1508 __x._M_data[__i] /= __y._M_data[__i];
1509 return __x;
1510 }
1511 else
1512 __y1 = __select_impl(mask_type::_S_init(mask_type::_S_implicit_mask),
1513 __y, basic_vec(value_type(1)));
1514 }
1515 __x._M_data /= __y1._M_data;
1516 return __x;
1517 }
1518
1519 [[__gnu__::__always_inline__]]
1520 friend constexpr basic_vec&
1521 operator%=(basic_vec& __x, const basic_vec& __y) noexcept
1522 requires requires(value_type __a) { __a % __a; }
1523 {
1524 static_assert(is_integral_v<value_type>);
1525 if constexpr (_S_is_partial)
1526 {
1527 const basic_vec __y1 = __select_impl(mask_type::_S_init(mask_type::_S_implicit_mask),
1528 __y, basic_vec(value_type(1)));
1529 if (__is_const_known(__y1))
1530 __x._M_data %= __y1._M_data;
1531 else
1532 {
1533 // Assume integral division doesn't have SIMD instructions and must be done per
1534 // element anyway. Partial vectors should skip their padding elements.
1535 for (int __i = 0; __i < _S_size; ++__i)
1536 __x._M_data[__i] %= __y._M_data[__i];
1537 }
1538 }
1539 else
1540 __x._M_data %= __y._M_data;
1541 return __x;
1542 }
1543
1544 [[__gnu__::__always_inline__]]
1545 friend constexpr basic_vec&
1546 operator<<=(basic_vec& __x, const basic_vec& __y) _GLIBCXX_SIMD_NOEXCEPT
1547 requires requires(value_type __a) { __a << __a; }
1548 {
1549 __glibcxx_simd_precondition(is_unsigned_v<value_type> || all_of(__y >= value_type()),
1550 "negative shift is undefined behavior");
1551 __glibcxx_simd_precondition(all_of(__y < __max_shift<value_type>),
1552 "too large shift invokes undefined behavior");
1553 __x._M_data <<= __y._M_data;
1554 return __x;
1555 }
1556
1557 [[__gnu__::__always_inline__]]
1558 friend constexpr basic_vec&
1559 operator>>=(basic_vec& __x, const basic_vec& __y) _GLIBCXX_SIMD_NOEXCEPT
1560 requires requires(value_type __a) { __a >> __a; }
1561 {
1562 __glibcxx_simd_precondition(is_unsigned_v<value_type> || all_of(__y >= value_type()),
1563 "negative shift is undefined behavior");
1564 __glibcxx_simd_precondition(all_of(__y < __max_shift<value_type>),
1565 "too large shift invokes undefined behavior");
1566 __x._M_data >>= __y._M_data;
1567 return __x;
1568 }
1569
1570 [[__gnu__::__always_inline__]]
1571 friend constexpr basic_vec&
1572 operator<<=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
1573 requires requires(value_type __a, __simd_size_type __b) { __a << __b; }
1574 {
1575 __glibcxx_simd_precondition(__y >= 0, "negative shift is undefined behavior");
1576 __glibcxx_simd_precondition(__y < int(__max_shift<value_type>),
1577 "too large shift invokes undefined behavior");
1578 __x._M_data <<= __y;
1579 return __x;
1580 }
1581
1582 [[__gnu__::__always_inline__]]
1583 friend constexpr basic_vec&
1584 operator>>=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
1585 requires requires(value_type __a, __simd_size_type __b) { __a >> __b; }
1586 {
1587 __glibcxx_simd_precondition(__y >= 0, "negative shift is undefined behavior");
1588 __glibcxx_simd_precondition(__y < int(__max_shift<value_type>),
1589 "too large shift invokes undefined behavior");
1590 __x._M_data >>= __y;
1591 return __x;
1592 }
1593
1594 // [simd.comparison] ----------------------------------------------------
1595#if _GLIBCXX_X86
1596 template <_X86Cmp _Cmp>
1597 [[__gnu__::__always_inline__]]
1598 constexpr mask_type
1599 _M_bitmask_cmp(_DataType __y) const
1600 {
1601 static_assert(_S_use_bitmask);
1602 if (__is_const_known(_M_data, __y))
1603 {
1604 constexpr auto [...__is] = _IotaArray<_S_size>;
1605 constexpr auto __cmp_op = [] [[__gnu__::__always_inline__]]
1606 (value_type __a, value_type __b) {
1607 if constexpr (_Cmp == _X86Cmp::_Eq)
1608 return __a == __b;
1609 else if constexpr (_Cmp == _X86Cmp::_Lt)
1610 return __a < __b;
1611 else if constexpr (_Cmp == _X86Cmp::_Le)
1612 return __a <= __b;
1613 else if constexpr (_Cmp == _X86Cmp::_Unord)
1614 return std::isunordered(__a, __b);
1615 else if constexpr (_Cmp == _X86Cmp::_Neq)
1616 return __a != __b;
1617 else if constexpr (_Cmp == _X86Cmp::_Nlt)
1618 return !(__a < __b);
1619 else if constexpr (_Cmp == _X86Cmp::_Nle)
1620 return !(__a <= __b);
1621 else
1622 static_assert(false);
1623 };
1624 const _Bitmask<_S_size> __bits
1625 = ((__cmp_op(__vec_get(_M_data, __is), __vec_get(__y, __is))
1626 ? (1ULL << __is) : 0) | ...);
1627 return mask_type::_S_init(__bits);
1628 }
1629 else
1630 return mask_type::_S_init(__x86_bitmask_cmp<_Cmp>(_M_data, __y));
1631 }
1632#endif
1633
1634 [[__gnu__::__always_inline__]]
1635 friend constexpr mask_type
1636 operator==(const basic_vec& __x, const basic_vec& __y) noexcept
1637 {
1638#if _GLIBCXX_X86
1639 if constexpr (_S_use_bitmask)
1640 return __x._M_bitmask_cmp<_X86Cmp::_Eq>(__y._M_data);
1641 else
1642#endif
1643 return mask_type::_S_init(__x._M_data == __y._M_data);
1644 }
1645
1646 [[__gnu__::__always_inline__]]
1647 friend constexpr mask_type
1648 operator!=(const basic_vec& __x, const basic_vec& __y) noexcept
1649 {
1650#if _GLIBCXX_X86
1651 if constexpr (_S_use_bitmask)
1652 return __x._M_bitmask_cmp<_X86Cmp::_Neq>(__y._M_data);
1653 else
1654#endif
1655 return mask_type::_S_init(__x._M_data != __y._M_data);
1656 }
1657
1658 [[__gnu__::__always_inline__]]
1659 friend constexpr mask_type
1660 operator<(const basic_vec& __x, const basic_vec& __y) noexcept
1661 {
1662#if _GLIBCXX_X86
1663 if constexpr (_S_use_bitmask)
1664 return __x._M_bitmask_cmp<_X86Cmp::_Lt>(__y._M_data);
1665 else
1666#endif
1667 return mask_type::_S_init(__x._M_data < __y._M_data);
1668 }
1669
1670 [[__gnu__::__always_inline__]]
1671 friend constexpr mask_type
1672 operator<=(const basic_vec& __x, const basic_vec& __y) noexcept
1673 {
1674#if _GLIBCXX_X86
1675 if constexpr (_S_use_bitmask)
1676 return __x._M_bitmask_cmp<_X86Cmp::_Le>(__y._M_data);
1677 else
1678#endif
1679 return mask_type::_S_init(__x._M_data <= __y._M_data);
1680 }
1681
1682 [[__gnu__::__always_inline__]]
1683 friend constexpr mask_type
1684 operator>(const basic_vec& __x, const basic_vec& __y) noexcept
1685 { return __y < __x; }
1686
1687 [[__gnu__::__always_inline__]]
1688 friend constexpr mask_type
1689 operator>=(const basic_vec& __x, const basic_vec& __y) noexcept
1690 { return __y <= __x; }
1691
1692 // [simd.cond] ---------------------------------------------------------
1693 template <_TargetTraits _Traits = {}>
1694 [[__gnu__::__always_inline__]]
1695 friend constexpr basic_vec
1696 __select_impl(const mask_type& __k, const basic_vec& __t, const basic_vec& __f) noexcept
1697 {
1698 if constexpr (_S_size == 1)
1699 return __k[0] ? __t : __f;
1700 else if constexpr (_S_use_bitmask)
1701 {
1702#if _GLIBCXX_X86
1703 if (__is_const_known(__k, __t, __f))
1704 return basic_vec([&](int __i) { return __k[__i] ? __t[__i] : __f[__i]; });
1705 else
1706 return __x86_bitmask_blend(__k._M_data, __t._M_data, __f._M_data);
1707#else
1708 static_assert(false, "TODO");
1709#endif
1710 }
1711 else if consteval
1712 {
1713 return __k._M_data ? __t._M_data : __f._M_data;
1714 }
1715 else
1716 {
1717 constexpr bool __uses_simd_register = sizeof(_M_data) >= 8;
1718 using _VO = _VecOps<_DataType>;
1719 if (_VO::_S_is_const_known_equal_to(__f._M_data, 0))
1720 {
1721 if (is_integral_v<value_type> && __uses_simd_register
1722 && _VO::_S_is_const_known_equal_to(__t._M_data, 1))
1723 // This is equivalent to converting the mask into a vec of 0s and 1s. So +__k.
1724 // However, basic_mask::operator+ arrives here; returning +__k would be
1725 // recursive. Instead we use -__k (which is a no-op for vector-masks) and then
1726 // flip all -1 elements to +1 by taking the absolute value.
1727 return basic_vec((-__k)._M_abs());
1728 else
1729 return __vec_and(reinterpret_cast<_DataType>(__k._M_data), __t._M_data);
1730 }
1731 else if (_VecOps<_DataType>::_S_is_const_known_equal_to(__t._M_data, 0))
1732 {
1733 if (is_integral_v<value_type> && __uses_simd_register
1734 && _VO::_S_is_const_known_equal_to(__f._M_data, 1))
1735 return value_type(1) + basic_vec(-__k);
1736 else
1737 return __vec_and(reinterpret_cast<_DataType>(__vec_not(__k._M_data)), __f._M_data);
1738 }
1739 else
1740 {
1741#if _GLIBCXX_X86
1742 // this works around bad code-gen when the compiler can't see that __k is a vector-mask.
1743 // This pattern, is recognized to match the x86 blend instructions, which only consider
1744 // the sign bit of the mask register. Also, without SSE4, if the compiler knows that __k
1745 // is a vector-mask, then the '< 0' is elided.
1746 return __k._M_data < 0 ? __t._M_data : __f._M_data;
1747#endif
1748 return __k._M_data ? __t._M_data : __f._M_data;
1749 }
1750 }
1751 }
1752 };
1753
1754 template <__vectorizable _Tp, __abi_tag _Ap>
1755 requires (_Ap::_S_nreg > 1)
1756 && (!__complex_like<_Tp>)
1757 class basic_vec<_Tp, _Ap>
1758 : public _VecBase<_Tp, _Ap>
1759 {
1760 template <typename, typename>
1761 friend class basic_vec;
1762
1763 template <size_t, typename>
1764 friend class basic_mask;
1765
1766 static constexpr int _S_size = _Ap::_S_size;
1767
1768 static constexpr int _N0 = __bit_ceil(unsigned(_S_size)) / 2;
1769
1770 static constexpr int _N1 = _S_size - _N0;
1771
1772 using _DataType0 = __similar_vec<_Tp, _N0, _Ap>;
1773
1774 // the implementation (and users) depend on elements being contiguous in memory
1775 static_assert(_N0 * sizeof(_Tp) == sizeof(_DataType0));
1776
1777 using _DataType1 = __similar_vec<_Tp, _N1, _Ap>;
1778
1779 static_assert(_DataType0::abi_type::_S_nreg + _DataType1::abi_type::_S_nreg == _Ap::_S_nreg);
1780
1781 static constexpr bool _S_is_scalar = _DataType0::_S_is_scalar;
1782
1783 _DataType0 _M_data0;
1784
1785 _DataType1 _M_data1;
1786
1787 static constexpr bool _S_use_bitmask = _DataType0::_S_use_bitmask;
1788
1789 static constexpr bool _S_is_partial = _DataType1::_S_is_partial;
1790
1791 public:
1792 using value_type = _Tp;
1793
1794 using mask_type = _VecBase<_Tp, _Ap>::mask_type;
1795
1796 [[__gnu__::__always_inline__]]
1797 static constexpr basic_vec
1798 _S_init(const _DataType0& __x, const _DataType1& __y)
1799 {
1800 basic_vec __r;
1801 __r._M_data0 = __x;
1802 __r._M_data1 = __y;
1803 return __r;
1804 }
1805
1806 [[__gnu__::__always_inline__]]
1807 constexpr _DataType0&
1808 _M_get_low() noexcept
1809 { return _M_data0; }
1810
1811 [[__gnu__::__always_inline__]]
1812 constexpr const _DataType0&
1813 _M_get_low() const noexcept
1814 { return _M_data0; }
1815
1816 [[__gnu__::__always_inline__]]
1817 constexpr _DataType1&
1818 _M_get_high() noexcept
1819 { return _M_data1; }
1820
1821 [[__gnu__::__always_inline__]]
1822 constexpr const _DataType1&
1823 _M_get_high() const noexcept
1824 { return _M_data1; }
1825
1826 [[__gnu__::__always_inline__]]
1827 friend constexpr bool
1828 __is_const_known(const basic_vec& __x)
1829 { return __is_const_known(__x._M_data0) && __is_const_known(__x._M_data1); }
1830
1831 [[__gnu__::__always_inline__]]
1832 constexpr auto
1833 _M_concat_data([[maybe_unused]] bool __do_sanitize = false) const
1834 {
1835 return __vec_concat(_M_data0._M_concat_data(false),
1836 __vec_zero_pad_to<sizeof(_M_data0)>(
1837 _M_data1._M_concat_data(__do_sanitize)));
1838 }
1839
1840 template <int _Size = _S_size, int _Offset = 0, typename _A0, typename _Fp>
1841 [[__gnu__::__always_inline__]]
1842 static constexpr basic_vec
1843 _S_static_permute(const basic_vec<value_type, _A0>& __x, _Fp&& __idxmap)
1844 {
1845 return _S_init(
1846 _DataType0::template _S_static_permute<_Size, _Offset>(__x, __idxmap),
1847 _DataType1::template _S_static_permute<_Size, _Offset + _N0>(__x, __idxmap));
1848 }
1849
1850 template <typename _Vp>
1851 [[__gnu__::__always_inline__]]
1852 constexpr auto
1853 _M_chunk() const noexcept
1854 {
1855 constexpr int __n = _S_size / _Vp::_S_size;
1856 constexpr int __rem = _S_size % _Vp::_S_size;
1857 constexpr auto [...__is] = _IotaArray<__n>;
1858 if constexpr (__rem == 0)
1859 return array<_Vp, __n>{__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>,
1860 _M_data0, _M_data1)...};
1861 else
1862 {
1863 using _Rest = resize_t<__rem, _Vp>;
1864 return tuple(__extract_simd_at<_Vp>(cw<_Vp::_S_size * __is>, _M_data0, _M_data1)...,
1865 __extract_simd_at<_Rest>(cw<_Vp::_S_size * __n>, _M_data0, _M_data1));
1866 }
1867 }
1868
1869 [[__gnu__::__always_inline__]]
1870 static constexpr const basic_vec&
1871 _S_concat(const basic_vec& __x0) noexcept
1872 { return __x0; }
1873
1874 template <typename... _As>
1875 requires (sizeof...(_As) >= 2)
1876 [[__gnu__::__always_inline__]]
1877 static constexpr basic_vec
1878 _S_concat(const basic_vec<value_type, _As>&... __xs) noexcept
1879 {
1880 static_assert(_S_size == (_As::_S_size + ...));
1881 return _S_init(__extract_simd_at<_DataType0>(cw<0>, __xs...),
1882 __extract_simd_at<_DataType1>(cw<_N0>, __xs...));
1883 }
1884
1885 [[__gnu__::__always_inline__]]
1886 constexpr auto
1887 _M_reduce_to_half(auto __binary_op) const requires (_N0 == _N1)
1888 { return __binary_op(_M_data0, _M_data1); }
1889
1890 [[__gnu__::__always_inline__]]
1891 constexpr value_type
1892 _M_reduce_tail(const auto& __rest, auto __binary_op) const
1893 {
1894 if constexpr (__rest.size() > _S_size)
1895 {
1896 auto [__a, __b] = __rest.template _M_chunk<basic_vec>();
1897 return __binary_op(*this, __a)._M_reduce_tail(__b, __binary_op);
1898 }
1899 else if constexpr (__rest.size() == _S_size)
1900 return __binary_op(*this, __rest)._M_reduce(__binary_op);
1901 else
1902 return _M_reduce_to_half(__binary_op)._M_reduce_tail(__rest, __binary_op);
1903 }
1904
1905 template <typename _BinaryOp, _TargetTraits _Traits = {}>
1906 [[__gnu__::__always_inline__]]
1907 constexpr value_type
1908 _M_reduce(_BinaryOp __binary_op) const
1909 {
1910 if constexpr (_Traits.template _M_eval_as_f32<value_type>()
1911 && (is_same_v<_BinaryOp, plus<>>
1912 || is_same_v<_BinaryOp, multiplies<>>))
1913 return value_type(rebind_t<float, basic_vec>(*this)._M_reduce(__binary_op));
1914#ifdef __SSE2__
1915 else if constexpr (is_integral_v<value_type> && sizeof(value_type) == 1
1916 && is_same_v<decltype(__binary_op), multiplies<>>)
1917 {
1918 // convert to unsigned short because of missing 8-bit mul instruction
1919 // we don't need to preserve the order of elements
1920 //
1921 // The left columns under Latency and Throughput show bit-cast to ushort with shift by
1922 // 8. The right column uses the alternative in the else branch.
1923 // Benchmark on Intel Ultra 7 165U (AVX2)
1924 // TYPE Latency Throughput
1925 // [cycles/call] [cycles/call]
1926 //schar, 64 59.9 70.7 10.5 13.3
1927 //schar, 128 81.4 97.2 12.2 21
1928 //schar, 256 92.4 129 17.2 35.2
1929 if constexpr (_DataType1::_S_is_scalar)
1930 return __binary_op(_DataType1(_M_data0._M_reduce(__binary_op)), _M_data1)[0];
1931 // TODO: optimize trailing scalar (e.g. (8+8)+(8+1))
1932 else if constexpr (_S_size % 2 == 0)
1933 { // If all elements participate in the reduction we can take this shortcut
1934 using _V16 = resize_t<_S_size / 2, rebind_t<unsigned short, basic_vec>>;
1935 auto __a = __builtin_bit_cast(_V16, *this);
1936 return __binary_op(__a, __a >> __CHAR_BIT__)._M_reduce(__binary_op);
1937 }
1938 else
1939 {
1940 using _V16 = rebind_t<unsigned short, basic_vec>;
1941 return _V16(*this)._M_reduce(__binary_op);
1942 }
1943 }
1944#endif
1945 else
1946 return _M_data0._M_reduce_tail(_M_data1, __binary_op);
1947 }
1948
1949 [[__gnu__::__always_inline__]]
1950 constexpr mask_type
1951 _M_isnan() const requires is_floating_point_v<value_type>
1952 { return mask_type::_S_init(_M_data0._M_isnan(), _M_data1._M_isnan()); }
1953
1954 [[__gnu__::__always_inline__]]
1955 constexpr mask_type
1956 _M_isinf() const requires is_floating_point_v<value_type>
1957 { return mask_type::_S_init(_M_data0._M_isinf(), _M_data1._M_isinf()); }
1958
1959 [[__gnu__::__always_inline__]]
1960 constexpr mask_type
1961 _M_isunordered(basic_vec __y) const requires is_floating_point_v<value_type>
1962 {
1963 return mask_type::_S_init(_M_data0._M_isunordered(__y._M_data0),
1964 _M_data1._M_isunordered(__y._M_data1));
1965 }
1966
1967 [[__gnu__::__always_inline__]]
1968 constexpr basic_vec
1969 _M_abs() const requires signed_integral<value_type>
1970 { return _S_init(_M_data0._M_abs(), _M_data1._M_abs()); }
1971
1972 [[__gnu__::__always_inline__]]
1973 constexpr basic_vec
1974 _M_fabs() const requires floating_point<value_type>
1975 { return _S_init(_M_data0._M_fabs(), _M_data1._M_fabs()); }
1976
1977 template <typename _Up>
1978 [[__gnu__::__always_inline__]]
1979 static inline basic_vec
1980 _S_partial_load(const _Up* __mem, size_t __n)
1981 {
1982 if (__n >= _N0)
1983 return _S_init(_DataType0(_LoadCtorTag(), __mem),
1984 _DataType1::_S_partial_load(__mem + _N0, __n - _N0));
1985 else
1986 return _S_init(_DataType0::_S_partial_load(__mem, __n),
1987 _DataType1());
1988 }
1989
1990 template <typename _Up, _ArchTraits _Traits = {}>
1991 static inline basic_vec
1992 _S_masked_load(const _Up* __mem, mask_type __k)
1993 {
1994 return _S_init(_DataType0::_S_masked_load(__mem, __k._M_data0),
1995 _DataType1::_S_masked_load(__mem + _N0, __k._M_data1));
1996 }
1997
1998 template <typename _Up>
1999 [[__gnu__::__always_inline__]]
2000 inline void
2001 _M_store(_Up* __mem) const
2002 {
2003 _M_data0._M_store(__mem);
2004 _M_data1._M_store(__mem + _N0);
2005 }
2006
2007 template <typename _Up>
2008 [[__gnu__::__always_inline__]]
2009 static inline void
2010 _S_partial_store(const basic_vec& __v, _Up* __mem, size_t __n)
2011 {
2012 if (__n >= _N0)
2013 {
2014 __v._M_data0._M_store(__mem);
2015 _DataType1::_S_partial_store(__v._M_data1, __mem + _N0, __n - _N0);
2016 }
2017 else
2018 {
2019 _DataType0::_S_partial_store(__v._M_data0, __mem, __n);
2020 }
2021 }
2022
2023 template <typename _Up>
2024 [[__gnu__::__always_inline__]]
2025 static inline void
2026 _S_masked_store(const basic_vec& __v, _Up* __mem, const mask_type& __k)
2027 {
2028 _DataType0::_S_masked_store(__v._M_data0, __mem, __k._M_data0);
2029 _DataType1::_S_masked_store(__v._M_data1, __mem + _N0, __k._M_data1);
2030 }
2031
2032 basic_vec() = default;
2033
2034 // [simd.overview] p2 impl-def conversions ------------------------------
2035 using _NativeVecType = __vec_builtin_type<value_type, __bit_ceil(unsigned(_S_size))>;
2036
2037 [[__gnu__::__always_inline__]]
2038 constexpr
2039 basic_vec(const _NativeVecType& __x)
2040 : _M_data0(_VecOps<__vec_builtin_type<value_type, _N0>>::_S_extract(__x)),
2041 _M_data1(_VecOps<__vec_builtin_type<value_type, __bit_ceil(unsigned(_N1))>>
2042 ::_S_extract(__x, integral_constant<int, _N0>()))
2043 {}
2044
2045 [[__gnu__::__always_inline__]]
2046 constexpr
2047 operator _NativeVecType() const
2048 { return _M_concat_data(); }
2049
2050 // [simd.ctor] broadcast constructor ------------------------------------
2051 template <__broadcast_constructible<value_type> _Up>
2052 [[__gnu__::__always_inline__]]
2053 constexpr
2054 basic_vec(_Up&& __x) noexcept
2055 : _M_data0(static_cast<value_type>(__x)), _M_data1(static_cast<value_type>(__x))
2056 {}
2057
2058 // [simd.ctor] conversion constructor -----------------------------------
2059 template <typename _Up, typename _UAbi>
2060 requires (_S_size == _UAbi::_S_size)
2061 && __explicitly_convertible_to<_Up, value_type>
2062 [[__gnu__::__always_inline__]]
2063 constexpr
2064 explicit(!__value_preserving_convertible_to<_Up, value_type>
2065 || __higher_rank_than<_Up, value_type>)
2066 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
2067 : _M_data0(get<0>(chunk<_N0>(__x))),
2068 _M_data1(get<1>(chunk<_N0>(__x)))
2069 {}
2070
2071 using _VecBase<_Tp, _Ap>::_VecBase;
2072
2073 // [simd.ctor] generator constructor ------------------------------------
2074 template <__simd_generator_invokable<value_type, _S_size> _Fp>
2075 [[__gnu__::__always_inline__]]
2076 constexpr explicit
2077 basic_vec(_Fp&& __gen)
2078 : _M_data0(__gen), _M_data1([&] [[__gnu__::__always_inline__]] (auto __i) {
2079 return __gen(__simd_size_c<__i + _N0>);
2080 })
2081 {}
2082
2083 // [simd.ctor] load constructor -----------------------------------------
2084 template <typename _Up>
2085 [[__gnu__::__always_inline__]]
2086 constexpr
2087 basic_vec(_LoadCtorTag, const _Up* __ptr)
2088 : _M_data0(_LoadCtorTag(), __ptr),
2089 _M_data1(_LoadCtorTag(), __ptr + _N0)
2090 {}
2091
2092 template <ranges::contiguous_range _Rg, typename... _Flags>
2093 requires ranges::__static_sized_range<_Rg>
2094 && __vectorizable<ranges::range_value_t<_Rg>>
2095 && __explicitly_convertible_to<ranges::range_value_t<_Rg>, value_type>
2096 constexpr
2097 basic_vec(_Rg&& __range, flags<_Flags...> __flags = {})
2098 requires (ranges::size(__range) == _S_size)
2099 : basic_vec(_LoadCtorTag(),
2100 __flags.template _S_adjust_pointer<basic_vec>(ranges::data(__range)))
2101 {
2102 static_assert(__loadstore_convertible_to<ranges::range_value_t<_Rg>, value_type,
2103 _Flags...>);
2104 }
2105
2106 // [simd.subscr] --------------------------------------------------------
2107 [[__gnu__::__always_inline__]]
2108 constexpr value_type
2109 operator[](__simd_size_type __i) const
2110 {
2111 __glibcxx_simd_precondition(__i >= 0 && __i < _S_size, "subscript is out of bounds");
2112 if (__is_const_known(__i))
2113 return __i < _N0 ? _M_data0[__i] : _M_data1[__i - _N0];
2114 else
2115 {
2116 using _AliasingT [[__gnu__::__may_alias__]] = value_type;
2117 return reinterpret_cast<const _AliasingT*>(this)[__i];
2118 }
2119 }
2120
2121 // [simd.unary] unary operators -----------------------------------------
2122 [[__gnu__::__always_inline__]]
2123 constexpr basic_vec&
2124 operator++() noexcept requires requires(value_type __a) { ++__a; }
2125 {
2126 ++_M_data0;
2127 ++_M_data1;
2128 return *this;
2129 }
2130
2131 [[__gnu__::__always_inline__]]
2132 constexpr basic_vec
2133 operator++(int) noexcept requires requires(value_type __a) { __a++; }
2134 {
2135 basic_vec __r = *this;
2136 ++_M_data0;
2137 ++_M_data1;
2138 return __r;
2139 }
2140
2141 [[__gnu__::__always_inline__]]
2142 constexpr basic_vec&
2143 operator--() noexcept requires requires(value_type __a) { --__a; }
2144 {
2145 --_M_data0;
2146 --_M_data1;
2147 return *this;
2148 }
2149
2150 [[__gnu__::__always_inline__]]
2151 constexpr basic_vec
2152 operator--(int) noexcept requires requires(value_type __a) { __a--; }
2153 {
2154 basic_vec __r = *this;
2155 --_M_data0;
2156 --_M_data1;
2157 return __r;
2158 }
2159
2160 [[__gnu__::__always_inline__]]
2161 constexpr mask_type
2162 operator!() const noexcept requires requires(value_type __a) { !__a; }
2163 { return mask_type::_S_init(!_M_data0, !_M_data1); }
2164
2165 [[__gnu__::__always_inline__]]
2166 constexpr basic_vec
2167 operator+() const noexcept requires requires(value_type __a) { +__a; }
2168 { return *this; }
2169
2170 [[__gnu__::__always_inline__]]
2171 constexpr basic_vec
2172 operator-() const noexcept requires requires(value_type __a) { -__a; }
2173 { return _S_init(-_M_data0, -_M_data1); }
2174
2175 [[__gnu__::__always_inline__]]
2176 constexpr basic_vec
2177 operator~() const noexcept requires requires(value_type __a) { ~__a; }
2178 { return _S_init(~_M_data0, ~_M_data1); }
2179
2180 // [simd.cassign] -------------------------------------------------------
2181#define _GLIBCXX_SIMD_DEFINE_OP(sym) \
2182 [[__gnu__::__always_inline__]] \
2183 friend constexpr basic_vec& \
2184 operator sym##=(basic_vec& __x, const basic_vec& __y) _GLIBCXX_SIMD_NOEXCEPT \
2185 { \
2186 __x._M_data0 sym##= __y._M_data0; \
2187 __x._M_data1 sym##= __y._M_data1; \
2188 return __x; \
2189 }
2190
2191 _GLIBCXX_SIMD_DEFINE_OP(+)
2192 _GLIBCXX_SIMD_DEFINE_OP(-)
2193 _GLIBCXX_SIMD_DEFINE_OP(*)
2194 _GLIBCXX_SIMD_DEFINE_OP(/)
2195 _GLIBCXX_SIMD_DEFINE_OP(%)
2196 _GLIBCXX_SIMD_DEFINE_OP(&)
2197 _GLIBCXX_SIMD_DEFINE_OP(|)
2198 _GLIBCXX_SIMD_DEFINE_OP(^)
2199 _GLIBCXX_SIMD_DEFINE_OP(<<)
2200 _GLIBCXX_SIMD_DEFINE_OP(>>)
2201
2202#undef _GLIBCXX_SIMD_DEFINE_OP
2203
2204 [[__gnu__::__always_inline__]]
2205 friend constexpr basic_vec&
2206 operator<<=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
2207 requires requires(value_type __a, __simd_size_type __b) { __a << __b; }
2208 {
2209 __x._M_data0 <<= __y;
2210 __x._M_data1 <<= __y;
2211 return __x;
2212 }
2213
2214 [[__gnu__::__always_inline__]]
2215 friend constexpr basic_vec&
2216 operator>>=(basic_vec& __x, __simd_size_type __y) _GLIBCXX_SIMD_NOEXCEPT
2217 requires requires(value_type __a, __simd_size_type __b) { __a >> __b; }
2218 {
2219 __x._M_data0 >>= __y;
2220 __x._M_data1 >>= __y;
2221 return __x;
2222 }
2223
2224 // [simd.comparison] ----------------------------------------------------
2225 [[__gnu__::__always_inline__]]
2226 friend constexpr mask_type
2227 operator==(const basic_vec& __x, const basic_vec& __y) noexcept
2228 { return mask_type::_S_init(__x._M_data0 == __y._M_data0, __x._M_data1 == __y._M_data1); }
2229
2230 [[__gnu__::__always_inline__]]
2231 friend constexpr mask_type
2232 operator!=(const basic_vec& __x, const basic_vec& __y) noexcept
2233 { return mask_type::_S_init(__x._M_data0 != __y._M_data0, __x._M_data1 != __y._M_data1); }
2234
2235 [[__gnu__::__always_inline__]]
2236 friend constexpr mask_type
2237 operator<(const basic_vec& __x, const basic_vec& __y) noexcept
2238 { return mask_type::_S_init(__x._M_data0 < __y._M_data0, __x._M_data1 < __y._M_data1); }
2239
2240 [[__gnu__::__always_inline__]]
2241 friend constexpr mask_type
2242 operator<=(const basic_vec& __x, const basic_vec& __y) noexcept
2243 { return mask_type::_S_init(__x._M_data0 <= __y._M_data0, __x._M_data1 <= __y._M_data1); }
2244
2245 [[__gnu__::__always_inline__]]
2246 friend constexpr mask_type
2247 operator>(const basic_vec& __x, const basic_vec& __y) noexcept
2248 { return mask_type::_S_init(__x._M_data0 > __y._M_data0, __x._M_data1 > __y._M_data1); }
2249
2250 [[__gnu__::__always_inline__]]
2251 friend constexpr mask_type
2252 operator>=(const basic_vec& __x, const basic_vec& __y) noexcept
2253 { return mask_type::_S_init(__x._M_data0 >= __y._M_data0, __x._M_data1 >= __y._M_data1); }
2254
2255 // [simd.cond] ---------------------------------------------------------
2256 [[__gnu__::__always_inline__]]
2257 friend constexpr basic_vec
2258 __select_impl(const mask_type& __k, const basic_vec& __t, const basic_vec& __f) noexcept
2259 {
2260 return _S_init(__select_impl(__k._M_data0, __t._M_data0, __f._M_data0),
2261 __select_impl(__k._M_data1, __t._M_data1, __f._M_data1));
2262 }
2263 };
2264
2265 // [simd.overview] deduction guide ------------------------------------------
2266 template <ranges::contiguous_range _Rg, typename... _Ts>
2267 requires ranges::__static_sized_range<_Rg>
2268 basic_vec(_Rg&& __r, _Ts...)
2269 -> basic_vec<ranges::range_value_t<_Rg>,
2270 __deduce_abi_t<ranges::range_value_t<_Rg>,
2271#if 0 // PR117849
2272 static_cast<__simd_size_type>(ranges::size(__r))>>;
2273#else
2274 static_cast<__simd_size_type>(decltype(std::span(__r))::extent)>>;
2275#endif
2276
2277 template <size_t _Bytes, typename _Ap>
2278 basic_vec(basic_mask<_Bytes, _Ap>)
2279 -> basic_vec<__integer_from<_Bytes>,
2280 decltype(__abi_rebind<__integer_from<_Bytes>, basic_mask<_Bytes, _Ap>::size.value,
2281 _Ap>())>;
2282
2283 // [P3319R5] ----------------------------------------------------------------
2284 template <__vectorizable _Tp>
2285 requires is_arithmetic_v<_Tp>
2286 inline constexpr _Tp
2287 __iota<_Tp> = _Tp();
2288
2289 template <typename _Tp, typename _Ap>
2290 inline constexpr basic_vec<_Tp, _Ap>
2291 __iota<basic_vec<_Tp, _Ap>> = basic_vec<_Tp, _Ap>([](_Tp __i) -> _Tp {
2292 static_assert(_Ap::_S_size - 1 <= numeric_limits<_Tp>::max(),
2293 "iota object would overflow");
2294 return __i;
2295 });
2296} // namespace simd
2297_GLIBCXX_END_NAMESPACE_VERSION
2298} // namespace std
2299
2300#pragma GCC diagnostic pop
2301#endif // C++26
2302#endif // _GLIBCXX_SIMD_VEC_H
std::chrono::operator<=
constexpr bool operator<=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:863
std::chrono::operator>=
constexpr bool operator>=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:877
std::chrono::operator<
constexpr bool operator<(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:830
std::chrono::operator>
constexpr bool operator>(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:870
std::operator-
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
Definition complex:404
std::operator+
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
Definition complex:374
std::is_sufficiently_aligned
bool is_sufficiently_aligned(_Tp *__ptr)
Is __ptr aligned to an _Align byte boundary?
Definition align.h:118
std
ISO C++ entities toplevel namespace is std.
std::fabs
_Tp fabs(const std::complex< _Tp > &__z)
fabs(__z) TR1 8.1.8 [tr.c99.cmplx.fabs]
Definition complex:2525
std::data
constexpr auto data(_Container &__cont) noexcept(noexcept(__cont.data())) -> decltype(__cont.data())
Return the data pointer of a container.
Definition range_access.h:354
std::numeric_limits::max
static constexpr _Tp max() noexcept
Definition limits:336
std::numeric_limits::infinity
static constexpr _Tp infinity() noexcept
Definition limits:356