libstdc++
simd_complex.h
1// Implementation of <simd> -*- C++ -*-
2
3// Copyright The GNU Toolchain Authors.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_SIMD_COMPLEX_H
26#define _GLIBCXX_SIMD_COMPLEX_H 1
27
28#ifdef _GLIBCXX_SYSHDR
29#pragma GCC system_header
30#endif
31
32#if __cplusplus >= 202400L
33
34#include "simd_vec.h"
35#include <complex>
36
37// psabi warnings are bogus because the ABI of the internal types never leaks into user code
38#pragma GCC diagnostic push
39#pragma GCC diagnostic ignored "-Wpsabi"
40
41namespace std _GLIBCXX_VISIBILITY(default)
42{
43_GLIBCXX_BEGIN_NAMESPACE_VERSION
44namespace simd
45{
46 /** @internal
47 * @brief Return a _CxIleav mask that holds @p __k as its data member.
48 *
49 * @note If the resulting mask type has size 1, then it will actually store a single bool, rather
50 * than the given mask object.
51 */
52 template <size_t _Bytes, typename _Ap, __abi_tag _Aret
53 = decltype(__abi_rebind<complex<__float_from<_Bytes>>, _Ap::_S_size / 2, _Ap>())>
54 [[__gnu__::__always_inline__]]
55 constexpr basic_mask<_Bytes * 2, _Aret>
56 __to_cx_ileav(const basic_mask<_Bytes, _Ap>& __k)
57 {
58 static_assert(_Ap::_S_size % 2 == 0
59 && (__filter_abi_variant(_Ap::_S_variant, _AbiVariant::_CxVariants)
60 == _AbiVariant()));
61 if constexpr (_Aret::_S_size == 1)
62 return basic_mask<_Bytes * 2, _Aret>(__k[0]);
63 else
64 return basic_mask<_Bytes * 2, _Aret>::_S_init(__k);
65 }
66
67 constexpr void
68 __check_hi_bits_for_zero(unsigned_integral auto __x)
69 {
70 __glibcxx_simd_precondition(__x == 0,
71 "to_ullong called on mask with 'true' elements at indices"
72 "higher than 64");
73 }
74
75 template <typename _T0, typename _T1>
76 constexpr void
77 __check_hi_bits_for_zero(const __trivial_pair<_T0, _T1>& __p)
78 {
79 std::simd::__check_hi_bits_for_zero(__p._M_first);
80 std::simd::__check_hi_bits_for_zero(__p._M_second);
81 }
82
83 constexpr unsigned long long
84 __unwrap_pairs_to_ullong(unsigned_integral auto __x)
85 { return __x; }
86
87 template <typename _T0, typename _T1>
88 constexpr unsigned long long
89 __unwrap_pairs_to_ullong(const __trivial_pair<_T0, _T1>& __p)
90 {
91 std::simd::__check_hi_bits_for_zero(__p._M_second);
92 return std::simd::__unwrap_pairs_to_ullong(__p._M_first);
93 }
94
95 template <int _Np>
96 constexpr bitset<_Np>
97 __unwrap_pairs_to_bitset(unsigned_integral auto __x)
98 {
99 static_assert(_Np <= 64);
100 return __x;
101 }
102
103 template <size_t _Np, typename _T0, typename _T1>
104 constexpr bitset<_Np>
105 __unwrap_pairs_to_bitset(const __trivial_pair<_T0, _T1>& __p)
106 {
107 constexpr size_t _N0 = __bit_floor(_Np);
108 constexpr size_t _N1 = _Np - _N0;
109 static_assert(_N0 % 64 == 0);
110 struct _Tmp
111 {
112 bitset<__bit_floor(_Np)> _M_lo;
113 bitset<_Np - __bit_floor(_Np)> _M_hi;
114 };
115 _Tmp __tmp = {std::simd::__unwrap_pairs_to_bitset<_N0>(__p._M_first),
116 std::simd::__unwrap_pairs_to_bitset<_N1>(__p._M_second)};
117 return __builtin_bit_cast(bitset<_Np>, __tmp);
118 }
119
120 template <size_t _Bytes>
121 consteval auto
122 __tree_of_ulong()
123 {
124 static constexpr size_t _N0 = __bit_floor(_Bytes - 1);
125 static constexpr size_t _N1 = _Bytes - _N0;
126 if constexpr (_Bytes <= sizeof(unsigned long))
127 return 0ul;
128 else
129 return __trivial_pair {__tree_of_ulong<_N0>(), __tree_of_ulong<_N1>()};
130 }
131
132 template <size_t _Bytes>
133 using __tree_of_ulong_t = decltype(__tree_of_ulong<_Bytes>());
134
135 template <size_t _Np>
136 constexpr auto
137 __bitset_to_pairs(const bitset<_Np>& __b) noexcept
138 {
139 if constexpr (_Np <= 64)
140 return __b.to_ullong();
141 else
142 return __builtin_bit_cast(__tree_of_ulong_t<__div_ceil(_Np, size_t(__CHAR_BIT__))>, __b);
143 }
144
145 // complex interleaved (_CxIleav) -------------------------------------------
146
147 /** @internal
148 * @brief Functions acting on / recursing into the non-complex fp vec objects, interpreting even
149 * elements as real and odd elements as imaginary.
150 */
151 namespace __cxileav
152 {
153 /** @internal
154 * @brief Set even (real) elements in @p __x to the values in @p __re.
155 */
156 template <typename _Tp, typename _Ap>
157 [[__gnu__::__always_inline__]]
158 constexpr void
159 __set_real(basic_vec<_Tp, _Ap>& __x,
160 const __similar_vec<_Tp, _Ap::_S_size / 2, _Ap>& __re) noexcept
161 {
162 if constexpr (__scalar_abi_tag<_Ap> && _Ap::_S_size == 2)
163 __x._M_get_low() = __re;
164 else if constexpr (_Ap::_S_nreg >= 2)
165 { // recurse
166 constexpr int __n0 = __x._M_get_low().size();
167 const auto& [__lo, __hi] = __re.template _M_chunk<
168 __similar_vec<_Tp, __n0 / 2, _Ap>>();
169 __set_real(__x._M_get_low(), __lo);
170 __set_real(__x._M_get_high(), __hi);
171 }
172 else
173 {
174 using _DataType = typename _Ap::template _DataType<_Tp>;
175 _DataType& __xv = __x._M_get();
176 const auto __rv = __re._M_get();
177 if constexpr (_Ap::_S_size == 2)
178 __vec_set(__xv, 0, __rv);
179 else if (__is_const_known(__x, __re))
180 {
181 constexpr auto [...__is] = _IotaArray<_Ap::_S_size>;
182 __xv = _DataType {((__is & 1) == 0 ? __rv[__is / 2] : __xv[__is])...};
183 }
184 else
185 _VecOps<_DataType>::_S_overwrite_even_elements(__xv, __rv);
186 }
187 }
188
189 /** @internal
190 * @brief Set odd (imaginary) elements in @p __x to the values in @p __im.
191 */
192 template <typename _Tp, typename _Ap>
193 [[__gnu__::__always_inline__]]
194 constexpr void
195 __set_imag(basic_vec<_Tp, _Ap>& __x,
196 const __similar_vec<_Tp, _Ap::_S_size / 2, _Ap>& __im) noexcept
197 {
198 if constexpr (__scalar_abi_tag<_Ap> && _Ap::_S_size == 2)
199 __x._M_get_high() = __im;
200 else if constexpr (_Ap::_S_nreg >= 2)
201 { // recurse
202 constexpr int __n0 = __x._M_get_low().size();
203 const auto& [__lo, __hi] = __im.template _M_chunk<
204 __similar_vec<_Tp, __n0 / 2, _Ap>>();
205 __set_imag(__x._M_get_low(), __lo);
206 __set_imag(__x._M_get_high(), __hi);
207 }
208 else
209 {
210 using _DataType = typename _Ap::template _DataType<_Tp>;
211 _DataType& __xv = __x._M_get();
212 const auto __iv = __im._M_get();
213 if constexpr (_Ap::_S_size == 2)
214 __vec_set(__xv, 1, __iv);
215 else if (__is_const_known(__x, __im))
216 {
217 constexpr auto [...__is] = _IotaArray<_Ap::_S_size>;
218 __xv = _DataType {((__is & 1) == 1 ? __iv[__is / 2] : __xv[__is])...};
219 }
220 else
221 _VecOps<_DataType>::_S_overwrite_odd_elements(__xv, __iv);
222 }
223 }
224
225 /** @internal
226 * @brief Return @p __x after flipping the sign of odd (imaginary) elements.
227 */
228 template <typename _Tp, typename _Ap>
229 [[__gnu__::__always_inline__]]
230 constexpr basic_vec<_Tp, _Ap>
231 __negate_imag(const basic_vec<_Tp, _Ap>& __x)
232 {
233 if constexpr (__scalar_abi_tag<_Ap> && _Ap::_S_size == 2)
234 return basic_vec<_Tp, _Ap>::_S_init(__x._M_get_low(), -__x._M_get_high());
235 else if constexpr (_Ap::_S_nreg >= 2) // recurse
236 return basic_vec<_Tp, _Ap>::_S_init(__negate_imag(__x._M_get_low()),
237 __negate_imag(__x._M_get_high()));
238 else
239 return _VecOps<typename _Ap::template _DataType<_Tp>>
240 ::_S_complex_negate_imag(__x._M_get());
241 }
242
243 /** @internal
244 * @brief Recompute all complex multiplications where @p __nan is true using @p _Cx's
245 * multiplication operator.
246 *
247 * @todo use coarser _TargetTraits and move into .so
248 */
249 template <typename _Cx, _TargetTraits, __vec_builtin _TV>
250 [[__gnu__::__cold__]]
251 constexpr _TV
252 __redo_mul(_TV __r, const _TV __x, const _TV __y, const auto __nan, const int __n)
253 {
254 // redo multiplication using scalar complex-mul on (NaN, NaN) results
255 for (int __i = 0; __i < __n; __i += 2)
256 {
257 if (__nan[__i] && __nan[__i + 1])
258 {
259 using _Tc = typename _Cx::value_type;
260 const _Cx __cx(_Tc(__x[__i]), _Tc(__x[__i + 1]));
261 const _Cx __cy(_Tc(__y[__i]), _Tc(__y[__i + 1]));
262 const _Cx __cr = __cx * __cy;
263 __vec_set(__r, __i, __cr.real());
264 __vec_set(__r, __i + 1, __cr.imag());
265 }
266 }
267 return __r;
268 }
269
270 /** @internal
271 * @brief Complex multiplication of @p __x and @p __y, returning the result in @p __x.
272 */
273 template <typename _Cx, _TargetTraits _Traits, typename _Tp, typename _Ap>
274 [[__gnu__::__always_inline__]]
275 constexpr void
276 __mul(basic_vec<_Tp, _Ap>& __x, const basic_vec<_Tp, _Ap>& __y)
277 {
278 static_assert(__complex_like<_Cx>);
279 if constexpr (__scalar_abi_tag<_Ap> && _Ap::_S_size == 2)
280 {
281 const _Cx __c = _Cx(__x[0], __x[1]) * _Cx(__y[0], __y[1]);
282 __x._M_get_low() = __c.real();
283 __x._M_get_high() = __c.imag();
284 }
285 else if constexpr (_Ap::_S_nreg >= 2)
286 { // recurse
287 __mul<_Cx, _Traits>(__x._M_get_low(), __y._M_get_low());
288 __mul<_Cx, _Traits>(__x._M_get_high(), __y._M_get_high());
289 }
290 else if constexpr (_Traits.template _M_eval_as_f32<_Tp>())
291 { // eval float16_t as float
292 using _Vf32 = rebind_t<float, basic_vec<_Tp, _Ap>>;
293 _Vf32 __xf32(__x);
294 __mul<_Cx, _Traits>(__xf32, _Vf32(__y));
295 __x = static_cast<basic_vec<_Tp, _Ap>>(__xf32);
296 }
297 else
298 {
299 using _DataType = typename _Ap::template _DataType<_Tp>;
300 const _DataType __xv = __x._M_get();
301 const _DataType __yv = __y._M_get();
302 using _VO = _VecOps<_DataType>; // don't care for actual numer of elements
303 using _VOS = _VecOps<_DataType, _Ap::_S_size>; // to check for const-prop values
304 if (_VOS::_S_complex_imag_is_const_known_zero(__xv))
305 {
306 if (_VOS::_S_complex_imag_is_const_known_zero(__yv))
307 __x = __xv * __yv;
308 else
309 {
310 if (_Traits._M_conforming_to_STDC_annex_G())
311 { // handle negative zero (0 * y can be -0)
312 auto __a = _VO::_S_dup_even(__xv) * __yv;
313 auto __b = _DataType() * _VO::_S_swap_neighbors(__yv);
314 __x = _VO::_S_addsub(__a, __b);
315 }
316 else
317 __x = _VO::_S_dup_even(__xv) * __yv;
318 }
319 }
320 else if (_VOS::_S_complex_imag_is_const_known_zero(__yv))
321 {
322 if (_Traits._M_conforming_to_STDC_annex_G())
323 __x = _VO::_S_addsub(_VO::_S_dup_even(__yv) * __xv,
324 _DataType() * _VO::_S_swap_neighbors(__xv));
325 else
326 __x = _VO::_S_dup_even(__yv) * __xv;
327 }
328 else if (_VOS::_S_complex_real_is_const_known_zero(__yv))
329 {
330 if (_Traits._M_conforming_to_STDC_annex_G())
331 __x = _VO::_S_addsub(_DataType(), _VO::_S_dup_odd(__yv)
332 * _VO::_S_swap_neighbors(__xv));
333 else
334 __x = _VO::_S_dup_odd(__yv)
335 * _VO::_S_complex_negate_real(_VO::_S_swap_neighbors(__xv));
336 }
337 else if (_VOS::_S_complex_real_is_const_known_zero(__xv))
338 {
339 if (_Traits._M_conforming_to_STDC_annex_G())
340 __x = _VO::_S_addsub(_DataType(), _VO::_S_dup_odd(__xv)
341 * _VO::_S_swap_neighbors(__yv));
342 else
343 __x = _VO::_S_dup_odd(__xv)
344 * _VO::_S_complex_negate_real(_VO::_S_swap_neighbors(__yv));
345 }
346 else
347 {
348#if _GLIBCXX_X86
349 if (_Traits._M_have_fma() && !__is_const_known(__xv, __yv))
350 {
351 if constexpr (_Traits._M_have_fma())
352 __x = __x86_complex_multiplies(__xv, __yv);
353 }
354 else
355#endif
356 __x = _VO::_S_addsub(_VO::_S_dup_even(__xv) * __yv,
357 _VO::_S_dup_odd(__xv) * _VO::_S_swap_neighbors(__yv));
358 const auto __nan = __x._M_isnan();
359 if (_Traits._M_conforming_to_STDC_annex_G() && __nan._M_any_of())
360 __x = __redo_mul<_Cx, _Traits>(__x._M_get(), __xv, __yv, __nan, _Ap::_S_size);
361 }
362 }
363 }
364 }
365
366 template <size_t _Bytes, __abi_tag _Ap>
367 requires _Ap::_S_is_cx_ileav && (_Ap::_S_size >= 2) // size 1 is in simd_mask.h
368 class basic_mask<_Bytes, _Ap>
369 : public _MaskBase<_Bytes, _Ap>
370 {
371 using _Base = _MaskBase<_Bytes, _Ap>;
372
373 using _VecType = _Base::_VecType;
374
375 template <size_t, typename>
376 friend class basic_mask;
377
378 template <typename, typename>
379 friend class basic_vec;
380
381 static constexpr int _S_size = _Ap::_S_size;
382
383 using _DataType = __component_mask_for_ileav<_Bytes, _Ap>;
384
385 static constexpr bool _S_is_scalar = _DataType::_S_is_scalar;
386
387 static constexpr bool _S_use_bitmask = _DataType::_S_use_bitmask;
388
389 static constexpr int _S_full_size = _DataType::_S_full_size / 2;
390
391 static constexpr bool _S_is_partial = _DataType::_S_is_partial;
392
393 static constexpr bool _S_has_bool_member = _DataType::_S_has_bool_member;
394
395 static constexpr size_t _S_padding_bytes = _DataType::_S_padding_bytes;
396
397 _DataType _M_data;
398
399 public:
400 using value_type = bool;
401
402 using abi_type = _Ap;
403
404 using iterator = _Base::iterator;
405
406 using const_iterator = _Base::const_iterator;
407
408 // internal but public API ----------------------------------------------
409 [[__gnu__::__always_inline__]]
410 static constexpr basic_mask
411 _S_init(const _DataType& __x)
412 {
413 basic_mask __r;
414 __r._M_data = __x;
415 return __r;
416 }
417
418 [[__gnu__::__always_inline__]]
419 constexpr auto
420 _M_concat_data() const
421 { return _M_data._M_concat_data(); }
422
423 [[__gnu__::__always_inline__]]
424 constexpr const _DataType&
425 _M_get_ileav_data() const
426 { return _M_data; }
427
428 template <_ArchTraits _Traits = {}>
429 [[__gnu__::__always_inline__]]
430 static constexpr basic_mask
431 _S_partial_mask_of_n(int __n)
432 { return _S_init(_DataType::_S_partial_mask_of_n(__n * 2)); }
433
434 [[__gnu__::__always_inline__]]
435 static constexpr basic_mask
436 _S_and_neighbors(_DataType __k)
437 { return _S_init(__k._M_and_neighbors()); }
438
439 [[__gnu__::__always_inline__]]
440 static constexpr basic_mask
441 _S_or_neighbors(_DataType __k)
442 { return _S_init(__k._M_or_neighbors()); }
443
444 template <typename _Mp>
445 [[__gnu__::__always_inline__]]
446 constexpr auto
447 _M_chunk() const noexcept
448 {
449 if constexpr (_Mp::abi_type::_S_variant != _Ap::_S_variant)
450 {
451 using _M2 = resize_t<_S_size, _Mp>;
452 static_assert(!is_same_v<_M2, basic_mask>);
453 return static_cast<_M2>(*this).template _M_chunk<_Mp>();
454 }
455 else if constexpr (_Mp::_S_size == 1)
456 {
457 constexpr auto [...__is] = _IotaArray<_S_size>;
458 return array{_Mp(_M_data[__is])...};
459 }
460 else // _Mp is the same partial specialization
461 {
462 constexpr int __rem = _S_size % _Mp::_S_size;
463 const auto [...__xs] = _M_data.template _M_chunk<typename _Mp::_DataType>();
464 static_assert(is_same_v<decltype(__to_cx_ileav(__xs...[0])), _Mp>);
465 if constexpr (__rem == 0)
466 return array{__to_cx_ileav(__xs)...};
467 else
468 return tuple(__to_cx_ileav(__xs)...);
469 }
470 }
471
472 [[__gnu__::__always_inline__]]
473 static constexpr const basic_mask&
474 _S_concat(const basic_mask& __x0) noexcept
475 { return __x0; }
476
477 template <typename... _As>
478 requires (sizeof...(_As) > 1)
479 [[__gnu__::__always_inline__]]
480 static constexpr basic_mask
481 _S_concat(const basic_mask<_Bytes, _As>&... __xs) noexcept
482 { return basic_mask::_S_init(_DataType::_S_concat(__xs._M_get_ileav_data()...)); }
483
484 // [simd.mask.overview] default constructor -----------------------------
485 basic_mask() = default;
486
487 // [simd.mask.overview] conversion extensions ---------------------------
488 template <__vec_builtin _TV>
489 [[__gnu__::__always_inline__]]
490 constexpr
491 basic_mask(const _TV& __x) requires convertible_to<_TV, _DataType>
492 : _M_data(__x)
493 {}
494
495 template <__vec_builtin _TV>
496 [[__gnu__::__always_inline__]]
497 constexpr
498 operator _TV() requires convertible_to<_DataType, _TV>
499 { return _M_data; }
500
501 // [simd.mask.ctor] broadcast constructor -------------------------------
502 [[__gnu__::__always_inline__]]
503 constexpr explicit
504 basic_mask(same_as<bool> auto __x) noexcept // LWG 4382.
505 : _M_data(__x)
506 {}
507
508 // [simd.mask.ctor] conversion constructor ------------------------------
509 template <size_t _UBytes, typename _UAbi>
510 requires (_S_size == _UAbi::_S_size)
511 [[__gnu__::__always_inline__]]
512 constexpr explicit(__is_mask_conversion_explicit<_Ap, _UAbi>(_Bytes, _UBytes))
513 basic_mask(const basic_mask<_UBytes, _UAbi>& __x) noexcept
514 : _M_data([&] {
515 using _UV = basic_mask<_UBytes, _UAbi>;
516 if constexpr (_UAbi::_S_is_cx_ileav)
517 // _CxIleav -> _CxIleav => we can simply convert the contained mask
518 return __x._M_data; // calls conversion ctor on _DataType
519
520 // __x is not _CxIleav from here on
521 else if constexpr (_S_use_bitmask || _UV::_S_use_bitmask)
522 return _DataType::_S_init(__duplicate_each_bit<_S_size>(__x._M_to_uint()));
523
524 // vec-mask to vec-mask from here on
525 else if constexpr (_UAbi::_S_is_cx_ctgus)
526 // unwrap _CxCtgus mask and recurse
527 return basic_mask(__x._M_data)._M_data;
528
529 else if constexpr (_UV::_S_is_scalar || _S_is_scalar)
530 // need to duplicate & convert one vector element into two bools
531 return _DataType([&](int __i) { return __x[__i / 2]; }); // TODO: optimize
532
533 else if constexpr (_Bytes == _UBytes)
534 return _DataType::_S_recursive_bit_cast(__x);
535 else if constexpr (_Bytes <= sizeof(0ll))
536 {
537 using _U2 = __similar_mask<__integer_from<_Bytes>, _S_size, _UAbi>;
538 return _DataType::_S_recursive_bit_cast(_U2(__x));
539 }
540 else if constexpr (_UBytes > 1)
541 {
542 using _U2 = __similar_mask<__integer_from<_UBytes / 2>, _S_size * 2, _UAbi>;
543 return _U2::_S_recursive_bit_cast(__x); // calls conversion ctor on _DataType
544 }
545 else // _Bytes == 16 && _UBytes == 1
546 // convert twice (1 -> 2 -> 16)
547 // The conversion to short keeps the intermediate mask as small as possible and thus
548 // requires fewer across-128bit boundary shuffles.
549 return basic_mask(__similar_mask<short, _UV::_S_size, _UAbi>(__x))._M_data;
550 }())
551 {}
552
553 using _Base::_MaskBase;
554
555 // [simd.mask.ctor] generator constructor -------------------------------
556 template <__simd_generator_invokable<bool, _S_size> _Fp>
557 [[__gnu__::__always_inline__]]
558 constexpr explicit
559 basic_mask(_Fp&& __gen)
560 : _M_data([&] [[__gnu__::__always_inline__]] {
561 // for _CxIleav, the results of each __gen call need to initialize two
562 // neighboring elements
563 constexpr auto [...__is] = _IotaArray<_S_size>;
564 bool __tmp[_S_size] = {__gen(__simd_size_c<__is>)...};
565 return _DataType([&] [[__gnu__::__always_inline__]] (size_t __i) {
566 return __tmp[__i / 2];
567 });
568 }())
569 {}
570
571 // [simd.mask.ctor] bitset constructor ----------------------------------
572 [[__gnu__::__always_inline__]]
573 constexpr
574 basic_mask(const same_as<bitset<_S_size>> auto& __b) noexcept // LWG 4382.
575 : _M_data(_DataType::_S_init(__duplicate_each_bit<_S_size>(
576 std::simd::__bitset_to_pairs(__b))))
577 {}
578
579 // [simd.mask.ctor] uint constructor ------------------------------------
580 template <unsigned_integral _Tp>
581 requires (!same_as<_Tp, bool>) // LWG 4382.
582 [[__gnu__::__always_inline__]]
583 constexpr explicit
584 basic_mask(_Tp __val) noexcept
585 : _M_data(__duplicate_each_bit<_S_size>(__val))
586 {}
587
588 // [simd.mask.subscr] ---------------------------------------------------
589 [[__gnu__::__always_inline__]]
590 constexpr value_type
591 operator[](__simd_size_type __i) const
592 { return _M_data[__i * 2]; }
593
594 // [simd.mask.unary] ----------------------------------------------------
595 [[__gnu__::__always_inline__]]
596 constexpr basic_mask
597 operator!() const noexcept
598 { return _S_init(!_M_data); }
599
600 [[__gnu__::__always_inline__]]
601 constexpr _VecType
602 operator+() const noexcept requires destructible<_VecType>
603 { return operator _VecType(); }
604
605 constexpr _VecType
606 operator+() const noexcept = delete;
607
608 [[__gnu__::__always_inline__]]
609 constexpr _VecType
610 operator-() const noexcept requires destructible<_VecType>
611 {
612 using _Ip = typename _VecType::value_type;
613 if constexpr (_S_use_bitmask)
614 return __select_impl(*this, _Ip(-1), _Ip());
615 else
616 return __builtin_bit_cast(_VecType, -_M_data);
617 }
618
619 constexpr _VecType
620 operator-() const noexcept = delete;
621
622 [[__gnu__::__always_inline__]]
623 constexpr _VecType
624 operator~() const noexcept requires destructible<_VecType>
625 {
626 using _Ip = typename _VecType::value_type;
627 if constexpr (_S_use_bitmask)
628 return __select_impl(*this, _Ip(-2), _Ip(-1));
629 else
630 return __builtin_bit_cast(_VecType, _M_data) - _Ip(1);
631 }
632
633 constexpr _VecType
634 operator~() const noexcept = delete;
635
636 // [simd.mask.conv] -----------------------------------------------------
637 template <typename _Up, typename _UAbi>
638 requires (_UAbi::_S_size == _S_size)
639 [[__gnu__::__always_inline__]]
640 constexpr explicit(sizeof(_Up) != _Bytes)
641 operator basic_vec<_Up, _UAbi>() const noexcept
642 {
643 using _Mp = typename basic_vec<_Up, _UAbi>::mask_type;
644 return __select_impl(_Mp(*this), basic_vec<_Up, _UAbi>(1), basic_vec<_Up, _UAbi>(0));
645 }
646
647 using _Base::operator basic_vec;
648
649 // [simd.mask.namedconv] ------------------------------------------------
650 [[__gnu__::__always_inline__]]
651 constexpr bitset<_S_size>
652 to_bitset() const noexcept
653 { return std::simd::__unwrap_pairs_to_bitset<_S_size>(_M_to_uint()); }
654
655 template <int _Offset = 0, _ArchTraits _Traits = {}>
656 [[__gnu__::__always_inline__]]
657 constexpr auto
658 _M_to_uint() const
659 { return _M_data.template _M_to_uint<_Offset, true>(); }
660
661 [[__gnu__::__always_inline__]]
662 constexpr unsigned long long
663 to_ullong() const
664 { return std::simd::__unwrap_pairs_to_ullong(_M_to_uint()); }
665
666 // [simd.mask.binary] ---------------------------------------------------
667 [[__gnu__::__always_inline__]]
668 friend constexpr basic_mask
669 operator&&(const basic_mask& __x, const basic_mask& __y) noexcept
670 { return _S_init(__x._M_data & __y._M_data); }
671
672 [[__gnu__::__always_inline__]]
673 friend constexpr basic_mask
674 operator||(const basic_mask& __x, const basic_mask& __y) noexcept
675 { return _S_init(__x._M_data | __y._M_data); }
676
677 [[__gnu__::__always_inline__]]
678 friend constexpr basic_mask
679 operator&(const basic_mask& __x, const basic_mask& __y) noexcept
680 { return _S_init(__x._M_data & __y._M_data); }
681
682 [[__gnu__::__always_inline__]]
683 friend constexpr basic_mask
684 operator|(const basic_mask& __x, const basic_mask& __y) noexcept
685 { return _S_init(__x._M_data | __y._M_data); }
686
687 [[__gnu__::__always_inline__]]
688 friend constexpr basic_mask
689 operator^(const basic_mask& __x, const basic_mask& __y) noexcept
690 { return _S_init(__x._M_data ^ __y._M_data); }
691
692 // [simd.mask.cassign] --------------------------------------------------
693 [[__gnu__::__always_inline__]]
694 friend constexpr basic_mask&
695 operator&=(basic_mask& __x, const basic_mask& __y) noexcept
696 {
697 __x._M_data &= __y._M_data;
698 return __x;
699 }
700
701 [[__gnu__::__always_inline__]]
702 friend constexpr basic_mask&
703 operator|=(basic_mask& __x, const basic_mask& __y) noexcept
704 {
705 __x._M_data |= __y._M_data;
706 return __x;
707 }
708
709 [[__gnu__::__always_inline__]]
710 friend constexpr basic_mask&
711 operator^=(basic_mask& __x, const basic_mask& __y) noexcept
712 {
713 __x._M_data ^= __y._M_data;
714 return __x;
715 }
716
717 // [simd.mask.comparison] -----------------------------------------------
718 [[__gnu__::__always_inline__]]
719 friend constexpr basic_mask
720 operator==(const basic_mask& __x, const basic_mask& __y) noexcept
721 { return _S_init(__x._M_data == __y._M_data); }
722
723 [[__gnu__::__always_inline__]]
724 friend constexpr basic_mask
725 operator!=(const basic_mask& __x, const basic_mask& __y) noexcept
726 { return _S_init(__x._M_data != __y._M_data); }
727
728 [[__gnu__::__always_inline__]]
729 friend constexpr basic_mask
730 operator>=(const basic_mask& __x, const basic_mask& __y) noexcept
731 { return _S_init(__x._M_data >= __y._M_data); }
732
733 [[__gnu__::__always_inline__]]
734 friend constexpr basic_mask
735 operator<=(const basic_mask& __x, const basic_mask& __y) noexcept
736 { return _S_init(__x._M_data <= __y._M_data); }
737
738 [[__gnu__::__always_inline__]]
739 friend constexpr basic_mask
740 operator>(const basic_mask& __x, const basic_mask& __y) noexcept
741 { return _S_init(__x._M_data > __y._M_data); }
742
743 [[__gnu__::__always_inline__]]
744 friend constexpr basic_mask
745 operator<(const basic_mask& __x, const basic_mask& __y) noexcept
746 { return _S_init(__x._M_data < __y._M_data); }
747
748 // [simd.mask.cond] -----------------------------------------------------
749 [[__gnu__::__always_inline__]]
750 friend constexpr basic_mask
751 __select_impl(const basic_mask& __k, const basic_mask& __t, const basic_mask& __f) noexcept
752 { return _S_init(__select_impl(__k._M_data, __t._M_data, __f._M_data)); }
753
754 [[__gnu__::__always_inline__]]
755 friend constexpr basic_mask
756 __select_impl(const basic_mask& __k, same_as<bool> auto __t, same_as<bool> auto __f) noexcept
757 { return _S_init(__select_impl(__k._M_data, __t, __f)); }
758
759 template <__vectorizable _T0, same_as<_T0> _T1>
760 requires (sizeof(_T0) == _Bytes)
761 [[__gnu__::__always_inline__]]
762 friend constexpr vec<_T0, _S_size>
763 __select_impl(const basic_mask& __k, const _T0& __t, const _T1& __f) noexcept
764 {
765 using _Vp = vec<_T0, _S_size>;
766 return __select_impl(static_cast<typename _Vp::mask_type>(__k), _Vp(__t), _Vp(__f));
767 }
768
769 // [simd.mask.reductions] implementation --------------------------------
770 [[__gnu__::__always_inline__]]
771 constexpr bool
772 _M_all_of() const noexcept
773 { return _M_data._M_all_of(); }
774
775 [[__gnu__::__always_inline__]]
776 constexpr bool
777 _M_any_of() const noexcept
778 { return _M_data._M_any_of(); }
779
780 [[__gnu__::__always_inline__]]
781 constexpr bool
782 _M_none_of() const noexcept
783 { return _M_data._M_none_of(); }
784
785 [[__gnu__::__always_inline__]]
786 constexpr __simd_size_type
787 _M_reduce_count() const noexcept
788 { return _M_data._M_reduce_count() / 2; }
789
790 [[__gnu__::__always_inline__]]
791 constexpr __simd_size_type
792 _M_reduce_min_index() const
793 { return _M_data._M_reduce_min_index() / 2; }
794
795 [[__gnu__::__always_inline__]]
796 constexpr __simd_size_type
797 _M_reduce_max_index() const
798 { return _M_data._M_reduce_max_index() / 2; }
799
800 [[__gnu__::__always_inline__]]
801 friend constexpr bool
802 __is_const_known(const basic_mask& __x)
803 { return __is_const_known(__x._M_data); }
804 };
805
806 template <__vectorizable _Tp, __abi_tag _Ap>
807 requires __complex_like<_Tp> && _Ap::_S_is_cx_ileav && (_Ap::_S_size >= 2) // size 1 is below
808 class basic_vec<_Tp, _Ap>
809 : public _VecBase<_Tp, _Ap>
810 {
811 template <typename, typename>
812 friend class basic_vec;
813
814 static constexpr int _S_size = _Ap::_S_size;
815
816 static constexpr int _S_full_size = __bit_ceil(unsigned(_S_size));
817
818 using _T0 = typename _Tp::value_type;
819
820 using _TSimd = __similar_vec<_T0, 2 * _S_size, _Ap>;
821
822 using _RealSimd = __similar_vec<_T0, _S_size, _Ap>;
823
824 _TSimd _M_data = {};
825
826 static constexpr bool _S_use_bitmask = _TSimd::_S_use_bitmask;
827
828 static constexpr bool _S_is_partial = sizeof(_M_data) > sizeof(_Tp) * _S_size;
829
830 [[__gnu__::__always_inline__]]
831 static constexpr basic_vec
832 _S_init(const _TSimd& __x)
833 {
834 basic_vec __r;
835 __r._M_data = __x;
836 return __r;
837 }
838
839 public:
840 using value_type = _Tp;
841
842 using mask_type = _VecBase<_Tp, _Ap>::mask_type;
843
844 // internal but public API ----------------------------------------------
845 [[__gnu__::__always_inline__]]
846 constexpr const _TSimd&
847 _M_get_ileav_data() const
848 { return _M_data; }
849
850 [[__gnu__::__always_inline__]]
851 constexpr const auto&
852 _M_get_low() const requires (_Ap::_S_nreg >= 2)
853 { return _M_data._M_get_low(); }
854
855 [[__gnu__::__always_inline__]]
856 constexpr const auto&
857 _M_get_high() const requires (_Ap::_S_nreg >= 2)
858 { return _M_data._M_get_high(); }
859
860 [[__gnu__::__always_inline__]]
861 friend constexpr bool
862 __is_const_known(const basic_vec& __x)
863 { return __is_const_known(__x._M_data); }
864
865 template <typename _Vp>
866 [[__gnu__::__always_inline__]]
867 constexpr auto
868 _M_chunk() const noexcept
869 {
870 if constexpr (_Vp::abi_type::_S_is_cx_ileav)
871 {
872 constexpr int __n = _S_size / _Vp::_S_size;
873 constexpr int __rem = _S_size % _Vp::_S_size;
874 const auto __chunked = _M_data.template _M_chunk<resize_t<_Vp::_S_size * 2,
875 _TSimd>>();
876 constexpr auto [...__is] = _IotaArray<__n>;
877 if constexpr (__rem == 0)
878 return array<_Vp, __n> {_Vp::_S_init(__chunked[__is])...};
879 else
880 {
881 using _Rest = resize_t<__rem, _Vp>;
882 return tuple(_Vp::_S_init(get<__is>(__chunked))...,
883 _Rest::_S_init(get<__n>(__chunked)));
884 }
885 }
886 else
887 return resize_t<_S_size, _Vp>(*this).template _M_chunk<_Vp>();
888 }
889
890 [[__gnu__::__always_inline__]]
891 static constexpr const basic_vec&
892 _S_concat(const basic_vec& __x0) noexcept
893 { return __x0; }
894
895 template <typename... _As>
896 requires (sizeof...(_As) > 1)
897 [[__gnu__::__always_inline__]]
898 static constexpr basic_vec
899 _S_concat(const basic_vec<value_type, _As>&... __xs) noexcept
900 { return basic_vec::_S_init(_TSimd::_S_concat(__xs._M_get_ileav_data()...)); }
901
902 template <typename _BinaryOp>
903 [[__gnu__::__always_inline__]]
904 constexpr auto
905 _M_reduce_to_register(_BinaryOp __binary_op) const
906 {
907 if constexpr (_TSimd::abi_type::_S_nreg == 1)
908 return *this;
909 else
910 {
911 auto [__lo, __hi] = _M_chunk<resize_t<__bit_ceil(unsigned(_S_size)) / 2,
912 basic_vec>>();
913 auto __a = __lo._M_reduce_to_register(__binary_op);
914 auto __b = __hi._M_reduce_to_register(__binary_op);
915 if constexpr (__a._S_size == __b._S_size)
916 return __binary_op(__a, __b);
917 else
918 {
919 using _V1 = resize_t<1, basic_vec>;
920 return __binary_op(_V1(__a._M_reduce(__binary_op)),
921 _V1(__b._M_reduce(__binary_op)));
922 }
923 }
924 }
925
926 template <typename _BinaryOp, _ArchTraits _Traits = {}>
927 [[__gnu__::__always_inline__]]
928 constexpr value_type
929 _M_reduce(_BinaryOp __binary_op) const
930 {
931 if constexpr (_S_size == 1)
932 return operator[](0);
933 else if constexpr (_Traits.template _M_eval_as_f32<_T0>())
934 return value_type(rebind_t<complex<float>, basic_vec>(*this)._M_reduce(__binary_op));
935 else if constexpr (_TSimd::abi_type::_S_nreg >= 2)
936 return _M_reduce_to_register(__binary_op)._M_reduce(__binary_op);
937 else if constexpr (__has_single_bit(unsigned(_S_size)))
938 {
939 const auto [__a, __b] = _M_chunk<resize_t<_S_size / 2, basic_vec>>();
940 return __binary_op(__a, __b)._M_reduce(__binary_op);
941 }
942 else
943 {
944 const auto [__a, __b, __c, ...__rest]
945 = _M_chunk<resize_t<__bit_floor(unsigned(_S_size)) / 2, basic_vec>>();
946 const auto __ab = __binary_op(__a, __b);
947 static_assert(sizeof...(__rest) <= 1);
948 if constexpr (__a._S_size != __c._S_size)
949 return cat(__ab, __c)._M_reduce(__binary_op);
950 else
951 return cat(__binary_op(__ab, __c), __rest...)._M_reduce(__binary_op);
952 }
953 }
954
955 template <typename _Up>
956 [[__gnu__::__always_inline__]]
957 static inline basic_vec
958 _S_partial_load(const _Up* __mem, size_t __n)
959 {
960 if constexpr (__complex_like<_Up>)
961 return _S_init(_TSimd::_S_partial_load(
962 reinterpret_cast<const typename _Up::value_type*>(__mem), __n * 2));
963 else
964 return basic_vec(_RealSimd::_S_partial_load(__mem, __n));
965 }
966
967 template <typename _Up, _ArchTraits _Traits = {}>
968 static inline basic_vec
969 _S_masked_load(const _Up* __mem, mask_type __k)
970 {
971 if constexpr (__complex_like<_Up>)
972 return _S_init(_TSimd::_S_masked_load(
973 reinterpret_cast<const typename _Up::value_type*>(__mem),
974 __k._M_data));
975 else
976 return basic_vec(_RealSimd::_S_masked_load(__mem, typename _RealSimd::mask_type(__k)));
977 }
978
979 template <typename _Up>
980 [[__gnu__::__always_inline__]]
981 inline void
982 _M_store(_Up* __mem) const
983 {
984 static_assert(__complex_like<_Up>);
985 _M_data._M_store(reinterpret_cast<typename _Up::value_type*>(__mem));
986 }
987
988 template <typename _Up>
989 [[__gnu__::__always_inline__]]
990 static inline void
991 _S_partial_store(const basic_vec& __v, _Up* __mem, size_t __n)
992 {
993 static_assert(__complex_like<_Up>);
994 _TSimd::_S_partial_store(__v._M_data, reinterpret_cast<typename _Up::value_type*>(__mem),
995 __n * 2);
996 }
997
998 template <typename _Up>
999 [[__gnu__::__always_inline__]]
1000 static inline void
1001 _S_masked_store(const basic_vec& __v, _Up* __mem, const mask_type& __k)
1002 {
1003 static_assert(__complex_like<_Up>);
1004 _TSimd::_S_masked_store(__v._M_data, reinterpret_cast<typename _Up::value_type*>(__mem),
1005 __k._M_data);
1006 }
1007
1008 basic_vec() = default;
1009
1010 // TODO: conversion extensions
1011
1012 // [simd.ctor] broadcast constructor ------------------------------------
1013 template <__broadcast_constructible<value_type> _Up>
1014 [[__gnu__::__always_inline__]]
1015 constexpr
1016 basic_vec(_Up&& __x) noexcept
1017 : _M_data([&](int __i) {
1018 if constexpr (__complex_like<_Up>)
1019 return (__i & 1) == 0 ? __x.real() : __x.imag();
1020 else
1021 return (__i & 1) == 0 ? __x : _T0();
1022 })
1023 {}
1024
1025 // [simd.ctor] conversion constructor -----------------------------------
1026 template <__complex_like _Up, typename _UAbi>
1027 requires (_S_size == _UAbi::_S_size)
1028 && __explicitly_convertible_to<_Up, value_type>
1029 && _UAbi::_S_is_cx_ileav
1030 [[__gnu__::__always_inline__]]
1031 constexpr
1032 explicit(!convertible_to<_Up, value_type>)
1033 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
1034 : _M_data(__x._M_data)
1035 {}
1036
1037 template <__complex_like _Up, typename _UAbi>
1038 requires (_S_size == _UAbi::_S_size)
1039 && __explicitly_convertible_to<_Up, value_type>
1040 && (!_UAbi::_S_is_cx_ileav)
1041 [[__gnu__::__always_inline__]]
1042 constexpr
1043 explicit(!convertible_to<_Up, value_type>)
1044 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
1045 : basic_vec(static_cast<_RealSimd>(__x._M_real), static_cast<_RealSimd>(__x._M_imag))
1046 {}
1047
1048 template <typename _Up, typename _UAbi>
1049 requires (!__complex_like<_Up>)
1050 && (_S_size == _UAbi::_S_size)
1051 && __explicitly_convertible_to<_Up, value_type>
1052 [[__gnu__::__always_inline__]]
1053 constexpr
1054 explicit(!convertible_to<_Up, value_type>)
1055 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
1056 : basic_vec(_RealSimd(__x))
1057 {}
1058
1059 using _VecBase<_Tp, _Ap>::_VecBase;
1060
1061 // [simd.ctor] generator constructor ------------------------------------
1062 template <__simd_generator_invokable<value_type, _S_size> _Fp>
1063 [[__gnu__::__always_inline__]]
1064 constexpr explicit
1065 basic_vec(_Fp&& __gen)
1066 : _M_data([&] {
1067 using _Arr = std::array<value_type, sizeof(_TSimd) / sizeof(value_type)>;
1068 constexpr auto [...__is] = _IotaArray<_S_size>;
1069 const _Arr __tmp = { static_cast<value_type>(__gen(__simd_size_c<__is>))... };
1070 return __builtin_bit_cast(_TSimd, __tmp);
1071 }())
1072 {}
1073
1074 // [simd.ctor] load constructor -----------------------------------------
1075 template <__complex_like _Up>
1076 [[__gnu__::__always_inline__]]
1077 constexpr
1078 basic_vec(_LoadCtorTag, const _Up* __ptr)
1079 : _M_data([&] {
1080 if consteval
1081 {
1082 return _TSimd([&](int __i) {
1083 const _Up& __cx = __ptr[__i / 2];
1084 return static_cast<_T0>(__i % 2 == 0 ? __cx.real() : __cx.imag());
1085 });
1086 }
1087 else
1088 {
1089 return _TSimd(_LoadCtorTag(),
1090 reinterpret_cast<const typename _Up::value_type*>(__ptr));
1091 }
1092 }())
1093 {}
1094
1095 template <typename _Up>
1096 [[__gnu__::__always_inline__]]
1097 constexpr
1098 basic_vec(_LoadCtorTag, const _Up* __ptr)
1099 : basic_vec(_RealSimd(_LoadCtorTag(), __ptr))
1100 {}
1101
1102 template <ranges::contiguous_range _Rg, typename... _Flags>
1103 requires ranges::__static_sized_range<_Rg> && (ranges::__static_size<_Rg>() == _S_size)
1104 && __vectorizable<ranges::range_value_t<_Rg>>
1105 && __explicitly_convertible_to<ranges::range_value_t<_Rg>, value_type>
1106 [[__gnu__::__always_inline__]]
1107 constexpr
1108 basic_vec(_Rg&& __range, flags<_Flags...> __flags = {})
1109 : basic_vec(_LoadCtorTag(), __flags.template _S_adjust_pointer<basic_vec>(
1110 ranges::data(__range)))
1111 {
1112 static_assert(__loadstore_convertible_to<ranges::range_value_t<_Rg>, value_type,
1113 _Flags...>);
1114 }
1115
1116 // [simd.ctor] complex init ---------------------------------------------
1117 // This uses _RealSimd as proposed in LWG4230
1118 [[__gnu__::__always_inline__]]
1119 constexpr
1120 basic_vec(const _RealSimd& __re, const _RealSimd& __im = {}) noexcept
1121 {
1122 __cxileav::__set_real(_M_data, __re);
1123 __cxileav::__set_imag(_M_data, __im);
1124 }
1125
1126 // [simd.subscr] --------------------------------------------------------
1127 [[__gnu__::__always_inline__]]
1128 constexpr value_type
1129 operator[](__simd_size_type __i) const
1130 { return value_type(_M_data[__i * 2], _M_data[__i * 2 + 1]); }
1131
1132 // [simd.unary] unary operators -----------------------------------------
1133 [[__gnu__::__always_inline__]]
1134 constexpr basic_vec&
1135 operator++() noexcept requires requires(value_type __a) { ++__a; }
1136 {
1137 _M_data += value_type(_T0(1));
1138 return *this;
1139 }
1140
1141 [[__gnu__::__always_inline__]]
1142 constexpr basic_vec
1143 operator++(int) noexcept requires requires(value_type __a) { __a++; }
1144 {
1145 basic_vec __r = *this;
1146 _M_data += value_type(_T0(1));
1147 return __r;
1148 }
1149
1150 [[__gnu__::__always_inline__]]
1151 constexpr basic_vec&
1152 operator--() noexcept requires requires(value_type __a) { --__a; }
1153 {
1154 _M_data -= value_type(_T0(1));
1155 return *this;
1156 }
1157
1158 [[__gnu__::__always_inline__]]
1159 constexpr basic_vec
1160 operator--(int) noexcept requires requires(value_type __a) { __a--; }
1161 {
1162 basic_vec __r = *this;
1163 _M_data -= value_type(_T0(1));
1164 return __r;
1165 }
1166
1167 [[__gnu__::__always_inline__]]
1168 constexpr mask_type
1169 operator!() const noexcept requires requires(value_type __a) { !__a; }
1170 { return _S_init(!_M_data); }
1171
1172 [[__gnu__::__always_inline__]]
1173 constexpr basic_vec
1174 operator+() const noexcept requires requires(value_type __a) { +__a; }
1175 { return *this; }
1176
1177 [[__gnu__::__always_inline__]]
1178 constexpr basic_vec
1179 operator-() const noexcept requires requires(value_type __a) { -__a; }
1180 {
1181 basic_vec __r = *this;
1182 __r._M_data = -_M_data;
1183 return __r;
1184 }
1185
1186 // [simd.cassign] compound assignment -----------------------------------
1187 [[__gnu__::__always_inline__]]
1188 friend constexpr basic_vec&
1189 operator+=(basic_vec& __x, const basic_vec& __y) noexcept
1190 requires requires(value_type __a) { __a + __a; }
1191 {
1192 __x._M_data += __y._M_data;
1193 return __x;
1194 }
1195
1196 [[__gnu__::__always_inline__]]
1197 friend constexpr basic_vec&
1198 operator-=(basic_vec& __x, const basic_vec& __y) noexcept
1199 requires requires(value_type __a) { __a - __a; }
1200 {
1201 __x._M_data -= __y._M_data;
1202 return __x;
1203 }
1204
1205 template <_TargetTraits _Traits = {}>
1206 [[__gnu__::__always_inline__]]
1207 friend constexpr basic_vec&
1208 operator*=(basic_vec& __x, const basic_vec& __y) noexcept
1209 requires requires(value_type __a) { __a * __a; }
1210 {
1211 __cxileav::__mul<value_type, _Traits>(__x._M_data, __y._M_data);
1212 return __x;
1213 }
1214
1215 template <int _RemoveMe = 0>
1216 [[__gnu__::__always_inline__]]
1217 friend constexpr basic_vec&
1218 operator/=(basic_vec& __x, const basic_vec& __y) noexcept
1219 requires requires(value_type __a) { __a / __a; }
1220 {
1221 static_assert(false, "TODO");
1222 }
1223
1224 // [simd.comparison] compare operators ----------------------------------
1225 [[__gnu__::__always_inline__]]
1226 friend constexpr mask_type
1227 operator==(const basic_vec& __x, const basic_vec& __y) noexcept
1228 { return mask_type::_S_and_neighbors(__x._M_data == __y._M_data); }
1229
1230 [[__gnu__::__always_inline__]]
1231 friend constexpr mask_type
1232 operator!=(const basic_vec& __x, const basic_vec& __y) noexcept
1233 { return mask_type::_S_or_neighbors(__x._M_data != __y._M_data); }
1234
1235 // [simd.complex.access] complex-value accessors ------------------------
1236 // LWG4230: returns _RealSimd instead of auto
1237 [[__gnu__::__always_inline__]]
1238 constexpr _RealSimd
1239 real() const noexcept
1240 { return permute<_S_size>(_M_data, [](int __i) { return __i * 2; }); }
1241
1242 [[__gnu__::__always_inline__]]
1243 constexpr _RealSimd
1244 imag() const noexcept
1245 { return permute<_S_size>(_M_data, [](int __i) { return __i * 2 + 1; }); }
1246
1247 [[__gnu__::__always_inline__]]
1248 constexpr void
1249 real(const _RealSimd& __x) noexcept
1250 { __cxileav::__set_real(_M_data, __x); }
1251
1252 [[__gnu__::__always_inline__]]
1253 constexpr void
1254 imag(const _RealSimd& __x) noexcept
1255 { __cxileav::__set_imag(_M_data, __x); }
1256
1257 // [simd.cond] ---------------------------------------------------------
1258 [[__gnu__::__always_inline__]]
1259 friend constexpr basic_vec
1260 __select_impl(const mask_type& __k, const basic_vec& __t, const basic_vec& __f) noexcept
1261 { return _S_init(__select_impl(__k._M_data, __t._M_data, __f._M_data)); }
1262
1263 // [simd.complex.math] internals ---------------------------------------
1264 [[__gnu__::__always_inline__]]
1265 constexpr _RealSimd
1266 _M_abs() const; // TODO: depends on [simd.math]
1267
1268 // associated functions
1269 [[__gnu__::__always_inline__]]
1270 constexpr _RealSimd
1271 _M_norm() const
1272 {
1273 auto __re = real();
1274 auto __im = imag();
1275 return __re * __re + __im * __im;
1276 }
1277
1278 [[__gnu__::__always_inline__]]
1279 constexpr basic_vec
1280 _M_conj() const
1281 { return _S_init(__cxileav::__negate_imag(_M_data)); }
1282 };
1283
1284 // complex contiguous (_CxCtgus) --------------------------------------------
1285 // (and _CxIleav basic_vec with size 1)
1286
1287 /** @internal
1288 * @brief Functions acting on / recursing into the non-complex fp vec objects, where real and
1289 * imaginary parts are stored in separate vec objects.
1290 */
1291 namespace __cxctgus
1292 {
1293 /** @internal
1294 * @brief Recompute all complex multiplications where @p __nan is true using @p _Cx's
1295 * multiplication operator.
1296 *
1297 * @todo use coarser _TargetTraits and move into .so
1298 */
1299 template <typename _Cx, _TargetTraits, __vec_builtin _TV, typename _Kp>
1300 [[__gnu__::__cold__, __gnu__::__noinline__]]
1301 constexpr void
1302 __redo_mul(_TV& __re, _TV& __im, const _TV __re0, const _TV __im0,
1303 const _TV __re1, const _TV __im1, const _Kp __nan, int __n)
1304 {
1305 for (int __i = 0; __i < __n; ++__i)
1306 {
1307 bool __isnan;
1308 if constexpr (is_integral_v<_Kp>)
1309 __isnan = (__nan & (_Kp(1) << __i)) != 0;
1310 else
1311 __isnan = __nan[__i] != 0;
1312 if (__isnan)
1313 {
1314 const _Cx __c0(__re0[__i], __im0[__i]);
1315 const _Cx __c1(__re1[__i], __im1[__i]);
1316 const _Cx __cr = __c0 * __c1;
1317 __vec_set(__re, __i, __cr.real());
1318 __vec_set(__im, __i, __cr.imag());
1319 }
1320 }
1321 }
1322
1323 /** @internal
1324 * @brief Complex multiplication of (@p __re0, @p __im0) and (@p __re1, @p __im1), returning the
1325 * result in @p __re0 and @p __im0.
1326 */
1327 template <typename _Cx, _TargetTraits _Traits, typename _Tp, typename _Ap>
1328 [[__gnu__::__always_inline__]]
1329 constexpr void
1330 __mul(basic_vec<_Tp, _Ap>& __re0, basic_vec<_Tp, _Ap>& __im0,
1331 const basic_vec<_Tp, _Ap>& __re1, const basic_vec<_Tp, _Ap>& __im1)
1332 {
1333 static_assert(__complex_like<_Cx>);
1334 if constexpr (_Ap::_S_nreg >= 2)
1335 {
1336 __mul<_Cx, _Traits>(__re0._M_get_low(), __im0._M_get_low(),
1337 __re1._M_get_low(), __im1._M_get_low());
1338 __mul<_Cx, _Traits>(__re0._M_get_high(), __im0._M_get_high(),
1339 __re1._M_get_high(), __im1._M_get_high());
1340 }
1341 else if constexpr (_Ap::_S_size == 1)
1342 { // use _Cx::operator*
1343 const _Cx __c0(__re0._M_get(), __im0._M_get());
1344 const _Cx __c1(__re1._M_get(), __im1._M_get());
1345 const _Cx __cr = __c0 * __c1;
1346 __re0._M_get() = __cr.real();
1347 __im0._M_get() = __cr.imag();
1348 }
1349 else if constexpr (_Traits.template _M_eval_as_f32<_Tp>())
1350 {
1351 using _Vf = rebind_t<float, basic_vec<_Tp, _Ap>>;
1352 using _Cf = complex<float>;
1353 _Vf __re0f = __re0;
1354 _Vf __im0f = __im0;
1355 __mul<_Cf, _Traits, float, typename _Vf::abi_type>(__re0f, __im0f, __re1, __im1);
1356 __re0 = static_cast<basic_vec<_Tp, _Ap>>(__re0f);
1357 __im0 = static_cast<basic_vec<_Tp, _Ap>>(__im0f);
1358 }
1359 else
1360 {
1361 basic_vec<_Tp, _Ap> __re = __re0 * __re1 - __im0 * __im1;
1362 basic_vec<_Tp, _Ap> __im = __re0 * __im1 + __im0 * __re1;
1363 const auto __nan = __re._M_isunordered(__im);
1364 if (__nan._M_any_of()) [[unlikely]]
1365 __redo_mul<_Cx, _Traits>(__re._M_get(), __im._M_get(), __re0._M_get(), __im0._M_get(),
1366 __re1._M_get(), __im1._M_get(),
1367 __nan._M_concat_data(), _Ap::_S_size);
1368 __re0 = __re;
1369 __im0 = __im;
1370 }
1371 }
1372 }
1373
1374 template <size_t _Bytes, __abi_tag _Ap>
1375 requires _Ap::_S_is_cx_ctgus && (_Ap::_S_size >= 2) // size 1 is in simd_mask.h
1376 class basic_mask<_Bytes, _Ap>
1377 : public _MaskBase<_Bytes, _Ap>
1378 {
1379 using _Base = _MaskBase<_Bytes, _Ap>;
1380
1381 using _VecType = _Base::_VecType;
1382
1383 template <size_t, typename>
1384 friend class basic_mask;
1385
1386 template <typename, typename>
1387 friend class basic_vec;
1388
1389 static constexpr int _S_size = _Ap::_S_size;
1390
1391 using _DataType = __component_mask_for_ctgus<_Bytes, _Ap>;
1392
1393 static_assert(_DataType::abi_type::_S_nreg == _Ap::_S_nreg);
1394
1395 static constexpr bool _S_is_scalar = _DataType::_S_is_scalar;
1396
1397 static constexpr bool _S_use_bitmask = _DataType::_S_use_bitmask;
1398
1399 static constexpr int _S_full_size = _DataType::_S_full_size;
1400
1401 static constexpr bool _S_is_partial = _DataType::_S_is_partial;
1402
1403 static constexpr bool _S_has_bool_member = _DataType::_S_has_bool_member;
1404
1405 static constexpr size_t _S_padding_bytes = _DataType::_S_padding_bytes;
1406
1407 _DataType _M_data;
1408
1409 public:
1410 using value_type = bool;
1411
1412 using abi_type = _Ap;
1413
1414 // internal but public API ----------------------------------------------
1415 [[__gnu__::__always_inline__]]
1416 static constexpr basic_mask
1417 _S_init(const _DataType& __x)
1418 {
1419 basic_mask __r;
1420 __r._M_data = __x;
1421 return __r;
1422 }
1423
1424 [[__gnu__::__always_inline__]]
1425 constexpr const _DataType&
1426 _M_get() const
1427 { return _M_data; }
1428
1429 [[__gnu__::__always_inline__]]
1430 constexpr auto
1431 _M_concat_data() const
1432 { return _M_data._M_concat_data(); }
1433
1434 template <_ArchTraits _Traits = {}>
1435 [[__gnu__::__always_inline__]]
1436 static constexpr basic_mask
1437 _S_partial_mask_of_n(int __n)
1438 { return _S_init(_DataType::_S_partial_mask_of_n(__n)); }
1439
1440 template <typename _Mp>
1441 [[__gnu__::__always_inline__]]
1442 constexpr auto
1443 _M_chunk() const noexcept
1444 {
1445 if constexpr (_Mp::abi_type::_S_variant != _Ap::_S_variant)
1446 {
1447 using _M2 = resize_t<_S_size, _Mp>;
1448 static_assert(!is_same_v<_M2, basic_mask>);
1449 return static_cast<_M2>(*this).template _M_chunk<_Mp>();
1450 }
1451 else if constexpr (_Mp::_S_size == 1)
1452 {
1453 constexpr auto [...__is] = _IotaArray<_S_size>;
1454 return array{_Mp(_M_data[__is])...};
1455 }
1456 else // _Mp is the same partial specialization
1457 {
1458 constexpr int __rem = _S_size % _Mp::_S_size;
1459 const auto [...__xs, __last] = _M_data.template _M_chunk<typename _Mp::_DataType>();
1460 if constexpr (__rem == 0)
1461 return array{_Mp::_S_init(__xs)..., _Mp::_S_init(__last)};
1462 else
1463 return tuple(_Mp::_S_init(__xs)..., resize_t<__rem, _Mp>(__last));
1464 }
1465 }
1466
1467 [[__gnu__::__always_inline__]]
1468 static constexpr const basic_mask&
1469 _S_concat(const basic_mask& __x0) noexcept
1470 { return __x0; }
1471
1472 /** @internal
1473 * @brief Adjust the mask type to match _RealSimd.
1474 *
1475 * This is a trivial unwrap for this partial specialization of basic_mask. However, for
1476 * _Abi<1, 1, _CxCtgus> _M_data is the bool object and needs to be converted.
1477 */
1478 [[__gnu__::__always_inline__]]
1479 constexpr const _DataType&
1480 _M_get_ctgus_data() const noexcept
1481 { return _M_data; }
1482
1483 template <typename... _As>
1484 requires (sizeof...(_As) > 1)
1485 [[__gnu__::__always_inline__]]
1486 static constexpr basic_mask
1487 _S_concat(const basic_mask<_Bytes, _As>&... __xs) noexcept
1488 { return basic_mask::_S_init(_DataType::_S_concat(__xs._M_get_ctgus_data()...)); }
1489
1490 // [simd.mask.overview] default constructor -----------------------------
1491 basic_mask() = default;
1492
1493 // [simd.mask.overview] conversion extensions ---------------------------
1494 template <__vec_builtin _TV>
1495 [[__gnu__::__always_inline__]]
1496 constexpr
1497 basic_mask(const _TV& __x) requires convertible_to<_TV, _DataType>
1498 : _M_data(__x)
1499 {}
1500
1501 template <__vec_builtin _TV>
1502 [[__gnu__::__always_inline__]]
1503 constexpr
1504 operator _TV() requires convertible_to<_DataType, _TV>
1505 { return _M_data; }
1506
1507 // [simd.mask.ctor] broadcast constructor -------------------------------
1508 [[__gnu__::__always_inline__]]
1509 constexpr explicit
1510 basic_mask(same_as<bool> auto __x) noexcept // LWG 4382.
1511 : _M_data(__x)
1512 {}
1513
1514 // [simd.mask.ctor] conversion constructor ------------------------------
1515 template <size_t _UBytes, typename _UAbi>
1516 requires (_S_size == _UAbi::_S_size)
1517 [[__gnu__::__always_inline__]]
1518 constexpr explicit(__is_mask_conversion_explicit<_Ap, _UAbi>(_Bytes, _UBytes))
1519 basic_mask(const basic_mask<_UBytes, _UAbi>& __x) noexcept
1520 : _M_data(__x)
1521 {}
1522
1523 using _Base::_MaskBase;
1524
1525 // [simd.mask.ctor] generator constructor -------------------------------
1526 template <__simd_generator_invokable<bool, _S_size> _Fp>
1527 [[__gnu__::__always_inline__]]
1528 constexpr explicit
1529 basic_mask(_Fp&& __gen)
1530 : _M_data(__gen)
1531 {}
1532
1533 // [simd.mask.ctor] bitset constructor ----------------------------------
1534 [[__gnu__::__always_inline__]]
1535 constexpr
1536 basic_mask(const same_as<bitset<_S_size>> auto& __b) noexcept // LWG 4382.
1537 : _M_data(__b)
1538 {}
1539
1540 // [simd.mask.ctor] uint constructor ------------------------------------
1541 template <unsigned_integral _Tp>
1542 requires (!same_as<_Tp, bool>) // LWG 4382.
1543 [[__gnu__::__always_inline__]]
1544 constexpr explicit
1545 basic_mask(_Tp __val) noexcept
1546 : _M_data(__val)
1547 {}
1548
1549 // [simd.mask.subscr] ---------------------------------------------------
1550 [[__gnu__::__always_inline__]]
1551 constexpr value_type
1552 operator[](__simd_size_type __i) const
1553 { return _M_data[__i]; }
1554
1555 // [simd.mask.unary] ----------------------------------------------------
1556 [[__gnu__::__always_inline__]]
1557 constexpr basic_mask
1558 operator!() const noexcept
1559 { return _S_init(!_M_data); }
1560
1561 [[__gnu__::__always_inline__]]
1562 constexpr _VecType
1563 operator+() const noexcept requires destructible<_VecType>
1564 { return static_cast<_VecType>(_M_data); }
1565
1566 constexpr _VecType
1567 operator+() const noexcept = delete;
1568
1569 [[__gnu__::__always_inline__]]
1570 constexpr _VecType
1571 operator-() const noexcept requires destructible<_VecType>
1572 {
1573 using _Ip = typename _VecType::value_type;
1574 if constexpr (_S_use_bitmask)
1575 return __select_impl(*this, _Ip(-1), _Ip());
1576 else
1577 return -_M_data; // sign-extends
1578 }
1579
1580 constexpr _VecType
1581 operator-() const noexcept = delete;
1582
1583 [[__gnu__::__always_inline__]]
1584 constexpr _VecType
1585 operator~() const noexcept requires destructible<_VecType>
1586 {
1587 using _Ip = typename _VecType::value_type;
1588 if constexpr (_S_use_bitmask)
1589 return __select_impl(*this, _Ip(-2), _Ip(-1));
1590 else
1591 return ~_M_data; // sign-extends
1592 }
1593
1594 constexpr _VecType
1595 operator~() const noexcept = delete;
1596
1597 // [simd.mask.conv] -----------------------------------------------------
1598 template <typename _Up, typename _UAbi>
1599 requires (_UAbi::_S_size == _S_size)
1600 [[__gnu__::__always_inline__]]
1601 constexpr explicit(sizeof(_Up) != _Bytes)
1602 operator basic_vec<_Up, _UAbi>() const noexcept
1603 {
1604 using _UV = basic_vec<_Up, _UAbi>;
1605 using _Mp = typename _UV::mask_type;
1606 return __select_impl(static_cast<_Mp>(_M_data), _UV(1), _UV(0));
1607 }
1608
1609 using _Base::operator basic_vec;
1610
1611 // [simd.mask.namedconv] ------------------------------------------------
1612 [[__gnu__::__always_inline__]]
1613 constexpr bitset<_S_size>
1614 to_bitset() const noexcept
1615 { return _M_data.to_bitset(); }
1616
1617 template <int _Offset = 0, _ArchTraits _Traits = {}>
1618 [[__gnu__::__always_inline__]]
1619 constexpr auto
1620 _M_to_uint() const
1621 { return _M_data.template _M_to_uint<_Offset>(); }
1622
1623 [[__gnu__::__always_inline__]]
1624 constexpr unsigned long long
1625 to_ullong() const
1626 { return _M_data.to_ullong(); }
1627
1628 // [simd.mask.binary] ---------------------------------------------------
1629 [[__gnu__::__always_inline__]]
1630 friend constexpr basic_mask
1631 operator&&(const basic_mask& __x, const basic_mask& __y) noexcept
1632 { return _S_init(__x._M_data & __y._M_data); }
1633
1634 [[__gnu__::__always_inline__]]
1635 friend constexpr basic_mask
1636 operator||(const basic_mask& __x, const basic_mask& __y) noexcept
1637 { return _S_init(__x._M_data | __y._M_data); }
1638
1639 [[__gnu__::__always_inline__]]
1640 friend constexpr basic_mask
1641 operator&(const basic_mask& __x, const basic_mask& __y) noexcept
1642 { return _S_init(__x._M_data & __y._M_data); }
1643
1644 [[__gnu__::__always_inline__]]
1645 friend constexpr basic_mask
1646 operator|(const basic_mask& __x, const basic_mask& __y) noexcept
1647 { return _S_init(__x._M_data | __y._M_data); }
1648
1649 [[__gnu__::__always_inline__]]
1650 friend constexpr basic_mask
1651 operator^(const basic_mask& __x, const basic_mask& __y) noexcept
1652 { return _S_init(__x._M_data ^ __y._M_data); }
1653
1654 // [simd.mask.cassign] --------------------------------------------------
1655 [[__gnu__::__always_inline__]]
1656 friend constexpr basic_mask&
1657 operator&=(basic_mask& __x, const basic_mask& __y) noexcept
1658 {
1659 __x._M_data &= __y._M_data;
1660 return __x;
1661 }
1662
1663 [[__gnu__::__always_inline__]]
1664 friend constexpr basic_mask&
1665 operator|=(basic_mask& __x, const basic_mask& __y) noexcept
1666 {
1667 __x._M_data |= __y._M_data;
1668 return __x;
1669 }
1670
1671 [[__gnu__::__always_inline__]]
1672 friend constexpr basic_mask&
1673 operator^=(basic_mask& __x, const basic_mask& __y) noexcept
1674 {
1675 __x._M_data ^= __y._M_data;
1676 return __x;
1677 }
1678
1679 // [simd.mask.comparison] -----------------------------------------------
1680 [[__gnu__::__always_inline__]]
1681 friend constexpr basic_mask
1682 operator==(const basic_mask& __x, const basic_mask& __y) noexcept
1683 { return _S_init(__x._M_data == __y._M_data); }
1684
1685 [[__gnu__::__always_inline__]]
1686 friend constexpr basic_mask
1687 operator!=(const basic_mask& __x, const basic_mask& __y) noexcept
1688 { return _S_init(__x._M_data != __y._M_data); }
1689
1690 [[__gnu__::__always_inline__]]
1691 friend constexpr basic_mask
1692 operator>=(const basic_mask& __x, const basic_mask& __y) noexcept
1693 { return _S_init(__x._M_data >= __y._M_data); }
1694
1695 [[__gnu__::__always_inline__]]
1696 friend constexpr basic_mask
1697 operator<=(const basic_mask& __x, const basic_mask& __y) noexcept
1698 { return _S_init(__x._M_data <= __y._M_data); }
1699
1700 [[__gnu__::__always_inline__]]
1701 friend constexpr basic_mask
1702 operator>(const basic_mask& __x, const basic_mask& __y) noexcept
1703 { return _S_init(__x._M_data > __y._M_data); }
1704
1705 [[__gnu__::__always_inline__]]
1706 friend constexpr basic_mask
1707 operator<(const basic_mask& __x, const basic_mask& __y) noexcept
1708 { return _S_init(__x._M_data < __y._M_data); }
1709
1710 // [simd.mask.cond] -----------------------------------------------------
1711 [[__gnu__::__always_inline__]]
1712 friend constexpr basic_mask
1713 __select_impl(const basic_mask& __k, const basic_mask& __t, const basic_mask& __f) noexcept
1714 { return __select_impl(__k._M_data, __t._M_data, __f._M_data); }
1715
1716 [[__gnu__::__always_inline__]]
1717 friend constexpr basic_mask
1718 __select_impl(const basic_mask& __k, same_as<bool> auto __t, same_as<bool> auto __f) noexcept
1719 { return _S_init(__select_impl(__k._M_data, __t, __f)); }
1720
1721 template <__vectorizable _T0, same_as<_T0> _T1>
1722 requires (sizeof(_T0) == _Bytes)
1723 [[__gnu__::__always_inline__]]
1724 friend constexpr vec<_T0, _S_size>
1725 __select_impl(const basic_mask& __k, const _T0& __t, const _T1& __f) noexcept
1726 {
1727 using _Vp = vec<_T0, _S_size>;
1728 return __select_impl(static_cast<typename _Vp::mask_type>(__k), _Vp(__t), _Vp(__f));
1729 }
1730
1731 // [simd.mask.reductions] implementation --------------------------------
1732 [[__gnu__::__always_inline__]]
1733 constexpr bool
1734 _M_all_of() const noexcept
1735 { return _M_data._M_all_of(); }
1736
1737 [[__gnu__::__always_inline__]]
1738 constexpr bool
1739 _M_any_of() const noexcept
1740 { return _M_data._M_any_of(); }
1741
1742 [[__gnu__::__always_inline__]]
1743 constexpr bool
1744 _M_none_of() const noexcept
1745 { return _M_data._M_none_of(); }
1746
1747 [[__gnu__::__always_inline__]]
1748 constexpr __simd_size_type
1749 _M_reduce_count() const noexcept
1750 { return _M_data._M_reduce_count(); }
1751
1752 [[__gnu__::__always_inline__]]
1753 constexpr __simd_size_type
1754 _M_reduce_min_index() const
1755 { return _M_data._M_reduce_min_index(); }
1756
1757 [[__gnu__::__always_inline__]]
1758 constexpr __simd_size_type
1759 _M_reduce_max_index() const
1760 { return _M_data._M_reduce_max_index(); }
1761
1762 [[__gnu__::__always_inline__]]
1763 friend constexpr bool
1764 __is_const_known(const basic_mask& __x)
1765 { return __is_const_known(__x._M_data); }
1766 };
1767
1768 template <__vectorizable _Tp, __abi_tag _Ap>
1769 requires __complex_like<_Tp> && (_Ap::_S_is_cx_ctgus || _Ap::_S_size == 1)
1770 class basic_vec<_Tp, _Ap>
1771 : public _VecBase<_Tp, _Ap>
1772 {
1773 template <typename, typename>
1774 friend class basic_vec;
1775
1776 static constexpr int _S_size = _Ap::_S_size;
1777
1778 static constexpr int _S_full_size = __bit_ceil(unsigned(_S_size));
1779
1780 using _T0 = typename _Tp::value_type;
1781
1782 using _RealSimd = __similar_vec<_T0, _S_size, _Ap>;
1783
1784 _RealSimd _M_real = {};
1785
1786 _RealSimd _M_imag = {};
1787
1788 static constexpr bool _S_is_scalar = _RealSimd::_S_is_scalar;
1789
1790 static constexpr bool _S_use_bitmask = _RealSimd::_S_use_bitmask;
1791
1792 static constexpr bool _S_is_partial = _RealSimd::_S_is_partial;
1793
1794 public:
1795 using value_type = _Tp;
1796
1797 using mask_type = _VecBase<_Tp, _Ap>::mask_type;
1798
1799 // internal but public API ----------------------------------------------
1800 [[__gnu__::__always_inline__]]
1801 constexpr _RealSimd&
1802 _M_get_real() noexcept
1803 { return _M_real; }
1804
1805 [[__gnu__::__always_inline__]]
1806 constexpr const _RealSimd&
1807 _M_get_real() const noexcept
1808 { return _M_real; }
1809
1810 [[__gnu__::__always_inline__]]
1811 constexpr _RealSimd&
1812 _M_get_imag() noexcept
1813 { return _M_imag; }
1814
1815 [[__gnu__::__always_inline__]]
1816 constexpr const _RealSimd&
1817 _M_get_imag() const noexcept
1818 { return _M_imag; }
1819
1820 [[__gnu__::__always_inline__]]
1821 constexpr auto
1822 _M_get_low() const requires (_Ap::_S_nreg >= 2)
1823 {
1824 return resize_t<_M_real._N0, basic_vec>(
1825 _M_real._M_get_low(), _M_imag._M_get_low());
1826 }
1827
1828 [[__gnu__::__always_inline__]]
1829 constexpr auto
1830 _M_get_high() const requires (_Ap::_S_nreg >= 2)
1831 {
1832 return resize_t<_M_real._N1, basic_vec>(
1833 _M_real._M_get_high(), _M_imag._M_get_high());
1834 }
1835
1836 [[__gnu__::__always_inline__]]
1837 constexpr auto
1838 _M_concat_data(bool /*do_sanitize*/ = false) const
1839 requires (_S_size == 1) // only for _CxCtgus of size 1
1840 {
1841 return __vec_builtin_type<__canonical_vec_type_t<_T0>, 2>{
1842 _M_real._M_data, _M_imag._M_data
1843 };
1844 }
1845
1846 [[__gnu__::__always_inline__]]
1847 constexpr auto
1848 _M_get_ileav_data() const
1849 requires (_S_size == 1 && _Ap::_S_is_cx_ileav)
1850 { return __builtin_bit_cast(__similar_vec<_T0, 2, _Ap>, *this); }
1851
1852 [[__gnu__::__always_inline__]]
1853 static constexpr basic_vec
1854 _S_init(const __similar_vec<_T0, 2, _Ap>& __x)
1855 requires (_S_size == 1 && _Ap::_S_is_cx_ileav)
1856 { return __builtin_bit_cast(basic_vec, __x); }
1857
1858 [[__gnu__::__always_inline__]]
1859 friend constexpr bool
1860 __is_const_known(const basic_vec& __x)
1861 { return __is_const_known(__x._M_real) && __is_const_known(__x._M_imag); }
1862
1863 template <typename _Vp>
1864 [[__gnu__::__always_inline__]]
1865 constexpr auto
1866 _M_chunk() const noexcept
1867 {
1868 constexpr int __n = _S_size / _Vp::_S_size;
1869 constexpr int __rem = _S_size % _Vp::_S_size;
1870 const auto [...__rs, __rN] = _M_real.template _M_chunk<typename _Vp::_RealSimd>();
1871 const auto [...__is, __iN] = _M_imag.template _M_chunk<typename _Vp::_RealSimd>();
1872 if constexpr (__rem == 0)
1873 return array<_Vp, __n>{_Vp(__rs, __is)..., _Vp(__rN, __iN)};
1874 else
1875 return tuple(_Vp(__rs, __is)..., resize_t<__rem, _Vp>(__rN, __iN));
1876 }
1877
1878 template <typename _A0>
1879 [[__gnu__::__always_inline__]]
1880 static constexpr basic_vec
1881 _S_concat(const basic_vec<value_type, _A0>& __x0) noexcept
1882 { return static_cast<basic_vec>(__x0); }
1883
1884 template <typename... _As>
1885 requires (sizeof...(_As) > 1)
1886 [[__gnu__::__always_inline__]]
1887 static constexpr basic_vec
1888 _S_concat(const basic_vec<value_type, _As>&... __xs) noexcept
1889 { return {_RealSimd::_S_concat(__xs._M_real...), _RealSimd::_S_concat(__xs._M_imag...) }; }
1890
1891 template <typename _BinaryOp>
1892 [[__gnu__::__always_inline__]]
1893 constexpr auto
1894 _M_reduce_to_register(_BinaryOp __binary_op) const
1895 {
1896 if constexpr (_RealSimd::abi_type::_S_nreg == 1)
1897 return *this;
1898 else
1899 {
1900 auto [__lo, __hi] = _M_chunk<resize_t<_RealSimd::_N0, basic_vec>>();
1901 auto __a = __lo._M_reduce_to_register(__binary_op);
1902 auto __b = __hi._M_reduce_to_register(__binary_op);
1903 if constexpr (__a._S_size == __b._S_size)
1904 return __binary_op(__a, __b);
1905 else
1906 {
1907 using _V1 = resize_t<1, basic_vec>;
1908 return __binary_op(_V1(__a._M_reduce(__binary_op)),
1909 _V1(__b._M_reduce(__binary_op)));
1910 }
1911 }
1912 }
1913
1914 template <typename _BinaryOp, _ArchTraits _Traits = {}>
1915 [[__gnu__::__always_inline__]]
1916 constexpr value_type
1917 _M_reduce(_BinaryOp __binary_op) const
1918 {
1919 if constexpr (_S_size == 1)
1920 return operator[](0);
1921 else if constexpr (_Traits.template _M_eval_as_f32<_T0>())
1922 return value_type(rebind_t<complex<float>, basic_vec>(*this)._M_reduce(__binary_op));
1923 else if constexpr (_RealSimd::abi_type::_S_nreg >= 2)
1924 return _M_reduce_to_register(__binary_op)._M_reduce(__binary_op);
1925 else if constexpr (__has_single_bit(unsigned(_S_size)))
1926 {
1927 const auto [__a, __b] = _M_chunk<resize_t<_S_size / 2, basic_vec>>();
1928 return __binary_op(__a, __b)._M_reduce(__binary_op);
1929 }
1930 else
1931 {
1932 const auto [__a, __b, __c, ...__rest]
1933 = _M_chunk<resize_t<__bit_floor(unsigned(_S_size)) / 2, basic_vec>>();
1934 const auto __ab = __binary_op(__a, __b);
1935 static_assert(sizeof...(__rest) <= 1);
1936 if constexpr (__a._S_size != __c._S_size)
1937 return cat(__ab, __c)._M_reduce(__binary_op);
1938 else
1939 return cat(__binary_op(__ab, __c), __rest...)._M_reduce(__binary_op);
1940 }
1941 }
1942
1943 /** @internal
1944 * Implementation of @ref partial_load.
1945 *
1946 * If @p __mem stores complex numbers, this needs to load @c abcdefgh from memory into two
1947 * basic_vec: @c aceg and @c bdfh.
1948 *
1949 * @param __mem A pointer to an array of @p __n values. Can be complex or real.
1950 * @param __n Read no more than @p __n values from memory.
1951 *
1952 * @todo Optimize with deinterleaving loads or loads + deinterleaving fixup.
1953 */
1954 template <typename _Up>
1955 [[__gnu__::__always_inline__]]
1956 static inline basic_vec
1957 _S_partial_load(const _Up* __mem, size_t __n)
1958 {
1959 if constexpr (__complex_like<_Up>)
1960 return basic_vec(
1961 _RealSimd([&](size_t __i) -> _T0 {
1962 return __i < __n ? __mem[__i].real() : _T0();
1963 }),
1964 _RealSimd([&](size_t __i) -> _T0 {
1965 return __i < __n ? __mem[__i].imag() : _T0();
1966 }));
1967 else
1968 return basic_vec(_RealSimd::_S_partial_load(__mem, __n));
1969 }
1970
1971 /** @internal
1972 *
1973 * @todo Optimize with deinterleaving loads or loads + deinterleaving fixup.
1974 */
1975 template <typename _Up, _ArchTraits _Traits = {}>
1976 static inline basic_vec
1977 _S_masked_load(const _Up* __mem, mask_type __k)
1978 {
1979 if constexpr (__complex_like<_Up>)
1980 { // TODO: optimize
1981 return basic_vec(_RealSimd([&](int __i) {
1982 return __k[__i] ? __mem[__i].real() : _T0();
1983 }),
1984 _RealSimd([&](int __i) {
1985 return __k[__i] ? __mem[__i].imag() : _T0();
1986 }));
1987 }
1988 else
1989 return basic_vec(_RealSimd::_S_masked_load(__mem, typename _RealSimd::mask_type(__k)));
1990 }
1991
1992 template <typename _Up>
1993 [[__gnu__::__always_inline__]]
1994 inline void
1995 _M_store(_Up* __mem) const
1996 {
1997 static_assert(__complex_like<_Up>);
1998 for (int __i = 0; __i < _S_size; ++__i)
1999 {
2000 __mem[__i].real(_M_real[__i]);
2001 __mem[__i].imag(_M_imag[__i]);
2002 }
2003 }
2004
2005 template <typename _Up>
2006 [[__gnu__::__always_inline__]]
2007 static inline void
2008 _S_partial_store(const basic_vec& __v, _Up* __mem, size_t __n)
2009 {
2010 static_assert(__complex_like<_Up>);
2011 for (size_t __i = 0; __i < std::min(__n, size_t(_S_size)); ++__i)
2012 {
2013 __mem[__i].real(__v._M_real[__i]);
2014 __mem[__i].imag(__v._M_imag[__i]);
2015 }
2016 }
2017
2018 template <typename _Up>
2019 [[__gnu__::__always_inline__]]
2020 static inline void
2021 _S_masked_store(const basic_vec& __v, _Up* __mem, const mask_type& __k)
2022 {
2023 // TODO: optimize
2024 static_assert(__complex_like<_Up>);
2025 for (int __i = 0; __i < _S_size; ++__i)
2026 {
2027 if (__k[__i])
2028 __mem[__i] = __v[__i];
2029 }
2030 }
2031
2032 basic_vec() = default;
2033
2034 // TODO: conversion extensions
2035
2036 // [simd.ctor] broadcast constructor ------------------------------------
2037 template <__broadcast_constructible<value_type> _Up>
2038 requires __complex_like<_Up>
2039 [[__gnu__::__always_inline__]]
2040 constexpr
2041 basic_vec(_Up&& __x) noexcept
2042 : _M_real(__x.real()), _M_imag(__x.imag())
2043 {}
2044
2045 template <__broadcast_constructible<value_type> _Up>
2046 [[__gnu__::__always_inline__]]
2047 constexpr
2048 basic_vec(_Up&& __x) noexcept
2049 : _M_real(__x), _M_imag()
2050 {}
2051
2052 // [simd.ctor] conversion constructor -----------------------------------
2053 template <__complex_like _Up, typename _UAbi>
2054 requires (_S_size == _UAbi::_S_size)
2055 && __explicitly_convertible_to<_Up, value_type>
2056 && _UAbi::_S_is_cx_ileav
2057 [[__gnu__::__always_inline__]]
2058 constexpr
2059 explicit(!convertible_to<_Up, value_type>)
2060 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
2061 : _M_real(__x.real()), _M_imag(__x.imag())
2062 {}
2063
2064 template <__complex_like _Up, typename _UAbi>
2065 requires (_S_size == _UAbi::_S_size)
2066 && __explicitly_convertible_to<_Up, value_type>
2067 && (!_UAbi::_S_is_cx_ileav)
2068 [[__gnu__::__always_inline__]]
2069 constexpr
2070 explicit(!convertible_to<_Up, value_type>)
2071 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
2072 : _M_real(__x._M_real), _M_imag(__x._M_imag) // using real() instead of _M_real is possible
2073 // but potentially leads to memcpy because of oversized _M_real (likewise for imag)
2074 {}
2075
2076 template <typename _Up, typename _UAbi> // _Up is not complex!
2077 requires (!__complex_like<_Up>)
2078 && (_S_size == _UAbi::_S_size)
2079 && __explicitly_convertible_to<_Up, value_type>
2080 [[__gnu__::__always_inline__]]
2081 constexpr
2082 explicit(!convertible_to<_Up, value_type>)
2083 basic_vec(const basic_vec<_Up, _UAbi>& __x) noexcept
2084 : _M_real(__x), _M_imag()
2085 {}
2086
2087 using _VecBase<_Tp, _Ap>::_VecBase;
2088
2089 // [simd.ctor] generator constructor ------------------------------------
2090 template <__simd_generator_invokable<value_type, _S_size> _Fp>
2091 [[__gnu__::__always_inline__]]
2092 constexpr explicit
2093 basic_vec(_Fp&& __gen)
2094 : _M_real(),
2095 _M_imag([&] {
2096 _T0 __re[sizeof(_RealSimd) / sizeof(_T0)] = {};
2097 _T0 __im[sizeof(_RealSimd) / sizeof(_T0)] = {};
2098 template for (constexpr int __i : _IotaArray<_S_size>)
2099 {
2100 const value_type __c = static_cast<value_type>(__gen(__simd_size_c<__i>));
2101 __re[__i] = __c.real();
2102 __im[__i] = __c.imag();
2103 }
2104 _M_real = __builtin_bit_cast(_RealSimd, __re);
2105 return __builtin_bit_cast(_RealSimd, __im);
2106 }())
2107 {}
2108
2109 // [simd.ctor] load constructor -----------------------------------------
2110 template <__complex_like _Up>
2111 [[__gnu__::__always_inline__]]
2112 constexpr
2113 basic_vec(_LoadCtorTag, const _Up* __ptr)
2114 : _M_real([&](int __i) -> _T0 { return __ptr[__i].real(); }),
2115 _M_imag([&](int __i) -> _T0 { return __ptr[__i].imag(); })
2116 {}
2117
2118 template <typename _Up>
2119 [[__gnu__::__always_inline__]]
2120 constexpr
2121 basic_vec(_LoadCtorTag, const _Up* __ptr)
2122 : _M_real(_LoadCtorTag(), __ptr), _M_imag()
2123 {}
2124
2125 template <ranges::contiguous_range _Rg, typename... _Flags>
2126 requires ranges::__static_sized_range<_Rg> && (ranges::__static_size<_Rg>() == _S_size)
2127 && __vectorizable<ranges::range_value_t<_Rg>>
2128 && __explicitly_convertible_to<ranges::range_value_t<_Rg>, value_type>
2129 [[__gnu__::__always_inline__]]
2130 constexpr
2131 basic_vec(_Rg&& __range, flags<_Flags...> __flags = {})
2132 : basic_vec(_LoadCtorTag(), __flags.template _S_adjust_pointer<basic_vec>(
2133 ranges::data(__range)))
2134 {
2135 static_assert(__loadstore_convertible_to<ranges::range_value_t<_Rg>, value_type,
2136 _Flags...>);
2137 }
2138
2139 // [simd.ctor] complex init ---------------------------------------------
2140 // This uses _RealSimd as proposed in LWG4230
2141 [[__gnu__::__always_inline__]]
2142 constexpr
2143 basic_vec(const _RealSimd& __re, const _RealSimd& __im = {}) noexcept
2144 : _M_real(__re), _M_imag(__im)
2145 {}
2146
2147 // [simd.subscr] --------------------------------------------------------
2148 [[__gnu__::__always_inline__]]
2149 constexpr value_type
2150 operator[](__simd_size_type __i) const
2151 { return value_type(_M_real[__i], _M_imag[__i]); }
2152
2153 // [simd.unary] unary operators -----------------------------------------
2154 [[__gnu__::__always_inline__]]
2155 constexpr basic_vec&
2156 operator++() noexcept requires requires(value_type __a) { ++__a; }
2157 {
2158 ++_M_real;
2159 return *this;
2160 }
2161
2162 [[__gnu__::__always_inline__]]
2163 constexpr basic_vec
2164 operator++(int) noexcept requires requires(value_type __a) { __a++; }
2165 {
2166 basic_vec __r = *this;
2167 ++_M_real;
2168 return __r;
2169 }
2170
2171 [[__gnu__::__always_inline__]]
2172 constexpr basic_vec&
2173 operator--() noexcept requires requires(value_type __a) { --__a; }
2174 {
2175 --_M_real;
2176 return *this;
2177 }
2178
2179 [[__gnu__::__always_inline__]]
2180 constexpr basic_vec
2181 operator--(int) noexcept requires requires(value_type __a) { __a--; }
2182 {
2183 basic_vec __r = *this;
2184 --_M_real;
2185 return __r;
2186 }
2187
2188 [[__gnu__::__always_inline__]]
2189 constexpr mask_type
2190 operator!() const noexcept requires requires(value_type __a) { !__a; }
2191 { return !_M_real && !_M_imag; }
2192
2193 [[__gnu__::__always_inline__]]
2194 constexpr basic_vec
2195 operator+() const noexcept requires requires(value_type __a) { +__a; }
2196 { return *this; }
2197
2198 [[__gnu__::__always_inline__]]
2199 constexpr basic_vec
2200 operator-() const noexcept requires requires(value_type __a) { -__a; }
2201 { return basic_vec(-_M_real, -_M_imag); }
2202
2203 // [simd.cassign] compound assignment -----------------------------------
2204 [[__gnu__::__always_inline__]]
2205 friend constexpr basic_vec&
2206 operator+=(basic_vec& __x, const basic_vec& __y) noexcept
2207 requires requires(value_type __a) { __a + __a; }
2208 {
2209 __x._M_real += __y._M_real;
2210 __x._M_imag += __y._M_imag;
2211 return __x;
2212 }
2213
2214 [[__gnu__::__always_inline__]]
2215 friend constexpr basic_vec&
2216 operator-=(basic_vec& __x, const basic_vec& __y) noexcept
2217 requires requires(value_type __a) { __a - __a; }
2218 {
2219 __x._M_real -= __y._M_real;
2220 __x._M_imag -= __y._M_imag;
2221 return __x;
2222 }
2223
2224
2225 template <_TargetTraits _Traits = {}>
2226 [[__gnu__::__always_inline__]]
2227 friend constexpr basic_vec&
2228 operator*=(basic_vec& __x, const basic_vec& __y) noexcept
2229 requires requires(value_type __a) { __a * __a; }
2230 {
2231 __cxctgus::__mul<value_type, _Traits>(__x._M_real, __x._M_imag, __y._M_real, __y._M_imag);
2232 return __x;
2233 }
2234
2235 [[__gnu__::__always_inline__]]
2236 friend constexpr basic_vec&
2237 operator/=(basic_vec& __x, const basic_vec& __y) noexcept
2238 requires requires(value_type __a) { __a / __a; }
2239 {
2240 const _RealSimd __r = __x._M_real * __y._M_real + __x._M_imag * __y._M_imag;
2241 const _RealSimd __n = __y._M_norm();
2242 __x._M_imag = (__x._M_imag * __y._M_real - __x._M_real * __y._M_imag) / __n;
2243 __x._M_real = __r / __n;
2244 return __x;
2245 }
2246
2247 // [simd.comparison] compare operators ----------------------------------
2248 [[__gnu__::__always_inline__]]
2249 friend constexpr mask_type
2250 operator==(const basic_vec& __x, const basic_vec& __y) noexcept
2251 { return mask_type(__x._M_real == __y._M_real && __x._M_imag == __y._M_imag); }
2252
2253 [[__gnu__::__always_inline__]]
2254 friend constexpr mask_type
2255 operator!=(const basic_vec& __x, const basic_vec& __y) noexcept
2256 { return mask_type(__x._M_real != __y._M_real || __x._M_imag != __y._M_imag); }
2257
2258 // [simd.complex.access] complex-value accessors ------------------------
2259 // LWG4230: returns _RealSimd instead of auto
2260 [[__gnu__::__always_inline__]]
2261 constexpr _RealSimd
2262 real() const noexcept
2263 { return _M_real; }
2264
2265 [[__gnu__::__always_inline__]]
2266 constexpr _RealSimd
2267 imag() const noexcept
2268 { return _M_imag; }
2269
2270 [[__gnu__::__always_inline__]]
2271 constexpr void
2272 real(const _RealSimd& __x) noexcept
2273 { _M_real = __x; }
2274
2275 [[__gnu__::__always_inline__]]
2276 constexpr void
2277 imag(const _RealSimd& __x) noexcept
2278 { _M_imag = __x; }
2279
2280 // [simd.cond] ---------------------------------------------------------
2281 [[__gnu__::__always_inline__]]
2282 friend constexpr basic_vec
2283 __select_impl(const mask_type& __k, const basic_vec& __t, const basic_vec& __f) noexcept
2284 {
2285 typename basic_vec::_RealSimd::mask_type __kk(__k);
2286 return basic_vec(__select_impl(__kk, __t._M_real, __f._M_real),
2287 __select_impl(__kk, __t._M_imag, __f._M_imag));
2288 }
2289
2290 // [simd.complex.math] internals ---------------------------------------
2291 [[__gnu__::__always_inline__]]
2292 constexpr _RealSimd
2293 _M_abs() const; // TODO: depends on [simd.math]
2294
2295 // associated functions
2296 [[__gnu__::__always_inline__]]
2297 constexpr _RealSimd
2298 _M_norm() const
2299 { return _M_real * _M_real + _M_imag * _M_imag; }
2300
2301 [[__gnu__::__always_inline__]]
2302 constexpr basic_vec
2303 _M_conj() const
2304 { return basic_vec(_M_real, -_M_imag); }
2305 };
2306
2307 // [P3319R5] (extension) ----------------------------------------------------
2308 template <__complex_like _Tp, typename _Ap>
2309 inline constexpr basic_vec<_Tp, _Ap>
2310 __iota<basic_vec<_Tp, _Ap>> = basic_vec<_Tp, _Ap>([](typename _Tp::value_type __i)
2311 -> typename _Tp::value_type {
2312 static_assert(_Ap::_S_size - 1 <= numeric_limits<typename _Tp::value_type>::max(),
2313 "iota object would overflow");
2314 return __i;
2315 });
2316} // namespace simd
2317_GLIBCXX_END_NAMESPACE_VERSION
2318} // namespace std
2319
2320#pragma GCC diagnostic pop
2321#endif // C++26
2322#endif // _GLIBCXX_SIMD_COMPLEX_H
std::chrono::operator<=
constexpr bool operator<=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:863
std::chrono::operator>=
constexpr bool operator>=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:877
std::chrono::operator<
constexpr bool operator<(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:830
std::chrono::operator>
constexpr bool operator>(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:870
std::operator-
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
Definition complex:404
std::operator+
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
Definition complex:374
std::min
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition stl_algobase.h:232
std
ISO C++ entities toplevel namespace is std.
std::operator^
constexpr bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1682
std::operator|
constexpr bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1672
std::operator&
constexpr bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1662
std::array
A standard container for storing a fixed size sequence of elements.
Definition array:104