libstdc++
simd_mask.h
1// Implementation of <simd> -*- C++ -*-
2
3// Copyright The GNU Toolchain Authors.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_SIMD_MASK_H
26#define _GLIBCXX_SIMD_MASK_H 1
27
28#ifdef _GLIBCXX_SYSHDR
29#pragma GCC system_header
30#endif
31
32#if __cplusplus >= 202400L
33
34#include "simd_iterator.h"
35#include "vec_ops.h"
36#if _GLIBCXX_X86
37#include "simd_x86.h"
38#endif
39
40#include <bit>
41#include <bitset>
42
43// psabi warnings are bogus because the ABI of the internal types never leaks into user code
44#pragma GCC diagnostic push
45#pragma GCC diagnostic ignored "-Wpsabi"
46
47namespace std _GLIBCXX_VISIBILITY(default)
48{
49_GLIBCXX_BEGIN_NAMESPACE_VERSION
50namespace simd
51{
52 template <unsigned _Np>
53 struct _SwapNeighbors
54 {
55 consteval unsigned
56 operator()(unsigned __i, unsigned __size) const
57 {
58 if (__size % (2 * _Np) != 0)
59 __builtin_abort(); // swap_neighbors<N> permutation requires a multiple of 2N elements
60 else if (std::has_single_bit(_Np))
61 return __i ^ _Np;
62 else if (__i % (2 * _Np) >= _Np)
63 return __i - _Np;
64 else
65 return __i + _Np;
66 }
67 };
68
69 template <size_t _Np, size_t _Mp>
70 constexpr auto
71 __bitset_split(const bitset<_Mp>& __b)
72 {
73 constexpr auto __bits_per_word = __CHAR_BIT__ * __SIZEOF_LONG__;
74 if constexpr (_Np % __bits_per_word == 0)
75 {
76 struct _Tmp
77 {
78 bitset<_Np> _M_lo;
79 bitset<_Mp - _Np> _M_hi;
80 };
81 return __builtin_bit_cast(_Tmp, __b);
82 }
83 else
84 {
85 constexpr auto __bits_per_ullong = __CHAR_BIT__ * __SIZEOF_LONG_LONG__;
86 static_assert(_Mp <= __bits_per_ullong);
87 using _Lo = _Bitmask<_Np>;
88 using _Hi = _Bitmask<_Mp - _Np>;
89 struct _Tmp
90 {
91 _Lo _M_lo;
92 _Hi _M_hi;
93 };
94 return _Tmp {static_cast<_Lo>(__b.to_ullong()), static_cast<_Hi>(__b.to_ullong() >> _Np)};
95 }
96 }
97
98 static_assert(__bitset_split<64>(bitset<128>(1))._M_lo == bitset<64>(1));
99 static_assert(__bitset_split<64>(bitset<128>(1))._M_hi == bitset<64>(0));
100
101 // [simd.traits]
102 // --- rebind ---
103 template <typename _Tp, typename _Vp, _ArchTraits _Traits = {}>
104 struct rebind
105 {};
106
107 /**
108 * Computes a member @c type `basic_vec<_Tp, Abi>`, where @c Abi is chosen such that the
109 * number of elements is equal to `_Vp::size()` and features of the ABI tag (such as the
110 * internal representation of masks, or storage order of complex components) are preserved.
111 */
112 template <__vectorizable _Tp, __simd_vec_type _Vp, _ArchTraits _Traits>
113 //requires requires { typename __deduce_abi_t<_Tp, _Vp::size()>; }
114 struct rebind<_Tp, _Vp, _Traits>
115 { using type = __similar_vec<_Tp, _Vp::size(), typename _Vp::abi_type>; };
116
117 /**
118 * As above, except for @c basic_mask.
119 */
120 template <__vectorizable _Tp, __simd_mask_type _Mp, _ArchTraits _Traits>
121 //requires requires { typename __deduce_abi_t<_Tp, _Mp::size()>; }
122 struct rebind<_Tp, _Mp, _Traits>
123 { using type = __similar_mask<_Tp, _Mp::size(), typename _Mp::abi_type>; };
124
125 template <typename _Tp, typename _Vp>
126 using rebind_t = typename rebind<_Tp, _Vp>::type;
127
128 // --- resize ---
129 template <__simd_size_type _Np, typename _Vp, _ArchTraits _Traits = {}>
130 struct resize
131 {};
132
133 template <__simd_size_type _Np, __simd_vec_type _Vp, _ArchTraits _Traits>
134 requires (_Np >= 1)
135 //requires requires { typename __deduce_abi_t<typename _Vp::value_type, _Np>; }
136 struct resize<_Np, _Vp, _Traits>
137 { using type = __similar_vec<typename _Vp::value_type, _Np, typename _Vp::abi_type>; };
138
139 template <__simd_size_type _Np, __simd_mask_type _Mp, _ArchTraits _Traits>
140 requires (_Np >= 1)
141 //requires requires { typename __deduce_abi_t<typename _Mp::value_type, _Np>; }
142 struct resize<_Np, _Mp, _Traits>
143 {
144 using _A1 = decltype(__abi_rebind<__mask_element_size<_Mp>, _Np, typename _Mp::abi_type,
145 true>());
146
147 static_assert(__abi_tag<_A1>);
148
149 static_assert(_Mp::abi_type::_S_variant == _A1::_S_variant || __scalar_abi_tag<_A1>
150 || __scalar_abi_tag<typename _Mp::abi_type>);
151
152 using type = basic_mask<__mask_element_size<_Mp>, _A1>;
153 };
154
155 template <__simd_size_type _Np, typename _Vp>
156 using resize_t = typename resize<_Np, _Vp>::type;
157
158 // [simd.syn]
159 inline constexpr __simd_size_type zero_element = numeric_limits<int>::min();
160
161 inline constexpr __simd_size_type uninit_element = zero_element + 1;
162
163 // [simd.permute.static]
164 template<__simd_size_type _Np = 0, __simd_vec_or_mask_type _Vp,
165 __index_permutation_function<_Vp> _IdxMap>
166 [[__gnu__::__always_inline__]]
167 constexpr resize_t<_Np == 0 ? _Vp::size() : _Np, _Vp>
168 permute(const _Vp& __v, _IdxMap&& __idxmap)
169 { return resize_t<_Np == 0 ? _Vp::size() : _Np, _Vp>::_S_static_permute(__v, __idxmap); }
170
171 // [simd.permute.dynamic]
172 template<__simd_vec_or_mask_type _Vp, __simd_integral _Ip>
173 [[__gnu__::__always_inline__]]
174 constexpr resize_t<_Ip::size(), _Vp>
175 permute(const _Vp& __v, const _Ip& __indices)
176 { return __v[__indices]; }
177
178 // [simd.creation] ----------------------------------------------------------
179 template<__simd_vec_type _Vp, typename _Ap>
180 [[__gnu__::__always_inline__]]
181 constexpr auto
182 chunk(const basic_vec<typename _Vp::value_type, _Ap>& __x) noexcept
183 { return __x.template _M_chunk<_Vp>(); }
184
185 template<__simd_mask_type _Mp, typename _Ap>
186 [[__gnu__::__always_inline__]]
187 constexpr auto
188 chunk(const basic_mask<__mask_element_size<_Mp>, _Ap>& __x) noexcept
189 { return __x.template _M_chunk<_Mp>(); }
190
191 template<__simd_size_type _Np, typename _Tp, typename _Ap>
192 [[__gnu__::__always_inline__]]
193 constexpr auto
194 chunk(const basic_vec<_Tp, _Ap>& __x) noexcept
195 -> decltype(chunk<resize_t<_Np, basic_vec<_Tp, _Ap>>>(__x))
196 { return chunk<resize_t<_Np, basic_vec<_Tp, _Ap>>>(__x); }
197
198 template<__simd_size_type _Np, size_t _Bytes, typename _Ap>
199 [[__gnu__::__always_inline__]]
200 constexpr auto
201 chunk(const basic_mask<_Bytes, _Ap>& __x) noexcept
202 -> decltype(chunk<resize_t<_Np, basic_mask<_Bytes, _Ap>>>(__x))
203 { return chunk<resize_t<_Np, basic_mask<_Bytes, _Ap>>>(__x); }
204
205 // LWG???? (reported 2025-11-25)
206 template<typename _Tp, typename _A0, typename... _Abis>
207 constexpr resize_t<(_A0::_S_size + ... + _Abis::_S_size), basic_vec<_Tp, _A0>>
208 cat(const basic_vec<_Tp, _A0>& __x0, const basic_vec<_Tp, _Abis>&... __xs) noexcept
209 {
210 return resize_t<(_A0::_S_size + ... + _Abis::_S_size), basic_vec<_Tp, _A0>>
211 ::_S_concat(__x0, __xs...);
212 }
213
214 // LWG???? (reported 2025-11-25)
215 template<size_t _Bytes, typename _A0, typename... _Abis>
216 constexpr resize_t<(_A0::_S_size + ... + _Abis::_S_size), basic_mask<_Bytes, _A0>>
217 cat(const basic_mask<_Bytes, _A0>& __x0, const basic_mask<_Bytes, _Abis>&... __xs) noexcept
218 {
219 return resize_t<(_A0::_S_size + ... + _Abis::_S_size), basic_mask<_Bytes, _A0>>
220 ::_S_concat(__x0, __xs...);
221 }
222
223 // implementation helper for chunk and cat
224 consteval int
225 __packs_to_skip_at_front(int __offset, initializer_list<int> __sizes)
226 {
227 int __i = 0;
228 int __n = 0;
229 for (int __s : __sizes)
230 {
231 __n += __s;
232 if (__n > __offset)
233 return __i;
234 ++__i;
235 }
236 __builtin_trap(); // called out of contract
237 }
238
239 consteval int
240 __packs_to_skip_at_back(int __offset, int __max, initializer_list<int> __sizes)
241 {
242 int __i = 0;
243 int __n = -__offset;
244 for (int __s : __sizes)
245 {
246 ++__i;
247 __n += __s;
248 if (__n >= __max)
249 return int(__sizes.size()) - __i;
250 }
251 return 0;
252 }
253
254 // in principle, this overload allows conversions to _Dst - and it wouldn't be wrong - but the
255 // general overload below is still a better candidate in overload resolution
256 template <typename _Dst>
257 [[__gnu__::__always_inline__]]
258 constexpr _Dst
259 __extract_simd_at(auto _Offset, const _Dst& __r, const auto&...)
260 requires(_Offset.value == 0)
261 { return __r; }
262
263 template <typename _Dst, typename _V0>
264 [[__gnu__::__always_inline__]]
265 constexpr _Dst
266 __extract_simd_at(auto _Offset, const _V0&, const _Dst& __r, const auto&...)
267 requires(_Offset.value == _V0::size.value)
268 { return __r; }
269
270 template <typename _Dst, typename... _Vs>
271 [[__gnu__::__always_inline__]]
272 constexpr _Dst
273 __extract_simd_at(auto _Offset, const _Vs&... __xs)
274 {
275 using _Adst = typename _Dst::abi_type;
276 if constexpr (_Adst::_S_nreg >= 2)
277 {
278 using _Dst0 = remove_cvref_t<decltype(declval<_Dst>()._M_get_low())>;
279 using _Dst1 = remove_cvref_t<decltype(declval<_Dst>()._M_get_high())>;
280 return _Dst::_S_init(__extract_simd_at<_Dst0>(_Offset, __xs...),
281 __extract_simd_at<_Dst1>(_Offset + _Dst0::size, __xs...));
282 }
283 else
284 {
285 using _Ret = remove_cvref_t<decltype(declval<_Dst>()._M_get())>;
286 constexpr bool __use_bitmask = __simd_mask_type<_Dst> && _Adst::_S_is_bitmask;
287 constexpr int __dst_full_size = __bit_ceil(unsigned(_Adst::_S_size));
288 constexpr int __nargs = sizeof...(__xs);
289 using _Afirst = typename _Vs...[0]::abi_type;
290 using _Alast = typename _Vs...[__nargs - 1]::abi_type;
291 const auto& __x0 = __xs...[0];
292 const auto& __xlast = __xs...[__nargs - 1];
293 constexpr int __ninputs = (_Vs::size.value + ...);
294 if constexpr (_Offset.value >= _Afirst::_S_size
295 || __ninputs - _Offset.value - _Alast::_S_size >= _Adst::_S_size)
296 { // can drop inputs at the front and/or back of the pack
297 constexpr int __skip_front = __packs_to_skip_at_front(_Offset.value,
298 {_Vs::size.value...});
299 constexpr int __skip_back = __packs_to_skip_at_back(_Offset.value, _Adst::_S_size,
300 {_Vs::size.value...});
301 static_assert(__skip_front > 0 || __skip_back > 0);
302 constexpr auto [...__skip] = _IotaArray<__skip_front>;
303 constexpr auto [...__is] = _IotaArray<__nargs - __skip_front - __skip_back>;
304 constexpr int __new_offset = _Offset.value - (0 + ... + _Vs...[__skip]::size.value);
305 return __extract_simd_at<_Dst>(cw<__new_offset>, __xs...[__is + __skip_front]...);
306 }
307 else if constexpr (_Adst::_S_size == 1)
308 { // trivial conversion to one value_type
309 return _Dst(__x0[_Offset.value]);
310 }
311 else if constexpr (_Afirst::_S_nreg >= 2 || _Alast::_S_nreg >= 2)
312 { // flatten first and/or last multi-register argument
313 constexpr bool __flatten_first = _Afirst::_S_nreg >= 2;
314 constexpr bool __flatten_last = __nargs > 1 && _Alast::_S_nreg >= 2;
315 constexpr auto [...__is] = _IotaArray<__nargs - __flatten_first - __flatten_last>;
316 if constexpr (__flatten_first && __flatten_last)
317 return __extract_simd_at<_Dst>(
318 _Offset, __x0._M_get_low(), __x0._M_get_high(), __xs...[__is + 1]...,
319 __xlast._M_get_low(), __xlast._M_get_high());
320 else if constexpr (__flatten_first)
321 return __extract_simd_at<_Dst>(
322 _Offset, __x0._M_get_low(), __x0._M_get_high(), __xs...[__is + 1]...);
323 else
324 return __extract_simd_at<_Dst>(
325 _Offset, __xs...[__is]..., __xlast._M_get_low(), __xlast._M_get_high());
326 }
327 else if constexpr (__simd_mask_type<_Dst>
328 && ((_Adst::_S_variant != _Vs::abi_type::_S_variant
329 && !__scalar_abi_tag<typename _Vs::abi_type>) || ...))
330 { // convert ABI tag if incompatible
331 return __extract_simd_at<_Dst>(
332 _Offset, static_cast<const resize_t<_Vs::size.value, _Dst>&>(__xs)...);
333 }
334
335 // at this point __xs should be as small as possible; there may be some corner cases left
336
337 else if constexpr (__nargs == 1)
338 { // simple and optimal
339 if constexpr (__use_bitmask)
340 return _Dst(_Ret(__x0._M_to_uint() >> _Offset.value));
341 else
342 return _VecOps<_Ret>::_S_extract(__x0._M_concat_data(false), _Offset);
343 }
344 else if constexpr (__use_bitmask)
345 { // fairly simple and optimal bit shifting solution
346 static_assert(_Afirst::_S_nreg == 1);
347 static_assert(_Offset.value < _Afirst::_S_size);
348 int __offset = -_Offset.value;
349 _Ret __r;
350 template for (const auto& __x : {__xs...})
351 {
352 if (__offset <= 0)
353 __r = _Ret(__x._M_to_uint() >> -__offset);
354 else if (__offset < _Adst::_S_size)
355 __r |= _Ret(_Ret(__x._M_to_uint()) << __offset);
356 __offset += __x.size.value;
357 }
358 return _Dst(__r);
359 }
360 else if constexpr (__nargs == 2 && _Offset == 0 && _Adst::_S_nreg == 1
361 && _Afirst::_S_size >= _Alast::_S_size
362 && __has_single_bit(unsigned(_Afirst::_S_size)))
363 { // simple __vec_concat
364 if constexpr (_Afirst::_S_size == 1)
365 // even simpler init from two values
366 return _Ret{__x0._M_concat_data()[0], __xlast._M_concat_data()[0]};
367 else
368 {
369 const auto __v0 = __x0._M_concat_data();
370 const auto __v1 = __vec_zero_pad_to<sizeof(__v0)>(__xlast._M_concat_data());
371 return __vec_concat(__v0, __v1);
372 }
373 }
374 else if constexpr (__nargs == 2 && _Adst::_S_nreg == 1 && _Offset == 0
375 && _Afirst::_S_nreg == 1 && _Alast::_S_size == 1)
376 { // optimize insertion of one element at the end
377 _Ret __r = __vec_zero_pad_to<sizeof(_Ret)>(__x0._M_get());
378 __vec_set(__r, _Afirst::_S_size, __xlast._M_concat_data()[0]);
379 return __r;
380 }
381 else if constexpr (__nargs == 2 && _Adst::_S_nreg == 1 && _Offset == 0
382 && _Afirst::_S_nreg == 1 && _Alast::_S_size == 2)
383 { // optimize insertion of two elements at the end
384 _Ret __r = __vec_zero_pad_to<sizeof(_Ret)>(__x0._M_concat_data());
385 const auto __x1 = __xlast._M_concat_data();
386 if constexpr (sizeof(__x1) <= sizeof(double) && (_Afirst::_S_size & 1) == 0)
387 { // can use a single insert instruction
388 using _Up = __conditional_t<
389 is_floating_point_v<__vec_value_type<_Ret>>,
390 __conditional_t<sizeof(__x1) == sizeof(double), double, float>,
391 __integer_from<sizeof(__x1)>>;
392 auto __r2 = __vec_bit_cast<_Up>(__r);
393 __vec_set(__r2, _Afirst::_S_size / 2, __vec_bit_cast<_Up>(__x1)[0]);
394 __r = reinterpret_cast<_Ret>(__r2);
395 }
396 else
397 {
398 __vec_set(__r, _Afirst::_S_size, __x1[0]);
399 __vec_set(__r, _Afirst::_S_size + 1, __x1[1]);
400 }
401 return __r;
402 }
403 else if constexpr (__nargs == 2 && _Afirst::_S_nreg == 1 && _Alast::_S_nreg == 1)
404 { // optimize concat of two input vectors (e.g. using palignr)
405 constexpr auto [...__is] = _IotaArray<__dst_full_size>;
406 constexpr int __v2_offset = __width_of<decltype(__x0._M_concat_data())>;
407 return __builtin_shufflevector(
408 __x0._M_concat_data(), __xlast._M_concat_data(), [](int __i) consteval {
409 if (__i < _Afirst::_S_size)
410 return __i;
411 __i -= _Afirst::_S_size;
412 if (__i < _Alast::_S_size)
413 return __i + __v2_offset;
414 else
415 return -1;
416 }(__is + _Offset.value)...);
417 }
418 else if (__is_const_known(__xs...) || __ninputs == _Adst::_S_size)
419 { // hard to optimize for the compiler, but necessary in constant expressions
420 return _VecOps<_Ret>::_S_extract(
421 __vec_concat_sized<__xs.size.value...>(__xs._M_concat_data(false)...),
422 _Offset);
423 }
424 else
425 { // fallback to concatenation in memory => load the result
426 alignas(_Ret) __vec_value_type<_Ret>
427 __tmp[std::max(__ninputs, _Offset.value + __dst_full_size)] = {};
428 int __offset = 0;
429 template for (const auto& __x : {__xs...})
430 {
431 if constexpr (__simd_mask_type<_Dst>)
432 (-__x)._M_store(__tmp + __offset);
433 else
434 __x._M_store(__tmp + __offset);
435 __offset += __x.size.value;
436 }
437 _Ret __r;
438 __builtin_memcpy(&__r, __tmp + _Offset.value, sizeof(_Ret));
439 return __r;
440 }
441 }
442 }
443
444 // [simd.mask] --------------------------------------------------------------
445 template <size_t _Bytes, typename _Ap>
446 class basic_mask
447 {
448 public:
449 using value_type = bool;
450
451 using abi_type = _Ap;
452
453#define _GLIBCXX_DELETE_SIMD "This specialization is disabled because of an invalid combination " \
454 "of template arguments to basic_mask."
455
456 basic_mask() = delete(_GLIBCXX_DELETE_SIMD);
457
458 ~basic_mask() = delete(_GLIBCXX_DELETE_SIMD);
459
460 basic_mask(const basic_mask&) = delete(_GLIBCXX_DELETE_SIMD);
461
462 basic_mask& operator=(const basic_mask&) = delete(_GLIBCXX_DELETE_SIMD);
463
464#undef _GLIBCXX_DELETE_SIMD
465 };
466
467 template <size_t _Bytes, typename _Ap>
468 class _MaskBase
469 {
470 using _Mp = basic_mask<_Bytes, _Ap>;
471
472 protected:
473 using _VecType = __simd_vec_from_mask_t<_Bytes, _Ap>;
474
475 static_assert(destructible<_VecType> || _Bytes > sizeof(0ull));
476
477 public:
478 using iterator = __iterator<_Mp>;
479
480 using const_iterator = __iterator<const _Mp>;
481
482 constexpr iterator
483 begin() noexcept
484 { return {static_cast<_Mp&>(*this), 0}; }
485
486 constexpr const_iterator
487 begin() const noexcept
488 { return cbegin(); }
489
490 constexpr const_iterator
491 cbegin() const noexcept
492 { return {static_cast<const _Mp&>(*this), 0}; }
493
494 constexpr default_sentinel_t
495 end() const noexcept
496 { return {}; }
497
498 constexpr default_sentinel_t
499 cend() const noexcept
500 { return {}; }
501
502 static constexpr auto size = __simd_size_c<_Ap::_S_size>;
503
504 _MaskBase() = default;
505
506 // LWG issue from 2026-03-04 / P4042R0
507 template <size_t _UBytes, typename _UAbi>
508 requires (_Ap::_S_size != _UAbi::_S_size)
509 explicit
510 _MaskBase(const basic_mask<_UBytes, _UAbi>&) = delete("size mismatch");
511
512 template <typename _Up, typename _UAbi>
513 explicit
514 _MaskBase(const basic_vec<_Up, _UAbi>&)
515 = delete("use operator! or a comparison to convert a vec into a mask");
516
517 template <typename _Up, typename _UAbi>
518 requires (_Ap::_S_size != _UAbi::_S_size)
519 operator basic_vec<_Up, _UAbi>() const
520 = delete("size mismatch");
521 };
522
523 template <size_t _Bytes, __abi_tag _Ap>
524 requires (_Ap::_S_nreg == 1)
525 && (__filter_abi_variant(_Ap::_S_variant, _AbiVariant::_CxVariants) == _AbiVariant()
526 || _Ap::_S_size == 1) // _Abi<1, 1, _CxIleav> and _Abi<1, 1, _CxCtgus> go here
527 class basic_mask<_Bytes, _Ap>
528 : public _MaskBase<_Bytes, _Ap>
529 {
530 using _Base = _MaskBase<_Bytes, _Ap>;
531
532 using _VecType = _Base::_VecType;
533
534 template <size_t, typename>
535 friend class basic_mask;
536
537 template <typename, typename>
538 friend class basic_vec;
539
540 static constexpr int _S_size = _Ap::_S_size;
541
542 using _DataType = typename _Ap::template _MaskDataType<_Bytes>;
543
544 static constexpr bool _S_has_bool_member = is_same_v<_DataType, bool>;
545
546 static constexpr bool _S_is_scalar = _S_has_bool_member;
547
548 static constexpr bool _S_use_bitmask = _Ap::_S_is_bitmask && !_S_is_scalar;
549
550 static constexpr int _S_full_size = [] {
551 if constexpr (_S_is_scalar)
552 return _S_size;
553 else if constexpr (_S_use_bitmask && _S_size < __CHAR_BIT__)
554 return __CHAR_BIT__;
555 else
556 return __bit_ceil(unsigned(_S_size));
557 }();
558
559 static constexpr bool _S_is_partial = _S_size != _S_full_size;
560
561 static constexpr _DataType _S_implicit_mask = [] {
562 if constexpr (_S_is_scalar)
563 return true;
564 else if (!_S_is_partial)
565 return _DataType(~_DataType());
566 else if constexpr (_S_use_bitmask)
567 return _DataType((_DataType(1) << _S_size) - 1);
568 else
569 {
570 constexpr auto [...__is] = _IotaArray<_S_full_size>;
571 return _DataType{ (__is < _S_size ? -1 : 0)... };
572 }
573 }();
574
575 // Actual padding bytes, not padding elements.
576 // => _S_padding_bytes is 0 even if _S_is_partial is true.
577 static constexpr size_t _S_padding_bytes = 0;
578
579 _DataType _M_data;
580
581 public:
582 using value_type = bool;
583
584 using abi_type = _Ap;
585
586 using iterator = _Base::iterator;
587
588 using const_iterator = _Base::const_iterator;
589
590 // internal but public API ----------------------------------------------
591 [[__gnu__::__always_inline__]]
592 static constexpr basic_mask
593 _S_init(_DataType __x)
594 {
595 basic_mask __r;
596 __r._M_data = __x;
597 return __r;
598 }
599
600 [[__gnu__::__always_inline__]]
601 static constexpr basic_mask
602 _S_init(unsigned_integral auto __bits)
603 { return basic_mask(__bits); }
604
605 [[__gnu__::__always_inline__]]
606 constexpr const _DataType&
607 _M_get() const
608 { return _M_data; }
609
610 /** @internal
611 * @brief Converts the type of the mask without changing the data member.
612 *
613 * Since _Abi<1, 1, _CxCtgus> uses this partial specialization of basic_mask, the _M_data
614 * member cannot be used as mask that matches the basic_vec elements.
615 */
616 [[__gnu__::__always_inline__]]
617 constexpr auto
618 _M_get_ileav_data() const noexcept
619 requires _Ap::_S_is_cx_ileav
620 { return __component_mask_for_ileav<_Bytes, _Ap>(_M_data); }
621
622 /** @internal
623 * @brief Converts the type of the mask from a scalar (bool) into a mask of 2 elements.
624 *
625 * Since _Abi<1, 1, _CxIleav> uses this partial specialization of basic_mask, the _M_data
626 * member cannot be used as mask that matches the basic_vec elements.
627 */
628 [[__gnu__::__always_inline__]]
629 constexpr auto
630 _M_get_ctgus_data() const noexcept
631 requires _Ap::_S_is_cx_ctgus
632 { return __component_mask_for_ctgus<_Bytes, _Ap>(_M_data); }
633
634 /** @internal
635 * Bit-cast the given object @p __x to basic_mask.
636 *
637 * This is necessary for _S_nreg > 1 where the last element can be bool or when the sizeof
638 * doesn't match because of different alignment requirements of the sub-masks.
639 */
640 template <size_t _UBytes, typename _UAbi>
641 [[__gnu__::__always_inline__]]
642 static constexpr basic_mask
643 _S_recursive_bit_cast(const basic_mask<_UBytes, _UAbi>& __x)
644 { return __builtin_bit_cast(basic_mask, __x._M_concat_data()); }
645
646 [[__gnu__::__always_inline__]]
647 constexpr auto
648 _M_concat_data(bool __do_sanitize = _S_is_partial) const
649 {
650 if constexpr (_S_is_scalar)
651 return __vec_builtin_type<__integer_from<_Bytes>, 1>{__integer_from<_Bytes>(-_M_data)};
652 else
653 {
654 if constexpr (_S_is_partial)
655 if (__do_sanitize)
656 return _DataType(_M_data & _S_implicit_mask);
657 return _M_data;
658 }
659 }
660
661 /** @internal
662 * Returns a mask where the first @p __n elements are true. All remaining elements are false.
663 *
664 * @pre @p __n > 0 && @p __n < _S_size
665 */
666 template <_ArchTraits _Traits = {}>
667 [[__gnu__::__always_inline__]]
668 static constexpr basic_mask
669 _S_partial_mask_of_n(int __n)
670 {
671 static_assert(!_S_is_scalar);
672 if constexpr (!_S_use_bitmask)
673 {
674 using _Ip = __integer_from<_Bytes>;
675 __glibcxx_simd_precondition(__n >= 0 && __n <= numeric_limits<_Ip>::max(),
676 "_S_partial_mask_of_n without _S_use_bitmask requires "
677 "positive __n that does not overflow.");
678 constexpr _DataType __0123
679 = __builtin_bit_cast(_DataType, _IotaArray<_Ip(_S_full_size)>);
680 return basic_mask(__0123 < _Ip(__n));
681 }
682 else
683 {
684 __glibcxx_simd_precondition(__n >= 0 && __n <= 255,
685 "The x86 BZHI instruction requires __n to "
686 "only use bits 0:7");
687#if __has_builtin(__builtin_ia32_bzhi_si)
688 if constexpr (_S_size <= 32 && _Traits._M_have_bmi2())
689 return _S_init(_Bitmask<_S_size>(
690 __builtin_ia32_bzhi_si(~0u >> (32 - _S_size), unsigned(__n))));
691#endif
692#if __has_builtin(__builtin_ia32_bzhi_di)
693 else if constexpr (_S_size <= 64 && _Traits._M_have_bmi2())
694 return _S_init(__builtin_ia32_bzhi_di(~0ull >> (64 - _S_size), unsigned(__n)));
695#endif
696 if constexpr (_S_size <= 32)
697 {
698 __glibcxx_simd_precondition(__n < 32, "invalid shift");
699 return _S_init(_Bitmask<_S_size>((1u << unsigned(__n)) - 1));
700 }
701 else if constexpr (_S_size <= 64)
702 {
703 __glibcxx_simd_precondition(__n < 64, "invalid shift");
704 return _S_init((1ull << unsigned(__n)) - 1);
705 }
706 else
707 static_assert(false);
708 }
709 }
710
711 [[__gnu__::__always_inline__]]
712 constexpr basic_mask&
713 _M_and_neighbors()
714 {
715 if constexpr (_S_use_bitmask)
716 _M_data &= ((_M_data >> 1) & 0x5555'5555'5555'5555ull)
717 | ((_M_data << 1) & ~0x5555'5555'5555'5555ull);
718 else
719 _M_data &= _VecOps<_DataType>::_S_swap_neighbors(_M_data);
720 return *this;
721 }
722
723 [[__gnu__::__always_inline__]]
724 constexpr basic_mask&
725 _M_or_neighbors()
726 {
727 if constexpr (_S_use_bitmask)
728 _M_data |= ((_M_data >> 1) & 0x5555'5555'5555'5555ull)
729 | ((_M_data << 1) & ~0x5555'5555'5555'5555ull);
730 else
731 _M_data |= _VecOps<_DataType>::_S_swap_neighbors(_M_data);
732 return *this;
733 }
734
735 template <typename _Mp>
736 [[__gnu__::__always_inline__]]
737 constexpr auto _M_chunk() const noexcept
738 {
739 constexpr int __n = _S_size / _Mp::_S_size;
740 constexpr int __rem = _S_size % _Mp::_S_size;
741 constexpr auto [...__is] = _IotaArray<__n>;
742 if constexpr (__rem == 0)
743 return array<_Mp, __n>{__extract_simd_at<_Mp>(cw<_Mp::_S_size * __is>, *this)...};
744 else
745 {
746 using _Rest = resize_t<__rem, _Mp>;
747 return tuple(__extract_simd_at<_Mp>(cw<_Mp::_S_size * __is>, *this)...,
748 __extract_simd_at<_Rest>(cw<_Mp::_S_size * __n>, *this));
749 }
750 }
751
752 [[__gnu__::__always_inline__]]
753 static constexpr const basic_mask&
754 _S_concat(const basic_mask& __x0) noexcept
755 { return __x0; }
756
757 template <typename... _As>
758 requires (sizeof...(_As) > 1)
759 [[__gnu__::__always_inline__]]
760 static constexpr basic_mask
761 _S_concat(const basic_mask<_Bytes, _As>&... __xs) noexcept
762 {
763 static_assert(_S_size == (_As::_S_size + ...));
764 return __extract_simd_at<basic_mask>(cw<0>, __xs...);
765 }
766
767 // [simd.mask.overview] default constructor -----------------------------
768 basic_mask() = default;
769
770 // [simd.mask.overview] conversion extensions ---------------------------
771 [[__gnu__::__always_inline__]]
772 constexpr
773 basic_mask(_DataType __x) requires(!_S_is_scalar && !_S_use_bitmask)
774 : _M_data(__x)
775 {}
776
777 [[__gnu__::__always_inline__]]
778 constexpr
779 operator _DataType() requires(!_S_is_scalar && !_S_use_bitmask)
780 { return _M_data; }
781
782 // [simd.mask.ctor] broadcast constructor -------------------------------
783 [[__gnu__::__always_inline__]]
784 constexpr explicit
785 basic_mask(same_as<bool> auto __x) noexcept // LWG 4382.
786 : _M_data(__x ? _S_implicit_mask : _DataType())
787 {}
788
789 // [simd.mask.ctor] conversion constructor ------------------------------
790 template <size_t _UBytes, typename _UAbi>
791 requires (_S_size == _UAbi::_S_size)
792 [[__gnu__::__always_inline__]]
793 constexpr explicit(__is_mask_conversion_explicit<_Ap, _UAbi>(_Bytes, _UBytes))
794 basic_mask(const basic_mask<_UBytes, _UAbi>& __x) noexcept
795 : _M_data([&] [[__gnu__::__always_inline__]] {
796 using _UV = basic_mask<_UBytes, _UAbi>;
797 // bool to bool
798 if constexpr (_S_is_scalar)
799 return __x[0];
800
801 // converting from an "array of bool"
802 else if constexpr (_UV::_S_is_scalar)
803 {
804 constexpr auto [...__is] = _IotaArray<_S_size>;
805 if constexpr (_S_use_bitmask)
806 return ((_DataType(__x[__is]) << __is) | ...);
807 else
808 return _DataType{__vec_value_type<_DataType>(-__x[__is])...};
809 }
810
811 // vec-/bit-mask to bit-mask | bit-mask to vec-mask
812 else if constexpr (_S_use_bitmask || _UV::_S_use_bitmask)
813 return basic_mask(__x.to_bitset())._M_data;
814
815 // _CxCtgus stores its masks matching the complex::value_type (_UBytes/2)
816 else if constexpr (_UAbi::_S_is_cx_ctgus)
817 return basic_mask(__x._M_data)._M_data;
818
819 // vec-mask to vec-mask
820 else if constexpr (_Bytes == _UBytes)
821 return _S_recursive_bit_cast(__x)._M_data;
822
823 // 2-mask-elements wrapper to plain mask
824 else if constexpr (_UAbi::_S_is_cx_ileav)
825 {
826 if constexpr (_UBytes <= sizeof(0ll))
827 // two step (bit-cast -> convert)
828 return basic_mask(__similar_mask<__integer_from<_UBytes>, _S_size, _UAbi>(__x))
829 ._M_data;
830 else if constexpr (_Bytes == 1)
831 { // 16 -> 1
832 constexpr auto [...__is] = _IotaArray<_S_size>;
833 using _Ip = __vec_value_type<_DataType>;
834 return _DataType {_Ip(__x._M_data._M_concat_data()[__is * 2])...};
835 }
836 else // from complex<double>
837 {
838 const auto __k2 = __similar_mask<__integer_from<_Bytes / 2>, 2 * _S_size,
839 _UAbi>(__x._M_data);
840 return _S_recursive_bit_cast(__k2);
841 }
842 }
843 else
844 {
845#if _GLIBCXX_X86
846 // TODO: turn this into a __vec_mask_cast overload in simd_x86.h
847 if constexpr (_Bytes == 1 && _UBytes == 2)
848 if (!__is_const_known(__x))
849 {
850 if constexpr (_UAbi::_S_nreg == 1)
851 return __x86_cvt_vecmask<_DataType>(__x._M_data);
852 else if constexpr (_UAbi::_S_nreg == 2)
853 {
854 auto __lo = __x._M_data0._M_data;
855 auto __hi = __vec_zero_pad_to<sizeof(__lo)>(
856 __x._M_data1._M_concat_data());
857 return __x86_cvt_vecmask<_DataType>(__lo, __hi);
858 }
859 }
860#endif
861 return __vec_mask_cast<_DataType>(__x._M_concat_data());
862 }
863 }())
864 {}
865
866 using _Base::_MaskBase;
867
868 // [simd.mask.ctor] generator constructor -------------------------------
869 template <__simd_generator_invokable<bool, _S_size> _Fp>
870 [[__gnu__::__always_inline__]]
871 constexpr explicit
872 basic_mask(_Fp&& __gen)
873 : _M_data([&] [[__gnu__::__always_inline__]] {
874 constexpr auto [...__is] = _IotaArray<_S_size>;
875 if constexpr (_S_is_scalar)
876 return __gen(__simd_size_c<0>);
877 else if constexpr (_S_use_bitmask)
878 return _DataType(((_DataType(__gen(__simd_size_c<__is>)) << __is)
879 | ...));
880 else
881 return _DataType{__vec_value_type<_DataType>(
882 __gen(__simd_size_c<__is>) ? -1 : 0)...};
883 }())
884 {}
885
886 // [simd.mask.ctor] bitset constructor ----------------------------------
887 [[__gnu__::__always_inline__]]
888 constexpr
889 basic_mask(const same_as<bitset<_S_size>> auto& __b) noexcept // LWG 4382.
890 : basic_mask(static_cast<_Bitmask<_S_size>>(__b.to_ullong()))
891 {
892 // more than 64 elements in one register? not yet.
893 static_assert(_S_size <= numeric_limits<unsigned long long>::digits);
894 }
895
896 // [simd.mask.ctor] uint constructor ------------------------------------
897 template <unsigned_integral _Tp>
898 requires (!same_as<_Tp, bool>) // LWG 4382.
899 [[__gnu__::__always_inline__]]
900 constexpr explicit
901 basic_mask(_Tp __val) noexcept
902 : _M_data([&] [[__gnu__::__always_inline__]] () {
903 if constexpr (_S_use_bitmask)
904 return __val;
905 else if constexpr (_S_is_scalar)
906 return bool(__val & 1);
907 else if (__is_const_known(__val))
908 {
909 constexpr auto [...__is] = _IotaArray<_S_size>;
910 return _DataType {__vec_value_type<_DataType>((__val & (1ull << __is)) == 0
911 ? 0 : -1)...};
912 }
913 else
914 {
915 using _Ip = typename _VecType::value_type;
916 _VecType __v0 = _Ip(__val);
917 constexpr int __bits_per_element = sizeof(_Ip) * __CHAR_BIT__;
918 constexpr _VecType __pow2 = _VecType(cw<1>)
919 << (__iota<_VecType> % cw<__bits_per_element>);
920 if constexpr (_S_size < __bits_per_element)
921 return ((__v0 & __pow2) > cw<0>)._M_concat_data();
922 else if constexpr (_S_size == __bits_per_element)
923 return ((__v0 & __pow2) != cw<0>)._M_concat_data();
924 else
925 {
926 static_assert(_Bytes == 1);
927 static_assert(sizeof(_Ip) == 1);
928 _Bitmask<_S_size> __bits = __val;
929 static_assert(sizeof(_VecType) % sizeof(__bits) == 0);
930 if constexpr (sizeof(_DataType) == 32)
931 {
932 __vec_builtin_type<_UInt<8>, 4> __v1 = {
933 0xffu & (__bits >> (0 * __CHAR_BIT__)),
934 0xffu & (__bits >> (1 * __CHAR_BIT__)),
935 0xffu & (__bits >> (2 * __CHAR_BIT__)),
936 0xffu & (__bits >> (3 * __CHAR_BIT__)),
937 };
938 __v1 *= 0x0101'0101'0101'0101ull;
939 __v0 = __builtin_bit_cast(_VecType, __v1);
940 return ((__v0 & __pow2) != cw<0>)._M_data;
941 }
942 else
943 {
944 using _V1 = vec<_Ip, sizeof(__bits)>;
945 _V1 __v1 = __builtin_bit_cast(_V1, __bits);
946 __v0 = _VecType::_S_static_permute(__v1, [](int __i) {
947 return __i / __CHAR_BIT__;
948 });
949 return ((__v0 & __pow2) != cw<0>)._M_data;
950 }
951 }
952 }
953 }())
954 {}
955
956 //Effects: Initializes the first M elements to the corresponding bit values in val, where M is
957 //the smaller of size() and the number of bits in the value representation
958 //([basic.types.general]) of the type of val. If M is less than size(), the remaining elements
959 //are initialized to zero.
960
961
962 // [simd.mask.subscr] ---------------------------------------------------
963 [[__gnu__::__always_inline__]]
964 constexpr value_type
965 operator[](__simd_size_type __i) const
966 {
967 __glibcxx_simd_precondition(__i >= 0 && __i < _S_size, "subscript is out of bounds");
968 if constexpr (_S_is_scalar)
969 return _M_data;
970 else if constexpr (_S_use_bitmask)
971 return bool((_M_data >> __i) & 1);
972 else
973 return _M_data[__i] & 1;
974 }
975
976 // [simd.mask.unary] ----------------------------------------------------
977 [[__gnu__::__always_inline__]]
978 constexpr basic_mask
979 operator!() const noexcept
980 {
981 if constexpr (_S_is_scalar)
982 return _S_init(!_M_data);
983 else
984 return _S_init(~_M_data);
985 }
986
987 [[__gnu__::__always_inline__]]
988 constexpr _VecType
989 operator+() const noexcept requires destructible<_VecType>
990 { return operator _VecType(); }
991
992 constexpr _VecType
993 operator+() const noexcept = delete;
994
995 [[__gnu__::__always_inline__]]
996 constexpr _VecType
997 operator-() const noexcept requires destructible<_VecType>
998 {
999 using _Ip = typename _VecType::value_type;
1000 if constexpr (_S_is_scalar)
1001 return _Ip(-int(_M_data));
1002 else if constexpr (_S_use_bitmask)
1003 return __select_impl(*this, _Ip(-1), _Ip());
1004 else
1005 {
1006 static_assert(sizeof(_VecType) == sizeof(_M_data));
1007 return __builtin_bit_cast(_VecType, _M_data);
1008 }
1009 }
1010
1011 constexpr _VecType
1012 operator-() const noexcept = delete;
1013
1014 [[__gnu__::__always_inline__]]
1015 constexpr _VecType
1016 operator~() const noexcept requires destructible<_VecType>
1017 {
1018 using _Ip = typename _VecType::value_type;
1019 if constexpr (_S_is_scalar)
1020 return _Ip(~int(_M_data));
1021 else if constexpr (_S_use_bitmask)
1022 return __select_impl(*this, _Ip(-2), _Ip(-1));
1023 else
1024 {
1025 static_assert(sizeof(_VecType) == sizeof(_M_data));
1026 return __builtin_bit_cast(_VecType, _M_data) - _Ip(1);
1027 }
1028 }
1029
1030 constexpr _VecType
1031 operator~() const noexcept = delete;
1032
1033 // [simd.mask.conv] -----------------------------------------------------
1034 template <typename _Up, typename _UAbi>
1035 requires (_UAbi::_S_size == _S_size)
1036 [[__gnu__::__always_inline__]]
1037 constexpr explicit(sizeof(_Up) != _Bytes)
1038 operator basic_vec<_Up, _UAbi>() const noexcept
1039 {
1040 if constexpr (_S_is_scalar)
1041 return _Up(_M_data);
1042 else
1043 {
1044 using _UV = basic_vec<_Up, _UAbi>;
1045 return __select_impl(static_cast<_UV::mask_type>(*this), _Up(1), _UV());
1046 }
1047 }
1048
1049 using _Base::operator basic_vec;
1050
1051 // [simd.mask.namedconv] ------------------------------------------------
1052 [[__gnu__::__always_inline__]]
1053 constexpr bitset<_S_size>
1054 to_bitset() const noexcept
1055 {
1056 // more than 64 elements in one register? not yet.
1057 static_assert(_S_size <= numeric_limits<unsigned long long>::digits);
1058 return to_ullong();
1059 }
1060
1061 /** @internal
1062 * Return the mask as the smallest possible unsigned integer (up to 64 bits).
1063 *
1064 * @tparam _Offset Adjust the return type & value to start at bit @p _Offset.
1065 * @tparam _Use_2_for_1 Store the value of every second element into one bit of the result.
1066 * (precondition: each even/odd pair stores the same value)
1067 */
1068 template <int _Offset = 0, bool _Use_2_for_1 = false, _ArchTraits _Traits = {}>
1069 [[__gnu__::__always_inline__]]
1070 constexpr _Bitmask<_S_size / (_Use_2_for_1 + 1) + _Offset>
1071 _M_to_uint() const
1072 {
1073 constexpr int __nbits = _S_size / (_Use_2_for_1 + 1);
1074 static_assert(__nbits + _Offset <= numeric_limits<unsigned long long>::digits);
1075 static_assert(!(_S_is_scalar && _Use_2_for_1));
1076 // before shifting
1077 using _U0 = _Bitmask<__nbits>;
1078 // potentially wider type needed for shift by _Offset
1079 using _Ur = _Bitmask<__nbits + _Offset>;
1080 if constexpr (_S_is_scalar || _S_use_bitmask)
1081 {
1082 auto __bits = _M_data;
1083 if constexpr (_S_is_partial)
1084 __bits &= _S_implicit_mask;
1085 if constexpr (_Use_2_for_1)
1086 __bits = __bit_extract_even<__nbits>(__bits);
1087 return _Ur(__bits) << _Offset;
1088 }
1089 else if constexpr (_Bytes == sizeof(0ll) && _Use_2_for_1)
1090 {
1091 const auto __u32 = __vec_bit_cast<unsigned>(_M_data);
1092 if constexpr (sizeof(_M_data) == 16)
1093 {
1094 if constexpr (_Offset < 32)
1095 return __u32[0] & (1u << _Offset);
1096 else
1097 return _M_data[0] & (1ull << _Offset);
1098 }
1099 else if constexpr (sizeof(_M_data) == 32)
1100 {
1101 if constexpr (_Offset < 31)
1102 return (__u32[4] & (2u << _Offset)) | (__u32[0] & (1u << _Offset));
1103 else
1104 return (_M_data[2] & (2ull << _Offset)) | (_M_data[0] & (1ull << _Offset));
1105 }
1106 else
1107 static_assert(false);
1108 }
1109 else if constexpr (_Use_2_for_1 && __nbits == 1)
1110 return _Ur(operator[](0)) << _Offset;
1111 else
1112 {
1113#if _GLIBCXX_X86
1114 if (!__is_const_known(*this))
1115 {
1116 _U0 __uint;
1117 if constexpr (_Use_2_for_1)
1118 __uint = __x86_cvt_vecmask_to_bitmask<_Traits>(
1119 __vec_bit_cast<__integer_from<_Bytes * 2>>(_M_data));
1120 else
1121 __uint = __x86_cvt_vecmask_to_bitmask<_Traits>( _M_data);
1122 if constexpr (_S_is_partial)
1123 __uint &= (_U0(1) << _S_size) - 1;
1124 return _Ur(__uint) << _Offset;
1125 }
1126#endif
1127 using _IV = conditional_t<_Use_2_for_1,
1128 __similar_vec<__integer_from<_Bytes * 2>, __nbits, _Ap>,
1129 _VecType>;
1130 static_assert(destructible<_IV>);
1131 const typename _IV::mask_type& __k = [&] [[__gnu__::__always_inline__]] () {
1132 if constexpr (_Use_2_for_1)
1133 return typename _IV::mask_type(__to_cx_ileav(*this));
1134 else if constexpr (is_same_v<typename _IV::mask_type, basic_mask>)
1135 return *this;
1136 else
1137 return typename _IV::mask_type(*this);
1138 }();
1139 constexpr int __n = _IV::size();
1140 if constexpr (_Bytes * __CHAR_BIT__ >= __n) // '1 << __iota' cannot overflow
1141 { // reduce(select(k, powers_of_2, 0))
1142 constexpr _IV __pow2 = _IV(cw<1>) << __iota<_IV>;
1143 return _Ur(_U0(__select_impl(__k, __pow2, _IV())
1144 ._M_reduce(bit_or<>()))) << _Offset;
1145 }
1146 else if constexpr (__n % __CHAR_BIT__ != 0)
1147 { // recurse after splitting in two
1148 constexpr int __n_lo = __n - __n % __CHAR_BIT__;
1149 const auto [__lo, __hi] = chunk<__n_lo>(__k);
1150 _Ur __bits = __hi.template _M_to_uint<_Offset + __n_lo, _Use_2_for_1>();
1151 return __bits | __lo.template _M_to_uint<_Offset, _Use_2_for_1>();
1152 }
1153 else
1154 { // limit powers_of_2 to 1, 2, 4, ..., 128
1155 constexpr _IV __pow2 = _IV(cw<1>) << (__iota<_IV> % _IV(cw<__CHAR_BIT__>));
1156 _IV __x = __select_impl(__k, __pow2, _IV());
1157 // partial reductions of 8 neighboring elements
1158 __x |= _IV::_S_static_permute(__x, _SwapNeighbors<4>());
1159 __x |= _IV::_S_static_permute(__x, _SwapNeighbors<2>());
1160 __x |= _IV::_S_static_permute(__x, _SwapNeighbors<1>());
1161 // permute partial reduction results to the front
1162 __x = _IV::_S_static_permute(__x, [](int __i) {
1163 return __i * 8 < __n ? __i * 8 : uninit_element;
1164 });
1165 // extract front as scalar unsigned
1166 _U0 __bits = __builtin_bit_cast(
1167 __similar_vec<_U0, __n * _Bytes / sizeof(_U0), _Ap>, __x)[0];
1168 // mask off unused bits
1169 if constexpr (!__has_single_bit(unsigned(__nbits)))
1170 __bits &= (_U0(1) << __nbits) - 1;
1171 return _Ur(__bits) << _Offset;
1172 }
1173 }
1174 }
1175
1176 [[__gnu__::__always_inline__]]
1177 constexpr unsigned long long
1178 to_ullong() const
1179 { return _M_to_uint(); }
1180
1181 // [simd.mask.binary] ---------------------------------------------------
1182 [[__gnu__::__always_inline__]]
1183 friend constexpr basic_mask
1184 operator&&(const basic_mask& __x, const basic_mask& __y) noexcept
1185 { return _S_init(__x._M_data & __y._M_data); }
1186
1187 [[__gnu__::__always_inline__]]
1188 friend constexpr basic_mask
1189 operator||(const basic_mask& __x, const basic_mask& __y) noexcept
1190 { return _S_init(__x._M_data | __y._M_data); }
1191
1192 [[__gnu__::__always_inline__]]
1193 friend constexpr basic_mask
1194 operator&(const basic_mask& __x, const basic_mask& __y) noexcept
1195 { return _S_init(__x._M_data & __y._M_data); }
1196
1197 [[__gnu__::__always_inline__]]
1198 friend constexpr basic_mask
1199 operator|(const basic_mask& __x, const basic_mask& __y) noexcept
1200 { return _S_init(__x._M_data | __y._M_data); }
1201
1202 [[__gnu__::__always_inline__]]
1203 friend constexpr basic_mask
1204 operator^(const basic_mask& __x, const basic_mask& __y) noexcept
1205 { return _S_init(__x._M_data ^ __y._M_data); }
1206
1207 // [simd.mask.cassign] --------------------------------------------------
1208 [[__gnu__::__always_inline__]]
1209 friend constexpr basic_mask&
1210 operator&=(basic_mask& __x, const basic_mask& __y) noexcept
1211 {
1212 __x._M_data &= __y._M_data;
1213 return __x;
1214 }
1215
1216 [[__gnu__::__always_inline__]]
1217 friend constexpr basic_mask&
1218 operator|=(basic_mask& __x, const basic_mask& __y) noexcept
1219 {
1220 __x._M_data |= __y._M_data;
1221 return __x;
1222 }
1223
1224 [[__gnu__::__always_inline__]]
1225 friend constexpr basic_mask&
1226 operator^=(basic_mask& __x, const basic_mask& __y) noexcept
1227 {
1228 __x._M_data ^= __y._M_data;
1229 return __x;
1230 }
1231
1232 // [simd.mask.comparison] -----------------------------------------------
1233 [[__gnu__::__always_inline__]]
1234 friend constexpr basic_mask
1235 operator==(const basic_mask& __x, const basic_mask& __y) noexcept
1236 { return !(__x ^ __y); }
1237
1238 [[__gnu__::__always_inline__]]
1239 friend constexpr basic_mask
1240 operator!=(const basic_mask& __x, const basic_mask& __y) noexcept
1241 { return __x ^ __y; }
1242
1243 [[__gnu__::__always_inline__]]
1244 friend constexpr basic_mask
1245 operator>=(const basic_mask& __x, const basic_mask& __y) noexcept
1246 { return __x || !__y; }
1247
1248 [[__gnu__::__always_inline__]]
1249 friend constexpr basic_mask
1250 operator<=(const basic_mask& __x, const basic_mask& __y) noexcept
1251 { return !__x || __y; }
1252
1253 [[__gnu__::__always_inline__]]
1254 friend constexpr basic_mask
1255 operator>(const basic_mask& __x, const basic_mask& __y) noexcept
1256 { return __x && !__y; }
1257
1258 [[__gnu__::__always_inline__]]
1259 friend constexpr basic_mask
1260 operator<(const basic_mask& __x, const basic_mask& __y) noexcept
1261 { return !__x && __y; }
1262
1263 // [simd.mask.cond] -----------------------------------------------------
1264 [[__gnu__::__always_inline__]]
1265 friend constexpr basic_mask
1266 __select_impl(const basic_mask& __k, const basic_mask& __t, const basic_mask& __f) noexcept
1267 {
1268 if constexpr (!_S_use_bitmask)
1269 {
1270#if _GLIBCXX_X86
1271 // this works around bad code-gen when the compiler can't see that __k is a vector-mask.
1272 // This pattern, is recognized to match the x86 blend instructions, which only consider
1273 // the sign bit of the mask register. Also, without SSE4, if the compiler knows that __k
1274 // is a vector-mask, then the '< 0' is elided.
1275 return __k._M_data < 0 ? __t._M_data : __f._M_data;
1276#endif
1277 return __k._M_data ? __t._M_data : __f._M_data;
1278 }
1279 else
1280 return (__k._M_data & __t._M_data) | (~__k._M_data & __f._M_data);
1281 }
1282
1283 [[__gnu__::__always_inline__]]
1284 friend constexpr basic_mask
1285 __select_impl(const basic_mask& __k, same_as<bool> auto __t, same_as<bool> auto __f) noexcept
1286 {
1287 if (__t == __f)
1288 return basic_mask(__t);
1289 else
1290 return __t ? __k : !__k;
1291 }
1292
1293 template <__vectorizable _T0, same_as<_T0> _T1>
1294 requires (sizeof(_T0) == _Bytes)
1295 [[__gnu__::__always_inline__]]
1296 friend constexpr vec<_T0, _S_size>
1297 __select_impl(const basic_mask& __k, const _T0& __t, const _T1& __f) noexcept
1298 {
1299 if constexpr (_S_is_scalar)
1300 return __k._M_data ? __t : __f;
1301 else
1302 {
1303 using _Vp = vec<_T0, _S_size>;
1304 using _Mp = typename _Vp::mask_type;
1305 return __select_impl(_Mp(__k), _Vp(__t), _Vp(__f));
1306 }
1307 }
1308
1309 // [simd.mask.reductions] implementation --------------------------------
1310 [[__gnu__::__always_inline__]]
1311 constexpr bool
1312 _M_all_of() const noexcept
1313 {
1314 if constexpr (_S_is_scalar)
1315 return _M_data;
1316 else if constexpr (_S_use_bitmask)
1317 {
1318 if constexpr (_S_is_partial)
1319 // PR120925 (partial kortest pattern not recognized)
1320 return (_M_data & _S_implicit_mask) == _S_implicit_mask;
1321 else
1322 return _M_data == _S_implicit_mask;
1323 }
1324#if _GLIBCXX_X86
1325 else if (!__is_const_known(_M_data))
1326 return __x86_vecmask_all<_S_size>(_M_data);
1327#endif
1328 else
1329 return _VecOps<_DataType, _S_size>::_S_all_of(_M_data);
1330 }
1331
1332 [[__gnu__::__always_inline__]]
1333 constexpr bool
1334 _M_any_of() const noexcept
1335 {
1336 if constexpr (_S_is_scalar)
1337 return _M_data;
1338 else if constexpr (_S_use_bitmask)
1339 {
1340 if constexpr (_S_is_partial)
1341 // PR120925 (partial kortest pattern not recognized)
1342 return (_M_data & _S_implicit_mask) != 0;
1343 else
1344 return _M_data != 0;
1345 }
1346#if _GLIBCXX_X86
1347 else if (!__is_const_known(_M_data))
1348 return __x86_vecmask_any<_S_size>(_M_data);
1349#endif
1350 else
1351 return _VecOps<_DataType, _S_size>::_S_any_of(_M_data);
1352 }
1353
1354 [[__gnu__::__always_inline__]]
1355 constexpr bool
1356 _M_none_of() const noexcept
1357 {
1358 if constexpr (_S_is_scalar)
1359 return !_M_data;
1360 else if constexpr (_S_use_bitmask)
1361 {
1362 if constexpr (_S_is_partial)
1363 // PR120925 (partial kortest pattern not recognized)
1364 return (_M_data & _S_implicit_mask) == 0;
1365 else
1366 return _M_data == 0;
1367 }
1368#if _GLIBCXX_X86
1369 else if (!__is_const_known(_M_data))
1370 return __x86_vecmask_none<_S_size>(_M_data);
1371#endif
1372 else
1373 return _VecOps<_DataType, _S_size>::_S_none_of(_M_data);
1374 }
1375
1376 [[__gnu__::__always_inline__]]
1377 constexpr __simd_size_type
1378 _M_reduce_count() const noexcept
1379 {
1380 if constexpr (_S_is_scalar)
1381 return int(_M_data);
1382 else if constexpr (_S_size <= numeric_limits<unsigned>::digits)
1383 return __builtin_popcount(_M_to_uint());
1384 else
1385 return __builtin_popcountll(to_ullong());
1386 }
1387
1388 [[__gnu__::__always_inline__]]
1389 constexpr __simd_size_type
1390 _M_reduce_min_index() const
1391 {
1392 const auto __bits = _M_to_uint();
1393 __glibcxx_simd_precondition(__bits, "An empty mask does not have a min_index.");
1394 if constexpr (_S_size == 1)
1395 return 0;
1396 else
1397 return __countr_zero(__bits);
1398 }
1399
1400 [[__gnu__::__always_inline__]]
1401 constexpr __simd_size_type
1402 _M_reduce_max_index() const
1403 {
1404 const auto __bits = _M_to_uint();
1405 __glibcxx_simd_precondition(__bits, "An empty mask does not have a max_index.");
1406 if constexpr (_S_size == 1)
1407 return 0;
1408 else
1409 return __highest_bit(__bits);
1410 }
1411
1412 [[__gnu__::__always_inline__]]
1413 friend constexpr bool
1414 __is_const_known(const basic_mask& __x)
1415 { return __builtin_constant_p(__x._M_data); }
1416 };
1417
1418 template <size_t _Bytes, __abi_tag _Ap>
1419 requires (_Ap::_S_nreg > 1)
1420 && (__filter_abi_variant(_Ap::_S_variant, _AbiVariant::_CxVariants) == _AbiVariant())
1421 class basic_mask<_Bytes, _Ap>
1422 : public _MaskBase<_Bytes, _Ap>
1423 {
1424 using _Base = _MaskBase<_Bytes, _Ap>;
1425
1426 using _VecType = _Base::_VecType;
1427
1428 template <size_t, typename>
1429 friend class basic_mask;
1430
1431 template <typename, typename>
1432 friend class basic_vec;
1433
1434 static constexpr int _S_size = _Ap::_S_size;
1435
1436 static constexpr int _N0 = __bit_ceil(unsigned(_S_size)) / 2;
1437
1438 static constexpr int _N1 = _S_size - _N0;
1439
1440 static constexpr int _Nreg0 = __bit_ceil(unsigned(_Ap::_S_nreg)) / 2;
1441
1442 static constexpr int _Nreg1 = _Ap::_S_nreg - _Nreg0;
1443
1444 // explicitly request _Nreg0 rather than use __abi_rebind. This way _Float16 can use half
1445 // of native registers (since they convert to full float32 registers).
1446 using _Abi0 = decltype(_Ap::template _S_resize<_N0, _Nreg0>());
1447
1448 using _Abi1 = decltype(_Ap::template _S_resize<_N1, _Nreg1>());
1449
1450 using _Mask0 = basic_mask<_Bytes, _Abi0>;
1451
1452 // the implementation (and users) depend on elements being contiguous in memory
1453 static_assert(_Mask0::_S_padding_bytes == 0 && !_Mask0::_S_is_partial);
1454
1455 using _Mask1 = basic_mask<_Bytes, _Abi1>;
1456
1457 static constexpr bool _S_is_partial = _Mask1::_S_is_partial;
1458
1459 // _Ap::_S_nreg determines how deep the recursion goes. E.g. basic_mask<4, _Abi<8, 4>> cannot
1460 // use basic_mask<4, _Abi<4, 1>> as _Mask0/1 types.
1461 static_assert(_Mask0::abi_type::_S_nreg + _Mask1::abi_type::_S_nreg == _Ap::_S_nreg);
1462
1463 static constexpr bool _S_use_bitmask = _Mask0::_S_use_bitmask;
1464
1465 static constexpr bool _S_is_scalar = _Mask0::_S_is_scalar;
1466
1467 _Mask0 _M_data0;
1468
1469 _Mask1 _M_data1;
1470
1471 static constexpr bool _S_has_bool_member = _Mask1::_S_has_bool_member;
1472
1473 // by construction _N0 >= _N1
1474 // => sizeof(_Mask0) >= sizeof(_Mask1)
1475 // and __alignof__(_Mask0) >= __alignof__(_Mask1)
1476 static constexpr size_t _S_padding_bytes
1477 = (__alignof__(_Mask0) == __alignof__(_Mask1)
1478 ? 0 : __alignof__(_Mask0) - (sizeof(_Mask1) % __alignof__(_Mask0)))
1479 + _Mask1::_S_padding_bytes;
1480
1481 public:
1482 using value_type = bool;
1483
1484 using abi_type = _Ap;
1485
1486 using iterator = _Base::iterator;
1487
1488 using const_iterator = _Base::const_iterator;
1489
1490 [[__gnu__::__always_inline__]]
1491 static constexpr basic_mask
1492 _S_init(const _Mask0& __x, const _Mask1& __y)
1493 {
1494 basic_mask __r;
1495 __r._M_data0 = __x;
1496 __r._M_data1 = __y;
1497 return __r;
1498 }
1499
1500 [[__gnu__::__always_inline__]]
1501 static constexpr basic_mask
1502 _S_init(unsigned_integral auto __bits)
1503 { return basic_mask(__bits); }
1504
1505 template <typename _U0, typename _U1>
1506 [[__gnu__::__always_inline__]]
1507 static constexpr basic_mask
1508 _S_init(const __trivial_pair<_U0, _U1>& __bits)
1509 {
1510 if constexpr (is_unsigned_v<_U0>)
1511 {
1512 static_assert(is_unsigned_v<_U1>);
1513 return _S_init(_Mask0(__bits._M_first), _Mask1(__bits._M_second));
1514 }
1515 else if constexpr (is_unsigned_v<_U1>)
1516 return _S_init(_Mask0::_S_init(__bits._M_first), _Mask1(__bits._M_second));
1517 else
1518 return _S_init(_Mask0::_S_init(__bits._M_first), _Mask1::_S_init(__bits._M_second));
1519 }
1520
1521 [[__gnu__::__always_inline__]]
1522 constexpr const _Mask0&
1523 _M_get_low() const
1524 { return _M_data0; }
1525
1526 [[__gnu__::__always_inline__]]
1527 constexpr const _Mask1&
1528 _M_get_high() const
1529 { return _M_data1; }
1530
1531 template <size_t _UBytes, typename _UAbi>
1532 [[__gnu__::__always_inline__]]
1533 static constexpr basic_mask
1534 _S_recursive_bit_cast(const basic_mask<_UBytes, _UAbi>& __x)
1535 {
1536 using _Mp = basic_mask<_UBytes, _UAbi>;
1537 if constexpr (_Mp::_S_has_bool_member || sizeof(basic_mask) > sizeof(__x)
1538 || _Mp::_S_padding_bytes != 0)
1539 return _S_init(__builtin_bit_cast(_Mask0, __x._M_data0),
1540 _Mask1::_S_recursive_bit_cast(__x._M_data1));
1541 else if constexpr (sizeof(basic_mask) == sizeof(__x))
1542 return __builtin_bit_cast(basic_mask, __x);
1543 else
1544 { // e.g. on IvyBridge (different alignment => different sizeof)
1545 struct _Tmp { alignas(_Mp) basic_mask _M_data; };
1546 return __builtin_bit_cast(_Tmp, __x)._M_data;
1547 }
1548 }
1549
1550 [[__gnu__::__always_inline__]]
1551 constexpr auto
1552 _M_concat_data(bool __do_sanitize = _S_is_partial) const
1553 {
1554 if constexpr (_S_use_bitmask)
1555 {
1556 static_assert(_S_size <= numeric_limits<unsigned long long>::digits,
1557 "cannot concat more than 64 bits");
1558 using _Up = _Bitmask<_S_size>;
1559 return _Up(_M_data0._M_concat_data() | (_Up(_M_data1._M_concat_data(__do_sanitize)) << _N0));
1560 }
1561 else
1562 {
1563 auto __lo = _M_data0._M_concat_data();
1564 auto __hi = __vec_zero_pad_to<sizeof(__lo)>(_M_data1._M_concat_data(__do_sanitize));
1565 return __vec_concat(__lo, __hi);
1566 }
1567 }
1568
1569 template <_ArchTraits _Traits = {}>
1570 [[__gnu__::__always_inline__]]
1571 static constexpr basic_mask
1572 _S_partial_mask_of_n(int __n)
1573 {
1574#if __has_builtin(__builtin_ia32_bzhi_di)
1575 if constexpr (_S_use_bitmask && _S_size <= 64 && _Traits._M_have_bmi2())
1576 return basic_mask(__builtin_ia32_bzhi_di(~0ull >> (64 - _S_size), unsigned(__n)));
1577#endif
1578 if constexpr (_N0 == 1)
1579 {
1580 static_assert(_S_size == 2); // => __n == 1
1581 return _S_init(_Mask0(true), _Mask1(false));
1582 }
1583 else if (__n < _N0)
1584 return _S_init(_Mask0::_S_partial_mask_of_n(__n), _Mask1(false));
1585 else if (__n == _N0 || _N1 == 1)
1586 return _S_init(_Mask0(true), _Mask1(false));
1587 else if constexpr (_N1 != 1)
1588 return _S_init(_Mask0(true), _Mask1::_S_partial_mask_of_n(__n - _N0));
1589 }
1590
1591 [[__gnu__::__always_inline__]]
1592 constexpr basic_mask&
1593 _M_and_neighbors()
1594 {
1595 if constexpr (_S_size == 2)
1596 {
1597 static_assert(_S_is_scalar);
1598 _M_data0 = _M_data1 = _M_data0 && _M_data1;
1599 }
1600 else
1601 {
1602 _M_data0._M_and_neighbors();
1603 _M_data1._M_and_neighbors();
1604 }
1605 return *this;
1606 }
1607
1608 [[__gnu__::__always_inline__]]
1609 constexpr basic_mask&
1610 _M_or_neighbors()
1611 {
1612 if constexpr (_S_size == 2)
1613 {
1614 static_assert(_S_is_scalar);
1615 _M_data0 = _M_data1 = _M_data0 || _M_data1;
1616 }
1617 else
1618 {
1619 _M_data0._M_or_neighbors();
1620 _M_data1._M_or_neighbors();
1621 }
1622 return *this;
1623 }
1624
1625 template <typename _Mp>
1626 [[__gnu__::__always_inline__]]
1627 constexpr auto
1628 _M_chunk() const noexcept
1629 {
1630 constexpr int __n = _S_size / _Mp::_S_size;
1631 constexpr int __rem = _S_size % _Mp::_S_size;
1632 constexpr auto [...__is] = _IotaArray<__n>;
1633 if constexpr (__rem == 0)
1634 return array<_Mp, __n>{__extract_simd_at<_Mp>(cw<_Mp::_S_size * __is>,
1635 _M_data0, _M_data1)...};
1636 else
1637 {
1638 using _Rest = resize_t<__rem, _Mp>;
1639 return tuple(__extract_simd_at<_Mp>(cw<_Mp::_S_size * __is>, _M_data0, _M_data1)...,
1640 __extract_simd_at<_Rest>(cw<_Mp::_S_size * __n>, _M_data0, _M_data1));
1641 }
1642 }
1643
1644 [[__gnu__::__always_inline__]]
1645 static constexpr basic_mask
1646 _S_concat(const basic_mask& __x0) noexcept
1647 { return __x0; }
1648
1649 template <typename... _As>
1650 requires (sizeof...(_As) >= 2)
1651 [[__gnu__::__always_inline__]]
1652 static constexpr basic_mask
1653 _S_concat(const basic_mask<_Bytes, _As>&... __xs) noexcept
1654 {
1655 static_assert(_S_size == (_As::_S_size + ...));
1656 return _S_init(__extract_simd_at<_Mask0>(cw<0>, __xs...),
1657 __extract_simd_at<_Mask1>(cw<_N0>, __xs...));
1658 }
1659
1660 // [simd.mask.overview] default constructor -----------------------------
1661 basic_mask() = default;
1662
1663 // [simd.mask.overview] conversion extensions ---------------------------
1664 // TODO: any?
1665
1666 // [simd.mask.ctor] broadcast constructor -------------------------------
1667 [[__gnu__::__always_inline__]]
1668 constexpr explicit
1669 basic_mask(same_as<bool> auto __x) noexcept // LWG 4382.
1670 : _M_data0(__x), _M_data1(__x)
1671 {}
1672
1673 // [simd.mask.ctor] conversion constructor ------------------------------
1674 template <size_t _UBytes, typename _UAbi>
1675 requires (_S_size == _UAbi::_S_size) && (_UAbi::_S_is_cx_ctgus)
1676 [[__gnu__::__always_inline__]]
1677 constexpr explicit(__is_mask_conversion_explicit<_Ap, _UAbi>(_Bytes, _UBytes))
1678 basic_mask(const basic_mask<_UBytes, _UAbi>& __x) noexcept
1679 : basic_mask(__x._M_data) // unwrap _CxCtgus basic_mask partial specialization
1680 {}
1681
1682
1683 template <size_t _UBytes, typename _UAbi>
1684 requires (_S_size == _UAbi::_S_size) && (!_UAbi::_S_is_cx_ctgus)
1685 [[__gnu__::__always_inline__]]
1686 constexpr explicit(__is_mask_conversion_explicit<_Ap, _UAbi>(_Bytes, _UBytes))
1687 basic_mask(const basic_mask<_UBytes, _UAbi>& __x) noexcept
1688 : _M_data0([&] {
1689 if constexpr (_UAbi::_S_nreg > 1)
1690 {
1691 if constexpr (_UAbi::_S_is_cx_ileav)
1692 return __to_cx_ileav(__x._M_data._M_data0);
1693 else
1694 return __x._M_data0;
1695 }
1696 else if constexpr (_N0 == 1)
1697 return _Mask0(__x[0]);
1698 else
1699 return get<0>(chunk<_N0>(__x));
1700 }()),
1701 _M_data1([&] {
1702 if constexpr (_UAbi::_S_nreg > 1)
1703 {
1704 if constexpr (_UAbi::_S_is_cx_ileav)
1705 return __to_cx_ileav(__x._M_data._M_data1);
1706 else
1707 return __x._M_data1;
1708 }
1709 else if constexpr (_N1 == 1)
1710 return _Mask1(__x[_N0]);
1711 else
1712 return get<1>(chunk<_N0>(__x));
1713 }())
1714 {}
1715
1716 using _Base::_MaskBase;
1717
1718 // [simd.mask.ctor] generator constructor -------------------------------
1719 template <__simd_generator_invokable<bool, _S_size> _Fp>
1720 [[__gnu__::__always_inline__]]
1721 constexpr explicit
1722 basic_mask(_Fp&& __gen)
1723 : _M_data0(__gen), _M_data1([&] [[__gnu__::__always_inline__]] (auto __i) {
1724 return __gen(__simd_size_c<__i + _N0>);
1725 })
1726 {}
1727
1728 // [simd.mask.ctor] bitset constructor ----------------------------------
1729 [[__gnu__::__always_inline__]]
1730 constexpr
1731 basic_mask(const same_as<bitset<_S_size>> auto& __b) noexcept // LWG 4382.
1732 : _M_data0(__bitset_split<_N0>(__b)._M_lo), _M_data1(__bitset_split<_N0>(__b)._M_hi)
1733 {}
1734
1735 // [simd.mask.ctor] uint constructor ------------------------------------------
1736 template <unsigned_integral _Tp>
1737 requires (!same_as<_Tp, bool>) // LWG 4382.
1738 [[__gnu__::__always_inline__]]
1739 constexpr explicit
1740 basic_mask(_Tp __val) noexcept
1741 : _M_data0(static_cast<_Bitmask<_N0>>(__val)),
1742 _M_data1(sizeof(_Tp) * __CHAR_BIT__ > _N0
1743 ? static_cast<_Bitmask<_N1>>(__val >> _N0) : _Bitmask<_N1>())
1744 {}
1745
1746 // [simd.mask.subscr] ---------------------------------------------------
1747 [[__gnu__::__always_inline__]]
1748 constexpr value_type
1749 operator[](__simd_size_type __i) const
1750 {
1751 __glibcxx_simd_precondition(__i >= 0 && __i < _S_size, "subscript is out of bounds");
1752 if (__is_const_known(__i))
1753 return __i < _N0 ? _M_data0[__i] : _M_data1[__i - _N0];
1754 else if constexpr (_M_data1._S_has_bool_member)
1755 // in some cases the last element can be 'bool' instead of bit-/vector-mask;
1756 // e.g. mask<short, 17> is {mask<short, 16>, mask<short, 1>}, where the latter uses
1757 // _Abi<1, 1>, which is stored as 'bool'
1758 return __i < _N0 ? _M_data0[__i] : _M_data1[__i - _N0];
1759 else if constexpr (abi_type::_S_is_bitmask)
1760 {
1761 using _AliasingByte [[__gnu__::__may_alias__]] = unsigned char;
1762 return bool((reinterpret_cast<const _AliasingByte*>(this)
1763 [__i / __CHAR_BIT__] >> (__i % __CHAR_BIT__)) & 1);
1764 }
1765 else
1766 {
1767 using _AliasingInt [[__gnu__::__may_alias__]] = __integer_from<_Bytes>;
1768 return reinterpret_cast<const _AliasingInt*>(this)[__i] != 0;
1769 }
1770 }
1771
1772 // [simd.mask.unary] ----------------------------------------------------
1773 [[__gnu__::__always_inline__]]
1774 constexpr basic_mask
1775 operator!() const noexcept
1776 { return _S_init(!_M_data0, !_M_data1); }
1777
1778 [[__gnu__::__always_inline__]]
1779 constexpr _VecType
1780 operator+() const noexcept requires destructible<_VecType>
1781 { return _VecType::_S_concat(+_M_data0, +_M_data1); }
1782
1783 constexpr _VecType
1784 operator+() const noexcept = delete;
1785
1786 [[__gnu__::__always_inline__]]
1787 constexpr _VecType
1788 operator-() const noexcept requires destructible<_VecType>
1789 { return _VecType::_S_concat(-_M_data0, -_M_data1); }
1790
1791 constexpr _VecType
1792 operator-() const noexcept = delete;
1793
1794 [[__gnu__::__always_inline__]]
1795 constexpr _VecType
1796 operator~() const noexcept requires destructible<_VecType>
1797 { return _VecType::_S_concat(~_M_data0, ~_M_data1); }
1798
1799 constexpr _VecType
1800 operator~() const noexcept = delete;
1801
1802 // [simd.mask.conv] -----------------------------------------------------
1803 template <typename _Up, typename _UAbi>
1804 requires (_UAbi::_S_size == _S_size)
1805 [[__gnu__::__always_inline__]]
1806 constexpr explicit(sizeof(_Up) != _Bytes)
1807 operator basic_vec<_Up, _UAbi>() const noexcept
1808 {
1809 using _Rp = basic_vec<_Up, _UAbi>;
1810 return _Rp::_S_init(static_cast<_Rp::_DataType0>(_M_data0),
1811 static_cast<_Rp::_DataType1>(_M_data1));
1812 }
1813
1814 using _Base::operator basic_vec;
1815
1816 // [simd.mask.namedconv] ------------------------------------------------
1817 [[__gnu__::__always_inline__]]
1818 constexpr bitset<_S_size>
1819 to_bitset() const noexcept
1820 {
1821 if constexpr (_S_size <= numeric_limits<unsigned long long>::digits)
1822 return to_ullong();
1823 else
1824 {
1825 static_assert(_N0 % numeric_limits<unsigned long long>::digits == 0);
1826 struct _Tmp
1827 {
1828 bitset<_N0> _M_lo;
1829 bitset<_N1> _M_hi;
1830 } __tmp = {_M_data0.to_bitset(), _M_data1.to_bitset()};
1831 return __builtin_bit_cast(bitset<_S_size>, __tmp);
1832 }
1833 }
1834
1835 template <int _Offset = 0, bool _Use_2_for_1 = false, _ArchTraits _Traits = {}>
1836 [[__gnu__::__always_inline__]]
1837 constexpr auto
1838 _M_to_uint() const
1839 {
1840 constexpr int _N0x = _Use_2_for_1 ? _N0 / 2 : _N0;
1841 if constexpr (_Use_2_for_1 && _S_is_scalar && _S_size == 2)
1842 return _M_data1.template _M_to_uint<_Offset>();
1843 else if constexpr (_N0x >= numeric_limits<unsigned long long>::digits)
1844 {
1845 static_assert(_Offset == 0);
1846 return __trivial_pair {
1847 _M_data0.template _M_to_uint<0, _Use_2_for_1>(),
1848 _M_data1.template _M_to_uint<0, _Use_2_for_1>()
1849 };
1850 }
1851 else
1852 {
1853#if _GLIBCXX_X86
1854 if constexpr (_Bytes == 2 && !_Traits._M_have_bmi2() && _Ap::_S_nreg == 2
1855 && !_S_is_scalar && !_S_use_bitmask && !_Use_2_for_1)
1856 return __similar_mask<char, _S_size, _Ap>(*this).template _M_to_uint<_Offset>();
1857#endif
1858 auto __uint = _M_data1.template _M_to_uint<_N0x + _Offset, _Use_2_for_1>();
1859 __uint |= _M_data0.template _M_to_uint<_Offset, _Use_2_for_1>();
1860 return __uint;
1861 }
1862 }
1863
1864 [[__gnu__::__always_inline__]]
1865 constexpr unsigned long long
1866 to_ullong() const
1867 {
1868 if constexpr (_S_size <= numeric_limits<unsigned long long>::digits)
1869 return _M_to_uint();
1870 else
1871 {
1872 __glibcxx_simd_precondition(_M_data1.to_ullong() == 0,
1873 "to_ullong called on mask with 'true' elements at indices"
1874 "higher than representable in a ullong");
1875 return _M_data0.to_ullong();
1876 }
1877 }
1878
1879 // [simd.mask.binary]
1880 [[__gnu__::__always_inline__]]
1881 friend constexpr basic_mask
1882 operator&&(const basic_mask& __x, const basic_mask& __y) noexcept
1883 { return _S_init(__x._M_data0 && __y._M_data0, __x._M_data1 && __y._M_data1); }
1884
1885 [[__gnu__::__always_inline__]]
1886 friend constexpr basic_mask
1887 operator||(const basic_mask& __x, const basic_mask& __y) noexcept
1888 { return _S_init(__x._M_data0 || __y._M_data0, __x._M_data1 || __y._M_data1); }
1889
1890 [[__gnu__::__always_inline__]]
1891 friend constexpr basic_mask
1892 operator&(const basic_mask& __x, const basic_mask& __y) noexcept
1893 { return _S_init(__x._M_data0 & __y._M_data0, __x._M_data1 & __y._M_data1); }
1894
1895 [[__gnu__::__always_inline__]]
1896 friend constexpr basic_mask
1897 operator|(const basic_mask& __x, const basic_mask& __y) noexcept
1898 { return _S_init(__x._M_data0 | __y._M_data0, __x._M_data1 | __y._M_data1); }
1899
1900 [[__gnu__::__always_inline__]]
1901 friend constexpr basic_mask
1902 operator^(const basic_mask& __x, const basic_mask& __y) noexcept
1903 { return _S_init(__x._M_data0 ^ __y._M_data0, __x._M_data1 ^ __y._M_data1); }
1904
1905 // [simd.mask.cassign]
1906 [[__gnu__::__always_inline__]]
1907 friend constexpr basic_mask&
1908 operator&=(basic_mask& __x, const basic_mask& __y) noexcept
1909 {
1910 __x._M_data0 &= __y._M_data0;
1911 __x._M_data1 &= __y._M_data1;
1912 return __x;
1913 }
1914
1915 [[__gnu__::__always_inline__]]
1916 friend constexpr basic_mask&
1917 operator|=(basic_mask& __x, const basic_mask& __y) noexcept
1918 {
1919 __x._M_data0 |= __y._M_data0;
1920 __x._M_data1 |= __y._M_data1;
1921 return __x;
1922 }
1923
1924 [[__gnu__::__always_inline__]]
1925 friend constexpr basic_mask&
1926 operator^=(basic_mask& __x, const basic_mask& __y) noexcept
1927 {
1928 __x._M_data0 ^= __y._M_data0;
1929 __x._M_data1 ^= __y._M_data1;
1930 return __x;
1931 }
1932
1933 // [simd.mask.comparison] -----------------------------------------------
1934 [[__gnu__::__always_inline__]]
1935 friend constexpr basic_mask
1936 operator==(const basic_mask& __x, const basic_mask& __y) noexcept
1937 { return !(__x ^ __y); }
1938
1939 [[__gnu__::__always_inline__]]
1940 friend constexpr basic_mask
1941 operator!=(const basic_mask& __x, const basic_mask& __y) noexcept
1942 { return __x ^ __y; }
1943
1944 [[__gnu__::__always_inline__]]
1945 friend constexpr basic_mask
1946 operator>=(const basic_mask& __x, const basic_mask& __y) noexcept
1947 { return __x || !__y; }
1948
1949 [[__gnu__::__always_inline__]]
1950 friend constexpr basic_mask
1951 operator<=(const basic_mask& __x, const basic_mask& __y) noexcept
1952 { return !__x || __y; }
1953
1954 [[__gnu__::__always_inline__]]
1955 friend constexpr basic_mask
1956 operator>(const basic_mask& __x, const basic_mask& __y) noexcept
1957 { return __x && !__y; }
1958
1959 [[__gnu__::__always_inline__]]
1960 friend constexpr basic_mask
1961 operator<(const basic_mask& __x, const basic_mask& __y) noexcept
1962 { return !__x && __y; }
1963
1964 // [simd.mask.cond] -----------------------------------------------------
1965 [[__gnu__::__always_inline__]]
1966 friend constexpr basic_mask
1967 __select_impl(const basic_mask& __k, const basic_mask& __t, const basic_mask& __f) noexcept
1968 {
1969 return _S_init(__select_impl(__k._M_data0, __t._M_data0, __f._M_data0),
1970 __select_impl(__k._M_data1, __t._M_data1, __f._M_data1));
1971 }
1972
1973 [[__gnu__::__always_inline__]]
1974 friend constexpr basic_mask
1975 __select_impl(const basic_mask& __k, same_as<bool> auto __t, same_as<bool> auto __f) noexcept
1976 {
1977 if (__t == __f)
1978 return basic_mask(__t);
1979 else
1980 return __t ? __k : !__k;
1981 }
1982
1983 template <__vectorizable _T0, same_as<_T0> _T1>
1984 requires (sizeof(_T0) == _Bytes)
1985 [[__gnu__::__always_inline__]]
1986 friend constexpr vec<_T0, _S_size>
1987 __select_impl(const basic_mask& __k, const _T0& __t, const _T1& __f) noexcept
1988 {
1989 using _Vp = vec<_T0, _S_size>;
1990 if constexpr (!is_same_v<basic_mask, typename _Vp::mask_type>)
1991 return __select_impl(static_cast<_Vp::mask_type>(__k), __t, __f);
1992 else if constexpr (__complex_like<_T0>)
1993 return _Vp::_S_concat(__select_impl(__k._M_data0, __t, __f),
1994 __select_impl(__k._M_data1, __t, __f));
1995 else
1996 return _Vp::_S_init(__select_impl(__k._M_data0, __t, __f),
1997 __select_impl(__k._M_data1, __t, __f));
1998 }
1999
2000 template <_ArchTraits _Traits = {}>
2001 [[__gnu__::__always_inline__]]
2002 constexpr bool
2003 _M_all_of() const
2004 {
2005 if constexpr (_N0 == _N1)
2006 return (_M_data0 && _M_data1)._M_all_of();
2007 else
2008 return _M_data0._M_all_of() && _M_data1._M_all_of();
2009 }
2010
2011 template <_ArchTraits _Traits = {}>
2012 [[__gnu__::__always_inline__]]
2013 constexpr bool
2014 _M_any_of() const
2015 {
2016 if constexpr (_N0 == _N1)
2017 return (_M_data0 || _M_data1)._M_any_of();
2018 else
2019 return _M_data0._M_any_of() || _M_data1._M_any_of();
2020 }
2021
2022 template <_ArchTraits _Traits = {}>
2023 [[__gnu__::__always_inline__]]
2024 constexpr bool
2025 _M_none_of() const
2026 {
2027 if constexpr (_N0 == _N1)
2028 return (_M_data0 || _M_data1)._M_none_of();
2029 else
2030 return _M_data0._M_none_of() && _M_data1._M_none_of();
2031 }
2032
2033 [[__gnu__::__always_inline__]]
2034 constexpr __simd_size_type
2035 _M_reduce_count() const noexcept
2036 {
2037 if constexpr (_S_is_scalar)
2038 // SWAR could help. I don't think we care at the moment.
2039 return _M_data0._M_reduce_count() + _M_data1._M_reduce_count();
2040 else if constexpr (_S_size <= numeric_limits<unsigned>::digits)
2041 return __builtin_popcount(_M_to_uint());
2042 else if constexpr (_S_size <= numeric_limits<unsigned long long>::digits)
2043 return __builtin_popcountll(to_ullong());
2044 else
2045 return _M_data0._M_reduce_count() + _M_data1._M_reduce_count();
2046 }
2047
2048 [[__gnu__::__always_inline__]]
2049 constexpr __simd_size_type
2050 _M_reduce_min_index() const
2051 {
2052 if constexpr (_S_size <= numeric_limits<unsigned long long>::digits)
2053 {
2054 const auto __bits = _M_to_uint();
2055 __glibcxx_simd_precondition(__bits, "An empty mask does not have a min_index.");
2056 return __countr_zero(_M_to_uint());
2057 }
2058 else if (_M_data0._M_none_of())
2059 return _M_data1._M_reduce_min_index() + _N0;
2060 else
2061 return _M_data0._M_reduce_min_index();
2062 }
2063
2064 [[__gnu__::__always_inline__]]
2065 constexpr __simd_size_type
2066 _M_reduce_max_index() const
2067 {
2068 if constexpr (_S_size <= numeric_limits<unsigned long long>::digits)
2069 {
2070 const auto __bits = _M_to_uint();
2071 __glibcxx_simd_precondition(__bits, "An empty mask does not have a max_index.");
2072 return __highest_bit(_M_to_uint());
2073 }
2074 else if (_M_data1._M_none_of())
2075 return _M_data0._M_reduce_max_index();
2076 else
2077 return _M_data1._M_reduce_max_index() + _N0;
2078 }
2079
2080 [[__gnu__::__always_inline__]]
2081 friend constexpr bool
2082 __is_const_known(const basic_mask& __x)
2083 { return __is_const_known(__x._M_data0) && __is_const_known(__x._M_data1); }
2084 };
2085} // namespace simd
2086_GLIBCXX_END_NAMESPACE_VERSION
2087} // namespace std
2088
2089#pragma GCC diagnostic pop
2090#endif // C++26
2091#endif // _GLIBCXX_SIMD_MASK_H
std::chrono::operator<=
constexpr bool operator<=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:863
std::chrono::operator>=
constexpr bool operator>=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:877
std::chrono::operator<
constexpr bool operator<(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:830
std::chrono::operator>
constexpr bool operator>(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition chrono.h:870
std::operator-
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
Definition complex:404
std::operator+
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
Definition complex:374
std::conditional_t
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition type_traits:2971
std::max
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
Definition stl_algobase.h:256
std
ISO C++ entities toplevel namespace is std.
std::cend
constexpr auto cend(const _Container &__cont) noexcept(noexcept(std::end(__cont))) -> decltype(std::end(__cont))
Return an iterator pointing to one past the last element of the const container.
Definition range_access.h:153
std::end
constexpr auto end(_Container &__cont) noexcept(noexcept(__cont.end())) -> decltype(__cont.end())
Return an iterator pointing to one past the last element of the container.
Definition range_access.h:80
std::size
constexpr auto size(const _Container &__cont) noexcept(noexcept(__cont.size())) -> decltype(__cont.size())
Return the size of a container.
Definition range_access.h:294
std::operator^
constexpr bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1682
std::cbegin
constexpr auto cbegin(const _Container &__cont) noexcept(noexcept(std::begin(__cont))) -> decltype(std::begin(__cont))
Return an iterator pointing to the first element of the const container.
Definition range_access.h:140
std::operator|
constexpr bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1672
std::begin
constexpr auto begin(_Container &__cont) noexcept(noexcept(__cont.begin())) -> decltype(__cont.begin())
Return an iterator pointing to the first element of the container.
Definition range_access.h:55
std::operator&
constexpr bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition bitset:1662
std::numeric_limits::max
static constexpr _Tp max() noexcept
Definition limits:336
std::numeric_limits::min
static constexpr _Tp min() noexcept
Definition limits:332