libstdc++: simd_converter.h Source File

libstdc++

simd_converter.h
1// Generic simd conversions -*- C++ -*-
2
3// Copyright (C) 2020-2026 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_
26#define _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_
27
28#if __cplusplus >= 201703L
29
30_GLIBCXX_SIMD_BEGIN_NAMESPACE
31
32template <typename _Arg, typename _Ret, typename _To, size_t _Np>
33 _Ret __converter_fallback(_Arg __a)
34 {
35 _Ret __ret{};
36 __execute_n_times<_Np>(
37 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
38 __ret._M_set(__i, static_cast<_To>(__a[__i]));
39 });
40 return __ret;
41 }
42
43// _SimdConverter scalar -> scalar {{{
44template <typename _From, typename _To>
45 struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::scalar,
46 enable_if_t<!is_same_v<_From, _To>>>
47 {
48 _GLIBCXX_SIMD_INTRINSIC constexpr _To operator()(_From __a) const noexcept
49 { return static_cast<_To>(__a); }
50 };
51
52// }}}
53// _SimdConverter scalar -> "native" {{{
54template <typename _From, typename _To, typename _Abi>
55 struct _SimdConverter<_From, simd_abi::scalar, _To, _Abi,
56 enable_if_t<!is_same_v<_Abi, simd_abi::scalar>>>
57 {
58 using _Ret = typename _Abi::template __traits<_To>::_SimdMember;
59
60 template <typename... _More>
61 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret
62 operator()(_From __a, _More... __more) const noexcept
63 {
64 static_assert(sizeof...(_More) + 1 == _Abi::template _S_size<_To>);
65 static_assert(conjunction_v<is_same<_From, _More>...>);
66 return __make_vector<_To>(__a, __more...);
67 }
68 };
69
70// }}}
71// _SimdConverter "native non-sve 1" -> "native non-sve 2" {{{
72template <typename _From, typename _To, typename _AFrom, typename _ATo>
73 struct _SimdConverter<
74 _From, _AFrom, _To, _ATo,
75 enable_if_t<!disjunction_v<
76 __is_fixed_size_abi<_AFrom>, __is_fixed_size_abi<_ATo>,
77 is_same<_AFrom, simd_abi::scalar>, is_same<_ATo, simd_abi::scalar>,
78 conjunction<is_same<_From, _To>, is_same<_AFrom, _ATo>>>
79 && !(__is_sve_abi<_AFrom>() || __is_sve_abi<_ATo>())>>
80 {
81 using _Arg = typename _AFrom::template __traits<_From>::_SimdMember;
82 using _Ret = typename _ATo::template __traits<_To>::_SimdMember;
83 using _V = __vector_type_t<_To, simd_size_v<_To, _ATo>>;
84
85 template <typename... _More>
86 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret
87 operator()(_Arg __a, _More... __more) const noexcept
88 { return __vector_convert<_V>(__a, __more...); }
89 };
90
91// }}}
92// _SimdConverter "native 1" -> "native 2" {{{
93template <typename _From, typename _To, typename _AFrom, typename _ATo>
94 struct _SimdConverter<
95 _From, _AFrom, _To, _ATo,
96 enable_if_t<!disjunction_v<
97 __is_fixed_size_abi<_AFrom>, __is_fixed_size_abi<_ATo>,
98 is_same<_AFrom, simd_abi::scalar>, is_same<_ATo, simd_abi::scalar>,
99 conjunction<is_same<_From, _To>, is_same<_AFrom, _ATo>>>
100 && (__is_sve_abi<_AFrom>() || __is_sve_abi<_ATo>())
101 >>
102 {
103 using _Arg = typename _AFrom::template __traits<_From>::_SimdMember;
104 using _Ret = typename _ATo::template __traits<_To>::_SimdMember;
105
106 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret
107 operator()(_Arg __x) const noexcept
108 { return __converter_fallback<_Arg, _Ret, _To, simd_size_v<_To, _ATo>>(__x); }
109 };
110
111// }}}
112// _SimdConverter scalar -> fixed_size<1> {{{1
113template <typename _From, typename _To>
114 struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::fixed_size<1>,
115 void>
116 {
117 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_To, simd_abi::scalar>
118 operator()(_From __x) const noexcept
119 { return {static_cast<_To>(__x)}; }
120 };
121
122// _SimdConverter fixed_size<1> -> scalar {{{1
123template <typename _From, typename _To>
124 struct _SimdConverter<_From, simd_abi::fixed_size<1>, _To, simd_abi::scalar,
125 void>
126 {
127 _GLIBCXX_SIMD_INTRINSIC constexpr _To
128 operator()(_SimdTuple<_From, simd_abi::scalar> __x) const noexcept
129 { return {static_cast<_To>(__x.first)}; }
130 };
131
132// _SimdConverter fixed_size<_Np> -> fixed_size<_Np> {{{1
133template <typename _From, typename _To, int _Np>
134 struct _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To,
135 simd_abi::fixed_size<_Np>,
136 enable_if_t<!is_same_v<_From, _To>>>
137 {
138 using _Ret = __fixed_size_storage_t<_To, _Np>;
139 using _Arg = __fixed_size_storage_t<_From, _Np>;
140
141 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret
142 operator()(const _Arg& __x) const noexcept
143 {
144 if constexpr (is_same_v<_From, _To>)
145 return __x;
146
147 // fallback to sequential when sve is available
148 else if constexpr (__have_sve)
149 return __converter_fallback<_Arg, _Ret, _To, _Np>(__x);
150
151 // special case (optimize) int signedness casts
152 else if constexpr (sizeof(_From) == sizeof(_To)
153 && is_integral_v<_From> && is_integral_v<_To>)
154 return __bit_cast<_Ret>(__x);
155
156 // special case if all ABI tags in _Ret are scalar
157 else if constexpr (__is_scalar_abi<typename _Ret::_FirstAbi>())
158 {
159 return __call_with_subscripts(
160 __x, make_index_sequence<_Np>(),
161 [](auto... __values) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA -> _Ret {
162 return __make_simd_tuple<_To, decltype((void) __values,
163 simd_abi::scalar())...>(
164 static_cast<_To>(__values)...);
165 });
166 }
167
168 // from one vector to one vector
169 else if constexpr (_Arg::_S_first_size == _Ret::_S_first_size)
170 {
171 _SimdConverter<_From, typename _Arg::_FirstAbi, _To,
172 typename _Ret::_FirstAbi>
173 __native_cvt;
174 if constexpr (_Arg::_S_tuple_size == 1)
175 return {__native_cvt(__x.first)};
176 else
177 {
178 constexpr size_t _NRemain = _Np - _Arg::_S_first_size;
179 _SimdConverter<_From, simd_abi::fixed_size<_NRemain>, _To,
180 simd_abi::fixed_size<_NRemain>>
181 __remainder_cvt;
182 return {__native_cvt(__x.first), __remainder_cvt(__x.second)};
183 }
184 }
185
186 // from one vector to multiple vectors
187 else if constexpr (_Arg::_S_first_size > _Ret::_S_first_size)
188 {
189 const auto __multiple_return_chunks
190 = __convert_all<__vector_type_t<_To, _Ret::_S_first_size>>(
191 __x.first);
192 constexpr auto __converted = __multiple_return_chunks.size()
193 * _Ret::_FirstAbi::template _S_size<_To>;
194 constexpr auto __remaining = _Np - __converted;
195 if constexpr (_Arg::_S_tuple_size == 1 && __remaining == 0)
196 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks);
197 else if constexpr (_Arg::_S_tuple_size == 1)
198 { // e.g. <int, 3> -> <double, 2, 1> or <short, 7> -> <double, 4, 2,
199 // 1>
200 using _RetRem
201 = __remove_cvref_t<decltype(__simd_tuple_pop_front<__converted>(
202 _Ret()))>;
203 const auto __return_chunks2
204 = __convert_all<__vector_type_t<_To, _RetRem::_S_first_size>, 0,
205 __converted>(__x.first);
206 constexpr auto __converted2
207 = __converted
208 + __return_chunks2.size() * _RetRem::_S_first_size;
209 if constexpr (__converted2 == _Np)
210 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks,
211 __return_chunks2);
212 else
213 {
214 using _RetRem2 = __remove_cvref_t<
215 decltype(__simd_tuple_pop_front<__return_chunks2.size()
216 * _RetRem::_S_first_size>(
217 _RetRem()))>;
218 const auto __return_chunks3 = __convert_all<
219 __vector_type_t<_To, _RetRem2::_S_first_size>, 0,
220 __converted2>(__x.first);
221 constexpr auto __converted3
222 = __converted2
223 + __return_chunks3.size() * _RetRem2::_S_first_size;
224 if constexpr (__converted3 == _Np)
225 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks,
226 __return_chunks2,
227 __return_chunks3);
228 else
229 {
230 using _RetRem3
231 = __remove_cvref_t<decltype(__simd_tuple_pop_front<
232 __return_chunks3.size()
233 * _RetRem2::_S_first_size>(
234 _RetRem2()))>;
235 const auto __return_chunks4 = __convert_all<
236 __vector_type_t<_To, _RetRem3::_S_first_size>, 0,
237 __converted3>(__x.first);
238 constexpr auto __converted4
239 = __converted3
240 + __return_chunks4.size() * _RetRem3::_S_first_size;
241 if constexpr (__converted4 == _Np)
242 return __to_simd_tuple<_To, _Np>(
243 __multiple_return_chunks, __return_chunks2,
244 __return_chunks3, __return_chunks4);
245 else
246 __assert_unreachable<_To>();
247 }
248 }
249 }
250 else
251 {
252 constexpr size_t _NRemain = _Np - _Arg::_S_first_size;
253 _SimdConverter<_From, simd_abi::fixed_size<_NRemain>, _To,
254 simd_abi::fixed_size<_NRemain>>
255 __remainder_cvt;
256 return __simd_tuple_concat(
257 __to_simd_tuple<_To, _Arg::_S_first_size>(
258 __multiple_return_chunks),
259 __remainder_cvt(__x.second));
260 }
261 }
262
263 // from multiple vectors to one vector
264 // _Arg::_S_first_size < _Ret::_S_first_size
265 // a) heterogeneous input at the end of the tuple (possible with partial
266 // native registers in _Ret)
267 else if constexpr (_Ret::_S_tuple_size == 1
268 && _Np % _Arg::_S_first_size != 0)
269 {
270 static_assert(_Ret::_FirstAbi::template _S_is_partial<_To>);
271 return _Ret{__generate_from_n_evaluations<
272 _Np, typename _VectorTraits<typename _Ret::_FirstType>::type>(
273 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
274 return static_cast<_To>(__x[__i]);
275 })};
276 }
277 else
278 {
279 static_assert(_Arg::_S_tuple_size > 1);
280 constexpr auto __n
281 = __div_roundup(_Ret::_S_first_size, _Arg::_S_first_size);
282 return __call_with_n_evaluations<__n>(
283 [&__x](auto... __uncvted) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
284 // assuming _Arg Abi tags for all __i are _Arg::_FirstAbi
285 _SimdConverter<_From, typename _Arg::_FirstAbi, _To,
286 typename _Ret::_FirstAbi>
287 __native_cvt;
288 if constexpr (_Ret::_S_tuple_size == 1)
289 return _Ret{__native_cvt(__uncvted...)};
290 else
291 return _Ret{
292 __native_cvt(__uncvted...),
293 _SimdConverter<
294 _From, simd_abi::fixed_size<_Np - _Ret::_S_first_size>, _To,
295 simd_abi::fixed_size<_Np - _Ret::_S_first_size>>()(
296 __simd_tuple_pop_front<_Ret::_S_first_size>(__x))};
297 }, [&__x](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
298 return __get_tuple_at<__i>(__x);
299 });
300 }
301 }
302 };
303
304// _SimdConverter "native" -> fixed_size<_Np> {{{1
305// i.e. 1 register to ? registers
306template <typename _From, typename _Ap, typename _To, int _Np>
307 struct _SimdConverter<_From, _Ap, _To, simd_abi::fixed_size<_Np>,
308 enable_if_t<!__is_fixed_size_abi_v<_Ap>>>
309 {
310 static_assert(
311 _Np == simd_size_v<_From, _Ap>,
312 "_SimdConverter to fixed_size only works for equal element counts");
313
314 using _Ret = __fixed_size_storage_t<_To, _Np>;
315 using _Arg = typename _SimdTraits<_From, _Ap>::_SimdMember;
316
317 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret
318 operator()(_Arg __x) const noexcept
319 {
320 if constexpr (__have_sve)
321 return __converter_fallback<_Arg, _Ret, _To, _Np>(__x);
322 else if constexpr (_Ret::_S_tuple_size == 1)
323 return {__vector_convert<typename _Ret::_FirstType::_BuiltinType>(__x)};
324 else
325 {
326 using _FixedNp = simd_abi::fixed_size<_Np>;
327 _SimdConverter<_From, _FixedNp, _To, _FixedNp> __fixed_cvt;
328 using _FromFixedStorage = __fixed_size_storage_t<_From, _Np>;
329 if constexpr (_FromFixedStorage::_S_tuple_size == 1)
330 return __fixed_cvt(_FromFixedStorage{__x});
331 else if constexpr (_FromFixedStorage::_S_tuple_size == 2)
332 {
333 _FromFixedStorage __tmp;
334 static_assert(sizeof(__tmp) <= sizeof(__x));
335 __builtin_memcpy(&__tmp.first, &__x, sizeof(__tmp.first));
336 __builtin_memcpy(&__tmp.second.first,
337 reinterpret_cast<const char*>(&__x)
338 + sizeof(__tmp.first),
339 sizeof(__tmp.second.first));
340 return __fixed_cvt(__tmp);
341 }
342 else
343 __assert_unreachable<_From>();
344 }
345 }
346 };
347
348// _SimdConverter fixed_size<_Np> -> "native" {{{1
349// i.e. ? register to 1 registers
350template <typename _From, int _Np, typename _To, typename _Ap>
351 struct _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To, _Ap,
352 enable_if_t<!__is_fixed_size_abi_v<_Ap>>>
353 {
354 static_assert(
355 _Np == simd_size_v<_To, _Ap>,
356 "_SimdConverter to fixed_size only works for equal element counts");
357
358 using _Arg = __fixed_size_storage_t<_From, _Np>;
359 using _Ret = typename _SimdTraits<_To, _Ap>::_SimdMember;
360
361 _GLIBCXX_SIMD_INTRINSIC constexpr
362 _Ret
363 operator()(const _Arg& __x) const noexcept
364 {
365 if constexpr(__have_sve)
366 return __converter_fallback<_Arg, _Ret, _To, _Np>(__x);
367 else if constexpr (_Arg::_S_tuple_size == 1)
368 return __vector_convert<__vector_type_t<_To, _Np>>(__x.first);
369 else if constexpr (_Arg::_S_is_homogeneous)
370 return __call_with_n_evaluations<_Arg::_S_tuple_size>(
371 [](auto... __members) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
372 if constexpr ((is_convertible_v<decltype(__members), _To> && ...))
373 return __vector_type_t<_To, _Np>{static_cast<_To>(__members)...};
374 else
375 return __vector_convert<__vector_type_t<_To, _Np>>(__members...);
376 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
377 return __get_tuple_at<__i>(__x);
378 });
379 else if constexpr (__fixed_size_storage_t<_To, _Np>::_S_tuple_size == 1)
380 {
381 _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To,
382 simd_abi::fixed_size<_Np>>
383 __fixed_cvt;
384 return __fixed_cvt(__x).first;
385 }
386 else
387 {
388 const _SimdWrapper<_From, _Np> __xv
389 = __generate_from_n_evaluations<_Np, __vector_type_t<_From, _Np>>(
390 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
391 return __vector_convert<__vector_type_t<_To, _Np>>(__xv);
392 }
393 }
394 };
395
396// }}}1
397_GLIBCXX_SIMD_END_NAMESPACE
398#endif // __cplusplus >= 201703L
399#endif // _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_
400
401// vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80
std::enable_if_t
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
Definition type_traits:2969