39#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_
40#define _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_
42#if __cplusplus >= 201703L
46_GLIBCXX_SIMD_BEGIN_NAMESPACE
49template <
size_t _I,
typename _Tp>
50 struct __simd_tuple_element;
52template <
typename _Tp,
typename _A0,
typename... _As>
53 struct __simd_tuple_element<0, _SimdTuple<_Tp, _A0, _As...>>
54 {
using type = simd<_Tp, _A0>; };
56template <
size_t _I,
typename _Tp,
typename _A0,
typename... _As>
57 struct __simd_tuple_element<_I, _SimdTuple<_Tp, _A0, _As...>>
58 {
using type =
typename __simd_tuple_element<_I - 1, _SimdTuple<_Tp, _As...>>::type; };
60template <
size_t _I,
typename _Tp>
61 using __simd_tuple_element_t =
typename __simd_tuple_element<_I, _Tp>::type;
66template <
typename _Tp,
typename... _A0s,
typename... _A1s>
67 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdTuple<_Tp, _A0s..., _A1s...>
68 __simd_tuple_concat(
const _SimdTuple<_Tp, _A0s...>& __left,
69 const _SimdTuple<_Tp, _A1s...>& __right)
71 if constexpr (
sizeof...(_A0s) == 0)
73 else if constexpr (
sizeof...(_A1s) == 0)
76 return {__left.first, __simd_tuple_concat(__left.second, __right)};
79template <
typename _Tp,
typename _A10,
typename... _A1s>
80 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdTuple<_Tp, simd_abi::scalar, _A10, _A1s...>
81 __simd_tuple_concat(
const _Tp& __left,
const _SimdTuple<_Tp, _A10, _A1s...>& __right)
82 {
return {__left, __right}; }
88template <
size_t _Np,
typename _Tp>
89 _GLIBCXX_SIMD_INTRINSIC
constexpr decltype(
auto)
90 __simd_tuple_pop_front(_Tp&& __x)
92 if constexpr (_Np == 0)
93 return static_cast<_Tp&&
>(__x);
96 using _Up = __remove_cvref_t<_Tp>;
97 static_assert(_Np >= _Up::_S_first_size);
98 return __simd_tuple_pop_front<_Np - _Up::_S_first_size>(__x.second);
106struct __as_simd_tuple {};
108template <
typename _Tp,
typename _A0,
typename... _Abis>
109 _GLIBCXX_SIMD_INTRINSIC
constexpr simd<_Tp, _A0>
110 __simd_tuple_get_impl(__as_simd,
const _SimdTuple<_Tp, _A0, _Abis...>& __t, _SizeConstant<0>)
111 {
return {__private_init, __t.first}; }
113template <
typename _Tp,
typename _A0,
typename... _Abis>
114 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
115 __simd_tuple_get_impl(__as_simd_tuple,
const _SimdTuple<_Tp, _A0, _Abis...>& __t,
117 {
return __t.first; }
119template <
typename _Tp,
typename _A0,
typename... _Abis>
120 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
121 __simd_tuple_get_impl(__as_simd_tuple, _SimdTuple<_Tp, _A0, _Abis...>& __t, _SizeConstant<0>)
122 {
return __t.first; }
124template <
typename _R,
size_t _Np,
typename _Tp,
typename... _Abis>
125 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
126 __simd_tuple_get_impl(_R,
const _SimdTuple<_Tp, _Abis...>& __t, _SizeConstant<_Np>)
127 {
return __simd_tuple_get_impl(_R(), __t.second, _SizeConstant<_Np - 1>()); }
129template <
size_t _Np,
typename _Tp,
typename... _Abis>
130 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
131 __simd_tuple_get_impl(__as_simd_tuple, _SimdTuple<_Tp, _Abis...>& __t, _SizeConstant<_Np>)
132 {
return __simd_tuple_get_impl(__as_simd_tuple(), __t.second, _SizeConstant<_Np - 1>()); }
134template <
size_t _Np,
typename _Tp,
typename... _Abis>
135 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
136 __get_simd_at(
const _SimdTuple<_Tp, _Abis...>& __t)
137 {
return __simd_tuple_get_impl(__as_simd(), __t, _SizeConstant<_Np>()); }
141template <
size_t _Np,
typename _Tp,
typename... _Abis>
142 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
143 __get_tuple_at(
const _SimdTuple<_Tp, _Abis...>& __t)
144 {
return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_Np>()); }
146template <
size_t _Np,
typename _Tp,
typename... _Abis>
147 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
148 __get_tuple_at(_SimdTuple<_Tp, _Abis...>& __t)
149 {
return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_Np>()); }
152template <
typename _Tp,
typename _Abi,
size_t _Offset>
153 struct __tuple_element_meta :
public _Abi::_SimdImpl
155 static_assert(is_same_v<
typename _Abi::_SimdImpl::abi_type,
158 using value_type = _Tp;
159 using abi_type = _Abi;
160 using _Traits = _SimdTraits<_Tp, _Abi>;
161 using _MaskImpl =
typename _Abi::_MaskImpl;
162 using _MaskMember =
typename _Traits::_MaskMember;
163 using simd_type = simd<_Tp, _Abi>;
164 static constexpr size_t _S_offset = _Offset;
165 static constexpr size_t _S_size() {
return simd_size<_Tp, _Abi>::value; }
166 static constexpr _MaskImpl _S_mask_impl = {};
168 template <
size_t _Np,
bool _Sanitized>
169 _GLIBCXX_SIMD_INTRINSIC
static constexpr auto
170 _S_submask(_BitMask<_Np, _Sanitized> __bits)
171 {
return __bits.template _M_extract<_Offset, _S_size()>(); }
173 template <
size_t _Np,
bool _Sanitized>
174 _GLIBCXX_SIMD_INTRINSIC
static constexpr _MaskMember
175 _S_make_mask(_BitMask<_Np, _Sanitized> __bits)
177 return _MaskImpl::template _S_convert<_Tp>(
178 __bits.template _M_extract<_Offset, _S_size()>()._M_sanitized());
181 _GLIBCXX_SIMD_INTRINSIC
static constexpr _ULLong
182 _S_mask_to_shifted_ullong(_MaskMember __k)
183 {
return _MaskImpl::_S_to_bits(__k).to_ullong() << _Offset; }
186template <
size_t _Offset,
typename _Tp,
typename _Abi,
typename... _As>
187 _GLIBCXX_SIMD_INTRINSIC
constexpr
188 __tuple_element_meta<_Tp, _Abi, _Offset>
189 __make_meta(
const _SimdTuple<_Tp, _Abi, _As...>&)
194template <
size_t _Offset,
typename _Base>
195 struct _WithOffset :
public _Base
197 static inline constexpr size_t _S_offset = _Offset;
199 _GLIBCXX_SIMD_INTRINSIC
char*
201 {
return reinterpret_cast<char*
>(
this) + _S_offset *
sizeof(
typename _Base::value_type); }
203 _GLIBCXX_SIMD_INTRINSIC
const char*
204 _M_as_charptr()
const
205 {
return reinterpret_cast<const char*
>(
this) + _S_offset *
sizeof(
typename _Base::value_type); }
209template <
size_t _O0,
size_t _O1,
typename _Base>
210 struct _WithOffset<_O0, _WithOffset<_O1, _Base>> {};
212template <
size_t _Offset,
typename _Tp>
213 _GLIBCXX_SIMD_INTRINSIC
215 __add_offset(_Tp& __base)
216 {
return static_cast<_WithOffset<_Offset, __remove_cvref_t<_Tp>
>&>(
__base); }
218template <
size_t _Offset,
typename _Tp>
219 _GLIBCXX_SIMD_INTRINSIC
221 __add_offset(
const _Tp& __base)
222 {
return static_cast<const _WithOffset<_Offset, __remove_cvref_t<_Tp>
>&>(
__base); }
224template <
size_t _Offset,
size_t _ExistingOffset,
typename _Tp>
225 _GLIBCXX_SIMD_INTRINSIC
227 __add_offset(_WithOffset<_ExistingOffset, _Tp>& __base)
228 {
return static_cast<_WithOffset<_Offset + _ExistingOffset, _Tp>&
>(
static_cast<_Tp&
>(
__base)); }
230template <
size_t _Offset,
size_t _ExistingOffset,
typename _Tp>
231 _GLIBCXX_SIMD_INTRINSIC
233 __add_offset(
const _WithOffset<_ExistingOffset, _Tp>& __base)
235 return static_cast<const _WithOffset<_Offset + _ExistingOffset, _Tp>&
>(
236 static_cast<const _Tp&
>(
__base));
239template <
typename _Tp>
240 constexpr inline size_t __offset = 0;
242template <
size_t _Offset,
typename _Tp>
243 constexpr inline size_t __offset<_WithOffset<_Offset, _Tp>>
244 = _WithOffset<_Offset, _Tp>::_S_offset;
246template <
typename _Tp>
247 constexpr inline size_t __offset<const _Tp> = __offset<_Tp>;
249template <
typename _Tp>
250 constexpr inline size_t __offset<_Tp&> = __offset<_Tp>;
252template <
typename _Tp>
253 constexpr inline size_t __offset<_Tp&&> = __offset<_Tp>;
258template <
typename _Tp>
259 struct _SimdTuple<_Tp>
261 using value_type = _Tp;
262 static constexpr size_t _S_tuple_size = 0;
263 static constexpr size_t _S_size() {
return 0; }
267template <
typename _FirstType,
typename _SecondType>
268 struct _SimdTupleData
273 _GLIBCXX_SIMD_INTRINSIC
275 _M_is_constprop()
const
277 if constexpr (is_class_v<_FirstType>)
278 return first._M_is_constprop() && second._M_is_constprop();
280 return __builtin_constant_p(first) && second._M_is_constprop();
284template <
typename _FirstType,
typename _Tp>
285 struct _SimdTupleData<_FirstType, _SimdTuple<_Tp>>
288 static constexpr _SimdTuple<_Tp> second = {};
290 _GLIBCXX_SIMD_INTRINSIC
292 _M_is_constprop()
const
294 if constexpr (is_class_v<_FirstType>)
295 return first._M_is_constprop();
297 return __builtin_constant_p(first);
302template <
typename _Tp,
typename _Abi0,
typename... _Abis>
303 struct _SimdTuple<_Tp, _Abi0, _Abis...>
304 : _SimdTupleData<typename _SimdTraits<_Tp, _Abi0>::_SimdMember,
305 _SimdTuple<_Tp, _Abis...>>
307 static_assert(!__is_fixed_size_abi_v<_Abi0>);
308 using value_type = _Tp;
309 using _FirstType =
typename _SimdTraits<_Tp, _Abi0>::_SimdMember;
310 using _FirstAbi = _Abi0;
311 using _SecondType = _SimdTuple<_Tp, _Abis...>;
312 static constexpr size_t _S_tuple_size =
sizeof...(_Abis) + 1;
314 static constexpr size_t _S_size()
315 {
return simd_size_v<_Tp, _Abi0> + _SecondType::_S_size(); }
317 static constexpr size_t _S_first_size = simd_size_v<_Tp, _Abi0>;
318 static constexpr bool _S_is_homogeneous = (is_same_v<_Abi0, _Abis> && ...);
320 using _Base = _SimdTupleData<typename _SimdTraits<_Tp, _Abi0>::_SimdMember,
321 _SimdTuple<_Tp, _Abis...>>;
325 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdTuple() =
default;
326 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdTuple(
const _SimdTuple&) =
default;
327 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdTuple& operator=(
const _SimdTuple&)
330 template <
typename _Up>
331 _GLIBCXX_SIMD_INTRINSIC
constexpr
332 _SimdTuple(_Up&& __x)
333 : _Base{static_cast<_Up&&>(__x)} {}
335 template <
typename _Up,
typename _Up2>
336 _GLIBCXX_SIMD_INTRINSIC
constexpr
337 _SimdTuple(_Up&& __x, _Up2&& __y)
338 : _Base{static_cast<_Up&&>(__x), static_cast<_Up2&&>(__y)} {}
340 template <
typename _Up>
341 _GLIBCXX_SIMD_INTRINSIC
constexpr
342 _SimdTuple(_Up&& __x, _SimdTuple<_Tp>)
343 : _Base{static_cast<_Up&&>(__x)} {}
345 _GLIBCXX_SIMD_INTRINSIC
char*
347 {
return reinterpret_cast<char*
>(
this); }
349 _GLIBCXX_SIMD_INTRINSIC
const char*
350 _M_as_charptr()
const
351 {
return reinterpret_cast<const char*
>(
this); }
353 template <
size_t _Np>
354 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
357 if constexpr (_Np == 0)
360 return second.template _M_at<_Np - 1>();
363 template <
size_t _Np>
364 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
367 if constexpr (_Np == 0)
370 return second.template _M_at<_Np - 1>();
373 template <
size_t _Np>
374 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
377 if constexpr (_Np == 0)
378 return simd<_Tp, _Abi0>(__private_init, first);
380 return second.template _M_simd_at<_Np - 1>();
383 template <
size_t _Offset = 0,
typename _Fp>
384 _GLIBCXX_SIMD_INTRINSIC
static constexpr _SimdTuple
385 _S_generate(_Fp&& __gen, _SizeConstant<_Offset> = {})
387 auto&& __first = __gen(__tuple_element_meta<_Tp, _Abi0, _Offset>());
388 if constexpr (_S_tuple_size == 1)
392 _SecondType::_S_generate(
393 static_cast<_Fp&&
>(__gen),
394 _SizeConstant<_Offset + simd_size_v<_Tp, _Abi0>>())};
397 template <
size_t _Offset = 0,
typename _Fp,
typename... _More>
398 _GLIBCXX_SIMD_INTRINSIC _SimdTuple
399 _M_apply_wrapped(_Fp&& __fun,
const _More&... __more)
const
402 = __fun(__make_meta<_Offset>(*
this), first, __more.first...);
403 if constexpr (_S_tuple_size == 1)
408 second.template _M_apply_wrapped<_Offset + simd_size_v<_Tp, _Abi0>>(
409 static_cast<_Fp&&
>(__fun), __more.second...)};
412 template <
typename _Tup>
413 _GLIBCXX_SIMD_INTRINSIC
constexpr decltype(
auto)
414 _M_extract_argument(_Tup&& __tup)
const
416 using _TupT =
typename __remove_cvref_t<_Tup>::value_type;
417 if constexpr (is_same_v<_SimdTuple, __remove_cvref_t<_Tup>>)
419 else if (__builtin_is_constant_evaluated())
420 return __fixed_size_storage_t<_TupT, _S_first_size>::_S_generate(
421 [&](
auto __meta)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
422 return __meta._S_generator(
423 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
425 },
static_cast<_TupT*
>(
nullptr));
429 __fixed_size_storage_t<_TupT, _S_first_size> __r;
430 __builtin_memcpy(__r._M_as_charptr(), __tup._M_as_charptr(),
436 template <
typename _Tup>
437 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
438 _M_skip_argument(_Tup&& __tup)
const
440 static_assert(_S_tuple_size > 1);
441 using _Up = __remove_cvref_t<_Tup>;
442 constexpr size_t __off = __offset<_Up>;
443 if constexpr (_S_first_size == _Up::_S_first_size && __off == 0)
445 else if constexpr (_S_first_size > _Up::_S_first_size
446 && _S_first_size % _Up::_S_first_size == 0
448 return __simd_tuple_pop_front<_S_first_size>(__tup);
449 else if constexpr (_S_first_size + __off < _Up::_S_first_size)
450 return __add_offset<_S_first_size>(__tup);
451 else if constexpr (_S_first_size + __off == _Up::_S_first_size)
454 __assert_unreachable<_Tup>();
457 template <
size_t _Offset,
typename... _More>
458 _GLIBCXX_SIMD_INTRINSIC
constexpr void
459 _M_assign_front(
const _SimdTuple<_Tp, _Abi0, _More...>& __x) &
461 static_assert(_Offset == 0);
463 if constexpr (
sizeof...(_More) > 0)
465 static_assert(
sizeof...(_Abis) >=
sizeof...(_More));
466 second.template _M_assign_front<0>(__x.second);
470 template <
size_t _Offset>
471 _GLIBCXX_SIMD_INTRINSIC
constexpr void
472 _M_assign_front(
const _FirstType& __x) &
474 static_assert(_Offset == 0);
478 template <
size_t _Offset,
typename... _As>
479 _GLIBCXX_SIMD_INTRINSIC
constexpr void
480 _M_assign_front(
const _SimdTuple<_Tp, _As...>& __x) &
482 __builtin_memcpy(_M_as_charptr() + _Offset *
sizeof(value_type),
484 sizeof(_Tp) * _SimdTuple<_Tp, _As...>::_S_size());
492 template <
typename _Fp,
typename... _More>
493 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdTuple
494 _M_apply_per_chunk(_Fp&& __fun, _More&&... __more)
const
498 is_lvalue_reference<_More>,
499 negation<is_const<remove_reference_t<_More>>>>) )
502 auto&& __first = [&](
auto... __args)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
503 auto __r = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
505 [[maybe_unused]]
auto&& __ignore_me = {(
506 [](
auto&& __dst,
const auto& __src) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
507 if constexpr (is_assignable_v<
decltype(__dst),
510 __dst.template _M_assign_front<__offset<
decltype(__dst)>>(
513 }(
static_cast<_More&&
>(__more), __args),
516 }(_M_extract_argument(__more)...);
517 if constexpr (_S_tuple_size == 1)
521 second._M_apply_per_chunk(
static_cast<_Fp&&
>(__fun),
522 _M_skip_argument(__more)...)};
526 auto&& __first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
527 _M_extract_argument(__more)...);
528 if constexpr (_S_tuple_size == 1)
532 second._M_apply_per_chunk(
static_cast<_Fp&&
>(__fun),
533 _M_skip_argument(__more)...)};
537 template <
typename _R = _Tp,
typename _Fp,
typename... _More>
538 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
539 _M_apply_r(_Fp&& __fun,
const _More&... __more)
const
541 auto&& __first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
543 if constexpr (_S_tuple_size == 1)
546 return __simd_tuple_concat<_R>(
547 __first, second.template _M_apply_r<_R>(
static_cast<_Fp&&
>(__fun),
551 template <
typename _Fp,
typename... _More>
552 _GLIBCXX_SIMD_INTRINSIC
constexpr friend _SanitizedBitMask<_S_size()>
553 _M_test(
const _Fp& __fun,
const _SimdTuple& __x,
const _More&... __more)
555 const _SanitizedBitMask<_S_first_size> __first
556 = _Abi0::_MaskImpl::_S_to_bits(
557 __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first,
559 if constexpr (_S_tuple_size == 1)
562 return _M_test(__fun, __x.second, __more.second...)
563 ._M_prepend(__first);
566 template <
typename _Up, _Up _I>
567 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
568 operator[](integral_constant<_Up, _I>)
const noexcept
570 if constexpr (_I < simd_size_v<_Tp, _Abi0>)
571 return _M_subscript_read(_I);
573 return second[integral_constant<_Up, _I - simd_size_v<_Tp, _Abi0>>()];
576 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
577 operator[](
size_t __i)
const noexcept
579 if constexpr (_S_tuple_size == 1)
580 return _M_subscript_read(__i);
581#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
582 else if (not __builtin_is_constant_evaluated())
583 return reinterpret_cast<const __may_alias<_Tp>*
>(
this)[__i];
585 else if constexpr (__is_scalar_abi<_Abi0>())
587 const _Tp* ptr = &first;
591 return __i < simd_size_v<_Tp, _Abi0> ? _M_subscript_read(__i)
592 : second[__i - simd_size_v<_Tp, _Abi0>];
595 _GLIBCXX_SIMD_INTRINSIC
constexpr void
596 _M_set(
size_t __i, _Tp __val)
noexcept
598 if constexpr (_S_tuple_size == 1)
599 return _M_subscript_write(__i, __val);
600#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
601 else if (not __builtin_is_constant_evaluated())
602 reinterpret_cast<__may_alias<_Tp>*
>(
this)[__i] = __val;
604 else if (__i < simd_size_v<_Tp, _Abi0>)
605 _M_subscript_write(__i, __val);
607 second._M_set(__i - simd_size_v<_Tp, _Abi0>, __val);
612 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
613 _M_subscript_read([[maybe_unused]]
size_t __i)
const noexcept
615 if constexpr (__is_vectorizable_v<_FirstType>)
621 _GLIBCXX_SIMD_INTRINSIC
constexpr void
622 _M_subscript_write([[maybe_unused]]
size_t __i, _Tp __y)
noexcept
624 if constexpr (__is_vectorizable_v<_FirstType>)
627 first._M_set(__i, __y);
634template <
typename _Tp,
typename _A0>
635 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdTuple<_Tp, _A0>
636 __make_simd_tuple(simd<_Tp, _A0> __x0)
637 {
return {__data(__x0)}; }
639template <
typename _Tp,
typename _A0,
typename... _As>
640 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdTuple<_Tp, _A0, _As...>
641 __make_simd_tuple(
const simd<_Tp, _A0>& __x0,
const simd<_Tp, _As>&... __xs)
642 {
return {__data(__x0), __make_simd_tuple(__xs...)}; }
644template <
typename _Tp,
typename _A0>
645 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdTuple<_Tp, _A0>
646 __make_simd_tuple(
const typename _SimdTraits<_Tp, _A0>::_SimdMember& __arg0)
649template <
typename _Tp,
typename _A0,
typename _A1,
typename... _Abis>
650 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdTuple<_Tp, _A0, _A1, _Abis...>
652 const typename _SimdTraits<_Tp, _A0>::_SimdMember& __arg0,
653 const typename _SimdTraits<_Tp, _A1>::_SimdMember& __arg1,
654 const typename _SimdTraits<_Tp, _Abis>::_SimdMember&... __args)
655 {
return {__arg0, __make_simd_tuple<_Tp, _A1, _Abis...>(__arg1, __args...)}; }
658template <
typename _Tp,
size_t _Np,
typename _V,
size_t _NV,
typename... _VX>
659 _GLIBCXX_SIMD_INTRINSIC
constexpr __fixed_size_storage_t<_Tp, _Np>
660 __to_simd_tuple(
const array<_V, _NV>& __from,
const _VX... __fromX);
662template <
typename _Tp,
size_t _Np,
664 typename _R = __fixed_size_storage_t<_Tp, _Np>,
typename _V0,
665 typename _V0VT = _VectorTraits<_V0>,
typename... _VX>
666 _GLIBCXX_SIMD_INTRINSIC _R
constexpr __to_simd_tuple(
const _V0 __from0,
const _VX... __fromX)
668 static_assert(is_same_v<typename _V0VT::value_type, _Tp>);
669 static_assert(_Offset < _V0VT::_S_full_size);
670 using _R0 = __vector_type_t<_Tp, _R::_S_first_size>;
671 if constexpr (_R::_S_tuple_size == 1)
673 if constexpr (_Np == 1)
674 return _R{__from0[_Offset]};
675 else if constexpr (_Offset == 0 && _V0VT::_S_full_size >= _Np)
676 return _R{__intrin_bitcast<_R0>(__from0)};
677 else if constexpr (_Offset * 2 == _V0VT::_S_full_size
678 && _V0VT::_S_full_size / 2 >= _Np)
679 return _R{__intrin_bitcast<_R0>(__extract_part<1, 2>(__from0))};
680 else if constexpr (_Offset * 4 == _V0VT::_S_full_size
681 && _V0VT::_S_full_size / 4 >= _Np)
682 return _R{__intrin_bitcast<_R0>(__extract_part<1, 4>(__from0))};
684 __assert_unreachable<_Tp>();
688 if constexpr (1 == _R::_S_first_size)
690 if constexpr (_Offset + 1 < _V0VT::_S_full_size)
691 return _R{__from0[_Offset],
692 __to_simd_tuple<_Tp, _Np - 1, _Offset + 1>(__from0,
695 return _R{__from0[_Offset],
696 __to_simd_tuple<_Tp, _Np - 1, 0>(__fromX...)};
700 else if constexpr (_V0VT::_S_full_size == _R::_S_first_size
703 __to_simd_tuple<_Tp, _Np - _R::_S_first_size>(__fromX...)};
706 else if constexpr (_V0VT::_S_full_size > _R::_S_first_size
708 return _R{__intrin_bitcast<_R0>(__from0),
709 __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
710 _R::_S_first_size>(__from0, __fromX...)};
714 else if constexpr (_Offset * 4 == _V0VT::_S_full_size
715 && _V0VT::_S_full_size >= 4 * _R::_S_first_size)
716 return _R{__intrin_bitcast<_R0>(__extract_part<2, 4>(__from0)),
717 __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
718 _Offset + _R::_S_first_size>(__from0,
723 else if constexpr (_Offset * 2 == _V0VT::_S_full_size
724 && _V0VT::_S_full_size >= 4 * _R::_S_first_size)
725 return _R{__intrin_bitcast<_R0>(__extract_part<2, 4>(__from0)),
726 __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
727 _Offset + _R::_S_first_size>(__from0,
731 else if constexpr (_Offset * 2 == _V0VT::_S_full_size
732 && _V0VT::_S_full_size / 2 >= _R::_S_first_size)
733 return _R{__intrin_bitcast<_R0>(__extract_part<1, 2>(__from0)),
734 __to_simd_tuple<_Tp, _Np - _R::_S_first_size, 0>(
739 __assert_unreachable<_Tp>();
743template <
typename _Tp,
size_t _Np,
typename _V,
size_t _NV,
typename... _VX>
744 _GLIBCXX_SIMD_INTRINSIC
constexpr __fixed_size_storage_t<_Tp, _Np>
745 __to_simd_tuple(
const array<_V, _NV>& __from,
const _VX... __fromX)
747 if constexpr (is_same_v<_Tp, _V>)
751 "An array of scalars must be the last argument to __to_simd_tuple");
752 return __call_with_subscripts(
753 __from, make_index_sequence<_NV>(),
754 [&](
const auto... __args)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
755 return __simd_tuple_concat(
756 _SimdTuple<_Tp, simd_abi::scalar>{__args}..., _SimdTuple<_Tp>());
760 return __call_with_subscripts(
761 __from, make_index_sequence<_NV>(),
762 [&](
const auto... __args)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
763 return __to_simd_tuple<_Tp, _Np>(__args..., __fromX...);
767template <
size_t,
typename _Tp>
768 using __to_tuple_helper = _Tp;
770template <
typename _Tp,
typename _A0,
size_t _NOut,
size_t _Np,
772 _GLIBCXX_SIMD_INTRINSIC __fixed_size_storage_t<_Tp, _NOut>
773 __to_simd_tuple_impl(index_sequence<_Indexes...>,
774 const array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>, _Np>& __args)
776 return __make_simd_tuple<_Tp, __to_tuple_helper<_Indexes, _A0>...>(
777 __args[_Indexes]...);
780template <
typename _Tp,
typename _A0,
size_t _NOut,
size_t _Np,
781 typename _R = __fixed_size_storage_t<_Tp, _NOut>>
782 _GLIBCXX_SIMD_INTRINSIC _R
783 __to_simd_tuple_sized(
784 const array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>, _Np>& __args)
786 static_assert(_Np * simd_size_v<_Tp, _A0> >= _NOut);
787 return __to_simd_tuple_impl<_Tp, _A0, _NOut>(
788 make_index_sequence<_R::_S_tuple_size>(), __args);
792template <
typename _Tp>
793 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdTuple<_Tp>
794 __optimize_simd_tuple(
const _SimdTuple<_Tp>)
797template <
typename _Tp,
typename _Ap>
798 _GLIBCXX_SIMD_INTRINSIC
constexpr const _SimdTuple<_Tp, _Ap>&
799 __optimize_simd_tuple(
const _SimdTuple<_Tp, _Ap>& __x)
802template <
typename _Tp,
typename _A0,
typename _A1,
typename... _Abis,
803 typename _R = __fixed_size_storage_t<
804 _Tp, _SimdTuple<_Tp, _A0, _A1, _Abis...>::_S_size()>>
805 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
806 __optimize_simd_tuple(
const _SimdTuple<_Tp, _A0, _A1, _Abis...>& __x)
808 using _Tup = _SimdTuple<_Tp, _A0, _A1, _Abis...>;
809 if constexpr (is_same_v<_R, _Tup>)
811 else if constexpr (is_same_v<
typename _R::_FirstType,
812 typename _Tup::_FirstType>)
813 return {__x.first, __optimize_simd_tuple(__x.second)};
814 else if constexpr (__is_scalar_abi<_A0>()
815 || _A0::template _S_is_partial<_Tp>)
816 return {__generate_from_n_evaluations<_R::_S_first_size,
817 typename _R::_FirstType>(
818 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; }),
819 __optimize_simd_tuple(
820 __simd_tuple_pop_front<_R::_S_first_size>(__x))};
821 else if constexpr (is_same_v<_A0, _A1>
822 && _R::_S_first_size == simd_size_v<_Tp, _A0> + simd_size_v<_Tp, _A1>)
823 return {__concat(__x.template _M_at<0>(), __x.template _M_at<1>()),
824 __optimize_simd_tuple(__x.second.second)};
825 else if constexpr (
sizeof...(_Abis) >= 2
826 && _R::_S_first_size == (4 * simd_size_v<_Tp, _A0>)
827 && simd_size_v<_Tp, _A0> == __simd_tuple_element_t<
828 (
sizeof...(_Abis) >= 2 ? 3 : 0), _Tup>::size())
830 __concat(__concat(__x.template _M_at<0>(), __x.template _M_at<1>()),
831 __concat(__x.template _M_at<2>(), __x.template _M_at<3>())),
832 __optimize_simd_tuple(__x.second.second.second.second)};
835 static_assert(
sizeof(_R) ==
sizeof(__x));
837 __builtin_memcpy(__r._M_as_charptr(), __x._M_as_charptr(),
838 sizeof(_Tp) * _R::_S_size());
844template <
size_t _Offset = 0,
typename _Tp,
typename _A0,
typename _Fp>
845 _GLIBCXX_SIMD_INTRINSIC
constexpr void
846 __for_each(
const _SimdTuple<_Tp, _A0>& __t, _Fp&& __fun)
847 {
static_cast<_Fp&&
>(__fun)(__make_meta<_Offset>(__t), __t.first); }
849template <
size_t _Offset = 0,
typename _Tp,
typename _A0,
typename _A1,
850 typename... _As,
typename _Fp>
851 _GLIBCXX_SIMD_INTRINSIC
constexpr void
852 __for_each(
const _SimdTuple<_Tp, _A0, _A1, _As...>& __t, _Fp&& __fun)
854 __fun(__make_meta<_Offset>(__t), __t.first);
855 __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second,
856 static_cast<_Fp&&
>(__fun));
860template <
size_t _Offset = 0,
typename _Tp,
typename _A0,
typename _Fp>
861 _GLIBCXX_SIMD_INTRINSIC
constexpr void
862 __for_each(_SimdTuple<_Tp, _A0>& __t, _Fp&& __fun)
863 {
static_cast<_Fp&&
>(__fun)(__make_meta<_Offset>(__t), __t.first); }
865template <
size_t _Offset = 0,
typename _Tp,
typename _A0,
typename _A1,
866 typename... _As,
typename _Fp>
867 _GLIBCXX_SIMD_INTRINSIC
constexpr void
868 __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __t, _Fp&& __fun)
870 __fun(__make_meta<_Offset>(__t), __t.first);
871 __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second,
872 static_cast<_Fp&&
>(__fun));
876template <
size_t _Offset = 0,
typename _Tp,
typename _A0,
typename _Fp>
877 _GLIBCXX_SIMD_INTRINSIC
constexpr void
878 __for_each(_SimdTuple<_Tp, _A0>& __a,
const _SimdTuple<_Tp, _A0>& __b, _Fp&& __fun)
879 {
static_cast<_Fp&&
>(__fun)(__make_meta<_Offset>(__a), __a.first, __b.first); }
881template <
size_t _Offset = 0,
typename _Tp,
typename _A0,
typename _A1,
882 typename... _As,
typename _Fp>
883 _GLIBCXX_SIMD_INTRINSIC
constexpr void
884 __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __a,
885 const _SimdTuple<_Tp, _A0, _A1, _As...>& __b, _Fp&& __fun)
887 __fun(__make_meta<_Offset>(__a), __a.first, __b.first);
888 __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second,
889 static_cast<_Fp&&
>(__fun));
893template <
size_t _Offset = 0,
typename _Tp,
typename _A0,
typename _Fp>
894 _GLIBCXX_SIMD_INTRINSIC
constexpr void
895 __for_each(
const _SimdTuple<_Tp, _A0>& __a,
const _SimdTuple<_Tp, _A0>& __b, _Fp&& __fun)
896 {
static_cast<_Fp&&
>(__fun)(__make_meta<_Offset>(__a), __a.first, __b.first); }
898template <
size_t _Offset = 0,
typename _Tp,
typename _A0,
typename _A1,
899 typename... _As,
typename _Fp>
900 _GLIBCXX_SIMD_INTRINSIC
constexpr void
901 __for_each(
const _SimdTuple<_Tp, _A0, _A1, _As...>& __a,
902 const _SimdTuple<_Tp, _A0, _A1, _As...>& __b, _Fp&& __fun)
904 __fun(__make_meta<_Offset>(__a), __a.first, __b.first);
905 __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second,
906 static_cast<_Fp&&
>(__fun));
911template <
int _Index,
int _Total,
int _Combine,
typename _Tp,
typename _A0,
typename... _As>
912 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
913 __extract_part(
const _SimdTuple<_Tp, _A0, _As...>& __x)
919 using _Tuple = _SimdTuple<_Tp, _A0, _As...>;
920 static_assert(_Index + _Combine <= _Total && _Index >= 0 && _Total >= 1);
921 constexpr size_t _Np = _Tuple::_S_size();
922 static_assert(_Np >= _Total && _Np % _Total == 0);
923 constexpr size_t __values_per_part = _Np / _Total;
924 [[maybe_unused]]
constexpr size_t __values_to_skip
925 = _Index * __values_per_part;
926 constexpr size_t __return_size = __values_per_part * _Combine;
927 using _RetAbi = simd_abi::deduce_t<_Tp, __return_size>;
930 if constexpr (_Index == 0 && _Tuple::_S_first_size == __return_size)
931 return __x.first._M_data;
932 else if constexpr (_Index == 0 && _Total == _Combine)
934 else if constexpr (_Index == 0 && _Tuple::_S_first_size >= __return_size)
935 return __intrin_bitcast<__vector_type_t<_Tp, __return_size>>(
936 __as_vector(__x.first));
939 else if constexpr (__values_to_skip >= _Tuple::_S_first_size)
941 if constexpr (_Tuple::_S_first_size % __values_per_part == 0)
943 constexpr int __parts_in_first
944 = _Tuple::_S_first_size / __values_per_part;
945 return __extract_part<_Index - __parts_in_first,
946 _Total - __parts_in_first, _Combine>(
950 return __extract_part<__values_to_skip - _Tuple::_S_first_size,
951 _Np - _Tuple::_S_first_size, __return_size>(
956 else if constexpr (__return_size > _Tuple::_S_first_size - __values_to_skip)
958#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
959 const __may_alias<_Tp>*
const element_ptr
960 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__x) + __values_to_skip;
961 return __as_vector(simd<_Tp, _RetAbi>(element_ptr, element_aligned));
963 [[maybe_unused]]
constexpr size_t __offset = __values_to_skip;
964 return __as_vector(simd<_Tp, _RetAbi>(
965 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
966 constexpr _SizeConstant<__i + __offset> __k;
973 else if constexpr (_Tuple::_S_first_size % __values_per_part == 0)
974 return __extract_part<_Index, _Tuple::_S_first_size / __values_per_part,
975 _Combine>(__x.first);
977 return __extract_part<__values_to_skip, _Tuple::_S_first_size,
978 _Combine * __values_per_part>(__x.first);
983template <
typename _Tp,
int _Np,
typename _Tuple,
984 typename _Next = simd<_Tp, _AllNativeAbis::_BestAbi<_Tp, _Np>>,
985 int _Remain = _Np - int(_Next::size())>
986 struct __fixed_size_storage_builder;
988template <
typename _Tp,
int _Np>
989 struct __fixed_size_storage
990 :
public __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp>> {};
992template <
typename _Tp,
int _Np,
typename... _As,
typename _Next>
993 struct __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp, _As...>, _Next,
995 {
using type = _SimdTuple<_Tp, _As...,
typename _Next::abi_type>; };
997template <
typename _Tp,
int _Np,
typename... _As,
typename _Next,
int _Remain>
998 struct __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp, _As...>, _Next,
1001 using type =
typename __fixed_size_storage_builder<
1002 _Tp, _Remain, _SimdTuple<_Tp, _As...,
typename _Next::abi_type>>::type;
1007template <
typename _Tp,
bool = is_arithmetic_v<__remove_cvref_t<_Tp>>>
1008 struct __autocvt_to_simd
1011 using _TT = __remove_cvref_t<_Tp>;
1013 _GLIBCXX_SIMD_INTRINSIC
constexpr
1017 _GLIBCXX_SIMD_INTRINSIC
constexpr
1020 static_assert(is_lvalue_reference<_Tp>::value,
"");
1021 static_assert(!is_const<_Tp>::value,
"");
1025 _GLIBCXX_SIMD_INTRINSIC
constexpr
1028 static_assert(is_lvalue_reference<_Tp>::value,
"");
1029 static_assert(!is_const<_Tp>::value,
"");
1033 _GLIBCXX_SIMD_INTRINSIC
constexpr
1034 __autocvt_to_simd(_Tp dd) : _M_data(dd) {}
1036 template <
typename _Abi>
1037 _GLIBCXX_SIMD_INTRINSIC
constexpr
1038 operator simd<typename _TT::value_type, _Abi>()
1039 {
return {__private_init, _M_data}; }
1041 template <
typename _Abi>
1042 _GLIBCXX_SIMD_INTRINSIC
constexpr
1043 operator simd<typename _TT::value_type, _Abi>&()
1044 {
return *
reinterpret_cast<simd<typename _TT::value_type, _Abi>*
>(&_M_data); }
1046 template <
typename _Abi>
1047 _GLIBCXX_SIMD_INTRINSIC
constexpr
1048 operator simd<typename _TT::value_type, _Abi>*()
1049 {
return reinterpret_cast<simd<typename _TT::value_type, _Abi>*
>(&_M_data); }
1052template <
typename _Tp>
1053 __autocvt_to_simd(_Tp &&) -> __autocvt_to_simd<_Tp>;
1055template <
typename _Tp>
1056 struct __autocvt_to_simd<_Tp, true>
1058 using _TT = __remove_cvref_t<_Tp>;
1060 fixed_size_simd<_TT, 1> _M_fd;
1062 _GLIBCXX_SIMD_INTRINSIC
1063 constexpr __autocvt_to_simd(_Tp dd) : _M_data(dd), _M_fd(_M_data) {}
1065 _GLIBCXX_SIMD_INTRINSIC
1066 ~__autocvt_to_simd()
1067 { _M_data = __data(_M_fd).first; }
1069 _GLIBCXX_SIMD_INTRINSIC
constexpr
1070 operator fixed_size_simd<_TT, 1>()
1073 _GLIBCXX_SIMD_INTRINSIC
constexpr
1074 operator fixed_size_simd<_TT, 1> &()
1076 static_assert(is_lvalue_reference<_Tp>::value,
"");
1077 static_assert(!is_const<_Tp>::value,
"");
1081 _GLIBCXX_SIMD_INTRINSIC
constexpr
1082 operator fixed_size_simd<_TT, 1> *()
1084 static_assert(is_lvalue_reference<_Tp>::value,
"");
1085 static_assert(!is_const<_Tp>::value,
"");
1092struct _CommonImplFixedSize;
1093template <
int _Np,
typename = __detail::__odr_helper>
struct _SimdImplFixedSize;
1094template <
int _Np,
typename = __detail::__odr_helper>
struct _MaskImplFixedSize;
1097 struct simd_abi::_Fixed
1099 template <
typename _Tp>
static constexpr size_t _S_size = _Np;
1100 template <
typename _Tp>
static constexpr size_t _S_full_size = _Np;
1102 struct _IsValidAbiTag :
public __bool_constant<(_Np > 0)> {};
1104 template <
typename _Tp>
1105 struct _IsValidSizeFor
1106 : __bool_constant<(_Np <= simd_abi::max_fixed_size<_Tp>)> {};
1108 template <typename _Tp>
1109 struct _IsValid : conjunction<_IsValidAbiTag, __is_vectorizable<_Tp>,
1110 _IsValidSizeFor<_Tp>> {};
1112 template <typename _Tp>
1113 static constexpr bool _S_is_valid_v = _IsValid<_Tp>::value;
1117 _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
1118 _S_masked(_BitMask<_Np> __x)
1119 { return __x._M_sanitized(); }
1121 _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
1122 _S_masked(_SanitizedBitMask<_Np> __x)
1127 using _CommonImpl = _CommonImplFixedSize;
1128 using _SimdImpl = _SimdImplFixedSize<_Np>;
1129 using _MaskImpl = _MaskImplFixedSize<_Np>;
1133 template <typename _Tp, bool = _S_is_valid_v<_Tp>>
1134 struct __traits : _InvalidTraits {};
1136 template <typename _Tp>
1137 struct __traits<_Tp, true>
1139 using _IsValid = true_type;
1140 using _SimdImpl = _SimdImplFixedSize<_Np>;
1141 using _MaskImpl = _MaskImplFixedSize<_Np>;
1144 using _SimdMember = __fixed_size_storage_t<_Tp, _Np>;
1145 using _MaskMember = _SanitizedBitMask<_Np>;
1147 static constexpr size_t _S_simd_align
1148 = std::__bit_ceil(_Np * sizeof(_Tp));
1150 static constexpr size_t _S_mask_align = alignof(_MaskMember);
1158 _GLIBCXX_SIMD_ALWAYS_INLINE constexpr
1159 _SimdBase(const _SimdBase&) {}
1161 _SimdBase() = default;
1163 _GLIBCXX_SIMD_ALWAYS_INLINE constexpr explicit
1164 operator const _SimdMember &() const
1165 { return static_cast<const simd<_Tp, _Fixed>*>(this)->_M_data; }
1167 _GLIBCXX_SIMD_ALWAYS_INLINE constexpr explicit
1168 operator array<_Tp, _Np>() const
1170 array<_Tp, _Np> __r;
1172 static_assert(
sizeof(__r) <=
sizeof(_SimdMember),
"");
1173 __builtin_memcpy(__r.data(), &
static_cast<const _SimdMember&
>(*
this),
1182 struct _MaskBase {};
1186 struct _SimdCastType
1188 _GLIBCXX_SIMD_ALWAYS_INLINE
constexpr
1189 _SimdCastType(
const array<_Tp, _Np>&);
1191 _GLIBCXX_SIMD_ALWAYS_INLINE
constexpr
1192 _SimdCastType(
const _SimdMember& dd) : _M_data(dd) {}
1194 _GLIBCXX_SIMD_ALWAYS_INLINE
constexpr explicit
1195 operator const _SimdMember &()
const {
return _M_data; }
1198 const _SimdMember& _M_data;
1205 _MaskCastType() =
delete;
1214struct _CommonImplFixedSize
1217 template <
typename _Tp,
typename... _As>
1218 _GLIBCXX_SIMD_INTRINSIC
static void
1219 _S_store(
const _SimdTuple<_Tp, _As...>& __x,
void* __addr)
1221 constexpr size_t _Np = _SimdTuple<_Tp, _As...>::_S_size();
1222 __builtin_memcpy(__addr, &__x, _Np *
sizeof(_Tp));
1232template <
int _Np,
typename>
1233 struct _SimdImplFixedSize
1236 using _MaskMember = _SanitizedBitMask<_Np>;
1238 template <
typename _Tp>
1239 using _SimdMember = __fixed_size_storage_t<_Tp, _Np>;
1241 template <
typename _Tp>
1242 static constexpr size_t _S_tuple_size = _SimdMember<_Tp>::_S_tuple_size;
1244 template <
typename _Tp>
1245 using _Simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
1247 template <
typename _Tp>
1248 using _TypeTag = _Tp*;
1251 template <
typename _Tp>
1252 _GLIBCXX_SIMD_INTRINSIC
static constexpr _SimdMember<_Tp>
1253 _S_broadcast(_Tp __x)
noexcept
1255 return _SimdMember<_Tp>::_S_generate(
1256 [&](
auto __meta)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1257 return __meta._S_broadcast(__x);
1262 template <
typename _Fp,
typename _Tp>
1263 _GLIBCXX_SIMD_INTRINSIC
static constexpr _SimdMember<_Tp>
1264 _S_generator(_Fp&& __gen, _TypeTag<_Tp>)
1266 return _SimdMember<_Tp>::_S_generate(
1267 [&__gen](
auto __meta)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1268 return __meta._S_generator(
1269 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1270 return __i < _Np ? __gen(_SizeConstant<__meta._S_offset + __i>())
1278 template <
typename _Tp,
typename _Up>
1279 _GLIBCXX_SIMD_INTRINSIC
static constexpr _SimdMember<_Tp>
1280 _S_load(
const _Up* __mem, _TypeTag<_Tp>)
noexcept
1282 return _SimdMember<_Tp>::_S_generate(
1283 [&](
auto __meta) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1284 return __meta._S_load(&__mem[__meta._S_offset], _TypeTag<_Tp>());
1289 template <
typename _Tp,
typename... _As,
typename _Up>
1290 _GLIBCXX_SIMD_INTRINSIC
static _SimdTuple<_Tp, _As...>
1291 _S_masked_load(
const _SimdTuple<_Tp, _As...>& __old,
1292 const _MaskMember __bits,
const _Up* __mem)
noexcept
1294 auto __merge = __old;
1295 __for_each(__merge, [&](
auto __meta,
auto& __native) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1296 if (__meta._S_submask(__bits).any())
1297#pragma GCC diagnostic push
1302#pragma GCC diagnostic ignored
"-Warray-bounds"
1304 = __meta._S_masked_load(__native, __meta._S_make_mask(__bits),
1305 __mem + __meta._S_offset);
1306#pragma GCC diagnostic pop
1312 template <
typename _Tp,
typename _Up>
1313 _GLIBCXX_SIMD_INTRINSIC
static constexpr void
1314 _S_store(
const _SimdMember<_Tp>& __v, _Up* __mem, _TypeTag<_Tp>)
noexcept
1316 __for_each(__v, [&](
auto __meta,
auto __native) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1317 __meta._S_store(__native, &__mem[__meta._S_offset], _TypeTag<_Tp>());
1322 template <
typename _Tp,
typename... _As,
typename _Up>
1323 _GLIBCXX_SIMD_INTRINSIC
static void
1324 _S_masked_store(
const _SimdTuple<_Tp, _As...>& __v, _Up* __mem,
1325 const _MaskMember __bits)
noexcept
1327 __for_each(__v, [&](
auto __meta,
auto __native) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1328 if (__meta._S_submask(__bits).any())
1329#pragma GCC diagnostic push
1334#pragma GCC diagnostic ignored
"-Warray-bounds"
1335 __meta._S_masked_store(__native, __mem + __meta._S_offset,
1336 __meta._S_make_mask(__bits));
1337#pragma GCC diagnostic pop
1342 template <
typename _Tp,
typename... _As>
1343 static constexpr inline _MaskMember
1344 _S_negate(
const _SimdTuple<_Tp, _As...>& __x)
noexcept
1346 _MaskMember __bits = 0;
1348 __x, [&__bits](
auto __meta,
auto __native)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1350 |= __meta._S_mask_to_shifted_ullong(__meta._S_negate(__native));
1356 template <
typename _Tp,
typename _BinaryOperation>
1357 static constexpr inline _Tp _S_reduce(
const _Simd<_Tp>& __x,
1358 const _BinaryOperation& __binary_op)
1360 using _Tup = _SimdMember<_Tp>;
1361 const _Tup& __tup = __data(__x);
1362 if constexpr (_Tup::_S_tuple_size == 1)
1363 return _Tup::_FirstAbi::_SimdImpl::_S_reduce(
1364 __tup.template _M_simd_at<0>(), __binary_op);
1365 else if constexpr (_Tup::_S_tuple_size == 2 && _Tup::_S_size() > 2
1366 && _Tup::_SecondType::_S_size() == 1)
1368 return __binary_op(simd<_Tp, simd_abi::scalar>(
1369 reduce(__tup.template _M_simd_at<0>(),
1371 __tup.template _M_simd_at<1>())[0];
1373 else if constexpr (_Tup::_S_tuple_size == 2 && _Tup::_S_size() > 4
1374 && _Tup::_SecondType::_S_size() == 2)
1377 simd<_Tp, simd_abi::scalar>(
1378 reduce(__tup.template _M_simd_at<0>(), __binary_op)),
1379 simd<_Tp, simd_abi::scalar>(
1380 reduce(__tup.template _M_simd_at<1>(), __binary_op)))[0];
1384 const auto& __x2 = __call_with_n_evaluations<
1385 __div_roundup(_Tup::_S_tuple_size, 2)>(
1386 [](
auto __first_simd,
auto... __remaining) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1387 if constexpr (
sizeof...(__remaining) == 0)
1388 return __first_simd;
1393 typename decltype(__first_simd)::abi_type,
1394 typename decltype(__remaining)::abi_type...>;
1395 return fixed_size_simd<_Tp, _Tup2::_S_size()>(
1397 __make_simd_tuple(__first_simd, __remaining...));
1400 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1401 auto __left = __tup.template _M_simd_at<2 * __i>();
1402 if constexpr (2 * __i + 1 == _Tup::_S_tuple_size)
1406 auto __right = __tup.template _M_simd_at<2 * __i + 1>();
1407 using _LT =
decltype(__left);
1408 using _RT =
decltype(__right);
1409 if constexpr (_LT::size() == _RT::size())
1410 return __binary_op(__left, __right);
1413 _GLIBCXX_SIMD_USE_CONSTEXPR_API
1414 typename _LT::mask_type __k(
1416 [](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1417 return __j < _RT::size();
1419 _LT __ext_right = __left;
1420 where(__k, __ext_right)
1421 = __proposed::resizing_simd_cast<_LT>(__right);
1422 where(__k, __left) = __binary_op(__left, __ext_right);
1427 return reduce(__x2, __binary_op);
1432 template <
typename _Tp,
typename... _As>
1433 _GLIBCXX_SIMD_INTRINSIC
static constexpr _SimdTuple<_Tp, _As...>
1434 _S_min(
const _SimdTuple<_Tp, _As...>& __a,
const _SimdTuple<_Tp, _As...>& __b)
1436 return __a._M_apply_per_chunk(
1437 [](
auto __impl,
auto __aa,
auto __bb)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1438 return __impl._S_min(__aa, __bb);
1443 template <
typename _Tp,
typename... _As>
1444 _GLIBCXX_SIMD_INTRINSIC
static constexpr _SimdTuple<_Tp, _As...>
1445 _S_max(
const _SimdTuple<_Tp, _As...>& __a,
const _SimdTuple<_Tp, _As...>& __b)
1447 return __a._M_apply_per_chunk(
1448 [](
auto __impl,
auto __aa,
auto __bb)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1449 return __impl._S_max(__aa, __bb);
1455 template <
typename _Tp,
typename... _As>
1456 _GLIBCXX_SIMD_INTRINSIC
static constexpr _SimdTuple<_Tp, _As...>
1457 _S_complement(
const _SimdTuple<_Tp, _As...>& __x)
noexcept
1459 return __x._M_apply_per_chunk(
1460 [](
auto __impl,
auto __xx)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1461 return __impl._S_complement(__xx);
1466 template <
typename _Tp,
typename... _As>
1467 _GLIBCXX_SIMD_INTRINSIC
static constexpr _SimdTuple<_Tp, _As...>
1468 _S_unary_minus(
const _SimdTuple<_Tp, _As...>& __x)
noexcept
1470 return __x._M_apply_per_chunk(
1471 [](
auto __impl,
auto __xx)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1472 return __impl._S_unary_minus(__xx);
1478#define _GLIBCXX_SIMD_FIXED_OP(name_, op_) \
1479 template <typename _Tp, typename... _As> \
1480 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...> name_( \
1481 const _SimdTuple<_Tp, _As...>& __x, const _SimdTuple<_Tp, _As...>& __y) \
1483 return __x._M_apply_per_chunk( \
1484 [](auto __impl, auto __xx, auto __yy) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \
1485 return __impl.name_(__xx, __yy); \
1490 _GLIBCXX_SIMD_FIXED_OP(_S_plus, +)
1491 _GLIBCXX_SIMD_FIXED_OP(_S_minus, -)
1492 _GLIBCXX_SIMD_FIXED_OP(_S_multiplies, *)
1493 _GLIBCXX_SIMD_FIXED_OP(_S_divides, /)
1494 _GLIBCXX_SIMD_FIXED_OP(_S_modulus, %)
1495 _GLIBCXX_SIMD_FIXED_OP(_S_bit_and, &)
1496 _GLIBCXX_SIMD_FIXED_OP(_S_bit_or, |)
1497 _GLIBCXX_SIMD_FIXED_OP(_S_bit_xor, ^)
1498 _GLIBCXX_SIMD_FIXED_OP(_S_bit_shift_left, <<)
1499 _GLIBCXX_SIMD_FIXED_OP(_S_bit_shift_right, >>)
1500#undef _GLIBCXX_SIMD_FIXED_OP
1502 template <
typename _Tp,
typename... _As>
1503 _GLIBCXX_SIMD_INTRINSIC
static constexpr _SimdTuple<_Tp, _As...>
1504 _S_bit_shift_left(
const _SimdTuple<_Tp, _As...>& __x,
int __y)
1506 return __x._M_apply_per_chunk(
1507 [__y](
auto __impl,
auto __xx)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1508 return __impl._S_bit_shift_left(__xx, __y);
1512 template <
typename _Tp,
typename... _As>
1513 _GLIBCXX_SIMD_INTRINSIC
static constexpr _SimdTuple<_Tp, _As...>
1514 _S_bit_shift_right(
const _SimdTuple<_Tp, _As...>& __x,
int __y)
1516 return __x._M_apply_per_chunk(
1517 [__y](
auto __impl,
auto __xx)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1518 return __impl._S_bit_shift_right(__xx, __y);
1523#define _GLIBCXX_SIMD_APPLY_ON_TUPLE(_RetTp, __name) \
1524 template <typename _Tp, typename... _As, typename... _More> \
1525 static inline __fixed_size_storage_t<_RetTp, _Np> \
1526 _S_##__name(const _SimdTuple<_Tp, _As...>& __x, \
1527 const _More&... __more) \
1529 if constexpr (sizeof...(_More) == 0) \
1531 if constexpr (is_same_v<_Tp, _RetTp>) \
1532 return __x._M_apply_per_chunk( \
1533 [](auto __impl, auto __xx) \
1534 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
1536 using _V = typename decltype(__impl)::simd_type; \
1537 return __data(__name(_V(__private_init, __xx))); \
1540 return __optimize_simd_tuple( \
1541 __x.template _M_apply_r<_RetTp>( \
1542 [](auto __impl, auto __xx) \
1543 _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
1544 { return __impl._S_##__name(__xx); })); \
1546 else if constexpr ( \
1549 _RetTp> && (... && is_same_v<_SimdTuple<_Tp, _As...>, _More>) ) \
1550 return __x._M_apply_per_chunk( \
1551 [](auto __impl, auto __xx, auto... __pack) \
1552 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
1554 using _V = typename decltype(__impl)::simd_type; \
1555 return __data(__name(_V(__private_init, __xx), \
1556 _V(__private_init, __pack)...)); \
1558 else if constexpr (is_same_v<_Tp, _RetTp>) \
1559 return __x._M_apply_per_chunk( \
1560 [](auto __impl, auto __xx, auto... __pack) \
1561 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
1563 using _V = typename decltype(__impl)::simd_type; \
1564 return __data(__name(_V(__private_init, __xx), \
1565 __autocvt_to_simd(__pack)...)); \
1568 __assert_unreachable<_Tp>(); \
1571 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, acos)
1572 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, asin)
1573 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atan)
1574 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atan2)
1575 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cos)
1576 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sin)
1577 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tan)
1578 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, acosh)
1579 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, asinh)
1580 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atanh)
1581 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cosh)
1582 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sinh)
1583 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tanh)
1584 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, exp)
1585 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, exp2)
1586 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, expm1)
1587 _GLIBCXX_SIMD_APPLY_ON_TUPLE(
int, ilogb)
1588 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log)
1589 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log10)
1590 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log1p)
1591 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log2)
1592 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, logb)
1594 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp,
1596 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, scalbln)
1597 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cbrt)
1598 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, abs)
1599 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fabs)
1600 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, pow)
1601 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sqrt)
1602 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, erf)
1603 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, erfc)
1604 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, lgamma)
1605 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tgamma)
1606 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, trunc)
1607 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, ceil)
1608 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, floor)
1609 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, nearbyint)
1611 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, rint)
1612 _GLIBCXX_SIMD_APPLY_ON_TUPLE(
long, lrint)
1613 _GLIBCXX_SIMD_APPLY_ON_TUPLE(
long long, llrint)
1615 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, round)
1616 _GLIBCXX_SIMD_APPLY_ON_TUPLE(
long, lround)
1617 _GLIBCXX_SIMD_APPLY_ON_TUPLE(
long long, llround)
1619 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, ldexp)
1620 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmod)
1621 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, remainder)
1622 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, copysign)
1623 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, nextafter)
1624 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fdim)
1625 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmax)
1626 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmin)
1627 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fma)
1628 _GLIBCXX_SIMD_APPLY_ON_TUPLE(
int, fpclassify)
1629#undef _GLIBCXX_SIMD_APPLY_ON_TUPLE
1631 template <
typename _Tp,
typename... _Abis>
1632 static inline _SimdTuple<_Tp, _Abis...>
1633 _S_remquo(
const _SimdTuple<_Tp, _Abis...>& __x,
const _SimdTuple<_Tp, _Abis...>& __y,
1634 __fixed_size_storage_t<
int, _SimdTuple<_Tp, _Abis...>::_S_size()>* __z)
1636 return __x._M_apply_per_chunk(
1637 [](
auto __impl,
const auto __xx,
const auto __yy,
auto& __zz)
1638 _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
1639 {
return __impl._S_remquo(__xx, __yy, &__zz); },
1643 template <
typename _Tp,
typename... _As>
1644 static inline _SimdTuple<_Tp, _As...>
1645 _S_frexp(
const _SimdTuple<_Tp, _As...>& __x,
1646 __fixed_size_storage_t<int, _Np>& __exp)
noexcept
1648 return __x._M_apply_per_chunk(
1649 [](
auto __impl,
const auto& __a,
auto& __b) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1650 return __data(frexp(
typename decltype(__impl)::simd_type(__private_init, __a),
1651 __autocvt_to_simd(__b)));
1655#define _GLIBCXX_SIMD_TEST_ON_TUPLE_(name_) \
1656 template <typename _Tp, typename... _As> \
1657 static inline _MaskMember \
1658 _S_##name_(const _SimdTuple<_Tp, _As...>& __x) noexcept \
1660 return _M_test([] (auto __impl, auto __xx) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \
1661 return __impl._S_##name_(__xx); \
1665 _GLIBCXX_SIMD_TEST_ON_TUPLE_(isinf)
1666 _GLIBCXX_SIMD_TEST_ON_TUPLE_(isfinite)
1667 _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnan)
1668 _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnormal)
1669 _GLIBCXX_SIMD_TEST_ON_TUPLE_(signbit)
1670#undef _GLIBCXX_SIMD_TEST_ON_TUPLE_
1673 template <
typename... _Ts>
1674 _GLIBCXX_SIMD_INTRINSIC
static constexpr void
1675 _S_increment(_SimdTuple<_Ts...>& __x)
1678 __x, [](
auto __meta,
auto& native)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1679 __meta._S_increment(native);
1683 template <
typename... _Ts>
1684 _GLIBCXX_SIMD_INTRINSIC
static constexpr void
1685 _S_decrement(_SimdTuple<_Ts...>& __x)
1688 __x, [](
auto __meta,
auto& native)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1689 __meta._S_decrement(native);
1694#define _GLIBCXX_SIMD_CMP_OPERATIONS(__cmp) \
1695 template <typename _Tp, typename... _As> \
1696 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember \
1697 __cmp(const _SimdTuple<_Tp, _As...>& __x, \
1698 const _SimdTuple<_Tp, _As...>& __y) \
1700 return _M_test([](auto __impl, auto __xx, auto __yy) \
1701 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
1702 { return __impl.__cmp(__xx, __yy); }, \
1706 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_equal_to)
1707 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_not_equal_to)
1708 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_less)
1709 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_less_equal)
1710 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isless)
1711 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_islessequal)
1712 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isgreater)
1713 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isgreaterequal)
1714 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_islessgreater)
1715 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isunordered)
1716#undef _GLIBCXX_SIMD_CMP_OPERATIONS
1719 template <
typename _Tp,
typename... _As,
typename _Up>
1720 _GLIBCXX_SIMD_INTRINSIC
static constexpr void
1721 _S_set(_SimdTuple<_Tp, _As...>& __v,
int __i, _Up&& __x)
noexcept
1722 { __v._M_set(__i,
static_cast<_Up&&
>(__x)); }
1725 template <
typename _Tp,
typename... _As>
1726 _GLIBCXX_SIMD_INTRINSIC
static constexpr void
1727 _S_masked_assign(
const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
1728 const __type_identity_t<_SimdTuple<_Tp, _As...>>& __rhs)
1730 __for_each(__lhs, __rhs,
1731 [&](
auto __meta,
auto& __native_lhs,
auto __native_rhs)
1732 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
1734 __meta._S_masked_assign(__meta._S_make_mask(__bits), __native_lhs,
1741 template <
typename _Tp,
typename... _As>
1742 _GLIBCXX_SIMD_INTRINSIC
static constexpr void
1743 _S_masked_assign(
const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
1744 const __type_identity_t<_Tp> __rhs)
1747 __lhs, [&](
auto __meta,
auto& __native_lhs)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1748 __meta._S_masked_assign(__meta._S_make_mask(__bits), __native_lhs,
1754 template <
typename _Op,
typename _Tp,
typename... _As>
1755 static constexpr inline void
1756 _S_masked_cassign(
const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
1757 const _SimdTuple<_Tp, _As...>& __rhs, _Op __op)
1759 __for_each(__lhs, __rhs,
1760 [&](
auto __meta,
auto& __native_lhs,
auto __native_rhs)
1761 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
1763 __meta.template _S_masked_cassign(__meta._S_make_mask(__bits),
1764 __native_lhs, __native_rhs, __op);
1770 template <
typename _Op,
typename _Tp,
typename... _As>
1771 static constexpr inline void
1772 _S_masked_cassign(
const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
1773 const _Tp& __rhs, _Op __op)
1776 __lhs, [&](
auto __meta,
auto& __native_lhs)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1777 __meta.template _S_masked_cassign(__meta._S_make_mask(__bits),
1778 __native_lhs, __rhs, __op);
1783 template <
template <
typename>
class _Op,
typename _Tp,
typename... _As>
1784 static constexpr inline _SimdTuple<_Tp, _As...>
1785 _S_masked_unary(
const _MaskMember __bits,
const _SimdTuple<_Tp, _As...>& __v)
1787 return __v._M_apply_wrapped([&__bits](
auto __meta,
1788 auto __native)
constexpr {
1789 return __meta.template _S_masked_unary<_Op>(__meta._S_make_mask(
1799template <
int _Np,
typename>
1800 struct _MaskImplFixedSize
1803 sizeof(_ULLong) * __CHAR_BIT__ >= _Np,
1804 "The fixed_size implementation relies on one _ULLong being able to store "
1805 "all boolean elements.");
1808 using _Abi = simd_abi::fixed_size<_Np>;
1810 using _MaskMember = _SanitizedBitMask<_Np>;
1812 template <
typename _Tp>
1813 using _FirstAbi =
typename __fixed_size_storage_t<_Tp, _Np>::_FirstAbi;
1815 template <
typename _Tp>
1816 using _TypeTag = _Tp*;
1821 _GLIBCXX_SIMD_INTRINSIC
static constexpr _MaskMember
1822 _S_broadcast(
bool __x)
1823 {
return __x ? ~_MaskMember() : _MaskMember(); }
1828 _GLIBCXX_SIMD_INTRINSIC
static constexpr _MaskMember
1829 _S_load(
const bool* __mem)
1831 if (__builtin_is_constant_evaluated())
1834 for (
size_t __i = 0; __i < _Np; ++__i)
1835 __r.set(__i, __mem[__i]);
1838 using _Ip = __int_for_sizeof_t<bool>;
1842 const simd<_Ip, _Abi> __bools(
reinterpret_cast<const __may_alias<_Ip>*
>(
1845 return __data(__bools != 0);
1850 template <
bool _Sanitized>
1851 _GLIBCXX_SIMD_INTRINSIC
static constexpr _SanitizedBitMask<_Np>
1852 _S_to_bits(_BitMask<_Np, _Sanitized> __x)
1854 if constexpr (_Sanitized)
1857 return __x._M_sanitized();
1862 template <
typename _Tp,
typename _Up,
typename _UAbi>
1863 _GLIBCXX_SIMD_INTRINSIC
static constexpr _MaskMember
1864 _S_convert(simd_mask<_Up, _UAbi> __x)
1866 return _UAbi::_MaskImpl::_S_to_bits(__data(__x))
1867 .template _M_extract<0, _Np>();
1872 template <
typename _Tp>
1873 _GLIBCXX_SIMD_INTRINSIC
static constexpr _MaskMember
1874 _S_from_bitmask(_MaskMember __bits, _TypeTag<_Tp>)
noexcept
1878 static constexpr inline _MaskMember
1879 _S_load(
const bool* __mem)
noexcept
1884 using _Vs = __fixed_size_storage_t<_UChar, _Np>;
1885 __for_each(_Vs{}, [&](
auto __meta,
auto) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1886 __r |= __meta._S_mask_to_shifted_ullong(
1887 __meta._S_mask_impl._S_load(&__mem[__meta._S_offset],
1888 _SizeConstant<__meta._S_size()>()));
1894 static constexpr inline _MaskMember
1895 _S_masked_load(_MaskMember __merge, _MaskMember __mask,
const bool* __mem)
noexcept
1897 _BitOps::_S_bit_iteration(__mask.to_ullong(),
1898 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1899 __merge.set(__i, __mem[__i]);
1905 static constexpr inline void
1906 _S_store(
const _MaskMember __bitmask,
bool* __mem)
noexcept
1908 if constexpr (_Np == 1)
1909 __mem[0] = __bitmask[0];
1911 _FirstAbi<_UChar>::_CommonImpl::_S_store_bool_array(__bitmask, __mem);
1915 static constexpr inline void
1916 _S_masked_store(
const _MaskMember __v,
bool* __mem,
const _MaskMember __k)
noexcept
1918 _BitOps::_S_bit_iteration(
1919 __k, [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { __mem[__i] = __v[__i]; });
1923 _GLIBCXX_SIMD_INTRINSIC
static constexpr _MaskMember
1924 _S_logical_and(
const _MaskMember& __x,
const _MaskMember& __y)
noexcept
1925 {
return __x & __y; }
1927 _GLIBCXX_SIMD_INTRINSIC
static constexpr _MaskMember
1928 _S_logical_or(
const _MaskMember& __x,
const _MaskMember& __y)
noexcept
1929 {
return __x | __y; }
1931 _GLIBCXX_SIMD_INTRINSIC
static constexpr _MaskMember
1932 _S_bit_not(
const _MaskMember& __x)
noexcept
1935 _GLIBCXX_SIMD_INTRINSIC
static constexpr _MaskMember
1936 _S_bit_and(
const _MaskMember& __x,
const _MaskMember& __y)
noexcept
1937 {
return __x & __y; }
1939 _GLIBCXX_SIMD_INTRINSIC
static constexpr _MaskMember
1940 _S_bit_or(
const _MaskMember& __x,
const _MaskMember& __y)
noexcept
1941 {
return __x | __y; }
1943 _GLIBCXX_SIMD_INTRINSIC
static constexpr _MaskMember
1944 _S_bit_xor(
const _MaskMember& __x,
const _MaskMember& __y)
noexcept
1945 {
return __x ^ __y; }
1948 _GLIBCXX_SIMD_INTRINSIC
static constexpr void
1949 _S_set(_MaskMember& __k,
int __i,
bool __x)
noexcept
1950 { __k.set(__i, __x); }
1953 _GLIBCXX_SIMD_INTRINSIC
static constexpr void
1954 _S_masked_assign(
const _MaskMember __k, _MaskMember& __lhs,
const _MaskMember __rhs)
1955 { __lhs = (__lhs & ~__k) | (__rhs & __k); }
1958 _GLIBCXX_SIMD_INTRINSIC
static constexpr void
1959 _S_masked_assign(
const _MaskMember __k, _MaskMember& __lhs,
const bool __rhs)
1969 template <
typename _Tp>
1970 _GLIBCXX_SIMD_INTRINSIC
static constexpr bool
1971 _S_all_of(simd_mask<_Tp, _Abi> __k)
1972 {
return __data(__k).all(); }
1976 template <
typename _Tp>
1977 _GLIBCXX_SIMD_INTRINSIC
static constexpr bool
1978 _S_any_of(simd_mask<_Tp, _Abi> __k)
1979 {
return __data(__k).any(); }
1983 template <
typename _Tp>
1984 _GLIBCXX_SIMD_INTRINSIC
static constexpr bool
1985 _S_none_of(simd_mask<_Tp, _Abi> __k)
1986 {
return __data(__k).none(); }
1990 template <
typename _Tp>
1991 _GLIBCXX_SIMD_INTRINSIC
static constexpr bool
1992 _S_some_of([[maybe_unused]] simd_mask<_Tp, _Abi> __k)
1994 if constexpr (_Np == 1)
1997 return __data(__k).any() && !__data(__k).all();
2002 template <
typename _Tp>
2003 _GLIBCXX_SIMD_INTRINSIC
static constexpr int
2004 _S_popcount(simd_mask<_Tp, _Abi> __k)
2005 {
return __data(__k).count(); }
2009 template <
typename _Tp>
2010 _GLIBCXX_SIMD_INTRINSIC
static constexpr int
2011 _S_find_first_set(simd_mask<_Tp, _Abi> __k)
2012 {
return std::__countr_zero(__data(__k).to_ullong()); }
2016 template <
typename _Tp>
2017 _GLIBCXX_SIMD_INTRINSIC
static constexpr int
2018 _S_find_last_set(simd_mask<_Tp, _Abi> __k)
2019 {
return std::__bit_width(__data(__k).to_ullong()) - 1; }
2025_GLIBCXX_SIMD_END_NAMESPACE
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
constexpr _Iterator __base(_Iterator __it)