25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26 #define _GLIBCXX_EXPERIMENTAL_SIMD_H
28 #if __cplusplus >= 201703L
30 #include "simd_detail.h"
34 #ifdef _GLIBCXX_DEBUG_UB
43 #if _GLIBCXX_SIMD_X86INTRIN
44 #include <x86intrin.h>
45 #elif _GLIBCXX_SIMD_HAVE_NEON
72 _GLIBCXX_SIMD_BEGIN_NAMESPACE
74 #if !_GLIBCXX_SIMD_X86INTRIN
75 using __m128 [[__gnu__::__vector_size__(16)]] = float;
76 using __m128d [[__gnu__::__vector_size__(16)]] = double;
77 using __m128i [[__gnu__::__vector_size__(16)]] =
long long;
78 using __m256 [[__gnu__::__vector_size__(32)]] = float;
79 using __m256d [[__gnu__::__vector_size__(32)]] = double;
80 using __m256i [[__gnu__::__vector_size__(32)]] =
long long;
81 using __m512 [[__gnu__::__vector_size__(64)]] = float;
82 using __m512d [[__gnu__::__vector_size__(64)]] = double;
83 using __m512i [[__gnu__::__vector_size__(64)]] =
long long;
105 template <
int _UsedBytes>
108 template <
int _UsedBytes>
109 struct _VecBltnBtmsk;
111 template <
typename _Tp,
int _Np>
112 using _VecN = _VecBuiltin<
sizeof(_Tp) * _Np>;
114 template <
int _UsedBytes = 16>
115 using _Sse = _VecBuiltin<_UsedBytes>;
117 template <
int _UsedBytes = 32>
118 using _Avx = _VecBuiltin<_UsedBytes>;
120 template <
int _UsedBytes = 64>
121 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
123 template <
int _UsedBytes = 16>
124 using _Neon = _VecBuiltin<_UsedBytes>;
127 using __sse = _Sse<>;
128 using __avx = _Avx<>;
129 using __avx512 = _Avx512<>;
130 using __neon = _Neon<>;
131 using __neon128 = _Neon<16>;
132 using __neon64 = _Neon<8>;
135 template <
typename _Tp,
size_t _Np,
typename...>
139 using fixed_size = _Fixed<_Np>;
141 using scalar = _Scalar;
146 template <
typename _Tp>
149 template <
typename _Tp>
152 template <
typename _Tp,
typename _Abi>
155 template <
typename _Tp,
typename _Abi>
158 template <
typename _Tp,
typename _Abi>
163 struct element_aligned_tag
165 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
166 static constexpr
size_t _S_alignment =
alignof(_Up);
168 template <
typename _Tp,
typename _Up>
169 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
174 struct vector_aligned_tag
176 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
177 static constexpr
size_t _S_alignment
178 = std::__bit_ceil(
sizeof(_Up) * _Tp::size());
180 template <
typename _Tp,
typename _Up>
181 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
184 return static_cast<_Up*
>(
185 __builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>));
189 template <
size_t _Np>
struct overaligned_tag
191 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
192 static constexpr
size_t _S_alignment = _Np;
194 template <
typename _Tp,
typename _Up>
195 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
197 {
return static_cast<_Up*
>(__builtin_assume_aligned(__ptr, _Np)); }
200 inline constexpr element_aligned_tag element_aligned = {};
202 inline constexpr vector_aligned_tag vector_aligned = {};
204 template <
size_t _Np>
205 inline constexpr overaligned_tag<_Np> overaligned = {};
208 template <
size_t _Xp>
209 using _SizeConstant = integral_constant<size_t, _Xp>;
211 constexpr
inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
212 constexpr
inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
213 constexpr
inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
214 constexpr
inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
215 constexpr
inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
216 constexpr
inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
217 constexpr
inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
218 constexpr
inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
219 constexpr
inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
220 constexpr
inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
221 constexpr
inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
222 constexpr
inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
223 constexpr
inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
224 constexpr
inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
225 constexpr
inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
226 constexpr
inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
227 constexpr
inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
228 constexpr
inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
229 constexpr
inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
230 constexpr
inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
231 constexpr
inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
232 constexpr
inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
233 constexpr
inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
234 constexpr
inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
235 constexpr
inline bool __have_avx512bitalg = _GLIBCXX_SIMD_HAVE_AVX512BITALG;
236 constexpr
inline bool __have_avx512vbmi2 = _GLIBCXX_SIMD_HAVE_AVX512VBMI2;
237 constexpr
inline bool __have_avx512vbmi = _GLIBCXX_SIMD_HAVE_AVX512VBMI;
238 constexpr
inline bool __have_avx512ifma = _GLIBCXX_SIMD_HAVE_AVX512IFMA;
239 constexpr
inline bool __have_avx512cd = _GLIBCXX_SIMD_HAVE_AVX512CD;
240 constexpr
inline bool __have_avx512vnni = _GLIBCXX_SIMD_HAVE_AVX512VNNI;
241 constexpr
inline bool __have_avx512vpopcntdq = _GLIBCXX_SIMD_HAVE_AVX512VPOPCNTDQ;
242 constexpr
inline bool __have_avx512vp2intersect = _GLIBCXX_SIMD_HAVE_AVX512VP2INTERSECT;
244 constexpr
inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
245 constexpr
inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
246 constexpr
inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
247 constexpr
inline bool __support_neon_float =
248 #if defined __GCC_IEC_559
250 #elif defined __FAST_MATH__
257 constexpr
inline bool __have_power10vec =
true;
259 constexpr
inline bool __have_power10vec =
false;
261 #ifdef __POWER9_VECTOR__
262 constexpr
inline bool __have_power9vec =
true;
264 constexpr
inline bool __have_power9vec =
false;
266 #if defined __POWER8_VECTOR__
267 constexpr
inline bool __have_power8vec =
true;
269 constexpr
inline bool __have_power8vec = __have_power9vec;
272 constexpr
inline bool __have_power_vsx =
true;
274 constexpr
inline bool __have_power_vsx = __have_power8vec;
276 #if defined __ALTIVEC__
277 constexpr
inline bool __have_power_vmx =
true;
279 constexpr
inline bool __have_power_vmx = __have_power_vsx;
286 constexpr
bool __handle_fpexcept =
287 #ifdef math_errhandling
288 math_errhandling & MATH_ERREXCEPT;
289 #elif defined __FAST_MATH__
295 constexpr std::uint_least64_t
296 __floating_point_flags()
298 std::uint_least64_t __flags = 0;
299 if constexpr (__handle_fpexcept)
303 #elif __FINITE_MATH_ONLY__
305 #elif __GCC_IEC_559 < 2
308 __flags |= (__FLT_EVAL_METHOD__ + 1) << 3;
312 constexpr std::uint_least64_t
315 if constexpr (__have_mmx || __have_sse)
320 | (__have_ssse3 << 4)
321 | (__have_sse4_1 << 5)
322 | (__have_sse4_2 << 6)
327 | (__have_bmi2 << 11)
328 | (__have_lzcnt << 12)
329 | (__have_sse4a << 13)
331 | (__have_fma4 << 15)
332 | (__have_f16c << 16)
333 | (__have_popcnt << 17)
334 | (__have_avx512f << 18)
335 | (__have_avx512dq << 19)
336 | (__have_avx512vl << 20)
337 | (__have_avx512bw << 21)
338 | (__have_avx512bitalg << 22)
339 | (__have_avx512vbmi2 << 23)
340 | (__have_avx512vbmi << 24)
341 | (__have_avx512ifma << 25)
342 | (__have_avx512cd << 26)
343 | (__have_avx512vnni << 27)
344 | (__have_avx512vpopcntdq << 28)
345 | (__have_avx512vp2intersect << 29);
346 else if constexpr (__have_neon)
348 | (__have_neon_a32 << 1)
349 | (__have_neon_a64 << 2)
350 | (__have_neon_a64 << 2)
351 | (__support_neon_float << 3);
352 else if constexpr (__have_power_vmx)
353 return __have_power_vmx
354 | (__have_power_vsx << 1)
355 | (__have_power8vec << 2)
356 | (__have_power9vec << 3)
357 | (__have_power10vec << 4);
364 struct _OdrEnforcer {};
367 template <std::uint_least64_t...>
368 struct _MachineFlagsTemplate {};
381 _MachineFlagsTemplate<__machine_flags(), __floating_point_flags()>>;
385 template <
typename _Tp>
386 _GLIBCXX_SIMD_INTRINSIC constexpr
388 operator()(_Tp __a, _Tp __b)
const
391 return min(__a, __b);
397 template <
typename _Tp>
398 _GLIBCXX_SIMD_INTRINSIC constexpr
400 operator()(_Tp __a, _Tp __b)
const
403 return max(__a, __b);
410 template <
typename _Fp,
size_t... _I>
411 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
413 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
414 { ((void)__f(_SizeConstant<_I>()), ...); }
416 template <
typename _Fp>
417 _GLIBCXX_SIMD_INTRINSIC constexpr
void
418 __execute_on_index_sequence(_Fp&&, index_sequence<>)
421 template <
size_t _Np,
typename _Fp>
422 _GLIBCXX_SIMD_INTRINSIC constexpr
void
423 __execute_n_times(_Fp&& __f)
425 __execute_on_index_sequence(
static_cast<_Fp&&
>(__f),
426 make_index_sequence<_Np>{});
431 template <
typename _R,
typename _Fp,
size_t... _I>
432 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
434 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
435 {
return _R{__f(_SizeConstant<_I>())...}; }
437 template <
size_t _Np,
typename _R,
typename _Fp>
438 _GLIBCXX_SIMD_INTRINSIC constexpr _R
439 __generate_from_n_evaluations(_Fp&& __f)
441 return __execute_on_index_sequence_with_return<_R>(
442 static_cast<_Fp&&
>(__f), make_index_sequence<_Np>{});
447 template <
size_t... _I,
typename _F0,
typename _FArgs>
448 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
450 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
451 {
return __f0(__fargs(_SizeConstant<_I>())...); }
453 template <
size_t _Np,
typename _F0,
typename _FArgs>
454 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
455 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
457 return __call_with_n_evaluations(make_index_sequence<_Np>{},
458 static_cast<_F0&&
>(__f0),
459 static_cast<_FArgs&&
>(__fargs));
464 template <
size_t _First = 0,
size_t... _It,
typename _Tp,
typename _Fp>
465 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
467 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
468 {
return __fun(__x[_First + _It]...); }
470 template <
size_t _Np,
size_t _First = 0,
typename _Tp,
typename _Fp>
471 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
472 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
474 return __call_with_subscripts<_First>(
static_cast<_Tp&&
>(__x),
475 make_index_sequence<_Np>(),
476 static_cast<_Fp&&
>(__fun));
483 using _UChar =
unsigned char;
484 using _SChar =
signed char;
485 using _UShort =
unsigned short;
486 using _UInt =
unsigned int;
487 using _ULong =
unsigned long;
488 using _ULLong =
unsigned long long;
489 using _LLong =
long long;
493 template <
typename _T0,
typename...>
494 struct __first_of_pack
495 {
using type = _T0; };
497 template <
typename... _Ts>
498 using __first_of_pack_t =
typename __first_of_pack<_Ts...>::type;
502 template <
typename _Tp>
503 typename _Tp::value_type
504 __value_type_or_identity_impl(
int);
506 template <
typename _Tp>
508 __value_type_or_identity_impl(
float);
510 template <
typename _Tp>
511 using __value_type_or_identity_t
512 = decltype(__value_type_or_identity_impl<_Tp>(
int()));
516 template <
typename _Tp>
517 struct __is_vectorizable :
public is_arithmetic<_Tp> {};
520 struct __is_vectorizable<bool> :
public false_type {};
522 template <
typename _Tp>
523 inline constexpr
bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
526 template <
typename _Tp,
typename = enable_if_t<__is_vectorizable_v<_Tp>>>
527 using _Vectorizable = _Tp;
531 template <
typename _Ptr,
typename _ValueType>
532 struct __is_possible_loadstore_conversion
533 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
536 struct __is_possible_loadstore_conversion<bool, bool> :
true_type {};
539 template <
typename _Ptr,
typename _ValueType,
541 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
542 using _LoadStorePtr = _Ptr;
546 template <
typename _Tp,
typename =
void_t<>>
549 template <
typename _Tp>
550 inline constexpr
bool __is_bitmask_v = __is_bitmask<_Tp>::value;
553 template <
typename _Tp>
554 struct __is_bitmask<_Tp,
555 void_t<decltype(
declval<unsigned&>() = declval<_Tp>() & 1u)>>
560 #pragma GCC diagnostic push
561 #pragma GCC diagnostic ignored "-Wpedantic"
562 template <
size_t _Bytes>
566 if constexpr (_Bytes ==
sizeof(
int))
569 else if constexpr (_Bytes ==
sizeof(
char))
572 else if constexpr (_Bytes ==
sizeof(_SChar))
575 else if constexpr (_Bytes ==
sizeof(
short))
578 else if constexpr (_Bytes ==
sizeof(
long))
581 else if constexpr (_Bytes ==
sizeof(_LLong))
583 #ifdef __SIZEOF_INT128__
584 else if constexpr (_Bytes ==
sizeof(__int128))
587 else if constexpr (_Bytes %
sizeof(
int) == 0)
589 constexpr
size_t _Np = _Bytes /
sizeof(int);
594 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
597 return __generate_from_n_evaluations<_Np, _Ip>(
598 [&](
auto __i) {
return __rhs._M_data[__i] & _M_data[__i]; });
601 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
604 return __generate_from_n_evaluations<_Np, _Ip>(
605 [&](
auto __i) {
return __rhs._M_data[__i] | _M_data[__i]; });
608 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
611 return __generate_from_n_evaluations<_Np, _Ip>(
612 [&](
auto __i) {
return __rhs._M_data[__i] ^ _M_data[__i]; });
615 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
618 return __generate_from_n_evaluations<_Np, _Ip>(
619 [&](
auto __i) {
return ~_M_data[__i]; });
625 static_assert(_Bytes != _Bytes,
"this should be unreachable");
627 #pragma GCC diagnostic pop
629 template <
typename _Tp>
630 using __int_for_sizeof_t = decltype(__int_for_sizeof<
sizeof(_Tp)>());
632 template <
size_t _Np>
633 using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
637 template <
typename _Tp>
641 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> :
true_type {};
643 template <
typename _Tp>
644 inline constexpr
bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
648 template <
typename _Abi>
651 {
return is_same_v<simd_abi::scalar, _Abi>; }
655 template <
template <
int>
class _Abi,
int _Bytes>
657 __abi_bytes_impl(_Abi<_Bytes>*)
660 template <
typename _Tp>
662 __abi_bytes_impl(_Tp*)
665 template <
typename _Abi>
666 inline constexpr
int __abi_bytes_v
667 = __abi_bytes_impl(
static_cast<_Abi*
>(
nullptr));
671 template <
typename _Abi>
673 __is_builtin_bitmask_abi()
674 {
return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
678 template <
typename _Abi>
682 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
683 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
688 template <
typename _Abi>
692 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
693 return _Bytes > 16 && _Bytes <= 32
694 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
699 template <
typename _Abi>
703 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
704 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
709 template <
typename _Abi>
713 constexpr
auto _Bytes = __abi_bytes_v<_Abi>;
714 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
719 template <
typename,
typename _Up>
720 struct __make_dependent
721 {
using type = _Up; };
723 template <
typename _Tp,
typename _Up>
724 using __make_dependent_t =
typename __make_dependent<_Tp, _Up>::type;
730 template <
typename... _Args>
731 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE
void
732 __invoke_ub([[maybe_unused]]
const char* __msg,
733 [[maybe_unused]]
const _Args&... __args)
735 #ifdef _GLIBCXX_DEBUG_UB
736 __builtin_fprintf(stderr, __msg, __args...);
739 __builtin_unreachable();
745 template <
typename _Tp>
746 struct __assert_unreachable
747 { static_assert(!is_same_v<_Tp, _Tp>,
"this should be unreachable"); };
751 template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
753 __size_or_zero_dispatch(
int)
756 template <
typename _Tp,
typename _Ap>
758 __size_or_zero_dispatch(
float)
761 template <
typename _Tp,
typename _Ap>
762 inline constexpr
size_t __size_or_zero_v
763 = __size_or_zero_dispatch<_Tp, _Ap>(0);
767 inline constexpr
size_t
768 __div_roundup(
size_t __a,
size_t __b)
769 {
return (__a + __b - 1) / __b; }
778 _GLIBCXX_SIMD_INTRINSIC constexpr _ExactBool(
bool __b) : _M_data(__b) {}
780 _ExactBool(
int) =
delete;
782 _GLIBCXX_SIMD_INTRINSIC constexpr
operator bool()
const {
return _M_data; }
792 template <
typename _Tp>
793 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
798 struct _UnsupportedBase
800 _UnsupportedBase() =
delete;
801 _UnsupportedBase(
const _UnsupportedBase&) =
delete;
802 _UnsupportedBase& operator=(
const _UnsupportedBase&) =
delete;
803 ~_UnsupportedBase() =
delete;
816 struct _InvalidTraits
819 using _SimdBase = _UnsupportedBase;
820 using _MaskBase = _UnsupportedBase;
822 static constexpr
size_t _S_full_size = 0;
823 static constexpr
bool _S_is_partial =
false;
825 static constexpr
size_t _S_simd_align = 1;
827 struct _SimdMember {};
828 struct _SimdCastType;
830 static constexpr
size_t _S_mask_align = 1;
832 struct _MaskMember {};
833 struct _MaskCastType;
838 template <
typename _Tp,
typename _Abi,
typename =
void_t<>>
839 struct _SimdTraits : _InvalidTraits {};
847 inline constexpr
struct _PrivateInit {} __private_init = {};
849 inline constexpr
struct _BitsetInit {} __bitset_init = {};
853 template <
typename _From,
typename _To,
bool = is_arithmetic_v<_From>,
854 bool = is_arithmetic_v<_To>>
855 struct __is_narrowing_conversion;
859 template <
typename _From,
typename _To>
860 struct __is_narrowing_conversion<_From, _To, true, true>
861 :
public __bool_constant<(
862 __digits_v<_From> > __digits_v<_To>
863 || __finite_max_v<_From> > __finite_max_v<_To>
864 || __finite_min_v<_From> < __finite_min_v<_To>
865 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
867 template <
typename _Tp>
868 struct __is_narrowing_conversion<_Tp, bool, true, true>
872 struct __is_narrowing_conversion<bool, bool, true, true>
875 template <
typename _Tp>
876 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
879 template <
typename _From,
typename _To>
880 struct __is_narrowing_conversion<_From, _To, false, true>
881 :
public negation<is_convertible<_From, _To>> {};
885 template <
typename _From,
typename _To,
bool = (sizeof(_From) < sizeof(_To))>
886 struct __converts_to_higher_
integer_rank : public true_type {};
889 template <
typename _From,
typename _To>
890 struct __converts_to_higher_integer_rank<_From, _To, false>
891 :
public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
895 template <
typename _Tp,
typename _Ap>
896 _GLIBCXX_SIMD_INTRINSIC constexpr
const auto&
897 __data(
const simd<_Tp, _Ap>& __x);
899 template <
typename _Tp,
typename _Ap>
900 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
901 __data(simd<_Tp, _Ap>& __x);
903 template <
typename _Tp,
typename _Ap>
904 _GLIBCXX_SIMD_INTRINSIC constexpr
const auto&
905 __data(
const simd_mask<_Tp, _Ap>& __x);
907 template <
typename _Tp,
typename _Ap>
908 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
909 __data(simd_mask<_Tp, _Ap>& __x);
913 template <
typename _FromT,
typename _FromA,
typename _ToT,
typename _ToA,
915 struct _SimdConverter;
917 template <
typename _Tp,
typename _Ap>
918 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
920 template <
typename _Up>
921 _GLIBCXX_SIMD_INTRINSIC
const _Up&
922 operator()(
const _Up& __x)
928 template <
typename _V>
929 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
930 __to_value_type_or_member_type(
const _V& __x) -> decltype(__data(__x))
931 {
return __data(__x); }
933 template <
typename _V>
934 _GLIBCXX_SIMD_INTRINSIC constexpr
const typename _V::value_type&
935 __to_value_type_or_member_type(
const typename _V::value_type& __x)
940 template <
size_t _Size>
941 struct __bool_storage_member_type;
943 template <
size_t _Size>
944 using __bool_storage_member_type_t =
945 typename __bool_storage_member_type<_Size>::type;
955 template <
typename _Tp,
typename... _Abis>
960 template <
typename _Tp,
int _Np>
961 struct __fixed_size_storage;
963 template <
typename _Tp,
int _Np>
964 using __fixed_size_storage_t =
typename __fixed_size_storage<_Tp, _Np>::type;
968 template <
typename _Tp,
size_t _Size,
typename =
void_t<>>
971 template <
typename _Tp>
972 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 /
sizeof(_Tp)>;
973 template <
typename _Tp>
974 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 /
sizeof(_Tp)>;
975 template <
typename _Tp>
976 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 /
sizeof(_Tp)>;
977 template <
typename _Tp>
978 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 /
sizeof(_Tp)>;
982 template <
typename _Tp>
985 template <
typename _Tp,
size_t _Np>
986 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> :
true_type {};
988 template <
typename _Tp>
989 inline constexpr
bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
996 template <
typename _Tp,
typename _Fp>
998 _S_bit_iteration(_Tp __mask, _Fp&& __f)
1000 static_assert(
sizeof(_ULLong) >=
sizeof(_Tp));
1001 conditional_t<
sizeof(_Tp) <=
sizeof(_UInt), _UInt, _ULLong> __k;
1002 if constexpr (is_convertible_v<_Tp, decltype(__k)>)
1005 __k = __mask.to_ullong();
1008 __f(std::__countr_zero(__k));
1018 template <
typename _Tp =
void>
1020 { constexpr _Tp operator()(_Tp __a)
const {
return ++__a; } };
1023 struct __increment<void>
1025 template <
typename _Tp>
1027 operator()(_Tp __a)
const
1031 template <
typename _Tp =
void>
1033 { constexpr _Tp operator()(_Tp __a)
const {
return --__a; } };
1036 struct __decrement<void>
1038 template <
typename _Tp>
1040 operator()(_Tp __a)
const
1046 template <
typename _From,
typename _To,
1048 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
1049 using _ValuePreserving = _From;
1051 template <
typename _From,
typename _To,
1052 typename _DecayedFrom = __remove_cvref_t<_From>,
1054 is_convertible<_From, _To>,
1056 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
1057 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
1058 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
1059 using _ValuePreservingOrInt = _From;
1063 template <
typename _Tp,
size_t _Bytes,
typename =
void_t<>>
1064 struct __intrinsic_type;
1066 template <
typename _Tp,
size_t _Size>
1067 using __intrinsic_type_t =
1068 typename __intrinsic_type<_Tp, _Size *
sizeof(_Tp)>::type;
1070 template <
typename _Tp>
1071 using __intrinsic_type2_t =
typename __intrinsic_type<_Tp, 2>::type;
1072 template <
typename _Tp>
1073 using __intrinsic_type4_t =
typename __intrinsic_type<_Tp, 4>::type;
1074 template <
typename _Tp>
1075 using __intrinsic_type8_t =
typename __intrinsic_type<_Tp, 8>::type;
1076 template <
typename _Tp>
1077 using __intrinsic_type16_t =
typename __intrinsic_type<_Tp, 16>::type;
1078 template <
typename _Tp>
1079 using __intrinsic_type32_t =
typename __intrinsic_type<_Tp, 32>::type;
1080 template <
typename _Tp>
1081 using __intrinsic_type64_t =
typename __intrinsic_type<_Tp, 64>::type;
1085 template <
size_t _Np,
bool _Sanitized = false>
1088 template <
size_t _Np,
bool _Sanitized>
1089 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> :
true_type {};
1091 template <
size_t _Np>
1092 using _SanitizedBitMask = _BitMask<_Np, true>;
1094 template <
size_t _Np,
bool _Sanitized>
1097 static_assert(_Np > 0);
1099 static constexpr
size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
1103 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
1105 static constexpr
int _S_array_size = __div_roundup(_NBytes,
sizeof(_Tp));
1107 _Tp _M_bits[_S_array_size];
1109 static constexpr
int _S_unused_bits
1110 = _Np == 1 ? 0 : _S_array_size *
sizeof(_Tp) * __CHAR_BIT__ - _Np;
1112 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1114 constexpr _BitMask() noexcept = default;
1116 constexpr _BitMask(
unsigned long long __x) noexcept
1117 : _M_bits{
static_cast<_Tp
>(__x)} {}
1119 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1121 constexpr _BitMask(
const _BitMask&) noexcept =
default;
1123 template <
bool _RhsSanitized,
typename =
enable_if_t<_RhsSanitized ==
false
1124 && _Sanitized ==
true>>
1125 constexpr _BitMask(
const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1126 : _BitMask(__rhs._M_sanitized()) {}
1128 constexpr
operator _SimdWrapper<bool, _Np>() const noexcept
1130 static_assert(_S_array_size == 1);
1136 _M_to_bits() const noexcept
1138 static_assert(_S_array_size == 1);
1143 constexpr
unsigned long long
1144 to_ullong() const noexcept
1146 static_assert(_S_array_size == 1);
1151 constexpr
unsigned long
1152 to_ulong() const noexcept
1154 static_assert(_S_array_size == 1);
1158 constexpr bitset<_Np>
1159 _M_to_bitset() const noexcept
1161 static_assert(_S_array_size == 1);
1165 constexpr decltype(
auto)
1166 _M_sanitized() const noexcept
1168 if constexpr (_Sanitized)
1170 else if constexpr (_Np == 1)
1171 return _SanitizedBitMask<_Np>(_M_bits[0]);
1174 _SanitizedBitMask<_Np> __r = {};
1175 for (
int __i = 0; __i < _S_array_size; ++__i)
1176 __r._M_bits[__i] = _M_bits[__i];
1177 if constexpr (_S_unused_bits > 0)
1178 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1183 template <
size_t _Mp,
bool _LSanitized>
1184 constexpr _BitMask<_Np + _Mp, _Sanitized>
1185 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb)
const noexcept
1187 constexpr
size_t _RN = _Np + _Mp;
1188 using _Rp = _BitMask<_RN, _Sanitized>;
1189 if constexpr (_Rp::_S_array_size == 1)
1191 _Rp __r{{_M_bits[0]}};
1192 __r._M_bits[0] <<= _Mp;
1193 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1197 __assert_unreachable<_Rp>();
1203 template <
size_t _DropLsb,
size_t _NewSize = _Np - _DropLsb>
1205 _M_extract() const noexcept
1207 static_assert(_Np > _DropLsb);
1208 static_assert(_DropLsb + _NewSize <=
sizeof(_ULLong) * __CHAR_BIT__,
1209 "not implemented for bitmasks larger than one ullong");
1210 if constexpr (_NewSize == 1)
1212 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1214 return _BitMask<_NewSize,
1215 ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1216 && _NewSize + _DropLsb <= _Np)
1217 || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1218 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1224 all() const noexcept
1226 if constexpr (_Np == 1)
1228 else if constexpr (!_Sanitized)
1229 return _M_sanitized().all();
1232 constexpr _Tp __allbits = ~_Tp();
1233 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1234 if (_M_bits[__i] != __allbits)
1236 return _M_bits[_S_array_size - 1] == _S_bitmask;
1243 any() const noexcept
1245 if constexpr (_Np == 1)
1247 else if constexpr (!_Sanitized)
1248 return _M_sanitized().any();
1251 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1252 if (_M_bits[__i] != 0)
1254 return _M_bits[_S_array_size - 1] != 0;
1260 none() const noexcept
1262 if constexpr (_Np == 1)
1264 else if constexpr (!_Sanitized)
1265 return _M_sanitized().none();
1268 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1269 if (_M_bits[__i] != 0)
1271 return _M_bits[_S_array_size - 1] == 0;
1278 count() const noexcept
1280 if constexpr (_Np == 1)
1282 else if constexpr (!_Sanitized)
1283 return _M_sanitized().none();
1286 int __result = __builtin_popcountll(_M_bits[0]);
1287 for (
int __i = 1; __i < _S_array_size; ++__i)
1288 __result += __builtin_popcountll(_M_bits[__i]);
1295 operator[](
size_t __i)
const noexcept
1297 if constexpr (_Np == 1)
1299 else if constexpr (_S_array_size == 1)
1300 return (_M_bits[0] >> __i) & 1;
1303 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1304 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1305 return (_M_bits[__j] >> __shift) & 1;
1309 template <
size_t __i>
1311 operator[](_SizeConstant<__i>)
const noexcept
1313 static_assert(__i < _Np);
1314 constexpr
size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1315 constexpr
size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1316 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1321 set(
size_t __i,
bool __x) noexcept
1323 if constexpr (_Np == 1)
1325 else if constexpr (_S_array_size == 1)
1327 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1328 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1332 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1333 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1334 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1335 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1339 template <
size_t __i>
1341 set(_SizeConstant<__i>,
bool __x) noexcept
1343 static_assert(__i < _Np);
1344 if constexpr (_Np == 1)
1348 constexpr
size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1349 constexpr
size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1350 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1351 _M_bits[__j] &= __mask;
1352 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1358 operator~() const noexcept
1360 if constexpr (_Np == 1)
1364 _BitMask __result{};
1365 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1366 __result._M_bits[__i] = ~_M_bits[__i];
1367 if constexpr (_Sanitized)
1368 __result._M_bits[_S_array_size - 1]
1369 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1371 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1377 operator^=(
const _BitMask& __b) & noexcept
1379 __execute_n_times<_S_array_size>(
1380 [&](
auto __i) { _M_bits[__i] ^= __b._M_bits[__i]; });
1385 operator|=(
const _BitMask& __b) & noexcept
1387 __execute_n_times<_S_array_size>(
1388 [&](
auto __i) { _M_bits[__i] |= __b._M_bits[__i]; });
1393 operator&=(
const _BitMask& __b) & noexcept
1395 __execute_n_times<_S_array_size>(
1396 [&](
auto __i) { _M_bits[__i] &= __b._M_bits[__i]; });
1400 friend constexpr _BitMask
1401 operator^(
const _BitMask& __a,
const _BitMask& __b) noexcept
1408 friend constexpr _BitMask
1409 operator|(
const _BitMask& __a,
const _BitMask& __b) noexcept
1416 friend constexpr _BitMask
1417 operator&(
const _BitMask& __a,
const _BitMask& __b) noexcept
1424 _GLIBCXX_SIMD_INTRINSIC
1426 _M_is_constprop()
const
1428 if constexpr (_S_array_size == 0)
1429 return __builtin_constant_p(_M_bits[0]);
1432 for (
int __i = 0; __i < _S_array_size; ++__i)
1433 if (!__builtin_constant_p(_M_bits[__i]))
1444 template <
typename _Tp =
void>
1445 static inline constexpr
int __min_vector_size = 2 *
sizeof(_Tp);
1447 #if _GLIBCXX_SIMD_HAVE_NEON
1449 inline constexpr
int __min_vector_size<void> = 8;
1452 inline constexpr
int __min_vector_size<void> = 16;
1457 template <
typename _Tp,
size_t _Np,
typename =
void>
1458 struct __vector_type_n {};
1461 template <
typename _Tp>
1462 struct __vector_type_n<_Tp, 0, void> {};
1465 template <
typename _Tp>
1466 struct __vector_type_n<_Tp, 1,
enable_if_t<__is_vectorizable_v<_Tp>>>
1467 {
using type = _Tp; };
1470 template <
typename _Tp,
size_t _Np>
1471 struct __vector_type_n<_Tp, _Np,
1472 enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1474 static constexpr
size_t _S_Np2 = std::__bit_ceil(_Np *
sizeof(_Tp));
1476 static constexpr
size_t _S_Bytes =
1482 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1485 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1488 template <
typename _Tp,
size_t _Bytes,
size_t = _Bytes % sizeof(_Tp)>
1489 struct __vector_type;
1491 template <
typename _Tp,
size_t _Bytes>
1492 struct __vector_type<_Tp, _Bytes, 0>
1493 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1495 template <
typename _Tp,
size_t _Size>
1496 using __vector_type_t =
typename __vector_type_n<_Tp, _Size>::type;
1498 template <
typename _Tp>
1499 using __vector_type2_t =
typename __vector_type<_Tp, 2>::type;
1500 template <
typename _Tp>
1501 using __vector_type4_t =
typename __vector_type<_Tp, 4>::type;
1502 template <
typename _Tp>
1503 using __vector_type8_t =
typename __vector_type<_Tp, 8>::type;
1504 template <
typename _Tp>
1505 using __vector_type16_t =
typename __vector_type<_Tp, 16>::type;
1506 template <
typename _Tp>
1507 using __vector_type32_t =
typename __vector_type<_Tp, 32>::type;
1508 template <
typename _Tp>
1509 using __vector_type64_t =
typename __vector_type<_Tp, 64>::type;
1513 template <
typename _Tp,
typename =
void_t<>>
1516 template <
typename _Tp>
1517 struct __is_vector_type<
1518 _Tp,
void_t<typename __vector_type<
1520 : is_same<_Tp, typename __vector_type<
1521 remove_reference_t<decltype(declval<_Tp>()[0])>,
1522 sizeof(_Tp)>::type> {};
1524 template <
typename _Tp>
1525 inline constexpr
bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1529 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
1530 template <
typename _Tp>
1531 using __is_intrinsic_type = __is_vector_type<_Tp>;
1533 template <
typename _Tp,
typename =
void_t<>>
1536 template <
typename _Tp>
1537 struct __is_intrinsic_type<
1538 _Tp,
void_t<typename __intrinsic_type<
1540 : is_same<_Tp, typename __intrinsic_type<
1541 remove_reference_t<decltype(declval<_Tp>()[0])>,
1542 sizeof(_Tp)>::type> {};
1545 template <
typename _Tp>
1546 inline constexpr
bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1550 template <
typename _Tp,
typename =
void_t<>>
1551 struct _VectorTraitsImpl;
1553 template <
typename _Tp>
1554 struct _VectorTraitsImpl<_Tp,
enable_if_t<__is_vector_type_v<_Tp>
1555 || __is_intrinsic_type_v<_Tp>>>
1558 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1559 static constexpr
int _S_full_size =
sizeof(_Tp) /
sizeof(value_type);
1560 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1561 template <
typename _Up,
int _W = _S_full_size>
1562 static constexpr
bool _S_is
1563 = is_same_v<value_type, _Up> && _W == _S_full_size;
1566 template <
typename _Tp,
size_t _Np>
1567 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1568 void_t<__vector_type_t<_Tp, _Np>>>
1570 using type = __vector_type_t<_Tp, _Np>;
1571 using value_type = _Tp;
1572 static constexpr
int _S_full_size =
sizeof(type) /
sizeof(value_type);
1573 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1574 static constexpr
bool _S_is_partial = (_Np == _S_full_size);
1575 static constexpr
int _S_partial_width = _Np;
1576 template <
typename _Up,
int _W = _S_full_size>
1577 static constexpr
bool _S_is
1578 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1581 template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1582 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1586 template <
typename _V>
1587 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
1590 if constexpr (__is_vector_type_v<_V>)
1592 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1593 return __data(__x)._M_data;
1594 else if constexpr (__is_vectorizable_v<_V>)
1595 return __vector_type_t<_V, 2>{__x};
1602 template <
size_t _Np = 0,
typename _V>
1603 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
1604 __as_wrapper(_V __x)
1606 if constexpr (__is_vector_type_v<_V>)
1607 return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1608 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1609 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1611 static_assert(_V::size() == _Np);
1616 static_assert(_V::_S_size == _Np);
1623 template <
typename _To,
typename _From>
1624 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1625 __intrin_bitcast(_From __v)
1627 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1628 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1629 if constexpr (
sizeof(_To) ==
sizeof(_From))
1630 return reinterpret_cast<_To
>(__v);
1631 else if constexpr (
sizeof(_From) >
sizeof(_To))
1632 if constexpr (
sizeof(_To) >= 16)
1633 return reinterpret_cast<const __may_alias<_To>&>(__v);
1637 __builtin_memcpy(&__r, &__v,
sizeof(_To));
1640 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1641 else if constexpr (__have_avx &&
sizeof(_From) == 16 &&
sizeof(_To) == 32)
1642 return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1643 reinterpret_cast<__vector_type_t<
float, 4>>(__v)));
1644 else if constexpr (__have_avx512f && sizeof(_From) == 16
1645 && sizeof(_To) == 64)
1646 return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1647 reinterpret_cast<__vector_type_t<
float, 4>>(__v)));
1648 else if constexpr (__have_avx512f && sizeof(_From) == 32
1649 && sizeof(_To) == 64)
1650 return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1651 reinterpret_cast<__vector_type_t<
float, 8>>(__v)));
1653 else if constexpr (
sizeof(__v) <= 8)
1654 return reinterpret_cast<_To>(
1655 __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1656 reinterpret_cast<__int_for_sizeof_t<_From>
>(__v)});
1659 static_assert(
sizeof(_To) >
sizeof(_From));
1661 __builtin_memcpy(&__r, &__v,
sizeof(_From));
1668 template <
typename _To,
size_t _NN = 0,
typename _From,
1669 typename _FromVT = _VectorTraits<_From>,
1670 size_t _Np = _NN == 0 ?
sizeof(_From) /
sizeof(_To) : _NN>
1671 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1672 __vector_bitcast(_From __x)
1674 using _R = __vector_type_t<_To, _Np>;
1675 return __intrin_bitcast<_R>(__x);
1678 template <
typename _To,
size_t _NN = 0,
typename _Tp,
size_t _Nx,
1680 = _NN == 0 ?
sizeof(_SimdWrapper<_Tp, _Nx>) /
sizeof(_To) : _NN>
1681 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1682 __vector_bitcast(
const _SimdWrapper<_Tp, _Nx>& __x)
1684 static_assert(_Np > 1);
1685 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1690 #ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1691 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1692 _To __convert_x86(_Tp);
1694 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1695 _To __convert_x86(_Tp, _Tp);
1697 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1698 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1700 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1701 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1703 template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1704 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1705 _Tp, _Tp, _Tp, _Tp);
1710 template <
typename _To,
typename _From>
1711 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1712 __bit_cast(
const _From __x)
1714 #if __has_builtin(__builtin_bit_cast)
1715 return __builtin_bit_cast(_To, __x);
1717 static_assert(
sizeof(_To) ==
sizeof(_From));
1718 constexpr
bool __to_is_vectorizable
1719 = is_arithmetic_v<_To> || is_enum_v<_To>;
1720 constexpr
bool __from_is_vectorizable
1721 = is_arithmetic_v<_From> || is_enum_v<_From>;
1722 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1723 return reinterpret_cast<_To
>(__x);
1724 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1726 using _FV [[gnu::vector_size(
sizeof(_From))]] = _From;
1727 return reinterpret_cast<_To
>(_FV{__x});
1729 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1731 using _TV [[gnu::vector_size(
sizeof(_To))]] = _To;
1732 using _FV [[gnu::vector_size(
sizeof(_From))]] = _From;
1733 return reinterpret_cast<_TV
>(_FV{__x})[0];
1735 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1737 using _TV [[gnu::vector_size(
sizeof(_To))]] = _To;
1738 return reinterpret_cast<_TV
>(__x)[0];
1743 __builtin_memcpy(
reinterpret_cast<char*
>(&__r),
1744 reinterpret_cast<const char*
>(&__x),
sizeof(_To));
1752 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
1754 = __
intrinsic_type_t<
typename _TVT::value_type, _TVT::_S_full_size>>
1755 _GLIBCXX_SIMD_INTRINSIC constexpr _R
1756 __to_intrin(_Tp __x)
1758 static_assert(
sizeof(__x) <=
sizeof(_R),
1759 "__to_intrin may never drop values off the end");
1760 if constexpr (
sizeof(__x) ==
sizeof(_R))
1761 return reinterpret_cast<_R
>(__as_vector(__x));
1764 using _Up = __int_for_sizeof_t<_Tp>;
1765 return reinterpret_cast<_R
>(
1766 __vector_type_t<_Up,
sizeof(_R) /
sizeof(_Up)>{__bit_cast<_Up>(__x)});
1772 template <
typename _Tp,
typename... _Args>
1773 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp,
sizeof...(_Args)>
1774 __make_vector(
const _Args&... __args)
1776 return __vector_type_t<_Tp,
sizeof...(_Args)>{
static_cast<_Tp
>(__args)...};
1781 template <
size_t _Np,
typename _Tp>
1782 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1783 __vector_broadcast(_Tp __x)
1785 return __call_with_n_evaluations<_Np>(
1786 [](
auto... __xx) {
return __vector_type_t<_Tp, _Np>{__xx...}; },
1787 [&__x](int) {
return __x; });
1792 template <
typename _Tp,
size_t _Np,
typename _Gp,
size_t... _I>
1793 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1794 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1796 return __vector_type_t<_Tp, _Np>{
1797 static_cast<_Tp
>(__gen(_SizeConstant<_I>()))...};
1800 template <
typename _V,
typename _VVT = _VectorTraits<_V>,
typename _Gp>
1801 _GLIBCXX_SIMD_INTRINSIC constexpr _V
1802 __generate_vector(_Gp&& __gen)
1804 if constexpr (__is_vector_type_v<_V>)
1805 return __generate_vector_impl<
typename _VVT::value_type,
1806 _VVT::_S_full_size>(
1807 static_cast<_Gp&&
>(__gen), make_index_sequence<_VVT::_S_full_size>());
1809 return __generate_vector_impl<
typename _VVT::value_type,
1810 _VVT::_S_partial_width>(
1811 static_cast<_Gp&&
>(__gen),
1812 make_index_sequence<_VVT::_S_partial_width>());
1815 template <
typename _Tp,
size_t _Np,
typename _Gp>
1816 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1817 __generate_vector(_Gp&& __gen)
1819 return __generate_vector_impl<_Tp, _Np>(
static_cast<_Gp&&
>(__gen),
1820 make_index_sequence<_Np>());
1825 template <
typename _TW>
1826 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1827 __xor(_TW __a, _TW __b) noexcept
1829 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1831 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1832 _VectorTraitsImpl<_TW>>::value_type;
1833 if constexpr (is_floating_point_v<_Tp>)
1835 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1836 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1837 ^ __vector_bitcast<_Ip>(__b));
1839 else if constexpr (__is_vector_type_v<_TW>)
1842 return __a._M_data ^ __b._M_data;
1850 template <
typename _TW>
1851 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1852 __or(_TW __a, _TW __b) noexcept
1854 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1856 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1857 _VectorTraitsImpl<_TW>>::value_type;
1858 if constexpr (is_floating_point_v<_Tp>)
1860 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1861 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1862 | __vector_bitcast<_Ip>(__b));
1864 else if constexpr (__is_vector_type_v<_TW>)
1867 return __a._M_data | __b._M_data;
1875 template <
typename _TW>
1876 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1877 __and(_TW __a, _TW __b) noexcept
1879 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1881 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1882 _VectorTraitsImpl<_TW>>::value_type;
1883 if constexpr (is_floating_point_v<_Tp>)
1885 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1886 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1887 & __vector_bitcast<_Ip>(__b));
1889 else if constexpr (__is_vector_type_v<_TW>)
1892 return __a._M_data & __b._M_data;
1900 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1901 static constexpr
struct
1903 _GLIBCXX_SIMD_INTRINSIC __v4sf
1904 operator()(__v4sf __a, __v4sf __b)
const noexcept
1905 {
return __builtin_ia32_andnps(__a, __b); }
1907 _GLIBCXX_SIMD_INTRINSIC __v2df
1908 operator()(__v2df __a, __v2df __b)
const noexcept
1909 {
return __builtin_ia32_andnpd(__a, __b); }
1911 _GLIBCXX_SIMD_INTRINSIC __v2di
1912 operator()(__v2di __a, __v2di __b)
const noexcept
1913 {
return __builtin_ia32_pandn128(__a, __b); }
1915 _GLIBCXX_SIMD_INTRINSIC __v8sf
1916 operator()(__v8sf __a, __v8sf __b)
const noexcept
1917 {
return __builtin_ia32_andnps256(__a, __b); }
1919 _GLIBCXX_SIMD_INTRINSIC __v4df
1920 operator()(__v4df __a, __v4df __b)
const noexcept
1921 {
return __builtin_ia32_andnpd256(__a, __b); }
1923 _GLIBCXX_SIMD_INTRINSIC __v4di
1924 operator()(__v4di __a, __v4di __b)
const noexcept
1926 if constexpr (__have_avx2)
1927 return __builtin_ia32_andnotsi256(__a, __b);
1929 return reinterpret_cast<__v4di
>(
1930 __builtin_ia32_andnpd256(
reinterpret_cast<__v4df
>(__a),
1931 reinterpret_cast<__v4df
>(__b)));
1934 _GLIBCXX_SIMD_INTRINSIC __v16sf
1935 operator()(__v16sf __a, __v16sf __b)
const noexcept
1937 if constexpr (__have_avx512dq)
1938 return _mm512_andnot_ps(__a, __b);
1940 return reinterpret_cast<__v16sf
>(
1941 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1942 reinterpret_cast<__v8di
>(__b)));
1945 _GLIBCXX_SIMD_INTRINSIC __v8df
1946 operator()(__v8df __a, __v8df __b)
const noexcept
1948 if constexpr (__have_avx512dq)
1949 return _mm512_andnot_pd(__a, __b);
1951 return reinterpret_cast<__v8df
>(
1952 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1953 reinterpret_cast<__v8di
>(__b)));
1956 _GLIBCXX_SIMD_INTRINSIC __v8di
1957 operator()(__v8di __a, __v8di __b)
const noexcept
1958 {
return _mm512_andnot_si512(__a, __b); }
1962 template <
typename _TW>
1963 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1964 __andnot(_TW __a, _TW __b) noexcept
1966 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1968 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1969 _VectorTraitsImpl<_TW>>;
1970 using _Tp =
typename _TVT::value_type;
1971 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1972 if constexpr (
sizeof(_TW) >= 16)
1974 const auto __ai = __to_intrin(__a);
1975 const auto __bi = __to_intrin(__b);
1976 if (!__builtin_is_constant_evaluated()
1977 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1979 const auto __r = _S_x86_andnot(__ai, __bi);
1980 if constexpr (is_convertible_v<decltype(__r), _TW>)
1983 return reinterpret_cast<typename _TVT::type>(__r);
1987 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1988 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
1989 & __vector_bitcast<_Ip>(__b));
1997 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1998 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
1999 __not(_Tp __a) noexcept
2001 if constexpr (is_floating_point_v<typename _TVT::value_type>)
2002 return reinterpret_cast<typename _TVT::type
>(
2003 ~__vector_bitcast<unsigned>(__a));
2010 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
2011 typename _R = __vector_type_t<
typename _TVT::value_type,
2012 _TVT::_S_full_size * 2>>
2014 __concat(_Tp a_, _Tp b_)
2016 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2018 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
2020 long long,
typename _TVT::value_type>>;
2021 constexpr
int input_width =
sizeof(_Tp) /
sizeof(_W);
2022 const auto __a = __vector_bitcast<_W>(a_);
2023 const auto __b = __vector_bitcast<_W>(b_);
2024 using _Up = __vector_type_t<_W,
sizeof(_R) /
sizeof(_W)>;
2026 constexpr
int input_width = _TVT::_S_full_size;
2027 const _Tp& __a = a_;
2028 const _Tp& __b = b_;
2031 if constexpr (input_width == 2)
2032 return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
2033 else if constexpr (input_width == 4)
2034 return reinterpret_cast<_R>(
2035 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
2036 else if constexpr (input_width == 8)
2037 return reinterpret_cast<_R>(
2038 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
2039 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
2040 else if constexpr (input_width == 16)
2041 return reinterpret_cast<_R>(
2042 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2043 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2044 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
2045 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
2046 __b[12], __b[13], __b[14], __b[15]});
2047 else if constexpr (input_width == 32)
2048 return reinterpret_cast<_R>(
2049 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2050 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2051 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
2052 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
2053 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
2054 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
2055 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
2056 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
2057 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
2063 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2064 struct _ZeroExtendProxy
2066 using value_type =
typename _TVT::value_type;
2067 static constexpr
size_t _Np = _TVT::_S_full_size;
2070 template <
typename _To,
typename _ToVT = _VectorTraits<_To>,
2072 = enable_if_t<is_same_v<
typename _ToVT::value_type, value_type>>>
2073 _GLIBCXX_SIMD_INTRINSIC
operator _To()
const
2075 constexpr
size_t _ToN = _ToVT::_S_full_size;
2076 if constexpr (_ToN == _Np)
2078 else if constexpr (_ToN == 2 * _Np)
2080 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2081 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
2082 return __vector_bitcast<value_type>(
2083 _mm256_insertf128_ps(__m256(), __x, 0));
2084 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
2085 return __vector_bitcast<value_type>(
2086 _mm256_insertf128_pd(__m256d(), __x, 0));
2087 else if constexpr (__have_avx2 && _Np *
sizeof(value_type) == 16)
2088 return __vector_bitcast<value_type>(
2089 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
2090 else if constexpr (__have_avx512f && _TVT::template _S_is<
float, 8>)
2092 if constexpr (__have_avx512dq)
2093 return __vector_bitcast<value_type>(
2094 _mm512_insertf32x8(__m512(), __x, 0));
2096 return reinterpret_cast<__m512
>(
2097 _mm512_insertf64x4(__m512d(),
2098 reinterpret_cast<__m256d
>(__x), 0));
2100 else if constexpr (__have_avx512f
2101 && _TVT::template _S_is<double, 4>)
2102 return __vector_bitcast<value_type>(
2103 _mm512_insertf64x4(__m512d(), __x, 0));
2104 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 32)
2105 return __vector_bitcast<value_type>(
2106 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
2108 return __concat(__x, _Tp());
2110 else if constexpr (_ToN == 4 * _Np)
2112 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2113 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2115 return __vector_bitcast<value_type>(
2116 _mm512_insertf64x2(__m512d(), __x, 0));
2118 else if constexpr (__have_avx512f
2119 && is_floating_point_v<value_type>)
2121 return __vector_bitcast<value_type>(
2122 _mm512_insertf32x4(__m512(),
reinterpret_cast<__m128
>(__x),
2125 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 16)
2127 return __vector_bitcast<value_type>(
2128 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2131 return __concat(__concat(__x, _Tp()),
2132 __vector_type_t<value_type, _Np * 2>());
2134 else if constexpr (_ToN == 8 * _Np)
2135 return __concat(
operator __vector_type_t<value_type, _Np * 4>(),
2136 __vector_type_t<value_type, _Np * 4>());
2137 else if constexpr (_ToN == 16 * _Np)
2138 return __concat(
operator __vector_type_t<value_type, _Np * 8>(),
2139 __vector_type_t<value_type, _Np * 8>());
2141 __assert_unreachable<_Tp>();
2145 template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2146 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2147 __zero_extend(_Tp __x)
2152 template <
int _Offset,
2155 typename _TVT = _VectorTraits<_Tp>,
2156 typename _R = __vector_type_t<
typename _TVT::value_type,
2157 _TVT::_S_full_size / _SplitBy>>
2158 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2161 using value_type =
typename _TVT::value_type;
2162 #if _GLIBCXX_SIMD_X86INTRIN
2163 if constexpr (
sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2165 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2166 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2167 else if constexpr (is_floating_point_v<value_type>)
2168 return __vector_bitcast<value_type>(
2169 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2171 return reinterpret_cast<_R
>(
2172 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2178 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2180 is_floating_point_v<value_type>, double,
2182 static_assert(
sizeof(_R) %
sizeof(_W) == 0);
2183 constexpr
int __return_width =
sizeof(_R) /
sizeof(_W);
2184 using _Up = __vector_type_t<_W, __return_width>;
2185 const auto __x = __vector_bitcast<_W>(__in);
2187 constexpr
int __return_width = _TVT::_S_full_size / _SplitBy;
2189 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2192 constexpr
int _O = _Offset * __return_width;
2193 return __call_with_subscripts<__return_width, _O>(
2194 __x, [](
auto... __entries) {
2195 return reinterpret_cast<_R
>(_Up{__entries...});
2202 template <
typename _Tp,
2204 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2205 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2209 __builtin_memcpy(&__r, &__x, 8);
2213 template <
typename _Tp,
2215 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2216 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2219 static_assert(
sizeof(_Tp) == 16,
"use __hi64z if you meant it");
2221 __builtin_memcpy(&__r,
reinterpret_cast<const char*
>(&__x) + 8, 8);
2225 template <
typename _Tp,
2227 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2228 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2229 __hi64z([[maybe_unused]] _Tp __x)
2232 if constexpr (
sizeof(_Tp) == 16)
2233 __builtin_memcpy(&__r, reinterpret_cast<const
char*>(&__x) + 8, 8);
2239 template <typename _Tp>
2240 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2242 {
return __extract<0,
sizeof(_Tp) / 16>(__x); }
2244 template <
typename _Tp>
2245 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
2248 static_assert(
sizeof(__x) == 32);
2249 return __extract<1, 2>(__x);
2254 template <
typename _Tp>
2255 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
2258 static_assert(
sizeof(__x) == 64);
2259 return __extract<0, 2>(__x);
2262 template <
typename _Tp>
2263 _GLIBCXX_SIMD_INTRINSIC constexpr
auto
2266 static_assert(
sizeof(__x) == 64);
2267 return __extract<1, 2>(__x);
2272 template <
typename _Tp>
2275 static_assert(__is_vector_type_v<_Tp>);
2279 template <
typename _Up,
typename _UVT = _VectorTraits<_Up>>
2280 _GLIBCXX_SIMD_INTRINSIC constexpr
operator _Up()
const
2281 {
return __intrin_bitcast<typename _UVT::type>(__x); }
2284 template <
typename _Tp>
2285 _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2286 __auto_bitcast(
const _Tp& __x)
2289 template <
typename _Tp,
size_t _Np>
2290 _GLIBCXX_SIMD_INTRINSIC constexpr
2291 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2292 __auto_bitcast(
const _SimdWrapper<_Tp, _Np>& __x)
2293 {
return {__x._M_data}; }
2298 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
2300 #if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2301 template <
size_t _Size>
2302 struct __bool_storage_member_type
2304 static_assert((_Size & (_Size - 1)) != 0,
2305 "This trait may only be used for non-power-of-2 sizes. "
2306 "Power-of-2 sizes must be specialized.");
2308 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2312 struct __bool_storage_member_type<1> {
using type = bool; };
2315 struct __bool_storage_member_type<2> {
using type = __mmask8; };
2318 struct __bool_storage_member_type<4> {
using type = __mmask8; };
2321 struct __bool_storage_member_type<8> {
using type = __mmask8; };
2324 struct __bool_storage_member_type<16> {
using type = __mmask16; };
2327 struct __bool_storage_member_type<32> {
using type = __mmask32; };
2330 struct __bool_storage_member_type<64> {
using type = __mmask64; };
2336 #if _GLIBCXX_SIMD_HAVE_SSE
2337 template <
typename _Tp,
size_t _Bytes>
2338 struct __intrinsic_type<_Tp, _Bytes,
2339 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2341 static_assert(!is_same_v<_Tp, long double>,
2342 "no __intrinsic_type support for long double on x86");
2344 static constexpr
size_t _S_VBytes = _Bytes <= 16 ? 16
2348 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2349 = conditional_t<is_integral_v<_Tp>,
long long int, _Tp>;
2356 #if _GLIBCXX_SIMD_HAVE_NEON
2358 struct __intrinsic_type<float, 8, void>
2359 {
using type = float32x2_t; };
2362 struct __intrinsic_type<float, 16, void>
2363 {
using type = float32x4_t; };
2365 #if _GLIBCXX_SIMD_HAVE_NEON_A64
2367 struct __intrinsic_type<double, 8, void>
2368 {
using type = float64x1_t; };
2371 struct __intrinsic_type<double, 16, void>
2372 {
using type = float64x2_t; };
2375 #define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2377 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2378 _Np * _Bits / 8, void> \
2379 { using type = int##_Bits##x##_Np##_t; }; \
2381 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2382 _Np * _Bits / 8, void> \
2383 { using type = uint##_Bits##x##_Np##_t; }
2384 _GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2385 _GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2386 _GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2387 _GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2388 _GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2389 _GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2390 _GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2391 _GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2392 #undef _GLIBCXX_SIMD_ARM_INTRIN
2394 template <
typename _Tp,
size_t _Bytes>
2395 struct __intrinsic_type<_Tp, _Bytes,
2396 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2398 static constexpr
int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2399 using _Ip = __int_for_sizeof_t<_Tp>;
2401 is_floating_point_v<_Tp>, _Tp,
2402 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2403 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2404 "should use explicit specialization above");
2405 using type =
typename __intrinsic_type<_Up, _SVecBytes>::type;
2412 template <
typename _Tp>
2413 struct __intrinsic_type_impl;
2415 #define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2417 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2418 _GLIBCXX_SIMD_PPC_INTRIN(
float);
2419 _GLIBCXX_SIMD_PPC_INTRIN(
double);
2420 _GLIBCXX_SIMD_PPC_INTRIN(
signed char);
2421 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned char);
2422 _GLIBCXX_SIMD_PPC_INTRIN(
signed short);
2423 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned short);
2424 _GLIBCXX_SIMD_PPC_INTRIN(
signed int);
2425 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned int);
2426 _GLIBCXX_SIMD_PPC_INTRIN(
signed long);
2427 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned long);
2428 _GLIBCXX_SIMD_PPC_INTRIN(
signed long long);
2429 _GLIBCXX_SIMD_PPC_INTRIN(
unsigned long long);
2430 #undef _GLIBCXX_SIMD_PPC_INTRIN
2432 template <
typename _Tp,
size_t _Bytes>
2433 struct __intrinsic_type<_Tp, _Bytes,
2434 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2436 static constexpr
bool _S_is_ldouble = is_same_v<_Tp, long double>;
2438 static_assert(!(_S_is_ldouble &&
sizeof(
long double) >
sizeof(
double)),
2439 "no __intrinsic_type support for long double on PPC");
2441 static_assert(!is_same_v<_Tp, double>,
2442 "no __intrinsic_type support for double on PPC w/o VSX");
2445 typename __intrinsic_type_impl<
2446 conditional_t<is_floating_point_v<_Tp>,
2447 conditional_t<_S_is_ldouble, double, _Tp>,
2448 __int_for_sizeof_t<_Tp>>>::type;
2454 template <
size_t _W
idth>
2455 struct _SimdWrapper<bool, _Width,
2456 void_t<typename __bool_storage_member_type<_Width>::type>>
2458 using _BuiltinType =
typename __bool_storage_member_type<_Width>::type;
2459 using value_type = bool;
2461 static constexpr
size_t _S_full_size =
sizeof(_BuiltinType) * __CHAR_BIT__;
2463 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2464 __as_full_vector()
const {
return _M_data; }
2466 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() =
default;
2467 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_BuiltinType __k)
2470 _GLIBCXX_SIMD_INTRINSIC
operator const _BuiltinType&()
const
2473 _GLIBCXX_SIMD_INTRINSIC
operator _BuiltinType&()
2476 _GLIBCXX_SIMD_INTRINSIC _BuiltinType __intrin()
const
2479 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator[](
size_t __i)
const
2480 {
return _M_data & (_BuiltinType(1) << __i); }
2482 template <
size_t __i>
2483 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2484 operator[](_SizeConstant<__i>)
const
2485 {
return _M_data & (_BuiltinType(1) << __i); }
2487 _GLIBCXX_SIMD_INTRINSIC constexpr
void _M_set(
size_t __i, value_type __x)
2490 _M_data |= (_BuiltinType(1) << __i);
2492 _M_data &= ~(_BuiltinType(1) << __i);
2495 _GLIBCXX_SIMD_INTRINSIC
2496 constexpr
bool _M_is_constprop()
const
2497 {
return __builtin_constant_p(_M_data); }
2499 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_none_of()
const
2501 if (__builtin_constant_p(_M_data))
2503 constexpr
int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2504 constexpr _BuiltinType __active_mask
2505 = ~_BuiltinType() >> (__nbits - _Width);
2506 return (_M_data & __active_mask) == 0;
2511 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_all_of()
const
2513 if (__builtin_constant_p(_M_data))
2515 constexpr
int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2516 constexpr _BuiltinType __active_mask
2517 = ~_BuiltinType() >> (__nbits - _Width);
2518 return (_M_data & __active_mask) == __active_mask;
2523 _BuiltinType _M_data;
2527 template <
bool _MustZeroInitPadding,
typename _BuiltinType>
2528 struct _SimdWrapperBase;
2530 template <
typename _BuiltinType>
2531 struct _SimdWrapperBase<false, _BuiltinType>
2533 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() =
default;
2534 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2538 _BuiltinType _M_data;
2541 template <
typename _BuiltinType>
2542 struct _SimdWrapperBase<true, _BuiltinType>
2545 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() : _M_data() {}
2546 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2550 _BuiltinType _M_data;
2555 template <
typename _Tp,
size_t _W
idth>
2556 struct _SimdWrapper<
2558 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2559 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2560 && sizeof(_Tp) * _Width
2561 == sizeof(__vector_type_t<_Tp, _Width>),
2562 __vector_type_t<_Tp, _Width>>
2565 = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2566 &&
sizeof(_Tp) * _Width
2567 ==
sizeof(__vector_type_t<_Tp, _Width>),
2568 __vector_type_t<_Tp, _Width>>;
2570 static_assert(__is_vectorizable_v<_Tp>);
2571 static_assert(_Width >= 2);
2573 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2574 using value_type = _Tp;
2576 static inline constexpr
size_t _S_full_size
2577 =
sizeof(_BuiltinType) /
sizeof(value_type);
2578 static inline constexpr
int _S_size = _Width;
2579 static inline constexpr
bool _S_is_partial = _S_full_size != _S_size;
2581 using _Base::_M_data;
2583 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2584 __as_full_vector()
const
2587 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(initializer_list<_Tp> __init)
2588 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2589 [&](auto __i) {
return __init.begin()[__i.value]; })) {}
2591 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() =
default;
2592 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(
const _SimdWrapper&)
2594 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_SimdWrapper&&) =
default;
2596 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2597 operator=(
const _SimdWrapper&) =
default;
2598 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2599 operator=(_SimdWrapper&&) =
default;
2601 template <
typename _V,
typename =
enable_if_t<disjunction_v<
2602 is_same<_V, __vector_type_t<_Tp, _Width>>,
2603 is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2604 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_V __x)
2606 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2608 template <
typename... _As,
2609 typename =
enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2610 &&
sizeof...(_As) <= _Width)>>
2611 _GLIBCXX_SIMD_INTRINSIC constexpr
2612 operator _SimdTuple<_Tp, _As...>()
const
2614 const auto& dd = _M_data;
2615 return __generate_from_n_evaluations<
sizeof...(_As),
2616 _SimdTuple<_Tp, _As...>>([&](
2617 auto __i) constexpr {
return dd[int(__i)]; });
2620 _GLIBCXX_SIMD_INTRINSIC constexpr
operator const _BuiltinType&()
const
2623 _GLIBCXX_SIMD_INTRINSIC constexpr
operator _BuiltinType&()
2626 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](
size_t __i)
const
2627 {
return _M_data[__i]; }
2629 template <
size_t __i>
2630 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](_SizeConstant<__i>)
const
2631 {
return _M_data[__i]; }
2633 _GLIBCXX_SIMD_INTRINSIC constexpr
void _M_set(
size_t __i, _Tp __x)
2634 { _M_data[__i] = __x; }
2636 _GLIBCXX_SIMD_INTRINSIC
2637 constexpr
bool _M_is_constprop()
const
2638 {
return __builtin_constant_p(_M_data); }
2640 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_none_of()
const
2642 if (__builtin_constant_p(_M_data))
2645 if constexpr (is_floating_point_v<_Tp>)
2647 using _Ip = __int_for_sizeof_t<_Tp>;
2648 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2649 __execute_n_times<_Width>(
2650 [&](
auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2653 __execute_n_times<_Width>(
2654 [&](
auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2660 _GLIBCXX_SIMD_INTRINSIC constexpr
bool _M_is_constprop_all_of()
const
2662 if (__builtin_constant_p(_M_data))
2665 if constexpr (is_floating_point_v<_Tp>)
2667 using _Ip = __int_for_sizeof_t<_Tp>;
2668 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2669 __execute_n_times<_Width>(
2670 [&](
auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2673 __execute_n_times<_Width>(
2674 [&](
auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2684 template <
typename _Tp>
2686 __vectorized_sizeof()
2688 if constexpr (!__is_vectorizable_v<_Tp>)
2691 if constexpr (
sizeof(_Tp) <= 8)
2694 if constexpr (__have_avx512bw)
2696 if constexpr (__have_avx512f &&
sizeof(_Tp) >= 4)
2698 if constexpr (__have_avx2)
2700 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2702 if constexpr (__have_sse2)
2704 if constexpr (__have_sse && is_same_v<_Tp,
float>)
2714 if constexpr (__have_power8vec
2715 || (__have_power_vmx && (sizeof(_Tp) < 8))
2716 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2720 if constexpr (__have_neon_a64
2721 || (__have_neon_a32 && !is_same_v<_Tp,
double>) )
2723 if constexpr (__have_neon
2728 && (__support_neon_float || !is_floating_point_v<_Tp>))
2736 namespace simd_abi {
2738 template <
typename _Tp>
2739 inline constexpr
int max_fixed_size
2740 = (__have_avx512bw &&
sizeof(_Tp) == 1) ? 64 : 32;
2743 #if defined __x86_64__ || defined __aarch64__
2744 template <
typename _Tp>
2745 using compatible =
conditional_t<(
sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2746 #elif defined __ARM_NEON
2749 template <
typename _Tp>
2752 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2753 _VecBuiltin<16>, scalar>;
2756 using compatible = scalar;
2761 template <
typename _Tp>
2763 __determine_native_abi()
2765 constexpr
size_t __bytes = __vectorized_sizeof<_Tp>();
2766 if constexpr (__bytes ==
sizeof(_Tp))
2767 return static_cast<scalar*
>(
nullptr);
2768 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2769 return static_cast<_VecBltnBtmsk<__bytes>*>(
nullptr);
2771 return static_cast<_VecBuiltin<__bytes>*>(
nullptr);
2774 template <typename _Tp, typename =
enable_if_t<__is_vectorizable_v<_Tp>>>
2779 #if defined _GLIBCXX_SIMD_DEFAULT_ABI
2780 template <
typename _Tp>
2781 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2783 template <
typename _Tp>
2784 using __default_abi = compatible<_Tp>;
2792 template <
typename _Tp,
typename =
void_t<>>
2795 template <
typename _Tp>
2796 struct is_abi_tag<_Tp,
void_t<typename _Tp::_IsValidAbiTag>>
2797 :
public _Tp::_IsValidAbiTag {};
2799 template <
typename _Tp>
2800 inline constexpr
bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2803 template <
typename _Tp>
2806 template <
typename _Tp>
2807 inline constexpr
bool is_simd_v = is_simd<_Tp>::value;
2809 template <
typename _Tp>
2812 template <
typename _Tp>
2813 inline constexpr
bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2816 template <
typename _Tp,
typename _Abi,
typename =
void>
2817 struct __simd_size_impl {};
2819 template <
typename _Tp,
typename _Abi>
2820 struct __simd_size_impl<
2822 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2823 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2825 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2826 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2828 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2829 inline constexpr
size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2832 template <
typename _Tp,
size_t _Np,
typename =
void>
2833 struct __deduce_impl;
2835 namespace simd_abi {
2844 template <
typename _Tp,
size_t _Np,
typename...>
2847 template <
typename _Tp,
size_t _Np,
typename... _Abis>
2848 using deduce_t =
typename deduce<_Tp, _Np, _Abis...>::type;
2853 template <
typename _Tp,
typename _V,
typename =
void>
2856 template <
typename _Tp,
typename _Up,
typename _Abi>
2858 _Tp, simd<_Up, _Abi>,
2859 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2862 = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2865 template <
typename _Tp,
typename _Up,
typename _Abi>
2867 _Tp, simd_mask<_Up, _Abi>,
2868 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2871 = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2874 template <
typename _Tp,
typename _V>
2875 using rebind_simd_t =
typename rebind_simd<_Tp, _V>::type;
2878 template <
int _Np,
typename _V,
typename =
void>
2881 template <
int _Np,
typename _Tp,
typename _Abi>
2882 struct resize_simd<_Np, simd<_Tp, _Abi>,
2883 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2884 {
using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2886 template <
int _Np,
typename _Tp,
typename _Abi>
2887 struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
2888 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2889 {
using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2891 template <
int _Np,
typename _V>
2892 using resize_simd_t =
typename resize_simd<_Np, _V>::type;
2896 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
2897 struct memory_alignment
2898 :
public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2900 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
2901 inline constexpr
size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2904 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2907 template <
typename _Tp,
typename _Abi>
2908 struct is_simd<simd<_Tp, _Abi>> :
public true_type {};
2910 template <
typename _Tp>
2911 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2913 template <
typename _Tp,
int _Np>
2914 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2916 template <
typename _Tp,
size_t _Np>
2917 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2920 template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2923 template <
typename _Tp,
typename _Abi>
2924 struct is_simd_mask<simd_mask<_Tp, _Abi>> :
public true_type {};
2926 template <
typename _Tp>
2927 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2929 template <
typename _Tp,
int _Np>
2930 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2932 template <
typename _Tp,
size_t _Np>
2933 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2937 template <
typename _Tp,
typename _Up,
typename _Ap,
bool = is_simd_v<_Tp>,
2939 struct __static_simd_cast_return_type;
2941 template <
typename _Tp,
typename _A0,
typename _Up,
typename _Ap>
2942 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false,
2944 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2946 template <
typename _Tp,
typename _Up,
typename _Ap>
2947 struct __static_simd_cast_return_type<
2948 _Tp, _Up, _Ap, true,
enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
2949 {
using type = _Tp; };
2951 template <
typename _Tp,
typename _Ap>
2952 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2953 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2959 {
using type = simd<_Tp, _Ap>; };
2961 template <
typename _Tp,
typename =
void>
2962 struct __safe_make_signed {
using type = _Tp;};
2964 template <
typename _Tp>
2965 struct __safe_make_signed<_Tp,
enable_if_t<is_integral_v<_Tp>>>
2968 using type = make_signed_t<make_unsigned_t<_Tp>>;
2971 template <
typename _Tp>
2972 using safe_make_signed_t =
typename __safe_make_signed<_Tp>::type;
2974 template <
typename _Tp,
typename _Up,
typename _Ap>
2975 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
2976 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2984 (is_integral_v<_Up> && is_integral_v<_Tp> &&
2985 #ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
2986 is_signed_v<_Up> != is_signed_v<_Tp> &&
2988 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
2989 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
2992 template <
typename _Tp,
typename _Up,
typename _Ap,
2994 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2995 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
2996 static_simd_cast(
const simd<_Up, _Ap>& __x)
2998 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
3002 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
3004 return _R(__private_init, __c(__data(__x)));
3008 namespace __proposed {
3009 template <
typename _Tp,
typename _Up,
typename _Ap,
3011 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3012 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
typename _R::mask_type
3013 static_simd_cast(
const simd_mask<_Up, _Ap>& __x)
3015 using _RM =
typename _R::mask_type;
3016 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
3017 typename _RM::simd_type::value_type>(__x)};
3020 template <
typename _To,
typename _Up,
typename _Abi>
3021 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3023 simd_bit_cast(
const simd<_Up, _Abi>& __x)
3025 using _Tp =
typename _To::value_type;
3026 using _ToMember =
typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3027 using _From = simd<_Up, _Abi>;
3028 using _FromMember =
typename _SimdTraits<_Up, _Abi>::_SimdMember;
3030 static_assert(
sizeof(_To) ==
sizeof(_From));
3031 static_assert(is_trivially_copyable_v<_Tp> && is_trivially_copyable_v<_Up>);
3032 static_assert(is_trivially_copyable_v<_ToMember> && is_trivially_copyable_v<_FromMember>);
3033 #if __has_builtin(__builtin_bit_cast)
3034 return {__private_init, __builtin_bit_cast(_ToMember, __data(__x))};
3036 return {__private_init, __bit_cast<_ToMember>(__data(__x))};
3040 template <
typename _To,
typename _Up,
typename _Abi>
3041 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3043 simd_bit_cast(
const simd_mask<_Up, _Abi>& __x)
3045 using _From = simd_mask<_Up, _Abi>;
3046 static_assert(
sizeof(_To) ==
sizeof(_From));
3047 static_assert(is_trivially_copyable_v<_From>);
3050 if constexpr (is_simd_v<_To>)
3052 using _Tp =
typename _To::value_type;
3053 using _ToMember =
typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3054 static_assert(is_trivially_copyable_v<_ToMember>);
3055 #if __has_builtin(__builtin_bit_cast)
3056 return {__private_init, __builtin_bit_cast(_ToMember, __x)};
3058 return {__private_init, __bit_cast<_ToMember>(__x)};
3063 static_assert(is_trivially_copyable_v<_To>);
3064 #if __has_builtin(__builtin_bit_cast)
3065 return __builtin_bit_cast(_To, __x);
3067 return __bit_cast<_To>(__x);
3074 template <
typename _Tp,
typename _Up,
typename _Ap,
3075 typename _To = __value_type_or_identity_t<_Tp>>
3076 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
3077 simd_cast(
const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
3078 -> decltype(static_simd_cast<_Tp>(__x))
3079 {
return static_simd_cast<_Tp>(__x); }
3081 namespace __proposed {
3082 template <
typename _Tp,
typename _Up,
typename _Ap,
3083 typename _To = __value_type_or_identity_t<_Tp>>
3084 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
3085 simd_cast(
const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
3086 -> decltype(static_simd_cast<_Tp>(__x))
3087 {
return static_simd_cast<_Tp>(__x); }
3092 namespace __proposed {
3119 template <
typename _Tp,
typename _Up,
typename _Ap>
3120 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
enable_if_t<
3121 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3122 resizing_simd_cast(
const simd<_Up, _Ap>& __x)
3124 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3126 else if constexpr (simd_size_v<_Up, _Ap> == 1)
3132 else if constexpr (_Tp::size() == 1)
3134 else if constexpr (sizeof(_Tp) == sizeof(__x)
3135 && !__is_fixed_size_abi_v<_Ap>)
3136 return {__private_init,
3137 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3138 _Ap::_S_masked(__data(__x))._M_data)};
3142 __builtin_memcpy(&__data(__r), &__data(__x),
3144 *
std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3149 template <
typename _Tp,
typename _Up,
typename _Ap>
3150 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3151 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3152 resizing_simd_cast(
const simd_mask<_Up, _Ap>& __x)
3154 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3155 typename _Tp::simd_type::value_type>(__x)};
3161 template <
typename _Tp,
int _Np>
3162 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3163 to_fixed_size(
const fixed_size_simd<_Tp, _Np>& __x)
3166 template <
typename _Tp,
int _Np>
3167 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3168 to_fixed_size(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3171 template <
typename _Tp,
typename _Ap>
3172 _GLIBCXX_SIMD_INTRINSIC
auto
3173 to_fixed_size(
const simd<_Tp, _Ap>& __x)
3175 return simd<_Tp, simd_abi::fixed_size<simd_size_v<_Tp, _Ap>>>([&__x](
3176 auto __i) constexpr {
return __x[__i]; });
3179 template <
typename _Tp,
typename _Ap>
3180 _GLIBCXX_SIMD_INTRINSIC
auto
3181 to_fixed_size(
const simd_mask<_Tp, _Ap>& __x)
3183 constexpr
int _Np = simd_mask<_Tp, _Ap>::size();
3184 fixed_size_simd_mask<_Tp, _Np> __r;
3185 __execute_n_times<_Np>([&](
auto __i) constexpr { __r[__i] = __x[__i]; });
3190 template <
typename _Tp,
int _Np>
3191 _GLIBCXX_SIMD_INTRINSIC
3192 enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3193 to_native(
const fixed_size_simd<_Tp, _Np>& __x)
3195 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3196 __x.copy_to(__mem, vector_aligned);
3197 return {__mem, vector_aligned};
3200 template <
typename _Tp,
size_t _Np>
3201 _GLIBCXX_SIMD_INTRINSIC
3202 enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3203 to_native(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3205 return native_simd_mask<_Tp>([&](
auto __i) constexpr {
return __x[__i]; });
3209 template <
typename _Tp,
size_t _Np>
3210 _GLIBCXX_SIMD_INTRINSIC
enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3211 to_compatible(
const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3213 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3214 __x.copy_to(__mem, vector_aligned);
3215 return {__mem, vector_aligned};
3218 template <
typename _Tp,
size_t _Np>
3219 _GLIBCXX_SIMD_INTRINSIC
3220 enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3221 to_compatible(
const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3222 {
return simd_mask<_Tp>([&](
auto __i) constexpr {
return __x[__i]; }); }
3228 template <
typename _M,
typename _Tp>
3229 class const_where_expression
3232 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3234 struct _Wrapper {
using value_type = _V; };
3237 using _Impl =
typename _V::_Impl;
3240 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3242 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3243 __get_mask(
const const_where_expression& __x)
3244 {
return __x._M_k; }
3246 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3247 __get_lvalue(
const const_where_expression& __x)
3248 {
return __x._M_value; }
3254 const_where_expression(
const const_where_expression&) =
delete;
3255 const_where_expression& operator=(
const const_where_expression&) =
delete;
3257 _GLIBCXX_SIMD_INTRINSIC const_where_expression(
const _M& __kk,
const _Tp& dd)
3258 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3260 _GLIBCXX_SIMD_INTRINSIC _V
3263 return {__private_init,
3264 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3268 template <
typename _Up,
typename _Flags>
3269 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3270 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3272 return {__private_init,
3273 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3274 _Flags::template _S_apply<_V>(__mem))};
3277 template <
typename _Up,
typename _Flags>
3278 _GLIBCXX_SIMD_INTRINSIC
void
3279 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3281 _Impl::_S_masked_store(__data(_M_value),
3282 _Flags::template _S_apply<_V>(__mem),
3288 template <
typename _Tp>
3289 class const_where_expression<bool, _Tp>
3294 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3296 struct _Wrapper {
using value_type = _V; };
3300 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3302 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3303 __get_mask(
const const_where_expression& __x)
3304 {
return __x._M_k; }
3306 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3307 __get_lvalue(
const const_where_expression& __x)
3308 {
return __x._M_value; }
3314 const_where_expression(
const const_where_expression&) =
delete;
3315 const_where_expression& operator=(
const const_where_expression&) =
delete;
3317 _GLIBCXX_SIMD_INTRINSIC const_where_expression(
const bool __kk,
const _Tp& dd)
3318 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3320 _GLIBCXX_SIMD_INTRINSIC _V
operator-() const&&
3321 {
return _M_k ? -_M_value : _M_value; }
3323 template <
typename _Up,
typename _Flags>
3324 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3325 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3326 {
return _M_k ?
static_cast<_V
>(__mem[0]) : _M_value; }
3328 template <
typename _Up,
typename _Flags>
3329 _GLIBCXX_SIMD_INTRINSIC
void
3330 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3333 __mem[0] = _M_value;
3338 template <
typename _M,
typename _Tp>
3339 class where_expression :
public const_where_expression<_M, _Tp>
3341 using _Impl =
typename const_where_expression<_M, _Tp>::_Impl;
3343 static_assert(!is_const<_Tp>::value,
3344 "where_expression may only be instantiated with __a non-const "
3347 using typename const_where_expression<_M, _Tp>::value_type;
3348 using const_where_expression<_M, _Tp>::_M_k;
3349 using const_where_expression<_M, _Tp>::_M_value;
3352 is_same<typename _M::abi_type, typename _Tp::abi_type>::value,
"");
3353 static_assert(_M::size() == _Tp::size(),
"");
3355 _GLIBCXX_SIMD_INTRINSIC
friend _Tp& __get_lvalue(where_expression& __x)
3356 {
return __x._M_value; }
3359 where_expression(
const where_expression&) =
delete;
3360 where_expression& operator=(
const where_expression&) =
delete;
3362 _GLIBCXX_SIMD_INTRINSIC where_expression(
const _M& __kk, _Tp& dd)
3363 : const_where_expression<_M, _Tp>(__kk, dd) {}
3365 template <
typename _Up>
3366 _GLIBCXX_SIMD_INTRINSIC
void operator=(_Up&& __x) &&
3368 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3369 __to_value_type_or_member_type<_Tp>(
3370 static_cast<_Up&&
>(__x)));
3373 #define _GLIBCXX_SIMD_OP_(__op, __name) \
3374 template <typename _Up> \
3375 _GLIBCXX_SIMD_INTRINSIC void operator __op##=(_Up&& __x)&& \
3377 _Impl::template _S_masked_cassign( \
3378 __data(_M_k), __data(_M_value), \
3379 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3380 [](auto __impl, auto __lhs, auto __rhs) constexpr { \
3381 return __impl.__name(__lhs, __rhs); \
3385 _GLIBCXX_SIMD_OP_(+, _S_plus);
3386 _GLIBCXX_SIMD_OP_(-, _S_minus);
3387 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3388 _GLIBCXX_SIMD_OP_(/, _S_divides);
3389 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3390 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3391 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3392 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3393 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3394 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3395 #undef _GLIBCXX_SIMD_OP_
3397 _GLIBCXX_SIMD_INTRINSIC
void operator++() &&
3400 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3404 _GLIBCXX_SIMD_INTRINSIC
void operator++(
int) &&
3407 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3411 _GLIBCXX_SIMD_INTRINSIC
void operator--() &&
3414 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3418 _GLIBCXX_SIMD_INTRINSIC
void operator--(
int) &&
3421 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3426 template <
typename _Up,
typename _Flags>
3427 _GLIBCXX_SIMD_INTRINSIC
void
3428 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3431 = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3432 _Flags::template _S_apply<_Tp>(__mem));
3437 template <
typename _Tp>
3438 class where_expression<bool, _Tp> :
public const_where_expression<bool, _Tp>
3441 using typename const_where_expression<_M, _Tp>::value_type;
3442 using const_where_expression<_M, _Tp>::_M_k;
3443 using const_where_expression<_M, _Tp>::_M_value;
3446 where_expression(
const where_expression&) =
delete;
3447 where_expression& operator=(
const where_expression&) =
delete;
3449 _GLIBCXX_SIMD_INTRINSIC where_expression(
const _M& __kk, _Tp& dd)
3450 : const_where_expression<_M, _Tp>(__kk, dd) {}
3452 #define _GLIBCXX_SIMD_OP_(__op) \
3453 template <typename _Up> \
3454 _GLIBCXX_SIMD_INTRINSIC void operator __op(_Up&& __x)&& \
3455 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3457 _GLIBCXX_SIMD_OP_(=)
3458 _GLIBCXX_SIMD_OP_(+=)
3459 _GLIBCXX_SIMD_OP_(-=)
3460 _GLIBCXX_SIMD_OP_(*=)
3461 _GLIBCXX_SIMD_OP_(/=)
3462 _GLIBCXX_SIMD_OP_(%=)
3463 _GLIBCXX_SIMD_OP_(&=)
3464 _GLIBCXX_SIMD_OP_(|=)
3465 _GLIBCXX_SIMD_OP_(^=)
3466 _GLIBCXX_SIMD_OP_(<<=)
3467 _GLIBCXX_SIMD_OP_(>>=)
3468 #undef _GLIBCXX_SIMD_OP_
3470 _GLIBCXX_SIMD_INTRINSIC
void operator++() &&
3471 {
if (_M_k) ++_M_value; }
3473 _GLIBCXX_SIMD_INTRINSIC
void operator++(
int) &&
3474 {
if (_M_k) ++_M_value; }
3476 _GLIBCXX_SIMD_INTRINSIC
void operator--() &&
3477 {
if (_M_k) --_M_value; }
3479 _GLIBCXX_SIMD_INTRINSIC
void operator--(
int) &&
3480 {
if (_M_k) --_M_value; }
3483 template <
typename _Up,
typename _Flags>
3484 _GLIBCXX_SIMD_INTRINSIC
void
3485 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3486 {
if (_M_k) _M_value = __mem[0]; }
3490 template <
typename _Tp,
typename _Ap>
3491 _GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3492 where(
const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3493 {
return {__k, __value}; }
3495 template <
typename _Tp,
typename _Ap>
3496 _GLIBCXX_SIMD_INTRINSIC
3497 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3498 where(
const typename simd<_Tp, _Ap>::mask_type& __k,
3499 const simd<_Tp, _Ap>& __value)
3500 {
return {__k, __value}; }
3502 template <
typename _Tp,
typename _Ap>
3503 _GLIBCXX_SIMD_INTRINSIC
3504 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3506 simd_mask<_Tp, _Ap>& __value)
3507 {
return {__k, __value}; }
3509 template <
typename _Tp,
typename _Ap>
3510 _GLIBCXX_SIMD_INTRINSIC
3511 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3513 const simd_mask<_Tp, _Ap>& __value)
3514 {
return {__k, __value}; }
3516 template <
typename _Tp>
3517 _GLIBCXX_SIMD_INTRINSIC where_expression<bool, _Tp>
3518 where(_ExactBool __k, _Tp& __value)
3519 {
return {__k, __value}; }
3521 template <
typename _Tp>
3522 _GLIBCXX_SIMD_INTRINSIC const_where_expression<bool, _Tp>
3523 where(_ExactBool __k,
const _Tp& __value)
3524 {
return {__k, __value}; }
3526 template <
typename _Tp,
typename _Ap>
3527 void where(
bool __k, simd<_Tp, _Ap>& __value) =
delete;
3529 template <
typename _Tp,
typename _Ap>
3530 void where(
bool __k,
const simd<_Tp, _Ap>& __value) =
delete;
3533 namespace __proposed {
3534 template <
size_t _Np>
3537 const bitset<_Np> __bits;
3540 where_range(bitset<_Np> __b) : __bits(__b) {}
3547 _GLIBCXX_SIMD_INTRINSIC
void __next_bit()
3548 { __bit = __builtin_ctzl(__mask); }
3550 _GLIBCXX_SIMD_INTRINSIC
void __reset_lsb()
3553 __mask &= (__mask - 1);
3558 iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3559 iterator(
const iterator&) =
default;
3560 iterator(iterator&&) =
default;
3562 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t operator->()
const
3565 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t operator*()
const
3568 _GLIBCXX_SIMD_ALWAYS_INLINE iterator& operator++()
3575 _GLIBCXX_SIMD_ALWAYS_INLINE iterator operator++(
int)
3577 iterator __tmp = *
this;
3583 _GLIBCXX_SIMD_ALWAYS_INLINE
bool operator==(
const iterator& __rhs)
const
3584 {
return __mask == __rhs.__mask; }
3586 _GLIBCXX_SIMD_ALWAYS_INLINE
bool operator!=(
const iterator& __rhs)
const
3587 {
return __mask != __rhs.__mask; }
3590 iterator begin()
const
3591 {
return __bits.to_ullong(); }
3593 iterator end()
const
3597 template <
typename _Tp,
typename _Ap>
3598 where_range<simd_size_v<_Tp, _Ap>>
3599 where(
const simd_mask<_Tp, _Ap>& __k)
3600 {
return __k.__to_bitset(); }
3606 template <
typename _Tp,
typename _Abi,
typename _BinaryOperation = plus<>>
3607 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3608 reduce(
const simd<_Tp, _Abi>& __v,
3609 _BinaryOperation __binary_op = _BinaryOperation())
3610 {
return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3612 template <
typename _M,
typename _V,
typename _BinaryOperation = plus<>>
3613 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3614 reduce(
const const_where_expression<_M, _V>& __x,
3615 typename _V::value_type __identity_element,
3616 _BinaryOperation __binary_op)
3618 if (__builtin_expect(none_of(__get_mask(__x)),
false))
3619 return __identity_element;
3621 _V __tmp = __identity_element;
3622 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3623 __data(__get_lvalue(__x)));
3624 return reduce(__tmp, __binary_op);
3627 template <
typename _M,
typename _V>
3628 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3629 reduce(
const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3630 {
return reduce(__x, 0, __binary_op); }
3632 template <
typename _M,
typename _V>
3633 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3634 reduce(
const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3635 {
return reduce(__x, 1, __binary_op); }
3637 template <
typename _M,
typename _V>
3638 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3639 reduce(
const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3640 {
return reduce(__x, ~
typename _V::value_type(), __binary_op); }
3642 template <
typename _M,
typename _V>
3643 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3644 reduce(
const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3645 {
return reduce(__x, 0, __binary_op); }
3647 template <
typename _M,
typename _V>
3648 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3649 reduce(
const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3650 {
return reduce(__x, 0, __binary_op); }
3652 template <
typename _Tp,
typename _Abi>
3653 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3654 hmin(
const simd<_Tp, _Abi>& __v) noexcept
3656 return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum());
3659 template <
typename _Tp,
typename _Abi>
3660 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3661 hmax(
const simd<_Tp, _Abi>& __v) noexcept
3663 return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum());
3666 template <
typename _M,
typename _V>
3667 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3668 typename _V::value_type
3669 hmin(
const const_where_expression<_M, _V>& __x) noexcept
3671 using _Tp =
typename _V::value_type;
3672 constexpr _Tp __id_elem =
3673 #ifdef __FINITE_MATH_ONLY__
3674 __finite_max_v<_Tp>;
3676 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3678 _V __tmp = __id_elem;
3679 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3680 __data(__get_lvalue(__x)));
3681 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3684 template <
typename _M,
typename _V>
3685 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3686 typename _V::value_type
3687 hmax(
const const_where_expression<_M, _V>& __x) noexcept
3689 using _Tp =
typename _V::value_type;
3690 constexpr _Tp __id_elem =
3691 #ifdef __FINITE_MATH_ONLY__
3692 __finite_min_v<_Tp>;
3695 if constexpr (__value_exists_v<__infinity, _Tp>)
3696 return -__infinity_v<_Tp>;
3698 return __finite_min_v<_Tp>;
3701 _V __tmp = __id_elem;
3702 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3703 __data(__get_lvalue(__x)));
3704 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3709 template <
typename _Tp,
typename _Ap>
3710 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3711 min(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3712 {
return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3714 template <
typename _Tp,
typename _Ap>
3715 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3716 max(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3717 {
return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3719 template <
typename _Tp,
typename _Ap>
3720 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3721 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3722 minmax(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3724 const auto pair_of_members
3725 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3726 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3727 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3730 template <
typename _Tp,
typename _Ap>
3731 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3732 clamp(
const simd<_Tp, _Ap>& __v,
const simd<_Tp, _Ap>& __lo,
3733 const simd<_Tp, _Ap>& __hi)
3735 using _Impl =
typename _Ap::_SimdImpl;
3736 return {__private_init,
3737 _Impl::_S_min(__data(__hi),
3738 _Impl::_S_max(__data(__lo), __data(__v)))};
3743 template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
3744 typename =
enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3745 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3746 split(
const simd<_Tp, _Ap>&);
3749 template <
int _Index,
int _Total,
int _Combine = 1,
typename _Tp,
size_t _Np>
3750 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
3751 _SimdWrapper<_Tp, _Np / _Total * _Combine>
3752 __extract_part(
const _SimdWrapper<_Tp, _Np> __x);
3754 template <
int Index,
int Parts,
int _Combine = 1,
typename _Tp,
typename _A0,
3756 _GLIBCXX_SIMD_INTRINSIC
auto
3757 __extract_part(
const _SimdTuple<_Tp, _A0, _As...>& __x);
3761 template <
size_t _V0,
size_t... _Values>
3764 template <
size_t _I>
3765 static constexpr
size_t _S_at(_SizeConstant<_I> = {})
3767 if constexpr (_I == 0)
3770 return _SizeList<_Values...>::template _S_at<_I - 1>();
3773 template <
size_t _I>
3774 static constexpr auto _S_before(_SizeConstant<_I> = {})
3776 if constexpr (_I == 0)
3777 return _SizeConstant<0>();
3779 return _SizeConstant<
3780 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3783 template <
size_t _Np>
3784 static constexpr auto _S_pop_front(_SizeConstant<_Np> = {})
3786 if constexpr (_Np == 0)
3789 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3795 template <typename _Tp,
size_t _Np>
3796 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3797 __extract_center(_SimdWrapper<_Tp, _Np> __x)
3799 static_assert(_Np >= 4);
3800 static_assert(_Np % 4 == 0);
3801 #if _GLIBCXX_SIMD_X86INTRIN
3802 if constexpr (__have_avx512f &&
sizeof(_Tp) * _Np == 64)
3804 const auto __intrin = __to_intrin(__x);
3805 if constexpr (is_integral_v<_Tp>)
3806 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3807 _mm512_shuffle_i32x4(__intrin, __intrin,
3808 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3809 else if constexpr (
sizeof(_Tp) == 4)
3810 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3811 _mm512_shuffle_f32x4(__intrin, __intrin,
3812 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3813 else if constexpr (sizeof(_Tp) == 8)
3814 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3815 _mm512_shuffle_f64x2(__intrin, __intrin,
3816 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3818 __assert_unreachable<_Tp>();
3820 else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3821 return __vector_bitcast<_Tp>(
3822 _mm_shuffle_pd(__lo128(__vector_bitcast<
double>(__x)),
3823 __hi128(__vector_bitcast<
double>(__x)), 1));
3824 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
3825 return __vector_bitcast<_Tp>(
3826 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3827 __lo128(__vector_bitcast<_LLong>(__x)),
3828 sizeof(_Tp) * _Np / 4));
3832 __vector_type_t<_Tp, _Np / 2> __r;
3833 __builtin_memcpy(&__r,
3834 reinterpret_cast<const char*
>(&__x)
3835 +
sizeof(_Tp) * _Np / 4,
3836 sizeof(_Tp) * _Np / 2);
3841 template <
typename _Tp,
typename _A0,
typename... _As>
3842 _GLIBCXX_SIMD_INTRINSIC
3843 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3844 __extract_center(
const _SimdTuple<_Tp, _A0, _As...>& __x)
3846 if constexpr (
sizeof...(_As) == 0)
3847 return __extract_center(__x.first);
3849 return __extract_part<1, 4, 2>(__x);
3854 template <
size_t... _Sizes, typename _Tp, typename... _As>
3856 __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
3858 return split<_Sizes...>(
3859 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3866 template <
typename _V,
typename _Ap,
3867 size_t Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
3868 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == Parts * _V::size()
3869 && is_simd_v<_V>, array<_V, Parts>>
3870 split(
const simd<typename _V::value_type, _Ap>& __x)
3872 using _Tp =
typename _V::value_type;
3873 if constexpr (Parts == 1)
3875 return {simd_cast<_V>(__x)};
3877 else if (__x._M_is_constprop())
3879 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3880 auto __i) constexpr {
3881 return _V([&](
auto __j) constexpr {
3882 return __x[__i * _V::size() + __j];
3887 __is_fixed_size_abi_v<_Ap>
3888 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3889 || (__is_fixed_size_abi_v<typename _V::abi_type>
3890 &&
sizeof(_V) ==
sizeof(_Tp) * _V::size()
3894 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3895 const __may_alias<_Tp>*
const __element_ptr
3896 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__data(__x));
3897 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3898 auto __i) constexpr {
3899 return _V(__element_ptr + __i * _V::size(), vector_aligned);
3902 const auto& __xx = __data(__x);
3903 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3904 auto __i) constexpr {
3905 [[maybe_unused]] constexpr
size_t __offset
3906 = decltype(__i)::value * _V::size();
3907 return _V([&](
auto __j) constexpr {
3908 constexpr _SizeConstant<__j + __offset> __k;
3914 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3917 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3918 auto __i) constexpr {
return __x[__i]; });
3922 return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3923 auto __i) constexpr {
3924 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3925 return _V([&](
auto __j) constexpr {
3926 return __x[__i * _V::size() + __j];
3929 return _V(__private_init,
3930 __extract_part<decltype(__i)::value, Parts>(__data(__x)));
3937 template <
typename _V,
typename _Ap,
3939 = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
3940 enable_if_t<is_simd_mask_v<_V> && simd_size_v<
typename
3941 _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
3942 split(
const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3944 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3946 else if constexpr (_Parts == 1)
3947 return {__proposed::static_simd_cast<_V>(__x)};
3948 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3949 && __is_avx_abi<_Ap>())
3950 return {_V(__private_init, __lo128(__data(__x))),
3951 _V(__private_init, __hi128(__data(__x)))};
3952 else if constexpr (_V::size() <= __CHAR_BIT__ *
sizeof(_ULLong))
3954 const bitset __bits = __x.__to_bitset();
3955 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3956 auto __i) constexpr {
3957 constexpr
size_t __offset = __i * _V::size();
3958 return _V(__bitset_init, (__bits >> __offset).to_ullong());
3963 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3964 auto __i) constexpr {
3965 constexpr
size_t __offset = __i * _V::size();
3967 __private_init, [&](
auto __j) constexpr {
3968 return __x[__j + __offset];
3976 template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
typename>
3977 _GLIBCXX_SIMD_ALWAYS_INLINE
3978 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3979 split(
const simd<_Tp, _Ap>& __x)
3981 using _SL = _SizeList<_Sizes...>;
3982 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
3983 constexpr
size_t _Np = simd_size_v<_Tp, _Ap>;
3984 constexpr
size_t _N0 = _SL::template _S_at<0>();
3985 using _V = __deduced_simd<_Tp, _N0>;
3987 if (__x._M_is_constprop())
3988 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
3989 auto __i) constexpr {
3990 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3991 constexpr
size_t __offset = _SL::_S_before(__i);
3992 return _Vi([&](
auto __j) constexpr {
return __x[__offset + __j]; });
3994 else if constexpr (_Np == _N0)
3996 static_assert(
sizeof...(_Sizes) == 1);
3997 return {simd_cast<_V>(__x)};
4000 (__is_fixed_size_abi_v<
4001 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
4004 !__is_fixed_size_abi_v<typename _V::abi_type>,
4005 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
4010 return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
4011 __split_wrapper(_SL::template _S_pop_front<1>(),
4012 __data(__x).second));
4014 else if constexpr ((!is_same_v<simd_abi::scalar,
4015 simd_abi::deduce_t<_Tp, _Sizes>> && ...)
4016 && (!__is_fixed_size_abi_v<
4017 simd_abi::deduce_t<_Tp, _Sizes>> && ...))
4019 if constexpr (((_Sizes * 2 == _Np) && ...))
4020 return {{__private_init, __extract_part<0, 2>(__data(__x))},
4021 {__private_init, __extract_part<1, 2>(__data(__x))}};
4022 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4023 _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
4024 return {{__private_init, __extract_part<0, 3>(__data(__x))},
4025 {__private_init, __extract_part<1, 3>(__data(__x))},
4026 {__private_init, __extract_part<2, 3>(__data(__x))}};
4027 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4028 _SizeList<2 * _Np / 3, _Np / 3>>)
4029 return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
4030 {__private_init, __extract_part<2, 3>(__data(__x))}};
4031 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4032 _SizeList<_Np / 3, 2 * _Np / 3>>)
4033 return {{__private_init, __extract_part<0, 3>(__data(__x))},
4034 {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
4035 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4036 _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
4037 return {{__private_init, __extract_part<0, 2>(__data(__x))},
4038 {__private_init, __extract_part<2, 4>(__data(__x))},
4039 {__private_init, __extract_part<3, 4>(__data(__x))}};
4040 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4041 _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
4042 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4043 {__private_init, __extract_part<1, 4>(__data(__x))},
4044 {__private_init, __extract_part<1, 2>(__data(__x))}};
4045 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4046 _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
4047 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4048 {__private_init, __extract_center(__data(__x))},
4049 {__private_init, __extract_part<3, 4>(__data(__x))}};
4050 else if constexpr (((_Sizes * 4 == _Np) && ...))
4051 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4052 {__private_init, __extract_part<1, 4>(__data(__x))},
4053 {__private_init, __extract_part<2, 4>(__data(__x))},
4054 {__private_init, __extract_part<3, 4>(__data(__x))}};
4057 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4058 const __may_alias<_Tp>*
const __element_ptr
4059 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__x);
4060 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
4061 auto __i) constexpr {
4062 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4063 constexpr
size_t __offset = _SL::_S_before(__i);
4064 constexpr
size_t __base_align =
alignof(simd<_Tp, _Ap>);
4065 constexpr
size_t __a
4066 = __base_align - ((__offset *
sizeof(_Tp)) % __base_align);
4067 constexpr
size_t __b = ((__a - 1) & __a) ^ __a;
4068 constexpr
size_t __alignment = __b == 0 ? __a : __b;
4069 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
4072 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>([&](
4073 auto __i) constexpr {
4074 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4075 const auto& __xx = __data(__x);
4076 using _Offset = decltype(_SL::_S_before(__i));
4077 return _Vi([&](
auto __j) constexpr {
4078 constexpr _SizeConstant<_Offset::value + __j> __k;
4088 template <
size_t _I,
typename _Tp,
typename _Ap,
typename... _As>
4089 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
4090 __subscript_in_pack(
const simd<_Tp, _Ap>& __x,
const simd<_Tp, _As>&... __xs)
4092 if constexpr (_I < simd_size_v<_Tp, _Ap>)
4095 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
4100 template <
typename _Tp,
typename _A0,
typename... _As>
4101 _GLIBCXX_SIMD_INTRINSIC
void
4102 __store_pack_of_simd(
char* __mem,
const simd<_Tp, _A0>& __x0,
4103 const simd<_Tp, _As>&... __xs)
4105 constexpr
size_t __n_bytes =
sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4106 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4107 if constexpr (
sizeof...(__xs) > 0)
4108 __store_pack_of_simd(__mem + __n_bytes, __xs...);
4113 template <typename _Tp, typename... _As, typename = __detail::__odr_helper>
4114 inline _GLIBCXX_SIMD_CONSTEXPR
4115 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4116 concat(const simd<_Tp, _As>&... __xs)
4118 using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
4119 if constexpr (
sizeof...(__xs) == 1)
4120 return simd_cast<_Rp>(__xs...);
4121 else if ((... && __xs._M_is_constprop()))
4123 simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>([&](
4124 auto __i) constexpr {
return __subscript_in_pack<__i>(__xs...); });
4128 __store_pack_of_simd(
reinterpret_cast<char*
>(&__data(__r)), __xs...);
4135 template <
typename _Tp,
typename _Abi,
size_t _Np>
4136 _GLIBCXX_SIMD_ALWAYS_INLINE
4137 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4138 concat(
const array<simd<_Tp, _Abi>, _Np>& __x)
4140 return __call_with_subscripts<_Np>(__x, [](
const auto&... __xs) {
4141 return concat(__xs...);
4149 template <
typename _Up,
typename _Accessor = _Up,
4150 typename _ValueType =
typename _Up::value_type>
4151 class _SmartReference
4157 _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType _M_read() const noexcept
4159 if constexpr (is_arithmetic_v<_Up>)
4162 return _M_obj[_M_index];
4165 template <
typename _Tp>
4166 _GLIBCXX_SIMD_INTRINSIC constexpr
void _M_write(_Tp&& __x)
const
4167 { _Accessor::_S_set(_M_obj, _M_index,
static_cast<_Tp&&
>(__x)); }
4170 _GLIBCXX_SIMD_INTRINSIC constexpr
4171 _SmartReference(_Up& __o,
int __i) noexcept
4172 : _M_index(__i), _M_obj(__o) {}
4174 using value_type = _ValueType;
4176 _GLIBCXX_SIMD_INTRINSIC _SmartReference(
const _SmartReference&) =
delete;
4178 _GLIBCXX_SIMD_INTRINSIC constexpr
operator value_type() const noexcept
4179 {
return _M_read(); }
4181 template <
typename _Tp,
4183 = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4184 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator=(_Tp&& __x) &&
4186 _M_write(
static_cast<_Tp&&
>(__x));
4187 return {_M_obj, _M_index};
4190 #define _GLIBCXX_SIMD_OP_(__op) \
4191 template <typename _Tp, \
4193 = decltype(declval<value_type>() __op declval<_Tp>()), \
4194 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4195 typename = _ValuePreservingOrInt<_TT, value_type>> \
4196 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4197 operator __op##=(_Tp&& __x) && \
4199 const value_type& __lhs = _M_read(); \
4200 _M_write(__lhs __op __x); \
4201 return {_M_obj, _M_index}; \
4203 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4204 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4205 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4206 #undef _GLIBCXX_SIMD_OP_
4208 template <
typename _Tp = void,
4210 = decltype(++
declval<conditional_t<true, value_type, _Tp>&>())>
4211 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator++() &&
4213 value_type __x = _M_read();
4215 return {_M_obj, _M_index};
4218 template <
typename _Tp = void,
4220 = decltype(
declval<conditional_t<true, value_type, _Tp>&>()++)>
4221 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator++(
int) &&
4223 const value_type __r = _M_read();
4224 value_type __x = __r;
4229 template <
typename _Tp = void,
4231 = decltype(--
declval<conditional_t<true, value_type, _Tp>&>())>
4232 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator--() &&
4234 value_type __x = _M_read();
4236 return {_M_obj, _M_index};
4239 template <
typename _Tp = void,
4241 = decltype(
declval<conditional_t<true, value_type, _Tp>&>()--)>
4242 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator--(
int) &&
4244 const value_type __r = _M_read();
4245 value_type __x = __r;
4250 _GLIBCXX_SIMD_INTRINSIC
friend void
4251 swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4253 is_nothrow_constructible<value_type, _SmartReference&&>,
4254 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4256 value_type __tmp =
static_cast<_SmartReference&&
>(__a);
4257 static_cast<_SmartReference&&
>(__a) =
static_cast<value_type
>(__b);
4258 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4261 _GLIBCXX_SIMD_INTRINSIC
friend void
4262 swap(value_type& __a, _SmartReference&& __b) noexcept(
4264 is_nothrow_constructible<value_type, value_type&&>,
4265 is_nothrow_assignable<value_type&, value_type&&>,
4266 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4269 __a =
static_cast<value_type
>(__b);
4270 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4273 _GLIBCXX_SIMD_INTRINSIC
friend void
4274 swap(_SmartReference&& __a, value_type& __b) noexcept(
4276 is_nothrow_constructible<value_type, _SmartReference&&>,
4277 is_nothrow_assignable<value_type&, value_type&&>,
4278 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4280 value_type __tmp(__a);
4281 static_cast<_SmartReference&&
>(__a) =
std::move(__b);
4288 template <
int _Bytes>
4289 struct __scalar_abi_wrapper
4291 template <
typename _Tp>
static constexpr
size_t _S_full_size = 1;
4292 template <
typename _Tp>
static constexpr
size_t _S_size = 1;
4293 template <
typename _Tp>
static constexpr
size_t _S_is_partial =
false;
4295 template <
typename _Tp,
typename _Abi = simd_abi::scalar>
4296 static constexpr
bool _S_is_valid_v
4297 = _Abi::template _IsValid<_Tp>::value &&
sizeof(_Tp) == _Bytes;
4302 template <
typename _Tp>
4303 struct __decay_abi {
using type = _Tp; };
4305 template <
int _Bytes>
4306 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4307 {
using type = simd_abi::scalar; };
4315 template <
template <
int>
class _Abi,
int _Bytes,
typename _Tp>
4316 struct __find_next_valid_abi
4318 static constexpr
auto _S_choose()
4320 constexpr
int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4321 using _NextAbi = _Abi<_NextBytes>;
4322 if constexpr (_NextBytes <
sizeof(_Tp) * 2)
4323 return _Abi<_Bytes>();
4324 else if constexpr (_NextAbi::template _S_is_partial<_Tp> ==
false
4325 && _NextAbi::template _S_is_valid_v<_Tp>)
4328 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4331 using type = decltype(_S_choose());
4334 template <
int _Bytes,
typename _Tp>
4335 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4336 {
using type = simd_abi::scalar; };
4339 template <
template <
int>
class...>
4342 template <
typename,
int>
static constexpr
bool _S_has_valid_abi =
false;
4343 template <
typename,
int>
using _FirstValidAbi = void;
4344 template <
typename,
int>
using _BestAbi = void;
4347 template <
template <
int>
class _A0,
template <
int>
class... _Rest>
4348 struct _AbiList<_A0, _Rest...>
4350 template <
typename _Tp,
int _Np>
4351 static constexpr
bool _S_has_valid_abi
4352 = _A0<
sizeof(_Tp) * _Np>::
template _S_is_valid_v<
4353 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4355 template <
typename _Tp,
int _Np>
4357 _A0<
sizeof(_Tp) * _Np>::
template _S_is_valid_v<_Tp>,
4358 typename __decay_abi<_A0<
sizeof(_Tp) * _Np>>::type,
4359 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4361 template <
typename _Tp,
int _Np>
4362 static constexpr
auto _S_determine_best_abi()
4364 static_assert(_Np >= 1);
4365 constexpr
int _Bytes =
sizeof(_Tp) * _Np;
4366 if constexpr (_Np == 1)
4367 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4370 constexpr
int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4375 if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4376 _Tp> && __fullsize / 2 < _Np)
4377 return typename __decay_abi<_A0<_Bytes>>::type{};
4381 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4382 if constexpr (_Bp::template _S_is_valid_v<
4383 _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4387 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4392 template <
typename _Tp,
int _Np>
4393 using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4401 using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4402 __scalar_abi_wrapper>;
4405 template <
typename _Tp,
typename _Abi>
4406 struct _SimdTraits<_Tp, _Abi,
void_t<typename _Abi::template _IsValid<_Tp>>>
4407 : _Abi::template __traits<_Tp> {};
4411 template <
typename _Tp,
size_t _Np>
4412 struct __deduce_impl<
4413 _Tp, _Np,
enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4414 {
using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4417 template <
typename _Tp,
size_t _Np,
typename =
void>
4418 struct __deduce_fixed_size_fallback {};
4420 template <
typename _Tp,
size_t _Np>
4421 struct __deduce_fixed_size_fallback<_Tp, _Np,
4422 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4423 {
using type = simd_abi::fixed_size<_Np>; };
4425 template <
typename _Tp,
size_t _Np,
typename>
4426 struct __deduce_impl :
public __deduce_fixed_size_fallback<_Tp, _Np> {};
4432 template <
typename _Tp,
typename _Abi>
4433 class simd_mask :
public _SimdTraits<_Tp, _Abi>::_MaskBase
4436 using _Traits = _SimdTraits<_Tp, _Abi>;
4437 using _MemberType =
typename _Traits::_MaskMember;
4443 using _Ip = __int_for_sizeof_t<_Tp>;
4444 static constexpr _Ip* _S_type_tag =
nullptr;
4446 friend typename _Traits::_MaskBase;
4447 friend class simd<_Tp, _Abi>;
4448 friend typename _Traits::_SimdImpl;
4451 using _Impl =
typename _Traits::_MaskImpl;
4456 using value_type = bool;
4457 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4458 using simd_type = simd<_Tp, _Abi>;
4459 using abi_type = _Abi;
4462 static constexpr
size_t size()
4463 {
return __size_or_zero_v<_Tp, _Abi>; }
4467 simd_mask() =
default;
4468 simd_mask(
const simd_mask&) =
default;
4469 simd_mask(simd_mask&&) =
default;
4470 simd_mask& operator=(
const simd_mask&) =
default;
4471 simd_mask& operator=(simd_mask&&) =
default;
4475 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit
4476 simd_mask(
typename _Traits::_MaskCastType __init)
4477 : _M_data{__init} {}
4486 _GLIBCXX_SIMD_ALWAYS_INLINE
static simd_mask
4487 __from_bitset(bitset<size()> bs)
4488 {
return {__bitset_init, bs}; }
4490 _GLIBCXX_SIMD_ALWAYS_INLINE bitset<size()>
4492 {
return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4496 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4497 simd_mask(value_type __x)
4498 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4502 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4504 template <
typename _Up,
typename _A2,
4505 typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4506 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit(
sizeof(_MemberType)
4507 !=
sizeof(
typename _SimdTraits<_Up, _A2>::_MaskMember))
4508 simd_mask(
const simd_mask<_Up, _A2>& __x)
4509 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4512 template <
typename _Up,
typename =
enable_if_t<conjunction<
4513 is_same<abi_type, simd_abi::fixed_size<size()>>,
4514 is_same<_Up, _Up>>::value>>
4515 _GLIBCXX_SIMD_ALWAYS_INLINE
4516 simd_mask(
const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4517 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4522 template <
typename _Flags>
4523 _GLIBCXX_SIMD_ALWAYS_INLINE
4524 simd_mask(
const value_type* __mem, _Flags)
4525 : _M_data(_Impl::template _S_load<_Ip>(
4526 _Flags::template _S_apply<simd_mask>(__mem))) {}
4528 template <
typename _Flags>
4529 _GLIBCXX_SIMD_ALWAYS_INLINE
4530 simd_mask(
const value_type* __mem, simd_mask __k, _Flags)
4534 = _Impl::_S_masked_load(_M_data, __k._M_data,
4535 _Flags::template _S_apply<simd_mask>(__mem));
4540 template <
typename _Flags>
4541 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4542 copy_from(
const value_type* __mem, _Flags)
4544 _M_data = _Impl::template _S_load<_Ip>(
4545 _Flags::template _S_apply<simd_mask>(__mem));
4550 template <
typename _Flags>
4551 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4552 copy_to(value_type* __mem, _Flags)
const
4553 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4557 _GLIBCXX_SIMD_ALWAYS_INLINE reference
4558 operator[](
size_t __i)
4561 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i, size() - 1);
4562 return {_M_data, int(__i)};
4565 _GLIBCXX_SIMD_ALWAYS_INLINE value_type
4566 operator[](
size_t __i)
const
4569 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i, size() - 1);
4570 if constexpr (__is_scalar_abi<_Abi>())
4573 return static_cast<
bool>(_M_data[__i]);
4578 _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask
4580 {
return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4584 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4586 template <
typename _Up,
typename _A2,
4588 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4589 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4590 operator&&(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4592 return {__private_init,
4593 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4596 template <
typename _Up,
typename _A2,
4598 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4599 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4600 operator||(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4602 return {__private_init,
4603 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4607 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4608 operator&&(
const simd_mask& __x,
const simd_mask& __y)
4610 return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)};
4613 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4614 operator||(
const simd_mask& __x,
const simd_mask& __y)
4616 return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)};
4619 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4620 operator&(
const simd_mask& __x,
const simd_mask& __y)
4621 {
return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4623 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4624 operator|(
const simd_mask& __x,
const simd_mask& __y)
4625 {
return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4627 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4628 operator^(
const simd_mask& __x,
const simd_mask& __y)
4629 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4631 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4632 operator&=(simd_mask& __x,
const simd_mask& __y)
4634 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4638 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4639 operator|=(simd_mask& __x,
const simd_mask& __y)
4641 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4645 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4646 operator^=(simd_mask& __x,
const simd_mask& __y)
4648 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4654 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4655 operator==(
const simd_mask& __x,
const simd_mask& __y)
4656 {
return !operator!=(__x, __y); }
4658 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4659 operator!=(
const simd_mask& __x,
const simd_mask& __y)
4660 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4664 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4665 simd_mask(_PrivateInit,
typename _Traits::_MaskMember __init)
4666 : _M_data(__init) {}
4670 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4671 _GLIBCXX_SIMD_INTRINSIC constexpr
4672 simd_mask(_PrivateInit, _Fp&& __gen)
4675 __execute_n_times<size()>([&](
auto __i) constexpr {
4676 _Impl::_S_set(_M_data, __i, __gen(__i));
4682 _GLIBCXX_SIMD_INTRINSIC simd_mask(_BitsetInit, bitset<size()> __init)
4684 _Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4695 template <
typename _Up,
typename _A2,
4697 = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4698 _GLIBCXX_SIMD_ALWAYS_INLINE
4699 operator simd_mask<_Up, _A2>() &&
4701 using namespace std::experimental::__proposed;
4702 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4705 const simd_mask<_Tp, _Abi>& _M_data;
4708 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4714 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4715 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4716 operator?:(
const simd_mask& __k,
const simd_mask& __where_true,
4717 const simd_mask& __where_false)
4719 auto __ret = __where_false;
4720 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4724 template <
typename _U1,
typename _U2,
4725 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4727 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4728 is_convertible<simd_mask, typename _Rp::mask_type>>>>
4729 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend _Rp
4730 operator?:(
const simd_mask& __k,
const _U1& __where_true,
4731 const _U2& __where_false)
4733 _Rp __ret = __where_false;
4734 _Rp::_Impl::_S_masked_assign(
4735 __data(
static_cast<typename _Rp::mask_type
>(__k)), __data(__ret),
4736 __data(
static_cast<_Rp
>(__where_true)));
4740 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4741 template <
typename _Kp,
typename _Ak,
typename _Up,
typename _Au,
4743 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4744 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4745 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4746 operator?:(
const simd_mask<_Kp, _Ak>& __k,
const simd_mask& __where_true,
4747 const simd_mask<_Up, _Au>& __where_false)
4749 simd_mask __ret = __where_false;
4750 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4751 __where_true._M_data);
4759 _GLIBCXX_SIMD_INTRINSIC constexpr
bool
4760 _M_is_constprop()
const
4762 if constexpr (__is_scalar_abi<_Abi>())
4763 return __builtin_constant_p(_M_data);
4765 return _M_data._M_is_constprop();
4771 friend const auto& __data<_Tp, abi_type>(const simd_mask&);
4772 friend auto& __data<_Tp, abi_type>(simd_mask&);
4773 alignas(_Traits::_S_mask_align) _MemberType _M_data;
4780 template <typename _Tp, typename _Ap>
4781 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
4782 __data(const simd_mask<_Tp, _Ap>& __x)
4783 {
return __x._M_data; }
4785 template <
typename _Tp,
typename _Ap>
4786 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
4787 __data(simd_mask<_Tp, _Ap>& __x)
4788 {
return __x._M_data; }
4794 template <
typename _Tp,
typename _Abi>
4795 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4796 all_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4798 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4800 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4806 return _Abi::_MaskImpl::_S_all_of(__k);
4809 template <
typename _Tp,
typename _Abi>
4810 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4811 any_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4813 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4815 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4821 return _Abi::_MaskImpl::_S_any_of(__k);
4824 template <
typename _Tp,
typename _Abi>
4825 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4826 none_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4828 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4830 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4836 return _Abi::_MaskImpl::_S_none_of(__k);
4839 template <
typename _Tp,
typename _Abi>
4840 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4841 some_of(
const simd_mask<_Tp, _Abi>& __k) noexcept
4843 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4845 for (
size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4846 if (__k[__i] != __k[__i - 1])
4851 return _Abi::_MaskImpl::_S_some_of(__k);
4854 template <
typename _Tp,
typename _Abi>
4855 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4856 popcount(
const simd_mask<_Tp, _Abi>& __k) noexcept
4858 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4860 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4861 __k, [](
auto... __elements) {
return ((__elements != 0) + ...); });
4862 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4865 return _Abi::_MaskImpl::_S_popcount(__k);
4868 template <
typename _Tp,
typename _Abi>
4869 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4870 find_first_set(
const simd_mask<_Tp, _Abi>& __k)
4872 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4874 constexpr
size_t _Np = simd_size_v<_Tp, _Abi>;
4875 const size_t _Idx = __call_with_n_evaluations<_Np>(
4876 [](
auto... __indexes) {
return std::min({__indexes...}); },
4877 [&](
auto __i) {
return __k[__i] ? +__i : _Np; });
4879 __invoke_ub(
"find_first_set(empty mask) is UB");
4880 if (__builtin_constant_p(_Idx))
4883 return _Abi::_MaskImpl::_S_find_first_set(__k);
4886 template <
typename _Tp,
typename _Abi>
4887 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4888 find_last_set(
const simd_mask<_Tp, _Abi>& __k)
4890 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4892 constexpr
size_t _Np = simd_size_v<_Tp, _Abi>;
4893 const int _Idx = __call_with_n_evaluations<_Np>(
4894 [](
auto... __indexes) {
return std::max({__indexes...}); },
4895 [&](
auto __i) {
return __k[__i] ? int(__i) : -1; });
4897 __invoke_ub(
"find_first_set(empty mask) is UB");
4898 if (__builtin_constant_p(_Idx))
4901 return _Abi::_MaskImpl::_S_find_last_set(__k);
4904 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4905 all_of(_ExactBool __x) noexcept
4908 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4909 any_of(_ExactBool __x) noexcept
4912 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4913 none_of(_ExactBool __x) noexcept
4916 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4917 some_of(_ExactBool) noexcept
4920 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4924 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4925 find_first_set(_ExactBool)
4928 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4929 find_last_set(_ExactBool)
4936 template <
typename _V,
typename _Tp,
typename _Abi,
bool>
4937 class _SimdIntOperators {};
4939 template <
typename _V,
typename _Tp,
typename _Abi>
4940 class _SimdIntOperators<_V, _Tp, _Abi, true>
4942 using _Impl =
typename _SimdTraits<_Tp, _Abi>::_SimdImpl;
4944 _GLIBCXX_SIMD_INTRINSIC
const _V& __derived()
const
4945 {
return *
static_cast<const _V*
>(
this); }
4947 template <
typename _Up>
4948 _GLIBCXX_SIMD_INTRINSIC
static _GLIBCXX_SIMD_CONSTEXPR _V
4949 _S_make_derived(_Up&& __d)
4950 {
return {__private_init,
static_cast<_Up&&
>(__d)}; }
4953 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
4955 operator%=(_V& __lhs,
const _V& __x)
4956 {
return __lhs = __lhs % __x; }
4958 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
4960 operator&=(_V& __lhs,
const _V& __x)
4961 {
return __lhs = __lhs & __x; }
4963 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
4965 operator|=(_V& __lhs,
const _V& __x)
4966 {
return __lhs = __lhs | __x; }
4968 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
4970 operator^=(_V& __lhs,
const _V& __x)
4971 {
return __lhs = __lhs ^ __x; }
4973 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
4975 operator<<=(_V& __lhs,
const _V& __x)
4976 {
return __lhs = __lhs << __x; }
4978 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
4980 operator>>=(_V& __lhs,
const _V& __x)
4981 {
return __lhs = __lhs >> __x; }
4983 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
4985 operator<<=(_V& __lhs,
int __x)
4986 {
return __lhs = __lhs << __x; }
4988 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
4990 operator>>=(_V& __lhs,
int __x)
4991 {
return __lhs = __lhs >> __x; }
4993 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
4995 operator%(
const _V& __x,
const _V& __y)
4997 return _SimdIntOperators::_S_make_derived(
4998 _Impl::_S_modulus(__data(__x), __data(__y)));
5001 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5005 return _SimdIntOperators::_S_make_derived(
5006 _Impl::_S_bit_and(__data(__x), __data(__y)));
5009 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5013 return _SimdIntOperators::_S_make_derived(
5014 _Impl::_S_bit_or(__data(__x), __data(__y)));
5017 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5021 return _SimdIntOperators::_S_make_derived(
5022 _Impl::_S_bit_xor(__data(__x), __data(__y)));
5025 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5029 return _SimdIntOperators::_S_make_derived(
5030 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
5033 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5037 return _SimdIntOperators::_S_make_derived(
5038 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
5041 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5046 __invoke_ub(
"The behavior is undefined if the right operand of a "
5047 "shift operation is negative. [expr.shift]\nA shift by "
5050 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5052 "The behavior is undefined if the right operand of a "
5053 "shift operation is greater than or equal to the width of the "
5054 "promoted left operand. [expr.shift]\nA shift by %d was requested",
5056 return _SimdIntOperators::_S_make_derived(
5057 _Impl::_S_bit_shift_left(__data(__x), __y));
5060 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5066 "The behavior is undefined if the right operand of a shift "
5067 "operation is negative. [expr.shift]\nA shift by %d was requested",
5069 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5071 "The behavior is undefined if the right operand of a shift "
5072 "operation is greater than or equal to the width of the promoted "
5073 "left operand. [expr.shift]\nA shift by %d was requested",
5075 return _SimdIntOperators::_S_make_derived(
5076 _Impl::_S_bit_shift_right(__data(__x), __y));
5080 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5083 {
return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
5090 template <
typename _Tp,
typename _Abi>
5091 class simd :
public _SimdIntOperators<
5092 simd<_Tp, _Abi>, _Tp, _Abi,
5093 conjunction<is_integral<_Tp>,
5094 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
5095 public _SimdTraits<_Tp, _Abi>::_SimdBase
5097 using _Traits = _SimdTraits<_Tp, _Abi>;
5098 using _MemberType =
typename _Traits::_SimdMember;
5099 using _CastType =
typename _Traits::_SimdCastType;
5100 static constexpr _Tp* _S_type_tag =
nullptr;
5101 friend typename _Traits::_SimdBase;
5104 using _Impl =
typename _Traits::_SimdImpl;
5106 friend _SimdIntOperators<simd, _Tp, _Abi, true>;
5108 using value_type = _Tp;
5109 using reference = _SmartReference<_MemberType, _Impl, value_type>;
5110 using mask_type = simd_mask<_Tp, _Abi>;
5111 using abi_type = _Abi;
5113 static constexpr
size_t size()
5114 {
return __size_or_zero_v<_Tp, _Abi>; }
5116 _GLIBCXX_SIMD_CONSTEXPR simd() =
default;
5117 _GLIBCXX_SIMD_CONSTEXPR simd(
const simd&) =
default;
5118 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
5119 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
5120 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
5123 template <typename _Up,
5124 typename =
enable_if_t<!is_same_v<__remove_cvref_t<_Up>,
bool>>>
5125 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5126 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
5128 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
5133 template <
typename _Up>
5134 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5135 simd(
const simd<_Up, simd_abi::fixed_size<size()>>& __x,
5138 is_same<simd_abi::fixed_size<size()>, abi_type>,
5139 negation<__is_narrowing_conversion<_Up, value_type>>,
5140 __converts_to_higher_integer_rank<_Up, value_type>>::value,
5142 : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5145 #ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5146 template <
typename _Up,
typename _A2,
5147 typename = decltype(static_simd_cast<simd>(
5148 declval<
const simd<_Up, _A2>&>()))>
5149 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5150 simd(
const simd<_Up, _A2>& __x)
5151 : simd(static_simd_cast<simd>(__x)) {}
5155 template <
typename _Fp>
5156 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5157 simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
5158 declval<_SizeConstant<0>&>())),
5159 value_type>* =
nullptr)
5160 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5163 template <
typename _Up,
typename _Flags>
5164 _GLIBCXX_SIMD_ALWAYS_INLINE
5165 simd(
const _Up* __mem, _Flags)
5167 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5171 template <
typename _Up,
typename _Flags>
5172 _GLIBCXX_SIMD_ALWAYS_INLINE
void
5173 copy_from(
const _Vectorizable<_Up>* __mem, _Flags)
5175 _M_data =
static_cast<decltype(_M_data)
>(
5176 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5180 template <
typename _Up,
typename _Flags>
5181 _GLIBCXX_SIMD_ALWAYS_INLINE
void
5182 copy_to(_Vectorizable<_Up>* __mem, _Flags)
const
5184 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5189 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5190 operator[](
size_t __i)
5191 {
return {_M_data, int(__i)}; }
5193 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5194 operator[]([[maybe_unused]]
size_t __i)
const
5196 if constexpr (__is_scalar_abi<_Abi>())
5198 _GLIBCXX_DEBUG_ASSERT(__i == 0);
5202 return _M_data[__i];
5206 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5209 _Impl::_S_increment(_M_data);
5213 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5217 _Impl::_S_increment(_M_data);
5221 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5224 _Impl::_S_decrement(_M_data);
5228 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5232 _Impl::_S_decrement(_M_data);
5237 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5239 {
return {__private_init, _Impl::_S_negate(_M_data)}; }
5241 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5245 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5247 {
return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5250 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5251 simd(_CastType __init) : _M_data(__init) {}
5254 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5255 operator+=(simd& __lhs,
const simd& __x)
5256 {
return __lhs = __lhs + __x; }
5258 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5259 operator-=(simd& __lhs,
const simd& __x)
5260 {
return __lhs = __lhs - __x; }
5262 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5263 operator*=(simd& __lhs,
const simd& __x)
5264 {
return __lhs = __lhs * __x; }
5266 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5267 operator/=(simd& __lhs,
const simd& __x)
5268 {
return __lhs = __lhs / __x; }
5271 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5272 operator+(
const simd& __x,
const simd& __y)
5273 {
return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5275 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5276 operator-(
const simd& __x,
const simd& __y)
5277 {
return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5279 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5280 operator*(
const simd& __x,
const simd& __y)
5281 {
return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5283 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5284 operator/(
const simd& __x,
const simd& __y)
5285 {
return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5288 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5289 operator==(
const simd& __x,
const simd& __y)
5290 {
return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5292 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5293 operator!=(
const simd& __x,
const simd& __y)
5295 return simd::_S_make_mask(
5296 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5299 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5300 operator<(
const simd& __x,
const simd& __y)
5301 {
return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5303 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5304 operator<=(
const simd& __x,
const simd& __y)
5306 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5309 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5310 operator>(
const simd& __x,
const simd& __y)
5311 {
return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5313 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5314 operator>=(
const simd& __x,
const simd& __y)
5316 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5320 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5321 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5322 operator?:(
const mask_type& __k,
const simd& __where_true,
5323 const simd& __where_false)
5325 auto __ret = __where_false;
5326 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5334 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5335 simd(_PrivateInit,
const _MemberType& __init)
5336 : _M_data(__init) {}
5339 _GLIBCXX_SIMD_INTRINSIC
5340 simd(_BitsetInit, bitset<size()> __init) : _M_data()
5341 { where(mask_type(__bitset_init, __init), *
this) = ~*
this; }
5343 _GLIBCXX_SIMD_INTRINSIC constexpr
bool
5344 _M_is_constprop()
const
5346 if constexpr (__is_scalar_abi<_Abi>())
5347 return __builtin_constant_p(_M_data);
5349 return _M_data._M_is_constprop();
5353 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR static mask_type
5354 _S_make_mask(typename mask_type::_MemberType __k)
5355 {
return {__private_init, __k}; }
5357 friend const auto& __data<value_type, abi_type>(
const simd&);
5358 friend auto& __data<value_type, abi_type>(simd&);
5359 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5365 template <
typename _Tp,
typename _Ap>
5366 _GLIBCXX_SIMD_INTRINSIC constexpr
const auto&
5367 __data(
const simd<_Tp, _Ap>& __x)
5368 {
return __x._M_data; }
5370 template <
typename _Tp,
typename _Ap>
5371 _GLIBCXX_SIMD_INTRINSIC constexpr
auto&
5372 __data(simd<_Tp, _Ap>& __x)
5373 {
return __x._M_data; }
5376 namespace __float_bitwise_operators {
5377 template <
typename _Tp,
typename _Ap>
5378 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5379 operator^(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5381 return {__private_init,
5382 _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))};
5385 template <
typename _Tp,
typename _Ap>
5386 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5387 operator|(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5389 return {__private_init,
5390 _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))};
5393 template <
typename _Tp,
typename _Ap>
5394 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5395 operator&(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5397 return {__private_init,
5398 _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))};
5401 template <
typename _Tp,
typename _Ap>
5402 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5403 enable_if_t<is_floating_point_v<_Tp>, simd<_Tp, _Ap>>
5404 operator~(
const simd<_Tp, _Ap>& __a)
5405 {
return {__private_init, _Ap::_SimdImpl::_S_complement(__data(__a))}; }
5410 _GLIBCXX_SIMD_END_NAMESPACE
constexpr _If_is_unsigned_integer< _Tp, int > popcount(_Tp __x) noexcept
The number of bits set in x.
constexpr complex< _Tp > operator*(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x times y.
constexpr complex< _Tp > operator/(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x divided by y.
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
void void_t
A metafunction that always yields void, used for detecting valid types.
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
typename remove_pointer< _Tp >::type remove_pointer_t
Alias template for remove_pointer.
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
typename remove_const< _Tp >::type remove_const_t
Alias template for remove_const.
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
tuple_cat
auto declval() noexcept -> decltype(__declval< _Tp >(0))
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.