25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26#define _GLIBCXX_EXPERIMENTAL_SIMD_H
28#if __cplusplus >= 201703L
30#include "simd_detail.h"
34#ifdef _GLIBCXX_DEBUG_UB
43#if _GLIBCXX_SIMD_X86INTRIN
45#elif _GLIBCXX_SIMD_HAVE_NEON
72_GLIBCXX_SIMD_BEGIN_NAMESPACE
74#if !_GLIBCXX_SIMD_X86INTRIN
75using __m128 [[__gnu__::__vector_size__(16)]] = float;
76using __m128d [[__gnu__::__vector_size__(16)]] = double;
77using __m128i [[__gnu__::__vector_size__(16)]] =
long long;
78using __m256 [[__gnu__::__vector_size__(32)]] = float;
79using __m256d [[__gnu__::__vector_size__(32)]] = double;
80using __m256i [[__gnu__::__vector_size__(32)]] =
long long;
81using __m512 [[__gnu__::__vector_size__(64)]] = float;
82using __m512d [[__gnu__::__vector_size__(64)]] = double;
83using __m512i [[__gnu__::__vector_size__(64)]] =
long long;
105template <
int _UsedBytes>
108template <
int _UsedBytes>
109 struct _VecBltnBtmsk;
111template <
typename _Tp,
int _Np>
112 using _VecN = _VecBuiltin<
sizeof(_Tp) * _Np>;
114template <
int _UsedBytes = 16>
115 using _Sse = _VecBuiltin<_UsedBytes>;
117template <
int _UsedBytes = 32>
118 using _Avx = _VecBuiltin<_UsedBytes>;
120template <
int _UsedBytes = 64>
121 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
123template <
int _UsedBytes = 16>
124 using _Neon = _VecBuiltin<_UsedBytes>;
129using __avx512 = _Avx512<>;
130using __neon = _Neon<>;
131using __neon128 = _Neon<16>;
132using __neon64 = _Neon<8>;
135template <
typename _Tp,
size_t _Np,
typename...>
139 using fixed_size = _Fixed<_Np>;
141using scalar = _Scalar;
146template <
typename _Tp>
149template <
typename _Tp>
152template <
typename _Tp,
typename _Abi>
155template <
typename _Tp,
typename _Abi>
158template <
typename _Tp,
typename _Abi>
163struct element_aligned_tag
165 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
166 static constexpr size_t _S_alignment =
alignof(_Up);
168 template <
typename _Tp,
typename _Up>
169 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
174struct vector_aligned_tag
176 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
177 static constexpr size_t _S_alignment
178 = std::__bit_ceil(
sizeof(_Up) * _Tp::size());
180 template <
typename _Tp,
typename _Up>
181 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
183 {
return static_cast<_Up*
>(__builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>)); }
186template <
size_t _Np>
struct overaligned_tag
188 template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
189 static constexpr size_t _S_alignment = _Np;
191 template <
typename _Tp,
typename _Up>
192 _GLIBCXX_SIMD_INTRINSIC
static constexpr _Up*
194 {
return static_cast<_Up*
>(__builtin_assume_aligned(__ptr, _Np)); }
197inline constexpr element_aligned_tag element_aligned = {};
199inline constexpr vector_aligned_tag vector_aligned = {};
202 inline constexpr overaligned_tag<_Np> overaligned = {};
206 using _SizeConstant = integral_constant<size_t, _Xp>;
208constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
209constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
210constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
211constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
212constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
213constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
214constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
215constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
216constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
217constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
218constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
219constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
220constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
221constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
222constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
223constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
224constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
225constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
226constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
227constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
228constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
229constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
230constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
231constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
232constexpr inline bool __have_avx512bitalg = _GLIBCXX_SIMD_HAVE_AVX512BITALG;
233constexpr inline bool __have_avx512vbmi2 = _GLIBCXX_SIMD_HAVE_AVX512VBMI2;
234constexpr inline bool __have_avx512vbmi = _GLIBCXX_SIMD_HAVE_AVX512VBMI;
235constexpr inline bool __have_avx512ifma = _GLIBCXX_SIMD_HAVE_AVX512IFMA;
236constexpr inline bool __have_avx512cd = _GLIBCXX_SIMD_HAVE_AVX512CD;
237constexpr inline bool __have_avx512vnni = _GLIBCXX_SIMD_HAVE_AVX512VNNI;
238constexpr inline bool __have_avx512vpopcntdq = _GLIBCXX_SIMD_HAVE_AVX512VPOPCNTDQ;
239constexpr inline bool __have_avx512vp2intersect = _GLIBCXX_SIMD_HAVE_AVX512VP2INTERSECT;
241constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
242constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
243constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
244constexpr inline bool __support_neon_float =
245#if defined __GCC_IEC_559
247#elif defined __FAST_MATH__
254constexpr inline bool __have_power10vec =
true;
256constexpr inline bool __have_power10vec =
false;
258#ifdef __POWER9_VECTOR__
259constexpr inline bool __have_power9vec =
true;
261constexpr inline bool __have_power9vec =
false;
263#if defined __POWER8_VECTOR__
264constexpr inline bool __have_power8vec =
true;
266constexpr inline bool __have_power8vec = __have_power9vec;
269constexpr inline bool __have_power_vsx =
true;
271constexpr inline bool __have_power_vsx = __have_power8vec;
273#if defined __ALTIVEC__
274constexpr inline bool __have_power_vmx =
true;
276constexpr inline bool __have_power_vmx = __have_power_vsx;
283#ifdef math_errhandling
287 template <
int = math_errhandling>
289 __handle_fpexcept_impl(
int)
290 {
return math_errhandling & MATH_ERREXCEPT; }
296 __handle_fpexcept_impl(
float)
298#if defined __FAST_MATH__
306 static constexpr bool _S_handle_fpexcept = __handle_fpexcept_impl(0);
308 constexpr std::uint_least64_t
309 __floating_point_flags()
311 std::uint_least64_t __flags = 0;
312 if constexpr (_S_handle_fpexcept)
316#elif __FINITE_MATH_ONLY__
318#elif __GCC_IEC_559 < 2
321 __flags |= (__FLT_EVAL_METHOD__ + 1) << 3;
325 constexpr std::uint_least64_t
328 if constexpr (__have_mmx || __have_sse)
333 | (__have_ssse3 << 4)
334 | (__have_sse4_1 << 5)
335 | (__have_sse4_2 << 6)
340 | (__have_bmi2 << 11)
341 | (__have_lzcnt << 12)
342 | (__have_sse4a << 13)
344 | (__have_fma4 << 15)
345 | (__have_f16c << 16)
346 | (__have_popcnt << 17)
347 | (__have_avx512f << 18)
348 | (__have_avx512dq << 19)
349 | (__have_avx512vl << 20)
350 | (__have_avx512bw << 21)
351 | (__have_avx512bitalg << 22)
352 | (__have_avx512vbmi2 << 23)
353 | (__have_avx512vbmi << 24)
354 | (__have_avx512ifma << 25)
355 | (__have_avx512cd << 26)
356 | (__have_avx512vnni << 27)
357 | (__have_avx512vpopcntdq << 28)
358 | (__have_avx512vp2intersect << 29);
359 else if constexpr (__have_neon)
361 | (__have_neon_a32 << 1)
362 | (__have_neon_a64 << 2)
363 | (__have_neon_a64 << 2)
364 | (__support_neon_float << 3);
365 else if constexpr (__have_power_vmx)
366 return __have_power_vmx
367 | (__have_power_vsx << 1)
368 | (__have_power8vec << 2)
369 | (__have_power9vec << 3)
370 | (__have_power10vec << 4);
377 struct _OdrEnforcer {};
380 template <std::uint_least64_t...>
381 struct _MachineFlagsTemplate {};
394 _MachineFlagsTemplate<__machine_flags(), __floating_point_flags()>>;
398 template <
typename _Tp>
399 _GLIBCXX_SIMD_INTRINSIC
constexpr
401 operator()(_Tp __a, _Tp __b)
const
404 return min(__a, __b);
410 template <
typename _Tp>
411 _GLIBCXX_SIMD_INTRINSIC
constexpr
413 operator()(_Tp __a, _Tp __b)
const
416 return max(__a, __b);
423template <
typename _Fp,
size_t... _I>
424 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
426 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
427 { ((void)__f(_SizeConstant<_I>()), ...); }
429template <
typename _Fp>
430 _GLIBCXX_SIMD_INTRINSIC
constexpr void
431 __execute_on_index_sequence(_Fp&&, index_sequence<>)
434template <
size_t _Np,
typename _Fp>
435 _GLIBCXX_SIMD_INTRINSIC
constexpr void
436 __execute_n_times(_Fp&& __f)
438 __execute_on_index_sequence(
static_cast<_Fp&&
>(__f),
439 make_index_sequence<_Np>{});
444template <
typename _R,
typename _Fp,
size_t... _I>
445 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
447 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
448 {
return _R{__f(_SizeConstant<_I>())...}; }
450template <
size_t _Np,
typename _R,
typename _Fp>
451 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
452 __generate_from_n_evaluations(_Fp&& __f)
454 return __execute_on_index_sequence_with_return<_R>(
455 static_cast<_Fp&&
>(__f), make_index_sequence<_Np>{});
460template <
size_t... _I,
typename _F0,
typename _FArgs>
461 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
463 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
464 {
return __f0(__fargs(_SizeConstant<_I>())...); }
466template <
size_t _Np,
typename _F0,
typename _FArgs>
467 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
468 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
470 return __call_with_n_evaluations(make_index_sequence<_Np>{},
471 static_cast<_F0&&
>(__f0),
472 static_cast<_FArgs&&
>(__fargs));
477template <
size_t _First = 0,
size_t... _It,
typename _Tp,
typename _Fp>
478 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC
constexpr
480 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
481 {
return __fun(__x[_First + _It]...); }
483template <
size_t _Np,
size_t _First = 0,
typename _Tp,
typename _Fp>
484 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
485 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
487 return __call_with_subscripts<_First>(
static_cast<_Tp&&
>(__x),
488 make_index_sequence<_Np>(),
489 static_cast<_Fp&&
>(__fun));
496using _UChar =
unsigned char;
497using _SChar =
signed char;
498using _UShort =
unsigned short;
499using _UInt =
unsigned int;
500using _ULong =
unsigned long;
501using _ULLong =
unsigned long long;
502using _LLong =
long long;
506template <
typename _T0,
typename...>
507 struct __first_of_pack
508 {
using type = _T0; };
510template <
typename... _Ts>
511 using __first_of_pack_t =
typename __first_of_pack<_Ts...>::type;
515template <
typename _Tp>
516 typename _Tp::value_type
517 __value_type_or_identity_impl(
int);
519template <
typename _Tp>
521 __value_type_or_identity_impl(
float);
523template <
typename _Tp>
524 using __value_type_or_identity_t
525 =
decltype(__value_type_or_identity_impl<_Tp>(
int()));
529template <
typename _Tp>
530 struct __is_vectorizable :
public is_arithmetic<_Tp> {};
533 struct __is_vectorizable<bool> :
public false_type {};
535template <
typename _Tp>
536 inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
539template <
typename _Tp,
typename = enable_if_t<__is_vectorizable_v<_Tp>>>
540 using _Vectorizable = _Tp;
544template <
typename _Ptr,
typename _ValueType>
545 struct __is_possible_loadstore_conversion
546 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
549 struct __is_possible_loadstore_conversion<bool, bool> :
true_type {};
552template <
typename _Ptr,
typename _ValueType,
554 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
555 using _LoadStorePtr = _Ptr;
559template <
typename _Tp,
typename =
void_t<>>
562template <
typename _Tp>
563 inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
566template <
typename _Tp>
567 struct __is_bitmask<_Tp,
568 void_t<decltype(
declval<unsigned&>() = declval<_Tp>() & 1u)>>
573#pragma GCC diagnostic push
574#pragma GCC diagnostic ignored "-Wpedantic"
575template <
size_t _Bytes>
579 if constexpr (_Bytes ==
sizeof(int))
582 else if constexpr (_Bytes ==
sizeof(char))
585 else if constexpr (_Bytes ==
sizeof(_SChar))
588 else if constexpr (_Bytes ==
sizeof(short))
591 else if constexpr (_Bytes ==
sizeof(long))
594 else if constexpr (_Bytes ==
sizeof(_LLong))
596 #ifdef __SIZEOF_INT128__
597 else if constexpr (_Bytes ==
sizeof(__int128))
600 else if constexpr (_Bytes %
sizeof(int) == 0)
602 constexpr size_t _Np = _Bytes /
sizeof(int);
607 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
610 return __generate_from_n_evaluations<_Np, _Ip>(
611 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
612 return __rhs._M_data[__i] & _M_data[__i];
616 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
619 return __generate_from_n_evaluations<_Np, _Ip>(
620 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
621 return __rhs._M_data[__i] | _M_data[__i];
625 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
628 return __generate_from_n_evaluations<_Np, _Ip>(
629 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
630 return __rhs._M_data[__i] ^ _M_data[__i];
634 _GLIBCXX_SIMD_INTRINSIC
constexpr _Ip
637 return __generate_from_n_evaluations<_Np, _Ip>(
638 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return ~_M_data[__i]; });
644 static_assert(_Bytes != _Bytes,
"this should be unreachable");
646#pragma GCC diagnostic pop
648template <
typename _Tp>
649 using __int_for_sizeof_t =
decltype(__int_for_sizeof<sizeof(_Tp)>());
652 using __int_with_sizeof_t =
decltype(__int_for_sizeof<_Np>());
656template <
typename _Tp>
660 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> :
true_type {};
662template <
typename _Tp>
663 inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
667template <
typename _Abi>
670 {
return is_same_v<simd_abi::scalar, _Abi>; }
674template <
template <
int>
class _Abi,
int _Bytes>
676 __abi_bytes_impl(_Abi<_Bytes>*)
679template <
typename _Tp>
681 __abi_bytes_impl(_Tp*)
684template <
typename _Abi>
685 inline constexpr int __abi_bytes_v
686 = __abi_bytes_impl(
static_cast<_Abi*
>(
nullptr));
690template <
typename _Abi>
692 __is_builtin_bitmask_abi()
693 {
return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
697template <
typename _Abi>
701 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
702 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
707template <
typename _Abi>
711 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
712 return _Bytes > 16 && _Bytes <= 32
713 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
718template <
typename _Abi>
722 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
723 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
728template <
typename _Abi>
732 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
733 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
738template <
typename,
typename _Up>
739 struct __make_dependent
740 {
using type = _Up; };
742template <
typename _Tp,
typename _Up>
743 using __make_dependent_t =
typename __make_dependent<_Tp, _Up>::type;
749template <
typename... _Args>
750 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE
void
751 __invoke_ub([[maybe_unused]]
const char* __msg, [[maybe_unused]]
const _Args&... __args)
753#ifdef _GLIBCXX_DEBUG_UB
754 __builtin_fprintf(stderr, __msg, __args...);
757 __builtin_unreachable();
763template <
typename _Tp>
764 struct __assert_unreachable
765 {
static_assert(!is_same_v<_Tp, _Tp>,
"this should be unreachable"); };
769template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
771 __size_or_zero_dispatch(
int)
774template <
typename _Tp,
typename _Ap>
776 __size_or_zero_dispatch(
float)
779template <
typename _Tp,
typename _Ap>
780 inline constexpr size_t __size_or_zero_v
781 = __size_or_zero_dispatch<_Tp, _Ap>(0);
785inline constexpr size_t
786__div_roundup(
size_t __a,
size_t __b)
787{
return (__a + __b - 1) / __b; }
796 _GLIBCXX_SIMD_INTRINSIC
constexpr
797 _ExactBool(
bool __b) : _M_data(__b) {}
799 _ExactBool(
int) =
delete;
801 _GLIBCXX_SIMD_INTRINSIC
constexpr
802 operator bool()
const
813template <
typename _Tp>
814 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
819struct _UnsupportedBase
821 _UnsupportedBase() =
delete;
822 _UnsupportedBase(
const _UnsupportedBase&) =
delete;
823 _UnsupportedBase& operator=(
const _UnsupportedBase&) =
delete;
824 ~_UnsupportedBase() =
delete;
840 using _SimdBase = _UnsupportedBase;
841 using _MaskBase = _UnsupportedBase;
843 static constexpr size_t _S_full_size = 0;
844 static constexpr bool _S_is_partial =
false;
846 static constexpr size_t _S_simd_align = 1;
848 struct _SimdMember {};
849 struct _SimdCastType;
851 static constexpr size_t _S_mask_align = 1;
853 struct _MaskMember {};
854 struct _MaskCastType;
859template <
typename _Tp,
typename _Abi,
typename =
void_t<>>
860 struct _SimdTraits : _InvalidTraits {};
868inline constexpr struct _PrivateInit {} __private_init = {};
870inline constexpr struct _BitsetInit {} __bitset_init = {};
874template <
typename _From,
typename _To,
bool = is_arithmetic_v<_From>,
875 bool = is_arithmetic_v<_To>>
876 struct __is_narrowing_conversion;
880template <
typename _From,
typename _To>
881 struct __is_narrowing_conversion<_From, _To, true, true>
882 :
public __bool_constant<(
883 __digits_v<_From> > __digits_v<_To>
884 || __finite_max_v<_From> > __finite_max_v<_To>
885 || __finite_min_v<_From> < __finite_min_v<_To>
886 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
888template <
typename _Tp>
889 struct __is_narrowing_conversion<_Tp, bool, true, true>
893 struct __is_narrowing_conversion<bool, bool, true, true>
896template <
typename _Tp>
897 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
900template <
typename _From,
typename _To>
901 struct __is_narrowing_conversion<_From, _To, false, true>
902 :
public negation<is_convertible<_From, _To>> {};
906template <
typename _From,
typename _To,
bool = (sizeof(_From) < sizeof(_To))>
907 struct __converts_to_higher_
integer_rank : public true_type {};
910template <
typename _From,
typename _To>
911 struct __converts_to_higher_integer_rank<_From, _To, false>
912 :
public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
916template <
typename _Tp,
typename _Ap>
917 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
918 __data(
const simd<_Tp, _Ap>& __x);
920template <
typename _Tp,
typename _Ap>
921 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
922 __data(simd<_Tp, _Ap>& __x);
924template <
typename _Tp,
typename _Ap>
925 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
926 __data(
const simd_mask<_Tp, _Ap>& __x);
928template <
typename _Tp,
typename _Ap>
929 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
930 __data(simd_mask<_Tp, _Ap>& __x);
934template <
typename _FromT,
typename _FromA,
typename _ToT,
typename _ToA,
936 struct _SimdConverter;
938template <
typename _Tp,
typename _Ap>
939 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
941 template <
typename _Up>
942 _GLIBCXX_SIMD_INTRINSIC
const _Up&
943 operator()(
const _Up& __x)
949template <
typename _V>
950 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
951 __to_value_type_or_member_type(
const _V& __x) ->
decltype(__data(__x))
952 {
return __data(__x); }
954template <
typename _V>
955 _GLIBCXX_SIMD_INTRINSIC
constexpr const typename _V::value_type&
956 __to_value_type_or_member_type(
const typename _V::value_type& __x)
961template <
size_t _Size>
962 struct __bool_storage_member_type;
964template <
size_t _Size>
965 using __bool_storage_member_type_t =
966 typename __bool_storage_member_type<_Size>::type;
976template <
typename _Tp,
typename... _Abis>
981template <
typename _Tp,
int _Np>
982 struct __fixed_size_storage;
984template <
typename _Tp,
int _Np>
985 using __fixed_size_storage_t =
typename __fixed_size_storage<_Tp, _Np>::type;
989template <
typename _Tp,
size_t _Size,
typename =
void_t<>>
992template <
typename _Tp>
993 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 /
sizeof(_Tp)>;
994template <
typename _Tp>
995 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 /
sizeof(_Tp)>;
996template <
typename _Tp>
997 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 /
sizeof(_Tp)>;
998template <
typename _Tp>
999 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 /
sizeof(_Tp)>;
1003template <
typename _Tp>
1006template <
typename _Tp,
size_t _Np>
1007 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> :
true_type {};
1009template <
typename _Tp>
1010 inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
1017 template <
typename _Tp,
typename _Fp>
1019 _S_bit_iteration(_Tp __mask, _Fp&& __f)
1021 static_assert(
sizeof(_ULLong) >=
sizeof(_Tp));
1022 conditional_t<
sizeof(_Tp) <=
sizeof(_UInt), _UInt, _ULLong> __k;
1023 if constexpr (is_convertible_v<_Tp,
decltype(__k)>)
1026 __k = __mask.to_ullong();
1029 __f(std::__countr_zero(__k));
1039template <
typename _Tp =
void>
1041 {
constexpr _Tp operator()(_Tp __a)
const {
return ++__a; } };
1044 struct __increment<void>
1046 template <
typename _Tp>
1048 operator()(_Tp __a)
const
1052template <
typename _Tp =
void>
1054 {
constexpr _Tp operator()(_Tp __a)
const {
return --__a; } };
1057 struct __decrement<void>
1059 template <
typename _Tp>
1061 operator()(_Tp __a)
const
1067template <
typename _From,
typename _To,
1069 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
1070 using _ValuePreserving = _From;
1072template <
typename _From,
typename _To,
1073 typename _DecayedFrom = __remove_cvref_t<_From>,
1075 is_convertible<_From, _To>,
1077 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
1078 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
1079 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
1080 using _ValuePreservingOrInt = _From;
1084template <
typename _Tp,
size_t _Bytes,
typename =
void_t<>>
1085 struct __intrinsic_type;
1087template <
typename _Tp,
size_t _Size>
1088 using __intrinsic_type_t =
1089 typename __intrinsic_type<_Tp, _Size *
sizeof(_Tp)>::type;
1091template <
typename _Tp>
1092 using __intrinsic_type2_t =
typename __intrinsic_type<_Tp, 2>::type;
1093template <
typename _Tp>
1094 using __intrinsic_type4_t =
typename __intrinsic_type<_Tp, 4>::type;
1095template <
typename _Tp>
1096 using __intrinsic_type8_t =
typename __intrinsic_type<_Tp, 8>::type;
1097template <
typename _Tp>
1098 using __intrinsic_type16_t =
typename __intrinsic_type<_Tp, 16>::type;
1099template <
typename _Tp>
1100 using __intrinsic_type32_t =
typename __intrinsic_type<_Tp, 32>::type;
1101template <
typename _Tp>
1102 using __intrinsic_type64_t =
typename __intrinsic_type<_Tp, 64>::type;
1106template <
size_t _Np,
bool _Sanitized = false>
1109template <
size_t _Np,
bool _Sanitized>
1110 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> :
true_type {};
1112template <
size_t _Np>
1113 using _SanitizedBitMask = _BitMask<_Np, true>;
1115template <
size_t _Np,
bool _Sanitized>
1118 static_assert(_Np > 0);
1120 static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
1124 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
1126 static constexpr int _S_array_size = __div_roundup(_NBytes,
sizeof(_Tp));
1128 _Tp _M_bits[_S_array_size];
1130 static constexpr int _S_unused_bits
1131 = _Np == 1 ? 0 : _S_array_size *
sizeof(_Tp) * __CHAR_BIT__ - _Np;
1133 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1135 constexpr _BitMask() noexcept = default;
1137 constexpr _BitMask(
unsigned long long __x) noexcept
1138 : _M_bits{
static_cast<_Tp
>(__x)} {}
1140 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1142 constexpr _BitMask(
const _BitMask&)
noexcept =
default;
1144 template <
bool _RhsSanitized,
typename =
enable_if_t<_RhsSanitized ==
false
1145 && _Sanitized ==
true>>
1146 constexpr _BitMask(
const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1147 : _BitMask(__rhs._M_sanitized()) {}
1149 constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1151 static_assert(_S_array_size == 1);
1157 _M_to_bits() const noexcept
1159 static_assert(_S_array_size == 1);
1164 constexpr unsigned long long
1165 to_ullong() const noexcept
1167 static_assert(_S_array_size == 1);
1172 constexpr unsigned long
1173 to_ulong() const noexcept
1175 static_assert(_S_array_size == 1);
1179 constexpr bitset<_Np>
1180 _M_to_bitset() const noexcept
1182 static_assert(_S_array_size == 1);
1186 constexpr decltype(
auto)
1187 _M_sanitized()
const noexcept
1189 if constexpr (_Sanitized)
1191 else if constexpr (_Np == 1)
1192 return _SanitizedBitMask<_Np>(_M_bits[0]);
1195 _SanitizedBitMask<_Np> __r = {};
1196 for (
int __i = 0; __i < _S_array_size; ++__i)
1197 __r._M_bits[__i] = _M_bits[__i];
1198 if constexpr (_S_unused_bits > 0)
1199 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1204 template <
size_t _Mp,
bool _LSanitized>
1205 constexpr _BitMask<_Np + _Mp, _Sanitized>
1206 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb)
const noexcept
1208 constexpr size_t _RN = _Np + _Mp;
1209 using _Rp = _BitMask<_RN, _Sanitized>;
1210 if constexpr (_Rp::_S_array_size == 1)
1212 _Rp __r{{_M_bits[0]}};
1213 __r._M_bits[0] <<= _Mp;
1214 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1218 __assert_unreachable<_Rp>();
1224 template <
size_t _DropLsb,
size_t _NewSize = _Np - _DropLsb>
1226 _M_extract() const noexcept
1228 static_assert(_Np > _DropLsb);
1229 static_assert(_DropLsb + _NewSize <=
sizeof(_ULLong) * __CHAR_BIT__,
1230 "not implemented for bitmasks larger than one ullong");
1231 if constexpr (_NewSize == 1)
1233 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1235 return _BitMask<_NewSize,
1236 ((_NewSize + _DropLsb ==
sizeof(_Tp) * __CHAR_BIT__
1237 && _NewSize + _DropLsb <= _Np)
1238 || ((_Sanitized || _Np ==
sizeof(_Tp) * __CHAR_BIT__)
1239 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1245 all() const noexcept
1247 if constexpr (_Np == 1)
1249 else if constexpr (!_Sanitized)
1250 return _M_sanitized().all();
1253 constexpr _Tp __allbits = ~_Tp();
1254 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1255 if (_M_bits[__i] != __allbits)
1257 return _M_bits[_S_array_size - 1] == _S_bitmask;
1264 any() const noexcept
1266 if constexpr (_Np == 1)
1268 else if constexpr (!_Sanitized)
1269 return _M_sanitized().any();
1272 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1273 if (_M_bits[__i] != 0)
1275 return _M_bits[_S_array_size - 1] != 0;
1281 none() const noexcept
1283 if constexpr (_Np == 1)
1285 else if constexpr (!_Sanitized)
1286 return _M_sanitized().none();
1289 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1290 if (_M_bits[__i] != 0)
1292 return _M_bits[_S_array_size - 1] == 0;
1299 count() const noexcept
1301 if constexpr (_Np == 1)
1303 else if constexpr (!_Sanitized)
1304 return _M_sanitized().none();
1307 int __result = __builtin_popcountll(_M_bits[0]);
1308 for (
int __i = 1; __i < _S_array_size; ++__i)
1309 __result += __builtin_popcountll(_M_bits[__i]);
1316 operator[](
size_t __i)
const noexcept
1318 if constexpr (_Np == 1)
1320 else if constexpr (_S_array_size == 1)
1321 return (_M_bits[0] >> __i) & 1;
1324 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1325 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1326 return (_M_bits[__j] >> __shift) & 1;
1330 template <
size_t __i>
1332 operator[](_SizeConstant<__i>)
const noexcept
1334 static_assert(__i < _Np);
1335 constexpr size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1336 constexpr size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1337 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1342 set(
size_t __i,
bool __x)
noexcept
1344 if constexpr (_Np == 1)
1346 else if constexpr (_S_array_size == 1)
1348 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1349 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1353 const size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1354 const size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1355 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1356 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1360 template <
size_t __i>
1362 set(_SizeConstant<__i>,
bool __x)
noexcept
1364 static_assert(__i < _Np);
1365 if constexpr (_Np == 1)
1369 constexpr size_t __j = __i / (
sizeof(_Tp) * __CHAR_BIT__);
1370 constexpr size_t __shift = __i % (
sizeof(_Tp) * __CHAR_BIT__);
1371 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1372 _M_bits[__j] &= __mask;
1373 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1379 operator~() const noexcept
1381 if constexpr (_Np == 1)
1385 _BitMask __result{};
1386 for (
int __i = 0; __i < _S_array_size - 1; ++__i)
1387 __result._M_bits[__i] = ~_M_bits[__i];
1388 if constexpr (_Sanitized)
1389 __result._M_bits[_S_array_size - 1]
1390 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1392 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1398 operator^=(
const _BitMask& __b) &
noexcept
1400 __execute_n_times<_S_array_size>(
1401 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] ^= __b._M_bits[__i]; });
1406 operator|=(
const _BitMask& __b) &
noexcept
1408 __execute_n_times<_S_array_size>(
1409 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] |= __b._M_bits[__i]; });
1414 operator&=(
const _BitMask& __b) &
noexcept
1416 __execute_n_times<_S_array_size>(
1417 [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] &= __b._M_bits[__i]; });
1421 friend constexpr _BitMask
1422 operator^(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1429 friend constexpr _BitMask
1430 operator|(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1437 friend constexpr _BitMask
1438 operator&(
const _BitMask& __a,
const _BitMask& __b)
noexcept
1445 _GLIBCXX_SIMD_INTRINSIC
1447 _M_is_constprop()
const
1449 if constexpr (_S_array_size == 0)
1450 return __builtin_constant_p(_M_bits[0]);
1453 for (
int __i = 0; __i < _S_array_size; ++__i)
1454 if (!__builtin_constant_p(_M_bits[__i]))
1465template <
typename _Tp =
void>
1466 static inline constexpr int __min_vector_size = 2 *
sizeof(_Tp);
1468#if _GLIBCXX_SIMD_HAVE_NEON
1470 inline constexpr int __min_vector_size<void> = 8;
1473 inline constexpr int __min_vector_size<void> = 16;
1478template <
typename _Tp,
size_t _Np,
typename =
void>
1479 struct __vector_type_n {};
1482template <
typename _Tp>
1483 struct __vector_type_n<_Tp, 0, void> {};
1486template <
typename _Tp>
1487 struct __vector_type_n<_Tp, 1,
enable_if_t<__is_vectorizable_v<_Tp>>>
1488 {
using type = _Tp; };
1491template <
typename _Tp,
size_t _Np>
1492 struct __vector_type_n<_Tp, _Np,
enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1494 static constexpr size_t _S_Np2 = std::__bit_ceil(_Np *
sizeof(_Tp));
1496 static constexpr size_t _S_Bytes =
1502 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1505 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1508template <
typename _Tp,
size_t _Bytes,
size_t = _Bytes % sizeof(_Tp)>
1509 struct __vector_type;
1511template <
typename _Tp,
size_t _Bytes>
1512 struct __vector_type<_Tp, _Bytes, 0>
1513 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1515template <
typename _Tp,
size_t _Size>
1516 using __vector_type_t =
typename __vector_type_n<_Tp, _Size>::type;
1518template <
typename _Tp>
1519 using __vector_type2_t =
typename __vector_type<_Tp, 2>::type;
1520template <
typename _Tp>
1521 using __vector_type4_t =
typename __vector_type<_Tp, 4>::type;
1522template <
typename _Tp>
1523 using __vector_type8_t =
typename __vector_type<_Tp, 8>::type;
1524template <
typename _Tp>
1525 using __vector_type16_t =
typename __vector_type<_Tp, 16>::type;
1526template <
typename _Tp>
1527 using __vector_type32_t =
typename __vector_type<_Tp, 32>::type;
1528template <
typename _Tp>
1529 using __vector_type64_t =
typename __vector_type<_Tp, 64>::type;
1533template <
typename _Tp,
typename =
void_t<>>
1536template <
typename _Tp>
1537 struct __is_vector_type<
1538 _Tp,
void_t<typename __vector_type<
1540 : is_same<_Tp, typename __vector_type<
1541 remove_reference_t<decltype(declval<_Tp>()[0])>,
1542 sizeof(_Tp)>::type> {};
1544template <
typename _Tp>
1545 inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1549#if _GLIBCXX_SIMD_HAVE_SSE_ABI
1550template <
typename _Tp>
1551 using __is_intrinsic_type = __is_vector_type<_Tp>;
1553template <
typename _Tp,
typename =
void_t<>>
1556template <
typename _Tp>
1557 struct __is_intrinsic_type<
1558 _Tp,
void_t<typename __intrinsic_type<
1560 : is_same<_Tp, typename __intrinsic_type<
1561 remove_reference_t<decltype(declval<_Tp>()[0])>,
1562 sizeof(_Tp)>::type> {};
1565template <
typename _Tp>
1566 inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1570template <
typename _Tp,
typename =
void_t<>>
1571 struct _VectorTraitsImpl;
1573template <
typename _Tp>
1574 struct _VectorTraitsImpl<_Tp,
enable_if_t<__is_vector_type_v<_Tp>
1575 || __is_intrinsic_type_v<_Tp>>>
1578 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1579 static constexpr int _S_full_size =
sizeof(_Tp) /
sizeof(value_type);
1580 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1581 template <
typename _Up,
int _W = _S_full_size>
1582 static constexpr bool _S_is
1583 = is_same_v<value_type, _Up> && _W == _S_full_size;
1586template <
typename _Tp,
size_t _Np>
1587 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1588 void_t<__vector_type_t<_Tp, _Np>>>
1590 using type = __vector_type_t<_Tp, _Np>;
1591 using value_type = _Tp;
1592 static constexpr int _S_full_size =
sizeof(type) /
sizeof(value_type);
1593 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1594 static constexpr bool _S_is_partial = (_Np == _S_full_size);
1595 static constexpr int _S_partial_width = _Np;
1596 template <
typename _Up,
int _W = _S_full_size>
1597 static constexpr bool _S_is
1598 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1601template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1602 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1606template <
typename _V>
1607 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
1610 if constexpr (__is_vector_type_v<_V>)
1612 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1613 return __data(__x)._M_data;
1614 else if constexpr (__is_vectorizable_v<_V>)
1615 return __vector_type_t<_V, 2>{__x};
1622template <
size_t _Np = 0,
typename _V>
1623 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
1624 __as_wrapper(_V __x)
1626 if constexpr (__is_vector_type_v<_V>)
1627 return _SimdWrapper<
typename _VectorTraits<_V>::value_type,
1628 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1629 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1631 static_assert(_V::size() == _Np);
1636 static_assert(_V::_S_size == _Np);
1643template <
typename _To,
typename _From>
1644 _GLIBCXX_SIMD_INTRINSIC
constexpr _To
1645 __intrin_bitcast(_From __v)
1647 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1648 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1649 if constexpr (
sizeof(_To) ==
sizeof(_From))
1650 return reinterpret_cast<_To
>(__v);
1651 else if constexpr (
sizeof(_From) >
sizeof(_To))
1652 if constexpr (
sizeof(_To) >= 16)
1653 return reinterpret_cast<const __may_alias<_To>&
>(__v);
1657 __builtin_memcpy(&__r, &__v,
sizeof(_To));
1660#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1661 else if constexpr (__have_avx &&
sizeof(_From) == 16 &&
sizeof(_To) == 32)
1662 return reinterpret_cast<_To
>(__builtin_ia32_ps256_ps(
1663 reinterpret_cast<__vector_type_t<float, 4>
>(__v)));
1664 else if constexpr (__have_avx512f &&
sizeof(_From) == 16
1665 &&
sizeof(_To) == 64)
1666 return reinterpret_cast<_To
>(__builtin_ia32_ps512_ps(
1667 reinterpret_cast<__vector_type_t<float, 4>
>(__v)));
1668 else if constexpr (__have_avx512f &&
sizeof(_From) == 32
1669 &&
sizeof(_To) == 64)
1670 return reinterpret_cast<_To
>(__builtin_ia32_ps512_256ps(
1671 reinterpret_cast<__vector_type_t<float, 8>
>(__v)));
1673 else if constexpr (
sizeof(__v) <= 8)
1674 return reinterpret_cast<_To
>(
1675 __vector_type_t<__int_for_sizeof_t<_From>,
sizeof(_To) /
sizeof(_From)>{
1676 reinterpret_cast<__int_for_sizeof_t<_From>
>(__v)});
1679 static_assert(
sizeof(_To) >
sizeof(_From));
1681 __builtin_memcpy(&__r, &__v,
sizeof(_From));
1688template <
typename _To,
size_t _NN = 0,
typename _From,
1689 typename _FromVT = _VectorTraits<_From>,
1690 size_t _Np = _NN == 0 ?
sizeof(_From) /
sizeof(_To) : _NN>
1691 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_To, _Np>
1692 __vector_bitcast(_From __x)
1694 using _R = __vector_type_t<_To, _Np>;
1695 return __intrin_bitcast<_R>(__x);
1698template <
typename _To,
size_t _NN = 0,
typename _Tp,
size_t _Nx,
1700 = _NN == 0 ?
sizeof(_SimdWrapper<_Tp, _Nx>) /
sizeof(_To) : _NN>
1701 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_To, _Np>
1702 __vector_bitcast(
const _SimdWrapper<_Tp, _Nx>& __x)
1704 static_assert(_Np > 1);
1705 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1710#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1711template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1712 _To __convert_x86(_Tp);
1714template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1715 _To __convert_x86(_Tp, _Tp);
1717template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1718 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1720template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1721 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1723template <
typename _To,
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
1724 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1725 _Tp, _Tp, _Tp, _Tp);
1730template <
typename _To,
typename _From>
1731 _GLIBCXX_SIMD_INTRINSIC
constexpr _To
1732 __bit_cast(
const _From __x)
1734#if __has_builtin(__builtin_bit_cast)
1735 return __builtin_bit_cast(_To, __x);
1737 static_assert(
sizeof(_To) ==
sizeof(_From));
1738 constexpr bool __to_is_vectorizable
1739 = is_arithmetic_v<_To> || is_enum_v<_To>;
1740 constexpr bool __from_is_vectorizable
1741 = is_arithmetic_v<_From> || is_enum_v<_From>;
1742 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1743 return reinterpret_cast<_To
>(__x);
1744 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1746 using _FV [[gnu::vector_size(
sizeof(_From))]] = _From;
1747 return reinterpret_cast<_To
>(_FV{__x});
1749 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1751 using _TV [[gnu::vector_size(
sizeof(_To))]] = _To;
1752 using _FV [[gnu::vector_size(
sizeof(_From))]] = _From;
1753 return reinterpret_cast<_TV
>(_FV{__x})[0];
1755 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1757 using _TV [[gnu::vector_size(
sizeof(_To))]] = _To;
1758 return reinterpret_cast<_TV
>(__x)[0];
1763 __builtin_memcpy(
reinterpret_cast<char*
>(&__r),
1764 reinterpret_cast<const char*
>(&__x),
sizeof(_To));
1772template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
1773 typename _R = __
intrinsic_type_t<
typename _TVT::value_type, _TVT::_S_full_size>>
1774 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
1775 __to_intrin(_Tp __x)
1777 static_assert(
sizeof(__x) <=
sizeof(_R),
1778 "__to_intrin may never drop values off the end");
1779 if constexpr (
sizeof(__x) ==
sizeof(_R))
1780 return reinterpret_cast<_R
>(__as_vector(__x));
1783 using _Up = __int_for_sizeof_t<_Tp>;
1784 return reinterpret_cast<_R
>(
1785 __vector_type_t<_Up,
sizeof(_R) /
sizeof(_Up)>{__bit_cast<_Up>(__x)});
1791template <
typename _Tp,
typename... _Args>
1792 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp,
sizeof...(_Args)>
1793 __make_vector(
const _Args&... __args)
1794 {
return __vector_type_t<_Tp,
sizeof...(_Args)>{
static_cast<_Tp
>(__args)...}; }
1798template <
size_t _Np,
typename _Tp,
size_t... _I>
1799 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1800 __vector_broadcast_impl(_Tp __x, index_sequence<_I...>)
1801 {
return __vector_type_t<_Tp, _Np>{((void)_I, __x)...}; }
1803template <
size_t _Np,
typename _Tp>
1804 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1805 __vector_broadcast(_Tp __x)
1806 {
return __vector_broadcast_impl<_Np, _Tp>(__x, make_index_sequence<_Np>()); }
1810 template <
typename _Tp,
size_t _Np,
typename _Gp,
size_t... _I>
1811 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1812 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1813 {
return __vector_type_t<_Tp, _Np>{
static_cast<_Tp
>(__gen(_SizeConstant<_I>()))...}; }
1815template <
typename _V,
typename _VVT = _VectorTraits<_V>,
typename _Gp>
1816 _GLIBCXX_SIMD_INTRINSIC
constexpr _V
1817 __generate_vector(_Gp&& __gen)
1819 if constexpr (__is_vector_type_v<_V>)
1820 return __generate_vector_impl<
typename _VVT::value_type,
1821 _VVT::_S_full_size>(
1822 static_cast<_Gp&&
>(__gen), make_index_sequence<_VVT::_S_full_size>());
1824 return __generate_vector_impl<
typename _VVT::value_type,
1825 _VVT::_S_partial_width>(
1826 static_cast<_Gp&&
>(__gen),
1827 make_index_sequence<_VVT::_S_partial_width>());
1830template <
typename _Tp,
size_t _Np,
typename _Gp>
1831 _GLIBCXX_SIMD_INTRINSIC
constexpr __vector_type_t<_Tp, _Np>
1832 __generate_vector(_Gp&& __gen)
1834 return __generate_vector_impl<_Tp, _Np>(
static_cast<_Gp&&
>(__gen),
1835 make_index_sequence<_Np>());
1840template <
typename _TW>
1841 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1842 __xor(_TW __a, _TW __b)
noexcept
1844 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1846 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1847 _VectorTraitsImpl<_TW>>::value_type;
1848 if constexpr (is_floating_point_v<_Tp>)
1850 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1851 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1852 ^ __vector_bitcast<_Ip>(__b));
1854 else if constexpr (__is_vector_type_v<_TW>)
1857 return __a._M_data ^ __b._M_data;
1865template <
typename _TW>
1866 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1867 __or(_TW __a, _TW __b)
noexcept
1869 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1871 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1872 _VectorTraitsImpl<_TW>>::value_type;
1873 if constexpr (is_floating_point_v<_Tp>)
1875 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1876 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1877 | __vector_bitcast<_Ip>(__b));
1879 else if constexpr (__is_vector_type_v<_TW>)
1882 return __a._M_data | __b._M_data;
1890template <
typename _TW>
1891 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1892 __and(_TW __a, _TW __b)
noexcept
1894 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1896 using _Tp =
typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1897 _VectorTraitsImpl<_TW>>::value_type;
1898 if constexpr (is_floating_point_v<_Tp>)
1900 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1901 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1902 & __vector_bitcast<_Ip>(__b));
1904 else if constexpr (__is_vector_type_v<_TW>)
1907 return __a._M_data & __b._M_data;
1915#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1916static constexpr struct
1918 _GLIBCXX_SIMD_INTRINSIC __v4sf
1919 operator()(__v4sf __a, __v4sf __b)
const noexcept
1920 {
return __builtin_ia32_andnps(__a, __b); }
1922 _GLIBCXX_SIMD_INTRINSIC __v2df
1923 operator()(__v2df __a, __v2df __b)
const noexcept
1924 {
return __builtin_ia32_andnpd(__a, __b); }
1926 _GLIBCXX_SIMD_INTRINSIC __v2di
1927 operator()(__v2di __a, __v2di __b)
const noexcept
1928 {
return __builtin_ia32_pandn128(__a, __b); }
1930 _GLIBCXX_SIMD_INTRINSIC __v8sf
1931 operator()(__v8sf __a, __v8sf __b)
const noexcept
1932 {
return __builtin_ia32_andnps256(__a, __b); }
1934 _GLIBCXX_SIMD_INTRINSIC __v4df
1935 operator()(__v4df __a, __v4df __b)
const noexcept
1936 {
return __builtin_ia32_andnpd256(__a, __b); }
1938 _GLIBCXX_SIMD_INTRINSIC __v4di
1939 operator()(__v4di __a, __v4di __b)
const noexcept
1941 if constexpr (__have_avx2)
1942 return __builtin_ia32_andnotsi256(__a, __b);
1944 return reinterpret_cast<__v4di
>(
1945 __builtin_ia32_andnpd256(
reinterpret_cast<__v4df
>(__a),
1946 reinterpret_cast<__v4df
>(__b)));
1949 _GLIBCXX_SIMD_INTRINSIC __v16sf
1950 operator()(__v16sf __a, __v16sf __b)
const noexcept
1952 if constexpr (__have_avx512dq)
1953 return _mm512_andnot_ps(__a, __b);
1955 return reinterpret_cast<__v16sf
>(
1956 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1957 reinterpret_cast<__v8di
>(__b)));
1960 _GLIBCXX_SIMD_INTRINSIC __v8df
1961 operator()(__v8df __a, __v8df __b)
const noexcept
1963 if constexpr (__have_avx512dq)
1964 return _mm512_andnot_pd(__a, __b);
1966 return reinterpret_cast<__v8df
>(
1967 _mm512_andnot_si512(
reinterpret_cast<__v8di
>(__a),
1968 reinterpret_cast<__v8di
>(__b)));
1971 _GLIBCXX_SIMD_INTRINSIC __v8di
1972 operator()(__v8di __a, __v8di __b)
const noexcept
1973 {
return _mm512_andnot_si512(__a, __b); }
1977template <
typename _TW>
1978 _GLIBCXX_SIMD_INTRINSIC
constexpr _TW
1979 __andnot(_TW __a, _TW __b)
noexcept
1981 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1983 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1984 _VectorTraitsImpl<_TW>>;
1985 using _Tp =
typename _TVT::value_type;
1986#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1987 if constexpr (
sizeof(_TW) >= 16)
1989 const auto __ai = __to_intrin(__a);
1990 const auto __bi = __to_intrin(__b);
1991 if (!__builtin_is_constant_evaluated()
1992 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1994 const auto __r = _S_x86_andnot(__ai, __bi);
1995 if constexpr (is_convertible_v<
decltype(__r), _TW>)
1998 return reinterpret_cast<typename _TVT::type
>(__r);
2002 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
2003 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
2004 & __vector_bitcast<_Ip>(__b));
2012template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2013 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
2014 __not(_Tp __a)
noexcept
2016 if constexpr (is_floating_point_v<typename _TVT::value_type>)
2017 return reinterpret_cast<typename _TVT::type
>(
2018 ~__vector_bitcast<unsigned>(__a));
2025template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>,
2026 typename _R = __vector_type_t<
typename _TVT::value_type, _TVT::_S_full_size * 2>>
2028 __concat(_Tp a_, _Tp b_)
2030#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2032 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
2034 long long,
typename _TVT::value_type>>;
2035 constexpr int input_width =
sizeof(_Tp) /
sizeof(_W);
2036 const auto __a = __vector_bitcast<_W>(a_);
2037 const auto __b = __vector_bitcast<_W>(b_);
2038 using _Up = __vector_type_t<_W,
sizeof(_R) /
sizeof(_W)>;
2040 constexpr int input_width = _TVT::_S_full_size;
2041 const _Tp& __a = a_;
2042 const _Tp& __b = b_;
2045 if constexpr (input_width == 2)
2046 return reinterpret_cast<_R
>(_Up{__a[0], __a[1], __b[0], __b[1]});
2047 else if constexpr (input_width == 4)
2048 return reinterpret_cast<_R
>(
2049 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
2050 else if constexpr (input_width == 8)
2051 return reinterpret_cast<_R
>(
2052 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
2053 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
2054 else if constexpr (input_width == 16)
2055 return reinterpret_cast<_R
>(
2056 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2057 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2058 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
2059 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
2060 __b[12], __b[13], __b[14], __b[15]});
2061 else if constexpr (input_width == 32)
2062 return reinterpret_cast<_R
>(
2063 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2064 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2065 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
2066 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
2067 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
2068 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
2069 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
2070 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
2071 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
2077template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2078 struct _ZeroExtendProxy
2080 using value_type =
typename _TVT::value_type;
2081 static constexpr size_t _Np = _TVT::_S_full_size;
2084 template <
typename _To,
typename _ToVT = _VectorTraits<_To>,
2086 = enable_if_t<is_same_v<
typename _ToVT::value_type, value_type>>>
2087 _GLIBCXX_SIMD_INTRINSIC
operator _To()
const
2089 constexpr size_t _ToN = _ToVT::_S_full_size;
2090 if constexpr (_ToN == _Np)
2092 else if constexpr (_ToN == 2 * _Np)
2094#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2095 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
2096 return __vector_bitcast<value_type>(
2097 _mm256_insertf128_ps(__m256(), __x, 0));
2098 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
2099 return __vector_bitcast<value_type>(
2100 _mm256_insertf128_pd(__m256d(), __x, 0));
2101 else if constexpr (__have_avx2 && _Np *
sizeof(value_type) == 16)
2102 return __vector_bitcast<value_type>(
2103 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
2104 else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
2106 if constexpr (__have_avx512dq)
2107 return __vector_bitcast<value_type>(
2108 _mm512_insertf32x8(__m512(), __x, 0));
2110 return reinterpret_cast<__m512
>(
2111 _mm512_insertf64x4(__m512d(),
2112 reinterpret_cast<__m256d
>(__x), 0));
2114 else if constexpr (__have_avx512f
2115 && _TVT::template _S_is<double, 4>)
2116 return __vector_bitcast<value_type>(
2117 _mm512_insertf64x4(__m512d(), __x, 0));
2118 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 32)
2119 return __vector_bitcast<value_type>(
2120 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
2122 return __concat(__x, _Tp());
2124 else if constexpr (_ToN == 4 * _Np)
2126#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2127 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2129 return __vector_bitcast<value_type>(
2130 _mm512_insertf64x2(__m512d(), __x, 0));
2132 else if constexpr (__have_avx512f
2133 && is_floating_point_v<value_type>)
2135 return __vector_bitcast<value_type>(
2136 _mm512_insertf32x4(__m512(),
reinterpret_cast<__m128
>(__x),
2139 else if constexpr (__have_avx512f && _Np *
sizeof(value_type) == 16)
2141 return __vector_bitcast<value_type>(
2142 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2145 return __concat(__concat(__x, _Tp()),
2146 __vector_type_t<value_type, _Np * 2>());
2148 else if constexpr (_ToN == 8 * _Np)
2149 return __concat(
operator __vector_type_t<value_type, _Np * 4>(),
2150 __vector_type_t<value_type, _Np * 4>());
2151 else if constexpr (_ToN == 16 * _Np)
2152 return __concat(
operator __vector_type_t<value_type, _Np * 8>(),
2153 __vector_type_t<value_type, _Np * 8>());
2155 __assert_unreachable<_Tp>();
2159template <
typename _Tp,
typename _TVT = _VectorTraits<_Tp>>
2160 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2161 __zero_extend(_Tp __x)
2166template <
int _Offset,
2169 typename _TVT = _VectorTraits<_Tp>,
2170 typename _R = __vector_type_t<
typename _TVT::value_type, _TVT::_S_full_size / _SplitBy>>
2171 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2174 using value_type =
typename _TVT::value_type;
2175#if _GLIBCXX_SIMD_X86INTRIN
2176 if constexpr (
sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2178 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2179 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2180 else if constexpr (is_floating_point_v<value_type>)
2181 return __vector_bitcast<value_type>(
2182 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2184 return reinterpret_cast<_R
>(
2185 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2191#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2193 is_floating_point_v<value_type>, double,
2195 static_assert(
sizeof(_R) %
sizeof(_W) == 0);
2196 constexpr int __return_width =
sizeof(_R) /
sizeof(_W);
2197 using _Up = __vector_type_t<_W, __return_width>;
2198 const auto __x = __vector_bitcast<_W>(__in);
2200 constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2202 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2205 constexpr int _O = _Offset * __return_width;
2206 return __call_with_subscripts<__return_width, _O>(
2207 __x, [](
auto... __entries) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2208 return reinterpret_cast<_R
>(_Up{__entries...});
2215template <
typename _Tp,
2216 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2217 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2221 __builtin_memcpy(&__r, &__x, 8);
2225template <
typename _Tp,
2226 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2227 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2230 static_assert(
sizeof(_Tp) == 16,
"use __hi64z if you meant it");
2232 __builtin_memcpy(&__r,
reinterpret_cast<const char*
>(&__x) + 8, 8);
2236template <
typename _Tp,
2237 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2238 _GLIBCXX_SIMD_INTRINSIC
constexpr _R
2239 __hi64z([[maybe_unused]] _Tp __x)
2242 if constexpr (
sizeof(_Tp) == 16)
2243 __builtin_memcpy(&__r,
reinterpret_cast<const char*
>(&__x) + 8, 8);
2249template <
typename _Tp>
2250 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2252 {
return __extract<0,
sizeof(_Tp) / 16>(__x); }
2254template <
typename _Tp>
2255 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2258 static_assert(
sizeof(__x) == 32);
2259 return __extract<1, 2>(__x);
2264template <
typename _Tp>
2265 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2268 static_assert(
sizeof(__x) == 64);
2269 return __extract<0, 2>(__x);
2272template <
typename _Tp>
2273 _GLIBCXX_SIMD_INTRINSIC
constexpr auto
2276 static_assert(
sizeof(__x) == 64);
2277 return __extract<1, 2>(__x);
2282template <
typename _Tp>
2285 static_assert(__is_vector_type_v<_Tp>);
2289 template <
typename _Up,
typename _UVT = _VectorTraits<_Up>>
2290 _GLIBCXX_SIMD_INTRINSIC
constexpr operator _Up()
const
2291 {
return __intrin_bitcast<typename _UVT::type>(__x); }
2294template <
typename _Tp>
2295 _GLIBCXX_SIMD_INTRINSIC
constexpr _AutoCast<_Tp>
2296 __auto_bitcast(
const _Tp& __x)
2299template <
typename _Tp,
size_t _Np>
2300 _GLIBCXX_SIMD_INTRINSIC
constexpr
2301 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2302 __auto_bitcast(
const _SimdWrapper<_Tp, _Np>& __x)
2303 {
return {__x._M_data}; }
2308#if _GLIBCXX_SIMD_HAVE_SSE_ABI
2310#if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2311template <
size_t _Size>
2312 struct __bool_storage_member_type
2314 static_assert((_Size & (_Size - 1)) != 0,
2315 "This trait may only be used for non-power-of-2 sizes. "
2316 "Power-of-2 sizes must be specialized.");
2318 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2322 struct __bool_storage_member_type<1> {
using type = bool; };
2325 struct __bool_storage_member_type<2> {
using type = __mmask8; };
2328 struct __bool_storage_member_type<4> {
using type = __mmask8; };
2331 struct __bool_storage_member_type<8> {
using type = __mmask8; };
2334 struct __bool_storage_member_type<16> {
using type = __mmask16; };
2337 struct __bool_storage_member_type<32> {
using type = __mmask32; };
2340 struct __bool_storage_member_type<64> {
using type = __mmask64; };
2346#if _GLIBCXX_SIMD_HAVE_SSE
2347template <
typename _Tp,
size_t _Bytes>
2348 struct __intrinsic_type<_Tp, _Bytes,
enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2350 static_assert(!is_same_v<_Tp, long double>,
2351 "no __intrinsic_type support for long double on x86");
2353 static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16 : _Bytes <= 32 ? 32 : 64;
2355 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2356 = conditional_t<is_integral_v<_Tp>,
long long int, _Tp>;
2363#if _GLIBCXX_SIMD_HAVE_NEON
2365 struct __intrinsic_type<float, 8, void>
2366 {
using type = float32x2_t; };
2369 struct __intrinsic_type<float, 16, void>
2370 {
using type = float32x4_t; };
2372#if _GLIBCXX_SIMD_HAVE_NEON_A64
2374 struct __intrinsic_type<double, 8, void>
2375 {
using type = float64x1_t; };
2378 struct __intrinsic_type<double, 16, void>
2379 {
using type = float64x2_t; };
2382#define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2384 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2385 _Np * _Bits / 8, void> \
2386 { using type = int##_Bits##x##_Np##_t; }; \
2388 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2389 _Np * _Bits / 8, void> \
2390 { using type = uint##_Bits##x##_Np##_t; }
2391_GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2392_GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2393_GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2394_GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2395_GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2396_GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2397_GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2398_GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2399#undef _GLIBCXX_SIMD_ARM_INTRIN
2401template <
typename _Tp,
size_t _Bytes>
2402 struct __intrinsic_type<_Tp, _Bytes,
enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2404 static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2406 using _Ip = __int_for_sizeof_t<_Tp>;
2409 is_floating_point_v<_Tp>, _Tp,
2410 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2412 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2413 "should use explicit specialization above");
2415 using type =
typename __intrinsic_type<_Up, _SVecBytes>::type;
2422template <
typename _Tp>
2423 struct __intrinsic_type_impl;
2425#define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2427 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2428_GLIBCXX_SIMD_PPC_INTRIN(
float);
2430_GLIBCXX_SIMD_PPC_INTRIN(
double);
2432_GLIBCXX_SIMD_PPC_INTRIN(
signed char);
2433_GLIBCXX_SIMD_PPC_INTRIN(
unsigned char);
2434_GLIBCXX_SIMD_PPC_INTRIN(
signed short);
2435_GLIBCXX_SIMD_PPC_INTRIN(
unsigned short);
2436_GLIBCXX_SIMD_PPC_INTRIN(
signed int);
2437_GLIBCXX_SIMD_PPC_INTRIN(
unsigned int);
2438#if defined __VSX__ || __SIZEOF_LONG__ == 4
2439_GLIBCXX_SIMD_PPC_INTRIN(
signed long);
2440_GLIBCXX_SIMD_PPC_INTRIN(
unsigned long);
2443_GLIBCXX_SIMD_PPC_INTRIN(
signed long long);
2444_GLIBCXX_SIMD_PPC_INTRIN(
unsigned long long);
2446#undef _GLIBCXX_SIMD_PPC_INTRIN
2448template <
typename _Tp,
size_t _Bytes>
2449 struct __intrinsic_type<_Tp, _Bytes,
enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2451 static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2454 static_assert(!(_S_is_ldouble &&
sizeof(
long double) >
sizeof(
double)),
2455 "no __intrinsic_type support for 128-bit floating point on PowerPC");
2458 static_assert(!(is_same_v<_Tp, double>
2459 || (_S_is_ldouble &&
sizeof(
long double) ==
sizeof(
double))),
2460 "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX");
2464 typename __intrinsic_type_impl<
2465 conditional_t<is_floating_point_v<_Tp>,
2466 conditional_t<_S_is_ldouble, double, _Tp>,
2467 __int_for_sizeof_t<_Tp>>>::type;
2473template <
size_t _W
idth>
2474 struct _SimdWrapper<bool, _Width,
2475 void_t<typename __bool_storage_member_type<_Width>::type>>
2477 using _BuiltinType =
typename __bool_storage_member_type<_Width>::type;
2478 using value_type = bool;
2480 static constexpr size_t _S_full_size =
sizeof(_BuiltinType) * __CHAR_BIT__;
2482 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper<bool, _S_full_size>
2483 __as_full_vector()
const
2486 _GLIBCXX_SIMD_INTRINSIC
constexpr
2487 _SimdWrapper() =
default;
2489 _GLIBCXX_SIMD_INTRINSIC
constexpr
2490 _SimdWrapper(_BuiltinType __k) : _M_data(__k) {};
2492 _GLIBCXX_SIMD_INTRINSIC
2493 operator const _BuiltinType&()
const
2496 _GLIBCXX_SIMD_INTRINSIC
2497 operator _BuiltinType&()
2500 _GLIBCXX_SIMD_INTRINSIC _BuiltinType
2504 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
2505 operator[](
size_t __i)
const
2506 {
return _M_data & (_BuiltinType(1) << __i); }
2508 template <
size_t __i>
2509 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
2510 operator[](_SizeConstant<__i>)
const
2511 {
return _M_data & (_BuiltinType(1) << __i); }
2513 _GLIBCXX_SIMD_INTRINSIC
constexpr void
2514 _M_set(
size_t __i, value_type __x)
2517 _M_data |= (_BuiltinType(1) << __i);
2519 _M_data &= ~(_BuiltinType(1) << __i);
2522 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2523 _M_is_constprop()
const
2524 {
return __builtin_constant_p(_M_data); }
2526 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2527 _M_is_constprop_none_of()
const
2529 if (__builtin_constant_p(_M_data))
2531 constexpr int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2532 constexpr _BuiltinType __active_mask
2533 = ~_BuiltinType() >> (__nbits - _Width);
2534 return (_M_data & __active_mask) == 0;
2539 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2540 _M_is_constprop_all_of()
const
2542 if (__builtin_constant_p(_M_data))
2544 constexpr int __nbits =
sizeof(_BuiltinType) * __CHAR_BIT__;
2545 constexpr _BuiltinType __active_mask
2546 = ~_BuiltinType() >> (__nbits - _Width);
2547 return (_M_data & __active_mask) == __active_mask;
2552 _BuiltinType _M_data;
2556template <
bool _MustZeroInitPadding,
typename _BuiltinType>
2557 struct _SimdWrapperBase;
2559template <
typename _BuiltinType>
2560 struct _SimdWrapperBase<false, _BuiltinType>
2562 _GLIBCXX_SIMD_INTRINSIC
constexpr
2563 _SimdWrapperBase() =
default;
2565 _GLIBCXX_SIMD_INTRINSIC
constexpr
2566 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2568 _BuiltinType _M_data;
2571template <
typename _BuiltinType>
2572 struct _SimdWrapperBase<true, _BuiltinType>
2575 _GLIBCXX_SIMD_INTRINSIC
constexpr
2576 _SimdWrapperBase() : _M_data() {}
2578 _GLIBCXX_SIMD_INTRINSIC
constexpr
2579 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {}
2581 _BuiltinType _M_data;
2586template <
typename _Tp,
size_t _W
idth>
2587 struct _SimdWrapper<
2589 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2590 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2591 && sizeof(_Tp) * _Width
2592 == sizeof(__vector_type_t<_Tp, _Width>),
2593 __vector_type_t<_Tp, _Width>>
2596 = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2597 &&
sizeof(_Tp) * _Width
2598 ==
sizeof(__vector_type_t<_Tp, _Width>),
2599 __vector_type_t<_Tp, _Width>>;
2601 static_assert(__is_vectorizable_v<_Tp>);
2602 static_assert(_Width >= 2);
2604 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2605 using value_type = _Tp;
2607 static inline constexpr size_t _S_full_size
2608 =
sizeof(_BuiltinType) /
sizeof(value_type);
2609 static inline constexpr int _S_size = _Width;
2610 static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2612 using _Base::_M_data;
2614 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper<_Tp, _S_full_size>
2615 __as_full_vector()
const
2618 _GLIBCXX_SIMD_INTRINSIC
constexpr
2619 _SimdWrapper(initializer_list<_Tp> __init)
2620 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2621 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2622 return __init.begin()[__i.value];
2625 _GLIBCXX_SIMD_INTRINSIC
constexpr
2626 _SimdWrapper() =
default;
2628 _GLIBCXX_SIMD_INTRINSIC
constexpr
2629 _SimdWrapper(
const _SimdWrapper&) =
default;
2631 _GLIBCXX_SIMD_INTRINSIC
constexpr
2632 _SimdWrapper(_SimdWrapper&&) =
default;
2634 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper&
2635 operator=(
const _SimdWrapper&) =
default;
2637 _GLIBCXX_SIMD_INTRINSIC
constexpr _SimdWrapper&
2638 operator=(_SimdWrapper&&) =
default;
2640 template <
typename _V,
typename =
enable_if_t<disjunction_v<
2641 is_same<_V, __vector_type_t<_Tp, _Width>>,
2642 is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2643 _GLIBCXX_SIMD_INTRINSIC
constexpr
2644 _SimdWrapper(_V __x)
2646 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2648 template <
typename... _As,
2649 typename =
enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2650 &&
sizeof...(_As) <= _Width)>>
2651 _GLIBCXX_SIMD_INTRINSIC
constexpr
2652 operator _SimdTuple<_Tp, _As...>()
const
2654 return __generate_from_n_evaluations<
sizeof...(_As), _SimdTuple<_Tp, _As...>>(
2655 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2656 {
return _M_data[int(__i)]; });
2659 _GLIBCXX_SIMD_INTRINSIC
constexpr
2660 operator const _BuiltinType&()
const
2663 _GLIBCXX_SIMD_INTRINSIC
constexpr
2664 operator _BuiltinType&()
2667 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
2668 operator[](
size_t __i)
const
2669 {
return _M_data[__i]; }
2671 template <
size_t __i>
2672 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
2673 operator[](_SizeConstant<__i>)
const
2674 {
return _M_data[__i]; }
2676 _GLIBCXX_SIMD_INTRINSIC
constexpr void
2677 _M_set(
size_t __i, _Tp __x)
2678 { _M_data[__i] = __x; }
2680 _GLIBCXX_SIMD_INTRINSIC
2682 _M_is_constprop()
const
2683 {
return __builtin_constant_p(_M_data); }
2685 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2686 _M_is_constprop_none_of()
const
2688 if (__builtin_constant_p(_M_data))
2691 if constexpr (is_floating_point_v<_Tp>)
2693 using _Ip = __int_for_sizeof_t<_Tp>;
2694 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2695 __execute_n_times<_Width>(
2696 [&](
auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2699 __execute_n_times<_Width>(
2700 [&](
auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2701 if (__builtin_constant_p(__r))
2707 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
2708 _M_is_constprop_all_of()
const
2710 if (__builtin_constant_p(_M_data))
2713 if constexpr (is_floating_point_v<_Tp>)
2715 using _Ip = __int_for_sizeof_t<_Tp>;
2716 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2717 __execute_n_times<_Width>(
2718 [&](
auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2721 __execute_n_times<_Width>(
2722 [&](
auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2723 if (__builtin_constant_p(__r))
2733template <
typename _Tp>
2735 __vectorized_sizeof()
2737 if constexpr (!__is_vectorizable_v<_Tp>)
2740 if constexpr (
sizeof(_Tp) <= 8)
2743 if constexpr (__have_avx512bw)
2745 if constexpr (__have_avx512f &&
sizeof(_Tp) >= 4)
2747 if constexpr (__have_avx2)
2749 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2751 if constexpr (__have_sse2)
2753 if constexpr (__have_sse && is_same_v<_Tp, float>)
2763 if constexpr (__have_power8vec
2764 || (__have_power_vmx && (
sizeof(_Tp) < 8))
2765 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2769 if constexpr (__have_neon_a64
2770 || (__have_neon_a32 && !is_same_v<_Tp, double>) )
2772 if constexpr (__have_neon
2777 && (__support_neon_float || !is_floating_point_v<_Tp>))
2787template <
typename _Tp>
2788 inline constexpr int max_fixed_size
2789 = (__have_avx512bw &&
sizeof(_Tp) == 1) ? 64 : 32;
2792#if defined __x86_64__ || defined __aarch64__
2793template <
typename _Tp>
2794 using compatible =
conditional_t<(
sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2795#elif defined __ARM_NEON
2798template <
typename _Tp>
2801 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2802 _VecBuiltin<16>, scalar>;
2805 using compatible = scalar;
2810template <
typename _Tp>
2812 __determine_native_abi()
2814 constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2815 if constexpr (__bytes ==
sizeof(_Tp))
2816 return static_cast<scalar*
>(
nullptr);
2817 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2818 return static_cast<_VecBltnBtmsk<__bytes>*
>(
nullptr);
2820 return static_cast<_VecBuiltin<__bytes>*
>(
nullptr);
2823template <
typename _Tp,
typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2824 using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2828#if defined _GLIBCXX_SIMD_DEFAULT_ABI
2829template <
typename _Tp>
2830 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2832template <
typename _Tp>
2833 using __default_abi = compatible<_Tp>;
2841template <
typename _Tp,
typename =
void_t<>>
2844template <
typename _Tp>
2845 struct is_abi_tag<_Tp,
void_t<typename _Tp::_IsValidAbiTag>>
2846 :
public _Tp::_IsValidAbiTag {};
2848template <
typename _Tp>
2849 inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2852template <
typename _Tp>
2855template <
typename _Tp>
2856 inline constexpr bool is_simd_v = is_simd<_Tp>::value;
2858template <
typename _Tp>
2861template <
typename _Tp>
2862inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2865template <
typename _Tp,
typename _Abi,
typename =
void>
2866 struct __simd_size_impl {};
2868template <
typename _Tp,
typename _Abi>
2869 struct __simd_size_impl<
2871 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2872 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2874template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2875 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2877template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2878 inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2881template <
typename _Tp,
size_t _Np,
typename =
void>
2882 struct __deduce_impl;
2893template <
typename _Tp,
size_t _Np,
typename...>
2896template <
typename _Tp,
size_t _Np,
typename... _Abis>
2897 using deduce_t =
typename deduce<_Tp, _Np, _Abis...>::type;
2902template <
typename _Tp,
typename _V,
typename =
void>
2905template <
typename _Tp,
typename _Up,
typename _Abi>
2906 struct rebind_simd<_Tp, simd<_Up, _Abi>,
2907 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2908 {
using type = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; };
2910template <
typename _Tp,
typename _Up,
typename _Abi>
2911 struct rebind_simd<_Tp, simd_mask<_Up, _Abi>,
2912 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2913 {
using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; };
2915template <
typename _Tp,
typename _V>
2916 using rebind_simd_t =
typename rebind_simd<_Tp, _V>::type;
2919template <
int _Np,
typename _V,
typename =
void>
2922template <
int _Np,
typename _Tp,
typename _Abi>
2923 struct resize_simd<_Np, simd<_Tp, _Abi>,
void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2924 {
using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2926template <
int _Np,
typename _Tp,
typename _Abi>
2927 struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2928 {
using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2930template <
int _Np,
typename _V>
2931 using resize_simd_t =
typename resize_simd<_Np, _V>::type;
2935template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
2936 struct memory_alignment
2937 :
public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2939template <
typename _Tp,
typename _Up =
typename _Tp::value_type>
2940 inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2943template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2946template <
typename _Tp,
typename _Abi>
2947 struct is_simd<simd<_Tp, _Abi>> :
public true_type {};
2949template <
typename _Tp>
2950 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2952template <
typename _Tp,
int _Np>
2953 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2955template <
typename _Tp,
size_t _Np>
2956 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2959template <
typename _Tp,
typename _Abi = simd_abi::__default_abi<_Tp>>
2962template <
typename _Tp,
typename _Abi>
2963 struct is_simd_mask<simd_mask<_Tp, _Abi>> :
public true_type {};
2965template <
typename _Tp>
2966 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2968template <
typename _Tp,
int _Np>
2969 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2971template <
typename _Tp,
size_t _Np>
2972 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2976template <
typename _Tp,
typename _Up,
typename _Ap,
bool = is_simd_v<_Tp>,
typename =
void>
2977 struct __static_simd_cast_return_type;
2979template <
typename _Tp,
typename _A0,
typename _Up,
typename _Ap>
2980 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false, void>
2981 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2983template <
typename _Tp,
typename _Up,
typename _Ap>
2984 struct __static_simd_cast_return_type<
2985 _Tp, _Up, _Ap, true,
enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
2986 {
using type = _Tp; };
2988template <
typename _Tp,
typename _Ap>
2989 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2990#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2996 {
using type = simd<_Tp, _Ap>; };
2998template <
typename _Tp,
typename =
void>
2999 struct __safe_make_signed {
using type = _Tp;};
3001template <
typename _Tp>
3002 struct __safe_make_signed<_Tp,
enable_if_t<is_integral_v<_Tp>>>
3005 using type = make_signed_t<make_unsigned_t<_Tp>>;
3008template <
typename _Tp>
3009 using safe_make_signed_t =
typename __safe_make_signed<_Tp>::type;
3011template <
typename _Tp,
typename _Up,
typename _Ap>
3012 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
3013#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
3021 (is_integral_v<_Up> && is_integral_v<_Tp> &&
3022#ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
3023 is_signed_v<_Up> != is_signed_v<_Tp> &&
3025 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
3026 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
3029template <
typename _Tp,
typename _Up,
typename _Ap,
3031 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3032 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
3033 static_simd_cast(
const simd<_Up, _Ap>& __x)
3035 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
3039 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
3041 return _R(__private_init, __c(__data(__x)));
3045namespace __proposed {
3046template <
typename _Tp,
typename _Up,
typename _Ap,
3048 =
typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3049 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
typename _R::mask_type
3050 static_simd_cast(
const simd_mask<_Up, _Ap>& __x)
3052 using _RM =
typename _R::mask_type;
3053 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
3054 typename _RM::simd_type::value_type>(__x)};
3057template <
typename _To,
typename _Up,
typename _Abi>
3058 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3060 simd_bit_cast(
const simd<_Up, _Abi>& __x)
3062 using _Tp =
typename _To::value_type;
3063 using _ToMember =
typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3064 using _From = simd<_Up, _Abi>;
3065 using _FromMember =
typename _SimdTraits<_Up, _Abi>::_SimdMember;
3067 static_assert(
sizeof(_To) ==
sizeof(_From));
3068 static_assert(is_trivially_copyable_v<_Tp> && is_trivially_copyable_v<_Up>);
3069 static_assert(is_trivially_copyable_v<_ToMember> && is_trivially_copyable_v<_FromMember>);
3070#if __has_builtin(__builtin_bit_cast)
3071 return {__private_init, __builtin_bit_cast(_ToMember, __data(__x))};
3073 return {__private_init, __bit_cast<_ToMember>(__data(__x))};
3077template <
typename _To,
typename _Up,
typename _Abi>
3078 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3080 simd_bit_cast(
const simd_mask<_Up, _Abi>& __x)
3082 using _From = simd_mask<_Up, _Abi>;
3083 static_assert(
sizeof(_To) ==
sizeof(_From));
3084 static_assert(is_trivially_copyable_v<_From>);
3087 if constexpr (is_simd_v<_To>)
3089 using _Tp =
typename _To::value_type;
3090 using _ToMember =
typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3091 static_assert(is_trivially_copyable_v<_ToMember>);
3092#if __has_builtin(__builtin_bit_cast)
3093 return {__private_init, __builtin_bit_cast(_ToMember, __x)};
3095 return {__private_init, __bit_cast<_ToMember>(__x)};
3100 static_assert(is_trivially_copyable_v<_To>);
3101#if __has_builtin(__builtin_bit_cast)
3102 return __builtin_bit_cast(_To, __x);
3104 return __bit_cast<_To>(__x);
3111template <
typename _Tp,
typename _Up,
typename _Ap,
3112 typename _To = __value_type_or_identity_t<_Tp>>
3113 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
3114 simd_cast(
const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
3115 ->
decltype(static_simd_cast<_Tp>(__x))
3116 {
return static_simd_cast<_Tp>(__x); }
3118namespace __proposed {
3119template <
typename _Tp,
typename _Up,
typename _Ap,
3120 typename _To = __value_type_or_identity_t<_Tp>>
3121 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
auto
3122 simd_cast(
const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
3123 ->
decltype(static_simd_cast<_Tp>(__x))
3124 {
return static_simd_cast<_Tp>(__x); }
3129namespace __proposed {
3156template <
typename _Tp,
typename _Up,
typename _Ap>
3157 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
enable_if_t<
3158 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3159 resizing_simd_cast(
const simd<_Up, _Ap>& __x)
3161 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3163 else if constexpr (simd_size_v<_Up, _Ap> == 1)
3169 else if constexpr (_Tp::size() == 1)
3171 else if constexpr (
sizeof(_Tp) ==
sizeof(__x)
3172 && !__is_fixed_size_abi_v<_Ap>)
3173 return {__private_init,
3174 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3175 _Ap::_S_masked(__data(__x))._M_data)};
3179 __builtin_memcpy(&__data(__r), &__data(__x),
3181 *
std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3186template <
typename _Tp,
typename _Up,
typename _Ap>
3187 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3188 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3189 resizing_simd_cast(
const simd_mask<_Up, _Ap>& __x)
3191 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3192 typename _Tp::simd_type::value_type>(__x)};
3198template <
typename _Tp,
int _Np>
3199 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3200 to_fixed_size(
const fixed_size_simd<_Tp, _Np>& __x)
3203template <
typename _Tp,
int _Np>
3204 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3205 to_fixed_size(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3208template <
typename _Tp,
typename _Ap>
3209 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>
3210 to_fixed_size(
const simd<_Tp, _Ap>& __x)
3212 using _Rp = fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>;
3213 return _Rp([&__x](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
3216template <
typename _Tp,
typename _Ap>
3217 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, simd_size_v<_Tp, _Ap>>
3218 to_fixed_size(
const simd_mask<_Tp, _Ap>& __x)
3220 return {__private_init,
3221 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; }};
3225template <
typename _Tp,
int _Np>
3226 _GLIBCXX_SIMD_INTRINSIC
3227 enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3228 to_native(
const fixed_size_simd<_Tp, _Np>& __x)
3230 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3231 __x.copy_to(__mem, vector_aligned);
3232 return {__mem, vector_aligned};
3235template <
typename _Tp,
size_t _Np>
3236 _GLIBCXX_SIMD_INTRINSIC
3237 enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3238 to_native(
const fixed_size_simd_mask<_Tp, _Np>& __x)
3240 return native_simd_mask<_Tp>(
3242 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
3246template <
typename _Tp,
size_t _Np>
3247 _GLIBCXX_SIMD_INTRINSIC
enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3248 to_compatible(
const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3250 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3251 __x.copy_to(__mem, vector_aligned);
3252 return {__mem, vector_aligned};
3255template <
typename _Tp,
size_t _Np>
3256 _GLIBCXX_SIMD_INTRINSIC
3257 enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3258 to_compatible(
const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3260 return simd_mask<_Tp>(
3261 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
3268template <
typename _M,
typename _Tp>
3269 class const_where_expression
3272 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3274 struct _Wrapper {
using value_type = _V; };
3277 using _Impl =
typename _V::_Impl;
3280 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3282 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3283 __get_mask(
const const_where_expression& __x)
3284 {
return __x._M_k; }
3286 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3287 __get_lvalue(
const const_where_expression& __x)
3288 {
return __x._M_value; }
3294 const_where_expression(
const const_where_expression&) =
delete;
3296 const_where_expression& operator=(
const const_where_expression&) =
delete;
3298 _GLIBCXX_SIMD_INTRINSIC const_where_expression(
const _M& __kk,
const _Tp& dd)
3299 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3301 _GLIBCXX_SIMD_INTRINSIC _V
3304 return {__private_init,
3305 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3309 template <
typename _Up,
typename _Flags>
3310 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3311 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3313 return {__private_init,
3314 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3315 _Flags::template _S_apply<_V>(__mem))};
3318 template <
typename _Up,
typename _Flags>
3319 _GLIBCXX_SIMD_INTRINSIC
void
3320 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3322 _Impl::_S_masked_store(__data(_M_value),
3323 _Flags::template _S_apply<_V>(__mem),
3329template <
typename _Tp>
3330 class const_where_expression<bool, _Tp>
3335 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3337 struct _Wrapper {
using value_type = _V; };
3341 =
typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3343 _GLIBCXX_SIMD_INTRINSIC
friend const _M&
3344 __get_mask(
const const_where_expression& __x)
3345 {
return __x._M_k; }
3347 _GLIBCXX_SIMD_INTRINSIC
friend const _Tp&
3348 __get_lvalue(
const const_where_expression& __x)
3349 {
return __x._M_value; }
3355 const_where_expression(
const const_where_expression&) =
delete;
3356 const_where_expression& operator=(
const const_where_expression&) =
delete;
3358 _GLIBCXX_SIMD_INTRINSIC const_where_expression(
const bool __kk,
const _Tp& dd)
3359 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3361 _GLIBCXX_SIMD_INTRINSIC _V
operator-() const&&
3362 {
return _M_k ? -_M_value : _M_value; }
3364 template <
typename _Up,
typename _Flags>
3365 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3366 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3367 {
return _M_k ?
static_cast<_V
>(__mem[0]) : _M_value; }
3369 template <
typename _Up,
typename _Flags>
3370 _GLIBCXX_SIMD_INTRINSIC
void
3371 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags)
const&&
3374 __mem[0] = _M_value;
3379template <
typename _M,
typename _Tp>
3380 class where_expression :
public const_where_expression<_M, _Tp>
3382 using _Impl =
typename const_where_expression<_M, _Tp>::_Impl;
3384 static_assert(!is_const<_Tp>::value,
3385 "where_expression may only be instantiated with __a non-const "
3388 using typename const_where_expression<_M, _Tp>::value_type;
3389 using const_where_expression<_M, _Tp>::_M_k;
3390 using const_where_expression<_M, _Tp>::_M_value;
3393 is_same<typename _M::abi_type, typename _Tp::abi_type>::value,
"");
3394 static_assert(_M::size() == _Tp::size(),
"");
3396 _GLIBCXX_SIMD_INTRINSIC
friend _Tp& __get_lvalue(where_expression& __x)
3397 {
return __x._M_value; }
3400 where_expression(
const where_expression&) =
delete;
3401 where_expression& operator=(
const where_expression&) =
delete;
3403 _GLIBCXX_SIMD_INTRINSIC where_expression(
const _M& __kk, _Tp& dd)
3404 : const_where_expression<_M, _Tp>(__kk, dd) {}
3406 template <
typename _Up>
3407 _GLIBCXX_SIMD_INTRINSIC
void operator=(_Up&& __x) &&
3409 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3410 __to_value_type_or_member_type<_Tp>(
3411 static_cast<_Up&&
>(__x)));
3414#define _GLIBCXX_SIMD_OP_(__op, __name) \
3415 template <typename _Up> \
3416 _GLIBCXX_SIMD_INTRINSIC void operator __op##=(_Up&& __x)&& \
3418 _Impl::template _S_masked_cassign( \
3419 __data(_M_k), __data(_M_value), \
3420 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3421 [](auto __impl, auto __lhs, auto __rhs) \
3422 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
3423 { return __impl.__name(__lhs, __rhs); }); \
3426 _GLIBCXX_SIMD_OP_(+, _S_plus);
3427 _GLIBCXX_SIMD_OP_(-, _S_minus);
3428 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3429 _GLIBCXX_SIMD_OP_(/, _S_divides);
3430 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3431 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3432 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3433 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3434 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3435 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3436#undef _GLIBCXX_SIMD_OP_
3438 _GLIBCXX_SIMD_INTRINSIC
void
3442 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3445 _GLIBCXX_SIMD_INTRINSIC
void
3449 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value));
3452 _GLIBCXX_SIMD_INTRINSIC
void
3456 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3459 _GLIBCXX_SIMD_INTRINSIC
void
3463 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value));
3467 template <
typename _Up,
typename _Flags>
3468 _GLIBCXX_SIMD_INTRINSIC
void
3469 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3471 __data(_M_value) = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3472 _Flags::template _S_apply<_Tp>(__mem));
3477template <
typename _Tp>
3478 class where_expression<bool, _Tp>
3479 :
public const_where_expression<bool, _Tp>
3482 using typename const_where_expression<_M, _Tp>::value_type;
3483 using const_where_expression<_M, _Tp>::_M_k;
3484 using const_where_expression<_M, _Tp>::_M_value;
3487 where_expression(
const where_expression&) =
delete;
3488 where_expression& operator=(
const where_expression&) =
delete;
3490 _GLIBCXX_SIMD_INTRINSIC
3491 where_expression(
const _M& __kk, _Tp& dd)
3492 : const_where_expression<_M, _Tp>(__kk, dd) {}
3494#define _GLIBCXX_SIMD_OP_(__op) \
3495 template <typename _Up> \
3496 _GLIBCXX_SIMD_INTRINSIC void \
3497 operator __op(_Up&& __x)&& \
3498 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3500 _GLIBCXX_SIMD_OP_(=)
3501 _GLIBCXX_SIMD_OP_(+=)
3502 _GLIBCXX_SIMD_OP_(-=)
3503 _GLIBCXX_SIMD_OP_(*=)
3504 _GLIBCXX_SIMD_OP_(/=)
3505 _GLIBCXX_SIMD_OP_(%=)
3506 _GLIBCXX_SIMD_OP_(&=)
3507 _GLIBCXX_SIMD_OP_(|=)
3508 _GLIBCXX_SIMD_OP_(^=)
3509 _GLIBCXX_SIMD_OP_(<<=)
3510 _GLIBCXX_SIMD_OP_(>>=)
3511 #undef _GLIBCXX_SIMD_OP_
3513 _GLIBCXX_SIMD_INTRINSIC
void
3515 {
if (_M_k) ++_M_value; }
3517 _GLIBCXX_SIMD_INTRINSIC
void
3519 {
if (_M_k) ++_M_value; }
3521 _GLIBCXX_SIMD_INTRINSIC
void
3523 {
if (_M_k) --_M_value; }
3525 _GLIBCXX_SIMD_INTRINSIC
void
3527 {
if (_M_k) --_M_value; }
3530 template <
typename _Up,
typename _Flags>
3531 _GLIBCXX_SIMD_INTRINSIC
void
3532 copy_from(
const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3533 {
if (_M_k) _M_value = __mem[0]; }
3537template <
typename _Tp,
typename _Ap>
3538 _GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3539 where(
const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3540 {
return {__k, __value}; }
3542template <
typename _Tp,
typename _Ap>
3543 _GLIBCXX_SIMD_INTRINSIC
3544 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3545 where(
const typename simd<_Tp, _Ap>::mask_type& __k,
const simd<_Tp, _Ap>& __value)
3546 {
return {__k, __value}; }
3548template <
typename _Tp,
typename _Ap>
3549 _GLIBCXX_SIMD_INTRINSIC
3550 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3551 where(
const remove_const_t<simd_mask<_Tp, _Ap>>& __k, simd_mask<_Tp, _Ap>& __value)
3552 {
return {__k, __value}; }
3554template <
typename _Tp,
typename _Ap>
3555 _GLIBCXX_SIMD_INTRINSIC
3556 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3557 where(
const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
const simd_mask<_Tp, _Ap>& __value)
3558 {
return {__k, __value}; }
3560template <
typename _Tp>
3561 _GLIBCXX_SIMD_INTRINSIC where_expression<bool, _Tp>
3562 where(_ExactBool __k, _Tp& __value)
3563 {
return {__k, __value}; }
3565template <
typename _Tp>
3566 _GLIBCXX_SIMD_INTRINSIC const_where_expression<bool, _Tp>
3567 where(_ExactBool __k,
const _Tp& __value)
3568 {
return {__k, __value}; }
3570template <
typename _Tp,
typename _Ap>
3571 void where(
bool __k, simd<_Tp, _Ap>& __value) =
delete;
3573template <
typename _Tp,
typename _Ap>
3574 void where(
bool __k,
const simd<_Tp, _Ap>& __value) =
delete;
3577namespace __proposed {
3578template <
size_t _Np>
3581 const bitset<_Np> __bits;
3584 where_range(bitset<_Np> __b) : __bits(__b) {}
3591 _GLIBCXX_SIMD_INTRINSIC
void
3593 { __bit = __builtin_ctzl(__mask); }
3595 _GLIBCXX_SIMD_INTRINSIC
void
3599 __mask &= (__mask - 1);
3604 iterator(
decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3605 iterator(
const iterator&) =
default;
3606 iterator(iterator&&) =
default;
3608 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t
3612 _GLIBCXX_SIMD_ALWAYS_INLINE
size_t
3616 _GLIBCXX_SIMD_ALWAYS_INLINE iterator&
3624 _GLIBCXX_SIMD_ALWAYS_INLINE iterator
3627 iterator __tmp = *
this;
3633 _GLIBCXX_SIMD_ALWAYS_INLINE
bool
3634 operator==(
const iterator& __rhs)
const
3635 {
return __mask == __rhs.__mask; }
3637 _GLIBCXX_SIMD_ALWAYS_INLINE
bool
3638 operator!=(
const iterator& __rhs)
const
3639 {
return __mask != __rhs.__mask; }
3644 {
return __bits.to_ullong(); }
3651template <
typename _Tp,
typename _Ap>
3652 where_range<simd_size_v<_Tp, _Ap>>
3653 where(
const simd_mask<_Tp, _Ap>& __k)
3654 {
return __k.__to_bitset(); }
3660template <
typename _Tp,
typename _Abi,
typename _BinaryOperation = plus<>>
3661 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3662 reduce(
const simd<_Tp, _Abi>& __v, _BinaryOperation __binary_op = _BinaryOperation())
3663 {
return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3665template <
typename _M,
typename _V,
typename _BinaryOperation = plus<>>
3666 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3667 reduce(
const const_where_expression<_M, _V>& __x,
3668 typename _V::value_type __identity_element, _BinaryOperation __binary_op)
3670 if (__builtin_expect(none_of(__get_mask(__x)),
false))
3671 return __identity_element;
3673 _V __tmp = __identity_element;
3674 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3675 __data(__get_lvalue(__x)));
3676 return reduce(__tmp, __binary_op);
3679template <
typename _M,
typename _V>
3680 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3681 reduce(
const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3682 {
return reduce(__x, 0, __binary_op); }
3684template <
typename _M,
typename _V>
3685 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3686 reduce(
const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3687 {
return reduce(__x, 1, __binary_op); }
3689template <
typename _M,
typename _V>
3690 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3691 reduce(
const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3692 {
return reduce(__x, ~
typename _V::value_type(), __binary_op); }
3694template <
typename _M,
typename _V>
3695 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3696 reduce(
const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3697 {
return reduce(__x, 0, __binary_op); }
3699template <
typename _M,
typename _V>
3700 _GLIBCXX_SIMD_INTRINSIC
typename _V::value_type
3701 reduce(
const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3702 {
return reduce(__x, 0, __binary_op); }
3704template <
typename _Tp,
typename _Abi>
3705 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3706 hmin(
const simd<_Tp, _Abi>& __v)
noexcept
3707 {
return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum()); }
3709template <
typename _Tp,
typename _Abi>
3710 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3711 hmax(
const simd<_Tp, _Abi>& __v)
noexcept
3712 {
return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum()); }
3714template <
typename _M,
typename _V>
3715 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3716 typename _V::value_type
3717 hmin(
const const_where_expression<_M, _V>& __x)
noexcept
3719 using _Tp =
typename _V::value_type;
3720 constexpr _Tp __id_elem =
3721#ifdef __FINITE_MATH_ONLY__
3722 __finite_max_v<_Tp>;
3724 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3726 _V __tmp = __id_elem;
3727 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3728 __data(__get_lvalue(__x)));
3729 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3732template <
typename _M,
typename _V>
3733 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3734 typename _V::value_type
3735 hmax(
const const_where_expression<_M, _V>& __x)
noexcept
3737 using _Tp =
typename _V::value_type;
3738 constexpr _Tp __id_elem =
3739#ifdef __FINITE_MATH_ONLY__
3740 __finite_min_v<_Tp>;
3743 if constexpr (__value_exists_v<__infinity, _Tp>)
3744 return -__infinity_v<_Tp>;
3746 return __finite_min_v<_Tp>;
3749 _V __tmp = __id_elem;
3750 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3751 __data(__get_lvalue(__x)));
3752 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3757template <
typename _Tp,
typename _Ap>
3758 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3759 min(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3760 {
return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3762template <
typename _Tp,
typename _Ap>
3763 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3764 max(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3765 {
return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3767template <
typename _Tp,
typename _Ap>
3768 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3769 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3770 minmax(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
3772 const auto pair_of_members
3773 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3774 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3775 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3778template <
typename _Tp,
typename _Ap>
3779 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3780 clamp(
const simd<_Tp, _Ap>& __v,
const simd<_Tp, _Ap>& __lo,
const simd<_Tp, _Ap>& __hi)
3782 using _Impl =
typename _Ap::_SimdImpl;
3783 return {__private_init,
3784 _Impl::_S_min(__data(__hi),
3785 _Impl::_S_max(__data(__lo), __data(__v)))};
3790template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
3791 typename =
enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3792 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3793 split(
const simd<_Tp, _Ap>&);
3796template <
int _Index,
int _Total,
int _Combine = 1,
typename _Tp,
size_t _Np>
3797 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
3798 _SimdWrapper<_Tp, _Np / _Total * _Combine>
3799 __extract_part(
const _SimdWrapper<_Tp, _Np> __x);
3801template <
int _Index,
int _Parts,
int _Combine = 1,
typename _Tp,
typename _A0,
typename... _As>
3802 _GLIBCXX_SIMD_INTRINSIC
auto
3803 __extract_part(
const _SimdTuple<_Tp, _A0, _As...>& __x);
3807template <
size_t _V0,
size_t... _Values>
3810 template <
size_t _I>
3811 static constexpr size_t
3812 _S_at(_SizeConstant<_I> = {})
3814 if constexpr (_I == 0)
3817 return _SizeList<_Values...>::template _S_at<_I - 1>();
3820 template <
size_t _I>
3821 static constexpr auto
3822 _S_before(_SizeConstant<_I> = {})
3824 if constexpr (_I == 0)
3825 return _SizeConstant<0>();
3827 return _SizeConstant<
3828 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3831 template <
size_t _Np>
3832 static constexpr auto
3833 _S_pop_front(_SizeConstant<_Np> = {})
3835 if constexpr (_Np == 0)
3838 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3844template <
typename _Tp,
size_t _Np>
3845 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3846 __extract_center(_SimdWrapper<_Tp, _Np> __x)
3848 static_assert(_Np >= 4);
3849 static_assert(_Np % 4 == 0);
3850#if _GLIBCXX_SIMD_X86INTRIN
3851 if constexpr (__have_avx512f &&
sizeof(_Tp) * _Np == 64)
3853 const auto __intrin = __to_intrin(__x);
3854 if constexpr (is_integral_v<_Tp>)
3855 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3856 _mm512_shuffle_i32x4(__intrin, __intrin,
3857 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3858 else if constexpr (
sizeof(_Tp) == 4)
3859 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3860 _mm512_shuffle_f32x4(__intrin, __intrin,
3861 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3862 else if constexpr (
sizeof(_Tp) == 8)
3863 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3864 _mm512_shuffle_f64x2(__intrin, __intrin,
3865 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3867 __assert_unreachable<_Tp>();
3869 else if constexpr (
sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3870 return __vector_bitcast<_Tp>(
3871 _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
3872 __hi128(__vector_bitcast<double>(__x)), 1));
3873 else if constexpr (
sizeof(__x) == 32 &&
sizeof(_Tp) * _Np <= 32)
3874 return __vector_bitcast<_Tp>(
3875 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3876 __lo128(__vector_bitcast<_LLong>(__x)),
3877 sizeof(_Tp) * _Np / 4));
3881 __vector_type_t<_Tp, _Np / 2> __r;
3882 __builtin_memcpy(&__r,
3883 reinterpret_cast<const char*
>(&__x)
3884 +
sizeof(_Tp) * _Np / 4,
3885 sizeof(_Tp) * _Np / 2);
3890template <
typename _Tp,
typename _A0,
typename... _As>
3891 _GLIBCXX_SIMD_INTRINSIC
3892 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3893 __extract_center(
const _SimdTuple<_Tp, _A0, _As...>& __x)
3895 if constexpr (
sizeof...(_As) == 0)
3896 return __extract_center(__x.first);
3898 return __extract_part<1, 4, 2>(__x);
3903template <
size_t... _Sizes,
typename _Tp,
typename... _As>
3905 __split_wrapper(_SizeList<_Sizes...>,
const _SimdTuple<_Tp, _As...>& __x)
3907 return split<_Sizes...>(
3908 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3915template <
typename _V,
typename _Ap,
3916 size_t _Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
3917 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == _Parts * _V::size()
3918 && is_simd_v<_V>, array<_V, _Parts>>
3919 split(
const simd<typename _V::value_type, _Ap>& __x)
3921 using _Tp =
typename _V::value_type;
3922 if constexpr (_Parts == 1)
3924 return {simd_cast<_V>(__x)};
3926 else if (__x._M_is_constprop())
3928 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3929 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3930 return _V([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
3931 {
return __x[__i * _V::size() + __j]; });
3935 __is_fixed_size_abi_v<_Ap>
3936 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3937 || (__is_fixed_size_abi_v<typename _V::abi_type>
3938 &&
sizeof(_V) ==
sizeof(_Tp) * _V::size()
3942#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3943 const __may_alias<_Tp>*
const __element_ptr
3944 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__data(__x));
3945 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3946 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
3947 {
return _V(__element_ptr + __i * _V::size(), vector_aligned); });
3949 const auto& __xx = __data(__x);
3950 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3951 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3952 [[maybe_unused]]
constexpr size_t __offset
3953 =
decltype(__i)::value * _V::size();
3954 return _V([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3955 constexpr _SizeConstant<__j + __offset> __k;
3961 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3964 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3965 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
return __x[__i]; });
3969 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3970 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3971 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3972 return _V([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3973 return __x[__i * _V::size() + __j];
3976 return _V(__private_init,
3977 __extract_part<
decltype(__i)::value, _Parts>(__data(__x)));
3984template <
typename _V,
typename _Ap,
3985 size_t _Parts = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
3986 enable_if_t<is_simd_mask_v<_V> && simd_size_v<
typename
3987 _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
3988 split(
const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3990 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3992 else if constexpr (_Parts == 1)
3993 return {__proposed::static_simd_cast<_V>(__x)};
3994 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3995 && __is_avx_abi<_Ap>())
3996 return {_V(__private_init, __lo128(__data(__x))),
3997 _V(__private_init, __hi128(__data(__x)))};
3998 else if constexpr (_V::size() <= __CHAR_BIT__ *
sizeof(_ULLong))
4000 const bitset __bits = __x.__to_bitset();
4001 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4002 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4003 constexpr size_t __offset = __i * _V::size();
4004 return _V(__bitset_init, (__bits >> __offset).to_ullong());
4009 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
4010 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4011 constexpr size_t __offset = __i * _V::size();
4012 return _V(__private_init,
4013 [&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4014 return __x[__j + __offset];
4022template <
size_t... _Sizes,
typename _Tp,
typename _Ap,
typename>
4023 _GLIBCXX_SIMD_ALWAYS_INLINE
4024 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
4025 split(
const simd<_Tp, _Ap>& __x)
4027 using _SL = _SizeList<_Sizes...>;
4028 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
4029 constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
4030 constexpr size_t _N0 = _SL::template _S_at<0>();
4031 using _V = __deduced_simd<_Tp, _N0>;
4033 if (__x._M_is_constprop())
4034 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>(
4035 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4036 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4037 constexpr size_t __offset = _SL::_S_before(__i);
4038 return _Vi([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4039 return __x[__offset + __j];
4042 else if constexpr (_Np == _N0)
4044 static_assert(
sizeof...(_Sizes) == 1);
4045 return {simd_cast<_V>(__x)};
4048 (__is_fixed_size_abi_v<
4049 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
4052 !__is_fixed_size_abi_v<typename _V::abi_type>,
4053 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
4059 __split_wrapper(_SL::template _S_pop_front<1>(),
4060 __data(__x).second));
4062 else if constexpr ((!is_same_v<simd_abi::scalar,
4063 simd_abi::deduce_t<_Tp, _Sizes>> && ...)
4064 && (!__is_fixed_size_abi_v<
4065 simd_abi::deduce_t<_Tp, _Sizes>> && ...))
4067 if constexpr (((_Sizes * 2 == _Np) && ...))
4068 return {{__private_init, __extract_part<0, 2>(__data(__x))},
4069 {__private_init, __extract_part<1, 2>(__data(__x))}};
4070 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4071 _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
4072 return {{__private_init, __extract_part<0, 3>(__data(__x))},
4073 {__private_init, __extract_part<1, 3>(__data(__x))},
4074 {__private_init, __extract_part<2, 3>(__data(__x))}};
4075 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4076 _SizeList<2 * _Np / 3, _Np / 3>>)
4077 return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
4078 {__private_init, __extract_part<2, 3>(__data(__x))}};
4079 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4080 _SizeList<_Np / 3, 2 * _Np / 3>>)
4081 return {{__private_init, __extract_part<0, 3>(__data(__x))},
4082 {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
4083 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4084 _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
4085 return {{__private_init, __extract_part<0, 2>(__data(__x))},
4086 {__private_init, __extract_part<2, 4>(__data(__x))},
4087 {__private_init, __extract_part<3, 4>(__data(__x))}};
4088 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4089 _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
4090 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4091 {__private_init, __extract_part<1, 4>(__data(__x))},
4092 {__private_init, __extract_part<1, 2>(__data(__x))}};
4093 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4094 _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
4095 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4096 {__private_init, __extract_center(__data(__x))},
4097 {__private_init, __extract_part<3, 4>(__data(__x))}};
4098 else if constexpr (((_Sizes * 4 == _Np) && ...))
4099 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4100 {__private_init, __extract_part<1, 4>(__data(__x))},
4101 {__private_init, __extract_part<2, 4>(__data(__x))},
4102 {__private_init, __extract_part<3, 4>(__data(__x))}};
4105#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4106 const __may_alias<_Tp>*
const __element_ptr
4107 =
reinterpret_cast<const __may_alias<_Tp>*
>(&__x);
4108 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>(
4109 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4110 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4111 constexpr size_t __offset = _SL::_S_before(__i);
4112 constexpr size_t __base_align =
alignof(simd<_Tp, _Ap>);
4113 constexpr size_t __a
4114 = __base_align - ((__offset *
sizeof(_Tp)) % __base_align);
4115 constexpr size_t __b = ((__a - 1) & __a) ^ __a;
4116 constexpr size_t __alignment = __b == 0 ? __a : __b;
4117 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
4120 return __generate_from_n_evaluations<
sizeof...(_Sizes), _Tuple>(
4121 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4122 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4123 const auto& __xx = __data(__x);
4124 using _Offset =
decltype(_SL::_S_before(__i));
4125 return _Vi([&](
auto __j)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4126 constexpr _SizeConstant<_Offset::value + __j> __k;
4136template <
size_t _I,
typename _Tp,
typename _Ap,
typename... _As>
4137 _GLIBCXX_SIMD_INTRINSIC
constexpr _Tp
4138 __subscript_in_pack(
const simd<_Tp, _Ap>& __x,
const simd<_Tp, _As>&... __xs)
4140 if constexpr (_I < simd_size_v<_Tp, _Ap>)
4143 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
4148template <
typename _Tp,
typename _A0,
typename... _As>
4149 _GLIBCXX_SIMD_INTRINSIC
void
4150 __store_pack_of_simd(
char* __mem,
const simd<_Tp, _A0>& __x0,
const simd<_Tp, _As>&... __xs)
4152 constexpr size_t __n_bytes =
sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4153 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4154 if constexpr (
sizeof...(__xs) > 0)
4155 __store_pack_of_simd(__mem + __n_bytes, __xs...);
4160template <
typename _Tp,
typename... _As,
typename = __detail::__odr_helper>
4161 inline _GLIBCXX_SIMD_CONSTEXPR
4162 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4163 concat(
const simd<_Tp, _As>&... __xs)
4165 using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
4166 if constexpr (
sizeof...(__xs) == 1)
4167 return simd_cast<_Rp>(__xs...);
4168 else if ((... && __xs._M_is_constprop()))
4170 simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>(
4171 [&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4172 {
return __subscript_in_pack<__i>(__xs...); });
4176 __store_pack_of_simd(
reinterpret_cast<char*
>(&__data(__r)), __xs...);
4183template <
typename _Tp,
typename _Abi,
size_t _Np>
4184 _GLIBCXX_SIMD_ALWAYS_INLINE
4185 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4186 concat(
const array<simd<_Tp, _Abi>, _Np>& __x)
4188 return __call_with_subscripts<_Np>(
4189 __x, [](
const auto&... __xs) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4190 return concat(__xs...);
4198template <
typename _Up,
typename _Accessor = _Up,
4199 typename _ValueType =
typename _Up::value_type>
4200 class _SmartReference
4206 _GLIBCXX_SIMD_INTRINSIC
constexpr _ValueType
4207 _M_read() const noexcept
4209 if constexpr (is_arithmetic_v<_Up>)
4212 return _M_obj[_M_index];
4215 template <
typename _Tp>
4216 _GLIBCXX_SIMD_INTRINSIC
constexpr void
4217 _M_write(_Tp&& __x)
const
4218 { _Accessor::_S_set(_M_obj, _M_index,
static_cast<_Tp&&
>(__x)); }
4221 _GLIBCXX_SIMD_INTRINSIC
constexpr
4222 _SmartReference(_Up& __o,
int __i) noexcept
4223 : _M_index(__i), _M_obj(__o) {}
4225 using value_type = _ValueType;
4227 _GLIBCXX_SIMD_INTRINSIC
4228 _SmartReference(
const _SmartReference&) =
delete;
4230 _GLIBCXX_SIMD_INTRINSIC
constexpr
4231 operator value_type() const noexcept
4232 {
return _M_read(); }
4234 template <
typename _Tp,
typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4235 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference
4236 operator=(_Tp&& __x) &&
4238 _M_write(
static_cast<_Tp&&
>(__x));
4239 return {_M_obj, _M_index};
4242#define _GLIBCXX_SIMD_OP_(__op) \
4243 template <typename _Tp, \
4244 typename _TT = decltype(declval<value_type>() __op declval<_Tp>()), \
4245 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4246 typename = _ValuePreservingOrInt<_TT, value_type>> \
4247 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4248 operator __op##=(_Tp&& __x) && \
4250 const value_type& __lhs = _M_read(); \
4251 _M_write(__lhs __op __x); \
4252 return {_M_obj, _M_index}; \
4254 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4255 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4256 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4257#undef _GLIBCXX_SIMD_OP_
4259 template <
typename _Tp = void,
4260 typename =
decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4261 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference
4264 value_type __x = _M_read();
4266 return {_M_obj, _M_index};
4269 template <
typename _Tp = void,
4270 typename =
decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4271 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
4274 const value_type __r = _M_read();
4275 value_type __x = __r;
4280 template <
typename _Tp = void,
4281 typename =
decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4282 _GLIBCXX_SIMD_INTRINSIC
constexpr _SmartReference
4285 value_type __x = _M_read();
4287 return {_M_obj, _M_index};
4290 template <
typename _Tp = void,
4291 typename =
decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4292 _GLIBCXX_SIMD_INTRINSIC
constexpr value_type
4295 const value_type __r = _M_read();
4296 value_type __x = __r;
4301 _GLIBCXX_SIMD_INTRINSIC
friend void
4302 swap(_SmartReference&& __a, _SmartReference&& __b)
noexcept(
4304 is_nothrow_constructible<value_type, _SmartReference&&>,
4305 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4307 value_type __tmp =
static_cast<_SmartReference&&
>(__a);
4308 static_cast<_SmartReference&&
>(__a) =
static_cast<value_type
>(__b);
4309 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4312 _GLIBCXX_SIMD_INTRINSIC
friend void
4313 swap(value_type& __a, _SmartReference&& __b)
noexcept(
4315 is_nothrow_constructible<value_type, value_type&&>,
4316 is_nothrow_assignable<value_type&, value_type&&>,
4317 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4320 __a =
static_cast<value_type
>(__b);
4321 static_cast<_SmartReference&&
>(__b) =
std::move(__tmp);
4324 _GLIBCXX_SIMD_INTRINSIC
friend void
4325 swap(_SmartReference&& __a, value_type& __b)
noexcept(
4327 is_nothrow_constructible<value_type, _SmartReference&&>,
4328 is_nothrow_assignable<value_type&, value_type&&>,
4329 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4331 value_type __tmp(__a);
4332 static_cast<_SmartReference&&
>(__a) =
std::move(__b);
4339template <
int _Bytes>
4340 struct __scalar_abi_wrapper
4342 template <
typename _Tp>
static constexpr size_t _S_full_size = 1;
4343 template <
typename _Tp>
static constexpr size_t _S_size = 1;
4344 template <
typename _Tp>
static constexpr size_t _S_is_partial =
false;
4346 template <
typename _Tp,
typename _Abi = simd_abi::scalar>
4347 static constexpr bool _S_is_valid_v
4348 = _Abi::template _IsValid<_Tp>::value &&
sizeof(_Tp) == _Bytes;
4353template <
typename _Tp>
4354 struct __decay_abi {
using type = _Tp; };
4356template <
int _Bytes>
4357 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4358 {
using type = simd_abi::scalar; };
4366template <
template <
int>
class _Abi,
int _Bytes,
typename _Tp>
4367 struct __find_next_valid_abi
4369 static constexpr auto
4372 constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4373 using _NextAbi = _Abi<_NextBytes>;
4374 if constexpr (_NextBytes <
sizeof(_Tp) * 2)
4375 return _Abi<_Bytes>();
4376 else if constexpr (_NextAbi::template _S_is_partial<_Tp> ==
false
4377 && _NextAbi::template _S_is_valid_v<_Tp>)
4380 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4383 using type =
decltype(_S_choose());
4386template <
int _Bytes,
typename _Tp>
4387 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4388 {
using type = simd_abi::scalar; };
4391template <
template <
int>
class...>
4394 template <
typename,
int>
static constexpr bool _S_has_valid_abi =
false;
4395 template <
typename,
int>
using _FirstValidAbi = void;
4396 template <
typename,
int>
using _BestAbi = void;
4399template <
template <
int>
class _A0,
template <
int>
class... _Rest>
4400 struct _AbiList<_A0, _Rest...>
4402 template <
typename _Tp,
int _Np>
4403 static constexpr bool _S_has_valid_abi
4404 = _A0<
sizeof(_Tp) * _Np>::template _S_is_valid_v<
4405 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4407 template <
typename _Tp,
int _Np>
4409 _A0<
sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4410 typename __decay_abi<_A0<
sizeof(_Tp) * _Np>>::type,
4411 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4413 template <
typename _Tp,
int _Np>
4414 static constexpr auto
4415 _S_determine_best_abi()
4417 static_assert(_Np >= 1);
4418 constexpr int _Bytes =
sizeof(_Tp) * _Np;
4419 if constexpr (_Np == 1)
4420 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4423 constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4428 if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4429 _Tp> && __fullsize / 2 < _Np)
4430 return typename __decay_abi<_A0<_Bytes>>::type{};
4434 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4435 if constexpr (_Bp::template _S_is_valid_v<
4436 _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4440 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4445 template <
typename _Tp,
int _Np>
4446 using _BestAbi =
decltype(_S_determine_best_abi<_Tp, _Np>());
4454using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4455 __scalar_abi_wrapper>;
4458template <
typename _Tp,
typename _Abi>
4459 struct _SimdTraits<_Tp, _Abi,
void_t<typename _Abi::template _IsValid<_Tp>>>
4460 : _Abi::template __traits<_Tp> {};
4464template <
typename _Tp,
size_t _Np>
4465 struct __deduce_impl<
4466 _Tp, _Np,
enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4467 {
using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4470template <
typename _Tp,
size_t _Np,
typename =
void>
4471 struct __deduce_fixed_size_fallback {};
4473template <
typename _Tp,
size_t _Np>
4474 struct __deduce_fixed_size_fallback<_Tp, _Np,
4475 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4476 {
using type = simd_abi::fixed_size<_Np>; };
4478template <
typename _Tp,
size_t _Np,
typename>
4479 struct __deduce_impl :
public __deduce_fixed_size_fallback<_Tp, _Np> {};
4485template <
typename _Tp,
typename _Abi>
4486 class simd_mask :
public _SimdTraits<_Tp, _Abi>::_MaskBase
4489 using _Traits = _SimdTraits<_Tp, _Abi>;
4490 using _MemberType =
typename _Traits::_MaskMember;
4496 using _Ip = __int_for_sizeof_t<_Tp>;
4497 static constexpr _Ip* _S_type_tag =
nullptr;
4499 friend typename _Traits::_MaskBase;
4500 friend class simd<_Tp, _Abi>;
4501 friend typename _Traits::_SimdImpl;
4504 using _Impl =
typename _Traits::_MaskImpl;
4509 using value_type = bool;
4510 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4511 using simd_type = simd<_Tp, _Abi>;
4512 using abi_type = _Abi;
4515 static constexpr size_t size()
4516 {
return __size_or_zero_v<_Tp, _Abi>; }
4520 simd_mask() =
default;
4521 simd_mask(
const simd_mask&) =
default;
4522 simd_mask(simd_mask&&) =
default;
4523 simd_mask& operator=(
const simd_mask&) =
default;
4524 simd_mask& operator=(simd_mask&&) =
default;
4528 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit
4529 simd_mask(
typename _Traits::_MaskCastType __init)
4530 : _M_data{__init} {}
4539 _GLIBCXX_SIMD_ALWAYS_INLINE
static simd_mask
4540 __from_bitset(bitset<size()> bs)
4541 {
return {__bitset_init, bs}; }
4543 _GLIBCXX_SIMD_ALWAYS_INLINE bitset<size()>
4545 {
return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4549 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
4550 simd_mask(value_type __x)
4551 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4555 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4557 template <
typename _Up,
typename _A2,
4558 typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4559 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit(
sizeof(_MemberType)
4560 !=
sizeof(
typename _SimdTraits<_Up, _A2>::_MaskMember))
4561 simd_mask(
const simd_mask<_Up, _A2>& __x)
4562 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4565 template <
typename _Up,
typename =
enable_if_t<conjunction<
4566 is_same<abi_type, simd_abi::fixed_size<size()>>,
4567 is_same<_Up, _Up>>::value>>
4568 _GLIBCXX_SIMD_ALWAYS_INLINE
4569 simd_mask(
const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4570 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4575 template <
typename _Flags>
4576 _GLIBCXX_SIMD_ALWAYS_INLINE
4577 simd_mask(
const value_type* __mem, _Flags)
4578 : _M_data(_Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem))) {}
4580 template <
typename _Flags>
4581 _GLIBCXX_SIMD_ALWAYS_INLINE
4582 simd_mask(
const value_type* __mem, simd_mask __k, _Flags)
4585 _M_data = _Impl::_S_masked_load(_M_data, __k._M_data,
4586 _Flags::template _S_apply<simd_mask>(__mem));
4591 template <
typename _Flags>
4592 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4593 copy_from(
const value_type* __mem, _Flags)
4594 { _M_data = _Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem)); }
4598 template <
typename _Flags>
4599 _GLIBCXX_SIMD_ALWAYS_INLINE
void
4600 copy_to(value_type* __mem, _Flags)
const
4601 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4605 _GLIBCXX_SIMD_ALWAYS_INLINE reference
4606 operator[](
size_t __i)
4609 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i, size() - 1);
4610 return {_M_data, int(__i)};
4613 _GLIBCXX_SIMD_ALWAYS_INLINE value_type
4614 operator[](
size_t __i)
const
4617 __invoke_ub(
"Subscript %d is out of range [0, %d]", __i, size() - 1);
4618 if constexpr (__is_scalar_abi<_Abi>())
4621 return static_cast<bool>(_M_data[__i]);
4626 _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask
4628 {
return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4632 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4634 template <
typename _Up,
typename _A2,
4635 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4636 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4637 operator&&(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4639 return {__private_init,
4640 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4643 template <
typename _Up,
typename _A2,
4644 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4645 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4646 operator||(
const simd_mask& __x,
const simd_mask<_Up, _A2>& __y)
4648 return {__private_init,
4649 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4653 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4654 operator&&(
const simd_mask& __x,
const simd_mask& __y)
4655 {
return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)}; }
4657 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4658 operator||(
const simd_mask& __x,
const simd_mask& __y)
4659 {
return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)}; }
4661 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4662 operator&(
const simd_mask& __x,
const simd_mask& __y)
4663 {
return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4665 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4666 operator|(
const simd_mask& __x,
const simd_mask& __y)
4667 {
return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4669 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask
4670 operator^(
const simd_mask& __x,
const simd_mask& __y)
4671 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4673 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4674 operator&=(simd_mask& __x,
const simd_mask& __y)
4676 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4680 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4681 operator|=(simd_mask& __x,
const simd_mask& __y)
4683 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4687 _GLIBCXX_SIMD_ALWAYS_INLINE
friend simd_mask&
4688 operator^=(simd_mask& __x,
const simd_mask& __y)
4690 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4696 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4697 operator==(
const simd_mask& __x,
const simd_mask& __y)
4698 {
return !operator!=(__x, __y); }
4700 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4701 operator!=(
const simd_mask& __x,
const simd_mask& __y)
4702 {
return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4706 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4707 simd_mask(_PrivateInit,
typename _Traits::_MaskMember __init)
4708 : _M_data(__init) {}
4712 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4713 _GLIBCXX_SIMD_INTRINSIC
constexpr
4714 simd_mask(_PrivateInit, _Fp&& __gen)
4717 __execute_n_times<size()>([&](
auto __i)
constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4718 _Impl::_S_set(_M_data, __i, __gen(__i));
4724 _GLIBCXX_SIMD_INTRINSIC simd_mask(_BitsetInit, bitset<size()> __init)
4725 : _M_data(_Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4736 template <
typename _Up,
typename _A2,
4737 typename = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4738 _GLIBCXX_SIMD_ALWAYS_INLINE
4739 operator simd_mask<_Up, _A2>() &&
4741 using namespace std::experimental::__proposed;
4742 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4745 const simd_mask<_Tp, _Abi>& _M_data;
4748 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4754 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4755 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4756 operator?:(
const simd_mask& __k,
const simd_mask& __where_true,
4757 const simd_mask& __where_false)
4759 auto __ret = __where_false;
4760 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4764 template <
typename _U1,
typename _U2,
4765 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4767 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4768 is_convertible<simd_mask, typename _Rp::mask_type>>>>
4769 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend _Rp
4770 operator?:(
const simd_mask& __k,
const _U1& __where_true,
4771 const _U2& __where_false)
4773 _Rp __ret = __where_false;
4774 _Rp::_Impl::_S_masked_assign(
4775 __data(
static_cast<typename _Rp::mask_type
>(__k)), __data(__ret),
4776 __data(
static_cast<_Rp
>(__where_true)));
4780 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4781 template <
typename _Kp,
typename _Ak,
typename _Up,
typename _Au,
4783 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4784 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4785 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd_mask
4786 operator?:(
const simd_mask<_Kp, _Ak>& __k,
const simd_mask& __where_true,
4787 const simd_mask<_Up, _Au>& __where_false)
4789 simd_mask __ret = __where_false;
4790 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4791 __where_true._M_data);
4799 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
4800 _M_is_constprop()
const
4802 if constexpr (__is_scalar_abi<_Abi>())
4803 return __builtin_constant_p(_M_data);
4805 return _M_data._M_is_constprop();
4811 friend const auto& __data<_Tp, abi_type>(
const simd_mask&);
4812 friend auto& __data<_Tp, abi_type>(simd_mask&);
4813 alignas(_Traits::_S_mask_align) _MemberType _M_data;
4820template <
typename _Tp,
typename _Ap>
4821 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
4822 __data(
const simd_mask<_Tp, _Ap>& __x)
4823 {
return __x._M_data; }
4825template <
typename _Tp,
typename _Ap>
4826 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
4827 __data(simd_mask<_Tp, _Ap>& __x)
4828 {
return __x._M_data; }
4834template <
typename _Tp,
typename _Abi>
4835 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4836 all_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
4838 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4840 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4846 return _Abi::_MaskImpl::_S_all_of(__k);
4849template <
typename _Tp,
typename _Abi>
4850 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4851 any_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
4853 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4855 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4861 return _Abi::_MaskImpl::_S_any_of(__k);
4864template <
typename _Tp,
typename _Abi>
4865 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4866 none_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
4868 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4870 for (
size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4876 return _Abi::_MaskImpl::_S_none_of(__k);
4879template <
typename _Tp,
typename _Abi>
4880 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4881 some_of(
const simd_mask<_Tp, _Abi>& __k)
noexcept
4883 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4885 for (
size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4886 if (__k[__i] != __k[__i - 1])
4891 return _Abi::_MaskImpl::_S_some_of(__k);
4894template <
typename _Tp,
typename _Abi>
4895 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4896 popcount(
const simd_mask<_Tp, _Abi>& __k)
noexcept
4898 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4900 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4901 __k, [](
auto... __elements) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4902 return ((__elements != 0) + ...);
4904 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4907 return _Abi::_MaskImpl::_S_popcount(__k);
4910template <
typename _Tp,
typename _Abi>
4911 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4912 find_first_set(
const simd_mask<_Tp, _Abi>& __k)
4914 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4916 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4917 const size_t _Idx = __call_with_n_evaluations<_Np>(
4918 [](
auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4920 }, [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4921 return __k[__i] ? +__i : _Np;
4924 __invoke_ub(
"find_first_set(empty mask) is UB");
4925 if (__builtin_constant_p(_Idx))
4928 return _Abi::_MaskImpl::_S_find_first_set(__k);
4931template <
typename _Tp,
typename _Abi>
4932 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4933 find_last_set(
const simd_mask<_Tp, _Abi>& __k)
4935 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4937 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4938 const int _Idx = __call_with_n_evaluations<_Np>(
4939 [](
auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4941 }, [&](
auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4942 return __k[__i] ? int(__i) : -1;
4945 __invoke_ub(
"find_first_set(empty mask) is UB");
4946 if (__builtin_constant_p(_Idx))
4949 return _Abi::_MaskImpl::_S_find_last_set(__k);
4952_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4953all_of(_ExactBool __x)
noexcept
4956_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4957any_of(_ExactBool __x)
noexcept
4960_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4961none_of(_ExactBool __x)
noexcept
4964_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
bool
4965some_of(_ExactBool)
noexcept
4968_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4972_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4973find_first_set(_ExactBool)
4976_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
int
4977find_last_set(_ExactBool)
4984template <
typename _V,
typename _Tp,
typename _Abi,
bool>
4985 class _SimdIntOperators {};
4987template <
typename _V,
typename _Tp,
typename _Abi>
4988 class _SimdIntOperators<_V, _Tp, _Abi, true>
4990 using _Impl =
typename _SimdTraits<_Tp, _Abi>::_SimdImpl;
4992 _GLIBCXX_SIMD_INTRINSIC
const _V& __derived()
const
4993 {
return *
static_cast<const _V*
>(
this); }
4995 template <
typename _Up>
4996 _GLIBCXX_SIMD_INTRINSIC
static _GLIBCXX_SIMD_CONSTEXPR _V
4997 _S_make_derived(_Up&& __d)
4998 {
return {__private_init,
static_cast<_Up&&
>(__d)}; }
5001 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5003 operator%=(_V& __lhs,
const _V& __x)
5004 {
return __lhs = __lhs % __x; }
5006 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5008 operator&=(_V& __lhs,
const _V& __x)
5009 {
return __lhs = __lhs & __x; }
5011 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5013 operator|=(_V& __lhs,
const _V& __x)
5014 {
return __lhs = __lhs | __x; }
5016 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5018 operator^=(_V& __lhs,
const _V& __x)
5019 {
return __lhs = __lhs ^ __x; }
5021 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5023 operator<<=(_V& __lhs,
const _V& __x)
5024 {
return __lhs = __lhs << __x; }
5026 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5028 operator>>=(_V& __lhs,
const _V& __x)
5029 {
return __lhs = __lhs >> __x; }
5031 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5033 operator<<=(_V& __lhs,
int __x)
5034 {
return __lhs = __lhs << __x; }
5036 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5038 operator>>=(_V& __lhs,
int __x)
5039 {
return __lhs = __lhs >> __x; }
5041 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5043 operator%(
const _V& __x,
const _V& __y)
5045 return _SimdIntOperators::_S_make_derived(
5046 _Impl::_S_modulus(__data(__x), __data(__y)));
5049 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5053 return _SimdIntOperators::_S_make_derived(
5054 _Impl::_S_bit_and(__data(__x), __data(__y)));
5057 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5061 return _SimdIntOperators::_S_make_derived(
5062 _Impl::_S_bit_or(__data(__x), __data(__y)));
5065 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5069 return _SimdIntOperators::_S_make_derived(
5070 _Impl::_S_bit_xor(__data(__x), __data(__y)));
5073 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5077 return _SimdIntOperators::_S_make_derived(
5078 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
5081 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5085 return _SimdIntOperators::_S_make_derived(
5086 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
5089 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5094 __invoke_ub(
"The behavior is undefined if the right operand of a "
5095 "shift operation is negative. [expr.shift]\nA shift by "
5098 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5100 "The behavior is undefined if the right operand of a "
5101 "shift operation is greater than or equal to the width of the "
5102 "promoted left operand. [expr.shift]\nA shift by %d was requested",
5104 return _SimdIntOperators::_S_make_derived(
5105 _Impl::_S_bit_shift_left(__data(__x), __y));
5108 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend
5114 "The behavior is undefined if the right operand of a shift "
5115 "operation is negative. [expr.shift]\nA shift by %d was requested",
5117 if (
size_t(__y) >=
sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5119 "The behavior is undefined if the right operand of a shift "
5120 "operation is greater than or equal to the width of the promoted "
5121 "left operand. [expr.shift]\nA shift by %d was requested",
5123 return _SimdIntOperators::_S_make_derived(
5124 _Impl::_S_bit_shift_right(__data(__x), __y));
5128 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5131 {
return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
5138template <
typename _Tp,
typename _Abi>
5139 class simd :
public _SimdIntOperators<
5140 simd<_Tp, _Abi>, _Tp, _Abi,
5141 conjunction<is_integral<_Tp>,
5142 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
5143 public _SimdTraits<_Tp, _Abi>::_SimdBase
5145 using _Traits = _SimdTraits<_Tp, _Abi>;
5146 using _MemberType =
typename _Traits::_SimdMember;
5147 using _CastType =
typename _Traits::_SimdCastType;
5148 static constexpr _Tp* _S_type_tag =
nullptr;
5149 friend typename _Traits::_SimdBase;
5152 using _Impl =
typename _Traits::_SimdImpl;
5154 friend _SimdIntOperators<simd, _Tp, _Abi, true>;
5156 using value_type = _Tp;
5157 using reference = _SmartReference<_MemberType, _Impl, value_type>;
5158 using mask_type = simd_mask<_Tp, _Abi>;
5159 using abi_type = _Abi;
5161 static constexpr size_t size()
5162 {
return __size_or_zero_v<_Tp, _Abi>; }
5164 _GLIBCXX_SIMD_CONSTEXPR simd() =
default;
5165 _GLIBCXX_SIMD_CONSTEXPR simd(
const simd&) =
default;
5166 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
5167 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
5168 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
5171 template <typename _Up,
5172 typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>,
bool>>>
5173 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5174 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
5176 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
5181 template <
typename _Up>
5182 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5183 simd(
const simd<_Up, simd_abi::fixed_size<size()>>& __x,
5186 is_same<simd_abi::fixed_size<size()>, abi_type>,
5187 negation<__is_narrowing_conversion<_Up, value_type>>,
5188 __converts_to_higher_integer_rank<_Up, value_type>>::value,
5190 : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5193#ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5194 template <
typename _Up,
typename _A2,
5195 typename =
decltype(static_simd_cast<simd>(
5196 declval<
const simd<_Up, _A2>&>()))>
5197 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5198 simd(
const simd<_Up, _A2>& __x)
5199 : simd(static_simd_cast<simd>(__x)) {}
5203 template <
typename _Fp>
5204 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5205 simd(_Fp&& __gen, _ValuePreservingOrInt<
decltype(declval<_Fp>()(
5206 declval<_SizeConstant<0>&>())),
5207 value_type>* =
nullptr)
5208 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5211 template <
typename _Up,
typename _Flags>
5212 _GLIBCXX_SIMD_ALWAYS_INLINE
5213 simd(
const _Up* __mem, _Flags)
5215 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5219 template <
typename _Up,
typename _Flags>
5220 _GLIBCXX_SIMD_ALWAYS_INLINE
void
5221 copy_from(
const _Vectorizable<_Up>* __mem, _Flags)
5223 _M_data =
static_cast<decltype(_M_data)
>(
5224 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5228 template <
typename _Up,
typename _Flags>
5229 _GLIBCXX_SIMD_ALWAYS_INLINE
void
5230 copy_to(_Vectorizable<_Up>* __mem, _Flags)
const
5232 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5237 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5238 operator[](
size_t __i)
5239 {
return {_M_data, int(__i)}; }
5241 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5242 operator[]([[maybe_unused]]
size_t __i)
const
5244 if constexpr (__is_scalar_abi<_Abi>())
5246 _GLIBCXX_DEBUG_ASSERT(__i == 0);
5250 return _M_data[__i];
5254 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5257 _Impl::_S_increment(_M_data);
5261 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5265 _Impl::_S_increment(_M_data);
5269 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5272 _Impl::_S_decrement(_M_data);
5276 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5280 _Impl::_S_decrement(_M_data);
5285 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5287 {
return {__private_init, _Impl::_S_negate(_M_data)}; }
5289 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5293 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5295 {
return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5298 _GLIBCXX_SIMD_ALWAYS_INLINE
explicit _GLIBCXX_SIMD_CONSTEXPR
5299 simd(_CastType __init) : _M_data(__init) {}
5302 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5303 operator+=(simd& __lhs,
const simd& __x)
5304 {
return __lhs = __lhs + __x; }
5306 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5307 operator-=(simd& __lhs,
const simd& __x)
5308 {
return __lhs = __lhs - __x; }
5310 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5311 operator*=(simd& __lhs,
const simd& __x)
5312 {
return __lhs = __lhs * __x; }
5314 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd&
5315 operator/=(simd& __lhs,
const simd& __x)
5316 {
return __lhs = __lhs / __x; }
5319 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5320 operator+(
const simd& __x,
const simd& __y)
5321 {
return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5323 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5324 operator-(
const simd& __x,
const simd& __y)
5325 {
return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5327 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5328 operator*(
const simd& __x,
const simd& __y)
5329 {
return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5331 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5332 operator/(
const simd& __x,
const simd& __y)
5333 {
return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5336 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5337 operator==(
const simd& __x,
const simd& __y)
5338 {
return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5340 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5341 operator!=(
const simd& __x,
const simd& __y)
5343 return simd::_S_make_mask(
5344 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5347 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5348 operator<(
const simd& __x,
const simd& __y)
5349 {
return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5351 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5352 operator<=(
const simd& __x,
const simd& __y)
5354 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5357 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5358 operator>(
const simd& __x,
const simd& __y)
5359 {
return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5361 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend mask_type
5362 operator>=(
const simd& __x,
const simd& __y)
5364 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5368#ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5369 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
friend simd
5370 operator?:(
const mask_type& __k,
const simd& __where_true,
5371 const simd& __where_false)
5373 auto __ret = __where_false;
5374 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5382 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5383 simd(_PrivateInit,
const _MemberType& __init)
5384 : _M_data(__init) {}
5387 _GLIBCXX_SIMD_INTRINSIC
5388 simd(_BitsetInit, bitset<size()> __init) : _M_data()
5389 { where(mask_type(__bitset_init, __init), *
this) = ~*
this; }
5391 _GLIBCXX_SIMD_INTRINSIC
constexpr bool
5392 _M_is_constprop()
const
5394 if constexpr (__is_scalar_abi<_Abi>())
5395 return __builtin_constant_p(_M_data);
5397 return _M_data._M_is_constprop();
5401 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
static mask_type
5402 _S_make_mask(
typename mask_type::_MemberType __k)
5403 {
return {__private_init, __k}; }
5405 friend const auto& __data<value_type, abi_type>(
const simd&);
5406 friend auto& __data<value_type, abi_type>(simd&);
5407 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5413template <
typename _Tp,
typename _Ap>
5414 _GLIBCXX_SIMD_INTRINSIC
constexpr const auto&
5415 __data(
const simd<_Tp, _Ap>& __x)
5416 {
return __x._M_data; }
5418template <
typename _Tp,
typename _Ap>
5419 _GLIBCXX_SIMD_INTRINSIC
constexpr auto&
5420 __data(simd<_Tp, _Ap>& __x)
5421 {
return __x._M_data; }
5424namespace __float_bitwise_operators {
5425template <
typename _Tp,
typename _Ap>
5426 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5427 operator^(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5428 {
return {__private_init, _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))}; }
5430template <
typename _Tp,
typename _Ap>
5431 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5432 operator|(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5433 {
return {__private_init, _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))}; }
5435template <
typename _Tp,
typename _Ap>
5436 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5437 operator&(
const simd<_Tp, _Ap>& __a,
const simd<_Tp, _Ap>& __b)
5438 {
return {__private_init, _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))}; }
5440template <
typename _Tp,
typename _Ap>
5441 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5442 enable_if_t<is_floating_point_v<_Tp>, simd<_Tp, _Ap>>
5443 operator~(
const simd<_Tp, _Ap>& __a)
5444 {
return {__private_init, _Ap::_SimdImpl::_S_complement(__data(__a))}; }
5449_GLIBCXX_SIMD_END_NAMESPACE
constexpr _If_is_unsigned_integer< _Tp, int > popcount(_Tp __x) noexcept
The number of bits set in x.
constexpr complex< _Tp > operator*(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x times y.
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
constexpr complex< _Tp > operator/(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x divided by y.
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
void void_t
A metafunction that always yields void, used for detecting valid types.
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
Create a tuple containing all elements from multiple tuple-like objects.
auto declval() noexcept -> decltype(__declval< _Tp >(0))
constexpr tuple< typename __decay_and_strip< _Elements >::__type... > make_tuple(_Elements &&... __args)
Create a tuple containing copies of the arguments.
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
constexpr bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
constexpr bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
constexpr bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.