libstdc++
simd.h
1// Definition of the public simd interfaces -*- C++ -*-
2
3// Copyright (C) 2020-2023 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library. This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.
10
11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14// GNU General Public License for more details.
15
16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.
19
20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23// <http://www.gnu.org/licenses/>.
24
25#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26#define _GLIBCXX_EXPERIMENTAL_SIMD_H
27
28#if __cplusplus >= 201703L
29
30#include "simd_detail.h"
31#include "numeric_traits.h"
32#include <bit>
33#include <bitset>
34#ifdef _GLIBCXX_DEBUG_UB
35#include <cstdio> // for stderr
36#endif
37#include <cstring>
38#include <cmath>
39#include <functional>
40#include <iosfwd>
41#include <utility>
42
43#if _GLIBCXX_SIMD_X86INTRIN
44#include <x86intrin.h>
45#elif _GLIBCXX_SIMD_HAVE_NEON
46#include <arm_neon.h>
47#endif
48
49/** @ingroup ts_simd
50 * @{
51 */
52/* There are several closely related types, with the following naming
53 * convention:
54 * _Tp: vectorizable (arithmetic) type (or any type)
55 * _TV: __vector_type_t<_Tp, _Np>
56 * _TW: _SimdWrapper<_Tp, _Np>
57 * _TI: __intrinsic_type_t<_Tp, _Np>
58 * _TVT: _VectorTraits<_TV> or _VectorTraits<_TW>
59 * If one additional type is needed use _U instead of _T.
60 * Otherwise use _T\d, _TV\d, _TW\d, TI\d, _TVT\d.
61 *
62 * More naming conventions:
63 * _Ap or _Abi: An ABI tag from the simd_abi namespace
64 * _Ip: often used for integer types with sizeof(_Ip) == sizeof(_Tp),
65 * _IV, _IW as for _TV, _TW
66 * _Np: number of elements (not bytes)
67 * _Bytes: number of bytes
68 *
69 * Variable names:
70 * __k: mask object (vector- or bitmask)
71 */
72_GLIBCXX_SIMD_BEGIN_NAMESPACE
73
74#if !_GLIBCXX_SIMD_X86INTRIN
75using __m128 [[__gnu__::__vector_size__(16)]] = float;
76using __m128d [[__gnu__::__vector_size__(16)]] = double;
77using __m128i [[__gnu__::__vector_size__(16)]] = long long;
78using __m256 [[__gnu__::__vector_size__(32)]] = float;
79using __m256d [[__gnu__::__vector_size__(32)]] = double;
80using __m256i [[__gnu__::__vector_size__(32)]] = long long;
81using __m512 [[__gnu__::__vector_size__(64)]] = float;
82using __m512d [[__gnu__::__vector_size__(64)]] = double;
83using __m512i [[__gnu__::__vector_size__(64)]] = long long;
84#endif
85
86namespace simd_abi {
87// simd_abi forward declarations {{{
88// implementation details:
89struct _Scalar;
90
91template <int _Np>
92 struct _Fixed;
93
94// There are two major ABIs that appear on different architectures.
95// Both have non-boolean values packed into an N Byte register
96// -> #elements = N / sizeof(T)
97// Masks differ:
98// 1. Use value vector registers for masks (all 0 or all 1)
99// 2. Use bitmasks (mask registers) with one bit per value in the corresponding
100// value vector
101//
102// Both can be partially used, masking off the rest when doing horizontal
103// operations or operations that can trap (e.g. FP_INVALID or integer division
104// by 0). This is encoded as the number of used bytes.
105template <int _UsedBytes>
106 struct _VecBuiltin;
107
108template <int _UsedBytes>
109 struct _VecBltnBtmsk;
110
111template <typename _Tp, int _Np>
112 using _VecN = _VecBuiltin<sizeof(_Tp) * _Np>;
113
114template <int _UsedBytes = 16>
115 using _Sse = _VecBuiltin<_UsedBytes>;
116
117template <int _UsedBytes = 32>
118 using _Avx = _VecBuiltin<_UsedBytes>;
119
120template <int _UsedBytes = 64>
121 using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
122
123template <int _UsedBytes = 16>
124 using _Neon = _VecBuiltin<_UsedBytes>;
125
126// implementation-defined:
127using __sse = _Sse<>;
128using __avx = _Avx<>;
129using __avx512 = _Avx512<>;
130using __neon = _Neon<>;
131using __neon128 = _Neon<16>;
132using __neon64 = _Neon<8>;
133
134// standard:
135template <typename _Tp, size_t _Np, typename...>
136 struct deduce;
137
138template <int _Np>
139 using fixed_size = _Fixed<_Np>;
140
141using scalar = _Scalar;
142
143// }}}
144} // namespace simd_abi
145// forward declarations is_simd(_mask), simd(_mask), simd_size {{{
146template <typename _Tp>
147 struct is_simd;
148
149template <typename _Tp>
150 struct is_simd_mask;
151
152template <typename _Tp, typename _Abi>
153 class simd;
154
155template <typename _Tp, typename _Abi>
156 class simd_mask;
157
158template <typename _Tp, typename _Abi>
159 struct simd_size;
160
161// }}}
162// load/store flags {{{
163struct element_aligned_tag
164{
165 template <typename _Tp, typename _Up = typename _Tp::value_type>
166 static constexpr size_t _S_alignment = alignof(_Up);
167
168 template <typename _Tp, typename _Up>
169 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
170 _S_apply(_Up* __ptr)
171 { return __ptr; }
172};
173
174struct vector_aligned_tag
175{
176 template <typename _Tp, typename _Up = typename _Tp::value_type>
177 static constexpr size_t _S_alignment
178 = std::__bit_ceil(sizeof(_Up) * _Tp::size());
179
180 template <typename _Tp, typename _Up>
181 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
182 _S_apply(_Up* __ptr)
183 {
184 return static_cast<_Up*>(
185 __builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>));
186 }
187};
188
189template <size_t _Np> struct overaligned_tag
190{
191 template <typename _Tp, typename _Up = typename _Tp::value_type>
192 static constexpr size_t _S_alignment = _Np;
193
194 template <typename _Tp, typename _Up>
195 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
196 _S_apply(_Up* __ptr)
197 { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); }
198};
199
200inline constexpr element_aligned_tag element_aligned = {};
201
202inline constexpr vector_aligned_tag vector_aligned = {};
203
204template <size_t _Np>
205 inline constexpr overaligned_tag<_Np> overaligned = {};
206
207// }}}
208template <size_t _Xp>
209 using _SizeConstant = integral_constant<size_t, _Xp>;
210// constexpr feature detection{{{
211constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
212constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
213constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
214constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
215constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
216constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
217constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
218constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
219constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
220constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
221constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
222constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
223constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
224constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
225constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
226constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
227constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
228constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
229constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
230constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
231constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
232constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
233constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
234constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
235constexpr inline bool __have_avx512bitalg = _GLIBCXX_SIMD_HAVE_AVX512BITALG;
236constexpr inline bool __have_avx512vbmi2 = _GLIBCXX_SIMD_HAVE_AVX512VBMI2;
237constexpr inline bool __have_avx512vbmi = _GLIBCXX_SIMD_HAVE_AVX512VBMI;
238constexpr inline bool __have_avx512ifma = _GLIBCXX_SIMD_HAVE_AVX512IFMA;
239constexpr inline bool __have_avx512cd = _GLIBCXX_SIMD_HAVE_AVX512CD;
240constexpr inline bool __have_avx512vnni = _GLIBCXX_SIMD_HAVE_AVX512VNNI;
241constexpr inline bool __have_avx512vpopcntdq = _GLIBCXX_SIMD_HAVE_AVX512VPOPCNTDQ;
242constexpr inline bool __have_avx512vp2intersect = _GLIBCXX_SIMD_HAVE_AVX512VP2INTERSECT;
243
244constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
245constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
246constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
247constexpr inline bool __support_neon_float =
248#if defined __GCC_IEC_559
249 __GCC_IEC_559 == 0;
250#elif defined __FAST_MATH__
251 true;
252#else
253 false;
254#endif
255
256#ifdef _ARCH_PWR10
257constexpr inline bool __have_power10vec = true;
258#else
259constexpr inline bool __have_power10vec = false;
260#endif
261#ifdef __POWER9_VECTOR__
262constexpr inline bool __have_power9vec = true;
263#else
264constexpr inline bool __have_power9vec = false;
265#endif
266#if defined __POWER8_VECTOR__
267constexpr inline bool __have_power8vec = true;
268#else
269constexpr inline bool __have_power8vec = __have_power9vec;
270#endif
271#if defined __VSX__
272constexpr inline bool __have_power_vsx = true;
273#else
274constexpr inline bool __have_power_vsx = __have_power8vec;
275#endif
276#if defined __ALTIVEC__
277constexpr inline bool __have_power_vmx = true;
278#else
279constexpr inline bool __have_power_vmx = __have_power_vsx;
280#endif
281
282// }}}
283
284namespace __detail
285{
286#ifdef math_errhandling
287 // Determines _S_handle_fpexcept from math_errhandling if it is defined and expands to a constant
288 // expression. math_errhandling may expand to an extern symbol, in which case a constexpr value
289 // must be guessed.
290 template <int = math_errhandling>
291 constexpr bool __handle_fpexcept_impl(int)
292 { return math_errhandling & MATH_ERREXCEPT; }
293#endif
294
295 // Fallback if math_errhandling doesn't work: with fast-math assume floating-point exceptions are
296 // ignored, otherwise implement correct exception behavior.
297 constexpr bool __handle_fpexcept_impl(float)
298 {
299#if defined __FAST_MATH__
300 return false;
301#else
302 return true;
303#endif
304 }
305
306 /// True if math functions must raise floating-point exceptions as specified by C17.
307 static constexpr bool _S_handle_fpexcept = __handle_fpexcept_impl(0);
308
309 constexpr std::uint_least64_t
310 __floating_point_flags()
311 {
312 std::uint_least64_t __flags = 0;
313 if constexpr (_S_handle_fpexcept)
314 __flags |= 1;
315#ifdef __FAST_MATH__
316 __flags |= 1 << 1;
317#elif __FINITE_MATH_ONLY__
318 __flags |= 2 << 1;
319#elif __GCC_IEC_559 < 2
320 __flags |= 3 << 1;
321#endif
322 __flags |= (__FLT_EVAL_METHOD__ + 1) << 3;
323 return __flags;
324 }
325
326 constexpr std::uint_least64_t
327 __machine_flags()
328 {
329 if constexpr (__have_mmx || __have_sse)
330 return __have_mmx
331 | (__have_sse << 1)
332 | (__have_sse2 << 2)
333 | (__have_sse3 << 3)
334 | (__have_ssse3 << 4)
335 | (__have_sse4_1 << 5)
336 | (__have_sse4_2 << 6)
337 | (__have_xop << 7)
338 | (__have_avx << 8)
339 | (__have_avx2 << 9)
340 | (__have_bmi << 10)
341 | (__have_bmi2 << 11)
342 | (__have_lzcnt << 12)
343 | (__have_sse4a << 13)
344 | (__have_fma << 14)
345 | (__have_fma4 << 15)
346 | (__have_f16c << 16)
347 | (__have_popcnt << 17)
348 | (__have_avx512f << 18)
349 | (__have_avx512dq << 19)
350 | (__have_avx512vl << 20)
351 | (__have_avx512bw << 21)
352 | (__have_avx512bitalg << 22)
353 | (__have_avx512vbmi2 << 23)
354 | (__have_avx512vbmi << 24)
355 | (__have_avx512ifma << 25)
356 | (__have_avx512cd << 26)
357 | (__have_avx512vnni << 27)
358 | (__have_avx512vpopcntdq << 28)
359 | (__have_avx512vp2intersect << 29);
360 else if constexpr (__have_neon)
361 return __have_neon
362 | (__have_neon_a32 << 1)
363 | (__have_neon_a64 << 2)
364 | (__have_neon_a64 << 2)
365 | (__support_neon_float << 3);
366 else if constexpr (__have_power_vmx)
367 return __have_power_vmx
368 | (__have_power_vsx << 1)
369 | (__have_power8vec << 2)
370 | (__have_power9vec << 3)
371 | (__have_power10vec << 4);
372 else
373 return 0;
374 }
375
376 namespace
377 {
378 struct _OdrEnforcer {};
379 }
380
381 template <std::uint_least64_t...>
382 struct _MachineFlagsTemplate {};
383
384 /**@internal
385 * Use this type as default template argument to all function templates that
386 * are not declared always_inline. It ensures, that a function
387 * specialization, which the compiler decides not to inline, has a unique symbol
388 * (_OdrEnforcer) or a symbol matching the machine/architecture flags
389 * (_MachineFlagsTemplate). This helps to avoid ODR violations in cases where
390 * users link TUs compiled with different flags. This is especially important
391 * for using simd in libraries.
392 */
393 using __odr_helper
394 = conditional_t<__machine_flags() == 0, _OdrEnforcer,
395 _MachineFlagsTemplate<__machine_flags(), __floating_point_flags()>>;
396
397 struct _Minimum
398 {
399 template <typename _Tp>
400 _GLIBCXX_SIMD_INTRINSIC constexpr
401 _Tp
402 operator()(_Tp __a, _Tp __b) const
403 {
404 using std::min;
405 return min(__a, __b);
406 }
407 };
408
409 struct _Maximum
410 {
411 template <typename _Tp>
412 _GLIBCXX_SIMD_INTRINSIC constexpr
413 _Tp
414 operator()(_Tp __a, _Tp __b) const
415 {
416 using std::max;
417 return max(__a, __b);
418 }
419 };
420} // namespace __detail
421
422// unrolled/pack execution helpers
423// __execute_n_times{{{
424template <typename _Fp, size_t... _I>
425 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
426 void
427 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
428 { ((void)__f(_SizeConstant<_I>()), ...); }
429
430template <typename _Fp>
431 _GLIBCXX_SIMD_INTRINSIC constexpr void
432 __execute_on_index_sequence(_Fp&&, index_sequence<>)
433 { }
434
435template <size_t _Np, typename _Fp>
436 _GLIBCXX_SIMD_INTRINSIC constexpr void
437 __execute_n_times(_Fp&& __f)
438 {
439 __execute_on_index_sequence(static_cast<_Fp&&>(__f),
440 make_index_sequence<_Np>{});
441 }
442
443// }}}
444// __generate_from_n_evaluations{{{
445template <typename _R, typename _Fp, size_t... _I>
446 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
447 _R
448 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
449 { return _R{__f(_SizeConstant<_I>())...}; }
450
451template <size_t _Np, typename _R, typename _Fp>
452 _GLIBCXX_SIMD_INTRINSIC constexpr _R
453 __generate_from_n_evaluations(_Fp&& __f)
454 {
455 return __execute_on_index_sequence_with_return<_R>(
456 static_cast<_Fp&&>(__f), make_index_sequence<_Np>{});
457 }
458
459// }}}
460// __call_with_n_evaluations{{{
461template <size_t... _I, typename _F0, typename _FArgs>
462 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
463 auto
464 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
465 { return __f0(__fargs(_SizeConstant<_I>())...); }
466
467template <size_t _Np, typename _F0, typename _FArgs>
468 _GLIBCXX_SIMD_INTRINSIC constexpr auto
469 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
470 {
471 return __call_with_n_evaluations(make_index_sequence<_Np>{},
472 static_cast<_F0&&>(__f0),
473 static_cast<_FArgs&&>(__fargs));
474 }
475
476// }}}
477// __call_with_subscripts{{{
478template <size_t _First = 0, size_t... _It, typename _Tp, typename _Fp>
479 [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
480 auto
481 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
482 { return __fun(__x[_First + _It]...); }
483
484template <size_t _Np, size_t _First = 0, typename _Tp, typename _Fp>
485 _GLIBCXX_SIMD_INTRINSIC constexpr auto
486 __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
487 {
488 return __call_with_subscripts<_First>(static_cast<_Tp&&>(__x),
489 make_index_sequence<_Np>(),
490 static_cast<_Fp&&>(__fun));
491 }
492
493// }}}
494
495// vvv ---- type traits ---- vvv
496// integer type aliases{{{
497using _UChar = unsigned char;
498using _SChar = signed char;
499using _UShort = unsigned short;
500using _UInt = unsigned int;
501using _ULong = unsigned long;
502using _ULLong = unsigned long long;
503using _LLong = long long;
504
505//}}}
506// __first_of_pack{{{
507template <typename _T0, typename...>
508 struct __first_of_pack
509 { using type = _T0; };
510
511template <typename... _Ts>
512 using __first_of_pack_t = typename __first_of_pack<_Ts...>::type;
513
514//}}}
515// __value_type_or_identity_t {{{
516template <typename _Tp>
517 typename _Tp::value_type
518 __value_type_or_identity_impl(int);
519
520template <typename _Tp>
521 _Tp
522 __value_type_or_identity_impl(float);
523
524template <typename _Tp>
525 using __value_type_or_identity_t
526 = decltype(__value_type_or_identity_impl<_Tp>(int()));
527
528// }}}
529// __is_vectorizable {{{
530template <typename _Tp>
531 struct __is_vectorizable : public is_arithmetic<_Tp> {};
532
533template <>
534 struct __is_vectorizable<bool> : public false_type {};
535
536template <typename _Tp>
537 inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
538
539// Deduces to a vectorizable type
540template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
541 using _Vectorizable = _Tp;
542
543// }}}
544// _LoadStorePtr / __is_possible_loadstore_conversion {{{
545template <typename _Ptr, typename _ValueType>
546 struct __is_possible_loadstore_conversion
547 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
548
549template <>
550 struct __is_possible_loadstore_conversion<bool, bool> : true_type {};
551
552// Deduces to a type allowed for load/store with the given value type.
553template <typename _Ptr, typename _ValueType,
554 typename = enable_if_t<
555 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
556 using _LoadStorePtr = _Ptr;
557
558// }}}
559// __is_bitmask{{{
560template <typename _Tp, typename = void_t<>>
561 struct __is_bitmask : false_type {};
562
563template <typename _Tp>
564 inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
565
566// the __mmaskXX case:
567template <typename _Tp>
568 struct __is_bitmask<_Tp,
569 void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>>
570 : true_type {};
571
572// }}}
573// __int_for_sizeof{{{
574#pragma GCC diagnostic push
575#pragma GCC diagnostic ignored "-Wpedantic"
576template <size_t _Bytes>
577 constexpr auto
578 __int_for_sizeof()
579 {
580 if constexpr (_Bytes == sizeof(int))
581 return int();
582 #ifdef __clang__
583 else if constexpr (_Bytes == sizeof(char))
584 return char();
585 #else
586 else if constexpr (_Bytes == sizeof(_SChar))
587 return _SChar();
588 #endif
589 else if constexpr (_Bytes == sizeof(short))
590 return short();
591 #ifndef __clang__
592 else if constexpr (_Bytes == sizeof(long))
593 return long();
594 #endif
595 else if constexpr (_Bytes == sizeof(_LLong))
596 return _LLong();
597 #ifdef __SIZEOF_INT128__
598 else if constexpr (_Bytes == sizeof(__int128))
599 return __int128();
600 #endif // __SIZEOF_INT128__
601 else if constexpr (_Bytes % sizeof(int) == 0)
602 {
603 constexpr size_t _Np = _Bytes / sizeof(int);
604 struct _Ip
605 {
606 int _M_data[_Np];
607
608 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
609 operator&(_Ip __rhs) const
610 {
611 return __generate_from_n_evaluations<_Np, _Ip>(
612 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
613 return __rhs._M_data[__i] & _M_data[__i];
614 });
615 }
616
617 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
618 operator|(_Ip __rhs) const
619 {
620 return __generate_from_n_evaluations<_Np, _Ip>(
621 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
622 return __rhs._M_data[__i] | _M_data[__i];
623 });
624 }
625
626 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
627 operator^(_Ip __rhs) const
628 {
629 return __generate_from_n_evaluations<_Np, _Ip>(
630 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
631 return __rhs._M_data[__i] ^ _M_data[__i];
632 });
633 }
634
635 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
636 operator~() const
637 {
638 return __generate_from_n_evaluations<_Np, _Ip>(
639 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return ~_M_data[__i]; });
640 }
641 };
642 return _Ip{};
643 }
644 else
645 static_assert(_Bytes != _Bytes, "this should be unreachable");
646 }
647#pragma GCC diagnostic pop
648
649template <typename _Tp>
650 using __int_for_sizeof_t = decltype(__int_for_sizeof<sizeof(_Tp)>());
651
652template <size_t _Np>
653 using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
654
655// }}}
656// __is_fixed_size_abi{{{
657template <typename _Tp>
658 struct __is_fixed_size_abi : false_type {};
659
660template <int _Np>
661 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> : true_type {};
662
663template <typename _Tp>
664 inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
665
666// }}}
667// __is_scalar_abi {{{
668template <typename _Abi>
669 constexpr bool
670 __is_scalar_abi()
671 { return is_same_v<simd_abi::scalar, _Abi>; }
672
673// }}}
674// __abi_bytes_v {{{
675template <template <int> class _Abi, int _Bytes>
676 constexpr int
677 __abi_bytes_impl(_Abi<_Bytes>*)
678 { return _Bytes; }
679
680template <typename _Tp>
681 constexpr int
682 __abi_bytes_impl(_Tp*)
683 { return -1; }
684
685template <typename _Abi>
686 inline constexpr int __abi_bytes_v
687 = __abi_bytes_impl(static_cast<_Abi*>(nullptr));
688
689// }}}
690// __is_builtin_bitmask_abi {{{
691template <typename _Abi>
692 constexpr bool
693 __is_builtin_bitmask_abi()
694 { return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
695
696// }}}
697// __is_sse_abi {{{
698template <typename _Abi>
699 constexpr bool
700 __is_sse_abi()
701 {
702 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
703 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
704 }
705
706// }}}
707// __is_avx_abi {{{
708template <typename _Abi>
709 constexpr bool
710 __is_avx_abi()
711 {
712 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
713 return _Bytes > 16 && _Bytes <= 32
714 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
715 }
716
717// }}}
718// __is_avx512_abi {{{
719template <typename _Abi>
720 constexpr bool
721 __is_avx512_abi()
722 {
723 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
724 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
725 }
726
727// }}}
728// __is_neon_abi {{{
729template <typename _Abi>
730 constexpr bool
731 __is_neon_abi()
732 {
733 constexpr auto _Bytes = __abi_bytes_v<_Abi>;
734 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
735 }
736
737// }}}
738// __make_dependent_t {{{
739template <typename, typename _Up>
740 struct __make_dependent
741 { using type = _Up; };
742
743template <typename _Tp, typename _Up>
744 using __make_dependent_t = typename __make_dependent<_Tp, _Up>::type;
745
746// }}}
747// ^^^ ---- type traits ---- ^^^
748
749// __invoke_ub{{{
750template <typename... _Args>
751 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE void
752 __invoke_ub([[maybe_unused]] const char* __msg,
753 [[maybe_unused]] const _Args&... __args)
754 {
755#ifdef _GLIBCXX_DEBUG_UB
756 __builtin_fprintf(stderr, __msg, __args...);
757 __builtin_trap();
758#else
759 __builtin_unreachable();
760#endif
761 }
762
763// }}}
764// __assert_unreachable{{{
765template <typename _Tp>
766 struct __assert_unreachable
767 { static_assert(!is_same_v<_Tp, _Tp>, "this should be unreachable"); };
768
769// }}}
770// __size_or_zero_v {{{
771template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
772 constexpr size_t
773 __size_or_zero_dispatch(int)
774 { return _Np; }
775
776template <typename _Tp, typename _Ap>
777 constexpr size_t
778 __size_or_zero_dispatch(float)
779 { return 0; }
780
781template <typename _Tp, typename _Ap>
782 inline constexpr size_t __size_or_zero_v
783 = __size_or_zero_dispatch<_Tp, _Ap>(0);
784
785// }}}
786// __div_roundup {{{
787inline constexpr size_t
788__div_roundup(size_t __a, size_t __b)
789{ return (__a + __b - 1) / __b; }
790
791// }}}
792// _ExactBool{{{
793class _ExactBool
794{
795 const bool _M_data;
796
797public:
798 _GLIBCXX_SIMD_INTRINSIC constexpr _ExactBool(bool __b) : _M_data(__b) {}
799
800 _ExactBool(int) = delete;
801
802 _GLIBCXX_SIMD_INTRINSIC constexpr operator bool() const { return _M_data; }
803};
804
805// }}}
806// __may_alias{{{
807/**@internal
808 * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an
809 * aliasing pointer. This adds the __may_alias attribute to _Tp (with compilers
810 * that support it).
811 */
812template <typename _Tp>
813 using __may_alias [[__gnu__::__may_alias__]] = _Tp;
814
815// }}}
816// _UnsupportedBase {{{
817// simd and simd_mask base for unsupported <_Tp, _Abi>
818struct _UnsupportedBase
819{
820 _UnsupportedBase() = delete;
821 _UnsupportedBase(const _UnsupportedBase&) = delete;
822 _UnsupportedBase& operator=(const _UnsupportedBase&) = delete;
823 ~_UnsupportedBase() = delete;
824};
825
826// }}}
827// _InvalidTraits {{{
828/**
829 * @internal
830 * Defines the implementation of __a given <_Tp, _Abi>.
831 *
832 * Implementations must ensure that only valid <_Tp, _Abi> instantiations are
833 * possible. Static assertions in the type definition do not suffice. It is
834 * important that SFINAE works.
835 */
836struct _InvalidTraits
837{
838 using _IsValid = false_type;
839 using _SimdBase = _UnsupportedBase;
840 using _MaskBase = _UnsupportedBase;
841
842 static constexpr size_t _S_full_size = 0;
843 static constexpr bool _S_is_partial = false;
844
845 static constexpr size_t _S_simd_align = 1;
846 struct _SimdImpl;
847 struct _SimdMember {};
848 struct _SimdCastType;
849
850 static constexpr size_t _S_mask_align = 1;
851 struct _MaskImpl;
852 struct _MaskMember {};
853 struct _MaskCastType;
854};
855
856// }}}
857// _SimdTraits {{{
858template <typename _Tp, typename _Abi, typename = void_t<>>
859 struct _SimdTraits : _InvalidTraits {};
860
861// }}}
862// __private_init, __bitset_init{{{
863/**
864 * @internal
865 * Tag used for private init constructor of simd and simd_mask
866 */
867inline constexpr struct _PrivateInit {} __private_init = {};
868
869inline constexpr struct _BitsetInit {} __bitset_init = {};
870
871// }}}
872// __is_narrowing_conversion<_From, _To>{{{
873template <typename _From, typename _To, bool = is_arithmetic_v<_From>,
874 bool = is_arithmetic_v<_To>>
875 struct __is_narrowing_conversion;
876
877// ignore "signed/unsigned mismatch" in the following trait.
878// The implicit conversions will do the right thing here.
879template <typename _From, typename _To>
880 struct __is_narrowing_conversion<_From, _To, true, true>
881 : public __bool_constant<(
882 __digits_v<_From> > __digits_v<_To>
883 || __finite_max_v<_From> > __finite_max_v<_To>
884 || __finite_min_v<_From> < __finite_min_v<_To>
885 || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
886
887template <typename _Tp>
888 struct __is_narrowing_conversion<_Tp, bool, true, true>
889 : public true_type {};
890
891template <>
892 struct __is_narrowing_conversion<bool, bool, true, true>
893 : public false_type {};
894
895template <typename _Tp>
896 struct __is_narrowing_conversion<_Tp, _Tp, true, true>
897 : public false_type {};
898
899template <typename _From, typename _To>
900 struct __is_narrowing_conversion<_From, _To, false, true>
901 : public negation<is_convertible<_From, _To>> {};
902
903// }}}
904// __converts_to_higher_integer_rank{{{
905template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))>
906 struct __converts_to_higher_integer_rank : public true_type {};
907
908// this may fail for char -> short if sizeof(char) == sizeof(short)
909template <typename _From, typename _To>
910 struct __converts_to_higher_integer_rank<_From, _To, false>
911 : public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
912
913// }}}
914// __data(simd/simd_mask) {{{
915template <typename _Tp, typename _Ap>
916 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
917 __data(const simd<_Tp, _Ap>& __x);
918
919template <typename _Tp, typename _Ap>
920 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
921 __data(simd<_Tp, _Ap>& __x);
922
923template <typename _Tp, typename _Ap>
924 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
925 __data(const simd_mask<_Tp, _Ap>& __x);
926
927template <typename _Tp, typename _Ap>
928 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
929 __data(simd_mask<_Tp, _Ap>& __x);
930
931// }}}
932// _SimdConverter {{{
933template <typename _FromT, typename _FromA, typename _ToT, typename _ToA,
934 typename = void>
935 struct _SimdConverter;
936
937template <typename _Tp, typename _Ap>
938 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
939 {
940 template <typename _Up>
941 _GLIBCXX_SIMD_INTRINSIC const _Up&
942 operator()(const _Up& __x)
943 { return __x; }
944 };
945
946// }}}
947// __to_value_type_or_member_type {{{
948template <typename _V>
949 _GLIBCXX_SIMD_INTRINSIC constexpr auto
950 __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x))
951 { return __data(__x); }
952
953template <typename _V>
954 _GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type&
955 __to_value_type_or_member_type(const typename _V::value_type& __x)
956 { return __x; }
957
958// }}}
959// __bool_storage_member_type{{{
960template <size_t _Size>
961 struct __bool_storage_member_type;
962
963template <size_t _Size>
964 using __bool_storage_member_type_t =
965 typename __bool_storage_member_type<_Size>::type;
966
967// }}}
968// _SimdTuple {{{
969// why not tuple?
970// 1. tuple gives no guarantee about the storage order, but I require
971// storage
972// equivalent to array<_Tp, _Np>
973// 2. direct access to the element type (first template argument)
974// 3. enforces equal element type, only different _Abi types are allowed
975template <typename _Tp, typename... _Abis>
976 struct _SimdTuple;
977
978//}}}
979// __fixed_size_storage_t {{{
980template <typename _Tp, int _Np>
981 struct __fixed_size_storage;
982
983template <typename _Tp, int _Np>
984 using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _Np>::type;
985
986// }}}
987// _SimdWrapper fwd decl{{{
988template <typename _Tp, size_t _Size, typename = void_t<>>
989 struct _SimdWrapper;
990
991template <typename _Tp>
992 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>;
993template <typename _Tp>
994 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>;
995template <typename _Tp>
996 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>;
997template <typename _Tp>
998 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>;
999
1000// }}}
1001// __is_simd_wrapper {{{
1002template <typename _Tp>
1003 struct __is_simd_wrapper : false_type {};
1004
1005template <typename _Tp, size_t _Np>
1006 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> : true_type {};
1007
1008template <typename _Tp>
1009 inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
1010
1011// }}}
1012// _BitOps {{{
1013struct _BitOps
1014{
1015 // _S_bit_iteration {{{
1016 template <typename _Tp, typename _Fp>
1017 static void
1018 _S_bit_iteration(_Tp __mask, _Fp&& __f)
1019 {
1020 static_assert(sizeof(_ULLong) >= sizeof(_Tp));
1021 conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k;
1022 if constexpr (is_convertible_v<_Tp, decltype(__k)>)
1023 __k = __mask;
1024 else
1025 __k = __mask.to_ullong();
1026 while(__k)
1027 {
1028 __f(std::__countr_zero(__k));
1029 __k &= (__k - 1);
1030 }
1031 }
1032
1033 //}}}
1034};
1035
1036//}}}
1037// __increment, __decrement {{{
1038template <typename _Tp = void>
1039 struct __increment
1040 { constexpr _Tp operator()(_Tp __a) const { return ++__a; } };
1041
1042template <>
1043 struct __increment<void>
1044 {
1045 template <typename _Tp>
1046 constexpr _Tp
1047 operator()(_Tp __a) const
1048 { return ++__a; }
1049 };
1050
1051template <typename _Tp = void>
1052 struct __decrement
1053 { constexpr _Tp operator()(_Tp __a) const { return --__a; } };
1054
1055template <>
1056 struct __decrement<void>
1057 {
1058 template <typename _Tp>
1059 constexpr _Tp
1060 operator()(_Tp __a) const
1061 { return --__a; }
1062 };
1063
1064// }}}
1065// _ValuePreserving(OrInt) {{{
1066template <typename _From, typename _To,
1067 typename = enable_if_t<negation<
1068 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
1069 using _ValuePreserving = _From;
1070
1071template <typename _From, typename _To,
1072 typename _DecayedFrom = __remove_cvref_t<_From>,
1073 typename = enable_if_t<conjunction<
1074 is_convertible<_From, _To>,
1075 disjunction<
1076 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
1077 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
1078 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
1079 using _ValuePreservingOrInt = _From;
1080
1081// }}}
1082// __intrinsic_type {{{
1083template <typename _Tp, size_t _Bytes, typename = void_t<>>
1084 struct __intrinsic_type;
1085
1086template <typename _Tp, size_t _Size>
1087 using __intrinsic_type_t =
1088 typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type;
1089
1090template <typename _Tp>
1091 using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type;
1092template <typename _Tp>
1093 using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type;
1094template <typename _Tp>
1095 using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type;
1096template <typename _Tp>
1097 using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type;
1098template <typename _Tp>
1099 using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type;
1100template <typename _Tp>
1101 using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type;
1102
1103// }}}
1104// _BitMask {{{
1105template <size_t _Np, bool _Sanitized = false>
1106 struct _BitMask;
1107
1108template <size_t _Np, bool _Sanitized>
1109 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> : true_type {};
1110
1111template <size_t _Np>
1112 using _SanitizedBitMask = _BitMask<_Np, true>;
1113
1114template <size_t _Np, bool _Sanitized>
1115 struct _BitMask
1116 {
1117 static_assert(_Np > 0);
1118
1119 static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
1120
1121 using _Tp = conditional_t<_Np == 1, bool,
1122 make_unsigned_t<__int_with_sizeof_t<std::min(
1123 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
1124
1125 static constexpr int _S_array_size = __div_roundup(_NBytes, sizeof(_Tp));
1126
1127 _Tp _M_bits[_S_array_size];
1128
1129 static constexpr int _S_unused_bits
1130 = _Np == 1 ? 0 : _S_array_size * sizeof(_Tp) * __CHAR_BIT__ - _Np;
1131
1132 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1133
1134 constexpr _BitMask() noexcept = default;
1135
1136 constexpr _BitMask(unsigned long long __x) noexcept
1137 : _M_bits{static_cast<_Tp>(__x)} {}
1138
1139 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1140
1141 constexpr _BitMask(const _BitMask&) noexcept = default;
1142
1143 template <bool _RhsSanitized, typename = enable_if_t<_RhsSanitized == false
1144 && _Sanitized == true>>
1145 constexpr _BitMask(const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1146 : _BitMask(__rhs._M_sanitized()) {}
1147
1148 constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1149 {
1150 static_assert(_S_array_size == 1);
1151 return _M_bits[0];
1152 }
1153
1154 // precondition: is sanitized
1155 constexpr _Tp
1156 _M_to_bits() const noexcept
1157 {
1158 static_assert(_S_array_size == 1);
1159 return _M_bits[0];
1160 }
1161
1162 // precondition: is sanitized
1163 constexpr unsigned long long
1164 to_ullong() const noexcept
1165 {
1166 static_assert(_S_array_size == 1);
1167 return _M_bits[0];
1168 }
1169
1170 // precondition: is sanitized
1171 constexpr unsigned long
1172 to_ulong() const noexcept
1173 {
1174 static_assert(_S_array_size == 1);
1175 return _M_bits[0];
1176 }
1177
1178 constexpr bitset<_Np>
1179 _M_to_bitset() const noexcept
1180 {
1181 static_assert(_S_array_size == 1);
1182 return _M_bits[0];
1183 }
1184
1185 constexpr decltype(auto)
1186 _M_sanitized() const noexcept
1187 {
1188 if constexpr (_Sanitized)
1189 return *this;
1190 else if constexpr (_Np == 1)
1191 return _SanitizedBitMask<_Np>(_M_bits[0]);
1192 else
1193 {
1194 _SanitizedBitMask<_Np> __r = {};
1195 for (int __i = 0; __i < _S_array_size; ++__i)
1196 __r._M_bits[__i] = _M_bits[__i];
1197 if constexpr (_S_unused_bits > 0)
1198 __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1199 return __r;
1200 }
1201 }
1202
1203 template <size_t _Mp, bool _LSanitized>
1204 constexpr _BitMask<_Np + _Mp, _Sanitized>
1205 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb) const noexcept
1206 {
1207 constexpr size_t _RN = _Np + _Mp;
1208 using _Rp = _BitMask<_RN, _Sanitized>;
1209 if constexpr (_Rp::_S_array_size == 1)
1210 {
1211 _Rp __r{{_M_bits[0]}};
1212 __r._M_bits[0] <<= _Mp;
1213 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1214 return __r;
1215 }
1216 else
1217 __assert_unreachable<_Rp>();
1218 }
1219
1220 // Return a new _BitMask with size _NewSize while dropping _DropLsb least
1221 // significant bits. If the operation implicitly produces a sanitized bitmask,
1222 // the result type will have _Sanitized set.
1223 template <size_t _DropLsb, size_t _NewSize = _Np - _DropLsb>
1224 constexpr auto
1225 _M_extract() const noexcept
1226 {
1227 static_assert(_Np > _DropLsb);
1228 static_assert(_DropLsb + _NewSize <= sizeof(_ULLong) * __CHAR_BIT__,
1229 "not implemented for bitmasks larger than one ullong");
1230 if constexpr (_NewSize == 1)
1231 // must sanitize because the return _Tp is bool
1232 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1233 else
1234 return _BitMask<_NewSize,
1235 ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1236 && _NewSize + _DropLsb <= _Np)
1237 || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1238 && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1239 >> _DropLsb);
1240 }
1241
1242 // True if all bits are set. Implicitly sanitizes if _Sanitized == false.
1243 constexpr bool
1244 all() const noexcept
1245 {
1246 if constexpr (_Np == 1)
1247 return _M_bits[0];
1248 else if constexpr (!_Sanitized)
1249 return _M_sanitized().all();
1250 else
1251 {
1252 constexpr _Tp __allbits = ~_Tp();
1253 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1254 if (_M_bits[__i] != __allbits)
1255 return false;
1256 return _M_bits[_S_array_size - 1] == _S_bitmask;
1257 }
1258 }
1259
1260 // True if at least one bit is set. Implicitly sanitizes if _Sanitized ==
1261 // false.
1262 constexpr bool
1263 any() const noexcept
1264 {
1265 if constexpr (_Np == 1)
1266 return _M_bits[0];
1267 else if constexpr (!_Sanitized)
1268 return _M_sanitized().any();
1269 else
1270 {
1271 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1272 if (_M_bits[__i] != 0)
1273 return true;
1274 return _M_bits[_S_array_size - 1] != 0;
1275 }
1276 }
1277
1278 // True if no bit is set. Implicitly sanitizes if _Sanitized == false.
1279 constexpr bool
1280 none() const noexcept
1281 {
1282 if constexpr (_Np == 1)
1283 return !_M_bits[0];
1284 else if constexpr (!_Sanitized)
1285 return _M_sanitized().none();
1286 else
1287 {
1288 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1289 if (_M_bits[__i] != 0)
1290 return false;
1291 return _M_bits[_S_array_size - 1] == 0;
1292 }
1293 }
1294
1295 // Returns the number of set bits. Implicitly sanitizes if _Sanitized ==
1296 // false.
1297 constexpr int
1298 count() const noexcept
1299 {
1300 if constexpr (_Np == 1)
1301 return _M_bits[0];
1302 else if constexpr (!_Sanitized)
1303 return _M_sanitized().none();
1304 else
1305 {
1306 int __result = __builtin_popcountll(_M_bits[0]);
1307 for (int __i = 1; __i < _S_array_size; ++__i)
1308 __result += __builtin_popcountll(_M_bits[__i]);
1309 return __result;
1310 }
1311 }
1312
1313 // Returns the bit at offset __i as bool.
1314 constexpr bool
1315 operator[](size_t __i) const noexcept
1316 {
1317 if constexpr (_Np == 1)
1318 return _M_bits[0];
1319 else if constexpr (_S_array_size == 1)
1320 return (_M_bits[0] >> __i) & 1;
1321 else
1322 {
1323 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1324 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1325 return (_M_bits[__j] >> __shift) & 1;
1326 }
1327 }
1328
1329 template <size_t __i>
1330 constexpr bool
1331 operator[](_SizeConstant<__i>) const noexcept
1332 {
1333 static_assert(__i < _Np);
1334 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1335 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1336 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1337 }
1338
1339 // Set the bit at offset __i to __x.
1340 constexpr void
1341 set(size_t __i, bool __x) noexcept
1342 {
1343 if constexpr (_Np == 1)
1344 _M_bits[0] = __x;
1345 else if constexpr (_S_array_size == 1)
1346 {
1347 _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1348 _M_bits[0] |= _Tp(_Tp(__x) << __i);
1349 }
1350 else
1351 {
1352 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1353 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1354 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1355 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1356 }
1357 }
1358
1359 template <size_t __i>
1360 constexpr void
1361 set(_SizeConstant<__i>, bool __x) noexcept
1362 {
1363 static_assert(__i < _Np);
1364 if constexpr (_Np == 1)
1365 _M_bits[0] = __x;
1366 else
1367 {
1368 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1369 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1370 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1371 _M_bits[__j] &= __mask;
1372 _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1373 }
1374 }
1375
1376 // Inverts all bits. Sanitized input leads to sanitized output.
1377 constexpr _BitMask
1378 operator~() const noexcept
1379 {
1380 if constexpr (_Np == 1)
1381 return !_M_bits[0];
1382 else
1383 {
1384 _BitMask __result{};
1385 for (int __i = 0; __i < _S_array_size - 1; ++__i)
1386 __result._M_bits[__i] = ~_M_bits[__i];
1387 if constexpr (_Sanitized)
1388 __result._M_bits[_S_array_size - 1]
1389 = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1390 else
1391 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1392 return __result;
1393 }
1394 }
1395
1396 constexpr _BitMask&
1397 operator^=(const _BitMask& __b) & noexcept
1398 {
1399 __execute_n_times<_S_array_size>(
1400 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] ^= __b._M_bits[__i]; });
1401 return *this;
1402 }
1403
1404 constexpr _BitMask&
1405 operator|=(const _BitMask& __b) & noexcept
1406 {
1407 __execute_n_times<_S_array_size>(
1408 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] |= __b._M_bits[__i]; });
1409 return *this;
1410 }
1411
1412 constexpr _BitMask&
1413 operator&=(const _BitMask& __b) & noexcept
1414 {
1415 __execute_n_times<_S_array_size>(
1416 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] &= __b._M_bits[__i]; });
1417 return *this;
1418 }
1419
1420 friend constexpr _BitMask
1421 operator^(const _BitMask& __a, const _BitMask& __b) noexcept
1422 {
1423 _BitMask __r = __a;
1424 __r ^= __b;
1425 return __r;
1426 }
1427
1428 friend constexpr _BitMask
1429 operator|(const _BitMask& __a, const _BitMask& __b) noexcept
1430 {
1431 _BitMask __r = __a;
1432 __r |= __b;
1433 return __r;
1434 }
1435
1436 friend constexpr _BitMask
1437 operator&(const _BitMask& __a, const _BitMask& __b) noexcept
1438 {
1439 _BitMask __r = __a;
1440 __r &= __b;
1441 return __r;
1442 }
1443
1444 _GLIBCXX_SIMD_INTRINSIC
1445 constexpr bool
1446 _M_is_constprop() const
1447 {
1448 if constexpr (_S_array_size == 0)
1449 return __builtin_constant_p(_M_bits[0]);
1450 else
1451 {
1452 for (int __i = 0; __i < _S_array_size; ++__i)
1453 if (!__builtin_constant_p(_M_bits[__i]))
1454 return false;
1455 return true;
1456 }
1457 }
1458 };
1459
1460// }}}
1461
1462// vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv
1463// __min_vector_size {{{
1464template <typename _Tp = void>
1465 static inline constexpr int __min_vector_size = 2 * sizeof(_Tp);
1466
1467#if _GLIBCXX_SIMD_HAVE_NEON
1468template <>
1469 inline constexpr int __min_vector_size<void> = 8;
1470#else
1471template <>
1472 inline constexpr int __min_vector_size<void> = 16;
1473#endif
1474
1475// }}}
1476// __vector_type {{{
1477template <typename _Tp, size_t _Np, typename = void>
1478 struct __vector_type_n {};
1479
1480// substition failure for 0-element case
1481template <typename _Tp>
1482 struct __vector_type_n<_Tp, 0, void> {};
1483
1484// special case 1-element to be _Tp itself
1485template <typename _Tp>
1486 struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>>
1487 { using type = _Tp; };
1488
1489// else, use GNU-style builtin vector types
1490template <typename _Tp, size_t _Np>
1491 struct __vector_type_n<_Tp, _Np,
1492 enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1493 {
1494 static constexpr size_t _S_Np2 = std::__bit_ceil(_Np * sizeof(_Tp));
1495
1496 static constexpr size_t _S_Bytes =
1497#ifdef __i386__
1498 // Using [[gnu::vector_size(8)]] would wreak havoc on the FPU because
1499 // those objects are passed via MMX registers and nothing ever calls EMMS.
1500 _S_Np2 == 8 ? 16 :
1501#endif
1502 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1503 : _S_Np2;
1504
1505 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1506 };
1507
1508template <typename _Tp, size_t _Bytes, size_t = _Bytes % sizeof(_Tp)>
1509 struct __vector_type;
1510
1511template <typename _Tp, size_t _Bytes>
1512 struct __vector_type<_Tp, _Bytes, 0>
1513 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1514
1515template <typename _Tp, size_t _Size>
1516 using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type;
1517
1518template <typename _Tp>
1519 using __vector_type2_t = typename __vector_type<_Tp, 2>::type;
1520template <typename _Tp>
1521 using __vector_type4_t = typename __vector_type<_Tp, 4>::type;
1522template <typename _Tp>
1523 using __vector_type8_t = typename __vector_type<_Tp, 8>::type;
1524template <typename _Tp>
1525 using __vector_type16_t = typename __vector_type<_Tp, 16>::type;
1526template <typename _Tp>
1527 using __vector_type32_t = typename __vector_type<_Tp, 32>::type;
1528template <typename _Tp>
1529 using __vector_type64_t = typename __vector_type<_Tp, 64>::type;
1530
1531// }}}
1532// __is_vector_type {{{
1533template <typename _Tp, typename = void_t<>>
1534 struct __is_vector_type : false_type {};
1535
1536template <typename _Tp>
1537 struct __is_vector_type<
1538 _Tp, void_t<typename __vector_type<
1539 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1540 : is_same<_Tp, typename __vector_type<
1541 remove_reference_t<decltype(declval<_Tp>()[0])>,
1542 sizeof(_Tp)>::type> {};
1543
1544template <typename _Tp>
1545 inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1546
1547// }}}
1548// __is_intrinsic_type {{{
1549#if _GLIBCXX_SIMD_HAVE_SSE_ABI
1550template <typename _Tp>
1551 using __is_intrinsic_type = __is_vector_type<_Tp>;
1552#else // not SSE (x86)
1553template <typename _Tp, typename = void_t<>>
1554 struct __is_intrinsic_type : false_type {};
1555
1556template <typename _Tp>
1557 struct __is_intrinsic_type<
1558 _Tp, void_t<typename __intrinsic_type<
1559 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1560 : is_same<_Tp, typename __intrinsic_type<
1561 remove_reference_t<decltype(declval<_Tp>()[0])>,
1562 sizeof(_Tp)>::type> {};
1563#endif
1564
1565template <typename _Tp>
1566 inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1567
1568// }}}
1569// _VectorTraits{{{
1570template <typename _Tp, typename = void_t<>>
1571 struct _VectorTraitsImpl;
1572
1573template <typename _Tp>
1574 struct _VectorTraitsImpl<_Tp, enable_if_t<__is_vector_type_v<_Tp>
1575 || __is_intrinsic_type_v<_Tp>>>
1576 {
1577 using type = _Tp;
1578 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1579 static constexpr int _S_full_size = sizeof(_Tp) / sizeof(value_type);
1580 using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1581 template <typename _Up, int _W = _S_full_size>
1582 static constexpr bool _S_is
1583 = is_same_v<value_type, _Up> && _W == _S_full_size;
1584 };
1585
1586template <typename _Tp, size_t _Np>
1587 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1588 void_t<__vector_type_t<_Tp, _Np>>>
1589 {
1590 using type = __vector_type_t<_Tp, _Np>;
1591 using value_type = _Tp;
1592 static constexpr int _S_full_size = sizeof(type) / sizeof(value_type);
1593 using _Wrapper = _SimdWrapper<_Tp, _Np>;
1594 static constexpr bool _S_is_partial = (_Np == _S_full_size);
1595 static constexpr int _S_partial_width = _Np;
1596 template <typename _Up, int _W = _S_full_size>
1597 static constexpr bool _S_is
1598 = is_same_v<value_type, _Up>&& _W == _S_full_size;
1599 };
1600
1601template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1602 using _VectorTraits = _VectorTraitsImpl<_Tp>;
1603
1604// }}}
1605// __as_vector{{{
1606template <typename _V>
1607 _GLIBCXX_SIMD_INTRINSIC constexpr auto
1608 __as_vector(_V __x)
1609 {
1610 if constexpr (__is_vector_type_v<_V>)
1611 return __x;
1612 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1613 return __data(__x)._M_data;
1614 else if constexpr (__is_vectorizable_v<_V>)
1615 return __vector_type_t<_V, 2>{__x};
1616 else
1617 return __x._M_data;
1618 }
1619
1620// }}}
1621// __as_wrapper{{{
1622template <size_t _Np = 0, typename _V>
1623 _GLIBCXX_SIMD_INTRINSIC constexpr auto
1624 __as_wrapper(_V __x)
1625 {
1626 if constexpr (__is_vector_type_v<_V>)
1627 return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1628 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1629 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1630 {
1631 static_assert(_V::size() == _Np);
1632 return __data(__x);
1633 }
1634 else
1635 {
1636 static_assert(_V::_S_size == _Np);
1637 return __x;
1638 }
1639 }
1640
1641// }}}
1642// __intrin_bitcast{{{
1643template <typename _To, typename _From>
1644 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1645 __intrin_bitcast(_From __v)
1646 {
1647 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1648 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1649 if constexpr (sizeof(_To) == sizeof(_From))
1650 return reinterpret_cast<_To>(__v);
1651 else if constexpr (sizeof(_From) > sizeof(_To))
1652 if constexpr (sizeof(_To) >= 16)
1653 return reinterpret_cast<const __may_alias<_To>&>(__v);
1654 else
1655 {
1656 _To __r;
1657 __builtin_memcpy(&__r, &__v, sizeof(_To));
1658 return __r;
1659 }
1660#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1661 else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32)
1662 return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1663 reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1664 else if constexpr (__have_avx512f && sizeof(_From) == 16
1665 && sizeof(_To) == 64)
1666 return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1667 reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1668 else if constexpr (__have_avx512f && sizeof(_From) == 32
1669 && sizeof(_To) == 64)
1670 return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1671 reinterpret_cast<__vector_type_t<float, 8>>(__v)));
1672#endif // _GLIBCXX_SIMD_X86INTRIN
1673 else if constexpr (sizeof(__v) <= 8)
1674 return reinterpret_cast<_To>(
1675 __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1676 reinterpret_cast<__int_for_sizeof_t<_From>>(__v)});
1677 else
1678 {
1679 static_assert(sizeof(_To) > sizeof(_From));
1680 _To __r = {};
1681 __builtin_memcpy(&__r, &__v, sizeof(_From));
1682 return __r;
1683 }
1684 }
1685
1686// }}}
1687// __vector_bitcast{{{
1688template <typename _To, size_t _NN = 0, typename _From,
1689 typename _FromVT = _VectorTraits<_From>,
1690 size_t _Np = _NN == 0 ? sizeof(_From) / sizeof(_To) : _NN>
1691 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1692 __vector_bitcast(_From __x)
1693 {
1694 using _R = __vector_type_t<_To, _Np>;
1695 return __intrin_bitcast<_R>(__x);
1696 }
1697
1698template <typename _To, size_t _NN = 0, typename _Tp, size_t _Nx,
1699 size_t _Np
1700 = _NN == 0 ? sizeof(_SimdWrapper<_Tp, _Nx>) / sizeof(_To) : _NN>
1701 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1702 __vector_bitcast(const _SimdWrapper<_Tp, _Nx>& __x)
1703 {
1704 static_assert(_Np > 1);
1705 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1706 }
1707
1708// }}}
1709// __convert_x86 declarations {{{
1710#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1711template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1712 _To __convert_x86(_Tp);
1713
1714template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1715 _To __convert_x86(_Tp, _Tp);
1716
1717template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1718 _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1719
1720template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1721 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1722
1723template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1724 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1725 _Tp, _Tp, _Tp, _Tp);
1726#endif // _GLIBCXX_SIMD_WORKAROUND_PR85048
1727
1728//}}}
1729// __bit_cast {{{
1730template <typename _To, typename _From>
1731 _GLIBCXX_SIMD_INTRINSIC constexpr _To
1732 __bit_cast(const _From __x)
1733 {
1734#if __has_builtin(__builtin_bit_cast)
1735 return __builtin_bit_cast(_To, __x);
1736#else
1737 static_assert(sizeof(_To) == sizeof(_From));
1738 constexpr bool __to_is_vectorizable
1739 = is_arithmetic_v<_To> || is_enum_v<_To>;
1740 constexpr bool __from_is_vectorizable
1741 = is_arithmetic_v<_From> || is_enum_v<_From>;
1742 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1743 return reinterpret_cast<_To>(__x);
1744 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1745 {
1746 using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1747 return reinterpret_cast<_To>(_FV{__x});
1748 }
1749 else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1750 {
1751 using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1752 using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1753 return reinterpret_cast<_TV>(_FV{__x})[0];
1754 }
1755 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1756 {
1757 using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1758 return reinterpret_cast<_TV>(__x)[0];
1759 }
1760 else
1761 {
1762 _To __r;
1763 __builtin_memcpy(reinterpret_cast<char*>(&__r),
1764 reinterpret_cast<const char*>(&__x), sizeof(_To));
1765 return __r;
1766 }
1767#endif
1768 }
1769
1770// }}}
1771// __to_intrin {{{
1772template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1773 typename _R
1774 = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_full_size>>
1775 _GLIBCXX_SIMD_INTRINSIC constexpr _R
1776 __to_intrin(_Tp __x)
1777 {
1778 static_assert(sizeof(__x) <= sizeof(_R),
1779 "__to_intrin may never drop values off the end");
1780 if constexpr (sizeof(__x) == sizeof(_R))
1781 return reinterpret_cast<_R>(__as_vector(__x));
1782 else
1783 {
1784 using _Up = __int_for_sizeof_t<_Tp>;
1785 return reinterpret_cast<_R>(
1786 __vector_type_t<_Up, sizeof(_R) / sizeof(_Up)>{__bit_cast<_Up>(__x)});
1787 }
1788 }
1789
1790// }}}
1791// __make_vector{{{
1792template <typename _Tp, typename... _Args>
1793 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)>
1794 __make_vector(const _Args&... __args)
1795 {
1796 return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(__args)...};
1797 }
1798
1799// }}}
1800// __vector_broadcast{{{
1801template <size_t _Np, typename _Tp>
1802 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1803 __vector_broadcast(_Tp __x)
1804 {
1805 return __call_with_n_evaluations<_Np>(
1806 [](auto... __xx) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
1807 return __vector_type_t<_Tp, _Np>{__xx...};
1808 }, [&__x](int) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x; });
1809 }
1810
1811// }}}
1812// __generate_vector{{{
1813 template <typename _Tp, size_t _Np, typename _Gp, size_t... _I>
1814 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1815 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1816 {
1817 return __vector_type_t<_Tp, _Np>{
1818 static_cast<_Tp>(__gen(_SizeConstant<_I>()))...};
1819 }
1820
1821template <typename _V, typename _VVT = _VectorTraits<_V>, typename _Gp>
1822 _GLIBCXX_SIMD_INTRINSIC constexpr _V
1823 __generate_vector(_Gp&& __gen)
1824 {
1825 if constexpr (__is_vector_type_v<_V>)
1826 return __generate_vector_impl<typename _VVT::value_type,
1827 _VVT::_S_full_size>(
1828 static_cast<_Gp&&>(__gen), make_index_sequence<_VVT::_S_full_size>());
1829 else
1830 return __generate_vector_impl<typename _VVT::value_type,
1831 _VVT::_S_partial_width>(
1832 static_cast<_Gp&&>(__gen),
1833 make_index_sequence<_VVT::_S_partial_width>());
1834 }
1835
1836template <typename _Tp, size_t _Np, typename _Gp>
1837 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1838 __generate_vector(_Gp&& __gen)
1839 {
1840 return __generate_vector_impl<_Tp, _Np>(static_cast<_Gp&&>(__gen),
1841 make_index_sequence<_Np>());
1842 }
1843
1844// }}}
1845// __xor{{{
1846template <typename _TW>
1847 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1848 __xor(_TW __a, _TW __b) noexcept
1849 {
1850 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1851 {
1852 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1853 _VectorTraitsImpl<_TW>>::value_type;
1854 if constexpr (is_floating_point_v<_Tp>)
1855 {
1856 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1857 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1858 ^ __vector_bitcast<_Ip>(__b));
1859 }
1860 else if constexpr (__is_vector_type_v<_TW>)
1861 return __a ^ __b;
1862 else
1863 return __a._M_data ^ __b._M_data;
1864 }
1865 else
1866 return __a ^ __b;
1867 }
1868
1869// }}}
1870// __or{{{
1871template <typename _TW>
1872 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1873 __or(_TW __a, _TW __b) noexcept
1874 {
1875 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1876 {
1877 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1878 _VectorTraitsImpl<_TW>>::value_type;
1879 if constexpr (is_floating_point_v<_Tp>)
1880 {
1881 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1882 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1883 | __vector_bitcast<_Ip>(__b));
1884 }
1885 else if constexpr (__is_vector_type_v<_TW>)
1886 return __a | __b;
1887 else
1888 return __a._M_data | __b._M_data;
1889 }
1890 else
1891 return __a | __b;
1892 }
1893
1894// }}}
1895// __and{{{
1896template <typename _TW>
1897 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1898 __and(_TW __a, _TW __b) noexcept
1899 {
1900 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1901 {
1902 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1903 _VectorTraitsImpl<_TW>>::value_type;
1904 if constexpr (is_floating_point_v<_Tp>)
1905 {
1906 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1907 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1908 & __vector_bitcast<_Ip>(__b));
1909 }
1910 else if constexpr (__is_vector_type_v<_TW>)
1911 return __a & __b;
1912 else
1913 return __a._M_data & __b._M_data;
1914 }
1915 else
1916 return __a & __b;
1917 }
1918
1919// }}}
1920// __andnot{{{
1921#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1922static constexpr struct
1923{
1924 _GLIBCXX_SIMD_INTRINSIC __v4sf
1925 operator()(__v4sf __a, __v4sf __b) const noexcept
1926 { return __builtin_ia32_andnps(__a, __b); }
1927
1928 _GLIBCXX_SIMD_INTRINSIC __v2df
1929 operator()(__v2df __a, __v2df __b) const noexcept
1930 { return __builtin_ia32_andnpd(__a, __b); }
1931
1932 _GLIBCXX_SIMD_INTRINSIC __v2di
1933 operator()(__v2di __a, __v2di __b) const noexcept
1934 { return __builtin_ia32_pandn128(__a, __b); }
1935
1936 _GLIBCXX_SIMD_INTRINSIC __v8sf
1937 operator()(__v8sf __a, __v8sf __b) const noexcept
1938 { return __builtin_ia32_andnps256(__a, __b); }
1939
1940 _GLIBCXX_SIMD_INTRINSIC __v4df
1941 operator()(__v4df __a, __v4df __b) const noexcept
1942 { return __builtin_ia32_andnpd256(__a, __b); }
1943
1944 _GLIBCXX_SIMD_INTRINSIC __v4di
1945 operator()(__v4di __a, __v4di __b) const noexcept
1946 {
1947 if constexpr (__have_avx2)
1948 return __builtin_ia32_andnotsi256(__a, __b);
1949 else
1950 return reinterpret_cast<__v4di>(
1951 __builtin_ia32_andnpd256(reinterpret_cast<__v4df>(__a),
1952 reinterpret_cast<__v4df>(__b)));
1953 }
1954
1955 _GLIBCXX_SIMD_INTRINSIC __v16sf
1956 operator()(__v16sf __a, __v16sf __b) const noexcept
1957 {
1958 if constexpr (__have_avx512dq)
1959 return _mm512_andnot_ps(__a, __b);
1960 else
1961 return reinterpret_cast<__v16sf>(
1962 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1963 reinterpret_cast<__v8di>(__b)));
1964 }
1965
1966 _GLIBCXX_SIMD_INTRINSIC __v8df
1967 operator()(__v8df __a, __v8df __b) const noexcept
1968 {
1969 if constexpr (__have_avx512dq)
1970 return _mm512_andnot_pd(__a, __b);
1971 else
1972 return reinterpret_cast<__v8df>(
1973 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1974 reinterpret_cast<__v8di>(__b)));
1975 }
1976
1977 _GLIBCXX_SIMD_INTRINSIC __v8di
1978 operator()(__v8di __a, __v8di __b) const noexcept
1979 { return _mm512_andnot_si512(__a, __b); }
1980} _S_x86_andnot;
1981#endif // _GLIBCXX_SIMD_X86INTRIN && !__clang__
1982
1983template <typename _TW>
1984 _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1985 __andnot(_TW __a, _TW __b) noexcept
1986 {
1987 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1988 {
1989 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1990 _VectorTraitsImpl<_TW>>;
1991 using _Tp = typename _TVT::value_type;
1992#if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1993 if constexpr (sizeof(_TW) >= 16)
1994 {
1995 const auto __ai = __to_intrin(__a);
1996 const auto __bi = __to_intrin(__b);
1997 if (!__builtin_is_constant_evaluated()
1998 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1999 {
2000 const auto __r = _S_x86_andnot(__ai, __bi);
2001 if constexpr (is_convertible_v<decltype(__r), _TW>)
2002 return __r;
2003 else
2004 return reinterpret_cast<typename _TVT::type>(__r);
2005 }
2006 }
2007#endif // _GLIBCXX_SIMD_X86INTRIN
2008 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
2009 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
2010 & __vector_bitcast<_Ip>(__b));
2011 }
2012 else
2013 return ~__a & __b;
2014 }
2015
2016// }}}
2017// __not{{{
2018template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2019 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
2020 __not(_Tp __a) noexcept
2021 {
2022 if constexpr (is_floating_point_v<typename _TVT::value_type>)
2023 return reinterpret_cast<typename _TVT::type>(
2024 ~__vector_bitcast<unsigned>(__a));
2025 else
2026 return ~__a;
2027 }
2028
2029// }}}
2030// __concat{{{
2031template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
2032 typename _R = __vector_type_t<typename _TVT::value_type,
2033 _TVT::_S_full_size * 2>>
2034 constexpr _R
2035 __concat(_Tp a_, _Tp b_)
2036 {
2037#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2038 using _W
2039 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
2040 conditional_t<(sizeof(_Tp) >= 2 * sizeof(long long)),
2041 long long, typename _TVT::value_type>>;
2042 constexpr int input_width = sizeof(_Tp) / sizeof(_W);
2043 const auto __a = __vector_bitcast<_W>(a_);
2044 const auto __b = __vector_bitcast<_W>(b_);
2045 using _Up = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>;
2046#else
2047 constexpr int input_width = _TVT::_S_full_size;
2048 const _Tp& __a = a_;
2049 const _Tp& __b = b_;
2050 using _Up = _R;
2051#endif
2052 if constexpr (input_width == 2)
2053 return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
2054 else if constexpr (input_width == 4)
2055 return reinterpret_cast<_R>(
2056 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
2057 else if constexpr (input_width == 8)
2058 return reinterpret_cast<_R>(
2059 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
2060 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
2061 else if constexpr (input_width == 16)
2062 return reinterpret_cast<_R>(
2063 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2064 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2065 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
2066 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
2067 __b[12], __b[13], __b[14], __b[15]});
2068 else if constexpr (input_width == 32)
2069 return reinterpret_cast<_R>(
2070 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2071 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2072 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
2073 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
2074 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
2075 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
2076 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
2077 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
2078 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
2079 __b[31]});
2080 }
2081
2082// }}}
2083// __zero_extend {{{
2084template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2085 struct _ZeroExtendProxy
2086 {
2087 using value_type = typename _TVT::value_type;
2088 static constexpr size_t _Np = _TVT::_S_full_size;
2089 const _Tp __x;
2090
2091 template <typename _To, typename _ToVT = _VectorTraits<_To>,
2092 typename
2093 = enable_if_t<is_same_v<typename _ToVT::value_type, value_type>>>
2094 _GLIBCXX_SIMD_INTRINSIC operator _To() const
2095 {
2096 constexpr size_t _ToN = _ToVT::_S_full_size;
2097 if constexpr (_ToN == _Np)
2098 return __x;
2099 else if constexpr (_ToN == 2 * _Np)
2100 {
2101#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2102 if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
2103 return __vector_bitcast<value_type>(
2104 _mm256_insertf128_ps(__m256(), __x, 0));
2105 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
2106 return __vector_bitcast<value_type>(
2107 _mm256_insertf128_pd(__m256d(), __x, 0));
2108 else if constexpr (__have_avx2 && _Np * sizeof(value_type) == 16)
2109 return __vector_bitcast<value_type>(
2110 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
2111 else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
2112 {
2113 if constexpr (__have_avx512dq)
2114 return __vector_bitcast<value_type>(
2115 _mm512_insertf32x8(__m512(), __x, 0));
2116 else
2117 return reinterpret_cast<__m512>(
2118 _mm512_insertf64x4(__m512d(),
2119 reinterpret_cast<__m256d>(__x), 0));
2120 }
2121 else if constexpr (__have_avx512f
2122 && _TVT::template _S_is<double, 4>)
2123 return __vector_bitcast<value_type>(
2124 _mm512_insertf64x4(__m512d(), __x, 0));
2125 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 32)
2126 return __vector_bitcast<value_type>(
2127 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
2128#endif
2129 return __concat(__x, _Tp());
2130 }
2131 else if constexpr (_ToN == 4 * _Np)
2132 {
2133#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2134 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2135 {
2136 return __vector_bitcast<value_type>(
2137 _mm512_insertf64x2(__m512d(), __x, 0));
2138 }
2139 else if constexpr (__have_avx512f
2140 && is_floating_point_v<value_type>)
2141 {
2142 return __vector_bitcast<value_type>(
2143 _mm512_insertf32x4(__m512(), reinterpret_cast<__m128>(__x),
2144 0));
2145 }
2146 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 16)
2147 {
2148 return __vector_bitcast<value_type>(
2149 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2150 }
2151#endif
2152 return __concat(__concat(__x, _Tp()),
2153 __vector_type_t<value_type, _Np * 2>());
2154 }
2155 else if constexpr (_ToN == 8 * _Np)
2156 return __concat(operator __vector_type_t<value_type, _Np * 4>(),
2157 __vector_type_t<value_type, _Np * 4>());
2158 else if constexpr (_ToN == 16 * _Np)
2159 return __concat(operator __vector_type_t<value_type, _Np * 8>(),
2160 __vector_type_t<value_type, _Np * 8>());
2161 else
2162 __assert_unreachable<_Tp>();
2163 }
2164 };
2165
2166template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2167 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2168 __zero_extend(_Tp __x)
2169 { return {__x}; }
2170
2171// }}}
2172// __extract<_Np, By>{{{
2173template <int _Offset,
2174 int _SplitBy,
2175 typename _Tp,
2176 typename _TVT = _VectorTraits<_Tp>,
2177 typename _R = __vector_type_t<typename _TVT::value_type,
2178 _TVT::_S_full_size / _SplitBy>>
2179 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2180 __extract(_Tp __in)
2181 {
2182 using value_type = typename _TVT::value_type;
2183#if _GLIBCXX_SIMD_X86INTRIN // {{{
2184 if constexpr (sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2185 {
2186 if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2187 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2188 else if constexpr (is_floating_point_v<value_type>)
2189 return __vector_bitcast<value_type>(
2190 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2191 else
2192 return reinterpret_cast<_R>(
2193 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2194 _Offset));
2195 }
2196 else
2197#endif // _GLIBCXX_SIMD_X86INTRIN }}}
2198 {
2199#ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2200 using _W = conditional_t<
2201 is_floating_point_v<value_type>, double,
2202 conditional_t<(sizeof(_R) >= 16), long long, value_type>>;
2203 static_assert(sizeof(_R) % sizeof(_W) == 0);
2204 constexpr int __return_width = sizeof(_R) / sizeof(_W);
2205 using _Up = __vector_type_t<_W, __return_width>;
2206 const auto __x = __vector_bitcast<_W>(__in);
2207#else
2208 constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2209 using _Up = _R;
2210 const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2211 = __in; // only needed for _Tp = _SimdWrapper<value_type, _Np>
2212#endif
2213 constexpr int _O = _Offset * __return_width;
2214 return __call_with_subscripts<__return_width, _O>(
2215 __x, [](auto... __entries) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2216 return reinterpret_cast<_R>(_Up{__entries...});
2217 });
2218 }
2219 }
2220
2221// }}}
2222// __lo/__hi64[z]{{{
2223template <typename _Tp,
2224 typename _R
2225 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2226 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2227 __lo64(_Tp __x)
2228 {
2229 _R __r{};
2230 __builtin_memcpy(&__r, &__x, 8);
2231 return __r;
2232 }
2233
2234template <typename _Tp,
2235 typename _R
2236 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2237 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2238 __hi64(_Tp __x)
2239 {
2240 static_assert(sizeof(_Tp) == 16, "use __hi64z if you meant it");
2241 _R __r{};
2242 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2243 return __r;
2244 }
2245
2246template <typename _Tp,
2247 typename _R
2248 = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2249 _GLIBCXX_SIMD_INTRINSIC constexpr _R
2250 __hi64z([[maybe_unused]] _Tp __x)
2251 {
2252 _R __r{};
2253 if constexpr (sizeof(_Tp) == 16)
2254 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2255 return __r;
2256 }
2257
2258// }}}
2259// __lo/__hi128{{{
2260template <typename _Tp>
2261 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2262 __lo128(_Tp __x)
2263 { return __extract<0, sizeof(_Tp) / 16>(__x); }
2264
2265template <typename _Tp>
2266 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2267 __hi128(_Tp __x)
2268 {
2269 static_assert(sizeof(__x) == 32);
2270 return __extract<1, 2>(__x);
2271 }
2272
2273// }}}
2274// __lo/__hi256{{{
2275template <typename _Tp>
2276 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2277 __lo256(_Tp __x)
2278 {
2279 static_assert(sizeof(__x) == 64);
2280 return __extract<0, 2>(__x);
2281 }
2282
2283template <typename _Tp>
2284 _GLIBCXX_SIMD_INTRINSIC constexpr auto
2285 __hi256(_Tp __x)
2286 {
2287 static_assert(sizeof(__x) == 64);
2288 return __extract<1, 2>(__x);
2289 }
2290
2291// }}}
2292// __auto_bitcast{{{
2293template <typename _Tp>
2294 struct _AutoCast
2295 {
2296 static_assert(__is_vector_type_v<_Tp>);
2297
2298 const _Tp __x;
2299
2300 template <typename _Up, typename _UVT = _VectorTraits<_Up>>
2301 _GLIBCXX_SIMD_INTRINSIC constexpr operator _Up() const
2302 { return __intrin_bitcast<typename _UVT::type>(__x); }
2303 };
2304
2305template <typename _Tp>
2306 _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2307 __auto_bitcast(const _Tp& __x)
2308 { return {__x}; }
2309
2310template <typename _Tp, size_t _Np>
2311 _GLIBCXX_SIMD_INTRINSIC constexpr
2312 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2313 __auto_bitcast(const _SimdWrapper<_Tp, _Np>& __x)
2314 { return {__x._M_data}; }
2315
2316// }}}
2317// ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^
2318
2319#if _GLIBCXX_SIMD_HAVE_SSE_ABI
2320// __bool_storage_member_type{{{
2321#if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2322template <size_t _Size>
2323 struct __bool_storage_member_type
2324 {
2325 static_assert((_Size & (_Size - 1)) != 0,
2326 "This trait may only be used for non-power-of-2 sizes. "
2327 "Power-of-2 sizes must be specialized.");
2328 using type =
2329 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2330 };
2331
2332template <>
2333 struct __bool_storage_member_type<1> { using type = bool; };
2334
2335template <>
2336 struct __bool_storage_member_type<2> { using type = __mmask8; };
2337
2338template <>
2339 struct __bool_storage_member_type<4> { using type = __mmask8; };
2340
2341template <>
2342 struct __bool_storage_member_type<8> { using type = __mmask8; };
2343
2344template <>
2345 struct __bool_storage_member_type<16> { using type = __mmask16; };
2346
2347template <>
2348 struct __bool_storage_member_type<32> { using type = __mmask32; };
2349
2350template <>
2351 struct __bool_storage_member_type<64> { using type = __mmask64; };
2352#endif // _GLIBCXX_SIMD_HAVE_AVX512F
2353
2354// }}}
2355// __intrinsic_type (x86){{{
2356// the following excludes bool via __is_vectorizable
2357#if _GLIBCXX_SIMD_HAVE_SSE
2358template <typename _Tp, size_t _Bytes>
2359 struct __intrinsic_type<_Tp, _Bytes,
2360 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2361 {
2362 static_assert(!is_same_v<_Tp, long double>,
2363 "no __intrinsic_type support for long double on x86");
2364
2365 static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16
2366 : _Bytes <= 32 ? 32
2367 : 64;
2368
2369 using type [[__gnu__::__vector_size__(_S_VBytes)]]
2370 = conditional_t<is_integral_v<_Tp>, long long int, _Tp>;
2371 };
2372#endif // _GLIBCXX_SIMD_HAVE_SSE
2373
2374// }}}
2375#endif // _GLIBCXX_SIMD_HAVE_SSE_ABI
2376// __intrinsic_type (ARM){{{
2377#if _GLIBCXX_SIMD_HAVE_NEON
2378template <>
2379 struct __intrinsic_type<float, 8, void>
2380 { using type = float32x2_t; };
2381
2382template <>
2383 struct __intrinsic_type<float, 16, void>
2384 { using type = float32x4_t; };
2385
2386#if _GLIBCXX_SIMD_HAVE_NEON_A64
2387template <>
2388 struct __intrinsic_type<double, 8, void>
2389 { using type = float64x1_t; };
2390
2391template <>
2392 struct __intrinsic_type<double, 16, void>
2393 { using type = float64x2_t; };
2394#endif
2395
2396#define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2397template <> \
2398 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2399 _Np * _Bits / 8, void> \
2400 { using type = int##_Bits##x##_Np##_t; }; \
2401template <> \
2402 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2403 _Np * _Bits / 8, void> \
2404 { using type = uint##_Bits##x##_Np##_t; }
2405_GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2406_GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2407_GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2408_GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2409_GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2410_GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2411_GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2412_GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2413#undef _GLIBCXX_SIMD_ARM_INTRIN
2414
2415template <typename _Tp, size_t _Bytes>
2416 struct __intrinsic_type<_Tp, _Bytes,
2417 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2418 {
2419 static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2420 using _Ip = __int_for_sizeof_t<_Tp>;
2421 using _Up = conditional_t<
2422 is_floating_point_v<_Tp>, _Tp,
2423 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2424 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2425 "should use explicit specialization above");
2426 using type = typename __intrinsic_type<_Up, _SVecBytes>::type;
2427 };
2428#endif // _GLIBCXX_SIMD_HAVE_NEON
2429
2430// }}}
2431// __intrinsic_type (PPC){{{
2432#ifdef __ALTIVEC__
2433template <typename _Tp>
2434 struct __intrinsic_type_impl;
2435
2436#define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2437 template <> \
2438 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2439_GLIBCXX_SIMD_PPC_INTRIN(float);
2440#ifdef __VSX__
2441_GLIBCXX_SIMD_PPC_INTRIN(double);
2442#endif
2443_GLIBCXX_SIMD_PPC_INTRIN(signed char);
2444_GLIBCXX_SIMD_PPC_INTRIN(unsigned char);
2445_GLIBCXX_SIMD_PPC_INTRIN(signed short);
2446_GLIBCXX_SIMD_PPC_INTRIN(unsigned short);
2447_GLIBCXX_SIMD_PPC_INTRIN(signed int);
2448_GLIBCXX_SIMD_PPC_INTRIN(unsigned int);
2449#if defined __VSX__ || __SIZEOF_LONG__ == 4
2450_GLIBCXX_SIMD_PPC_INTRIN(signed long);
2451_GLIBCXX_SIMD_PPC_INTRIN(unsigned long);
2452#endif
2453#ifdef __VSX__
2454_GLIBCXX_SIMD_PPC_INTRIN(signed long long);
2455_GLIBCXX_SIMD_PPC_INTRIN(unsigned long long);
2456#endif
2457#undef _GLIBCXX_SIMD_PPC_INTRIN
2458
2459template <typename _Tp, size_t _Bytes>
2460 struct __intrinsic_type<_Tp, _Bytes,
2461 enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2462 {
2463 static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2464 // allow _Tp == long double with -mlong-double-64
2465 static_assert(!(_S_is_ldouble && sizeof(long double) > sizeof(double)),
2466 "no __intrinsic_type support for 128-bit floating point on PowerPC");
2467#ifndef __VSX__
2468 static_assert(!(is_same_v<_Tp, double>
2469 || (_S_is_ldouble && sizeof(long double) == sizeof(double))),
2470 "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX");
2471#endif
2472 using type =
2473 typename __intrinsic_type_impl<
2474 conditional_t<is_floating_point_v<_Tp>,
2475 conditional_t<_S_is_ldouble, double, _Tp>,
2476 __int_for_sizeof_t<_Tp>>>::type;
2477 };
2478#endif // __ALTIVEC__
2479
2480// }}}
2481// _SimdWrapper<bool>{{{1
2482template <size_t _Width>
2483 struct _SimdWrapper<bool, _Width,
2484 void_t<typename __bool_storage_member_type<_Width>::type>>
2485 {
2486 using _BuiltinType = typename __bool_storage_member_type<_Width>::type;
2487 using value_type = bool;
2488
2489 static constexpr size_t _S_full_size = sizeof(_BuiltinType) * __CHAR_BIT__;
2490
2491 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2492 __as_full_vector() const { return _M_data; }
2493
2494 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
2495 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_BuiltinType __k)
2496 : _M_data(__k) {};
2497
2498 _GLIBCXX_SIMD_INTRINSIC operator const _BuiltinType&() const
2499 { return _M_data; }
2500
2501 _GLIBCXX_SIMD_INTRINSIC operator _BuiltinType&()
2502 { return _M_data; }
2503
2504 _GLIBCXX_SIMD_INTRINSIC _BuiltinType __intrin() const
2505 { return _M_data; }
2506
2507 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator[](size_t __i) const
2508 { return _M_data & (_BuiltinType(1) << __i); }
2509
2510 template <size_t __i>
2511 _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2512 operator[](_SizeConstant<__i>) const
2513 { return _M_data & (_BuiltinType(1) << __i); }
2514
2515 _GLIBCXX_SIMD_INTRINSIC constexpr void _M_set(size_t __i, value_type __x)
2516 {
2517 if (__x)
2518 _M_data |= (_BuiltinType(1) << __i);
2519 else
2520 _M_data &= ~(_BuiltinType(1) << __i);
2521 }
2522
2523 _GLIBCXX_SIMD_INTRINSIC
2524 constexpr bool _M_is_constprop() const
2525 { return __builtin_constant_p(_M_data); }
2526
2527 _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_none_of() const
2528 {
2529 if (__builtin_constant_p(_M_data))
2530 {
2531 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2532 constexpr _BuiltinType __active_mask
2533 = ~_BuiltinType() >> (__nbits - _Width);
2534 return (_M_data & __active_mask) == 0;
2535 }
2536 return false;
2537 }
2538
2539 _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_all_of() const
2540 {
2541 if (__builtin_constant_p(_M_data))
2542 {
2543 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2544 constexpr _BuiltinType __active_mask
2545 = ~_BuiltinType() >> (__nbits - _Width);
2546 return (_M_data & __active_mask) == __active_mask;
2547 }
2548 return false;
2549 }
2550
2551 _BuiltinType _M_data;
2552 };
2553
2554// _SimdWrapperBase{{{1
2555template <bool _MustZeroInitPadding, typename _BuiltinType>
2556 struct _SimdWrapperBase;
2557
2558template <typename _BuiltinType>
2559 struct _SimdWrapperBase<false, _BuiltinType> // no padding or no SNaNs
2560 {
2561 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() = default;
2562 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2563 : _M_data(__init)
2564 {}
2565
2566 _BuiltinType _M_data;
2567 };
2568
2569template <typename _BuiltinType>
2570 struct _SimdWrapperBase<true, _BuiltinType> // with padding that needs to
2571 // never become SNaN
2572 {
2573 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() : _M_data() {}
2574 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2575 : _M_data(__init)
2576 {}
2577
2578 _BuiltinType _M_data;
2579 };
2580
2581// }}}
2582// _SimdWrapper{{{
2583template <typename _Tp, size_t _Width>
2584 struct _SimdWrapper<
2585 _Tp, _Width,
2586 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2587 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2588 && sizeof(_Tp) * _Width
2589 == sizeof(__vector_type_t<_Tp, _Width>),
2590 __vector_type_t<_Tp, _Width>>
2591 {
2592 using _Base
2593 = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2594 && sizeof(_Tp) * _Width
2595 == sizeof(__vector_type_t<_Tp, _Width>),
2596 __vector_type_t<_Tp, _Width>>;
2597
2598 static_assert(__is_vectorizable_v<_Tp>);
2599 static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then
2600
2601 using _BuiltinType = __vector_type_t<_Tp, _Width>;
2602 using value_type = _Tp;
2603
2604 static inline constexpr size_t _S_full_size
2605 = sizeof(_BuiltinType) / sizeof(value_type);
2606 static inline constexpr int _S_size = _Width;
2607 static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2608
2609 using _Base::_M_data;
2610
2611 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2612 __as_full_vector() const
2613 { return _M_data; }
2614
2615 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(initializer_list<_Tp> __init)
2616 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2617 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __init.begin()[__i.value]; })) {}
2618
2619 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
2620 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(const _SimdWrapper&)
2621 = default;
2622 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_SimdWrapper&&) = default;
2623
2624 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2625 operator=(const _SimdWrapper&) = default;
2626 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2627 operator=(_SimdWrapper&&) = default;
2628
2629 template <typename _V, typename = enable_if_t<disjunction_v<
2630 is_same<_V, __vector_type_t<_Tp, _Width>>,
2631 is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2632 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_V __x)
2633 // __vector_bitcast can convert e.g. __m128 to __vector(2) float
2634 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2635
2636 template <typename... _As,
2637 typename = enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2638 && sizeof...(_As) <= _Width)>>
2639 _GLIBCXX_SIMD_INTRINSIC constexpr
2640 operator _SimdTuple<_Tp, _As...>() const
2641 {
2642 return __generate_from_n_evaluations<sizeof...(_As), _SimdTuple<_Tp, _As...>>(
2643 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
2644 { return _M_data[int(__i)]; });
2645 }
2646
2647 _GLIBCXX_SIMD_INTRINSIC constexpr operator const _BuiltinType&() const
2648 { return _M_data; }
2649
2650 _GLIBCXX_SIMD_INTRINSIC constexpr operator _BuiltinType&()
2651 { return _M_data; }
2652
2653 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](size_t __i) const
2654 { return _M_data[__i]; }
2655
2656 template <size_t __i>
2657 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](_SizeConstant<__i>) const
2658 { return _M_data[__i]; }
2659
2660 _GLIBCXX_SIMD_INTRINSIC constexpr void _M_set(size_t __i, _Tp __x)
2661 { _M_data[__i] = __x; }
2662
2663 _GLIBCXX_SIMD_INTRINSIC
2664 constexpr bool _M_is_constprop() const
2665 { return __builtin_constant_p(_M_data); }
2666
2667 _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_none_of() const
2668 {
2669 if (__builtin_constant_p(_M_data))
2670 {
2671 bool __r = true;
2672 if constexpr (is_floating_point_v<_Tp>)
2673 {
2674 using _Ip = __int_for_sizeof_t<_Tp>;
2675 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2676 __execute_n_times<_Width>(
2677 [&](auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2678 }
2679 else
2680 __execute_n_times<_Width>(
2681 [&](auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2682 if (__builtin_constant_p(__r))
2683 return __r;
2684 }
2685 return false;
2686 }
2687
2688 _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_all_of() const
2689 {
2690 if (__builtin_constant_p(_M_data))
2691 {
2692 bool __r = true;
2693 if constexpr (is_floating_point_v<_Tp>)
2694 {
2695 using _Ip = __int_for_sizeof_t<_Tp>;
2696 const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2697 __execute_n_times<_Width>(
2698 [&](auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2699 }
2700 else
2701 __execute_n_times<_Width>(
2702 [&](auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2703 if (__builtin_constant_p(__r))
2704 return __r;
2705 }
2706 return false;
2707 }
2708 };
2709
2710// }}}
2711
2712// __vectorized_sizeof {{{
2713template <typename _Tp>
2714 constexpr size_t
2715 __vectorized_sizeof()
2716 {
2717 if constexpr (!__is_vectorizable_v<_Tp>)
2718 return 0;
2719
2720 if constexpr (sizeof(_Tp) <= 8)
2721 {
2722 // X86:
2723 if constexpr (__have_avx512bw)
2724 return 64;
2725 if constexpr (__have_avx512f && sizeof(_Tp) >= 4)
2726 return 64;
2727 if constexpr (__have_avx2)
2728 return 32;
2729 if constexpr (__have_avx && is_floating_point_v<_Tp>)
2730 return 32;
2731 if constexpr (__have_sse2)
2732 return 16;
2733 if constexpr (__have_sse && is_same_v<_Tp, float>)
2734 return 16;
2735 /* The following is too much trouble because of mixed MMX and x87 code.
2736 * While nothing here explicitly calls MMX instructions of registers,
2737 * they are still emitted but no EMMS cleanup is done.
2738 if constexpr (__have_mmx && sizeof(_Tp) <= 4 && is_integral_v<_Tp>)
2739 return 8;
2740 */
2741
2742 // PowerPC:
2743 if constexpr (__have_power8vec
2744 || (__have_power_vmx && (sizeof(_Tp) < 8))
2745 || (__have_power_vsx && is_floating_point_v<_Tp>) )
2746 return 16;
2747
2748 // ARM:
2749 if constexpr (__have_neon_a64
2750 || (__have_neon_a32 && !is_same_v<_Tp, double>) )
2751 return 16;
2752 if constexpr (__have_neon
2753 && sizeof(_Tp) < 8
2754 // Only allow fp if the user allows non-ICE559 fp (e.g.
2755 // via -ffast-math). ARMv7 NEON fp is not conforming to
2756 // IEC559.
2757 && (__support_neon_float || !is_floating_point_v<_Tp>))
2758 return 16;
2759 }
2760
2761 return sizeof(_Tp);
2762 }
2763
2764// }}}
2765namespace simd_abi {
2766// most of simd_abi is defined in simd_detail.h
2767template <typename _Tp>
2768 inline constexpr int max_fixed_size
2769 = (__have_avx512bw && sizeof(_Tp) == 1) ? 64 : 32;
2770
2771// compatible {{{
2772#if defined __x86_64__ || defined __aarch64__
2773template <typename _Tp>
2774 using compatible = conditional_t<(sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2775#elif defined __ARM_NEON
2776// FIXME: not sure, probably needs to be scalar (or dependent on the hard-float
2777// ABI?)
2778template <typename _Tp>
2779 using compatible
2780 = conditional_t<(sizeof(_Tp) < 8
2781 && (__support_neon_float || !is_floating_point_v<_Tp>)),
2782 _VecBuiltin<16>, scalar>;
2783#else
2784template <typename>
2785 using compatible = scalar;
2786#endif
2787
2788// }}}
2789// native {{{
2790template <typename _Tp>
2791 constexpr auto
2792 __determine_native_abi()
2793 {
2794 constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2795 if constexpr (__bytes == sizeof(_Tp))
2796 return static_cast<scalar*>(nullptr);
2797 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2798 return static_cast<_VecBltnBtmsk<__bytes>*>(nullptr);
2799 else
2800 return static_cast<_VecBuiltin<__bytes>*>(nullptr);
2801 }
2802
2803template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2804 using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2805
2806// }}}
2807// __default_abi {{{
2808#if defined _GLIBCXX_SIMD_DEFAULT_ABI
2809template <typename _Tp>
2810 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2811#else
2812template <typename _Tp>
2813 using __default_abi = compatible<_Tp>;
2814#endif
2815
2816// }}}
2817} // namespace simd_abi
2818
2819// traits {{{1
2820// is_abi_tag {{{2
2821template <typename _Tp, typename = void_t<>>
2822 struct is_abi_tag : false_type {};
2823
2824template <typename _Tp>
2825 struct is_abi_tag<_Tp, void_t<typename _Tp::_IsValidAbiTag>>
2826 : public _Tp::_IsValidAbiTag {};
2827
2828template <typename _Tp>
2829 inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2830
2831// is_simd(_mask) {{{2
2832template <typename _Tp>
2833 struct is_simd : public false_type {};
2834
2835template <typename _Tp>
2836 inline constexpr bool is_simd_v = is_simd<_Tp>::value;
2837
2838template <typename _Tp>
2839 struct is_simd_mask : public false_type {};
2840
2841template <typename _Tp>
2842inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2843
2844// simd_size {{{2
2845template <typename _Tp, typename _Abi, typename = void>
2846 struct __simd_size_impl {};
2847
2848template <typename _Tp, typename _Abi>
2849 struct __simd_size_impl<
2850 _Tp, _Abi,
2851 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2852 : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2853
2854template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2855 struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2856
2857template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2858 inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2859
2860// simd_abi::deduce {{{2
2861template <typename _Tp, size_t _Np, typename = void>
2862 struct __deduce_impl;
2863
2864namespace simd_abi {
2865/**
2866 * @tparam _Tp The requested `value_type` for the elements.
2867 * @tparam _Np The requested number of elements.
2868 * @tparam _Abis This parameter is ignored, since this implementation cannot
2869 * make any use of it. Either __a good native ABI is matched and used as `type`
2870 * alias, or the `fixed_size<_Np>` ABI is used, which internally is built from
2871 * the best matching native ABIs.
2872 */
2873template <typename _Tp, size_t _Np, typename...>
2874 struct deduce : __deduce_impl<_Tp, _Np> {};
2875
2876template <typename _Tp, size_t _Np, typename... _Abis>
2877 using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type;
2878} // namespace simd_abi
2879
2880// }}}2
2881// rebind_simd {{{2
2882template <typename _Tp, typename _V, typename = void>
2883 struct rebind_simd;
2884
2885template <typename _Tp, typename _Up, typename _Abi>
2886 struct rebind_simd<
2887 _Tp, simd<_Up, _Abi>,
2888 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2889 {
2890 using type
2891 = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2892 };
2893
2894template <typename _Tp, typename _Up, typename _Abi>
2895 struct rebind_simd<
2896 _Tp, simd_mask<_Up, _Abi>,
2897 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2898 {
2899 using type
2900 = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2901 };
2902
2903template <typename _Tp, typename _V>
2904 using rebind_simd_t = typename rebind_simd<_Tp, _V>::type;
2905
2906// resize_simd {{{2
2907template <int _Np, typename _V, typename = void>
2908 struct resize_simd;
2909
2910template <int _Np, typename _Tp, typename _Abi>
2911 struct resize_simd<_Np, simd<_Tp, _Abi>,
2912 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2913 { using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2914
2915template <int _Np, typename _Tp, typename _Abi>
2916 struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
2917 void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2918 { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2919
2920template <int _Np, typename _V>
2921 using resize_simd_t = typename resize_simd<_Np, _V>::type;
2922
2923// }}}2
2924// memory_alignment {{{2
2925template <typename _Tp, typename _Up = typename _Tp::value_type>
2926 struct memory_alignment
2927 : public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2928
2929template <typename _Tp, typename _Up = typename _Tp::value_type>
2930 inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2931
2932// class template simd [simd] {{{1
2933template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2934 class simd;
2935
2936template <typename _Tp, typename _Abi>
2937 struct is_simd<simd<_Tp, _Abi>> : public true_type {};
2938
2939template <typename _Tp>
2940 using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2941
2942template <typename _Tp, int _Np>
2943 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2944
2945template <typename _Tp, size_t _Np>
2946 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2947
2948// class template simd_mask [simd_mask] {{{1
2949template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2950 class simd_mask;
2951
2952template <typename _Tp, typename _Abi>
2953 struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {};
2954
2955template <typename _Tp>
2956 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2957
2958template <typename _Tp, int _Np>
2959 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2960
2961template <typename _Tp, size_t _Np>
2962 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2963
2964// casts [simd.casts] {{{1
2965// static_simd_cast {{{2
2966template <typename _Tp, typename _Up, typename _Ap, bool = is_simd_v<_Tp>,
2967 typename = void>
2968 struct __static_simd_cast_return_type;
2969
2970template <typename _Tp, typename _A0, typename _Up, typename _Ap>
2971 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false,
2972 void>
2973 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2974
2975template <typename _Tp, typename _Up, typename _Ap>
2976 struct __static_simd_cast_return_type<
2977 _Tp, _Up, _Ap, true, enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
2978 { using type = _Tp; };
2979
2980template <typename _Tp, typename _Ap>
2981 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2982#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2983 enable_if_t<__is_vectorizable_v<_Tp>>
2984#else
2985 void
2986#endif
2987 >
2988 { using type = simd<_Tp, _Ap>; };
2989
2990template <typename _Tp, typename = void>
2991 struct __safe_make_signed { using type = _Tp;};
2992
2993template <typename _Tp>
2994 struct __safe_make_signed<_Tp, enable_if_t<is_integral_v<_Tp>>>
2995 {
2996 // the extra make_unsigned_t is because of PR85951
2997 using type = make_signed_t<make_unsigned_t<_Tp>>;
2998 };
2999
3000template <typename _Tp>
3001 using safe_make_signed_t = typename __safe_make_signed<_Tp>::type;
3002
3003template <typename _Tp, typename _Up, typename _Ap>
3004 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
3005#ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
3006 enable_if_t<__is_vectorizable_v<_Tp>>
3007#else
3008 void
3009#endif
3010 >
3011 {
3012 using type = conditional_t<
3013 (is_integral_v<_Up> && is_integral_v<_Tp> &&
3014#ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
3015 is_signed_v<_Up> != is_signed_v<_Tp> &&
3016#endif
3017 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
3018 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
3019 };
3020
3021template <typename _Tp, typename _Up, typename _Ap,
3022 typename _R
3023 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3024 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
3025 static_simd_cast(const simd<_Up, _Ap>& __x)
3026 {
3027 if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
3028 return __x;
3029 else
3030 {
3031 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
3032 __c;
3033 return _R(__private_init, __c(__data(__x)));
3034 }
3035 }
3036
3037namespace __proposed {
3038template <typename _Tp, typename _Up, typename _Ap,
3039 typename _R
3040 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3041 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR typename _R::mask_type
3042 static_simd_cast(const simd_mask<_Up, _Ap>& __x)
3043 {
3044 using _RM = typename _R::mask_type;
3045 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
3046 typename _RM::simd_type::value_type>(__x)};
3047 }
3048
3049template <typename _To, typename _Up, typename _Abi>
3050 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3051 _To
3052 simd_bit_cast(const simd<_Up, _Abi>& __x)
3053 {
3054 using _Tp = typename _To::value_type;
3055 using _ToMember = typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3056 using _From = simd<_Up, _Abi>;
3057 using _FromMember = typename _SimdTraits<_Up, _Abi>::_SimdMember;
3058 // with concepts, the following should be constraints
3059 static_assert(sizeof(_To) == sizeof(_From));
3060 static_assert(is_trivially_copyable_v<_Tp> && is_trivially_copyable_v<_Up>);
3061 static_assert(is_trivially_copyable_v<_ToMember> && is_trivially_copyable_v<_FromMember>);
3062#if __has_builtin(__builtin_bit_cast)
3063 return {__private_init, __builtin_bit_cast(_ToMember, __data(__x))};
3064#else
3065 return {__private_init, __bit_cast<_ToMember>(__data(__x))};
3066#endif
3067 }
3068
3069template <typename _To, typename _Up, typename _Abi>
3070 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3071 _To
3072 simd_bit_cast(const simd_mask<_Up, _Abi>& __x)
3073 {
3074 using _From = simd_mask<_Up, _Abi>;
3075 static_assert(sizeof(_To) == sizeof(_From));
3076 static_assert(is_trivially_copyable_v<_From>);
3077 // _To can be simd<T, A>, specifically simd<T, fixed_size<N>> in which case _To is not trivially
3078 // copyable.
3079 if constexpr (is_simd_v<_To>)
3080 {
3081 using _Tp = typename _To::value_type;
3082 using _ToMember = typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3083 static_assert(is_trivially_copyable_v<_ToMember>);
3084#if __has_builtin(__builtin_bit_cast)
3085 return {__private_init, __builtin_bit_cast(_ToMember, __x)};
3086#else
3087 return {__private_init, __bit_cast<_ToMember>(__x)};
3088#endif
3089 }
3090 else
3091 {
3092 static_assert(is_trivially_copyable_v<_To>);
3093#if __has_builtin(__builtin_bit_cast)
3094 return __builtin_bit_cast(_To, __x);
3095#else
3096 return __bit_cast<_To>(__x);
3097#endif
3098 }
3099 }
3100} // namespace __proposed
3101
3102// simd_cast {{{2
3103template <typename _Tp, typename _Up, typename _Ap,
3104 typename _To = __value_type_or_identity_t<_Tp>>
3105 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3106 simd_cast(const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
3107 -> decltype(static_simd_cast<_Tp>(__x))
3108 { return static_simd_cast<_Tp>(__x); }
3109
3110namespace __proposed {
3111template <typename _Tp, typename _Up, typename _Ap,
3112 typename _To = __value_type_or_identity_t<_Tp>>
3113 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3114 simd_cast(const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
3115 -> decltype(static_simd_cast<_Tp>(__x))
3116 { return static_simd_cast<_Tp>(__x); }
3117} // namespace __proposed
3118
3119// }}}2
3120// resizing_simd_cast {{{
3121namespace __proposed {
3122/* Proposed spec:
3123
3124template <class T, class U, class Abi>
3125T resizing_simd_cast(const simd<U, Abi>& x)
3126
3127p1 Constraints:
3128 - is_simd_v<T> is true and
3129 - T::value_type is the same type as U
3130
3131p2 Returns:
3132 A simd object with the i^th element initialized to x[i] for all i in the
3133 range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3134 than simd_size_v<U, Abi>, the remaining elements are value-initialized.
3135
3136template <class T, class U, class Abi>
3137T resizing_simd_cast(const simd_mask<U, Abi>& x)
3138
3139p1 Constraints: is_simd_mask_v<T> is true
3140
3141p2 Returns:
3142 A simd_mask object with the i^th element initialized to x[i] for all i in
3143the range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3144 than simd_size_v<U, Abi>, the remaining elements are initialized to false.
3145
3146 */
3147
3148template <typename _Tp, typename _Up, typename _Ap>
3149 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR enable_if_t<
3150 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3151 resizing_simd_cast(const simd<_Up, _Ap>& __x)
3152 {
3153 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3154 return __x;
3155 else if constexpr (simd_size_v<_Up, _Ap> == 1)
3156 {
3157 _Tp __r{};
3158 __r[0] = __x[0];
3159 return __r;
3160 }
3161 else if constexpr (_Tp::size() == 1)
3162 return __x[0];
3163 else if constexpr (sizeof(_Tp) == sizeof(__x)
3164 && !__is_fixed_size_abi_v<_Ap>)
3165 return {__private_init,
3166 __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3167 _Ap::_S_masked(__data(__x))._M_data)};
3168 else
3169 {
3170 _Tp __r{};
3171 __builtin_memcpy(&__data(__r), &__data(__x),
3172 sizeof(_Up)
3173 * std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3174 return __r;
3175 }
3176 }
3177
3178template <typename _Tp, typename _Up, typename _Ap>
3179 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3180 enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3181 resizing_simd_cast(const simd_mask<_Up, _Ap>& __x)
3182 {
3183 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3184 typename _Tp::simd_type::value_type>(__x)};
3185 }
3186} // namespace __proposed
3187
3188// }}}
3189// to_fixed_size {{{2
3190template <typename _Tp, int _Np>
3191 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3192 to_fixed_size(const fixed_size_simd<_Tp, _Np>& __x)
3193 { return __x; }
3194
3195template <typename _Tp, int _Np>
3196 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3197 to_fixed_size(const fixed_size_simd_mask<_Tp, _Np>& __x)
3198 { return __x; }
3199
3200template <typename _Tp, typename _Ap>
3201 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>
3202 to_fixed_size(const simd<_Tp, _Ap>& __x)
3203 {
3204 using _Rp = fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>;
3205 return _Rp([&__x](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3206 }
3207
3208template <typename _Tp, typename _Ap>
3209 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, simd_size_v<_Tp, _Ap>>
3210 to_fixed_size(const simd_mask<_Tp, _Ap>& __x)
3211 {
3212 return {__private_init,
3213 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }};
3214 }
3215
3216// to_native {{{2
3217template <typename _Tp, int _Np>
3218 _GLIBCXX_SIMD_INTRINSIC
3219 enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3220 to_native(const fixed_size_simd<_Tp, _Np>& __x)
3221 {
3222 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3223 __x.copy_to(__mem, vector_aligned);
3224 return {__mem, vector_aligned};
3225 }
3226
3227template <typename _Tp, size_t _Np>
3228 _GLIBCXX_SIMD_INTRINSIC
3229 enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3230 to_native(const fixed_size_simd_mask<_Tp, _Np>& __x)
3231 {
3232 return native_simd_mask<_Tp>(
3233 __private_init,
3234 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3235 }
3236
3237// to_compatible {{{2
3238template <typename _Tp, size_t _Np>
3239 _GLIBCXX_SIMD_INTRINSIC enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3240 to_compatible(const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3241 {
3242 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3243 __x.copy_to(__mem, vector_aligned);
3244 return {__mem, vector_aligned};
3245 }
3246
3247template <typename _Tp, size_t _Np>
3248 _GLIBCXX_SIMD_INTRINSIC
3249 enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3250 to_compatible(const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3251 {
3252 return simd_mask<_Tp>(
3253 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3254 }
3255
3256// masked assignment [simd_mask.where] {{{1
3257
3258// where_expression {{{1
3259// const_where_expression<M, T> {{{2
3260template <typename _M, typename _Tp>
3261 class const_where_expression
3262 {
3263 using _V = _Tp;
3264 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3265
3266 struct _Wrapper { using value_type = _V; };
3267
3268 protected:
3269 using _Impl = typename _V::_Impl;
3270
3271 using value_type =
3272 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3273
3274 _GLIBCXX_SIMD_INTRINSIC friend const _M&
3275 __get_mask(const const_where_expression& __x)
3276 { return __x._M_k; }
3277
3278 _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3279 __get_lvalue(const const_where_expression& __x)
3280 { return __x._M_value; }
3281
3282 const _M& _M_k;
3283 _Tp& _M_value;
3284
3285 public:
3286 const_where_expression(const const_where_expression&) = delete;
3287 const_where_expression& operator=(const const_where_expression&) = delete;
3288
3289 _GLIBCXX_SIMD_INTRINSIC const_where_expression(const _M& __kk, const _Tp& dd)
3290 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3291
3292 _GLIBCXX_SIMD_INTRINSIC _V
3293 operator-() const&&
3294 {
3295 return {__private_init,
3296 _Impl::template _S_masked_unary<negate>(__data(_M_k),
3297 __data(_M_value))};
3298 }
3299
3300 template <typename _Up, typename _Flags>
3301 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3302 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3303 {
3304 return {__private_init,
3305 _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3306 _Flags::template _S_apply<_V>(__mem))};
3307 }
3308
3309 template <typename _Up, typename _Flags>
3310 _GLIBCXX_SIMD_INTRINSIC void
3311 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3312 {
3313 _Impl::_S_masked_store(__data(_M_value),
3314 _Flags::template _S_apply<_V>(__mem),
3315 __data(_M_k));
3316 }
3317 };
3318
3319// const_where_expression<bool, T> {{{2
3320template <typename _Tp>
3321 class const_where_expression<bool, _Tp>
3322 {
3323 using _M = bool;
3324 using _V = _Tp;
3325
3326 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3327
3328 struct _Wrapper { using value_type = _V; };
3329
3330 protected:
3331 using value_type =
3332 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3333
3334 _GLIBCXX_SIMD_INTRINSIC friend const _M&
3335 __get_mask(const const_where_expression& __x)
3336 { return __x._M_k; }
3337
3338 _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3339 __get_lvalue(const const_where_expression& __x)
3340 { return __x._M_value; }
3341
3342 const bool _M_k;
3343 _Tp& _M_value;
3344
3345 public:
3346 const_where_expression(const const_where_expression&) = delete;
3347 const_where_expression& operator=(const const_where_expression&) = delete;
3348
3349 _GLIBCXX_SIMD_INTRINSIC const_where_expression(const bool __kk, const _Tp& dd)
3350 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3351
3352 _GLIBCXX_SIMD_INTRINSIC _V operator-() const&&
3353 { return _M_k ? -_M_value : _M_value; }
3354
3355 template <typename _Up, typename _Flags>
3356 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3357 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3358 { return _M_k ? static_cast<_V>(__mem[0]) : _M_value; }
3359
3360 template <typename _Up, typename _Flags>
3361 _GLIBCXX_SIMD_INTRINSIC void
3362 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3363 {
3364 if (_M_k)
3365 __mem[0] = _M_value;
3366 }
3367 };
3368
3369// where_expression<M, T> {{{2
3370template <typename _M, typename _Tp>
3371 class where_expression : public const_where_expression<_M, _Tp>
3372 {
3373 using _Impl = typename const_where_expression<_M, _Tp>::_Impl;
3374
3375 static_assert(!is_const<_Tp>::value,
3376 "where_expression may only be instantiated with __a non-const "
3377 "_Tp parameter");
3378
3379 using typename const_where_expression<_M, _Tp>::value_type;
3380 using const_where_expression<_M, _Tp>::_M_k;
3381 using const_where_expression<_M, _Tp>::_M_value;
3382
3383 static_assert(
3384 is_same<typename _M::abi_type, typename _Tp::abi_type>::value, "");
3385 static_assert(_M::size() == _Tp::size(), "");
3386
3387 _GLIBCXX_SIMD_INTRINSIC friend _Tp& __get_lvalue(where_expression& __x)
3388 { return __x._M_value; }
3389
3390 public:
3391 where_expression(const where_expression&) = delete;
3392 where_expression& operator=(const where_expression&) = delete;
3393
3394 _GLIBCXX_SIMD_INTRINSIC where_expression(const _M& __kk, _Tp& dd)
3395 : const_where_expression<_M, _Tp>(__kk, dd) {}
3396
3397 template <typename _Up>
3398 _GLIBCXX_SIMD_INTRINSIC void operator=(_Up&& __x) &&
3399 {
3400 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3401 __to_value_type_or_member_type<_Tp>(
3402 static_cast<_Up&&>(__x)));
3403 }
3404
3405#define _GLIBCXX_SIMD_OP_(__op, __name) \
3406 template <typename _Up> \
3407 _GLIBCXX_SIMD_INTRINSIC void operator __op##=(_Up&& __x)&& \
3408 { \
3409 _Impl::template _S_masked_cassign( \
3410 __data(_M_k), __data(_M_value), \
3411 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3412 [](auto __impl, auto __lhs, auto __rhs) \
3413 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \
3414 { return __impl.__name(__lhs, __rhs); }); \
3415 } \
3416 static_assert(true)
3417 _GLIBCXX_SIMD_OP_(+, _S_plus);
3418 _GLIBCXX_SIMD_OP_(-, _S_minus);
3419 _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3420 _GLIBCXX_SIMD_OP_(/, _S_divides);
3421 _GLIBCXX_SIMD_OP_(%, _S_modulus);
3422 _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3423 _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3424 _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3425 _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3426 _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3427#undef _GLIBCXX_SIMD_OP_
3428
3429 _GLIBCXX_SIMD_INTRINSIC void operator++() &&
3430 {
3431 __data(_M_value)
3432 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3433 __data(_M_value));
3434 }
3435
3436 _GLIBCXX_SIMD_INTRINSIC void operator++(int) &&
3437 {
3438 __data(_M_value)
3439 = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3440 __data(_M_value));
3441 }
3442
3443 _GLIBCXX_SIMD_INTRINSIC void operator--() &&
3444 {
3445 __data(_M_value)
3446 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3447 __data(_M_value));
3448 }
3449
3450 _GLIBCXX_SIMD_INTRINSIC void operator--(int) &&
3451 {
3452 __data(_M_value)
3453 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3454 __data(_M_value));
3455 }
3456
3457 // intentionally hides const_where_expression::copy_from
3458 template <typename _Up, typename _Flags>
3459 _GLIBCXX_SIMD_INTRINSIC void
3460 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3461 {
3462 __data(_M_value)
3463 = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3464 _Flags::template _S_apply<_Tp>(__mem));
3465 }
3466 };
3467
3468// where_expression<bool, T> {{{2
3469template <typename _Tp>
3470 class where_expression<bool, _Tp> : public const_where_expression<bool, _Tp>
3471 {
3472 using _M = bool;
3473 using typename const_where_expression<_M, _Tp>::value_type;
3474 using const_where_expression<_M, _Tp>::_M_k;
3475 using const_where_expression<_M, _Tp>::_M_value;
3476
3477 public:
3478 where_expression(const where_expression&) = delete;
3479 where_expression& operator=(const where_expression&) = delete;
3480
3481 _GLIBCXX_SIMD_INTRINSIC where_expression(const _M& __kk, _Tp& dd)
3482 : const_where_expression<_M, _Tp>(__kk, dd) {}
3483
3484#define _GLIBCXX_SIMD_OP_(__op) \
3485 template <typename _Up> \
3486 _GLIBCXX_SIMD_INTRINSIC void operator __op(_Up&& __x)&& \
3487 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3488
3489 _GLIBCXX_SIMD_OP_(=)
3490 _GLIBCXX_SIMD_OP_(+=)
3491 _GLIBCXX_SIMD_OP_(-=)
3492 _GLIBCXX_SIMD_OP_(*=)
3493 _GLIBCXX_SIMD_OP_(/=)
3494 _GLIBCXX_SIMD_OP_(%=)
3495 _GLIBCXX_SIMD_OP_(&=)
3496 _GLIBCXX_SIMD_OP_(|=)
3497 _GLIBCXX_SIMD_OP_(^=)
3498 _GLIBCXX_SIMD_OP_(<<=)
3499 _GLIBCXX_SIMD_OP_(>>=)
3500 #undef _GLIBCXX_SIMD_OP_
3501
3502 _GLIBCXX_SIMD_INTRINSIC void operator++() &&
3503 { if (_M_k) ++_M_value; }
3504
3505 _GLIBCXX_SIMD_INTRINSIC void operator++(int) &&
3506 { if (_M_k) ++_M_value; }
3507
3508 _GLIBCXX_SIMD_INTRINSIC void operator--() &&
3509 { if (_M_k) --_M_value; }
3510
3511 _GLIBCXX_SIMD_INTRINSIC void operator--(int) &&
3512 { if (_M_k) --_M_value; }
3513
3514 // intentionally hides const_where_expression::copy_from
3515 template <typename _Up, typename _Flags>
3516 _GLIBCXX_SIMD_INTRINSIC void
3517 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3518 { if (_M_k) _M_value = __mem[0]; }
3519 };
3520
3521// where {{{1
3522template <typename _Tp, typename _Ap>
3523 _GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3524 where(const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3525 { return {__k, __value}; }
3526
3527template <typename _Tp, typename _Ap>
3528 _GLIBCXX_SIMD_INTRINSIC
3529 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3530 where(const typename simd<_Tp, _Ap>::mask_type& __k,
3531 const simd<_Tp, _Ap>& __value)
3532 { return {__k, __value}; }
3533
3534template <typename _Tp, typename _Ap>
3535 _GLIBCXX_SIMD_INTRINSIC
3536 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3537 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
3538 simd_mask<_Tp, _Ap>& __value)
3539 { return {__k, __value}; }
3540
3541template <typename _Tp, typename _Ap>
3542 _GLIBCXX_SIMD_INTRINSIC
3543 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3544 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
3545 const simd_mask<_Tp, _Ap>& __value)
3546 { return {__k, __value}; }
3547
3548template <typename _Tp>
3549 _GLIBCXX_SIMD_INTRINSIC where_expression<bool, _Tp>
3550 where(_ExactBool __k, _Tp& __value)
3551 { return {__k, __value}; }
3552
3553template <typename _Tp>
3554 _GLIBCXX_SIMD_INTRINSIC const_where_expression<bool, _Tp>
3555 where(_ExactBool __k, const _Tp& __value)
3556 { return {__k, __value}; }
3557
3558 template <typename _Tp, typename _Ap>
3559 void where(bool __k, simd<_Tp, _Ap>& __value) = delete;
3560
3561 template <typename _Tp, typename _Ap>
3562 void where(bool __k, const simd<_Tp, _Ap>& __value) = delete;
3563
3564// proposed mask iterations {{{1
3565namespace __proposed {
3566template <size_t _Np>
3567 class where_range
3568 {
3569 const bitset<_Np> __bits;
3570
3571 public:
3572 where_range(bitset<_Np> __b) : __bits(__b) {}
3573
3574 class iterator
3575 {
3576 size_t __mask;
3577 size_t __bit;
3578
3579 _GLIBCXX_SIMD_INTRINSIC void __next_bit()
3580 { __bit = __builtin_ctzl(__mask); }
3581
3582 _GLIBCXX_SIMD_INTRINSIC void __reset_lsb()
3583 {
3584 // 01100100 - 1 = 01100011
3585 __mask &= (__mask - 1);
3586 // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit));
3587 }
3588
3589 public:
3590 iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3591 iterator(const iterator&) = default;
3592 iterator(iterator&&) = default;
3593
3594 _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator->() const
3595 { return __bit; }
3596
3597 _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator*() const
3598 { return __bit; }
3599
3600 _GLIBCXX_SIMD_ALWAYS_INLINE iterator& operator++()
3601 {
3602 __reset_lsb();
3603 __next_bit();
3604 return *this;
3605 }
3606
3607 _GLIBCXX_SIMD_ALWAYS_INLINE iterator operator++(int)
3608 {
3609 iterator __tmp = *this;
3610 __reset_lsb();
3611 __next_bit();
3612 return __tmp;
3613 }
3614
3615 _GLIBCXX_SIMD_ALWAYS_INLINE bool operator==(const iterator& __rhs) const
3616 { return __mask == __rhs.__mask; }
3617
3618 _GLIBCXX_SIMD_ALWAYS_INLINE bool operator!=(const iterator& __rhs) const
3619 { return __mask != __rhs.__mask; }
3620 };
3621
3622 iterator begin() const
3623 { return __bits.to_ullong(); }
3624
3625 iterator end() const
3626 { return 0; }
3627 };
3628
3629template <typename _Tp, typename _Ap>
3630 where_range<simd_size_v<_Tp, _Ap>>
3631 where(const simd_mask<_Tp, _Ap>& __k)
3632 { return __k.__to_bitset(); }
3633
3634} // namespace __proposed
3635
3636// }}}1
3637// reductions [simd.reductions] {{{1
3638template <typename _Tp, typename _Abi, typename _BinaryOperation = plus<>>
3639 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3640 reduce(const simd<_Tp, _Abi>& __v,
3641 _BinaryOperation __binary_op = _BinaryOperation())
3642 { return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3643
3644template <typename _M, typename _V, typename _BinaryOperation = plus<>>
3645 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3646 reduce(const const_where_expression<_M, _V>& __x,
3647 typename _V::value_type __identity_element,
3648 _BinaryOperation __binary_op)
3649 {
3650 if (__builtin_expect(none_of(__get_mask(__x)), false))
3651 return __identity_element;
3652
3653 _V __tmp = __identity_element;
3654 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3655 __data(__get_lvalue(__x)));
3656 return reduce(__tmp, __binary_op);
3657 }
3658
3659template <typename _M, typename _V>
3660 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3661 reduce(const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3662 { return reduce(__x, 0, __binary_op); }
3663
3664template <typename _M, typename _V>
3665 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3666 reduce(const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3667 { return reduce(__x, 1, __binary_op); }
3668
3669template <typename _M, typename _V>
3670 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3671 reduce(const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3672 { return reduce(__x, ~typename _V::value_type(), __binary_op); }
3673
3674template <typename _M, typename _V>
3675 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3676 reduce(const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3677 { return reduce(__x, 0, __binary_op); }
3678
3679template <typename _M, typename _V>
3680 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3681 reduce(const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3682 { return reduce(__x, 0, __binary_op); }
3683
3684template <typename _Tp, typename _Abi>
3685 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3686 hmin(const simd<_Tp, _Abi>& __v) noexcept
3687 {
3688 return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum());
3689 }
3690
3691template <typename _Tp, typename _Abi>
3692 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3693 hmax(const simd<_Tp, _Abi>& __v) noexcept
3694 {
3695 return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum());
3696 }
3697
3698template <typename _M, typename _V>
3699 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3700 typename _V::value_type
3701 hmin(const const_where_expression<_M, _V>& __x) noexcept
3702 {
3703 using _Tp = typename _V::value_type;
3704 constexpr _Tp __id_elem =
3705#ifdef __FINITE_MATH_ONLY__
3706 __finite_max_v<_Tp>;
3707#else
3708 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3709#endif
3710 _V __tmp = __id_elem;
3711 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3712 __data(__get_lvalue(__x)));
3713 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3714 }
3715
3716template <typename _M, typename _V>
3717 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3718 typename _V::value_type
3719 hmax(const const_where_expression<_M, _V>& __x) noexcept
3720 {
3721 using _Tp = typename _V::value_type;
3722 constexpr _Tp __id_elem =
3723#ifdef __FINITE_MATH_ONLY__
3724 __finite_min_v<_Tp>;
3725#else
3726 [] {
3727 if constexpr (__value_exists_v<__infinity, _Tp>)
3728 return -__infinity_v<_Tp>;
3729 else
3730 return __finite_min_v<_Tp>;
3731 }();
3732#endif
3733 _V __tmp = __id_elem;
3734 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3735 __data(__get_lvalue(__x)));
3736 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3737 }
3738
3739// }}}1
3740// algorithms [simd.alg] {{{
3741template <typename _Tp, typename _Ap>
3742 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3743 min(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3744 { return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3745
3746template <typename _Tp, typename _Ap>
3747 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3748 max(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3749 { return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3750
3751template <typename _Tp, typename _Ap>
3752 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3753 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3754 minmax(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3755 {
3756 const auto pair_of_members
3757 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3758 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3759 simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3760 }
3761
3762template <typename _Tp, typename _Ap>
3763 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3764 clamp(const simd<_Tp, _Ap>& __v, const simd<_Tp, _Ap>& __lo,
3765 const simd<_Tp, _Ap>& __hi)
3766 {
3767 using _Impl = typename _Ap::_SimdImpl;
3768 return {__private_init,
3769 _Impl::_S_min(__data(__hi),
3770 _Impl::_S_max(__data(__lo), __data(__v)))};
3771 }
3772
3773// }}}
3774
3775template <size_t... _Sizes, typename _Tp, typename _Ap,
3776 typename = enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3777 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3778 split(const simd<_Tp, _Ap>&);
3779
3780// __extract_part {{{
3781template <int _Index, int _Total, int _Combine = 1, typename _Tp, size_t _Np>
3782 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
3783 _SimdWrapper<_Tp, _Np / _Total * _Combine>
3784 __extract_part(const _SimdWrapper<_Tp, _Np> __x);
3785
3786template <int _Index, int _Parts, int _Combine = 1, typename _Tp, typename _A0,
3787 typename... _As>
3788 _GLIBCXX_SIMD_INTRINSIC auto
3789 __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x);
3790
3791// }}}
3792// _SizeList {{{
3793template <size_t _V0, size_t... _Values>
3794 struct _SizeList
3795 {
3796 template <size_t _I>
3797 static constexpr size_t _S_at(_SizeConstant<_I> = {})
3798 {
3799 if constexpr (_I == 0)
3800 return _V0;
3801 else
3802 return _SizeList<_Values...>::template _S_at<_I - 1>();
3803 }
3804
3805 template <size_t _I>
3806 static constexpr auto _S_before(_SizeConstant<_I> = {})
3807 {
3808 if constexpr (_I == 0)
3809 return _SizeConstant<0>();
3810 else
3811 return _SizeConstant<
3812 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3813 }
3814
3815 template <size_t _Np>
3816 static constexpr auto _S_pop_front(_SizeConstant<_Np> = {})
3817 {
3818 if constexpr (_Np == 0)
3819 return _SizeList();
3820 else
3821 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3822 }
3823 };
3824
3825// }}}
3826// __extract_center {{{
3827template <typename _Tp, size_t _Np>
3828 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3829 __extract_center(_SimdWrapper<_Tp, _Np> __x)
3830 {
3831 static_assert(_Np >= 4);
3832 static_assert(_Np % 4 == 0); // x0 - x1 - x2 - x3 -> return {x1, x2}
3833#if _GLIBCXX_SIMD_X86INTRIN // {{{
3834 if constexpr (__have_avx512f && sizeof(_Tp) * _Np == 64)
3835 {
3836 const auto __intrin = __to_intrin(__x);
3837 if constexpr (is_integral_v<_Tp>)
3838 return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3839 _mm512_shuffle_i32x4(__intrin, __intrin,
3840 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3841 else if constexpr (sizeof(_Tp) == 4)
3842 return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3843 _mm512_shuffle_f32x4(__intrin, __intrin,
3844 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3845 else if constexpr (sizeof(_Tp) == 8)
3846 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3847 _mm512_shuffle_f64x2(__intrin, __intrin,
3848 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3849 else
3850 __assert_unreachable<_Tp>();
3851 }
3852 else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3853 return __vector_bitcast<_Tp>(
3854 _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
3855 __hi128(__vector_bitcast<double>(__x)), 1));
3856 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
3857 return __vector_bitcast<_Tp>(
3858 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3859 __lo128(__vector_bitcast<_LLong>(__x)),
3860 sizeof(_Tp) * _Np / 4));
3861 else
3862#endif // _GLIBCXX_SIMD_X86INTRIN }}}
3863 {
3864 __vector_type_t<_Tp, _Np / 2> __r;
3865 __builtin_memcpy(&__r,
3866 reinterpret_cast<const char*>(&__x)
3867 + sizeof(_Tp) * _Np / 4,
3868 sizeof(_Tp) * _Np / 2);
3869 return __r;
3870 }
3871 }
3872
3873template <typename _Tp, typename _A0, typename... _As>
3874 _GLIBCXX_SIMD_INTRINSIC
3875 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3876 __extract_center(const _SimdTuple<_Tp, _A0, _As...>& __x)
3877 {
3878 if constexpr (sizeof...(_As) == 0)
3879 return __extract_center(__x.first);
3880 else
3881 return __extract_part<1, 4, 2>(__x);
3882 }
3883
3884// }}}
3885// __split_wrapper {{{
3886template <size_t... _Sizes, typename _Tp, typename... _As>
3887 auto
3888 __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
3889 {
3890 return split<_Sizes...>(
3891 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3892 __x));
3893 }
3894
3895// }}}
3896
3897// split<simd>(simd) {{{
3898template <typename _V, typename _Ap,
3899 size_t _Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
3900 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == _Parts * _V::size()
3901 && is_simd_v<_V>, array<_V, _Parts>>
3902 split(const simd<typename _V::value_type, _Ap>& __x)
3903 {
3904 using _Tp = typename _V::value_type;
3905 if constexpr (_Parts == 1)
3906 {
3907 return {simd_cast<_V>(__x)};
3908 }
3909 else if (__x._M_is_constprop())
3910 {
3911 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3912 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3913 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
3914 { return __x[__i * _V::size() + __j]; });
3915 });
3916 }
3917 else if constexpr (
3918 __is_fixed_size_abi_v<_Ap>
3919 && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3920 || (__is_fixed_size_abi_v<typename _V::abi_type>
3921 && sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding
3922 )))
3923 {
3924 // fixed_size -> fixed_size (w/o padding) or scalar
3925#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3926 const __may_alias<_Tp>* const __element_ptr
3927 = reinterpret_cast<const __may_alias<_Tp>*>(&__data(__x));
3928 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3929 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
3930 { return _V(__element_ptr + __i * _V::size(), vector_aligned); });
3931#else
3932 const auto& __xx = __data(__x);
3933 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3934 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3935 [[maybe_unused]] constexpr size_t __offset
3936 = decltype(__i)::value * _V::size();
3937 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3938 constexpr _SizeConstant<__j + __offset> __k;
3939 return __xx[__k];
3940 });
3941 });
3942#endif
3943 }
3944 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3945 {
3946 // normally memcpy should work here as well
3947 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3948 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; });
3949 }
3950 else
3951 {
3952 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3953 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3954 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3955 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3956 return __x[__i * _V::size() + __j];
3957 });
3958 else
3959 return _V(__private_init,
3960 __extract_part<decltype(__i)::value, _Parts>(__data(__x)));
3961 });
3962 }
3963 }
3964
3965// }}}
3966// split<simd_mask>(simd_mask) {{{
3967template <typename _V, typename _Ap,
3968 size_t _Parts
3969 = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
3970 enable_if_t<is_simd_mask_v<_V> && simd_size_v<typename
3971 _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
3972 split(const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3973 {
3974 if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3975 return {__x};
3976 else if constexpr (_Parts == 1)
3977 return {__proposed::static_simd_cast<_V>(__x)};
3978 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3979 && __is_avx_abi<_Ap>())
3980 return {_V(__private_init, __lo128(__data(__x))),
3981 _V(__private_init, __hi128(__data(__x)))};
3982 else if constexpr (_V::size() <= __CHAR_BIT__ * sizeof(_ULLong))
3983 {
3984 const bitset __bits = __x.__to_bitset();
3985 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3986 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3987 constexpr size_t __offset = __i * _V::size();
3988 return _V(__bitset_init, (__bits >> __offset).to_ullong());
3989 });
3990 }
3991 else
3992 {
3993 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>(
3994 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3995 constexpr size_t __offset = __i * _V::size();
3996 return _V(__private_init,
3997 [&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
3998 return __x[__j + __offset];
3999 });
4000 });
4001 }
4002 }
4003
4004// }}}
4005// split<_Sizes...>(simd) {{{
4006template <size_t... _Sizes, typename _Tp, typename _Ap, typename>
4007 _GLIBCXX_SIMD_ALWAYS_INLINE
4008 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
4009 split(const simd<_Tp, _Ap>& __x)
4010 {
4011 using _SL = _SizeList<_Sizes...>;
4012 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
4013 constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
4014 constexpr size_t _N0 = _SL::template _S_at<0>();
4015 using _V = __deduced_simd<_Tp, _N0>;
4016
4017 if (__x._M_is_constprop())
4018 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4019 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4020 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4021 constexpr size_t __offset = _SL::_S_before(__i);
4022 return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4023 return __x[__offset + __j];
4024 });
4025 });
4026 else if constexpr (_Np == _N0)
4027 {
4028 static_assert(sizeof...(_Sizes) == 1);
4029 return {simd_cast<_V>(__x)};
4030 }
4031 else if constexpr // split from fixed_size, such that __x::first.size == _N0
4032 (__is_fixed_size_abi_v<
4033 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
4034 {
4035 static_assert(
4036 !__is_fixed_size_abi_v<typename _V::abi_type>,
4037 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
4038 "fixed_size_simd "
4039 "when deduced?");
4040 // extract first and recurse (__split_wrapper is needed to deduce a new
4041 // _Sizes pack)
4042 return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
4043 __split_wrapper(_SL::template _S_pop_front<1>(),
4044 __data(__x).second));
4045 }
4046 else if constexpr ((!is_same_v<simd_abi::scalar,
4047 simd_abi::deduce_t<_Tp, _Sizes>> && ...)
4048 && (!__is_fixed_size_abi_v<
4049 simd_abi::deduce_t<_Tp, _Sizes>> && ...))
4050 {
4051 if constexpr (((_Sizes * 2 == _Np) && ...))
4052 return {{__private_init, __extract_part<0, 2>(__data(__x))},
4053 {__private_init, __extract_part<1, 2>(__data(__x))}};
4054 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4055 _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
4056 return {{__private_init, __extract_part<0, 3>(__data(__x))},
4057 {__private_init, __extract_part<1, 3>(__data(__x))},
4058 {__private_init, __extract_part<2, 3>(__data(__x))}};
4059 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4060 _SizeList<2 * _Np / 3, _Np / 3>>)
4061 return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
4062 {__private_init, __extract_part<2, 3>(__data(__x))}};
4063 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4064 _SizeList<_Np / 3, 2 * _Np / 3>>)
4065 return {{__private_init, __extract_part<0, 3>(__data(__x))},
4066 {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
4067 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4068 _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
4069 return {{__private_init, __extract_part<0, 2>(__data(__x))},
4070 {__private_init, __extract_part<2, 4>(__data(__x))},
4071 {__private_init, __extract_part<3, 4>(__data(__x))}};
4072 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4073 _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
4074 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4075 {__private_init, __extract_part<1, 4>(__data(__x))},
4076 {__private_init, __extract_part<1, 2>(__data(__x))}};
4077 else if constexpr (is_same_v<_SizeList<_Sizes...>,
4078 _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
4079 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4080 {__private_init, __extract_center(__data(__x))},
4081 {__private_init, __extract_part<3, 4>(__data(__x))}};
4082 else if constexpr (((_Sizes * 4 == _Np) && ...))
4083 return {{__private_init, __extract_part<0, 4>(__data(__x))},
4084 {__private_init, __extract_part<1, 4>(__data(__x))},
4085 {__private_init, __extract_part<2, 4>(__data(__x))},
4086 {__private_init, __extract_part<3, 4>(__data(__x))}};
4087 // else fall through
4088 }
4089#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4090 const __may_alias<_Tp>* const __element_ptr
4091 = reinterpret_cast<const __may_alias<_Tp>*>(&__x);
4092 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4093 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4094 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4095 constexpr size_t __offset = _SL::_S_before(__i);
4096 constexpr size_t __base_align = alignof(simd<_Tp, _Ap>);
4097 constexpr size_t __a
4098 = __base_align - ((__offset * sizeof(_Tp)) % __base_align);
4099 constexpr size_t __b = ((__a - 1) & __a) ^ __a;
4100 constexpr size_t __alignment = __b == 0 ? __a : __b;
4101 return _Vi(__element_ptr + __offset, overaligned<__alignment>);
4102 });
4103#else
4104 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>(
4105 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4106 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4107 const auto& __xx = __data(__x);
4108 using _Offset = decltype(_SL::_S_before(__i));
4109 return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4110 constexpr _SizeConstant<_Offset::value + __j> __k;
4111 return __xx[__k];
4112 });
4113 });
4114#endif
4115 }
4116
4117// }}}
4118
4119// __subscript_in_pack {{{
4120template <size_t _I, typename _Tp, typename _Ap, typename... _As>
4121 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
4122 __subscript_in_pack(const simd<_Tp, _Ap>& __x, const simd<_Tp, _As>&... __xs)
4123 {
4124 if constexpr (_I < simd_size_v<_Tp, _Ap>)
4125 return __x[_I];
4126 else
4127 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
4128 }
4129
4130// }}}
4131// __store_pack_of_simd {{{
4132template <typename _Tp, typename _A0, typename... _As>
4133 _GLIBCXX_SIMD_INTRINSIC void
4134 __store_pack_of_simd(char* __mem, const simd<_Tp, _A0>& __x0,
4135 const simd<_Tp, _As>&... __xs)
4136 {
4137 constexpr size_t __n_bytes = sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4138 __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4139 if constexpr (sizeof...(__xs) > 0)
4140 __store_pack_of_simd(__mem + __n_bytes, __xs...);
4141 }
4142
4143// }}}
4144// concat(simd...) {{{
4145template <typename _Tp, typename... _As, typename = __detail::__odr_helper>
4146 inline _GLIBCXX_SIMD_CONSTEXPR
4147 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4148 concat(const simd<_Tp, _As>&... __xs)
4149 {
4150 using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
4151 if constexpr (sizeof...(__xs) == 1)
4152 return simd_cast<_Rp>(__xs...);
4153 else if ((... && __xs._M_is_constprop()))
4154 return simd<_Tp,
4155 simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>(
4156 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA
4157 { return __subscript_in_pack<__i>(__xs...); });
4158 else
4159 {
4160 _Rp __r{};
4161 __store_pack_of_simd(reinterpret_cast<char*>(&__data(__r)), __xs...);
4162 return __r;
4163 }
4164 }
4165
4166// }}}
4167// concat(array<simd>) {{{
4168template <typename _Tp, typename _Abi, size_t _Np>
4169 _GLIBCXX_SIMD_ALWAYS_INLINE
4170 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4171 concat(const array<simd<_Tp, _Abi>, _Np>& __x)
4172 {
4173 return __call_with_subscripts<_Np>(
4174 __x, [](const auto&... __xs) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4175 return concat(__xs...);
4176 });
4177 }
4178
4179// }}}
4180
4181/// @cond undocumented
4182// _SmartReference {{{
4183template <typename _Up, typename _Accessor = _Up,
4184 typename _ValueType = typename _Up::value_type>
4185 class _SmartReference
4186 {
4187 friend _Accessor;
4188 int _M_index;
4189 _Up& _M_obj;
4190
4191 _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType _M_read() const noexcept
4192 {
4193 if constexpr (is_arithmetic_v<_Up>)
4194 return _M_obj;
4195 else
4196 return _M_obj[_M_index];
4197 }
4198
4199 template <typename _Tp>
4200 _GLIBCXX_SIMD_INTRINSIC constexpr void _M_write(_Tp&& __x) const
4201 { _Accessor::_S_set(_M_obj, _M_index, static_cast<_Tp&&>(__x)); }
4202
4203 public:
4204 _GLIBCXX_SIMD_INTRINSIC constexpr
4205 _SmartReference(_Up& __o, int __i) noexcept
4206 : _M_index(__i), _M_obj(__o) {}
4207
4208 using value_type = _ValueType;
4209
4210 _GLIBCXX_SIMD_INTRINSIC _SmartReference(const _SmartReference&) = delete;
4211
4212 _GLIBCXX_SIMD_INTRINSIC constexpr operator value_type() const noexcept
4213 { return _M_read(); }
4214
4215 template <typename _Tp,
4216 typename
4217 = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4218 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator=(_Tp&& __x) &&
4219 {
4220 _M_write(static_cast<_Tp&&>(__x));
4221 return {_M_obj, _M_index};
4222 }
4223
4224#define _GLIBCXX_SIMD_OP_(__op) \
4225 template <typename _Tp, \
4226 typename _TT \
4227 = decltype(declval<value_type>() __op declval<_Tp>()), \
4228 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4229 typename = _ValuePreservingOrInt<_TT, value_type>> \
4230 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4231 operator __op##=(_Tp&& __x) && \
4232 { \
4233 const value_type& __lhs = _M_read(); \
4234 _M_write(__lhs __op __x); \
4235 return {_M_obj, _M_index}; \
4236 }
4237 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4238 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4239 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4240#undef _GLIBCXX_SIMD_OP_
4241
4242 template <typename _Tp = void,
4243 typename
4244 = decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4245 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator++() &&
4246 {
4247 value_type __x = _M_read();
4248 _M_write(++__x);
4249 return {_M_obj, _M_index};
4250 }
4251
4252 template <typename _Tp = void,
4253 typename
4254 = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4255 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator++(int) &&
4256 {
4257 const value_type __r = _M_read();
4258 value_type __x = __r;
4259 _M_write(++__x);
4260 return __r;
4261 }
4262
4263 template <typename _Tp = void,
4264 typename
4265 = decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4266 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator--() &&
4267 {
4268 value_type __x = _M_read();
4269 _M_write(--__x);
4270 return {_M_obj, _M_index};
4271 }
4272
4273 template <typename _Tp = void,
4274 typename
4275 = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4276 _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator--(int) &&
4277 {
4278 const value_type __r = _M_read();
4279 value_type __x = __r;
4280 _M_write(--__x);
4281 return __r;
4282 }
4283
4284 _GLIBCXX_SIMD_INTRINSIC friend void
4285 swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4286 conjunction<
4287 is_nothrow_constructible<value_type, _SmartReference&&>,
4288 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4289 {
4290 value_type __tmp = static_cast<_SmartReference&&>(__a);
4291 static_cast<_SmartReference&&>(__a) = static_cast<value_type>(__b);
4292 static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4293 }
4294
4295 _GLIBCXX_SIMD_INTRINSIC friend void
4296 swap(value_type& __a, _SmartReference&& __b) noexcept(
4297 conjunction<
4298 is_nothrow_constructible<value_type, value_type&&>,
4299 is_nothrow_assignable<value_type&, value_type&&>,
4300 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4301 {
4302 value_type __tmp(std::move(__a));
4303 __a = static_cast<value_type>(__b);
4304 static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4305 }
4306
4307 _GLIBCXX_SIMD_INTRINSIC friend void
4308 swap(_SmartReference&& __a, value_type& __b) noexcept(
4309 conjunction<
4310 is_nothrow_constructible<value_type, _SmartReference&&>,
4311 is_nothrow_assignable<value_type&, value_type&&>,
4312 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4313 {
4314 value_type __tmp(__a);
4315 static_cast<_SmartReference&&>(__a) = std::move(__b);
4316 __b = std::move(__tmp);
4317 }
4318 };
4319
4320// }}}
4321// __scalar_abi_wrapper {{{
4322template <int _Bytes>
4323 struct __scalar_abi_wrapper
4324 {
4325 template <typename _Tp> static constexpr size_t _S_full_size = 1;
4326 template <typename _Tp> static constexpr size_t _S_size = 1;
4327 template <typename _Tp> static constexpr size_t _S_is_partial = false;
4328
4329 template <typename _Tp, typename _Abi = simd_abi::scalar>
4330 static constexpr bool _S_is_valid_v
4331 = _Abi::template _IsValid<_Tp>::value && sizeof(_Tp) == _Bytes;
4332 };
4333
4334// }}}
4335// __decay_abi metafunction {{{
4336template <typename _Tp>
4337 struct __decay_abi { using type = _Tp; };
4338
4339template <int _Bytes>
4340 struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4341 { using type = simd_abi::scalar; };
4342
4343// }}}
4344// __find_next_valid_abi metafunction {{{1
4345// Given an ABI tag A<N>, find an N2 < N such that A<N2>::_S_is_valid_v<_Tp> ==
4346// true, N2 is a power-of-2, and A<N2>::_S_is_partial<_Tp> is false. Break
4347// recursion at 2 elements in the resulting ABI tag. In this case
4348// type::_S_is_valid_v<_Tp> may be false.
4349template <template <int> class _Abi, int _Bytes, typename _Tp>
4350 struct __find_next_valid_abi
4351 {
4352 static constexpr auto _S_choose()
4353 {
4354 constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4355 using _NextAbi = _Abi<_NextBytes>;
4356 if constexpr (_NextBytes < sizeof(_Tp) * 2) // break recursion
4357 return _Abi<_Bytes>();
4358 else if constexpr (_NextAbi::template _S_is_partial<_Tp> == false
4359 && _NextAbi::template _S_is_valid_v<_Tp>)
4360 return _NextAbi();
4361 else
4362 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4363 }
4364
4365 using type = decltype(_S_choose());
4366 };
4367
4368template <int _Bytes, typename _Tp>
4369 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4370 { using type = simd_abi::scalar; };
4371
4372// _AbiList {{{1
4373template <template <int> class...>
4374 struct _AbiList
4375 {
4376 template <typename, int> static constexpr bool _S_has_valid_abi = false;
4377 template <typename, int> using _FirstValidAbi = void;
4378 template <typename, int> using _BestAbi = void;
4379 };
4380
4381template <template <int> class _A0, template <int> class... _Rest>
4382 struct _AbiList<_A0, _Rest...>
4383 {
4384 template <typename _Tp, int _Np>
4385 static constexpr bool _S_has_valid_abi
4386 = _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<
4387 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4388
4389 template <typename _Tp, int _Np>
4390 using _FirstValidAbi = conditional_t<
4391 _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4392 typename __decay_abi<_A0<sizeof(_Tp) * _Np>>::type,
4393 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4394
4395 template <typename _Tp, int _Np>
4396 static constexpr auto _S_determine_best_abi()
4397 {
4398 static_assert(_Np >= 1);
4399 constexpr int _Bytes = sizeof(_Tp) * _Np;
4400 if constexpr (_Np == 1)
4401 return __make_dependent_t<_Tp, simd_abi::scalar>{};
4402 else
4403 {
4404 constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4405 // _A0<_Bytes> is good if:
4406 // 1. The ABI tag is valid for _Tp
4407 // 2. The storage overhead is no more than padding to fill the next
4408 // power-of-2 number of bytes
4409 if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4410 _Tp> && __fullsize / 2 < _Np)
4411 return typename __decay_abi<_A0<_Bytes>>::type{};
4412 else
4413 {
4414 using _Bp =
4415 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4416 if constexpr (_Bp::template _S_is_valid_v<
4417 _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4418 return _Bp{};
4419 else
4420 return
4421 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4422 }
4423 }
4424 }
4425
4426 template <typename _Tp, int _Np>
4427 using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4428 };
4429
4430// }}}1
4431
4432// the following lists all native ABIs, which makes them accessible to
4433// simd_abi::deduce and select_best_vector_type_t (for fixed_size). Order
4434// matters: Whatever comes first has higher priority.
4435using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4436 __scalar_abi_wrapper>;
4437
4438// valid _SimdTraits specialization {{{1
4439template <typename _Tp, typename _Abi>
4440 struct _SimdTraits<_Tp, _Abi, void_t<typename _Abi::template _IsValid<_Tp>>>
4441 : _Abi::template __traits<_Tp> {};
4442
4443// __deduce_impl specializations {{{1
4444// try all native ABIs (including scalar) first
4445template <typename _Tp, size_t _Np>
4446 struct __deduce_impl<
4447 _Tp, _Np, enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4448 { using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4449
4450// fall back to fixed_size only if scalar and native ABIs don't match
4451template <typename _Tp, size_t _Np, typename = void>
4452 struct __deduce_fixed_size_fallback {};
4453
4454template <typename _Tp, size_t _Np>
4455 struct __deduce_fixed_size_fallback<_Tp, _Np,
4456 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4457 { using type = simd_abi::fixed_size<_Np>; };
4458
4459template <typename _Tp, size_t _Np, typename>
4460 struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _Np> {};
4461
4462//}}}1
4463/// @endcond
4464
4465// simd_mask {{{
4466template <typename _Tp, typename _Abi>
4467 class simd_mask : public _SimdTraits<_Tp, _Abi>::_MaskBase
4468 {
4469 // types, tags, and friends {{{
4470 using _Traits = _SimdTraits<_Tp, _Abi>;
4471 using _MemberType = typename _Traits::_MaskMember;
4472
4473 // We map all masks with equal element sizeof to a single integer type, the
4474 // one given by __int_for_sizeof_t<_Tp>. This is the approach
4475 // [[gnu::vector_size(N)]] types take as well and it reduces the number of
4476 // template specializations in the implementation classes.
4477 using _Ip = __int_for_sizeof_t<_Tp>;
4478 static constexpr _Ip* _S_type_tag = nullptr;
4479
4480 friend typename _Traits::_MaskBase;
4481 friend class simd<_Tp, _Abi>; // to construct masks on return
4482 friend typename _Traits::_SimdImpl; // to construct masks on return and
4483 // inspect data on masked operations
4484 public:
4485 using _Impl = typename _Traits::_MaskImpl;
4486 friend _Impl;
4487
4488 // }}}
4489 // member types {{{
4490 using value_type = bool;
4491 using reference = _SmartReference<_MemberType, _Impl, value_type>;
4492 using simd_type = simd<_Tp, _Abi>;
4493 using abi_type = _Abi;
4494
4495 // }}}
4496 static constexpr size_t size() // {{{
4497 { return __size_or_zero_v<_Tp, _Abi>; }
4498
4499 // }}}
4500 // constructors & assignment {{{
4501 simd_mask() = default;
4502 simd_mask(const simd_mask&) = default;
4503 simd_mask(simd_mask&&) = default;
4504 simd_mask& operator=(const simd_mask&) = default;
4505 simd_mask& operator=(simd_mask&&) = default;
4506
4507 // }}}
4508 // access to internal representation (optional feature) {{{
4509 _GLIBCXX_SIMD_ALWAYS_INLINE explicit
4510 simd_mask(typename _Traits::_MaskCastType __init)
4511 : _M_data{__init} {}
4512 // conversions to internal type is done in _MaskBase
4513
4514 // }}}
4515 // bitset interface (extension to be proposed) {{{
4516 // TS_FEEDBACK:
4517 // Conversion of simd_mask to and from bitset makes it much easier to
4518 // interface with other facilities. I suggest adding `static
4519 // simd_mask::from_bitset` and `simd_mask::to_bitset`.
4520 _GLIBCXX_SIMD_ALWAYS_INLINE static simd_mask
4521 __from_bitset(bitset<size()> bs)
4522 { return {__bitset_init, bs}; }
4523
4524 _GLIBCXX_SIMD_ALWAYS_INLINE bitset<size()>
4525 __to_bitset() const
4526 { return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4527
4528 // }}}
4529 // explicit broadcast constructor {{{
4530 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4531 simd_mask(value_type __x)
4532 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4533
4534 // }}}
4535 // implicit type conversion constructor {{{
4536 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4537 // proposed improvement
4538 template <typename _Up, typename _A2,
4539 typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4540 _GLIBCXX_SIMD_ALWAYS_INLINE explicit(sizeof(_MemberType)
4541 != sizeof(typename _SimdTraits<_Up, _A2>::_MaskMember))
4542 simd_mask(const simd_mask<_Up, _A2>& __x)
4543 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4544 #else
4545 // conforming to ISO/IEC 19570:2018
4546 template <typename _Up, typename = enable_if_t<conjunction<
4547 is_same<abi_type, simd_abi::fixed_size<size()>>,
4548 is_same<_Up, _Up>>::value>>
4549 _GLIBCXX_SIMD_ALWAYS_INLINE
4550 simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4551 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4552 #endif
4553
4554 // }}}
4555 // load constructor {{{
4556 template <typename _Flags>
4557 _GLIBCXX_SIMD_ALWAYS_INLINE
4558 simd_mask(const value_type* __mem, _Flags)
4559 : _M_data(_Impl::template _S_load<_Ip>(
4560 _Flags::template _S_apply<simd_mask>(__mem))) {}
4561
4562 template <typename _Flags>
4563 _GLIBCXX_SIMD_ALWAYS_INLINE
4564 simd_mask(const value_type* __mem, simd_mask __k, _Flags)
4565 : _M_data{}
4566 {
4567 _M_data
4568 = _Impl::_S_masked_load(_M_data, __k._M_data,
4569 _Flags::template _S_apply<simd_mask>(__mem));
4570 }
4571
4572 // }}}
4573 // loads [simd_mask.load] {{{
4574 template <typename _Flags>
4575 _GLIBCXX_SIMD_ALWAYS_INLINE void
4576 copy_from(const value_type* __mem, _Flags)
4577 {
4578 _M_data = _Impl::template _S_load<_Ip>(
4579 _Flags::template _S_apply<simd_mask>(__mem));
4580 }
4581
4582 // }}}
4583 // stores [simd_mask.store] {{{
4584 template <typename _Flags>
4585 _GLIBCXX_SIMD_ALWAYS_INLINE void
4586 copy_to(value_type* __mem, _Flags) const
4587 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4588
4589 // }}}
4590 // scalar access {{{
4591 _GLIBCXX_SIMD_ALWAYS_INLINE reference
4592 operator[](size_t __i)
4593 {
4594 if (__i >= size())
4595 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4596 return {_M_data, int(__i)};
4597 }
4598
4599 _GLIBCXX_SIMD_ALWAYS_INLINE value_type
4600 operator[](size_t __i) const
4601 {
4602 if (__i >= size())
4603 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4604 if constexpr (__is_scalar_abi<_Abi>())
4605 return _M_data;
4606 else
4607 return static_cast<bool>(_M_data[__i]);
4608 }
4609
4610 // }}}
4611 // negation {{{
4612 _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask
4613 operator!() const
4614 { return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4615
4616 // }}}
4617 // simd_mask binary operators [simd_mask.binary] {{{
4618 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4619 // simd_mask<int> && simd_mask<uint> needs disambiguation
4620 template <typename _Up, typename _A2,
4621 typename
4622 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4623 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4624 operator&&(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4625 {
4626 return {__private_init,
4627 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4628 }
4629
4630 template <typename _Up, typename _A2,
4631 typename
4632 = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4633 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4634 operator||(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4635 {
4636 return {__private_init,
4637 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4638 }
4639 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4640
4641 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4642 operator&&(const simd_mask& __x, const simd_mask& __y)
4643 {
4644 return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)};
4645 }
4646
4647 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4648 operator||(const simd_mask& __x, const simd_mask& __y)
4649 {
4650 return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)};
4651 }
4652
4653 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4654 operator&(const simd_mask& __x, const simd_mask& __y)
4655 { return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4656
4657 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4658 operator|(const simd_mask& __x, const simd_mask& __y)
4659 { return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4660
4661 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4662 operator^(const simd_mask& __x, const simd_mask& __y)
4663 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4664
4665 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4666 operator&=(simd_mask& __x, const simd_mask& __y)
4667 {
4668 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4669 return __x;
4670 }
4671
4672 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4673 operator|=(simd_mask& __x, const simd_mask& __y)
4674 {
4675 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4676 return __x;
4677 }
4678
4679 _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4680 operator^=(simd_mask& __x, const simd_mask& __y)
4681 {
4682 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4683 return __x;
4684 }
4685
4686 // }}}
4687 // simd_mask compares [simd_mask.comparison] {{{
4688 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4689 operator==(const simd_mask& __x, const simd_mask& __y)
4690 { return !operator!=(__x, __y); }
4691
4692 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4693 operator!=(const simd_mask& __x, const simd_mask& __y)
4694 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4695
4696 // }}}
4697 // private_init ctor {{{
4698 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4699 simd_mask(_PrivateInit, typename _Traits::_MaskMember __init)
4700 : _M_data(__init) {}
4701
4702 // }}}
4703 // private_init generator ctor {{{
4704 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4705 _GLIBCXX_SIMD_INTRINSIC constexpr
4706 simd_mask(_PrivateInit, _Fp&& __gen)
4707 : _M_data()
4708 {
4709 __execute_n_times<size()>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4710 _Impl::_S_set(_M_data, __i, __gen(__i));
4711 });
4712 }
4713
4714 // }}}
4715 // bitset_init ctor {{{
4716 _GLIBCXX_SIMD_INTRINSIC simd_mask(_BitsetInit, bitset<size()> __init)
4717 : _M_data(
4718 _Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4719 {}
4720
4721 // }}}
4722 // __cvt {{{
4723 // TS_FEEDBACK:
4724 // The conversion operator this implements should be a ctor on simd_mask.
4725 // Once you call .__cvt() on a simd_mask it converts conveniently.
4726 // A useful variation: add `explicit(sizeof(_Tp) != sizeof(_Up))`
4727 struct _CvtProxy
4728 {
4729 template <typename _Up, typename _A2,
4730 typename
4731 = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4732 _GLIBCXX_SIMD_ALWAYS_INLINE
4733 operator simd_mask<_Up, _A2>() &&
4734 {
4735 using namespace std::experimental::__proposed;
4736 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4737 }
4738
4739 const simd_mask<_Tp, _Abi>& _M_data;
4740 };
4741
4742 _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4743 __cvt() const
4744 { return {*this}; }
4745
4746 // }}}
4747 // operator?: overloads (suggested extension) {{{
4748 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4749 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4750 operator?:(const simd_mask& __k, const simd_mask& __where_true,
4751 const simd_mask& __where_false)
4752 {
4753 auto __ret = __where_false;
4754 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4755 return __ret;
4756 }
4757
4758 template <typename _U1, typename _U2,
4759 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4760 typename = enable_if_t<conjunction_v<
4761 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4762 is_convertible<simd_mask, typename _Rp::mask_type>>>>
4763 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend _Rp
4764 operator?:(const simd_mask& __k, const _U1& __where_true,
4765 const _U2& __where_false)
4766 {
4767 _Rp __ret = __where_false;
4768 _Rp::_Impl::_S_masked_assign(
4769 __data(static_cast<typename _Rp::mask_type>(__k)), __data(__ret),
4770 __data(static_cast<_Rp>(__where_true)));
4771 return __ret;
4772 }
4773
4774 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4775 template <typename _Kp, typename _Ak, typename _Up, typename _Au,
4776 typename = enable_if_t<
4777 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4778 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4779 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4780 operator?:(const simd_mask<_Kp, _Ak>& __k, const simd_mask& __where_true,
4781 const simd_mask<_Up, _Au>& __where_false)
4782 {
4783 simd_mask __ret = __where_false;
4784 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4785 __where_true._M_data);
4786 return __ret;
4787 }
4788 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4789 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
4790
4791 // }}}
4792 // _M_is_constprop {{{
4793 _GLIBCXX_SIMD_INTRINSIC constexpr bool
4794 _M_is_constprop() const
4795 {
4796 if constexpr (__is_scalar_abi<_Abi>())
4797 return __builtin_constant_p(_M_data);
4798 else
4799 return _M_data._M_is_constprop();
4800 }
4801
4802 // }}}
4803
4804 private:
4805 friend const auto& __data<_Tp, abi_type>(const simd_mask&);
4806 friend auto& __data<_Tp, abi_type>(simd_mask&);
4807 alignas(_Traits::_S_mask_align) _MemberType _M_data;
4808 };
4809
4810// }}}
4811
4812/// @cond undocumented
4813// __data(simd_mask) {{{
4814template <typename _Tp, typename _Ap>
4815 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
4816 __data(const simd_mask<_Tp, _Ap>& __x)
4817 { return __x._M_data; }
4818
4819template <typename _Tp, typename _Ap>
4820 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
4821 __data(simd_mask<_Tp, _Ap>& __x)
4822 { return __x._M_data; }
4823
4824// }}}
4825/// @endcond
4826
4827// simd_mask reductions [simd_mask.reductions] {{{
4828template <typename _Tp, typename _Abi>
4829 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4830 all_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4831 {
4832 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4833 {
4834 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4835 if (!__k[__i])
4836 return false;
4837 return true;
4838 }
4839 else
4840 return _Abi::_MaskImpl::_S_all_of(__k);
4841 }
4842
4843template <typename _Tp, typename _Abi>
4844 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4845 any_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4846 {
4847 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4848 {
4849 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4850 if (__k[__i])
4851 return true;
4852 return false;
4853 }
4854 else
4855 return _Abi::_MaskImpl::_S_any_of(__k);
4856 }
4857
4858template <typename _Tp, typename _Abi>
4859 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4860 none_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4861 {
4862 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4863 {
4864 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4865 if (__k[__i])
4866 return false;
4867 return true;
4868 }
4869 else
4870 return _Abi::_MaskImpl::_S_none_of(__k);
4871 }
4872
4873template <typename _Tp, typename _Abi>
4874 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4875 some_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4876 {
4877 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4878 {
4879 for (size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4880 if (__k[__i] != __k[__i - 1])
4881 return true;
4882 return false;
4883 }
4884 else
4885 return _Abi::_MaskImpl::_S_some_of(__k);
4886 }
4887
4888template <typename _Tp, typename _Abi>
4889 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4890 popcount(const simd_mask<_Tp, _Abi>& __k) noexcept
4891 {
4892 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4893 {
4894 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4895 __k, [](auto... __elements) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4896 return ((__elements != 0) + ...);
4897 });
4898 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4899 return __r;
4900 }
4901 return _Abi::_MaskImpl::_S_popcount(__k);
4902 }
4903
4904template <typename _Tp, typename _Abi>
4905 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4906 find_first_set(const simd_mask<_Tp, _Abi>& __k)
4907 {
4908 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4909 {
4910 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4911 const size_t _Idx = __call_with_n_evaluations<_Np>(
4912 [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4913 return std::min({__indexes...});
4914 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4915 return __k[__i] ? +__i : _Np;
4916 });
4917 if (_Idx >= _Np)
4918 __invoke_ub("find_first_set(empty mask) is UB");
4919 if (__builtin_constant_p(_Idx))
4920 return _Idx;
4921 }
4922 return _Abi::_MaskImpl::_S_find_first_set(__k);
4923 }
4924
4925template <typename _Tp, typename _Abi>
4926 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4927 find_last_set(const simd_mask<_Tp, _Abi>& __k)
4928 {
4929 if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4930 {
4931 constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4932 const int _Idx = __call_with_n_evaluations<_Np>(
4933 [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4934 return std::max({__indexes...});
4935 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
4936 return __k[__i] ? int(__i) : -1;
4937 });
4938 if (_Idx < 0)
4939 __invoke_ub("find_first_set(empty mask) is UB");
4940 if (__builtin_constant_p(_Idx))
4941 return _Idx;
4942 }
4943 return _Abi::_MaskImpl::_S_find_last_set(__k);
4944 }
4945
4946_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4947all_of(_ExactBool __x) noexcept
4948{ return __x; }
4949
4950_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4951any_of(_ExactBool __x) noexcept
4952{ return __x; }
4953
4954_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4955none_of(_ExactBool __x) noexcept
4956{ return !__x; }
4957
4958_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4959some_of(_ExactBool) noexcept
4960{ return false; }
4961
4962_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4963popcount(_ExactBool __x) noexcept
4964{ return __x; }
4965
4966_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4967find_first_set(_ExactBool)
4968{ return 0; }
4969
4970_GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4971find_last_set(_ExactBool)
4972{ return 0; }
4973
4974// }}}
4975
4976/// @cond undocumented
4977// _SimdIntOperators{{{1
4978template <typename _V, typename _Tp, typename _Abi, bool>
4979 class _SimdIntOperators {};
4980
4981template <typename _V, typename _Tp, typename _Abi>
4982 class _SimdIntOperators<_V, _Tp, _Abi, true>
4983 {
4984 using _Impl = typename _SimdTraits<_Tp, _Abi>::_SimdImpl;
4985
4986 _GLIBCXX_SIMD_INTRINSIC const _V& __derived() const
4987 { return *static_cast<const _V*>(this); }
4988
4989 template <typename _Up>
4990 _GLIBCXX_SIMD_INTRINSIC static _GLIBCXX_SIMD_CONSTEXPR _V
4991 _S_make_derived(_Up&& __d)
4992 { return {__private_init, static_cast<_Up&&>(__d)}; }
4993
4994 public:
4995 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
4996 _V&
4997 operator%=(_V& __lhs, const _V& __x)
4998 { return __lhs = __lhs % __x; }
4999
5000 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5001 _V&
5002 operator&=(_V& __lhs, const _V& __x)
5003 { return __lhs = __lhs & __x; }
5004
5005 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5006 _V&
5007 operator|=(_V& __lhs, const _V& __x)
5008 { return __lhs = __lhs | __x; }
5009
5010 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5011 _V&
5012 operator^=(_V& __lhs, const _V& __x)
5013 { return __lhs = __lhs ^ __x; }
5014
5015 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5016 _V&
5017 operator<<=(_V& __lhs, const _V& __x)
5018 { return __lhs = __lhs << __x; }
5019
5020 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5021 _V&
5022 operator>>=(_V& __lhs, const _V& __x)
5023 { return __lhs = __lhs >> __x; }
5024
5025 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5026 _V&
5027 operator<<=(_V& __lhs, int __x)
5028 { return __lhs = __lhs << __x; }
5029
5030 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5031 _V&
5032 operator>>=(_V& __lhs, int __x)
5033 { return __lhs = __lhs >> __x; }
5034
5035 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5036 _V
5037 operator%(const _V& __x, const _V& __y)
5038 {
5039 return _SimdIntOperators::_S_make_derived(
5040 _Impl::_S_modulus(__data(__x), __data(__y)));
5041 }
5042
5043 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5044 _V
5045 operator&(const _V& __x, const _V& __y)
5046 {
5047 return _SimdIntOperators::_S_make_derived(
5048 _Impl::_S_bit_and(__data(__x), __data(__y)));
5049 }
5050
5051 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5052 _V
5053 operator|(const _V& __x, const _V& __y)
5054 {
5055 return _SimdIntOperators::_S_make_derived(
5056 _Impl::_S_bit_or(__data(__x), __data(__y)));
5057 }
5058
5059 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5060 _V
5061 operator^(const _V& __x, const _V& __y)
5062 {
5063 return _SimdIntOperators::_S_make_derived(
5064 _Impl::_S_bit_xor(__data(__x), __data(__y)));
5065 }
5066
5067 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5068 _V
5069 operator<<(const _V& __x, const _V& __y)
5070 {
5071 return _SimdIntOperators::_S_make_derived(
5072 _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
5073 }
5074
5075 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5076 _V
5077 operator>>(const _V& __x, const _V& __y)
5078 {
5079 return _SimdIntOperators::_S_make_derived(
5080 _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
5081 }
5082
5083 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5084 _V
5085 operator<<(const _V& __x, int __y)
5086 {
5087 if (__y < 0)
5088 __invoke_ub("The behavior is undefined if the right operand of a "
5089 "shift operation is negative. [expr.shift]\nA shift by "
5090 "%d was requested",
5091 __y);
5092 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5093 __invoke_ub(
5094 "The behavior is undefined if the right operand of a "
5095 "shift operation is greater than or equal to the width of the "
5096 "promoted left operand. [expr.shift]\nA shift by %d was requested",
5097 __y);
5098 return _SimdIntOperators::_S_make_derived(
5099 _Impl::_S_bit_shift_left(__data(__x), __y));
5100 }
5101
5102 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5103 _V
5104 operator>>(const _V& __x, int __y)
5105 {
5106 if (__y < 0)
5107 __invoke_ub(
5108 "The behavior is undefined if the right operand of a shift "
5109 "operation is negative. [expr.shift]\nA shift by %d was requested",
5110 __y);
5111 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5112 __invoke_ub(
5113 "The behavior is undefined if the right operand of a shift "
5114 "operation is greater than or equal to the width of the promoted "
5115 "left operand. [expr.shift]\nA shift by %d was requested",
5116 __y);
5117 return _SimdIntOperators::_S_make_derived(
5118 _Impl::_S_bit_shift_right(__data(__x), __y));
5119 }
5120
5121 // unary operators (for integral _Tp)
5122 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5123 _V
5124 operator~() const
5125 { return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
5126 };
5127
5128//}}}1
5129/// @endcond
5130
5131// simd {{{
5132template <typename _Tp, typename _Abi>
5133 class simd : public _SimdIntOperators<
5134 simd<_Tp, _Abi>, _Tp, _Abi,
5135 conjunction<is_integral<_Tp>,
5136 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
5137 public _SimdTraits<_Tp, _Abi>::_SimdBase
5138 {
5139 using _Traits = _SimdTraits<_Tp, _Abi>;
5140 using _MemberType = typename _Traits::_SimdMember;
5141 using _CastType = typename _Traits::_SimdCastType;
5142 static constexpr _Tp* _S_type_tag = nullptr;
5143 friend typename _Traits::_SimdBase;
5144
5145 public:
5146 using _Impl = typename _Traits::_SimdImpl;
5147 friend _Impl;
5148 friend _SimdIntOperators<simd, _Tp, _Abi, true>;
5149
5150 using value_type = _Tp;
5151 using reference = _SmartReference<_MemberType, _Impl, value_type>;
5152 using mask_type = simd_mask<_Tp, _Abi>;
5153 using abi_type = _Abi;
5154
5155 static constexpr size_t size()
5156 { return __size_or_zero_v<_Tp, _Abi>; }
5157
5158 _GLIBCXX_SIMD_CONSTEXPR simd() = default;
5159 _GLIBCXX_SIMD_CONSTEXPR simd(const simd&) = default;
5160 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
5161 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
5162 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
5163
5164 // implicit broadcast constructor
5165 template <typename _Up,
5166 typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>, bool>>>
5167 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5168 simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
5169 : _M_data(
5170 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
5171 {}
5172
5173 // implicit type conversion constructor (convert from fixed_size to
5174 // fixed_size)
5175 template <typename _Up>
5176 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5177 simd(const simd<_Up, simd_abi::fixed_size<size()>>& __x,
5178 enable_if_t<
5179 conjunction<
5180 is_same<simd_abi::fixed_size<size()>, abi_type>,
5181 negation<__is_narrowing_conversion<_Up, value_type>>,
5182 __converts_to_higher_integer_rank<_Up, value_type>>::value,
5183 void*> = nullptr)
5184 : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5185
5186 // explicit type conversion constructor
5187#ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5188 template <typename _Up, typename _A2,
5189 typename = decltype(static_simd_cast<simd>(
5190 declval<const simd<_Up, _A2>&>()))>
5191 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5192 simd(const simd<_Up, _A2>& __x)
5193 : simd(static_simd_cast<simd>(__x)) {}
5194#endif // _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5195
5196 // generator constructor
5197 template <typename _Fp>
5198 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5199 simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
5200 declval<_SizeConstant<0>&>())),
5201 value_type>* = nullptr)
5202 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5203
5204 // load constructor
5205 template <typename _Up, typename _Flags>
5206 _GLIBCXX_SIMD_ALWAYS_INLINE
5207 simd(const _Up* __mem, _Flags)
5208 : _M_data(
5209 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5210 {}
5211
5212 // loads [simd.load]
5213 template <typename _Up, typename _Flags>
5214 _GLIBCXX_SIMD_ALWAYS_INLINE void
5215 copy_from(const _Vectorizable<_Up>* __mem, _Flags)
5216 {
5217 _M_data = static_cast<decltype(_M_data)>(
5218 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5219 }
5220
5221 // stores [simd.store]
5222 template <typename _Up, typename _Flags>
5223 _GLIBCXX_SIMD_ALWAYS_INLINE void
5224 copy_to(_Vectorizable<_Up>* __mem, _Flags) const
5225 {
5226 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5227 _S_type_tag);
5228 }
5229
5230 // scalar access
5231 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5232 operator[](size_t __i)
5233 { return {_M_data, int(__i)}; }
5234
5235 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5236 operator[]([[maybe_unused]] size_t __i) const
5237 {
5238 if constexpr (__is_scalar_abi<_Abi>())
5239 {
5240 _GLIBCXX_DEBUG_ASSERT(__i == 0);
5241 return _M_data;
5242 }
5243 else
5244 return _M_data[__i];
5245 }
5246
5247 // increment and decrement:
5248 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5249 operator++()
5250 {
5251 _Impl::_S_increment(_M_data);
5252 return *this;
5253 }
5254
5255 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5256 operator++(int)
5257 {
5258 simd __r = *this;
5259 _Impl::_S_increment(_M_data);
5260 return __r;
5261 }
5262
5263 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5264 operator--()
5265 {
5266 _Impl::_S_decrement(_M_data);
5267 return *this;
5268 }
5269
5270 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5271 operator--(int)
5272 {
5273 simd __r = *this;
5274 _Impl::_S_decrement(_M_data);
5275 return __r;
5276 }
5277
5278 // unary operators (for any _Tp)
5279 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5280 operator!() const
5281 { return {__private_init, _Impl::_S_negate(_M_data)}; }
5282
5283 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5284 operator+() const
5285 { return *this; }
5286
5287 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5288 operator-() const
5289 { return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5290
5291 // access to internal representation (suggested extension)
5292 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5293 simd(_CastType __init) : _M_data(__init) {}
5294
5295 // compound assignment [simd.cassign]
5296 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5297 operator+=(simd& __lhs, const simd& __x)
5298 { return __lhs = __lhs + __x; }
5299
5300 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5301 operator-=(simd& __lhs, const simd& __x)
5302 { return __lhs = __lhs - __x; }
5303
5304 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5305 operator*=(simd& __lhs, const simd& __x)
5306 { return __lhs = __lhs * __x; }
5307
5308 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5309 operator/=(simd& __lhs, const simd& __x)
5310 { return __lhs = __lhs / __x; }
5311
5312 // binary operators [simd.binary]
5313 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5314 operator+(const simd& __x, const simd& __y)
5315 { return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5316
5317 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5318 operator-(const simd& __x, const simd& __y)
5319 { return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5320
5321 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5322 operator*(const simd& __x, const simd& __y)
5323 { return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5324
5325 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5326 operator/(const simd& __x, const simd& __y)
5327 { return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5328
5329 // compares [simd.comparison]
5330 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5331 operator==(const simd& __x, const simd& __y)
5332 { return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5333
5334 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5335 operator!=(const simd& __x, const simd& __y)
5336 {
5337 return simd::_S_make_mask(
5338 _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5339 }
5340
5341 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5342 operator<(const simd& __x, const simd& __y)
5343 { return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5344
5345 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5346 operator<=(const simd& __x, const simd& __y)
5347 {
5348 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5349 }
5350
5351 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5352 operator>(const simd& __x, const simd& __y)
5353 { return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5354
5355 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5356 operator>=(const simd& __x, const simd& __y)
5357 {
5358 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5359 }
5360
5361 // operator?: overloads (suggested extension) {{{
5362#ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5363 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5364 operator?:(const mask_type& __k, const simd& __where_true,
5365 const simd& __where_false)
5366 {
5367 auto __ret = __where_false;
5368 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5369 return __ret;
5370 }
5371
5372#endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5373 // }}}
5374
5375 // "private" because of the first arguments's namespace
5376 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5377 simd(_PrivateInit, const _MemberType& __init)
5378 : _M_data(__init) {}
5379
5380 // "private" because of the first arguments's namespace
5381 _GLIBCXX_SIMD_INTRINSIC
5382 simd(_BitsetInit, bitset<size()> __init) : _M_data()
5383 { where(mask_type(__bitset_init, __init), *this) = ~*this; }
5384
5385 _GLIBCXX_SIMD_INTRINSIC constexpr bool
5386 _M_is_constprop() const
5387 {
5388 if constexpr (__is_scalar_abi<_Abi>())
5389 return __builtin_constant_p(_M_data);
5390 else
5391 return _M_data._M_is_constprop();
5392 }
5393
5394 private:
5395 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR static mask_type
5396 _S_make_mask(typename mask_type::_MemberType __k)
5397 { return {__private_init, __k}; }
5398
5399 friend const auto& __data<value_type, abi_type>(const simd&);
5400 friend auto& __data<value_type, abi_type>(simd&);
5401 alignas(_Traits::_S_simd_align) _MemberType _M_data;
5402 };
5403
5404// }}}
5405/// @cond undocumented
5406// __data {{{
5407template <typename _Tp, typename _Ap>
5408 _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5409 __data(const simd<_Tp, _Ap>& __x)
5410 { return __x._M_data; }
5411
5412template <typename _Tp, typename _Ap>
5413 _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5414 __data(simd<_Tp, _Ap>& __x)
5415 { return __x._M_data; }
5416
5417// }}}
5418namespace __float_bitwise_operators { //{{{
5419template <typename _Tp, typename _Ap>
5420 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5421 operator^(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5422 {
5423 return {__private_init,
5424 _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))};
5425 }
5426
5427template <typename _Tp, typename _Ap>
5428 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5429 operator|(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5430 {
5431 return {__private_init,
5432 _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))};
5433 }
5434
5435template <typename _Tp, typename _Ap>
5436 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5437 operator&(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5438 {
5439 return {__private_init,
5440 _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))};
5441 }
5442
5443template <typename _Tp, typename _Ap>
5444 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5445 enable_if_t<is_floating_point_v<_Tp>, simd<_Tp, _Ap>>
5446 operator~(const simd<_Tp, _Ap>& __a)
5447 { return {__private_init, _Ap::_SimdImpl::_S_complement(__data(__a))}; }
5448} // namespace __float_bitwise_operators }}}
5449/// @endcond
5450
5451/// @}
5452_GLIBCXX_SIMD_END_NAMESPACE
5453
5454#endif // __cplusplus >= 201703L
5455#endif // _GLIBCXX_EXPERIMENTAL_SIMD_H
5456
5457// vim: foldmethod=marker foldmarker={{{,}}}
constexpr _If_is_unsigned_integer< _Tp, int > popcount(_Tp __x) noexcept
The number of bits set in x.
Definition: bit:426
constexpr complex< _Tp > operator*(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x times y.
Definition: complex:395
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
Definition: complex:365
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
Definition: complex:335
constexpr complex< _Tp > operator/(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x divided by y.
Definition: complex:425
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
Definition: type_traits:1640
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition: type_traits:1983
void void_t
A metafunction that always yields void, used for detecting valid types.
Definition: type_traits:2632
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
Definition: type_traits:82
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition: type_traits:2614
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
Definition: type_traits:85
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
Definition: type_traits:2610
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
Create a tuple containing all elements from multiple tuple-like objects.
Definition: tuple:2141
auto declval() noexcept -> decltype(__declval< _Tp >(0))
Definition: type_traits:2386
constexpr tuple< typename __decay_and_strip< _Elements >::__type... > make_tuple(_Elements &&... __args)
Create a tuple containing copies of the arguments.
Definition: tuple:1987
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
Definition: move.h:104
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:257
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:233
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
Definition: numeric:287
constexpr bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1573
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1593
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1683
constexpr bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1563
constexpr bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1553