libstdc++
simd.h
1 // Definition of the public simd interfaces -*- C++ -*-
2 
3 // Copyright (C) 2020-2022 Free Software Foundation, Inc.
4 //
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 3, or (at your option)
9 // any later version.
10 
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
15 
16 // Under Section 7 of GPL version 3, you are granted additional
17 // permissions described in the GCC Runtime Library Exception, version
18 // 3.1, as published by the Free Software Foundation.
19 
20 // You should have received a copy of the GNU General Public License and
21 // a copy of the GCC Runtime Library Exception along with this program;
22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 // <http://www.gnu.org/licenses/>.
24 
25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H
26 #define _GLIBCXX_EXPERIMENTAL_SIMD_H
27 
28 #if __cplusplus >= 201703L
29 
30 #include "simd_detail.h"
31 #include "numeric_traits.h"
32 #include <bit>
33 #include <bitset>
34 #ifdef _GLIBCXX_DEBUG_UB
35 #include <cstdio> // for stderr
36 #endif
37 #include <cstring>
38 #include <cmath>
39 #include <functional>
40 #include <iosfwd>
41 #include <utility>
42 
43 #if _GLIBCXX_SIMD_X86INTRIN
44 #include <x86intrin.h>
45 #elif _GLIBCXX_SIMD_HAVE_NEON
46 #include <arm_neon.h>
47 #endif
48 
49 /** @ingroup ts_simd
50  * @{
51  */
52 /* There are several closely related types, with the following naming
53  * convention:
54  * _Tp: vectorizable (arithmetic) type (or any type)
55  * _TV: __vector_type_t<_Tp, _Np>
56  * _TW: _SimdWrapper<_Tp, _Np>
57  * _TI: __intrinsic_type_t<_Tp, _Np>
58  * _TVT: _VectorTraits<_TV> or _VectorTraits<_TW>
59  * If one additional type is needed use _U instead of _T.
60  * Otherwise use _T\d, _TV\d, _TW\d, TI\d, _TVT\d.
61  *
62  * More naming conventions:
63  * _Ap or _Abi: An ABI tag from the simd_abi namespace
64  * _Ip: often used for integer types with sizeof(_Ip) == sizeof(_Tp),
65  * _IV, _IW as for _TV, _TW
66  * _Np: number of elements (not bytes)
67  * _Bytes: number of bytes
68  *
69  * Variable names:
70  * __k: mask object (vector- or bitmask)
71  */
72 _GLIBCXX_SIMD_BEGIN_NAMESPACE
73 
74 #if !_GLIBCXX_SIMD_X86INTRIN
75 using __m128 [[__gnu__::__vector_size__(16)]] = float;
76 using __m128d [[__gnu__::__vector_size__(16)]] = double;
77 using __m128i [[__gnu__::__vector_size__(16)]] = long long;
78 using __m256 [[__gnu__::__vector_size__(32)]] = float;
79 using __m256d [[__gnu__::__vector_size__(32)]] = double;
80 using __m256i [[__gnu__::__vector_size__(32)]] = long long;
81 using __m512 [[__gnu__::__vector_size__(64)]] = float;
82 using __m512d [[__gnu__::__vector_size__(64)]] = double;
83 using __m512i [[__gnu__::__vector_size__(64)]] = long long;
84 #endif
85 
86 namespace simd_abi {
87 // simd_abi forward declarations {{{
88 // implementation details:
89 struct _Scalar;
90 
91 template <int _Np>
92  struct _Fixed;
93 
94 // There are two major ABIs that appear on different architectures.
95 // Both have non-boolean values packed into an N Byte register
96 // -> #elements = N / sizeof(T)
97 // Masks differ:
98 // 1. Use value vector registers for masks (all 0 or all 1)
99 // 2. Use bitmasks (mask registers) with one bit per value in the corresponding
100 // value vector
101 //
102 // Both can be partially used, masking off the rest when doing horizontal
103 // operations or operations that can trap (e.g. FP_INVALID or integer division
104 // by 0). This is encoded as the number of used bytes.
105 template <int _UsedBytes>
106  struct _VecBuiltin;
107 
108 template <int _UsedBytes>
109  struct _VecBltnBtmsk;
110 
111 template <typename _Tp, int _Np>
112  using _VecN = _VecBuiltin<sizeof(_Tp) * _Np>;
113 
114 template <int _UsedBytes = 16>
115  using _Sse = _VecBuiltin<_UsedBytes>;
116 
117 template <int _UsedBytes = 32>
118  using _Avx = _VecBuiltin<_UsedBytes>;
119 
120 template <int _UsedBytes = 64>
121  using _Avx512 = _VecBltnBtmsk<_UsedBytes>;
122 
123 template <int _UsedBytes = 16>
124  using _Neon = _VecBuiltin<_UsedBytes>;
125 
126 // implementation-defined:
127 using __sse = _Sse<>;
128 using __avx = _Avx<>;
129 using __avx512 = _Avx512<>;
130 using __neon = _Neon<>;
131 using __neon128 = _Neon<16>;
132 using __neon64 = _Neon<8>;
133 
134 // standard:
135 template <typename _Tp, size_t _Np, typename...>
136  struct deduce;
137 
138 template <int _Np>
139  using fixed_size = _Fixed<_Np>;
140 
141 using scalar = _Scalar;
142 
143 // }}}
144 } // namespace simd_abi
145 // forward declarations is_simd(_mask), simd(_mask), simd_size {{{
146 template <typename _Tp>
147  struct is_simd;
148 
149 template <typename _Tp>
150  struct is_simd_mask;
151 
152 template <typename _Tp, typename _Abi>
153  class simd;
154 
155 template <typename _Tp, typename _Abi>
156  class simd_mask;
157 
158 template <typename _Tp, typename _Abi>
159  struct simd_size;
160 
161 // }}}
162 // load/store flags {{{
163 struct element_aligned_tag
164 {
165  template <typename _Tp, typename _Up = typename _Tp::value_type>
166  static constexpr size_t _S_alignment = alignof(_Up);
167 
168  template <typename _Tp, typename _Up>
169  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
170  _S_apply(_Up* __ptr)
171  { return __ptr; }
172 };
173 
174 struct vector_aligned_tag
175 {
176  template <typename _Tp, typename _Up = typename _Tp::value_type>
177  static constexpr size_t _S_alignment
178  = std::__bit_ceil(sizeof(_Up) * _Tp::size());
179 
180  template <typename _Tp, typename _Up>
181  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
182  _S_apply(_Up* __ptr)
183  {
184  return static_cast<_Up*>(
185  __builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>));
186  }
187 };
188 
189 template <size_t _Np> struct overaligned_tag
190 {
191  template <typename _Tp, typename _Up = typename _Tp::value_type>
192  static constexpr size_t _S_alignment = _Np;
193 
194  template <typename _Tp, typename _Up>
195  _GLIBCXX_SIMD_INTRINSIC static constexpr _Up*
196  _S_apply(_Up* __ptr)
197  { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); }
198 };
199 
200 inline constexpr element_aligned_tag element_aligned = {};
201 
202 inline constexpr vector_aligned_tag vector_aligned = {};
203 
204 template <size_t _Np>
205  inline constexpr overaligned_tag<_Np> overaligned = {};
206 
207 // }}}
208 template <size_t _Xp>
209  using _SizeConstant = integral_constant<size_t, _Xp>;
210 // constexpr feature detection{{{
211 constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX;
212 constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE;
213 constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2;
214 constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3;
215 constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3;
216 constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1;
217 constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2;
218 constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP;
219 constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX;
220 constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2;
221 constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1;
222 constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2;
223 constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT;
224 constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A;
225 constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA;
226 constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4;
227 constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C;
228 constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT;
229 constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F;
230 constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ;
231 constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL;
232 constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW;
233 constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl;
234 constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl;
235 constexpr inline bool __have_avx512bitalg = _GLIBCXX_SIMD_HAVE_AVX512BITALG;
236 constexpr inline bool __have_avx512vbmi2 = _GLIBCXX_SIMD_HAVE_AVX512VBMI2;
237 constexpr inline bool __have_avx512vbmi = _GLIBCXX_SIMD_HAVE_AVX512VBMI;
238 constexpr inline bool __have_avx512ifma = _GLIBCXX_SIMD_HAVE_AVX512IFMA;
239 constexpr inline bool __have_avx512cd = _GLIBCXX_SIMD_HAVE_AVX512CD;
240 constexpr inline bool __have_avx512vnni = _GLIBCXX_SIMD_HAVE_AVX512VNNI;
241 constexpr inline bool __have_avx512vpopcntdq = _GLIBCXX_SIMD_HAVE_AVX512VPOPCNTDQ;
242 constexpr inline bool __have_avx512vp2intersect = _GLIBCXX_SIMD_HAVE_AVX512VP2INTERSECT;
243 
244 constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON;
245 constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32;
246 constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64;
247 constexpr inline bool __support_neon_float =
248 #if defined __GCC_IEC_559
249  __GCC_IEC_559 == 0;
250 #elif defined __FAST_MATH__
251  true;
252 #else
253  false;
254 #endif
255 
256 #ifdef _ARCH_PWR10
257 constexpr inline bool __have_power10vec = true;
258 #else
259 constexpr inline bool __have_power10vec = false;
260 #endif
261 #ifdef __POWER9_VECTOR__
262 constexpr inline bool __have_power9vec = true;
263 #else
264 constexpr inline bool __have_power9vec = false;
265 #endif
266 #if defined __POWER8_VECTOR__
267 constexpr inline bool __have_power8vec = true;
268 #else
269 constexpr inline bool __have_power8vec = __have_power9vec;
270 #endif
271 #if defined __VSX__
272 constexpr inline bool __have_power_vsx = true;
273 #else
274 constexpr inline bool __have_power_vsx = __have_power8vec;
275 #endif
276 #if defined __ALTIVEC__
277 constexpr inline bool __have_power_vmx = true;
278 #else
279 constexpr inline bool __have_power_vmx = __have_power_vsx;
280 #endif
281 
282 // }}}
283 
284 namespace __detail
285 {
286  constexpr bool __handle_fpexcept =
287 #ifdef math_errhandling
288  math_errhandling & MATH_ERREXCEPT;
289 #elif defined __FAST_MATH__
290  false;
291 #else
292  true;
293 #endif
294 
295  constexpr std::uint_least64_t
296  __floating_point_flags()
297  {
298  std::uint_least64_t __flags = 0;
299  if constexpr (__handle_fpexcept)
300  __flags |= 1;
301 #ifdef __FAST_MATH__
302  __flags |= 1 << 1;
303 #elif __FINITE_MATH_ONLY__
304  __flags |= 2 << 1;
305 #elif __GCC_IEC_559 < 2
306  __flags |= 3 << 1;
307 #endif
308  __flags |= (__FLT_EVAL_METHOD__ + 1) << 3;
309  return __flags;
310  }
311 
312  constexpr std::uint_least64_t
313  __machine_flags()
314  {
315  if constexpr (__have_mmx || __have_sse)
316  return __have_mmx
317  | (__have_sse << 1)
318  | (__have_sse2 << 2)
319  | (__have_sse3 << 3)
320  | (__have_ssse3 << 4)
321  | (__have_sse4_1 << 5)
322  | (__have_sse4_2 << 6)
323  | (__have_xop << 7)
324  | (__have_avx << 8)
325  | (__have_avx2 << 9)
326  | (__have_bmi << 10)
327  | (__have_bmi2 << 11)
328  | (__have_lzcnt << 12)
329  | (__have_sse4a << 13)
330  | (__have_fma << 14)
331  | (__have_fma4 << 15)
332  | (__have_f16c << 16)
333  | (__have_popcnt << 17)
334  | (__have_avx512f << 18)
335  | (__have_avx512dq << 19)
336  | (__have_avx512vl << 20)
337  | (__have_avx512bw << 21)
338  | (__have_avx512bitalg << 22)
339  | (__have_avx512vbmi2 << 23)
340  | (__have_avx512vbmi << 24)
341  | (__have_avx512ifma << 25)
342  | (__have_avx512cd << 26)
343  | (__have_avx512vnni << 27)
344  | (__have_avx512vpopcntdq << 28)
345  | (__have_avx512vp2intersect << 29);
346  else if constexpr (__have_neon)
347  return __have_neon
348  | (__have_neon_a32 << 1)
349  | (__have_neon_a64 << 2)
350  | (__have_neon_a64 << 2)
351  | (__support_neon_float << 3);
352  else if constexpr (__have_power_vmx)
353  return __have_power_vmx
354  | (__have_power_vsx << 1)
355  | (__have_power8vec << 2)
356  | (__have_power9vec << 3)
357  | (__have_power10vec << 4);
358  else
359  return 0;
360  }
361 
362  namespace
363  {
364  struct _OdrEnforcer {};
365  }
366 
367  template <std::uint_least64_t...>
368  struct _MachineFlagsTemplate {};
369 
370  /**@internal
371  * Use this type as default template argument to all function templates that
372  * are not declared always_inline. It ensures, that a function
373  * specialization, which the compiler decides not to inline, has a unique symbol
374  * (_OdrEnforcer) or a symbol matching the machine/architecture flags
375  * (_MachineFlagsTemplate). This helps to avoid ODR violations in cases where
376  * users link TUs compiled with different flags. This is especially important
377  * for using simd in libraries.
378  */
379  using __odr_helper
380  = conditional_t<__machine_flags() == 0, _OdrEnforcer,
381  _MachineFlagsTemplate<__machine_flags(), __floating_point_flags()>>;
382 
383  struct _Minimum
384  {
385  template <typename _Tp>
386  _GLIBCXX_SIMD_INTRINSIC constexpr
387  _Tp
388  operator()(_Tp __a, _Tp __b) const
389  {
390  using std::min;
391  return min(__a, __b);
392  }
393  };
394 
395  struct _Maximum
396  {
397  template <typename _Tp>
398  _GLIBCXX_SIMD_INTRINSIC constexpr
399  _Tp
400  operator()(_Tp __a, _Tp __b) const
401  {
402  using std::max;
403  return max(__a, __b);
404  }
405  };
406 } // namespace __detail
407 
408 // unrolled/pack execution helpers
409 // __execute_n_times{{{
410 template <typename _Fp, size_t... _I>
411  [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
412  void
413  __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>)
414  { ((void)__f(_SizeConstant<_I>()), ...); }
415 
416 template <typename _Fp>
417  _GLIBCXX_SIMD_INTRINSIC constexpr void
418  __execute_on_index_sequence(_Fp&&, index_sequence<>)
419  { }
420 
421 template <size_t _Np, typename _Fp>
422  _GLIBCXX_SIMD_INTRINSIC constexpr void
423  __execute_n_times(_Fp&& __f)
424  {
425  __execute_on_index_sequence(static_cast<_Fp&&>(__f),
426  make_index_sequence<_Np>{});
427  }
428 
429 // }}}
430 // __generate_from_n_evaluations{{{
431 template <typename _R, typename _Fp, size_t... _I>
432  [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
433  _R
434  __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>)
435  { return _R{__f(_SizeConstant<_I>())...}; }
436 
437 template <size_t _Np, typename _R, typename _Fp>
438  _GLIBCXX_SIMD_INTRINSIC constexpr _R
439  __generate_from_n_evaluations(_Fp&& __f)
440  {
441  return __execute_on_index_sequence_with_return<_R>(
442  static_cast<_Fp&&>(__f), make_index_sequence<_Np>{});
443  }
444 
445 // }}}
446 // __call_with_n_evaluations{{{
447 template <size_t... _I, typename _F0, typename _FArgs>
448  [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
449  auto
450  __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs)
451  { return __f0(__fargs(_SizeConstant<_I>())...); }
452 
453 template <size_t _Np, typename _F0, typename _FArgs>
454  _GLIBCXX_SIMD_INTRINSIC constexpr auto
455  __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs)
456  {
457  return __call_with_n_evaluations(make_index_sequence<_Np>{},
458  static_cast<_F0&&>(__f0),
459  static_cast<_FArgs&&>(__fargs));
460  }
461 
462 // }}}
463 // __call_with_subscripts{{{
464 template <size_t _First = 0, size_t... _It, typename _Tp, typename _Fp>
465  [[__gnu__::__flatten__]] _GLIBCXX_SIMD_INTRINSIC constexpr
466  auto
467  __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun)
468  { return __fun(__x[_First + _It]...); }
469 
470 template <size_t _Np, size_t _First = 0, typename _Tp, typename _Fp>
471  _GLIBCXX_SIMD_INTRINSIC constexpr auto
472  __call_with_subscripts(_Tp&& __x, _Fp&& __fun)
473  {
474  return __call_with_subscripts<_First>(static_cast<_Tp&&>(__x),
475  make_index_sequence<_Np>(),
476  static_cast<_Fp&&>(__fun));
477  }
478 
479 // }}}
480 
481 // vvv ---- type traits ---- vvv
482 // integer type aliases{{{
483 using _UChar = unsigned char;
484 using _SChar = signed char;
485 using _UShort = unsigned short;
486 using _UInt = unsigned int;
487 using _ULong = unsigned long;
488 using _ULLong = unsigned long long;
489 using _LLong = long long;
490 
491 //}}}
492 // __first_of_pack{{{
493 template <typename _T0, typename...>
494  struct __first_of_pack
495  { using type = _T0; };
496 
497 template <typename... _Ts>
498  using __first_of_pack_t = typename __first_of_pack<_Ts...>::type;
499 
500 //}}}
501 // __value_type_or_identity_t {{{
502 template <typename _Tp>
503  typename _Tp::value_type
504  __value_type_or_identity_impl(int);
505 
506 template <typename _Tp>
507  _Tp
508  __value_type_or_identity_impl(float);
509 
510 template <typename _Tp>
511  using __value_type_or_identity_t
512  = decltype(__value_type_or_identity_impl<_Tp>(int()));
513 
514 // }}}
515 // __is_vectorizable {{{
516 template <typename _Tp>
517  struct __is_vectorizable : public is_arithmetic<_Tp> {};
518 
519 template <>
520  struct __is_vectorizable<bool> : public false_type {};
521 
522 template <typename _Tp>
523  inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value;
524 
525 // Deduces to a vectorizable type
526 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
527  using _Vectorizable = _Tp;
528 
529 // }}}
530 // _LoadStorePtr / __is_possible_loadstore_conversion {{{
531 template <typename _Ptr, typename _ValueType>
532  struct __is_possible_loadstore_conversion
533  : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {};
534 
535 template <>
536  struct __is_possible_loadstore_conversion<bool, bool> : true_type {};
537 
538 // Deduces to a type allowed for load/store with the given value type.
539 template <typename _Ptr, typename _ValueType,
540  typename = enable_if_t<
541  __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>>
542  using _LoadStorePtr = _Ptr;
543 
544 // }}}
545 // __is_bitmask{{{
546 template <typename _Tp, typename = void_t<>>
547  struct __is_bitmask : false_type {};
548 
549 template <typename _Tp>
550  inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value;
551 
552 // the __mmaskXX case:
553 template <typename _Tp>
554  struct __is_bitmask<_Tp,
555  void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>>
556  : true_type {};
557 
558 // }}}
559 // __int_for_sizeof{{{
560 #pragma GCC diagnostic push
561 #pragma GCC diagnostic ignored "-Wpedantic"
562 template <size_t _Bytes>
563  constexpr auto
564  __int_for_sizeof()
565  {
566  if constexpr (_Bytes == sizeof(int))
567  return int();
568  #ifdef __clang__
569  else if constexpr (_Bytes == sizeof(char))
570  return char();
571  #else
572  else if constexpr (_Bytes == sizeof(_SChar))
573  return _SChar();
574  #endif
575  else if constexpr (_Bytes == sizeof(short))
576  return short();
577  #ifndef __clang__
578  else if constexpr (_Bytes == sizeof(long))
579  return long();
580  #endif
581  else if constexpr (_Bytes == sizeof(_LLong))
582  return _LLong();
583  #ifdef __SIZEOF_INT128__
584  else if constexpr (_Bytes == sizeof(__int128))
585  return __int128();
586  #endif // __SIZEOF_INT128__
587  else if constexpr (_Bytes % sizeof(int) == 0)
588  {
589  constexpr size_t _Np = _Bytes / sizeof(int);
590  struct _Ip
591  {
592  int _M_data[_Np];
593 
594  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
595  operator&(_Ip __rhs) const
596  {
597  return __generate_from_n_evaluations<_Np, _Ip>(
598  [&](auto __i) { return __rhs._M_data[__i] & _M_data[__i]; });
599  }
600 
601  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
602  operator|(_Ip __rhs) const
603  {
604  return __generate_from_n_evaluations<_Np, _Ip>(
605  [&](auto __i) { return __rhs._M_data[__i] | _M_data[__i]; });
606  }
607 
608  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
609  operator^(_Ip __rhs) const
610  {
611  return __generate_from_n_evaluations<_Np, _Ip>(
612  [&](auto __i) { return __rhs._M_data[__i] ^ _M_data[__i]; });
613  }
614 
615  _GLIBCXX_SIMD_INTRINSIC constexpr _Ip
616  operator~() const
617  {
618  return __generate_from_n_evaluations<_Np, _Ip>(
619  [&](auto __i) { return ~_M_data[__i]; });
620  }
621  };
622  return _Ip{};
623  }
624  else
625  static_assert(_Bytes != _Bytes, "this should be unreachable");
626  }
627 #pragma GCC diagnostic pop
628 
629 template <typename _Tp>
630  using __int_for_sizeof_t = decltype(__int_for_sizeof<sizeof(_Tp)>());
631 
632 template <size_t _Np>
633  using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>());
634 
635 // }}}
636 // __is_fixed_size_abi{{{
637 template <typename _Tp>
638  struct __is_fixed_size_abi : false_type {};
639 
640 template <int _Np>
641  struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> : true_type {};
642 
643 template <typename _Tp>
644  inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value;
645 
646 // }}}
647 // __is_scalar_abi {{{
648 template <typename _Abi>
649  constexpr bool
650  __is_scalar_abi()
651  { return is_same_v<simd_abi::scalar, _Abi>; }
652 
653 // }}}
654 // __abi_bytes_v {{{
655 template <template <int> class _Abi, int _Bytes>
656  constexpr int
657  __abi_bytes_impl(_Abi<_Bytes>*)
658  { return _Bytes; }
659 
660 template <typename _Tp>
661  constexpr int
662  __abi_bytes_impl(_Tp*)
663  { return -1; }
664 
665 template <typename _Abi>
666  inline constexpr int __abi_bytes_v
667  = __abi_bytes_impl(static_cast<_Abi*>(nullptr));
668 
669 // }}}
670 // __is_builtin_bitmask_abi {{{
671 template <typename _Abi>
672  constexpr bool
673  __is_builtin_bitmask_abi()
674  { return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; }
675 
676 // }}}
677 // __is_sse_abi {{{
678 template <typename _Abi>
679  constexpr bool
680  __is_sse_abi()
681  {
682  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
683  return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
684  }
685 
686 // }}}
687 // __is_avx_abi {{{
688 template <typename _Abi>
689  constexpr bool
690  __is_avx_abi()
691  {
692  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
693  return _Bytes > 16 && _Bytes <= 32
694  && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
695  }
696 
697 // }}}
698 // __is_avx512_abi {{{
699 template <typename _Abi>
700  constexpr bool
701  __is_avx512_abi()
702  {
703  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
704  return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>;
705  }
706 
707 // }}}
708 // __is_neon_abi {{{
709 template <typename _Abi>
710  constexpr bool
711  __is_neon_abi()
712  {
713  constexpr auto _Bytes = __abi_bytes_v<_Abi>;
714  return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>;
715  }
716 
717 // }}}
718 // __make_dependent_t {{{
719 template <typename, typename _Up>
720  struct __make_dependent
721  { using type = _Up; };
722 
723 template <typename _Tp, typename _Up>
724  using __make_dependent_t = typename __make_dependent<_Tp, _Up>::type;
725 
726 // }}}
727 // ^^^ ---- type traits ---- ^^^
728 
729 // __invoke_ub{{{
730 template <typename... _Args>
731  [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE void
732  __invoke_ub([[maybe_unused]] const char* __msg,
733  [[maybe_unused]] const _Args&... __args)
734  {
735 #ifdef _GLIBCXX_DEBUG_UB
736  __builtin_fprintf(stderr, __msg, __args...);
737  __builtin_trap();
738 #else
739  __builtin_unreachable();
740 #endif
741  }
742 
743 // }}}
744 // __assert_unreachable{{{
745 template <typename _Tp>
746  struct __assert_unreachable
747  { static_assert(!is_same_v<_Tp, _Tp>, "this should be unreachable"); };
748 
749 // }}}
750 // __size_or_zero_v {{{
751 template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value>
752  constexpr size_t
753  __size_or_zero_dispatch(int)
754  { return _Np; }
755 
756 template <typename _Tp, typename _Ap>
757  constexpr size_t
758  __size_or_zero_dispatch(float)
759  { return 0; }
760 
761 template <typename _Tp, typename _Ap>
762  inline constexpr size_t __size_or_zero_v
763  = __size_or_zero_dispatch<_Tp, _Ap>(0);
764 
765 // }}}
766 // __div_roundup {{{
767 inline constexpr size_t
768 __div_roundup(size_t __a, size_t __b)
769 { return (__a + __b - 1) / __b; }
770 
771 // }}}
772 // _ExactBool{{{
773 class _ExactBool
774 {
775  const bool _M_data;
776 
777 public:
778  _GLIBCXX_SIMD_INTRINSIC constexpr _ExactBool(bool __b) : _M_data(__b) {}
779 
780  _ExactBool(int) = delete;
781 
782  _GLIBCXX_SIMD_INTRINSIC constexpr operator bool() const { return _M_data; }
783 };
784 
785 // }}}
786 // __may_alias{{{
787 /**@internal
788  * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an
789  * aliasing pointer. This adds the __may_alias attribute to _Tp (with compilers
790  * that support it).
791  */
792 template <typename _Tp>
793  using __may_alias [[__gnu__::__may_alias__]] = _Tp;
794 
795 // }}}
796 // _UnsupportedBase {{{
797 // simd and simd_mask base for unsupported <_Tp, _Abi>
798 struct _UnsupportedBase
799 {
800  _UnsupportedBase() = delete;
801  _UnsupportedBase(const _UnsupportedBase&) = delete;
802  _UnsupportedBase& operator=(const _UnsupportedBase&) = delete;
803  ~_UnsupportedBase() = delete;
804 };
805 
806 // }}}
807 // _InvalidTraits {{{
808 /**
809  * @internal
810  * Defines the implementation of __a given <_Tp, _Abi>.
811  *
812  * Implementations must ensure that only valid <_Tp, _Abi> instantiations are
813  * possible. Static assertions in the type definition do not suffice. It is
814  * important that SFINAE works.
815  */
816 struct _InvalidTraits
817 {
818  using _IsValid = false_type;
819  using _SimdBase = _UnsupportedBase;
820  using _MaskBase = _UnsupportedBase;
821 
822  static constexpr size_t _S_full_size = 0;
823  static constexpr bool _S_is_partial = false;
824 
825  static constexpr size_t _S_simd_align = 1;
826  struct _SimdImpl;
827  struct _SimdMember {};
828  struct _SimdCastType;
829 
830  static constexpr size_t _S_mask_align = 1;
831  struct _MaskImpl;
832  struct _MaskMember {};
833  struct _MaskCastType;
834 };
835 
836 // }}}
837 // _SimdTraits {{{
838 template <typename _Tp, typename _Abi, typename = void_t<>>
839  struct _SimdTraits : _InvalidTraits {};
840 
841 // }}}
842 // __private_init, __bitset_init{{{
843 /**
844  * @internal
845  * Tag used for private init constructor of simd and simd_mask
846  */
847 inline constexpr struct _PrivateInit {} __private_init = {};
848 
849 inline constexpr struct _BitsetInit {} __bitset_init = {};
850 
851 // }}}
852 // __is_narrowing_conversion<_From, _To>{{{
853 template <typename _From, typename _To, bool = is_arithmetic_v<_From>,
854  bool = is_arithmetic_v<_To>>
855  struct __is_narrowing_conversion;
856 
857 // ignore "signed/unsigned mismatch" in the following trait.
858 // The implicit conversions will do the right thing here.
859 template <typename _From, typename _To>
860  struct __is_narrowing_conversion<_From, _To, true, true>
861  : public __bool_constant<(
862  __digits_v<_From> > __digits_v<_To>
863  || __finite_max_v<_From> > __finite_max_v<_To>
864  || __finite_min_v<_From> < __finite_min_v<_To>
865  || (is_signed_v<_From> && is_unsigned_v<_To>))> {};
866 
867 template <typename _Tp>
868  struct __is_narrowing_conversion<_Tp, bool, true, true>
869  : public true_type {};
870 
871 template <>
872  struct __is_narrowing_conversion<bool, bool, true, true>
873  : public false_type {};
874 
875 template <typename _Tp>
876  struct __is_narrowing_conversion<_Tp, _Tp, true, true>
877  : public false_type {};
878 
879 template <typename _From, typename _To>
880  struct __is_narrowing_conversion<_From, _To, false, true>
881  : public negation<is_convertible<_From, _To>> {};
882 
883 // }}}
884 // __converts_to_higher_integer_rank{{{
885 template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))>
886  struct __converts_to_higher_integer_rank : public true_type {};
887 
888 // this may fail for char -> short if sizeof(char) == sizeof(short)
889 template <typename _From, typename _To>
890  struct __converts_to_higher_integer_rank<_From, _To, false>
891  : public is_same<decltype(declval<_From>() + declval<_To>()), _To> {};
892 
893 // }}}
894 // __data(simd/simd_mask) {{{
895 template <typename _Tp, typename _Ap>
896  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
897  __data(const simd<_Tp, _Ap>& __x);
898 
899 template <typename _Tp, typename _Ap>
900  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
901  __data(simd<_Tp, _Ap>& __x);
902 
903 template <typename _Tp, typename _Ap>
904  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
905  __data(const simd_mask<_Tp, _Ap>& __x);
906 
907 template <typename _Tp, typename _Ap>
908  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
909  __data(simd_mask<_Tp, _Ap>& __x);
910 
911 // }}}
912 // _SimdConverter {{{
913 template <typename _FromT, typename _FromA, typename _ToT, typename _ToA,
914  typename = void>
915  struct _SimdConverter;
916 
917 template <typename _Tp, typename _Ap>
918  struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void>
919  {
920  template <typename _Up>
921  _GLIBCXX_SIMD_INTRINSIC const _Up&
922  operator()(const _Up& __x)
923  { return __x; }
924  };
925 
926 // }}}
927 // __to_value_type_or_member_type {{{
928 template <typename _V>
929  _GLIBCXX_SIMD_INTRINSIC constexpr auto
930  __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x))
931  { return __data(__x); }
932 
933 template <typename _V>
934  _GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type&
935  __to_value_type_or_member_type(const typename _V::value_type& __x)
936  { return __x; }
937 
938 // }}}
939 // __bool_storage_member_type{{{
940 template <size_t _Size>
941  struct __bool_storage_member_type;
942 
943 template <size_t _Size>
944  using __bool_storage_member_type_t =
945  typename __bool_storage_member_type<_Size>::type;
946 
947 // }}}
948 // _SimdTuple {{{
949 // why not tuple?
950 // 1. tuple gives no guarantee about the storage order, but I require
951 // storage
952 // equivalent to array<_Tp, _Np>
953 // 2. direct access to the element type (first template argument)
954 // 3. enforces equal element type, only different _Abi types are allowed
955 template <typename _Tp, typename... _Abis>
956  struct _SimdTuple;
957 
958 //}}}
959 // __fixed_size_storage_t {{{
960 template <typename _Tp, int _Np>
961  struct __fixed_size_storage;
962 
963 template <typename _Tp, int _Np>
964  using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _Np>::type;
965 
966 // }}}
967 // _SimdWrapper fwd decl{{{
968 template <typename _Tp, size_t _Size, typename = void_t<>>
969  struct _SimdWrapper;
970 
971 template <typename _Tp>
972  using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>;
973 template <typename _Tp>
974  using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>;
975 template <typename _Tp>
976  using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>;
977 template <typename _Tp>
978  using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>;
979 
980 // }}}
981 // __is_simd_wrapper {{{
982 template <typename _Tp>
983  struct __is_simd_wrapper : false_type {};
984 
985 template <typename _Tp, size_t _Np>
986  struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> : true_type {};
987 
988 template <typename _Tp>
989  inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value;
990 
991 // }}}
992 // _BitOps {{{
993 struct _BitOps
994 {
995  // _S_bit_iteration {{{
996  template <typename _Tp, typename _Fp>
997  static void
998  _S_bit_iteration(_Tp __mask, _Fp&& __f)
999  {
1000  static_assert(sizeof(_ULLong) >= sizeof(_Tp));
1001  conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k;
1002  if constexpr (is_convertible_v<_Tp, decltype(__k)>)
1003  __k = __mask;
1004  else
1005  __k = __mask.to_ullong();
1006  while(__k)
1007  {
1008  __f(std::__countr_zero(__k));
1009  __k &= (__k - 1);
1010  }
1011  }
1012 
1013  //}}}
1014 };
1015 
1016 //}}}
1017 // __increment, __decrement {{{
1018 template <typename _Tp = void>
1019  struct __increment
1020  { constexpr _Tp operator()(_Tp __a) const { return ++__a; } };
1021 
1022 template <>
1023  struct __increment<void>
1024  {
1025  template <typename _Tp>
1026  constexpr _Tp
1027  operator()(_Tp __a) const
1028  { return ++__a; }
1029  };
1030 
1031 template <typename _Tp = void>
1032  struct __decrement
1033  { constexpr _Tp operator()(_Tp __a) const { return --__a; } };
1034 
1035 template <>
1036  struct __decrement<void>
1037  {
1038  template <typename _Tp>
1039  constexpr _Tp
1040  operator()(_Tp __a) const
1041  { return --__a; }
1042  };
1043 
1044 // }}}
1045 // _ValuePreserving(OrInt) {{{
1046 template <typename _From, typename _To,
1047  typename = enable_if_t<negation<
1048  __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>>
1049  using _ValuePreserving = _From;
1050 
1051 template <typename _From, typename _To,
1052  typename _DecayedFrom = __remove_cvref_t<_From>,
1053  typename = enable_if_t<conjunction<
1054  is_convertible<_From, _To>,
1055  disjunction<
1056  is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>,
1057  conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>,
1058  negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>>
1059  using _ValuePreservingOrInt = _From;
1060 
1061 // }}}
1062 // __intrinsic_type {{{
1063 template <typename _Tp, size_t _Bytes, typename = void_t<>>
1064  struct __intrinsic_type;
1065 
1066 template <typename _Tp, size_t _Size>
1067  using __intrinsic_type_t =
1068  typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type;
1069 
1070 template <typename _Tp>
1071  using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type;
1072 template <typename _Tp>
1073  using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type;
1074 template <typename _Tp>
1075  using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type;
1076 template <typename _Tp>
1077  using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type;
1078 template <typename _Tp>
1079  using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type;
1080 template <typename _Tp>
1081  using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type;
1082 
1083 // }}}
1084 // _BitMask {{{
1085 template <size_t _Np, bool _Sanitized = false>
1086  struct _BitMask;
1087 
1088 template <size_t _Np, bool _Sanitized>
1089  struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> : true_type {};
1090 
1091 template <size_t _Np>
1092  using _SanitizedBitMask = _BitMask<_Np, true>;
1093 
1094 template <size_t _Np, bool _Sanitized>
1095  struct _BitMask
1096  {
1097  static_assert(_Np > 0);
1098 
1099  static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__);
1100 
1101  using _Tp = conditional_t<_Np == 1, bool,
1102  make_unsigned_t<__int_with_sizeof_t<std::min(
1103  sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>;
1104 
1105  static constexpr int _S_array_size = __div_roundup(_NBytes, sizeof(_Tp));
1106 
1107  _Tp _M_bits[_S_array_size];
1108 
1109  static constexpr int _S_unused_bits
1110  = _Np == 1 ? 0 : _S_array_size * sizeof(_Tp) * __CHAR_BIT__ - _Np;
1111 
1112  static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits;
1113 
1114  constexpr _BitMask() noexcept = default;
1115 
1116  constexpr _BitMask(unsigned long long __x) noexcept
1117  : _M_bits{static_cast<_Tp>(__x)} {}
1118 
1119  _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {}
1120 
1121  constexpr _BitMask(const _BitMask&) noexcept = default;
1122 
1123  template <bool _RhsSanitized, typename = enable_if_t<_RhsSanitized == false
1124  && _Sanitized == true>>
1125  constexpr _BitMask(const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept
1126  : _BitMask(__rhs._M_sanitized()) {}
1127 
1128  constexpr operator _SimdWrapper<bool, _Np>() const noexcept
1129  {
1130  static_assert(_S_array_size == 1);
1131  return _M_bits[0];
1132  }
1133 
1134  // precondition: is sanitized
1135  constexpr _Tp
1136  _M_to_bits() const noexcept
1137  {
1138  static_assert(_S_array_size == 1);
1139  return _M_bits[0];
1140  }
1141 
1142  // precondition: is sanitized
1143  constexpr unsigned long long
1144  to_ullong() const noexcept
1145  {
1146  static_assert(_S_array_size == 1);
1147  return _M_bits[0];
1148  }
1149 
1150  // precondition: is sanitized
1151  constexpr unsigned long
1152  to_ulong() const noexcept
1153  {
1154  static_assert(_S_array_size == 1);
1155  return _M_bits[0];
1156  }
1157 
1158  constexpr bitset<_Np>
1159  _M_to_bitset() const noexcept
1160  {
1161  static_assert(_S_array_size == 1);
1162  return _M_bits[0];
1163  }
1164 
1165  constexpr decltype(auto)
1166  _M_sanitized() const noexcept
1167  {
1168  if constexpr (_Sanitized)
1169  return *this;
1170  else if constexpr (_Np == 1)
1171  return _SanitizedBitMask<_Np>(_M_bits[0]);
1172  else
1173  {
1174  _SanitizedBitMask<_Np> __r = {};
1175  for (int __i = 0; __i < _S_array_size; ++__i)
1176  __r._M_bits[__i] = _M_bits[__i];
1177  if constexpr (_S_unused_bits > 0)
1178  __r._M_bits[_S_array_size - 1] &= _S_bitmask;
1179  return __r;
1180  }
1181  }
1182 
1183  template <size_t _Mp, bool _LSanitized>
1184  constexpr _BitMask<_Np + _Mp, _Sanitized>
1185  _M_prepend(_BitMask<_Mp, _LSanitized> __lsb) const noexcept
1186  {
1187  constexpr size_t _RN = _Np + _Mp;
1188  using _Rp = _BitMask<_RN, _Sanitized>;
1189  if constexpr (_Rp::_S_array_size == 1)
1190  {
1191  _Rp __r{{_M_bits[0]}};
1192  __r._M_bits[0] <<= _Mp;
1193  __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0];
1194  return __r;
1195  }
1196  else
1197  __assert_unreachable<_Rp>();
1198  }
1199 
1200  // Return a new _BitMask with size _NewSize while dropping _DropLsb least
1201  // significant bits. If the operation implicitly produces a sanitized bitmask,
1202  // the result type will have _Sanitized set.
1203  template <size_t _DropLsb, size_t _NewSize = _Np - _DropLsb>
1204  constexpr auto
1205  _M_extract() const noexcept
1206  {
1207  static_assert(_Np > _DropLsb);
1208  static_assert(_DropLsb + _NewSize <= sizeof(_ULLong) * __CHAR_BIT__,
1209  "not implemented for bitmasks larger than one ullong");
1210  if constexpr (_NewSize == 1)
1211  // must sanitize because the return _Tp is bool
1212  return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb));
1213  else
1214  return _BitMask<_NewSize,
1215  ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__
1216  && _NewSize + _DropLsb <= _Np)
1217  || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__)
1218  && _NewSize + _DropLsb >= _Np))>(_M_bits[0]
1219  >> _DropLsb);
1220  }
1221 
1222  // True if all bits are set. Implicitly sanitizes if _Sanitized == false.
1223  constexpr bool
1224  all() const noexcept
1225  {
1226  if constexpr (_Np == 1)
1227  return _M_bits[0];
1228  else if constexpr (!_Sanitized)
1229  return _M_sanitized().all();
1230  else
1231  {
1232  constexpr _Tp __allbits = ~_Tp();
1233  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1234  if (_M_bits[__i] != __allbits)
1235  return false;
1236  return _M_bits[_S_array_size - 1] == _S_bitmask;
1237  }
1238  }
1239 
1240  // True if at least one bit is set. Implicitly sanitizes if _Sanitized ==
1241  // false.
1242  constexpr bool
1243  any() const noexcept
1244  {
1245  if constexpr (_Np == 1)
1246  return _M_bits[0];
1247  else if constexpr (!_Sanitized)
1248  return _M_sanitized().any();
1249  else
1250  {
1251  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1252  if (_M_bits[__i] != 0)
1253  return true;
1254  return _M_bits[_S_array_size - 1] != 0;
1255  }
1256  }
1257 
1258  // True if no bit is set. Implicitly sanitizes if _Sanitized == false.
1259  constexpr bool
1260  none() const noexcept
1261  {
1262  if constexpr (_Np == 1)
1263  return !_M_bits[0];
1264  else if constexpr (!_Sanitized)
1265  return _M_sanitized().none();
1266  else
1267  {
1268  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1269  if (_M_bits[__i] != 0)
1270  return false;
1271  return _M_bits[_S_array_size - 1] == 0;
1272  }
1273  }
1274 
1275  // Returns the number of set bits. Implicitly sanitizes if _Sanitized ==
1276  // false.
1277  constexpr int
1278  count() const noexcept
1279  {
1280  if constexpr (_Np == 1)
1281  return _M_bits[0];
1282  else if constexpr (!_Sanitized)
1283  return _M_sanitized().none();
1284  else
1285  {
1286  int __result = __builtin_popcountll(_M_bits[0]);
1287  for (int __i = 1; __i < _S_array_size; ++__i)
1288  __result += __builtin_popcountll(_M_bits[__i]);
1289  return __result;
1290  }
1291  }
1292 
1293  // Returns the bit at offset __i as bool.
1294  constexpr bool
1295  operator[](size_t __i) const noexcept
1296  {
1297  if constexpr (_Np == 1)
1298  return _M_bits[0];
1299  else if constexpr (_S_array_size == 1)
1300  return (_M_bits[0] >> __i) & 1;
1301  else
1302  {
1303  const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1304  const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1305  return (_M_bits[__j] >> __shift) & 1;
1306  }
1307  }
1308 
1309  template <size_t __i>
1310  constexpr bool
1311  operator[](_SizeConstant<__i>) const noexcept
1312  {
1313  static_assert(__i < _Np);
1314  constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1315  constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1316  return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift));
1317  }
1318 
1319  // Set the bit at offset __i to __x.
1320  constexpr void
1321  set(size_t __i, bool __x) noexcept
1322  {
1323  if constexpr (_Np == 1)
1324  _M_bits[0] = __x;
1325  else if constexpr (_S_array_size == 1)
1326  {
1327  _M_bits[0] &= ~_Tp(_Tp(1) << __i);
1328  _M_bits[0] |= _Tp(_Tp(__x) << __i);
1329  }
1330  else
1331  {
1332  const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1333  const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1334  _M_bits[__j] &= ~_Tp(_Tp(1) << __shift);
1335  _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1336  }
1337  }
1338 
1339  template <size_t __i>
1340  constexpr void
1341  set(_SizeConstant<__i>, bool __x) noexcept
1342  {
1343  static_assert(__i < _Np);
1344  if constexpr (_Np == 1)
1345  _M_bits[0] = __x;
1346  else
1347  {
1348  constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__);
1349  constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__);
1350  constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift);
1351  _M_bits[__j] &= __mask;
1352  _M_bits[__j] |= _Tp(_Tp(__x) << __shift);
1353  }
1354  }
1355 
1356  // Inverts all bits. Sanitized input leads to sanitized output.
1357  constexpr _BitMask
1358  operator~() const noexcept
1359  {
1360  if constexpr (_Np == 1)
1361  return !_M_bits[0];
1362  else
1363  {
1364  _BitMask __result{};
1365  for (int __i = 0; __i < _S_array_size - 1; ++__i)
1366  __result._M_bits[__i] = ~_M_bits[__i];
1367  if constexpr (_Sanitized)
1368  __result._M_bits[_S_array_size - 1]
1369  = _M_bits[_S_array_size - 1] ^ _S_bitmask;
1370  else
1371  __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1];
1372  return __result;
1373  }
1374  }
1375 
1376  constexpr _BitMask&
1377  operator^=(const _BitMask& __b) & noexcept
1378  {
1379  __execute_n_times<_S_array_size>(
1380  [&](auto __i) { _M_bits[__i] ^= __b._M_bits[__i]; });
1381  return *this;
1382  }
1383 
1384  constexpr _BitMask&
1385  operator|=(const _BitMask& __b) & noexcept
1386  {
1387  __execute_n_times<_S_array_size>(
1388  [&](auto __i) { _M_bits[__i] |= __b._M_bits[__i]; });
1389  return *this;
1390  }
1391 
1392  constexpr _BitMask&
1393  operator&=(const _BitMask& __b) & noexcept
1394  {
1395  __execute_n_times<_S_array_size>(
1396  [&](auto __i) { _M_bits[__i] &= __b._M_bits[__i]; });
1397  return *this;
1398  }
1399 
1400  friend constexpr _BitMask
1401  operator^(const _BitMask& __a, const _BitMask& __b) noexcept
1402  {
1403  _BitMask __r = __a;
1404  __r ^= __b;
1405  return __r;
1406  }
1407 
1408  friend constexpr _BitMask
1409  operator|(const _BitMask& __a, const _BitMask& __b) noexcept
1410  {
1411  _BitMask __r = __a;
1412  __r |= __b;
1413  return __r;
1414  }
1415 
1416  friend constexpr _BitMask
1417  operator&(const _BitMask& __a, const _BitMask& __b) noexcept
1418  {
1419  _BitMask __r = __a;
1420  __r &= __b;
1421  return __r;
1422  }
1423 
1424  _GLIBCXX_SIMD_INTRINSIC
1425  constexpr bool
1426  _M_is_constprop() const
1427  {
1428  if constexpr (_S_array_size == 0)
1429  return __builtin_constant_p(_M_bits[0]);
1430  else
1431  {
1432  for (int __i = 0; __i < _S_array_size; ++__i)
1433  if (!__builtin_constant_p(_M_bits[__i]))
1434  return false;
1435  return true;
1436  }
1437  }
1438  };
1439 
1440 // }}}
1441 
1442 // vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv
1443 // __min_vector_size {{{
1444 template <typename _Tp = void>
1445  static inline constexpr int __min_vector_size = 2 * sizeof(_Tp);
1446 
1447 #if _GLIBCXX_SIMD_HAVE_NEON
1448 template <>
1449  inline constexpr int __min_vector_size<void> = 8;
1450 #else
1451 template <>
1452  inline constexpr int __min_vector_size<void> = 16;
1453 #endif
1454 
1455 // }}}
1456 // __vector_type {{{
1457 template <typename _Tp, size_t _Np, typename = void>
1458  struct __vector_type_n {};
1459 
1460 // substition failure for 0-element case
1461 template <typename _Tp>
1462  struct __vector_type_n<_Tp, 0, void> {};
1463 
1464 // special case 1-element to be _Tp itself
1465 template <typename _Tp>
1466  struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>>
1467  { using type = _Tp; };
1468 
1469 // else, use GNU-style builtin vector types
1470 template <typename _Tp, size_t _Np>
1471  struct __vector_type_n<_Tp, _Np,
1472  enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>>
1473  {
1474  static constexpr size_t _S_Np2 = std::__bit_ceil(_Np * sizeof(_Tp));
1475 
1476  static constexpr size_t _S_Bytes =
1477 #ifdef __i386__
1478  // Using [[gnu::vector_size(8)]] would wreak havoc on the FPU because
1479  // those objects are passed via MMX registers and nothing ever calls EMMS.
1480  _S_Np2 == 8 ? 16 :
1481 #endif
1482  _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp>
1483  : _S_Np2;
1484 
1485  using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp;
1486  };
1487 
1488 template <typename _Tp, size_t _Bytes, size_t = _Bytes % sizeof(_Tp)>
1489  struct __vector_type;
1490 
1491 template <typename _Tp, size_t _Bytes>
1492  struct __vector_type<_Tp, _Bytes, 0>
1493  : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {};
1494 
1495 template <typename _Tp, size_t _Size>
1496  using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type;
1497 
1498 template <typename _Tp>
1499  using __vector_type2_t = typename __vector_type<_Tp, 2>::type;
1500 template <typename _Tp>
1501  using __vector_type4_t = typename __vector_type<_Tp, 4>::type;
1502 template <typename _Tp>
1503  using __vector_type8_t = typename __vector_type<_Tp, 8>::type;
1504 template <typename _Tp>
1505  using __vector_type16_t = typename __vector_type<_Tp, 16>::type;
1506 template <typename _Tp>
1507  using __vector_type32_t = typename __vector_type<_Tp, 32>::type;
1508 template <typename _Tp>
1509  using __vector_type64_t = typename __vector_type<_Tp, 64>::type;
1510 
1511 // }}}
1512 // __is_vector_type {{{
1513 template <typename _Tp, typename = void_t<>>
1514  struct __is_vector_type : false_type {};
1515 
1516 template <typename _Tp>
1517  struct __is_vector_type<
1518  _Tp, void_t<typename __vector_type<
1519  remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1520  : is_same<_Tp, typename __vector_type<
1521  remove_reference_t<decltype(declval<_Tp>()[0])>,
1522  sizeof(_Tp)>::type> {};
1523 
1524 template <typename _Tp>
1525  inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value;
1526 
1527 // }}}
1528 // __is_intrinsic_type {{{
1529 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
1530 template <typename _Tp>
1531  using __is_intrinsic_type = __is_vector_type<_Tp>;
1532 #else // not SSE (x86)
1533 template <typename _Tp, typename = void_t<>>
1534  struct __is_intrinsic_type : false_type {};
1535 
1536 template <typename _Tp>
1537  struct __is_intrinsic_type<
1538  _Tp, void_t<typename __intrinsic_type<
1539  remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>>
1540  : is_same<_Tp, typename __intrinsic_type<
1541  remove_reference_t<decltype(declval<_Tp>()[0])>,
1542  sizeof(_Tp)>::type> {};
1543 #endif
1544 
1545 template <typename _Tp>
1546  inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value;
1547 
1548 // }}}
1549 // _VectorTraits{{{
1550 template <typename _Tp, typename = void_t<>>
1551  struct _VectorTraitsImpl;
1552 
1553 template <typename _Tp>
1554  struct _VectorTraitsImpl<_Tp, enable_if_t<__is_vector_type_v<_Tp>
1555  || __is_intrinsic_type_v<_Tp>>>
1556  {
1557  using type = _Tp;
1558  using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>;
1559  static constexpr int _S_full_size = sizeof(_Tp) / sizeof(value_type);
1560  using _Wrapper = _SimdWrapper<value_type, _S_full_size>;
1561  template <typename _Up, int _W = _S_full_size>
1562  static constexpr bool _S_is
1563  = is_same_v<value_type, _Up> && _W == _S_full_size;
1564  };
1565 
1566 template <typename _Tp, size_t _Np>
1567  struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>,
1568  void_t<__vector_type_t<_Tp, _Np>>>
1569  {
1570  using type = __vector_type_t<_Tp, _Np>;
1571  using value_type = _Tp;
1572  static constexpr int _S_full_size = sizeof(type) / sizeof(value_type);
1573  using _Wrapper = _SimdWrapper<_Tp, _Np>;
1574  static constexpr bool _S_is_partial = (_Np == _S_full_size);
1575  static constexpr int _S_partial_width = _Np;
1576  template <typename _Up, int _W = _S_full_size>
1577  static constexpr bool _S_is
1578  = is_same_v<value_type, _Up>&& _W == _S_full_size;
1579  };
1580 
1581 template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type>
1582  using _VectorTraits = _VectorTraitsImpl<_Tp>;
1583 
1584 // }}}
1585 // __as_vector{{{
1586 template <typename _V>
1587  _GLIBCXX_SIMD_INTRINSIC constexpr auto
1588  __as_vector(_V __x)
1589  {
1590  if constexpr (__is_vector_type_v<_V>)
1591  return __x;
1592  else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1593  return __data(__x)._M_data;
1594  else if constexpr (__is_vectorizable_v<_V>)
1595  return __vector_type_t<_V, 2>{__x};
1596  else
1597  return __x._M_data;
1598  }
1599 
1600 // }}}
1601 // __as_wrapper{{{
1602 template <size_t _Np = 0, typename _V>
1603  _GLIBCXX_SIMD_INTRINSIC constexpr auto
1604  __as_wrapper(_V __x)
1605  {
1606  if constexpr (__is_vector_type_v<_V>)
1607  return _SimdWrapper<typename _VectorTraits<_V>::value_type,
1608  (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x);
1609  else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value)
1610  {
1611  static_assert(_V::size() == _Np);
1612  return __data(__x);
1613  }
1614  else
1615  {
1616  static_assert(_V::_S_size == _Np);
1617  return __x;
1618  }
1619  }
1620 
1621 // }}}
1622 // __intrin_bitcast{{{
1623 template <typename _To, typename _From>
1624  _GLIBCXX_SIMD_INTRINSIC constexpr _To
1625  __intrin_bitcast(_From __v)
1626  {
1627  static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>)
1628  && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>));
1629  if constexpr (sizeof(_To) == sizeof(_From))
1630  return reinterpret_cast<_To>(__v);
1631  else if constexpr (sizeof(_From) > sizeof(_To))
1632  if constexpr (sizeof(_To) >= 16)
1633  return reinterpret_cast<const __may_alias<_To>&>(__v);
1634  else
1635  {
1636  _To __r;
1637  __builtin_memcpy(&__r, &__v, sizeof(_To));
1638  return __r;
1639  }
1640 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1641  else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32)
1642  return reinterpret_cast<_To>(__builtin_ia32_ps256_ps(
1643  reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1644  else if constexpr (__have_avx512f && sizeof(_From) == 16
1645  && sizeof(_To) == 64)
1646  return reinterpret_cast<_To>(__builtin_ia32_ps512_ps(
1647  reinterpret_cast<__vector_type_t<float, 4>>(__v)));
1648  else if constexpr (__have_avx512f && sizeof(_From) == 32
1649  && sizeof(_To) == 64)
1650  return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps(
1651  reinterpret_cast<__vector_type_t<float, 8>>(__v)));
1652 #endif // _GLIBCXX_SIMD_X86INTRIN
1653  else if constexpr (sizeof(__v) <= 8)
1654  return reinterpret_cast<_To>(
1655  __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{
1656  reinterpret_cast<__int_for_sizeof_t<_From>>(__v)});
1657  else
1658  {
1659  static_assert(sizeof(_To) > sizeof(_From));
1660  _To __r = {};
1661  __builtin_memcpy(&__r, &__v, sizeof(_From));
1662  return __r;
1663  }
1664  }
1665 
1666 // }}}
1667 // __vector_bitcast{{{
1668 template <typename _To, size_t _NN = 0, typename _From,
1669  typename _FromVT = _VectorTraits<_From>,
1670  size_t _Np = _NN == 0 ? sizeof(_From) / sizeof(_To) : _NN>
1671  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1672  __vector_bitcast(_From __x)
1673  {
1674  using _R = __vector_type_t<_To, _Np>;
1675  return __intrin_bitcast<_R>(__x);
1676  }
1677 
1678 template <typename _To, size_t _NN = 0, typename _Tp, size_t _Nx,
1679  size_t _Np
1680  = _NN == 0 ? sizeof(_SimdWrapper<_Tp, _Nx>) / sizeof(_To) : _NN>
1681  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np>
1682  __vector_bitcast(const _SimdWrapper<_Tp, _Nx>& __x)
1683  {
1684  static_assert(_Np > 1);
1685  return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data);
1686  }
1687 
1688 // }}}
1689 // __convert_x86 declarations {{{
1690 #ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
1691 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1692  _To __convert_x86(_Tp);
1693 
1694 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1695  _To __convert_x86(_Tp, _Tp);
1696 
1697 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1698  _To __convert_x86(_Tp, _Tp, _Tp, _Tp);
1699 
1700 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1701  _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp);
1702 
1703 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1704  _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp,
1705  _Tp, _Tp, _Tp, _Tp);
1706 #endif // _GLIBCXX_SIMD_WORKAROUND_PR85048
1707 
1708 //}}}
1709 // __bit_cast {{{
1710 template <typename _To, typename _From>
1711  _GLIBCXX_SIMD_INTRINSIC constexpr _To
1712  __bit_cast(const _From __x)
1713  {
1714 #if __has_builtin(__builtin_bit_cast)
1715  return __builtin_bit_cast(_To, __x);
1716 #else
1717  static_assert(sizeof(_To) == sizeof(_From));
1718  constexpr bool __to_is_vectorizable
1719  = is_arithmetic_v<_To> || is_enum_v<_To>;
1720  constexpr bool __from_is_vectorizable
1721  = is_arithmetic_v<_From> || is_enum_v<_From>;
1722  if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>)
1723  return reinterpret_cast<_To>(__x);
1724  else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable)
1725  {
1726  using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1727  return reinterpret_cast<_To>(_FV{__x});
1728  }
1729  else if constexpr (__to_is_vectorizable && __from_is_vectorizable)
1730  {
1731  using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1732  using _FV [[gnu::vector_size(sizeof(_From))]] = _From;
1733  return reinterpret_cast<_TV>(_FV{__x})[0];
1734  }
1735  else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>)
1736  {
1737  using _TV [[gnu::vector_size(sizeof(_To))]] = _To;
1738  return reinterpret_cast<_TV>(__x)[0];
1739  }
1740  else
1741  {
1742  _To __r;
1743  __builtin_memcpy(reinterpret_cast<char*>(&__r),
1744  reinterpret_cast<const char*>(&__x), sizeof(_To));
1745  return __r;
1746  }
1747 #endif
1748  }
1749 
1750 // }}}
1751 // __to_intrin {{{
1752 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
1753  typename _R
1754  = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_full_size>>
1755  _GLIBCXX_SIMD_INTRINSIC constexpr _R
1756  __to_intrin(_Tp __x)
1757  {
1758  static_assert(sizeof(__x) <= sizeof(_R),
1759  "__to_intrin may never drop values off the end");
1760  if constexpr (sizeof(__x) == sizeof(_R))
1761  return reinterpret_cast<_R>(__as_vector(__x));
1762  else
1763  {
1764  using _Up = __int_for_sizeof_t<_Tp>;
1765  return reinterpret_cast<_R>(
1766  __vector_type_t<_Up, sizeof(_R) / sizeof(_Up)>{__bit_cast<_Up>(__x)});
1767  }
1768  }
1769 
1770 // }}}
1771 // __make_vector{{{
1772 template <typename _Tp, typename... _Args>
1773  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)>
1774  __make_vector(const _Args&... __args)
1775  {
1776  return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(__args)...};
1777  }
1778 
1779 // }}}
1780 // __vector_broadcast{{{
1781 template <size_t _Np, typename _Tp>
1782  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1783  __vector_broadcast(_Tp __x)
1784  {
1785  return __call_with_n_evaluations<_Np>(
1786  [](auto... __xx) { return __vector_type_t<_Tp, _Np>{__xx...}; },
1787  [&__x](int) { return __x; });
1788  }
1789 
1790 // }}}
1791 // __generate_vector{{{
1792  template <typename _Tp, size_t _Np, typename _Gp, size_t... _I>
1793  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1794  __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>)
1795  {
1796  return __vector_type_t<_Tp, _Np>{
1797  static_cast<_Tp>(__gen(_SizeConstant<_I>()))...};
1798  }
1799 
1800 template <typename _V, typename _VVT = _VectorTraits<_V>, typename _Gp>
1801  _GLIBCXX_SIMD_INTRINSIC constexpr _V
1802  __generate_vector(_Gp&& __gen)
1803  {
1804  if constexpr (__is_vector_type_v<_V>)
1805  return __generate_vector_impl<typename _VVT::value_type,
1806  _VVT::_S_full_size>(
1807  static_cast<_Gp&&>(__gen), make_index_sequence<_VVT::_S_full_size>());
1808  else
1809  return __generate_vector_impl<typename _VVT::value_type,
1810  _VVT::_S_partial_width>(
1811  static_cast<_Gp&&>(__gen),
1812  make_index_sequence<_VVT::_S_partial_width>());
1813  }
1814 
1815 template <typename _Tp, size_t _Np, typename _Gp>
1816  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np>
1817  __generate_vector(_Gp&& __gen)
1818  {
1819  return __generate_vector_impl<_Tp, _Np>(static_cast<_Gp&&>(__gen),
1820  make_index_sequence<_Np>());
1821  }
1822 
1823 // }}}
1824 // __xor{{{
1825 template <typename _TW>
1826  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1827  __xor(_TW __a, _TW __b) noexcept
1828  {
1829  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1830  {
1831  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1832  _VectorTraitsImpl<_TW>>::value_type;
1833  if constexpr (is_floating_point_v<_Tp>)
1834  {
1835  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1836  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1837  ^ __vector_bitcast<_Ip>(__b));
1838  }
1839  else if constexpr (__is_vector_type_v<_TW>)
1840  return __a ^ __b;
1841  else
1842  return __a._M_data ^ __b._M_data;
1843  }
1844  else
1845  return __a ^ __b;
1846  }
1847 
1848 // }}}
1849 // __or{{{
1850 template <typename _TW>
1851  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1852  __or(_TW __a, _TW __b) noexcept
1853  {
1854  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1855  {
1856  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1857  _VectorTraitsImpl<_TW>>::value_type;
1858  if constexpr (is_floating_point_v<_Tp>)
1859  {
1860  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1861  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1862  | __vector_bitcast<_Ip>(__b));
1863  }
1864  else if constexpr (__is_vector_type_v<_TW>)
1865  return __a | __b;
1866  else
1867  return __a._M_data | __b._M_data;
1868  }
1869  else
1870  return __a | __b;
1871  }
1872 
1873 // }}}
1874 // __and{{{
1875 template <typename _TW>
1876  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1877  __and(_TW __a, _TW __b) noexcept
1878  {
1879  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1880  {
1881  using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1882  _VectorTraitsImpl<_TW>>::value_type;
1883  if constexpr (is_floating_point_v<_Tp>)
1884  {
1885  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1886  return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a)
1887  & __vector_bitcast<_Ip>(__b));
1888  }
1889  else if constexpr (__is_vector_type_v<_TW>)
1890  return __a & __b;
1891  else
1892  return __a._M_data & __b._M_data;
1893  }
1894  else
1895  return __a & __b;
1896  }
1897 
1898 // }}}
1899 // __andnot{{{
1900 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1901 static constexpr struct
1902 {
1903  _GLIBCXX_SIMD_INTRINSIC __v4sf
1904  operator()(__v4sf __a, __v4sf __b) const noexcept
1905  { return __builtin_ia32_andnps(__a, __b); }
1906 
1907  _GLIBCXX_SIMD_INTRINSIC __v2df
1908  operator()(__v2df __a, __v2df __b) const noexcept
1909  { return __builtin_ia32_andnpd(__a, __b); }
1910 
1911  _GLIBCXX_SIMD_INTRINSIC __v2di
1912  operator()(__v2di __a, __v2di __b) const noexcept
1913  { return __builtin_ia32_pandn128(__a, __b); }
1914 
1915  _GLIBCXX_SIMD_INTRINSIC __v8sf
1916  operator()(__v8sf __a, __v8sf __b) const noexcept
1917  { return __builtin_ia32_andnps256(__a, __b); }
1918 
1919  _GLIBCXX_SIMD_INTRINSIC __v4df
1920  operator()(__v4df __a, __v4df __b) const noexcept
1921  { return __builtin_ia32_andnpd256(__a, __b); }
1922 
1923  _GLIBCXX_SIMD_INTRINSIC __v4di
1924  operator()(__v4di __a, __v4di __b) const noexcept
1925  {
1926  if constexpr (__have_avx2)
1927  return __builtin_ia32_andnotsi256(__a, __b);
1928  else
1929  return reinterpret_cast<__v4di>(
1930  __builtin_ia32_andnpd256(reinterpret_cast<__v4df>(__a),
1931  reinterpret_cast<__v4df>(__b)));
1932  }
1933 
1934  _GLIBCXX_SIMD_INTRINSIC __v16sf
1935  operator()(__v16sf __a, __v16sf __b) const noexcept
1936  {
1937  if constexpr (__have_avx512dq)
1938  return _mm512_andnot_ps(__a, __b);
1939  else
1940  return reinterpret_cast<__v16sf>(
1941  _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1942  reinterpret_cast<__v8di>(__b)));
1943  }
1944 
1945  _GLIBCXX_SIMD_INTRINSIC __v8df
1946  operator()(__v8df __a, __v8df __b) const noexcept
1947  {
1948  if constexpr (__have_avx512dq)
1949  return _mm512_andnot_pd(__a, __b);
1950  else
1951  return reinterpret_cast<__v8df>(
1952  _mm512_andnot_si512(reinterpret_cast<__v8di>(__a),
1953  reinterpret_cast<__v8di>(__b)));
1954  }
1955 
1956  _GLIBCXX_SIMD_INTRINSIC __v8di
1957  operator()(__v8di __a, __v8di __b) const noexcept
1958  { return _mm512_andnot_si512(__a, __b); }
1959 } _S_x86_andnot;
1960 #endif // _GLIBCXX_SIMD_X86INTRIN && !__clang__
1961 
1962 template <typename _TW>
1963  _GLIBCXX_SIMD_INTRINSIC constexpr _TW
1964  __andnot(_TW __a, _TW __b) noexcept
1965  {
1966  if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>)
1967  {
1968  using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW,
1969  _VectorTraitsImpl<_TW>>;
1970  using _Tp = typename _TVT::value_type;
1971 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__
1972  if constexpr (sizeof(_TW) >= 16)
1973  {
1974  const auto __ai = __to_intrin(__a);
1975  const auto __bi = __to_intrin(__b);
1976  if (!__builtin_is_constant_evaluated()
1977  && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi)))
1978  {
1979  const auto __r = _S_x86_andnot(__ai, __bi);
1980  if constexpr (is_convertible_v<decltype(__r), _TW>)
1981  return __r;
1982  else
1983  return reinterpret_cast<typename _TVT::type>(__r);
1984  }
1985  }
1986 #endif // _GLIBCXX_SIMD_X86INTRIN
1987  using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>;
1988  return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a)
1989  & __vector_bitcast<_Ip>(__b));
1990  }
1991  else
1992  return ~__a & __b;
1993  }
1994 
1995 // }}}
1996 // __not{{{
1997 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
1998  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
1999  __not(_Tp __a) noexcept
2000  {
2001  if constexpr (is_floating_point_v<typename _TVT::value_type>)
2002  return reinterpret_cast<typename _TVT::type>(
2003  ~__vector_bitcast<unsigned>(__a));
2004  else
2005  return ~__a;
2006  }
2007 
2008 // }}}
2009 // __concat{{{
2010 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>,
2011  typename _R = __vector_type_t<typename _TVT::value_type,
2012  _TVT::_S_full_size * 2>>
2013  constexpr _R
2014  __concat(_Tp a_, _Tp b_)
2015  {
2016 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2017  using _W
2018  = conditional_t<is_floating_point_v<typename _TVT::value_type>, double,
2019  conditional_t<(sizeof(_Tp) >= 2 * sizeof(long long)),
2020  long long, typename _TVT::value_type>>;
2021  constexpr int input_width = sizeof(_Tp) / sizeof(_W);
2022  const auto __a = __vector_bitcast<_W>(a_);
2023  const auto __b = __vector_bitcast<_W>(b_);
2024  using _Up = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>;
2025 #else
2026  constexpr int input_width = _TVT::_S_full_size;
2027  const _Tp& __a = a_;
2028  const _Tp& __b = b_;
2029  using _Up = _R;
2030 #endif
2031  if constexpr (input_width == 2)
2032  return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]});
2033  else if constexpr (input_width == 4)
2034  return reinterpret_cast<_R>(
2035  _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]});
2036  else if constexpr (input_width == 8)
2037  return reinterpret_cast<_R>(
2038  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7],
2039  __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]});
2040  else if constexpr (input_width == 16)
2041  return reinterpret_cast<_R>(
2042  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2043  __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2044  __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4],
2045  __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11],
2046  __b[12], __b[13], __b[14], __b[15]});
2047  else if constexpr (input_width == 32)
2048  return reinterpret_cast<_R>(
2049  _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6],
2050  __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13],
2051  __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20],
2052  __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27],
2053  __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2],
2054  __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9],
2055  __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16],
2056  __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23],
2057  __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30],
2058  __b[31]});
2059  }
2060 
2061 // }}}
2062 // __zero_extend {{{
2063 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2064  struct _ZeroExtendProxy
2065  {
2066  using value_type = typename _TVT::value_type;
2067  static constexpr size_t _Np = _TVT::_S_full_size;
2068  const _Tp __x;
2069 
2070  template <typename _To, typename _ToVT = _VectorTraits<_To>,
2071  typename
2072  = enable_if_t<is_same_v<typename _ToVT::value_type, value_type>>>
2073  _GLIBCXX_SIMD_INTRINSIC operator _To() const
2074  {
2075  constexpr size_t _ToN = _ToVT::_S_full_size;
2076  if constexpr (_ToN == _Np)
2077  return __x;
2078  else if constexpr (_ToN == 2 * _Np)
2079  {
2080 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2081  if constexpr (__have_avx && _TVT::template _S_is<float, 4>)
2082  return __vector_bitcast<value_type>(
2083  _mm256_insertf128_ps(__m256(), __x, 0));
2084  else if constexpr (__have_avx && _TVT::template _S_is<double, 2>)
2085  return __vector_bitcast<value_type>(
2086  _mm256_insertf128_pd(__m256d(), __x, 0));
2087  else if constexpr (__have_avx2 && _Np * sizeof(value_type) == 16)
2088  return __vector_bitcast<value_type>(
2089  _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0));
2090  else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>)
2091  {
2092  if constexpr (__have_avx512dq)
2093  return __vector_bitcast<value_type>(
2094  _mm512_insertf32x8(__m512(), __x, 0));
2095  else
2096  return reinterpret_cast<__m512>(
2097  _mm512_insertf64x4(__m512d(),
2098  reinterpret_cast<__m256d>(__x), 0));
2099  }
2100  else if constexpr (__have_avx512f
2101  && _TVT::template _S_is<double, 4>)
2102  return __vector_bitcast<value_type>(
2103  _mm512_insertf64x4(__m512d(), __x, 0));
2104  else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 32)
2105  return __vector_bitcast<value_type>(
2106  _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0));
2107 #endif
2108  return __concat(__x, _Tp());
2109  }
2110  else if constexpr (_ToN == 4 * _Np)
2111  {
2112 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3
2113  if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>)
2114  {
2115  return __vector_bitcast<value_type>(
2116  _mm512_insertf64x2(__m512d(), __x, 0));
2117  }
2118  else if constexpr (__have_avx512f
2119  && is_floating_point_v<value_type>)
2120  {
2121  return __vector_bitcast<value_type>(
2122  _mm512_insertf32x4(__m512(), reinterpret_cast<__m128>(__x),
2123  0));
2124  }
2125  else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 16)
2126  {
2127  return __vector_bitcast<value_type>(
2128  _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0));
2129  }
2130 #endif
2131  return __concat(__concat(__x, _Tp()),
2132  __vector_type_t<value_type, _Np * 2>());
2133  }
2134  else if constexpr (_ToN == 8 * _Np)
2135  return __concat(operator __vector_type_t<value_type, _Np * 4>(),
2136  __vector_type_t<value_type, _Np * 4>());
2137  else if constexpr (_ToN == 16 * _Np)
2138  return __concat(operator __vector_type_t<value_type, _Np * 8>(),
2139  __vector_type_t<value_type, _Np * 8>());
2140  else
2141  __assert_unreachable<_Tp>();
2142  }
2143  };
2144 
2145 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
2146  _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT>
2147  __zero_extend(_Tp __x)
2148  { return {__x}; }
2149 
2150 // }}}
2151 // __extract<_Np, By>{{{
2152 template <int _Offset,
2153  int _SplitBy,
2154  typename _Tp,
2155  typename _TVT = _VectorTraits<_Tp>,
2156  typename _R = __vector_type_t<typename _TVT::value_type,
2157  _TVT::_S_full_size / _SplitBy>>
2158  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2159  __extract(_Tp __in)
2160  {
2161  using value_type = typename _TVT::value_type;
2162 #if _GLIBCXX_SIMD_X86INTRIN // {{{
2163  if constexpr (sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0)
2164  {
2165  if constexpr (__have_avx512dq && is_same_v<double, value_type>)
2166  return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset);
2167  else if constexpr (is_floating_point_v<value_type>)
2168  return __vector_bitcast<value_type>(
2169  _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset));
2170  else
2171  return reinterpret_cast<_R>(
2172  _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in),
2173  _Offset));
2174  }
2175  else
2176 #endif // _GLIBCXX_SIMD_X86INTRIN }}}
2177  {
2178 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1
2179  using _W = conditional_t<
2180  is_floating_point_v<value_type>, double,
2181  conditional_t<(sizeof(_R) >= 16), long long, value_type>>;
2182  static_assert(sizeof(_R) % sizeof(_W) == 0);
2183  constexpr int __return_width = sizeof(_R) / sizeof(_W);
2184  using _Up = __vector_type_t<_W, __return_width>;
2185  const auto __x = __vector_bitcast<_W>(__in);
2186 #else
2187  constexpr int __return_width = _TVT::_S_full_size / _SplitBy;
2188  using _Up = _R;
2189  const __vector_type_t<value_type, _TVT::_S_full_size>& __x
2190  = __in; // only needed for _Tp = _SimdWrapper<value_type, _Np>
2191 #endif
2192  constexpr int _O = _Offset * __return_width;
2193  return __call_with_subscripts<__return_width, _O>(
2194  __x, [](auto... __entries) {
2195  return reinterpret_cast<_R>(_Up{__entries...});
2196  });
2197  }
2198  }
2199 
2200 // }}}
2201 // __lo/__hi64[z]{{{
2202 template <typename _Tp,
2203  typename _R
2204  = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2205  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2206  __lo64(_Tp __x)
2207  {
2208  _R __r{};
2209  __builtin_memcpy(&__r, &__x, 8);
2210  return __r;
2211  }
2212 
2213 template <typename _Tp,
2214  typename _R
2215  = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2216  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2217  __hi64(_Tp __x)
2218  {
2219  static_assert(sizeof(_Tp) == 16, "use __hi64z if you meant it");
2220  _R __r{};
2221  __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2222  return __r;
2223  }
2224 
2225 template <typename _Tp,
2226  typename _R
2227  = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>>
2228  _GLIBCXX_SIMD_INTRINSIC constexpr _R
2229  __hi64z([[maybe_unused]] _Tp __x)
2230  {
2231  _R __r{};
2232  if constexpr (sizeof(_Tp) == 16)
2233  __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8);
2234  return __r;
2235  }
2236 
2237 // }}}
2238 // __lo/__hi128{{{
2239 template <typename _Tp>
2240  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2241  __lo128(_Tp __x)
2242  { return __extract<0, sizeof(_Tp) / 16>(__x); }
2243 
2244 template <typename _Tp>
2245  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2246  __hi128(_Tp __x)
2247  {
2248  static_assert(sizeof(__x) == 32);
2249  return __extract<1, 2>(__x);
2250  }
2251 
2252 // }}}
2253 // __lo/__hi256{{{
2254 template <typename _Tp>
2255  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2256  __lo256(_Tp __x)
2257  {
2258  static_assert(sizeof(__x) == 64);
2259  return __extract<0, 2>(__x);
2260  }
2261 
2262 template <typename _Tp>
2263  _GLIBCXX_SIMD_INTRINSIC constexpr auto
2264  __hi256(_Tp __x)
2265  {
2266  static_assert(sizeof(__x) == 64);
2267  return __extract<1, 2>(__x);
2268  }
2269 
2270 // }}}
2271 // __auto_bitcast{{{
2272 template <typename _Tp>
2273  struct _AutoCast
2274  {
2275  static_assert(__is_vector_type_v<_Tp>);
2276 
2277  const _Tp __x;
2278 
2279  template <typename _Up, typename _UVT = _VectorTraits<_Up>>
2280  _GLIBCXX_SIMD_INTRINSIC constexpr operator _Up() const
2281  { return __intrin_bitcast<typename _UVT::type>(__x); }
2282  };
2283 
2284 template <typename _Tp>
2285  _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp>
2286  __auto_bitcast(const _Tp& __x)
2287  { return {__x}; }
2288 
2289 template <typename _Tp, size_t _Np>
2290  _GLIBCXX_SIMD_INTRINSIC constexpr
2291  _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType>
2292  __auto_bitcast(const _SimdWrapper<_Tp, _Np>& __x)
2293  { return {__x._M_data}; }
2294 
2295 // }}}
2296 // ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^
2297 
2298 #if _GLIBCXX_SIMD_HAVE_SSE_ABI
2299 // __bool_storage_member_type{{{
2300 #if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN
2301 template <size_t _Size>
2302  struct __bool_storage_member_type
2303  {
2304  static_assert((_Size & (_Size - 1)) != 0,
2305  "This trait may only be used for non-power-of-2 sizes. "
2306  "Power-of-2 sizes must be specialized.");
2307  using type =
2308  typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type;
2309  };
2310 
2311 template <>
2312  struct __bool_storage_member_type<1> { using type = bool; };
2313 
2314 template <>
2315  struct __bool_storage_member_type<2> { using type = __mmask8; };
2316 
2317 template <>
2318  struct __bool_storage_member_type<4> { using type = __mmask8; };
2319 
2320 template <>
2321  struct __bool_storage_member_type<8> { using type = __mmask8; };
2322 
2323 template <>
2324  struct __bool_storage_member_type<16> { using type = __mmask16; };
2325 
2326 template <>
2327  struct __bool_storage_member_type<32> { using type = __mmask32; };
2328 
2329 template <>
2330  struct __bool_storage_member_type<64> { using type = __mmask64; };
2331 #endif // _GLIBCXX_SIMD_HAVE_AVX512F
2332 
2333 // }}}
2334 // __intrinsic_type (x86){{{
2335 // the following excludes bool via __is_vectorizable
2336 #if _GLIBCXX_SIMD_HAVE_SSE
2337 template <typename _Tp, size_t _Bytes>
2338  struct __intrinsic_type<_Tp, _Bytes,
2339  enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>>
2340  {
2341  static_assert(!is_same_v<_Tp, long double>,
2342  "no __intrinsic_type support for long double on x86");
2343 
2344  static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16
2345  : _Bytes <= 32 ? 32
2346  : 64;
2347 
2348  using type [[__gnu__::__vector_size__(_S_VBytes)]]
2349  = conditional_t<is_integral_v<_Tp>, long long int, _Tp>;
2350  };
2351 #endif // _GLIBCXX_SIMD_HAVE_SSE
2352 
2353 // }}}
2354 #endif // _GLIBCXX_SIMD_HAVE_SSE_ABI
2355 // __intrinsic_type (ARM){{{
2356 #if _GLIBCXX_SIMD_HAVE_NEON
2357 template <>
2358  struct __intrinsic_type<float, 8, void>
2359  { using type = float32x2_t; };
2360 
2361 template <>
2362  struct __intrinsic_type<float, 16, void>
2363  { using type = float32x4_t; };
2364 
2365 #if _GLIBCXX_SIMD_HAVE_NEON_A64
2366 template <>
2367  struct __intrinsic_type<double, 8, void>
2368  { using type = float64x1_t; };
2369 
2370 template <>
2371  struct __intrinsic_type<double, 16, void>
2372  { using type = float64x2_t; };
2373 #endif
2374 
2375 #define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \
2376 template <> \
2377  struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \
2378  _Np * _Bits / 8, void> \
2379  { using type = int##_Bits##x##_Np##_t; }; \
2380 template <> \
2381  struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \
2382  _Np * _Bits / 8, void> \
2383  { using type = uint##_Bits##x##_Np##_t; }
2384 _GLIBCXX_SIMD_ARM_INTRIN(8, 8);
2385 _GLIBCXX_SIMD_ARM_INTRIN(8, 16);
2386 _GLIBCXX_SIMD_ARM_INTRIN(16, 4);
2387 _GLIBCXX_SIMD_ARM_INTRIN(16, 8);
2388 _GLIBCXX_SIMD_ARM_INTRIN(32, 2);
2389 _GLIBCXX_SIMD_ARM_INTRIN(32, 4);
2390 _GLIBCXX_SIMD_ARM_INTRIN(64, 1);
2391 _GLIBCXX_SIMD_ARM_INTRIN(64, 2);
2392 #undef _GLIBCXX_SIMD_ARM_INTRIN
2393 
2394 template <typename _Tp, size_t _Bytes>
2395  struct __intrinsic_type<_Tp, _Bytes,
2396  enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2397  {
2398  static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16;
2399  using _Ip = __int_for_sizeof_t<_Tp>;
2400  using _Up = conditional_t<
2401  is_floating_point_v<_Tp>, _Tp,
2402  conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>;
2403  static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes,
2404  "should use explicit specialization above");
2405  using type = typename __intrinsic_type<_Up, _SVecBytes>::type;
2406  };
2407 #endif // _GLIBCXX_SIMD_HAVE_NEON
2408 
2409 // }}}
2410 // __intrinsic_type (PPC){{{
2411 #ifdef __ALTIVEC__
2412 template <typename _Tp>
2413  struct __intrinsic_type_impl;
2414 
2415 #define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \
2416  template <> \
2417  struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; }
2418 _GLIBCXX_SIMD_PPC_INTRIN(float);
2419 _GLIBCXX_SIMD_PPC_INTRIN(double);
2420 _GLIBCXX_SIMD_PPC_INTRIN(signed char);
2421 _GLIBCXX_SIMD_PPC_INTRIN(unsigned char);
2422 _GLIBCXX_SIMD_PPC_INTRIN(signed short);
2423 _GLIBCXX_SIMD_PPC_INTRIN(unsigned short);
2424 _GLIBCXX_SIMD_PPC_INTRIN(signed int);
2425 _GLIBCXX_SIMD_PPC_INTRIN(unsigned int);
2426 _GLIBCXX_SIMD_PPC_INTRIN(signed long);
2427 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long);
2428 _GLIBCXX_SIMD_PPC_INTRIN(signed long long);
2429 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long long);
2430 #undef _GLIBCXX_SIMD_PPC_INTRIN
2431 
2432 template <typename _Tp, size_t _Bytes>
2433  struct __intrinsic_type<_Tp, _Bytes,
2434  enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>>
2435  {
2436  static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>;
2437  // allow _Tp == long double with -mlong-double-64
2438  static_assert(!(_S_is_ldouble && sizeof(long double) > sizeof(double)),
2439  "no __intrinsic_type support for long double on PPC");
2440 #ifndef __VSX__
2441  static_assert(!is_same_v<_Tp, double>,
2442  "no __intrinsic_type support for double on PPC w/o VSX");
2443 #endif
2444  using type =
2445  typename __intrinsic_type_impl<
2446  conditional_t<is_floating_point_v<_Tp>,
2447  conditional_t<_S_is_ldouble, double, _Tp>,
2448  __int_for_sizeof_t<_Tp>>>::type;
2449  };
2450 #endif // __ALTIVEC__
2451 
2452 // }}}
2453 // _SimdWrapper<bool>{{{1
2454 template <size_t _Width>
2455  struct _SimdWrapper<bool, _Width,
2456  void_t<typename __bool_storage_member_type<_Width>::type>>
2457  {
2458  using _BuiltinType = typename __bool_storage_member_type<_Width>::type;
2459  using value_type = bool;
2460 
2461  static constexpr size_t _S_full_size = sizeof(_BuiltinType) * __CHAR_BIT__;
2462 
2463  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size>
2464  __as_full_vector() const { return _M_data; }
2465 
2466  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
2467  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_BuiltinType __k)
2468  : _M_data(__k) {};
2469 
2470  _GLIBCXX_SIMD_INTRINSIC operator const _BuiltinType&() const
2471  { return _M_data; }
2472 
2473  _GLIBCXX_SIMD_INTRINSIC operator _BuiltinType&()
2474  { return _M_data; }
2475 
2476  _GLIBCXX_SIMD_INTRINSIC _BuiltinType __intrin() const
2477  { return _M_data; }
2478 
2479  _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator[](size_t __i) const
2480  { return _M_data & (_BuiltinType(1) << __i); }
2481 
2482  template <size_t __i>
2483  _GLIBCXX_SIMD_INTRINSIC constexpr value_type
2484  operator[](_SizeConstant<__i>) const
2485  { return _M_data & (_BuiltinType(1) << __i); }
2486 
2487  _GLIBCXX_SIMD_INTRINSIC constexpr void _M_set(size_t __i, value_type __x)
2488  {
2489  if (__x)
2490  _M_data |= (_BuiltinType(1) << __i);
2491  else
2492  _M_data &= ~(_BuiltinType(1) << __i);
2493  }
2494 
2495  _GLIBCXX_SIMD_INTRINSIC
2496  constexpr bool _M_is_constprop() const
2497  { return __builtin_constant_p(_M_data); }
2498 
2499  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_none_of() const
2500  {
2501  if (__builtin_constant_p(_M_data))
2502  {
2503  constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2504  constexpr _BuiltinType __active_mask
2505  = ~_BuiltinType() >> (__nbits - _Width);
2506  return (_M_data & __active_mask) == 0;
2507  }
2508  return false;
2509  }
2510 
2511  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_all_of() const
2512  {
2513  if (__builtin_constant_p(_M_data))
2514  {
2515  constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__;
2516  constexpr _BuiltinType __active_mask
2517  = ~_BuiltinType() >> (__nbits - _Width);
2518  return (_M_data & __active_mask) == __active_mask;
2519  }
2520  return false;
2521  }
2522 
2523  _BuiltinType _M_data;
2524  };
2525 
2526 // _SimdWrapperBase{{{1
2527 template <bool _MustZeroInitPadding, typename _BuiltinType>
2528  struct _SimdWrapperBase;
2529 
2530 template <typename _BuiltinType>
2531  struct _SimdWrapperBase<false, _BuiltinType> // no padding or no SNaNs
2532  {
2533  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() = default;
2534  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2535  : _M_data(__init)
2536  {}
2537 
2538  _BuiltinType _M_data;
2539  };
2540 
2541 template <typename _BuiltinType>
2542  struct _SimdWrapperBase<true, _BuiltinType> // with padding that needs to
2543  // never become SNaN
2544  {
2545  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase() : _M_data() {}
2546  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapperBase(_BuiltinType __init)
2547  : _M_data(__init)
2548  {}
2549 
2550  _BuiltinType _M_data;
2551  };
2552 
2553 // }}}
2554 // _SimdWrapper{{{
2555 template <typename _Tp, size_t _Width>
2556  struct _SimdWrapper<
2557  _Tp, _Width,
2558  void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>>
2559  : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2560  && sizeof(_Tp) * _Width
2561  == sizeof(__vector_type_t<_Tp, _Width>),
2562  __vector_type_t<_Tp, _Width>>
2563  {
2564  using _Base
2565  = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value
2566  && sizeof(_Tp) * _Width
2567  == sizeof(__vector_type_t<_Tp, _Width>),
2568  __vector_type_t<_Tp, _Width>>;
2569 
2570  static_assert(__is_vectorizable_v<_Tp>);
2571  static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then
2572 
2573  using _BuiltinType = __vector_type_t<_Tp, _Width>;
2574  using value_type = _Tp;
2575 
2576  static inline constexpr size_t _S_full_size
2577  = sizeof(_BuiltinType) / sizeof(value_type);
2578  static inline constexpr int _S_size = _Width;
2579  static inline constexpr bool _S_is_partial = _S_full_size != _S_size;
2580 
2581  using _Base::_M_data;
2582 
2583  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size>
2584  __as_full_vector() const
2585  { return _M_data; }
2586 
2587  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(initializer_list<_Tp> __init)
2588  : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>(
2589  [&](auto __i) { return __init.begin()[__i.value]; })) {}
2590 
2591  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper() = default;
2592  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(const _SimdWrapper&)
2593  = default;
2594  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_SimdWrapper&&) = default;
2595 
2596  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2597  operator=(const _SimdWrapper&) = default;
2598  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper&
2599  operator=(_SimdWrapper&&) = default;
2600 
2601  template <typename _V, typename = enable_if_t<disjunction_v<
2602  is_same<_V, __vector_type_t<_Tp, _Width>>,
2603  is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>>
2604  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper(_V __x)
2605  // __vector_bitcast can convert e.g. __m128 to __vector(2) float
2606  : _Base(__vector_bitcast<_Tp, _Width>(__x)) {}
2607 
2608  template <typename... _As,
2609  typename = enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...)
2610  && sizeof...(_As) <= _Width)>>
2611  _GLIBCXX_SIMD_INTRINSIC constexpr
2612  operator _SimdTuple<_Tp, _As...>() const
2613  {
2614  const auto& dd = _M_data; // workaround for GCC7 ICE
2615  return __generate_from_n_evaluations<sizeof...(_As),
2616  _SimdTuple<_Tp, _As...>>([&](
2617  auto __i) constexpr { return dd[int(__i)]; });
2618  }
2619 
2620  _GLIBCXX_SIMD_INTRINSIC constexpr operator const _BuiltinType&() const
2621  { return _M_data; }
2622 
2623  _GLIBCXX_SIMD_INTRINSIC constexpr operator _BuiltinType&()
2624  { return _M_data; }
2625 
2626  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](size_t __i) const
2627  { return _M_data[__i]; }
2628 
2629  template <size_t __i>
2630  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp operator[](_SizeConstant<__i>) const
2631  { return _M_data[__i]; }
2632 
2633  _GLIBCXX_SIMD_INTRINSIC constexpr void _M_set(size_t __i, _Tp __x)
2634  { _M_data[__i] = __x; }
2635 
2636  _GLIBCXX_SIMD_INTRINSIC
2637  constexpr bool _M_is_constprop() const
2638  { return __builtin_constant_p(_M_data); }
2639 
2640  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_none_of() const
2641  {
2642  if (__builtin_constant_p(_M_data))
2643  {
2644  bool __r = true;
2645  if constexpr (is_floating_point_v<_Tp>)
2646  {
2647  using _Ip = __int_for_sizeof_t<_Tp>;
2648  const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2649  __execute_n_times<_Width>(
2650  [&](auto __i) { __r &= __intdata[__i.value] == _Ip(); });
2651  }
2652  else
2653  __execute_n_times<_Width>(
2654  [&](auto __i) { __r &= _M_data[__i.value] == _Tp(); });
2655  return __r;
2656  }
2657  return false;
2658  }
2659 
2660  _GLIBCXX_SIMD_INTRINSIC constexpr bool _M_is_constprop_all_of() const
2661  {
2662  if (__builtin_constant_p(_M_data))
2663  {
2664  bool __r = true;
2665  if constexpr (is_floating_point_v<_Tp>)
2666  {
2667  using _Ip = __int_for_sizeof_t<_Tp>;
2668  const auto __intdata = __vector_bitcast<_Ip>(_M_data);
2669  __execute_n_times<_Width>(
2670  [&](auto __i) { __r &= __intdata[__i.value] == ~_Ip(); });
2671  }
2672  else
2673  __execute_n_times<_Width>(
2674  [&](auto __i) { __r &= _M_data[__i.value] == ~_Tp(); });
2675  return __r;
2676  }
2677  return false;
2678  }
2679  };
2680 
2681 // }}}
2682 
2683 // __vectorized_sizeof {{{
2684 template <typename _Tp>
2685  constexpr size_t
2686  __vectorized_sizeof()
2687  {
2688  if constexpr (!__is_vectorizable_v<_Tp>)
2689  return 0;
2690 
2691  if constexpr (sizeof(_Tp) <= 8)
2692  {
2693  // X86:
2694  if constexpr (__have_avx512bw)
2695  return 64;
2696  if constexpr (__have_avx512f && sizeof(_Tp) >= 4)
2697  return 64;
2698  if constexpr (__have_avx2)
2699  return 32;
2700  if constexpr (__have_avx && is_floating_point_v<_Tp>)
2701  return 32;
2702  if constexpr (__have_sse2)
2703  return 16;
2704  if constexpr (__have_sse && is_same_v<_Tp, float>)
2705  return 16;
2706  /* The following is too much trouble because of mixed MMX and x87 code.
2707  * While nothing here explicitly calls MMX instructions of registers,
2708  * they are still emitted but no EMMS cleanup is done.
2709  if constexpr (__have_mmx && sizeof(_Tp) <= 4 && is_integral_v<_Tp>)
2710  return 8;
2711  */
2712 
2713  // PowerPC:
2714  if constexpr (__have_power8vec
2715  || (__have_power_vmx && (sizeof(_Tp) < 8))
2716  || (__have_power_vsx && is_floating_point_v<_Tp>) )
2717  return 16;
2718 
2719  // ARM:
2720  if constexpr (__have_neon_a64
2721  || (__have_neon_a32 && !is_same_v<_Tp, double>) )
2722  return 16;
2723  if constexpr (__have_neon
2724  && sizeof(_Tp) < 8
2725  // Only allow fp if the user allows non-ICE559 fp (e.g.
2726  // via -ffast-math). ARMv7 NEON fp is not conforming to
2727  // IEC559.
2728  && (__support_neon_float || !is_floating_point_v<_Tp>))
2729  return 16;
2730  }
2731 
2732  return sizeof(_Tp);
2733  }
2734 
2735 // }}}
2736 namespace simd_abi {
2737 // most of simd_abi is defined in simd_detail.h
2738 template <typename _Tp>
2739  inline constexpr int max_fixed_size
2740  = (__have_avx512bw && sizeof(_Tp) == 1) ? 64 : 32;
2741 
2742 // compatible {{{
2743 #if defined __x86_64__ || defined __aarch64__
2744 template <typename _Tp>
2745  using compatible = conditional_t<(sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>;
2746 #elif defined __ARM_NEON
2747 // FIXME: not sure, probably needs to be scalar (or dependent on the hard-float
2748 // ABI?)
2749 template <typename _Tp>
2750  using compatible
2751  = conditional_t<(sizeof(_Tp) < 8
2752  && (__support_neon_float || !is_floating_point_v<_Tp>)),
2753  _VecBuiltin<16>, scalar>;
2754 #else
2755 template <typename>
2756  using compatible = scalar;
2757 #endif
2758 
2759 // }}}
2760 // native {{{
2761 template <typename _Tp>
2762  constexpr auto
2763  __determine_native_abi()
2764  {
2765  constexpr size_t __bytes = __vectorized_sizeof<_Tp>();
2766  if constexpr (__bytes == sizeof(_Tp))
2767  return static_cast<scalar*>(nullptr);
2768  else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64))
2769  return static_cast<_VecBltnBtmsk<__bytes>*>(nullptr);
2770  else
2771  return static_cast<_VecBuiltin<__bytes>*>(nullptr);
2772  }
2773 
2774 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>>
2775  using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>;
2776 
2777 // }}}
2778 // __default_abi {{{
2779 #if defined _GLIBCXX_SIMD_DEFAULT_ABI
2780 template <typename _Tp>
2781  using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>;
2782 #else
2783 template <typename _Tp>
2784  using __default_abi = compatible<_Tp>;
2785 #endif
2786 
2787 // }}}
2788 } // namespace simd_abi
2789 
2790 // traits {{{1
2791 // is_abi_tag {{{2
2792 template <typename _Tp, typename = void_t<>>
2793  struct is_abi_tag : false_type {};
2794 
2795 template <typename _Tp>
2796  struct is_abi_tag<_Tp, void_t<typename _Tp::_IsValidAbiTag>>
2797  : public _Tp::_IsValidAbiTag {};
2798 
2799 template <typename _Tp>
2800  inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value;
2801 
2802 // is_simd(_mask) {{{2
2803 template <typename _Tp>
2804  struct is_simd : public false_type {};
2805 
2806 template <typename _Tp>
2807  inline constexpr bool is_simd_v = is_simd<_Tp>::value;
2808 
2809 template <typename _Tp>
2810  struct is_simd_mask : public false_type {};
2811 
2812 template <typename _Tp>
2813 inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value;
2814 
2815 // simd_size {{{2
2816 template <typename _Tp, typename _Abi, typename = void>
2817  struct __simd_size_impl {};
2818 
2819 template <typename _Tp, typename _Abi>
2820  struct __simd_size_impl<
2821  _Tp, _Abi,
2822  enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>>
2823  : _SizeConstant<_Abi::template _S_size<_Tp>> {};
2824 
2825 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2826  struct simd_size : __simd_size_impl<_Tp, _Abi> {};
2827 
2828 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2829  inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value;
2830 
2831 // simd_abi::deduce {{{2
2832 template <typename _Tp, size_t _Np, typename = void>
2833  struct __deduce_impl;
2834 
2835 namespace simd_abi {
2836 /**
2837  * @tparam _Tp The requested `value_type` for the elements.
2838  * @tparam _Np The requested number of elements.
2839  * @tparam _Abis This parameter is ignored, since this implementation cannot
2840  * make any use of it. Either __a good native ABI is matched and used as `type`
2841  * alias, or the `fixed_size<_Np>` ABI is used, which internally is built from
2842  * the best matching native ABIs.
2843  */
2844 template <typename _Tp, size_t _Np, typename...>
2845  struct deduce : __deduce_impl<_Tp, _Np> {};
2846 
2847 template <typename _Tp, size_t _Np, typename... _Abis>
2848  using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type;
2849 } // namespace simd_abi
2850 
2851 // }}}2
2852 // rebind_simd {{{2
2853 template <typename _Tp, typename _V, typename = void>
2854  struct rebind_simd;
2855 
2856 template <typename _Tp, typename _Up, typename _Abi>
2857  struct rebind_simd<
2858  _Tp, simd<_Up, _Abi>,
2859  void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2860  {
2861  using type
2862  = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2863  };
2864 
2865 template <typename _Tp, typename _Up, typename _Abi>
2866  struct rebind_simd<
2867  _Tp, simd_mask<_Up, _Abi>,
2868  void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>>
2869  {
2870  using type
2871  = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>;
2872  };
2873 
2874 template <typename _Tp, typename _V>
2875  using rebind_simd_t = typename rebind_simd<_Tp, _V>::type;
2876 
2877 // resize_simd {{{2
2878 template <int _Np, typename _V, typename = void>
2879  struct resize_simd;
2880 
2881 template <int _Np, typename _Tp, typename _Abi>
2882  struct resize_simd<_Np, simd<_Tp, _Abi>,
2883  void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2884  { using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2885 
2886 template <int _Np, typename _Tp, typename _Abi>
2887  struct resize_simd<_Np, simd_mask<_Tp, _Abi>,
2888  void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>>
2889  { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; };
2890 
2891 template <int _Np, typename _V>
2892  using resize_simd_t = typename resize_simd<_Np, _V>::type;
2893 
2894 // }}}2
2895 // memory_alignment {{{2
2896 template <typename _Tp, typename _Up = typename _Tp::value_type>
2897  struct memory_alignment
2898  : public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {};
2899 
2900 template <typename _Tp, typename _Up = typename _Tp::value_type>
2901  inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value;
2902 
2903 // class template simd [simd] {{{1
2904 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2905  class simd;
2906 
2907 template <typename _Tp, typename _Abi>
2908  struct is_simd<simd<_Tp, _Abi>> : public true_type {};
2909 
2910 template <typename _Tp>
2911  using native_simd = simd<_Tp, simd_abi::native<_Tp>>;
2912 
2913 template <typename _Tp, int _Np>
2914  using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>;
2915 
2916 template <typename _Tp, size_t _Np>
2917  using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2918 
2919 // class template simd_mask [simd_mask] {{{1
2920 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>>
2921  class simd_mask;
2922 
2923 template <typename _Tp, typename _Abi>
2924  struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {};
2925 
2926 template <typename _Tp>
2927  using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>;
2928 
2929 template <typename _Tp, int _Np>
2930  using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>;
2931 
2932 template <typename _Tp, size_t _Np>
2933  using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>;
2934 
2935 // casts [simd.casts] {{{1
2936 // static_simd_cast {{{2
2937 template <typename _Tp, typename _Up, typename _Ap, bool = is_simd_v<_Tp>,
2938  typename = void>
2939  struct __static_simd_cast_return_type;
2940 
2941 template <typename _Tp, typename _A0, typename _Up, typename _Ap>
2942  struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false,
2943  void>
2944  : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {};
2945 
2946 template <typename _Tp, typename _Up, typename _Ap>
2947  struct __static_simd_cast_return_type<
2948  _Tp, _Up, _Ap, true, enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>>
2949  { using type = _Tp; };
2950 
2951 template <typename _Tp, typename _Ap>
2952  struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false,
2953 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2954  enable_if_t<__is_vectorizable_v<_Tp>>
2955 #else
2956  void
2957 #endif
2958  >
2959  { using type = simd<_Tp, _Ap>; };
2960 
2961 template <typename _Tp, typename = void>
2962  struct __safe_make_signed { using type = _Tp;};
2963 
2964 template <typename _Tp>
2965  struct __safe_make_signed<_Tp, enable_if_t<is_integral_v<_Tp>>>
2966  {
2967  // the extra make_unsigned_t is because of PR85951
2968  using type = make_signed_t<make_unsigned_t<_Tp>>;
2969  };
2970 
2971 template <typename _Tp>
2972  using safe_make_signed_t = typename __safe_make_signed<_Tp>::type;
2973 
2974 template <typename _Tp, typename _Up, typename _Ap>
2975  struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false,
2976 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66
2977  enable_if_t<__is_vectorizable_v<_Tp>>
2978 #else
2979  void
2980 #endif
2981  >
2982  {
2983  using type = conditional_t<
2984  (is_integral_v<_Up> && is_integral_v<_Tp> &&
2985 #ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65
2986  is_signed_v<_Up> != is_signed_v<_Tp> &&
2987 #endif
2988  is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>),
2989  simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>;
2990  };
2991 
2992 template <typename _Tp, typename _Up, typename _Ap,
2993  typename _R
2994  = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
2995  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R
2996  static_simd_cast(const simd<_Up, _Ap>& __x)
2997  {
2998  if constexpr (is_same<_R, simd<_Up, _Ap>>::value)
2999  return __x;
3000  else
3001  {
3002  _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type>
3003  __c;
3004  return _R(__private_init, __c(__data(__x)));
3005  }
3006  }
3007 
3008 namespace __proposed {
3009 template <typename _Tp, typename _Up, typename _Ap,
3010  typename _R
3011  = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type>
3012  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR typename _R::mask_type
3013  static_simd_cast(const simd_mask<_Up, _Ap>& __x)
3014  {
3015  using _RM = typename _R::mask_type;
3016  return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert<
3017  typename _RM::simd_type::value_type>(__x)};
3018  }
3019 
3020 template <typename _To, typename _Up, typename _Abi>
3021  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3022  _To
3023  simd_bit_cast(const simd<_Up, _Abi>& __x)
3024  {
3025  using _Tp = typename _To::value_type;
3026  using _ToMember = typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3027  using _From = simd<_Up, _Abi>;
3028  using _FromMember = typename _SimdTraits<_Up, _Abi>::_SimdMember;
3029  // with concepts, the following should be constraints
3030  static_assert(sizeof(_To) == sizeof(_From));
3031  static_assert(is_trivially_copyable_v<_Tp> && is_trivially_copyable_v<_Up>);
3032  static_assert(is_trivially_copyable_v<_ToMember> && is_trivially_copyable_v<_FromMember>);
3033 #if __has_builtin(__builtin_bit_cast)
3034  return {__private_init, __builtin_bit_cast(_ToMember, __data(__x))};
3035 #else
3036  return {__private_init, __bit_cast<_ToMember>(__data(__x))};
3037 #endif
3038  }
3039 
3040 template <typename _To, typename _Up, typename _Abi>
3041  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3042  _To
3043  simd_bit_cast(const simd_mask<_Up, _Abi>& __x)
3044  {
3045  using _From = simd_mask<_Up, _Abi>;
3046  static_assert(sizeof(_To) == sizeof(_From));
3047  static_assert(is_trivially_copyable_v<_From>);
3048  // _To can be simd<T, A>, specifically simd<T, fixed_size<N>> in which case _To is not trivially
3049  // copyable.
3050  if constexpr (is_simd_v<_To>)
3051  {
3052  using _Tp = typename _To::value_type;
3053  using _ToMember = typename _SimdTraits<_Tp, typename _To::abi_type>::_SimdMember;
3054  static_assert(is_trivially_copyable_v<_ToMember>);
3055 #if __has_builtin(__builtin_bit_cast)
3056  return {__private_init, __builtin_bit_cast(_ToMember, __x)};
3057 #else
3058  return {__private_init, __bit_cast<_ToMember>(__x)};
3059 #endif
3060  }
3061  else
3062  {
3063  static_assert(is_trivially_copyable_v<_To>);
3064 #if __has_builtin(__builtin_bit_cast)
3065  return __builtin_bit_cast(_To, __x);
3066 #else
3067  return __bit_cast<_To>(__x);
3068 #endif
3069  }
3070  }
3071 } // namespace __proposed
3072 
3073 // simd_cast {{{2
3074 template <typename _Tp, typename _Up, typename _Ap,
3075  typename _To = __value_type_or_identity_t<_Tp>>
3076  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3077  simd_cast(const simd<_ValuePreserving<_Up, _To>, _Ap>& __x)
3078  -> decltype(static_simd_cast<_Tp>(__x))
3079  { return static_simd_cast<_Tp>(__x); }
3080 
3081 namespace __proposed {
3082 template <typename _Tp, typename _Up, typename _Ap,
3083  typename _To = __value_type_or_identity_t<_Tp>>
3084  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto
3085  simd_cast(const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x)
3086  -> decltype(static_simd_cast<_Tp>(__x))
3087  { return static_simd_cast<_Tp>(__x); }
3088 } // namespace __proposed
3089 
3090 // }}}2
3091 // resizing_simd_cast {{{
3092 namespace __proposed {
3093 /* Proposed spec:
3094 
3095 template <class T, class U, class Abi>
3096 T resizing_simd_cast(const simd<U, Abi>& x)
3097 
3098 p1 Constraints:
3099  - is_simd_v<T> is true and
3100  - T::value_type is the same type as U
3101 
3102 p2 Returns:
3103  A simd object with the i^th element initialized to x[i] for all i in the
3104  range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3105  than simd_size_v<U, Abi>, the remaining elements are value-initialized.
3106 
3107 template <class T, class U, class Abi>
3108 T resizing_simd_cast(const simd_mask<U, Abi>& x)
3109 
3110 p1 Constraints: is_simd_mask_v<T> is true
3111 
3112 p2 Returns:
3113  A simd_mask object with the i^th element initialized to x[i] for all i in
3114 the range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger
3115  than simd_size_v<U, Abi>, the remaining elements are initialized to false.
3116 
3117  */
3118 
3119 template <typename _Tp, typename _Up, typename _Ap>
3120  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR enable_if_t<
3121  conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp>
3122  resizing_simd_cast(const simd<_Up, _Ap>& __x)
3123  {
3124  if constexpr (is_same_v<typename _Tp::abi_type, _Ap>)
3125  return __x;
3126  else if constexpr (simd_size_v<_Up, _Ap> == 1)
3127  {
3128  _Tp __r{};
3129  __r[0] = __x[0];
3130  return __r;
3131  }
3132  else if constexpr (_Tp::size() == 1)
3133  return __x[0];
3134  else if constexpr (sizeof(_Tp) == sizeof(__x)
3135  && !__is_fixed_size_abi_v<_Ap>)
3136  return {__private_init,
3137  __vector_bitcast<typename _Tp::value_type, _Tp::size()>(
3138  _Ap::_S_masked(__data(__x))._M_data)};
3139  else
3140  {
3141  _Tp __r{};
3142  __builtin_memcpy(&__data(__r), &__data(__x),
3143  sizeof(_Up)
3144  * std::min(_Tp::size(), simd_size_v<_Up, _Ap>));
3145  return __r;
3146  }
3147  }
3148 
3149 template <typename _Tp, typename _Up, typename _Ap>
3150  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3151  enable_if_t<is_simd_mask_v<_Tp>, _Tp>
3152  resizing_simd_cast(const simd_mask<_Up, _Ap>& __x)
3153  {
3154  return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert<
3155  typename _Tp::simd_type::value_type>(__x)};
3156  }
3157 } // namespace __proposed
3158 
3159 // }}}
3160 // to_fixed_size {{{2
3161 template <typename _Tp, int _Np>
3162  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np>
3163  to_fixed_size(const fixed_size_simd<_Tp, _Np>& __x)
3164  { return __x; }
3165 
3166 template <typename _Tp, int _Np>
3167  _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np>
3168  to_fixed_size(const fixed_size_simd_mask<_Tp, _Np>& __x)
3169  { return __x; }
3170 
3171 template <typename _Tp, typename _Ap>
3172  _GLIBCXX_SIMD_INTRINSIC auto
3173  to_fixed_size(const simd<_Tp, _Ap>& __x)
3174  {
3175  return simd<_Tp, simd_abi::fixed_size<simd_size_v<_Tp, _Ap>>>([&__x](
3176  auto __i) constexpr { return __x[__i]; });
3177  }
3178 
3179 template <typename _Tp, typename _Ap>
3180  _GLIBCXX_SIMD_INTRINSIC auto
3181  to_fixed_size(const simd_mask<_Tp, _Ap>& __x)
3182  {
3183  constexpr int _Np = simd_mask<_Tp, _Ap>::size();
3184  fixed_size_simd_mask<_Tp, _Np> __r;
3185  __execute_n_times<_Np>([&](auto __i) constexpr { __r[__i] = __x[__i]; });
3186  return __r;
3187  }
3188 
3189 // to_native {{{2
3190 template <typename _Tp, int _Np>
3191  _GLIBCXX_SIMD_INTRINSIC
3192  enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>>
3193  to_native(const fixed_size_simd<_Tp, _Np>& __x)
3194  {
3195  alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np];
3196  __x.copy_to(__mem, vector_aligned);
3197  return {__mem, vector_aligned};
3198  }
3199 
3200 template <typename _Tp, size_t _Np>
3201  _GLIBCXX_SIMD_INTRINSIC
3202  enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>>
3203  to_native(const fixed_size_simd_mask<_Tp, _Np>& __x)
3204  {
3205  return native_simd_mask<_Tp>([&](auto __i) constexpr { return __x[__i]; });
3206  }
3207 
3208 // to_compatible {{{2
3209 template <typename _Tp, size_t _Np>
3210  _GLIBCXX_SIMD_INTRINSIC enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>>
3211  to_compatible(const simd<_Tp, simd_abi::fixed_size<_Np>>& __x)
3212  {
3213  alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np];
3214  __x.copy_to(__mem, vector_aligned);
3215  return {__mem, vector_aligned};
3216  }
3217 
3218 template <typename _Tp, size_t _Np>
3219  _GLIBCXX_SIMD_INTRINSIC
3220  enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>>
3221  to_compatible(const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x)
3222  { return simd_mask<_Tp>([&](auto __i) constexpr { return __x[__i]; }); }
3223 
3224 // masked assignment [simd_mask.where] {{{1
3225 
3226 // where_expression {{{1
3227 // const_where_expression<M, T> {{{2
3228 template <typename _M, typename _Tp>
3229  class const_where_expression
3230  {
3231  using _V = _Tp;
3232  static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3233 
3234  struct _Wrapper { using value_type = _V; };
3235 
3236  protected:
3237  using _Impl = typename _V::_Impl;
3238 
3239  using value_type =
3240  typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3241 
3242  _GLIBCXX_SIMD_INTRINSIC friend const _M&
3243  __get_mask(const const_where_expression& __x)
3244  { return __x._M_k; }
3245 
3246  _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3247  __get_lvalue(const const_where_expression& __x)
3248  { return __x._M_value; }
3249 
3250  const _M& _M_k;
3251  _Tp& _M_value;
3252 
3253  public:
3254  const_where_expression(const const_where_expression&) = delete;
3255  const_where_expression& operator=(const const_where_expression&) = delete;
3256 
3257  _GLIBCXX_SIMD_INTRINSIC const_where_expression(const _M& __kk, const _Tp& dd)
3258  : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3259 
3260  _GLIBCXX_SIMD_INTRINSIC _V
3261  operator-() const&&
3262  {
3263  return {__private_init,
3264  _Impl::template _S_masked_unary<negate>(__data(_M_k),
3265  __data(_M_value))};
3266  }
3267 
3268  template <typename _Up, typename _Flags>
3269  [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3270  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3271  {
3272  return {__private_init,
3273  _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3274  _Flags::template _S_apply<_V>(__mem))};
3275  }
3276 
3277  template <typename _Up, typename _Flags>
3278  _GLIBCXX_SIMD_INTRINSIC void
3279  copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3280  {
3281  _Impl::_S_masked_store(__data(_M_value),
3282  _Flags::template _S_apply<_V>(__mem),
3283  __data(_M_k));
3284  }
3285  };
3286 
3287 // const_where_expression<bool, T> {{{2
3288 template <typename _Tp>
3289  class const_where_expression<bool, _Tp>
3290  {
3291  using _M = bool;
3292  using _V = _Tp;
3293 
3294  static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>);
3295 
3296  struct _Wrapper { using value_type = _V; };
3297 
3298  protected:
3299  using value_type =
3300  typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type;
3301 
3302  _GLIBCXX_SIMD_INTRINSIC friend const _M&
3303  __get_mask(const const_where_expression& __x)
3304  { return __x._M_k; }
3305 
3306  _GLIBCXX_SIMD_INTRINSIC friend const _Tp&
3307  __get_lvalue(const const_where_expression& __x)
3308  { return __x._M_value; }
3309 
3310  const bool _M_k;
3311  _Tp& _M_value;
3312 
3313  public:
3314  const_where_expression(const const_where_expression&) = delete;
3315  const_where_expression& operator=(const const_where_expression&) = delete;
3316 
3317  _GLIBCXX_SIMD_INTRINSIC const_where_expression(const bool __kk, const _Tp& dd)
3318  : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {}
3319 
3320  _GLIBCXX_SIMD_INTRINSIC _V operator-() const&&
3321  { return _M_k ? -_M_value : _M_value; }
3322 
3323  template <typename _Up, typename _Flags>
3324  [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _V
3325  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3326  { return _M_k ? static_cast<_V>(__mem[0]) : _M_value; }
3327 
3328  template <typename _Up, typename _Flags>
3329  _GLIBCXX_SIMD_INTRINSIC void
3330  copy_to(_LoadStorePtr<_Up, value_type>* __mem, _Flags) const&&
3331  {
3332  if (_M_k)
3333  __mem[0] = _M_value;
3334  }
3335  };
3336 
3337 // where_expression<M, T> {{{2
3338 template <typename _M, typename _Tp>
3339  class where_expression : public const_where_expression<_M, _Tp>
3340  {
3341  using _Impl = typename const_where_expression<_M, _Tp>::_Impl;
3342 
3343  static_assert(!is_const<_Tp>::value,
3344  "where_expression may only be instantiated with __a non-const "
3345  "_Tp parameter");
3346 
3347  using typename const_where_expression<_M, _Tp>::value_type;
3348  using const_where_expression<_M, _Tp>::_M_k;
3349  using const_where_expression<_M, _Tp>::_M_value;
3350 
3351  static_assert(
3352  is_same<typename _M::abi_type, typename _Tp::abi_type>::value, "");
3353  static_assert(_M::size() == _Tp::size(), "");
3354 
3355  _GLIBCXX_SIMD_INTRINSIC friend _Tp& __get_lvalue(where_expression& __x)
3356  { return __x._M_value; }
3357 
3358  public:
3359  where_expression(const where_expression&) = delete;
3360  where_expression& operator=(const where_expression&) = delete;
3361 
3362  _GLIBCXX_SIMD_INTRINSIC where_expression(const _M& __kk, _Tp& dd)
3363  : const_where_expression<_M, _Tp>(__kk, dd) {}
3364 
3365  template <typename _Up>
3366  _GLIBCXX_SIMD_INTRINSIC void operator=(_Up&& __x) &&
3367  {
3368  _Impl::_S_masked_assign(__data(_M_k), __data(_M_value),
3369  __to_value_type_or_member_type<_Tp>(
3370  static_cast<_Up&&>(__x)));
3371  }
3372 
3373 #define _GLIBCXX_SIMD_OP_(__op, __name) \
3374  template <typename _Up> \
3375  _GLIBCXX_SIMD_INTRINSIC void operator __op##=(_Up&& __x)&& \
3376  { \
3377  _Impl::template _S_masked_cassign( \
3378  __data(_M_k), __data(_M_value), \
3379  __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \
3380  [](auto __impl, auto __lhs, auto __rhs) constexpr { \
3381  return __impl.__name(__lhs, __rhs); \
3382  }); \
3383  } \
3384  static_assert(true)
3385  _GLIBCXX_SIMD_OP_(+, _S_plus);
3386  _GLIBCXX_SIMD_OP_(-, _S_minus);
3387  _GLIBCXX_SIMD_OP_(*, _S_multiplies);
3388  _GLIBCXX_SIMD_OP_(/, _S_divides);
3389  _GLIBCXX_SIMD_OP_(%, _S_modulus);
3390  _GLIBCXX_SIMD_OP_(&, _S_bit_and);
3391  _GLIBCXX_SIMD_OP_(|, _S_bit_or);
3392  _GLIBCXX_SIMD_OP_(^, _S_bit_xor);
3393  _GLIBCXX_SIMD_OP_(<<, _S_shift_left);
3394  _GLIBCXX_SIMD_OP_(>>, _S_shift_right);
3395 #undef _GLIBCXX_SIMD_OP_
3396 
3397  _GLIBCXX_SIMD_INTRINSIC void operator++() &&
3398  {
3399  __data(_M_value)
3400  = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3401  __data(_M_value));
3402  }
3403 
3404  _GLIBCXX_SIMD_INTRINSIC void operator++(int) &&
3405  {
3406  __data(_M_value)
3407  = _Impl::template _S_masked_unary<__increment>(__data(_M_k),
3408  __data(_M_value));
3409  }
3410 
3411  _GLIBCXX_SIMD_INTRINSIC void operator--() &&
3412  {
3413  __data(_M_value)
3414  = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3415  __data(_M_value));
3416  }
3417 
3418  _GLIBCXX_SIMD_INTRINSIC void operator--(int) &&
3419  {
3420  __data(_M_value)
3421  = _Impl::template _S_masked_unary<__decrement>(__data(_M_k),
3422  __data(_M_value));
3423  }
3424 
3425  // intentionally hides const_where_expression::copy_from
3426  template <typename _Up, typename _Flags>
3427  _GLIBCXX_SIMD_INTRINSIC void
3428  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3429  {
3430  __data(_M_value)
3431  = _Impl::_S_masked_load(__data(_M_value), __data(_M_k),
3432  _Flags::template _S_apply<_Tp>(__mem));
3433  }
3434  };
3435 
3436 // where_expression<bool, T> {{{2
3437 template <typename _Tp>
3438  class where_expression<bool, _Tp> : public const_where_expression<bool, _Tp>
3439  {
3440  using _M = bool;
3441  using typename const_where_expression<_M, _Tp>::value_type;
3442  using const_where_expression<_M, _Tp>::_M_k;
3443  using const_where_expression<_M, _Tp>::_M_value;
3444 
3445  public:
3446  where_expression(const where_expression&) = delete;
3447  where_expression& operator=(const where_expression&) = delete;
3448 
3449  _GLIBCXX_SIMD_INTRINSIC where_expression(const _M& __kk, _Tp& dd)
3450  : const_where_expression<_M, _Tp>(__kk, dd) {}
3451 
3452 #define _GLIBCXX_SIMD_OP_(__op) \
3453  template <typename _Up> \
3454  _GLIBCXX_SIMD_INTRINSIC void operator __op(_Up&& __x)&& \
3455  { if (_M_k) _M_value __op static_cast<_Up&&>(__x); }
3456 
3457  _GLIBCXX_SIMD_OP_(=)
3458  _GLIBCXX_SIMD_OP_(+=)
3459  _GLIBCXX_SIMD_OP_(-=)
3460  _GLIBCXX_SIMD_OP_(*=)
3461  _GLIBCXX_SIMD_OP_(/=)
3462  _GLIBCXX_SIMD_OP_(%=)
3463  _GLIBCXX_SIMD_OP_(&=)
3464  _GLIBCXX_SIMD_OP_(|=)
3465  _GLIBCXX_SIMD_OP_(^=)
3466  _GLIBCXX_SIMD_OP_(<<=)
3467  _GLIBCXX_SIMD_OP_(>>=)
3468  #undef _GLIBCXX_SIMD_OP_
3469 
3470  _GLIBCXX_SIMD_INTRINSIC void operator++() &&
3471  { if (_M_k) ++_M_value; }
3472 
3473  _GLIBCXX_SIMD_INTRINSIC void operator++(int) &&
3474  { if (_M_k) ++_M_value; }
3475 
3476  _GLIBCXX_SIMD_INTRINSIC void operator--() &&
3477  { if (_M_k) --_M_value; }
3478 
3479  _GLIBCXX_SIMD_INTRINSIC void operator--(int) &&
3480  { if (_M_k) --_M_value; }
3481 
3482  // intentionally hides const_where_expression::copy_from
3483  template <typename _Up, typename _Flags>
3484  _GLIBCXX_SIMD_INTRINSIC void
3485  copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _Flags) &&
3486  { if (_M_k) _M_value = __mem[0]; }
3487  };
3488 
3489 // where {{{1
3490 template <typename _Tp, typename _Ap>
3491  _GLIBCXX_SIMD_INTRINSIC where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3492  where(const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value)
3493  { return {__k, __value}; }
3494 
3495 template <typename _Tp, typename _Ap>
3496  _GLIBCXX_SIMD_INTRINSIC
3497  const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>>
3498  where(const typename simd<_Tp, _Ap>::mask_type& __k,
3499  const simd<_Tp, _Ap>& __value)
3500  { return {__k, __value}; }
3501 
3502 template <typename _Tp, typename _Ap>
3503  _GLIBCXX_SIMD_INTRINSIC
3504  where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3505  where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
3506  simd_mask<_Tp, _Ap>& __value)
3507  { return {__k, __value}; }
3508 
3509 template <typename _Tp, typename _Ap>
3510  _GLIBCXX_SIMD_INTRINSIC
3511  const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>>
3512  where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k,
3513  const simd_mask<_Tp, _Ap>& __value)
3514  { return {__k, __value}; }
3515 
3516 template <typename _Tp>
3517  _GLIBCXX_SIMD_INTRINSIC where_expression<bool, _Tp>
3518  where(_ExactBool __k, _Tp& __value)
3519  { return {__k, __value}; }
3520 
3521 template <typename _Tp>
3522  _GLIBCXX_SIMD_INTRINSIC const_where_expression<bool, _Tp>
3523  where(_ExactBool __k, const _Tp& __value)
3524  { return {__k, __value}; }
3525 
3526  template <typename _Tp, typename _Ap>
3527  void where(bool __k, simd<_Tp, _Ap>& __value) = delete;
3528 
3529  template <typename _Tp, typename _Ap>
3530  void where(bool __k, const simd<_Tp, _Ap>& __value) = delete;
3531 
3532 // proposed mask iterations {{{1
3533 namespace __proposed {
3534 template <size_t _Np>
3535  class where_range
3536  {
3537  const bitset<_Np> __bits;
3538 
3539  public:
3540  where_range(bitset<_Np> __b) : __bits(__b) {}
3541 
3542  class iterator
3543  {
3544  size_t __mask;
3545  size_t __bit;
3546 
3547  _GLIBCXX_SIMD_INTRINSIC void __next_bit()
3548  { __bit = __builtin_ctzl(__mask); }
3549 
3550  _GLIBCXX_SIMD_INTRINSIC void __reset_lsb()
3551  {
3552  // 01100100 - 1 = 01100011
3553  __mask &= (__mask - 1);
3554  // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit));
3555  }
3556 
3557  public:
3558  iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); }
3559  iterator(const iterator&) = default;
3560  iterator(iterator&&) = default;
3561 
3562  _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator->() const
3563  { return __bit; }
3564 
3565  _GLIBCXX_SIMD_ALWAYS_INLINE size_t operator*() const
3566  { return __bit; }
3567 
3568  _GLIBCXX_SIMD_ALWAYS_INLINE iterator& operator++()
3569  {
3570  __reset_lsb();
3571  __next_bit();
3572  return *this;
3573  }
3574 
3575  _GLIBCXX_SIMD_ALWAYS_INLINE iterator operator++(int)
3576  {
3577  iterator __tmp = *this;
3578  __reset_lsb();
3579  __next_bit();
3580  return __tmp;
3581  }
3582 
3583  _GLIBCXX_SIMD_ALWAYS_INLINE bool operator==(const iterator& __rhs) const
3584  { return __mask == __rhs.__mask; }
3585 
3586  _GLIBCXX_SIMD_ALWAYS_INLINE bool operator!=(const iterator& __rhs) const
3587  { return __mask != __rhs.__mask; }
3588  };
3589 
3590  iterator begin() const
3591  { return __bits.to_ullong(); }
3592 
3593  iterator end() const
3594  { return 0; }
3595  };
3596 
3597 template <typename _Tp, typename _Ap>
3598  where_range<simd_size_v<_Tp, _Ap>>
3599  where(const simd_mask<_Tp, _Ap>& __k)
3600  { return __k.__to_bitset(); }
3601 
3602 } // namespace __proposed
3603 
3604 // }}}1
3605 // reductions [simd.reductions] {{{1
3606 template <typename _Tp, typename _Abi, typename _BinaryOperation = plus<>>
3607  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3608  reduce(const simd<_Tp, _Abi>& __v,
3609  _BinaryOperation __binary_op = _BinaryOperation())
3610  { return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); }
3611 
3612 template <typename _M, typename _V, typename _BinaryOperation = plus<>>
3613  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3614  reduce(const const_where_expression<_M, _V>& __x,
3615  typename _V::value_type __identity_element,
3616  _BinaryOperation __binary_op)
3617  {
3618  if (__builtin_expect(none_of(__get_mask(__x)), false))
3619  return __identity_element;
3620 
3621  _V __tmp = __identity_element;
3622  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3623  __data(__get_lvalue(__x)));
3624  return reduce(__tmp, __binary_op);
3625  }
3626 
3627 template <typename _M, typename _V>
3628  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3629  reduce(const const_where_expression<_M, _V>& __x, plus<> __binary_op = {})
3630  { return reduce(__x, 0, __binary_op); }
3631 
3632 template <typename _M, typename _V>
3633  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3634  reduce(const const_where_expression<_M, _V>& __x, multiplies<> __binary_op)
3635  { return reduce(__x, 1, __binary_op); }
3636 
3637 template <typename _M, typename _V>
3638  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3639  reduce(const const_where_expression<_M, _V>& __x, bit_and<> __binary_op)
3640  { return reduce(__x, ~typename _V::value_type(), __binary_op); }
3641 
3642 template <typename _M, typename _V>
3643  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3644  reduce(const const_where_expression<_M, _V>& __x, bit_or<> __binary_op)
3645  { return reduce(__x, 0, __binary_op); }
3646 
3647 template <typename _M, typename _V>
3648  _GLIBCXX_SIMD_INTRINSIC typename _V::value_type
3649  reduce(const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op)
3650  { return reduce(__x, 0, __binary_op); }
3651 
3652 template <typename _Tp, typename _Abi>
3653  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3654  hmin(const simd<_Tp, _Abi>& __v) noexcept
3655  {
3656  return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum());
3657  }
3658 
3659 template <typename _Tp, typename _Abi>
3660  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp
3661  hmax(const simd<_Tp, _Abi>& __v) noexcept
3662  {
3663  return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum());
3664  }
3665 
3666 template <typename _M, typename _V>
3667  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3668  typename _V::value_type
3669  hmin(const const_where_expression<_M, _V>& __x) noexcept
3670  {
3671  using _Tp = typename _V::value_type;
3672  constexpr _Tp __id_elem =
3673 #ifdef __FINITE_MATH_ONLY__
3674  __finite_max_v<_Tp>;
3675 #else
3676  __value_or<__infinity, _Tp>(__finite_max_v<_Tp>);
3677 #endif
3678  _V __tmp = __id_elem;
3679  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3680  __data(__get_lvalue(__x)));
3681  return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum());
3682  }
3683 
3684 template <typename _M, typename _V>
3685  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3686  typename _V::value_type
3687  hmax(const const_where_expression<_M, _V>& __x) noexcept
3688  {
3689  using _Tp = typename _V::value_type;
3690  constexpr _Tp __id_elem =
3691 #ifdef __FINITE_MATH_ONLY__
3692  __finite_min_v<_Tp>;
3693 #else
3694  [] {
3695  if constexpr (__value_exists_v<__infinity, _Tp>)
3696  return -__infinity_v<_Tp>;
3697  else
3698  return __finite_min_v<_Tp>;
3699  }();
3700 #endif
3701  _V __tmp = __id_elem;
3702  _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp),
3703  __data(__get_lvalue(__x)));
3704  return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum());
3705  }
3706 
3707 // }}}1
3708 // algorithms [simd.alg] {{{
3709 template <typename _Tp, typename _Ap>
3710  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3711  min(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3712  { return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; }
3713 
3714 template <typename _Tp, typename _Ap>
3715  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3716  max(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3717  { return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; }
3718 
3719 template <typename _Tp, typename _Ap>
3720  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
3721  pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>>
3722  minmax(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
3723  {
3724  const auto pair_of_members
3725  = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b));
3726  return {simd<_Tp, _Ap>(__private_init, pair_of_members.first),
3727  simd<_Tp, _Ap>(__private_init, pair_of_members.second)};
3728  }
3729 
3730 template <typename _Tp, typename _Ap>
3731  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
3732  clamp(const simd<_Tp, _Ap>& __v, const simd<_Tp, _Ap>& __lo,
3733  const simd<_Tp, _Ap>& __hi)
3734  {
3735  using _Impl = typename _Ap::_SimdImpl;
3736  return {__private_init,
3737  _Impl::_S_min(__data(__hi),
3738  _Impl::_S_max(__data(__lo), __data(__v)))};
3739  }
3740 
3741 // }}}
3742 
3743 template <size_t... _Sizes, typename _Tp, typename _Ap,
3744  typename = enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>>
3745  inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3746  split(const simd<_Tp, _Ap>&);
3747 
3748 // __extract_part {{{
3749 template <int _Index, int _Total, int _Combine = 1, typename _Tp, size_t _Np>
3750  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST
3751  _SimdWrapper<_Tp, _Np / _Total * _Combine>
3752  __extract_part(const _SimdWrapper<_Tp, _Np> __x);
3753 
3754 template <int Index, int Parts, int _Combine = 1, typename _Tp, typename _A0,
3755  typename... _As>
3756  _GLIBCXX_SIMD_INTRINSIC auto
3757  __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x);
3758 
3759 // }}}
3760 // _SizeList {{{
3761 template <size_t _V0, size_t... _Values>
3762  struct _SizeList
3763  {
3764  template <size_t _I>
3765  static constexpr size_t _S_at(_SizeConstant<_I> = {})
3766  {
3767  if constexpr (_I == 0)
3768  return _V0;
3769  else
3770  return _SizeList<_Values...>::template _S_at<_I - 1>();
3771  }
3772 
3773  template <size_t _I>
3774  static constexpr auto _S_before(_SizeConstant<_I> = {})
3775  {
3776  if constexpr (_I == 0)
3777  return _SizeConstant<0>();
3778  else
3779  return _SizeConstant<
3780  _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>();
3781  }
3782 
3783  template <size_t _Np>
3784  static constexpr auto _S_pop_front(_SizeConstant<_Np> = {})
3785  {
3786  if constexpr (_Np == 0)
3787  return _SizeList();
3788  else
3789  return _SizeList<_Values...>::template _S_pop_front<_Np - 1>();
3790  }
3791  };
3792 
3793 // }}}
3794 // __extract_center {{{
3795 template <typename _Tp, size_t _Np>
3796  _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2>
3797  __extract_center(_SimdWrapper<_Tp, _Np> __x)
3798  {
3799  static_assert(_Np >= 4);
3800  static_assert(_Np % 4 == 0); // x0 - x1 - x2 - x3 -> return {x1, x2}
3801 #if _GLIBCXX_SIMD_X86INTRIN // {{{
3802  if constexpr (__have_avx512f && sizeof(_Tp) * _Np == 64)
3803  {
3804  const auto __intrin = __to_intrin(__x);
3805  if constexpr (is_integral_v<_Tp>)
3806  return __vector_bitcast<_Tp>(_mm512_castsi512_si256(
3807  _mm512_shuffle_i32x4(__intrin, __intrin,
3808  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3809  else if constexpr (sizeof(_Tp) == 4)
3810  return __vector_bitcast<_Tp>(_mm512_castps512_ps256(
3811  _mm512_shuffle_f32x4(__intrin, __intrin,
3812  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3813  else if constexpr (sizeof(_Tp) == 8)
3814  return __vector_bitcast<_Tp>(_mm512_castpd512_pd256(
3815  _mm512_shuffle_f64x2(__intrin, __intrin,
3816  1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40)));
3817  else
3818  __assert_unreachable<_Tp>();
3819  }
3820  else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>)
3821  return __vector_bitcast<_Tp>(
3822  _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)),
3823  __hi128(__vector_bitcast<double>(__x)), 1));
3824  else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32)
3825  return __vector_bitcast<_Tp>(
3826  _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)),
3827  __lo128(__vector_bitcast<_LLong>(__x)),
3828  sizeof(_Tp) * _Np / 4));
3829  else
3830 #endif // _GLIBCXX_SIMD_X86INTRIN }}}
3831  {
3832  __vector_type_t<_Tp, _Np / 2> __r;
3833  __builtin_memcpy(&__r,
3834  reinterpret_cast<const char*>(&__x)
3835  + sizeof(_Tp) * _Np / 4,
3836  sizeof(_Tp) * _Np / 2);
3837  return __r;
3838  }
3839  }
3840 
3841 template <typename _Tp, typename _A0, typename... _As>
3842  _GLIBCXX_SIMD_INTRINSIC
3843  _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2>
3844  __extract_center(const _SimdTuple<_Tp, _A0, _As...>& __x)
3845  {
3846  if constexpr (sizeof...(_As) == 0)
3847  return __extract_center(__x.first);
3848  else
3849  return __extract_part<1, 4, 2>(__x);
3850  }
3851 
3852 // }}}
3853 // __split_wrapper {{{
3854 template <size_t... _Sizes, typename _Tp, typename... _As>
3855  auto
3856  __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x)
3857  {
3858  return split<_Sizes...>(
3859  fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init,
3860  __x));
3861  }
3862 
3863 // }}}
3864 
3865 // split<simd>(simd) {{{
3866 template <typename _V, typename _Ap,
3867  size_t Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()>
3868  enable_if_t<simd_size_v<typename _V::value_type, _Ap> == Parts * _V::size()
3869  && is_simd_v<_V>, array<_V, Parts>>
3870  split(const simd<typename _V::value_type, _Ap>& __x)
3871  {
3872  using _Tp = typename _V::value_type;
3873  if constexpr (Parts == 1)
3874  {
3875  return {simd_cast<_V>(__x)};
3876  }
3877  else if (__x._M_is_constprop())
3878  {
3879  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3880  auto __i) constexpr {
3881  return _V([&](auto __j) constexpr {
3882  return __x[__i * _V::size() + __j];
3883  });
3884  });
3885  }
3886  else if constexpr (
3887  __is_fixed_size_abi_v<_Ap>
3888  && (is_same_v<typename _V::abi_type, simd_abi::scalar>
3889  || (__is_fixed_size_abi_v<typename _V::abi_type>
3890  && sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding
3891  )))
3892  {
3893  // fixed_size -> fixed_size (w/o padding) or scalar
3894 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
3895  const __may_alias<_Tp>* const __element_ptr
3896  = reinterpret_cast<const __may_alias<_Tp>*>(&__data(__x));
3897  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3898  auto __i) constexpr {
3899  return _V(__element_ptr + __i * _V::size(), vector_aligned);
3900  });
3901 #else
3902  const auto& __xx = __data(__x);
3903  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3904  auto __i) constexpr {
3905  [[maybe_unused]] constexpr size_t __offset
3906  = decltype(__i)::value * _V::size();
3907  return _V([&](auto __j) constexpr {
3908  constexpr _SizeConstant<__j + __offset> __k;
3909  return __xx[__k];
3910  });
3911  });
3912 #endif
3913  }
3914  else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>)
3915  {
3916  // normally memcpy should work here as well
3917  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3918  auto __i) constexpr { return __x[__i]; });
3919  }
3920  else
3921  {
3922  return __generate_from_n_evaluations<Parts, array<_V, Parts>>([&](
3923  auto __i) constexpr {
3924  if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>)
3925  return _V([&](auto __j) constexpr {
3926  return __x[__i * _V::size() + __j];
3927  });
3928  else
3929  return _V(__private_init,
3930  __extract_part<decltype(__i)::value, Parts>(__data(__x)));
3931  });
3932  }
3933  }
3934 
3935 // }}}
3936 // split<simd_mask>(simd_mask) {{{
3937 template <typename _V, typename _Ap,
3938  size_t _Parts
3939  = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()>
3940  enable_if_t<is_simd_mask_v<_V> && simd_size_v<typename
3941  _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>>
3942  split(const simd_mask<typename _V::simd_type::value_type, _Ap>& __x)
3943  {
3944  if constexpr (is_same_v<_Ap, typename _V::abi_type>)
3945  return {__x};
3946  else if constexpr (_Parts == 1)
3947  return {__proposed::static_simd_cast<_V>(__x)};
3948  else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>()
3949  && __is_avx_abi<_Ap>())
3950  return {_V(__private_init, __lo128(__data(__x))),
3951  _V(__private_init, __hi128(__data(__x)))};
3952  else if constexpr (_V::size() <= __CHAR_BIT__ * sizeof(_ULLong))
3953  {
3954  const bitset __bits = __x.__to_bitset();
3955  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3956  auto __i) constexpr {
3957  constexpr size_t __offset = __i * _V::size();
3958  return _V(__bitset_init, (__bits >> __offset).to_ullong());
3959  });
3960  }
3961  else
3962  {
3963  return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>([&](
3964  auto __i) constexpr {
3965  constexpr size_t __offset = __i * _V::size();
3966  return _V(
3967  __private_init, [&](auto __j) constexpr {
3968  return __x[__j + __offset];
3969  });
3970  });
3971  }
3972  }
3973 
3974 // }}}
3975 // split<_Sizes...>(simd) {{{
3976 template <size_t... _Sizes, typename _Tp, typename _Ap, typename>
3977  _GLIBCXX_SIMD_ALWAYS_INLINE
3978  tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...>
3979  split(const simd<_Tp, _Ap>& __x)
3980  {
3981  using _SL = _SizeList<_Sizes...>;
3982  using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>;
3983  constexpr size_t _Np = simd_size_v<_Tp, _Ap>;
3984  constexpr size_t _N0 = _SL::template _S_at<0>();
3985  using _V = __deduced_simd<_Tp, _N0>;
3986 
3987  if (__x._M_is_constprop())
3988  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
3989  auto __i) constexpr {
3990  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
3991  constexpr size_t __offset = _SL::_S_before(__i);
3992  return _Vi([&](auto __j) constexpr { return __x[__offset + __j]; });
3993  });
3994  else if constexpr (_Np == _N0)
3995  {
3996  static_assert(sizeof...(_Sizes) == 1);
3997  return {simd_cast<_V>(__x)};
3998  }
3999  else if constexpr // split from fixed_size, such that __x::first.size == _N0
4000  (__is_fixed_size_abi_v<
4001  _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0)
4002  {
4003  static_assert(
4004  !__is_fixed_size_abi_v<typename _V::abi_type>,
4005  "How can <_Tp, _Np> be __a single _SimdTuple entry but __a "
4006  "fixed_size_simd "
4007  "when deduced?");
4008  // extract first and recurse (__split_wrapper is needed to deduce a new
4009  // _Sizes pack)
4010  return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)),
4011  __split_wrapper(_SL::template _S_pop_front<1>(),
4012  __data(__x).second));
4013  }
4014  else if constexpr ((!is_same_v<simd_abi::scalar,
4015  simd_abi::deduce_t<_Tp, _Sizes>> && ...)
4016  && (!__is_fixed_size_abi_v<
4017  simd_abi::deduce_t<_Tp, _Sizes>> && ...))
4018  {
4019  if constexpr (((_Sizes * 2 == _Np) && ...))
4020  return {{__private_init, __extract_part<0, 2>(__data(__x))},
4021  {__private_init, __extract_part<1, 2>(__data(__x))}};
4022  else if constexpr (is_same_v<_SizeList<_Sizes...>,
4023  _SizeList<_Np / 3, _Np / 3, _Np / 3>>)
4024  return {{__private_init, __extract_part<0, 3>(__data(__x))},
4025  {__private_init, __extract_part<1, 3>(__data(__x))},
4026  {__private_init, __extract_part<2, 3>(__data(__x))}};
4027  else if constexpr (is_same_v<_SizeList<_Sizes...>,
4028  _SizeList<2 * _Np / 3, _Np / 3>>)
4029  return {{__private_init, __extract_part<0, 3, 2>(__data(__x))},
4030  {__private_init, __extract_part<2, 3>(__data(__x))}};
4031  else if constexpr (is_same_v<_SizeList<_Sizes...>,
4032  _SizeList<_Np / 3, 2 * _Np / 3>>)
4033  return {{__private_init, __extract_part<0, 3>(__data(__x))},
4034  {__private_init, __extract_part<1, 3, 2>(__data(__x))}};
4035  else if constexpr (is_same_v<_SizeList<_Sizes...>,
4036  _SizeList<_Np / 2, _Np / 4, _Np / 4>>)
4037  return {{__private_init, __extract_part<0, 2>(__data(__x))},
4038  {__private_init, __extract_part<2, 4>(__data(__x))},
4039  {__private_init, __extract_part<3, 4>(__data(__x))}};
4040  else if constexpr (is_same_v<_SizeList<_Sizes...>,
4041  _SizeList<_Np / 4, _Np / 4, _Np / 2>>)
4042  return {{__private_init, __extract_part<0, 4>(__data(__x))},
4043  {__private_init, __extract_part<1, 4>(__data(__x))},
4044  {__private_init, __extract_part<1, 2>(__data(__x))}};
4045  else if constexpr (is_same_v<_SizeList<_Sizes...>,
4046  _SizeList<_Np / 4, _Np / 2, _Np / 4>>)
4047  return {{__private_init, __extract_part<0, 4>(__data(__x))},
4048  {__private_init, __extract_center(__data(__x))},
4049  {__private_init, __extract_part<3, 4>(__data(__x))}};
4050  else if constexpr (((_Sizes * 4 == _Np) && ...))
4051  return {{__private_init, __extract_part<0, 4>(__data(__x))},
4052  {__private_init, __extract_part<1, 4>(__data(__x))},
4053  {__private_init, __extract_part<2, 4>(__data(__x))},
4054  {__private_init, __extract_part<3, 4>(__data(__x))}};
4055  // else fall through
4056  }
4057 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
4058  const __may_alias<_Tp>* const __element_ptr
4059  = reinterpret_cast<const __may_alias<_Tp>*>(&__x);
4060  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
4061  auto __i) constexpr {
4062  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4063  constexpr size_t __offset = _SL::_S_before(__i);
4064  constexpr size_t __base_align = alignof(simd<_Tp, _Ap>);
4065  constexpr size_t __a
4066  = __base_align - ((__offset * sizeof(_Tp)) % __base_align);
4067  constexpr size_t __b = ((__a - 1) & __a) ^ __a;
4068  constexpr size_t __alignment = __b == 0 ? __a : __b;
4069  return _Vi(__element_ptr + __offset, overaligned<__alignment>);
4070  });
4071 #else
4072  return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>([&](
4073  auto __i) constexpr {
4074  using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>;
4075  const auto& __xx = __data(__x);
4076  using _Offset = decltype(_SL::_S_before(__i));
4077  return _Vi([&](auto __j) constexpr {
4078  constexpr _SizeConstant<_Offset::value + __j> __k;
4079  return __xx[__k];
4080  });
4081  });
4082 #endif
4083  }
4084 
4085 // }}}
4086 
4087 // __subscript_in_pack {{{
4088 template <size_t _I, typename _Tp, typename _Ap, typename... _As>
4089  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
4090  __subscript_in_pack(const simd<_Tp, _Ap>& __x, const simd<_Tp, _As>&... __xs)
4091  {
4092  if constexpr (_I < simd_size_v<_Tp, _Ap>)
4093  return __x[_I];
4094  else
4095  return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...);
4096  }
4097 
4098 // }}}
4099 // __store_pack_of_simd {{{
4100 template <typename _Tp, typename _A0, typename... _As>
4101  _GLIBCXX_SIMD_INTRINSIC void
4102  __store_pack_of_simd(char* __mem, const simd<_Tp, _A0>& __x0,
4103  const simd<_Tp, _As>&... __xs)
4104  {
4105  constexpr size_t __n_bytes = sizeof(_Tp) * simd_size_v<_Tp, _A0>;
4106  __builtin_memcpy(__mem, &__data(__x0), __n_bytes);
4107  if constexpr (sizeof...(__xs) > 0)
4108  __store_pack_of_simd(__mem + __n_bytes, __xs...);
4109  }
4110 
4111 // }}}
4112 // concat(simd...) {{{
4113 template <typename _Tp, typename... _As, typename = __detail::__odr_helper>
4114  inline _GLIBCXX_SIMD_CONSTEXPR
4115  simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>
4116  concat(const simd<_Tp, _As>&... __xs)
4117  {
4118  using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>;
4119  if constexpr (sizeof...(__xs) == 1)
4120  return simd_cast<_Rp>(__xs...);
4121  else if ((... && __xs._M_is_constprop()))
4122  return simd<_Tp,
4123  simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>([&](
4124  auto __i) constexpr { return __subscript_in_pack<__i>(__xs...); });
4125  else
4126  {
4127  _Rp __r{};
4128  __store_pack_of_simd(reinterpret_cast<char*>(&__data(__r)), __xs...);
4129  return __r;
4130  }
4131  }
4132 
4133 // }}}
4134 // concat(array<simd>) {{{
4135 template <typename _Tp, typename _Abi, size_t _Np>
4136  _GLIBCXX_SIMD_ALWAYS_INLINE
4137  _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np>
4138  concat(const array<simd<_Tp, _Abi>, _Np>& __x)
4139  {
4140  return __call_with_subscripts<_Np>(__x, [](const auto&... __xs) {
4141  return concat(__xs...);
4142  });
4143  }
4144 
4145 // }}}
4146 
4147 /// @cond undocumented
4148 // _SmartReference {{{
4149 template <typename _Up, typename _Accessor = _Up,
4150  typename _ValueType = typename _Up::value_type>
4151  class _SmartReference
4152  {
4153  friend _Accessor;
4154  int _M_index;
4155  _Up& _M_obj;
4156 
4157  _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType _M_read() const noexcept
4158  {
4159  if constexpr (is_arithmetic_v<_Up>)
4160  return _M_obj;
4161  else
4162  return _M_obj[_M_index];
4163  }
4164 
4165  template <typename _Tp>
4166  _GLIBCXX_SIMD_INTRINSIC constexpr void _M_write(_Tp&& __x) const
4167  { _Accessor::_S_set(_M_obj, _M_index, static_cast<_Tp&&>(__x)); }
4168 
4169  public:
4170  _GLIBCXX_SIMD_INTRINSIC constexpr
4171  _SmartReference(_Up& __o, int __i) noexcept
4172  : _M_index(__i), _M_obj(__o) {}
4173 
4174  using value_type = _ValueType;
4175 
4176  _GLIBCXX_SIMD_INTRINSIC _SmartReference(const _SmartReference&) = delete;
4177 
4178  _GLIBCXX_SIMD_INTRINSIC constexpr operator value_type() const noexcept
4179  { return _M_read(); }
4180 
4181  template <typename _Tp,
4182  typename
4183  = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>>
4184  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator=(_Tp&& __x) &&
4185  {
4186  _M_write(static_cast<_Tp&&>(__x));
4187  return {_M_obj, _M_index};
4188  }
4189 
4190 #define _GLIBCXX_SIMD_OP_(__op) \
4191  template <typename _Tp, \
4192  typename _TT \
4193  = decltype(declval<value_type>() __op declval<_Tp>()), \
4194  typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \
4195  typename = _ValuePreservingOrInt<_TT, value_type>> \
4196  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \
4197  operator __op##=(_Tp&& __x) && \
4198  { \
4199  const value_type& __lhs = _M_read(); \
4200  _M_write(__lhs __op __x); \
4201  return {_M_obj, _M_index}; \
4202  }
4203  _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_);
4204  _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_);
4205  _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_);
4206 #undef _GLIBCXX_SIMD_OP_
4207 
4208  template <typename _Tp = void,
4209  typename
4210  = decltype(++declval<conditional_t<true, value_type, _Tp>&>())>
4211  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator++() &&
4212  {
4213  value_type __x = _M_read();
4214  _M_write(++__x);
4215  return {_M_obj, _M_index};
4216  }
4217 
4218  template <typename _Tp = void,
4219  typename
4220  = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)>
4221  _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator++(int) &&
4222  {
4223  const value_type __r = _M_read();
4224  value_type __x = __r;
4225  _M_write(++__x);
4226  return __r;
4227  }
4228 
4229  template <typename _Tp = void,
4230  typename
4231  = decltype(--declval<conditional_t<true, value_type, _Tp>&>())>
4232  _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference operator--() &&
4233  {
4234  value_type __x = _M_read();
4235  _M_write(--__x);
4236  return {_M_obj, _M_index};
4237  }
4238 
4239  template <typename _Tp = void,
4240  typename
4241  = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)>
4242  _GLIBCXX_SIMD_INTRINSIC constexpr value_type operator--(int) &&
4243  {
4244  const value_type __r = _M_read();
4245  value_type __x = __r;
4246  _M_write(--__x);
4247  return __r;
4248  }
4249 
4250  _GLIBCXX_SIMD_INTRINSIC friend void
4251  swap(_SmartReference&& __a, _SmartReference&& __b) noexcept(
4252  conjunction<
4253  is_nothrow_constructible<value_type, _SmartReference&&>,
4254  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4255  {
4256  value_type __tmp = static_cast<_SmartReference&&>(__a);
4257  static_cast<_SmartReference&&>(__a) = static_cast<value_type>(__b);
4258  static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4259  }
4260 
4261  _GLIBCXX_SIMD_INTRINSIC friend void
4262  swap(value_type& __a, _SmartReference&& __b) noexcept(
4263  conjunction<
4264  is_nothrow_constructible<value_type, value_type&&>,
4265  is_nothrow_assignable<value_type&, value_type&&>,
4266  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4267  {
4268  value_type __tmp(std::move(__a));
4269  __a = static_cast<value_type>(__b);
4270  static_cast<_SmartReference&&>(__b) = std::move(__tmp);
4271  }
4272 
4273  _GLIBCXX_SIMD_INTRINSIC friend void
4274  swap(_SmartReference&& __a, value_type& __b) noexcept(
4275  conjunction<
4276  is_nothrow_constructible<value_type, _SmartReference&&>,
4277  is_nothrow_assignable<value_type&, value_type&&>,
4278  is_nothrow_assignable<_SmartReference&&, value_type&&>>::value)
4279  {
4280  value_type __tmp(__a);
4281  static_cast<_SmartReference&&>(__a) = std::move(__b);
4282  __b = std::move(__tmp);
4283  }
4284  };
4285 
4286 // }}}
4287 // __scalar_abi_wrapper {{{
4288 template <int _Bytes>
4289  struct __scalar_abi_wrapper
4290  {
4291  template <typename _Tp> static constexpr size_t _S_full_size = 1;
4292  template <typename _Tp> static constexpr size_t _S_size = 1;
4293  template <typename _Tp> static constexpr size_t _S_is_partial = false;
4294 
4295  template <typename _Tp, typename _Abi = simd_abi::scalar>
4296  static constexpr bool _S_is_valid_v
4297  = _Abi::template _IsValid<_Tp>::value && sizeof(_Tp) == _Bytes;
4298  };
4299 
4300 // }}}
4301 // __decay_abi metafunction {{{
4302 template <typename _Tp>
4303  struct __decay_abi { using type = _Tp; };
4304 
4305 template <int _Bytes>
4306  struct __decay_abi<__scalar_abi_wrapper<_Bytes>>
4307  { using type = simd_abi::scalar; };
4308 
4309 // }}}
4310 // __find_next_valid_abi metafunction {{{1
4311 // Given an ABI tag A<N>, find an N2 < N such that A<N2>::_S_is_valid_v<_Tp> ==
4312 // true, N2 is a power-of-2, and A<N2>::_S_is_partial<_Tp> is false. Break
4313 // recursion at 2 elements in the resulting ABI tag. In this case
4314 // type::_S_is_valid_v<_Tp> may be false.
4315 template <template <int> class _Abi, int _Bytes, typename _Tp>
4316  struct __find_next_valid_abi
4317  {
4318  static constexpr auto _S_choose()
4319  {
4320  constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2;
4321  using _NextAbi = _Abi<_NextBytes>;
4322  if constexpr (_NextBytes < sizeof(_Tp) * 2) // break recursion
4323  return _Abi<_Bytes>();
4324  else if constexpr (_NextAbi::template _S_is_partial<_Tp> == false
4325  && _NextAbi::template _S_is_valid_v<_Tp>)
4326  return _NextAbi();
4327  else
4328  return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose();
4329  }
4330 
4331  using type = decltype(_S_choose());
4332  };
4333 
4334 template <int _Bytes, typename _Tp>
4335  struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp>
4336  { using type = simd_abi::scalar; };
4337 
4338 // _AbiList {{{1
4339 template <template <int> class...>
4340  struct _AbiList
4341  {
4342  template <typename, int> static constexpr bool _S_has_valid_abi = false;
4343  template <typename, int> using _FirstValidAbi = void;
4344  template <typename, int> using _BestAbi = void;
4345  };
4346 
4347 template <template <int> class _A0, template <int> class... _Rest>
4348  struct _AbiList<_A0, _Rest...>
4349  {
4350  template <typename _Tp, int _Np>
4351  static constexpr bool _S_has_valid_abi
4352  = _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<
4353  _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>;
4354 
4355  template <typename _Tp, int _Np>
4356  using _FirstValidAbi = conditional_t<
4357  _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>,
4358  typename __decay_abi<_A0<sizeof(_Tp) * _Np>>::type,
4359  typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>;
4360 
4361  template <typename _Tp, int _Np>
4362  static constexpr auto _S_determine_best_abi()
4363  {
4364  static_assert(_Np >= 1);
4365  constexpr int _Bytes = sizeof(_Tp) * _Np;
4366  if constexpr (_Np == 1)
4367  return __make_dependent_t<_Tp, simd_abi::scalar>{};
4368  else
4369  {
4370  constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>;
4371  // _A0<_Bytes> is good if:
4372  // 1. The ABI tag is valid for _Tp
4373  // 2. The storage overhead is no more than padding to fill the next
4374  // power-of-2 number of bytes
4375  if constexpr (_A0<_Bytes>::template _S_is_valid_v<
4376  _Tp> && __fullsize / 2 < _Np)
4377  return typename __decay_abi<_A0<_Bytes>>::type{};
4378  else
4379  {
4380  using _Bp =
4381  typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type;
4382  if constexpr (_Bp::template _S_is_valid_v<
4383  _Tp> && _Bp::template _S_size<_Tp> <= _Np)
4384  return _Bp{};
4385  else
4386  return
4387  typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{};
4388  }
4389  }
4390  }
4391 
4392  template <typename _Tp, int _Np>
4393  using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>());
4394  };
4395 
4396 // }}}1
4397 
4398 // the following lists all native ABIs, which makes them accessible to
4399 // simd_abi::deduce and select_best_vector_type_t (for fixed_size). Order
4400 // matters: Whatever comes first has higher priority.
4401 using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin,
4402  __scalar_abi_wrapper>;
4403 
4404 // valid _SimdTraits specialization {{{1
4405 template <typename _Tp, typename _Abi>
4406  struct _SimdTraits<_Tp, _Abi, void_t<typename _Abi::template _IsValid<_Tp>>>
4407  : _Abi::template __traits<_Tp> {};
4408 
4409 // __deduce_impl specializations {{{1
4410 // try all native ABIs (including scalar) first
4411 template <typename _Tp, size_t _Np>
4412  struct __deduce_impl<
4413  _Tp, _Np, enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>>
4414  { using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; };
4415 
4416 // fall back to fixed_size only if scalar and native ABIs don't match
4417 template <typename _Tp, size_t _Np, typename = void>
4418  struct __deduce_fixed_size_fallback {};
4419 
4420 template <typename _Tp, size_t _Np>
4421  struct __deduce_fixed_size_fallback<_Tp, _Np,
4422  enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>>
4423  { using type = simd_abi::fixed_size<_Np>; };
4424 
4425 template <typename _Tp, size_t _Np, typename>
4426  struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _Np> {};
4427 
4428 //}}}1
4429 /// @endcond
4430 
4431 // simd_mask {{{
4432 template <typename _Tp, typename _Abi>
4433  class simd_mask : public _SimdTraits<_Tp, _Abi>::_MaskBase
4434  {
4435  // types, tags, and friends {{{
4436  using _Traits = _SimdTraits<_Tp, _Abi>;
4437  using _MemberType = typename _Traits::_MaskMember;
4438 
4439  // We map all masks with equal element sizeof to a single integer type, the
4440  // one given by __int_for_sizeof_t<_Tp>. This is the approach
4441  // [[gnu::vector_size(N)]] types take as well and it reduces the number of
4442  // template specializations in the implementation classes.
4443  using _Ip = __int_for_sizeof_t<_Tp>;
4444  static constexpr _Ip* _S_type_tag = nullptr;
4445 
4446  friend typename _Traits::_MaskBase;
4447  friend class simd<_Tp, _Abi>; // to construct masks on return
4448  friend typename _Traits::_SimdImpl; // to construct masks on return and
4449  // inspect data on masked operations
4450  public:
4451  using _Impl = typename _Traits::_MaskImpl;
4452  friend _Impl;
4453 
4454  // }}}
4455  // member types {{{
4456  using value_type = bool;
4457  using reference = _SmartReference<_MemberType, _Impl, value_type>;
4458  using simd_type = simd<_Tp, _Abi>;
4459  using abi_type = _Abi;
4460 
4461  // }}}
4462  static constexpr size_t size() // {{{
4463  { return __size_or_zero_v<_Tp, _Abi>; }
4464 
4465  // }}}
4466  // constructors & assignment {{{
4467  simd_mask() = default;
4468  simd_mask(const simd_mask&) = default;
4469  simd_mask(simd_mask&&) = default;
4470  simd_mask& operator=(const simd_mask&) = default;
4471  simd_mask& operator=(simd_mask&&) = default;
4472 
4473  // }}}
4474  // access to internal representation (optional feature) {{{
4475  _GLIBCXX_SIMD_ALWAYS_INLINE explicit
4476  simd_mask(typename _Traits::_MaskCastType __init)
4477  : _M_data{__init} {}
4478  // conversions to internal type is done in _MaskBase
4479 
4480  // }}}
4481  // bitset interface (extension to be proposed) {{{
4482  // TS_FEEDBACK:
4483  // Conversion of simd_mask to and from bitset makes it much easier to
4484  // interface with other facilities. I suggest adding `static
4485  // simd_mask::from_bitset` and `simd_mask::to_bitset`.
4486  _GLIBCXX_SIMD_ALWAYS_INLINE static simd_mask
4487  __from_bitset(bitset<size()> bs)
4488  { return {__bitset_init, bs}; }
4489 
4490  _GLIBCXX_SIMD_ALWAYS_INLINE bitset<size()>
4491  __to_bitset() const
4492  { return _Impl::_S_to_bits(_M_data)._M_to_bitset(); }
4493 
4494  // }}}
4495  // explicit broadcast constructor {{{
4496  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
4497  simd_mask(value_type __x)
4498  : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {}
4499 
4500  // }}}
4501  // implicit type conversion constructor {{{
4502  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4503  // proposed improvement
4504  template <typename _Up, typename _A2,
4505  typename = enable_if_t<simd_size_v<_Up, _A2> == size()>>
4506  _GLIBCXX_SIMD_ALWAYS_INLINE explicit(sizeof(_MemberType)
4507  != sizeof(typename _SimdTraits<_Up, _A2>::_MaskMember))
4508  simd_mask(const simd_mask<_Up, _A2>& __x)
4509  : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {}
4510  #else
4511  // conforming to ISO/IEC 19570:2018
4512  template <typename _Up, typename = enable_if_t<conjunction<
4513  is_same<abi_type, simd_abi::fixed_size<size()>>,
4514  is_same<_Up, _Up>>::value>>
4515  _GLIBCXX_SIMD_ALWAYS_INLINE
4516  simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x)
4517  : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {}
4518  #endif
4519 
4520  // }}}
4521  // load constructor {{{
4522  template <typename _Flags>
4523  _GLIBCXX_SIMD_ALWAYS_INLINE
4524  simd_mask(const value_type* __mem, _Flags)
4525  : _M_data(_Impl::template _S_load<_Ip>(
4526  _Flags::template _S_apply<simd_mask>(__mem))) {}
4527 
4528  template <typename _Flags>
4529  _GLIBCXX_SIMD_ALWAYS_INLINE
4530  simd_mask(const value_type* __mem, simd_mask __k, _Flags)
4531  : _M_data{}
4532  {
4533  _M_data
4534  = _Impl::_S_masked_load(_M_data, __k._M_data,
4535  _Flags::template _S_apply<simd_mask>(__mem));
4536  }
4537 
4538  // }}}
4539  // loads [simd_mask.load] {{{
4540  template <typename _Flags>
4541  _GLIBCXX_SIMD_ALWAYS_INLINE void
4542  copy_from(const value_type* __mem, _Flags)
4543  {
4544  _M_data = _Impl::template _S_load<_Ip>(
4545  _Flags::template _S_apply<simd_mask>(__mem));
4546  }
4547 
4548  // }}}
4549  // stores [simd_mask.store] {{{
4550  template <typename _Flags>
4551  _GLIBCXX_SIMD_ALWAYS_INLINE void
4552  copy_to(value_type* __mem, _Flags) const
4553  { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); }
4554 
4555  // }}}
4556  // scalar access {{{
4557  _GLIBCXX_SIMD_ALWAYS_INLINE reference
4558  operator[](size_t __i)
4559  {
4560  if (__i >= size())
4561  __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4562  return {_M_data, int(__i)};
4563  }
4564 
4565  _GLIBCXX_SIMD_ALWAYS_INLINE value_type
4566  operator[](size_t __i) const
4567  {
4568  if (__i >= size())
4569  __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1);
4570  if constexpr (__is_scalar_abi<_Abi>())
4571  return _M_data;
4572  else
4573  return static_cast<bool>(_M_data[__i]);
4574  }
4575 
4576  // }}}
4577  // negation {{{
4578  _GLIBCXX_SIMD_ALWAYS_INLINE simd_mask
4579  operator!() const
4580  { return {__private_init, _Impl::_S_bit_not(_M_data)}; }
4581 
4582  // }}}
4583  // simd_mask binary operators [simd_mask.binary] {{{
4584  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4585  // simd_mask<int> && simd_mask<uint> needs disambiguation
4586  template <typename _Up, typename _A2,
4587  typename
4588  = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4589  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4590  operator&&(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4591  {
4592  return {__private_init,
4593  _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)};
4594  }
4595 
4596  template <typename _Up, typename _A2,
4597  typename
4598  = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>>
4599  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4600  operator||(const simd_mask& __x, const simd_mask<_Up, _A2>& __y)
4601  {
4602  return {__private_init,
4603  _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)};
4604  }
4605  #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4606 
4607  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4608  operator&&(const simd_mask& __x, const simd_mask& __y)
4609  {
4610  return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)};
4611  }
4612 
4613  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4614  operator||(const simd_mask& __x, const simd_mask& __y)
4615  {
4616  return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)};
4617  }
4618 
4619  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4620  operator&(const simd_mask& __x, const simd_mask& __y)
4621  { return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; }
4622 
4623  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4624  operator|(const simd_mask& __x, const simd_mask& __y)
4625  { return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; }
4626 
4627  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask
4628  operator^(const simd_mask& __x, const simd_mask& __y)
4629  { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4630 
4631  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4632  operator&=(simd_mask& __x, const simd_mask& __y)
4633  {
4634  __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data);
4635  return __x;
4636  }
4637 
4638  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4639  operator|=(simd_mask& __x, const simd_mask& __y)
4640  {
4641  __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data);
4642  return __x;
4643  }
4644 
4645  _GLIBCXX_SIMD_ALWAYS_INLINE friend simd_mask&
4646  operator^=(simd_mask& __x, const simd_mask& __y)
4647  {
4648  __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data);
4649  return __x;
4650  }
4651 
4652  // }}}
4653  // simd_mask compares [simd_mask.comparison] {{{
4654  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4655  operator==(const simd_mask& __x, const simd_mask& __y)
4656  { return !operator!=(__x, __y); }
4657 
4658  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4659  operator!=(const simd_mask& __x, const simd_mask& __y)
4660  { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; }
4661 
4662  // }}}
4663  // private_init ctor {{{
4664  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
4665  simd_mask(_PrivateInit, typename _Traits::_MaskMember __init)
4666  : _M_data(__init) {}
4667 
4668  // }}}
4669  // private_init generator ctor {{{
4670  template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))>
4671  _GLIBCXX_SIMD_INTRINSIC constexpr
4672  simd_mask(_PrivateInit, _Fp&& __gen)
4673  : _M_data()
4674  {
4675  __execute_n_times<size()>([&](auto __i) constexpr {
4676  _Impl::_S_set(_M_data, __i, __gen(__i));
4677  });
4678  }
4679 
4680  // }}}
4681  // bitset_init ctor {{{
4682  _GLIBCXX_SIMD_INTRINSIC simd_mask(_BitsetInit, bitset<size()> __init)
4683  : _M_data(
4684  _Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag))
4685  {}
4686 
4687  // }}}
4688  // __cvt {{{
4689  // TS_FEEDBACK:
4690  // The conversion operator this implements should be a ctor on simd_mask.
4691  // Once you call .__cvt() on a simd_mask it converts conveniently.
4692  // A useful variation: add `explicit(sizeof(_Tp) != sizeof(_Up))`
4693  struct _CvtProxy
4694  {
4695  template <typename _Up, typename _A2,
4696  typename
4697  = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>>
4698  _GLIBCXX_SIMD_ALWAYS_INLINE
4699  operator simd_mask<_Up, _A2>() &&
4700  {
4701  using namespace std::experimental::__proposed;
4702  return static_simd_cast<simd_mask<_Up, _A2>>(_M_data);
4703  }
4704 
4705  const simd_mask<_Tp, _Abi>& _M_data;
4706  };
4707 
4708  _GLIBCXX_SIMD_INTRINSIC _CvtProxy
4709  __cvt() const
4710  { return {*this}; }
4711 
4712  // }}}
4713  // operator?: overloads (suggested extension) {{{
4714  #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
4715  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4716  operator?:(const simd_mask& __k, const simd_mask& __where_true,
4717  const simd_mask& __where_false)
4718  {
4719  auto __ret = __where_false;
4720  _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data);
4721  return __ret;
4722  }
4723 
4724  template <typename _U1, typename _U2,
4725  typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>,
4726  typename = enable_if_t<conjunction_v<
4727  is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>,
4728  is_convertible<simd_mask, typename _Rp::mask_type>>>>
4729  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend _Rp
4730  operator?:(const simd_mask& __k, const _U1& __where_true,
4731  const _U2& __where_false)
4732  {
4733  _Rp __ret = __where_false;
4734  _Rp::_Impl::_S_masked_assign(
4735  __data(static_cast<typename _Rp::mask_type>(__k)), __data(__ret),
4736  __data(static_cast<_Rp>(__where_true)));
4737  return __ret;
4738  }
4739 
4740  #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4741  template <typename _Kp, typename _Ak, typename _Up, typename _Au,
4742  typename = enable_if_t<
4743  conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>,
4744  is_convertible<simd_mask<_Up, _Au>, simd_mask>>>>
4745  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask
4746  operator?:(const simd_mask<_Kp, _Ak>& __k, const simd_mask& __where_true,
4747  const simd_mask<_Up, _Au>& __where_false)
4748  {
4749  simd_mask __ret = __where_false;
4750  _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data,
4751  __where_true._M_data);
4752  return __ret;
4753  }
4754  #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST
4755  #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
4756 
4757  // }}}
4758  // _M_is_constprop {{{
4759  _GLIBCXX_SIMD_INTRINSIC constexpr bool
4760  _M_is_constprop() const
4761  {
4762  if constexpr (__is_scalar_abi<_Abi>())
4763  return __builtin_constant_p(_M_data);
4764  else
4765  return _M_data._M_is_constprop();
4766  }
4767 
4768  // }}}
4769 
4770  private:
4771  friend const auto& __data<_Tp, abi_type>(const simd_mask&);
4772  friend auto& __data<_Tp, abi_type>(simd_mask&);
4773  alignas(_Traits::_S_mask_align) _MemberType _M_data;
4774  };
4775 
4776 // }}}
4777 
4778 /// @cond undocumented
4779 // __data(simd_mask) {{{
4780 template <typename _Tp, typename _Ap>
4781  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
4782  __data(const simd_mask<_Tp, _Ap>& __x)
4783  { return __x._M_data; }
4784 
4785 template <typename _Tp, typename _Ap>
4786  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
4787  __data(simd_mask<_Tp, _Ap>& __x)
4788  { return __x._M_data; }
4789 
4790 // }}}
4791 /// @endcond
4792 
4793 // simd_mask reductions [simd_mask.reductions] {{{
4794 template <typename _Tp, typename _Abi>
4795  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4796  all_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4797  {
4798  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4799  {
4800  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4801  if (!__k[__i])
4802  return false;
4803  return true;
4804  }
4805  else
4806  return _Abi::_MaskImpl::_S_all_of(__k);
4807  }
4808 
4809 template <typename _Tp, typename _Abi>
4810  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4811  any_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4812  {
4813  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4814  {
4815  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4816  if (__k[__i])
4817  return true;
4818  return false;
4819  }
4820  else
4821  return _Abi::_MaskImpl::_S_any_of(__k);
4822  }
4823 
4824 template <typename _Tp, typename _Abi>
4825  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4826  none_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4827  {
4828  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4829  {
4830  for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i)
4831  if (__k[__i])
4832  return false;
4833  return true;
4834  }
4835  else
4836  return _Abi::_MaskImpl::_S_none_of(__k);
4837  }
4838 
4839 template <typename _Tp, typename _Abi>
4840  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4841  some_of(const simd_mask<_Tp, _Abi>& __k) noexcept
4842  {
4843  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4844  {
4845  for (size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i)
4846  if (__k[__i] != __k[__i - 1])
4847  return true;
4848  return false;
4849  }
4850  else
4851  return _Abi::_MaskImpl::_S_some_of(__k);
4852  }
4853 
4854 template <typename _Tp, typename _Abi>
4855  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4856  popcount(const simd_mask<_Tp, _Abi>& __k) noexcept
4857  {
4858  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4859  {
4860  const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>(
4861  __k, [](auto... __elements) { return ((__elements != 0) + ...); });
4862  if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r))
4863  return __r;
4864  }
4865  return _Abi::_MaskImpl::_S_popcount(__k);
4866  }
4867 
4868 template <typename _Tp, typename _Abi>
4869  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4870  find_first_set(const simd_mask<_Tp, _Abi>& __k)
4871  {
4872  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4873  {
4874  constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4875  const size_t _Idx = __call_with_n_evaluations<_Np>(
4876  [](auto... __indexes) { return std::min({__indexes...}); },
4877  [&](auto __i) { return __k[__i] ? +__i : _Np; });
4878  if (_Idx >= _Np)
4879  __invoke_ub("find_first_set(empty mask) is UB");
4880  if (__builtin_constant_p(_Idx))
4881  return _Idx;
4882  }
4883  return _Abi::_MaskImpl::_S_find_first_set(__k);
4884  }
4885 
4886 template <typename _Tp, typename _Abi>
4887  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4888  find_last_set(const simd_mask<_Tp, _Abi>& __k)
4889  {
4890  if (__builtin_is_constant_evaluated() || __k._M_is_constprop())
4891  {
4892  constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
4893  const int _Idx = __call_with_n_evaluations<_Np>(
4894  [](auto... __indexes) { return std::max({__indexes...}); },
4895  [&](auto __i) { return __k[__i] ? int(__i) : -1; });
4896  if (_Idx < 0)
4897  __invoke_ub("find_first_set(empty mask) is UB");
4898  if (__builtin_constant_p(_Idx))
4899  return _Idx;
4900  }
4901  return _Abi::_MaskImpl::_S_find_last_set(__k);
4902  }
4903 
4904 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4905 all_of(_ExactBool __x) noexcept
4906 { return __x; }
4907 
4908 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4909 any_of(_ExactBool __x) noexcept
4910 { return __x; }
4911 
4912 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4913 none_of(_ExactBool __x) noexcept
4914 { return !__x; }
4915 
4916 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool
4917 some_of(_ExactBool) noexcept
4918 { return false; }
4919 
4920 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4921 popcount(_ExactBool __x) noexcept
4922 { return __x; }
4923 
4924 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4925 find_first_set(_ExactBool)
4926 { return 0; }
4927 
4928 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int
4929 find_last_set(_ExactBool)
4930 { return 0; }
4931 
4932 // }}}
4933 
4934 /// @cond undocumented
4935 // _SimdIntOperators{{{1
4936 template <typename _V, typename _Tp, typename _Abi, bool>
4937  class _SimdIntOperators {};
4938 
4939 template <typename _V, typename _Tp, typename _Abi>
4940  class _SimdIntOperators<_V, _Tp, _Abi, true>
4941  {
4942  using _Impl = typename _SimdTraits<_Tp, _Abi>::_SimdImpl;
4943 
4944  _GLIBCXX_SIMD_INTRINSIC const _V& __derived() const
4945  { return *static_cast<const _V*>(this); }
4946 
4947  template <typename _Up>
4948  _GLIBCXX_SIMD_INTRINSIC static _GLIBCXX_SIMD_CONSTEXPR _V
4949  _S_make_derived(_Up&& __d)
4950  { return {__private_init, static_cast<_Up&&>(__d)}; }
4951 
4952  public:
4953  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
4954  _V&
4955  operator%=(_V& __lhs, const _V& __x)
4956  { return __lhs = __lhs % __x; }
4957 
4958  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
4959  _V&
4960  operator&=(_V& __lhs, const _V& __x)
4961  { return __lhs = __lhs & __x; }
4962 
4963  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
4964  _V&
4965  operator|=(_V& __lhs, const _V& __x)
4966  { return __lhs = __lhs | __x; }
4967 
4968  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
4969  _V&
4970  operator^=(_V& __lhs, const _V& __x)
4971  { return __lhs = __lhs ^ __x; }
4972 
4973  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
4974  _V&
4975  operator<<=(_V& __lhs, const _V& __x)
4976  { return __lhs = __lhs << __x; }
4977 
4978  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
4979  _V&
4980  operator>>=(_V& __lhs, const _V& __x)
4981  { return __lhs = __lhs >> __x; }
4982 
4983  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
4984  _V&
4985  operator<<=(_V& __lhs, int __x)
4986  { return __lhs = __lhs << __x; }
4987 
4988  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
4989  _V&
4990  operator>>=(_V& __lhs, int __x)
4991  { return __lhs = __lhs >> __x; }
4992 
4993  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
4994  _V
4995  operator%(const _V& __x, const _V& __y)
4996  {
4997  return _SimdIntOperators::_S_make_derived(
4998  _Impl::_S_modulus(__data(__x), __data(__y)));
4999  }
5000 
5001  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5002  _V
5003  operator&(const _V& __x, const _V& __y)
5004  {
5005  return _SimdIntOperators::_S_make_derived(
5006  _Impl::_S_bit_and(__data(__x), __data(__y)));
5007  }
5008 
5009  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5010  _V
5011  operator|(const _V& __x, const _V& __y)
5012  {
5013  return _SimdIntOperators::_S_make_derived(
5014  _Impl::_S_bit_or(__data(__x), __data(__y)));
5015  }
5016 
5017  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5018  _V
5019  operator^(const _V& __x, const _V& __y)
5020  {
5021  return _SimdIntOperators::_S_make_derived(
5022  _Impl::_S_bit_xor(__data(__x), __data(__y)));
5023  }
5024 
5025  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5026  _V
5027  operator<<(const _V& __x, const _V& __y)
5028  {
5029  return _SimdIntOperators::_S_make_derived(
5030  _Impl::_S_bit_shift_left(__data(__x), __data(__y)));
5031  }
5032 
5033  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5034  _V
5035  operator>>(const _V& __x, const _V& __y)
5036  {
5037  return _SimdIntOperators::_S_make_derived(
5038  _Impl::_S_bit_shift_right(__data(__x), __data(__y)));
5039  }
5040 
5041  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5042  _V
5043  operator<<(const _V& __x, int __y)
5044  {
5045  if (__y < 0)
5046  __invoke_ub("The behavior is undefined if the right operand of a "
5047  "shift operation is negative. [expr.shift]\nA shift by "
5048  "%d was requested",
5049  __y);
5050  if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5051  __invoke_ub(
5052  "The behavior is undefined if the right operand of a "
5053  "shift operation is greater than or equal to the width of the "
5054  "promoted left operand. [expr.shift]\nA shift by %d was requested",
5055  __y);
5056  return _SimdIntOperators::_S_make_derived(
5057  _Impl::_S_bit_shift_left(__data(__x), __y));
5058  }
5059 
5060  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend
5061  _V
5062  operator>>(const _V& __x, int __y)
5063  {
5064  if (__y < 0)
5065  __invoke_ub(
5066  "The behavior is undefined if the right operand of a shift "
5067  "operation is negative. [expr.shift]\nA shift by %d was requested",
5068  __y);
5069  if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__)
5070  __invoke_ub(
5071  "The behavior is undefined if the right operand of a shift "
5072  "operation is greater than or equal to the width of the promoted "
5073  "left operand. [expr.shift]\nA shift by %d was requested",
5074  __y);
5075  return _SimdIntOperators::_S_make_derived(
5076  _Impl::_S_bit_shift_right(__data(__x), __y));
5077  }
5078 
5079  // unary operators (for integral _Tp)
5080  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5081  _V
5082  operator~() const
5083  { return {__private_init, _Impl::_S_complement(__derived()._M_data)}; }
5084  };
5085 
5086 //}}}1
5087 /// @endcond
5088 
5089 // simd {{{
5090 template <typename _Tp, typename _Abi>
5091  class simd : public _SimdIntOperators<
5092  simd<_Tp, _Abi>, _Tp, _Abi,
5093  conjunction<is_integral<_Tp>,
5094  typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>,
5095  public _SimdTraits<_Tp, _Abi>::_SimdBase
5096  {
5097  using _Traits = _SimdTraits<_Tp, _Abi>;
5098  using _MemberType = typename _Traits::_SimdMember;
5099  using _CastType = typename _Traits::_SimdCastType;
5100  static constexpr _Tp* _S_type_tag = nullptr;
5101  friend typename _Traits::_SimdBase;
5102 
5103  public:
5104  using _Impl = typename _Traits::_SimdImpl;
5105  friend _Impl;
5106  friend _SimdIntOperators<simd, _Tp, _Abi, true>;
5107 
5108  using value_type = _Tp;
5109  using reference = _SmartReference<_MemberType, _Impl, value_type>;
5110  using mask_type = simd_mask<_Tp, _Abi>;
5111  using abi_type = _Abi;
5112 
5113  static constexpr size_t size()
5114  { return __size_or_zero_v<_Tp, _Abi>; }
5115 
5116  _GLIBCXX_SIMD_CONSTEXPR simd() = default;
5117  _GLIBCXX_SIMD_CONSTEXPR simd(const simd&) = default;
5118  _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default;
5119  _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default;
5120  _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default;
5121 
5122  // implicit broadcast constructor
5123  template <typename _Up,
5124  typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>, bool>>>
5125  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5126  simd(_ValuePreservingOrInt<_Up, value_type>&& __x)
5127  : _M_data(
5128  _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x))))
5129  {}
5130 
5131  // implicit type conversion constructor (convert from fixed_size to
5132  // fixed_size)
5133  template <typename _Up>
5134  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR
5135  simd(const simd<_Up, simd_abi::fixed_size<size()>>& __x,
5136  enable_if_t<
5137  conjunction<
5138  is_same<simd_abi::fixed_size<size()>, abi_type>,
5139  negation<__is_narrowing_conversion<_Up, value_type>>,
5140  __converts_to_higher_integer_rank<_Up, value_type>>::value,
5141  void*> = nullptr)
5142  : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {}
5143 
5144  // explicit type conversion constructor
5145 #ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5146  template <typename _Up, typename _A2,
5147  typename = decltype(static_simd_cast<simd>(
5148  declval<const simd<_Up, _A2>&>()))>
5149  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5150  simd(const simd<_Up, _A2>& __x)
5151  : simd(static_simd_cast<simd>(__x)) {}
5152 #endif // _GLIBCXX_SIMD_ENABLE_STATIC_CAST
5153 
5154  // generator constructor
5155  template <typename _Fp>
5156  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5157  simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()(
5158  declval<_SizeConstant<0>&>())),
5159  value_type>* = nullptr)
5160  : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {}
5161 
5162  // load constructor
5163  template <typename _Up, typename _Flags>
5164  _GLIBCXX_SIMD_ALWAYS_INLINE
5165  simd(const _Up* __mem, _Flags)
5166  : _M_data(
5167  _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag))
5168  {}
5169 
5170  // loads [simd.load]
5171  template <typename _Up, typename _Flags>
5172  _GLIBCXX_SIMD_ALWAYS_INLINE void
5173  copy_from(const _Vectorizable<_Up>* __mem, _Flags)
5174  {
5175  _M_data = static_cast<decltype(_M_data)>(
5176  _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag));
5177  }
5178 
5179  // stores [simd.store]
5180  template <typename _Up, typename _Flags>
5181  _GLIBCXX_SIMD_ALWAYS_INLINE void
5182  copy_to(_Vectorizable<_Up>* __mem, _Flags) const
5183  {
5184  _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem),
5185  _S_type_tag);
5186  }
5187 
5188  // scalar access
5189  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference
5190  operator[](size_t __i)
5191  { return {_M_data, int(__i)}; }
5192 
5193  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type
5194  operator[]([[maybe_unused]] size_t __i) const
5195  {
5196  if constexpr (__is_scalar_abi<_Abi>())
5197  {
5198  _GLIBCXX_DEBUG_ASSERT(__i == 0);
5199  return _M_data;
5200  }
5201  else
5202  return _M_data[__i];
5203  }
5204 
5205  // increment and decrement:
5206  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5207  operator++()
5208  {
5209  _Impl::_S_increment(_M_data);
5210  return *this;
5211  }
5212 
5213  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5214  operator++(int)
5215  {
5216  simd __r = *this;
5217  _Impl::_S_increment(_M_data);
5218  return __r;
5219  }
5220 
5221  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd&
5222  operator--()
5223  {
5224  _Impl::_S_decrement(_M_data);
5225  return *this;
5226  }
5227 
5228  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5229  operator--(int)
5230  {
5231  simd __r = *this;
5232  _Impl::_S_decrement(_M_data);
5233  return __r;
5234  }
5235 
5236  // unary operators (for any _Tp)
5237  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type
5238  operator!() const
5239  { return {__private_init, _Impl::_S_negate(_M_data)}; }
5240 
5241  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5242  operator+() const
5243  { return *this; }
5244 
5245  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd
5246  operator-() const
5247  { return {__private_init, _Impl::_S_unary_minus(_M_data)}; }
5248 
5249  // access to internal representation (suggested extension)
5250  _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR
5251  simd(_CastType __init) : _M_data(__init) {}
5252 
5253  // compound assignment [simd.cassign]
5254  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5255  operator+=(simd& __lhs, const simd& __x)
5256  { return __lhs = __lhs + __x; }
5257 
5258  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5259  operator-=(simd& __lhs, const simd& __x)
5260  { return __lhs = __lhs - __x; }
5261 
5262  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5263  operator*=(simd& __lhs, const simd& __x)
5264  { return __lhs = __lhs * __x; }
5265 
5266  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd&
5267  operator/=(simd& __lhs, const simd& __x)
5268  { return __lhs = __lhs / __x; }
5269 
5270  // binary operators [simd.binary]
5271  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5272  operator+(const simd& __x, const simd& __y)
5273  { return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; }
5274 
5275  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5276  operator-(const simd& __x, const simd& __y)
5277  { return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; }
5278 
5279  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5280  operator*(const simd& __x, const simd& __y)
5281  { return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; }
5282 
5283  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5284  operator/(const simd& __x, const simd& __y)
5285  { return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; }
5286 
5287  // compares [simd.comparison]
5288  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5289  operator==(const simd& __x, const simd& __y)
5290  { return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); }
5291 
5292  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5293  operator!=(const simd& __x, const simd& __y)
5294  {
5295  return simd::_S_make_mask(
5296  _Impl::_S_not_equal_to(__x._M_data, __y._M_data));
5297  }
5298 
5299  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5300  operator<(const simd& __x, const simd& __y)
5301  { return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); }
5302 
5303  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5304  operator<=(const simd& __x, const simd& __y)
5305  {
5306  return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data));
5307  }
5308 
5309  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5310  operator>(const simd& __x, const simd& __y)
5311  { return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); }
5312 
5313  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type
5314  operator>=(const simd& __x, const simd& __y)
5315  {
5316  return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data));
5317  }
5318 
5319  // operator?: overloads (suggested extension) {{{
5320 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__
5321  _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd
5322  operator?:(const mask_type& __k, const simd& __where_true,
5323  const simd& __where_false)
5324  {
5325  auto __ret = __where_false;
5326  _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true));
5327  return __ret;
5328  }
5329 
5330 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__
5331  // }}}
5332 
5333  // "private" because of the first arguments's namespace
5334  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5335  simd(_PrivateInit, const _MemberType& __init)
5336  : _M_data(__init) {}
5337 
5338  // "private" because of the first arguments's namespace
5339  _GLIBCXX_SIMD_INTRINSIC
5340  simd(_BitsetInit, bitset<size()> __init) : _M_data()
5341  { where(mask_type(__bitset_init, __init), *this) = ~*this; }
5342 
5343  _GLIBCXX_SIMD_INTRINSIC constexpr bool
5344  _M_is_constprop() const
5345  {
5346  if constexpr (__is_scalar_abi<_Abi>())
5347  return __builtin_constant_p(_M_data);
5348  else
5349  return _M_data._M_is_constprop();
5350  }
5351 
5352  private:
5353  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR static mask_type
5354  _S_make_mask(typename mask_type::_MemberType __k)
5355  { return {__private_init, __k}; }
5356 
5357  friend const auto& __data<value_type, abi_type>(const simd&);
5358  friend auto& __data<value_type, abi_type>(simd&);
5359  alignas(_Traits::_S_simd_align) _MemberType _M_data;
5360  };
5361 
5362 // }}}
5363 /// @cond undocumented
5364 // __data {{{
5365 template <typename _Tp, typename _Ap>
5366  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
5367  __data(const simd<_Tp, _Ap>& __x)
5368  { return __x._M_data; }
5369 
5370 template <typename _Tp, typename _Ap>
5371  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
5372  __data(simd<_Tp, _Ap>& __x)
5373  { return __x._M_data; }
5374 
5375 // }}}
5376 namespace __float_bitwise_operators { //{{{
5377 template <typename _Tp, typename _Ap>
5378  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5379  operator^(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5380  {
5381  return {__private_init,
5382  _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))};
5383  }
5384 
5385 template <typename _Tp, typename _Ap>
5386  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5387  operator|(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5388  {
5389  return {__private_init,
5390  _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))};
5391  }
5392 
5393 template <typename _Tp, typename _Ap>
5394  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap>
5395  operator&(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b)
5396  {
5397  return {__private_init,
5398  _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))};
5399  }
5400 
5401 template <typename _Tp, typename _Ap>
5402  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR
5403  enable_if_t<is_floating_point_v<_Tp>, simd<_Tp, _Ap>>
5404  operator~(const simd<_Tp, _Ap>& __a)
5405  { return {__private_init, _Ap::_SimdImpl::_S_complement(__data(__a))}; }
5406 } // namespace __float_bitwise_operators }}}
5407 /// @endcond
5408 
5409 /// @}
5410 _GLIBCXX_SIMD_END_NAMESPACE
5411 
5412 #endif // __cplusplus >= 201703L
5413 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_H
5414 
5415 // vim: foldmethod=marker foldmarker={{{,}}}
constexpr _If_is_unsigned_integer< _Tp, int > popcount(_Tp __x) noexcept
The number of bits set in x.
Definition: bit:410
constexpr complex< _Tp > operator*(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x times y.
Definition: complex:392
constexpr complex< _Tp > operator/(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x divided by y.
Definition: complex:422
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
Definition: complex:362
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
Definition: complex:332
typename remove_reference< _Tp >::type remove_reference_t
Alias template for remove_reference.
Definition: type_traits:1670
typename make_unsigned< _Tp >::type make_unsigned_t
Alias template for make_unsigned.
Definition: type_traits:2009
void void_t
A metafunction that always yields void, used for detecting valid types.
Definition: type_traits:2636
integral_constant< bool, true > true_type
The type used as a compile-time boolean with true value.
Definition: type_traits:82
typename conditional< _Cond, _Iftrue, _Iffalse >::type conditional_t
Alias template for conditional.
Definition: type_traits:2618
typename remove_pointer< _Tp >::type remove_pointer_t
Alias template for remove_pointer.
Definition: type_traits:2084
integral_constant< bool, false > false_type
The type used as a compile-time boolean with false value.
Definition: type_traits:85
typename remove_const< _Tp >::type remove_const_t
Alias template for remove_const.
Definition: type_traits:1601
typename enable_if< _Cond, _Tp >::type enable_if_t
Alias template for enable_if.
Definition: type_traits:2614
constexpr auto tuple_cat(_Tpls &&... __tpls) -> typename __tuple_cat_result< _Tpls... >::__type
tuple_cat
Definition: tuple:1730
auto declval() noexcept -> decltype(__declval< _Tp >(0))
Definition: type_traits:2393
constexpr std::remove_reference< _Tp >::type && move(_Tp &&__t) noexcept
Convert a value to an rvalue.
Definition: move.h:104
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:254
constexpr const _Tp & min(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:230
constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
Calculate reduction of values in a range.
Definition: numeric:278
std::basic_istream< _CharT, _Traits > & operator>>(std::basic_istream< _CharT, _Traits > &__is, bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1472
bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1453
std::basic_ostream< _CharT, _Traits > & operator<<(std::basic_ostream< _CharT, _Traits > &__os, const bitset< _Nb > &__x)
Global I/O operators for bitsets.
Definition: bitset:1540
bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1444
bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1435