Intel(R) Threading Building Blocks Doxygen Documentation  version 4.2.3
atomic.h
Go to the documentation of this file.
1 /*
2  Copyright (c) 2005-2019 Intel Corporation
3 
4  Licensed under the Apache License, Version 2.0 (the "License");
5  you may not use this file except in compliance with the License.
6  You may obtain a copy of the License at
7 
8  http://www.apache.org/licenses/LICENSE-2.0
9 
10  Unless required by applicable law or agreed to in writing, software
11  distributed under the License is distributed on an "AS IS" BASIS,
12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  See the License for the specific language governing permissions and
14  limitations under the License.
15 
16 
17 
18 
19 */
20 
21 #ifndef __TBB_atomic_H
22 #define __TBB_atomic_H
23 
24 #include <cstddef>
25 
26 #if _MSC_VER
27 #define __TBB_LONG_LONG __int64
28 #else
29 #define __TBB_LONG_LONG long long
30 #endif /* _MSC_VER */
31 
32 #include "tbb_machine.h"
33 
34 #if _MSC_VER && !__INTEL_COMPILER
35  // Suppress overzealous compiler warnings till the end of the file
36  #pragma warning (push)
37  #pragma warning (disable: 4244 4267 4512)
38 #endif
39 
40 namespace tbb {
41 
52 };
53 
55 namespace internal {
56 
57 #if __TBB_ALIGNAS_PRESENT
58  #define __TBB_DECL_ATOMIC_FIELD(t,f,a) alignas(a) t f;
59 #elif __TBB_ATTRIBUTE_ALIGNED_PRESENT
60  #define __TBB_DECL_ATOMIC_FIELD(t,f,a) t f __attribute__ ((aligned(a)));
61 #elif __TBB_DECLSPEC_ALIGN_PRESENT
62  #define __TBB_DECL_ATOMIC_FIELD(t,f,a) __declspec(align(a)) t f;
63 #else
64  #error Do not know syntax for forcing alignment.
65 #endif
66 
67 template<size_t S>
68 struct atomic_rep; // Primary template declared, but never defined.
69 
70 template<>
71 struct atomic_rep<1> { // Specialization
72  typedef int8_t word;
73 };
74 template<>
75 struct atomic_rep<2> { // Specialization
76  typedef int16_t word;
77 };
78 template<>
79 struct atomic_rep<4> { // Specialization
80 #if _MSC_VER && !_WIN64
81  // Work-around that avoids spurious /Wp64 warnings
82  typedef intptr_t word;
83 #else
84  typedef int32_t word;
85 #endif
86 };
87 #if __TBB_64BIT_ATOMICS
88 template<>
89 struct atomic_rep<8> { // Specialization
90  typedef int64_t word;
91 };
92 #endif
93 
94 template<typename value_type, size_t size>
96 
97 //the specializations are needed to please MSVC syntax of __declspec(align()) which accept _literal_ constants only
98 #if __TBB_ATOMIC_CTORS
99  #define ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(S) \
100  template<typename value_type> \
101  struct aligned_storage<value_type,S> { \
102  __TBB_DECL_ATOMIC_FIELD(value_type,my_value,S) \
103  aligned_storage() = default ; \
104  constexpr aligned_storage(value_type value):my_value(value){} \
105  }; \
106 
107 #else
108  #define ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(S) \
109  template<typename value_type> \
110  struct aligned_storage<value_type,S> { \
111  __TBB_DECL_ATOMIC_FIELD(value_type,my_value,S) \
112  }; \
113 
114 #endif
115 
116 template<typename value_type>
117 struct aligned_storage<value_type,1> {
118  value_type my_value;
119 #if __TBB_ATOMIC_CTORS
120  aligned_storage() = default ;
121  constexpr aligned_storage(value_type value):my_value(value){}
122 #endif
123 };
124 
127 #if __TBB_64BIT_ATOMICS
129 #endif
130 
131 template<size_t Size, memory_semantics M>
132 struct atomic_traits; // Primary template declared, but not defined.
133 
134 #define __TBB_DECL_FENCED_ATOMIC_PRIMITIVES(S,M) \
135  template<> struct atomic_traits<S,M> { \
136  typedef atomic_rep<S>::word word; \
137  inline static word compare_and_swap( volatile void* location, word new_value, word comparand ) { \
138  return __TBB_machine_cmpswp##S##M(location,new_value,comparand); \
139  } \
140  inline static word fetch_and_add( volatile void* location, word addend ) { \
141  return __TBB_machine_fetchadd##S##M(location,addend); \
142  } \
143  inline static word fetch_and_store( volatile void* location, word value ) { \
144  return __TBB_machine_fetchstore##S##M(location,value); \
145  } \
146  };
147 
148 #define __TBB_DECL_ATOMIC_PRIMITIVES(S) \
149  template<memory_semantics M> \
150  struct atomic_traits<S,M> { \
151  typedef atomic_rep<S>::word word; \
152  inline static word compare_and_swap( volatile void* location, word new_value, word comparand ) { \
153  return __TBB_machine_cmpswp##S(location,new_value,comparand); \
154  } \
155  inline static word fetch_and_add( volatile void* location, word addend ) { \
156  return __TBB_machine_fetchadd##S(location,addend); \
157  } \
158  inline static word fetch_and_store( volatile void* location, word value ) { \
159  return __TBB_machine_fetchstore##S(location,value); \
160  } \
161  };
162 
163 template<memory_semantics M>
164 struct atomic_load_store_traits; // Primary template declaration
165 
166 #define __TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(M) \
167  template<> struct atomic_load_store_traits<M> { \
168  template <typename T> \
169  inline static T load( const volatile T& location ) { \
170  return __TBB_load_##M( location ); \
171  } \
172  template <typename T> \
173  inline static void store( volatile T& location, T value ) { \
174  __TBB_store_##M( location, value ); \
175  } \
176  }
177 
178 #if __TBB_USE_FENCED_ATOMICS
191 #if __TBB_64BIT_ATOMICS
196 #endif
197 #else /* !__TBB_USE_FENCED_ATOMICS */
201 #if __TBB_64BIT_ATOMICS
203 #endif
204 #endif /* !__TBB_USE_FENCED_ATOMICS */
205 
210 
212 
214 #define __TBB_MINUS_ONE(T) (T(T(0)-T(1)))
215 
217 
219 template<typename T>
220 struct atomic_impl {
221 protected:
223 private:
224  //TODO: rechecks on recent versions of gcc if union is still the _only_ way to do a conversion without warnings
226  template<typename value_type>
227  union converter {
228  typedef typename atomic_rep<sizeof(value_type)>::word bits_type;
230  converter(value_type a_value) : value(a_value) {}
233  };
234 
235  template<typename value_t>
236  static typename converter<value_t>::bits_type to_bits(value_t value){
237  return converter<value_t>(value).bits;
238  }
239  template<typename value_t>
240  static value_t to_value(typename converter<value_t>::bits_type bits){
241  converter<value_t> u;
242  u.bits = bits;
243  return u.value;
244  }
245 
246  template<typename value_t>
247  union ptr_converter; //Primary template declared, but never defined.
248 
249  template<typename value_t>
250  union ptr_converter<value_t *> {
252  ptr_converter(value_t* a_value) : value(a_value) {}
253  value_t* value;
254  uintptr_t bits;
255  };
256  //TODO: check if making to_bits accepting reference (thus unifying it with to_bits_ref)
257  //does not hurt performance
258  template<typename value_t>
259  static typename converter<value_t>::bits_type & to_bits_ref(value_t& value){
260  //TODO: this #ifdef is temporary workaround, as union conversion seems to fail
261  //on suncc for 64 bit types for 32 bit target
262  #if !__SUNPRO_CC
263  return *(typename converter<value_t>::bits_type*)ptr_converter<value_t*>(&value).bits;
264  #else
265  return *(typename converter<value_t>::bits_type*)(&value);
266  #endif
267  }
268 
269 
270 public:
271  typedef T value_type;
272 
273 #if __TBB_ATOMIC_CTORS
274  atomic_impl() = default ;
276 #endif
277  template<memory_semantics M>
279  return to_value<value_type>(
280  internal::atomic_traits<sizeof(value_type),M>::fetch_and_store( &my_storage.my_value, to_bits(value) )
281  );
282  }
283 
285  return fetch_and_store<full_fence>(value);
286  }
287 
288  template<memory_semantics M>
290  return to_value<value_type>(
291  internal::atomic_traits<sizeof(value_type),M>::compare_and_swap( &my_storage.my_value, to_bits(value), to_bits(comparand) )
292  );
293  }
294 
296  return compare_and_swap<full_fence>(value,comparand);
297  }
298 
299  operator value_type() const volatile { // volatile qualifier here for backwards compatibility
300  return to_value<value_type>(
302  );
303  }
304 
305  template<memory_semantics M>
306  value_type load () const {
307  return to_value<value_type>(
309  );
310  }
311 
312  value_type load () const {
313  return load<acquire>();
314  }
315 
316  template<memory_semantics M>
317  void store ( value_type value ) {
319  }
320 
321  void store ( value_type value ) {
322  store<release>( value );
323  }
324 
325 protected:
327  //TODO: unify with store<release>
329  return rhs;
330  }
331 };
332 
334 
337 template<typename I, typename D, typename StepType>
339 public:
340  typedef I value_type;
341 #if __TBB_ATOMIC_CTORS
342  atomic_impl_with_arithmetic() = default ;
344 #endif
345  template<memory_semantics M>
346  value_type fetch_and_add( D addend ) {
347  return value_type(internal::atomic_traits<sizeof(value_type),M>::fetch_and_add( &this->my_storage.my_value, addend*sizeof(StepType) ));
348  }
349 
350  value_type fetch_and_add( D addend ) {
351  return fetch_and_add<full_fence>(addend);
352  }
353 
354  template<memory_semantics M>
356  return fetch_and_add<M>(1);
357  }
358 
360  return fetch_and_add(1);
361  }
362 
363  template<memory_semantics M>
365  return fetch_and_add<M>(__TBB_MINUS_ONE(D));
366  }
367 
369  return fetch_and_add(__TBB_MINUS_ONE(D));
370  }
371 
372 public:
374  return fetch_and_add(value)+value;
375  }
376 
378  // Additive inverse of value computed using binary minus,
379  // instead of unary minus, for sake of avoiding compiler warnings.
380  return operator+=(D(0)-value);
381  }
382 
384  return fetch_and_add(1)+1;
385  }
386 
388  return fetch_and_add(__TBB_MINUS_ONE(D))-1;
389  }
390 
392  return fetch_and_add(1);
393  }
394 
396  return fetch_and_add(__TBB_MINUS_ONE(D));
397  }
398 };
399 
400 } /* Internal */
402 
404 
406 template<typename T>
407 struct atomic: internal::atomic_impl<T> {
408 #if __TBB_ATOMIC_CTORS
409  atomic() = default;
410  constexpr atomic(T arg): internal::atomic_impl<T>(arg) {}
411 #endif
412  T operator=( T rhs ) {
413  // "this" required here in strict ISO C++ because store_with_release is a dependent name
414  return this->store_with_release(rhs);
415  }
416  atomic<T>& operator=( const atomic<T>& rhs ) {this->store_with_release(rhs); return *this;}
417 };
418 
419 #if __TBB_ATOMIC_CTORS
420  #define __TBB_DECL_ATOMIC(T) \
421  template<> struct atomic<T>: internal::atomic_impl_with_arithmetic<T,T,char> { \
422  atomic() = default; \
423  constexpr atomic(T arg): internal::atomic_impl_with_arithmetic<T,T,char>(arg) {} \
424  \
425  T operator=( T rhs ) {return store_with_release(rhs);} \
426  atomic<T>& operator=( const atomic<T>& rhs ) {store_with_release(rhs); return *this;} \
427  };
428 #else
429  #define __TBB_DECL_ATOMIC(T) \
430  template<> struct atomic<T>: internal::atomic_impl_with_arithmetic<T,T,char> { \
431  T operator=( T rhs ) {return store_with_release(rhs);} \
432  atomic<T>& operator=( const atomic<T>& rhs ) {store_with_release(rhs); return *this;} \
433  };
434 #endif
435 
436 #if __TBB_64BIT_ATOMICS
437 //TODO: consider adding non-default (and atomic) copy constructor for 32bit platform
440 #else
441 // test_atomic will verify that sizeof(long long)==8
442 #endif
443 __TBB_DECL_ATOMIC(long)
444 __TBB_DECL_ATOMIC(unsigned long)
445 
446 #if _MSC_VER && !_WIN64
447 #if __TBB_ATOMIC_CTORS
448 /* Special version of __TBB_DECL_ATOMIC that avoids gratuitous warnings from cl /Wp64 option.
449  It is identical to __TBB_DECL_ATOMIC(unsigned) except that it replaces operator=(T)
450  with an operator=(U) that explicitly converts the U to a T. Types T and U should be
451  type synonyms on the platform. Type U should be the wider variant of T from the
452  perspective of /Wp64. */
453 #define __TBB_DECL_ATOMIC_ALT(T,U) \
454  template<> struct atomic<T>: internal::atomic_impl_with_arithmetic<T,T,char> { \
455  atomic() = default ; \
456  constexpr atomic(T arg): internal::atomic_impl_with_arithmetic<T,T,char>(arg) {} \
457  T operator=( U rhs ) {return store_with_release(T(rhs));} \
458  atomic<T>& operator=( const atomic<T>& rhs ) {store_with_release(rhs); return *this;} \
459  };
460 #else
461 #define __TBB_DECL_ATOMIC_ALT(T,U) \
462  template<> struct atomic<T>: internal::atomic_impl_with_arithmetic<T,T,char> { \
463  T operator=( U rhs ) {return store_with_release(T(rhs));} \
464  atomic<T>& operator=( const atomic<T>& rhs ) {store_with_release(rhs); return *this;} \
465  };
466 #endif
467 __TBB_DECL_ATOMIC_ALT(unsigned,size_t)
468 __TBB_DECL_ATOMIC_ALT(int,ptrdiff_t)
469 #else
470 __TBB_DECL_ATOMIC(unsigned)
472 #endif /* _MSC_VER && !_WIN64 */
473 
474 __TBB_DECL_ATOMIC(unsigned short)
475 __TBB_DECL_ATOMIC(short)
476 __TBB_DECL_ATOMIC(char)
477 __TBB_DECL_ATOMIC(signed char)
478 __TBB_DECL_ATOMIC(unsigned char)
479 
480 #if !_MSC_VER || defined(_NATIVE_WCHAR_T_DEFINED)
481 __TBB_DECL_ATOMIC(wchar_t)
482 #endif /* _MSC_VER||!defined(_NATIVE_WCHAR_T_DEFINED) */
483 
485 template<typename T> struct atomic<T*>: internal::atomic_impl_with_arithmetic<T*,ptrdiff_t,T> {
486 #if __TBB_ATOMIC_CTORS
487  atomic() = default ;
488  constexpr atomic(T* arg): internal::atomic_impl_with_arithmetic<T*,ptrdiff_t,T>(arg) {}
489 #endif
490  T* operator=( T* rhs ) {
491  // "this" required here in strict ISO C++ because store_with_release is a dependent name
492  return this->store_with_release(rhs);
493  }
494  atomic<T*>& operator=( const atomic<T*>& rhs ) {
495  this->store_with_release(rhs); return *this;
496  }
497  T* operator->() const {
498  return (*this);
499  }
500 };
501 
503 template<> struct atomic<void*>: internal::atomic_impl<void*> {
504 #if __TBB_ATOMIC_CTORS
505  atomic() = default ;
506  constexpr atomic(void* arg): internal::atomic_impl<void*>(arg) {}
507 #endif
508  void* operator=( void* rhs ) {
509  // "this" required here in strict ISO C++ because store_with_release is a dependent name
510  return this->store_with_release(rhs);
511  }
513  this->store_with_release(rhs); return *this;
514  }
515 };
516 
517 // Helpers to workaround ugly syntax of calling template member function of a
518 // template class with template argument dependent on template parameters.
519 
520 template <memory_semantics M, typename T>
521 T load ( const atomic<T>& a ) { return a.template load<M>(); }
522 
523 template <memory_semantics M, typename T>
524 void store ( atomic<T>& a, T value ) { a.template store<M>(value); }
525 
526 namespace interface6{
528 template<typename T>
530  atomic<T> a;
531  store<relaxed>(a,t);
532  return a;
533 }
534 }
536 
537 namespace internal {
538 template<memory_semantics M, typename T >
539 void swap(atomic<T> & lhs, atomic<T> & rhs){
540  T tmp = load<M>(lhs);
541  store<M>(lhs,load<M>(rhs));
542  store<M>(rhs,tmp);
543 }
544 
545 // only to aid in the gradual conversion of ordinary variables to proper atomics
546 template<typename T>
547 inline atomic<T>& as_atomic( T& t ) {
548  return (atomic<T>&)t;
549 }
550 } // namespace tbb::internal
551 
552 } // namespace tbb
553 
554 #if _MSC_VER && !__INTEL_COMPILER
555  #pragma warning (pop)
556 #endif // warnings are restored
557 
558 #endif /* __TBB_atomic_H */
Specialization for atomic<void*>, for sake of not allowing arithmetic or operator->.
Definition: atomic.h:503
Sequential consistency.
Definition: atomic.h:45
Base class that provides basic functionality for atomic<T> with fetch_and_add.
Definition: atomic.h:338
Union type used to convert type T to underlying integral type.
Definition: atomic.h:227
T * operator->() const
Definition: atomic.h:497
No ordering.
Definition: atomic.h:51
T __TBB_load_with_acquire(const volatile T &location)
Definition: tbb_machine.h:713
constexpr atomic(T *arg)
Definition: atomic.h:488
void * operator=(void *rhs)
Definition: atomic.h:508
static converter< value_t >::bits_type to_bits(value_t value)
Definition: atomic.h:236
atomic_rep< sizeof(value_type)>::word bits_type
Definition: atomic.h:228
Specialization for atomic<T*> with arithmetic and operator->.
Definition: atomic.h:485
atomic< T > make_atomic(T t)
Make an atomic for use in an initialization (list), as an alternative to zero-initialization or norma...
Definition: atomic.h:529
T operator=(T rhs)
Definition: atomic.h:412
#define ATOMIC_STORAGE_PARTIAL_SPECIALIZATION(S)
Definition: atomic.h:99
atomic< void * > & operator=(const atomic< void * > &rhs)
Definition: atomic.h:512
constexpr atomic_impl_with_arithmetic(value_type value)
Definition: atomic.h:343
void swap(atomic< T > &lhs, atomic< T > &rhs)
Definition: atomic.h:539
value_type fetch_and_add(D addend)
Definition: atomic.h:346
value_type compare_and_swap(value_type value, value_type comparand)
Definition: atomic.h:295
atomic()=default
void store(atomic< T > &a, T value)
Definition: atomic.h:524
constexpr aligned_storage(value_type value)
Definition: atomic.h:121
T load(const atomic< T > &a)
Definition: atomic.h:521
static converter< value_t >::bits_type & to_bits_ref(value_t &value)
Definition: atomic.h:259
memory_semantics
Specifies memory semantics.
Definition: atomic.h:43
Acquire.
Definition: atomic.h:47
#define __TBB_DECL_ATOMIC(T)
Definition: atomic.h:420
#define __TBB_DECL_ATOMIC_PRIMITIVES(S)
Definition: atomic.h:148
value_type load() const
Definition: atomic.h:306
#define __TBB_LONG_LONG
Definition: atomic.h:29
The graph class.
#define __TBB_MINUS_ONE(T)
Additive inverse of 1 for type T.
Definition: atomic.h:214
atomic< T > & operator=(const atomic< T > &rhs)
Definition: atomic.h:416
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long value
Base class that provides basic functionality for atomic<T> without fetch_and_add.
Definition: atomic.h:220
T * operator=(T *rhs)
Definition: atomic.h:490
constexpr atomic(void *arg)
Definition: atomic.h:506
value_type fetch_and_add(D addend)
Definition: atomic.h:350
value_type fetch_and_store(value_type value)
Definition: atomic.h:284
aligned_storage< T, sizeof(T)> my_storage
Definition: atomic.h:222
Primary template for atomic.
Definition: atomic.h:407
value_type load() const
Definition: atomic.h:312
value_type compare_and_swap(value_type value, value_type comparand)
Definition: atomic.h:289
void store(value_type value)
Definition: atomic.h:321
constexpr atomic(T arg)
Definition: atomic.h:410
converter(value_type a_value)
Definition: atomic.h:230
constexpr atomic_impl(value_type value)
Definition: atomic.h:275
__TBB_DECL_ATOMIC_LOAD_STORE_PRIMITIVES(full_fence)
atomic< T * > & operator=(const atomic< T * > &rhs)
Definition: atomic.h:494
void store(value_type value)
Definition: atomic.h:317
value_type store_with_release(value_type rhs)
Definition: atomic.h:326
#define __TBB_DECL_FENCED_ATOMIC_PRIMITIVES(S, M)
Definition: atomic.h:134
Release.
Definition: atomic.h:49
void __TBB_store_with_release(volatile T &location, V value)
Definition: tbb_machine.h:717
atomic< T > & as_atomic(T &t)
Definition: atomic.h:547
value_type fetch_and_store(value_type value)
Definition: atomic.h:278
static value_t to_value(typename converter< value_t >::bits_type bits)
Definition: atomic.h:240

Copyright © 2005-2019 Intel Corporation. All Rights Reserved.

Intel, Pentium, Intel Xeon, Itanium, Intel XScale and VTune are registered trademarks or trademarks of Intel Corporation or its subsidiaries in the United States and other countries.

* Other names and brands may be claimed as the property of others.