Atomic.h
24.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
#pragma once
#include "../C/Baselib_Atomic.h"
#include "Internal/TypeTraits.h"
// Note that aligning by type is not possible with the C compatible COMPILER_ALIGN_AS as MSVC's own alignment attribute does not allow evaluation of sizeof
#define ALIGN_ATOMIC(TYPE_) alignas(sizeof(TYPE_))
#define ALIGNED_ATOMIC(TYPE_) ALIGN_ATOMIC(TYPE_) TYPE_
// Atomic interface that sticks closely to std::atomic
// Major differences:
// * free functions that operate on types other than baselib::atomic
// * baselib::atomic allows access to its internal value
// * no zero initialization on baselib::atomic
// * no single parameter versions of compare_exchange
namespace baselib
{
BASELIB_CPP_INTERFACE
{
enum memory_order_relaxed_t { memory_order_relaxed = 0 }; // Equal to std::memory_order_relaxed
enum memory_order_acquire_t { memory_order_acquire = 2 }; // Equal to std::memory_order_acquire
enum memory_order_release_t { memory_order_release = 3 }; // Equal to std::memory_order_release
enum memory_order_acq_rel_t { memory_order_acq_rel = 4 }; // Equal to std::memory_order_acq_rel
enum memory_order_seq_cst_t { memory_order_seq_cst = 5 }; // Equal to std::memory_order_seq_cst
namespace detail
{
template<typename T, typename ... Rest>
struct is_any : std::false_type {};
template<typename T, typename First>
struct is_any<T, First> : std::is_same<T, First> {};
template<typename T, typename First, typename ... Rest>
struct is_any<T, First, Rest...>
: std::integral_constant<bool, std::is_same<T, First>::value || is_any<T, Rest...>::value>
{};
#define TEST_ATOMICS_PREREQUISITES(_TYPE) \
static_assert(baselib::is_trivially_copyable<_TYPE>::value, "atomic operation operands needs to be trivially copyable"); \
static_assert(sizeof(_TYPE) <= sizeof(void*) * 2, "atomic operation operands need to be smaller or equal than two pointers");
template<typename T> static inline T fail();
template<typename T, typename MemoryOrder, typename ... AllowedMemoryOrders> static inline T fail_prerequisites()
{
TEST_ATOMICS_PREREQUISITES(T);
static_assert(is_any<MemoryOrder, AllowedMemoryOrders...>::value, "the specified memory ordering is invalid for this atomic operation");
return fail<T>();
}
template<typename T, typename MemoryOrderSuccess, typename MemoryOrderFailure> static inline T fail_prerequisites_cmpxchg()
{
TEST_ATOMICS_PREREQUISITES(T);
static_assert(
// fail: relaxed, success: relaxed/acquire/release/seq_cst
(std::is_same<MemoryOrderFailure, baselib::memory_order_relaxed_t>::value &&
is_any<MemoryOrderSuccess, baselib::memory_order_relaxed_t, baselib::memory_order_acquire_t, baselib::memory_order_release_t, baselib::memory_order_seq_cst_t>::value) ||
// fail: acquire, success acquire/release/seq_cst
(std::is_same<MemoryOrderFailure, baselib::memory_order_relaxed_t>::value &&
is_any<MemoryOrderSuccess, baselib::memory_order_acquire_t, baselib::memory_order_release_t, baselib::memory_order_seq_cst_t>::value) ||
// fail: seq_cst, success: seq_cst
(std::is_same<MemoryOrderSuccess, baselib::memory_order_seq_cst_t>::value && std::is_same<MemoryOrderFailure, baselib::memory_order_seq_cst_t>::value),
"the specified combination of memory ordering is invalid for compare exchange operations");
return fail<T>();
}
template<typename T, typename MemoryOrder> static inline T fail_prerequisites_alu()
{
static_assert(std::is_integral<T>::value, "operands of arithmetic atomic operations need to be integral");
return fail_prerequisites<T, MemoryOrder,
baselib::memory_order_relaxed_t,
baselib::memory_order_acquire_t,
baselib::memory_order_release_t,
baselib::memory_order_acq_rel_t,
baselib::memory_order_seq_cst_t>();
}
}
// MACRO generated impl
// re-directs to Baselib_atomic_ API
// ----------------------------------------------------------------------------------------------------------------------------------
#define detail_THREAD_FENCE(order, ...) \
static FORCE_INLINE void atomic_thread_fence(memory_order_##order##_t order) \
{ \
return Baselib_atomic_thread_fence_##order(); \
}
#define detail_LOAD(op, order, id, bits, ...) \
template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
static FORCE_INLINE T atomic_load_explicit(const T& obj, memory_order_##order##_t order) \
{ \
T ret; \
Baselib_atomic_load_##id##_##order##_v(&obj, &ret); \
return ret; \
}
#define detail_LOAD128(op, order, id, bits, ...) \
template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
static FORCE_INLINE T atomic_load_explicit(const T& obj, memory_order_##order##_t order) \
{ \
T ret; \
Baselib_atomic_load_##id##_##order##_v(const_cast<T*>(&obj), &ret); \
return ret; \
}
#define detail_STORE(op, order, id, bits, ...) \
template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
static FORCE_INLINE void atomic_store_explicit(T& obj, typename std::common_type<T>::type value, memory_order_##order##_t order)\
{ \
return Baselib_atomic_store_##id##_##order##_v(&obj, &value); \
}
#define detail_LOAD_STORE(op, order, id, bits, ...) \
template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
static FORCE_INLINE T atomic_##op##_explicit(T& obj, typename std::common_type<T>::type value, memory_order_##order##_t order) \
{ \
T ret; \
Baselib_atomic_##op##_##id##_##order##_v(&obj, &value, &ret); \
return ret; \
}
#define detail_ALU(op, order, id, bits, ...) \
template<typename T, typename std::enable_if<baselib::is_integral_of_size<T, bits/8>::value, int>::type = 0> \
static FORCE_INLINE T atomic_##op##_explicit(T& obj, typename std::common_type<T>::type value, memory_order_##order##_t order) \
{ \
T ret; \
Baselib_atomic_##op##_##id##_##order##_v(&obj, &value, &ret); \
return ret; \
}
#define detail_CMP_XCHG(op, order1, order2, id, bits, ...) \
template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
static FORCE_INLINE bool atomic_##op##_explicit(T& obj, \
typename std::common_type<T>::type& expected, \
typename std::common_type<T>::type desired, \
memory_order_##order1##_t order_success, \
memory_order_##order2##_t order_failure) \
{ \
return Baselib_atomic_##op##_##id##_##order1##_##order2##_v(&obj, &expected, &desired); \
}
#define detail_NOT_SUPPORTED(...)
Baselib_Atomic_FOR_EACH_MEMORY_ORDER(
detail_THREAD_FENCE
)
Baselib_Atomic_FOR_EACH_ATOMIC_OP_MEMORY_ORDER_AND_INT_TYPE(
detail_LOAD, // load
detail_STORE, // store
detail_ALU, // add
detail_ALU, // and
detail_ALU, // or
detail_ALU, // xor
detail_LOAD_STORE, // exchange
detail_CMP_XCHG, // compare_exchange_weak
detail_CMP_XCHG // compare_exchange_strong
)
#if PLATFORM_ARCH_64
// 128bit atomics
Baselib_Atomic_FOR_EACH_ATOMIC_OP_AND_MEMORY_ORDER(
detail_LOAD128, // load
detail_STORE, // store
detail_NOT_SUPPORTED, // add
detail_NOT_SUPPORTED, // and
detail_NOT_SUPPORTED, // or
detail_NOT_SUPPORTED, // xor
detail_LOAD_STORE, // exchange
detail_CMP_XCHG, // compare_exchange_weak
detail_CMP_XCHG, // compare_exchange_strong
128, 128)
#endif
#undef detail_THREAD_FENCE
#undef detail_LOAD
#undef detail_STORE
#undef detail_LOAD_STORE
#undef detail_ALU
#undef detail_CMP_XCHG
#undef detail_NOT_SUPPORTED
template<typename T, typename MemoryOrder>
static FORCE_INLINE T atomic_fetch_sub_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
{
return atomic_fetch_add_explicit(obj, 0 - value, order);
}
// API documentation and default fallback for non-matching types
// ----------------------------------------------------------------------------------------------------------------------
template<typename T, typename MemoryOrder>
static FORCE_INLINE T atomic_load_explicit(const T& obj, MemoryOrder order)
{
return detail::fail_prerequisites<T, MemoryOrder, baselib::memory_order_relaxed_t, baselib::memory_order_acquire_t, baselib::memory_order_seq_cst_t>();
}
template<typename T, typename MemoryOrder>
static FORCE_INLINE void atomic_store_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
{
detail::fail_prerequisites<T, MemoryOrder, baselib::memory_order_relaxed_t, baselib::memory_order_release_t, baselib::memory_order_seq_cst_t>();
}
template<typename T, typename MemoryOrder>
static FORCE_INLINE T atomic_fetch_add_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
{
return detail::fail_prerequisites_alu<T, MemoryOrder>();
}
template<typename T, typename MemoryOrder>
static FORCE_INLINE T atomic_fetch_and_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
{
return detail::fail_prerequisites_alu<T, MemoryOrder>();
}
template<typename T, typename MemoryOrder>
static FORCE_INLINE T atomic_fetch_or_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
{
return detail::fail_prerequisites_alu<T, MemoryOrder>();
}
template<typename T, typename MemoryOrder>
static FORCE_INLINE T atomic_fetch_xor_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
{
return detail::fail_prerequisites_alu<T, MemoryOrder>();
}
template<typename T, typename MemoryOrder>
static FORCE_INLINE T atomic_exchange_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
{
return detail::fail_prerequisites<T, MemoryOrder>();
}
template<typename T, typename MemoryOrderSuccess, typename MemoryOrderFailure>
static FORCE_INLINE bool atomic_compare_exchange_weak_explicit(T& obj,
typename std::common_type<T>::type& expected,
typename std::common_type<T>::type desired,
MemoryOrderSuccess order_success,
MemoryOrderFailure order_failure)
{
detail::fail_prerequisites_cmpxchg<T, MemoryOrderSuccess, MemoryOrderFailure>();
return false;
}
template<typename T, typename MemoryOrderSuccess, typename MemoryOrderFailure>
static FORCE_INLINE bool atomic_compare_exchange_strong_explicit(T& obj,
typename std::common_type<T>::type& expected,
typename std::common_type<T>::type desired,
MemoryOrderSuccess order_success,
MemoryOrderFailure order_failure)
{
detail::fail_prerequisites_cmpxchg<T, MemoryOrderSuccess, MemoryOrderFailure>();
return false;
}
// default memory order (memory_order_seq_cst)
// ----------------------------------------------------------------------------------------------------------------------
template<typename T>
static FORCE_INLINE T atomic_load(const T& obj)
{
return atomic_load_explicit(obj, memory_order_seq_cst);
}
template<typename T>
static FORCE_INLINE void atomic_store(T& obj, typename std::common_type<T>::type value)
{
return atomic_store_explicit(obj, value, memory_order_seq_cst);
}
template<typename T>
static FORCE_INLINE T atomic_fetch_add(T& obj, typename std::common_type<T>::type value)
{
return atomic_fetch_add_explicit(obj, value, memory_order_seq_cst);
}
template<typename T>
static FORCE_INLINE T atomic_fetch_sub(T& obj, typename std::common_type<T>::type value)
{
return atomic_fetch_sub_explicit(obj, value, memory_order_seq_cst);
}
template<typename T>
static FORCE_INLINE T atomic_fetch_and(T& obj, typename std::common_type<T>::type value)
{
return atomic_fetch_and_explicit(obj, value, memory_order_seq_cst);
}
template<typename T>
static FORCE_INLINE T atomic_fetch_or(T& obj, typename std::common_type<T>::type value)
{
return atomic_fetch_or_explicit(obj, value, memory_order_seq_cst);
}
template<typename T>
static FORCE_INLINE T atomic_fetch_xor(T& obj, typename std::common_type<T>::type value)
{
return atomic_fetch_xor_explicit(obj, value, memory_order_seq_cst);
}
template<typename T>
static FORCE_INLINE T atomic_exchange(T& obj, typename std::common_type<T>::type value)
{
return atomic_exchange_explicit(obj, value, memory_order_seq_cst);
}
template<typename T>
static FORCE_INLINE bool atomic_compare_exchange_weak(T& obj,
typename std::common_type<T>::type& expected,
typename std::common_type<T>::type desired)
{
return atomic_compare_exchange_weak_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}
template<typename T>
static FORCE_INLINE bool atomic_compare_exchange_strong(T& obj,
typename std::common_type<T>::type& expected,
typename std::common_type<T>::type desired)
{
return atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}
template<typename T>
struct atomic_common
{
using value_type = T;
TEST_ATOMICS_PREREQUISITES(T);
ALIGNED_ATOMIC(T) obj;
FORCE_INLINE atomic_common() = default;
// Initializes atomic with a given value. Initialization is not atomic!
FORCE_INLINE atomic_common(T value)
{
obj = value;
}
FORCE_INLINE operator T() const { return atomic_load_explicit(obj, memory_order_seq_cst); }
FORCE_INLINE T operator=(T value) { atomic_store_explicit(obj, value, memory_order_seq_cst); return value; }
template<typename TMemoryOrder = memory_order_seq_cst_t>
FORCE_INLINE T load(TMemoryOrder order = memory_order_seq_cst) const
{
return atomic_load_explicit(obj, order);
}
template<typename TMemoryOrder = memory_order_seq_cst_t>
FORCE_INLINE void store(T value, TMemoryOrder order = memory_order_seq_cst)
{
return atomic_store_explicit(obj, value, order);
}
template<typename TMemoryOrder = memory_order_seq_cst_t>
FORCE_INLINE T exchange(T value, TMemoryOrder order = memory_order_seq_cst)
{
return atomic_exchange_explicit(obj, value, order);
}
template<typename TMemoryOrderSuccess, typename TMemoryOrderFailure>
FORCE_INLINE bool compare_exchange_weak(T& expected, T desired, TMemoryOrderSuccess order_success, TMemoryOrderFailure order_failure)
{
return atomic_compare_exchange_weak_explicit(obj, expected, desired, order_success, order_failure);
}
FORCE_INLINE bool compare_exchange_weak(T& expected, T desired)
{
return atomic_compare_exchange_weak_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}
template<typename TMemoryOrderSuccess, typename TMemoryOrderFailure>
FORCE_INLINE bool compare_exchange_strong(T& expected, T desired, TMemoryOrderSuccess order_success, TMemoryOrderFailure order_failure)
{
return atomic_compare_exchange_strong_explicit(obj, expected, desired, order_success, order_failure);
}
FORCE_INLINE bool compare_exchange_strong(T& expected, T desired)
{
return atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
}
};
template<typename T, bool IsIntegral>
struct atomic_base {};
// Atomic type for integral types.
template<typename T>
struct atomic_base<T, true> : atomic_common<T>
{
using atomic_common<T>::atomic_common;
template<typename TMemoryOrder = memory_order_seq_cst_t>
FORCE_INLINE T fetch_add(T value, TMemoryOrder order = memory_order_seq_cst)
{
return atomic_fetch_add_explicit(atomic_common<T>::obj, value, order);
}
template<typename TMemoryOrder = memory_order_seq_cst_t>
FORCE_INLINE T fetch_sub(T value, TMemoryOrder order = memory_order_seq_cst)
{
return atomic_fetch_sub_explicit(atomic_common<T>::obj, value, order);
}
template<typename TMemoryOrder = memory_order_seq_cst_t>
FORCE_INLINE T fetch_and(T value, TMemoryOrder order = memory_order_seq_cst)
{
return atomic_fetch_and_explicit(atomic_common<T>::obj, value, order);
}
template<typename TMemoryOrder = memory_order_seq_cst_t>
FORCE_INLINE T fetch_or(T value, TMemoryOrder order = memory_order_seq_cst)
{
return atomic_fetch_or_explicit(atomic_common<T>::obj, value, order);
}
template<typename TMemoryOrder = memory_order_seq_cst_t>
FORCE_INLINE T fetch_xor(T value, TMemoryOrder order = memory_order_seq_cst)
{
return atomic_fetch_xor_explicit(atomic_common<T>::obj, value, order);
}
FORCE_INLINE T operator++(int) { return atomic_fetch_add_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst); }
FORCE_INLINE T operator--(int) { return atomic_fetch_sub_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst); }
FORCE_INLINE T operator++() { return atomic_fetch_add_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst) + T(1); }
FORCE_INLINE T operator--() { return atomic_fetch_sub_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst) - T(1); }
FORCE_INLINE T operator+=(T value) { return atomic_fetch_add_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) + value; }
FORCE_INLINE T operator-=(T value) { return atomic_fetch_sub_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) - value; }
FORCE_INLINE T operator&=(T value) { return atomic_fetch_and_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) & value; }
FORCE_INLINE T operator|=(T value) { return atomic_fetch_or_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) | value; }
FORCE_INLINE T operator^=(T value) { return atomic_fetch_xor_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) ^ value; }
};
// Atomic type for non-integral types.
template<typename T>
struct atomic_base<T, false> : atomic_common<T>
{
using atomic_common<T>::atomic_common;
};
template<typename T>
struct atomic : atomic_base<T, std::is_integral<T>::value>
{
using atomic_base<T, std::is_integral<T>::value>::atomic_base;
};
#undef TEST_ATOMICS_PREREQUISITES
}
}