OLD | NEW |
1 #ifndef _ATOMIC_H | 1 #ifndef _ATOMIC_H |
2 #define _ATOMIC_H | 2 #define _ATOMIC_H |
3 | 3 |
4 #include <stdint.h> | 4 #include <stdint.h> |
5 | 5 |
6 #include "atomic_arch.h" | 6 #include "atomic_arch.h" |
7 | 7 |
8 #ifdef a_ll | 8 #ifdef a_ll |
9 | 9 |
10 #ifndef a_pre_llsc | 10 #ifndef a_pre_llsc |
11 #define a_pre_llsc() | 11 #define a_pre_llsc() |
12 #endif | 12 #endif |
13 | 13 |
14 #ifndef a_post_llsc | 14 #ifndef a_post_llsc |
15 #define a_post_llsc() | 15 #define a_post_llsc() |
16 #endif | 16 #endif |
17 | 17 |
18 #ifndef a_cas | 18 #ifndef a_cas |
19 #define a_cas a_cas | 19 #define a_cas a_cas |
20 static inline int a_cas(volatile int *p, int t, int s) | 20 static inline int a_cas(volatile int* p, int t, int s) { |
21 { | 21 int old; |
22 » int old; | 22 a_pre_llsc(); |
23 » a_pre_llsc(); | 23 do |
24 » do old = a_ll(p); | 24 old = a_ll(p); |
25 » while (old==t && !a_sc(p, s)); | 25 while (old == t && !a_sc(p, s)); |
26 » a_post_llsc(); | 26 a_post_llsc(); |
27 » return old; | 27 return old; |
28 } | 28 } |
29 #endif | 29 #endif |
30 | 30 |
31 #ifndef a_swap | 31 #ifndef a_swap |
32 #define a_swap a_swap | 32 #define a_swap a_swap |
33 static inline int a_swap(volatile int *p, int v) | 33 static inline int a_swap(volatile int* p, int v) { |
34 { | 34 int old; |
35 » int old; | 35 a_pre_llsc(); |
36 » a_pre_llsc(); | 36 do |
37 » do old = a_ll(p); | 37 old = a_ll(p); |
38 » while (!a_sc(p, v)); | 38 while (!a_sc(p, v)); |
39 » a_post_llsc(); | 39 a_post_llsc(); |
40 » return old; | 40 return old; |
41 } | 41 } |
42 #endif | 42 #endif |
43 | 43 |
44 #ifndef a_fetch_add | 44 #ifndef a_fetch_add |
45 #define a_fetch_add a_fetch_add | 45 #define a_fetch_add a_fetch_add |
46 static inline int a_fetch_add(volatile int *p, int v) | 46 static inline int a_fetch_add(volatile int* p, int v) { |
47 { | 47 int old; |
48 » int old; | 48 a_pre_llsc(); |
49 » a_pre_llsc(); | 49 do |
50 » do old = a_ll(p); | 50 old = a_ll(p); |
51 » while (!a_sc(p, (unsigned)old + v)); | 51 while (!a_sc(p, (unsigned)old + v)); |
52 » a_post_llsc(); | 52 a_post_llsc(); |
53 » return old; | 53 return old; |
54 } | 54 } |
55 #endif | 55 #endif |
56 | 56 |
57 #ifndef a_fetch_and | 57 #ifndef a_fetch_and |
58 #define a_fetch_and a_fetch_and | 58 #define a_fetch_and a_fetch_and |
59 static inline int a_fetch_and(volatile int *p, int v) | 59 static inline int a_fetch_and(volatile int* p, int v) { |
60 { | 60 int old; |
61 » int old; | 61 a_pre_llsc(); |
62 » a_pre_llsc(); | 62 do |
63 » do old = a_ll(p); | 63 old = a_ll(p); |
64 » while (!a_sc(p, old & v)); | 64 while (!a_sc(p, old & v)); |
65 » a_post_llsc(); | 65 a_post_llsc(); |
66 » return old; | 66 return old; |
67 } | 67 } |
68 #endif | 68 #endif |
69 | 69 |
70 #ifndef a_fetch_or | 70 #ifndef a_fetch_or |
71 #define a_fetch_or a_fetch_or | 71 #define a_fetch_or a_fetch_or |
72 static inline int a_fetch_or(volatile int *p, int v) | 72 static inline int a_fetch_or(volatile int* p, int v) { |
73 { | 73 int old; |
74 » int old; | 74 a_pre_llsc(); |
75 » a_pre_llsc(); | 75 do |
76 » do old = a_ll(p); | 76 old = a_ll(p); |
77 » while (!a_sc(p, old | v)); | 77 while (!a_sc(p, old | v)); |
78 » a_post_llsc(); | 78 a_post_llsc(); |
79 » return old; | 79 return old; |
80 } | 80 } |
81 #endif | 81 #endif |
82 | 82 |
83 #endif | 83 #endif |
84 | 84 |
85 #ifndef a_cas | 85 #ifndef a_cas |
86 #error missing definition of a_cas | 86 #error missing definition of a_cas |
87 #endif | 87 #endif |
88 | 88 |
89 #ifndef a_swap | 89 #ifndef a_swap |
90 #define a_swap a_swap | 90 #define a_swap a_swap |
91 static inline int a_swap(volatile int *p, int v) | 91 static inline int a_swap(volatile int* p, int v) { |
92 { | 92 int old; |
93 » int old; | 93 do |
94 » do old = *p; | 94 old = *p; |
95 » while (a_cas(p, old, v) != old); | 95 while (a_cas(p, old, v) != old); |
96 » return old; | 96 return old; |
97 } | 97 } |
98 #endif | 98 #endif |
99 | 99 |
100 #ifndef a_fetch_add | 100 #ifndef a_fetch_add |
101 #define a_fetch_add a_fetch_add | 101 #define a_fetch_add a_fetch_add |
102 static inline int a_fetch_add(volatile int *p, int v) | 102 static inline int a_fetch_add(volatile int* p, int v) { |
103 { | 103 int old; |
104 » int old; | 104 do |
105 » do old = *p; | 105 old = *p; |
106 » while (a_cas(p, old, (unsigned)old+v) != old); | 106 while (a_cas(p, old, (unsigned)old + v) != old); |
107 » return old; | 107 return old; |
108 } | 108 } |
109 #endif | 109 #endif |
110 | 110 |
111 #ifndef a_fetch_and | 111 #ifndef a_fetch_and |
112 #define a_fetch_and a_fetch_and | 112 #define a_fetch_and a_fetch_and |
113 static inline int a_fetch_and(volatile int *p, int v) | 113 static inline int a_fetch_and(volatile int* p, int v) { |
114 { | 114 int old; |
115 » int old; | 115 do |
116 » do old = *p; | 116 old = *p; |
117 » while (a_cas(p, old, old&v) != old); | 117 while (a_cas(p, old, old & v) != old); |
118 » return old; | 118 return old; |
119 } | 119 } |
120 #endif | 120 #endif |
121 #ifndef a_fetch_or | 121 #ifndef a_fetch_or |
122 #define a_fetch_or a_fetch_or | 122 #define a_fetch_or a_fetch_or |
123 static inline int a_fetch_or(volatile int *p, int v) | 123 static inline int a_fetch_or(volatile int* p, int v) { |
124 { | 124 int old; |
125 » int old; | 125 do |
126 » do old = *p; | 126 old = *p; |
127 » while (a_cas(p, old, old|v) != old); | 127 while (a_cas(p, old, old | v) != old); |
128 » return old; | 128 return old; |
129 } | 129 } |
130 #endif | 130 #endif |
131 | 131 |
132 #ifndef a_and | 132 #ifndef a_and |
133 #define a_and a_and | 133 #define a_and a_and |
134 static inline void a_and(volatile int *p, int v) | 134 static inline void a_and(volatile int* p, int v) { |
135 { | 135 a_fetch_and(p, v); |
136 » a_fetch_and(p, v); | |
137 } | 136 } |
138 #endif | 137 #endif |
139 | 138 |
140 #ifndef a_or | 139 #ifndef a_or |
141 #define a_or a_or | 140 #define a_or a_or |
142 static inline void a_or(volatile int *p, int v) | 141 static inline void a_or(volatile int* p, int v) { |
143 { | 142 a_fetch_or(p, v); |
144 » a_fetch_or(p, v); | |
145 } | 143 } |
146 #endif | 144 #endif |
147 | 145 |
148 #ifndef a_inc | 146 #ifndef a_inc |
149 #define a_inc a_inc | 147 #define a_inc a_inc |
150 static inline void a_inc(volatile int *p) | 148 static inline void a_inc(volatile int* p) { |
151 { | 149 a_fetch_add(p, 1); |
152 » a_fetch_add(p, 1); | |
153 } | 150 } |
154 #endif | 151 #endif |
155 | 152 |
156 #ifndef a_dec | 153 #ifndef a_dec |
157 #define a_dec a_dec | 154 #define a_dec a_dec |
158 static inline void a_dec(volatile int *p) | 155 static inline void a_dec(volatile int* p) { |
159 { | 156 a_fetch_add(p, -1); |
160 » a_fetch_add(p, -1); | |
161 } | 157 } |
162 #endif | 158 #endif |
163 | 159 |
164 #ifndef a_store | 160 #ifndef a_store |
165 #define a_store a_store | 161 #define a_store a_store |
166 static inline void a_store(volatile int *p, int v) | 162 static inline void a_store(volatile int* p, int v) { |
167 { | |
168 #ifdef a_barrier | 163 #ifdef a_barrier |
169 » a_barrier(); | 164 a_barrier(); |
170 » *p = v; | 165 *p = v; |
171 » a_barrier(); | 166 a_barrier(); |
172 #else | 167 #else |
173 » a_swap(p, v); | 168 a_swap(p, v); |
174 #endif | 169 #endif |
175 } | 170 } |
176 #endif | 171 #endif |
177 | 172 |
178 #ifndef a_barrier | 173 #ifndef a_barrier |
179 #define a_barrier a_barrier | 174 #define a_barrier a_barrier |
180 static void a_barrier() | 175 static void a_barrier() { |
181 { | 176 volatile int tmp = 0; |
182 » volatile int tmp = 0; | 177 a_cas(&tmp, 0, 0); |
183 » a_cas(&tmp, 0, 0); | |
184 } | 178 } |
185 #endif | 179 #endif |
186 | 180 |
187 #ifndef a_spin | 181 #ifndef a_spin |
188 #define a_spin a_barrier | 182 #define a_spin a_barrier |
189 #endif | 183 #endif |
190 | 184 |
191 #ifndef a_and_64 | 185 #ifndef a_and_64 |
192 #define a_and_64 a_and_64 | 186 #define a_and_64 a_and_64 |
193 static inline void a_and_64(volatile uint64_t *p, uint64_t v) | 187 static inline void a_and_64(volatile uint64_t* p, uint64_t v) { |
194 { | 188 union { |
195 » union { uint64_t v; uint32_t r[2]; } u = { v }; | 189 uint64_t v; |
196 » if (u.r[0]+1) a_and((int *)p, u.r[0]); | 190 uint32_t r[2]; |
197 » if (u.r[1]+1) a_and((int *)p+1, u.r[1]); | 191 } u = {v}; |
| 192 if (u.r[0] + 1) |
| 193 a_and((int*)p, u.r[0]); |
| 194 if (u.r[1] + 1) |
| 195 a_and((int*)p + 1, u.r[1]); |
198 } | 196 } |
199 #endif | 197 #endif |
200 | 198 |
201 #ifndef a_or_64 | 199 #ifndef a_or_64 |
202 #define a_or_64 a_or_64 | 200 #define a_or_64 a_or_64 |
203 static inline void a_or_64(volatile uint64_t *p, uint64_t v) | 201 static inline void a_or_64(volatile uint64_t* p, uint64_t v) { |
204 { | 202 union { |
205 » union { uint64_t v; uint32_t r[2]; } u = { v }; | 203 uint64_t v; |
206 » if (u.r[0]) a_or((int *)p, u.r[0]); | 204 uint32_t r[2]; |
207 » if (u.r[1]) a_or((int *)p+1, u.r[1]); | 205 } u = {v}; |
| 206 if (u.r[0]) |
| 207 a_or((int*)p, u.r[0]); |
| 208 if (u.r[1]) |
| 209 a_or((int*)p + 1, u.r[1]); |
208 } | 210 } |
209 #endif | 211 #endif |
210 | 212 |
211 #ifndef a_cas_p | 213 #ifndef a_cas_p |
212 #define a_cas_p a_cas_p | 214 #define a_cas_p a_cas_p |
213 static inline void *a_cas_p(volatile void *p, void *t, void *s) | 215 static inline void* a_cas_p(volatile void* p, void* t, void* s) { |
214 { | 216 return (void*)a_cas((volatile int*)p, (int)t, (int)s); |
215 » return (void *)a_cas((volatile int *)p, (int)t, (int)s); | |
216 } | 217 } |
217 #endif | 218 #endif |
218 | 219 |
219 #ifndef a_or_l | 220 #ifndef a_or_l |
220 #define a_or_l a_or_l | 221 #define a_or_l a_or_l |
221 static inline void a_or_l(volatile void *p, long v) | 222 static inline void a_or_l(volatile void* p, long v) { |
222 { | 223 if (sizeof(long) == sizeof(int)) |
223 » if (sizeof(long) == sizeof(int)) a_or(p, v); | 224 a_or(p, v); |
224 » else a_or_64(p, v); | 225 else |
| 226 a_or_64(p, v); |
225 } | 227 } |
226 #endif | 228 #endif |
227 | 229 |
228 #ifndef a_crash | 230 #ifndef a_crash |
229 #define a_crash a_crash | 231 #define a_crash a_crash |
230 static inline void a_crash() | 232 static inline void a_crash() { |
231 { | 233 *(volatile char*)0 = 0; |
232 » *(volatile char *)0=0; | |
233 } | 234 } |
234 #endif | 235 #endif |
235 | 236 |
236 #ifndef a_ctz_64 | 237 #ifndef a_ctz_64 |
237 #define a_ctz_64 a_ctz_64 | 238 #define a_ctz_64 a_ctz_64 |
238 static inline int a_ctz_64(uint64_t x) | 239 static inline int a_ctz_64(uint64_t x) { |
239 { | 240 static const char debruijn64[64] = { |
240 » static const char debruijn64[64] = { | 241 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, |
241 » » 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, | 242 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, |
242 » » 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, | 243 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, |
243 » » 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, | 244 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12}; |
244 » » 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12 | 245 static const char debruijn32[32] = { |
245 » }; | 246 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, |
246 » static const char debruijn32[32] = { | 247 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14}; |
247 » » 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, | 248 if (sizeof(long) < 8) { |
248 » » 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14 | 249 uint32_t y = x; |
249 » }; | 250 if (!y) { |
250 » if (sizeof(long) < 8) { | 251 y = x >> 32; |
251 » » uint32_t y = x; | 252 return 32 + debruijn32[(y & -y) * 0x076be629 >> 27]; |
252 » » if (!y) { | 253 } |
253 » » » y = x>>32; | 254 return debruijn32[(y & -y) * 0x076be629 >> 27]; |
254 » » » return 32 + debruijn32[(y&-y)*0x076be629 >> 27]; | 255 } |
255 » » } | 256 return debruijn64[(x & -x) * 0x022fdd63cc95386dull >> 58]; |
256 » » return debruijn32[(y&-y)*0x076be629 >> 27]; | |
257 » } | |
258 » return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58]; | |
259 } | 257 } |
260 #endif | 258 #endif |
261 | 259 |
262 #ifndef a_ctz_l | 260 #ifndef a_ctz_l |
263 #define a_ctz_l a_ctz_l | 261 #define a_ctz_l a_ctz_l |
264 static inline int a_ctz_l(unsigned long x) | 262 static inline int a_ctz_l(unsigned long x) { |
265 { | 263 static const char debruijn32[32] = { |
266 » static const char debruijn32[32] = { | 264 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, |
267 » » 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, | 265 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14}; |
268 » » 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14 | 266 if (sizeof(long) == 8) |
269 » }; | 267 return a_ctz_64(x); |
270 » if (sizeof(long) == 8) return a_ctz_64(x); | 268 return debruijn32[(x & -x) * 0x076be629 >> 27]; |
271 » return debruijn32[(x&-x)*0x076be629 >> 27]; | |
272 } | 269 } |
273 #endif | 270 #endif |
274 | 271 |
275 #endif | 272 #endif |
OLD | NEW |