| OLD | NEW |
| 1 #ifndef _ATOMIC_H | 1 #ifndef _ATOMIC_H |
| 2 #define _ATOMIC_H | 2 #define _ATOMIC_H |
| 3 | 3 |
| 4 #include <stdint.h> | 4 #include <stdint.h> |
| 5 | 5 |
| 6 #include "atomic_arch.h" | 6 #include "atomic_arch.h" |
| 7 | 7 |
| 8 #ifdef a_ll | 8 #ifdef a_ll |
| 9 | 9 |
| 10 #ifndef a_pre_llsc | 10 #ifndef a_pre_llsc |
| 11 #define a_pre_llsc() | 11 #define a_pre_llsc() |
| 12 #endif | 12 #endif |
| 13 | 13 |
| 14 #ifndef a_post_llsc | 14 #ifndef a_post_llsc |
| 15 #define a_post_llsc() | 15 #define a_post_llsc() |
| 16 #endif | 16 #endif |
| 17 | 17 |
| 18 #ifndef a_cas | 18 #ifndef a_cas |
| 19 #define a_cas a_cas | 19 #define a_cas a_cas |
| 20 static inline int a_cas(volatile int *p, int t, int s) | 20 static inline int a_cas(volatile int* p, int t, int s) { |
| 21 { | 21 int old; |
| 22 » int old; | 22 a_pre_llsc(); |
| 23 » a_pre_llsc(); | 23 do |
| 24 » do old = a_ll(p); | 24 old = a_ll(p); |
| 25 » while (old==t && !a_sc(p, s)); | 25 while (old == t && !a_sc(p, s)); |
| 26 » a_post_llsc(); | 26 a_post_llsc(); |
| 27 » return old; | 27 return old; |
| 28 } | 28 } |
| 29 #endif | 29 #endif |
| 30 | 30 |
| 31 #ifndef a_swap | 31 #ifndef a_swap |
| 32 #define a_swap a_swap | 32 #define a_swap a_swap |
| 33 static inline int a_swap(volatile int *p, int v) | 33 static inline int a_swap(volatile int* p, int v) { |
| 34 { | 34 int old; |
| 35 » int old; | 35 a_pre_llsc(); |
| 36 » a_pre_llsc(); | 36 do |
| 37 » do old = a_ll(p); | 37 old = a_ll(p); |
| 38 » while (!a_sc(p, v)); | 38 while (!a_sc(p, v)); |
| 39 » a_post_llsc(); | 39 a_post_llsc(); |
| 40 » return old; | 40 return old; |
| 41 } | 41 } |
| 42 #endif | 42 #endif |
| 43 | 43 |
| 44 #ifndef a_fetch_add | 44 #ifndef a_fetch_add |
| 45 #define a_fetch_add a_fetch_add | 45 #define a_fetch_add a_fetch_add |
| 46 static inline int a_fetch_add(volatile int *p, int v) | 46 static inline int a_fetch_add(volatile int* p, int v) { |
| 47 { | 47 int old; |
| 48 » int old; | 48 a_pre_llsc(); |
| 49 » a_pre_llsc(); | 49 do |
| 50 » do old = a_ll(p); | 50 old = a_ll(p); |
| 51 » while (!a_sc(p, (unsigned)old + v)); | 51 while (!a_sc(p, (unsigned)old + v)); |
| 52 » a_post_llsc(); | 52 a_post_llsc(); |
| 53 » return old; | 53 return old; |
| 54 } | 54 } |
| 55 #endif | 55 #endif |
| 56 | 56 |
| 57 #ifndef a_fetch_and | 57 #ifndef a_fetch_and |
| 58 #define a_fetch_and a_fetch_and | 58 #define a_fetch_and a_fetch_and |
| 59 static inline int a_fetch_and(volatile int *p, int v) | 59 static inline int a_fetch_and(volatile int* p, int v) { |
| 60 { | 60 int old; |
| 61 » int old; | 61 a_pre_llsc(); |
| 62 » a_pre_llsc(); | 62 do |
| 63 » do old = a_ll(p); | 63 old = a_ll(p); |
| 64 » while (!a_sc(p, old & v)); | 64 while (!a_sc(p, old & v)); |
| 65 » a_post_llsc(); | 65 a_post_llsc(); |
| 66 » return old; | 66 return old; |
| 67 } | 67 } |
| 68 #endif | 68 #endif |
| 69 | 69 |
| 70 #ifndef a_fetch_or | 70 #ifndef a_fetch_or |
| 71 #define a_fetch_or a_fetch_or | 71 #define a_fetch_or a_fetch_or |
| 72 static inline int a_fetch_or(volatile int *p, int v) | 72 static inline int a_fetch_or(volatile int* p, int v) { |
| 73 { | 73 int old; |
| 74 » int old; | 74 a_pre_llsc(); |
| 75 » a_pre_llsc(); | 75 do |
| 76 » do old = a_ll(p); | 76 old = a_ll(p); |
| 77 » while (!a_sc(p, old | v)); | 77 while (!a_sc(p, old | v)); |
| 78 » a_post_llsc(); | 78 a_post_llsc(); |
| 79 » return old; | 79 return old; |
| 80 } | 80 } |
| 81 #endif | 81 #endif |
| 82 | 82 |
| 83 #endif | 83 #endif |
| 84 | 84 |
| 85 #ifndef a_cas | 85 #ifndef a_cas |
| 86 #error missing definition of a_cas | 86 #error missing definition of a_cas |
| 87 #endif | 87 #endif |
| 88 | 88 |
| 89 #ifndef a_swap | 89 #ifndef a_swap |
| 90 #define a_swap a_swap | 90 #define a_swap a_swap |
| 91 static inline int a_swap(volatile int *p, int v) | 91 static inline int a_swap(volatile int* p, int v) { |
| 92 { | 92 int old; |
| 93 » int old; | 93 do |
| 94 » do old = *p; | 94 old = *p; |
| 95 » while (a_cas(p, old, v) != old); | 95 while (a_cas(p, old, v) != old); |
| 96 » return old; | 96 return old; |
| 97 } | 97 } |
| 98 #endif | 98 #endif |
| 99 | 99 |
| 100 #ifndef a_fetch_add | 100 #ifndef a_fetch_add |
| 101 #define a_fetch_add a_fetch_add | 101 #define a_fetch_add a_fetch_add |
| 102 static inline int a_fetch_add(volatile int *p, int v) | 102 static inline int a_fetch_add(volatile int* p, int v) { |
| 103 { | 103 int old; |
| 104 » int old; | 104 do |
| 105 » do old = *p; | 105 old = *p; |
| 106 » while (a_cas(p, old, (unsigned)old+v) != old); | 106 while (a_cas(p, old, (unsigned)old + v) != old); |
| 107 » return old; | 107 return old; |
| 108 } | 108 } |
| 109 #endif | 109 #endif |
| 110 | 110 |
| 111 #ifndef a_fetch_and | 111 #ifndef a_fetch_and |
| 112 #define a_fetch_and a_fetch_and | 112 #define a_fetch_and a_fetch_and |
| 113 static inline int a_fetch_and(volatile int *p, int v) | 113 static inline int a_fetch_and(volatile int* p, int v) { |
| 114 { | 114 int old; |
| 115 » int old; | 115 do |
| 116 » do old = *p; | 116 old = *p; |
| 117 » while (a_cas(p, old, old&v) != old); | 117 while (a_cas(p, old, old & v) != old); |
| 118 » return old; | 118 return old; |
| 119 } | 119 } |
| 120 #endif | 120 #endif |
| 121 #ifndef a_fetch_or | 121 #ifndef a_fetch_or |
| 122 #define a_fetch_or a_fetch_or | 122 #define a_fetch_or a_fetch_or |
| 123 static inline int a_fetch_or(volatile int *p, int v) | 123 static inline int a_fetch_or(volatile int* p, int v) { |
| 124 { | 124 int old; |
| 125 » int old; | 125 do |
| 126 » do old = *p; | 126 old = *p; |
| 127 » while (a_cas(p, old, old|v) != old); | 127 while (a_cas(p, old, old | v) != old); |
| 128 » return old; | 128 return old; |
| 129 } | 129 } |
| 130 #endif | 130 #endif |
| 131 | 131 |
| 132 #ifndef a_and | 132 #ifndef a_and |
| 133 #define a_and a_and | 133 #define a_and a_and |
| 134 static inline void a_and(volatile int *p, int v) | 134 static inline void a_and(volatile int* p, int v) { |
| 135 { | 135 a_fetch_and(p, v); |
| 136 » a_fetch_and(p, v); | |
| 137 } | 136 } |
| 138 #endif | 137 #endif |
| 139 | 138 |
| 140 #ifndef a_or | 139 #ifndef a_or |
| 141 #define a_or a_or | 140 #define a_or a_or |
| 142 static inline void a_or(volatile int *p, int v) | 141 static inline void a_or(volatile int* p, int v) { |
| 143 { | 142 a_fetch_or(p, v); |
| 144 » a_fetch_or(p, v); | |
| 145 } | 143 } |
| 146 #endif | 144 #endif |
| 147 | 145 |
| 148 #ifndef a_inc | 146 #ifndef a_inc |
| 149 #define a_inc a_inc | 147 #define a_inc a_inc |
| 150 static inline void a_inc(volatile int *p) | 148 static inline void a_inc(volatile int* p) { |
| 151 { | 149 a_fetch_add(p, 1); |
| 152 » a_fetch_add(p, 1); | |
| 153 } | 150 } |
| 154 #endif | 151 #endif |
| 155 | 152 |
| 156 #ifndef a_dec | 153 #ifndef a_dec |
| 157 #define a_dec a_dec | 154 #define a_dec a_dec |
| 158 static inline void a_dec(volatile int *p) | 155 static inline void a_dec(volatile int* p) { |
| 159 { | 156 a_fetch_add(p, -1); |
| 160 » a_fetch_add(p, -1); | |
| 161 } | 157 } |
| 162 #endif | 158 #endif |
| 163 | 159 |
| 164 #ifndef a_store | 160 #ifndef a_store |
| 165 #define a_store a_store | 161 #define a_store a_store |
| 166 static inline void a_store(volatile int *p, int v) | 162 static inline void a_store(volatile int* p, int v) { |
| 167 { | |
| 168 #ifdef a_barrier | 163 #ifdef a_barrier |
| 169 » a_barrier(); | 164 a_barrier(); |
| 170 » *p = v; | 165 *p = v; |
| 171 » a_barrier(); | 166 a_barrier(); |
| 172 #else | 167 #else |
| 173 » a_swap(p, v); | 168 a_swap(p, v); |
| 174 #endif | 169 #endif |
| 175 } | 170 } |
| 176 #endif | 171 #endif |
| 177 | 172 |
| 178 #ifndef a_barrier | 173 #ifndef a_barrier |
| 179 #define a_barrier a_barrier | 174 #define a_barrier a_barrier |
| 180 static void a_barrier() | 175 static void a_barrier() { |
| 181 { | 176 volatile int tmp = 0; |
| 182 » volatile int tmp = 0; | 177 a_cas(&tmp, 0, 0); |
| 183 » a_cas(&tmp, 0, 0); | |
| 184 } | 178 } |
| 185 #endif | 179 #endif |
| 186 | 180 |
| 187 #ifndef a_spin | 181 #ifndef a_spin |
| 188 #define a_spin a_barrier | 182 #define a_spin a_barrier |
| 189 #endif | 183 #endif |
| 190 | 184 |
| 191 #ifndef a_and_64 | 185 #ifndef a_and_64 |
| 192 #define a_and_64 a_and_64 | 186 #define a_and_64 a_and_64 |
| 193 static inline void a_and_64(volatile uint64_t *p, uint64_t v) | 187 static inline void a_and_64(volatile uint64_t* p, uint64_t v) { |
| 194 { | 188 union { |
| 195 » union { uint64_t v; uint32_t r[2]; } u = { v }; | 189 uint64_t v; |
| 196 » if (u.r[0]+1) a_and((int *)p, u.r[0]); | 190 uint32_t r[2]; |
| 197 » if (u.r[1]+1) a_and((int *)p+1, u.r[1]); | 191 } u = {v}; |
| 192 if (u.r[0] + 1) |
| 193 a_and((int*)p, u.r[0]); |
| 194 if (u.r[1] + 1) |
| 195 a_and((int*)p + 1, u.r[1]); |
| 198 } | 196 } |
| 199 #endif | 197 #endif |
| 200 | 198 |
| 201 #ifndef a_or_64 | 199 #ifndef a_or_64 |
| 202 #define a_or_64 a_or_64 | 200 #define a_or_64 a_or_64 |
| 203 static inline void a_or_64(volatile uint64_t *p, uint64_t v) | 201 static inline void a_or_64(volatile uint64_t* p, uint64_t v) { |
| 204 { | 202 union { |
| 205 » union { uint64_t v; uint32_t r[2]; } u = { v }; | 203 uint64_t v; |
| 206 » if (u.r[0]) a_or((int *)p, u.r[0]); | 204 uint32_t r[2]; |
| 207 » if (u.r[1]) a_or((int *)p+1, u.r[1]); | 205 } u = {v}; |
| 206 if (u.r[0]) |
| 207 a_or((int*)p, u.r[0]); |
| 208 if (u.r[1]) |
| 209 a_or((int*)p + 1, u.r[1]); |
| 208 } | 210 } |
| 209 #endif | 211 #endif |
| 210 | 212 |
| 211 #ifndef a_cas_p | 213 #ifndef a_cas_p |
| 212 #define a_cas_p a_cas_p | 214 #define a_cas_p a_cas_p |
| 213 static inline void *a_cas_p(volatile void *p, void *t, void *s) | 215 static inline void* a_cas_p(volatile void* p, void* t, void* s) { |
| 214 { | 216 return (void*)a_cas((volatile int*)p, (int)t, (int)s); |
| 215 » return (void *)a_cas((volatile int *)p, (int)t, (int)s); | |
| 216 } | 217 } |
| 217 #endif | 218 #endif |
| 218 | 219 |
| 219 #ifndef a_or_l | 220 #ifndef a_or_l |
| 220 #define a_or_l a_or_l | 221 #define a_or_l a_or_l |
| 221 static inline void a_or_l(volatile void *p, long v) | 222 static inline void a_or_l(volatile void* p, long v) { |
| 222 { | 223 if (sizeof(long) == sizeof(int)) |
| 223 » if (sizeof(long) == sizeof(int)) a_or(p, v); | 224 a_or(p, v); |
| 224 » else a_or_64(p, v); | 225 else |
| 226 a_or_64(p, v); |
| 225 } | 227 } |
| 226 #endif | 228 #endif |
| 227 | 229 |
| 228 #ifndef a_crash | 230 #ifndef a_crash |
| 229 #define a_crash a_crash | 231 #define a_crash a_crash |
| 230 static inline void a_crash() | 232 static inline void a_crash() { |
| 231 { | 233 *(volatile char*)0 = 0; |
| 232 » *(volatile char *)0=0; | |
| 233 } | 234 } |
| 234 #endif | 235 #endif |
| 235 | 236 |
| 236 #ifndef a_ctz_64 | 237 #ifndef a_ctz_64 |
| 237 #define a_ctz_64 a_ctz_64 | 238 #define a_ctz_64 a_ctz_64 |
| 238 static inline int a_ctz_64(uint64_t x) | 239 static inline int a_ctz_64(uint64_t x) { |
| 239 { | 240 static const char debruijn64[64] = { |
| 240 » static const char debruijn64[64] = { | 241 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, |
| 241 » » 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, | 242 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, |
| 242 » » 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, | 243 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, |
| 243 » » 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, | 244 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12}; |
| 244 » » 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12 | 245 static const char debruijn32[32] = { |
| 245 » }; | 246 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, |
| 246 » static const char debruijn32[32] = { | 247 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14}; |
| 247 » » 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, | 248 if (sizeof(long) < 8) { |
| 248 » » 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14 | 249 uint32_t y = x; |
| 249 » }; | 250 if (!y) { |
| 250 » if (sizeof(long) < 8) { | 251 y = x >> 32; |
| 251 » » uint32_t y = x; | 252 return 32 + debruijn32[(y & -y) * 0x076be629 >> 27]; |
| 252 » » if (!y) { | 253 } |
| 253 » » » y = x>>32; | 254 return debruijn32[(y & -y) * 0x076be629 >> 27]; |
| 254 » » » return 32 + debruijn32[(y&-y)*0x076be629 >> 27]; | 255 } |
| 255 » » } | 256 return debruijn64[(x & -x) * 0x022fdd63cc95386dull >> 58]; |
| 256 » » return debruijn32[(y&-y)*0x076be629 >> 27]; | |
| 257 » } | |
| 258 » return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58]; | |
| 259 } | 257 } |
| 260 #endif | 258 #endif |
| 261 | 259 |
| 262 #ifndef a_ctz_l | 260 #ifndef a_ctz_l |
| 263 #define a_ctz_l a_ctz_l | 261 #define a_ctz_l a_ctz_l |
| 264 static inline int a_ctz_l(unsigned long x) | 262 static inline int a_ctz_l(unsigned long x) { |
| 265 { | 263 static const char debruijn32[32] = { |
| 266 » static const char debruijn32[32] = { | 264 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, |
| 267 » » 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, | 265 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14}; |
| 268 » » 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14 | 266 if (sizeof(long) == 8) |
| 269 » }; | 267 return a_ctz_64(x); |
| 270 » if (sizeof(long) == 8) return a_ctz_64(x); | 268 return debruijn32[(x & -x) * 0x076be629 >> 27]; |
| 271 » return debruijn32[(x&-x)*0x076be629 >> 27]; | |
| 272 } | 269 } |
| 273 #endif | 270 #endif |
| 274 | 271 |
| 275 #endif | 272 #endif |
| OLD | NEW |