OLD | NEW |
1 // Copyright 2012 the V8 project authors. All rights reserved. | 1 // Copyright 2012 the V8 project authors. All rights reserved. |
2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
4 // met: | 4 // met: |
5 // | 5 // |
6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
(...skipping 16 matching lines...) Expand all Loading... |
27 | 27 |
28 // This file is an internal atomic implementation, use atomicops.h instead. | 28 // This file is an internal atomic implementation, use atomicops.h instead. |
29 | 29 |
30 #ifndef V8_ATOMICOPS_INTERNALS_ARM_GCC_H_ | 30 #ifndef V8_ATOMICOPS_INTERNALS_ARM_GCC_H_ |
31 #define V8_ATOMICOPS_INTERNALS_ARM_GCC_H_ | 31 #define V8_ATOMICOPS_INTERNALS_ARM_GCC_H_ |
32 | 32 |
33 namespace v8 { | 33 namespace v8 { |
34 namespace internal { | 34 namespace internal { |
35 | 35 |
36 inline void MemoryBarrier() { | 36 inline void MemoryBarrier() { |
37 __asm__ __volatile__ ( // NOLINT | 37 __asm__ __volatile__ ("dmb ish" ::: "memory"); // NOLINT |
38 "dmb ish \n\t" // Data memory barrier. | |
39 ::: "memory" | |
40 ); // NOLINT | |
41 } | 38 } |
42 | 39 |
| 40 // NoBarrier versions of the operation include "memory" in the clobber list. |
| 41 // This is not required for direct usage of the NoBarrier versions of the |
| 42 // operations. However this is required for correctness when they are used as |
| 43 // part of the Acquire or Release versions, to ensure that nothing from outside |
| 44 // the call is reordered between the operation and the memory barrier. This does |
| 45 // not change the code generated, so has no or minimal impact on the |
| 46 // NoBarrier operations. |
43 | 47 |
44 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, | 48 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, |
45 Atomic32 old_value, | 49 Atomic32 old_value, |
46 Atomic32 new_value) { | 50 Atomic32 new_value) { |
47 Atomic32 prev; | 51 Atomic32 prev; |
48 int32_t temp; | 52 int32_t temp; |
49 | 53 |
50 __asm__ __volatile__ ( // NOLINT | 54 __asm__ __volatile__ ( // NOLINT |
51 "0: \n\t" | 55 "0: \n\t" |
52 "ldxr %w[prev], %[ptr] \n\t" // Load the previous value. | 56 "ldxr %w[prev], %[ptr] \n\t" // Load the previous value. |
53 "cmp %w[prev], %w[old_value] \n\t" | 57 "cmp %w[prev], %w[old_value] \n\t" |
54 "bne 1f \n\t" | 58 "bne 1f \n\t" |
55 "stxr %w[temp], %w[new_value], %[ptr] \n\t" // Try to store the new value. | 59 "stxr %w[temp], %w[new_value], %[ptr] \n\t" // Try to store the new value. |
56 "cbnz %w[temp], 0b \n\t" // Retry if it did not work. | 60 "cbnz %w[temp], 0b \n\t" // Retry if it did not work. |
57 "1: \n\t" | 61 "1: \n\t" |
58 "clrex \n\t" // In case we didn't swap. | |
59 : [prev]"=&r" (prev), | 62 : [prev]"=&r" (prev), |
60 [temp]"=&r" (temp), | 63 [temp]"=&r" (temp), |
61 [ptr]"+Q" (*ptr) | 64 [ptr]"+Q" (*ptr) |
62 : [old_value]"r" (old_value), | 65 : [old_value]"IJr" (old_value), |
63 [new_value]"r" (new_value) | 66 [new_value]"r" (new_value) |
64 : "memory", "cc" | 67 : "cc", "memory" |
65 ); // NOLINT | 68 ); // NOLINT |
66 | 69 |
67 return prev; | 70 return prev; |
68 } | 71 } |
69 | 72 |
70 inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, | 73 inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, |
71 Atomic32 new_value) { | 74 Atomic32 new_value) { |
72 Atomic32 result; | 75 Atomic32 result; |
73 int32_t temp; | 76 int32_t temp; |
74 | 77 |
(...skipping 19 matching lines...) Expand all Loading... |
94 | 97 |
95 __asm__ __volatile__ ( // NOLINT | 98 __asm__ __volatile__ ( // NOLINT |
96 "0: \n\t" | 99 "0: \n\t" |
97 "ldxr %w[result], %[ptr] \n\t" // Load the previous value. | 100 "ldxr %w[result], %[ptr] \n\t" // Load the previous value. |
98 "add %w[result], %w[result], %w[increment]\n\t" | 101 "add %w[result], %w[result], %w[increment]\n\t" |
99 "stxr %w[temp], %w[result], %[ptr] \n\t" // Try to store the result. | 102 "stxr %w[temp], %w[result], %[ptr] \n\t" // Try to store the result. |
100 "cbnz %w[temp], 0b \n\t" // Retry on failure. | 103 "cbnz %w[temp], 0b \n\t" // Retry on failure. |
101 : [result]"=&r" (result), | 104 : [result]"=&r" (result), |
102 [temp]"=&r" (temp), | 105 [temp]"=&r" (temp), |
103 [ptr]"+Q" (*ptr) | 106 [ptr]"+Q" (*ptr) |
104 : [increment]"r" (increment) | 107 : [increment]"IJr" (increment) |
105 : "memory" | 108 : "memory" |
106 ); // NOLINT | 109 ); // NOLINT |
107 | 110 |
108 return result; | 111 return result; |
109 } | 112 } |
110 | 113 |
111 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, | 114 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, |
112 Atomic32 increment) { | 115 Atomic32 increment) { |
| 116 Atomic32 result; |
| 117 |
113 MemoryBarrier(); | 118 MemoryBarrier(); |
114 Atomic32 result = NoBarrier_AtomicIncrement(ptr, increment); | 119 result = NoBarrier_AtomicIncrement(ptr, increment); |
115 MemoryBarrier(); | 120 MemoryBarrier(); |
116 | 121 |
117 return result; | 122 return result; |
118 } | 123 } |
119 | 124 |
120 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, | 125 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, |
121 Atomic32 old_value, | 126 Atomic32 old_value, |
122 Atomic32 new_value) { | 127 Atomic32 new_value) { |
123 Atomic32 prev; | 128 Atomic32 prev; |
124 int32_t temp; | |
125 | 129 |
126 __asm__ __volatile__ ( // NOLINT | 130 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
127 "0: \n\t" | 131 MemoryBarrier(); |
128 "ldxr %w[prev], %[ptr] \n\t" // Load the previous value. | |
129 "cmp %w[prev], %w[old_value] \n\t" | |
130 "bne 1f \n\t" | |
131 "stxr %w[temp], %w[new_value], %[ptr] \n\t" // Try to store the new value. | |
132 "cbnz %w[temp], 0b \n\t" // Retry if it did not work. | |
133 "dmb ish \n\t" // Data memory barrier. | |
134 "1: \n\t" | |
135 // If the compare failed the 'dmb' is unnecessary, but we still need a | |
136 // 'clrex'. | |
137 "clrex \n\t" | |
138 : [prev]"=&r" (prev), | |
139 [temp]"=&r" (temp), | |
140 [ptr]"+Q" (*ptr) | |
141 : [old_value]"r" (old_value), | |
142 [new_value]"r" (new_value) | |
143 : "memory", "cc" | |
144 ); // NOLINT | |
145 | 132 |
146 return prev; | 133 return prev; |
147 } | 134 } |
148 | 135 |
149 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, | 136 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, |
150 Atomic32 old_value, | 137 Atomic32 old_value, |
151 Atomic32 new_value) { | 138 Atomic32 new_value) { |
152 Atomic32 prev; | 139 Atomic32 prev; |
153 int32_t temp; | |
154 | 140 |
155 MemoryBarrier(); | 141 MemoryBarrier(); |
156 | 142 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
157 __asm__ __volatile__ ( // NOLINT | |
158 "0: \n\t" | |
159 "ldxr %w[prev], %[ptr] \n\t" // Load the previous value. | |
160 "cmp %w[prev], %w[old_value] \n\t" | |
161 "bne 1f \n\t" | |
162 "stxr %w[temp], %w[new_value], %[ptr] \n\t" // Try to store the new value. | |
163 "cbnz %w[temp], 0b \n\t" // Retry if it did not work. | |
164 "1: \n\t" | |
165 // If the compare failed the we still need a 'clrex'. | |
166 "clrex \n\t" | |
167 : [prev]"=&r" (prev), | |
168 [temp]"=&r" (temp), | |
169 [ptr]"+Q" (*ptr) | |
170 : [old_value]"r" (old_value), | |
171 [new_value]"r" (new_value) | |
172 : "memory", "cc" | |
173 ); // NOLINT | |
174 | 143 |
175 return prev; | 144 return prev; |
176 } | 145 } |
177 | 146 |
178 inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { | 147 inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { |
179 *ptr = value; | 148 *ptr = value; |
180 } | 149 } |
181 | 150 |
182 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { | 151 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { |
183 *ptr = value; | 152 *ptr = value; |
184 MemoryBarrier(); | 153 MemoryBarrier(); |
185 } | 154 } |
186 | 155 |
187 inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { | 156 inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { |
188 MemoryBarrier(); | 157 __asm__ __volatile__ ( // NOLINT |
189 *ptr = value; | 158 "stlr %w[value], %[ptr] \n\t" |
| 159 : [ptr]"=Q" (*ptr) |
| 160 : [value]"r" (value) |
| 161 : "memory" |
| 162 ); // NOLINT |
190 } | 163 } |
191 | 164 |
192 inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { | 165 inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { |
193 return *ptr; | 166 return *ptr; |
194 } | 167 } |
195 | 168 |
196 inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { | 169 inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { |
197 Atomic32 value = *ptr; | 170 Atomic32 value; |
198 MemoryBarrier(); | 171 |
| 172 __asm__ __volatile__ ( // NOLINT |
| 173 "ldar %w[value], %[ptr] \n\t" |
| 174 : [value]"=r" (value) |
| 175 : [ptr]"Q" (*ptr) |
| 176 : "memory" |
| 177 ); // NOLINT |
| 178 |
199 return value; | 179 return value; |
200 } | 180 } |
201 | 181 |
202 inline Atomic32 Release_Load(volatile const Atomic32* ptr) { | 182 inline Atomic32 Release_Load(volatile const Atomic32* ptr) { |
203 MemoryBarrier(); | 183 MemoryBarrier(); |
204 return *ptr; | 184 return *ptr; |
205 } | 185 } |
206 | 186 |
207 // 64-bit versions of the operations. | 187 // 64-bit versions of the operations. |
208 // See the 32-bit versions for comments. | 188 // See the 32-bit versions for comments. |
209 | 189 |
210 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, | 190 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, |
211 Atomic64 old_value, | 191 Atomic64 old_value, |
212 Atomic64 new_value) { | 192 Atomic64 new_value) { |
213 Atomic64 prev; | 193 Atomic64 prev; |
214 int32_t temp; | 194 int32_t temp; |
215 | 195 |
216 __asm__ __volatile__ ( // NOLINT | 196 __asm__ __volatile__ ( // NOLINT |
217 "0: \n\t" | 197 "0: \n\t" |
218 "ldxr %[prev], %[ptr] \n\t" | 198 "ldxr %[prev], %[ptr] \n\t" |
219 "cmp %[prev], %[old_value] \n\t" | 199 "cmp %[prev], %[old_value] \n\t" |
220 "bne 1f \n\t" | 200 "bne 1f \n\t" |
221 "stxr %w[temp], %[new_value], %[ptr] \n\t" | 201 "stxr %w[temp], %[new_value], %[ptr] \n\t" |
222 "cbnz %w[temp], 0b \n\t" | 202 "cbnz %w[temp], 0b \n\t" |
223 "1: \n\t" | 203 "1: \n\t" |
224 "clrex \n\t" | |
225 : [prev]"=&r" (prev), | 204 : [prev]"=&r" (prev), |
226 [temp]"=&r" (temp), | 205 [temp]"=&r" (temp), |
227 [ptr]"+Q" (*ptr) | 206 [ptr]"+Q" (*ptr) |
228 : [old_value]"r" (old_value), | 207 : [old_value]"IJr" (old_value), |
229 [new_value]"r" (new_value) | 208 [new_value]"r" (new_value) |
230 : "memory", "cc" | 209 : "cc", "memory" |
231 ); // NOLINT | 210 ); // NOLINT |
232 | 211 |
233 return prev; | 212 return prev; |
234 } | 213 } |
235 | 214 |
236 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, | 215 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, |
237 Atomic64 new_value) { | 216 Atomic64 new_value) { |
238 Atomic64 result; | 217 Atomic64 result; |
239 int32_t temp; | 218 int32_t temp; |
240 | 219 |
(...skipping 19 matching lines...) Expand all Loading... |
260 | 239 |
261 __asm__ __volatile__ ( // NOLINT | 240 __asm__ __volatile__ ( // NOLINT |
262 "0: \n\t" | 241 "0: \n\t" |
263 "ldxr %[result], %[ptr] \n\t" | 242 "ldxr %[result], %[ptr] \n\t" |
264 "add %[result], %[result], %[increment] \n\t" | 243 "add %[result], %[result], %[increment] \n\t" |
265 "stxr %w[temp], %[result], %[ptr] \n\t" | 244 "stxr %w[temp], %[result], %[ptr] \n\t" |
266 "cbnz %w[temp], 0b \n\t" | 245 "cbnz %w[temp], 0b \n\t" |
267 : [result]"=&r" (result), | 246 : [result]"=&r" (result), |
268 [temp]"=&r" (temp), | 247 [temp]"=&r" (temp), |
269 [ptr]"+Q" (*ptr) | 248 [ptr]"+Q" (*ptr) |
270 : [increment]"r" (increment) | 249 : [increment]"IJr" (increment) |
271 : "memory" | 250 : "memory" |
272 ); // NOLINT | 251 ); // NOLINT |
273 | 252 |
274 return result; | 253 return result; |
275 } | 254 } |
276 | 255 |
277 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, | 256 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, |
278 Atomic64 increment) { | 257 Atomic64 increment) { |
| 258 Atomic64 result; |
| 259 |
279 MemoryBarrier(); | 260 MemoryBarrier(); |
280 Atomic64 result = NoBarrier_AtomicIncrement(ptr, increment); | 261 result = NoBarrier_AtomicIncrement(ptr, increment); |
281 MemoryBarrier(); | 262 MemoryBarrier(); |
282 | 263 |
283 return result; | 264 return result; |
284 } | 265 } |
285 | 266 |
286 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, | 267 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, |
287 Atomic64 old_value, | 268 Atomic64 old_value, |
288 Atomic64 new_value) { | 269 Atomic64 new_value) { |
289 Atomic64 prev; | 270 Atomic64 prev; |
290 int32_t temp; | |
291 | 271 |
292 __asm__ __volatile__ ( // NOLINT | 272 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
293 "0: \n\t" | 273 MemoryBarrier(); |
294 "ldxr %[prev], %[ptr] \n\t" | |
295 "cmp %[prev], %[old_value] \n\t" | |
296 "bne 1f \n\t" | |
297 "stxr %w[temp], %[new_value], %[ptr] \n\t" | |
298 "cbnz %w[temp], 0b \n\t" | |
299 "dmb ish \n\t" | |
300 "1: \n\t" | |
301 "clrex \n\t" | |
302 : [prev]"=&r" (prev), | |
303 [temp]"=&r" (temp), | |
304 [ptr]"+Q" (*ptr) | |
305 : [old_value]"r" (old_value), | |
306 [new_value]"r" (new_value) | |
307 : "memory", "cc" | |
308 ); // NOLINT | |
309 | 274 |
310 return prev; | 275 return prev; |
311 } | 276 } |
312 | 277 |
313 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, | 278 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, |
314 Atomic64 old_value, | 279 Atomic64 old_value, |
315 Atomic64 new_value) { | 280 Atomic64 new_value) { |
316 Atomic64 prev; | 281 Atomic64 prev; |
317 int32_t temp; | |
318 | 282 |
319 MemoryBarrier(); | 283 MemoryBarrier(); |
320 | 284 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
321 __asm__ __volatile__ ( // NOLINT | |
322 "0: \n\t" | |
323 "ldxr %[prev], %[ptr] \n\t" | |
324 "cmp %[prev], %[old_value] \n\t" | |
325 "bne 1f \n\t" | |
326 "stxr %w[temp], %[new_value], %[ptr] \n\t" | |
327 "cbnz %w[temp], 0b \n\t" | |
328 "1: \n\t" | |
329 "clrex \n\t" | |
330 : [prev]"=&r" (prev), | |
331 [temp]"=&r" (temp), | |
332 [ptr]"+Q" (*ptr) | |
333 : [old_value]"r" (old_value), | |
334 [new_value]"r" (new_value) | |
335 : "memory", "cc" | |
336 ); // NOLINT | |
337 | 285 |
338 return prev; | 286 return prev; |
339 } | 287 } |
340 | 288 |
341 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { | 289 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { |
342 *ptr = value; | 290 *ptr = value; |
343 } | 291 } |
344 | 292 |
345 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { | 293 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { |
346 *ptr = value; | 294 *ptr = value; |
347 MemoryBarrier(); | 295 MemoryBarrier(); |
348 } | 296 } |
349 | 297 |
350 inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { | 298 inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { |
351 MemoryBarrier(); | 299 __asm__ __volatile__ ( // NOLINT |
352 *ptr = value; | 300 "stlr %x[value], %[ptr] \n\t" |
| 301 : [ptr]"=Q" (*ptr) |
| 302 : [value]"r" (value) |
| 303 : "memory" |
| 304 ); // NOLINT |
353 } | 305 } |
354 | 306 |
355 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { | 307 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { |
356 return *ptr; | 308 return *ptr; |
357 } | 309 } |
358 | 310 |
359 inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { | 311 inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { |
360 Atomic64 value = *ptr; | 312 Atomic32 value; |
361 MemoryBarrier(); | 313 |
| 314 __asm__ __volatile__ ( // NOLINT |
| 315 "ldar %x[value], %[ptr] \n\t" |
| 316 : [value]"=r" (value) |
| 317 : [ptr]"Q" (*ptr) |
| 318 : "memory" |
| 319 ); // NOLINT |
| 320 |
362 return value; | 321 return value; |
363 } | 322 } |
364 | 323 |
365 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { | 324 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { |
366 MemoryBarrier(); | 325 MemoryBarrier(); |
367 return *ptr; | 326 return *ptr; |
368 } | 327 } |
369 | 328 |
370 } } // namespace v8::internal | 329 } } // namespace v8::internal |
371 | 330 |
372 #endif // V8_ATOMICOPS_INTERNALS_ARM_GCC_H_ | 331 #endif // V8_ATOMICOPS_INTERNALS_ARM_GCC_H_ |
OLD | NEW |