OLD | NEW |
| (Empty) |
1 // Copyright (c) 2011, Google Inc. | |
2 // All rights reserved. | |
3 // | |
4 // Redistribution and use in source and binary forms, with or without | |
5 // modification, are permitted provided that the following conditions are | |
6 // met: | |
7 // | |
8 // * Redistributions of source code must retain the above copyright | |
9 // notice, this list of conditions and the following disclaimer. | |
10 // * Redistributions in binary form must reproduce the above | |
11 // copyright notice, this list of conditions and the following disclaimer | |
12 // in the documentation and/or other materials provided with the | |
13 // distribution. | |
14 // * Neither the name of Google Inc. nor the names of its | |
15 // contributors may be used to endorse or promote products derived from | |
16 // this software without specific prior written permission. | |
17 // | |
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
29 // --- | |
30 // | |
31 // Author: Sasha Levitskiy | |
32 // based on atomicops-internals by Sanjay Ghemawat | |
33 // | |
34 // This file is an internal atomic implementation, use base/atomicops.h instead. | |
35 // | |
36 // This code implements ARM atomics for architectures V6 and newer. | |
37 | |
38 #ifndef BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ | |
39 #define BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ | |
40 | |
41 #include <stdio.h> | |
42 #include <stdlib.h> | |
43 #include "base/abort.h" | |
44 #include "base/basictypes.h" // For COMPILE_ASSERT | |
45 | |
46 // The LDREXD and STREXD instructions in ARM all v7 variants or above. In v6, | |
47 // only some variants support it. For simplicity, we only use exclusive | |
48 // 64-bit load/store in V7 or above. | |
49 #if defined(ARMV7) | |
50 # define BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
51 #endif | |
52 | |
53 typedef int32_t Atomic32; | |
54 | |
55 namespace base { | |
56 namespace subtle { | |
57 | |
58 typedef int64_t Atomic64; | |
59 | |
60 // 32-bit low-level ops | |
61 | |
62 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, | |
63 Atomic32 old_value, | |
64 Atomic32 new_value) { | |
65 Atomic32 oldval, res; | |
66 do { | |
67 __asm__ __volatile__( | |
68 "ldrex %1, [%3]\n" | |
69 "mov %0, #0\n" | |
70 "teq %1, %4\n" | |
71 // The following IT (if-then) instruction is needed for the subsequent | |
72 // conditional instruction STREXEQ when compiling in THUMB mode. | |
73 // In ARM mode, the compiler/assembler will not generate any code for it. | |
74 "it eq\n" | |
75 "strexeq %0, %5, [%3]\n" | |
76 : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) | |
77 : "r" (ptr), "Ir" (old_value), "r" (new_value) | |
78 : "cc"); | |
79 } while (res); | |
80 return oldval; | |
81 } | |
82 | |
83 inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, | |
84 Atomic32 new_value) { | |
85 Atomic32 tmp, old; | |
86 __asm__ __volatile__( | |
87 "1:\n" | |
88 "ldrex %1, [%2]\n" | |
89 "strex %0, %3, [%2]\n" | |
90 "teq %0, #0\n" | |
91 "bne 1b" | |
92 : "=&r" (tmp), "=&r" (old) | |
93 : "r" (ptr), "r" (new_value) | |
94 : "cc", "memory"); | |
95 return old; | |
96 } | |
97 | |
98 inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, | |
99 Atomic32 increment) { | |
100 Atomic32 tmp, res; | |
101 __asm__ __volatile__( | |
102 "1:\n" | |
103 "ldrex %1, [%2]\n" | |
104 "add %1, %1, %3\n" | |
105 "strex %0, %1, [%2]\n" | |
106 "teq %0, #0\n" | |
107 "bne 1b" | |
108 : "=&r" (tmp), "=&r"(res) | |
109 : "r" (ptr), "r"(increment) | |
110 : "cc", "memory"); | |
111 return res; | |
112 } | |
113 | |
114 inline void MemoryBarrier() { | |
115 __asm__ __volatile__("dmb" : : : "memory"); | |
116 } | |
117 | |
118 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, | |
119 Atomic32 increment) { | |
120 Atomic32 tmp, res; | |
121 __asm__ __volatile__( | |
122 "1:\n" | |
123 "ldrex %1, [%2]\n" | |
124 "add %1, %1, %3\n" | |
125 "dmb\n" | |
126 "strex %0, %1, [%2]\n" | |
127 "teq %0, #0\n" | |
128 "bne 1b" | |
129 : "=&r" (tmp), "=&r"(res) | |
130 : "r" (ptr), "r"(increment) | |
131 : "cc", "memory"); | |
132 return res; | |
133 } | |
134 | |
135 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, | |
136 Atomic32 old_value, | |
137 Atomic32 new_value) { | |
138 Atomic32 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
139 MemoryBarrier(); | |
140 return value; | |
141 } | |
142 | |
143 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, | |
144 Atomic32 old_value, | |
145 Atomic32 new_value) { | |
146 MemoryBarrier(); | |
147 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
148 } | |
149 | |
150 inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { | |
151 *ptr = value; | |
152 } | |
153 | |
154 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { | |
155 *ptr = value; | |
156 MemoryBarrier(); | |
157 } | |
158 | |
159 inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { | |
160 MemoryBarrier(); | |
161 *ptr = value; | |
162 } | |
163 | |
164 inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { | |
165 return *ptr; | |
166 } | |
167 | |
168 inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { | |
169 Atomic32 value = *ptr; | |
170 MemoryBarrier(); | |
171 return value; | |
172 } | |
173 | |
174 inline Atomic32 Release_Load(volatile const Atomic32* ptr) { | |
175 MemoryBarrier(); | |
176 return *ptr; | |
177 } | |
178 | |
179 // 64-bit versions are only available if LDREXD and STREXD instructions | |
180 // are available. | |
181 #ifdef BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
182 | |
183 #define BASE_HAS_ATOMIC64 1 | |
184 | |
185 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, | |
186 Atomic64 old_value, | |
187 Atomic64 new_value) { | |
188 Atomic64 oldval, res; | |
189 do { | |
190 __asm__ __volatile__( | |
191 "ldrexd %1, [%3]\n" | |
192 "mov %0, #0\n" | |
193 "teq %Q1, %Q4\n" | |
194 // The following IT (if-then) instructions are needed for the subsequent | |
195 // conditional instructions when compiling in THUMB mode. | |
196 // In ARM mode, the compiler/assembler will not generate any code for it. | |
197 "it eq\n" | |
198 "teqeq %R1, %R4\n" | |
199 "it eq\n" | |
200 "strexdeq %0, %5, [%3]\n" | |
201 : "=&r" (res), "=&r" (oldval), "+Q" (*ptr) | |
202 : "r" (ptr), "Ir" (old_value), "r" (new_value) | |
203 : "cc"); | |
204 } while (res); | |
205 return oldval; | |
206 } | |
207 | |
208 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, | |
209 Atomic64 new_value) { | |
210 int store_failed; | |
211 Atomic64 old; | |
212 __asm__ __volatile__( | |
213 "1:\n" | |
214 "ldrexd %1, [%2]\n" | |
215 "strexd %0, %3, [%2]\n" | |
216 "teq %0, #0\n" | |
217 "bne 1b" | |
218 : "=&r" (store_failed), "=&r" (old) | |
219 : "r" (ptr), "r" (new_value) | |
220 : "cc", "memory"); | |
221 return old; | |
222 } | |
223 | |
224 inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, | |
225 Atomic64 increment) { | |
226 int store_failed; | |
227 Atomic64 res; | |
228 __asm__ __volatile__( | |
229 "1:\n" | |
230 "ldrexd %1, [%2]\n" | |
231 "adds %Q1, %Q1, %Q3\n" | |
232 "adc %R1, %R1, %R3\n" | |
233 "strexd %0, %1, [%2]\n" | |
234 "teq %0, #0\n" | |
235 "bne 1b" | |
236 : "=&r" (store_failed), "=&r"(res) | |
237 : "r" (ptr), "r"(increment) | |
238 : "cc", "memory"); | |
239 return res; | |
240 } | |
241 | |
242 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, | |
243 Atomic64 increment) { | |
244 int store_failed; | |
245 Atomic64 res; | |
246 __asm__ __volatile__( | |
247 "1:\n" | |
248 "ldrexd %1, [%2]\n" | |
249 "adds %Q1, %Q1, %Q3\n" | |
250 "adc %R1, %R1, %R3\n" | |
251 "dmb\n" | |
252 "strexd %0, %1, [%2]\n" | |
253 "teq %0, #0\n" | |
254 "bne 1b" | |
255 : "=&r" (store_failed), "=&r"(res) | |
256 : "r" (ptr), "r"(increment) | |
257 : "cc", "memory"); | |
258 return res; | |
259 } | |
260 | |
261 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { | |
262 int store_failed; | |
263 Atomic64 dummy; | |
264 __asm__ __volatile__( | |
265 "1:\n" | |
266 // Dummy load to lock cache line. | |
267 "ldrexd %1, [%3]\n" | |
268 "strexd %0, %2, [%3]\n" | |
269 "teq %0, #0\n" | |
270 "bne 1b" | |
271 : "=&r" (store_failed), "=&r"(dummy) | |
272 : "r"(value), "r" (ptr) | |
273 : "cc", "memory"); | |
274 } | |
275 | |
276 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { | |
277 Atomic64 res; | |
278 __asm__ __volatile__( | |
279 "ldrexd %0, [%1]\n" | |
280 "clrex\n" | |
281 : "=r" (res) | |
282 : "r"(ptr), "Q"(*ptr)); | |
283 return res; | |
284 } | |
285 | |
286 #else // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
287 | |
288 inline void NotImplementedFatalError(const char *function_name) { | |
289 fprintf(stderr, "64-bit %s() not implemented on this platform\n", | |
290 function_name); | |
291 tcmalloc::Abort(); | |
292 } | |
293 | |
294 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, | |
295 Atomic64 old_value, | |
296 Atomic64 new_value) { | |
297 NotImplementedFatalError("NoBarrier_CompareAndSwap"); | |
298 return 0; | |
299 } | |
300 | |
301 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, | |
302 Atomic64 new_value) { | |
303 NotImplementedFatalError("NoBarrier_AtomicExchange"); | |
304 return 0; | |
305 } | |
306 | |
307 inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, | |
308 Atomic64 increment) { | |
309 NotImplementedFatalError("NoBarrier_AtomicIncrement"); | |
310 return 0; | |
311 } | |
312 | |
313 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, | |
314 Atomic64 increment) { | |
315 NotImplementedFatalError("Barrier_AtomicIncrement"); | |
316 return 0; | |
317 } | |
318 | |
319 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { | |
320 NotImplementedFatalError("NoBarrier_Store"); | |
321 } | |
322 | |
323 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { | |
324 NotImplementedFatalError("NoBarrier_Load"); | |
325 return 0; | |
326 } | |
327 | |
328 #endif // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
329 | |
330 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { | |
331 NoBarrier_Store(ptr, value); | |
332 MemoryBarrier(); | |
333 } | |
334 | |
335 inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { | |
336 MemoryBarrier(); | |
337 NoBarrier_Store(ptr, value); | |
338 } | |
339 | |
340 inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { | |
341 Atomic64 value = NoBarrier_Load(ptr); | |
342 MemoryBarrier(); | |
343 return value; | |
344 } | |
345 | |
346 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { | |
347 MemoryBarrier(); | |
348 return NoBarrier_Load(ptr); | |
349 } | |
350 | |
351 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, | |
352 Atomic64 old_value, | |
353 Atomic64 new_value) { | |
354 Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
355 MemoryBarrier(); | |
356 return value; | |
357 } | |
358 | |
359 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, | |
360 Atomic64 old_value, | |
361 Atomic64 new_value) { | |
362 MemoryBarrier(); | |
363 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
364 } | |
365 | |
366 } // namespace subtle ends | |
367 } // namespace base ends | |
368 | |
369 #endif // BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ | |
OLD | NEW |