OLD | NEW |
| (Empty) |
1 // Copyright 2014 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 // This file is an internal atomic implementation, use base/atomicops.h instead. | |
6 | |
7 // TODO(rmcilroy): Investigate whether we can use __sync__ intrinsics instead of | |
8 // the hand coded assembly without introducing perf regressions. | |
9 // TODO(rmcilroy): Investigate whether we can use acquire / release versions of | |
10 // exclusive load / store assembly instructions and do away with | |
11 // the barriers. | |
12 | |
13 #ifndef BASE_ATOMICOPS_INTERNALS_ARM64_GCC_H_ | |
14 #define BASE_ATOMICOPS_INTERNALS_ARM64_GCC_H_ | |
15 | |
16 #if defined(OS_QNX) | |
17 #include <sys/cpuinline.h> | |
18 #endif | |
19 | |
20 namespace base { | |
21 namespace subtle { | |
22 | |
23 inline void MemoryBarrier() { | |
24 __asm__ __volatile__ ("dmb ish" ::: "memory"); // NOLINT | |
25 } | |
26 | |
27 // NoBarrier versions of the operation include "memory" in the clobber list. | |
28 // This is not required for direct usage of the NoBarrier versions of the | |
29 // operations. However this is required for correctness when they are used as | |
30 // part of the Acquire or Release versions, to ensure that nothing from outside | |
31 // the call is reordered between the operation and the memory barrier. This does | |
32 // not change the code generated, so has no or minimal impact on the | |
33 // NoBarrier operations. | |
34 | |
35 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, | |
36 Atomic32 old_value, | |
37 Atomic32 new_value) { | |
38 Atomic32 prev; | |
39 int32_t temp; | |
40 | |
41 __asm__ __volatile__ ( // NOLINT | |
42 "0: \n\t" | |
43 "ldxr %w[prev], %[ptr] \n\t" // Load the previous value. | |
44 "cmp %w[prev], %w[old_value] \n\t" | |
45 "bne 1f \n\t" | |
46 "stxr %w[temp], %w[new_value], %[ptr] \n\t" // Try to store the new value. | |
47 "cbnz %w[temp], 0b \n\t" // Retry if it did not work. | |
48 "1: \n\t" | |
49 : [prev]"=&r" (prev), | |
50 [temp]"=&r" (temp), | |
51 [ptr]"+Q" (*ptr) | |
52 : [old_value]"IJr" (old_value), | |
53 [new_value]"r" (new_value) | |
54 : "cc", "memory" | |
55 ); // NOLINT | |
56 | |
57 return prev; | |
58 } | |
59 | |
60 inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, | |
61 Atomic32 new_value) { | |
62 Atomic32 result; | |
63 int32_t temp; | |
64 | |
65 __asm__ __volatile__ ( // NOLINT | |
66 "0: \n\t" | |
67 "ldxr %w[result], %[ptr] \n\t" // Load the previous value. | |
68 "stxr %w[temp], %w[new_value], %[ptr] \n\t" // Try to store the new value. | |
69 "cbnz %w[temp], 0b \n\t" // Retry if it did not work. | |
70 : [result]"=&r" (result), | |
71 [temp]"=&r" (temp), | |
72 [ptr]"+Q" (*ptr) | |
73 : [new_value]"r" (new_value) | |
74 : "memory" | |
75 ); // NOLINT | |
76 | |
77 return result; | |
78 } | |
79 | |
80 inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, | |
81 Atomic32 increment) { | |
82 Atomic32 result; | |
83 int32_t temp; | |
84 | |
85 __asm__ __volatile__ ( // NOLINT | |
86 "0: \n\t" | |
87 "ldxr %w[result], %[ptr] \n\t" // Load the previous value. | |
88 "add %w[result], %w[result], %w[increment]\n\t" | |
89 "stxr %w[temp], %w[result], %[ptr] \n\t" // Try to store the result. | |
90 "cbnz %w[temp], 0b \n\t" // Retry on failure. | |
91 : [result]"=&r" (result), | |
92 [temp]"=&r" (temp), | |
93 [ptr]"+Q" (*ptr) | |
94 : [increment]"IJr" (increment) | |
95 : "memory" | |
96 ); // NOLINT | |
97 | |
98 return result; | |
99 } | |
100 | |
101 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, | |
102 Atomic32 increment) { | |
103 MemoryBarrier(); | |
104 Atomic32 result = NoBarrier_AtomicIncrement(ptr, increment); | |
105 MemoryBarrier(); | |
106 | |
107 return result; | |
108 } | |
109 | |
110 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, | |
111 Atomic32 old_value, | |
112 Atomic32 new_value) { | |
113 Atomic32 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
114 MemoryBarrier(); | |
115 | |
116 return prev; | |
117 } | |
118 | |
119 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, | |
120 Atomic32 old_value, | |
121 Atomic32 new_value) { | |
122 MemoryBarrier(); | |
123 Atomic32 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
124 | |
125 return prev; | |
126 } | |
127 | |
128 inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { | |
129 *ptr = value; | |
130 } | |
131 | |
132 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { | |
133 *ptr = value; | |
134 MemoryBarrier(); | |
135 } | |
136 | |
137 inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { | |
138 __asm__ __volatile__ ( // NOLINT | |
139 "stlr %w[value], %[ptr] \n\t" | |
140 : [ptr]"=Q" (*ptr) | |
141 : [value]"r" (value) | |
142 : "memory" | |
143 ); // NOLINT | |
144 } | |
145 | |
146 inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { | |
147 return *ptr; | |
148 } | |
149 | |
150 inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { | |
151 Atomic32 value; | |
152 | |
153 __asm__ __volatile__ ( // NOLINT | |
154 "ldar %w[value], %[ptr] \n\t" | |
155 : [value]"=r" (value) | |
156 : [ptr]"Q" (*ptr) | |
157 : "memory" | |
158 ); // NOLINT | |
159 | |
160 return value; | |
161 } | |
162 | |
163 inline Atomic32 Release_Load(volatile const Atomic32* ptr) { | |
164 MemoryBarrier(); | |
165 return *ptr; | |
166 } | |
167 | |
168 // 64-bit versions of the operations. | |
169 // See the 32-bit versions for comments. | |
170 | |
171 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, | |
172 Atomic64 old_value, | |
173 Atomic64 new_value) { | |
174 Atomic64 prev; | |
175 int32_t temp; | |
176 | |
177 __asm__ __volatile__ ( // NOLINT | |
178 "0: \n\t" | |
179 "ldxr %[prev], %[ptr] \n\t" | |
180 "cmp %[prev], %[old_value] \n\t" | |
181 "bne 1f \n\t" | |
182 "stxr %w[temp], %[new_value], %[ptr] \n\t" | |
183 "cbnz %w[temp], 0b \n\t" | |
184 "1: \n\t" | |
185 : [prev]"=&r" (prev), | |
186 [temp]"=&r" (temp), | |
187 [ptr]"+Q" (*ptr) | |
188 : [old_value]"IJr" (old_value), | |
189 [new_value]"r" (new_value) | |
190 : "cc", "memory" | |
191 ); // NOLINT | |
192 | |
193 return prev; | |
194 } | |
195 | |
196 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, | |
197 Atomic64 new_value) { | |
198 Atomic64 result; | |
199 int32_t temp; | |
200 | |
201 __asm__ __volatile__ ( // NOLINT | |
202 "0: \n\t" | |
203 "ldxr %[result], %[ptr] \n\t" | |
204 "stxr %w[temp], %[new_value], %[ptr] \n\t" | |
205 "cbnz %w[temp], 0b \n\t" | |
206 : [result]"=&r" (result), | |
207 [temp]"=&r" (temp), | |
208 [ptr]"+Q" (*ptr) | |
209 : [new_value]"r" (new_value) | |
210 : "memory" | |
211 ); // NOLINT | |
212 | |
213 return result; | |
214 } | |
215 | |
216 inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, | |
217 Atomic64 increment) { | |
218 Atomic64 result; | |
219 int32_t temp; | |
220 | |
221 __asm__ __volatile__ ( // NOLINT | |
222 "0: \n\t" | |
223 "ldxr %[result], %[ptr] \n\t" | |
224 "add %[result], %[result], %[increment] \n\t" | |
225 "stxr %w[temp], %[result], %[ptr] \n\t" | |
226 "cbnz %w[temp], 0b \n\t" | |
227 : [result]"=&r" (result), | |
228 [temp]"=&r" (temp), | |
229 [ptr]"+Q" (*ptr) | |
230 : [increment]"IJr" (increment) | |
231 : "memory" | |
232 ); // NOLINT | |
233 | |
234 return result; | |
235 } | |
236 | |
237 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, | |
238 Atomic64 increment) { | |
239 MemoryBarrier(); | |
240 Atomic64 result = NoBarrier_AtomicIncrement(ptr, increment); | |
241 MemoryBarrier(); | |
242 | |
243 return result; | |
244 } | |
245 | |
246 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, | |
247 Atomic64 old_value, | |
248 Atomic64 new_value) { | |
249 Atomic64 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
250 MemoryBarrier(); | |
251 | |
252 return prev; | |
253 } | |
254 | |
255 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, | |
256 Atomic64 old_value, | |
257 Atomic64 new_value) { | |
258 MemoryBarrier(); | |
259 Atomic64 prev = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
260 | |
261 return prev; | |
262 } | |
263 | |
264 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { | |
265 *ptr = value; | |
266 } | |
267 | |
268 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { | |
269 *ptr = value; | |
270 MemoryBarrier(); | |
271 } | |
272 | |
273 inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { | |
274 __asm__ __volatile__ ( // NOLINT | |
275 "stlr %x[value], %[ptr] \n\t" | |
276 : [ptr]"=Q" (*ptr) | |
277 : [value]"r" (value) | |
278 : "memory" | |
279 ); // NOLINT | |
280 } | |
281 | |
282 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { | |
283 return *ptr; | |
284 } | |
285 | |
286 inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { | |
287 Atomic64 value; | |
288 | |
289 __asm__ __volatile__ ( // NOLINT | |
290 "ldar %x[value], %[ptr] \n\t" | |
291 : [value]"=r" (value) | |
292 : [ptr]"Q" (*ptr) | |
293 : "memory" | |
294 ); // NOLINT | |
295 | |
296 return value; | |
297 } | |
298 | |
299 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { | |
300 MemoryBarrier(); | |
301 return *ptr; | |
302 } | |
303 | |
304 } // namespace subtle | |
305 } // namespace base | |
306 | |
307 #endif // BASE_ATOMICOPS_INTERNALS_ARM64_GCC_H_ | |
OLD | NEW |