OLD | NEW |
(Empty) | |
| 1 // Copyright (c) 2011, Google Inc. |
| 2 // All rights reserved. |
| 3 // |
| 4 // Redistribution and use in source and binary forms, with or without |
| 5 // modification, are permitted provided that the following conditions are |
| 6 // met: |
| 7 // |
| 8 // * Redistributions of source code must retain the above copyright |
| 9 // notice, this list of conditions and the following disclaimer. |
| 10 // * Redistributions in binary form must reproduce the above |
| 11 // copyright notice, this list of conditions and the following disclaimer |
| 12 // in the documentation and/or other materials provided with the |
| 13 // distribution. |
| 14 // * Neither the name of Google Inc. nor the names of its |
| 15 // contributors may be used to endorse or promote products derived from |
| 16 // this software without specific prior written permission. |
| 17 // |
| 18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 29 // --- |
| 30 // |
| 31 // Author: Sasha Levitskiy |
| 32 // based on atomicops-internals by Sanjay Ghemawat |
| 33 // |
| 34 // This file is an internal atomic implementation, use base/atomicops.h instead. |
| 35 // |
| 36 // This code implements ARM atomics for architectures V6 and newer. |
| 37 |
| 38 #ifndef BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ |
| 39 #define BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ |
| 40 |
| 41 #include <stdio.h> |
| 42 #include <stdlib.h> |
| 43 #include "base/abort.h" |
| 44 #include "base/basictypes.h" // For COMPILE_ASSERT |
| 45 |
| 46 // The LDREXD and STREXD instructions in ARM all v7 variants or above. In v6, |
| 47 // only some variants support it. For simplicity, we only use exclusive |
| 48 // 64-bit load/store in V7 or above. |
| 49 #if defined(ARMV7) |
| 50 # define BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD |
| 51 #endif |
| 52 |
| 53 typedef int32_t Atomic32; |
| 54 |
| 55 namespace base { |
| 56 namespace subtle { |
| 57 |
| 58 typedef int64_t Atomic64; |
| 59 |
| 60 // 32-bit low-level ops |
| 61 |
| 62 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, |
| 63 Atomic32 old_value, |
| 64 Atomic32 new_value) { |
| 65 Atomic32 oldval, res; |
| 66 do { |
| 67 __asm__ __volatile__( |
| 68 "ldrex %1, [%3]\n" |
| 69 "mov %0, #0\n" |
| 70 "teq %1, %4\n" |
| 71 // The following IT (if-then) instruction is needed for the subsequent |
| 72 // conditional instruction STREXEQ when compiling in THUMB mode. |
| 73 // In ARM mode, the compiler/assembler will not generate any code for it. |
| 74 "it eq\n" |
| 75 "strexeq %0, %5, [%3]\n" |
| 76 : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) |
| 77 : "r" (ptr), "Ir" (old_value), "r" (new_value) |
| 78 : "cc"); |
| 79 } while (res); |
| 80 return oldval; |
| 81 } |
| 82 |
| 83 inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, |
| 84 Atomic32 new_value) { |
| 85 Atomic32 tmp, old; |
| 86 __asm__ __volatile__( |
| 87 "1:\n" |
| 88 "ldrex %1, [%2]\n" |
| 89 "strex %0, %3, [%2]\n" |
| 90 "teq %0, #0\n" |
| 91 "bne 1b" |
| 92 : "=&r" (tmp), "=&r" (old) |
| 93 : "r" (ptr), "r" (new_value) |
| 94 : "cc", "memory"); |
| 95 return old; |
| 96 } |
| 97 |
| 98 inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, |
| 99 Atomic32 increment) { |
| 100 Atomic32 tmp, res; |
| 101 __asm__ __volatile__( |
| 102 "1:\n" |
| 103 "ldrex %1, [%2]\n" |
| 104 "add %1, %1, %3\n" |
| 105 "strex %0, %1, [%2]\n" |
| 106 "teq %0, #0\n" |
| 107 "bne 1b" |
| 108 : "=&r" (tmp), "=&r"(res) |
| 109 : "r" (ptr), "r"(increment) |
| 110 : "cc", "memory"); |
| 111 return res; |
| 112 } |
| 113 |
| 114 inline void MemoryBarrier() { |
| 115 __asm__ __volatile__("dmb" : : : "memory"); |
| 116 } |
| 117 |
| 118 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, |
| 119 Atomic32 increment) { |
| 120 Atomic32 tmp, res; |
| 121 __asm__ __volatile__( |
| 122 "1:\n" |
| 123 "ldrex %1, [%2]\n" |
| 124 "add %1, %1, %3\n" |
| 125 "dmb\n" |
| 126 "strex %0, %1, [%2]\n" |
| 127 "teq %0, #0\n" |
| 128 "bne 1b" |
| 129 : "=&r" (tmp), "=&r"(res) |
| 130 : "r" (ptr), "r"(increment) |
| 131 : "cc", "memory"); |
| 132 return res; |
| 133 } |
| 134 |
| 135 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, |
| 136 Atomic32 old_value, |
| 137 Atomic32 new_value) { |
| 138 Atomic32 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
| 139 MemoryBarrier(); |
| 140 return value; |
| 141 } |
| 142 |
| 143 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, |
| 144 Atomic32 old_value, |
| 145 Atomic32 new_value) { |
| 146 MemoryBarrier(); |
| 147 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
| 148 } |
| 149 |
| 150 inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { |
| 151 *ptr = value; |
| 152 } |
| 153 |
| 154 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { |
| 155 *ptr = value; |
| 156 MemoryBarrier(); |
| 157 } |
| 158 |
| 159 inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { |
| 160 MemoryBarrier(); |
| 161 *ptr = value; |
| 162 } |
| 163 |
| 164 inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { |
| 165 return *ptr; |
| 166 } |
| 167 |
| 168 inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { |
| 169 Atomic32 value = *ptr; |
| 170 MemoryBarrier(); |
| 171 return value; |
| 172 } |
| 173 |
| 174 inline Atomic32 Release_Load(volatile const Atomic32* ptr) { |
| 175 MemoryBarrier(); |
| 176 return *ptr; |
| 177 } |
| 178 |
| 179 // 64-bit versions are only available if LDREXD and STREXD instructions |
| 180 // are available. |
| 181 #ifdef BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD |
| 182 |
| 183 #define BASE_HAS_ATOMIC64 1 |
| 184 |
| 185 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, |
| 186 Atomic64 old_value, |
| 187 Atomic64 new_value) { |
| 188 Atomic64 oldval, res; |
| 189 do { |
| 190 __asm__ __volatile__( |
| 191 "ldrexd %1, [%3]\n" |
| 192 "mov %0, #0\n" |
| 193 "teq %Q1, %Q4\n" |
| 194 // The following IT (if-then) instructions are needed for the subsequent |
| 195 // conditional instructions when compiling in THUMB mode. |
| 196 // In ARM mode, the compiler/assembler will not generate any code for it. |
| 197 "it eq\n" |
| 198 "teqeq %R1, %R4\n" |
| 199 "it eq\n" |
| 200 "strexdeq %0, %5, [%3]\n" |
| 201 : "=&r" (res), "=&r" (oldval), "+Q" (*ptr) |
| 202 : "r" (ptr), "Ir" (old_value), "r" (new_value) |
| 203 : "cc"); |
| 204 } while (res); |
| 205 return oldval; |
| 206 } |
| 207 |
| 208 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, |
| 209 Atomic64 new_value) { |
| 210 int store_failed; |
| 211 Atomic64 old; |
| 212 __asm__ __volatile__( |
| 213 "1:\n" |
| 214 "ldrexd %1, [%2]\n" |
| 215 "strexd %0, %3, [%2]\n" |
| 216 "teq %0, #0\n" |
| 217 "bne 1b" |
| 218 : "=&r" (store_failed), "=&r" (old) |
| 219 : "r" (ptr), "r" (new_value) |
| 220 : "cc", "memory"); |
| 221 return old; |
| 222 } |
| 223 |
| 224 inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, |
| 225 Atomic64 increment) { |
| 226 int store_failed; |
| 227 Atomic64 res; |
| 228 __asm__ __volatile__( |
| 229 "1:\n" |
| 230 "ldrexd %1, [%2]\n" |
| 231 "adds %Q1, %Q1, %Q3\n" |
| 232 "adc %R1, %R1, %R3\n" |
| 233 "strexd %0, %1, [%2]\n" |
| 234 "teq %0, #0\n" |
| 235 "bne 1b" |
| 236 : "=&r" (store_failed), "=&r"(res) |
| 237 : "r" (ptr), "r"(increment) |
| 238 : "cc", "memory"); |
| 239 return res; |
| 240 } |
| 241 |
| 242 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, |
| 243 Atomic64 increment) { |
| 244 int store_failed; |
| 245 Atomic64 res; |
| 246 __asm__ __volatile__( |
| 247 "1:\n" |
| 248 "ldrexd %1, [%2]\n" |
| 249 "adds %Q1, %Q1, %Q3\n" |
| 250 "adc %R1, %R1, %R3\n" |
| 251 "dmb\n" |
| 252 "strexd %0, %1, [%2]\n" |
| 253 "teq %0, #0\n" |
| 254 "bne 1b" |
| 255 : "=&r" (store_failed), "=&r"(res) |
| 256 : "r" (ptr), "r"(increment) |
| 257 : "cc", "memory"); |
| 258 return res; |
| 259 } |
| 260 |
| 261 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { |
| 262 int store_failed; |
| 263 Atomic64 dummy; |
| 264 __asm__ __volatile__( |
| 265 "1:\n" |
| 266 // Dummy load to lock cache line. |
| 267 "ldrexd %1, [%3]\n" |
| 268 "strexd %0, %2, [%3]\n" |
| 269 "teq %0, #0\n" |
| 270 "bne 1b" |
| 271 : "=&r" (store_failed), "=&r"(dummy) |
| 272 : "r"(value), "r" (ptr) |
| 273 : "cc", "memory"); |
| 274 } |
| 275 |
| 276 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { |
| 277 Atomic64 res; |
| 278 __asm__ __volatile__( |
| 279 "ldrexd %0, [%1]\n" |
| 280 "clrex\n" |
| 281 : "=r" (res) |
| 282 : "r"(ptr), "Q"(*ptr)); |
| 283 return res; |
| 284 } |
| 285 |
| 286 #else // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD |
| 287 |
| 288 inline void NotImplementedFatalError(const char *function_name) { |
| 289 fprintf(stderr, "64-bit %s() not implemented on this platform\n", |
| 290 function_name); |
| 291 tcmalloc::Abort(); |
| 292 } |
| 293 |
| 294 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, |
| 295 Atomic64 old_value, |
| 296 Atomic64 new_value) { |
| 297 NotImplementedFatalError("NoBarrier_CompareAndSwap"); |
| 298 return 0; |
| 299 } |
| 300 |
| 301 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, |
| 302 Atomic64 new_value) { |
| 303 NotImplementedFatalError("NoBarrier_AtomicExchange"); |
| 304 return 0; |
| 305 } |
| 306 |
| 307 inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, |
| 308 Atomic64 increment) { |
| 309 NotImplementedFatalError("NoBarrier_AtomicIncrement"); |
| 310 return 0; |
| 311 } |
| 312 |
| 313 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, |
| 314 Atomic64 increment) { |
| 315 NotImplementedFatalError("Barrier_AtomicIncrement"); |
| 316 return 0; |
| 317 } |
| 318 |
| 319 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { |
| 320 NotImplementedFatalError("NoBarrier_Store"); |
| 321 } |
| 322 |
| 323 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { |
| 324 NotImplementedFatalError("NoBarrier_Load"); |
| 325 return 0; |
| 326 } |
| 327 |
| 328 #endif // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD |
| 329 |
| 330 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { |
| 331 NoBarrier_Store(ptr, value); |
| 332 MemoryBarrier(); |
| 333 } |
| 334 |
| 335 inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { |
| 336 MemoryBarrier(); |
| 337 NoBarrier_Store(ptr, value); |
| 338 } |
| 339 |
| 340 inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { |
| 341 Atomic64 value = NoBarrier_Load(ptr); |
| 342 MemoryBarrier(); |
| 343 return value; |
| 344 } |
| 345 |
| 346 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { |
| 347 MemoryBarrier(); |
| 348 return NoBarrier_Load(ptr); |
| 349 } |
| 350 |
| 351 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, |
| 352 Atomic64 old_value, |
| 353 Atomic64 new_value) { |
| 354 Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
| 355 MemoryBarrier(); |
| 356 return value; |
| 357 } |
| 358 |
| 359 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, |
| 360 Atomic64 old_value, |
| 361 Atomic64 new_value) { |
| 362 MemoryBarrier(); |
| 363 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
| 364 } |
| 365 |
| 366 } // namespace subtle ends |
| 367 } // namespace base ends |
| 368 |
| 369 #endif // BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ |
OLD | NEW |