| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2011, Google Inc. | |
| 2 // All rights reserved. | |
| 3 // | |
| 4 // Redistribution and use in source and binary forms, with or without | |
| 5 // modification, are permitted provided that the following conditions are | |
| 6 // met: | |
| 7 // | |
| 8 // * Redistributions of source code must retain the above copyright | |
| 9 // notice, this list of conditions and the following disclaimer. | |
| 10 // * Redistributions in binary form must reproduce the above | |
| 11 // copyright notice, this list of conditions and the following disclaimer | |
| 12 // in the documentation and/or other materials provided with the | |
| 13 // distribution. | |
| 14 // * Neither the name of Google Inc. nor the names of its | |
| 15 // contributors may be used to endorse or promote products derived from | |
| 16 // this software without specific prior written permission. | |
| 17 // | |
| 18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 29 // --- | |
| 30 // | |
| 31 // Author: Sasha Levitskiy | |
| 32 // based on atomicops-internals by Sanjay Ghemawat | |
| 33 // | |
| 34 // This file is an internal atomic implementation, use base/atomicops.h instead. | |
| 35 // | |
| 36 // This code implements ARM atomics for architectures V6 and newer. | |
| 37 | |
| 38 #ifndef BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ | |
| 39 #define BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ | |
| 40 | |
| 41 #include <stdio.h> | |
| 42 #include <stdlib.h> | |
| 43 #include "base/abort.h" | |
| 44 #include "base/basictypes.h" // For COMPILE_ASSERT | |
| 45 | |
| 46 // The LDREXD and STREXD instructions in ARM all v7 variants or above. In v6, | |
| 47 // only some variants support it. For simplicity, we only use exclusive | |
| 48 // 64-bit load/store in V7 or above. | |
| 49 #if defined(ARMV7) | |
| 50 # define BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
| 51 #endif | |
| 52 | |
| 53 typedef int32_t Atomic32; | |
| 54 | |
| 55 namespace base { | |
| 56 namespace subtle { | |
| 57 | |
| 58 typedef int64_t Atomic64; | |
| 59 | |
| 60 // 32-bit low-level ops | |
| 61 | |
| 62 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, | |
| 63 Atomic32 old_value, | |
| 64 Atomic32 new_value) { | |
| 65 Atomic32 oldval, res; | |
| 66 do { | |
| 67 __asm__ __volatile__( | |
| 68 "ldrex %1, [%3]\n" | |
| 69 "mov %0, #0\n" | |
| 70 "teq %1, %4\n" | |
| 71 // The following IT (if-then) instruction is needed for the subsequent | |
| 72 // conditional instruction STREXEQ when compiling in THUMB mode. | |
| 73 // In ARM mode, the compiler/assembler will not generate any code for it. | |
| 74 "it eq\n" | |
| 75 "strexeq %0, %5, [%3]\n" | |
| 76 : "=&r" (res), "=&r" (oldval), "+Qo" (*ptr) | |
| 77 : "r" (ptr), "Ir" (old_value), "r" (new_value) | |
| 78 : "cc"); | |
| 79 } while (res); | |
| 80 return oldval; | |
| 81 } | |
| 82 | |
| 83 inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, | |
| 84 Atomic32 new_value) { | |
| 85 Atomic32 tmp, old; | |
| 86 __asm__ __volatile__( | |
| 87 "1:\n" | |
| 88 "ldrex %1, [%2]\n" | |
| 89 "strex %0, %3, [%2]\n" | |
| 90 "teq %0, #0\n" | |
| 91 "bne 1b" | |
| 92 : "=&r" (tmp), "=&r" (old) | |
| 93 : "r" (ptr), "r" (new_value) | |
| 94 : "cc", "memory"); | |
| 95 return old; | |
| 96 } | |
| 97 | |
| 98 inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, | |
| 99 Atomic32 increment) { | |
| 100 Atomic32 tmp, res; | |
| 101 __asm__ __volatile__( | |
| 102 "1:\n" | |
| 103 "ldrex %1, [%2]\n" | |
| 104 "add %1, %1, %3\n" | |
| 105 "strex %0, %1, [%2]\n" | |
| 106 "teq %0, #0\n" | |
| 107 "bne 1b" | |
| 108 : "=&r" (tmp), "=&r"(res) | |
| 109 : "r" (ptr), "r"(increment) | |
| 110 : "cc", "memory"); | |
| 111 return res; | |
| 112 } | |
| 113 | |
| 114 inline void MemoryBarrier() { | |
| 115 __asm__ __volatile__("dmb" : : : "memory"); | |
| 116 } | |
| 117 | |
| 118 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, | |
| 119 Atomic32 increment) { | |
| 120 Atomic32 tmp, res; | |
| 121 __asm__ __volatile__( | |
| 122 "1:\n" | |
| 123 "ldrex %1, [%2]\n" | |
| 124 "add %1, %1, %3\n" | |
| 125 "dmb\n" | |
| 126 "strex %0, %1, [%2]\n" | |
| 127 "teq %0, #0\n" | |
| 128 "bne 1b" | |
| 129 : "=&r" (tmp), "=&r"(res) | |
| 130 : "r" (ptr), "r"(increment) | |
| 131 : "cc", "memory"); | |
| 132 return res; | |
| 133 } | |
| 134 | |
| 135 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, | |
| 136 Atomic32 old_value, | |
| 137 Atomic32 new_value) { | |
| 138 Atomic32 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
| 139 MemoryBarrier(); | |
| 140 return value; | |
| 141 } | |
| 142 | |
| 143 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, | |
| 144 Atomic32 old_value, | |
| 145 Atomic32 new_value) { | |
| 146 MemoryBarrier(); | |
| 147 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
| 148 } | |
| 149 | |
| 150 inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { | |
| 151 *ptr = value; | |
| 152 } | |
| 153 | |
| 154 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { | |
| 155 *ptr = value; | |
| 156 MemoryBarrier(); | |
| 157 } | |
| 158 | |
| 159 inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { | |
| 160 MemoryBarrier(); | |
| 161 *ptr = value; | |
| 162 } | |
| 163 | |
| 164 inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { | |
| 165 return *ptr; | |
| 166 } | |
| 167 | |
| 168 inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { | |
| 169 Atomic32 value = *ptr; | |
| 170 MemoryBarrier(); | |
| 171 return value; | |
| 172 } | |
| 173 | |
| 174 inline Atomic32 Release_Load(volatile const Atomic32* ptr) { | |
| 175 MemoryBarrier(); | |
| 176 return *ptr; | |
| 177 } | |
| 178 | |
| 179 // 64-bit versions are only available if LDREXD and STREXD instructions | |
| 180 // are available. | |
| 181 #ifdef BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
| 182 | |
| 183 #define BASE_HAS_ATOMIC64 1 | |
| 184 | |
| 185 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, | |
| 186 Atomic64 old_value, | |
| 187 Atomic64 new_value) { | |
| 188 Atomic64 oldval, res; | |
| 189 do { | |
| 190 __asm__ __volatile__( | |
| 191 "ldrexd %1, [%3]\n" | |
| 192 "mov %0, #0\n" | |
| 193 "teq %Q1, %Q4\n" | |
| 194 // The following IT (if-then) instructions are needed for the subsequent | |
| 195 // conditional instructions when compiling in THUMB mode. | |
| 196 // In ARM mode, the compiler/assembler will not generate any code for it. | |
| 197 "it eq\n" | |
| 198 "teqeq %R1, %R4\n" | |
| 199 "it eq\n" | |
| 200 "strexdeq %0, %5, [%3]\n" | |
| 201 : "=&r" (res), "=&r" (oldval), "+Q" (*ptr) | |
| 202 : "r" (ptr), "Ir" (old_value), "r" (new_value) | |
| 203 : "cc"); | |
| 204 } while (res); | |
| 205 return oldval; | |
| 206 } | |
| 207 | |
| 208 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, | |
| 209 Atomic64 new_value) { | |
| 210 int store_failed; | |
| 211 Atomic64 old; | |
| 212 __asm__ __volatile__( | |
| 213 "1:\n" | |
| 214 "ldrexd %1, [%2]\n" | |
| 215 "strexd %0, %3, [%2]\n" | |
| 216 "teq %0, #0\n" | |
| 217 "bne 1b" | |
| 218 : "=&r" (store_failed), "=&r" (old) | |
| 219 : "r" (ptr), "r" (new_value) | |
| 220 : "cc", "memory"); | |
| 221 return old; | |
| 222 } | |
| 223 | |
| 224 inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, | |
| 225 Atomic64 increment) { | |
| 226 int store_failed; | |
| 227 Atomic64 res; | |
| 228 __asm__ __volatile__( | |
| 229 "1:\n" | |
| 230 "ldrexd %1, [%2]\n" | |
| 231 "adds %Q1, %Q1, %Q3\n" | |
| 232 "adc %R1, %R1, %R3\n" | |
| 233 "strexd %0, %1, [%2]\n" | |
| 234 "teq %0, #0\n" | |
| 235 "bne 1b" | |
| 236 : "=&r" (store_failed), "=&r"(res) | |
| 237 : "r" (ptr), "r"(increment) | |
| 238 : "cc", "memory"); | |
| 239 return res; | |
| 240 } | |
| 241 | |
| 242 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, | |
| 243 Atomic64 increment) { | |
| 244 int store_failed; | |
| 245 Atomic64 res; | |
| 246 __asm__ __volatile__( | |
| 247 "1:\n" | |
| 248 "ldrexd %1, [%2]\n" | |
| 249 "adds %Q1, %Q1, %Q3\n" | |
| 250 "adc %R1, %R1, %R3\n" | |
| 251 "dmb\n" | |
| 252 "strexd %0, %1, [%2]\n" | |
| 253 "teq %0, #0\n" | |
| 254 "bne 1b" | |
| 255 : "=&r" (store_failed), "=&r"(res) | |
| 256 : "r" (ptr), "r"(increment) | |
| 257 : "cc", "memory"); | |
| 258 return res; | |
| 259 } | |
| 260 | |
| 261 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { | |
| 262 int store_failed; | |
| 263 Atomic64 dummy; | |
| 264 __asm__ __volatile__( | |
| 265 "1:\n" | |
| 266 // Dummy load to lock cache line. | |
| 267 "ldrexd %1, [%3]\n" | |
| 268 "strexd %0, %2, [%3]\n" | |
| 269 "teq %0, #0\n" | |
| 270 "bne 1b" | |
| 271 : "=&r" (store_failed), "=&r"(dummy) | |
| 272 : "r"(value), "r" (ptr) | |
| 273 : "cc", "memory"); | |
| 274 } | |
| 275 | |
| 276 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { | |
| 277 Atomic64 res; | |
| 278 __asm__ __volatile__( | |
| 279 "ldrexd %0, [%1]\n" | |
| 280 "clrex\n" | |
| 281 : "=r" (res) | |
| 282 : "r"(ptr), "Q"(*ptr)); | |
| 283 return res; | |
| 284 } | |
| 285 | |
| 286 #else // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
| 287 | |
| 288 inline void NotImplementedFatalError(const char *function_name) { | |
| 289 fprintf(stderr, "64-bit %s() not implemented on this platform\n", | |
| 290 function_name); | |
| 291 tcmalloc::Abort(); | |
| 292 } | |
| 293 | |
| 294 inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, | |
| 295 Atomic64 old_value, | |
| 296 Atomic64 new_value) { | |
| 297 NotImplementedFatalError("NoBarrier_CompareAndSwap"); | |
| 298 return 0; | |
| 299 } | |
| 300 | |
| 301 inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, | |
| 302 Atomic64 new_value) { | |
| 303 NotImplementedFatalError("NoBarrier_AtomicExchange"); | |
| 304 return 0; | |
| 305 } | |
| 306 | |
| 307 inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, | |
| 308 Atomic64 increment) { | |
| 309 NotImplementedFatalError("NoBarrier_AtomicIncrement"); | |
| 310 return 0; | |
| 311 } | |
| 312 | |
| 313 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, | |
| 314 Atomic64 increment) { | |
| 315 NotImplementedFatalError("Barrier_AtomicIncrement"); | |
| 316 return 0; | |
| 317 } | |
| 318 | |
| 319 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { | |
| 320 NotImplementedFatalError("NoBarrier_Store"); | |
| 321 } | |
| 322 | |
| 323 inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { | |
| 324 NotImplementedFatalError("NoBarrier_Load"); | |
| 325 return 0; | |
| 326 } | |
| 327 | |
| 328 #endif // BASE_ATOMICOPS_HAS_LDREXD_AND_STREXD | |
| 329 | |
| 330 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { | |
| 331 NoBarrier_Store(ptr, value); | |
| 332 MemoryBarrier(); | |
| 333 } | |
| 334 | |
| 335 inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { | |
| 336 MemoryBarrier(); | |
| 337 NoBarrier_Store(ptr, value); | |
| 338 } | |
| 339 | |
| 340 inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { | |
| 341 Atomic64 value = NoBarrier_Load(ptr); | |
| 342 MemoryBarrier(); | |
| 343 return value; | |
| 344 } | |
| 345 | |
| 346 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { | |
| 347 MemoryBarrier(); | |
| 348 return NoBarrier_Load(ptr); | |
| 349 } | |
| 350 | |
| 351 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, | |
| 352 Atomic64 old_value, | |
| 353 Atomic64 new_value) { | |
| 354 Atomic64 value = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
| 355 MemoryBarrier(); | |
| 356 return value; | |
| 357 } | |
| 358 | |
| 359 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, | |
| 360 Atomic64 old_value, | |
| 361 Atomic64 new_value) { | |
| 362 MemoryBarrier(); | |
| 363 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); | |
| 364 } | |
| 365 | |
| 366 } // namespace subtle ends | |
| 367 } // namespace base ends | |
| 368 | |
| 369 #endif // BASE_ATOMICOPS_INTERNALS_ARM_V6PLUS_H_ | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1549913002:
tcmalloc: Use C++11 atomics where appropriate. (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master