| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (C) 2007, 2008, 2010, 2012 Apple Inc. All rights reserved. | 2 * Copyright (C) 2007, 2008, 2010, 2012 Apple Inc. All rights reserved. |
| 3 * Copyright (C) 2007 Justin Haygood (jhaygood@reaktix.com) | 3 * Copyright (C) 2007 Justin Haygood (jhaygood@reaktix.com) |
| 4 * | 4 * |
| 5 * Redistribution and use in source and binary forms, with or without | 5 * Redistribution and use in source and binary forms, with or without |
| 6 * modification, are permitted provided that the following conditions | 6 * modification, are permitted provided that the following conditions |
| 7 * are met: | 7 * are met: |
| 8 * | 8 * |
| 9 * 1. Redistributions of source code must retain the above copyright | 9 * 1. Redistributions of source code must retain the above copyright |
| 10 * notice, this list of conditions and the following disclaimer. | 10 * notice, this list of conditions and the following disclaimer. |
| (...skipping 17 matching lines...) Expand all Loading... |
| 28 */ | 28 */ |
| 29 | 29 |
| 30 #ifndef Atomics_h | 30 #ifndef Atomics_h |
| 31 #define Atomics_h | 31 #define Atomics_h |
| 32 | 32 |
| 33 #include "wtf/Assertions.h" | 33 #include "wtf/Assertions.h" |
| 34 #include "wtf/CPU.h" | 34 #include "wtf/CPU.h" |
| 35 | 35 |
| 36 #include <stdint.h> | 36 #include <stdint.h> |
| 37 | 37 |
| 38 #if COMPILER(MSVC) | |
| 39 #include <windows.h> | |
| 40 #endif | |
| 41 | |
| 42 #if defined(THREAD_SANITIZER) | 38 #if defined(THREAD_SANITIZER) |
| 43 #include <sanitizer/tsan_interface_atomic.h> | 39 #include <sanitizer/tsan_interface_atomic.h> |
| 44 #endif | 40 #endif |
| 45 | 41 |
| 46 namespace WTF { | 42 namespace WTF { |
| 47 | 43 |
| 48 #if COMPILER(MSVC) | |
| 49 | |
| 50 // atomicAdd returns the result of the addition. | |
| 51 ALWAYS_INLINE int atomicAdd(int volatile* addend, int increment) | |
| 52 { | |
| 53 return InterlockedExchangeAdd(reinterpret_cast<long volatile*>(addend), stat
ic_cast<long>(increment)) + increment; | |
| 54 } | |
| 55 | |
| 56 // atomicSubtract returns the result of the subtraction. | |
| 57 ALWAYS_INLINE int atomicSubtract(int volatile* addend, int decrement) | |
| 58 { | |
| 59 return InterlockedExchangeAdd(reinterpret_cast<long volatile*>(addend), stat
ic_cast<long>(-decrement)) - decrement; | |
| 60 } | |
| 61 | |
| 62 ALWAYS_INLINE int atomicIncrement(int volatile* addend) { return InterlockedIncr
ement(reinterpret_cast<long volatile*>(addend)); } | |
| 63 ALWAYS_INLINE int atomicDecrement(int volatile* addend) { return InterlockedDecr
ement(reinterpret_cast<long volatile*>(addend)); } | |
| 64 | |
| 65 ALWAYS_INLINE int64_t atomicIncrement(int64_t volatile* addend) { return Interlo
ckedIncrement64(reinterpret_cast<long long volatile*>(addend)); } | |
| 66 ALWAYS_INLINE int64_t atomicDecrement(int64_t volatile* addend) { return Interlo
ckedDecrement64(reinterpret_cast<long long volatile*>(addend)); } | |
| 67 | |
| 68 ALWAYS_INLINE int atomicTestAndSetToOne(int volatile* ptr) | |
| 69 { | |
| 70 int ret = InterlockedExchange(reinterpret_cast<long volatile*>(ptr), 1); | |
| 71 ASSERT(!ret || ret == 1); | |
| 72 return ret; | |
| 73 } | |
| 74 | |
| 75 ALWAYS_INLINE void atomicSetOneToZero(int volatile* ptr) | |
| 76 { | |
| 77 ASSERT(*ptr == 1); | |
| 78 InterlockedExchange(reinterpret_cast<long volatile*>(ptr), 0); | |
| 79 } | |
| 80 | |
| 81 #else | |
| 82 | |
| 83 // atomicAdd returns the result of the addition. | 44 // atomicAdd returns the result of the addition. |
| 84 ALWAYS_INLINE int atomicAdd(int volatile* addend, int increment) { return __sync
_add_and_fetch(addend, increment); } | 45 ALWAYS_INLINE int atomicAdd(int volatile* addend, int increment) { return __sync
_add_and_fetch(addend, increment); } |
| 85 // atomicSubtract returns the result of the subtraction. | 46 // atomicSubtract returns the result of the subtraction. |
| 86 ALWAYS_INLINE int atomicSubtract(int volatile* addend, int decrement) { return _
_sync_sub_and_fetch(addend, decrement); } | 47 ALWAYS_INLINE int atomicSubtract(int volatile* addend, int decrement) { return _
_sync_sub_and_fetch(addend, decrement); } |
| 87 | 48 |
| 88 ALWAYS_INLINE int atomicIncrement(int volatile* addend) { return atomicAdd(adden
d, 1); } | 49 ALWAYS_INLINE int atomicIncrement(int volatile* addend) { return atomicAdd(adden
d, 1); } |
| 89 ALWAYS_INLINE int atomicDecrement(int volatile* addend) { return atomicSubtract(
addend, 1); } | 50 ALWAYS_INLINE int atomicDecrement(int volatile* addend) { return atomicSubtract(
addend, 1); } |
| 90 | 51 |
| 91 ALWAYS_INLINE int64_t atomicIncrement(int64_t volatile* addend) { return __sync_
add_and_fetch(addend, 1); } | 52 ALWAYS_INLINE int64_t atomicIncrement(int64_t volatile* addend) { return __sync_
add_and_fetch(addend, 1); } |
| 92 ALWAYS_INLINE int64_t atomicDecrement(int64_t volatile* addend) { return __sync_
sub_and_fetch(addend, 1); } | 53 ALWAYS_INLINE int64_t atomicDecrement(int64_t volatile* addend) { return __sync_
sub_and_fetch(addend, 1); } |
| 93 | 54 |
| 94 ALWAYS_INLINE int atomicTestAndSetToOne(int volatile* ptr) | 55 ALWAYS_INLINE int atomicTestAndSetToOne(int volatile* ptr) |
| 95 { | 56 { |
| 96 int ret = __sync_lock_test_and_set(ptr, 1); | 57 int ret = __sync_lock_test_and_set(ptr, 1); |
| 97 ASSERT(!ret || ret == 1); | 58 ASSERT(!ret || ret == 1); |
| 98 return ret; | 59 return ret; |
| 99 } | 60 } |
| 100 | 61 |
| 101 ALWAYS_INLINE void atomicSetOneToZero(int volatile* ptr) | 62 ALWAYS_INLINE void atomicSetOneToZero(int volatile* ptr) |
| 102 { | 63 { |
| 103 ASSERT(*ptr == 1); | 64 ASSERT(*ptr == 1); |
| 104 __sync_lock_release(ptr); | 65 __sync_lock_release(ptr); |
| 105 } | 66 } |
| 106 #endif | |
| 107 | 67 |
| 108 #if defined(THREAD_SANITIZER) | 68 #if defined(THREAD_SANITIZER) |
| 109 ALWAYS_INLINE void releaseStore(volatile int* ptr, int value) | 69 ALWAYS_INLINE void releaseStore(volatile int* ptr, int value) |
| 110 { | 70 { |
| 111 __tsan_atomic32_store(ptr, value, __tsan_memory_order_release); | 71 __tsan_atomic32_store(ptr, value, __tsan_memory_order_release); |
| 112 } | 72 } |
| 113 | 73 |
| 114 ALWAYS_INLINE int acquireLoad(volatile const int* ptr) | 74 ALWAYS_INLINE int acquireLoad(volatile const int* ptr) |
| 115 { | 75 { |
| 116 return __tsan_atomic32_load(ptr, __tsan_memory_order_acquire); | 76 return __tsan_atomic32_load(ptr, __tsan_memory_order_acquire); |
| 117 } | 77 } |
| 118 | 78 |
| 119 ALWAYS_INLINE void releaseStore(volatile unsigned* ptr, unsigned value) | 79 ALWAYS_INLINE void releaseStore(volatile unsigned* ptr, unsigned value) |
| 120 { | 80 { |
| 121 __tsan_atomic32_store(reinterpret_cast<volatile int*>(ptr), static_cast<int>
(value), __tsan_memory_order_release); | 81 __tsan_atomic32_store(reinterpret_cast<volatile int*>(ptr), static_cast<int>
(value), __tsan_memory_order_release); |
| 122 } | 82 } |
| 123 | 83 |
| 124 ALWAYS_INLINE unsigned acquireLoad(volatile const unsigned* ptr) | 84 ALWAYS_INLINE unsigned acquireLoad(volatile const unsigned* ptr) |
| 125 { | 85 { |
| 126 return static_cast<unsigned>(__tsan_atomic32_load(reinterpret_cast<volatile
const int*>(ptr), __tsan_memory_order_acquire)); | 86 return static_cast<unsigned>(__tsan_atomic32_load(reinterpret_cast<volatile
const int*>(ptr), __tsan_memory_order_acquire)); |
| 127 } | 87 } |
| 128 #else | 88 #else |
| 129 | 89 |
| 130 #if CPU(X86) || CPU(X86_64) | 90 #if CPU(X86) || CPU(X86_64) |
| 131 // Only compiler barrier is needed. | 91 // Only compiler barrier is needed. |
| 132 #if COMPILER(MSVC) | |
| 133 // Starting from Visual Studio 2005 compiler guarantees acquire and release | |
| 134 // semantics for operations on volatile variables. See MSDN entry for | |
| 135 // MemoryBarrier macro. | |
| 136 #define MEMORY_BARRIER() | |
| 137 #else | |
| 138 #define MEMORY_BARRIER() __asm__ __volatile__("" : : : "memory") | 92 #define MEMORY_BARRIER() __asm__ __volatile__("" : : : "memory") |
| 139 #endif | |
| 140 #elif CPU(ARM) && (OS(LINUX) || OS(ANDROID)) | 93 #elif CPU(ARM) && (OS(LINUX) || OS(ANDROID)) |
| 141 // On ARM __sync_synchronize generates dmb which is very expensive on single | 94 // On ARM __sync_synchronize generates dmb which is very expensive on single |
| 142 // core devices which don't actually need it. Avoid the cost by calling into | 95 // core devices which don't actually need it. Avoid the cost by calling into |
| 143 // kuser_memory_barrier helper. | 96 // kuser_memory_barrier helper. |
| 144 inline void memoryBarrier() | 97 inline void memoryBarrier() |
| 145 { | 98 { |
| 146 // Note: This is a function call, which is also an implicit compiler barrier
. | 99 // Note: This is a function call, which is also an implicit compiler barrier
. |
| 147 typedef void (*KernelMemoryBarrierFunc)(); | 100 typedef void (*KernelMemoryBarrierFunc)(); |
| 148 ((KernelMemoryBarrierFunc)0xffff0fa0)(); | 101 ((KernelMemoryBarrierFunc)0xffff0fa0)(); |
| 149 } | 102 } |
| (...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 188 using WTF::atomicAdd; | 141 using WTF::atomicAdd; |
| 189 using WTF::atomicSubtract; | 142 using WTF::atomicSubtract; |
| 190 using WTF::atomicDecrement; | 143 using WTF::atomicDecrement; |
| 191 using WTF::atomicIncrement; | 144 using WTF::atomicIncrement; |
| 192 using WTF::atomicTestAndSetToOne; | 145 using WTF::atomicTestAndSetToOne; |
| 193 using WTF::atomicSetOneToZero; | 146 using WTF::atomicSetOneToZero; |
| 194 using WTF::acquireLoad; | 147 using WTF::acquireLoad; |
| 195 using WTF::releaseStore; | 148 using WTF::releaseStore; |
| 196 | 149 |
| 197 #endif // Atomics_h | 150 #endif // Atomics_h |
| OLD | NEW |