| OLD | NEW |
| 1 // Protocol Buffers - Google's data interchange format | 1 // Protocol Buffers - Google's data interchange format |
| 2 // Copyright 2012 Google Inc. All rights reserved. | 2 // Copyright 2012 Google Inc. All rights reserved. |
| 3 // https://developers.google.com/protocol-buffers/ | 3 // https://developers.google.com/protocol-buffers/ |
| 4 // | 4 // |
| 5 // Redistribution and use in source and binary forms, with or without | 5 // Redistribution and use in source and binary forms, with or without |
| 6 // modification, are permitted provided that the following conditions are | 6 // modification, are permitted provided that the following conditions are |
| 7 // met: | 7 // met: |
| 8 // | 8 // |
| 9 // * Redistributions of source code must retain the above copyright | 9 // * Redistributions of source code must retain the above copyright |
| 10 // notice, this list of conditions and the following disclaimer. | 10 // notice, this list of conditions and the following disclaimer. |
| (...skipping 28 matching lines...) Expand all Loading... |
| 39 | 39 |
| 40 // This struct is not part of the public API of this module; clients may not | 40 // This struct is not part of the public API of this module; clients may not |
| 41 // use it. | 41 // use it. |
| 42 // Features of this x86. Values may not be correct before main() is run, | 42 // Features of this x86. Values may not be correct before main() is run, |
| 43 // but are set conservatively. | 43 // but are set conservatively. |
| 44 struct AtomicOps_x86CPUFeatureStruct { | 44 struct AtomicOps_x86CPUFeatureStruct { |
| 45 bool has_amd_lock_mb_bug; // Processor has AMD memory-barrier bug; do lfence | 45 bool has_amd_lock_mb_bug; // Processor has AMD memory-barrier bug; do lfence |
| 46 // after acquire compare-and-swap. | 46 // after acquire compare-and-swap. |
| 47 bool has_sse2; // Processor has SSE2. | 47 bool has_sse2; // Processor has SSE2. |
| 48 }; | 48 }; |
| 49 extern struct AtomicOps_x86CPUFeatureStruct AtomicOps_Internalx86CPUFeatures; | 49 extern struct AtomicOps_x86CPUFeatureStruct cr_AtomicOps_Internalx86CPUFeatures; |
| 50 |
| 51 void AtomicOps_Internalx86CPUFeaturesInit(); |
| 50 | 52 |
| 51 #define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") | 53 #define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") |
| 52 | 54 |
| 53 // 32-bit low-level operations on any platform. | 55 // 32-bit low-level operations on any platform. |
| 54 | 56 |
| 55 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, | 57 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, |
| 56 Atomic32 old_value, | 58 Atomic32 old_value, |
| 57 Atomic32 new_value) { | 59 Atomic32 new_value) { |
| 58 Atomic32 prev; | 60 Atomic32 prev; |
| 59 __asm__ __volatile__("lock; cmpxchgl %1,%2" | 61 __asm__ __volatile__("lock; cmpxchgl %1,%2" |
| (...skipping 22 matching lines...) Expand all Loading... |
| 82 return temp + increment; | 84 return temp + increment; |
| 83 } | 85 } |
| 84 | 86 |
| 85 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, | 87 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, |
| 86 Atomic32 increment) { | 88 Atomic32 increment) { |
| 87 Atomic32 temp = increment; | 89 Atomic32 temp = increment; |
| 88 __asm__ __volatile__("lock; xaddl %0,%1" | 90 __asm__ __volatile__("lock; xaddl %0,%1" |
| 89 : "+r" (temp), "+m" (*ptr) | 91 : "+r" (temp), "+m" (*ptr) |
| 90 : : "memory"); | 92 : : "memory"); |
| 91 // temp now holds the old value of *ptr | 93 // temp now holds the old value of *ptr |
| 92 if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { | 94 if (cr_AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { |
| 93 __asm__ __volatile__("lfence" : : : "memory"); | 95 __asm__ __volatile__("lfence" : : : "memory"); |
| 94 } | 96 } |
| 95 return temp + increment; | 97 return temp + increment; |
| 96 } | 98 } |
| 97 | 99 |
| 98 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, | 100 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, |
| 99 Atomic32 old_value, | 101 Atomic32 old_value, |
| 100 Atomic32 new_value) { | 102 Atomic32 new_value) { |
| 101 Atomic32 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | 103 Atomic32 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
| 102 if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { | 104 if (cr_AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { |
| 103 __asm__ __volatile__("lfence" : : : "memory"); | 105 __asm__ __volatile__("lfence" : : : "memory"); |
| 104 } | 106 } |
| 105 return x; | 107 return x; |
| 106 } | 108 } |
| 107 | 109 |
| 108 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, | 110 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, |
| 109 Atomic32 old_value, | 111 Atomic32 old_value, |
| 110 Atomic32 new_value) { | 112 Atomic32 new_value) { |
| 111 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); | 113 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
| 112 } | 114 } |
| (...skipping 11 matching lines...) Expand all Loading... |
| 124 } | 126 } |
| 125 | 127 |
| 126 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { | 128 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { |
| 127 *ptr = value; | 129 *ptr = value; |
| 128 MemoryBarrier(); | 130 MemoryBarrier(); |
| 129 } | 131 } |
| 130 | 132 |
| 131 #else | 133 #else |
| 132 | 134 |
| 133 inline void MemoryBarrier() { | 135 inline void MemoryBarrier() { |
| 134 if (AtomicOps_Internalx86CPUFeatures.has_sse2) { | 136 if (cr_AtomicOps_Internalx86CPUFeatures.has_sse2) { |
| 135 __asm__ __volatile__("mfence" : : : "memory"); | 137 __asm__ __volatile__("mfence" : : : "memory"); |
| 136 } else { // mfence is faster but not present on PIII | 138 } else { // mfence is faster but not present on PIII |
| 137 Atomic32 x = 0; | 139 Atomic32 x = 0; |
| 138 NoBarrier_AtomicExchange(&x, 0); // acts as a barrier on PIII | 140 NoBarrier_AtomicExchange(&x, 0); // acts as a barrier on PIII |
| 139 } | 141 } |
| 140 } | 142 } |
| 141 | 143 |
| 142 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { | 144 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { |
| 143 if (AtomicOps_Internalx86CPUFeatures.has_sse2) { | 145 if (cr_AtomicOps_Internalx86CPUFeatures.has_sse2) { |
| 144 *ptr = value; | 146 *ptr = value; |
| 145 __asm__ __volatile__("mfence" : : : "memory"); | 147 __asm__ __volatile__("mfence" : : : "memory"); |
| 146 } else { | 148 } else { |
| 147 NoBarrier_AtomicExchange(ptr, value); | 149 NoBarrier_AtomicExchange(ptr, value); |
| 148 // acts as a barrier on PIII | 150 // acts as a barrier on PIII |
| 149 } | 151 } |
| 150 } | 152 } |
| 151 #endif | 153 #endif |
| 152 | 154 |
| 153 inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { | 155 inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { |
| (...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 206 return temp + increment; | 208 return temp + increment; |
| 207 } | 209 } |
| 208 | 210 |
| 209 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, | 211 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, |
| 210 Atomic64 increment) { | 212 Atomic64 increment) { |
| 211 Atomic64 temp = increment; | 213 Atomic64 temp = increment; |
| 212 __asm__ __volatile__("lock; xaddq %0,%1" | 214 __asm__ __volatile__("lock; xaddq %0,%1" |
| 213 : "+r" (temp), "+m" (*ptr) | 215 : "+r" (temp), "+m" (*ptr) |
| 214 : : "memory"); | 216 : : "memory"); |
| 215 // temp now contains the previous value of *ptr | 217 // temp now contains the previous value of *ptr |
| 216 if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { | 218 if (cr_AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { |
| 217 __asm__ __volatile__("lfence" : : : "memory"); | 219 __asm__ __volatile__("lfence" : : : "memory"); |
| 218 } | 220 } |
| 219 return temp + increment; | 221 return temp + increment; |
| 220 } | 222 } |
| 221 | 223 |
| 222 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { | 224 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { |
| 223 *ptr = value; | 225 *ptr = value; |
| 224 } | 226 } |
| 225 | 227 |
| 226 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { | 228 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { |
| (...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 263 | 265 |
| 264 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { | 266 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { |
| 265 MemoryBarrier(); | 267 MemoryBarrier(); |
| 266 return *ptr; | 268 return *ptr; |
| 267 } | 269 } |
| 268 | 270 |
| 269 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, | 271 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, |
| 270 Atomic64 old_value, | 272 Atomic64 old_value, |
| 271 Atomic64 new_value) { | 273 Atomic64 new_value) { |
| 272 Atomic64 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | 274 Atomic64 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
| 273 if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { | 275 if (cr_AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { |
| 274 __asm__ __volatile__("lfence" : : : "memory"); | 276 __asm__ __volatile__("lfence" : : : "memory"); |
| 275 } | 277 } |
| 276 return x; | 278 return x; |
| 277 } | 279 } |
| 278 | 280 |
| 279 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, | 281 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, |
| 280 Atomic64 old_value, | 282 Atomic64 old_value, |
| 281 Atomic64 new_value) { | 283 Atomic64 new_value) { |
| 282 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); | 284 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
| 283 } | 285 } |
| 284 | 286 |
| 285 #endif // defined(__x86_64__) | 287 #endif // defined(__x86_64__) |
| 286 | 288 |
| 287 } // namespace internal | 289 } // namespace internal |
| 288 } // namespace protobuf | 290 } // namespace protobuf |
| 289 } // namespace google | 291 } // namespace google |
| 290 | 292 |
| 291 #undef ATOMICOPS_COMPILER_BARRIER | 293 #undef ATOMICOPS_COMPILER_BARRIER |
| 292 | 294 |
| 293 #endif // GOOGLE_PROTOBUF_ATOMICOPS_INTERNALS_X86_GCC_H_ | 295 #endif // GOOGLE_PROTOBUF_ATOMICOPS_INTERNALS_X86_GCC_H_ |
| OLD | NEW |