OLD | NEW |
1 // Protocol Buffers - Google's data interchange format | 1 // Protocol Buffers - Google's data interchange format |
2 // Copyright 2012 Google Inc. All rights reserved. | 2 // Copyright 2012 Google Inc. All rights reserved. |
3 // https://developers.google.com/protocol-buffers/ | 3 // https://developers.google.com/protocol-buffers/ |
4 // | 4 // |
5 // Redistribution and use in source and binary forms, with or without | 5 // Redistribution and use in source and binary forms, with or without |
6 // modification, are permitted provided that the following conditions are | 6 // modification, are permitted provided that the following conditions are |
7 // met: | 7 // met: |
8 // | 8 // |
9 // * Redistributions of source code must retain the above copyright | 9 // * Redistributions of source code must retain the above copyright |
10 // notice, this list of conditions and the following disclaimer. | 10 // notice, this list of conditions and the following disclaimer. |
(...skipping 28 matching lines...) Expand all Loading... |
39 | 39 |
40 // This struct is not part of the public API of this module; clients may not | 40 // This struct is not part of the public API of this module; clients may not |
41 // use it. | 41 // use it. |
42 // Features of this x86. Values may not be correct before main() is run, | 42 // Features of this x86. Values may not be correct before main() is run, |
43 // but are set conservatively. | 43 // but are set conservatively. |
44 struct AtomicOps_x86CPUFeatureStruct { | 44 struct AtomicOps_x86CPUFeatureStruct { |
45 bool has_amd_lock_mb_bug; // Processor has AMD memory-barrier bug; do lfence | 45 bool has_amd_lock_mb_bug; // Processor has AMD memory-barrier bug; do lfence |
46 // after acquire compare-and-swap. | 46 // after acquire compare-and-swap. |
47 bool has_sse2; // Processor has SSE2. | 47 bool has_sse2; // Processor has SSE2. |
48 }; | 48 }; |
49 extern struct AtomicOps_x86CPUFeatureStruct AtomicOps_Internalx86CPUFeatures; | 49 extern struct AtomicOps_x86CPUFeatureStruct cr_AtomicOps_Internalx86CPUFeatures; |
| 50 |
| 51 void AtomicOps_Internalx86CPUFeaturesInit(); |
50 | 52 |
51 #define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") | 53 #define ATOMICOPS_COMPILER_BARRIER() __asm__ __volatile__("" : : : "memory") |
52 | 54 |
53 // 32-bit low-level operations on any platform. | 55 // 32-bit low-level operations on any platform. |
54 | 56 |
55 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, | 57 inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, |
56 Atomic32 old_value, | 58 Atomic32 old_value, |
57 Atomic32 new_value) { | 59 Atomic32 new_value) { |
58 Atomic32 prev; | 60 Atomic32 prev; |
59 __asm__ __volatile__("lock; cmpxchgl %1,%2" | 61 __asm__ __volatile__("lock; cmpxchgl %1,%2" |
(...skipping 22 matching lines...) Expand all Loading... |
82 return temp + increment; | 84 return temp + increment; |
83 } | 85 } |
84 | 86 |
85 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, | 87 inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, |
86 Atomic32 increment) { | 88 Atomic32 increment) { |
87 Atomic32 temp = increment; | 89 Atomic32 temp = increment; |
88 __asm__ __volatile__("lock; xaddl %0,%1" | 90 __asm__ __volatile__("lock; xaddl %0,%1" |
89 : "+r" (temp), "+m" (*ptr) | 91 : "+r" (temp), "+m" (*ptr) |
90 : : "memory"); | 92 : : "memory"); |
91 // temp now holds the old value of *ptr | 93 // temp now holds the old value of *ptr |
92 if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { | 94 if (cr_AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { |
93 __asm__ __volatile__("lfence" : : : "memory"); | 95 __asm__ __volatile__("lfence" : : : "memory"); |
94 } | 96 } |
95 return temp + increment; | 97 return temp + increment; |
96 } | 98 } |
97 | 99 |
98 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, | 100 inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, |
99 Atomic32 old_value, | 101 Atomic32 old_value, |
100 Atomic32 new_value) { | 102 Atomic32 new_value) { |
101 Atomic32 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | 103 Atomic32 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
102 if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { | 104 if (cr_AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { |
103 __asm__ __volatile__("lfence" : : : "memory"); | 105 __asm__ __volatile__("lfence" : : : "memory"); |
104 } | 106 } |
105 return x; | 107 return x; |
106 } | 108 } |
107 | 109 |
108 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, | 110 inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, |
109 Atomic32 old_value, | 111 Atomic32 old_value, |
110 Atomic32 new_value) { | 112 Atomic32 new_value) { |
111 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); | 113 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
112 } | 114 } |
(...skipping 11 matching lines...) Expand all Loading... |
124 } | 126 } |
125 | 127 |
126 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { | 128 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { |
127 *ptr = value; | 129 *ptr = value; |
128 MemoryBarrier(); | 130 MemoryBarrier(); |
129 } | 131 } |
130 | 132 |
131 #else | 133 #else |
132 | 134 |
133 inline void MemoryBarrier() { | 135 inline void MemoryBarrier() { |
134 if (AtomicOps_Internalx86CPUFeatures.has_sse2) { | 136 if (cr_AtomicOps_Internalx86CPUFeatures.has_sse2) { |
135 __asm__ __volatile__("mfence" : : : "memory"); | 137 __asm__ __volatile__("mfence" : : : "memory"); |
136 } else { // mfence is faster but not present on PIII | 138 } else { // mfence is faster but not present on PIII |
137 Atomic32 x = 0; | 139 Atomic32 x = 0; |
138 NoBarrier_AtomicExchange(&x, 0); // acts as a barrier on PIII | 140 NoBarrier_AtomicExchange(&x, 0); // acts as a barrier on PIII |
139 } | 141 } |
140 } | 142 } |
141 | 143 |
142 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { | 144 inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { |
143 if (AtomicOps_Internalx86CPUFeatures.has_sse2) { | 145 if (cr_AtomicOps_Internalx86CPUFeatures.has_sse2) { |
144 *ptr = value; | 146 *ptr = value; |
145 __asm__ __volatile__("mfence" : : : "memory"); | 147 __asm__ __volatile__("mfence" : : : "memory"); |
146 } else { | 148 } else { |
147 NoBarrier_AtomicExchange(ptr, value); | 149 NoBarrier_AtomicExchange(ptr, value); |
148 // acts as a barrier on PIII | 150 // acts as a barrier on PIII |
149 } | 151 } |
150 } | 152 } |
151 #endif | 153 #endif |
152 | 154 |
153 inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { | 155 inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
206 return temp + increment; | 208 return temp + increment; |
207 } | 209 } |
208 | 210 |
209 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, | 211 inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, |
210 Atomic64 increment) { | 212 Atomic64 increment) { |
211 Atomic64 temp = increment; | 213 Atomic64 temp = increment; |
212 __asm__ __volatile__("lock; xaddq %0,%1" | 214 __asm__ __volatile__("lock; xaddq %0,%1" |
213 : "+r" (temp), "+m" (*ptr) | 215 : "+r" (temp), "+m" (*ptr) |
214 : : "memory"); | 216 : : "memory"); |
215 // temp now contains the previous value of *ptr | 217 // temp now contains the previous value of *ptr |
216 if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { | 218 if (cr_AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { |
217 __asm__ __volatile__("lfence" : : : "memory"); | 219 __asm__ __volatile__("lfence" : : : "memory"); |
218 } | 220 } |
219 return temp + increment; | 221 return temp + increment; |
220 } | 222 } |
221 | 223 |
222 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { | 224 inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { |
223 *ptr = value; | 225 *ptr = value; |
224 } | 226 } |
225 | 227 |
226 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { | 228 inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
263 | 265 |
264 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { | 266 inline Atomic64 Release_Load(volatile const Atomic64* ptr) { |
265 MemoryBarrier(); | 267 MemoryBarrier(); |
266 return *ptr; | 268 return *ptr; |
267 } | 269 } |
268 | 270 |
269 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, | 271 inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, |
270 Atomic64 old_value, | 272 Atomic64 old_value, |
271 Atomic64 new_value) { | 273 Atomic64 new_value) { |
272 Atomic64 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); | 274 Atomic64 x = NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
273 if (AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { | 275 if (cr_AtomicOps_Internalx86CPUFeatures.has_amd_lock_mb_bug) { |
274 __asm__ __volatile__("lfence" : : : "memory"); | 276 __asm__ __volatile__("lfence" : : : "memory"); |
275 } | 277 } |
276 return x; | 278 return x; |
277 } | 279 } |
278 | 280 |
279 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, | 281 inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, |
280 Atomic64 old_value, | 282 Atomic64 old_value, |
281 Atomic64 new_value) { | 283 Atomic64 new_value) { |
282 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); | 284 return NoBarrier_CompareAndSwap(ptr, old_value, new_value); |
283 } | 285 } |
284 | 286 |
285 #endif // defined(__x86_64__) | 287 #endif // defined(__x86_64__) |
286 | 288 |
287 } // namespace internal | 289 } // namespace internal |
288 } // namespace protobuf | 290 } // namespace protobuf |
289 } // namespace google | 291 } // namespace google |
290 | 292 |
291 #undef ATOMICOPS_COMPILER_BARRIER | 293 #undef ATOMICOPS_COMPILER_BARRIER |
292 | 294 |
293 #endif // GOOGLE_PROTOBUF_ATOMICOPS_INTERNALS_X86_GCC_H_ | 295 #endif // GOOGLE_PROTOBUF_ATOMICOPS_INTERNALS_X86_GCC_H_ |
OLD | NEW |