OLD | NEW |
1 // Protocol Buffers - Google's data interchange format | 1 // Protocol Buffers - Google's data interchange format |
2 // Copyright 2008 Google Inc. All rights reserved. | 2 // Copyright 2008 Google Inc. All rights reserved. |
3 // https://developers.google.com/protocol-buffers/ | 3 // https://developers.google.com/protocol-buffers/ |
4 // | 4 // |
5 // Redistribution and use in source and binary forms, with or without | 5 // Redistribution and use in source and binary forms, with or without |
6 // modification, are permitted provided that the following conditions are | 6 // modification, are permitted provided that the following conditions are |
7 // met: | 7 // met: |
8 // | 8 // |
9 // * Redistributions of source code must retain the above copyright | 9 // * Redistributions of source code must retain the above copyright |
10 // notice, this list of conditions and the following disclaimer. | 10 // notice, this list of conditions and the following disclaimer. |
(...skipping 21 matching lines...) Expand all Loading... |
32 | 32 |
33 | 33 |
34 #ifdef ADDRESS_SANITIZER | 34 #ifdef ADDRESS_SANITIZER |
35 #include <sanitizer/asan_interface.h> | 35 #include <sanitizer/asan_interface.h> |
36 #endif | 36 #endif |
37 | 37 |
38 namespace google { | 38 namespace google { |
39 namespace protobuf { | 39 namespace protobuf { |
40 | 40 |
41 | 41 |
| 42 google::protobuf::internal::SequenceNumber Arena::lifecycle_id_generator_; |
42 #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) | 43 #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) |
43 Arena::ThreadCache& Arena::cr_thread_cache() { | 44 Arena::ThreadCache& Arena::thread_cache() { |
44 static internal::ThreadLocalStorage<ThreadCache>* thread_cache_ = | 45 static internal::ThreadLocalStorage<ThreadCache>* thread_cache_ = |
45 new internal::ThreadLocalStorage<ThreadCache>(); | 46 new internal::ThreadLocalStorage<ThreadCache>(); |
46 return *thread_cache_->Get(); | 47 return *thread_cache_->Get(); |
47 } | 48 } |
48 #elif !defined(PROTOBUF_USE_DLLS) | 49 #elif defined(PROTOBUF_USE_DLLS) |
| 50 Arena::ThreadCache& Arena::thread_cache() { |
| 51 static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_ = { -1, NULL }; |
| 52 return thread_cache_; |
| 53 } |
| 54 #else |
49 GOOGLE_THREAD_LOCAL Arena::ThreadCache Arena::thread_cache_ = { -1, NULL }; | 55 GOOGLE_THREAD_LOCAL Arena::ThreadCache Arena::thread_cache_ = { -1, NULL }; |
50 #endif | 56 #endif |
51 | 57 |
52 void Arena::Init() { | 58 void Arena::Init() { |
53 lifecycle_id_ = internal::cr_lifecycle_id_generator_.GetNext(); | 59 lifecycle_id_ = lifecycle_id_generator_.GetNext(); |
54 blocks_ = 0; | 60 blocks_ = 0; |
55 hint_ = 0; | 61 hint_ = 0; |
56 owns_first_block_ = true; | 62 owns_first_block_ = true; |
57 cleanup_list_ = 0; | 63 cleanup_list_ = 0; |
58 | 64 |
59 if (options_.initial_block != NULL && options_.initial_block_size > 0) { | 65 if (options_.initial_block != NULL && options_.initial_block_size > 0) { |
60 GOOGLE_CHECK_GE(options_.initial_block_size, sizeof(Block)) | 66 GOOGLE_CHECK_GE(options_.initial_block_size, sizeof(Block)) |
61 << ": Initial block size too small for header."; | 67 << ": Initial block size too small for header."; |
62 | 68 |
63 // Add first unowned block to list. | 69 // Add first unowned block to list. |
64 Block* first_block = reinterpret_cast<Block*>(options_.initial_block); | 70 Block* first_block = reinterpret_cast<Block*>(options_.initial_block); |
65 first_block->size = options_.initial_block_size; | 71 first_block->size = options_.initial_block_size; |
66 first_block->pos = kHeaderSize; | 72 first_block->pos = kHeaderSize; |
67 first_block->next = NULL; | 73 first_block->next = NULL; |
68 // Thread which calls Init() owns the first block. This allows the | 74 // Thread which calls Init() owns the first block. This allows the |
69 // single-threaded case to allocate on the first block without taking any | 75 // single-threaded case to allocate on the first block without taking any |
70 // locks. | 76 // locks. |
71 first_block->owner = &cr_thread_cache(); | 77 first_block->owner = &thread_cache(); |
72 SetThreadCacheBlock(first_block); | 78 SetThreadCacheBlock(first_block); |
73 AddBlockInternal(first_block); | 79 AddBlockInternal(first_block); |
74 owns_first_block_ = false; | 80 owns_first_block_ = false; |
75 } | 81 } |
76 | 82 |
77 // Call the initialization hook | 83 // Call the initialization hook |
78 if (options_.on_arena_init != NULL) { | 84 if (options_.on_arena_init != NULL) { |
79 hooks_cookie_ = options_.on_arena_init(this); | 85 hooks_cookie_ = options_.on_arena_init(this); |
80 } else { | 86 } else { |
81 hooks_cookie_ = NULL; | 87 hooks_cookie_ = NULL; |
82 } | 88 } |
83 } | 89 } |
84 | 90 |
85 Arena::~Arena() { | 91 Arena::~Arena() { |
86 uint64 space_allocated = ResetInternal(); | 92 uint64 space_allocated = ResetInternal(); |
87 | 93 |
88 // Call the destruction hook | 94 // Call the destruction hook |
89 if (options_.on_arena_destruction != NULL) { | 95 if (options_.on_arena_destruction != NULL) { |
90 options_.on_arena_destruction(this, hooks_cookie_, space_allocated); | 96 options_.on_arena_destruction(this, hooks_cookie_, space_allocated); |
91 } | 97 } |
92 } | 98 } |
93 | 99 |
94 uint64 Arena::Reset() { | 100 uint64 Arena::Reset() { |
95 // Invalidate any ThreadCaches pointing to any blocks we just destroyed. | 101 // Invalidate any ThreadCaches pointing to any blocks we just destroyed. |
96 lifecycle_id_ = internal::cr_lifecycle_id_generator_.GetNext(); | 102 lifecycle_id_ = lifecycle_id_generator_.GetNext(); |
97 return ResetInternal(); | 103 return ResetInternal(); |
98 } | 104 } |
99 | 105 |
100 uint64 Arena::ResetInternal() { | 106 uint64 Arena::ResetInternal() { |
101 CleanupList(); | 107 CleanupList(); |
102 uint64 space_allocated = FreeBlocks(); | 108 uint64 space_allocated = FreeBlocks(); |
103 | 109 |
104 // Call the reset hook | 110 // Call the reset hook |
105 if (options_.on_arena_reset != NULL) { | 111 if (options_.on_arena_reset != NULL) { |
106 options_.on_arena_reset(this, hooks_cookie_, space_allocated); | 112 options_.on_arena_reset(this, hooks_cookie_, space_allocated); |
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
171 | 177 |
172 // Monitor allocation if needed. | 178 // Monitor allocation if needed. |
173 if (GOOGLE_PREDICT_FALSE(hooks_cookie_ != NULL) && | 179 if (GOOGLE_PREDICT_FALSE(hooks_cookie_ != NULL) && |
174 options_.on_arena_allocation != NULL) { | 180 options_.on_arena_allocation != NULL) { |
175 options_.on_arena_allocation(allocated, n, hooks_cookie_); | 181 options_.on_arena_allocation(allocated, n, hooks_cookie_); |
176 } | 182 } |
177 | 183 |
178 // If this thread already owns a block in this arena then try to use that. | 184 // If this thread already owns a block in this arena then try to use that. |
179 // This fast path optimizes the case where multiple threads allocate from the | 185 // This fast path optimizes the case where multiple threads allocate from the |
180 // same arena. | 186 // same arena. |
181 if (cr_thread_cache().last_lifecycle_id_seen == lifecycle_id_ && | 187 if (thread_cache().last_lifecycle_id_seen == lifecycle_id_ && |
182 cr_thread_cache().last_block_used_ != NULL) { | 188 thread_cache().last_block_used_ != NULL) { |
183 if (cr_thread_cache().last_block_used_->avail() < n) { | 189 if (thread_cache().last_block_used_->avail() < n) { |
184 return SlowAlloc(n); | 190 return SlowAlloc(n); |
185 } | 191 } |
186 return AllocFromBlock(cr_thread_cache().last_block_used_, n); | 192 return AllocFromBlock(thread_cache().last_block_used_, n); |
187 } | 193 } |
188 | 194 |
189 // Check whether we own the last accessed block on this arena. | 195 // Check whether we own the last accessed block on this arena. |
190 // This fast path optimizes the case where a single thread uses multiple | 196 // This fast path optimizes the case where a single thread uses multiple |
191 // arenas. | 197 // arenas. |
192 void* me = &cr_thread_cache(); | 198 void* me = &thread_cache(); |
193 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&
hint_)); | 199 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&
hint_)); |
194 if (!b || b->owner != me || b->avail() < n) { | 200 if (!b || b->owner != me || b->avail() < n) { |
195 return SlowAlloc(n); | 201 return SlowAlloc(n); |
196 } | 202 } |
197 return AllocFromBlock(b, n); | 203 return AllocFromBlock(b, n); |
198 } | 204 } |
199 | 205 |
200 void* Arena::AllocFromBlock(Block* b, size_t n) { | 206 void* Arena::AllocFromBlock(Block* b, size_t n) { |
201 size_t p = b->pos; | 207 size_t p = b->pos; |
202 b->pos = p + n; | 208 b->pos = p + n; |
203 #ifdef ADDRESS_SANITIZER | 209 #ifdef ADDRESS_SANITIZER |
204 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b) + p, n); | 210 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b) + p, n); |
205 #endif | 211 #endif |
206 return reinterpret_cast<char*>(b) + p; | 212 return reinterpret_cast<char*>(b) + p; |
207 } | 213 } |
208 | 214 |
209 void* Arena::SlowAlloc(size_t n) { | 215 void* Arena::SlowAlloc(size_t n) { |
210 void* me = &cr_thread_cache(); | 216 void* me = &thread_cache(); |
211 Block* b = FindBlock(me); // Find block owned by me. | 217 Block* b = FindBlock(me); // Find block owned by me. |
212 // See if allocation fits in my latest block. | 218 // See if allocation fits in my latest block. |
213 if (b != NULL && b->avail() >= n) { | 219 if (b != NULL && b->avail() >= n) { |
214 SetThreadCacheBlock(b); | 220 SetThreadCacheBlock(b); |
215 google::protobuf::internal::NoBarrier_Store(&hint_, reinterpret_cast<google:
:protobuf::internal::AtomicWord>(b)); | 221 google::protobuf::internal::NoBarrier_Store(&hint_, reinterpret_cast<google:
:protobuf::internal::AtomicWord>(b)); |
216 return AllocFromBlock(b, n); | 222 return AllocFromBlock(b, n); |
217 } | 223 } |
218 b = NewBlock(me, b, n, options_.start_block_size, options_.max_block_size); | 224 b = NewBlock(me, b, n, options_.start_block_size, options_.max_block_size); |
219 AddBlock(b); | 225 AddBlock(b); |
220 if (b->owner == me) { // If this block can be reused (see NewBlock()). | 226 if (b->owner == me) { // If this block can be reused (see NewBlock()). |
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
277 } | 283 } |
278 blocks_ = 0; | 284 blocks_ = 0; |
279 hint_ = 0; | 285 hint_ = 0; |
280 if (!owns_first_block_) { | 286 if (!owns_first_block_) { |
281 // Make the first block that was passed in through ArenaOptions | 287 // Make the first block that was passed in through ArenaOptions |
282 // available for reuse. | 288 // available for reuse. |
283 first_block->pos = kHeaderSize; | 289 first_block->pos = kHeaderSize; |
284 // Thread which calls Reset() owns the first block. This allows the | 290 // Thread which calls Reset() owns the first block. This allows the |
285 // single-threaded case to allocate on the first block without taking any | 291 // single-threaded case to allocate on the first block without taking any |
286 // locks. | 292 // locks. |
287 first_block->owner = &cr_thread_cache(); | 293 first_block->owner = &thread_cache(); |
288 SetThreadCacheBlock(first_block); | 294 SetThreadCacheBlock(first_block); |
289 AddBlockInternal(first_block); | 295 AddBlockInternal(first_block); |
290 } | 296 } |
291 return space_allocated; | 297 return space_allocated; |
292 } | 298 } |
293 | 299 |
294 void Arena::CleanupList() { | 300 void Arena::CleanupList() { |
295 Node* head = | 301 Node* head = |
296 reinterpret_cast<Node*>(google::protobuf::internal::NoBarrier_Load(&cleanu
p_list_)); | 302 reinterpret_cast<Node*>(google::protobuf::internal::NoBarrier_Load(&cleanu
p_list_)); |
297 while (head != NULL) { | 303 while (head != NULL) { |
298 head->cleanup(head->elem); | 304 head->cleanup(head->elem); |
299 head = head->next; | 305 head = head->next; |
300 } | 306 } |
301 cleanup_list_ = 0; | 307 cleanup_list_ = 0; |
302 } | 308 } |
303 | 309 |
304 Arena::Block* Arena::FindBlock(void* me) { | 310 Arena::Block* Arena::FindBlock(void* me) { |
305 // TODO(sanjay): We might want to keep a separate list with one | 311 // TODO(sanjay): We might want to keep a separate list with one |
306 // entry per thread. | 312 // entry per thread. |
307 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&
blocks_)); | 313 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&
blocks_)); |
308 while (b != NULL && b->owner != me) { | 314 while (b != NULL && b->owner != me) { |
309 b = b->next; | 315 b = b->next; |
310 } | 316 } |
311 return b; | 317 return b; |
312 } | 318 } |
313 | 319 |
314 } // namespace protobuf | 320 } // namespace protobuf |
315 } // namespace google | 321 } // namespace google |
OLD | NEW |