| OLD | NEW |
| 1 // Protocol Buffers - Google's data interchange format | 1 // Protocol Buffers - Google's data interchange format |
| 2 // Copyright 2008 Google Inc. All rights reserved. | 2 // Copyright 2008 Google Inc. All rights reserved. |
| 3 // https://developers.google.com/protocol-buffers/ | 3 // https://developers.google.com/protocol-buffers/ |
| 4 // | 4 // |
| 5 // Redistribution and use in source and binary forms, with or without | 5 // Redistribution and use in source and binary forms, with or without |
| 6 // modification, are permitted provided that the following conditions are | 6 // modification, are permitted provided that the following conditions are |
| 7 // met: | 7 // met: |
| 8 // | 8 // |
| 9 // * Redistributions of source code must retain the above copyright | 9 // * Redistributions of source code must retain the above copyright |
| 10 // notice, this list of conditions and the following disclaimer. | 10 // notice, this list of conditions and the following disclaimer. |
| (...skipping 21 matching lines...) Expand all Loading... |
| 32 | 32 |
| 33 | 33 |
| 34 #ifdef ADDRESS_SANITIZER | 34 #ifdef ADDRESS_SANITIZER |
| 35 #include <sanitizer/asan_interface.h> | 35 #include <sanitizer/asan_interface.h> |
| 36 #endif | 36 #endif |
| 37 | 37 |
| 38 namespace google { | 38 namespace google { |
| 39 namespace protobuf { | 39 namespace protobuf { |
| 40 | 40 |
| 41 | 41 |
| 42 google::protobuf::internal::SequenceNumber Arena::lifecycle_id_generator_; |
| 42 #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) | 43 #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) |
| 43 Arena::ThreadCache& Arena::cr_thread_cache() { | 44 Arena::ThreadCache& Arena::thread_cache() { |
| 44 static internal::ThreadLocalStorage<ThreadCache>* thread_cache_ = | 45 static internal::ThreadLocalStorage<ThreadCache>* thread_cache_ = |
| 45 new internal::ThreadLocalStorage<ThreadCache>(); | 46 new internal::ThreadLocalStorage<ThreadCache>(); |
| 46 return *thread_cache_->Get(); | 47 return *thread_cache_->Get(); |
| 47 } | 48 } |
| 48 #elif !defined(PROTOBUF_USE_DLLS) | 49 #elif defined(PROTOBUF_USE_DLLS) |
| 50 Arena::ThreadCache& Arena::thread_cache() { |
| 51 static GOOGLE_THREAD_LOCAL ThreadCache thread_cache_ = { -1, NULL }; |
| 52 return thread_cache_; |
| 53 } |
| 54 #else |
| 49 GOOGLE_THREAD_LOCAL Arena::ThreadCache Arena::thread_cache_ = { -1, NULL }; | 55 GOOGLE_THREAD_LOCAL Arena::ThreadCache Arena::thread_cache_ = { -1, NULL }; |
| 50 #endif | 56 #endif |
| 51 | 57 |
| 52 void Arena::Init() { | 58 void Arena::Init() { |
| 53 lifecycle_id_ = internal::cr_lifecycle_id_generator_.GetNext(); | 59 lifecycle_id_ = lifecycle_id_generator_.GetNext(); |
| 54 blocks_ = 0; | 60 blocks_ = 0; |
| 55 hint_ = 0; | 61 hint_ = 0; |
| 56 owns_first_block_ = true; | 62 owns_first_block_ = true; |
| 57 cleanup_list_ = 0; | 63 cleanup_list_ = 0; |
| 58 | 64 |
| 59 if (options_.initial_block != NULL && options_.initial_block_size > 0) { | 65 if (options_.initial_block != NULL && options_.initial_block_size > 0) { |
| 60 GOOGLE_CHECK_GE(options_.initial_block_size, sizeof(Block)) | 66 GOOGLE_CHECK_GE(options_.initial_block_size, sizeof(Block)) |
| 61 << ": Initial block size too small for header."; | 67 << ": Initial block size too small for header."; |
| 62 | 68 |
| 63 // Add first unowned block to list. | 69 // Add first unowned block to list. |
| 64 Block* first_block = reinterpret_cast<Block*>(options_.initial_block); | 70 Block* first_block = reinterpret_cast<Block*>(options_.initial_block); |
| 65 first_block->size = options_.initial_block_size; | 71 first_block->size = options_.initial_block_size; |
| 66 first_block->pos = kHeaderSize; | 72 first_block->pos = kHeaderSize; |
| 67 first_block->next = NULL; | 73 first_block->next = NULL; |
| 68 // Thread which calls Init() owns the first block. This allows the | 74 // Thread which calls Init() owns the first block. This allows the |
| 69 // single-threaded case to allocate on the first block without taking any | 75 // single-threaded case to allocate on the first block without taking any |
| 70 // locks. | 76 // locks. |
| 71 first_block->owner = &cr_thread_cache(); | 77 first_block->owner = &thread_cache(); |
| 72 SetThreadCacheBlock(first_block); | 78 SetThreadCacheBlock(first_block); |
| 73 AddBlockInternal(first_block); | 79 AddBlockInternal(first_block); |
| 74 owns_first_block_ = false; | 80 owns_first_block_ = false; |
| 75 } | 81 } |
| 76 | 82 |
| 77 // Call the initialization hook | 83 // Call the initialization hook |
| 78 if (options_.on_arena_init != NULL) { | 84 if (options_.on_arena_init != NULL) { |
| 79 hooks_cookie_ = options_.on_arena_init(this); | 85 hooks_cookie_ = options_.on_arena_init(this); |
| 80 } else { | 86 } else { |
| 81 hooks_cookie_ = NULL; | 87 hooks_cookie_ = NULL; |
| 82 } | 88 } |
| 83 } | 89 } |
| 84 | 90 |
| 85 Arena::~Arena() { | 91 Arena::~Arena() { |
| 86 uint64 space_allocated = ResetInternal(); | 92 uint64 space_allocated = ResetInternal(); |
| 87 | 93 |
| 88 // Call the destruction hook | 94 // Call the destruction hook |
| 89 if (options_.on_arena_destruction != NULL) { | 95 if (options_.on_arena_destruction != NULL) { |
| 90 options_.on_arena_destruction(this, hooks_cookie_, space_allocated); | 96 options_.on_arena_destruction(this, hooks_cookie_, space_allocated); |
| 91 } | 97 } |
| 92 } | 98 } |
| 93 | 99 |
| 94 uint64 Arena::Reset() { | 100 uint64 Arena::Reset() { |
| 95 // Invalidate any ThreadCaches pointing to any blocks we just destroyed. | 101 // Invalidate any ThreadCaches pointing to any blocks we just destroyed. |
| 96 lifecycle_id_ = internal::cr_lifecycle_id_generator_.GetNext(); | 102 lifecycle_id_ = lifecycle_id_generator_.GetNext(); |
| 97 return ResetInternal(); | 103 return ResetInternal(); |
| 98 } | 104 } |
| 99 | 105 |
| 100 uint64 Arena::ResetInternal() { | 106 uint64 Arena::ResetInternal() { |
| 101 CleanupList(); | 107 CleanupList(); |
| 102 uint64 space_allocated = FreeBlocks(); | 108 uint64 space_allocated = FreeBlocks(); |
| 103 | 109 |
| 104 // Call the reset hook | 110 // Call the reset hook |
| 105 if (options_.on_arena_reset != NULL) { | 111 if (options_.on_arena_reset != NULL) { |
| 106 options_.on_arena_reset(this, hooks_cookie_, space_allocated); | 112 options_.on_arena_reset(this, hooks_cookie_, space_allocated); |
| (...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 171 | 177 |
| 172 // Monitor allocation if needed. | 178 // Monitor allocation if needed. |
| 173 if (GOOGLE_PREDICT_FALSE(hooks_cookie_ != NULL) && | 179 if (GOOGLE_PREDICT_FALSE(hooks_cookie_ != NULL) && |
| 174 options_.on_arena_allocation != NULL) { | 180 options_.on_arena_allocation != NULL) { |
| 175 options_.on_arena_allocation(allocated, n, hooks_cookie_); | 181 options_.on_arena_allocation(allocated, n, hooks_cookie_); |
| 176 } | 182 } |
| 177 | 183 |
| 178 // If this thread already owns a block in this arena then try to use that. | 184 // If this thread already owns a block in this arena then try to use that. |
| 179 // This fast path optimizes the case where multiple threads allocate from the | 185 // This fast path optimizes the case where multiple threads allocate from the |
| 180 // same arena. | 186 // same arena. |
| 181 if (cr_thread_cache().last_lifecycle_id_seen == lifecycle_id_ && | 187 if (thread_cache().last_lifecycle_id_seen == lifecycle_id_ && |
| 182 cr_thread_cache().last_block_used_ != NULL) { | 188 thread_cache().last_block_used_ != NULL) { |
| 183 if (cr_thread_cache().last_block_used_->avail() < n) { | 189 if (thread_cache().last_block_used_->avail() < n) { |
| 184 return SlowAlloc(n); | 190 return SlowAlloc(n); |
| 185 } | 191 } |
| 186 return AllocFromBlock(cr_thread_cache().last_block_used_, n); | 192 return AllocFromBlock(thread_cache().last_block_used_, n); |
| 187 } | 193 } |
| 188 | 194 |
| 189 // Check whether we own the last accessed block on this arena. | 195 // Check whether we own the last accessed block on this arena. |
| 190 // This fast path optimizes the case where a single thread uses multiple | 196 // This fast path optimizes the case where a single thread uses multiple |
| 191 // arenas. | 197 // arenas. |
| 192 void* me = &cr_thread_cache(); | 198 void* me = &thread_cache(); |
| 193 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&
hint_)); | 199 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&
hint_)); |
| 194 if (!b || b->owner != me || b->avail() < n) { | 200 if (!b || b->owner != me || b->avail() < n) { |
| 195 return SlowAlloc(n); | 201 return SlowAlloc(n); |
| 196 } | 202 } |
| 197 return AllocFromBlock(b, n); | 203 return AllocFromBlock(b, n); |
| 198 } | 204 } |
| 199 | 205 |
| 200 void* Arena::AllocFromBlock(Block* b, size_t n) { | 206 void* Arena::AllocFromBlock(Block* b, size_t n) { |
| 201 size_t p = b->pos; | 207 size_t p = b->pos; |
| 202 b->pos = p + n; | 208 b->pos = p + n; |
| 203 #ifdef ADDRESS_SANITIZER | 209 #ifdef ADDRESS_SANITIZER |
| 204 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b) + p, n); | 210 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b) + p, n); |
| 205 #endif | 211 #endif |
| 206 return reinterpret_cast<char*>(b) + p; | 212 return reinterpret_cast<char*>(b) + p; |
| 207 } | 213 } |
| 208 | 214 |
| 209 void* Arena::SlowAlloc(size_t n) { | 215 void* Arena::SlowAlloc(size_t n) { |
| 210 void* me = &cr_thread_cache(); | 216 void* me = &thread_cache(); |
| 211 Block* b = FindBlock(me); // Find block owned by me. | 217 Block* b = FindBlock(me); // Find block owned by me. |
| 212 // See if allocation fits in my latest block. | 218 // See if allocation fits in my latest block. |
| 213 if (b != NULL && b->avail() >= n) { | 219 if (b != NULL && b->avail() >= n) { |
| 214 SetThreadCacheBlock(b); | 220 SetThreadCacheBlock(b); |
| 215 google::protobuf::internal::NoBarrier_Store(&hint_, reinterpret_cast<google:
:protobuf::internal::AtomicWord>(b)); | 221 google::protobuf::internal::NoBarrier_Store(&hint_, reinterpret_cast<google:
:protobuf::internal::AtomicWord>(b)); |
| 216 return AllocFromBlock(b, n); | 222 return AllocFromBlock(b, n); |
| 217 } | 223 } |
| 218 b = NewBlock(me, b, n, options_.start_block_size, options_.max_block_size); | 224 b = NewBlock(me, b, n, options_.start_block_size, options_.max_block_size); |
| 219 AddBlock(b); | 225 AddBlock(b); |
| 220 if (b->owner == me) { // If this block can be reused (see NewBlock()). | 226 if (b->owner == me) { // If this block can be reused (see NewBlock()). |
| (...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 277 } | 283 } |
| 278 blocks_ = 0; | 284 blocks_ = 0; |
| 279 hint_ = 0; | 285 hint_ = 0; |
| 280 if (!owns_first_block_) { | 286 if (!owns_first_block_) { |
| 281 // Make the first block that was passed in through ArenaOptions | 287 // Make the first block that was passed in through ArenaOptions |
| 282 // available for reuse. | 288 // available for reuse. |
| 283 first_block->pos = kHeaderSize; | 289 first_block->pos = kHeaderSize; |
| 284 // Thread which calls Reset() owns the first block. This allows the | 290 // Thread which calls Reset() owns the first block. This allows the |
| 285 // single-threaded case to allocate on the first block without taking any | 291 // single-threaded case to allocate on the first block without taking any |
| 286 // locks. | 292 // locks. |
| 287 first_block->owner = &cr_thread_cache(); | 293 first_block->owner = &thread_cache(); |
| 288 SetThreadCacheBlock(first_block); | 294 SetThreadCacheBlock(first_block); |
| 289 AddBlockInternal(first_block); | 295 AddBlockInternal(first_block); |
| 290 } | 296 } |
| 291 return space_allocated; | 297 return space_allocated; |
| 292 } | 298 } |
| 293 | 299 |
| 294 void Arena::CleanupList() { | 300 void Arena::CleanupList() { |
| 295 Node* head = | 301 Node* head = |
| 296 reinterpret_cast<Node*>(google::protobuf::internal::NoBarrier_Load(&cleanu
p_list_)); | 302 reinterpret_cast<Node*>(google::protobuf::internal::NoBarrier_Load(&cleanu
p_list_)); |
| 297 while (head != NULL) { | 303 while (head != NULL) { |
| 298 head->cleanup(head->elem); | 304 head->cleanup(head->elem); |
| 299 head = head->next; | 305 head = head->next; |
| 300 } | 306 } |
| 301 cleanup_list_ = 0; | 307 cleanup_list_ = 0; |
| 302 } | 308 } |
| 303 | 309 |
| 304 Arena::Block* Arena::FindBlock(void* me) { | 310 Arena::Block* Arena::FindBlock(void* me) { |
| 305 // TODO(sanjay): We might want to keep a separate list with one | 311 // TODO(sanjay): We might want to keep a separate list with one |
| 306 // entry per thread. | 312 // entry per thread. |
| 307 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&
blocks_)); | 313 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&
blocks_)); |
| 308 while (b != NULL && b->owner != me) { | 314 while (b != NULL && b->owner != me) { |
| 309 b = b->next; | 315 b = b->next; |
| 310 } | 316 } |
| 311 return b; | 317 return b; |
| 312 } | 318 } |
| 313 | 319 |
| 314 } // namespace protobuf | 320 } // namespace protobuf |
| 315 } // namespace google | 321 } // namespace google |
| OLD | NEW |