| OLD | NEW |
| 1 // Protocol Buffers - Google's data interchange format | 1 // Protocol Buffers - Google's data interchange format |
| 2 // Copyright 2008 Google Inc. All rights reserved. | 2 // Copyright 2008 Google Inc. All rights reserved. |
| 3 // https://developers.google.com/protocol-buffers/ | 3 // https://developers.google.com/protocol-buffers/ |
| 4 // | 4 // |
| 5 // Redistribution and use in source and binary forms, with or without | 5 // Redistribution and use in source and binary forms, with or without |
| 6 // modification, are permitted provided that the following conditions are | 6 // modification, are permitted provided that the following conditions are |
| 7 // met: | 7 // met: |
| 8 // | 8 // |
| 9 // * Redistributions of source code must retain the above copyright | 9 // * Redistributions of source code must retain the above copyright |
| 10 // notice, this list of conditions and the following disclaimer. | 10 // notice, this list of conditions and the following disclaimer. |
| (...skipping 12 matching lines...) Expand all Loading... |
| 23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | 24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 30 | 30 |
| 31 #include <google/protobuf/arena.h> | 31 #include <google/protobuf/arena.h> |
| 32 | 32 |
| 33 #include <algorithm> |
| 34 #include <limits> |
| 35 |
| 33 | 36 |
| 34 #ifdef ADDRESS_SANITIZER | 37 #ifdef ADDRESS_SANITIZER |
| 35 #include <sanitizer/asan_interface.h> | 38 #include <sanitizer/asan_interface.h> |
| 36 #endif | 39 #endif // ADDRESS_SANITIZER |
| 37 | 40 |
| 38 namespace google { | 41 namespace google { |
| 39 namespace protobuf { | 42 namespace protobuf { |
| 40 | 43 |
| 41 | 44 |
| 42 google::protobuf::internal::SequenceNumber Arena::lifecycle_id_generator_; | 45 google::protobuf::internal::SequenceNumber Arena::lifecycle_id_generator_; |
| 43 #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) | 46 #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) |
| 44 Arena::ThreadCache& Arena::thread_cache() { | 47 Arena::ThreadCache& Arena::thread_cache() { |
| 45 static internal::ThreadLocalStorage<ThreadCache>* thread_cache_ = | 48 static internal::ThreadLocalStorage<ThreadCache>* thread_cache_ = |
| 46 new internal::ThreadLocalStorage<ThreadCache>(); | 49 new internal::ThreadLocalStorage<ThreadCache>(); |
| (...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 118 Arena::Block* Arena::NewBlock(void* me, Block* my_last_block, size_t n, | 121 Arena::Block* Arena::NewBlock(void* me, Block* my_last_block, size_t n, |
| 119 size_t start_block_size, size_t max_block_size) { | 122 size_t start_block_size, size_t max_block_size) { |
| 120 size_t size; | 123 size_t size; |
| 121 if (my_last_block != NULL) { | 124 if (my_last_block != NULL) { |
| 122 // Double the current block size, up to a limit. | 125 // Double the current block size, up to a limit. |
| 123 size = 2 * (my_last_block->size); | 126 size = 2 * (my_last_block->size); |
| 124 if (size > max_block_size) size = max_block_size; | 127 if (size > max_block_size) size = max_block_size; |
| 125 } else { | 128 } else { |
| 126 size = start_block_size; | 129 size = start_block_size; |
| 127 } | 130 } |
| 128 if (n > size - kHeaderSize) { | 131 // Verify that n + kHeaderSize won't overflow. |
| 129 // TODO(sanjay): Check if n + kHeaderSize would overflow | 132 GOOGLE_CHECK_LE(n, std::numeric_limits<size_t>::max() - kHeaderSize); |
| 130 size = kHeaderSize + n; | 133 size = std::max(size, kHeaderSize + n); |
| 131 } | |
| 132 | 134 |
| 133 Block* b = reinterpret_cast<Block*>(options_.block_alloc(size)); | 135 Block* b = reinterpret_cast<Block*>(options_.block_alloc(size)); |
| 134 b->pos = kHeaderSize + n; | 136 b->pos = kHeaderSize + n; |
| 135 b->size = size; | 137 b->size = size; |
| 136 if (b->avail() == 0) { | 138 b->owner = me; |
| 137 // Do not attempt to reuse this block. | |
| 138 b->owner = NULL; | |
| 139 } else { | |
| 140 b->owner = me; | |
| 141 } | |
| 142 #ifdef ADDRESS_SANITIZER | 139 #ifdef ADDRESS_SANITIZER |
| 143 // Poison the rest of the block for ASAN. It was unpoisoned by the underlying | 140 // Poison the rest of the block for ASAN. It was unpoisoned by the underlying |
| 144 // malloc but it's not yet usable until we return it as part of an allocation. | 141 // malloc but it's not yet usable until we return it as part of an allocation. |
| 145 ASAN_POISON_MEMORY_REGION( | 142 ASAN_POISON_MEMORY_REGION( |
| 146 reinterpret_cast<char*>(b) + b->pos, b->size - b->pos); | 143 reinterpret_cast<char*>(b) + b->pos, b->size - b->pos); |
| 147 #endif | 144 #endif // ADDRESS_SANITIZER |
| 148 return b; | 145 return b; |
| 149 } | 146 } |
| 150 | 147 |
| 151 void Arena::AddBlock(Block* b) { | 148 void Arena::AddBlock(Block* b) { |
| 152 MutexLock l(&blocks_lock_); | 149 MutexLock l(&blocks_lock_); |
| 153 AddBlockInternal(b); | 150 AddBlockInternal(b); |
| 154 } | 151 } |
| 155 | 152 |
| 156 void Arena::AddBlockInternal(Block* b) { | 153 void Arena::AddBlockInternal(Block* b) { |
| 157 b->next = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load(
&blocks_)); | 154 b->next = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load(
&blocks_)); |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 201 return SlowAlloc(n); | 198 return SlowAlloc(n); |
| 202 } | 199 } |
| 203 return AllocFromBlock(b, n); | 200 return AllocFromBlock(b, n); |
| 204 } | 201 } |
| 205 | 202 |
| 206 void* Arena::AllocFromBlock(Block* b, size_t n) { | 203 void* Arena::AllocFromBlock(Block* b, size_t n) { |
| 207 size_t p = b->pos; | 204 size_t p = b->pos; |
| 208 b->pos = p + n; | 205 b->pos = p + n; |
| 209 #ifdef ADDRESS_SANITIZER | 206 #ifdef ADDRESS_SANITIZER |
| 210 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b) + p, n); | 207 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b) + p, n); |
| 211 #endif | 208 #endif // ADDRESS_SANITIZER |
| 212 return reinterpret_cast<char*>(b) + p; | 209 return reinterpret_cast<char*>(b) + p; |
| 213 } | 210 } |
| 214 | 211 |
| 215 void* Arena::SlowAlloc(size_t n) { | 212 void* Arena::SlowAlloc(size_t n) { |
| 216 void* me = &thread_cache(); | 213 void* me = &thread_cache(); |
| 217 Block* b = FindBlock(me); // Find block owned by me. | 214 Block* b = FindBlock(me); // Find block owned by me. |
| 218 // See if allocation fits in my latest block. | 215 // See if allocation fits in my latest block. |
| 219 if (b != NULL && b->avail() >= n) { | 216 if (b != NULL && b->avail() >= n) { |
| 220 SetThreadCacheBlock(b); | 217 SetThreadCacheBlock(b); |
| 221 google::protobuf::internal::NoBarrier_Store(&hint_, reinterpret_cast<google:
:protobuf::internal::AtomicWord>(b)); | 218 google::protobuf::internal::NoBarrier_Store(&hint_, reinterpret_cast<google:
:protobuf::internal::AtomicWord>(b)); |
| 222 return AllocFromBlock(b, n); | 219 return AllocFromBlock(b, n); |
| 223 } | 220 } |
| 224 b = NewBlock(me, b, n, options_.start_block_size, options_.max_block_size); | 221 b = NewBlock(me, b, n, options_.start_block_size, options_.max_block_size); |
| 225 AddBlock(b); | 222 AddBlock(b); |
| 226 if (b->owner == me) { // If this block can be reused (see NewBlock()). | 223 SetThreadCacheBlock(b); |
| 227 SetThreadCacheBlock(b); | |
| 228 } | |
| 229 return reinterpret_cast<char*>(b) + kHeaderSize; | 224 return reinterpret_cast<char*>(b) + kHeaderSize; |
| 230 } | 225 } |
| 231 | 226 |
| 232 uint64 Arena::SpaceAllocated() const { | 227 uint64 Arena::SpaceAllocated() const { |
| 233 uint64 space_allocated = 0; | 228 uint64 space_allocated = 0; |
| 234 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load
(&blocks_)); | 229 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load
(&blocks_)); |
| 235 while (b != NULL) { | 230 while (b != NULL) { |
| 236 space_allocated += (b->size); | 231 space_allocated += (b->size); |
| 237 b = b->next; | 232 b = b->next; |
| 238 } | 233 } |
| 239 return space_allocated; | 234 return space_allocated; |
| 240 } | 235 } |
| 241 | 236 |
| 242 uint64 Arena::SpaceUsed() const { | 237 uint64 Arena::SpaceUsed() const { |
| 243 uint64 space_used = 0; | 238 uint64 space_used = 0; |
| 244 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load
(&blocks_)); | 239 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load
(&blocks_)); |
| 245 while (b != NULL) { | 240 while (b != NULL) { |
| 246 space_used += (b->pos - kHeaderSize); | 241 space_used += (b->pos - kHeaderSize); |
| 247 b = b->next; | 242 b = b->next; |
| 248 } | 243 } |
| 249 return space_used; | 244 return space_used; |
| 250 } | 245 } |
| 251 | 246 |
| 252 pair<uint64, uint64> Arena::SpaceAllocatedAndUsed() const { | 247 std::pair<uint64, uint64> Arena::SpaceAllocatedAndUsed() const { |
| 253 uint64 allocated = 0; | 248 uint64 allocated = 0; |
| 254 uint64 used = 0; | 249 uint64 used = 0; |
| 255 | 250 |
| 256 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load
(&blocks_)); | 251 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load
(&blocks_)); |
| 257 while (b != NULL) { | 252 while (b != NULL) { |
| 258 allocated += b->size; | 253 allocated += b->size; |
| 259 used += (b->pos - kHeaderSize); | 254 used += (b->pos - kHeaderSize); |
| 260 b = b->next; | 255 b = b->next; |
| 261 } | 256 } |
| 262 return std::make_pair(allocated, used); | 257 return std::make_pair(allocated, used); |
| 263 } | 258 } |
| 264 | 259 |
| 265 uint64 Arena::FreeBlocks() { | 260 uint64 Arena::FreeBlocks() { |
| 266 uint64 space_allocated = 0; | 261 uint64 space_allocated = 0; |
| 267 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load
(&blocks_)); | 262 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::NoBarrier_Load
(&blocks_)); |
| 268 Block* first_block = NULL; | 263 Block* first_block = NULL; |
| 269 while (b != NULL) { | 264 while (b != NULL) { |
| 270 space_allocated += (b->size); | 265 space_allocated += (b->size); |
| 271 Block* next = b->next; | 266 Block* next = b->next; |
| 272 if (next != NULL) { | 267 if (next != NULL) { |
| 268 #ifdef ADDRESS_SANITIZER |
| 269 // This memory was provided by the underlying allocator as unpoisoned, so |
| 270 // return it in an unpoisoned state. |
| 271 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b), b->size); |
| 272 #endif // ADDRESS_SANITIZER |
| 273 options_.block_dealloc(b, b->size); | 273 options_.block_dealloc(b, b->size); |
| 274 } else { | 274 } else { |
| 275 if (owns_first_block_) { | 275 if (owns_first_block_) { |
| 276 #ifdef ADDRESS_SANITIZER |
| 277 // This memory was provided by the underlying allocator as unpoisoned, |
| 278 // so return it in an unpoisoned state. |
| 279 ASAN_UNPOISON_MEMORY_REGION(reinterpret_cast<char*>(b), b->size); |
| 280 #endif // ADDRESS_SANITIZER |
| 276 options_.block_dealloc(b, b->size); | 281 options_.block_dealloc(b, b->size); |
| 277 } else { | 282 } else { |
| 278 // User passed in the first block, skip free'ing the memory. | 283 // User passed in the first block, skip free'ing the memory. |
| 279 first_block = b; | 284 first_block = b; |
| 280 } | 285 } |
| 281 } | 286 } |
| 282 b = next; | 287 b = next; |
| 283 } | 288 } |
| 284 blocks_ = 0; | 289 blocks_ = 0; |
| 285 hint_ = 0; | 290 hint_ = 0; |
| (...skipping 26 matching lines...) Expand all Loading... |
| 312 // entry per thread. | 317 // entry per thread. |
| 313 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&
blocks_)); | 318 Block* b = reinterpret_cast<Block*>(google::protobuf::internal::Acquire_Load(&
blocks_)); |
| 314 while (b != NULL && b->owner != me) { | 319 while (b != NULL && b->owner != me) { |
| 315 b = b->next; | 320 b = b->next; |
| 316 } | 321 } |
| 317 return b; | 322 return b; |
| 318 } | 323 } |
| 319 | 324 |
| 320 } // namespace protobuf | 325 } // namespace protobuf |
| 321 } // namespace google | 326 } // namespace google |
| OLD | NEW |