| OLD | NEW |
| 1 // Copyright (c) 2005, Google Inc. | 1 // Copyright (c) 2005, Google Inc. |
| 2 // All rights reserved. | 2 // All rights reserved. |
| 3 // | 3 // |
| 4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
| 5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
| 6 // met: | 6 // met: |
| 7 // | 7 // |
| 8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
| 9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
| 10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
| (...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 96 | 96 |
| 97 // Structure for discovering alignment | 97 // Structure for discovering alignment |
| 98 union MemoryAligner { | 98 union MemoryAligner { |
| 99 void* p; | 99 void* p; |
| 100 double d; | 100 double d; |
| 101 size_t s; | 101 size_t s; |
| 102 } CACHELINE_ALIGNED; | 102 } CACHELINE_ALIGNED; |
| 103 | 103 |
| 104 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED); | 104 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED); |
| 105 | 105 |
| 106 #if defined(HAVE_MMAP) || defined(MADV_DONTNEED) | 106 #ifdef HAVE_GETPAGESIZE |
| 107 // Page size is initialized on demand (only needed for mmap-based allocators) | |
| 108 static size_t pagesize = 0; | 107 static size_t pagesize = 0; |
| 109 #endif | 108 #endif |
| 110 | 109 |
| 111 // The current system allocator | 110 // The current system allocator |
| 112 SysAllocator* sys_alloc = NULL; | 111 SysAllocator* sys_alloc = NULL; |
| 113 | 112 |
| 114 // Configuration parameters. | 113 // Configuration parameters. |
| 115 DEFINE_int32(malloc_devmem_start, | 114 DEFINE_int32(malloc_devmem_start, |
| 116 EnvToInt("TCMALLOC_DEVMEM_START", 0), | 115 EnvToInt("TCMALLOC_DEVMEM_START", 0), |
| 117 "Physical memory starting location in MB for /dev/mem allocation." | 116 "Physical memory starting location in MB for /dev/mem allocation." |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 153 void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 152 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
| 154 void FlagsInitialized() {} | 153 void FlagsInitialized() {} |
| 155 }; | 154 }; |
| 156 | 155 |
| 157 class DefaultSysAllocator : public SysAllocator { | 156 class DefaultSysAllocator : public SysAllocator { |
| 158 public: | 157 public: |
| 159 DefaultSysAllocator() : SysAllocator() { | 158 DefaultSysAllocator() : SysAllocator() { |
| 160 for (int i = 0; i < kMaxAllocators; i++) { | 159 for (int i = 0; i < kMaxAllocators; i++) { |
| 161 failed_[i] = true; | 160 failed_[i] = true; |
| 162 allocs_[i] = NULL; | 161 allocs_[i] = NULL; |
| 162 names_[i] = NULL; |
| 163 } | 163 } |
| 164 } | 164 } |
| 165 void SetChildAllocator(SysAllocator* alloc, unsigned int index, | 165 void SetChildAllocator(SysAllocator* alloc, unsigned int index, |
| 166 const char* name) { | 166 const char* name) { |
| 167 if (index < kMaxAllocators && alloc != NULL) { | 167 if (index < kMaxAllocators && alloc != NULL) { |
| 168 allocs_[index] = alloc; | 168 allocs_[index] = alloc; |
| 169 failed_[index] = false; | 169 failed_[index] = false; |
| 170 names_[index] = name; |
| 170 } | 171 } |
| 171 } | 172 } |
| 172 void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 173 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
| 173 void FlagsInitialized() {} | 174 void FlagsInitialized() {} |
| 174 | 175 |
| 175 private: | 176 private: |
| 176 static const int kMaxAllocators = 2; | 177 static const int kMaxAllocators = 2; |
| 177 bool failed_[kMaxAllocators]; | 178 bool failed_[kMaxAllocators]; |
| 178 SysAllocator* allocs_[kMaxAllocators]; | 179 SysAllocator* allocs_[kMaxAllocators]; |
| 179 const char* names_[kMaxAllocators]; | 180 const char* names_[kMaxAllocators]; |
| 180 }; | 181 }; |
| 181 static char default_space[sizeof(DefaultSysAllocator)]; | 182 static char default_space[sizeof(DefaultSysAllocator)]; |
| 182 static const char sbrk_name[] = "SbrkSysAllocator"; | 183 static const char sbrk_name[] = "SbrkSysAllocator"; |
| 183 static const char mmap_name[] = "MmapSysAllocator"; | 184 static const char mmap_name[] = "MmapSysAllocator"; |
| 184 | 185 |
| 185 | 186 |
| 186 void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size, | 187 void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size, |
| 187 size_t alignment) { | 188 size_t alignment) { |
| 188 #ifndef HAVE_SBRK | 189 #ifndef HAVE_SBRK |
| 189 failed_ = true; | |
| 190 return NULL; | 190 return NULL; |
| 191 #else | 191 #else |
| 192 // Check if we should use sbrk allocation. | 192 // Check if we should use sbrk allocation. |
| 193 // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized | 193 // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized |
| 194 // state) and eventually gets initialized to the specified value. Note | 194 // state) and eventually gets initialized to the specified value. Note |
| 195 // that this code runs for a while before the flags are initialized. | 195 // that this code runs for a while before the flags are initialized. |
| 196 // That means that even if this flag is set to true, some (initial) | 196 // That means that even if this flag is set to true, some (initial) |
| 197 // memory will be allocated with sbrk before the flag takes effect. | 197 // memory will be allocated with sbrk before the flag takes effect. |
| 198 if (FLAGS_malloc_skip_sbrk) { | 198 if (FLAGS_malloc_skip_sbrk) { |
| 199 return NULL; | 199 return NULL; |
| 200 } | 200 } |
| 201 | 201 |
| 202 // sbrk will release memory if passed a negative number, so we do | 202 // sbrk will release memory if passed a negative number, so we do |
| 203 // a strict check here | 203 // a strict check here |
| 204 if (static_cast<ptrdiff_t>(size + alignment) < 0) return NULL; | 204 if (static_cast<std::ptrdiff_t>(size + alignment) < 0) return NULL; |
| 205 | 205 |
| 206 // This doesn't overflow because TCMalloc_SystemAlloc has already | 206 // This doesn't overflow because TCMalloc_SystemAlloc has already |
| 207 // tested for overflow at the alignment boundary. | 207 // tested for overflow at the alignment boundary. |
| 208 size = ((size + alignment - 1) / alignment) * alignment; | 208 size = ((size + alignment - 1) / alignment) * alignment; |
| 209 | 209 |
| 210 // "actual_size" indicates that the bytes from the returned pointer | 210 // "actual_size" indicates that the bytes from the returned pointer |
| 211 // p up to and including (p + actual_size - 1) have been allocated. | 211 // p up to and including (p + actual_size - 1) have been allocated. |
| 212 if (actual_size) { | 212 if (actual_size) { |
| 213 *actual_size = size; | 213 *actual_size = size; |
| 214 } | 214 } |
| (...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 251 if ((ptr & (alignment-1)) != 0) { | 251 if ((ptr & (alignment-1)) != 0) { |
| 252 ptr += alignment - (ptr & (alignment-1)); | 252 ptr += alignment - (ptr & (alignment-1)); |
| 253 } | 253 } |
| 254 return reinterpret_cast<void*>(ptr); | 254 return reinterpret_cast<void*>(ptr); |
| 255 #endif // HAVE_SBRK | 255 #endif // HAVE_SBRK |
| 256 } | 256 } |
| 257 | 257 |
| 258 void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size, | 258 void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size, |
| 259 size_t alignment) { | 259 size_t alignment) { |
| 260 #ifndef HAVE_MMAP | 260 #ifndef HAVE_MMAP |
| 261 failed_ = true; | |
| 262 return NULL; | 261 return NULL; |
| 263 #else | 262 #else |
| 264 // Check if we should use mmap allocation. | 263 // Check if we should use mmap allocation. |
| 265 // FLAGS_malloc_skip_mmap starts out as false (its uninitialized | 264 // FLAGS_malloc_skip_mmap starts out as false (its uninitialized |
| 266 // state) and eventually gets initialized to the specified value. Note | 265 // state) and eventually gets initialized to the specified value. Note |
| 267 // that this code runs for a while before the flags are initialized. | 266 // that this code runs for a while before the flags are initialized. |
| 268 // Chances are we never get here before the flags are initialized since | 267 // Chances are we never get here before the flags are initialized since |
| 269 // sbrk is used until the heap is exhausted (before mmap is used). | 268 // sbrk is used until the heap is exhausted (before mmap is used). |
| 270 if (FLAGS_malloc_skip_mmap) { | 269 if (FLAGS_malloc_skip_mmap) { |
| 271 return NULL; | 270 return NULL; |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 320 } | 319 } |
| 321 | 320 |
| 322 ptr += adjust; | 321 ptr += adjust; |
| 323 return reinterpret_cast<void*>(ptr); | 322 return reinterpret_cast<void*>(ptr); |
| 324 #endif // HAVE_MMAP | 323 #endif // HAVE_MMAP |
| 325 } | 324 } |
| 326 | 325 |
| 327 void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size, | 326 void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size, |
| 328 size_t alignment) { | 327 size_t alignment) { |
| 329 #ifndef HAVE_MMAP | 328 #ifndef HAVE_MMAP |
| 330 failed_ = true; | |
| 331 return NULL; | 329 return NULL; |
| 332 #else | 330 #else |
| 333 static bool initialized = false; | 331 static bool initialized = false; |
| 334 static off_t physmem_base; // next physical memory address to allocate | 332 static off_t physmem_base; // next physical memory address to allocate |
| 335 static off_t physmem_limit; // maximum physical address allowed | 333 static off_t physmem_limit; // maximum physical address allowed |
| 336 static int physmem_fd; // file descriptor for /dev/mem | 334 static int physmem_fd; // file descriptor for /dev/mem |
| 337 | 335 |
| 338 // Check if we should use /dev/mem allocation. Note that it may take | 336 // Check if we should use /dev/mem allocation. Note that it may take |
| 339 // a while to get this flag initialized, so meanwhile we fall back to | 337 // a while to get this flag initialized, so meanwhile we fall back to |
| 340 // the next allocator. (It looks like 7MB gets allocated before | 338 // the next allocator. (It looks like 7MB gets allocated before |
| (...skipping 137 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 478 CheckAddressBits<kAddressBits>( | 476 CheckAddressBits<kAddressBits>( |
| 479 reinterpret_cast<uintptr_t>(result) + *actual_size - 1); | 477 reinterpret_cast<uintptr_t>(result) + *actual_size - 1); |
| 480 } else { | 478 } else { |
| 481 CheckAddressBits<kAddressBits>( | 479 CheckAddressBits<kAddressBits>( |
| 482 reinterpret_cast<uintptr_t>(result) + size - 1); | 480 reinterpret_cast<uintptr_t>(result) + size - 1); |
| 483 } | 481 } |
| 484 } | 482 } |
| 485 return result; | 483 return result; |
| 486 } | 484 } |
| 487 | 485 |
| 486 size_t TCMalloc_SystemAddGuard(void* start, size_t size) { |
| 487 #ifdef HAVE_GETPAGESIZE |
| 488 if (pagesize == 0) |
| 489 pagesize = getpagesize(); |
| 490 |
| 491 if (size < pagesize || (reinterpret_cast<size_t>(start) % pagesize) != 0) |
| 492 return 0; |
| 493 |
| 494 if (!mprotect(start, pagesize, PROT_NONE)) |
| 495 return pagesize; |
| 496 #endif |
| 497 |
| 498 return 0; |
| 499 } |
| 500 |
| 488 void TCMalloc_SystemRelease(void* start, size_t length) { | 501 void TCMalloc_SystemRelease(void* start, size_t length) { |
| 489 #ifdef MADV_DONTNEED | 502 #ifdef MADV_DONTNEED |
| 490 if (FLAGS_malloc_devmem_start) { | 503 if (FLAGS_malloc_devmem_start) { |
| 491 // It's not safe to use MADV_DONTNEED if we've been mapping | 504 // It's not safe to use MADV_DONTNEED if we've been mapping |
| 492 // /dev/mem for heap memory | 505 // /dev/mem for heap memory |
| 493 return; | 506 return; |
| 494 } | 507 } |
| 495 if (pagesize == 0) pagesize = getpagesize(); | 508 if (pagesize == 0) pagesize = getpagesize(); |
| 496 const size_t pagemask = pagesize - 1; | 509 const size_t pagemask = pagesize - 1; |
| 497 | 510 |
| (...skipping 15 matching lines...) Expand all Loading... |
| 513 // Note -- ignoring most return codes, because if this fails it | 526 // Note -- ignoring most return codes, because if this fails it |
| 514 // doesn't matter... | 527 // doesn't matter... |
| 515 while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start, | 528 while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start, |
| 516 MADV_DONTNEED) == -1 && | 529 MADV_DONTNEED) == -1 && |
| 517 errno == EAGAIN) { | 530 errno == EAGAIN) { |
| 518 // NOP | 531 // NOP |
| 519 } | 532 } |
| 520 } | 533 } |
| 521 #endif | 534 #endif |
| 522 } | 535 } |
| 536 |
| 537 void TCMalloc_SystemCommit(void* start, size_t length) { |
| 538 // Nothing to do here. TCMalloc_SystemRelease does not alter pages |
| 539 // such that they need to be re-committed before they can be used by the |
| 540 // application. |
| 541 } |
| OLD | NEW |