OLD | NEW |
1 // Copyright (c) 2005, Google Inc. | 1 // Copyright (c) 2005, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
54 #include "base/spinlock.h" // for SpinLockHolder, SpinLock, etc | 54 #include "base/spinlock.h" // for SpinLockHolder, SpinLock, etc |
55 #include "common.h" | 55 #include "common.h" |
56 #include "internal_logging.h" | 56 #include "internal_logging.h" |
57 | 57 |
58 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old | 58 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old |
59 // form of the name instead. | 59 // form of the name instead. |
60 #ifndef MAP_ANONYMOUS | 60 #ifndef MAP_ANONYMOUS |
61 # define MAP_ANONYMOUS MAP_ANON | 61 # define MAP_ANONYMOUS MAP_ANON |
62 #endif | 62 #endif |
63 | 63 |
| 64 // MADV_FREE is specifically designed for use by malloc(), but only |
| 65 // FreeBSD supports it; in linux we fall back to the somewhat inferior |
| 66 // MADV_DONTNEED. |
| 67 #if !defined(MADV_FREE) && defined(MADV_DONTNEED) |
| 68 # define MADV_FREE MADV_DONTNEED |
| 69 #endif |
| 70 |
64 // Solaris has a bug where it doesn't declare madvise() for C++. | 71 // Solaris has a bug where it doesn't declare madvise() for C++. |
65 // http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0 | 72 // http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0 |
66 #if defined(__sun) && defined(__SVR4) | 73 #if defined(__sun) && defined(__SVR4) |
67 # include <sys/types.h> // for caddr_t | 74 # include <sys/types.h> // for caddr_t |
68 extern "C" { extern int madvise(caddr_t, size_t, int); } | 75 extern "C" { extern int madvise(caddr_t, size_t, int); } |
69 #endif | 76 #endif |
70 | 77 |
71 // Set kDebugMode mode so that we can have use C++ conditionals | 78 // Set kDebugMode mode so that we can have use C++ conditionals |
72 // instead of preprocessor conditionals. | 79 // instead of preprocessor conditionals. |
73 #ifdef NDEBUG | 80 #ifdef NDEBUG |
74 static const bool kDebugMode = false; | 81 static const bool kDebugMode = false; |
75 #else | 82 #else |
76 static const bool kDebugMode = true; | 83 static const bool kDebugMode = true; |
77 #endif | 84 #endif |
78 | 85 |
| 86 // TODO(sanjay): Move the code below into the tcmalloc namespace |
| 87 using tcmalloc::kLog; |
| 88 using tcmalloc::Log; |
| 89 |
79 // Anonymous namespace to avoid name conflicts on "CheckAddressBits". | 90 // Anonymous namespace to avoid name conflicts on "CheckAddressBits". |
80 namespace { | 91 namespace { |
81 | 92 |
82 // Check that no bit is set at position ADDRESS_BITS or higher. | 93 // Check that no bit is set at position ADDRESS_BITS or higher. |
83 template <int ADDRESS_BITS> bool CheckAddressBits(uintptr_t ptr) { | 94 template <int ADDRESS_BITS> bool CheckAddressBits(uintptr_t ptr) { |
84 return (ptr >> ADDRESS_BITS) == 0; | 95 return (ptr >> ADDRESS_BITS) == 0; |
85 } | 96 } |
86 | 97 |
87 // Specialize for the bit width of a pointer to avoid undefined shift. | 98 // Specialize for the bit width of a pointer to avoid undefined shift. |
88 template <> bool CheckAddressBits<8 * sizeof(void*)>(uintptr_t ptr) { | 99 template <> bool CheckAddressBits<8 * sizeof(void*)>(uintptr_t ptr) { |
89 return true; | 100 return true; |
90 } | 101 } |
91 | 102 |
92 } // Anonymous namespace to avoid name conflicts on "CheckAddressBits". | 103 } // Anonymous namespace to avoid name conflicts on "CheckAddressBits". |
93 | 104 |
94 COMPILE_ASSERT(kAddressBits <= 8 * sizeof(void*), | 105 COMPILE_ASSERT(kAddressBits <= 8 * sizeof(void*), |
95 address_bits_larger_than_pointer_size); | 106 address_bits_larger_than_pointer_size); |
96 | 107 |
97 // Structure for discovering alignment | 108 // Structure for discovering alignment |
98 union MemoryAligner { | 109 union MemoryAligner { |
99 void* p; | 110 void* p; |
100 double d; | 111 double d; |
101 size_t s; | 112 size_t s; |
102 } CACHELINE_ALIGNED; | 113 } CACHELINE_ALIGNED; |
103 | 114 |
104 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED); | 115 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED); |
105 | 116 |
106 #ifdef HAVE_GETPAGESIZE | 117 #if defined(HAVE_MMAP) || defined(MADV_FREE) |
| 118 // Page size is initialized on demand (only needed for mmap-based allocators) |
107 static size_t pagesize = 0; | 119 static size_t pagesize = 0; |
108 #endif | 120 #endif |
109 | 121 |
110 // The current system allocator | 122 // The current system allocator |
111 SysAllocator* sys_alloc = NULL; | 123 SysAllocator* sys_alloc = NULL; |
112 | 124 |
113 // Configuration parameters. | 125 // Configuration parameters. |
114 DEFINE_int32(malloc_devmem_start, | 126 DEFINE_int32(malloc_devmem_start, |
115 EnvToInt("TCMALLOC_DEVMEM_START", 0), | 127 EnvToInt("TCMALLOC_DEVMEM_START", 0), |
116 "Physical memory starting location in MB for /dev/mem allocation." | 128 "Physical memory starting location in MB for /dev/mem allocation." |
117 " Setting this to 0 disables /dev/mem allocation"); | 129 " Setting this to 0 disables /dev/mem allocation"); |
118 DEFINE_int32(malloc_devmem_limit, | 130 DEFINE_int32(malloc_devmem_limit, |
119 EnvToInt("TCMALLOC_DEVMEM_LIMIT", 0), | 131 EnvToInt("TCMALLOC_DEVMEM_LIMIT", 0), |
120 "Physical memory limit location in MB for /dev/mem allocation." | 132 "Physical memory limit location in MB for /dev/mem allocation." |
121 " Setting this to 0 means no limit."); | 133 " Setting this to 0 means no limit."); |
122 DEFINE_bool(malloc_skip_sbrk, | 134 DEFINE_bool(malloc_skip_sbrk, |
123 EnvToBool("TCMALLOC_SKIP_SBRK", false), | 135 EnvToBool("TCMALLOC_SKIP_SBRK", false), |
124 "Whether sbrk can be used to obtain memory."); | 136 "Whether sbrk can be used to obtain memory."); |
125 DEFINE_bool(malloc_skip_mmap, | 137 DEFINE_bool(malloc_skip_mmap, |
126 EnvToBool("TCMALLOC_SKIP_MMAP", false), | 138 EnvToBool("TCMALLOC_SKIP_MMAP", false), |
127 "Whether mmap can be used to obtain memory."); | 139 "Whether mmap can be used to obtain memory."); |
128 | 140 |
129 // static allocators | 141 // static allocators |
130 class SbrkSysAllocator : public SysAllocator { | 142 class SbrkSysAllocator : public SysAllocator { |
131 public: | 143 public: |
132 SbrkSysAllocator() : SysAllocator() { | 144 SbrkSysAllocator() : SysAllocator() { |
133 } | 145 } |
134 void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 146 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
135 void FlagsInitialized() {} | |
136 }; | 147 }; |
137 static char sbrk_space[sizeof(SbrkSysAllocator)]; | 148 static char sbrk_space[sizeof(SbrkSysAllocator)]; |
138 | 149 |
139 class MmapSysAllocator : public SysAllocator { | 150 class MmapSysAllocator : public SysAllocator { |
140 public: | 151 public: |
141 MmapSysAllocator() : SysAllocator() { | 152 MmapSysAllocator() : SysAllocator() { |
142 } | 153 } |
143 void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 154 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
144 void FlagsInitialized() {} | |
145 }; | 155 }; |
146 static char mmap_space[sizeof(MmapSysAllocator)]; | 156 static char mmap_space[sizeof(MmapSysAllocator)]; |
147 | 157 |
148 class DevMemSysAllocator : public SysAllocator { | 158 class DevMemSysAllocator : public SysAllocator { |
149 public: | 159 public: |
150 DevMemSysAllocator() : SysAllocator() { | 160 DevMemSysAllocator() : SysAllocator() { |
151 } | 161 } |
152 void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 162 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
153 void FlagsInitialized() {} | |
154 }; | 163 }; |
155 | 164 |
156 class DefaultSysAllocator : public SysAllocator { | 165 class DefaultSysAllocator : public SysAllocator { |
157 public: | 166 public: |
158 DefaultSysAllocator() : SysAllocator() { | 167 DefaultSysAllocator() : SysAllocator() { |
159 for (int i = 0; i < kMaxAllocators; i++) { | 168 for (int i = 0; i < kMaxAllocators; i++) { |
160 failed_[i] = true; | 169 failed_[i] = true; |
161 allocs_[i] = NULL; | 170 allocs_[i] = NULL; |
162 names_[i] = NULL; | 171 names_[i] = NULL; |
163 } | 172 } |
164 } | 173 } |
165 void SetChildAllocator(SysAllocator* alloc, unsigned int index, | 174 void SetChildAllocator(SysAllocator* alloc, unsigned int index, |
166 const char* name) { | 175 const char* name) { |
167 if (index < kMaxAllocators && alloc != NULL) { | 176 if (index < kMaxAllocators && alloc != NULL) { |
168 allocs_[index] = alloc; | 177 allocs_[index] = alloc; |
169 failed_[index] = false; | 178 failed_[index] = false; |
170 names_[index] = name; | 179 names_[index] = name; |
171 } | 180 } |
172 } | 181 } |
173 void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 182 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
174 void FlagsInitialized() {} | |
175 | 183 |
176 private: | 184 private: |
177 static const int kMaxAllocators = 2; | 185 static const int kMaxAllocators = 2; |
178 bool failed_[kMaxAllocators]; | 186 bool failed_[kMaxAllocators]; |
179 SysAllocator* allocs_[kMaxAllocators]; | 187 SysAllocator* allocs_[kMaxAllocators]; |
180 const char* names_[kMaxAllocators]; | 188 const char* names_[kMaxAllocators]; |
181 }; | 189 }; |
182 static char default_space[sizeof(DefaultSysAllocator)]; | 190 static char default_space[sizeof(DefaultSysAllocator)]; |
183 static const char sbrk_name[] = "SbrkSysAllocator"; | 191 static const char sbrk_name[] = "SbrkSysAllocator"; |
184 static const char mmap_name[] = "MmapSysAllocator"; | 192 static const char mmap_name[] = "MmapSysAllocator"; |
185 | 193 |
186 | 194 |
187 void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size, | 195 void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size, |
188 size_t alignment) { | 196 size_t alignment) { |
189 #ifndef HAVE_SBRK | 197 #ifndef HAVE_SBRK |
| 198 failed_ = true; |
190 return NULL; | 199 return NULL; |
191 #else | 200 #else |
192 // Check if we should use sbrk allocation. | 201 // Check if we should use sbrk allocation. |
193 // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized | 202 // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized |
194 // state) and eventually gets initialized to the specified value. Note | 203 // state) and eventually gets initialized to the specified value. Note |
195 // that this code runs for a while before the flags are initialized. | 204 // that this code runs for a while before the flags are initialized. |
196 // That means that even if this flag is set to true, some (initial) | 205 // That means that even if this flag is set to true, some (initial) |
197 // memory will be allocated with sbrk before the flag takes effect. | 206 // memory will be allocated with sbrk before the flag takes effect. |
198 if (FLAGS_malloc_skip_sbrk) { | 207 if (FLAGS_malloc_skip_sbrk) { |
199 return NULL; | 208 return NULL; |
200 } | 209 } |
201 | 210 |
202 // sbrk will release memory if passed a negative number, so we do | 211 // sbrk will release memory if passed a negative number, so we do |
203 // a strict check here | 212 // a strict check here |
204 if (static_cast<std::ptrdiff_t>(size + alignment) < 0) return NULL; | 213 if (static_cast<ptrdiff_t>(size + alignment) < 0) return NULL; |
205 | 214 |
206 // This doesn't overflow because TCMalloc_SystemAlloc has already | 215 // This doesn't overflow because TCMalloc_SystemAlloc has already |
207 // tested for overflow at the alignment boundary. | 216 // tested for overflow at the alignment boundary. |
208 size = ((size + alignment - 1) / alignment) * alignment; | 217 size = ((size + alignment - 1) / alignment) * alignment; |
209 | 218 |
210 // "actual_size" indicates that the bytes from the returned pointer | 219 // "actual_size" indicates that the bytes from the returned pointer |
211 // p up to and including (p + actual_size - 1) have been allocated. | 220 // p up to and including (p + actual_size - 1) have been allocated. |
212 if (actual_size) { | 221 if (actual_size) { |
213 *actual_size = size; | 222 *actual_size = size; |
214 } | 223 } |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
251 if ((ptr & (alignment-1)) != 0) { | 260 if ((ptr & (alignment-1)) != 0) { |
252 ptr += alignment - (ptr & (alignment-1)); | 261 ptr += alignment - (ptr & (alignment-1)); |
253 } | 262 } |
254 return reinterpret_cast<void*>(ptr); | 263 return reinterpret_cast<void*>(ptr); |
255 #endif // HAVE_SBRK | 264 #endif // HAVE_SBRK |
256 } | 265 } |
257 | 266 |
258 void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size, | 267 void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size, |
259 size_t alignment) { | 268 size_t alignment) { |
260 #ifndef HAVE_MMAP | 269 #ifndef HAVE_MMAP |
| 270 failed_ = true; |
261 return NULL; | 271 return NULL; |
262 #else | 272 #else |
263 // Check if we should use mmap allocation. | 273 // Check if we should use mmap allocation. |
264 // FLAGS_malloc_skip_mmap starts out as false (its uninitialized | 274 // FLAGS_malloc_skip_mmap starts out as false (its uninitialized |
265 // state) and eventually gets initialized to the specified value. Note | 275 // state) and eventually gets initialized to the specified value. Note |
266 // that this code runs for a while before the flags are initialized. | 276 // that this code runs for a while before the flags are initialized. |
267 // Chances are we never get here before the flags are initialized since | 277 // Chances are we never get here before the flags are initialized since |
268 // sbrk is used until the heap is exhausted (before mmap is used). | 278 // sbrk is used until the heap is exhausted (before mmap is used). |
269 if (FLAGS_malloc_skip_mmap) { | 279 if (FLAGS_malloc_skip_mmap) { |
270 return NULL; | 280 return NULL; |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
319 } | 329 } |
320 | 330 |
321 ptr += adjust; | 331 ptr += adjust; |
322 return reinterpret_cast<void*>(ptr); | 332 return reinterpret_cast<void*>(ptr); |
323 #endif // HAVE_MMAP | 333 #endif // HAVE_MMAP |
324 } | 334 } |
325 | 335 |
326 void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size, | 336 void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size, |
327 size_t alignment) { | 337 size_t alignment) { |
328 #ifndef HAVE_MMAP | 338 #ifndef HAVE_MMAP |
| 339 failed_ = true; |
329 return NULL; | 340 return NULL; |
330 #else | 341 #else |
331 static bool initialized = false; | 342 static bool initialized = false; |
332 static off_t physmem_base; // next physical memory address to allocate | 343 static off_t physmem_base; // next physical memory address to allocate |
333 static off_t physmem_limit; // maximum physical address allowed | 344 static off_t physmem_limit; // maximum physical address allowed |
334 static int physmem_fd; // file descriptor for /dev/mem | 345 static int physmem_fd; // file descriptor for /dev/mem |
335 | 346 |
336 // Check if we should use /dev/mem allocation. Note that it may take | 347 // Check if we should use /dev/mem allocation. Note that it may take |
337 // a while to get this flag initialized, so meanwhile we fall back to | 348 // a while to get this flag initialized, so meanwhile we fall back to |
338 // the next allocator. (It looks like 7MB gets allocated before | 349 // the next allocator. (It looks like 7MB gets allocated before |
(...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
413 } | 424 } |
414 | 425 |
415 void* DefaultSysAllocator::Alloc(size_t size, size_t *actual_size, | 426 void* DefaultSysAllocator::Alloc(size_t size, size_t *actual_size, |
416 size_t alignment) { | 427 size_t alignment) { |
417 for (int i = 0; i < kMaxAllocators; i++) { | 428 for (int i = 0; i < kMaxAllocators; i++) { |
418 if (!failed_[i] && allocs_[i] != NULL) { | 429 if (!failed_[i] && allocs_[i] != NULL) { |
419 void* result = allocs_[i]->Alloc(size, actual_size, alignment); | 430 void* result = allocs_[i]->Alloc(size, actual_size, alignment); |
420 if (result != NULL) { | 431 if (result != NULL) { |
421 return result; | 432 return result; |
422 } | 433 } |
423 TCMalloc_MESSAGE(__FILE__, __LINE__, "%s failed.\n", names_[i]); | |
424 failed_[i] = true; | 434 failed_[i] = true; |
425 } | 435 } |
426 } | 436 } |
427 // After both failed, reset "failed_" to false so that a single failed | 437 // After both failed, reset "failed_" to false so that a single failed |
428 // allocation won't make the allocator never work again. | 438 // allocation won't make the allocator never work again. |
429 for (int i = 0; i < kMaxAllocators; i++) { | 439 for (int i = 0; i < kMaxAllocators; i++) { |
430 failed_[i] = false; | 440 failed_[i] = false; |
431 } | 441 } |
432 return NULL; | 442 return NULL; |
433 } | 443 } |
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
476 CheckAddressBits<kAddressBits>( | 486 CheckAddressBits<kAddressBits>( |
477 reinterpret_cast<uintptr_t>(result) + *actual_size - 1); | 487 reinterpret_cast<uintptr_t>(result) + *actual_size - 1); |
478 } else { | 488 } else { |
479 CheckAddressBits<kAddressBits>( | 489 CheckAddressBits<kAddressBits>( |
480 reinterpret_cast<uintptr_t>(result) + size - 1); | 490 reinterpret_cast<uintptr_t>(result) + size - 1); |
481 } | 491 } |
482 } | 492 } |
483 return result; | 493 return result; |
484 } | 494 } |
485 | 495 |
486 size_t TCMalloc_SystemAddGuard(void* start, size_t size) { | |
487 #ifdef HAVE_GETPAGESIZE | |
488 if (pagesize == 0) | |
489 pagesize = getpagesize(); | |
490 | |
491 if (size < pagesize || (reinterpret_cast<size_t>(start) % pagesize) != 0) | |
492 return 0; | |
493 | |
494 if (!mprotect(start, pagesize, PROT_NONE)) | |
495 return pagesize; | |
496 #endif | |
497 | |
498 return 0; | |
499 } | |
500 | |
501 void TCMalloc_SystemRelease(void* start, size_t length) { | 496 void TCMalloc_SystemRelease(void* start, size_t length) { |
502 #ifdef MADV_DONTNEED | 497 #ifdef MADV_FREE |
503 if (FLAGS_malloc_devmem_start) { | 498 if (FLAGS_malloc_devmem_start) { |
504 // It's not safe to use MADV_DONTNEED if we've been mapping | 499 // It's not safe to use MADV_FREE/MADV_DONTNEED if we've been |
505 // /dev/mem for heap memory | 500 // mapping /dev/mem for heap memory. |
506 return; | 501 return; |
507 } | 502 } |
508 if (pagesize == 0) pagesize = getpagesize(); | 503 if (pagesize == 0) pagesize = getpagesize(); |
509 const size_t pagemask = pagesize - 1; | 504 const size_t pagemask = pagesize - 1; |
510 | 505 |
511 size_t new_start = reinterpret_cast<size_t>(start); | 506 size_t new_start = reinterpret_cast<size_t>(start); |
512 size_t end = new_start + length; | 507 size_t end = new_start + length; |
513 size_t new_end = end; | 508 size_t new_end = end; |
514 | 509 |
515 // Round up the starting address and round down the ending address | 510 // Round up the starting address and round down the ending address |
516 // to be page aligned: | 511 // to be page aligned: |
517 new_start = (new_start + pagesize - 1) & ~pagemask; | 512 new_start = (new_start + pagesize - 1) & ~pagemask; |
518 new_end = new_end & ~pagemask; | 513 new_end = new_end & ~pagemask; |
519 | 514 |
520 ASSERT((new_start & pagemask) == 0); | 515 ASSERT((new_start & pagemask) == 0); |
521 ASSERT((new_end & pagemask) == 0); | 516 ASSERT((new_end & pagemask) == 0); |
522 ASSERT(new_start >= reinterpret_cast<size_t>(start)); | 517 ASSERT(new_start >= reinterpret_cast<size_t>(start)); |
523 ASSERT(new_end <= end); | 518 ASSERT(new_end <= end); |
524 | 519 |
525 if (new_end > new_start) { | 520 if (new_end > new_start) { |
526 // Note -- ignoring most return codes, because if this fails it | 521 // Note -- ignoring most return codes, because if this fails it |
527 // doesn't matter... | 522 // doesn't matter... |
528 while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start, | 523 while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start, |
529 MADV_DONTNEED) == -1 && | 524 MADV_FREE) == -1 && |
530 errno == EAGAIN) { | 525 errno == EAGAIN) { |
531 // NOP | 526 // NOP |
532 } | 527 } |
533 } | 528 } |
534 #endif | 529 #endif |
535 } | 530 } |
536 | |
537 void TCMalloc_SystemCommit(void* start, size_t length) { | |
538 // Nothing to do here. TCMalloc_SystemRelease does not alter pages | |
539 // such that they need to be re-committed before they can be used by the | |
540 // application. | |
541 } | |
OLD | NEW |