OLD | NEW |
1 // Copyright (c) 2005, Google Inc. | 1 // Copyright (c) 2005, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 13 matching lines...) Expand all Loading... |
24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
29 | 29 |
30 // --- | 30 // --- |
31 // Author: Sanjay Ghemawat | 31 // Author: Sanjay Ghemawat |
32 | 32 |
33 #include <config.h> | 33 #include <config.h> |
| 34 #include <errno.h> // for EAGAIN, errno |
| 35 #include <fcntl.h> // for open, O_RDWR |
| 36 #include <stddef.h> // for size_t, NULL, ptrdiff_t |
34 #if defined HAVE_STDINT_H | 37 #if defined HAVE_STDINT_H |
35 #include <stdint.h> | 38 #include <stdint.h> // for uintptr_t, intptr_t |
36 #elif defined HAVE_INTTYPES_H | 39 #elif defined HAVE_INTTYPES_H |
37 #include <inttypes.h> | 40 #include <inttypes.h> |
38 #else | 41 #else |
39 #include <sys/types.h> | 42 #include <sys/types.h> |
40 #endif | 43 #endif |
| 44 #ifdef HAVE_MMAP |
| 45 #include <sys/mman.h> // for munmap, mmap, MADV_DONTNEED, etc |
| 46 #endif |
41 #ifdef HAVE_UNISTD_H | 47 #ifdef HAVE_UNISTD_H |
42 #include <unistd.h> | 48 #include <unistd.h> // for sbrk, getpagesize, off_t |
43 #endif | 49 #endif |
44 #include <fcntl.h> // for open() | 50 #include <new> // for operator new |
45 #ifdef HAVE_MMAP | 51 #include <google/malloc_extension.h> |
46 #include <sys/mman.h> | 52 #include "base/basictypes.h" |
47 #endif | 53 #include "base/commandlineflags.h" |
48 #include <errno.h> | 54 #include "base/spinlock.h" // for SpinLockHolder, SpinLock, etc |
49 #include "system-alloc.h" | 55 #include "common.h" |
50 #include "internal_logging.h" | 56 #include "internal_logging.h" |
51 #include "base/logging.h" | |
52 #include "base/commandlineflags.h" | |
53 #include "base/spinlock.h" | |
54 | 57 |
55 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old | 58 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old |
56 // form of the name instead. | 59 // form of the name instead. |
57 #ifndef MAP_ANONYMOUS | 60 #ifndef MAP_ANONYMOUS |
58 # define MAP_ANONYMOUS MAP_ANON | 61 # define MAP_ANONYMOUS MAP_ANON |
59 #endif | 62 #endif |
60 | 63 |
61 // Solaris has a bug where it doesn't declare madvise() for C++. | 64 // Solaris has a bug where it doesn't declare madvise() for C++. |
62 // http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0 | 65 // http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0 |
63 #if defined(__sun) && defined(__SVR4) | 66 #if defined(__sun) && defined(__SVR4) |
64 # include <sys/types.h> // for caddr_t | 67 # include <sys/types.h> // for caddr_t |
65 extern "C" { extern int madvise(caddr_t, size_t, int); } | 68 extern "C" { extern int madvise(caddr_t, size_t, int); } |
66 #endif | 69 #endif |
67 | 70 |
68 // Set kDebugMode mode so that we can have use C++ conditionals | 71 // Set kDebugMode mode so that we can have use C++ conditionals |
69 // instead of preprocessor conditionals. | 72 // instead of preprocessor conditionals. |
70 #ifdef NDEBUG | 73 #ifdef NDEBUG |
71 static const bool kDebugMode = false; | 74 static const bool kDebugMode = false; |
72 #else | 75 #else |
73 static const bool kDebugMode = true; | 76 static const bool kDebugMode = true; |
74 #endif | 77 #endif |
75 | 78 |
| 79 // Anonymous namespace to avoid name conflicts on "CheckAddressBits". |
| 80 namespace { |
| 81 |
| 82 // Check that no bit is set at position ADDRESS_BITS or higher. |
| 83 template <int ADDRESS_BITS> bool CheckAddressBits(uintptr_t ptr) { |
| 84 return (ptr >> ADDRESS_BITS) == 0; |
| 85 } |
| 86 |
| 87 // Specialize for the bit width of a pointer to avoid undefined shift. |
| 88 template <> bool CheckAddressBits<8 * sizeof(void*)>(uintptr_t ptr) { |
| 89 return true; |
| 90 } |
| 91 |
| 92 } // Anonymous namespace to avoid name conflicts on "CheckAddressBits". |
| 93 |
| 94 COMPILE_ASSERT(kAddressBits <= 8 * sizeof(void*), |
| 95 address_bits_larger_than_pointer_size); |
| 96 |
76 // Structure for discovering alignment | 97 // Structure for discovering alignment |
77 union MemoryAligner { | 98 union MemoryAligner { |
78 void* p; | 99 void* p; |
79 double d; | 100 double d; |
80 size_t s; | 101 size_t s; |
81 } CACHELINE_ALIGNED; | 102 } CACHELINE_ALIGNED; |
82 | 103 |
83 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED); | 104 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED); |
84 | 105 |
85 #if defined(HAVE_MMAP) || defined(MADV_DONTNEED) | 106 #if defined(HAVE_MMAP) || defined(MADV_DONTNEED) |
86 // Page size is initialized on demand (only needed for mmap-based allocators) | 107 // Page size is initialized on demand (only needed for mmap-based allocators) |
87 static size_t pagesize = 0; | 108 static size_t pagesize = 0; |
88 #endif | 109 #endif |
89 | 110 |
| 111 // The current system allocator |
| 112 SysAllocator* sys_alloc = NULL; |
| 113 |
90 // Configuration parameters. | 114 // Configuration parameters. |
91 | |
92 DEFINE_int32(malloc_devmem_start, | 115 DEFINE_int32(malloc_devmem_start, |
93 EnvToInt("TCMALLOC_DEVMEM_START", 0), | 116 EnvToInt("TCMALLOC_DEVMEM_START", 0), |
94 "Physical memory starting location in MB for /dev/mem allocation." | 117 "Physical memory starting location in MB for /dev/mem allocation." |
95 " Setting this to 0 disables /dev/mem allocation"); | 118 " Setting this to 0 disables /dev/mem allocation"); |
96 DEFINE_int32(malloc_devmem_limit, | 119 DEFINE_int32(malloc_devmem_limit, |
97 EnvToInt("TCMALLOC_DEVMEM_LIMIT", 0), | 120 EnvToInt("TCMALLOC_DEVMEM_LIMIT", 0), |
98 "Physical memory limit location in MB for /dev/mem allocation." | 121 "Physical memory limit location in MB for /dev/mem allocation." |
99 " Setting this to 0 means no limit."); | 122 " Setting this to 0 means no limit."); |
100 DEFINE_bool(malloc_skip_sbrk, | 123 DEFINE_bool(malloc_skip_sbrk, |
101 EnvToBool("TCMALLOC_SKIP_SBRK", false), | 124 EnvToBool("TCMALLOC_SKIP_SBRK", false), |
102 "Whether sbrk can be used to obtain memory."); | 125 "Whether sbrk can be used to obtain memory."); |
103 DEFINE_bool(malloc_skip_mmap, | 126 DEFINE_bool(malloc_skip_mmap, |
104 EnvToBool("TCMALLOC_SKIP_MMAP", false), | 127 EnvToBool("TCMALLOC_SKIP_MMAP", false), |
105 "Whether mmap can be used to obtain memory."); | 128 "Whether mmap can be used to obtain memory."); |
106 | 129 |
107 // static allocators | 130 // static allocators |
108 class SbrkSysAllocator : public SysAllocator { | 131 class SbrkSysAllocator : public SysAllocator { |
109 public: | 132 public: |
110 SbrkSysAllocator() : SysAllocator() { | 133 SbrkSysAllocator() : SysAllocator() { |
111 } | 134 } |
112 void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 135 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
113 void DumpStats(TCMalloc_Printer* printer); | 136 void FlagsInitialized() {} |
114 }; | 137 }; |
115 static char sbrk_space[sizeof(SbrkSysAllocator)]; | 138 static char sbrk_space[sizeof(SbrkSysAllocator)]; |
116 | 139 |
117 class MmapSysAllocator : public SysAllocator { | 140 class MmapSysAllocator : public SysAllocator { |
118 public: | 141 public: |
119 MmapSysAllocator() : SysAllocator() { | 142 MmapSysAllocator() : SysAllocator() { |
120 } | 143 } |
121 void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 144 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
122 void DumpStats(TCMalloc_Printer* printer); | 145 void FlagsInitialized() {} |
123 }; | 146 }; |
124 static char mmap_space[sizeof(MmapSysAllocator)]; | 147 static char mmap_space[sizeof(MmapSysAllocator)]; |
125 | 148 |
126 class DevMemSysAllocator : public SysAllocator { | 149 class DevMemSysAllocator : public SysAllocator { |
127 public: | 150 public: |
128 DevMemSysAllocator() : SysAllocator() { | 151 DevMemSysAllocator() : SysAllocator() { |
129 } | 152 } |
130 void* Alloc(size_t size, size_t *actual_size, size_t alignment); | 153 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
131 void DumpStats(TCMalloc_Printer* printer); | 154 void FlagsInitialized() {} |
132 }; | 155 }; |
133 static char devmem_space[sizeof(DevMemSysAllocator)]; | |
134 | 156 |
135 static const int kStaticAllocators = 3; | 157 class DefaultSysAllocator : public SysAllocator { |
136 // kMaxDynamicAllocators + kStaticAllocators; | 158 public: |
137 static const int kMaxAllocators = 5; | 159 DefaultSysAllocator() : SysAllocator() { |
138 static SysAllocator *allocators[kMaxAllocators]; | 160 for (int i = 0; i < kMaxAllocators; i++) { |
| 161 failed_[i] = true; |
| 162 allocs_[i] = NULL; |
| 163 } |
| 164 } |
| 165 void SetChildAllocator(SysAllocator* alloc, unsigned int index, |
| 166 const char* name) { |
| 167 if (index < kMaxAllocators && alloc != NULL) { |
| 168 allocs_[index] = alloc; |
| 169 failed_[index] = false; |
| 170 } |
| 171 } |
| 172 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
| 173 void FlagsInitialized() {} |
139 | 174 |
140 bool RegisterSystemAllocator(SysAllocator *a, int priority) { | 175 private: |
141 SpinLockHolder lock_holder(&spinlock); | 176 static const int kMaxAllocators = 2; |
142 | 177 bool failed_[kMaxAllocators]; |
143 // No two allocators should have a priority conflict, since the order | 178 SysAllocator* allocs_[kMaxAllocators]; |
144 // is determined at compile time. | 179 const char* names_[kMaxAllocators]; |
145 CHECK_CONDITION(allocators[priority] == NULL); | 180 }; |
146 allocators[priority] = a; | 181 static char default_space[sizeof(DefaultSysAllocator)]; |
147 return true; | 182 static const char sbrk_name[] = "SbrkSysAllocator"; |
148 } | 183 static const char mmap_name[] = "MmapSysAllocator"; |
149 | 184 |
150 | 185 |
151 void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size, | 186 void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size, |
152 size_t alignment) { | 187 size_t alignment) { |
153 #ifndef HAVE_SBRK | 188 #ifndef HAVE_SBRK |
154 failed_ = true; | |
155 return NULL; | 189 return NULL; |
156 #else | 190 #else |
157 // Check if we should use sbrk allocation. | 191 // Check if we should use sbrk allocation. |
158 // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized | 192 // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized |
159 // state) and eventually gets initialized to the specified value. Note | 193 // state) and eventually gets initialized to the specified value. Note |
160 // that this code runs for a while before the flags are initialized. | 194 // that this code runs for a while before the flags are initialized. |
161 // That means that even if this flag is set to true, some (initial) | 195 // That means that even if this flag is set to true, some (initial) |
162 // memory will be allocated with sbrk before the flag takes effect. | 196 // memory will be allocated with sbrk before the flag takes effect. |
163 if (FLAGS_malloc_skip_sbrk) { | 197 if (FLAGS_malloc_skip_sbrk) { |
164 return NULL; | 198 return NULL; |
(...skipping 14 matching lines...) Expand all Loading... |
179 } | 213 } |
180 | 214 |
181 // Check that we we're not asking for so much more memory that we'd | 215 // Check that we we're not asking for so much more memory that we'd |
182 // wrap around the end of the virtual address space. (This seems | 216 // wrap around the end of the virtual address space. (This seems |
183 // like something sbrk() should check for us, and indeed opensolaris | 217 // like something sbrk() should check for us, and indeed opensolaris |
184 // does, but glibc does not: | 218 // does, but glibc does not: |
185 // http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/lib/libc/p
ort/sys/sbrk.c?a=true | 219 // http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/lib/libc/p
ort/sys/sbrk.c?a=true |
186 // http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/libc/misc/sbrk.c?rev
=1.1.2.1&content-type=text/plain&cvsroot=glibc | 220 // http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/libc/misc/sbrk.c?rev
=1.1.2.1&content-type=text/plain&cvsroot=glibc |
187 // Without this check, sbrk may succeed when it ought to fail.) | 221 // Without this check, sbrk may succeed when it ought to fail.) |
188 if (reinterpret_cast<intptr_t>(sbrk(0)) + size < size) { | 222 if (reinterpret_cast<intptr_t>(sbrk(0)) + size < size) { |
189 failed_ = true; | |
190 return NULL; | 223 return NULL; |
191 } | 224 } |
192 | 225 |
193 void* result = sbrk(size); | 226 void* result = sbrk(size); |
194 if (result == reinterpret_cast<void*>(-1)) { | 227 if (result == reinterpret_cast<void*>(-1)) { |
195 failed_ = true; | |
196 return NULL; | 228 return NULL; |
197 } | 229 } |
198 | 230 |
199 // Is it aligned? | 231 // Is it aligned? |
200 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); | 232 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
201 if ((ptr & (alignment-1)) == 0) return result; | 233 if ((ptr & (alignment-1)) == 0) return result; |
202 | 234 |
203 // Try to get more memory for alignment | 235 // Try to get more memory for alignment |
204 size_t extra = alignment - (ptr & (alignment-1)); | 236 size_t extra = alignment - (ptr & (alignment-1)); |
205 void* r2 = sbrk(extra); | 237 void* r2 = sbrk(extra); |
206 if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) { | 238 if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) { |
207 // Contiguous with previous result | 239 // Contiguous with previous result |
208 return reinterpret_cast<void*>(ptr + extra); | 240 return reinterpret_cast<void*>(ptr + extra); |
209 } | 241 } |
210 | 242 |
211 // Give up and ask for "size + alignment - 1" bytes so | 243 // Give up and ask for "size + alignment - 1" bytes so |
212 // that we can find an aligned region within it. | 244 // that we can find an aligned region within it. |
213 result = sbrk(size + alignment - 1); | 245 result = sbrk(size + alignment - 1); |
214 if (result == reinterpret_cast<void*>(-1)) { | 246 if (result == reinterpret_cast<void*>(-1)) { |
215 failed_ = true; | |
216 return NULL; | 247 return NULL; |
217 } | 248 } |
218 ptr = reinterpret_cast<uintptr_t>(result); | 249 ptr = reinterpret_cast<uintptr_t>(result); |
219 if ((ptr & (alignment-1)) != 0) { | 250 if ((ptr & (alignment-1)) != 0) { |
220 ptr += alignment - (ptr & (alignment-1)); | 251 ptr += alignment - (ptr & (alignment-1)); |
221 } | 252 } |
222 return reinterpret_cast<void*>(ptr); | 253 return reinterpret_cast<void*>(ptr); |
223 #endif // HAVE_SBRK | 254 #endif // HAVE_SBRK |
224 } | 255 } |
225 | 256 |
226 void SbrkSysAllocator::DumpStats(TCMalloc_Printer* printer) { | |
227 printer->printf("SbrkSysAllocator: failed_=%d\n", failed_); | |
228 } | |
229 | |
230 void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size, | 257 void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size, |
231 size_t alignment) { | 258 size_t alignment) { |
232 #ifndef HAVE_MMAP | 259 #ifndef HAVE_MMAP |
233 failed_ = true; | |
234 return NULL; | 260 return NULL; |
235 #else | 261 #else |
236 // Check if we should use mmap allocation. | 262 // Check if we should use mmap allocation. |
237 // FLAGS_malloc_skip_mmap starts out as false (its uninitialized | 263 // FLAGS_malloc_skip_mmap starts out as false (its uninitialized |
238 // state) and eventually gets initialized to the specified value. Note | 264 // state) and eventually gets initialized to the specified value. Note |
239 // that this code runs for a while before the flags are initialized. | 265 // that this code runs for a while before the flags are initialized. |
240 // Chances are we never get here before the flags are initialized since | 266 // Chances are we never get here before the flags are initialized since |
241 // sbrk is used until the heap is exhausted (before mmap is used). | 267 // sbrk is used until the heap is exhausted (before mmap is used). |
242 if (FLAGS_malloc_skip_mmap) { | 268 if (FLAGS_malloc_skip_mmap) { |
243 return NULL; | 269 return NULL; |
(...skipping 22 matching lines...) Expand all Loading... |
266 | 292 |
267 // Note: size + extra does not overflow since: | 293 // Note: size + extra does not overflow since: |
268 // size + alignment < (1<<NBITS). | 294 // size + alignment < (1<<NBITS). |
269 // and extra <= alignment | 295 // and extra <= alignment |
270 // therefore size + extra < (1<<NBITS) | 296 // therefore size + extra < (1<<NBITS) |
271 void* result = mmap(NULL, size + extra, | 297 void* result = mmap(NULL, size + extra, |
272 PROT_READ|PROT_WRITE, | 298 PROT_READ|PROT_WRITE, |
273 MAP_PRIVATE|MAP_ANONYMOUS, | 299 MAP_PRIVATE|MAP_ANONYMOUS, |
274 -1, 0); | 300 -1, 0); |
275 if (result == reinterpret_cast<void*>(MAP_FAILED)) { | 301 if (result == reinterpret_cast<void*>(MAP_FAILED)) { |
276 failed_ = true; | |
277 return NULL; | 302 return NULL; |
278 } | 303 } |
279 | 304 |
280 // Adjust the return memory so it is aligned | 305 // Adjust the return memory so it is aligned |
281 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); | 306 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
282 size_t adjust = 0; | 307 size_t adjust = 0; |
283 if ((ptr & (alignment - 1)) != 0) { | 308 if ((ptr & (alignment - 1)) != 0) { |
284 adjust = alignment - (ptr & (alignment - 1)); | 309 adjust = alignment - (ptr & (alignment - 1)); |
285 } | 310 } |
286 | 311 |
287 // Return the unused memory to the system | 312 // Return the unused memory to the system |
288 if (adjust > 0) { | 313 if (adjust > 0) { |
289 munmap(reinterpret_cast<void*>(ptr), adjust); | 314 munmap(reinterpret_cast<void*>(ptr), adjust); |
290 } | 315 } |
291 if (adjust < extra) { | 316 if (adjust < extra) { |
292 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust); | 317 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust); |
293 } | 318 } |
294 | 319 |
295 ptr += adjust; | 320 ptr += adjust; |
296 return reinterpret_cast<void*>(ptr); | 321 return reinterpret_cast<void*>(ptr); |
297 #endif // HAVE_MMAP | 322 #endif // HAVE_MMAP |
298 } | 323 } |
299 | 324 |
300 void MmapSysAllocator::DumpStats(TCMalloc_Printer* printer) { | |
301 printer->printf("MmapSysAllocator: failed_=%d\n", failed_); | |
302 } | |
303 | |
304 void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size, | 325 void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size, |
305 size_t alignment) { | 326 size_t alignment) { |
306 #ifndef HAVE_MMAP | 327 #ifndef HAVE_MMAP |
307 failed_ = true; | |
308 return NULL; | 328 return NULL; |
309 #else | 329 #else |
310 static bool initialized = false; | 330 static bool initialized = false; |
311 static off_t physmem_base; // next physical memory address to allocate | 331 static off_t physmem_base; // next physical memory address to allocate |
312 static off_t physmem_limit; // maximum physical address allowed | 332 static off_t physmem_limit; // maximum physical address allowed |
313 static int physmem_fd; // file descriptor for /dev/mem | 333 static int physmem_fd; // file descriptor for /dev/mem |
314 | 334 |
315 // Check if we should use /dev/mem allocation. Note that it may take | 335 // Check if we should use /dev/mem allocation. Note that it may take |
316 // a while to get this flag initialized, so meanwhile we fall back to | 336 // a while to get this flag initialized, so meanwhile we fall back to |
317 // the next allocator. (It looks like 7MB gets allocated before | 337 // the next allocator. (It looks like 7MB gets allocated before |
318 // this flag gets initialized -khr.) | 338 // this flag gets initialized -khr.) |
319 if (FLAGS_malloc_devmem_start == 0) { | 339 if (FLAGS_malloc_devmem_start == 0) { |
320 // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to | 340 // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to |
321 // try us again next time. | 341 // try us again next time. |
322 return NULL; | 342 return NULL; |
323 } | 343 } |
324 | 344 |
325 if (!initialized) { | 345 if (!initialized) { |
326 physmem_fd = open("/dev/mem", O_RDWR); | 346 physmem_fd = open("/dev/mem", O_RDWR); |
327 if (physmem_fd < 0) { | 347 if (physmem_fd < 0) { |
328 failed_ = true; | |
329 return NULL; | 348 return NULL; |
330 } | 349 } |
331 physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL; | 350 physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL; |
332 physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL; | 351 physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL; |
333 initialized = true; | 352 initialized = true; |
334 } | 353 } |
335 | 354 |
336 // Enforce page alignment | 355 // Enforce page alignment |
337 if (pagesize == 0) pagesize = getpagesize(); | 356 if (pagesize == 0) pagesize = getpagesize(); |
338 if (alignment < pagesize) alignment = pagesize; | 357 if (alignment < pagesize) alignment = pagesize; |
(...skipping 11 matching lines...) Expand all Loading... |
350 | 369 |
351 // Ask for extra memory if alignment > pagesize | 370 // Ask for extra memory if alignment > pagesize |
352 size_t extra = 0; | 371 size_t extra = 0; |
353 if (alignment > pagesize) { | 372 if (alignment > pagesize) { |
354 extra = alignment - pagesize; | 373 extra = alignment - pagesize; |
355 } | 374 } |
356 | 375 |
357 // check to see if we have any memory left | 376 // check to see if we have any memory left |
358 if (physmem_limit != 0 && | 377 if (physmem_limit != 0 && |
359 ((size + extra) > (physmem_limit - physmem_base))) { | 378 ((size + extra) > (physmem_limit - physmem_base))) { |
360 failed_ = true; | |
361 return NULL; | 379 return NULL; |
362 } | 380 } |
363 | 381 |
364 // Note: size + extra does not overflow since: | 382 // Note: size + extra does not overflow since: |
365 // size + alignment < (1<<NBITS). | 383 // size + alignment < (1<<NBITS). |
366 // and extra <= alignment | 384 // and extra <= alignment |
367 // therefore size + extra < (1<<NBITS) | 385 // therefore size + extra < (1<<NBITS) |
368 void *result = mmap(0, size + extra, PROT_WRITE|PROT_READ, | 386 void *result = mmap(0, size + extra, PROT_WRITE|PROT_READ, |
369 MAP_SHARED, physmem_fd, physmem_base); | 387 MAP_SHARED, physmem_fd, physmem_base); |
370 if (result == reinterpret_cast<void*>(MAP_FAILED)) { | 388 if (result == reinterpret_cast<void*>(MAP_FAILED)) { |
371 failed_ = true; | |
372 return NULL; | 389 return NULL; |
373 } | 390 } |
374 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); | 391 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
375 | 392 |
376 // Adjust the return memory so it is aligned | 393 // Adjust the return memory so it is aligned |
377 size_t adjust = 0; | 394 size_t adjust = 0; |
378 if ((ptr & (alignment - 1)) != 0) { | 395 if ((ptr & (alignment - 1)) != 0) { |
379 adjust = alignment - (ptr & (alignment - 1)); | 396 adjust = alignment - (ptr & (alignment - 1)); |
380 } | 397 } |
381 | 398 |
382 // Return the unused virtual memory to the system | 399 // Return the unused virtual memory to the system |
383 if (adjust > 0) { | 400 if (adjust > 0) { |
384 munmap(reinterpret_cast<void*>(ptr), adjust); | 401 munmap(reinterpret_cast<void*>(ptr), adjust); |
385 } | 402 } |
386 if (adjust < extra) { | 403 if (adjust < extra) { |
387 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust); | 404 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust); |
388 } | 405 } |
389 | 406 |
390 ptr += adjust; | 407 ptr += adjust; |
391 physmem_base += adjust + size; | 408 physmem_base += adjust + size; |
392 | 409 |
393 return reinterpret_cast<void*>(ptr); | 410 return reinterpret_cast<void*>(ptr); |
394 #endif // HAVE_MMAP | 411 #endif // HAVE_MMAP |
395 } | 412 } |
396 | 413 |
397 void DevMemSysAllocator::DumpStats(TCMalloc_Printer* printer) { | 414 void* DefaultSysAllocator::Alloc(size_t size, size_t *actual_size, |
398 printer->printf("DevMemSysAllocator: failed_=%d\n", failed_); | 415 size_t alignment) { |
| 416 for (int i = 0; i < kMaxAllocators; i++) { |
| 417 if (!failed_[i] && allocs_[i] != NULL) { |
| 418 void* result = allocs_[i]->Alloc(size, actual_size, alignment); |
| 419 if (result != NULL) { |
| 420 return result; |
| 421 } |
| 422 TCMalloc_MESSAGE(__FILE__, __LINE__, "%s failed.\n", names_[i]); |
| 423 failed_[i] = true; |
| 424 } |
| 425 } |
| 426 // After both failed, reset "failed_" to false so that a single failed |
| 427 // allocation won't make the allocator never work again. |
| 428 for (int i = 0; i < kMaxAllocators; i++) { |
| 429 failed_[i] = false; |
| 430 } |
| 431 return NULL; |
399 } | 432 } |
400 | 433 |
401 static bool system_alloc_inited = false; | 434 static bool system_alloc_inited = false; |
402 void InitSystemAllocators(void) { | 435 void InitSystemAllocators(void) { |
403 // This determines the order in which system allocators are called | 436 MmapSysAllocator *mmap = new (mmap_space) MmapSysAllocator(); |
404 int i = kMaxDynamicAllocators; | 437 SbrkSysAllocator *sbrk = new (sbrk_space) SbrkSysAllocator(); |
405 allocators[i++] = new (devmem_space) DevMemSysAllocator(); | |
406 | 438 |
407 // In 64-bit debug mode, place the mmap allocator first since it | 439 // In 64-bit debug mode, place the mmap allocator first since it |
408 // allocates pointers that do not fit in 32 bits and therefore gives | 440 // allocates pointers that do not fit in 32 bits and therefore gives |
409 // us better testing of code's 64-bit correctness. It also leads to | 441 // us better testing of code's 64-bit correctness. It also leads to |
410 // less false negatives in heap-checking code. (Numbers are less | 442 // less false negatives in heap-checking code. (Numbers are less |
411 // likely to look like pointers and therefore the conservative gc in | 443 // likely to look like pointers and therefore the conservative gc in |
412 // the heap-checker is less likely to misinterpret a number as a | 444 // the heap-checker is less likely to misinterpret a number as a |
413 // pointer). | 445 // pointer). |
| 446 DefaultSysAllocator *sdef = new (default_space) DefaultSysAllocator(); |
414 if (kDebugMode && sizeof(void*) > 4) { | 447 if (kDebugMode && sizeof(void*) > 4) { |
415 allocators[i++] = new (mmap_space) MmapSysAllocator(); | 448 sdef->SetChildAllocator(mmap, 0, mmap_name); |
416 allocators[i++] = new (sbrk_space) SbrkSysAllocator(); | 449 sdef->SetChildAllocator(sbrk, 1, sbrk_name); |
417 } else { | 450 } else { |
418 allocators[i++] = new (sbrk_space) SbrkSysAllocator(); | 451 sdef->SetChildAllocator(sbrk, 0, sbrk_name); |
419 allocators[i++] = new (mmap_space) MmapSysAllocator(); | 452 sdef->SetChildAllocator(mmap, 1, mmap_name); |
420 } | 453 } |
| 454 sys_alloc = sdef; |
421 } | 455 } |
422 | 456 |
423 void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, | 457 void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, |
424 size_t alignment) { | 458 size_t alignment) { |
425 // Discard requests that overflow | 459 // Discard requests that overflow |
426 if (size + alignment < size) return NULL; | 460 if (size + alignment < size) return NULL; |
427 | 461 |
428 SpinLockHolder lock_holder(&spinlock); | 462 SpinLockHolder lock_holder(&spinlock); |
429 | 463 |
430 if (!system_alloc_inited) { | 464 if (!system_alloc_inited) { |
431 InitSystemAllocators(); | 465 InitSystemAllocators(); |
432 system_alloc_inited = true; | 466 system_alloc_inited = true; |
433 } | 467 } |
434 | 468 |
435 // Enforce minimum alignment | 469 // Enforce minimum alignment |
436 if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner); | 470 if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner); |
437 | 471 |
438 // Try twice, once avoiding allocators that failed before, and once | 472 void* result = sys_alloc->Alloc(size, actual_size, alignment); |
439 // more trying all allocators even if they failed before. | 473 if (result != NULL) { |
440 for (int i = 0; i < 2; i++) { | 474 if (actual_size) { |
441 for (int j = 0; j < kMaxAllocators; j++) { | 475 CheckAddressBits<kAddressBits>( |
442 SysAllocator *a = allocators[j]; | 476 reinterpret_cast<uintptr_t>(result) + *actual_size - 1); |
443 if (a == NULL) continue; | 477 } else { |
444 if (a->usable_ && !a->failed_) { | 478 CheckAddressBits<kAddressBits>( |
445 void* result = a->Alloc(size, actual_size, alignment); | 479 reinterpret_cast<uintptr_t>(result) + size - 1); |
446 if (result != NULL) return result; | |
447 } | |
448 } | |
449 | |
450 // nothing worked - reset failed_ flags and try again | |
451 for (int j = 0; j < kMaxAllocators; j++) { | |
452 SysAllocator *a = allocators[j]; | |
453 if (a == NULL) continue; | |
454 a->failed_ = false; | |
455 } | 480 } |
456 } | 481 } |
457 return NULL; | 482 return result; |
458 } | 483 } |
459 | 484 |
460 void TCMalloc_SystemRelease(void* start, size_t length) { | 485 void TCMalloc_SystemRelease(void* start, size_t length) { |
461 #ifdef MADV_DONTNEED | 486 #ifdef MADV_DONTNEED |
462 if (FLAGS_malloc_devmem_start) { | 487 if (FLAGS_malloc_devmem_start) { |
463 // It's not safe to use MADV_DONTNEED if we've been mapping | 488 // It's not safe to use MADV_DONTNEED if we've been mapping |
464 // /dev/mem for heap memory | 489 // /dev/mem for heap memory |
465 return; | 490 return; |
466 } | 491 } |
467 if (pagesize == 0) pagesize = getpagesize(); | 492 if (pagesize == 0) pagesize = getpagesize(); |
(...skipping 23 matching lines...) Expand all Loading... |
491 } | 516 } |
492 } | 517 } |
493 #endif | 518 #endif |
494 } | 519 } |
495 | 520 |
496 void TCMalloc_SystemCommit(void* start, size_t length) { | 521 void TCMalloc_SystemCommit(void* start, size_t length) { |
497 // Nothing to do here. TCMalloc_SystemRelease does not alter pages | 522 // Nothing to do here. TCMalloc_SystemRelease does not alter pages |
498 // such that they need to be re-committed before they can be used by the | 523 // such that they need to be re-committed before they can be used by the |
499 // application. | 524 // application. |
500 } | 525 } |
501 | |
502 void DumpSystemAllocatorStats(TCMalloc_Printer* printer) { | |
503 for (int j = 0; j < kMaxAllocators; j++) { | |
504 SysAllocator *a = allocators[j]; | |
505 if (a == NULL) continue; | |
506 if (a->usable_) { | |
507 a->DumpStats(printer); | |
508 } | |
509 } | |
510 } | |
OLD | NEW |