OLD | NEW |
(Empty) | |
| 1 // Copyright (c) 2005, Google Inc. |
| 2 // All rights reserved. |
| 3 // |
| 4 // Redistribution and use in source and binary forms, with or without |
| 5 // modification, are permitted provided that the following conditions are |
| 6 // met: |
| 7 // |
| 8 // * Redistributions of source code must retain the above copyright |
| 9 // notice, this list of conditions and the following disclaimer. |
| 10 // * Redistributions in binary form must reproduce the above |
| 11 // copyright notice, this list of conditions and the following disclaimer |
| 12 // in the documentation and/or other materials provided with the |
| 13 // distribution. |
| 14 // * Neither the name of Google Inc. nor the names of its |
| 15 // contributors may be used to endorse or promote products derived from |
| 16 // this software without specific prior written permission. |
| 17 // |
| 18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 29 |
| 30 // --- |
| 31 // Author: Sanjay Ghemawat |
| 32 |
| 33 #include <config.h> |
| 34 #if defined HAVE_STDINT_H |
| 35 #include <stdint.h> |
| 36 #elif defined HAVE_INTTYPES_H |
| 37 #include <inttypes.h> |
| 38 #else |
| 39 #include <sys/types.h> |
| 40 #endif |
| 41 #ifdef HAVE_UNISTD_H |
| 42 #include <unistd.h> |
| 43 #endif |
| 44 #include <fcntl.h> // for open() |
| 45 #ifdef HAVE_MMAP |
| 46 #include <sys/mman.h> |
| 47 #endif |
| 48 #include <errno.h> |
| 49 #include "system-alloc.h" |
| 50 #include "internal_logging.h" |
| 51 #include "base/logging.h" |
| 52 #include "base/commandlineflags.h" |
| 53 #include "base/spinlock.h" |
| 54 |
| 55 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old |
| 56 // form of the name instead. |
| 57 #ifndef MAP_ANONYMOUS |
| 58 # define MAP_ANONYMOUS MAP_ANON |
| 59 #endif |
| 60 |
| 61 // Solaris has a bug where it doesn't declare madvise() for C++. |
| 62 // http://www.opensolaris.org/jive/thread.jspa?threadID=21035&tstart=0 |
| 63 #if defined(__sun) && defined(__SVR4) |
| 64 # include <sys/types.h> // for caddr_t |
| 65 extern "C" { extern int madvise(caddr_t, size_t, int); } |
| 66 #endif |
| 67 |
| 68 // Set kDebugMode mode so that we can have use C++ conditionals |
| 69 // instead of preprocessor conditionals. |
| 70 #ifdef NDEBUG |
| 71 static const bool kDebugMode = false; |
| 72 #else |
| 73 static const bool kDebugMode = true; |
| 74 #endif |
| 75 |
| 76 // Structure for discovering alignment |
| 77 union MemoryAligner { |
| 78 void* p; |
| 79 double d; |
| 80 size_t s; |
| 81 }; |
| 82 |
| 83 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED); |
| 84 |
| 85 #if defined(HAVE_MMAP) || defined(MADV_DONTNEED) |
| 86 // Page size is initialized on demand (only needed for mmap-based allocators) |
| 87 static size_t pagesize = 0; |
| 88 #endif |
| 89 |
| 90 // Configuration parameters. |
| 91 |
| 92 DEFINE_int32(malloc_devmem_start, |
| 93 EnvToInt("TCMALLOC_DEVMEM_START", 0), |
| 94 "Physical memory starting location in MB for /dev/mem allocation." |
| 95 " Setting this to 0 disables /dev/mem allocation"); |
| 96 DEFINE_int32(malloc_devmem_limit, |
| 97 EnvToInt("TCMALLOC_DEVMEM_LIMIT", 0), |
| 98 "Physical memory limit location in MB for /dev/mem allocation." |
| 99 " Setting this to 0 means no limit."); |
| 100 DEFINE_bool(malloc_skip_sbrk, |
| 101 EnvToBool("TCMALLOC_SKIP_SBRK", false), |
| 102 "Whether sbrk can be used to obtain memory."); |
| 103 DEFINE_bool(malloc_skip_mmap, |
| 104 EnvToBool("TCMALLOC_SKIP_MMAP", false), |
| 105 "Whether mmap can be used to obtain memory."); |
| 106 |
| 107 // static allocators |
| 108 class SbrkSysAllocator : public SysAllocator { |
| 109 public: |
| 110 SbrkSysAllocator() : SysAllocator() { |
| 111 } |
| 112 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
| 113 void DumpStats(TCMalloc_Printer* printer); |
| 114 }; |
| 115 static char sbrk_space[sizeof(SbrkSysAllocator)]; |
| 116 |
| 117 class MmapSysAllocator : public SysAllocator { |
| 118 public: |
| 119 MmapSysAllocator() : SysAllocator() { |
| 120 } |
| 121 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
| 122 void DumpStats(TCMalloc_Printer* printer); |
| 123 }; |
| 124 static char mmap_space[sizeof(MmapSysAllocator)]; |
| 125 |
| 126 class DevMemSysAllocator : public SysAllocator { |
| 127 public: |
| 128 DevMemSysAllocator() : SysAllocator() { |
| 129 } |
| 130 void* Alloc(size_t size, size_t *actual_size, size_t alignment); |
| 131 void DumpStats(TCMalloc_Printer* printer); |
| 132 }; |
| 133 static char devmem_space[sizeof(DevMemSysAllocator)]; |
| 134 |
| 135 static const int kStaticAllocators = 3; |
| 136 // kMaxDynamicAllocators + kStaticAllocators; |
| 137 static const int kMaxAllocators = 5; |
| 138 static SysAllocator *allocators[kMaxAllocators]; |
| 139 |
| 140 bool RegisterSystemAllocator(SysAllocator *a, int priority) { |
| 141 SpinLockHolder lock_holder(&spinlock); |
| 142 |
| 143 // No two allocators should have a priority conflict, since the order |
| 144 // is determined at compile time. |
| 145 CHECK_CONDITION(allocators[priority] == NULL); |
| 146 allocators[priority] = a; |
| 147 return true; |
| 148 } |
| 149 |
| 150 |
| 151 void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size, |
| 152 size_t alignment) { |
| 153 // Check if we should use sbrk allocation. |
| 154 // FLAGS_malloc_skip_sbrk starts out as false (its uninitialized |
| 155 // state) and eventually gets initialized to the specified value. Note |
| 156 // that this code runs for a while before the flags are initialized. |
| 157 // That means that even if this flag is set to true, some (initial) |
| 158 // memory will be allocated with sbrk before the flag takes effect. |
| 159 if (FLAGS_malloc_skip_sbrk) { |
| 160 return NULL; |
| 161 } |
| 162 |
| 163 // sbrk will release memory if passed a negative number, so we do |
| 164 // a strict check here |
| 165 if (static_cast<ptrdiff_t>(size + alignment) < 0) return NULL; |
| 166 |
| 167 // could theoretically return the "extra" bytes here, but this |
| 168 // is simple and correct. |
| 169 if (actual_size) { |
| 170 *actual_size = size; |
| 171 } |
| 172 |
| 173 // This doesn't overflow because TCMalloc_SystemAlloc has already |
| 174 // tested for overflow at the alignment boundary. |
| 175 size = ((size + alignment - 1) / alignment) * alignment; |
| 176 |
| 177 // Check that we we're not asking for so much more memory that we'd |
| 178 // wrap around the end of the virtual address space. (This seems |
| 179 // like something sbrk() should check for us, and indeed opensolaris |
| 180 // does, but glibc does not: |
| 181 // http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/lib/libc/p
ort/sys/sbrk.c?a=true |
| 182 // http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/libc/misc/sbrk.c?rev
=1.1.2.1&content-type=text/plain&cvsroot=glibc |
| 183 // Without this check, sbrk may succeed when it ought to fail.) |
| 184 if (reinterpret_cast<intptr_t>(sbrk(0)) + size < size) { |
| 185 failed_ = true; |
| 186 return NULL; |
| 187 } |
| 188 |
| 189 void* result = sbrk(size); |
| 190 if (result == reinterpret_cast<void*>(-1)) { |
| 191 failed_ = true; |
| 192 return NULL; |
| 193 } |
| 194 |
| 195 // Is it aligned? |
| 196 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
| 197 if ((ptr & (alignment-1)) == 0) return result; |
| 198 |
| 199 // Try to get more memory for alignment |
| 200 size_t extra = alignment - (ptr & (alignment-1)); |
| 201 void* r2 = sbrk(extra); |
| 202 if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) { |
| 203 // Contiguous with previous result |
| 204 return reinterpret_cast<void*>(ptr + extra); |
| 205 } |
| 206 |
| 207 // Give up and ask for "size + alignment - 1" bytes so |
| 208 // that we can find an aligned region within it. |
| 209 result = sbrk(size + alignment - 1); |
| 210 if (result == reinterpret_cast<void*>(-1)) { |
| 211 failed_ = true; |
| 212 return NULL; |
| 213 } |
| 214 ptr = reinterpret_cast<uintptr_t>(result); |
| 215 if ((ptr & (alignment-1)) != 0) { |
| 216 ptr += alignment - (ptr & (alignment-1)); |
| 217 } |
| 218 return reinterpret_cast<void*>(ptr); |
| 219 } |
| 220 |
| 221 void SbrkSysAllocator::DumpStats(TCMalloc_Printer* printer) { |
| 222 printer->printf("SbrkSysAllocator: failed_=%d\n", failed_); |
| 223 } |
| 224 |
| 225 void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size, |
| 226 size_t alignment) { |
| 227 #ifndef HAVE_MMAP |
| 228 failed_ = true; |
| 229 return NULL; |
| 230 #else |
| 231 // Check if we should use mmap allocation. |
| 232 // FLAGS_malloc_skip_mmap starts out as false (its uninitialized |
| 233 // state) and eventually gets initialized to the specified value. Note |
| 234 // that this code runs for a while before the flags are initialized. |
| 235 // Chances are we never get here before the flags are initialized since |
| 236 // sbrk is used until the heap is exhausted (before mmap is used). |
| 237 if (FLAGS_malloc_skip_mmap) { |
| 238 return NULL; |
| 239 } |
| 240 |
| 241 // could theoretically return the "extra" bytes here, but this |
| 242 // is simple and correct. |
| 243 if (actual_size) { |
| 244 *actual_size = size; |
| 245 } |
| 246 |
| 247 // Enforce page alignment |
| 248 if (pagesize == 0) pagesize = getpagesize(); |
| 249 if (alignment < pagesize) alignment = pagesize; |
| 250 size_t aligned_size = ((size + alignment - 1) / alignment) * alignment; |
| 251 if (aligned_size < size) { |
| 252 return NULL; |
| 253 } |
| 254 size = aligned_size; |
| 255 |
| 256 // Ask for extra memory if alignment > pagesize |
| 257 size_t extra = 0; |
| 258 if (alignment > pagesize) { |
| 259 extra = alignment - pagesize; |
| 260 } |
| 261 |
| 262 // Note: size + extra does not overflow since: |
| 263 // size + alignment < (1<<NBITS). |
| 264 // and extra <= alignment |
| 265 // therefore size + extra < (1<<NBITS) |
| 266 void* result = mmap(NULL, size + extra, |
| 267 PROT_READ|PROT_WRITE, |
| 268 MAP_PRIVATE|MAP_ANONYMOUS, |
| 269 -1, 0); |
| 270 if (result == reinterpret_cast<void*>(MAP_FAILED)) { |
| 271 failed_ = true; |
| 272 return NULL; |
| 273 } |
| 274 |
| 275 // Adjust the return memory so it is aligned |
| 276 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
| 277 size_t adjust = 0; |
| 278 if ((ptr & (alignment - 1)) != 0) { |
| 279 adjust = alignment - (ptr & (alignment - 1)); |
| 280 } |
| 281 |
| 282 // Return the unused memory to the system |
| 283 if (adjust > 0) { |
| 284 munmap(reinterpret_cast<void*>(ptr), adjust); |
| 285 } |
| 286 if (adjust < extra) { |
| 287 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust); |
| 288 } |
| 289 |
| 290 ptr += adjust; |
| 291 return reinterpret_cast<void*>(ptr); |
| 292 #endif // HAVE_MMAP |
| 293 } |
| 294 |
| 295 void MmapSysAllocator::DumpStats(TCMalloc_Printer* printer) { |
| 296 printer->printf("MmapSysAllocator: failed_=%d\n", failed_); |
| 297 } |
| 298 |
| 299 void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size, |
| 300 size_t alignment) { |
| 301 #ifndef HAVE_MMAP |
| 302 failed_ = true; |
| 303 return NULL; |
| 304 #else |
| 305 static bool initialized = false; |
| 306 static off_t physmem_base; // next physical memory address to allocate |
| 307 static off_t physmem_limit; // maximum physical address allowed |
| 308 static int physmem_fd; // file descriptor for /dev/mem |
| 309 |
| 310 // Check if we should use /dev/mem allocation. Note that it may take |
| 311 // a while to get this flag initialized, so meanwhile we fall back to |
| 312 // the next allocator. (It looks like 7MB gets allocated before |
| 313 // this flag gets initialized -khr.) |
| 314 if (FLAGS_malloc_devmem_start == 0) { |
| 315 // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to |
| 316 // try us again next time. |
| 317 return NULL; |
| 318 } |
| 319 |
| 320 if (!initialized) { |
| 321 physmem_fd = open("/dev/mem", O_RDWR); |
| 322 if (physmem_fd < 0) { |
| 323 failed_ = true; |
| 324 return NULL; |
| 325 } |
| 326 physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL; |
| 327 physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL; |
| 328 initialized = true; |
| 329 } |
| 330 |
| 331 // could theoretically return the "extra" bytes here, but this |
| 332 // is simple and correct. |
| 333 if (actual_size) { |
| 334 *actual_size = size; |
| 335 } |
| 336 |
| 337 // Enforce page alignment |
| 338 if (pagesize == 0) pagesize = getpagesize(); |
| 339 if (alignment < pagesize) alignment = pagesize; |
| 340 size_t aligned_size = ((size + alignment - 1) / alignment) * alignment; |
| 341 if (aligned_size < size) { |
| 342 return NULL; |
| 343 } |
| 344 size = aligned_size; |
| 345 |
| 346 // Ask for extra memory if alignment > pagesize |
| 347 size_t extra = 0; |
| 348 if (alignment > pagesize) { |
| 349 extra = alignment - pagesize; |
| 350 } |
| 351 |
| 352 // check to see if we have any memory left |
| 353 if (physmem_limit != 0 && |
| 354 ((size + extra) > (physmem_limit - physmem_base))) { |
| 355 failed_ = true; |
| 356 return NULL; |
| 357 } |
| 358 |
| 359 // Note: size + extra does not overflow since: |
| 360 // size + alignment < (1<<NBITS). |
| 361 // and extra <= alignment |
| 362 // therefore size + extra < (1<<NBITS) |
| 363 void *result = mmap(0, size + extra, PROT_WRITE|PROT_READ, |
| 364 MAP_SHARED, physmem_fd, physmem_base); |
| 365 if (result == reinterpret_cast<void*>(MAP_FAILED)) { |
| 366 failed_ = true; |
| 367 return NULL; |
| 368 } |
| 369 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
| 370 |
| 371 // Adjust the return memory so it is aligned |
| 372 size_t adjust = 0; |
| 373 if ((ptr & (alignment - 1)) != 0) { |
| 374 adjust = alignment - (ptr & (alignment - 1)); |
| 375 } |
| 376 |
| 377 // Return the unused virtual memory to the system |
| 378 if (adjust > 0) { |
| 379 munmap(reinterpret_cast<void*>(ptr), adjust); |
| 380 } |
| 381 if (adjust < extra) { |
| 382 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust); |
| 383 } |
| 384 |
| 385 ptr += adjust; |
| 386 physmem_base += adjust + size; |
| 387 |
| 388 return reinterpret_cast<void*>(ptr); |
| 389 #endif // HAVE_MMAP |
| 390 } |
| 391 |
| 392 void DevMemSysAllocator::DumpStats(TCMalloc_Printer* printer) { |
| 393 printer->printf("DevMemSysAllocator: failed_=%d\n", failed_); |
| 394 } |
| 395 |
| 396 static bool system_alloc_inited = false; |
| 397 void InitSystemAllocators(void) { |
| 398 // This determines the order in which system allocators are called |
| 399 int i = kMaxDynamicAllocators; |
| 400 allocators[i++] = new (devmem_space) DevMemSysAllocator(); |
| 401 |
| 402 // In 64-bit debug mode, place the mmap allocator first since it |
| 403 // allocates pointers that do not fit in 32 bits and therefore gives |
| 404 // us better testing of code's 64-bit correctness. It also leads to |
| 405 // less false negatives in heap-checking code. (Numbers are less |
| 406 // likely to look like pointers and therefore the conservative gc in |
| 407 // the heap-checker is less likely to misinterpret a number as a |
| 408 // pointer). |
| 409 if (kDebugMode && sizeof(void*) > 4) { |
| 410 allocators[i++] = new (mmap_space) MmapSysAllocator(); |
| 411 allocators[i++] = new (sbrk_space) SbrkSysAllocator(); |
| 412 } else { |
| 413 allocators[i++] = new (sbrk_space) SbrkSysAllocator(); |
| 414 allocators[i++] = new (mmap_space) MmapSysAllocator(); |
| 415 } |
| 416 } |
| 417 |
| 418 void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, |
| 419 size_t alignment) { |
| 420 // Discard requests that overflow |
| 421 if (size + alignment < size) return NULL; |
| 422 |
| 423 SpinLockHolder lock_holder(&spinlock); |
| 424 |
| 425 if (!system_alloc_inited) { |
| 426 InitSystemAllocators(); |
| 427 system_alloc_inited = true; |
| 428 } |
| 429 |
| 430 // Enforce minimum alignment |
| 431 if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner); |
| 432 |
| 433 // Try twice, once avoiding allocators that failed before, and once |
| 434 // more trying all allocators even if they failed before. |
| 435 for (int i = 0; i < 2; i++) { |
| 436 for (int j = 0; j < kMaxAllocators; j++) { |
| 437 SysAllocator *a = allocators[j]; |
| 438 if (a == NULL) continue; |
| 439 if (a->usable_ && !a->failed_) { |
| 440 void* result = a->Alloc(size, actual_size, alignment); |
| 441 if (result != NULL) return result; |
| 442 } |
| 443 } |
| 444 |
| 445 // nothing worked - reset failed_ flags and try again |
| 446 for (int j = 0; j < kMaxAllocators; j++) { |
| 447 SysAllocator *a = allocators[j]; |
| 448 if (a == NULL) continue; |
| 449 a->failed_ = false; |
| 450 } |
| 451 } |
| 452 return NULL; |
| 453 } |
| 454 |
| 455 void TCMalloc_SystemRelease(void* start, size_t length) { |
| 456 #ifdef MADV_DONTNEED |
| 457 if (FLAGS_malloc_devmem_start) { |
| 458 // It's not safe to use MADV_DONTNEED if we've been mapping |
| 459 // /dev/mem for heap memory |
| 460 return; |
| 461 } |
| 462 if (pagesize == 0) pagesize = getpagesize(); |
| 463 const size_t pagemask = pagesize - 1; |
| 464 |
| 465 size_t new_start = reinterpret_cast<size_t>(start); |
| 466 size_t end = new_start + length; |
| 467 size_t new_end = end; |
| 468 |
| 469 // Round up the starting address and round down the ending address |
| 470 // to be page aligned: |
| 471 new_start = (new_start + pagesize - 1) & ~pagemask; |
| 472 new_end = new_end & ~pagemask; |
| 473 |
| 474 ASSERT((new_start & pagemask) == 0); |
| 475 ASSERT((new_end & pagemask) == 0); |
| 476 ASSERT(new_start >= reinterpret_cast<size_t>(start)); |
| 477 ASSERT(new_end <= end); |
| 478 |
| 479 if (new_end > new_start) { |
| 480 // Note -- ignoring most return codes, because if this fails it |
| 481 // doesn't matter... |
| 482 while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start, |
| 483 MADV_DONTNEED) == -1 && |
| 484 errno == EAGAIN) { |
| 485 // NOP |
| 486 } |
| 487 } |
| 488 #endif |
| 489 } |
| 490 |
| 491 void TCMalloc_SystemCommit(void* start, size_t length) { |
| 492 // Nothing to do here. TCMalloc_SystemRelease does not alter pages |
| 493 // such that they need to be re-committed before they can be used by the |
| 494 // application. |
| 495 } |
| 496 |
| 497 void DumpSystemAllocatorStats(TCMalloc_Printer* printer) { |
| 498 for (int j = 0; j < kMaxAllocators; j++) { |
| 499 SysAllocator *a = allocators[j]; |
| 500 if (a == NULL) continue; |
| 501 if (a->usable_) { |
| 502 a->DumpStats(printer); |
| 503 } |
| 504 } |
| 505 } |
OLD | NEW |