OLD | NEW |
1 // Copyright (c) 2000, Google Inc. | 1 // Copyright (c) 2000, Google Inc. |
2 // All rights reserved. | 2 // All rights reserved. |
3 // | 3 // |
4 // Redistribution and use in source and binary forms, with or without | 4 // Redistribution and use in source and binary forms, with or without |
5 // modification, are permitted provided that the following conditions are | 5 // modification, are permitted provided that the following conditions are |
6 // met: | 6 // met: |
7 // | 7 // |
8 // * Redistributions of source code must retain the above copyright | 8 // * Redistributions of source code must retain the above copyright |
9 // notice, this list of conditions and the following disclaimer. | 9 // notice, this list of conditions and the following disclaimer. |
10 // * Redistributions in binary form must reproduce the above | 10 // * Redistributions in binary form must reproduce the above |
(...skipping 13 matching lines...) Expand all Loading... |
24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
29 | 29 |
30 // --- | 30 // --- |
31 // Author: Urs Holzle <opensource@google.com> | 31 // Author: Urs Holzle <opensource@google.com> |
32 | 32 |
33 #include "config.h" | 33 #include "config.h" |
34 #ifdef HAVE_MALLOC_H | 34 // We only need malloc.h for struct mallinfo. |
35 #include <malloc.h> | 35 #ifdef HAVE_STRUCT_MALLINFO |
| 36 // Malloc can be in several places on older versions of OS X. |
| 37 # if defined(HAVE_MALLOC_H) |
| 38 # include <malloc.h> |
| 39 # elif defined(HAVE_MALLOC_MALLOC_H) |
| 40 # include <malloc/malloc.h> |
| 41 # elif defined(HAVE_SYS_MALLOC_H) |
| 42 # include <sys/malloc.h> |
| 43 # endif |
36 #endif | 44 #endif |
37 #include <pthread.h> | 45 #include <pthread.h> |
38 #include <stdio.h> | 46 #include <stdio.h> |
39 #ifdef HAVE_INTTYPES_H | 47 #ifdef HAVE_INTTYPES_H |
40 #include <inttypes.h> | 48 #include <inttypes.h> |
41 #endif | 49 #endif |
42 #include <stdarg.h> | 50 #include <stdarg.h> |
43 #ifdef HAVE_MMAP | 51 #ifdef HAVE_MMAP |
44 #include <sys/mman.h> | 52 #include <sys/mman.h> |
45 #endif | 53 #endif |
46 #include <sys/types.h> | 54 #include <sys/types.h> |
47 #include <sys/stat.h> | 55 #include <sys/stat.h> |
48 #ifdef HAVE_FCNTL_H | 56 #ifdef HAVE_FCNTL_H |
49 #include <fcntl.h> | 57 #include <fcntl.h> |
50 #endif | 58 #endif |
51 #ifdef HAVE_UNISTD_H | 59 #ifdef HAVE_UNISTD_H |
52 #include <unistd.h> | 60 #include <unistd.h> |
53 #endif | 61 #endif |
54 #include <errno.h> | 62 #include <errno.h> |
55 #include <string.h> | 63 #include <string.h> |
56 | 64 |
| 65 #include <google/malloc_extension.h> |
| 66 #include <google/malloc_hook.h> |
| 67 #include <google/stacktrace.h> |
57 #include "base/commandlineflags.h" | 68 #include "base/commandlineflags.h" |
58 #include "base/googleinit.h" | 69 #include "base/googleinit.h" |
59 #include "base/logging.h" | 70 #include "base/logging.h" |
60 #include "google/malloc_extension.h" | 71 #include "base/spinlock.h" |
61 #include "google/malloc_hook.h" | |
62 #include "google/stacktrace.h" | |
63 #include "addressmap-inl.h" | 72 #include "addressmap-inl.h" |
64 #include "malloc_hook-inl.h" | 73 #include "malloc_hook-inl.h" |
65 #include "symbolize.h" | 74 #include "symbolize.h" |
66 | 75 |
67 #ifdef TCMALLOC_FOR_DEBUGALLOCATION | 76 #define TCMALLOC_USING_DEBUGALLOCATION |
68 #include "tcmalloc.cc" | 77 #include "tcmalloc.cc" |
69 #else | |
70 #include "base/spinlock.h" | |
71 // Else we already have a SpinLock defined in tcmalloc/internal_spinlock.h | |
72 #endif | |
73 | 78 |
74 // __THROW is defined in glibc systems. It means, counter-intuitively, | 79 // __THROW is defined in glibc systems. It means, counter-intuitively, |
75 // "This function will never throw an exception." It's an optional | 80 // "This function will never throw an exception." It's an optional |
76 // optimization tool, but we may need to use it to match glibc prototypes. | 81 // optimization tool, but we may need to use it to match glibc prototypes. |
77 #ifndef __THROW // I guess we're not on a glibc system | 82 #ifndef __THROW // I guess we're not on a glibc system |
78 # define __THROW // __THROW is just an optimization, so ok to make it "" | 83 # define __THROW // __THROW is just an optimization, so ok to make it "" |
79 #endif | 84 #endif |
80 | 85 |
81 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old | 86 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old |
82 // form of the name instead. | 87 // form of the name instead. |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
119 EnvToBool("TCMALLOC_SYMBOLIZE_STACKTRACE", true), | 124 EnvToBool("TCMALLOC_SYMBOLIZE_STACKTRACE", true), |
120 "Symbolize the stack trace when provided (on some error exits)"); | 125 "Symbolize the stack trace when provided (on some error exits)"); |
121 | 126 |
122 // ========================================================================= // | 127 // ========================================================================= // |
123 | 128 |
124 // A safe version of printf() that does not do any allocation and | 129 // A safe version of printf() that does not do any allocation and |
125 // uses very little stack space. | 130 // uses very little stack space. |
126 static void TracePrintf(int fd, const char *fmt, ...) | 131 static void TracePrintf(int fd, const char *fmt, ...) |
127 __attribute__ ((__format__ (__printf__, 2, 3))); | 132 __attribute__ ((__format__ (__printf__, 2, 3))); |
128 | 133 |
129 // | 134 // The do_* functions are defined in tcmalloc/tcmalloc.cc, |
130 // GNU has some weird "weak aliasing" thing that permits us to define our | |
131 // own malloc(), free(), and realloc() which can use the normal versions of | |
132 // of themselves by calling __libc_malloc(), __libc_free(), and | |
133 // __libc_realloc(). | |
134 // | |
135 extern "C" { | |
136 extern void* __libc_malloc(size_t size); | |
137 extern void __libc_free(void* ptr); | |
138 extern void* __libc_realloc(void* ptr, size_t size); | |
139 extern void* __libc_calloc(size_t nmemb, size_t size); | |
140 extern int __libc_mallopt(int cmd, int value); | |
141 #ifdef HAVE_STRUCT_MALLINFO | |
142 extern struct mallinfo __libc_mallinfo(void); | |
143 #endif | |
144 } | |
145 | |
146 // Define the malloc/free/mallopt/mallinfo implementations | |
147 // we will be working on top of. | |
148 // TODO(csilvers): provide debugallocation on top of libc alloc, | |
149 // so this #ifdef might sometimes be false. | |
150 #ifdef TCMALLOC_FOR_DEBUGALLOCATION | |
151 | |
152 // The do_* functions are defined in tcmalloc.cc, | |
153 // which is included before this file | 135 // which is included before this file |
154 // when TCMALLOC_FOR_DEBUGALLOCATION is defined. | 136 // when TCMALLOC_FOR_DEBUGALLOCATION is defined |
155 #define BASE_MALLOC_NEW(size) cpp_alloc(size, false) | 137 #define BASE_MALLOC_NEW(size) cpp_alloc(size, false) |
156 #define BASE_MALLOC do_malloc_or_cpp_alloc | 138 #define BASE_MALLOC do_malloc |
157 #define BASE_FREE do_free | 139 #define BASE_FREE do_free |
158 #define BASE_MALLOPT do_mallopt | 140 #define BASE_MALLOC_STATS do_malloc_stats |
159 #define BASE_MALLINFO do_mallinfo | 141 #define BASE_MALLOPT do_mallopt |
160 | 142 #define BASE_MALLINFO do_mallinfo |
161 #else | 143 #define BASE_MALLOC_SIZE(ptr) GetSizeWithCallback(ptr, &InvalidGetAllocatedSize) |
162 | |
163 // We are working on top of standard libc's malloc library | |
164 #define BASE_MALLOC_NEW __libc_malloc | |
165 #define BASE_MALLOC __libc_malloc | |
166 #define BASE_FREE __libc_free | |
167 #define BASE_MALLOPT __libc_mallopt | |
168 #define BASE_MALLINFO __libc_mallinfo | |
169 | |
170 #endif | |
171 | 144 |
172 // ========================================================================= // | 145 // ========================================================================= // |
173 | 146 |
174 class MallocBlock; | 147 class MallocBlock; |
175 | 148 |
176 // A circular buffer to hold freed blocks of memory. MallocBlock::Deallocate | 149 // A circular buffer to hold freed blocks of memory. MallocBlock::Deallocate |
177 // (below) pushes blocks into this queue instead of returning them to the | 150 // (below) pushes blocks into this queue instead of returning them to the |
178 // underlying allocator immediately. See MallocBlock::Deallocate for more | 151 // underlying allocator immediately. See MallocBlock::Deallocate for more |
179 // information. | 152 // information. |
180 // | 153 // |
181 // We can't use an STL class for this because we need to be careful not to | 154 // We can't use an STL class for this because we need to be careful not to |
182 // perform any heap de-allocations in any of the code in this class, since the | 155 // perform any heap de-allocations in any of the code in this class, since the |
183 // code in MallocBlock::Deallocate is not re-entrant. | 156 // code in MallocBlock::Deallocate is not re-entrant. |
184 template <typename QueueEntry> | 157 template <typename QueueEntry> |
185 class FreeQueue { | 158 class FreeQueue { |
186 public: | 159 public: |
187 FreeQueue() : q_front_(0), q_back_(0) {} | 160 FreeQueue() : q_front_(0), q_back_(0) {} |
188 | 161 |
189 bool Full() { | 162 bool Full() { |
190 return (q_front_ + 1) % kFreeQueueSize == q_back_; | 163 return (q_front_ + 1) % kFreeQueueSize == q_back_; |
191 } | 164 } |
192 | 165 |
193 void Push(QueueEntry block) { | 166 void Push(const QueueEntry& block) { |
194 q_[q_front_] = block; | 167 q_[q_front_] = block; |
195 q_front_ = (q_front_ + 1) % kFreeQueueSize; | 168 q_front_ = (q_front_ + 1) % kFreeQueueSize; |
196 } | 169 } |
197 | 170 |
198 QueueEntry Pop() { | 171 QueueEntry Pop() { |
199 const QueueEntry& ret = q_[q_back_]; | 172 const QueueEntry& ret = q_[q_back_]; |
200 q_back_ = (q_back_ + 1) % kFreeQueueSize; | 173 q_back_ = (q_back_ + 1) % kFreeQueueSize; |
201 return ret; | 174 return ret; |
202 } | 175 } |
203 | 176 |
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
266 static const int kDeallocatedTypeBit = 0x4; | 239 static const int kDeallocatedTypeBit = 0x4; |
267 | 240 |
268 // For better memory debugging, we initialize all storage to known | 241 // For better memory debugging, we initialize all storage to known |
269 // values, and overwrite the storage when it's deallocated: | 242 // values, and overwrite the storage when it's deallocated: |
270 // Byte that fills uninitialized storage. | 243 // Byte that fills uninitialized storage. |
271 static const int kMagicUninitializedByte = 0xAB; | 244 static const int kMagicUninitializedByte = 0xAB; |
272 // Byte that fills deallocated storage. | 245 // Byte that fills deallocated storage. |
273 // NOTE: tcmalloc.cc depends on the value of kMagicDeletedByte | 246 // NOTE: tcmalloc.cc depends on the value of kMagicDeletedByte |
274 // to work around a bug in the pthread library. | 247 // to work around a bug in the pthread library. |
275 static const int kMagicDeletedByte = 0xCD; | 248 static const int kMagicDeletedByte = 0xCD; |
276 // An int (type of alloc_type_ below) in a deallocated storage | 249 // A size_t (type of alloc_type_ below) in a deallocated storage |
277 // filled with kMagicDeletedByte. | 250 // filled with kMagicDeletedByte. |
278 static const int kMagicDeletedInt = 0xCDCDCDCD | ((0xCDCDCDCD << 16) << 16); | 251 static const size_t kMagicDeletedSizeT = |
279 // Initializer works for 32 and 64 bit ints; | 252 0xCDCDCDCD | (((size_t)0xCDCDCDCD << 16) << 16); |
| 253 // Initializer works for 32 and 64 bit size_ts; |
280 // "<< 16 << 16" is to fool gcc from issuing a warning | 254 // "<< 16 << 16" is to fool gcc from issuing a warning |
281 // when ints are 32 bits. | 255 // when size_ts are 32 bits. |
282 | 256 |
283 // NOTE: on Linux, you can enable malloc debugging support in libc by | 257 // NOTE: on Linux, you can enable malloc debugging support in libc by |
284 // setting the environment variable MALLOC_CHECK_ to 1 before you | 258 // setting the environment variable MALLOC_CHECK_ to 1 before you |
285 // start the program (see man malloc). | 259 // start the program (see man malloc). |
286 | 260 |
287 // We use either BASE_MALLOC or mmap to make the actual allocation. In | 261 // We use either BASE_MALLOC or mmap to make the actual allocation. In |
288 // order to remember which one of the two was used for any block, we store an | 262 // order to remember which one of the two was used for any block, we store an |
289 // appropriate magic word next to the block. | 263 // appropriate magic word next to the block. |
290 static const int kMagicMalloc = 0xDEADBEEF; | 264 static const int kMagicMalloc = 0xDEADBEEF; |
291 static const int kMagicMMap = 0xABCDEFAB; | 265 static const int kMagicMMap = 0xABCDEFAB; |
292 | 266 |
293 // This array will be filled with 0xCD, for use with memcmp. | 267 // This array will be filled with 0xCD, for use with memcmp. |
294 static unsigned char kMagicDeletedBuffer[1024]; | 268 static unsigned char kMagicDeletedBuffer[1024]; |
295 static bool deleted_buffer_initialized_; | 269 static bool deleted_buffer_initialized_; |
296 | 270 |
297 private: // data layout | 271 private: // data layout |
298 | 272 |
299 // The four fields size1_,offset_,magic1_,alloc_type_ | 273 // The four fields size1_,offset_,magic1_,alloc_type_ |
300 // should together occupy a multiple of 8 bytes. | 274 // should together occupy a multiple of 16 bytes. (At the |
| 275 // moment, sizeof(size_t) == 4 or 8 depending on piii vs |
| 276 // k8, and 4 of those sum to 16 or 32 bytes). |
| 277 // This, combined with BASE_MALLOC's alignment guarantees, |
| 278 // ensures that SSE types can be stored into the returned |
| 279 // block, at &size2_. |
301 size_t size1_; | 280 size_t size1_; |
302 size_t offset_; // normally 0 unless memaligned memory | 281 size_t offset_; // normally 0 unless memaligned memory |
303 // see comments in memalign() and FromRawPointer(). | 282 // see comments in memalign() and FromRawPointer(). |
304 int magic1_; | 283 size_t magic1_; |
305 int alloc_type_; | 284 size_t alloc_type_; |
306 // here comes the actual data (variable length) | 285 // here comes the actual data (variable length) |
307 // ... | 286 // ... |
308 // then come the size2_ and magic2_, or a full page of mprotect-ed memory | 287 // then come the size2_ and magic2_, or a full page of mprotect-ed memory |
309 // if the malloc_page_fence feature is enabled. | 288 // if the malloc_page_fence feature is enabled. |
310 size_t size2_; | 289 size_t size2_; |
311 int magic2_; | 290 int magic2_; |
312 | 291 |
313 private: // static data and helpers | 292 private: // static data and helpers |
314 | 293 |
315 // Allocation map: stores the allocation type for each allocated object, | 294 // Allocation map: stores the allocation type for each allocated object, |
(...skipping 112 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
428 RAW_LOG(FATAL, "memory allocation bug: object at %p " | 407 RAW_LOG(FATAL, "memory allocation bug: object at %p " |
429 "has never been allocated", data_addr()); | 408 "has never been allocated", data_addr()); |
430 } else { | 409 } else { |
431 map_type = *found_type; | 410 map_type = *found_type; |
432 } | 411 } |
433 if ((map_type & kDeallocatedTypeBit) != 0) { | 412 if ((map_type & kDeallocatedTypeBit) != 0) { |
434 RAW_LOG(FATAL, "memory allocation bug: object at %p " | 413 RAW_LOG(FATAL, "memory allocation bug: object at %p " |
435 "has been already deallocated (it was allocated with %s)", | 414 "has been already deallocated (it was allocated with %s)", |
436 data_addr(), AllocName(map_type & ~kDeallocatedTypeBit)); | 415 data_addr(), AllocName(map_type & ~kDeallocatedTypeBit)); |
437 } | 416 } |
438 if (alloc_type_ == kMagicDeletedInt) { | 417 if (alloc_type_ == kMagicDeletedSizeT) { |
439 RAW_LOG(FATAL, "memory stomping bug: a word before object at %p " | 418 RAW_LOG(FATAL, "memory stomping bug: a word before object at %p " |
440 "has been corrupted; or else the object has been already " | 419 "has been corrupted; or else the object has been already " |
441 "deallocated and our memory map has been corrupted", | 420 "deallocated and our memory map has been corrupted", |
442 data_addr()); | 421 data_addr()); |
443 } | 422 } |
444 if (!IsValidMagicValue(magic1_)) { | 423 if (!IsValidMagicValue(magic1_)) { |
445 RAW_LOG(FATAL, "memory stomping bug: a word before object at %p " | 424 RAW_LOG(FATAL, "memory stomping bug: a word before object at %p " |
446 "has been corrupted; " | 425 "has been corrupted; " |
447 "or else our memory map has been corrupted and this is a " | 426 "or else our memory map has been corrupted and this is a " |
448 "deallocation for not (currently) heap-allocated object", | 427 "deallocation for not (currently) heap-allocated object", |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
490 | 469 |
491 public: // our main interface | 470 public: // our main interface |
492 | 471 |
493 static MallocBlock* Allocate(size_t size, int type) { | 472 static MallocBlock* Allocate(size_t size, int type) { |
494 // Prevent an integer overflow / crash with large allocation sizes. | 473 // Prevent an integer overflow / crash with large allocation sizes. |
495 // TODO - Note that for a e.g. 64-bit size_t, max_size_t may not actually | 474 // TODO - Note that for a e.g. 64-bit size_t, max_size_t may not actually |
496 // be the maximum value, depending on how the compiler treats ~0. The worst | 475 // be the maximum value, depending on how the compiler treats ~0. The worst |
497 // practical effect is that allocations are limited to 4Gb or so, even if | 476 // practical effect is that allocations are limited to 4Gb or so, even if |
498 // the address space could take more. | 477 // the address space could take more. |
499 static size_t max_size_t = ~0; | 478 static size_t max_size_t = ~0; |
500 if (size < 0 || size > max_size_t - sizeof(MallocBlock)) { | 479 if (size > max_size_t - sizeof(MallocBlock)) { |
501 RAW_LOG(ERROR, "Massive size passed to malloc: %"PRIuS"", size); | 480 RAW_LOG(ERROR, "Massive size passed to malloc: %"PRIuS"", size); |
502 return NULL; | 481 return NULL; |
503 } | 482 } |
504 MallocBlock* b = NULL; | 483 MallocBlock* b = NULL; |
505 const bool use_malloc_page_fence = FLAGS_malloc_page_fence; | 484 const bool use_malloc_page_fence = FLAGS_malloc_page_fence; |
506 #ifdef HAVE_MMAP | 485 #ifdef HAVE_MMAP |
507 if (use_malloc_page_fence) { | 486 if (use_malloc_page_fence) { |
508 // Put the block towards the end of the page and make the next page | 487 // Put the block towards the end of the page and make the next page |
509 // inaccessible. This will catch buffer overrun right when it happens. | 488 // inaccessible. This will catch buffer overrun right when it happens. |
510 size_t sz = real_mmapped_size(size); | 489 size_t sz = real_mmapped_size(size); |
(...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
655 RAW_LOG(ERROR, "Buffer too large to print corruption."); | 634 RAW_LOG(ERROR, "Buffer too large to print corruption."); |
656 } | 635 } |
657 | 636 |
658 const MallocBlock* b = queue_entry.block; | 637 const MallocBlock* b = queue_entry.block; |
659 const size_t size = queue_entry.size; | 638 const size_t size = queue_entry.size; |
660 if (queue_entry.num_deleter_pcs > 0) { | 639 if (queue_entry.num_deleter_pcs > 0) { |
661 TracePrintf(STDERR_FILENO, "Deleted by thread %p\n", | 640 TracePrintf(STDERR_FILENO, "Deleted by thread %p\n", |
662 reinterpret_cast<void*>( | 641 reinterpret_cast<void*>( |
663 PRINTABLE_PTHREAD(queue_entry.deleter_threadid))); | 642 PRINTABLE_PTHREAD(queue_entry.deleter_threadid))); |
664 | 643 |
665 SymbolTable symbolization_table; | 644 // We don't want to allocate or deallocate memory here, so we use |
666 const int num_symbols = queue_entry.num_deleter_pcs; // short alias name | 645 // placement-new. It's ok that we don't destroy this, since we're |
667 for (int i = 0; i < num_symbols; i++) { | 646 // just going to error-exit below anyway. Union is for alignment. |
| 647 union { void* alignment; char buf[sizeof(SymbolTable)]; } tablebuf; |
| 648 SymbolTable* symbolization_table = new (tablebuf.buf) SymbolTable; |
| 649 for (int i = 0; i < queue_entry.num_deleter_pcs; i++) { |
668 // Symbolizes the previous address of pc because pc may be in the | 650 // Symbolizes the previous address of pc because pc may be in the |
669 // next function. This may happen when the function ends with | 651 // next function. This may happen when the function ends with |
670 // a call to a function annotated noreturn (e.g. CHECK). | 652 // a call to a function annotated noreturn (e.g. CHECK). |
671 char* pc = | 653 char *pc = reinterpret_cast<char*>(queue_entry.deleter_pcs[i]); |
672 reinterpret_cast<char*>(queue_entry.deleter_pcs[i]) - 1; | 654 symbolization_table->Add(pc - 1); |
673 symbolization_table.Add(pc); | |
674 } | 655 } |
675 if (FLAGS_symbolize_stacktrace) | 656 if (FLAGS_symbolize_stacktrace) |
676 symbolization_table.Symbolize(); | 657 symbolization_table->Symbolize(); |
677 for (int i = 0; i < num_symbols; i++) { | 658 for (int i = 0; i < queue_entry.num_deleter_pcs; i++) { |
678 char *pc = | 659 char *pc = reinterpret_cast<char*>(queue_entry.deleter_pcs[i]); |
679 reinterpret_cast<char*>(queue_entry.deleter_pcs[i]) - 1; | 660 TracePrintf(STDERR_FILENO, " @ %p %s\n", |
680 TracePrintf(STDERR_FILENO, " @ %"PRIxPTR" %s\n", | 661 pc, symbolization_table->GetSymbol(pc - 1)); |
681 reinterpret_cast<uintptr_t>(pc), | |
682 symbolization_table.GetSymbol(pc)); | |
683 } | 662 } |
684 } else { | 663 } else { |
685 RAW_LOG(ERROR, | 664 RAW_LOG(ERROR, |
686 "Skipping the printing of the deleter's stack! Its stack was " | 665 "Skipping the printing of the deleter's stack! Its stack was " |
687 "not found; either the corruption occurred too early in " | 666 "not found; either the corruption occurred too early in " |
688 "execution to obtain a stack trace or --max_free_queue_size was " | 667 "execution to obtain a stack trace or --max_free_queue_size was " |
689 "set to 0."); | 668 "set to 0."); |
690 } | 669 } |
691 | 670 |
692 RAW_LOG(FATAL, | 671 RAW_LOG(FATAL, |
693 "Memory was written to after being freed. MallocBlock: %p, user " | 672 "Memory was written to after being freed. MallocBlock: %p, user " |
694 "ptr: %p, size: %zd. If you can't find the source of the error, " | 673 "ptr: %p, size: %zd. If you can't find the source of the error, " |
695 "try using valgrind or purify, or study the output of the " | 674 "try using valgrind or purify, or study the output of the " |
696 "deleter's stack printed above.", b, b->data_addr(), size); | 675 "deleter's stack printed above.", b, b->data_addr(), size); |
697 } | 676 } |
698 | 677 |
699 static MallocBlock* FromRawPointer(void* p) { | 678 static MallocBlock* FromRawPointer(void* p) { |
700 const size_t data_offset = MallocBlock::data_offset(); | 679 const size_t data_offset = MallocBlock::data_offset(); |
701 // Find the header just before client's memory. | 680 // Find the header just before client's memory. |
702 MallocBlock *mb = reinterpret_cast<MallocBlock *>( | 681 MallocBlock *mb = reinterpret_cast<MallocBlock *>( |
703 reinterpret_cast<char *>(p) - data_offset); | 682 reinterpret_cast<char *>(p) - data_offset); |
704 // If mb->alloc_type_ is kMagicDeletedInt, we're not an ok pointer. | 683 // If mb->alloc_type_ is kMagicDeletedSizeT, we're not an ok pointer. |
705 if (mb->alloc_type_ == kMagicDeletedInt) { | 684 if (mb->alloc_type_ == kMagicDeletedSizeT) { |
706 RAW_LOG(FATAL, "memory allocation bug: object at %p has been already" | 685 RAW_LOG(FATAL, "memory allocation bug: object at %p has been already" |
707 " deallocated; or else a word before the object has been" | 686 " deallocated; or else a word before the object has been" |
708 " corrupted (memory stomping bug)", p); | 687 " corrupted (memory stomping bug)", p); |
709 } | 688 } |
710 // If mb->offset_ is zero (common case), mb is the real header. If | 689 // If mb->offset_ is zero (common case), mb is the real header. If |
711 // mb->offset_ is non-zero, this block was allocated by memalign, and | 690 // mb->offset_ is non-zero, this block was allocated by memalign, and |
712 // mb->offset_ is the distance backwards to the real header from mb, | 691 // mb->offset_ is the distance backwards to the real header from mb, |
713 // which is a fake header. The following subtraction works for both zero | 692 // which is a fake header. The following subtraction works for both zero |
714 // and non-zero values. | 693 // and non-zero values. |
715 return reinterpret_cast<MallocBlock *>( | 694 return reinterpret_cast<MallocBlock *>( |
(...skipping 253 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
969 | 948 |
970 static inline void DebugDeallocate(void* ptr, int type) { | 949 static inline void DebugDeallocate(void* ptr, int type) { |
971 MALLOC_TRACE("free", | 950 MALLOC_TRACE("free", |
972 (ptr != 0 ? MallocBlock::FromRawPointer(ptr)->data_size() : 0), | 951 (ptr != 0 ? MallocBlock::FromRawPointer(ptr)->data_size() : 0), |
973 ptr); | 952 ptr); |
974 if (ptr) MallocBlock::FromRawPointer(ptr)->Deallocate(type); | 953 if (ptr) MallocBlock::FromRawPointer(ptr)->Deallocate(type); |
975 } | 954 } |
976 | 955 |
977 // ========================================================================= // | 956 // ========================================================================= // |
978 | 957 |
979 // Alloc/free stuff for debug hooks for malloc & friends | 958 // The following functions may be called via MallocExtension::instance() |
| 959 // for memory verification and statistics. |
| 960 class DebugMallocImplementation : public TCMallocImplementation { |
| 961 public: |
| 962 virtual bool GetNumericProperty(const char* name, size_t* value) { |
| 963 bool result = TCMallocImplementation::GetNumericProperty(name, value); |
| 964 if (result && (strcmp(name, "generic.current_allocated_bytes") == 0)) { |
| 965 // Subtract bytes kept in the free queue |
| 966 size_t qsize = MallocBlock::FreeQueueSize(); |
| 967 if (*value >= qsize) { |
| 968 *value -= qsize; |
| 969 } |
| 970 } |
| 971 return result; |
| 972 } |
980 | 973 |
981 // CAVEAT: The code structure below ensures that MallocHook methods are always | 974 virtual bool VerifyNewMemory(void* p) { |
982 // called from the stack frame of the invoked allocation function. | 975 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType); |
983 // heap-checker.cc depends on this to start a stack trace from | 976 return true; |
984 // the call to the (de)allocation function. | 977 } |
985 | 978 |
986 // Put all callers of MallocHook::Invoke* in this module into | 979 virtual bool VerifyArrayNewMemory(void* p) { |
987 // ATTRIBUTE_SECTION(google_malloc) section, | 980 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType); |
988 // so that MallocHook::GetCallerStackTrace can function accurately: | 981 return true; |
| 982 } |
989 | 983 |
990 extern "C" { | 984 virtual bool VerifyMallocMemory(void* p) { |
991 void* malloc(size_t size) __THROW ATTRIBUTE_SECTION(google_malloc); | 985 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType); |
992 void free(void* ptr) __THROW ATTRIBUTE_SECTION(google_malloc); | 986 return true; |
993 void* realloc(void* ptr, size_t size) __THROW | 987 } |
994 ATTRIBUTE_SECTION(google_malloc); | |
995 void* calloc(size_t nmemb, size_t size) __THROW | |
996 ATTRIBUTE_SECTION(google_malloc); | |
997 void cfree(void* ptr) __THROW ATTRIBUTE_SECTION(google_malloc); | |
998 | 988 |
999 void* memalign(size_t __alignment, size_t __size) __THROW | 989 virtual bool VerifyAllMemory() { |
1000 ATTRIBUTE_SECTION(google_malloc); | 990 return MallocBlock::CheckEverything(); |
1001 int posix_memalign(void** ptr, size_t align, size_t size) __THROW | 991 } |
1002 ATTRIBUTE_SECTION(google_malloc); | 992 |
1003 void* valloc(size_t __size) __THROW | 993 virtual bool MallocMemoryStats(int* blocks, size_t* total, |
1004 ATTRIBUTE_SECTION(google_malloc); | 994 int histogram[kMallocHistogramSize]) { |
1005 void* pvalloc(size_t __size) __THROW | 995 return MallocBlock::MemoryStats(blocks, total, histogram); |
1006 ATTRIBUTE_SECTION(google_malloc); | 996 } |
| 997 |
| 998 virtual size_t GetAllocatedSize(void* p) { |
| 999 if (p) { |
| 1000 return MallocBlock::FromRawPointer(p)->data_size(); |
| 1001 } |
| 1002 return 0; |
| 1003 } |
| 1004 virtual size_t GetEstimatedAllocatedSize(size_t size) { |
| 1005 return size; |
| 1006 } |
| 1007 |
| 1008 virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) { |
| 1009 static const char* kDebugFreeQueue = "debug.free_queue"; |
| 1010 |
| 1011 TCMallocImplementation::GetFreeListSizes(v); |
| 1012 |
| 1013 MallocExtension::FreeListInfo i; |
| 1014 i.type = kDebugFreeQueue; |
| 1015 i.min_object_size = 0; |
| 1016 i.max_object_size = numeric_limits<size_t>::max(); |
| 1017 i.total_bytes_free = MallocBlock::FreeQueueSize(); |
| 1018 v->push_back(i); |
| 1019 } |
| 1020 |
| 1021 }; |
| 1022 |
| 1023 static DebugMallocImplementation debug_malloc_implementation; |
| 1024 |
| 1025 REGISTER_MODULE_INITIALIZER(debugallocation, { |
| 1026 // Either we or valgrind will control memory management. We |
| 1027 // register our extension if we're the winner. |
| 1028 if (RunningOnValgrind()) { |
| 1029 // Let Valgrind uses its own malloc (so don't register our extension). |
| 1030 } else { |
| 1031 MallocExtension::Register(&debug_malloc_implementation); |
| 1032 // When the program exits, check all blocks still in the free |
| 1033 // queue for corruption. |
| 1034 atexit(DanglingWriteChecker); |
| 1035 } |
| 1036 }); |
| 1037 |
| 1038 // ========================================================================= // |
| 1039 |
| 1040 // This is mostly the same a cpp_alloc in tcmalloc.cc. |
| 1041 // TODO(csilvers): write a wrapper for new-handler so we don't have to |
| 1042 // copy this code so much. |
| 1043 inline void* debug_cpp_alloc(size_t size, int new_type, bool nothrow) { |
| 1044 for (;;) { |
| 1045 void* p = DebugAllocate(size, new_type); |
| 1046 #ifdef PREANSINEW |
| 1047 return p; |
| 1048 #else |
| 1049 if (p == NULL) { // allocation failed |
| 1050 // Get the current new handler. NB: this function is not |
| 1051 // thread-safe. We make a feeble stab at making it so here, but |
| 1052 // this lock only protects against tcmalloc interfering with |
| 1053 // itself, not with other libraries calling set_new_handler. |
| 1054 std::new_handler nh; |
| 1055 { |
| 1056 SpinLockHolder h(&set_new_handler_lock); |
| 1057 nh = std::set_new_handler(0); |
| 1058 (void) std::set_new_handler(nh); |
| 1059 } |
| 1060 #if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) &
& !_HAS_EXCEPTIONS) |
| 1061 if (nh) { |
| 1062 // Since exceptions are disabled, we don't really know if new_handler |
| 1063 // failed. Assume it will abort if it fails. |
| 1064 (*nh)(); |
| 1065 continue; |
| 1066 } |
| 1067 return 0; |
| 1068 #else |
| 1069 // If no new_handler is established, the allocation failed. |
| 1070 if (!nh) { |
| 1071 if (nothrow) return 0; |
| 1072 throw std::bad_alloc(); |
| 1073 } |
| 1074 // Otherwise, try the new_handler. If it returns, retry the |
| 1075 // allocation. If it throws std::bad_alloc, fail the allocation. |
| 1076 // if it throws something else, don't interfere. |
| 1077 try { |
| 1078 (*nh)(); |
| 1079 } catch (const std::bad_alloc&) { |
| 1080 if (!nothrow) throw; |
| 1081 return p; |
| 1082 } |
| 1083 #endif // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPT
IONS) && !_HAS_EXCEPTIONS) |
| 1084 } else { // allocation success |
| 1085 return p; |
| 1086 } |
| 1087 #endif // PREANSINEW |
| 1088 } |
1007 } | 1089 } |
1008 | 1090 |
1009 static void *MemalignOverride(size_t align, size_t size, | 1091 inline void* do_debug_malloc_or_debug_cpp_alloc(size_t size) { |
1010 const void *caller) __THROW | 1092 return tc_new_mode ? debug_cpp_alloc(size, MallocBlock::kMallocType, true) |
1011 ATTRIBUTE_SECTION(google_malloc); | 1093 : DebugAllocate(size, MallocBlock::kMallocType); |
| 1094 } |
1012 | 1095 |
1013 void* operator new(size_t size) throw (std::bad_alloc) | 1096 // Exported routines |
1014 ATTRIBUTE_SECTION(google_malloc); | |
1015 void* operator new(size_t size, const std::nothrow_t&) __THROW | |
1016 ATTRIBUTE_SECTION(google_malloc); | |
1017 void operator delete(void* p) __THROW | |
1018 ATTRIBUTE_SECTION(google_malloc); | |
1019 void operator delete(void* p, const std::nothrow_t&) __THROW | |
1020 ATTRIBUTE_SECTION(google_malloc); | |
1021 void* operator new[](size_t size) throw (std::bad_alloc) | |
1022 ATTRIBUTE_SECTION(google_malloc); | |
1023 void* operator new[](size_t size, const std::nothrow_t&) __THROW | |
1024 ATTRIBUTE_SECTION(google_malloc); | |
1025 void operator delete[](void* p) __THROW | |
1026 ATTRIBUTE_SECTION(google_malloc); | |
1027 void operator delete[](void* p, const std::nothrow_t&) __THROW | |
1028 ATTRIBUTE_SECTION(google_malloc); | |
1029 | 1097 |
1030 extern "C" void* malloc(size_t size) __THROW { | 1098 extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW { |
1031 void* ptr = DebugAllocate(size, MallocBlock::kMallocType); | 1099 void* ptr = do_debug_malloc_or_debug_cpp_alloc(size); |
1032 MallocHook::InvokeNewHook(ptr, size); | 1100 MallocHook::InvokeNewHook(ptr, size); |
1033 return ptr; | 1101 return ptr; |
1034 } | 1102 } |
1035 | 1103 |
1036 extern "C" void free(void* ptr) __THROW { | 1104 extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW { |
1037 MallocHook::InvokeDeleteHook(ptr); | 1105 MallocHook::InvokeDeleteHook(ptr); |
1038 DebugDeallocate(ptr, MallocBlock::kMallocType); | 1106 DebugDeallocate(ptr, MallocBlock::kMallocType); |
1039 } | 1107 } |
1040 | 1108 |
1041 extern "C" void* realloc(void* ptr, size_t size) __THROW { | 1109 extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t count, size_t size) __THROW
{ |
| 1110 // Overflow check |
| 1111 const size_t total_size = count * size; |
| 1112 if (size != 0 && total_size / size != count) return NULL; |
| 1113 |
| 1114 void* block = do_debug_malloc_or_debug_cpp_alloc(total_size); |
| 1115 MallocHook::InvokeNewHook(block, total_size); |
| 1116 if (block) memset(block, 0, total_size); |
| 1117 return block; |
| 1118 } |
| 1119 |
| 1120 extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW { |
| 1121 MallocHook::InvokeDeleteHook(ptr); |
| 1122 DebugDeallocate(ptr, MallocBlock::kMallocType); |
| 1123 } |
| 1124 |
| 1125 extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW { |
1042 if (ptr == NULL) { | 1126 if (ptr == NULL) { |
1043 ptr = DebugAllocate(size, MallocBlock::kMallocType); | 1127 ptr = do_debug_malloc_or_debug_cpp_alloc(size); |
1044 MallocHook::InvokeNewHook(ptr, size); | 1128 MallocHook::InvokeNewHook(ptr, size); |
1045 return ptr; | 1129 return ptr; |
1046 } | 1130 } |
1047 if (size == 0) { | 1131 if (size == 0) { |
1048 MallocHook::InvokeDeleteHook(ptr); | 1132 MallocHook::InvokeDeleteHook(ptr); |
1049 DebugDeallocate(ptr, MallocBlock::kMallocType); | 1133 DebugDeallocate(ptr, MallocBlock::kMallocType); |
1050 return NULL; | 1134 return NULL; |
1051 } | 1135 } |
1052 MallocBlock* old = MallocBlock::FromRawPointer(ptr); | 1136 MallocBlock* old = MallocBlock::FromRawPointer(ptr); |
1053 old->Check(MallocBlock::kMallocType); | 1137 old->Check(MallocBlock::kMallocType); |
1054 MallocBlock* p = MallocBlock::Allocate(size, MallocBlock::kMallocType); | 1138 MallocBlock* p = MallocBlock::Allocate(size, MallocBlock::kMallocType); |
1055 | 1139 |
1056 // If realloc fails we are to leave the old block untouched and | 1140 // If realloc fails we are to leave the old block untouched and |
1057 // return null | 1141 // return null |
1058 if (p == NULL) return NULL; | 1142 if (p == NULL) return NULL; |
1059 | 1143 |
1060 memcpy(p->data_addr(), old->data_addr(), | 1144 memcpy(p->data_addr(), old->data_addr(), |
1061 (old->data_size() < size) ? old->data_size() : size); | 1145 (old->data_size() < size) ? old->data_size() : size); |
1062 MallocHook::InvokeDeleteHook(ptr); | 1146 MallocHook::InvokeDeleteHook(ptr); |
1063 MallocHook::InvokeNewHook(p->data_addr(), size); | 1147 MallocHook::InvokeNewHook(p->data_addr(), size); |
1064 DebugDeallocate(ptr, MallocBlock::kMallocType); | 1148 DebugDeallocate(ptr, MallocBlock::kMallocType); |
1065 MALLOC_TRACE("realloc", p->data_size(), p->data_addr()); | 1149 MALLOC_TRACE("realloc", p->data_size(), p->data_addr()); |
1066 return p->data_addr(); | 1150 return p->data_addr(); |
1067 } | 1151 } |
1068 | 1152 |
1069 extern "C" void* calloc(size_t count, size_t size) __THROW { | 1153 extern "C" PERFTOOLS_DLL_DECL void* tc_new(size_t size) { |
1070 // Overflow check | 1154 void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, false); |
1071 const size_t total_size = count * size; | 1155 MallocHook::InvokeNewHook(ptr, size); |
1072 if (size != 0 && total_size / size != count) return NULL; | 1156 if (ptr == NULL) { |
1073 | 1157 RAW_LOG(FATAL, "Unable to allocate %"PRIuS" bytes: new failed.", size); |
1074 void* block = DebugAllocate(total_size, MallocBlock::kMallocType); | 1158 } |
1075 MallocHook::InvokeNewHook(block, total_size); | 1159 return ptr; |
1076 if (block) memset(block, 0, total_size); | |
1077 return block; | |
1078 } | 1160 } |
1079 | 1161 |
1080 extern "C" void cfree(void* ptr) __THROW { | 1162 extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, const std::nothr
ow_t&) __THROW { |
1081 MallocHook::InvokeDeleteHook(ptr); | 1163 void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, true); |
1082 DebugDeallocate(ptr, MallocBlock::kMallocType); | 1164 MallocHook::InvokeNewHook(ptr, size); |
| 1165 return ptr; |
| 1166 } |
| 1167 |
| 1168 extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) __THROW { |
| 1169 MallocHook::InvokeDeleteHook(p); |
| 1170 DebugDeallocate(p, MallocBlock::kNewType); |
| 1171 } |
| 1172 |
| 1173 // Some STL implementations explicitly invoke this. |
| 1174 // It is completely equivalent to a normal delete (delete never throws). |
| 1175 extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow
_t&) __THROW { |
| 1176 MallocHook::InvokeDeleteHook(p); |
| 1177 DebugDeallocate(p, MallocBlock::kNewType); |
| 1178 } |
| 1179 |
| 1180 extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) { |
| 1181 void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, false); |
| 1182 MallocHook::InvokeNewHook(ptr, size); |
| 1183 if (ptr == NULL) { |
| 1184 RAW_LOG(FATAL, "Unable to allocate %"PRIuS" bytes: new[] failed.", size); |
| 1185 } |
| 1186 return ptr; |
| 1187 } |
| 1188 |
| 1189 extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::
nothrow_t&) |
| 1190 __THROW { |
| 1191 void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, true); |
| 1192 MallocHook::InvokeNewHook(ptr, size); |
| 1193 return ptr; |
| 1194 } |
| 1195 |
| 1196 extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) __THROW { |
| 1197 MallocHook::InvokeDeleteHook(p); |
| 1198 DebugDeallocate(p, MallocBlock::kArrayNewType); |
| 1199 } |
| 1200 |
| 1201 // Some STL implementations explicitly invoke this. |
| 1202 // It is completely equivalent to a normal delete (delete never throws). |
| 1203 extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::no
throw_t&) __THROW { |
| 1204 MallocHook::InvokeDeleteHook(p); |
| 1205 DebugDeallocate(p, MallocBlock::kArrayNewType); |
1083 } | 1206 } |
1084 | 1207 |
1085 // Round "value" up to next "alignment" boundary. | 1208 // Round "value" up to next "alignment" boundary. |
1086 // Requires that "alignment" be a power of two. | 1209 // Requires that "alignment" be a power of two. |
1087 static intptr_t RoundUp(intptr_t value, intptr_t alignment) { | 1210 static intptr_t RoundUp(intptr_t value, intptr_t alignment) { |
1088 return (value + alignment - 1) & ~(alignment - 1); | 1211 return (value + alignment - 1) & ~(alignment - 1); |
1089 } | 1212 } |
1090 | 1213 |
| 1214 // This is mostly the same as do_memalign in tcmalloc.cc. |
1091 static void *do_debug_memalign(size_t alignment, size_t size) { | 1215 static void *do_debug_memalign(size_t alignment, size_t size) { |
1092 // Allocate >= size bytes aligned on "alignment" boundary | 1216 // Allocate >= size bytes aligned on "alignment" boundary |
1093 // "alignment" is a power of two. | 1217 // "alignment" is a power of two. |
1094 void *p = 0; | 1218 void *p = 0; |
1095 RAW_CHECK((alignment & (alignment-1)) == 0, "must be power of two"); | 1219 RAW_CHECK((alignment & (alignment-1)) == 0, "must be power of two"); |
1096 const size_t data_offset = MallocBlock::data_offset(); | 1220 const size_t data_offset = MallocBlock::data_offset(); |
1097 // Allocate "alignment-1" extra bytes to ensure alignment is possible, and | 1221 // Allocate "alignment-1" extra bytes to ensure alignment is possible, and |
1098 // a further data_offset bytes for an additional fake header. | 1222 // a further data_offset bytes for an additional fake header. |
1099 size_t extra_bytes = data_offset + alignment - 1; | 1223 size_t extra_bytes = data_offset + alignment - 1; |
1100 if (size + extra_bytes < size) return NULL; // Overflow | 1224 if (size + extra_bytes < size) return NULL; // Overflow |
1101 p = DebugAllocate(size + extra_bytes, MallocBlock::kMallocType); | 1225 p = DebugAllocate(size + extra_bytes, MallocBlock::kMallocType); |
1102 if (p != 0) { | 1226 if (p != 0) { |
1103 intptr_t orig_p = reinterpret_cast<intptr_t>(p); | 1227 intptr_t orig_p = reinterpret_cast<intptr_t>(p); |
1104 // Leave data_offset bytes for fake header, and round up to meet | 1228 // Leave data_offset bytes for fake header, and round up to meet |
1105 // alignment. | 1229 // alignment. |
1106 p = reinterpret_cast<void *>(RoundUp(orig_p + data_offset, alignment)); | 1230 p = reinterpret_cast<void *>(RoundUp(orig_p + data_offset, alignment)); |
1107 // Create a fake header block with an offset_ that points back to the | 1231 // Create a fake header block with an offset_ that points back to the |
1108 // real header. FromRawPointer uses this value. | 1232 // real header. FromRawPointer uses this value. |
1109 MallocBlock *fake_hdr = reinterpret_cast<MallocBlock *>( | 1233 MallocBlock *fake_hdr = reinterpret_cast<MallocBlock *>( |
1110 reinterpret_cast<char *>(p) - data_offset); | 1234 reinterpret_cast<char *>(p) - data_offset); |
1111 // offset_ is distance between real and fake headers. | 1235 // offset_ is distance between real and fake headers. |
1112 // p is now end of fake header (beginning of client area), | 1236 // p is now end of fake header (beginning of client area), |
1113 // and orig_p is the end of the real header, so offset_ | 1237 // and orig_p is the end of the real header, so offset_ |
1114 // is their difference. | 1238 // is their difference. |
1115 fake_hdr->set_offset(reinterpret_cast<intptr_t>(p) - orig_p); | 1239 fake_hdr->set_offset(reinterpret_cast<intptr_t>(p) - orig_p); |
1116 } | 1240 } |
1117 return p; | 1241 return p; |
1118 } | 1242 } |
1119 | 1243 |
1120 // Override __libc_memalign in libc on linux boxes. | 1244 // This is mostly the same as cpp_memalign in tcmalloc.cc. |
1121 // They have a bug in libc that causes them (very rarely) to allocate | 1245 static void* debug_cpp_memalign(size_t align, size_t size) { |
1122 // with __libc_memalign() yet deallocate with free(). | |
1123 // This function is an exception to the rule of calling MallocHook method | |
1124 // from the stack frame of the allocation function; | |
1125 // heap-checker handles this special case explicitly. | |
1126 static void *MemalignOverride(size_t align, size_t size, | |
1127 const void *caller) __THROW { | |
1128 void *p = do_debug_memalign(align, size); | |
1129 MallocHook::InvokeNewHook(p, size); | |
1130 return p; | |
1131 } | |
1132 void *(*__memalign_hook)(size_t, size_t, const void *) = MemalignOverride; | |
1133 | |
1134 extern "C" void* memalign(size_t align, size_t size) __THROW { | |
1135 void *p = do_debug_memalign(align, size); | |
1136 MallocHook::InvokeNewHook(p, size); | |
1137 return p; | |
1138 } | |
1139 | |
1140 // Implementation taken from tcmalloc/tcmalloc.cc | |
1141 extern "C" int posix_memalign(void** result_ptr, | |
1142 size_t align, size_t size) __THROW { | |
1143 if (((align % sizeof(void*)) != 0) || | |
1144 ((align & (align - 1)) != 0) || | |
1145 (align == 0)) { | |
1146 return EINVAL; | |
1147 } | |
1148 | |
1149 void* result = do_debug_memalign(align, size); | |
1150 MallocHook::InvokeNewHook(result, size); | |
1151 if (result == NULL) { | |
1152 return ENOMEM; | |
1153 } else { | |
1154 *result_ptr = result; | |
1155 return 0; | |
1156 } | |
1157 } | |
1158 | |
1159 extern "C" void* valloc(size_t size) __THROW { | |
1160 // Allocate >= size bytes starting on a page boundary | |
1161 void *p = do_debug_memalign(getpagesize(), size); | |
1162 MallocHook::InvokeNewHook(p, size); | |
1163 return p; | |
1164 } | |
1165 | |
1166 extern "C" void* pvalloc(size_t size) __THROW { | |
1167 // Round size up to a multiple of pages | |
1168 // then allocate memory on a page boundary | |
1169 int pagesize = getpagesize(); | |
1170 size = RoundUp(size, pagesize); | |
1171 if (size == 0) { // pvalloc(0) should allocate one page, according to | |
1172 size = pagesize; // http://man.free4web.biz/man3/libmpatrol.3.html | |
1173 } | |
1174 void *p = do_debug_memalign(pagesize, size); | |
1175 MallocHook::InvokeNewHook(p, size); | |
1176 return p; | |
1177 } | |
1178 | |
1179 extern "C" int mallopt(int cmd, int value) __THROW { | |
1180 return BASE_MALLOPT(cmd, value); | |
1181 } | |
1182 | |
1183 #ifdef HAVE_STRUCT_MALLINFO | |
1184 extern "C" struct mallinfo mallinfo(void) __THROW { | |
1185 return BASE_MALLINFO(); | |
1186 } | |
1187 #endif | |
1188 | |
1189 // ========================================================================= // | |
1190 | |
1191 // Alloc/free stuff for debug operator new & friends | |
1192 | |
1193 // This is mostly the same a cpp_alloc in tcmalloc.cc. | |
1194 inline void* cpp_debug_alloc(size_t size, int new_type, bool nothrow) { | |
1195 for (;;) { | 1246 for (;;) { |
1196 void* p = DebugAllocate(size, new_type); | 1247 void* p = do_debug_memalign(align, size); |
1197 #ifdef PREANSINEW | 1248 #ifdef PREANSINEW |
1198 return p; | 1249 return p; |
1199 #else | 1250 #else |
1200 if (p == NULL) { // allocation failed | 1251 if (p == NULL) { // allocation failed |
1201 // Get the current new handler. NB: this function is not | 1252 // Get the current new handler. NB: this function is not |
1202 // thread-safe. We make a feeble stab at making it so here, but | 1253 // thread-safe. We make a feeble stab at making it so here, but |
1203 // this lock only protects against tcmalloc interfering with | 1254 // this lock only protects against tcmalloc interfering with |
1204 // itself, not with other libraries calling set_new_handler. | 1255 // itself, not with other libraries calling set_new_handler. |
1205 std::new_handler nh; | 1256 std::new_handler nh; |
1206 { | 1257 { |
1207 SpinLockHolder h(&set_new_handler_lock); | 1258 SpinLockHolder h(&set_new_handler_lock); |
1208 nh = std::set_new_handler(0); | 1259 nh = std::set_new_handler(0); |
1209 (void) std::set_new_handler(nh); | 1260 (void) std::set_new_handler(nh); |
1210 } | 1261 } |
1211 #if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) &
& !_HAS_EXCEPTIONS) | 1262 #if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) &
& !_HAS_EXCEPTIONS) |
1212 if (nh) { | 1263 if (nh) { |
1213 // Since exceptions are disabled, we don't really know if new_handler | 1264 // Since exceptions are disabled, we don't really know if new_handler |
1214 // failed. Assume it will abort if it fails. | 1265 // failed. Assume it will abort if it fails. |
1215 (*nh)(); | 1266 (*nh)(); |
1216 continue; | 1267 continue; |
1217 } | 1268 } |
1218 return 0; | 1269 return 0; |
1219 #else | 1270 #else |
1220 // If no new_handler is established, the allocation failed. | 1271 // If no new_handler is established, the allocation failed. |
1221 if (!nh) { | 1272 if (!nh) |
1222 if (nothrow) return 0; | 1273 return 0; |
1223 throw std::bad_alloc(); | 1274 |
1224 } | |
1225 // Otherwise, try the new_handler. If it returns, retry the | 1275 // Otherwise, try the new_handler. If it returns, retry the |
1226 // allocation. If it throws std::bad_alloc, fail the allocation. | 1276 // allocation. If it throws std::bad_alloc, fail the allocation. |
1227 // if it throws something else, don't interfere. | 1277 // if it throws something else, don't interfere. |
1228 try { | 1278 try { |
1229 (*nh)(); | 1279 (*nh)(); |
1230 } catch (const std::bad_alloc&) { | 1280 } catch (const std::bad_alloc&) { |
1231 if (!nothrow) throw; | |
1232 return p; | 1281 return p; |
1233 } | 1282 } |
1234 #endif // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPT
IONS) && !_HAS_EXCEPTIONS) | 1283 #endif // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPT
IONS) && !_HAS_EXCEPTIONS) |
1235 } else { // allocation success | 1284 } else { // allocation success |
1236 return p; | 1285 return p; |
1237 } | 1286 } |
1238 #endif // PREANSINEW | 1287 #endif // PREANSINEW |
1239 } | 1288 } |
1240 } | 1289 } |
1241 | 1290 |
1242 void* operator new(size_t size) throw (std::bad_alloc) { | 1291 inline void* do_debug_memalign_or_debug_cpp_memalign(size_t align, |
1243 void* ptr = cpp_debug_alloc(size, MallocBlock::kNewType, false); | 1292 size_t size) { |
1244 MallocHook::InvokeNewHook(ptr, size); | 1293 return tc_new_mode ? debug_cpp_memalign(align, size) |
1245 if (ptr == NULL) { | 1294 : do_debug_memalign(align, size); |
1246 RAW_LOG(FATAL, "Unable to allocate %"PRIuS" bytes: new failed.", size); | |
1247 } | |
1248 return ptr; | |
1249 } | 1295 } |
1250 | 1296 |
1251 void* operator new(size_t size, const std::nothrow_t&) __THROW { | 1297 extern "C" PERFTOOLS_DLL_DECL void* tc_memalign(size_t align, size_t size) __THR
OW { |
1252 void* ptr = cpp_debug_alloc(size, MallocBlock::kNewType, true); | 1298 void *p = do_debug_memalign_or_debug_cpp_memalign(align, size); |
1253 MallocHook::InvokeNewHook(ptr, size); | 1299 MallocHook::InvokeNewHook(p, size); |
1254 return ptr; | 1300 return p; |
1255 } | 1301 } |
1256 | 1302 |
1257 void operator delete(void* ptr) __THROW { | 1303 // Implementation taken from tcmalloc/tcmalloc.cc |
1258 MallocHook::InvokeDeleteHook(ptr); | 1304 extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign(void** result_ptr, size_t al
ign, size_t size) |
1259 DebugDeallocate(ptr, MallocBlock::kNewType); | 1305 __THROW { |
1260 } | 1306 if (((align % sizeof(void*)) != 0) || |
1261 | 1307 ((align & (align - 1)) != 0) || |
1262 // Some STL implementations explicitly invoke this. | 1308 (align == 0)) { |
1263 // It is completely equivalent to a normal delete (delete never throws). | 1309 return EINVAL; |
1264 void operator delete(void* ptr, const std::nothrow_t&) __THROW { | |
1265 MallocHook::InvokeDeleteHook(ptr); | |
1266 DebugDeallocate(ptr, MallocBlock::kNewType); | |
1267 } | |
1268 | |
1269 // ========================================================================= // | |
1270 | |
1271 // Alloc/free stuff for debug operator new[] & friends | |
1272 | |
1273 void* operator new[](size_t size) throw (std::bad_alloc) { | |
1274 void* ptr = cpp_debug_alloc(size, MallocBlock::kArrayNewType, false); | |
1275 MallocHook::InvokeNewHook(ptr, size); | |
1276 if (ptr == NULL) { | |
1277 RAW_LOG(FATAL, "Unable to allocate %"PRIuS" bytes: new[] failed.", size); | |
1278 } | |
1279 return ptr; | |
1280 } | |
1281 | |
1282 void* operator new[](size_t size, const std::nothrow_t&) __THROW { | |
1283 void* ptr = cpp_debug_alloc(size, MallocBlock::kArrayNewType, true); | |
1284 MallocHook::InvokeNewHook(ptr, size); | |
1285 return ptr; | |
1286 } | |
1287 | |
1288 void operator delete[](void* ptr) __THROW { | |
1289 MallocHook::InvokeDeleteHook(ptr); | |
1290 DebugDeallocate(ptr, MallocBlock::kArrayNewType); | |
1291 } | |
1292 | |
1293 // Some STL implementations explicitly invoke this. | |
1294 // It is completely equivalent to a normal delete (delete never throws). | |
1295 void operator delete[](void* ptr, const std::nothrow_t&) __THROW { | |
1296 MallocHook::InvokeDeleteHook(ptr); | |
1297 DebugDeallocate(ptr, MallocBlock::kArrayNewType); | |
1298 } | |
1299 | |
1300 // ========================================================================= // | |
1301 | |
1302 // The following functions may be called via MallocExtension::instance() | |
1303 // for memory verification and statistics. | |
1304 #ifdef TCMALLOC_FOR_DEBUGALLOCATION | |
1305 // Inherit from tcmalloc's version | |
1306 typedef TCMallocImplementation ParentImplementation; | |
1307 #else | |
1308 // Inherit from default version | |
1309 typedef MallocExtension ParentImplementation; | |
1310 #endif | |
1311 | |
1312 class DebugMallocImplementation : public ParentImplementation { | |
1313 public: | |
1314 virtual bool GetNumericProperty(const char* name, size_t* value) { | |
1315 bool result = ParentImplementation::GetNumericProperty(name, value); | |
1316 if (result && (strcmp(name, "generic.current_allocated_bytes") == 0)) { | |
1317 // Subtract bytes kept in the free queue | |
1318 size_t qsize = MallocBlock::FreeQueueSize(); | |
1319 if (*value >= qsize) { | |
1320 *value -= qsize; | |
1321 } | |
1322 } | |
1323 return result; | |
1324 } | 1310 } |
1325 | 1311 |
1326 virtual bool VerifyNewMemory(void* p) { | 1312 void* result = do_debug_memalign_or_debug_cpp_memalign(align, size); |
1327 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType); | 1313 MallocHook::InvokeNewHook(result, size); |
1328 return true; | 1314 if (result == NULL) { |
1329 } | 1315 return ENOMEM; |
1330 | 1316 } else { |
1331 virtual bool VerifyArrayNewMemory(void* p) { | 1317 *result_ptr = result; |
1332 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType); | |
1333 return true; | |
1334 } | |
1335 | |
1336 virtual bool VerifyMallocMemory(void* p) { | |
1337 if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType); | |
1338 return true; | |
1339 } | |
1340 | |
1341 virtual bool VerifyAllMemory() { | |
1342 return MallocBlock::CheckEverything(); | |
1343 } | |
1344 | |
1345 virtual bool MallocMemoryStats(int* blocks, size_t* total, | |
1346 int histogram[kMallocHistogramSize]) { | |
1347 return MallocBlock::MemoryStats(blocks, total, histogram); | |
1348 } | |
1349 | |
1350 virtual size_t GetAllocatedSize(void* p) { | |
1351 if (p) { | |
1352 return MallocBlock::FromRawPointer(p)->data_size(); | |
1353 } | |
1354 return 0; | 1318 return 0; |
1355 } | 1319 } |
1356 virtual size_t GetEstimatedAllocatedSize(size_t size) { | |
1357 return size; | |
1358 } | |
1359 }; | |
1360 | |
1361 static DebugMallocImplementation debug_malloc_implementation; | |
1362 | |
1363 REGISTER_MODULE_INITIALIZER(debugallocation, { | |
1364 // Either we or valgrind will control memory management. We | |
1365 // register our extension if we're the winner. | |
1366 if (RunningOnValgrind()) { | |
1367 // Let Valgrind uses its own malloc (so don't register our extension). | |
1368 } else { | |
1369 MallocExtension::Register(&debug_malloc_implementation); | |
1370 // When the program exits, check all blocks still in the free | |
1371 // queue for corruption. | |
1372 atexit(DanglingWriteChecker); | |
1373 } | |
1374 }); | |
1375 | |
1376 #ifdef TCMALLOC_FOR_DEBUGALLOCATION | |
1377 | |
1378 // Redefine malloc_stats to use tcmalloc's implementation: | |
1379 extern "C" void malloc_stats(void) __THROW { | |
1380 do_malloc_stats(); | |
1381 } | |
1382 | |
1383 // Some library routines on RedHat 9 allocate memory using malloc() | |
1384 // and free it using __libc_free() (or vice-versa). Since we provide | |
1385 // our own implementations of malloc/free using tcmalloc.cc, | |
1386 // we need to make sure that the __libc_XXX variants | |
1387 // also point to the same implementations. | |
1388 // | |
1389 // Note: this might not override __libc_XXX calls withing libc itself, | |
1390 // but it can be important for other libraries that mention these functions | |
1391 // or when this code is LD_PRELOAD-ed. | |
1392 // TODO: In case these __libc_* definitions do not actually matter, | |
1393 // they should go away from here and from tcmalloc/tcmalloc.cc. | |
1394 // | |
1395 extern "C" { | |
1396 void* __libc_malloc(size_t size) { return malloc(size); } | |
1397 void __libc_free(void* ptr) { free(ptr); } | |
1398 void* __libc_realloc(void* ptr, size_t size) { return realloc(ptr, size); } | |
1399 void* __libc_calloc(size_t n, size_t size) { return calloc(n, size); } | |
1400 void __libc_cfree(void* ptr) { cfree(ptr); } | |
1401 void* __libc_memalign(size_t align, size_t s) { return memalign(align, s); } | |
1402 void* __libc_valloc(size_t size) { return valloc(size); } | |
1403 void* __libc_pvalloc(size_t size) { return pvalloc(size); } | |
1404 int __posix_memalign(void** r, size_t a, size_t s) { | |
1405 return posix_memalign(r, a, s); | |
1406 } | |
1407 } | 1320 } |
1408 | 1321 |
1409 #endif // #ifdef TCMALLOC_FOR_DEBUGALLOCATION | 1322 extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) __THROW { |
| 1323 // Allocate >= size bytes starting on a page boundary |
| 1324 void *p = do_debug_memalign_or_debug_cpp_memalign(getpagesize(), size); |
| 1325 MallocHook::InvokeNewHook(p, size); |
| 1326 return p; |
| 1327 } |
| 1328 |
| 1329 extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) __THROW { |
| 1330 // Round size up to a multiple of pages |
| 1331 // then allocate memory on a page boundary |
| 1332 int pagesize = getpagesize(); |
| 1333 size = RoundUp(size, pagesize); |
| 1334 if (size == 0) { // pvalloc(0) should allocate one page, according to |
| 1335 size = pagesize; // http://man.free4web.biz/man3/libmpatrol.3.html |
| 1336 } |
| 1337 void *p = do_debug_memalign_or_debug_cpp_memalign(pagesize, size); |
| 1338 MallocHook::InvokeNewHook(p, size); |
| 1339 return p; |
| 1340 } |
| 1341 |
| 1342 // malloc_stats just falls through to the base implementation. |
| 1343 extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) __THROW { |
| 1344 BASE_MALLOC_STATS(); |
| 1345 } |
| 1346 |
| 1347 extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) __THROW { |
| 1348 return BASE_MALLOPT(cmd, value); |
| 1349 } |
| 1350 |
| 1351 #ifdef HAVE_STRUCT_MALLINFO |
| 1352 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW { |
| 1353 return BASE_MALLINFO(); |
| 1354 } |
| 1355 #endif |
| 1356 |
| 1357 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW { |
| 1358 if (!ptr) { |
| 1359 return 0; |
| 1360 } |
| 1361 MallocBlock* mb = MallocBlock::FromRawPointer(ptr); |
| 1362 // This is just to make sure we actually own mb (and ptr). We don't |
| 1363 // use the actual value, just the 'exception' it raises on error. |
| 1364 (void)BASE_MALLOC_SIZE(mb); |
| 1365 return mb->data_size(); |
| 1366 } |
| 1367 |
| 1368 // Override __libc_memalign in libc on linux boxes. |
| 1369 // They have a bug in libc that causes them (very rarely) to allocate |
| 1370 // with __libc_memalign() yet deallocate with free(). |
| 1371 // This function is an exception to the rule of calling MallocHook method |
| 1372 // from the stack frame of the allocation function; |
| 1373 // heap-checker handles this special case explicitly. |
| 1374 static void *MemalignOverride(size_t align, size_t size, const void *caller) |
| 1375 __THROW ATTRIBUTE_SECTION(google_malloc); |
| 1376 |
| 1377 static void *MemalignOverride(size_t align, size_t size, const void *caller) |
| 1378 __THROW { |
| 1379 void *p = do_debug_memalign_or_debug_cpp_memalign(align, size); |
| 1380 MallocHook::InvokeNewHook(p, size); |
| 1381 return p; |
| 1382 } |
| 1383 void *(*__memalign_hook)(size_t, size_t, const void *) = MemalignOverride; |
OLD | NEW |