Merge google-perftools r109 (the current contents of third_party/tcmalloc/vendor)

third_party/tcmalloc/chromium/src/debugallocation.cc

 // Copyright (c) 2000, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 13 matching lines @@
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 // ---
 // Author: Urs Holzle <opensource@google.com>
 
 #include "config.h"
-#ifdef HAVE_MALLOC_H
-#include <malloc.h>
+// We only need malloc.h for struct mallinfo.
+#ifdef HAVE_STRUCT_MALLINFO
+// Malloc can be in several places on older versions of OS X.
+# if defined(HAVE_MALLOC_H)
+# include <malloc.h>
+# elif defined(HAVE_MALLOC_MALLOC_H)
+# include <malloc/malloc.h>
+# elif defined(HAVE_SYS_MALLOC_H)
+# include <sys/malloc.h>
+# endif
 #endif
 #include <pthread.h>
 #include <stdio.h>
 #ifdef HAVE_INTTYPES_H
 #include <inttypes.h>
 #endif
 #include <stdarg.h>
 #ifdef HAVE_MMAP
 #include <sys/mman.h>
 #endif
 #include <sys/types.h>
 #include <sys/stat.h>
 #ifdef HAVE_FCNTL_H
 #include <fcntl.h>
 #endif
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
 #include <errno.h>
 #include <string.h>
 
+#include <google/malloc_extension.h>
+#include <google/malloc_hook.h>
+#include <google/stacktrace.h>
 #include "base/commandlineflags.h"
 #include "base/googleinit.h"
 #include "base/logging.h"
-#include "google/malloc_extension.h"
-#include "google/malloc_hook.h"
-#include "google/stacktrace.h"
+#include "base/spinlock.h"
 #include "addressmap-inl.h"
 #include "malloc_hook-inl.h"
 #include "symbolize.h"
 
-#ifdef TCMALLOC_FOR_DEBUGALLOCATION
+#define TCMALLOC_USING_DEBUGALLOCATION
 #include "tcmalloc.cc"
-#else
-#include "base/spinlock.h"
-// Else we already have a SpinLock defined in tcmalloc/internal_spinlock.h
-#endif
 
 // __THROW is defined in glibc systems.  It means, counter-intuitively,
 // "This function will never throw an exception."  It's an optional
 // optimization tool, but we may need to use it to match glibc prototypes.
 #ifndef __THROW    // I guess we're not on a glibc system
 # define __THROW   // __THROW is just an optimization, so ok to make it ""
 #endif
 
 // On systems (like freebsd) that don't define MAP_ANONYMOUS, use the old
 // form of the name instead.
@@ skipping 36 matching lines @@
             EnvToBool("TCMALLOC_SYMBOLIZE_STACKTRACE", true),
             "Symbolize the stack trace when provided (on some error exits)");
 
 // ========================================================================= //
 
 // A safe version of printf() that does not do any allocation and
 // uses very little stack space.
 static void TracePrintf(int fd, const char *fmt, ...)
   __attribute__ ((__format__ (__printf__, 2, 3)));
 
-//
-// GNU has some weird "weak aliasing" thing that permits us to define our
-// own malloc(), free(), and realloc() which can use the normal versions of
-// of themselves by calling __libc_malloc(), __libc_free(), and
-// __libc_realloc().
-//
-extern "C" {
-  extern void* __libc_malloc(size_t size);
-  extern void __libc_free(void* ptr);
-  extern void* __libc_realloc(void* ptr, size_t size);
-  extern void* __libc_calloc(size_t nmemb, size_t size);
-  extern int __libc_mallopt(int cmd, int value);
-#ifdef HAVE_STRUCT_MALLINFO
-  extern struct mallinfo __libc_mallinfo(void);
-#endif
-}
-
-// Define the malloc/free/mallopt/mallinfo implementations
-// we will be working on top of.
-// TODO(csilvers): provide debugallocation on top of libc alloc,
-//                 so this #ifdef might sometimes be false.
-#ifdef TCMALLOC_FOR_DEBUGALLOCATION
-
-// The do_* functions are defined in tcmalloc.cc,
+// The do_* functions are defined in tcmalloc/tcmalloc.cc,
 // which is included before this file
-// when TCMALLOC_FOR_DEBUGALLOCATION is defined.
-#define BASE_MALLOC_NEW(size)  cpp_alloc(size, false)
-#define BASE_MALLOC      do_malloc_or_cpp_alloc
-#define BASE_FREE        do_free
-#define BASE_MALLOPT     do_mallopt
-#define BASE_MALLINFO    do_mallinfo
-
-#else
-
-// We are working on top of standard libc's malloc library
-#define BASE_MALLOC_NEW  __libc_malloc
-#define BASE_MALLOC      __libc_malloc
-#define BASE_FREE        __libc_free
-#define BASE_MALLOPT     __libc_mallopt
-#define BASE_MALLINFO    __libc_mallinfo
-
-#endif
+// when TCMALLOC_FOR_DEBUGALLOCATION is defined
+#define BASE_MALLOC_NEW(size)    cpp_alloc(size, false)
+#define BASE_MALLOC        do_malloc
+#define BASE_FREE          do_free
+#define BASE_MALLOC_STATS  do_malloc_stats
+#define BASE_MALLOPT       do_mallopt
+#define BASE_MALLINFO      do_mallinfo
+#define BASE_MALLOC_SIZE(ptr) GetSizeWithCallback(ptr, &InvalidGetAllocatedSize)
 
 // ========================================================================= //
 
 class MallocBlock;
 
 // A circular buffer to hold freed blocks of memory.  MallocBlock::Deallocate
 // (below) pushes blocks into this queue instead of returning them to the
 // underlying allocator immediately.  See MallocBlock::Deallocate for more
 // information.
 //
 // We can't use an STL class for this because we need to be careful not to
 // perform any heap de-allocations in any of the code in this class, since the
 // code in MallocBlock::Deallocate is not re-entrant.
 template <typename QueueEntry>
 class FreeQueue {
  public:
   FreeQueue() : q_front_(0), q_back_(0) {}
 
   bool Full() {
     return (q_front_ + 1) % kFreeQueueSize == q_back_;
   }
 
-  void Push(QueueEntry block) {
+  void Push(const QueueEntry& block) {
     q_[q_front_] = block;
     q_front_ = (q_front_ + 1) % kFreeQueueSize;
   }
 
   QueueEntry Pop() {
     const QueueEntry& ret = q_[q_back_];
     q_back_ = (q_back_ + 1) % kFreeQueueSize;
     return ret;
   }
 
@@ skipping 62 matching lines @@
   static const int kDeallocatedTypeBit = 0x4;
 
   // For better memory debugging, we initialize all storage to known
   // values, and overwrite the storage when it's deallocated:
   // Byte that fills uninitialized storage.
   static const int kMagicUninitializedByte = 0xAB;
   // Byte that fills deallocated storage.
   // NOTE: tcmalloc.cc depends on the value of kMagicDeletedByte
   //       to work around a bug in the pthread library.
   static const int kMagicDeletedByte = 0xCD;
-  // An int (type of alloc_type_ below) in a deallocated storage
+  // A size_t (type of alloc_type_ below) in a deallocated storage
   // filled with kMagicDeletedByte.
-  static const int kMagicDeletedInt = 0xCDCDCDCD | ((0xCDCDCDCD << 16) << 16);
-    // Initializer works for 32 and 64 bit ints;
+  static const size_t kMagicDeletedSizeT =
+      0xCDCDCDCD | (((size_t)0xCDCDCDCD << 16) << 16);
+    // Initializer works for 32 and 64 bit size_ts;
     // "<< 16 << 16" is to fool gcc from issuing a warning
-    // when ints are 32 bits.
+    // when size_ts are 32 bits.
 
   // NOTE: on Linux, you can enable malloc debugging support in libc by
   // setting the environment variable MALLOC_CHECK_ to 1 before you
   // start the program (see man malloc).
 
   // We use either BASE_MALLOC or mmap to make the actual allocation. In
   // order to remember which one of the two was used for any block, we store an
   // appropriate magic word next to the block.
   static const int kMagicMalloc = 0xDEADBEEF;
   static const int kMagicMMap = 0xABCDEFAB;
 
   // This array will be filled with 0xCD, for use with memcmp.
   static unsigned char kMagicDeletedBuffer[1024];
   static bool deleted_buffer_initialized_;
 
  private:  // data layout
 
                     // The four fields size1_,offset_,magic1_,alloc_type_
-                    // should together occupy a multiple of 8 bytes.
+                    // should together occupy a multiple of 16 bytes. (At the
+                    // moment, sizeof(size_t) == 4 or 8 depending on piii vs
+                    // k8, and 4 of those sum to 16 or 32 bytes).
+                    // This, combined with BASE_MALLOC's alignment guarantees,
+                    // ensures that SSE types can be stored into the returned
+                    // block, at &size2_.
   size_t size1_;
   size_t offset_;   // normally 0 unless memaligned memory
                     // see comments in memalign() and FromRawPointer().
-  int magic1_;
-  int alloc_type_;
+  size_t magic1_;
+  size_t alloc_type_;
   // here comes the actual data (variable length)
   // ...
   // then come the size2_ and magic2_, or a full page of mprotect-ed memory
   // if the malloc_page_fence feature is enabled.
   size_t size2_;
   int magic2_;
 
  private:  // static data and helpers
 
   // Allocation map: stores the allocation type for each allocated object,
@@ skipping 112 matching lines @@
       RAW_LOG(FATAL, "memory allocation bug: object at %p "
                      "has never been allocated", data_addr());
     } else {
       map_type = *found_type;
     }
     if ((map_type & kDeallocatedTypeBit) != 0) {
       RAW_LOG(FATAL, "memory allocation bug: object at %p "
                      "has been already deallocated (it was allocated with %s)",
                      data_addr(), AllocName(map_type & ~kDeallocatedTypeBit));
     }
-    if (alloc_type_ == kMagicDeletedInt) {
+    if (alloc_type_ == kMagicDeletedSizeT) {
       RAW_LOG(FATAL, "memory stomping bug: a word before object at %p "
                      "has been corrupted; or else the object has been already "
                      "deallocated and our memory map has been corrupted",
                      data_addr());
     }
     if (!IsValidMagicValue(magic1_)) {
       RAW_LOG(FATAL, "memory stomping bug: a word before object at %p "
                      "has been corrupted; "
                      "or else our memory map has been corrupted and this is a "
                      "deallocation for not (currently) heap-allocated object",
@@ skipping 41 matching lines @@
 
  public:  // our main interface
 
   static MallocBlock* Allocate(size_t size, int type) {
     // Prevent an integer overflow / crash with large allocation sizes.
     // TODO - Note that for a e.g. 64-bit size_t, max_size_t may not actually
     // be the maximum value, depending on how the compiler treats ~0. The worst
     // practical effect is that allocations are limited to 4Gb or so, even if
     // the address space could take more.
     static size_t max_size_t = ~0;
-    if (size < 0 || size > max_size_t - sizeof(MallocBlock)) {
+    if (size > max_size_t - sizeof(MallocBlock)) {
       RAW_LOG(ERROR, "Massive size passed to malloc: %"PRIuS"", size);
       return NULL;
     }
     MallocBlock* b = NULL;
     const bool use_malloc_page_fence = FLAGS_malloc_page_fence;
 #ifdef HAVE_MMAP
     if (use_malloc_page_fence) {
       // Put the block towards the end of the page and make the next page
       // inaccessible. This will catch buffer overrun right when it happens.
       size_t sz = real_mmapped_size(size);
@@ skipping 144 matching lines @@
       RAW_LOG(ERROR, "Buffer too large to print corruption.");
     }
 
     const MallocBlock* b = queue_entry.block;
     const size_t size = queue_entry.size;
     if (queue_entry.num_deleter_pcs > 0) {
       TracePrintf(STDERR_FILENO, "Deleted by thread %p\n",
                   reinterpret_cast<void*>(
                       PRINTABLE_PTHREAD(queue_entry.deleter_threadid)));
 
-      SymbolTable symbolization_table;
-      const int num_symbols = queue_entry.num_deleter_pcs;  // short alias name
-      for (int i = 0; i < num_symbols; i++) {
+      // We don't want to allocate or deallocate memory here, so we use
+      // placement-new.  It's ok that we don't destroy this, since we're
+      // just going to error-exit below anyway.  Union is for alignment.
+      union { void* alignment; char buf[sizeof(SymbolTable)]; } tablebuf;
+      SymbolTable* symbolization_table = new (tablebuf.buf) SymbolTable;
+      for (int i = 0; i < queue_entry.num_deleter_pcs; i++) {
         // Symbolizes the previous address of pc because pc may be in the
         // next function.  This may happen when the function ends with
         // a call to a function annotated noreturn (e.g. CHECK).
-        char* pc =
-            reinterpret_cast<char*>(queue_entry.deleter_pcs[i]) - 1;
-        symbolization_table.Add(pc);
+        char *pc = reinterpret_cast<char*>(queue_entry.deleter_pcs[i]);
+        symbolization_table->Add(pc - 1);
       }
       if (FLAGS_symbolize_stacktrace)
-        symbolization_table.Symbolize();
-      for (int i = 0; i < num_symbols; i++) {
-        char *pc =
-            reinterpret_cast<char*>(queue_entry.deleter_pcs[i]) - 1;
-        TracePrintf(STDERR_FILENO, "    @ %"PRIxPTR" %s\n",
-                    reinterpret_cast<uintptr_t>(pc),
-                    symbolization_table.GetSymbol(pc));
+        symbolization_table->Symbolize();
+      for (int i = 0; i < queue_entry.num_deleter_pcs; i++) {
+        char *pc = reinterpret_cast<char*>(queue_entry.deleter_pcs[i]);
+        TracePrintf(STDERR_FILENO, "    @ %p %s\n",
+                    pc, symbolization_table->GetSymbol(pc - 1));
       }
     } else {
       RAW_LOG(ERROR,
               "Skipping the printing of the deleter's stack!  Its stack was "
               "not found; either the corruption occurred too early in "
               "execution to obtain a stack trace or --max_free_queue_size was "
               "set to 0.");
     }
 
     RAW_LOG(FATAL,
             "Memory was written to after being freed.  MallocBlock: %p, user "
             "ptr: %p, size: %zd.  If you can't find the source of the error, "
             "try using valgrind or purify, or study the output of the "
             "deleter's stack printed above.", b, b->data_addr(), size);
   }
 
   static MallocBlock* FromRawPointer(void* p) {
     const size_t data_offset = MallocBlock::data_offset();
     // Find the header just before client's memory.
     MallocBlock *mb = reinterpret_cast<MallocBlock *>(
                 reinterpret_cast<char *>(p) - data_offset);
-    // If mb->alloc_type_ is kMagicDeletedInt, we're not an ok pointer.
-    if (mb->alloc_type_ == kMagicDeletedInt) {
+    // If mb->alloc_type_ is kMagicDeletedSizeT, we're not an ok pointer.
+    if (mb->alloc_type_ == kMagicDeletedSizeT) {
       RAW_LOG(FATAL, "memory allocation bug: object at %p has been already"
                      " deallocated; or else a word before the object has been"
                      " corrupted (memory stomping bug)", p);
     }
     // If mb->offset_ is zero (common case), mb is the real header.  If
     // mb->offset_ is non-zero, this block was allocated by memalign, and
     // mb->offset_ is the distance backwards to the real header from mb,
     // which is a fake header.  The following subtraction works for both zero
     // and non-zero values.
     return reinterpret_cast<MallocBlock *>(
@@ skipping 253 matching lines @@
 
 static inline void DebugDeallocate(void* ptr, int type) {
   MALLOC_TRACE("free",
                (ptr != 0 ? MallocBlock::FromRawPointer(ptr)->data_size() : 0),
                ptr);
   if (ptr)  MallocBlock::FromRawPointer(ptr)->Deallocate(type);
 }
 
 // ========================================================================= //
 
-// Alloc/free stuff for debug hooks for malloc & friends
+// The following functions may be called via MallocExtension::instance()
+// for memory verification and statistics.
+class DebugMallocImplementation : public TCMallocImplementation {
+ public:
+  virtual bool GetNumericProperty(const char* name, size_t* value) {
+    bool result = TCMallocImplementation::GetNumericProperty(name, value);
+    if (result && (strcmp(name, "generic.current_allocated_bytes") == 0)) {
+      // Subtract bytes kept in the free queue
+      size_t qsize = MallocBlock::FreeQueueSize();
+      if (*value >= qsize) {
+        *value -= qsize;
+      }
+    }
+    return result;
+  }
 
-// CAVEAT: The code structure below ensures that MallocHook methods are always
-//         called from the stack frame of the invoked allocation function.
-//         heap-checker.cc depends on this to start a stack trace from
-//         the call to the (de)allocation function.
+  virtual bool VerifyNewMemory(void* p) {
+    if (p)  MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType);
+    return true;
+  }
 
-// Put all callers of MallocHook::Invoke* in this module into
-// ATTRIBUTE_SECTION(google_malloc) section,
-// so that MallocHook::GetCallerStackTrace can function accurately:
+  virtual bool VerifyArrayNewMemory(void* p) {
+    if (p)  MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType);
+    return true;
+  }
 
-extern "C" {
-  void* malloc(size_t size) __THROW ATTRIBUTE_SECTION(google_malloc);
-  void free(void* ptr) __THROW ATTRIBUTE_SECTION(google_malloc);
-  void* realloc(void* ptr, size_t size) __THROW
-    ATTRIBUTE_SECTION(google_malloc);
-  void* calloc(size_t nmemb, size_t size) __THROW
-    ATTRIBUTE_SECTION(google_malloc);
-  void cfree(void* ptr) __THROW ATTRIBUTE_SECTION(google_malloc);
+  virtual bool VerifyMallocMemory(void* p) {
+    if (p)  MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType);
+    return true;
+  }
 
-  void* memalign(size_t __alignment, size_t __size) __THROW
-    ATTRIBUTE_SECTION(google_malloc);
-  int posix_memalign(void** ptr, size_t align, size_t size) __THROW
-    ATTRIBUTE_SECTION(google_malloc);
-  void* valloc(size_t __size) __THROW
-    ATTRIBUTE_SECTION(google_malloc);
-  void* pvalloc(size_t __size) __THROW
-    ATTRIBUTE_SECTION(google_malloc);
+  virtual bool VerifyAllMemory() {
+    return MallocBlock::CheckEverything();
+  }
+
+  virtual bool MallocMemoryStats(int* blocks, size_t* total,
+                                 int histogram[kMallocHistogramSize]) {
+    return MallocBlock::MemoryStats(blocks, total, histogram);
+  }
+
+  virtual size_t GetAllocatedSize(void* p) {
+    if (p) {
+      return MallocBlock::FromRawPointer(p)->data_size();
+    }
+    return 0;
+  }
+  virtual size_t GetEstimatedAllocatedSize(size_t size) {
+    return size;
+  }
+
+  virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) {
+    static const char* kDebugFreeQueue = "debug.free_queue";
+
+    TCMallocImplementation::GetFreeListSizes(v);
+
+    MallocExtension::FreeListInfo i;
+    i.type = kDebugFreeQueue;
+    i.min_object_size = 0;
+    i.max_object_size = numeric_limits<size_t>::max();
+    i.total_bytes_free = MallocBlock::FreeQueueSize();
+    v->push_back(i);
+  }
+
+ };
+
+static DebugMallocImplementation debug_malloc_implementation;
+
+REGISTER_MODULE_INITIALIZER(debugallocation, {
+  // Either we or valgrind will control memory management.  We
+  // register our extension if we're the winner.
+  if (RunningOnValgrind()) {
+    // Let Valgrind uses its own malloc (so don't register our extension).
+  } else {
+    MallocExtension::Register(&debug_malloc_implementation);
+    // When the program exits, check all blocks still in the free
+    // queue for corruption.
+    atexit(DanglingWriteChecker);
+  }
+});
+
+// ========================================================================= //
+
+// This is mostly the same a cpp_alloc in tcmalloc.cc.
+// TODO(csilvers): write a wrapper for new-handler so we don't have to
+// copy this code so much.
+inline void* debug_cpp_alloc(size_t size, int new_type, bool nothrow) {
+  for (;;) {
+    void* p = DebugAllocate(size, new_type);
+#ifdef PREANSINEW
+    return p;
+#else
+    if (p == NULL) {  // allocation failed
+      // Get the current new handler.  NB: this function is not
+      // thread-safe.  We make a feeble stab at making it so here, but
+      // this lock only protects against tcmalloc interfering with
+      // itself, not with other libraries calling set_new_handler.
+      std::new_handler nh;
+      {
+        SpinLockHolder h(&set_new_handler_lock);
+        nh = std::set_new_handler(0);
+        (void) std::set_new_handler(nh);
+      }
+#if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS)
+      if (nh) {
+        // Since exceptions are disabled, we don't really know if new_handler
+        // failed.  Assume it will abort if it fails.
+        (*nh)();
+        continue;
+      }
+      return 0;
+#else
+      // If no new_handler is established, the allocation failed.
+      if (!nh) {
+        if (nothrow) return 0;
+        throw std::bad_alloc();
+      }
+      // Otherwise, try the new_handler.  If it returns, retry the
+      // allocation.  If it throws std::bad_alloc, fail the allocation.
+      // if it throws something else, don't interfere.
+      try {
+        (*nh)();
+      } catch (const std::bad_alloc&) {
+        if (!nothrow) throw;
+        return p;
+      }
+#endif  // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS)
+    } else {  // allocation success
+      return p;
+    }
+#endif  // PREANSINEW
+  }
 }
 
-static void *MemalignOverride(size_t align, size_t size,
-                              const void *caller) __THROW
-  ATTRIBUTE_SECTION(google_malloc);
+inline void* do_debug_malloc_or_debug_cpp_alloc(size_t size) {
+  return tc_new_mode ? debug_cpp_alloc(size, MallocBlock::kMallocType, true)
+                     : DebugAllocate(size, MallocBlock::kMallocType);
+}
 
-void* operator new(size_t size) throw (std::bad_alloc)
-  ATTRIBUTE_SECTION(google_malloc);
-void* operator new(size_t size, const std::nothrow_t&) __THROW
-  ATTRIBUTE_SECTION(google_malloc);
-void operator delete(void* p) __THROW
-  ATTRIBUTE_SECTION(google_malloc);
-void operator delete(void* p, const std::nothrow_t&) __THROW
-  ATTRIBUTE_SECTION(google_malloc);
-void* operator new[](size_t size) throw (std::bad_alloc)
-  ATTRIBUTE_SECTION(google_malloc);
-void* operator new[](size_t size, const std::nothrow_t&) __THROW
-  ATTRIBUTE_SECTION(google_malloc);
-void operator delete[](void* p) __THROW
-  ATTRIBUTE_SECTION(google_malloc);
-void operator delete[](void* p, const std::nothrow_t&) __THROW
-  ATTRIBUTE_SECTION(google_malloc);
+// Exported routines
 
-extern "C" void* malloc(size_t size) __THROW {
-  void* ptr = DebugAllocate(size, MallocBlock::kMallocType);
+extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW {
+  void* ptr = do_debug_malloc_or_debug_cpp_alloc(size);
   MallocHook::InvokeNewHook(ptr, size);
   return ptr;
 }
 
-extern "C" void free(void* ptr) __THROW {
+extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW {
   MallocHook::InvokeDeleteHook(ptr);
   DebugDeallocate(ptr, MallocBlock::kMallocType);
 }
 
-extern "C" void* realloc(void* ptr, size_t size) __THROW {
+extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t count, size_t size) __THROW {
+  // Overflow check
+  const size_t total_size = count * size;
+  if (size != 0 && total_size / size != count) return NULL;
+
+  void* block = do_debug_malloc_or_debug_cpp_alloc(total_size);
+  MallocHook::InvokeNewHook(block, total_size);
+  if (block)  memset(block, 0, total_size);
+  return block;
+}
+
+extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW {
+  MallocHook::InvokeDeleteHook(ptr);
+  DebugDeallocate(ptr, MallocBlock::kMallocType);
+}
+
+extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW {
   if (ptr == NULL) {
-    ptr = DebugAllocate(size, MallocBlock::kMallocType);
+    ptr = do_debug_malloc_or_debug_cpp_alloc(size);
     MallocHook::InvokeNewHook(ptr, size);
     return ptr;
   }
   if (size == 0) {
     MallocHook::InvokeDeleteHook(ptr);
     DebugDeallocate(ptr, MallocBlock::kMallocType);
     return NULL;
   }
   MallocBlock* old = MallocBlock::FromRawPointer(ptr);
   old->Check(MallocBlock::kMallocType);
   MallocBlock* p = MallocBlock::Allocate(size, MallocBlock::kMallocType);
 
   // If realloc fails we are to leave the old block untouched and
   // return null
   if (p == NULL)  return NULL;
 
   memcpy(p->data_addr(), old->data_addr(),
          (old->data_size() < size) ? old->data_size() : size);
   MallocHook::InvokeDeleteHook(ptr);
   MallocHook::InvokeNewHook(p->data_addr(), size);
   DebugDeallocate(ptr, MallocBlock::kMallocType);
   MALLOC_TRACE("realloc", p->data_size(), p->data_addr());
   return p->data_addr();
 }
 
-extern "C" void* calloc(size_t count, size_t size) __THROW {
-  // Overflow check
-  const size_t total_size = count * size;
-  if (size != 0 && total_size / size != count) return NULL;
-
-  void* block = DebugAllocate(total_size, MallocBlock::kMallocType);
-  MallocHook::InvokeNewHook(block, total_size);
-  if (block)  memset(block, 0, total_size);
-  return block;
+extern "C" PERFTOOLS_DLL_DECL void* tc_new(size_t size) {
+  void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, false);
+  MallocHook::InvokeNewHook(ptr, size);
+  if (ptr == NULL) {
+    RAW_LOG(FATAL, "Unable to allocate %"PRIuS" bytes: new failed.", size);
+  }
+  return ptr;
 }
 
-extern "C" void cfree(void* ptr) __THROW {
-  MallocHook::InvokeDeleteHook(ptr);
-  DebugDeallocate(ptr, MallocBlock::kMallocType);
+extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, const std::nothrow_t&) __THROW {
+  void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, true);
+  MallocHook::InvokeNewHook(ptr, size);
+  return ptr;
+}
+
+extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) __THROW {
+  MallocHook::InvokeDeleteHook(p);
+  DebugDeallocate(p, MallocBlock::kNewType);
+}
+
+// Some STL implementations explicitly invoke this.
+// It is completely equivalent to a normal delete (delete never throws).
+extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) __THROW {
+  MallocHook::InvokeDeleteHook(p);
+  DebugDeallocate(p, MallocBlock::kNewType);
+}
+
+extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) {
+  void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, false);
+  MallocHook::InvokeNewHook(ptr, size);
+  if (ptr == NULL) {
+    RAW_LOG(FATAL, "Unable to allocate %"PRIuS" bytes: new[] failed.", size);
+  }
+  return ptr;
+}
+
+extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::nothrow_t&)
+    __THROW {
+  void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, true);
+  MallocHook::InvokeNewHook(ptr, size);
+  return ptr;
+}
+
+extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) __THROW {
+  MallocHook::InvokeDeleteHook(p);
+  DebugDeallocate(p, MallocBlock::kArrayNewType);
+}
+
+// Some STL implementations explicitly invoke this.
+// It is completely equivalent to a normal delete (delete never throws).
+extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) __THROW {
+  MallocHook::InvokeDeleteHook(p);
+  DebugDeallocate(p, MallocBlock::kArrayNewType);
 }
 
 // Round "value" up to next "alignment" boundary.
 // Requires that "alignment" be a power of two.
 static intptr_t RoundUp(intptr_t value, intptr_t alignment) {
   return (value + alignment - 1) & ~(alignment - 1);
 }
 
+// This is mostly the same as do_memalign in tcmalloc.cc.
 static void *do_debug_memalign(size_t alignment, size_t size) {
   // Allocate >= size bytes aligned on "alignment" boundary
   // "alignment" is a power of two.
   void *p = 0;
   RAW_CHECK((alignment & (alignment-1)) == 0, "must be power of two");
   const size_t data_offset = MallocBlock::data_offset();
   // Allocate "alignment-1" extra bytes to ensure alignment is possible, and
   // a further data_offset bytes for an additional fake header.
   size_t extra_bytes = data_offset + alignment - 1;
   if (size + extra_bytes < size) return NULL;         // Overflow
   p = DebugAllocate(size + extra_bytes, MallocBlock::kMallocType);
   if (p != 0) {
     intptr_t orig_p = reinterpret_cast<intptr_t>(p);
     // Leave data_offset bytes for fake header, and round up to meet
     // alignment.
     p = reinterpret_cast<void *>(RoundUp(orig_p + data_offset, alignment));
     // Create a fake header block with an offset_ that points back to the
     // real header.  FromRawPointer uses this value.
     MallocBlock *fake_hdr = reinterpret_cast<MallocBlock *>(
                 reinterpret_cast<char *>(p) - data_offset);
     // offset_ is distance between real and fake headers.
     // p is now end of fake header (beginning of client area),
     // and orig_p is the end of the real header, so offset_
     // is their difference.
     fake_hdr->set_offset(reinterpret_cast<intptr_t>(p) - orig_p);
   }
   return p;
 }
 
-// Override __libc_memalign in libc on linux boxes.
-// They have a bug in libc that causes them (very rarely) to allocate
-// with __libc_memalign() yet deallocate with free().
-// This function is an exception to the rule of calling MallocHook method
-// from the stack frame of the allocation function;
-// heap-checker handles this special case explicitly.
-static void *MemalignOverride(size_t align, size_t size,
-                              const void *caller) __THROW {
-  void *p = do_debug_memalign(align, size);
-  MallocHook::InvokeNewHook(p, size);
-  return p;
-}
-void *(*__memalign_hook)(size_t, size_t, const void *) = MemalignOverride;
-
-extern "C" void* memalign(size_t align, size_t size) __THROW {
-  void *p = do_debug_memalign(align, size);
-  MallocHook::InvokeNewHook(p, size);
-  return p;
-}
-
-// Implementation taken from tcmalloc/tcmalloc.cc
-extern "C" int posix_memalign(void** result_ptr,
-                              size_t align, size_t size) __THROW {
-  if (((align % sizeof(void*)) != 0) ||
-      ((align & (align - 1)) != 0) ||
-      (align == 0)) {
-    return EINVAL;
-  }
-
-  void* result = do_debug_memalign(align, size);
-  MallocHook::InvokeNewHook(result, size);
-  if (result == NULL) {
-    return ENOMEM;
-  } else {
-    *result_ptr = result;
-    return 0;
-  }
-}
-
-extern "C" void* valloc(size_t size) __THROW {
-  // Allocate >= size bytes starting on a page boundary
-  void *p = do_debug_memalign(getpagesize(), size);
-  MallocHook::InvokeNewHook(p, size);
-  return p;
-}
-
-extern "C" void* pvalloc(size_t size) __THROW {
-  // Round size up to a multiple of pages
-  // then allocate memory on a page boundary
-  int pagesize = getpagesize();
-  size = RoundUp(size, pagesize);
-  if (size == 0) {     // pvalloc(0) should allocate one page, according to
-    size = pagesize;   // http://man.free4web.biz/man3/libmpatrol.3.html
-  }
-  void *p = do_debug_memalign(pagesize, size);
-  MallocHook::InvokeNewHook(p, size);
-  return p;
-}
-
-extern "C" int mallopt(int cmd, int value) __THROW {
-  return BASE_MALLOPT(cmd, value);
-}
-
-#ifdef HAVE_STRUCT_MALLINFO
-extern "C" struct mallinfo mallinfo(void) __THROW {
-  return BASE_MALLINFO();
-}
-#endif
-
-// ========================================================================= //
-
-// Alloc/free stuff for debug operator new & friends
-
-// This is mostly the same a cpp_alloc in tcmalloc.cc.
-inline void* cpp_debug_alloc(size_t size, int new_type, bool nothrow) {
+// This is mostly the same as cpp_memalign in tcmalloc.cc.
+static void* debug_cpp_memalign(size_t align, size_t size) {
   for (;;) {
-    void* p = DebugAllocate(size, new_type);
+    void* p = do_debug_memalign(align, size);
 #ifdef PREANSINEW
     return p;
 #else
     if (p == NULL) {  // allocation failed
       // Get the current new handler.  NB: this function is not
       // thread-safe.  We make a feeble stab at making it so here, but
       // this lock only protects against tcmalloc interfering with
       // itself, not with other libraries calling set_new_handler.
       std::new_handler nh;
       {
         SpinLockHolder h(&set_new_handler_lock);
         nh = std::set_new_handler(0);
         (void) std::set_new_handler(nh);
       }
 #if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS)
       if (nh) {
         // Since exceptions are disabled, we don't really know if new_handler
         // failed.  Assume it will abort if it fails.
         (*nh)();
         continue;
       }
       return 0;
 #else
       // If no new_handler is established, the allocation failed.
-      if (!nh) {
-        if (nothrow) return 0;
-        throw std::bad_alloc();
-      }
+      if (!nh)
+        return 0;
+
       // Otherwise, try the new_handler.  If it returns, retry the
       // allocation.  If it throws std::bad_alloc, fail the allocation.
       // if it throws something else, don't interfere.
       try {
         (*nh)();
       } catch (const std::bad_alloc&) {
-        if (!nothrow) throw;
         return p;
       }
 #endif  // (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS)
     } else {  // allocation success
       return p;
     }
 #endif  // PREANSINEW
   }
 }
 
-void* operator new(size_t size) throw (std::bad_alloc) {
-  void* ptr = cpp_debug_alloc(size, MallocBlock::kNewType, false);
-  MallocHook::InvokeNewHook(ptr, size);
-  if (ptr == NULL) {
-    RAW_LOG(FATAL, "Unable to allocate %"PRIuS" bytes: new failed.", size);
-  }
-  return ptr;
+inline void* do_debug_memalign_or_debug_cpp_memalign(size_t align,
+                                                     size_t size) {
+  return tc_new_mode ? debug_cpp_memalign(align, size)
+                     : do_debug_memalign(align, size);
 }
 
-void* operator new(size_t size, const std::nothrow_t&) __THROW {
-  void* ptr = cpp_debug_alloc(size, MallocBlock::kNewType, true);
-  MallocHook::InvokeNewHook(ptr, size);
-  return ptr;
+extern "C" PERFTOOLS_DLL_DECL void* tc_memalign(size_t align, size_t size) __THROW {
+  void *p = do_debug_memalign_or_debug_cpp_memalign(align, size);
+  MallocHook::InvokeNewHook(p, size);
+  return p;
 }
 
-void operator delete(void* ptr) __THROW {
-  MallocHook::InvokeDeleteHook(ptr);
-  DebugDeallocate(ptr, MallocBlock::kNewType);
-}
-
-// Some STL implementations explicitly invoke this.
-// It is completely equivalent to a normal delete (delete never throws).
-void operator delete(void* ptr, const std::nothrow_t&) __THROW {
-  MallocHook::InvokeDeleteHook(ptr);
-  DebugDeallocate(ptr, MallocBlock::kNewType);
-}
-
-// ========================================================================= //
-
-// Alloc/free stuff for debug operator new[] & friends
-
-void* operator new[](size_t size) throw (std::bad_alloc) {
-  void* ptr = cpp_debug_alloc(size, MallocBlock::kArrayNewType, false);
-  MallocHook::InvokeNewHook(ptr, size);
-  if (ptr == NULL) {
-    RAW_LOG(FATAL, "Unable to allocate %"PRIuS" bytes: new[] failed.", size);
-  }
-  return ptr;
-}
-
-void* operator new[](size_t size, const std::nothrow_t&) __THROW {
-  void* ptr = cpp_debug_alloc(size, MallocBlock::kArrayNewType, true);
-  MallocHook::InvokeNewHook(ptr, size);
-  return ptr;
-}
-
-void operator delete[](void* ptr) __THROW {
-  MallocHook::InvokeDeleteHook(ptr);
-  DebugDeallocate(ptr, MallocBlock::kArrayNewType);
-}
-
-// Some STL implementations explicitly invoke this.
-// It is completely equivalent to a normal delete (delete never throws).
-void operator delete[](void* ptr, const std::nothrow_t&) __THROW {
-  MallocHook::InvokeDeleteHook(ptr);
-  DebugDeallocate(ptr, MallocBlock::kArrayNewType);
-}
-
-// ========================================================================= //
-
-// The following functions may be called via MallocExtension::instance()
-// for memory verification and statistics.
-#ifdef TCMALLOC_FOR_DEBUGALLOCATION
-// Inherit from tcmalloc's version
-typedef TCMallocImplementation ParentImplementation;
-#else
-// Inherit from default version
-typedef MallocExtension ParentImplementation;
-#endif
-
-class DebugMallocImplementation : public ParentImplementation {
- public:
-  virtual bool GetNumericProperty(const char* name, size_t* value) {
-    bool result = ParentImplementation::GetNumericProperty(name, value);
-    if (result && (strcmp(name, "generic.current_allocated_bytes") == 0)) {
-      // Subtract bytes kept in the free queue
-      size_t qsize = MallocBlock::FreeQueueSize();
-      if (*value >= qsize) {
-        *value -= qsize;
-      }
-    }
-    return result;
+// Implementation taken from tcmalloc/tcmalloc.cc
+extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign(void** result_ptr, size_t align, size_t size)
+    __THROW {
+  if (((align % sizeof(void*)) != 0) ||
+      ((align & (align - 1)) != 0) ||
+      (align == 0)) {
+    return EINVAL;
   }
 
-  virtual bool VerifyNewMemory(void* p) {
-    if (p)  MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType);
-    return true;
-  }
-
-  virtual bool VerifyArrayNewMemory(void* p) {
-    if (p)  MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType);
-    return true;
-  }
-
-  virtual bool VerifyMallocMemory(void* p) {
-    if (p)  MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType);
-    return true;
-  }
-
-  virtual bool VerifyAllMemory() {
-    return MallocBlock::CheckEverything();
-  }
-
-  virtual bool MallocMemoryStats(int* blocks, size_t* total,
-                                 int histogram[kMallocHistogramSize]) {
-    return MallocBlock::MemoryStats(blocks, total, histogram);
-  }
-
-  virtual size_t GetAllocatedSize(void* p) {
-    if (p) {
-      return MallocBlock::FromRawPointer(p)->data_size();
-    }
+  void* result = do_debug_memalign_or_debug_cpp_memalign(align, size);
+  MallocHook::InvokeNewHook(result, size);
+  if (result == NULL) {
+    return ENOMEM;
+  } else {
+    *result_ptr = result;
     return 0;
   }
-  virtual size_t GetEstimatedAllocatedSize(size_t size) {
-    return size;
-  }
- };
-
-static DebugMallocImplementation debug_malloc_implementation;
-
-REGISTER_MODULE_INITIALIZER(debugallocation, {
-  // Either we or valgrind will control memory management.  We
-  // register our extension if we're the winner.
-  if (RunningOnValgrind()) {
-    // Let Valgrind uses its own malloc (so don't register our extension).
-  } else {
-    MallocExtension::Register(&debug_malloc_implementation);
-    // When the program exits, check all blocks still in the free
-    // queue for corruption.
-    atexit(DanglingWriteChecker);
-  }
-});
-
-#ifdef TCMALLOC_FOR_DEBUGALLOCATION
-
-// Redefine malloc_stats to use tcmalloc's implementation:
-extern "C" void malloc_stats(void) __THROW {
-  do_malloc_stats();
-}
-
-// Some library routines on RedHat 9 allocate memory using malloc()
-// and free it using __libc_free() (or vice-versa).  Since we provide
-// our own implementations of malloc/free using tcmalloc.cc,
-// we need to make sure that the __libc_XXX variants
-// also point to the same implementations.
-//
-// Note: this might not override __libc_XXX calls withing libc itself,
-// but it can be important for other libraries that mention these functions
-// or when this code is LD_PRELOAD-ed.
-// TODO: In case these __libc_* definitions do not actually matter,
-// they should go away from here and from tcmalloc/tcmalloc.cc.
-//
-extern "C" {
-  void* __libc_malloc(size_t size)              { return malloc(size);       }
-  void  __libc_free(void* ptr)                  { free(ptr);                 }
-  void* __libc_realloc(void* ptr, size_t size)  { return realloc(ptr, size); }
-  void* __libc_calloc(size_t n, size_t size)    { return calloc(n, size);    }
-  void  __libc_cfree(void* ptr)                 { cfree(ptr);                }
-  void* __libc_memalign(size_t align, size_t s) { return memalign(align, s); }
-  void* __libc_valloc(size_t size)              { return valloc(size);       }
-  void* __libc_pvalloc(size_t size)             { return pvalloc(size);      }
-  int __posix_memalign(void** r, size_t a, size_t s) {
-    return posix_memalign(r, a, s);
-  }
 }
 
-#endif  // #ifdef TCMALLOC_FOR_DEBUGALLOCATION
+extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) __THROW {
+  // Allocate >= size bytes starting on a page boundary
+  void *p = do_debug_memalign_or_debug_cpp_memalign(getpagesize(), size);
+  MallocHook::InvokeNewHook(p, size);
+  return p;
+}
+
+extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) __THROW {
+  // Round size up to a multiple of pages
+  // then allocate memory on a page boundary
+  int pagesize = getpagesize();
+  size = RoundUp(size, pagesize);
+  if (size == 0) {     // pvalloc(0) should allocate one page, according to
+    size = pagesize;   // http://man.free4web.biz/man3/libmpatrol.3.html
+  }
+  void *p = do_debug_memalign_or_debug_cpp_memalign(pagesize, size);
+  MallocHook::InvokeNewHook(p, size);
+  return p;
+}
+
+// malloc_stats just falls through to the base implementation.
+extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) __THROW {
+  BASE_MALLOC_STATS();
+}
+
+extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) __THROW {
+  return BASE_MALLOPT(cmd, value);
+}
+
+#ifdef HAVE_STRUCT_MALLINFO
+extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW {
+  return BASE_MALLINFO();
+}
+#endif
+
+extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW {
+  if (!ptr) {
+    return 0;
+  }
+  MallocBlock* mb = MallocBlock::FromRawPointer(ptr);
+  // This is just to make sure we actually own mb (and ptr).  We don't
+  // use the actual value, just the 'exception' it raises on error.
+  (void)BASE_MALLOC_SIZE(mb);
+  return mb->data_size();
+}
+
+// Override __libc_memalign in libc on linux boxes.
+// They have a bug in libc that causes them (very rarely) to allocate
+// with __libc_memalign() yet deallocate with free().
+// This function is an exception to the rule of calling MallocHook method
+// from the stack frame of the allocation function;
+// heap-checker handles this special case explicitly.
+static void *MemalignOverride(size_t align, size_t size, const void *caller)
+    __THROW ATTRIBUTE_SECTION(google_malloc);
+
+static void *MemalignOverride(size_t align, size_t size, const void *caller)
+    __THROW {
+  void *p = do_debug_memalign_or_debug_cpp_memalign(align, size);
+  MallocHook::InvokeNewHook(p, size);
+  return p;
+}
+void *(*__memalign_hook)(size_t, size_t, const void *) = MemalignOverride;