Merge tcmalloc r111 (perftools v. 1.8) with the chromium/ branch.

third_party/tcmalloc/chromium/src/tcmalloc.cc

 // Copyright (c) 2005, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 75 matching lines @@
 // * allocation of a reasonably complicated struct
 //   goes from about 1100 ns to about 300 ns.
 
 #include "config.h"
 #include <google/tcmalloc.h>
 
 #include <errno.h>                      // for ENOMEM, EINVAL, errno
 #ifdef HAVE_SYS_CDEFS_H
 #include <sys/cdefs.h>                  // for __THROW
 #endif
-#ifdef HAVE_FEATURES_H
-#include <features.h>                   // for __GLIBC__
-#endif
 #if defined HAVE_STDINT_H
 #include <stdint.h>
 #elif defined HAVE_INTTYPES_H
 #include <inttypes.h>
 #else
 #include <sys/types.h>
 #endif
 #include <stddef.h>                     // for size_t, NULL
 #include <stdlib.h>                     // for getenv
 #include <string.h>                     // for strcmp, memset, strlen, etc
@@ skipping 34 matching lines @@
 # include <sys/malloc.h>
 # elif defined(HAVE_MALLOC_MALLOC_H)
 # include <malloc/malloc.h>
 # endif
 #endif
 
 #if (defined(_WIN32) && !defined(__CYGWIN__) && !defined(__CYGWIN32__)) && !defined(WIN32_OVERRIDE_ALLOCATORS)
 # define WIN32_DO_PATCHING 1
 #endif
 
-// GLibc 2.14+ requires the hook functions be declared volatile, based on the
-// value of the define __MALLOC_HOOK_VOLATILE. For compatibility with
-// older/non-GLibc implementations, provide an empty definition.
-#if !defined(__MALLOC_HOOK_VOLATILE)
-#define __MALLOC_HOOK_VOLATILE
-#endif
-
 using STL_NAMESPACE::max;
 using STL_NAMESPACE::numeric_limits;
 using STL_NAMESPACE::vector;
+
+#include "libc_override.h"
+
+// __THROW is defined in glibc (via <sys/cdefs.h>).  It means,
+// counter-intuitively, "This function will never throw an exception."
+// It's an optional optimization tool, but we may need to use it to
+// match glibc prototypes.
+#ifndef __THROW    // I guess we're not on a glibc system
+# define __THROW   // __THROW is just an optimization, so ok to make it ""
+#endif
+
 using tcmalloc::AlignmentForSize;
 using tcmalloc::PageHeap;
 using tcmalloc::PageHeapAllocator;
 using tcmalloc::SizeMap;
 using tcmalloc::Span;
 using tcmalloc::StackTrace;
 using tcmalloc::Static;
 using tcmalloc::ThreadCache;
 
-// __THROW is defined in glibc systems.  It means, counter-intuitively,
-// "This function will never throw an exception."  It's an optional
-// optimization tool, but we may need to use it to match glibc prototypes.
-#ifndef __THROW    // I guess we're not on a glibc system
-# define __THROW   // __THROW is just an optimization, so ok to make it ""
-#endif
-
 // ---- Double free debug declarations
 static size_t ExcludeSpaceForMark(size_t size);
 static void AddRoomForMark(size_t* size);
 static void ExcludeMarkFromSize(size_t* new_size);
 static void MarkAllocatedRegion(void* ptr);
 static void ValidateAllocatedRegion(void* ptr, size_t cl);
 // ---- End Double free debug declarations
 
 DECLARE_int64(tcmalloc_sample_parameter);
 DECLARE_double(tcmalloc_release_rate);
@@ skipping 84 matching lines @@
   // Some non-standard extensions that we support.
 
   // This is equivalent to
   //    OS X: malloc_size()
   //    glibc: malloc_usable_size()
   //    Windows: _msize()
   size_t tc_malloc_size(void* p) __THROW
       ATTRIBUTE_SECTION(google_malloc);
 }  // extern "C"
 
-// Override the libc functions to prefer our own instead.  This comes
-// first so code in tcmalloc.cc can use the overridden versions.  One
-// exception: in windows, by default, we patch our code into these
-// functions (via src/windows/patch_function.cc) rather than override
-// them.  In that case, we don't want to do this overriding here.
-#if !defined(WIN32_DO_PATCHING)
-
-// TODO(mbelshe):  Turn off TCMalloc's symbols for libc.  We do that
-//                 elsewhere.
-#ifndef _WIN32
-
-#if defined(__GNUC__) && !defined(__MACH__)
-  // Potentially faster variants that use the gcc alias extension.
-  // FreeBSD does support aliases, but apparently not correctly. :-(
-  // NOTE: we make many of these symbols weak, but do so in the makefile
-  //       (via objcopy -W) and not here.  That ends up being more portable.
-# define ALIAS(x) __attribute__ ((alias (x)))
-void* operator new(size_t size) throw (std::bad_alloc) ALIAS("tc_new");
-void operator delete(void* p) __THROW            ALIAS("tc_delete");
-void* operator new[](size_t size) throw (std::bad_alloc) ALIAS("tc_newarray");
-void operator delete[](void* p) __THROW          ALIAS("tc_deletearray");
-void* operator new(size_t size, const std::nothrow_t&) __THROW
-                                                 ALIAS("tc_new_nothrow");
-void* operator new[](size_t size, const std::nothrow_t&) __THROW
-                                                 ALIAS("tc_newarray_nothrow");
-void operator delete(void* size, const std::nothrow_t&) __THROW
-                                                 ALIAS("tc_delete_nothrow");
-void operator delete[](void* size, const std::nothrow_t&) __THROW
-                                                ALIAS("tc_deletearray_nothrow");
-extern "C" {
-  void* malloc(size_t size) __THROW              ALIAS("tc_malloc");
-  void  free(void* ptr) __THROW                  ALIAS("tc_free");
-  void* realloc(void* ptr, size_t size) __THROW  ALIAS("tc_realloc");
-  void* calloc(size_t n, size_t size) __THROW    ALIAS("tc_calloc");
-  void  cfree(void* ptr) __THROW                 ALIAS("tc_cfree");
-  void* memalign(size_t align, size_t s) __THROW ALIAS("tc_memalign");
-  void* valloc(size_t size) __THROW              ALIAS("tc_valloc");
-  void* pvalloc(size_t size) __THROW             ALIAS("tc_pvalloc");
-  int posix_memalign(void** r, size_t a, size_t s) __THROW
-      ALIAS("tc_posix_memalign");
-  void malloc_stats(void) __THROW                ALIAS("tc_malloc_stats");
-  int mallopt(int cmd, int value) __THROW        ALIAS("tc_mallopt");
-#ifdef HAVE_STRUCT_MALLINFO
-  struct mallinfo mallinfo(void) __THROW         ALIAS("tc_mallinfo");
-#endif
-  size_t malloc_size(void* p) __THROW            ALIAS("tc_malloc_size");
-  size_t malloc_usable_size(void* p) __THROW     ALIAS("tc_malloc_size");
-}   // extern "C"
-#else  // #if defined(__GNUC__) && !defined(__MACH__)
-// Portable wrappers
-void* operator new(size_t size)                  { return tc_new(size);       }
-void operator delete(void* p) __THROW            { tc_delete(p);              }
-void* operator new[](size_t size)                { return tc_newarray(size);  }
-void operator delete[](void* p) __THROW          { tc_deletearray(p);         }
-void* operator new(size_t size, const std::nothrow_t& nt) __THROW {
-  return tc_new_nothrow(size, nt);
-}
-void* operator new[](size_t size, const std::nothrow_t& nt) __THROW {
-  return tc_newarray_nothrow(size, nt);
-}
-void operator delete(void* ptr, const std::nothrow_t& nt) __THROW {
-  return tc_delete_nothrow(ptr, nt);
-}
-void operator delete[](void* ptr, const std::nothrow_t& nt) __THROW {
-  return tc_deletearray_nothrow(ptr, nt);
-}
-extern "C" {
-  void* malloc(size_t s) __THROW                 { return tc_malloc(s);       }
-  void  free(void* p) __THROW                    { tc_free(p);                }
-  void* realloc(void* p, size_t s) __THROW       { return tc_realloc(p, s);   }
-  void* calloc(size_t n, size_t s) __THROW       { return tc_calloc(n, s);    }
-  void  cfree(void* p) __THROW                   { tc_cfree(p);               }
-  void* memalign(size_t a, size_t s) __THROW     { return tc_memalign(a, s);  }
-  void* valloc(size_t s) __THROW                 { return tc_valloc(s);       }
-  void* pvalloc(size_t s) __THROW                { return tc_pvalloc(s);      }
-  int posix_memalign(void** r, size_t a, size_t s) __THROW {
-    return tc_posix_memalign(r, a, s);
-  }
-  void malloc_stats(void) __THROW                { tc_malloc_stats();         }
-  int mallopt(int cmd, int v) __THROW            { return tc_mallopt(cmd, v); }
-#ifdef HAVE_STRUCT_MALLINFO
-  struct mallinfo mallinfo(void) __THROW         { return tc_mallinfo();      }
-#endif
-  size_t malloc_size(void* p) __THROW            { return tc_malloc_size(p); }
-  size_t malloc_usable_size(void* p) __THROW     { return tc_malloc_size(p); }
-}  // extern "C"
-#endif  // #if defined(__GNUC__)
-
-// Some library routines on RedHat 9 allocate memory using malloc()
-// and free it using __libc_free() (or vice-versa).  Since we provide
-// our own implementations of malloc/free, we need to make sure that
-// the __libc_XXX variants (defined as part of glibc) also point to
-// the same implementations.
-#ifdef __GLIBC__       // only glibc defines __libc_*
-extern "C" {
-#ifdef ALIAS
-  void* __libc_malloc(size_t size)               ALIAS("tc_malloc");
-  void  __libc_free(void* ptr)                   ALIAS("tc_free");
-  void* __libc_realloc(void* ptr, size_t size)   ALIAS("tc_realloc");
-  void* __libc_calloc(size_t n, size_t size)     ALIAS("tc_calloc");
-  void  __libc_cfree(void* ptr)                  ALIAS("tc_cfree");
-  void* __libc_memalign(size_t align, size_t s)  ALIAS("tc_memalign");
-  void* __libc_valloc(size_t size)               ALIAS("tc_valloc");
-  void* __libc_pvalloc(size_t size)              ALIAS("tc_pvalloc");
-  int __posix_memalign(void** r, size_t a, size_t s) ALIAS("tc_posix_memalign");
-#else  // #ifdef ALIAS
-  void* __libc_malloc(size_t size)               { return malloc(size);       }
-  void  __libc_free(void* ptr)                   { free(ptr);                 }
-  void* __libc_realloc(void* ptr, size_t size)   { return realloc(ptr, size); }
-  void* __libc_calloc(size_t n, size_t size)     { return calloc(n, size);    }
-  void  __libc_cfree(void* ptr)                  { cfree(ptr);                }
-  void* __libc_memalign(size_t align, size_t s)  { return memalign(align, s); }
-  void* __libc_valloc(size_t size)               { return valloc(size);       }
-  void* __libc_pvalloc(size_t size)              { return pvalloc(size);      }
-  int __posix_memalign(void** r, size_t a, size_t s) {
-    return posix_memalign(r, a, s);
-  }
-#endif  // #ifdef ALIAS
-}   // extern "C"
-#endif  // ifdef __GLIBC__
-
-#if defined(__GLIBC__) && defined(HAVE_MALLOC_H)
-// If we're using glibc, then override glibc malloc hooks to make sure that even
-// if calls fall through to ptmalloc (due to dlopen() with RTLD_DEEPBIND or what
-// not), ptmalloc will use TCMalloc.
-
-static void* tc_ptmalloc_malloc_hook(size_t size, const void* caller) {
-  return tc_malloc(size);
-}
-
-void* (*__MALLOC_HOOK_VOLATILE __malloc_hook)(
-    size_t size, const void* caller) = tc_ptmalloc_malloc_hook;
-
-static void* tc_ptmalloc_realloc_hook(
-    void* ptr, size_t size, const void* caller) {
-  return tc_realloc(ptr, size);
-}
-
-void* (*__MALLOC_HOOK_VOLATILE __realloc_hook)(
-    void* ptr, size_t size, const void* caller) = tc_ptmalloc_realloc_hook;
-
-static void tc_ptmalloc_free_hook(void* ptr, const void* caller) {
-  tc_free(ptr);
-}
-
-void (*__MALLOC_HOOK_VOLATILE __free_hook)(void* ptr, const void* caller) = tc_ptmalloc_free_hook;
-
-#endif
-
-#endif  // #ifndef _WIN32
-#undef ALIAS
-
-#endif  // #ifndef(WIN32_DO_PATCHING)
-
-
 // ----------------------- IMPLEMENTATION -------------------------------
 
 static int tc_new_mode = 0;  // See tc_set_new_mode().
 
 // Routines such as free() and realloc() catch some erroneous pointers
 // passed to them, and invoke the below when they do.  (An erroneous pointer
 // won't be caught if it's within a valid span or a stale span for which
 // the pagemap cache has a non-zero sizeclass.) This is a cheap (source-editing
 // required) kind of exception handling for these routines.
 namespace {
@@ skipping 21 matching lines @@
   PageHeap::Stats pageheap;   // Stats from page heap
 };
 
 // Get stats into "r".  Also get per-size-class counts if class_count != NULL
 static void ExtractStats(TCMallocStats* r, uint64_t* class_count) {
   r->central_bytes = 0;
   r->transfer_bytes = 0;
   for (int cl = 0; cl < kNumClasses; ++cl) {
     const int length = Static::central_cache()[cl].length();
     const int tc_length = Static::central_cache()[cl].tc_length();
+    const size_t cache_overhead = Static::central_cache()[cl].OverheadBytes();
     const size_t size = static_cast<uint64_t>(
         Static::sizemap()->ByteSizeForClass(cl));
-    r->central_bytes += (size * length);
+    r->central_bytes += (size * length) + cache_overhead;
     r->transfer_bytes += (size * tc_length);
     if (class_count) class_count[cl] = length + tc_length;
   }
 
   // Add stats from per-thread heaps
   r->thread_bytes = 0;
   { // scope
     SpinLockHolder h(Static::pageheap_lock());
     ThreadCache::GetThreadStats(&r->thread_bytes, class_count);
     r->metadata_bytes = tcmalloc::metadata_system_bytes();
@@ skipping 369 matching lines @@
     } else {
       return tcmalloc::pages(size) << kPageShift;
     }
   }
 
   // This just calls GetSizeWithCallback, but because that's in an
   // unnamed namespace, we need to move the definition below it in the
   // file.
   virtual size_t GetAllocatedSize(void* ptr);
 
+  // This duplicates some of the logic in GetSizeWithCallback, but is
+  // faster.  This is important on OS X, where this function is called
+  // on every allocation operation.
+  virtual Ownership GetOwnership(const void* ptr) {
+    const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
+    // The rest of tcmalloc assumes that all allocated pointers use at
+    // most kAddressBits bits.  If ptr doesn't, then it definitely
+    // wasn't alloacted by tcmalloc.
+    if ((p >> (kAddressBits - kPageShift)) > 0) {
+      return kNotOwned;
+    }
+    size_t cl = Static::pageheap()->GetSizeClassIfCached(p);
+    if (cl != 0) {
+      return kOwned;
+    }
+    const Span *span = Static::pageheap()->GetDescriptor(p);
+    return span ? kOwned : kNotOwned;
+  }
+
   virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) {
     static const char* kCentralCacheType = "tcmalloc.central";
     static const char* kTransferCacheType = "tcmalloc.transfer";
     static const char* kThreadCacheType = "tcmalloc.thread";
     static const char* kPageHeapType = "tcmalloc.page";
     static const char* kPageHeapUnmappedType = "tcmalloc.page_unmapped";
     static const char* kLargeSpanType = "tcmalloc.large";
     static const char* kLargeUnmappedSpanType = "tcmalloc.large_unmapped";
 
     v->clear();
@@ skipping 93 matching lines @@
 // well for STL).
 //
 // The destructor prints stats when the program exits.
 static int tcmallocguard_refcount = 0;  // no lock needed: runs before main()
 TCMallocGuard::TCMallocGuard() {
   if (tcmallocguard_refcount++ == 0) {
 #ifdef HAVE_TLS    // this is true if the cc/ld/libc combo support TLS
     // Check whether the kernel also supports TLS (needs to happen at runtime)
     tcmalloc::CheckIfKernelSupportsTLS();
 #endif
-#ifdef WIN32_DO_PATCHING
-    // patch the windows VirtualAlloc, etc.
-    PatchWindowsFunctions();    // defined in windows/patch_functions.cc
+    ReplaceSystemAlloc();    // defined in libc_override_*.h
+#if defined(__APPLE__)
+    // To break the recursive call of malloc, as malloc -> TCMALLOC_MESSAGE
+    // -> snprintf -> localeconv_l -> malloc, on MacOS.
+    char buf[32];
+    snprintf(buf, sizeof(buf), "%d", tcmallocguard_refcount);
 #endif
     tc_free(tc_malloc(1));
     ThreadCache::InitTSD();
     tc_free(tc_malloc(1));
     // Either we, or debugallocation.cc, or valgrind will control memory
     // management.  We register our extension if we're the winner.
 #ifdef TCMALLOC_USING_DEBUGALLOCATION
     // Let debugallocation register its extension.
 #else
     if (RunningOnValgrind()) {
@@ skipping 77 matching lines @@
   (kPageSize > FLAGS_tcmalloc_large_alloc_report_threshold
    ? kPageSize : FLAGS_tcmalloc_large_alloc_report_threshold);
 
 static void ReportLargeAlloc(Length num_pages, void* result) {
   StackTrace stack;
   stack.depth = GetStackTrace(stack.stack, tcmalloc::kMaxStackDepth, 1);
 
   static const int N = 1000;
   char buffer[N];
   TCMalloc_Printer printer(buffer, N);
-  printer.printf("tcmalloc: large alloc %llu bytes == %p @ ",
-                 static_cast<unsigned long long>(num_pages) << kPageShift,
+  printer.printf("tcmalloc: large alloc %"PRIu64" bytes == %p @ ",
+                 static_cast<uint64>(num_pages) << kPageShift,
                  result);
   for (int i = 0; i < stack.depth; i++) {
     printer.printf(" %p", stack.stack[i]);
   }
   printer.printf("\n");
   write(STDERR_FILENO, buffer, strlen(buffer));
 }
 
 inline void* cpp_alloc(size_t size, bool nothrow);
 inline void* do_malloc(size_t size);
@@ skipping 146 matching lines @@
     }
     Static::pageheap()->Delete(span);
   }
 }
 
 // The default "do_free" that uses the default callback.
 inline void do_free(void* ptr) {
   return do_free_with_callback(ptr, &InvalidFree);
 }
 
+// NOTE: some logic here is duplicated in GetOwnership (above), for
+// speed.  If you change this function, look at that one too.
 inline size_t GetSizeWithCallback(void* ptr,
                                   size_t (*invalid_getsize_fn)(void*)) {
   if (ptr == NULL)
     return 0;
   const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
   size_t cl = Static::pageheap()->GetSizeClassIfCached(p);
   if (cl != 0) {
     return Static::sizemap()->ByteSizeForClass(cl);
   } else {
-    Span *span = Static::pageheap()->GetDescriptor(p);
+    const Span *span = Static::pageheap()->GetDescriptor(p);
     if (span == NULL) {  // means we do not own this memory
       return (*invalid_getsize_fn)(ptr);
     } else if (span->sizeclass != 0) {
       Static::pageheap()->CacheSizeClass(p, span->sizeclass);
       return Static::sizemap()->ByteSizeForClass(span->sizeclass);
     } else {
       return span->length << kPageShift;
     }
   }
 }
@@ skipping 267 matching lines @@
       return p;
     }
 #endif  // PREANSINEW
   }
 }
 
 }  // end unnamed namespace
 
 // As promised, the definition of this function, declared above.
 size_t TCMallocImplementation::GetAllocatedSize(void* ptr) {
+  ASSERT(TCMallocImplementation::GetOwnership(ptr)
+         != TCMallocImplementation::kNotOwned);
   return ExcludeSpaceForMark(
       GetSizeWithCallback(ptr, &InvalidGetAllocatedSize));
 }
 
 void TCMallocImplementation::MarkThreadBusy() {
   // Allocate to force the creation of a thread cache, but avoid
   // invoking any hooks.
   do_free(do_malloc(0));
 }
 
@@ skipping 178 matching lines @@
   return do_mallopt(cmd, value);
 }
 
 #ifdef HAVE_STRUCT_MALLINFO
 extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW {
   return do_mallinfo();
 }
 #endif
 
 extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW {
-  return GetSizeWithCallback(ptr, &InvalidGetAllocatedSize);
+  return MallocExtension::instance()->GetAllocatedSize(ptr);
 }
 
-
-// Override __libc_memalign in libc on linux boxes specially.
-// They have a bug in libc that causes them to (very rarely) allocate
-// with __libc_memalign() yet deallocate with free() and the
-// definitions above don't catch it.
-// This function is an exception to the rule of calling MallocHook method
-// from the stack frame of the allocation function;
-// heap-checker handles this special case explicitly.
-static void *MemalignOverride(size_t align, size_t size, const void *caller)
-    __THROW ATTRIBUTE_SECTION(google_malloc);
-
-static void *MemalignOverride(size_t align, size_t size, const void *caller)
-    __THROW {
-  void* result = do_memalign_or_cpp_memalign(align, size);
-  MallocHook::InvokeNewHook(result, size);
-  return result;
-}
-void *(*__MALLOC_HOOK_VOLATILE __memalign_hook)(size_t, size_t, const void *) = MemalignOverride;
 #endif  // TCMALLOC_USING_DEBUGALLOCATION
 
 // ---Double free() debugging implementation -----------------------------------
 // We will put a mark at the extreme end of each allocation block.  We make
 // sure that we always allocate enough "extra memory" that we can fit in the
 // mark, and still provide the requested usable region.  If ever that mark is
 // not as expected, then we know that the user is corrupting memory beyond their
 // request size, or that they have called free a second time without having
 // the memory allocated (again).  This allows us to spot most double free()s,
 // but some can "slip by" or confuse our logic if the caller reallocates memory
@@ skipping 148 matching lines @@
   *mark = ~allocated_mark;  //  Distinctively not allocated.
 }
 
 static void MarkAllocatedRegion(void* ptr) {
   if (ptr == NULL) return;
   MarkType* mark = GetMarkLocation(ptr);
   *mark = GetMarkValue(ptr, mark);
 }
 
 #endif  // TCMALLOC_VALIDATION