Merge google-perftools r109 (the current contents of third_party/tcmalloc/vendor)

third_party/tcmalloc/chromium/src/malloc_hook.cc

Index: third_party/tcmalloc/chromium/src/malloc_hook.cc
===================================================================
--- third_party/tcmalloc/chromium/src/malloc_hook.cc	(revision 88335)
+++ third_party/tcmalloc/chromium/src/malloc_hook.cc	(working copy)
@@ -41,9 +41,15 @@
 # undef mremap
 #endif
 
+#include <stddef.h>
+#ifdef HAVE_STDINT_H
+#include <stdint.h>
+#endif
 #include <algorithm>
 #include "base/basictypes.h"
 #include "base/logging.h"
+#include "base/spinlock.h"
+#include "maybe_threads.h"
 #include "malloc_hook-inl.h"
 #include <google/malloc_hook.h>
 
@@ -66,48 +72,93 @@
 using std::copy;
 
 
-// Declarations of three default weak hook functions, that can be overridden by
-// linking-in a strong definition (as heap-checker.cc does)
+// Declaration of default weak initialization function, that can be overridden
+// by linking-in a strong definition (as heap-checker.cc does).  This is
+// extern "C" so that it doesn't trigger gold's --detect-odr-violations warning,
+// which only looks at C++ symbols.
 //
-// These default hooks let some other library we link in
-// to define strong versions of InitialMallocHook_New, InitialMallocHook_MMap,
-// InitialMallocHook_PreMMap, InitialMallocHook_PreSbrk, and
-// InitialMallocHook_Sbrk to have a chance to hook into the very first
-// invocation of an allocation function call, mmap, or sbrk.
+// This function is declared here as weak, and defined later, rather than a more
+// straightforward simple weak definition, as a workround for an icc compiler
+// issue ((Intel reference 290819).  This issue causes icc to resolve weak
+// symbols too early, at compile rather than link time.  By declaring it (weak)
+// here, then defining it below after its use, we can avoid the problem.
+extern "C" {
+ATTRIBUTE_WEAK void MallocHook_InitAtFirstAllocation_HeapLeakChecker();
+}
+
+namespace {
+
+void RemoveInitialHooksAndCallInitializers();  // below.
+
+pthread_once_t once = PTHREAD_ONCE_INIT;
+
+// These hooks are installed in MallocHook as the only initial hooks.  The first
+// hook that is called will run RemoveInitialHooksAndCallInitializers (see the
+// definition below) and then redispatch to any malloc hooks installed by
+// RemoveInitialHooksAndCallInitializers.
 //
-// These functions are declared here as weak, and defined later, rather than a
-// more straightforward simple weak definition, as a workround for an icc
-// compiler issue ((Intel reference 290819).  This issue causes icc to resolve
-// weak symbols too early, at compile rather than link time.  By declaring it
-// (weak) here, then defining it below after its use, we can avoid the problem.
-//
-ATTRIBUTE_WEAK
-extern void InitialMallocHook_New(const void* ptr, size_t size);
+// Note(llib): there is a possibility of a race in the event that there are
+// multiple threads running before the first allocation.  This is pretty
+// difficult to achieve, but if it is then multiple threads may concurrently do
+// allocations.  The first caller will call
+// RemoveInitialHooksAndCallInitializers via one of the initial hooks.  A
+// concurrent allocation may, depending on timing either:
+// * still have its initial malloc hook installed, run that and block on waiting
+//   for the first caller to finish its call to
+//   RemoveInitialHooksAndCallInitializers, and proceed normally.
+// * occur some time during the RemoveInitialHooksAndCallInitializers call, at
+//   which point there could be no initial hooks and the subsequent hooks that
+//   are about to be set up by RemoveInitialHooksAndCallInitializers haven't
+//   been installed yet.  I think the worst we can get is that some allocations
+//   will not get reported to some hooks set by the initializers called from
+//   RemoveInitialHooksAndCallInitializers.
 
-ATTRIBUTE_WEAK
-extern void InitialMallocHook_PreMMap(const void* start,
-                                      size_t size,
-                                      int protection,
-                                      int flags,
-                                      int fd,
-                                      off_t offset);
+void InitialNewHook(const void* ptr, size_t size) {
+  perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers);
+  MallocHook::InvokeNewHook(ptr, size);
+}
 
-ATTRIBUTE_WEAK
-extern void InitialMallocHook_MMap(const void* result,
-                                   const void* start,
-                                   size_t size,
-                                   int protection,
-                                   int flags,
-                                   int fd,
-                                   off_t offset);
+void InitialPreMMapHook(const void* start,
+                               size_t size,
+                               int protection,
+                               int flags,
+                               int fd,
+                               off_t offset) {
+  perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers);
+  MallocHook::InvokePreMmapHook(start, size, protection, flags, fd, offset);
+}
 
-ATTRIBUTE_WEAK
-extern void InitialMallocHook_PreSbrk(std::ptrdiff_t increment);
+void InitialPreSbrkHook(std::ptrdiff_t increment) {
+  perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers);
+  MallocHook::InvokePreSbrkHook(increment);
+}
 
-ATTRIBUTE_WEAK
-extern void InitialMallocHook_Sbrk(const void* result, std::ptrdiff_t increment);
+// This function is called at most once by one of the above initial malloc
+// hooks.  It removes all initial hooks and initializes all other clients that
+// want to get control at the very first memory allocation.  The initializers
+// may assume that the initial malloc hooks have been removed.  The initializers
+// may set up malloc hooks and allocate memory.
+void RemoveInitialHooksAndCallInitializers() {
+  RAW_CHECK(MallocHook::RemoveNewHook(&InitialNewHook), "");
+  RAW_CHECK(MallocHook::RemovePreMmapHook(&InitialPreMMapHook), "");
+  RAW_CHECK(MallocHook::RemovePreSbrkHook(&InitialPreSbrkHook), "");
 
+  // HeapLeakChecker is currently the only module that needs to get control on
+  // the first memory allocation, but one can add other modules by following the
+  // same weak/strong function pattern.
+  MallocHook_InitAtFirstAllocation_HeapLeakChecker();
+}
+
+}  // namespace
+
+// Weak default initialization function that must go after its use.
+extern "C" void MallocHook_InitAtFirstAllocation_HeapLeakChecker() {
+  // Do nothing.
+}
+
 namespace base { namespace internal {
+
+// The code below is DEPRECATED.
 template<typename PtrT>
 PtrT AtomicPtr<PtrT>::Exchange(PtrT new_val) {
   base::subtle::MemoryBarrier();  // Release semantics.
@@ -123,22 +174,139 @@
   return old_val;
 }
 
-AtomicPtr<MallocHook::NewHook>    new_hook_ = {
-  reinterpret_cast<AtomicWord>(InitialMallocHook_New) };
+template<typename PtrT>
+PtrT AtomicPtr<PtrT>::CompareAndSwap(PtrT old_val, PtrT new_val) {
+  base::subtle::MemoryBarrier();  // Release semantics.
+  PtrT retval = reinterpret_cast<PtrT>(static_cast<AtomicWord>(
+      base::subtle::NoBarrier_CompareAndSwap(
+          &data_,
+          reinterpret_cast<AtomicWord>(old_val),
+          reinterpret_cast<AtomicWord>(new_val))));
+  base::subtle::MemoryBarrier();  // And acquire semantics.
+  return retval;
+}
+
+AtomicPtr<MallocHook::NewHook>    new_hook_ = { 0 };
 AtomicPtr<MallocHook::DeleteHook> delete_hook_ = { 0 };
-AtomicPtr<MallocHook::PreMmapHook> premmap_hook_ = {
-  reinterpret_cast<AtomicWord>(InitialMallocHook_PreMMap) };
-AtomicPtr<MallocHook::MmapHook>   mmap_hook_ = {
-  reinterpret_cast<AtomicWord>(InitialMallocHook_MMap) };
+AtomicPtr<MallocHook::PreMmapHook> premmap_hook_ = { 0 };
+AtomicPtr<MallocHook::MmapHook>   mmap_hook_ = { 0 };
 AtomicPtr<MallocHook::MunmapHook> munmap_hook_ = { 0 };
 AtomicPtr<MallocHook::MremapHook> mremap_hook_ = { 0 };
-AtomicPtr<MallocHook::PreSbrkHook> presbrk_hook_ = {
-  reinterpret_cast<AtomicWord>(InitialMallocHook_PreSbrk) };
-AtomicPtr<MallocHook::SbrkHook>   sbrk_hook_ = {
-  reinterpret_cast<AtomicWord>(InitialMallocHook_Sbrk) };
+AtomicPtr<MallocHook::PreSbrkHook> presbrk_hook_ = { 0 };
+AtomicPtr<MallocHook::SbrkHook>   sbrk_hook_ = { 0 };
+// End of DEPRECATED code section.
 
+// This lock is shared between all implementations of HookList::Add & Remove.
+// The potential for contention is very small.  This needs to be a SpinLock and
+// not a Mutex since it's possible for Mutex locking to allocate memory (e.g.,
+// per-thread allocation in debug builds), which could cause infinite recursion.
+static SpinLock hooklist_spinlock(base::LINKER_INITIALIZED);
+
+template <typename T>
+bool HookList<T>::Add(T value_as_t) {
+  // Note: we need to check this _before_ reinterpret_cast, since
+  // reinterpret_cast may include random junk from memory.
+  if (value_as_t == 0) {
+    return false;
+  }
+  AtomicWord value = reinterpret_cast<const AtomicWord&>(value_as_t);
+  if (value == 0) {
+    // This should not actually happen, but just to be sure...
+    return false;
+  }
+  SpinLockHolder l(&hooklist_spinlock);
+  // Find the first slot in data that is 0.
+  int index = 0;
+  while ((index < kHookListMaxValues) &&
+         (base::subtle::NoBarrier_Load(&priv_data[index]) != 0)) {
+    ++index;
+  }
+  if (index == kHookListMaxValues) {
+    return false;
+  }
+  AtomicWord prev_num_hooks = base::subtle::Acquire_Load(&priv_end);
+  base::subtle::Release_Store(&priv_data[index], value);
+  if (prev_num_hooks <= index) {
+    base::subtle::Release_Store(&priv_end, index + 1);
+  }
+  return true;
+}
+
+template <typename T>
+bool HookList<T>::Remove(T value_as_t) {
+  if (value_as_t == 0) {
+    return false;
+  }
+  SpinLockHolder l(&hooklist_spinlock);
+  AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end);
+  int index = 0;
+  // Note: we need to cast back to T since T may be smaller than AtomicWord.
+  while (index < hooks_end && value_as_t != reinterpret_cast<T>(
+             base::subtle::Acquire_Load(&priv_data[index]))) {
+    ++index;
+  }
+  if (index == hooks_end) {
+    return false;
+  }
+  base::subtle::Release_Store(&priv_data[index], 0);
+  if (hooks_end == index + 1) {
+    // Adjust hooks_end down to the lowest possible value.
+    hooks_end = index;
+    while ((hooks_end > 0) &&
+           (base::subtle::Acquire_Load(&priv_data[hooks_end - 1]) == 0)) {
+      --hooks_end;
+    }
+    base::subtle::Release_Store(&priv_end, hooks_end);
+  }
+  return true;
+}
+
+template <typename T>
+int HookList<T>::Traverse(T* output_array, int n) const {
+  AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end);
+  int actual_hooks_end = 0;
+  for (int i = 0; i < hooks_end && n > 0; ++i) {
+    AtomicWord data = base::subtle::Acquire_Load(&priv_data[i]);
+    if (data != 0) {
+      *output_array++ = reinterpret_cast<const T&>(data);
+      ++actual_hooks_end;
+      --n;
+    }
+  }
+  return actual_hooks_end;
+}
+
+// Initialize a HookList (optionally with the given initial_value in index 0).
+#define INIT_HOOK_LIST { 0 }
+#define INIT_HOOK_LIST_WITH_VALUE(initial_value)                \
+  { 1, { reinterpret_cast<AtomicWord>(initial_value) } }
+
+// Explicit instantiation for malloc_hook_test.cc.  This ensures all the methods
+// are instantiated.
+template class HookList<MallocHook::NewHook>;
+
+HookList<MallocHook::NewHook> new_hooks_ =
+    INIT_HOOK_LIST_WITH_VALUE(&InitialNewHook);
+HookList<MallocHook::DeleteHook> delete_hooks_ = INIT_HOOK_LIST;
+HookList<MallocHook::PreMmapHook> premmap_hooks_ =
+    INIT_HOOK_LIST_WITH_VALUE(&InitialPreMMapHook);
+HookList<MallocHook::MmapHook> mmap_hooks_ = INIT_HOOK_LIST;
+HookList<MallocHook::MunmapHook> munmap_hooks_ = INIT_HOOK_LIST;
+HookList<MallocHook::MremapHook> mremap_hooks_ = INIT_HOOK_LIST;
+HookList<MallocHook::PreSbrkHook> presbrk_hooks_ =
+    INIT_HOOK_LIST_WITH_VALUE(InitialPreSbrkHook);
+HookList<MallocHook::SbrkHook> sbrk_hooks_ = INIT_HOOK_LIST;
+
+// These lists contain either 0 or 1 hooks.
+HookList<MallocHook::MmapReplacement> mmap_replacement_ = { 0 };
+HookList<MallocHook::MunmapReplacement> munmap_replacement_ = { 0 };
+
+#undef INIT_HOOK_LIST_WITH_VALUE
+#undef INIT_HOOK_LIST
+
 } }  // namespace base::internal
 
+// The code below is DEPRECATED.
 using base::internal::new_hook_;
 using base::internal::delete_hook_;
 using base::internal::premmap_hook_;
@@ -147,109 +315,290 @@
 using base::internal::mremap_hook_;
 using base::internal::presbrk_hook_;
 using base::internal::sbrk_hook_;
+// End of DEPRECATED code section.
 
+using base::internal::kHookListMaxValues;
+using base::internal::new_hooks_;
+using base::internal::delete_hooks_;
+using base::internal::premmap_hooks_;
+using base::internal::mmap_hooks_;
+using base::internal::mmap_replacement_;
+using base::internal::munmap_hooks_;
+using base::internal::munmap_replacement_;
+using base::internal::mremap_hooks_;
+using base::internal::presbrk_hooks_;
+using base::internal::sbrk_hooks_;
 
 // These are available as C bindings as well as C++, hence their
 // definition outside the MallocHook class.
 extern "C"
+int MallocHook_AddNewHook(MallocHook_NewHook hook) {
+  RAW_VLOG(10, "AddNewHook(%p)", hook);
+  return new_hooks_.Add(hook);
+}
+
+extern "C"
+int MallocHook_RemoveNewHook(MallocHook_NewHook hook) {
+  RAW_VLOG(10, "RemoveNewHook(%p)", hook);
+  return new_hooks_.Remove(hook);
+}
+
+extern "C"
+int MallocHook_AddDeleteHook(MallocHook_DeleteHook hook) {
+  RAW_VLOG(10, "AddDeleteHook(%p)", hook);
+  return delete_hooks_.Add(hook);
+}
+
+extern "C"
+int MallocHook_RemoveDeleteHook(MallocHook_DeleteHook hook) {
+  RAW_VLOG(10, "RemoveDeleteHook(%p)", hook);
+  return delete_hooks_.Remove(hook);
+}
+
+extern "C"
+int MallocHook_AddPreMmapHook(MallocHook_PreMmapHook hook) {
+  RAW_VLOG(10, "AddPreMmapHook(%p)", hook);
+  return premmap_hooks_.Add(hook);
+}
+
+extern "C"
+int MallocHook_RemovePreMmapHook(MallocHook_PreMmapHook hook) {
+  RAW_VLOG(10, "RemovePreMmapHook(%p)", hook);
+  return premmap_hooks_.Remove(hook);
+}
+
+extern "C"
+int MallocHook_SetMmapReplacement(MallocHook_MmapReplacement hook) {
+  RAW_VLOG(10, "SetMmapReplacement(%p)", hook);
+  // NOTE this is a best effort CHECK. Concurrent sets could succeed since
+  // this test is outside of the Add spin lock.
+  RAW_CHECK(mmap_replacement_.empty(), "Only one MMapReplacement is allowed.");
+  return mmap_replacement_.Add(hook);
+}
+
+extern "C"
+int MallocHook_RemoveMmapReplacement(MallocHook_MmapReplacement hook) {
+  RAW_VLOG(10, "RemoveMmapReplacement(%p)", hook);
+  return mmap_replacement_.Remove(hook);
+}
+
+extern "C"
+int MallocHook_AddMmapHook(MallocHook_MmapHook hook) {
+  RAW_VLOG(10, "AddMmapHook(%p)", hook);
+  return mmap_hooks_.Add(hook);
+}
+
+extern "C"
+int MallocHook_RemoveMmapHook(MallocHook_MmapHook hook) {
+  RAW_VLOG(10, "RemoveMmapHook(%p)", hook);
+  return mmap_hooks_.Remove(hook);
+}
+
+extern "C"
+int MallocHook_AddMunmapHook(MallocHook_MunmapHook hook) {
+  RAW_VLOG(10, "AddMunmapHook(%p)", hook);
+  return munmap_hooks_.Add(hook);
+}
+
+extern "C"
+int MallocHook_RemoveMunmapHook(MallocHook_MunmapHook hook) {
+  RAW_VLOG(10, "RemoveMunmapHook(%p)", hook);
+  return munmap_hooks_.Remove(hook);
+}
+
+extern "C"
+int MallocHook_SetMunmapReplacement(MallocHook_MunmapReplacement hook) {
+  RAW_VLOG(10, "SetMunmapReplacement(%p)", hook);
+  // NOTE this is a best effort CHECK. Concurrent sets could succeed since
+  // this test is outside of the Add spin lock.
+  RAW_CHECK(munmap_replacement_.empty(),
+            "Only one MunmapReplacement is allowed.");
+  return munmap_replacement_.Add(hook);
+}
+
+extern "C"
+int MallocHook_RemoveMunmapReplacement(MallocHook_MunmapReplacement hook) {
+  RAW_VLOG(10, "RemoveMunmapReplacement(%p)", hook);
+  return munmap_replacement_.Remove(hook);
+}
+
+extern "C"
+int MallocHook_AddMremapHook(MallocHook_MremapHook hook) {
+  RAW_VLOG(10, "AddMremapHook(%p)", hook);
+  return mremap_hooks_.Add(hook);
+}
+
+extern "C"
+int MallocHook_RemoveMremapHook(MallocHook_MremapHook hook) {
+  RAW_VLOG(10, "RemoveMremapHook(%p)", hook);
+  return mremap_hooks_.Remove(hook);
+}
+
+extern "C"
+int MallocHook_AddPreSbrkHook(MallocHook_PreSbrkHook hook) {
+  RAW_VLOG(10, "AddPreSbrkHook(%p)", hook);
+  return presbrk_hooks_.Add(hook);
+}
+
+extern "C"
+int MallocHook_RemovePreSbrkHook(MallocHook_PreSbrkHook hook) {
+  RAW_VLOG(10, "RemovePreSbrkHook(%p)", hook);
+  return presbrk_hooks_.Remove(hook);
+}
+
+extern "C"
+int MallocHook_AddSbrkHook(MallocHook_SbrkHook hook) {
+  RAW_VLOG(10, "AddSbrkHook(%p)", hook);
+  return sbrk_hooks_.Add(hook);
+}
+
+extern "C"
+int MallocHook_RemoveSbrkHook(MallocHook_SbrkHook hook) {
+  RAW_VLOG(10, "RemoveSbrkHook(%p)", hook);
+  return sbrk_hooks_.Remove(hook);
+}
+
+// The code below is DEPRECATED.
+extern "C"
 MallocHook_NewHook MallocHook_SetNewHook(MallocHook_NewHook hook) {
+  RAW_VLOG(10, "SetNewHook(%p)", hook);
   return new_hook_.Exchange(hook);
 }
 
 extern "C"
 MallocHook_DeleteHook MallocHook_SetDeleteHook(MallocHook_DeleteHook hook) {
+  RAW_VLOG(10, "SetDeleteHook(%p)", hook);
   return delete_hook_.Exchange(hook);
 }
 
 extern "C"
 MallocHook_PreMmapHook MallocHook_SetPreMmapHook(MallocHook_PreMmapHook hook) {
+  RAW_VLOG(10, "SetPreMmapHook(%p)", hook);
   return premmap_hook_.Exchange(hook);
 }
 
 extern "C"
 MallocHook_MmapHook MallocHook_SetMmapHook(MallocHook_MmapHook hook) {
+  RAW_VLOG(10, "SetMmapHook(%p)", hook);
   return mmap_hook_.Exchange(hook);
 }
 
 extern "C"
 MallocHook_MunmapHook MallocHook_SetMunmapHook(MallocHook_MunmapHook hook) {
+  RAW_VLOG(10, "SetMunmapHook(%p)", hook);
   return munmap_hook_.Exchange(hook);
 }
 
 extern "C"
 MallocHook_MremapHook MallocHook_SetMremapHook(MallocHook_MremapHook hook) {
+  RAW_VLOG(10, "SetMremapHook(%p)", hook);
   return mremap_hook_.Exchange(hook);
 }
 
 extern "C"
 MallocHook_PreSbrkHook MallocHook_SetPreSbrkHook(MallocHook_PreSbrkHook hook) {
+  RAW_VLOG(10, "SetPreSbrkHook(%p)", hook);
   return presbrk_hook_.Exchange(hook);
 }
 
 extern "C"
 MallocHook_SbrkHook MallocHook_SetSbrkHook(MallocHook_SbrkHook hook) {
+  RAW_VLOG(10, "SetSbrkHook(%p)", hook);
   return sbrk_hook_.Exchange(hook);
 }
+// End of DEPRECATED code section.
 
+// Note: embedding the function calls inside the traversal of HookList would be
+// very confusing, as it is legal for a hook to remove itself and add other
+// hooks.  Doing traversal first, and then calling the hooks ensures we only
+// call the hooks registered at the start.
+#define INVOKE_HOOKS(HookType, hook_list, args) do {                    \
+    HookType hooks[kHookListMaxValues];                                 \
+    int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues);      \
+    for (int i = 0; i < num_hooks; ++i) {                               \
+      (*hooks[i])args;                                                  \
+    }                                                                   \
+  } while (0)
 
-// The definitions of weak default malloc hooks (New, MMap, and Sbrk)
-// that self deinstall on their first call.  This is entirely for
-// efficiency: the default version of these functions will be called a
-// maximum of one time.  If these functions were a no-op instead, they'd
-// be called every time, costing an extra function call per malloc.
-//
-// However, this 'delete self' isn't safe in general -- it's possible
-// that this function will be called via a daisy chain.  That is,
-// someone else might do
-//    old_hook = MallocHook::SetNewHook(&myhook);
-//    void myhook(void* ptr, size_t size) {
-//       do_my_stuff();
-//       old_hook(ptr, size);   // daisy-chain the hooks
-//    }
-// If old_hook is InitialMallocHook_New(), then this is broken code! --
-// after the first run it'll deregister not only InitialMallocHook_New()
-// but also myhook.  To protect against that, InitialMallocHook_New()
-// makes sure it's the 'top-level' hook before doing the deregistration.
-// This means the daisy-chain case will be less efficient because the
-// hook will be called, and do an if check, for every new.  Alas.
-// TODO(csilvers): add support for removing a hook from the middle of a chain.
+// There should only be one replacement. Return the result of the first
+// one, or false if there is none.
+#define INVOKE_REPLACEMENT(HookType, hook_list, args) do {              \
+    HookType hooks[kHookListMaxValues];                                 \
+    int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues);      \
+    return (num_hooks > 0 && (*hooks[0])args);                          \
+  } while (0)
 
-void InitialMallocHook_New(const void* ptr, size_t size) {
-   if (MallocHook::GetNewHook() == &InitialMallocHook_New)
-     MallocHook::SetNewHook(NULL);
+
+void MallocHook::InvokeNewHookSlow(const void* p, size_t s) {
+  INVOKE_HOOKS(NewHook, new_hooks_, (p, s));
 }
 
-void InitialMallocHook_PreMMap(const void* start,
-                               size_t size,
-                               int protection,
-                               int flags,
-                               int fd,
-                               off_t offset) {
-  if (MallocHook::GetPreMmapHook() == &InitialMallocHook_PreMMap)
-    MallocHook::SetPreMmapHook(NULL);
+void MallocHook::InvokeDeleteHookSlow(const void* p) {
+  INVOKE_HOOKS(DeleteHook, delete_hooks_, (p));
 }
 
-void InitialMallocHook_MMap(const void* result,
-                            const void* start,
-                            size_t size,
-                            int protection,
-                            int flags,
-                            int fd,
-                            off_t offset) {
-  if (MallocHook::GetMmapHook() == &InitialMallocHook_MMap)
-    MallocHook::SetMmapHook(NULL);
+void MallocHook::InvokePreMmapHookSlow(const void* start,
+                                       size_t size,
+                                       int protection,
+                                       int flags,
+                                       int fd,
+                                       off_t offset) {
+  INVOKE_HOOKS(PreMmapHook, premmap_hooks_, (start, size, protection, flags, fd,
+                                            offset));
 }
 
-void InitialMallocHook_PreSbrk(std::ptrdiff_t increment) {
-  if (MallocHook::GetPreSbrkHook() == &InitialMallocHook_PreSbrk)
-    MallocHook::SetPreSbrkHook(NULL);
+void MallocHook::InvokeMmapHookSlow(const void* result,
+                                    const void* start,
+                                    size_t size,
+                                    int protection,
+                                    int flags,
+                                    int fd,
+                                    off_t offset) {
+  INVOKE_HOOKS(MmapHook, mmap_hooks_, (result, start, size, protection, flags,
+                                       fd, offset));
 }
 
-void InitialMallocHook_Sbrk(const void* result, std::ptrdiff_t increment) {
-  if (MallocHook::GetSbrkHook() == &InitialMallocHook_Sbrk)
-    MallocHook::SetSbrkHook(NULL);
+bool MallocHook::InvokeMmapReplacementSlow(const void* start,
+                                           size_t size,
+                                           int protection,
+                                           int flags,
+                                           int fd,
+                                           off_t offset,
+                                           void** result) {
+  INVOKE_REPLACEMENT(MmapReplacement, mmap_replacement_,
+                      (start, size, protection, flags, fd, offset, result));
 }
 
+void MallocHook::InvokeMunmapHookSlow(const void* p, size_t s) {
+  INVOKE_HOOKS(MunmapHook, munmap_hooks_, (p, s));
+}
+
+bool MallocHook::InvokeMunmapReplacementSlow(const void* p,
+                                             size_t s,
+                                             int* result) {
+  INVOKE_REPLACEMENT(MunmapReplacement, munmap_replacement_, (p, s, result));
+}
+
+void MallocHook::InvokeMremapHookSlow(const void* result,
+                                      const void* old_addr,
+                                      size_t old_size,
+                                      size_t new_size,
+                                      int flags,
+                                      const void* new_addr) {
+  INVOKE_HOOKS(MremapHook, mremap_hooks_, (result, old_addr, old_size, new_size,
+                                           flags, new_addr));
+}
+
+void MallocHook::InvokePreSbrkHookSlow(std::ptrdiff_t increment) {
+  INVOKE_HOOKS(PreSbrkHook, presbrk_hooks_, (increment));
+}
+
+void MallocHook::InvokeSbrkHookSlow(const void* result, std::ptrdiff_t increment) {
+  INVOKE_HOOKS(SbrkHook, sbrk_hooks_, (result, increment));
+}
+
+#undef INVOKE_HOOKS
+
 DEFINE_ATTRIBUTE_SECTION_VARS(google_malloc);
 DECLARE_ATTRIBUTE_SECTION_VARS(google_malloc);
   // actual functions are in debugallocation.cc or tcmalloc.cc
@@ -398,7 +747,8 @@
     }
 
     result = (void *)syscall(SYS_mmap2, 
-                             start, length, prot, flags, fd, offset / pagesize);
+                       start, length, prot, flags, fd,
+                       (off_t) (offset / pagesize));
     if (result != MAP_FAILED || errno != ENOSYS)  goto out;
 
     // We don't have mmap2() after all - don't bother trying it in future
@@ -456,7 +806,11 @@
 extern "C" void* mmap64(void *start, size_t length, int prot, int flags,
                         int fd, __off64_t offset) __THROW {
   MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset);
-  void *result = do_mmap64(start, length, prot, flags, fd, offset);
+  void *result;
+  if (!MallocHook::InvokeMmapReplacement(
+          start, length, prot, flags, fd, offset, &result)) {
+    result = do_mmap64(start, length, prot, flags, fd, offset);
+  }
   MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset);
   return result;
 }
@@ -466,8 +820,12 @@
 extern "C" void* mmap(void *start, size_t length, int prot, int flags,
                       int fd, off_t offset) __THROW {
   MallocHook::InvokePreMmapHook(start, length, prot, flags, fd, offset);
-  void *result = do_mmap64(start, length, prot, flags, fd,
-                           static_cast<size_t>(offset)); // avoid sign extension
+  void *result;
+  if (!MallocHook::InvokeMmapReplacement(
+          start, length, prot, flags, fd, offset, &result)) {
+    result = do_mmap64(start, length, prot, flags, fd,
+                       static_cast<size_t>(offset)); // avoid sign extension
+  }
   MallocHook::InvokeMmapHook(result, start, length, prot, flags, fd, offset);
   return result;
 }
@@ -476,7 +834,11 @@
 
 extern "C" int munmap(void* start, size_t length) __THROW {
   MallocHook::InvokeMunmapHook(start, length);
-  return syscall(SYS_munmap, start, length);
+  int result;
+  if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) {
+    result = syscall(SYS_munmap, start, length);
+  }
+  return result;
 }
 
 extern "C" void* mremap(void* old_addr, size_t old_size, size_t new_size,
@@ -503,11 +865,20 @@
 
 /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot,
                                          int flags, int fd, off_t offset) {
-  return do_mmap64(start, length, prot, flags, fd, offset);
+  void* result;
+  if (!MallocHook::InvokeMmapReplacement(
+          start, length, prot, flags, fd, offset, &result)) {
+    result = do_mmap64(start, length, prot, flags, fd, offset);
+  }
+  return result;
 }
 
 /*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) {
-  return sys_munmap(start, length);
+  int result;
+  if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) {
+    result = sys_munmap(start, length);
+  }
+  return result;
 }
 
 #else   // defined(__linux) &&
@@ -515,11 +886,20 @@
 
 /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot,
                                          int flags, int fd, off_t offset) {
-  return mmap(start, length, prot, flags, fd, offset);
+  void* result;
+  if (!MallocHook::InvokeMmapReplacement(
+          start, length, prot, flags, fd, offset, &result)) {
+    result = mmap(start, length, prot, flags, fd, offset);
+  }
+  return result;
 }
 
 /*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) {
-  return munmap(start, length);
+  int result;
+  if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) {
+    result = munmap(start, length);
+  }
+  return result;
 }
 
 #endif  // defined(__linux) &&