mac: Hook up allocator shim during app startup.

base/allocator/allocator_interception_mac.mm

 // Copyright 2017 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // This file contains all the logic necessary to intercept allocations on
 // macOS. "malloc zones" are an abstraction that allows the process to intercept
 // all malloc-related functions.  There is no good mechanism [short of
 // interposition] to determine new malloc zones are added, so there's no clean
 // mechanism to intercept all malloc zones. This file contains logic to
 // intercept the default and purgeable zones, which always exist. A cursory
 // review of Chrome seems to imply that non-default zones are almost never used.
 //
 // This file also contains logic to intercept Core Foundation and Objective-C
 // allocations. The implementations forward to the default malloc zone, so the
 // only reason to intercept these calls is to re-label OOM crashes with slightly
 // more details.
 
 #include "base/allocator/allocator_interception_mac.h"
 
 #include <CoreFoundation/CoreFoundation.h>
 #import <Foundation/Foundation.h>
 #include <errno.h>
 #include <mach/mach.h>
 #include <mach/mach_vm.h>
 #import <objc/runtime.h>
 #include <stddef.h>
 
 #include <new>
 
+#include "base/allocator/allocator_shim.h"
+#include "base/allocator/features.h"
 #include "base/logging.h"
 #include "base/mac/mac_util.h"
 #include "base/mac/mach_logging.h"
 #include "base/process/memory.h"
 #include "base/scoped_clear_errno.h"
 #include "build/build_config.h"
 #include "third_party/apple_apsl/CFBase.h"
 
 namespace base {
 namespace allocator {
@@ skipping 399 matching lines @@
   // where memory pressure is not a concern. Finally, the only public interface
   // to batch_malloc is malloc_zone_batch_malloc, which is specific to the
   // system's malloc implementation. It's unlikely that anyone's even heard of
   // it.
 
   // === C++ operator new ===
 
   // Yes, operator new does call through to malloc, but this will catch failures
   // that our imperfect handling of malloc cannot.
 
   std::set_new_handler(oom_killer_new);

[Primiano Tucci (use gerrit), 2017/02/07 11:58:08]

(read this comment after the one below)
Honestly I think this is the misplaced thing that mislead you. I know that you
didn't touch this, but I think that this call should really happen on 
memory_mac.mm -> EnableTerminationOnOutOfMemory(), similarly to what happens in
Linux.
In other words, I think it would be cleaner if   memory_mac.mm  looked like
this:
EnableTerminationOnOutOfMemory() {
  // Step 1, enable OOM killer on C++ failures
  std::set_new_handler(oom_killer_new);

  // Step 2: enabled OOM killer on C-malloc failures for the default zone (if we
have a shim) 
  #if BUILDFLAG(USE_EXPERIMENTAL_ALLOCATOR_SHIM)
    allocator::SetCallNewHandlerOnMallocFailure(true);
  #endif

  // Step 3: enable OOM killer on all other malloc zones (or just "all" without
"other" if shim is disabled)
  allocator::InterceptAllocationsMac();   // (and this is the reason why in the
previous CL I was saying that this has a bad name now.. but we'll deal with
naming later)
}

Other than being cleaner, as a side effect this will look more consistent with
memory_linux.cc EnableTerminationOnOutOfMemory()

[erikchen, 2017/02/09 23:49:04]

Done.

 
+#if BUILDFLAG(USE_EXPERIMENTAL_ALLOCATOR_SHIM)
+  allocator::SetCallNewHandlerOnMallocFailure(true);

[Primiano Tucci (use gerrit), 2017/02/07 11:58:08]

why here and not in memory_mac.mm -> EnableTerminationOnOutOfMemory() ?
That would be consistent with what we do today in memory_linux.cc.

this is creating a logic dependency cycle. So far, allocator_interception_mac.mm
has been  a low level building block, which  knows how to deal with malloc
replacement stuff *
allocator_interception_mac.mm has no idea what the shim layer is. the shim layer
uses allocator_interception_mac.mm to implement malloc shimming on OSX.
Now with this line you are creating the following situation:
- the shim layer knows about allocator_interception_mac.mm and uses it to insert
itself in the malloc zone.
- allocator_interception_mac.mm knows about the shim and uses it to turn on the
suicide-on-oom flag
So if you do this, and look at the full picture, the end-to-end stacktrace will
look as follows (i hope that rietveld won't screw up my indentation ascii art
with non-monospace fonts):

content_main_runner.cc               ContentMainRunnerImpl::Initialize()
   allocator_shim.cc                 InitializeAllocatorShim()
     allocator_interception_mac.mm   ReplaceFunctionsForDefaultZone()
       allocator_shim.cc             SetCallNewHandlerOnMallocFailure()
   memory_mac.mm                     EnableTerminationOnOutOfMemory()
     allocator_interception_mac.mm   InterceptAllocationsMac()

(note the allocator_shim -> interception_mac -> allocator_shim sequence)

Instead what I was proposing is:

content_main_runner.cc               ContentMainRunnerImpl::Initialize()
   allocator_shim.cc                 InitializeAllocatorShim()
     allocator_interception_mac.mm   ReplaceFunctionsForDefaultZone()
   memory_mac.mm                     EnableTerminationOnOutOfMemory()
     allocator_interception_mac.mm   InterceptAllocationsMac()
     allocator_shim.cc               SetCallNewHandlerOnMallocFailure()

which looks a bit more cleaner to me.

* There is indeed a non-ideal thing in all this, the fact that the
suicide-on-OOM for the non-default zones happens in
allocator_interception_mac.mm while the one for the default zone happens in the
shim. but, as we discussed in the past, we have to live with it and the only
other option is refactoring all the shim to introduce the notions of zones
(which is probably going to create more headaches).

[erikchen, 2017/02/09 23:49:04]

Done.

+#endif
+
 #ifndef ADDRESS_SANITIZER
   // === Core Foundation CFAllocators ===
 
   // This will not catch allocation done by custom allocators, but will catch
   // all allocation done by system-provided ones.
 
   CHECK(!g_old_cfallocator_system_default && !g_old_cfallocator_malloc &&
         !g_old_cfallocator_malloc_zone)
       << "Old allocators unexpectedly non-null";
 
@@ skipping 41 matching lines @@
   g_old_allocWithZone =
       reinterpret_cast<allocWithZone_t>(method_getImplementation(orig_method));
   CHECK(g_old_allocWithZone)
       << "Failed to get allocWithZone allocation function.";
   method_setImplementation(orig_method,
                            reinterpret_cast<IMP>(oom_killer_allocWithZone));
 }
 
 }  // namespace allocator
 }  // namespace base