Hook memory tracing into Windows heap shim.

base/trace_event/malloc_dump_provider.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/trace_event/malloc_dump_provider.h"
 
 #include <stddef.h>
 
 #include "base/allocator/allocator_extension.h"
 #include "base/allocator/allocator_shim.h"
 #include "base/allocator/features.h"
 #include "base/trace_event/heap_profiler_allocation_context.h"
 #include "base/trace_event/heap_profiler_allocation_context_tracker.h"
 #include "base/trace_event/heap_profiler_allocation_register.h"
 #include "base/trace_event/heap_profiler_heap_dump_writer.h"
 #include "base/trace_event/process_memory_dump.h"
 #include "base/trace_event/trace_event_argument.h"
 #include "build/build_config.h"
 
 #if defined(OS_MACOSX)
 #include <malloc/malloc.h>
 #else
 #include <malloc.h>
 #endif
+#if defined(OS_WIN)
+#include <windows.h>
+#endif
 
 namespace base {
 namespace trace_event {
 
+namespace {
 #if BUILDFLAG(USE_EXPERIMENTAL_ALLOCATOR_SHIM)
-namespace {
 
 using allocator::AllocatorDispatch;
 
 void* HookAlloc(const AllocatorDispatch* self, size_t size) {
   const AllocatorDispatch* const next = self->next;
   void* ptr = next->alloc_function(next, size);
   if (ptr)
     MallocDumpProvider::GetInstance()->InsertAllocation(ptr, size);
   return ptr;
 }
@@ skipping 33 matching lines @@
 }
 
 AllocatorDispatch g_allocator_hooks = {
     &HookAlloc,         /* alloc_function */
     &HookZeroInitAlloc, /* alloc_zero_initialized_function */
     &HookllocAligned,   /* alloc_aligned_function */
     &HookRealloc,       /* realloc_function */
     &HookFree,          /* free_function */
     nullptr,            /* next */
 };
+#endif  // BUILDFLAG(USE_EXPERIMENTAL_ALLOCATOR_SHIM)
 
+#if defined(OS_WIN)
+// A structure containing some information about a given heap.
+struct WinHeapInfo {
+  HANDLE heap_id;
+  size_t committed_size;
+  size_t uncommitted_size;
+  size_t allocated_size;
+  size_t block_count;
+};
+
+bool GetHeapInformation(WinHeapInfo* heap_info,
+                        const std::set<void*>& block_to_skip) {
+  CHECK(::HeapLock(heap_info->heap_id) == TRUE);
+  PROCESS_HEAP_ENTRY heap_entry;
+  heap_entry.lpData = nullptr;
+  // Walk over all the entries in this heap.
+  while (::HeapWalk(heap_info->heap_id, &heap_entry) != FALSE) {
+    if (block_to_skip.count(heap_entry.lpData) == 1)
+      continue;
+    if ((heap_entry.wFlags & PROCESS_HEAP_ENTRY_BUSY) != 0) {
+      heap_info->allocated_size += heap_entry.cbData;
+      heap_info->block_count++;
+    } else if ((heap_entry.wFlags & PROCESS_HEAP_REGION) != 0) {
+      heap_info->committed_size += heap_entry.Region.dwCommittedSize;
+      heap_info->uncommitted_size += heap_entry.Region.dwUnCommittedSize;
+    }
+  }
+  CHECK(::HeapUnlock(heap_info->heap_id) == TRUE);
+  return true;
+}
+
+void WinHeapMemoryDumpImpl(WinHeapInfo* all_heap_info) {
+// This method might be flaky for 2 reasons:
+//   - GetProcessHeaps is racy by design. It returns a snapshot of the
+//     available heaps, but there's no guarantee that that snapshot remains
+//     valid. If a heap disappears between GetProcessHeaps() and HeapWalk()
+//     then chaos should be assumed. This flakyness is acceptable for tracing.
+//   - The MSDN page for HeapLock says: "If the HeapLock function is called on
+//     a heap created with the HEAP_NO_SERIALIZATION flag, the results are
+//     undefined."
+#if defined(SYZYASAN)
+  if (base::debug::IsBinaryInstrumented())
+    return;
+#endif
+
+  // Retrieves the number of heaps in the current process.
+  DWORD number_of_heaps = ::GetProcessHeaps(0, NULL);
+
+  // Try to retrieve a handle to all the heaps owned by this process. Returns
+  // false if the number of heaps has changed.
+  //
+  // This is inherently racy as is, but it's not something that we observe a lot
+  // in Chrome, the heaps tend to be created at startup only.
+  std::unique_ptr<HANDLE[]> all_heaps(new HANDLE[number_of_heaps]);
+  if (::GetProcessHeaps(number_of_heaps, all_heaps.get()) != number_of_heaps)
+    return;
+
+  // Skip the pointer to the heap array to avoid accounting the memory used by
+  // this dump provider.
+  std::set<void*> block_to_skip;
+  block_to_skip.insert(all_heaps.get());
+
+  // Retrieves some metrics about each heap.
+  for (size_t i = 0; i < number_of_heaps; ++i) {

[Primiano Tucci (use gerrit), 2016/08/25 16:49:07]

note for the future: I've no idea what these multiple heaps are (who creates
them? why there isn't just one) but if you feel is worth reporting them
individually, you can just create more entries and the UI will auto add them up.
In any case I wouldn't bother doing that in this CL.

[Sigurður Ásgeirsson, 2016/08/25 17:35:52]

Added a lengthy comment at the top of the function.

I think this is fine as-is and it matches the current reporting, let's see
whether anyone complains.

To answer your question, the Windows heap API allows for an arbitrary number of
heaps and supplies operations to create and destroy them. Any binary (DLL or
EXE) that links statically against a copy of the CRT (this includes the CRT
itself), will historically create their own private heap. When such a binary is
unloaded, it'll delete its heap and thus scuttle all the crap it allocated.
To confuse things, there will be multiple versions of the CRT, and even
different binaries that dynamically link the CRT will load different ones, each
(historically) with its own heap.

+    WinHeapInfo heap_info = {0};
+    heap_info.heap_id = all_heaps[i];
+    GetHeapInformation(&heap_info, block_to_skip);
+
+    all_heap_info->allocated_size += heap_info.allocated_size;
+    all_heap_info->committed_size += heap_info.committed_size;
+    all_heap_info->uncommitted_size += heap_info.uncommitted_size;
+    all_heap_info->block_count += heap_info.block_count;
+  }
+}
+#endif  // defined(OS_WIN)
 }  // namespace
-#endif  // BUILDFLAG(USE_EXPERIMENTAL_ALLOCATOR_SHIM)
 
 // static
 const char MallocDumpProvider::kAllocatedObjects[] = "malloc/allocated_objects";
 
 // static
 MallocDumpProvider* MallocDumpProvider::GetInstance() {
   return Singleton<MallocDumpProvider,
                    LeakySingletonTraits<MallocDumpProvider>>::get();
 }
 
 MallocDumpProvider::MallocDumpProvider()
     : heap_profiler_enabled_(false), tid_dumping_heap_(kInvalidThreadId) {}
 
 MallocDumpProvider::~MallocDumpProvider() {}
 
 // Called at trace dump point time. Creates a snapshot the memory counters for
 // the current process.
 bool MallocDumpProvider::OnMemoryDump(const MemoryDumpArgs& args,
                                       ProcessMemoryDump* pmd) {
   size_t total_virtual_size = 0;
   size_t resident_size = 0;
   size_t allocated_objects_size = 0;
+  size_t allocated_objects_count = 0;
 #if defined(USE_TCMALLOC)
   bool res =
       allocator::GetNumericProperty("generic.heap_size", &total_virtual_size);
   DCHECK(res);
   res = allocator::GetNumericProperty("generic.total_physical_bytes",
                                       &resident_size);
   DCHECK(res);
   res = allocator::GetNumericProperty("generic.current_allocated_bytes",
                                       &allocated_objects_size);
   DCHECK(res);
 #elif defined(OS_MACOSX) || defined(OS_IOS)
   malloc_statistics_t stats = {0};
   malloc_zone_statistics(nullptr, &stats);
   total_virtual_size = stats.size_allocated;
   allocated_objects_size = stats.size_in_use;
 
   // The resident size is approximated to the max size in use, which would count
   // the total size of all regions other than the free bytes at the end of each
   // region. In each allocation region the allocations are rounded off to a
   // fixed quantum, so the excess region will not be resident.
   // See crrev.com/1531463004 for detailed explanation.
   resident_size = stats.max_size_in_use;
+#elif defined(OS_WIN)
+  WinHeapInfo all_heap_info = {};
+  WinHeapMemoryDumpImpl(&all_heap_info);
+  total_virtual_size =
+      all_heap_info.committed_size + all_heap_info.uncommitted_size;
+  // Resident size is approximated with committed heap size. Note that it is
+  // possible to do this with better accuracy on windows by intersecting the
+  // working set with the virtual memory ranges occuipied by the heap. It's not
+  // clear that this is worth it, as it's fairly expensive to do.
+  resident_size = all_heap_info.committed_size;
+  allocated_objects_size = all_heap_info.allocated_size;
+  allocated_objects_count = all_heap_info.block_count;
 #else
   struct mallinfo info = mallinfo();
   DCHECK_GE(info.arena + info.hblkhd, info.uordblks);
 
   // In case of Android's jemalloc |arena| is 0 and the outer pages size is
   // reported by |hblkhd|. In case of dlmalloc the total is given by
   // |arena| + |hblkhd|. For more details see link: http://goo.gl/fMR8lF.
   total_virtual_size = info.arena + info.hblkhd;
   resident_size = info.uordblks;
+  // Total allocated space is given by |uordblks|.

[Primiano Tucci (use gerrit), 2016/08/25 16:49:07]

nit add newline before this line.

[Sigurður Ásgeirsson, 2016/08/25 17:35:52]

Done.

   allocated_objects_size = info.uordblks;
 #endif
 
   MemoryAllocatorDump* outer_dump = pmd->CreateAllocatorDump("malloc");
   outer_dump->AddScalar("virtual_size", MemoryAllocatorDump::kUnitsBytes,
                         total_virtual_size);
   outer_dump->AddScalar(MemoryAllocatorDump::kNameSize,
                         MemoryAllocatorDump::kUnitsBytes, resident_size);
 
-  // Total allocated space is given by |uordblks|.
   MemoryAllocatorDump* inner_dump = pmd->CreateAllocatorDump(kAllocatedObjects);
   inner_dump->AddScalar(MemoryAllocatorDump::kNameSize,
                         MemoryAllocatorDump::kUnitsBytes,
                         allocated_objects_size);
+  if (allocated_objects_count != 0) {
+    inner_dump->AddScalar(MemoryAllocatorDump::kNameSize,
+                          MemoryAllocatorDump::kUnitsObjects,
+                          allocated_objects_count);
+  }
 
   if (resident_size - allocated_objects_size > 0) {
     // Explicitly specify why is extra memory resident. In tcmalloc it accounts
     // for free lists and caches. In mac and ios it accounts for the
     // fragmentation and metadata.
     MemoryAllocatorDump* other_dump =
         pmd->CreateAllocatorDump("malloc/metadata_fragmentation_caches");
     other_dump->AddScalar(MemoryAllocatorDump::kNameSize,
                           MemoryAllocatorDump::kUnitsBytes,
                           resident_size - allocated_objects_size);
@@ skipping 84 matching lines @@
       tid_dumping_heap_ == PlatformThread::CurrentId())
     return;
   AutoLock lock(allocation_register_lock_);
   if (!allocation_register_)
     return;
   allocation_register_->Remove(address);
 }
 
 }  // namespace trace_event
 }  // namespace base