Fix HEAP_PROFILE_MMAP in google-perftools.

third_party/tcmalloc/chromium/src/heap-profile-table.cc

 // Copyright (c) 2006, Google Inc.
 // All rights reserved.
 // 
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 // 
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 41 matching lines @@
 #endif
 #include <errno.h>
 #include <stdarg.h>
 #include <string>
 #include <map>
 #include <algorithm>  // for sort(), equal(), and copy()
 
 #include "heap-profile-table.h"
 
 #include "base/logging.h"
+#include "memory_region_map.h"
 #include "raw_printer.h"
 #include "symbolize.h"
 #include <google/stacktrace.h>
 #include <google/malloc_hook.h>
 #include "base/commandlineflags.h"
 #include "base/logging.h"    // for the RawFD I/O commands
 #include "base/sysinfo.h"
 
 using std::sort;
 using std::equal;
@@ skipping 19 matching lines @@
 // header of the dumped heap profile
 static const char kProfileHeader[] = "heap profile: ";
 static const char kProcSelfMapsHeader[] = "\nMAPPED_LIBRARIES:\n";
 
 //----------------------------------------------------------------------
 
 const char HeapProfileTable::kFileExt[] = ".heap";
 
 //----------------------------------------------------------------------
 
-static const int kHashTableSize = 179999;   // Size for table_.
+// Size for alloc_table_ and mmap_table_.
+static const int kHashTableSize = 179999;
 /*static*/ const int HeapProfileTable::kMaxStackDepth;
 
 //----------------------------------------------------------------------
 
 // We strip out different number of stack frames in debug mode
 // because less inlining happens in that case
 #ifdef NDEBUG
 static const int kStripFrames = 2;
 #else
 static const int kStripFrames = 3;
 #endif
 
 // For sorting Stats or Buckets by in-use space
 static bool ByAllocatedSpace(HeapProfileTable::Stats* a,
                              HeapProfileTable::Stats* b) {
   // Return true iff "a" has more allocated space than "b"
   return (a->alloc_size - a->free_size) > (b->alloc_size - b->free_size);
 }
 
 //----------------------------------------------------------------------
 
 HeapProfileTable::HeapProfileTable(Allocator alloc, DeAllocator dealloc)
     : alloc_(alloc), dealloc_(dealloc) {
-  // Make the table
-  const int table_bytes = kHashTableSize * sizeof(*table_);
-  table_ = reinterpret_cast<Bucket**>(alloc_(table_bytes));
-  memset(table_, 0, table_bytes);
-  // Make allocation map
-  allocation_ =
-    new(alloc_(sizeof(AllocationMap))) AllocationMap(alloc_, dealloc_);
-  // init the rest:
+  // Initialize the overall profile stats.
   memset(&total_, 0, sizeof(total_));
-  num_buckets_ = 0;
+
+  // Make the malloc table.
+  const int alloc_table_bytes = kHashTableSize * sizeof(*alloc_table_);
+  alloc_table_ = reinterpret_cast<Bucket**>(alloc_(alloc_table_bytes));
+  memset(alloc_table_, 0, alloc_table_bytes);
+  num_alloc_buckets_ = 0;
+
+  // Initialize the mmap table.
+  mmap_table_ = NULL;
+  num_available_mmap_buckets_ = 0;
+
+  // Make malloc and mmap allocation maps.
+  alloc_address_map_ =
+      new(alloc_(sizeof(AllocationMap))) AllocationMap(alloc_, dealloc_);
+  mmap_address_map_ = NULL;
 }
 
 HeapProfileTable::~HeapProfileTable() {
-  // free allocation map
-  allocation_->~AllocationMap();
-  dealloc_(allocation_);
-  allocation_ = NULL;
-  // free hash table
-  for (int b = 0; b < kHashTableSize; b++) {
-    for (Bucket* x = table_[b]; x != 0; /**/) {
-      Bucket* b = x;
-      x = x->next;
-      dealloc_(b->stack);
-      dealloc_(b);
-    }
-  }
-  dealloc_(table_);
-  table_ = NULL;
+  DeallocateBucketTable(alloc_table_);
+  alloc_table_ = NULL;
+  DeallocateBucketTable(mmap_table_);
+  mmap_table_ = NULL;
+  DeallocateAllocationMap(alloc_address_map_);
+  alloc_address_map_ = NULL;
+  DeallocateAllocationMap(mmap_address_map_);
+  mmap_address_map_ = NULL;
 }
 
-HeapProfileTable::Bucket* HeapProfileTable::GetBucket(int depth,
-                                                      const void* const key[]) {
+void HeapProfileTable::DeallocateAllocationMap(AllocationMap* allocation) {
+  if (allocation != NULL) {
+    alloc_address_map_->~AllocationMap();
+    dealloc_(allocation);
+  }
+}
+
+void HeapProfileTable::DeallocateBucketTable(Bucket** table) {
+  if (table != NULL) {
+    for (int b = 0; b < kHashTableSize; b++) {
+      for (Bucket* x = table[b]; x != 0; /**/) {
+        Bucket* b = x;
+        x = x->next;
+        dealloc_(b->stack);
+        dealloc_(b);
+      }
+    }
+    dealloc_(table);
+  }
+}
+
+HeapProfileTable::Bucket* HeapProfileTable::GetBucket(
+    int depth, const void* const key[], Bucket** table,
+    int* bucket_count) {
   // Make hash-value
   uintptr_t h = 0;
   for (int i = 0; i < depth; i++) {
     h += reinterpret_cast<uintptr_t>(key[i]);
     h += h << 10;
     h ^= h >> 6;
   }
   h += h << 3;
   h ^= h >> 11;
 
   // Lookup stack trace in table
   unsigned int buck = ((unsigned int) h) % kHashTableSize;
-  for (Bucket* b = table_[buck]; b != 0; b = b->next) {
+  for (Bucket* b = table[buck]; b != 0; b = b->next) {
     if ((b->hash == h) &&
         (b->depth == depth) &&
         equal(key, key + depth, b->stack)) {
       return b;
     }
   }
 
   // Create new bucket
   const size_t key_size = sizeof(key[0]) * depth;
   const void** kcopy = reinterpret_cast<const void**>(alloc_(key_size));
   copy(key, key + depth, kcopy);
   Bucket* b = reinterpret_cast<Bucket*>(alloc_(sizeof(Bucket)));
   memset(b, 0, sizeof(*b));
   b->hash  = h;
   b->depth = depth;
   b->stack = kcopy;
-  b->next  = table_[buck];
-  table_[buck] = b;
-  num_buckets_++;
+  b->next  = table[buck];
+  table[buck] = b;
+  if (bucket_count != NULL) {
+    ++(*bucket_count);
+  }
   return b;
 }
 
 void HeapProfileTable::RecordAlloc(const void* ptr, size_t bytes,
                                    int skip_count) {
   void* key[kMaxStackDepth];
   int depth = MallocHook::GetCallerStackTrace(
     key, kMaxStackDepth, kStripFrames + skip_count + 1);
   RecordAllocWithStack(ptr, bytes, depth, key);
 }
 
 void HeapProfileTable::RecordAllocWithStack(
     const void* ptr, size_t bytes, int stack_depth,
     const void* const call_stack[]) {
-  Bucket* b = GetBucket(stack_depth, call_stack);
+  Bucket* b = GetBucket(stack_depth, call_stack, alloc_table_,
+                        &num_alloc_buckets_);
   b->allocs++;
   b->alloc_size += bytes;
   total_.allocs++;
   total_.alloc_size += bytes;
 
   AllocValue v;
   v.set_bucket(b);  // also did set_live(false); set_ignore(false)
   v.bytes = bytes;
-  allocation_->Insert(ptr, v);
+  alloc_address_map_->Insert(ptr, v);
 }
 
 void HeapProfileTable::RecordFree(const void* ptr) {
   AllocValue v;
-  if (allocation_->FindAndRemove(ptr, &v)) {
+  if (alloc_address_map_->FindAndRemove(ptr, &v)) {
     Bucket* b = v.bucket();
     b->frees++;
     b->free_size += v.bytes;
     total_.frees++;
     total_.free_size += v.bytes;
   }
 }
 
 bool HeapProfileTable::FindAlloc(const void* ptr, size_t* object_size) const {
-  const AllocValue* alloc_value = allocation_->Find(ptr);
+  const AllocValue* alloc_value = alloc_address_map_->Find(ptr);
   if (alloc_value != NULL) *object_size = alloc_value->bytes;
   return alloc_value != NULL;
 }
 
 bool HeapProfileTable::FindAllocDetails(const void* ptr,
                                         AllocInfo* info) const {
-  const AllocValue* alloc_value = allocation_->Find(ptr);
+  const AllocValue* alloc_value = alloc_address_map_->Find(ptr);
   if (alloc_value != NULL) {
     info->object_size = alloc_value->bytes;
     info->call_stack = alloc_value->bucket()->stack;
     info->stack_depth = alloc_value->bucket()->depth;
   }
   return alloc_value != NULL;
 }
 
 bool HeapProfileTable::FindInsideAlloc(const void* ptr,
                                        size_t max_size,
                                        const void** object_ptr,
                                        size_t* object_size) const {
   const AllocValue* alloc_value =
-    allocation_->FindInside(&AllocValueSize, max_size, ptr, object_ptr);
+    alloc_address_map_->FindInside(&AllocValueSize, max_size, ptr, object_ptr);
   if (alloc_value != NULL) *object_size = alloc_value->bytes;
   return alloc_value != NULL;
 }
 
 bool HeapProfileTable::MarkAsLive(const void* ptr) {
-  AllocValue* alloc = allocation_->FindMutable(ptr);
+  AllocValue* alloc = alloc_address_map_->FindMutable(ptr);
   if (alloc && !alloc->live()) {
     alloc->set_live(true);
     return true;
   }
   return false;
 }
 
 void HeapProfileTable::MarkAsIgnored(const void* ptr) {
-  AllocValue* alloc = allocation_->FindMutable(ptr);
+  AllocValue* alloc = alloc_address_map_->FindMutable(ptr);
   if (alloc) {
     alloc->set_ignore(true);
   }
 }
 
 // We'd be happier using snprintfer, but we don't to reduce dependencies.
 int HeapProfileTable::UnparseBucket(const Bucket& b,
                                     char* buf, int buflen, int bufsize,
                                     const char* extra,
                                     Stats* profile_stats) {
   if (profile_stats != NULL) {
     profile_stats->allocs += b.allocs;
     profile_stats->alloc_size += b.alloc_size;
     profile_stats->frees += b.frees;
     profile_stats->free_size += b.free_size;
   }
-  int printed =
-    snprintf(buf + buflen, bufsize - buflen, "%6d: %8"PRId64" [%6d: %8"PRId64"] @%s",
-             b.allocs - b.frees,
-             b.alloc_size - b.free_size,
-             b.allocs,
-             b.alloc_size,
-             extra);
+  int printed = snprintf(buf + buflen, bufsize - buflen,
+      "%6d: %8"PRId64" [%6d: %8"PRId64"] @%s",
+      b.allocs - b.frees, b.alloc_size - b.free_size,
+      b.allocs, b.alloc_size, extra);
   // If it looks like the snprintf failed, ignore the fact we printed anything
   if (printed < 0 || printed >= bufsize - buflen) return buflen;
   buflen += printed;
   for (int d = 0; d < b.depth; d++) {
     printed = snprintf(buf + buflen, bufsize - buflen, " 0x%08" PRIxPTR,
                        reinterpret_cast<uintptr_t>(b.stack[d]));
     if (printed < 0 || printed >= bufsize - buflen) return buflen;
     buflen += printed;
   }
   printed = snprintf(buf + buflen, bufsize - buflen, "\n");
   if (printed < 0 || printed >= bufsize - buflen) return buflen;
   buflen += printed;
   return buflen;
 }
 
 HeapProfileTable::Bucket**
 HeapProfileTable::MakeSortedBucketList() const {
-  Bucket** list =
-    reinterpret_cast<Bucket**>(alloc_(sizeof(Bucket) * num_buckets_));
+  Bucket** list = reinterpret_cast<Bucket**>(alloc_(sizeof(Bucket) *
+      (num_alloc_buckets_ + num_available_mmap_buckets_)));
+
+  RAW_DCHECK(mmap_table_ != NULL || num_available_mmap_buckets_ == 0, "");
 
   int n = 0;
+
   for (int b = 0; b < kHashTableSize; b++) {
-    for (Bucket* x = table_[b]; x != 0; x = x->next) {
+    for (Bucket* x = alloc_table_[b]; x != 0; x = x->next) {
       list[n++] = x;
     }
   }
-  RAW_DCHECK(n == num_buckets_, "");
+  RAW_DCHECK(n == num_alloc_buckets_, "");
 
-  sort(list, list + num_buckets_, ByAllocatedSpace);
+  if (mmap_table_ != NULL) {
+    for (int b = 0; b < kHashTableSize; b++) {
+      for (Bucket* x = mmap_table_[b]; x != 0; x = x->next) {
+        list[n++] = x;
+      }
+    }
+  }
+  RAW_DCHECK(n == num_alloc_buckets_ + num_available_mmap_buckets_, "");
+
+  sort(list, list + num_alloc_buckets_ + num_available_mmap_buckets_,
+       ByAllocatedSpace);
 
   return list;
 }
 
+void HeapProfileTable::RefreshMMapData() {
+  // Make the table
+  static const int mmap_table_bytes = kHashTableSize * sizeof(*mmap_table_);
+  if (mmap_table_ == NULL) {
+    mmap_table_ = reinterpret_cast<Bucket**>(alloc_(mmap_table_bytes));
+    memset(mmap_table_, 0, mmap_table_bytes);
+  }
+  num_available_mmap_buckets_ = 0;
+
+  ClearMMapData();
+  mmap_address_map_ =
+      new(alloc_(sizeof(AllocationMap))) AllocationMap(alloc_, dealloc_);
+
+  MemoryRegionMap::LockHolder l;
+  for (MemoryRegionMap::RegionIterator r =
+           MemoryRegionMap::BeginRegionLocked();
+       r != MemoryRegionMap::EndRegionLocked(); ++r) {
+    Bucket* b =
+        GetBucket(r->call_stack_depth, r->call_stack, mmap_table_, NULL);
+    if (b->alloc_size == 0) {
+      num_available_mmap_buckets_ += 1;
+    }
+    b->allocs += 1;
+    b->alloc_size += r->end_addr - r->start_addr;
+
+    AllocValue v;
+    v.set_bucket(b);
+    v.bytes = r->end_addr - r->start_addr;
+    mmap_address_map_->Insert(reinterpret_cast<const void*>(r->start_addr), v);
+  }
+}
+
+void HeapProfileTable::ClearMMapData() {
+  if (mmap_address_map_ != NULL) {
+    mmap_address_map_->Iterate(ZeroBucketCountsIterator, this);
+    mmap_address_map_->~AllocationMap();
+    dealloc_(mmap_address_map_);
+    mmap_address_map_ = NULL;
+  }
+}
+
 void HeapProfileTable::IterateOrderedAllocContexts(
     AllocContextIterator callback) const {
   Bucket** list = MakeSortedBucketList();
   AllocContextInfo info;
-  for (int i = 0; i < num_buckets_; ++i) {
+  for (int i = 0; i < num_alloc_buckets_; ++i) {
     *static_cast<Stats*>(&info) = *static_cast<Stats*>(list[i]);
     info.stack_depth = list[i]->depth;
     info.call_stack = list[i]->stack;
     callback(info);
   }
   dealloc_(list);
 }
 
 int HeapProfileTable::FillOrderedProfile(char buf[], int size) const {
   Bucket** list = MakeSortedBucketList();
@@ skipping 11 matching lines @@
   map_length += FillProcSelfMaps(buf + map_length, size - map_length, &dummy);
   RAW_DCHECK(map_length <= size, "");
   char* const map_start = buf + size - map_length;      // move to end
   memmove(map_start, buf, map_length);
   size -= map_length;
 
   Stats stats;
   memset(&stats, 0, sizeof(stats));
   int bucket_length = snprintf(buf, size, "%s", kProfileHeader);
   if (bucket_length < 0 || bucket_length >= size) return 0;
-  bucket_length = UnparseBucket(total_, buf, bucket_length, size,
+  Bucket total_with_mmap(total_);
+  if (mmap_table_ != NULL) {
+    total_with_mmap.alloc_size += MemoryRegionMap::MapSize();
+    total_with_mmap.free_size += MemoryRegionMap::UnmapSize();
+  }
+  bucket_length = UnparseBucket(total_with_mmap, buf, bucket_length, size,
                                 " heapprofile", &stats);
-  for (int i = 0; i < num_buckets_; i++) {
+  for (int i = 0; i < num_alloc_buckets_; i++) {
     bucket_length = UnparseBucket(*list[i], buf, bucket_length, size, "",
                                   &stats);
   }
   RAW_DCHECK(bucket_length < size, "");
 
   dealloc_(list);
 
   RAW_DCHECK(buf + bucket_length <= map_start, "");
   memmove(buf + bucket_length, map_start, map_length);  // close the gap
 
@@ skipping 14 matching lines @@
   memset(&b, 0, sizeof(b));
   b.allocs = 1;
   b.alloc_size = v->bytes;
   b.depth = v->bucket()->depth;
   b.stack = v->bucket()->stack;
   char buf[1024];
   int len = UnparseBucket(b, buf, 0, sizeof(buf), "", args.profile_stats);
   RawWrite(args.fd, buf, len);
 }
 
+inline void HeapProfileTable::ZeroBucketCountsIterator(
+    const void* ptr, AllocValue* v, HeapProfileTable* heap_profile) {
+  Bucket* b = v->bucket();
+  if (b != NULL) {
+    b->allocs = 0;
+    b->alloc_size = 0;
+    b->free_size = 0;
+    b->frees = 0;
+  }
+}
+
 // Callback from NonLiveSnapshot; adds entry to arg->dest
 // if not the entry is not live and is not present in arg->base.
 void HeapProfileTable::AddIfNonLive(const void* ptr, AllocValue* v,
                                     AddNonLiveArgs* arg) {
   if (v->live()) {
     v->set_live(false);
   } else {
     if (arg->base != NULL && arg->base->map_.Find(ptr) != NULL) {
       // Present in arg->base, so do not save
     } else {
@@ skipping 47 matching lines @@
     }
   }
   globfree(&g);
 #else   /* HAVE_GLOB_H */
   RAW_LOG(WARNING, "Unable to remove old heap profiles (can't run glob())");
 #endif
 }
 
 HeapProfileTable::Snapshot* HeapProfileTable::TakeSnapshot() {
   Snapshot* s = new (alloc_(sizeof(Snapshot))) Snapshot(alloc_, dealloc_);
-  allocation_->Iterate(AddToSnapshot, s);
+  alloc_address_map_->Iterate(AddToSnapshot, s);
   return s;
 }
 
 void HeapProfileTable::ReleaseSnapshot(Snapshot* s) {
   s->~Snapshot();
   dealloc_(s);
 }
 
 // Callback from TakeSnapshot; adds a single entry to snapshot
 void HeapProfileTable::AddToSnapshot(const void* ptr, AllocValue* v,
                                      Snapshot* snapshot) {
   snapshot->Add(ptr, *v);
 }
 
 HeapProfileTable::Snapshot* HeapProfileTable::NonLiveSnapshot(
     Snapshot* base) {
   RAW_VLOG(2, "NonLiveSnapshot input: %d %d\n",
            int(total_.allocs - total_.frees),
            int(total_.alloc_size - total_.free_size));
 
   Snapshot* s = new (alloc_(sizeof(Snapshot))) Snapshot(alloc_, dealloc_);
   AddNonLiveArgs args;
   args.dest = s;
   args.base = base;
-  allocation_->Iterate<AddNonLiveArgs*>(AddIfNonLive, &args);
+  alloc_address_map_->Iterate<AddNonLiveArgs*>(AddIfNonLive, &args);
   RAW_VLOG(2, "NonLiveSnapshot output: %d %d\n",
            int(s->total_.allocs - s->total_.frees),
            int(s->total_.alloc_size - s->total_.free_size));
   return s;
 }
 
 // Information kept per unique bucket seen
 struct HeapProfileTable::Snapshot::Entry {
   int count;
   int bytes;
@@ skipping 99 matching lines @@
                                               char* unused) {
   // Perhaps also log the allocation stack trace (unsymbolized)
   // on this line in case somebody finds it useful.
   RAW_LOG(ERROR, "leaked %"PRIuS" byte object %p", v->bytes, ptr);
 }
 
 void HeapProfileTable::Snapshot::ReportIndividualObjects() {
   char unused;
   map_.Iterate(ReportObject, &unused);
 }