Classify memory usage by allocated type in Deep Memory Profiler.

tools/deep_memory_profiler/dmprof

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """The deep heap profiler script for Chrome."""
 
 from datetime import datetime
 import json
 import optparse
 import os
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 
 FIND_RUNTIME_SYMBOLS_PATH = os.path.join(
     os.path.dirname(os.path.abspath(__file__)),
     os.pardir,
     'find_runtime_symbols')
 sys.path.append(FIND_RUNTIME_SYMBOLS_PATH)
 
 from find_runtime_symbols import find_runtime_symbols_list
+from find_runtime_symbols import find_runtime_typeinfo_symbols_list
+from find_runtime_symbols import RuntimeSymbolsInProcess
 from prepare_symbol_info import prepare_symbol_info
-from static_symbols import StaticSymbols
 
 BUCKET_ID = 5
 VIRTUAL = 0
 COMMITTED = 1
 ALLOC_COUNT = 2
 FREE_COUNT = 3
 NULL_REGEX = re.compile('')
 
 POLICIES_JSON_PATH = os.path.join(
     os.path.dirname(os.path.abspath(__file__)),
@@ skipping 37 matching lines @@
 # mmap regions are distincted w/ mmap frames in the pattern column.
 POLICY_DEEP_1 = 'POLICY_DEEP_1'
 
 # POLICY_DEEP_2 DOES include allocation_type columns.
 # mmap regions are distincted w/ the allocation_type column.
 POLICY_DEEP_2 = 'POLICY_DEEP_2'
 
 # POLICY_DEEP_3 is in JSON format.
 POLICY_DEEP_3 = 'POLICY_DEEP_3'
 
+# POLICY_DEEP_3 contains typeinfo.
+POLICY_DEEP_4 = 'POLICY_DEEP_4'
+
 
 class EmptyDumpException(Exception):
   def __init__(self, value):
     self.value = value
   def __str__(self):
     return repr(self.value)
 
 
 class ParsingException(Exception):
   def __init__(self, value):
@@ skipping 18 matching lines @@
 
 class DelayedStaticSymbols(object):
   """Represents static symbol information loaded lazily."""
 
   def __init__(self, prefix, keep=False):
     self.maps_path = prefix + '.maps'
     self.keep = keep
     if keep:
       self.prepared_data_dir = prefix + '.pre'
     self.loaded_static_symbols = None
+    self.loaded_symbols_in_process = None
 
   def get(self):
-    if not self.loaded_static_symbols:
+    if not self.loaded_symbols_in_process:
       if not self.keep:
         self.prepared_data_dir = tempfile.mkdtemp()
       try:
         prepare_symbol_info(self.maps_path, self.prepared_data_dir)
-        self.loaded_static_symbols = StaticSymbols.load(self.prepared_data_dir)
+        self.loaded_symbols_in_process = RuntimeSymbolsInProcess.load(
+            self.prepared_data_dir)
       finally:
         if not self.keep:
           shutil.rmtree(self.prepared_data_dir)
-    return self.loaded_static_symbols
+    return self.loaded_symbols_in_process
 
 
 class Rule(object):
   """Represents one matching rule in a policy file."""
 
-  def __init__(self, name, mmap, stacktrace_pattern):
+  def __init__(self, name, mmap, stacktrace_pattern, typeinfo_pattern=None):
     self.name = name
     self.mmap = mmap
     self.stacktrace_pattern = re.compile(stacktrace_pattern + r'\Z')
+    if typeinfo_pattern:
+      self.typeinfo_pattern = re.compile(typeinfo_pattern + r'\Z')
+    else:
+      self.typeinfo_pattern = None
 
 
 class Policy(object):
   """Represents a policy, a content of a policy file."""
 
   def __init__(self, rules, version, components):
     self.rules = rules
     self.version = version
     self.components = components
 
@@ skipping 11 matching lines @@
 
   Returns:
       A string representing a component name.
   """
   if not bucket:
     return 'no-bucket'
   if bucket.component_cache:
     return bucket.component_cache
 
   stacktrace = ''.join(symbols[a] + ' ' for a in bucket.stacktrace).strip()
+  typeinfo = bucket.typeinfo_symbol
+  if typeinfo.startswith('0x'):
+    typeinfo = bucket.typename
 
   for rule in rule_list:
-    if bucket.mmap == rule.mmap and rule.stacktrace_pattern.match(stacktrace):
+    if (bucket.mmap == rule.mmap and
+        rule.stacktrace_pattern.match(stacktrace) and
+        (not rule.typeinfo_pattern or rule.typeinfo_pattern.match(typeinfo))):
       bucket.component_cache = rule.name
       return rule.name
 
   assert False
 
 
 class Bucket(object):
   """Represents a bucket, which is a unit of memory classification."""
 
-  def __init__(self, stacktrace, mmap):
+  def __init__(self, stacktrace, mmap, typeinfo=0, typename=''):
     self.stacktrace = stacktrace
     self.mmap = mmap
+    self.typeinfo = typeinfo
+    self.typeinfo_symbol = typename
+    self.typename = typename
     self.component_cache = ''
 
   def clear_component_cache(self):
     self.component_cache = ''
 
 
 class Dump(object):
   """Represents one heap profile dump."""
 
   def __init__(self, dump_path):
@@ skipping 357 matching lines @@
       sizes['hour'] = (self.dump_time - first_dump_time) / 60.0 / 60.0
     if 'minute' in sizes:
       sizes['minute'] = (self.dump_time - first_dump_time) / 60.0
     if 'second' in sizes:
       sizes['second'] = self.dump_time - first_dump_time
 
     return sizes
 
   @staticmethod
   def accumulate_size_for_expand(stacktrace_lines, rule_list, buckets,
-                                 component_name, depth, sizes, symbols):
+                                 component_name, depth, sizes, symbols,
+                                 typeinfo_symbols):
     for line in stacktrace_lines:
       words = line.split()
       bucket = buckets.get(int(words[BUCKET_ID]))
       component_match = get_component(rule_list, bucket, symbols)
       if component_match == component_name:
-        stacktrace_sequence = ''
+        stacktrace_sequence = '(type=%s) ' % typeinfo_symbols[bucket.typeinfo]
+        stacktrace_sequence +='(type.name=%s) ' % bucket.typename
         for address in bucket.stacktrace[0 : min(len(bucket.stacktrace),
                                                  1 + depth)]:
           stacktrace_sequence += symbols[address] + ' '
         if not stacktrace_sequence in sizes:
           sizes[stacktrace_sequence] = 0
         sizes[stacktrace_sequence] += int(words[COMMITTED])
 
-  def expand(self, rule_list, buckets, component_name, depth, symbols):
+  def expand(
+      self, rule_list, buckets, component_name, depth, symbols,
+      typeinfo_symbols):
     """Prints all stacktraces in a given component of given depth.
 
     Args:
         rule_list: A list of Rule objects.
         buckets: A dict mapping bucket ids to Bucket objects.
         component_name: A name of component for filtering.
         depth: An integer representing depth to be printed.
         symbols: A dict mapping runtime addresses to symbol names.
     """
     sizes = {}
 
     self.accumulate_size_for_expand(
         self.stacktrace_lines, rule_list, buckets, component_name,
-        depth, sizes, symbols)
+        depth, sizes, symbols, typeinfo_symbols)
 
     sorted_sizes_list = sorted(
         sizes.iteritems(), key=(lambda x: x[1]), reverse=True)
     total = 0
     for size_pair in sorted_sizes_list:
       sys.stdout.write('%10d %s\n' % (size_pair[1], size_pair[0]))
       total += size_pair[1]
     sys.stderr.write('total: %d\n' % (total))
 
 
 def update_symbols(
-    symbol_path, delayed_static_symbols, appeared_addresses, symbols):
+    symbol_path, delayed_static_symbols, appeared_addresses,
+    parameter_find_runtime_symbols_list, symbols):
   """Updates address/symbol mapping on memory and in a .symbol cache file.
 
   It reads cached address/symbol mapping from a .symbol file if it exists.
   Then, it resolves unresolved addresses from a Chrome binary with pprof.
   Both mappings on memory and in a .symbol cache file are updated.
 
   Symbol files are formatted as follows:
     <Address> <Symbol>
     <Address> <Symbol>
     <Address> <Symbol>
     ...
 
   Args:
       symbol_path: A string representing a path for a .symbol file.
       delayed_static_symbols: A DelayedStaticSymbols object.
       appeared_addresses: A list of known addresses.
+      parameter_find_runtime_symbols_list: A function to find symbols.
       symbols: A dict mapping runtime addresses to symbol names.
   """
   with open(symbol_path, mode='a+') as symbol_f:
     symbol_lines = symbol_f.readlines()
     if symbol_lines:
       for line in symbol_lines:
         items = line.split(None, 1)
         if len(items) == 1:
           items.append('??')
         symbols[int(items[0], 16)] = items[1].rstrip()
     if symbols:
       sys.stderr.write('  Found %d symbols in cache.\n' % len(symbols))
     else:
       sys.stderr.write('  No symbols found in cache.\n')
 
     unresolved_addresses = sorted(
         a for a in appeared_addresses if a not in symbols)
 
     if not unresolved_addresses:
       sys.stderr.write('  No need to resolve any more addresses.\n')
     else:
       sys.stderr.write('  %d addresses unresolved.\n' %
                        len(unresolved_addresses))
-      static_symbols = delayed_static_symbols.get()
-      symbol_list = find_runtime_symbols_list(
-          static_symbols, unresolved_addresses)
+
+      sys.stderr.write('  Loading symbols\n')
+      symbols_in_process = delayed_static_symbols.get()
+      symbol_list = parameter_find_runtime_symbols_list(
+          symbols_in_process, unresolved_addresses)
+      sys.stderr.write('  Loaded\n')
 
       for address, symbol in zip(unresolved_addresses, symbol_list):
         if not symbol:
           symbol = '??'
         stripped_symbol = symbol.strip()
         symbols[address] = stripped_symbol
         symbol_f.write('%x %s\n' % (address, stripped_symbol))
 
     sys.stderr.write('  All symbols resolved.\n')
 
@@ skipping 61 matching lines @@
   Args:
        policy_path: A path for a policy file.
   Returns:
        A loaded policy object.
   """
   with open(policy_path, mode='r') as f:
     policy = json.load(f)
 
   rules = []
   for rule in policy['rules']:
-    rules.append(Rule(
-        rule['name'], rule['allocator'] == 'mmap', rule['stacktrace']))
+    if 'typeinfo' in rule:
+      rules.append(Rule(
+          rule['name'], rule['allocator'] == 'mmap', rule['stacktrace'],
+          rule['typeinfo']))
+    else:
+      rules.append(Rule(
+          rule['name'], rule['allocator'] == 'mmap', rule['stacktrace']))
   return Policy(rules, policy['version'], policy['components'])
 
 
 def find_prefix(path):
   return re.sub('\.[0-9][0-9][0-9][0-9]\.heap', '', path)
 
 
 def load_buckets(prefix):
   # Reading buckets
   sys.stderr.write('Loading bucket files.\n')
+  appeared_typeinfo_addresses = set()
   buckets = {}
   bucket_count = 0
   n = 0
   while True:
     buckets_path = '%s.%04d.buckets' % (prefix, n)
     if not os.path.exists(buckets_path):
       if n > 10:
         break
       n += 1
       continue
     sys.stderr.write('  %s\n' % buckets_path)
     with open(buckets_path, 'r') as buckets_f:
       for line in buckets_f:
         words = line.split()
-        stacktrace = [int(address, 16) for address in words[2:]]
-        buckets[int(words[0])] = Bucket(stacktrace, words[1] == 'mmap')
+        typeinfo = 0
+        typename = ''
+        stacktrace_begin = 2
+        for index, word in enumerate(words):
+          if index < 2:
+            continue
+          if word[0] == 't':
+            typeinfo = int(word[1:], 16)
+            appeared_typeinfo_addresses.add(typeinfo)
+          elif word[0] == 'n':
+            typename = word[1:]
+          else:
+            stacktrace_begin = index
+            break
+        stacktrace = [int(address, 16) for address in words[stacktrace_begin:]]
+        buckets[int(words[0])] = Bucket(
+            stacktrace, words[1] == 'mmap', typeinfo, typename)
     n += 1
 
-  return buckets
+  return buckets, appeared_typeinfo_addresses
 
 
 def determine_dump_path_list(dump_path, prefix):
   dump_path_list = [dump_path]
 
   # search for the sequence of files
   n = int(dump_path[len(dump_path) - 9 : len(dump_path) - 5])
   n += 1  # skip current file
   while True:
     p = '%s.%04d.heap' % (prefix, n)
@@ skipping 34 matching lines @@
   appeared_addresses = set()
   dumps = []
   for path in dump_path_list:
     sys.stderr.write('  %s' % path)
     dumps.append(load_single_dump(path, buckets, appeared_addresses))
     sys.stderr.write('\n')
   return dumps, appeared_addresses
 
 
 def load_and_update_symbol_cache(
-    prefix, appeared_addresses, delayed_static_symbols):
+    prefix, appeared_addresses, appeared_typeinfo_addresses,
+    delayed_static_symbols):
   symbol_path = prefix + '.symbols'
   sys.stderr.write('Loading and updating symbol cache: "%s".\n' % symbol_path)
   symbols = {}
   update_symbols(
-      symbol_path, delayed_static_symbols, appeared_addresses, symbols)
-  return symbols
+      symbol_path, delayed_static_symbols, appeared_addresses,
+      find_runtime_symbols_list, symbols)
+
+  typeinfo_symbol_path = prefix + '.tsymbols'
+  sys.stderr.write('Loading and updating typeinfo symbol cache: "%s".\n' %
+                   typeinfo_symbol_path)
+  typeinfo_symbols = {}
+  update_symbols(
+      typeinfo_symbol_path, delayed_static_symbols, appeared_typeinfo_addresses,
+      find_runtime_typeinfo_symbols_list, typeinfo_symbols)
+
+  return symbols, typeinfo_symbols
 
 
 def load_default_policies():
   with open(POLICIES_JSON_PATH, mode='r') as policies_f:
     default_policies = json.load(policies_f)
   return default_policies
 
 
 def load_policy(policies_dict, policy_label):
   policy_file = policies_dict[policy_label]['file']
@@ skipping 31 matching lines @@
         specified_policies[specified_policy] = (
             default_policies[specified_policy])
     policies = load_policies_dict(specified_policies)
   else:
     policies = load_policies_dict(default_policies)
   return policies
 
 
 def load_basic_files_with_multiple_dumps(dump_path, keep):
   prefix = find_prefix(dump_path)
-  buckets = load_buckets(prefix)
+  buckets, appeared_typeinfo_addresses = load_buckets(prefix)
   dumps, appeared_addresses = load_dumps(
       determine_dump_path_list(dump_path, prefix), buckets)
   delayed_static_symbols = DelayedStaticSymbols(prefix, keep)
-  symbols = load_and_update_symbol_cache(
-      prefix, appeared_addresses, delayed_static_symbols)
-  return buckets, dumps, appeared_addresses, delayed_static_symbols, symbols
+  symbols, typeinfo_symbols = load_and_update_symbol_cache(
+      prefix, appeared_addresses, appeared_typeinfo_addresses,
+      delayed_static_symbols)
+  for bucket in buckets:
+    if buckets[bucket].typeinfo != None:

[M-A Ruel, 2012/08/19 02:40:00]

if buckets[bucket].typeinfo:
?

[Dai Mikurube (NOT FULLTIME), 2012/08/20 10:35:27]

Actually, buckets[bucket].typeinfo can be 0, but it should hit this 'if'.

[M-A Ruel, 2012/08/20 18:12:13]

Ok

+      buckets[bucket].typeinfo_symbol = typeinfo_symbols[
+          buckets[bucket].typeinfo]
+
+  return (buckets, dumps, appeared_addresses, appeared_typeinfo_addresses,
+          delayed_static_symbols, symbols, typeinfo_symbols)
 
 
 def load_basic_files_with_single_dump(dump_path, keep):
   prefix = find_prefix(dump_path)
-  buckets = load_buckets(prefix)
+  buckets, appeared_typeinfo_addresses = load_buckets(prefix)
   dump, appeared_addresses = load_dump(dump_path, buckets)
   delayed_static_symbols = DelayedStaticSymbols(prefix, keep)
-  symbols = load_and_update_symbol_cache(
-      prefix, appeared_addresses, delayed_static_symbols)
-  return buckets, dump, appeared_addresses, delayed_static_symbols, symbols
+  symbols, typeinfo_symbols = load_and_update_symbol_cache(
+      prefix, appeared_addresses, appeared_typeinfo_addresses,
+      delayed_static_symbols)
+  for bucket in buckets:
+    if buckets[bucket].typeinfo != None:
+      buckets[bucket].typeinfo_symbol = typeinfo_symbols[
+          buckets[bucket].typeinfo]
+
+  return (buckets, dump, appeared_addresses, appeared_typeinfo_addresses,
+          delayed_static_symbols, symbols, typeinfo_symbols)
 
 
 def do_stacktrace(sys_argv):
   parser = optparse.OptionParser(
       'Usage: %prog stacktrace [--keep] <dump>')
   parser.add_option('--keep', dest='keep', action='store_true')
   options, args = parser.parse_args(sys_argv)
 
   if len(args) != 2:
     parser.error('needs 1 argument.')
     return 1
 
   dump_path = args[1]
 
-  buckets, dump, appeared_addresses, delayed_static_symbols, symbols = (
-      load_basic_files_with_single_dump(dump_path, options.keep))
+  (buckets, dump, appeared_addresses, appeared_typeinfo_addresses,
+   delayed_static_symbols, symbols, typeinfo_symbols) = (
+       load_basic_files_with_single_dump(dump_path, options.keep))
 
   dump.print_stacktrace(buckets, symbols)
 
   return 0
 
 
 def do_csv(sys_argv):
   parser = optparse.OptionParser(
       'Usage: %prog csv [-p POLICY] [--keep] <first-dump>')
   parser.add_option('-p', '--policy', type='string', dest='policy',
                     help='profile with POLICY', metavar='POLICY')
   parser.add_option('--keep', dest='keep', action='store_true')
   options, args = parser.parse_args(sys_argv)
 
   if len(args) != 2:
     parser.error('needs 1 argument.')
     return 1
 
   dump_path = args[1]
 
-  buckets, dumps, appeared_addresses, delayed_static_symbols, symbols = (
-      load_basic_files_with_multiple_dumps(dump_path, options.keep))
+  (buckets, dumps, appeared_addresses, appeared_typeinfo_addresses,
+   delayed_static_symbols, symbols, typeinfo_symbols) = (
+       load_basic_files_with_multiple_dumps(dump_path, options.keep))
+
   policies = load_policies(options.policy)
 
   max_components = 0
   for policy in policies:
     max_components = max(max_components, len(policies[policy].components))
 
   for policy in sorted(policies):
     rule_list = policies[policy].rules
     components = policies[policy].components
 
@@ skipping 27 matching lines @@
                     help='profile with POLICY', metavar='POLICY')
   parser.add_option('--keep', dest='keep', action='store_true')
   options, args = parser.parse_args(sys_argv)
 
   if len(args) != 2:
     parser.error('needs 1 argument.')
     return 1
 
   dump_path = args[1]
 
-  buckets, dumps, appeared_addresses, delayed_static_symbols, symbols = (
-      load_basic_files_with_multiple_dumps(dump_path, options.keep))
+  (buckets, dumps, appeared_addresses, appeared_typeinfo_addresses,
+   delayed_static_symbols, symbols, typeinfo_symbols) = (
+       load_basic_files_with_multiple_dumps(dump_path, options.keep))
   policies = load_policies(options.policy)
 
   json_base = {
     'version': 'JSON_DEEP_2',
     'policies': {},
   }
 
   for policy in sorted(policies):
     rule_list = policies[policy].rules
     components = policies[policy].components
@@ skipping 26 matching lines @@
                     help='profile with POLICY', metavar='POLICY')
   parser.add_option('--keep', dest='keep', action='store_true')
   options, args = parser.parse_args(sys_argv)
 
   if len(args) != 2:
     parser.error('needs 1 argument.')
     return 1
 
   dump_path = args[1]
 
-  buckets, dumps, appeared_addresses, delayed_static_symbols, symbols = (
-      load_basic_files_with_multiple_dumps(dump_path, options.keep))
+  (buckets, dumps, appeared_addresses, appeared_typeinfo_addresses,
+   delayed_static_symbols, symbols, typeinfo_symbols) = (
+       load_basic_files_with_multiple_dumps(dump_path, options.keep))
   policies = load_policies(options.policy)
 
   for policy in sorted(policies):
     rule_list = policies[policy].rules
     components = policies[policy].components
 
     component_sizes = dumps[0].apply_policy(
         rule_list, buckets, dumps[0].dump_time, components, symbols)
     sys.stdout.write('%s:\n' % policy)
     for c in components:
@@ skipping 17 matching lines @@
 
   if len(args) != 5:
     parser.error('needs 4 arguments.')
     return 1
 
   dump_path = args[1]
   target_policy = args[2]
   component_name = args[3]
   depth = args[4]
 
-  buckets, dump, appeared_addresses, delayed_static_symbols, symbols = (
-      load_basic_files_with_single_dump(dump_path, options.keep))
+  (buckets, dump, appeared_addresses, appeared_typeinfo_addresses,
+   delayed_static_symbols, symbols, typeinfo_symbols) = (
+       load_basic_files_with_single_dump(dump_path, options.keep))
   policies = load_policies(target_policy)
 
   rule_list = policies[target_policy].rules
 
-  dump.expand(rule_list, buckets, component_name, int(depth), symbols)
+  dump.expand(rule_list, buckets, component_name, int(depth), symbols,
+              typeinfo_symbols)
 
   return 0
 
 
 def do_pprof(sys_argv):
   parser = optparse.OptionParser(
       'Usage: %prog pprof [-c COMPONENT] [--keep] <dump> <policy>')
   parser.add_option('-c', '--component', type='string', dest='component',
                     help='restrict to COMPONENT', metavar='COMPONENT')
   parser.add_option('--keep', dest='keep', action='store_true')
   options, args = parser.parse_args(sys_argv)
 
   if len(args) != 3:
     parser.error('needs 2 arguments.')
     return 1
 
   dump_path = args[1]
   target_policy = args[2]
   component = options.component
 
-  buckets, dump, appeared_addresses, delayed_static_symbols, symbols = (
-      load_basic_files_with_single_dump(dump_path, options.keep))
+  (buckets, dump, appeared_addresses, appeared_typeinfo_addresses,
+   delayed_static_symbols, symbols, typeinfo_symbols) = (
+       load_basic_files_with_single_dump(dump_path, options.keep))
   policies = load_policies(target_policy)
 
   rule_list = policies[target_policy].rules
 
   with open(find_prefix(dump_path) + '.maps', 'r') as maps_f:
     maps_lines = maps_f.readlines()
   dump.print_for_pprof(rule_list, buckets, maps_lines, component, symbols)
 
   return 0
 
@@ skipping 38 matching lines @@
    dmprof stacktrace [--keep] <dump>
 """ % (sys.argv[0]))
     sys.exit(1)
   action = sys.argv.pop(1)
 
   return COMMANDS[action](sys.argv)
 
 
 if __name__ == '__main__':
   sys.exit(main())