Breakdown "unhooked" memory regions by VMA pathnames and permissions.

tools/deep_memory_profiler/dmprof.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """The deep heap profiler script for Chrome."""
 
 import copy
 import datetime
 import json
 import logging
@@ skipping 359 matching lines @@
                      len(self._symbol_mapping_caches[symbol_type]))
     except IOError as e:
       LOGGER.info('The symbol cache file is invalid: %s' % e)
 
 
 class Rule(object):
   """Represents one matching rule in a policy file."""
 
   def __init__(self,
                name,
-               mmap,
+               allocator_type,
                stackfunction_pattern=None,
                stacksourcefile_pattern=None,
-               typeinfo_pattern=None):
+               typeinfo_pattern=None,
+               mappedpathname_pattern=None,
+               mappedpermission_pattern=None):
     self._name = name
-    self._mmap = mmap
+    self._allocator_type = allocator_type
 
     self._stackfunction_pattern = None
     if stackfunction_pattern:
       self._stackfunction_pattern = re.compile(
           stackfunction_pattern + r'\Z')
 
     self._stacksourcefile_pattern = None
     if stacksourcefile_pattern:
       self._stacksourcefile_pattern = re.compile(
           stacksourcefile_pattern + r'\Z')
 
     self._typeinfo_pattern = None
     if typeinfo_pattern:
       self._typeinfo_pattern = re.compile(typeinfo_pattern + r'\Z')
 
+    self._mappedpathname_pattern = None
+    if mappedpathname_pattern:
+      self._mappedpathname_pattern = re.compile(mappedpathname_pattern + r'\Z')
+
+    self._mappedpermission_pattern = None
+    if mappedpermission_pattern:
+      self._mappedpermission_pattern = re.compile(
+          mappedpermission_pattern + r'\Z')
+
   @property
   def name(self):
     return self._name
 
   @property
-  def mmap(self):
-    return self._mmap
+  def allocator_type(self):
+    return self._allocator_type
 
   @property
   def stackfunction_pattern(self):
     return self._stackfunction_pattern
 
   @property
   def stacksourcefile_pattern(self):
     return self._stacksourcefile_pattern
 
   @property
   def typeinfo_pattern(self):
     return self._typeinfo_pattern
 
+  @property
+  def mappedpathname_pattern(self):
+    return self._mappedpathname_pattern
+
+  @property
+  def mappedpermission_pattern(self):
+    return self._mappedpermission_pattern
+
 
 class Policy(object):
   """Represents a policy, a content of a policy file."""
 
   def __init__(self, rules, version, components):
     self._rules = rules
     self._version = version
     self._components = components
 
   @property
@@ skipping 22 matching lines @@
     if bucket.component_cache:
       return bucket.component_cache
 
     stackfunction = bucket.symbolized_joined_stackfunction
     stacksourcefile = bucket.symbolized_joined_stacksourcefile
     typeinfo = bucket.symbolized_typeinfo
     if typeinfo.startswith('0x'):
       typeinfo = bucket.typeinfo_name
 
     for rule in self._rules:
-      if (bucket.mmap == rule.mmap and
+      if (bucket.allocator_type == rule.allocator_type and
           (not rule.stackfunction_pattern or
            rule.stackfunction_pattern.match(stackfunction)) and
           (not rule.stacksourcefile_pattern or
            rule.stacksourcefile_pattern.match(stacksourcefile)) and
           (not rule.typeinfo_pattern or rule.typeinfo_pattern.match(typeinfo))):
         bucket.component_cache = rule.name
         return rule.name
 
     assert False
 
+  def find_unhooked(self, region):
+    for rule in self._rules:
+      if (region[0] == 'unhooked' and
+          rule.allocator_type == 'unhooked' and
+          (not rule.mappedpathname_pattern or
+           rule.mappedpathname_pattern.match(region[1]['vma']['name'])) and
+          (not rule.mappedpermission_pattern or
+           rule.mappedpermission_pattern.match(
+               region[1]['vma']['readable'] +
+               region[1]['vma']['writable'] +
+               region[1]['vma']['executable'] +
+               region[1]['vma']['private']))):
+        return rule.name
+
+    assert False
+
   @staticmethod
   def load(filename, filetype):
     """Loads a policy file of |filename| in a |format|.
 
     Args:
         filename: A filename to be loaded.
         filetype: A string to specify a type of the file.  Only 'json' is
             supported for now.
 
     Returns:
@@ skipping 34 matching lines @@
          A loaded policy object.
     """
     policy = json.load(policy_f)
 
     rules = []
     for rule in policy['rules']:
       stackfunction = rule.get('stackfunction') or rule.get('stacktrace')
       stacksourcefile = rule.get('stacksourcefile')
       rules.append(Rule(
           rule['name'],
-          rule['allocator'] == 'mmap',
+          rule['allocator'],  # allocator_type
           stackfunction,
           stacksourcefile,
-          rule['typeinfo'] if 'typeinfo' in rule else None))
+          rule['typeinfo'] if 'typeinfo' in rule else None,
+          rule.get('mappedpathname'),
+          rule.get('mappedpermission')))
 
     return Policy(rules, policy['version'], policy['components'])
 
 
 class PolicySet(object):
   """Represents a set of policies."""
 
   def __init__(self, policy_directory):
     self._policy_directory = policy_directory
 
@@ skipping 49 matching lines @@
       LOGGER.info('  %s: %s' % (label, directory[label]['file']))
       loaded = Policy.load(directory[label]['file'], directory[label]['format'])
       if loaded:
         policies[label] = loaded
     return PolicySet(policies)
 
 
 class Bucket(object):
   """Represents a bucket, which is a unit of memory block classification."""
 
-  def __init__(self, stacktrace, mmap, typeinfo, typeinfo_name):
+  def __init__(self, stacktrace, allocator_type, typeinfo, typeinfo_name):
     self._stacktrace = stacktrace
-    self._mmap = mmap
+    self._allocator_type = allocator_type
     self._typeinfo = typeinfo
     self._typeinfo_name = typeinfo_name
 
     self._symbolized_stackfunction = stacktrace
     self._symbolized_joined_stackfunction = ''
     self._symbolized_stacksourcefile = stacktrace
     self._symbolized_joined_stacksourcefile = ''
     self._symbolized_typeinfo = typeinfo_name
 
     self.component_cache = ''
 
   def __str__(self):
     result = []
-    result.append('mmap' if self._mmap else 'malloc')
+    result.append(self._allocator_type)
     if self._symbolized_typeinfo == 'no typeinfo':
       result.append('tno_typeinfo')
     else:
       result.append('t' + self._symbolized_typeinfo)
     result.append('n' + self._typeinfo_name)
     result.extend(['%s(@%s)' % (function, sourcefile)
                    for function, sourcefile
                    in zip(self._symbolized_stackfunction,
                           self._symbolized_stacksourcefile)])
     return ' '.join(result)
@@ skipping 24 matching lines @@
         self._symbolized_typeinfo = 'no typeinfo'
 
   def clear_component_cache(self):
     self.component_cache = ''
 
   @property
   def stacktrace(self):
     return self._stacktrace
 
   @property
-  def mmap(self):
-    return self._mmap
+  def allocator_type(self):
+    return self._allocator_type
 
   @property
   def typeinfo(self):
     return self._typeinfo
 
   @property
   def typeinfo_name(self):
     return self._typeinfo_name
 
   @property
@@ skipping 59 matching lines @@
           self._typeinfo_addresses.add(typeinfo)
         elif word[0] == 'n':
           typeinfo_name = word[1:]
         else:
           stacktrace_begin = index
           break
       stacktrace = [int(address, 16) for address in words[stacktrace_begin:]]
       for frame in stacktrace:
         self._code_addresses.add(frame)
       self._buckets[int(words[0])] = Bucket(
-          stacktrace, words[1] == 'mmap', typeinfo, typeinfo_name)
+          stacktrace, words[1], typeinfo, typeinfo_name)
 
   def __iter__(self):
     for bucket_id, bucket_content in self._buckets.iteritems():
       yield bucket_id, bucket_content
 
   def __getitem__(self, bucket_id):
     return self._buckets[bucket_id]
 
   def get(self, bucket_id):
     return self._buckets.get(bucket_id)
@@ skipping 17 matching lines @@
 
 class Dump(object):
   """Represents a heap profile dump."""
 
   _PATH_PATTERN = re.compile(r'^(.*)\.([0-9]+)\.([0-9]+)\.heap$')
 
   _HOOK_PATTERN = re.compile(
       r'^ ([ \(])([a-f0-9]+)([ \)])-([ \(])([a-f0-9]+)([ \)])\s+'
       r'(hooked|unhooked)\s+(.+)$', re.IGNORECASE)
 
+  _HOOKED_PATTERN = re.compile(r'(?P<TYPE>.+ )?(?P<COMMITTED>[0-9]+) / '
+                               '(?P<RESERVED>[0-9]+) @ (?P<BUCKETID>[0-9]+)')
+  _UNHOOKED_PATTERN = re.compile(r'(?P<TYPE>.+ )?(?P<COMMITTED>[0-9]+) / '
+                                 '(?P<RESERVED>[0-9]+)')
+
+  _OLD_HOOKED_PATTERN = re.compile(r'(?P<TYPE>.+) @ (?P<BUCKETID>[0-9]+)')
+  _OLD_UNHOOKED_PATTERN = re.compile(r'(?P<TYPE>.+) (?P<COMMITTED>[0-9]+)')
+
   _TIME_PATTERN_FORMAT = re.compile(
       r'^Time: ([0-9]+/[0-9]+/[0-9]+ [0-9]+:[0-9]+:[0-9]+)(\.[0-9]+)?')
   _TIME_PATTERN_SECONDS = re.compile(r'^Time: ([0-9]+)$')
 
   def __init__(self, path, modified_time):
     self._path = path
     matched = self._PATH_PATTERN.match(path)
     self._pid = int(matched.group(2))
     self._count = int(matched.group(3))
     self._time = modified_time
@@ skipping 162 matching lines @@
   def _parse_mmap_list(self):
     """Parses lines in self._lines as a mmap list."""
     (ln, found) = skip_while(
         0, len(self._lines),
         lambda n: self._lines[n] != 'MMAP_LIST:\n')
     if not found:
       return {}
 
     ln += 1
     self._map = {}
+    current_vma = dict()
     while True:
       entry = proc_maps.ProcMaps.parse_line(self._lines[ln])
       if entry:
+        current_vma = dict()
         for _, _, attr in self._procmaps.iter_range(entry.begin, entry.end):
           for key, value in entry.as_dict().iteritems():
             attr[key] = value
+            current_vma[key] = value
         ln += 1
         continue
       matched = self._HOOK_PATTERN.match(self._lines[ln])
       if not matched:
         break
       # 2: starting address
       # 5: end address
       # 7: hooked or unhooked
       # 8: additional information
+      if matched.group(7) == 'hooked':
+        submatched = self._HOOKED_PATTERN.match(matched.group(8))
+        if not submatched:
+          submatched = self._OLD_HOOKED_PATTERN.match(matched.group(8))
+      elif matched.group(7) == 'unhooked':
+        submatched = self._UNHOOKED_PATTERN.match(matched.group(8))
+        if not submatched:
+          submatched = self._OLD_UNHOOKED_PATTERN.match(matched.group(8))
+      else:
+        assert matched.group(7) in ['hooked', 'unhooked']
+
+      submatched_dict = submatched.groupdict()
+      region_info = { 'vma': current_vma }
+      if 'TYPE' in submatched_dict:
+        region_info['type'] = submatched_dict['TYPE'].strip()
+      if 'COMMITTED' in submatched_dict:
+        region_info['committed'] = int(submatched_dict['COMMITTED'])
+      if 'RESERVED' in submatched_dict:
+        region_info['reserved'] = int(submatched_dict['RESERVED'])
+      if 'BUCKETID' in submatched_dict:
+        region_info['bucket_id'] = int(submatched_dict['BUCKETID'])
+
       self._map[(int(matched.group(2), 16),
-                 int(matched.group(5), 16))] = (matched.group(7),
-                                                matched.group(8))
+                 int(matched.group(5), 16))] = (matched.group(7), region_info)
       ln += 1
 
   def _extract_stacktrace_lines(self, line_number):
     """Extracts the position of stacktrace lines.
 
     Valid stacktrace lines are stored into self._stacktrace_lines.
 
     Args:
         line_number: A line number to start parsing in lines.
 
@@ skipping 292 matching lines @@
         first_dump_time: An integer representing time when the first dump is
             dumped.
 
     Returns:
         A dict mapping components and their corresponding sizes.
     """
     LOGGER.info('  %s' % dump.path)
     sizes = dict((c, 0) for c in policy.components)
 
     PolicyCommands._accumulate(dump, policy, bucket_set, sizes)
+    PolicyCommands._accumulate_maps(dump, policy, sizes)
 
     sizes['mmap-no-log'] = (
         dump.global_stat('profiled-mmap_committed') -
         sizes['mmap-total-log'])
     sizes['mmap-total-record'] = dump.global_stat('profiled-mmap_committed')
     sizes['mmap-total-record-vm'] = dump.global_stat('profiled-mmap_virtual')
 
     sizes['tc-no-log'] = (
         dump.global_stat('profiled-malloc_committed') -
         sizes['tc-total-log'])
     sizes['tc-total-record'] = dump.global_stat('profiled-malloc_committed')
     sizes['tc-unused'] = (
         sizes['mmap-tcmalloc'] -
         dump.global_stat('profiled-malloc_committed'))
+    if sizes['tc-unused'] < 0:
+      LOGGER.warn('    Assuming tc-unused=0 as it is negative: %d (bytes)' %
+                  sizes['tc-unused'])
+      sizes['tc-unused'] = 0
     sizes['tc-total'] = sizes['mmap-tcmalloc']
 
     for key, value in {
         'total': 'total_committed',
         'filemapped': 'file_committed',
         'absent': 'absent_committed',
         'file-exec': 'file-exec_committed',
         'file-nonexec': 'file-nonexec_committed',
         'anonymous': 'anonymous_committed',
         'stack': 'stack_committed',
         'other': 'other_committed',
         'unhooked-absent': 'nonprofiled-absent_committed',
-        'unhooked-anonymous': 'nonprofiled-anonymous_committed',
-        'unhooked-file-exec': 'nonprofiled-file-exec_committed',
-        'unhooked-file-nonexec': 'nonprofiled-file-nonexec_committed',
-        'unhooked-stack': 'nonprofiled-stack_committed',
-        'unhooked-other': 'nonprofiled-other_committed',
         'total-vm': 'total_virtual',
         'filemapped-vm': 'file_virtual',
         'anonymous-vm': 'anonymous_virtual',
         'other-vm': 'other_virtual' }.iteritems():
       if key in sizes:
         sizes[key] = dump.global_stat(value)
 
     if 'mustbezero' in sizes:
       removed_list = (
           'profiled-mmap_committed',
@@ skipping 27 matching lines @@
       component_match = policy.find(bucket)
       sizes[component_match] += int(words[COMMITTED])
 
       if component_match.startswith('tc-'):
         sizes['tc-total-log'] += int(words[COMMITTED])
       elif component_match.startswith('mmap-'):
         sizes['mmap-total-log'] += int(words[COMMITTED])
       else:
         sizes['other-total-log'] += int(words[COMMITTED])
 
+  @staticmethod
+  def _accumulate_maps(dump, policy, sizes):
+    for _, value in dump.iter_map:
+      if value[0] == 'unhooked':
+        component_match = policy.find_unhooked(value)
+        sizes[component_match] += int(value[1]['committed'])
+
 
 class CSVCommand(PolicyCommands):
   def __init__(self):
     super(CSVCommand, self).__init__('csv')
 
   def do(self, sys_argv):
     policy_set, dumps, bucket_set = self._set_up(sys_argv)
     return CSVCommand._output(policy_set, dumps, bucket_set, sys.stdout)
 
   @staticmethod
@@ skipping 126 matching lines @@
     max_dump_count_digit = len(str(max_dump_count))
     for begin, end, attr in range_dict.iter_range():
       out.write('%x-%x\n' % (begin, end))
       if len(attr) < max_dump_count:
         attr[max_dump_count] = None
       for index, x in enumerate(attr[1:]):
         out.write('  #%0*d: ' % (max_dump_count_digit, index + 1))
         if not x:
           out.write('None\n')
         elif x[0] == 'hooked':
-          attrs = x[1].split()
-          assert len(attrs) == 3
-          bucket_id = int(attrs[2])
+          region_info = x[1]
+          bucket_id = region_info['bucket_id']
           bucket = bucket_set.get(bucket_id)
           component = policy.find(bucket)
-          out.write('hooked %s: %s @ %d\n' % (attrs[0], component, bucket_id))
+          out.write('hooked %s: %s @ %d\n' % (
+              region_info['type'] if 'type' in region_info else 'None',
+              component, bucket_id))
         else:
-          attrs = x[1].split()
-          size = int(attrs[1])
-          out.write('unhooked %s: %d bytes committed\n' % (attrs[0], size))
+          region_info = x[1]
+          size = region_info['committed']
+          out.write('unhooked %s: %d bytes committed\n' % (
+              region_info['type'] if 'type' in region_info else 'None', size))
 
 
 class ExpandCommand(Command):
   def __init__(self):
     super(ExpandCommand, self).__init__(
         'Usage: %prog expand <dump> <policy> <component> <depth>')
 
   def do(self, sys_argv):
     _, args = self._parse_args(sys_argv, 4)
     dump_path = args[1]
@@ skipping 279 matching lines @@
     errorcode = COMMANDS[action]().do(sys.argv)
   except ParsingException, e:
     errorcode = 1
     sys.stderr.write('Exit by parsing error: %s\n' % e)
 
   return errorcode
 
 
 if __name__ == '__main__':
   sys.exit(main())