Retry: Add a first test for tools/deep_memory_profiler.

tools/deep_memory_profiler/dmprof.py

-#!/usr/bin/env python

[M-A Ruel, 2012/11/17 23:23:10]

Why are you removing the shebang and removing the executable bit on this file?

[Dai Mikurube (NOT FULLTIME), 2012/11/18 06:01:54]

It's because now dmprof can be executed by the shell script.  I thought we'd be
happy to have required minimum executable files.

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """The deep heap profiler script for Chrome."""
 
 from datetime import datetime
 import json
 import logging
 import optparse
 import os
 import re
-import shutil
-import subprocess
 import sys
-import tempfile
 
 BASE_PATH = os.path.dirname(os.path.abspath(__file__))
 FIND_RUNTIME_SYMBOLS_PATH = os.path.join(
     BASE_PATH, os.pardir, 'find_runtime_symbols')
 sys.path.append(FIND_RUNTIME_SYMBOLS_PATH)
 
 from find_runtime_symbols import find_runtime_symbols_list
 from find_runtime_symbols import find_runtime_typeinfo_symbols_list
 from find_runtime_symbols import RuntimeSymbolsInProcess
 from prepare_symbol_info import prepare_symbol_info
@@ skipping 44 matching lines @@
 
 # POLICY_DEEP_3 is in JSON format.
 POLICY_DEEP_3 = 'POLICY_DEEP_3'
 
 # POLICY_DEEP_3 contains typeinfo.
 POLICY_DEEP_4 = 'POLICY_DEEP_4'
 
 
 class EmptyDumpException(Exception):
   def __init__(self, value):
+    super(EmptyDumpException, self).__init__()
     self.value = value
   def __str__(self):
     return repr(self.value)
 
 
 class ParsingException(Exception):
   def __init__(self, value):
+    super(ParsingException, self).__init__()
     self.value = value
   def __str__(self):
     return repr(self.value)
 
 
 class InvalidDumpException(ParsingException):
   def __init__(self, value):
+    super(InvalidDumpException, self).__init__()
     self.value = value
   def __str__(self):
     return "invalid heap profile dump: %s" % repr(self.value)
 
 
 class ObsoleteDumpVersionException(ParsingException):
   def __init__(self, value):
+    super(ObsoleteDumpVersionException, self).__init__()
     self.value = value
   def __str__(self):
     return "obsolete heap profile dump version: %s" % repr(self.value)
 
 
 def skip_while(index, max_index, skipping_condition):
   """Increments |index| until |skipping_condition|(|index|) is False.
 
   Returns:
       A pair of an integer indicating a line number after skipped, and a
@@ skipping 148 matching lines @@
     try:
       with open(symbol_cache_path, mode='r') as symbol_f:
         for line in symbol_f:
           items = line.rstrip().split(None, 1)
           if len(items) == 1:
             items.append('??')
           self._symbol_caches[address_type][int(items[0], 16)] = items[1]
       LOGGER.info('Loaded %d entries from symbol cache.' %
                      len(self._symbol_caches[address_type]))
     except IOError as e:
-      LOGGER.info('No valid symbol cache file is found.')
+      LOGGER.info('No valid symbol cache file is found: %s' % e)
 
 
 class Rule(object):
   """Represents one matching rule in a policy file."""
 
   def __init__(self, name, mmap, stacktrace_pattern, typeinfo_pattern=None):
     self._name = name
     self._mmap = mmap
     self._stacktrace_pattern = re.compile(stacktrace_pattern + r'\Z')
     if typeinfo_pattern:
@@ skipping 60 matching lines @@
     for rule in self._rules:
       if (bucket.mmap == rule.mmap and
           rule.stacktrace_pattern.match(stacktrace) and
           (not rule.typeinfo_pattern or rule.typeinfo_pattern.match(typeinfo))):
         bucket.component_cache = rule.name
         return rule.name
 
     assert False
 
   @staticmethod
-  def load(filename, format):
+  def load(filename, filetype):
     """Loads a policy file of |filename| in a |format|.
 
     Args:
         filename: A filename to be loaded.
-        format: A string to specify a format of the file.  Only 'json' is
+        filetype: A string to specify a type of the file.  Only 'json' is
             supported for now.
 
     Returns:
         A loaded Policy object.
     """
     with open(os.path.join(BASE_PATH, filename)) as policy_f:
-      return Policy.parse(policy_f, format)
+      return Policy.parse(policy_f, filetype)
 
   @staticmethod
-  def parse(policy_f, format):
+  def parse(policy_f, filetype):
     """Parses a policy file content in a |format|.
 
     Args:
         policy_f: An IO object to be loaded.
-        format: A string to specify a format of the file.  Only 'json' is
+        filetype: A string to specify a type of the file.  Only 'json' is
             supported for now.
 
     Returns:
         A loaded Policy object.
     """
-    if format == 'json':
+    if filetype == 'json':
       return Policy._parse_json(policy_f)
     else:
       return None
 
   @staticmethod
   def _parse_json(policy_f):
     """Parses policy file in json format.
 
     A policy file contains component's names and their stacktrace pattern
     written in regular expression.  Those patterns are matched against each
@@ skipping 147 matching lines @@
 
 class BucketSet(object):
   """Represents a set of bucket."""
   def __init__(self):
     self._buckets = {}
     self._addresses = {
         FUNCTION_ADDRESS: set(),
         TYPEINFO_ADDRESS: set(),
         }
 
-  @staticmethod
-  def load(prefix):
+  def load(self, prefix):
     """Loads all related bucket files.
 
     Args:
         prefix: A prefix string for bucket file names.
-
-    Returns:
-        A loaded BucketSet object.
     """
     LOGGER.info('Loading bucket files.')
-    bucket_set = BucketSet()
 
     n = 0
     while True:
       path = '%s.%04d.buckets' % (prefix, n)
       if not os.path.exists(path):
         if n > 10:
           break
         n += 1
         continue
       LOGGER.info('  %s' % path)
       with open(path, 'r') as f:
-        bucket_set._load_file(f)
+        self._load_file(f)
       n += 1
 
-    return bucket_set
-
   def _load_file(self, bucket_f):
     for line in bucket_f:
       words = line.split()
       typeinfo = None
       typeinfo_name = ''
       stacktrace_begin = 2
       for index, word in enumerate(words):
         if index < 2:
           continue
         if word[0] == 't':
@@ skipping 29 matching lines @@
       bucket_content.clear_component_cache()
 
   def iter_addresses(self, address_type):
     for function in self._addresses[address_type]:
       yield function
 
 
 class Dump(object):
   """Represents a heap profile dump."""
 
-  def __init__(self):
-    self._path = ''
-    self._time = None
+  def __init__(self, path, time):
+    self._path = path
+    self._time = time
     self._stacktrace_lines = []
     self._global_stats = {} # used only in apply_policy
 
     self._version = ''
     self._lines = []
 
   @property
   def path(self):
     return self._path
 
@@ skipping 16 matching lines @@
     Args:
         path: A file path string to load.
         log_header: A preceding string for log messages.
 
     Returns:
         A loaded Dump object.
 
     Raises:
         ParsingException for invalid heap profile dumps.
     """
-    dump = Dump()
-    dump._path = path
-    dump._time = os.stat(dump._path).st_mtime
-    dump._version = ''
+    dump = Dump(path, os.stat(path).st_mtime)
+    with open(path, 'r') as f:
+      dump.load_file(f, log_header)
+    return dump
 
-    dump._lines = [line for line in open(dump._path, 'r')
+  def load_file(self, f, log_header):
+    self._lines = [line for line in f
                    if line and not line.startswith('#')]
 
     try:
-      dump._version, ln = dump._parse_version()
-      dump._parse_global_stats()
-      dump._extract_stacktrace_lines(ln)
+      self._version, ln = self._parse_version()
+      self._parse_global_stats()
+      self._extract_stacktrace_lines(ln)
     except EmptyDumpException:
-      LOGGER.info('%s%s ...ignored an empty dump.' % (log_header, path))
+      LOGGER.info('%s%s ...ignored an empty dump.' % (log_header, self._path))
     except ParsingException, e:
-      LOGGER.error('%s%s ...error %s' % (log_header, path, e))
+      LOGGER.error('%s%s ...error %s' % (log_header, self._path, e))
       raise
     else:
-      LOGGER.info('%s%s (version: %s)' % (log_header, path, dump._version))
-
-    return dump
+      LOGGER.info('%s%s (version:%s)' % (log_header, self._path, self._version))
 
   def _parse_version(self):
     """Parses a version string in self._lines.
 
     Returns:
         A pair of (a string representing a version of the stacktrace dump,
         and an integer indicating a line number next to the version string).
 
     Raises:
         ParsingException for invalid dump versions.
@@ skipping 125 matching lines @@
   See COMMANDS in main().
   """
   def __init__(self, usage):
     self._parser = optparse.OptionParser(usage)
 
   @staticmethod
   def load_basic_files(dump_path, multiple):
     prefix = Command._find_prefix(dump_path)
     symbol_mapping = SymbolMapping(prefix)
     symbol_mapping.prepare()
-    bucket_set = BucketSet.load(prefix)
+    bucket_set = BucketSet()
+    bucket_set.load(prefix)
     if multiple:
       dump_list = DumpList.load(Command._find_all_dumps(dump_path))
     else:
       dump = Dump.load(dump_path)
     symbol_cache = SymbolCache(prefix)
     symbol_cache.update(FUNCTION_ADDRESS, bucket_set, symbol_mapping)
     symbol_cache.update(TYPEINFO_ADDRESS, bucket_set, symbol_mapping)
     bucket_set.symbolize(symbol_cache)
     if multiple:
       return (bucket_set, dump_list)
@@ skipping 21 matching lines @@
 
     return dump_path_list
 
   def _parse_args(self, sys_argv, required):
     options, args = self._parser.parse_args(sys_argv)
     if len(args) != required + 1:
       self._parser.error('needs %d argument(s).\n' % required)
       return None
     return (options, args)
 
-  def _parse_policy_list(self, options_policy):
+  @staticmethod
+  def _parse_policy_list(options_policy):
     if options_policy:
       return options_policy.split(',')
     else:
       return None
 
 
 class StacktraceCommand(Command):
   def __init__(self):
     super(StacktraceCommand, self).__init__(
         'Usage: %prog stacktrace <dump>')
 
   def do(self, sys_argv):
-    options, args = self._parse_args(sys_argv, 1)
+    _, args = self._parse_args(sys_argv, 1)
     dump_path = args[1]
     (bucket_set, dump) = Command.load_basic_files(dump_path, False)
 
     StacktraceCommand._output(dump, bucket_set, sys.stdout)
     return 0
 
   @staticmethod
   def _output(dump, bucket_set, out):
     """Outputs a given stacktrace.
 
@@ skipping 18 matching lines @@
     super(PolicyCommands, self).__init__(
         'Usage: %%prog %s [-p POLICY] <first-dump>' % command)
     self._parser.add_option('-p', '--policy', type='string', dest='policy',
                             help='profile with POLICY', metavar='POLICY')
 
   def _set_up(self, sys_argv):
     options, args = self._parse_args(sys_argv, 1)
     dump_path = args[1]
     (bucket_set, dumps) = Command.load_basic_files(dump_path, True)
 
-    policy_set = PolicySet.load(self._parse_policy_list(options.policy))
+    policy_set = PolicySet.load(Command._parse_policy_list(options.policy))
     return policy_set, dumps, bucket_set
 
-  def _apply_policy(self, dump, policy, bucket_set, first_dump_time):
+  @staticmethod
+  def _apply_policy(dump, policy, bucket_set, first_dump_time):
     """Aggregates the total memory size of each component.
 
     Iterate through all stacktraces and attribute them to one of the components
     based on the policy.  It is important to apply policy in right order.
 
     Args:
         dump: A Dump object.
         policy: A Policy object.
         bucket_set: A BucketSet object.
         first_dump_time: An integer representing time when the first dump is
@@ skipping 83 matching lines @@
       else:
         sizes['other-total-log'] += int(words[COMMITTED])
 
 
 class CSVCommand(PolicyCommands):
   def __init__(self):
     super(CSVCommand, self).__init__('csv')
 
   def do(self, sys_argv):
     policy_set, dumps, bucket_set = self._set_up(sys_argv)
-    return self._output(policy_set, dumps, bucket_set, sys.stdout)
+    return CSVCommand._output(policy_set, dumps, bucket_set, sys.stdout)
 
-  def _output(self, policy_set, dumps, bucket_set, out):
+  @staticmethod
+  def _output(policy_set, dumps, bucket_set, out):
     max_components = 0
     for label in policy_set:
       max_components = max(max_components, len(policy_set[label].components))
 
     for label in sorted(policy_set):
       components = policy_set[label].components
       if len(policy_set) > 1:
         out.write('%s%s\n' % (label, ',' * (max_components - 1)))
       out.write('%s%s\n' % (
           ','.join(components), ',' * (max_components - len(components))))
 
       LOGGER.info('Applying a policy %s to...' % label)
       for dump in dumps:
-        component_sizes = self._apply_policy(
+        component_sizes = PolicyCommands._apply_policy(
             dump, policy_set[label], bucket_set, dumps[0].time)
         s = []
         for c in components:
           if c in ('hour', 'minute', 'second'):
             s.append('%05.5f' % (component_sizes[c]))
           else:
             s.append('%05.5f' % (component_sizes[c] / 1024.0 / 1024.0))
         out.write('%s%s\n' % (
               ','.join(s), ',' * (max_components - len(components))))
 
       bucket_set.clear_component_cache()
 
     return 0
 
 
 class JSONCommand(PolicyCommands):
   def __init__(self):
     super(JSONCommand, self).__init__('json')
 
   def do(self, sys_argv):
     policy_set, dumps, bucket_set = self._set_up(sys_argv)
-    return self._output(policy_set, dumps, bucket_set, sys.stdout)
+    return JSONCommand._output(policy_set, dumps, bucket_set, sys.stdout)
 
-  def _output(self, policy_set, dumps, bucket_set, out):
+  @staticmethod
+  def _output(policy_set, dumps, bucket_set, out):
     json_base = {
       'version': 'JSON_DEEP_2',
       'policies': {},
     }
 
     for label in sorted(policy_set):
       json_base['policies'][label] = {
         'legends': policy_set[label].components,
         'snapshots': [],
       }
 
       LOGGER.info('Applying a policy %s to...' % label)
       for dump in dumps:
-        component_sizes = self._apply_policy(
+        component_sizes = PolicyCommands._apply_policy(
             dump, policy_set[label], bucket_set, dumps[0].time)
         component_sizes['dump_path'] = dump.path
         component_sizes['dump_time'] = datetime.fromtimestamp(
             dump.time).strftime('%Y-%m-%d %H:%M:%S')
         json_base['policies'][label]['snapshots'].append(component_sizes)
 
       bucket_set.clear_component_cache()
 
     json.dump(json_base, out, indent=2, sort_keys=True)
 
     return 0
 
 
 class ListCommand(PolicyCommands):
   def __init__(self):
     super(ListCommand, self).__init__('list')
 
   def do(self, sys_argv):
     policy_set, dumps, bucket_set = self._set_up(sys_argv)
-    return self._output(policy_set, dumps, bucket_set, sys.stdout)
+    return ListCommand._output(policy_set, dumps, bucket_set, sys.stdout)
 
-  def _output(self, policy_set, dumps, bucket_set, out):
+  @staticmethod
+  def _output(policy_set, dumps, bucket_set, out):
     for label in sorted(policy_set):
       LOGGER.info('Applying a policy %s to...' % label)
       for dump in dumps:
-        component_sizes = self._apply_policy(
+        component_sizes = PolicyCommands._apply_policy(
             dump, policy_set[label], bucket_set, dump.time)
         out.write('%s for %s:\n' % (label, dump.path))
         for c in policy_set[label].components:
           if c in ['hour', 'minute', 'second']:
             out.write('%40s %12.3f\n' % (c, component_sizes[c]))
           else:
             out.write('%40s %12d\n' % (c, component_sizes[c]))
 
       bucket_set.clear_component_cache()
 
     return 0
 
 
 class ExpandCommand(Command):
   def __init__(self):
     super(ExpandCommand, self).__init__(
         'Usage: %prog expand <dump> <policy> <component> <depth>')
 
   def do(self, sys_argv):
-    options, args = self._parse_args(sys_argv, 4)
+    _, args = self._parse_args(sys_argv, 4)
     dump_path = args[1]
     target_policy = args[2]
     component_name = args[3]
     depth = args[4]
     (bucket_set, dump) = Command.load_basic_files(dump_path, False)
-    policy_set = PolicySet.load(self._parse_policy_list(target_policy))
+    policy_set = PolicySet.load(Command._parse_policy_list(target_policy))
 
-    self._output(dump, policy_set[target_policy], bucket_set,
-                 component_name, int(depth), sys.stdout)
+    ExpandCommand._output(dump, policy_set[target_policy], bucket_set,
+                          component_name, int(depth), sys.stdout)
     return 0
 
-  def _output(self, dump, policy, bucket_set, component_name, depth, out):
+  @staticmethod
+  def _output(dump, policy, bucket_set, component_name, depth, out):
     """Prints all stacktraces in a given component of given depth.
 
     Args:
         dump: A Dump object.
         policy: A Policy object.
         bucket_set: A BucketSet object.
         component_name: A name of component for filtering.
         depth: An integer representing depth to be printed.
         out: An IO object to output.
     """
@@ skipping 38 matching lines @@
                             help='restrict to COMPONENT', metavar='COMPONENT')
 
   def do(self, sys_argv):
     options, args = self._parse_args(sys_argv, 2)
 
     dump_path = args[1]
     target_policy = args[2]
     component = options.component
 
     (bucket_set, dump) = Command.load_basic_files(dump_path, False)
-    policy_set = PolicySet.load(self._parse_policy_list(target_policy))
+    policy_set = PolicySet.load(Command._parse_policy_list(target_policy))
 
     with open(Command._find_prefix(dump_path) + '.maps', 'r') as maps_f:
       maps_lines = maps_f.readlines()
     PProfCommand._output(
         dump, policy_set[target_policy], bucket_set, maps_lines, component,
         sys.stdout)
 
     return 0
 
   @staticmethod
@@ skipping 82 matching lines @@
   COMMANDS = {
     'csv': CSVCommand,
     'expand': ExpandCommand,
     'json': JSONCommand,
     'list': ListCommand,
     'pprof': PProfCommand,
     'stacktrace': StacktraceCommand,
   }
 
   if len(sys.argv) < 2 or (not sys.argv[1] in COMMANDS):
-    sys.stderr.write("""Usage: %s <command> [options] [<args>]
+    sys.stderr.write("""Usage: dmprof <command> [options] [<args>]
 
 Commands:
    csv          Classify memory usage in CSV
    expand       Show all stacktraces contained in the specified component
    json         Classify memory usage in JSON
    list         Classify memory usage in simple listing format
    pprof        Format the profile dump so that it can be processed by pprof
    stacktrace   Convert runtime addresses to symbol names
 
 Quick Reference:
    dmprof csv [-p POLICY] <first-dump>
    dmprof expand <dump> <policy> <component> <depth>
    dmprof json [-p POLICY] <first-dump>
    dmprof list [-p POLICY] <first-dump>
    dmprof pprof [-c COMPONENT] <dump> <policy>
    dmprof stacktrace <dump>
-""" % (sys.argv[0]))
+""")
     sys.exit(1)
   action = sys.argv.pop(1)
 
   LOGGER.setLevel(logging.DEBUG)
   handler = logging.StreamHandler()
   handler.setLevel(logging.INFO)
   formatter = logging.Formatter('%(message)s')
   handler.setFormatter(formatter)
   LOGGER.addHandler(handler)
 
   try:
     errorcode = COMMANDS[action]().do(sys.argv)
   except ParsingException, e:
     errorcode = 1
     sys.stderr.write('Exit by parsing error: %s\n' % e)
 
   return errorcode
 
 
 if __name__ == '__main__':
   sys.exit(main())