Improve parsing error handling in dmprof.

tools/deep_memory_profiler/dmprof

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """The deep heap profiler script for Chrome."""
 
 from datetime import datetime
 import json
 import os
@@ skipping 48 matching lines @@
 # POLICY_DEEP_2 DOES include allocation_type columns.
 # mmap regions are distincted w/ the allocation_type column.
 POLICY_DEEP_2 = 'POLICY_DEEP_2'
 
 # TODO(dmikurube): Avoid global variables.
 address_symbol_dict = {}
 appeared_addresses = set()
 components = []
 
 
+class ParsingException(Exception):
+  def __init__(self, value):
+    self.value = value
+  def __str__(self):
+    return repr(self.value)
+
+
 class Policy(object):
 
   def __init__(self, name, mmap, pattern):
     self.name = name
     self.mmap = mmap
     self.condition = re.compile(pattern + r'\Z')
 
 
 def get_component(policy_list, bucket, mmap):
   """Returns a component name which a given bucket belongs to.
@@ skipping 26 matching lines @@
 class Bucket(object):
 
   def __init__(self, stacktrace):
     self.stacktrace = stacktrace
     self.component = ''
 
 
 class Log(object):
 
   """A class representing one dumped log data."""
-  def __init__(self, log_path, buckets):
+  def __init__(self, log_path):
     self.log_path = log_path
     self.log_lines = [
         l for l in open(self.log_path, 'r') if l and not l.startswith('#')]
     self.log_version = ''
-    sys.stderr.write('parsing a log file:%s\n' % log_path)
+    sys.stderr.write('Loading a dump: %s\n' % log_path)
     self.mmap_stacktrace_lines = []
     self.malloc_stacktrace_lines = []
     self.counters = {}
     self.log_time = os.stat(self.log_path).st_mtime
-    self.parse_log(buckets)
 
   @staticmethod
   def dump_stacktrace_lines(stacktrace_lines, buckets):
     """Prints a given stacktrace.
 
     Args:
         stacktrace_lines: A list of strings which are valid as stacktraces.
         buckets: A dict mapping bucket ids and their corresponding Bucket
             objects.
     """
@@ skipping 136 matching lines @@
       return False
     bucket = buckets.get(int(words[BUCKET_ID]))
     if bucket:
       for address in bucket.stacktrace:
         appeared_addresses.add(address)
     return True
 
   @staticmethod
   def skip_lines_while(line_number, max_line_number, skipping_condition):
     """Increments line_number until skipping_condition(line_number) is false.
+
+    Returns:
+        A pair of an integer indicating a line number after skipped, and a
+        boolean value which is True if found a line which skipping_condition
+        is False for.
     """
     while skipping_condition(line_number):
       line_number += 1
       if line_number >= max_line_number:
-        sys.stderr.write('invalid heap profile dump.')
-        return line_number
-    return line_number
+        return line_number, False
+    return line_number, True
 
   def parse_stacktraces_while_valid(self, buckets, log_lines, line_number):
     """Parses stacktrace lines while the lines are valid.
 
     Args:
         buckets: A dict mapping bucket ids and their corresponding Bucket
             objects.
         log_lines: A list of lines to be parsed.
         line_number: An integer representing the starting line number in
             log_lines.
 
     Returns:
         A pair of a list of valid lines and an integer representing the last
         line number in log_lines.
     """
-    line_number = self.skip_lines_while(
+    (line_number, found) = self.skip_lines_while(

[M-A Ruel, 2012/04/16 12:20:55]

since found is not used, you can either:
line_number = self.skip_lines_while(...)[0]
or
(lin_number, _) = self.skip_lines_while(...)

[Dai Mikurube (NOT FULLTIME), 2012/04/17 03:26:32]

Done.

         line_number, len(log_lines),
         lambda n: not log_lines[n].split()[0].isdigit())
     stacktrace_lines_start = line_number
-    line_number = self.skip_lines_while(
+    (line_number, found) = self.skip_lines_while(
         line_number, len(log_lines),
         lambda n: self.check_stacktrace_line(log_lines[n], buckets))
     return (log_lines[stacktrace_lines_start:line_number], line_number)
 
   def parse_stacktraces(self, buckets, line_number):
     """Parses lines in self.log_lines as stacktrace.
 
     Valid stacktrace lines are stored into self.mmap_stacktrace_lines and
     self.malloc_stacktrace_lines.
 
     Args:
         buckets: A dict mapping bucket ids and their corresponding Bucket
             objects.
         line_number: An integer representing the starting line number in
             log_lines.
 
     Raises:
-        RuntimeException for invalid dump versions.
+        ParsingException for invalid dump versions.
     """
-    sys.stderr.write('  heap profile dump version: %s\n' % self.log_version)
+    sys.stderr.write('  Version: %s\n' % self.log_version)
 
     if self.log_version in (DUMP_DEEP_3, DUMP_DEEP_4):
       (self.mmap_stacktrace_lines, line_number) = (
           self.parse_stacktraces_while_valid(
               buckets, self.log_lines, line_number))
-      line_number = self.skip_lines_while(
+      (line_number, found) = self.skip_lines_while(
           line_number, len(self.log_lines),
           lambda n: self.log_lines[n] != 'MALLOC_STACKTRACES:\n')
       (self.malloc_stacktrace_lines, line_number) = (
           self.parse_stacktraces_while_valid(
               buckets, self.log_lines, line_number))
 
     elif self.log_version == DUMP_DEEP_2:
       (self.mmap_stacktrace_lines, line_number) = (
           self.parse_stacktraces_while_valid(
               buckets, self.log_lines, line_number))
-      line_number = self.skip_lines_while(
+      (line_number, found) = self.skip_lines_while(
           line_number, len(self.log_lines),
           lambda n: self.log_lines[n] != 'MALLOC_STACKTRACES:\n')
       (self.malloc_stacktrace_lines, line_number) = (
           self.parse_stacktraces_while_valid(
               buckets, self.log_lines, line_number))
       self.malloc_stacktrace_lines.extend(self.mmap_stacktrace_lines)
       self.mmap_stacktrace_lines = []
 
     elif self.log_version == DUMP_DEEP_1:
       (self.malloc_stacktrace_lines, line_number) = (
           self.parse_stacktraces_while_valid(
               buckets, self.log_lines, line_number))
 
     else:
-      raise RuntimeError('invalid heap profile dump version: %s' % (
+      raise ParsingException('invalid heap profile dump version: %s' % (
           self.log_version))
 
   def parse_global_stats(self):
     """Parses lines in self.log_lines as global stats."""
-    ln = self.skip_lines_while(
+    (ln, found) = self.skip_lines_while(
         0, len(self.log_lines),
         lambda n: self.log_lines[n] != 'GLOBAL_STATS:\n')
 
     if self.log_version == DUMP_DEEP_4:
       global_stat_names = [
           'total', 'file-exec', 'file-nonexec', 'anonymous', 'stack', 'other',
           'nonprofiled-absent', 'nonprofiled-anonymous',
           'nonprofiled-file-exec', 'nonprofiled-file-nonexec',
           'nonprofiled-stack', 'nonprofiled-other',
           'profiled-mmap', 'profiled-malloc']
     else:
       global_stat_names = [
           'total', 'file', 'anonymous', 'other', 'mmap', 'tcmalloc']
 
     for prefix in global_stat_names:
-      ln = self.skip_lines_while(
+      (ln, found) = self.skip_lines_while(
           ln, len(self.log_lines),
           lambda n: self.log_lines[n].split()[0] != prefix)
       words = self.log_lines[ln].split()
       self.counters[prefix + '_virtual'] = int(words[-2])
       self.counters[prefix + '_committed'] = int(words[-1])
 
   def parse_version(self):
     """Parses a version string in self.log_lines.
 
     Returns:
         A pair of (a string representing a version of the stacktrace dump,
         and an integer indicating a line number next to the version string).
 
     Raises:
-        RuntimeException for invalid dump versions.
+        ParsingException for invalid dump versions.
     """
     version = ''
 
     # Skip until an identifiable line.
     headers = ('STACKTRACES:\n', 'MMAP_STACKTRACES:\n', 'heap profile: ')
-    ln = self.skip_lines_while(
+    if len(self.log_lines) <= 0:

[M-A Ruel, 2012/04/16 12:20:55]

len() < 0?
Just use:
if not self.log_lines:

[Dai Mikurube (NOT FULLTIME), 2012/04/17 03:26:32]

Done.

+      raise ParsingException('Empty heap dump file.')
+    (ln, found) = self.skip_lines_while(
         0, len(self.log_lines),
         lambda n: not self.log_lines[n].startswith(headers))
+    if not found:
+      raise ParsingException('Invalid heap dump file (no version header).')
 
     # Identify a version.
     if self.log_lines[ln].startswith('heap profile: '):
       version = self.log_lines[ln][13:].strip()
       if (version == DUMP_DEEP_2 or version == DUMP_DEEP_3 or
           version == DUMP_DEEP_4):
-        ln = self.skip_lines_while(
+        (ln, found) = self.skip_lines_while(
             ln, len(self.log_lines),
             lambda n: self.log_lines[n] != 'MMAP_STACKTRACES:\n')
       else:
-        raise RuntimeError('invalid heap profile dump version: %s' % version)
+        raise ParsingException('invalid heap profile dump version: %s'
+                               % version)
     elif self.log_lines[ln] == 'STACKTRACES:\n':
       version = DUMP_DEEP_1
     elif self.log_lines[ln] == 'MMAP_STACKTRACES:\n':
       version = DUMP_DEEP_2
 
     return (version, ln)
 
   def parse_log(self, buckets):
     self.log_version, ln = self.parse_version()
     self.parse_global_stats()
@@ skipping 313 matching lines @@
         break
       n += 1
       continue
     sys.stderr.write('reading buckets from %s\n' % (buckets_path))
     with open(buckets_path, 'r') as buckets_f:
       for l in buckets_f:
         words = l.split()
         buckets[int(words[0])] = Bucket(words[1:])
     n += 1
 
-  sys.stderr.write('the number buckets: %d\n' % (bucket_count))
-
   log_path_list = [log_path]
 
   if action in ('--csv', '--json'):
     # search for the sequence of files
     n = int(log_path[len(log_path) - 9 : len(log_path) - 5])
     n += 1  # skip current file
     while True:
       p = '%s.%04d.heap' % (prefix, n)
       if os.path.exists(p):
         log_path_list.append(p)
       else:
         break
       n += 1
 
-  logs = [Log(path, buckets) for path in log_path_list]
+  logs = []
+  for path in log_path_list:
+    new_log = Log(path)
+    sys.stderr.write('Parsing a dump: %s\n' % path)
+    try:
+      new_log.parse_log(buckets)

[M-A Ruel, 2012/04/16 12:20:55]

The functional part of me dislike the fact that a member function 'loads' the
object and prefers something like:
new_log = Log.parse_log(path, buckets)
but I don't mind.

[Dai Mikurube (NOT FULLTIME), 2012/04/17 03:26:32]

I basically agree with you.  I'll do that in another change.

+    except ParsingException:
+      sys.stderr.write('  Ignored an invalid dump: %s\n' % path)
+    else:
+      logs.append(new_log)
 
   sys.stderr.write('getting symbols\n')
   update_symbols(symbol_path, maps_lines, chrome_path)
 
   # TODO(dmikurube): Many modes now.  Split them into separete functions.
   if action == '--stacktrace':
     logs[0].dump_stacktrace(buckets)
 
   elif action == '--csv':
     sys.stdout.write(','.join(components))
@@ skipping 41 matching lines @@
 
   elif action == '--pprof':
     if len(sys.argv) > 5:
       logs[0].dump_for_pprof(policy_list, buckets, maps_lines, sys.argv[5])
     else:
       logs[0].dump_for_pprof(policy_list, buckets, maps_lines, None)
 
 
 if __name__ == '__main__':
   sys.exit(main())