Refactor dmprof: split lib.Dump into lib.Dump and lib.DeepDump.

tools/deep_memory_profiler/lib/dump.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import copy
-import datetime
 import logging
 import os
-import re
-import time
-
-from lib.exceptions import EmptyDumpException, InvalidDumpException
-from lib.exceptions import ObsoleteDumpVersionException, ParsingException
-from lib.pageframe import PageFrame
-from lib.range_dict import ExclusiveRangeDict
-from lib.symbol import procfs
 
 
 LOGGER = logging.getLogger('dmprof')
-VIRTUAL, COMMITTED, ALLOC_COUNT, FREE_COUNT, _AT, BUCKET_ID = range(6)
-
-
-# Heap Profile Dump versions
-
-# DUMP_DEEP_[1-4] are obsolete.
-# DUMP_DEEP_2+ distinct mmap regions and malloc chunks.
-# DUMP_DEEP_3+ don't include allocation functions in their stack dumps.
-# DUMP_DEEP_4+ support comments with '#' and global stats "nonprofiled-*".
-# DUMP_DEEP_[1-2] should be processed by POLICY_DEEP_1.
-# DUMP_DEEP_[3-4] should be processed by POLICY_DEEP_2 or POLICY_DEEP_3.
-DUMP_DEEP_1 = 'DUMP_DEEP_1'
-DUMP_DEEP_2 = 'DUMP_DEEP_2'
-DUMP_DEEP_3 = 'DUMP_DEEP_3'
-DUMP_DEEP_4 = 'DUMP_DEEP_4'
-
-DUMP_DEEP_OBSOLETE = (DUMP_DEEP_1, DUMP_DEEP_2, DUMP_DEEP_3, DUMP_DEEP_4)
-
-# DUMP_DEEP_5 doesn't separate sections for malloc and mmap.
-# malloc and mmap are identified in bucket files.
-# DUMP_DEEP_5 should be processed by POLICY_DEEP_4.
-DUMP_DEEP_5 = 'DUMP_DEEP_5'
-
-# DUMP_DEEP_6 adds a mmap list to DUMP_DEEP_5.
-DUMP_DEEP_6 = 'DUMP_DEEP_6'
 
 
 class Dump(object):
   """Represents a heap profile dump."""
-
-  _PATH_PATTERN = re.compile(r'^(.*)\.([0-9]+)\.([0-9]+)\.heap$')
-
-  _HOOK_PATTERN = re.compile(
-      r'^ ([ \(])([a-f0-9]+)([ \)])-([ \(])([a-f0-9]+)([ \)])\s+'
-      r'(hooked|unhooked)\s+(.+)$', re.IGNORECASE)
-
-  _HOOKED_PATTERN = re.compile(r'(?P<TYPE>.+ )?(?P<COMMITTED>[0-9]+) / '
-                               '(?P<RESERVED>[0-9]+) @ (?P<BUCKETID>[0-9]+)')
-  _UNHOOKED_PATTERN = re.compile(r'(?P<TYPE>.+ )?(?P<COMMITTED>[0-9]+) / '
-                                 '(?P<RESERVED>[0-9]+)')
-
-  _OLD_HOOKED_PATTERN = re.compile(r'(?P<TYPE>.+) @ (?P<BUCKETID>[0-9]+)')
-  _OLD_UNHOOKED_PATTERN = re.compile(r'(?P<TYPE>.+) (?P<COMMITTED>[0-9]+)')
-
-  _TIME_PATTERN_FORMAT = re.compile(
-      r'^Time: ([0-9]+/[0-9]+/[0-9]+ [0-9]+:[0-9]+:[0-9]+)(\.[0-9]+)?')
-  _TIME_PATTERN_SECONDS = re.compile(r'^Time: ([0-9]+)$')
-
-  def __init__(self, path, modified_time):
-    self._path = path
-    matched = self._PATH_PATTERN.match(path)
-    self._pid = int(matched.group(2))
-    self._count = int(matched.group(3))
-    self._time = modified_time
-    self._map = {}
-    self._procmaps = ExclusiveRangeDict(ProcMapsEntryAttribute)
-    self._stacktrace_lines = []
-    self._global_stats = {} # used only in apply_policy
-
-    self._run_id = ''
-    self._pagesize = 4096
-    self._pageframe_length = 0
-    self._pageframe_encoding = ''
-    self._has_pagecount = False
-
-    self._version = ''
-    self._lines = []
+  def __init__(self):
+    pass
 
   @property
   def path(self):
-    return self._path
+    raise NotImplementedError
 
   @property
   def count(self):
-    return self._count
+    raise NotImplementedError
 
   @property
   def time(self):
-    return self._time
+    raise NotImplementedError
 
   @property
   def iter_map(self):
-    for region in sorted(self._map.iteritems()):
-      yield region[0], region[1]
+    raise NotImplementedError
 
   @property
   def iter_stacktrace(self):
-    for line in self._stacktrace_lines:
-      words = line.split()
-      yield (int(words[BUCKET_ID]),
-             int(words[VIRTUAL]),
-             int(words[COMMITTED]),
-             int(words[ALLOC_COUNT]),
-             int(words[FREE_COUNT]))
+    raise NotImplementedError
 
   def global_stat(self, name):
-    return self._global_stats[name]
+    raise NotImplementedError
 
   @property
   def run_id(self):
-    return self._run_id
+    raise NotImplementedError
 
   @property
   def pagesize(self):
-    return self._pagesize
+    raise NotImplementedError
 
   @property
   def pageframe_length(self):
-    return self._pageframe_length
+    raise NotImplementedError
 
   @property
   def pageframe_encoding(self):
-    return self._pageframe_encoding
+    raise NotImplementedError
 
   @property
   def has_pagecount(self):
-    return self._has_pagecount
+    raise NotImplementedError
 
   @staticmethod
   def load(path, log_header='Loading a heap profile dump: '):
     """Loads a heap profile dump.
 
     Args:
         path: A file path string to load.
         log_header: A preceding string for log messages.
 
     Returns:
         A loaded Dump object.
 
     Raises:
         ParsingException for invalid heap profile dumps.
     """
-    dump = Dump(path, os.stat(path).st_mtime)
+    from lib.deep_dump import DeepDump
+    dump = DeepDump(path, os.stat(path).st_mtime)
     with open(path, 'r') as f:
       dump.load_file(f, log_header)
     return dump
 
-  def load_file(self, f, log_header):
-    self._lines = [line for line in f
-                   if line and not line.startswith('#')]
-
-    try:
-      self._version, ln = self._parse_version()
-      self._parse_meta_information()
-      if self._version == DUMP_DEEP_6:
-        self._parse_mmap_list()
-      self._parse_global_stats()
-      self._extract_stacktrace_lines(ln)
-    except EmptyDumpException:
-      LOGGER.info('%s%s ...ignored an empty dump.' % (log_header, self._path))
-    except ParsingException, e:
-      LOGGER.error('%s%s ...error %s' % (log_header, self._path, e))
-      raise
-    else:
-      LOGGER.info('%s%s (version:%s)' % (log_header, self._path, self._version))
-
-  def _parse_version(self):
-    """Parses a version string in self._lines.
-
-    Returns:
-        A pair of (a string representing a version of the stacktrace dump,
-        and an integer indicating a line number next to the version string).
-
-    Raises:
-        ParsingException for invalid dump versions.
-    """
-    version = ''
-
-    # Skip until an identifiable line.
-    headers = ('STACKTRACES:\n', 'MMAP_STACKTRACES:\n', 'heap profile: ')
-    if not self._lines:
-      raise EmptyDumpException('Empty heap dump file.')
-    (ln, found) = skip_while(
-        0, len(self._lines),
-        lambda n: not self._lines[n].startswith(headers))
-    if not found:
-      raise InvalidDumpException('No version header.')
-
-    # Identify a version.
-    if self._lines[ln].startswith('heap profile: '):
-      version = self._lines[ln][13:].strip()
-      if version in (DUMP_DEEP_5, DUMP_DEEP_6):
-        (ln, _) = skip_while(
-            ln, len(self._lines),
-            lambda n: self._lines[n] != 'STACKTRACES:\n')
-      elif version in DUMP_DEEP_OBSOLETE:
-        raise ObsoleteDumpVersionException(version)
-      else:
-        raise InvalidDumpException('Invalid version: %s' % version)
-    elif self._lines[ln] == 'STACKTRACES:\n':
-      raise ObsoleteDumpVersionException(DUMP_DEEP_1)
-    elif self._lines[ln] == 'MMAP_STACKTRACES:\n':
-      raise ObsoleteDumpVersionException(DUMP_DEEP_2)
-
-    return (version, ln)
-
-  def _parse_global_stats(self):
-    """Parses lines in self._lines as global stats."""
-    (ln, _) = skip_while(
-        0, len(self._lines),
-        lambda n: self._lines[n] != 'GLOBAL_STATS:\n')
-
-    global_stat_names = [
-        'total', 'absent', 'file-exec', 'file-nonexec', 'anonymous', 'stack',
-        'other', 'nonprofiled-absent', 'nonprofiled-anonymous',
-        'nonprofiled-file-exec', 'nonprofiled-file-nonexec',
-        'nonprofiled-stack', 'nonprofiled-other',
-        'profiled-mmap', 'profiled-malloc']
-
-    for prefix in global_stat_names:
-      (ln, _) = skip_while(
-          ln, len(self._lines),
-          lambda n: self._lines[n].split()[0] != prefix)
-      words = self._lines[ln].split()
-      self._global_stats[prefix + '_virtual'] = int(words[-2])
-      self._global_stats[prefix + '_committed'] = int(words[-1])
-
-  def _parse_meta_information(self):
-    """Parses lines in self._lines for meta information."""
-    (ln, found) = skip_while(
-        0, len(self._lines),
-        lambda n: self._lines[n] != 'META:\n')
-    if not found:
-      return
-    ln += 1
-
-    while True:
-      if self._lines[ln].startswith('Time:'):
-        matched_seconds = self._TIME_PATTERN_SECONDS.match(self._lines[ln])
-        matched_format = self._TIME_PATTERN_FORMAT.match(self._lines[ln])
-        if matched_format:
-          self._time = time.mktime(datetime.datetime.strptime(
-              matched_format.group(1), '%Y/%m/%d %H:%M:%S').timetuple())
-          if matched_format.group(2):
-            self._time += float(matched_format.group(2)[1:]) / 1000.0
-        elif matched_seconds:
-          self._time = float(matched_seconds.group(1))
-      elif self._lines[ln].startswith('Reason:'):
-        pass  # Nothing to do for 'Reason:'
-      elif self._lines[ln].startswith('PageSize: '):
-        self._pagesize = int(self._lines[ln][10:])
-      elif self._lines[ln].startswith('CommandLine:'):
-        pass
-      elif (self._lines[ln].startswith('PageFrame: ') or
-            self._lines[ln].startswith('PFN: ')):
-        if self._lines[ln].startswith('PageFrame: '):
-          words = self._lines[ln][11:].split(',')
-        else:
-          words = self._lines[ln][5:].split(',')
-        for word in words:
-          if word == '24':
-            self._pageframe_length = 24
-          elif word == 'Base64':
-            self._pageframe_encoding = 'base64'
-          elif word == 'PageCount':
-            self._has_pagecount = True
-      elif self._lines[ln].startswith('RunID: '):
-        self._run_id = self._lines[ln][7:].strip()
-      elif (self._lines[ln].startswith('MMAP_LIST:') or
-            self._lines[ln].startswith('GLOBAL_STATS:')):
-        # Skip until "MMAP_LIST:" or "GLOBAL_STATS" is found.
-        break
-      else:
-        pass
-      ln += 1
-
-  def _parse_mmap_list(self):
-    """Parses lines in self._lines as a mmap list."""
-    (ln, found) = skip_while(
-        0, len(self._lines),
-        lambda n: self._lines[n] != 'MMAP_LIST:\n')
-    if not found:
-      return {}
-
-    ln += 1
-    self._map = {}
-    current_vma = {}
-    pageframe_list = []
-    while True:
-      entry = procfs.ProcMaps.parse_line(self._lines[ln])
-      if entry:
-        current_vma = {}
-        for _, _, attr in self._procmaps.iter_range(entry.begin, entry.end):
-          for key, value in entry.as_dict().iteritems():
-            attr[key] = value
-            current_vma[key] = value
-        ln += 1
-        continue
-
-      if self._lines[ln].startswith('  PF: '):
-        for pageframe in self._lines[ln][5:].split():
-          pageframe_list.append(PageFrame.parse(pageframe, self._pagesize))
-        ln += 1
-        continue
-
-      matched = self._HOOK_PATTERN.match(self._lines[ln])
-      if not matched:
-        break
-      # 2: starting address
-      # 5: end address
-      # 7: hooked or unhooked
-      # 8: additional information
-      if matched.group(7) == 'hooked':
-        submatched = self._HOOKED_PATTERN.match(matched.group(8))
-        if not submatched:
-          submatched = self._OLD_HOOKED_PATTERN.match(matched.group(8))
-      elif matched.group(7) == 'unhooked':
-        submatched = self._UNHOOKED_PATTERN.match(matched.group(8))
-        if not submatched:
-          submatched = self._OLD_UNHOOKED_PATTERN.match(matched.group(8))
-      else:
-        assert matched.group(7) in ['hooked', 'unhooked']
-
-      submatched_dict = submatched.groupdict()
-      region_info = { 'vma': current_vma }
-      if submatched_dict.get('TYPE'):
-        region_info['type'] = submatched_dict['TYPE'].strip()
-      if submatched_dict.get('COMMITTED'):
-        region_info['committed'] = int(submatched_dict['COMMITTED'])
-      if submatched_dict.get('RESERVED'):
-        region_info['reserved'] = int(submatched_dict['RESERVED'])
-      if submatched_dict.get('BUCKETID'):
-        region_info['bucket_id'] = int(submatched_dict['BUCKETID'])
-
-      if matched.group(1) == '(':
-        start = current_vma['begin']
-      else:
-        start = int(matched.group(2), 16)
-      if matched.group(4) == '(':
-        end = current_vma['end']
-      else:
-        end = int(matched.group(5), 16)
-
-      if pageframe_list and pageframe_list[0].start_truncated:
-        pageframe_list[0].set_size(
-            pageframe_list[0].size - start % self._pagesize)
-      if pageframe_list and pageframe_list[-1].end_truncated:
-        pageframe_list[-1].set_size(
-            pageframe_list[-1].size - (self._pagesize - end % self._pagesize))
-      region_info['pageframe'] = pageframe_list
-      pageframe_list = []
-
-      self._map[(start, end)] = (matched.group(7), region_info)
-      ln += 1
-
-  def _extract_stacktrace_lines(self, line_number):
-    """Extracts the position of stacktrace lines.
-
-    Valid stacktrace lines are stored into self._stacktrace_lines.
-
-    Args:
-        line_number: A line number to start parsing in lines.
-
-    Raises:
-        ParsingException for invalid dump versions.
-    """
-    if self._version in (DUMP_DEEP_5, DUMP_DEEP_6):
-      (line_number, _) = skip_while(
-          line_number, len(self._lines),
-          lambda n: not self._lines[n].split()[0].isdigit())
-      stacktrace_start = line_number
-      (line_number, _) = skip_while(
-          line_number, len(self._lines),
-          lambda n: self._check_stacktrace_line(self._lines[n]))
-      self._stacktrace_lines = self._lines[stacktrace_start:line_number]
-
-    elif self._version in DUMP_DEEP_OBSOLETE:
-      raise ObsoleteDumpVersionException(self._version)
-
-    else:
-      raise InvalidDumpException('Invalid version: %s' % self._version)
-
-  @staticmethod
-  def _check_stacktrace_line(stacktrace_line):
-    """Checks if a given stacktrace_line is valid as stacktrace.
-
-    Args:
-        stacktrace_line: A string to be checked.
-
-    Returns:
-        True if the given stacktrace_line is valid.
-    """
-    words = stacktrace_line.split()
-    if len(words) < BUCKET_ID + 1:
-      return False
-    if words[BUCKET_ID - 1] != '@':
-      return False
-    return True
-
 
 class DumpList(object):
   """Represents a sequence of heap profile dumps.
 
   Individual dumps are loaded into memory lazily as the sequence is accessed,
   either while being iterated through or randomly accessed.  Loaded dumps are
   not cached, meaning a newly loaded Dump object is returned every time an
   element in the list is accessed.
   """
 
   def __init__(self, dump_path_list):
     self._dump_path_list = dump_path_list
 
   @staticmethod
   def load(path_list):
     return DumpList(path_list)
 
   def __len__(self):
     return len(self._dump_path_list)
 
   def __iter__(self):
     for dump in self._dump_path_list:
       yield Dump.load(dump)
 
   def __getitem__(self, index):
     return Dump.load(self._dump_path_list[index])
-
-
-class ProcMapsEntryAttribute(ExclusiveRangeDict.RangeAttribute):
-  """Represents an entry of /proc/maps in range_dict.ExclusiveRangeDict."""
-  _DUMMY_ENTRY = procfs.ProcMapsEntry(
-      0,     # begin
-      0,     # end
-      '-',   # readable
-      '-',   # writable
-      '-',   # executable
-      '-',   # private
-      0,     # offset
-      '00',  # major
-      '00',  # minor
-      0,     # inode
-      ''     # name
-      )
-
-  def __init__(self):
-    super(ProcMapsEntryAttribute, self).__init__()
-    self._entry = self._DUMMY_ENTRY.as_dict()
-
-  def __str__(self):
-    return str(self._entry)
-
-  def __repr__(self):
-    return 'ProcMapsEntryAttribute' + str(self._entry)
-
-  def __getitem__(self, key):
-    return self._entry[key]
-
-  def __setitem__(self, key, value):
-    if key not in self._entry:
-      raise KeyError(key)
-    self._entry[key] = value
-
-  def copy(self):
-    new_entry = ProcMapsEntryAttribute()
-    for key, value in self._entry.iteritems():
-      new_entry[key] = copy.deepcopy(value)
-    return new_entry
-
-
-def skip_while(index, max_index, skipping_condition):
-  """Increments |index| until |skipping_condition|(|index|) is False.
-
-  Returns:
-      A pair of an integer indicating a line number after skipped, and a
-      boolean value which is True if found a line which skipping_condition
-      is False for.
-  """
-  while skipping_condition(index):
-    index += 1
-    if index >= max_index:
-      return index, False
-  return index, True