[telemetry] Add support for composable process dumps in memory-infra

tools/telemetry/telemetry/timeline/memory_dump_event.py

Index: tools/telemetry/telemetry/timeline/memory_dump_event.py
diff --git a/tools/telemetry/telemetry/timeline/memory_dump_event.py b/tools/telemetry/telemetry/timeline/memory_dump_event.py
index fff2a677588e98427422ed09fdb16514454a119a..cf69156efeea701fec83e4836a219fcc80ccbca3 100644
--- a/tools/telemetry/telemetry/timeline/memory_dump_event.py
+++ b/tools/telemetry/telemetry/timeline/memory_dump_event.py
@@ -146,25 +146,73 @@ class ProcessMemoryDumpEvent(timeline_event.TimelineEvent):
     has_mmaps: True if the memory dump has mmaps information. If False then
         GetMemoryUsage will report all zeros.
   """
-  def __init__(self, process, event):
-    assert event['ph'] == 'v' and process.pid == event['pid']
-
-    super(ProcessMemoryDumpEvent, self).__init__(
-        'memory', 'memory_dump', event['ts'] / 1000.0, 0.0)
+  def __init__(self, process, event_id, thread_start):
+    super(ProcessMemoryDumpEvent, self).__init__('memory', 'memory_dump',
+                                                 thread_start / 1000.0, 0.0)
 
     self.process = process
-    self.dump_id = event['id']
+    self.dump_id = event_id
+    self.has_mmaps = False
+    self._allocators = None
+    self._buckets = None
+    self._raw_allocator_dumps = {}
+    self._raw_vm_regions = []
+
+  @property
+  def process_name(self):
+    return self.process.name
+
+  def _AddRegion(self, vm_region):
+    path = ''
+    category = ROOT_CATEGORY
+    while category:
+      path = posixpath.join(path, category.name)
+      self.GetMemoryBucket(path).AddRegion(vm_region['bs'])
+      mapped_file = vm_region['mf']
+      category = category.GetMatchingChild(mapped_file)
+
+  def __repr__(self):
+    values = ['pid=%d' % self.process.pid]
+    for key, value in sorted(self.GetMemoryUsage().iteritems()):
+      values.append('%s=%d' % (key, value))
+    values = ', '.join(values)
+    return '%s[%s]' % (type(self).__name__, values)

[petrcermak, 2016/01/07 18:59:28]

idea: There's no reason to put the square brackets here because the string
representation of list already contains them:

< '[%s]' % [1, 2, 3]

[ssid, 2016/01/07 20:46:09]

discussed offline.

+
+  def AddRawEvent(self, event):
+    """Add the raw event without processing.

[petrcermak, 2016/01/07 18:59:28]

supernit: s/the/a/

+
+    Add the allocator dumps and vm regions from the timeline event without
+    processing."""

[petrcermak, 2016/01/07 18:59:28]

The three quotes should be on a separate line.

+    assert event['ph'] == 'v' and self.process.pid == event['pid']
 
     try:
-      allocators_dict = event['args']['dumps']['allocators']
+      self._raw_allocator_dumps.update(event['args']['dumps']['allocators'])
+    except KeyError:
+      pass  # It's ok if any of those keys are not present.
+    try:
+      vm_regions = event['args']['dumps']['process_mmaps']['vm_regions']
+      assert not self.has_mmaps
+      assert not self._raw_vm_regions
+      self._raw_vm_regions += vm_regions
+      self.has_mmaps = True
     except KeyError:
-      allocators_dict = {}
+      pass  # It's ok if any of those keys are not present.
+
+  def ProcessAllocatorDumps(self):
+    """Process the raw memory dump data into structured stats.
+
+    Parse and aggregate the allocator dumps values and the vm regions so that
+    it is easy to retrieve and add memory usage values to telemetry results.
+    This method needs to be called before making calls to GetMemory* functions

[petrcermak, 2016/01/07 18:59:28]

nit: s/functions/methods/

+    of this class.
+    """
     self._allocators = {}
     parent_path = ''
     parent_has_size = False
-    for allocator_name, size_values in sorted(allocators_dict.iteritems()):
-      if ((allocator_name.startswith(parent_path) and parent_has_size)
-          or allocator_name.startswith('global/')):
+    for allocator_name, size_values in sorted(
+        self._raw_allocator_dumps.iteritems()):
+      if ((allocator_name.startswith(parent_path) and parent_has_size) or
+          allocator_name.startswith('global/')):
         continue
       parent_path = allocator_name + '/'
       parent_has_size = 'size' in size_values['attrs']
@@ -173,8 +221,8 @@ class ProcessMemoryDumpEvent(timeline_event.TimelineEvent):
       # For 'gpu/android_memtrack/*' we want to keep track of individual
       # components. E.g. 'gpu/android_memtrack/gl' will be stored as
       # 'android_memtrack_gl' in the allocators dict.
-      if (len(name_parts) == 3 and allocator_name == 'gpu'
-          and name_parts[1] == 'android_memtrack'):
+      if (len(name_parts) == 3 and allocator_name == 'gpu' and
+          name_parts[1] == 'android_memtrack'):
         allocator_name = '_'.join(name_parts[1:3])
       allocator = self._allocators.setdefault(allocator_name, {})
       for size_key, size_value in size_values['attrs'].iteritems():
@@ -185,36 +233,13 @@ class ProcessMemoryDumpEvent(timeline_event.TimelineEvent):
     try:
       self._allocators['malloc']['size'] -= self._allocators['tracing']['size']
     except KeyError:
-      pass # it's ok if any of those keys are not present
+      pass  # It's ok if any of those keys are not present.
+    self._raw_allocator_dumps = None
 
     self._buckets = {}
-    try:
-      vm_regions = event['args']['dumps']['process_mmaps']['vm_regions']
-    except KeyError:
-      vm_regions = []
-    self.has_mmaps = bool(vm_regions)
-    for vm_region in vm_regions:
+    for vm_region in self._raw_vm_regions:
       self._AddRegion(vm_region)
-
-  @property
-  def process_name(self):
-    return self.process.name
-
-  def _AddRegion(self, vm_region):
-    path = ''
-    category = ROOT_CATEGORY
-    while category:
-      path = posixpath.join(path, category.name)
-      self.GetMemoryBucket(path).AddRegion(vm_region['bs'])
-      mapped_file = vm_region['mf']
-      category = category.GetMatchingChild(mapped_file)
-
-  def __repr__(self):
-    values = ['pid=%d' % self.process.pid]
-    for key, value in sorted(self.GetMemoryUsage().iteritems()):
-      values.append('%s=%d' % (key, value))
-    values = ', '.join(values)
-    return '%s[%s]' % (type(self).__name__, values)
+    self._raw_vm_regions = None
 
   def GetMemoryBucket(self, path):
     """Return the MemoryBucket associated with a category path.
@@ -259,8 +284,10 @@ class ProcessMemoryDumpEvent(timeline_event.TimelineEvent):
       if 'memtrack_pss' in values:
         usage[name] = values['memtrack_pss']
     if self.has_mmaps:
-      usage.update((key, self.GetMemoryValue(*value))
-                   for key, value in MMAPS_METRICS.iteritems())
+      for key, value in MMAPS_METRICS.iteritems():
+        result = self.GetMemoryValue(*value)
+        if result:
+          usage[key] = result
     return usage