[tracing] Add native allocation tracing mode.

tools/trace/symbolize.py

+#!/usr/bin/env python

[Primiano Tucci (use gerrit), 2016/04/07 15:51:57]

Did you figure out with perezju where to move this?
Maybe Let's take it out of this CL to make it more reviawable?

+# Copyright (c) 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import bisect
+import collections
+import gzip
+import json
+import os
+import posixpath
+import sys
+
+# This is how tools/binary_size/run_binary_size_analysis.py does it.
+# See http://crbug.com/375725
+elf_symbolizer_path = os.path.abspath(os.path.join(
+    os.path.dirname(__file__),
+    '..',
+    '..',
+    'build',
+    'android',
+    'pylib'))
+sys.path.append(elf_symbolizer_path)
+import symbols.elf_symbolizer as elf_symbolizer  # pylint: disable=F0401
+
+
+class ProcessMemoryMaps(object):
+  Region = collections.namedtuple(
+      'Region',
+      ['start_address', 'end_address', 'file_name'])
+
+  def __init__(self, value):
+    self._regions = []
+    for region_value in value['vm_regions']:
+      start_address = long(region_value['sa'], 16)
+      size = long(region_value['sz'], 16)
+      self._regions.append(self.Region(
+          start_address,
+          start_address + size,
+          region_value['mf']))
+
+    self._regions.sort()
+
+    # Remove duplicates; check for overlaps
+    region_index = 0
+    while region_index < len(self._regions):
+      region = self._regions[region_index]
+      if region_index != 0:
+        last_region = self._regions[region_index - 1]
+        if last_region == region:
+          del self._regions[region_index]
+          continue
+
+        assert region.start_address >= last_region.end_address, \
+            'Regions {} and {} overlap.'.format(last_region, region)
+      region_index += 1
+
+  @property
+  def regions(self):
+    return self._regions
+
+  def FindRegion(self, address):
+    if not self._regions:
+      return None
+    region_index = bisect.bisect_left(
+        self._regions, self.Region(address, None, None))
+    region = self._regions[region_index]
+    if region.start_address > address and region_index != 0:
+      region_index -= 1
+      region = self._regions[region_index]
+    if address >= region.start_address and address < region.end_address:
+      return region
+    return None
+
+
+class StackFrames(object):
+  def __init__(self, value):
+    self._frames_value = value
+
+  def CollectPCs(self):
+    return [pc for _, pc in self._IteratePCs()]
+
+  def SymbolizePCs(self, pc_symbol_map):
+    symbolized = False
+    for frame, pc in self._IteratePCs():
+      if pc in pc_symbol_map:
+        frame['name'] = pc_symbol_map[pc]
+        symbolized = True
+    return symbolized
+
+  # Details
+
+  _PC_TAG = 'pc:'
+
+  def _ParsePC(self, name):
+    if not name.startswith(self._PC_TAG):
+      return None
+    return long(name[len(self._PC_TAG):], 16)
+
+  def _IteratePCs(self):
+    for frame in self._frames_value.itervalues():
+      pc = self._ParsePC(frame['name'])
+      if pc is not None:
+        yield (frame, pc)
+
+
+class Symbolizer(object):
+  def __init__(self, binary_path, addr2line_path):
+    self._elf_symbolizer = None
+    self._pc_symbol_map = {}
+    self._failed_pcs = set()
+    self._queued_pcs = set()
+
+    if os.path.isfile(binary_path):
+      self._elf_symbolizer = elf_symbolizer.ELFSymbolizer(
+          binary_path,
+          addr2line_path,
+          self._SymbolizerCallback)
+
+  @property
+  def pc_symbol_map(self):
+    return self._pc_symbol_map
+
+  @property
+  def failed_pcs(self):
+      return self._failed_pcs
+
+  def SymbolizeAsync(self, pc, region):
+    if self._elf_symbolizer is None:
+      self._failed_pcs.add(pc)
+      return
+
+    if pc not in self._queued_pcs:
+      self._elf_symbolizer.SymbolizeAsync(int(pc - region.start_address), pc)
+      self._queued_pcs.add(pc)
+
+  def Join(self):
+    if self._elf_symbolizer is None:
+      return
+
+    self._elf_symbolizer.Join()
+
+  # Details
+
+  def _SymbolizerCallback(self, sym_info, pc):
+    self._queued_pcs.remove(pc)
+    if sym_info.name:
+      self._pc_symbol_map[pc] = sym_info.name
+    else:
+      self._failed_pcs.add(pc)
+
+
+class Process(object):
+  def __init__(self, pid):
+    self.pid = pid
+    self.name = None
+    self.mmaps = None
+    self.stack_frames = None
+
+
+def CollectProcesses(trace):
+  process_map = {}
+
+  for event in trace['traceEvents']:
+    name = event.get('name')
+    if not name:
+      continue
+
+    pid = event['pid']
+    process = process_map.get(pid)
+    if process is None:
+      process = Process(pid)
+      process_map[pid] = process
+
+    if name == 'process_name':
+      process.name = event['args']['name']
+    elif name == 'stackFrames':
+      value = event['args']['stackFrames']
+      process.stack_frames = StackFrames(value)
+    elif name == 'periodic_interval':
+      value = event['args']['dumps'].get('process_mmaps')
+      if value:
+        process.mmaps = ProcessMemoryMaps(value)
+
+  processes = []
+  for process in process_map.itervalues():
+    if process.mmaps is not None and process.stack_frames is not None:
+      processes.append(process)
+
+  return processes
+
+
+def SymbolizeProcess(process, addr2line_path):
+  pcs = process.stack_frames.CollectPCs()
+  if not pcs:
+    return False
+
+  print 'Symbolizing {} ({})...'.format(process.name, process.pid)
+
+  def _SubPrintf(message, *args):
+    print ('  ' + message).format(*args)
+
+  symbolizer_map = {}
+  unresolved_pcs = set()
+  for pc in pcs:
+    region = process.mmaps.FindRegion(pc)
+    if region is None:
+      unresolved_pcs.add(pc)
+      continue
+    symbolizer = symbolizer_map.get(region.file_name)
+    if symbolizer is None:
+      symbolizer = Symbolizer(region.file_name, addr2line_path)
+      symbolizer_map[region.file_name] = symbolizer
+    symbolizer.SymbolizeAsync(pc, region)
+
+  if unresolved_pcs:
+    _SubPrintf('{} PCs were not resolved.', len(unresolved_pcs))
+
+  pc_symbol_map = {}
+  for file_path, symbolizer in symbolizer_map.iteritems():
+    symbolizer.Join()
+    _SubPrintf('{}: {} PCs symbolized ({} failed)',
+               file_path,
+               len(symbolizer.pc_symbol_map),
+               len(symbolizer.failed_pcs))
+    pc_symbol_map.update(symbolizer.pc_symbol_map)
+
+  return process.stack_frames.SymbolizePCs(pc_symbol_map)
+
+
+def FindInSystemPath(binary_name):
+  paths = os.environ["PATH"].split(os.pathsep)
+  for path in paths:
+    binary_path = os.path.join(path, binary_name)
+    if os.path.isfile(binary_path):
+      return binary_path
+  return None
+
+
+def main():
+  BACKUP_FILE_TAG = '.BACKUP'
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument('file', nargs=1,
+                      help='Trace file to symbolize (.json or .json.gz)')
+  parser.add_argument('--no-backup', action='store_true',
+                      help="Don't create {} files".format(BACKUP_FILE_TAG))
+  options = parser.parse_args()
+
+  trace_file_path = options.file[0]
+  def _OpenTraceFile(mode):
+    if trace_file_path.endswith('.gz'):
+      return gzip.open(trace_file_path, mode)
+    else:
+      return open(trace_file_path, mode)
+
+  with _OpenTraceFile('rb') as trace_file:
+    trace = json.load(trace_file)
+
+  processes = CollectProcesses(trace)
+
+  addr2line_path = FindInSystemPath('addr2line')
+
+  update_trace = False
+  for process in processes:
+    if SymbolizeProcess(process, addr2line_path):
+      update_trace = True
+
+  if update_trace:
+    if not options.no_backup:
+      print 'Backing up trace file...'
+      os.rename(trace_file_path, trace_file_path + BACKUP_FILE_TAG)
+
+    print 'Updating trace file...'
+    with _OpenTraceFile('wb') as trace_file:
+      json.dump(trace, trace_file)
+
+
+if __name__ == '__main__':
+  main()