Integrate perfvis profiling mode, and temporary report generator.

tools/telemetry/telemetry/core/platform/profiler/perfvis_profiler.py

Index: tools/telemetry/telemetry/core/platform/profiler/perfvis_profiler.py
diff --git a/tools/telemetry/telemetry/core/platform/profiler/perfvis_profiler.py b/tools/telemetry/telemetry/core/platform/profiler/perfvis_profiler.py
new file mode 100644
index 0000000000000000000000000000000000000000..8092454265abffe6b80d6d0101bf90fbc9b883b2
--- /dev/null
+++ b/tools/telemetry/telemetry/core/platform/profiler/perfvis_profiler.py
@@ -0,0 +1,315 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import os
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import pdb
+import json
+from datetime import date
+
+from telemetry.core import util
+from telemetry.core.platform import profiler
+from telemetry.core.platform.profiler import android_prebuilt_profiler_helper
+from telemetry.util import support_binaries
+
+util.AddDirToPythonPath(util.GetChromiumSrcDir(), 'build', 'android')
+from pylib.perf import perf_control  # pylint: disable=F0401
+
+timeline_metadata = {}
+
+class _AllProcessPerfProfiler(object):
+  """An internal class for using perf for a given process.
+
+  On android, this profiler uses pre-built binaries from AOSP.
+  See more details in prebuilt/android/README.txt.
+  """
+  def __init__(self, pid, output_file, browser_backend, platform_backend):
+    self._pid = pid
+    self._browser_backend = browser_backend
+    self._platform_backend = platform_backend
+    self._output_file = output_file
+    self._tmp_output_file = tempfile.NamedTemporaryFile('w', 0)
+    self._is_android = platform_backend.GetOSName() == 'android'
+    cmd_prefix = []
+    if self._is_android:
+      perf_binary = android_prebuilt_profiler_helper.GetDevicePath(
+          'perf')
+      cmd_prefix = ['adb', '-s', browser_backend.adb.device_serial(), 'shell',
+                    perf_binary]
+      output_file = os.path.join('/sdcard', 'perf_profiles',
+                                 os.path.basename(output_file))
+      self._device_output_file = output_file
+      browser_backend.adb.RunShellCommand(
+          'mkdir -p ' + os.path.dirname(self._device_output_file))
+    else:
+      cmd_prefix = ['perf']
+    # In perf 3.13 --call-graph requires an argument, so use
+    # the -g short-hand which does not.
+    proc_cmd = cmd_prefix +[
+        'record', '-g', '-r', '80', '-e', 'cycles', '-F', '2000',
+        '-a', '-R', '--output', output_file]
+
+    print "Start perf:", proc_cmd
+    self._proc = subprocess.Popen(proc_cmd,
+        stdout=self._tmp_output_file, stderr=subprocess.STDOUT)
+
+  def StopProfile(self):
+    if ('renderer' in self._output_file and
+        not self._is_android and
+        not self._platform_backend.GetCommandLine(self._pid)):
+      logging.warning('Renderer was swapped out during profiling. '
+                      'To collect a full profile rerun with '
+                      '"--extra-browser-args=--single-process"')
+    if self._is_android:
+      device = self._browser_backend.adb.device()
+      perf_pids = device.old_interface.ExtractPid('perf')
+      device.old_interface.RunShellCommand(
+          'kill -SIGINT ' + ' '.join(perf_pids))
+      util.WaitFor(lambda: not device.old_interface.ExtractPid('perf'),
+                   timeout=2)
+    self._proc.send_signal(signal.SIGINT)
+    exit_code = self._proc.wait()
+    try:
+      if exit_code == 128:
+        raise Exception(
+            """perf failed with exit code 128.
+Try rerunning this script under sudo or setting
+/proc/sys/kernel/perf_event_paranoid to "-1".\nOutput:\n%s""" %
+            self._GetStdOut())
+      elif exit_code not in (0, -2):
+        raise Exception(
+            'perf failed with exit code %d. Output:\n%s' % (exit_code,
+                                                            self._GetStdOut()))
+    finally:
+      self._tmp_output_file.close()
+
+  def CollectProfile(self):
+    perfvis_path = os.path.abspath(os.path.dirname(__file__)) + '/perf_vis/'
+    perfhost_path = \
+        os.path.abspath(support_binaries.FindPath('perfhost', 'linux'))
+    # Make perfhost executable.
+    os.chmod(perfhost_path, 0755)
+
+    symfs = []
+    if self._is_android:
+      print 'On Android, assuming $CHROMIUM_OUT_DIR/Release/lib has a fresh'
+      print 'symbolized library matching the one on device.'
+      objdump_path = os.path.join(os.environ.get('ANDROID_TOOLCHAIN',
+                                                 '$ANDROID_TOOLCHAIN'),
+                                  'arm-linux-androideabi-objdump')
+      print 'If you have recent version of perf (3.10+), append the following '
+      print 'to see annotated source code (by pressing the \'a\' key): '
+      print '  --objdump %s' % objdump_path
+
+      # Pull perf output file.
+      device = self._browser_backend.adb.device()
+      device.old_interface.Adb().Pull(
+          self._device_output_file, self._output_file)
+
+      # Parse required symbol libs.
+      required_libs = []
+      with open(os.devnull, 'w') as null_file:
+        repcmd = [perfhost_path, 'script', '-s',
+            perfvis_path + 'perf_to_tracing.py', '-i', self._output_file]
+        print "Parsing required libs...", repcmd
+        proc = subprocess.Popen(
+            repcmd, stdout=null_file, stderr=subprocess.PIPE)
+        regex = re.compile('Failed to open ([^,]+), continuing.*')
+        required_libs = [
+            m.group(1) for m in [
+                regex.match(line) for line in proc.stderr.readlines()] if m]
+        proc.wait()
+
+      symfs = self._PrepareAndroidSymfs(required_libs)
+
+    # Output bottom-up perf report
+    with open(self._output_file + '.json', 'w') as report_file:
+      repcmd = [perfhost_path, 'script', '-s', perfvis_path +
+          'perf_to_tracing.py', '-i', self._output_file] + symfs
+      print 'Generating tracing json with: ' + ' '.join(repcmd)
+      print ' ->', self._output_file + '.json'
+      subprocess.call(repcmd, stdout=report_file, stderr=subprocess.PIPE)
+
+    with open(self._output_file + 'meta.json', 'w') as meta_file:
+      json.dump(timeline_metadata, meta_file, indent=1)
+
+    device = self._browser_backend.adb.device()
+    max_freq = int(device.old_interface.GetFileContents(
+        '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq')[0]) * 1000
+
+    today = date.today()
+    output_ext = '_%02d%02d%02d.html' % (today.day, today.month, today.year)
+
+    repcmd = [perfvis_path + 'perf_vis.py',
+              '-m', self._output_file + 'meta.json',
+              '-o', self._output_file + output_ext,
+              '-c', str(max_freq),
+              self._output_file + '.json']
+    print 'Generating perf-vis with: ' + ' '.join(repcmd)
+    subprocess.call(
+        repcmd, stderr=subprocess.STDOUT)
+
+    return self._output_file
+
+  def _PrepareAndroidSymfs(self, required_libs):
+    """Create a symfs directory using an Android device.
+
+    Create a symfs directory by pulling the necessary files from an Android
+    device.
+
+    Returns:
+      List of arguments to be passed to perf to point it to the created symfs.
+    """
+    assert self._is_android
+    device = self._browser_backend.adb.device()
+    device.old_interface.Adb().Pull(self._device_output_file, self._output_file)
+    symfs_dir = os.path.dirname(self._output_file)
+    host_app_symfs = os.path.join(symfs_dir, 'data', 'app-lib')
+    if not os.path.exists(host_app_symfs):
+      print "starts with", self._browser_backend.package
+      os.makedirs(host_app_symfs)
+      os.makedirs(os.path.join(symfs_dir, 'data', 'app'))
+      # On Android, the --symfs parameter needs to map a directory structure
+      # similar to the device, that is:
+      # --symfs=/tmp/foobar and then inside foobar there'll be something like
+      # /tmp/foobar/data/app-lib/$PACKAGE/libname.so
+      # Assume the symbolized library under out/Release/lib is equivalent to
+      # the one in the device, and symlink it in the host to match --symfs.
+
+      # Also pull copies of required libraries from the device so perf can
+      # resolve their symbols.
+
+      for lib in required_libs:
+        #print "Lib", lib
+        if lib.startswith('/system/lib/') or lib.startswith('/vendor/lib/'):
+          print 'Pulling', lib
+          device.old_interface.Adb().Pull('/%s' % lib,
+                                          symfs_dir + '/%s' % lib)
+        elif lib.startswith('/data/') and self._browser_backend.package in lib:
+          # Assume the symbolized library under out/Release/lib is equivalent to
+          # the one in the device, and symlink it in the host to match --symfs.
+          chromepath = os.path.abspath(symfs_dir + '/%s' % lib)
+          chromedir = os.path.dirname(chromepath)
+          lib_basename =  os.path.basename(chromepath)
+          os.makedirs(chromedir)
+          print "Symlink '%s' lib to" % lib_basename, chromepath
+          os.symlink(os.path.abspath(
+                      os.path.join(util.GetChromiumSrcDir(),
+                                   os.environ.get('CHROMIUM_OUT_DIR', 'out'),
+                                   'Release', 'lib', lib_basename)),
+                     chromepath)
+    # Pull a copy of the kernel symbols.
+    host_kallsyms = os.path.join(symfs_dir, 'kallsyms')
+    if not os.path.exists(host_kallsyms):
+      device.old_interface.Adb().Pull('/proc/kallsyms', host_kallsyms)
+    return ['--kallsyms', host_kallsyms, '--symfs', symfs_dir]
+
+  def _GetStdOut(self):
+    self._tmp_output_file.flush()
+    try:
+      with open(self._tmp_output_file.name) as f:
+        return f.read()
+    except IOError:
+      return ''
+
+
+class PerfVisProfiler(profiler.Profiler):
+
+  def __init__(self, browser_backend, platform_backend, output_path, state):
+    super(PerfVisProfiler, self).__init__(
+        browser_backend, platform_backend, output_path, state)
+    print "New PerfVisProfiler..."
+    self._perf_control = None
+    if platform_backend.GetOSName() == 'android':
+      print " ...installing 'perf' to device."
+
+      device = browser_backend.adb.device()
+      android_prebuilt_profiler_helper.InstallOnDevice(
+          device, 'perf')
+      # Make sure kernel pointers are not hidden.
+      browser_backend.adb.device().old_interface.SetProtectedFileContents(
+          '/proc/sys/kernel/kptr_restrict', '0')
+
+      self._perf_control = perf_control.PerfControl(device)
+      self._perf_control.SetPerfProfilingMode()
+
+    process_output_file_map = self._GetProcessOutputFileMap()
+    self._process_profilers = []
+    self._browser_backend = browser_backend
+    self._platform_backend = platform_backend
+    for pid, output_file in process_output_file_map.iteritems():
+      if output_file.find('browser0') >= 0:
+        self._process_profilers.append(
+            _AllProcessPerfProfiler(
+                pid, output_file, browser_backend, platform_backend))
+        break
+
+  @classmethod
+  def name(cls):
+    return 'perfvis'
+
+  @classmethod
+  def is_supported(cls, browser_type):
+    if sys.platform != 'linux2':
+      return False
+    if browser_type.startswith('cros'):
+      return False
+    return True
+
+  @classmethod
+  def CustomizeBrowserOptions(cls, browser_type, options):
+    options.AppendExtraBrowserArgs([
+        '--no-sandbox',
+        '--allow-sandbox-debugging',
+    ])
+
+  @classmethod
+  def WillProfile(cls, browser_backend, platform_backend):
+    pass
+
+  def StopProfile(self):
+    # Stop all process profilers together.
+    for single_process in self._process_profilers:
+      single_process.StopProfile()
+
+    if self._perf_control:
+      self._perf_control.SetDefaultPerfMode()
+
+  def CollectProfile(self):
+    output_files = []
+    for single_process in self._process_profilers:
+      output_files.append(single_process.CollectProfile())
+    return output_files
+
+  @classmethod
+  def GetTopSamples(cls, file_name, number):
+    """Parses the perf generated profile in |file_name| and returns a
+    {function: period} dict of the |number| hottests functions.
+    """
+    assert os.path.exists(file_name)
+    with open(os.devnull, 'w') as devnull:
+      report = subprocess.Popen(
+          ['perf', 'report', '--show-total-period', '-U', '-t', '^', '-i',
+           file_name],
+          stdout=subprocess.PIPE, stderr=devnull).communicate()[0]
+    period_by_function = {}
+    for line in report.split('\n'):
+      if not line or line.startswith('#'):
+        continue
+      fields = line.split('^')
+      if len(fields) != 5:
+        continue
+      period = int(fields[1])
+      function = fields[4].partition(' ')[2]
+      function = re.sub('<.*>', '', function)  # Strip template params.
+      function = re.sub('[(].*[)]', '', function)  # Strip function params.
+      period_by_function[function] = period
+      if len(period_by_function) == number:
+        break
+    return period_by_function