Make VR latency results compatible with perf dashboard, refactor into classes

chrome/test/vr/perf/latency/webvr_latency_test.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import motopho_thread as mt
+import robot_arm as ra
+
+import json
+import glob
+import logging
+import os
+import re
+import subprocess
+import sys
+import time
+
+MOTOPHO_THREAD_TIMEOUT = 30
+
+def GetTtyDevices(tty_pattern, vendor_ids):
+  """Find all devices connected to tty that match a pattern and device id.

[Lei Lei, 2017/04/17 22:57:13]

s/Find/Finds

[bsheedy, 2017/04/18 21:17:26]

Done.

+
+  If a serial device is connected to the computer via USB, this function
+  will check all tty devices that match tty_pattern, and return the ones
+  that have vendor identification number in the list vendor_ids.
+
+  Args:
+    tty_pattern: The search pattern, such as r'ttyACM\d+'.
+    vendor_ids: The list of 16-bit USB vendor ids, such as [0x2a03].
+
+  Returns:
+    A list of strings of tty devices, for example ['ttyACM0'].
+  """
+  product_string = 'PRODUCT='
+  sys_class_dir = '/sys/class/tty/'
+
+  tty_devices = glob.glob(sys_class_dir + '*')
+
+  matcher = re.compile('.*' + tty_pattern)
+  tty_matches = [x for x in tty_devices if matcher.search(x)]
+  tty_matches = [x[len(sys_class_dir):] for x in tty_matches]
+
+  found_devices = []
+  for match in tty_matches:
+    class_filename = sys_class_dir + match + '/device/uevent'
+    with open(class_filename, 'r') as uevent_file:
+      # Look for the desired product id in the uevent text.
+      for line in uevent_file:
+        if product_string in line:
+          ids = line[len(product_string):].split('/')
+          ids = [int(x, 16) for x in ids]
+
+          for desired_id in vendor_ids:
+            if desired_id in ids:
+              found_devices.append(match)
+
+  return found_devices
+
+class WebVrLatencyTest(object):
+  """Base class for all WebVR latency tests.
+
+  This is meant to be subclassed for each platform the test is run on. While
+  the latency test itself is cross-platform, the setup and teardown for
+  tests is platform-dependent.
+  """
+  def __init__(self, args):
+    self.args = args
+    self.device_name = 'generic_device'
+    # Connect to the Arduino that drives the servos
+    devices = GetTtyDevices(r'ttyACM\d+', [0x2a03, 0x2341])
+    assert (len(devices) == 1),'Found %d devices, expected 1' % len(devices)
+    self.robot_arm = ra.RobotArm(devices[0])
+
+  def RunTest(self):
+    """Runs the steps to start Chrome, measure/save latency, and clean up."""
+    self._Setup()
+    self._Run()
+    self._Teardown()
+
+  def _Setup(self):
+    """Perform any platform-specific setup."""
+    raise NotImplementedError(
+        'Platform-specific setup must be implemented in subclass')
+
+  def _Run(self):
+    """Run the latency test.
+
+    Handles the actual latency measurement, which is identical across
+    different platforms, as well as result saving.
+    """
+    # Motopho scripts use relative paths, so switch to the Motopho directory
+    os.chdir(self.args.motopho_path)
+    # Start the Motopho script
+    motopho_thread = mt.MotophoThread()
+    motopho_thread.start()
+    # Let the Motopho be stationary so the script can calculate the bias
+    time.sleep(3)
+    # Move so we can measure latency
+    self.robot_arm.StartMotophoMovement()
+    motopho_thread.join(MOTOPHO_THREAD_TIMEOUT)
+    if motopho_thread.isAlive():
+      # TODO(bsheedy): Look into ways to prevent Motopho from not sending any
+      # data until unplugged and replugged into the machine after a reboot.
+      logging.error('Motopho thread timeout, Motopho may need to be replugged.')
+    self.robot_arm.StopAllMovement()
+    logging.info('Latency: %s', motopho_thread.latency)
+    logging.info('Correlation: %s', motopho_thread.max_correlation)
+    self._SaveResults(motopho_thread.latency, motopho_thread.max_correlation)
+
+  def _Teardown(self):
+    """Performs any platform-specific teardown."""
+    raise NotImplementedError(
+        'Platform-specific setup must be implemented in subclass')
+
+  def _RunCommand(self, cmd):
+    """Runs the given cmd list and returns its output.
+
+    Prints the command's output and exits if any error occurs.
+
+    Returns:
+      A string containing the stdout and stderr of the command.
+    """
+    try:
+      return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+      logging.error('Failed command output: %s', e.output)
+      raise e
+
+  def _SetChromeCommandLineFlags(self, flags):
+    raise NotImplementedError(
+        'Command-line flag setting must be implemented in subclass')
+
+  def _SaveResults(self, latency, correlation):
+    """Saves the results to a JSON file.
+
+    Saved JSON object is compatible with Chrome perf dashboard if
+    put in as the 'chart_data' value.
+    """
+    if not (self.args.output_dir and os.path.isdir(self.args.output_dir)):
+      logging.warning('No output directory set, not saving results to file')
+      return
+
+    latency_trace = {
+      'improvement_direction': 'down',
+      'name': 'latency',
+      'type': 'scalar',
+      'units': 'ms',
+      'value': latency or 999,
+    }
+
+    results = {
+      'format_version': '1.0',
+      'benchmark_name': 'webvr_latency',
+      'benchmark_description': 'Measures the motion-to-photon latency of WebVR',
+      'charts': {
+        self.device_name: {

[Lei Lei, 2017/04/17 22:57:13]

I thought we got agreement to use bot name for device name, how about
constructing the charts json data as following to keep consistent with other
perf data from telemetry tests?

'charts' : {
  'correlation': {
    '<flicker app url>': {
       .....
    }
    'summary' : {
      same as above
    }
  }
  'latency' : {
     '<flicker app url>': {
       .....
    }
    'summary' : {
      same as above
    }
  }
}

Moreover, can we run latency test multiple times and generate average value and
std deviation? Similar to
https://omnibot-isolate-server.appspot.com/browse?namespace=default-gzip&dige...

[bsheedy, 2017/04/18 21:17:25]

Done. Sample dashboard is
https://chromeperf.appspot.com/report?sid=2e9079f933c1bd542c9a87bdf05718b3fb6...

[Lei Lei, 2017/04/20 04:56:20]

Nice!

+          'correlation': {
+            'improvement_direction': 'up',
+            'name': 'correlation',
+            'type': 'scalar',
+            'units': '',
+            'value': correlation or -1,
+          },
+          # Points aren't added unless you have a summary trace, and can't seem
+          # to change the summary's name away from the chart name, so duplicate
+          # the latency information
+          'latency': latency_trace,
+          'summary': latency_trace,
+        }
+      }
+    }
+
+    with file(os.path.join(self.args.output_dir,
+                           self.args.results_file), 'w') as outfile:
+      json.dump(results, outfile)
+