Add automated VR latency tester

chrome/test/vr/automated_motopho_latency/run_latency_test.py

Index: chrome/test/vr/automated_motopho_latency/run_latency_test.py
diff --git a/chrome/test/vr/automated_motopho_latency/run_latency_test.py b/chrome/test/vr/automated_motopho_latency/run_latency_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d0fec7f6b7d4298c66cfad920c49c98a23b0d7f
--- /dev/null
+++ b/chrome/test/vr/automated_motopho_latency/run_latency_test.py
@@ -0,0 +1,261 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Script for automatically measuring motion-to-photon latency for VR.
+
+Doing so requires two specialized pieces of hardware. The first is a Motopho,
+which when used with a VR flicker app, finds the delay between movement and
+the test device's screen updating in response to the movement. The second is
+a ShakerBot, which physically moves the test device and Motopho during the

[Lei Lei, 2017/04/07 21:36:35]

I am not sure if it is ok to mention shakerbot public, since it is our internal
tool. Better to remove it.

[bsheedy, 2017/04/07 23:24:52]

Removed from the comment, and renamed the class to a very generic "RobotArm".

+latency test.
+"""
+
+import argparse
+import glob
+import httplib
+import os
+import re
+import serial
+import subprocess
+import sys
+import threading
+import time
+
+# ShakerBot connection constants
+BAUD_RATE = 115200
+CONNECTION_TIMEOUT = 3.0
+NUM_TRIES = 5
+# Motopho constants
+DEFAULT_ADB_PATH = ('/home/gtvchrome/tools/android/android-sdk-linux/'
+                    'platform-tools/adb')
+DEFAULT_MOTOPHO_PATH = '/home/gtvchrome/motopho/Motopho'
+MOTOPHO_THREAD_TIMEOUT = 30
+
+class MotophoThread(threading.Thread):
+  """Handles the running of the Motopho script and extracting results."""
+  def __init__(self):
+    threading.Thread.__init__(self)
+    self._latency = None
+    self._max_correlation = None
+
+  def run(self):

[Lei Lei, 2017/04/07 21:36:35]

s/run/Run to keep consistent about naming within file.

[bsheedy, 2017/04/07 23:24:52]

It's overriding threading.Thread's run() function.

+    motopho_output = ""
+    try:
+      motopho_output = subprocess.check_output(["./motophopro_nograph"],
+                                               stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+      print e.output

[Lei Lei, 2017/04/07 21:36:35]

It is better to use logging module for logs instead of printing it out to
console.

And you can also control the log level through command line. See
run_tests_helper.SetLogLevel(args.verbose_count)

https://cs.chromium.org/chromium/src/build/android/test_runner.py?q=test_runn...

[bsheedy, 2017/04/07 23:24:52]

Done.

+      return 1
+
+    if "FAIL" in motopho_output:
+      print 'Failed to get latency, printing raw output'
+      print motopho_output
+      return 1
+
+    self._latency = None
+    self._max_correlation = None
+    for line in motopho_output.split("\n"):
+      if 'Motion-to-photon latency:' in line:
+        self._latency = float(line.split(" ")[-2])
+      if 'Max correlation is' in line:
+        self._max_correlation = float(line.split(' ')[-1])
+      if self._latency and self._max_correlation:
+        break;
+
+  @property
+  def latency(self):
+    return self._latency
+
+  @property
+  def max_correlation(self):
+    return self._max_correlation
+
+
+class ShakerBot():
+  """Handles the serial communication with the ShakerBot."""
+  def __init__(self, device_name, num_tries, baud, timeout):
+    self._connection = None
+    connected = False
+    for attempt in xrange(num_tries):

[Lei Lei, 2017/04/07 21:36:35]

nit: attempt is unused, you can add unused prefix for unused variable.

unused_attempt

[bsheedy, 2017/04/07 23:24:52]

Replaced with the more standard "_" for unused variables.

+      try:
+        self._connection = serial.Serial('/dev/' + device_name,
+                                         baud,
+                                         timeout=timeout)
+      except serial.SerialException as e:
+        pass
+      if self._connection and 'Enter parameters' in self._connection.read(1024):
+        connected = True
+        break
+    if not connected:
+      print 'Failed to connect to ShakerBot'
+      sys.exit(1)

[Lei Lei, 2017/04/07 21:36:35]

It is better to throw exception here instead of exiting directly.

[bsheedy, 2017/04/07 23:24:52]

Done.

+
+  def StartMotophoMovement(self):
+    if not self._connection:
+      return
+    self._connection.write('9\n')
+
+  def StopAllMovement(self):
+    if not self._connection:
+      return
+    self._connection.write('0\n')
+
+
+def GetParsedArgs():
+  """Parses the command line arguments passed to the script.
+
+  Fails if any unknown arguments are present.
+  """
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--adb-path',
+                      type=os.path.realpath,
+                      help='The absolute path to adb',
+                      default=DEFAULT_ADB_PATH)
+  parser.add_argument('--motopho-path',
+                      type=os.path.realpath,
+                      help='The absolute path to the directory with Motopho '
+                           'scripts',
+                      default=DEFAULT_MOTOPHO_PATH)
+  (args, unknown_args) = parser.parse_known_args()
+  if unknown_args:
+    print 'Received unknown args: ' + ' '.join(unknown_args)
+    sys.exit(1)

[Lei Lei, 2017/04/07 21:36:35]

Considering to use parser.error('unrecognized arguments: %s' % '
'.join(unknown_args)) here.

See test_runner.py.
https://cs.chromium.org/chromium/src/build/android/test_runner.py?q=test_runn...

[bsheedy, 2017/04/07 23:24:52]

Done.

+  return args
+
+
+# Taken from Daydream's perf test suite
+def GetTtyDevices(tty_pattern, vendor_ids):
+  """Find all devices connected to tty that match a pattern and device id.
+
+  If a serial device is connected to the computer via USB, this function
+  will check all tty devices that match tty_pattern, and return the ones
+  that have vendor identification number in the list vendor_ids.
+
+  Args:
+    tty_pattern: The search pattern, such as r'ttyACM\d+'.
+    vendor_ids: The list of 16-bit USB vendor ids, such as [0x2a03].
+
+  Returns:
+    A list of strings of tty devices, for example ['ttyACM0'].
+  """
+  product_string = 'PRODUCT='
+  sys_class_dir = '/sys/class/tty/'
+
+  tty_devices = glob.glob(sys_class_dir + '*')
+
+  matcher = re.compile('.*' + tty_pattern)
+  tty_matches = [x for x in tty_devices if matcher.search(x)]
+  tty_matches = [x[len(sys_class_dir):] for x in tty_matches]
+
+  found_devices = []
+  for match in tty_matches:
+    class_filename = sys_class_dir + match + '/device/uevent'
+    with open(class_filename, 'r') as uevent_file:
+      # Look for the desired product id in the uevent text.
+      for line in uevent_file:
+        if product_string in line:
+          ids = line[len(product_string):].split('/')
+          ids = [int(x, 16) for x in ids]
+
+          for desired_id in vendor_ids:
+            if desired_id in ids:
+              found_devices.append(match)
+
+  return found_devices
+
+
+def RunCmdOrFail(cmd):

[Lei Lei, 2017/04/07 21:36:35]

Consider to rename it to RunCommand, no need for fail at the end.

[bsheedy, 2017/04/07 23:24:52]

Done.

+  """Runs the given cmd list.
+
+  Prints the command's output and exits if any error occurs.
+  """
+  try:
+    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+  except subprocess.CalledProcessError as e:
+    print e.output
+    sys.exit(1)

[Lei Lei, 2017/04/07 21:36:35]

It is better to throw the original exception here.

except ...
  logging.exception(...)
  throw e

[bsheedy, 2017/04/07 23:24:52]

Done.

+
+
+def SetChromeCommandLineFlags(adb_path, flags):
+  """Sets the given Chrome command line flags.
+
+  Puts them in both valid file locations.
+  """
+  flag_string = "echo 'chrome " + ' '.join(flags) + "' > "
+  RunCmdOrFail([adb_path,
+                'shell', flag_string + '/data/local/chrome-command-line'])
+  RunCmdOrFail([adb_path,
+                'shell', flag_string + '/data/local/tmp/chrome-command-line'])

[Lei Lei, 2017/04/07 21:36:35]

Why does it need to set the flags in two files? According to this
instruction(https://www.chromium.org/developers/how-tos/run-chromium-with-flags),
it only needs to update /data/local/tmp/chrome-command-line.

[bsheedy, 2017/04/07 23:24:52]

You're right, /data/local/chrome-command-line was kept around in some places for
backwards compatibility since it was used before M33, but that's not necessary
here.

+
+
+def main():
+  args = GetParsedArgs()
+
+  RunCmdOrFail([args.adb_path, 'root'])
+  RunCmdOrFail([args.adb_path, 'install', '-r', 'apks/ChromePublic.apk'])
+  # Force WebVR support and don't have first run experience
+  SetChromeCommandLineFlags(args.adb_path, ['--enable-webvr', '--disable-fre'])
+
+  # Motopho scripts use relative paths, so switch to the Motopho directory
+  os.chdir(args.motopho_path)
+
+  # Connect to the ShakerBot
+  devices = GetTtyDevices(r'ttyACM\d+', [0x2a03, 0x2341])
+  if len(devices) != 1:
+    print 'Found ' + str(len(devices)) + ' devices, expected 1'
+    return 1
+  shaker_bot = ShakerBot(devices[0], NUM_TRIES, BAUD_RATE, CONNECTION_TIMEOUT)
+
+  # Wake the device
+  RunCmdOrFail([args.adb_path, 'shell', 'input', 'keyevent', 'KEYCODE_WAKEUP'])
+  # Sleep a bit, otherwise WebGL can crash when Canary starts
+  time.sleep(1)
+
+  # Start Chrome and go to the flicker app
+  # TODO(bsheedy): See about having versioned copies of the flicker app instead
+  # of using personal github.
+  RunCmdOrFail([args.adb_path, 'shell', 'am', 'start',
+               '-a', 'android.intent.action.MAIN',
+               '-n', 'org.chromium.chrome/com.google.android.apps.chrome.Main',
+               'https://weableandbob.github.io/Motopho/flicker_apps/webvr/webvr-flicker-app-klaus.html?polyfill=0\&canvasClickPresents=1'])
+  time.sleep(10)
+
+  # Tap the screen to start presenting
+  RunCmdOrFail([args.adb_path, 'shell', 'input', 'touchscreen',
+                'tap', '800', '800'])
+  # Wait for VR to fully start up
+  time.sleep(5)
+
+  # Start the Motopho script
+  motopho_thread = MotophoThread()
+  motopho_thread.start()
+  # Let the Motopho be stationary so the script can calculate its bias
+  time.sleep(3)
+
+  # Move so we can measure latency
+  shaker_bot.StartMotophoMovement()
+  motopho_thread.join(MOTOPHO_THREAD_TIMEOUT)
+  if motopho_thread.isAlive():
+    # TODO(bsheedy): Look into ways to prevent Motopho from not sending any
+    # data until unplugged and replugged into the machine after a reboot.
+    print "Motopho thread join timed out - Motopho might need to be replugged."
+  shaker_bot.StopAllMovement()
+
+  print "Latency: " + str(motopho_thread.latency)
+  print "Max correlation: " + str(motopho_thread.max_correlation)

[Lei Lei, 2017/04/07 21:36:35]

The latency and max correlation are also needed to be stored in a file, so it
can be used to upload to chrome perf dashboard later.

You can define --output_dir flag and update gn_isolate_map.pyl as following:
"--output-dir=${ISOLATED_OUTDIR}",

https://cs.chromium.org/chromium/src/testing/buildbot/gn_isolate_map.pyl?q=gn...


ISOLATED_OUTDIR is reserved environment variable, it points to a local folder on
swarming bot, but all the files under that folder will be uploaded to swarming
and finally copied to the client.

[bsheedy, 2017/04/07 23:24:52]

Done.

+
+  # Exit VR and Close Chrome
+  # TODO(bsheedy): See about closing current tab before exiting so they don't
+  # pile up over time.
+  RunCmdOrFail([args.adb_path, 'shell', 'input', 'keyevent', 'KEYCODE_BACK'])
+  RunCmdOrFail([args.adb_path, 'shell', 'am', 'force-stop',
+                'org.chromium.chrome'])
+
+  # Turn off the screen
+  RunCmdOrFail([args.adb_path, 'shell', 'input', 'keyevent', 'KEYCODE_POWER'])
+
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main())