Add script to process results from Android render tests.

build/android/render_tests/process_render_test_results.py

+#!/usr/bin/env python
+#
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import logging
+import os
+import posixpath
+import re
+import shutil
+import sys
+import tempfile
+import zipfile
+from PIL import ImageChops
+from PIL import Image
+from collections import defaultdict

[jbudorick, 2016/10/19 16:11:32]

nit: import collections, then use collections.defaultdict.

[mikecase (-- gone --), 2016/10/19 19:04:44]

Done

+
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))
+import devil_chromium
+from devil.android import device_utils
+from devil.utils import cmd_helper
+from pylib.constants import host_paths
+
+sys.path.append(os.path.abspath(

[jbudorick, 2016/10/19 16:11:32]

nit: This doesn't need to be abspath, DIR_SOURCE_ROOT is already absolute.

[mikecase (-- gone --), 2016/10/19 19:04:44]

Done

+    os.path.join(host_paths.DIR_SOURCE_ROOT, 'build')))
+import find_depot_tools  # pylint: disable=import-error,unused-import
+import download_from_google_storage
+
+sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'))
+import jinja2  # pylint: disable=import-error
+
+_RE_IMAGE_NAME = re.compile(
+    r'(?P<test_class>\w+)\.(?P<description>\w+)\.'

[jbudorick, 2016/10/19 16:11:32]

super nit: I would split this s.t. each capture is on its own line.

[mikecase (-- gone --), 2016/10/19 19:04:44]

Done

+    r'(?P<device_model>\w+)\.(?P<orientation>port|land)\.png')
+
+_RENDER_TEST_BASE_URL = 'https://storage.googleapis.com/chromium-render-tests/'
+_RENDER_TEST_BUCKET = 'gs://chromium-render-tests/'
+
+_JINJA_TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__))
+_JINJA_TEMPLATE_FILENAME = 'render_webpage.html.jinja2'
+
+_CHROMIUM_SRC = os.path.join(

[jbudorick, 2016/10/19 16:11:32]

This is host_paths.DIR_SOURCE_ROOT.

[mikecase (-- gone --), 2016/10/19 19:04:43]

Done

+    os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)
+
+
+def _UploadFiles(upload_dir, files):
+  """Upload files to the render tests GS bucket."""
+  google_storage_upload_dir = os.path.join(_RENDER_TEST_BUCKET, upload_dir)
+  cmd = [download_from_google_storage.GSUTIL_DEFAULT_PATH, '-m', 'cp']

[jbudorick, 2016/10/19 16:11:33]

using "download_from_google_storage" to upload files... 🤔

Why not just os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py') ?

[mikecase (-- gone --), 2016/10/19 19:04:44]

I thought it was silly too.

upload_to_google_storage imports this constant from download_from_google_storage
too.

https://cs.chromium.org/chromium/tools/depot_tools/upload_to_google_storage.p...

I can change it to suggestion though.

+  cmd.extend(files)
+  cmd.append(google_storage_upload_dir)
+  cmd_helper.RunCmd(cmd)
+
+
+def _GoogleStorageUrl(upload_dir, filename):
+  return os.path.join(
+      _RENDER_TEST_BASE_URL, upload_dir, os.path.basename(filename))
+
+
+def _ComputeImageDiff(failure_image, golden_image):
+  """Compute mask showing which pixels are different between two images."""
+  return (ImageChops.difference(failure_image, golden_image)
+      .convert('L')
+      .point(lambda i: 255 if i else 0))
+
+
+def ProcessRenderTestResults(devices, render_results_dir,
+                             upload_dir, html_file):
+  """Grabs render results from device and generates webpage displaying results.
+
+  Args:
+    devices: List of DeviceUtils objects to grab results from.
+    render_results_path: Path where render test results are storage.
+        Will look for failures render test results on the device in
+        /sdcard/chromium_tests_root/<render_results_path>/failures/
+        and will look for golden images at Chromium src/<render_results_path>/.
+    upload_dir: Directory to upload the render test results to.
+    html_file: File to write the test results to.
+  """
+  results_dict = defaultdict(lambda: defaultdict(list))
+
+  diff_upload_dir = os.path.join(upload_dir, 'diffs')
+  failure_upload_dir = os.path.join(upload_dir, 'failures')
+  golden_upload_dir = os.path.join(upload_dir, 'goldens')
+
+  image_files = {

[jbudorick, 2016/10/19 16:11:33]

Why aren't these just three separate lists?

[mikecase (-- gone --), 2016/10/19 19:04:44]

Done

+    'diffs': [],
+    'failures': [],
+    'goldens': [],
+  }
+
+  temp_dir = None
+  try:
+    temp_dir = tempfile.mkdtemp()
+
+    for device in devices:
+      failures_device_dir = posixpath.join(
+          device.GetExternalStoragePath(),
+          'chromium_tests_root', render_results_dir, 'failures')
+      device.adb.Pull(failures_device_dir, temp_dir)

[jbudorick, 2016/10/19 16:11:32]

device.PullFile plz

[mikecase (-- gone --), 2016/10/19 19:04:44]

Hmmm, didn't know this would work. Name and docstring suggested this function
only works on single File whereas I want to pull directory.

+
+    for failure_filename in os.listdir(temp_dir):
+      m = _RE_IMAGE_NAME.match(failure_filename)
+      if not m:

[jbudorick, 2016/10/19 16:11:33]

This should log something if we hit an unexpected file name.

[mikecase (-- gone --), 2016/10/19 19:04:44]

Done

+        continue
+      failure_file = os.path.join(temp_dir, failure_filename)
+
+      # Check to make sure we have golden image for this failure.
+      golden_file = os.path.join(
+          _CHROMIUM_SRC, render_results_dir, failure_filename)
+      if not os.path.exists(golden_file):
+        logging.error('Cannot find golden image for %s', failure_filename)
+        continue
+
+      # Compute image diff between failure and golden.
+      diff_image = _ComputeImageDiff(
+          Image.open(failure_file), Image.open(golden_file))
+      diff_filename = '_diff'.join(
+          os.path.splitext(os.path.basename(failure_file)))
+      diff_file = os.path.join(temp_dir, diff_filename)
+      diff_image.save(diff_file)
+
+      image_files['diffs'].append(diff_file)
+      image_files['failures'].append(failure_file)
+      image_files['goldens'].append(golden_file)

[mikecase (-- gone --), 2016/10/18 16:00:13]

I dont upload the file immediately since it is much much faster to upload them
all at once in parallel using "gsutil -m cp" command.

+
+      test_class = m.group('test_class')
+      device_model = m.group('device_model')
+      results_dict[test_class][device_model].append({
+          'description': m.group('description'),
+          'orientation': m.group('orientation'),
+          'diff_image': _GoogleStorageUrl(diff_upload_dir, diff_file),
+          'failure_image': _GoogleStorageUrl(failure_upload_dir, failure_file),
+          'golden_image': _GoogleStorageUrl(golden_upload_dir, golden_file),
+      })
+
+    failures_zipfile = os.path.join(temp_dir, 'failures.zip')

[mikecase (-- gone --), 2016/10/19 19:04:44]

Added check so that no zip is created/uploaded if there are no failures.

+    with zipfile.ZipFile(failures_zipfile, mode='w') as zf:
+      for failure_file in image_files['failures']:
+        zf.write(failure_file, os.path.join(
+            render_results_dir, os.path.basename(failure_file)))
+      failure_zip_url = _GoogleStorageUrl(upload_dir, failures_zipfile)
+
+    _UploadFiles(diff_upload_dir, image_files['diffs'])
+    _UploadFiles(failure_upload_dir, image_files['failures'])
+    _UploadFiles(golden_upload_dir, image_files['goldens'])
+    _UploadFiles(upload_dir, [failures_zipfile])
+
+    jinja2_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(_JINJA_TEMPLATE_DIR),
+        trim_blocks=True)
+    template = jinja2_env.get_template(_JINJA_TEMPLATE_FILENAME)
+    processed_template_output = template.render(
+        full_results=dict(results_dict), failure_zip_url=failure_zip_url)
+    with open(html_file, "wb") as f:

[jbudorick, 2016/10/19 16:11:33]

nit: single quotes

[mikecase (-- gone --), 2016/10/19 19:04:44]

Done

+      f.write(processed_template_output)
+  finally:
+    if temp_dir:

[jbudorick, 2016/10/19 16:11:32]

we really ought to extract
https://codesearch.chromium.org/chromium/src/build/android/gyp/util/build_uti...
into somewhere more generally usable.

[mikecase (-- gone --), 2016/10/19 19:04:44]

Ack

+      shutil.rmtree(temp_dir)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+
+  parser.add_argument('--render-results-dir',
+                      required=True,
+                      help='Path on device to look for render test images')
+  parser.add_argument('--output-html-file',
+                      required=True,
+                      help='File to output the results webpage.')
+  parser.add_argument('-d', '--device', dest='devices', action='append',
+                      default=[],
+                      help='Device to look for render test results on. '
+                           'Default is to look on all connected devices.')
+  parser.add_argument('--adb-path', type=os.path.abspath,
+                      help='Absolute path to the adb binary to use.')
+  parser.add_argument('--buildername', type=str, required=True,
+                      help='Bot buildername. Used to generate path to upload '
+                           'render test results')
+  parser.add_argument('--build-number', type=str, required=True,
+                      help='Bot build number. Used to generate path to upload '
+                           'render test results')
+
+  args = parser.parse_args()
+  devil_chromium.Initialize(adb_path=args.adb_path)
+  devices = device_utils.DeviceUtils.HealthyDevices(device_arg=args.devices)
+
+  upload_dir = os.path.join(args.buildername, args.build_number)
+  ProcessRenderTestResults(
+      devices, args.render_results_dir, upload_dir, args.output_html_file)
+
+
+if __name__ == '__main__':
+  sys.exit(main())