Add script to process results from Android render tests.

build/android/render_tests/process_render_test_results.py

+#!/usr/bin/env python
+#
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import logging
+import os
+import re
+import shutil
+import sys
+import tempfile
+from PIL import ImageChops
+from PIL import Image
+
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))
+import devil_chromium
+from devil.android import device_utils
+from devil.utils import cmd_helper
+from pylib.constants import host_paths
+
+sys.path.append(os.path.abspath(
+    os.path.join(host_paths.DIR_SOURCE_ROOT, 'build')))
+import find_depot_tools  # pylint: disable=import-error,unused-import
+import download_from_google_storage
+
+sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'))
+import jinja2  # pylint: disable=F0401
+
+_RE_IMAGE_NAME = re.compile(
+    r'(?P<test_class>\w+)\.(?P<description>\w+)\.'
+    r'(?P<device_model>\w+)\.(?P<orientation>port|land)\.png')
+
+_RENDER_TEST_BASE_URL = 'https://storage.googleapis.com/chromium-render-tests/'
+_RENDER_TEST_BUCKET = 'gs://chromium-render-tests/'
+
+_JINJA_TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__))
+_JINJA_TEMPLATE_FILENAME = 'render_webpage.html.jinja2'
+
+
+def _UploadImage(upload_dir, image_file):
+  """Upload image to the render tests GS bucket."""

[mikecase (-- gone --), 2016/10/07 20:37:28]

Add doc about Return value here. (its just a url to link to in the results
webpage)

+  google_storage_path = os.path.join(
+      _RENDER_TEST_BUCKET, upload_dir, os.path.basename(image_file))
+  google_storage_url = os.path.join(
+      _RENDER_TEST_BASE_URL, upload_dir, os.path.basename(image_file))
+  cmd_helper.RunCmd(
+    [download_from_google_storage.GSUTIL_DEFAULT_PATH,
+     'cp', image_file, google_storage_path])
+  return google_storage_url
+
+
+def _ComputeImageDiff(failure_image, golden_image):
+  """Compute mask showing which pixels are different between two images."""
+  diff_image = ImageChops.difference(failure_image, golden_image)
+  diff_image = diff_image.convert('L')
+  diff_image = diff_image.point(lambda i: 255 if i else 0)
+  return diff_image
+
+
+def ProcessRenderTestResults(devices, render_results_dir,
+                             upload_dir, html_file):
+  """Grabs render results from device and generates webpage displaying results.
+
+  Args:
+    devices: List of DeviceUtils objects to grab results from.
+    render_results_dir: Directory on the devices to look for results. Assumes
+        directory given contains the golden images for the tests and a directory
+        "failures" containing images for failed tests.
+    upload_dir: Directory to upload the render test results to.
+    html_file: File to write the test results to.
+  """
+  results_dict = {}
+
+  temp_dir = None
+  try:
+    temp_dir = tempfile.mkdtemp()
+    for device in devices:
+      device.adb.Pull(render_results_dir, temp_dir)
+
+    failure_dir = os.path.join(temp_dir, 'failures')
+
+    for failure_filename in os.listdir(failure_dir):
+      m = _RE_IMAGE_NAME.match(failure_filename)
+      if not m:
+        continue
+
+      # Check to make sure we have golden image for this failure.
+      golden_file = os.path.join(temp_dir, failure_filename)
+      if not os.path.exists(golden_file):
+        logging.error('Cannot find golden image for %s', failure_filename)
+        continue
+
+      failure_file = os.path.join(failure_dir, failure_filename)
+
+      # Compute image diff between failure and golden.
+      diff_image = _ComputeImageDiff(
+          Image.open(failure_file), Image.open(golden_file))
+      diff_filename = '_diff'.join(
+          os.path.splitext(os.path.basename(failure_file)))
+      diff_file = os.path.join(temp_dir, diff_filename)
+      diff_image.save(diff_file)
+
+      failure_image_url = _UploadImage(upload_dir, failure_file)
+      golden_image_url = _UploadImage(upload_dir, golden_file)
+      diff_image_url = _UploadImage(upload_dir, diff_file)
+
+      test_class = m.group('test_class')
+      device_model = m.group('device_model')
+
+      if test_class not in results_dict:
+        results_dict[test_class] = {}
+
+      if device_model not in results_dict[test_class]:
+        results_dict[test_class][device_model] = []
+
+      results_dict[test_class][device_model].append({
+          'description': m.group('description'),
+          'orientation': m.group('orientation'),
+          'failure_image': failure_image_url,
+          'golden_image': golden_image_url,
+          'diff_image': diff_image_url,
+      })
+
+    jinja2_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(_JINJA_TEMPLATE_DIR),
+        trim_blocks=True)
+    template = jinja2_env.get_template(_JINJA_TEMPLATE_FILENAME)
+    processed_template_output = template.render(full_results=results_dict)
+    with open(html_file, "wb") as f:
+      f.write(processed_template_output)
+  finally:
+    if temp_dir:
+      shutil.rmtree(temp_dir)
+
+def main():
+  parser = argparse.ArgumentParser()
+
+  parser.add_argument('--render-results-dir',
+                      required=True,
+                      help='Path on device to look for render test images')
+  parser.add_argument('--output-html-file',
+                      required=True,
+                      help='File to output the results webpage.')
+  parser.add_argument('-d', '--device', dest='devices', action='append',
+                      default=[],
+                      help='Device to look for render test results on. '
+                           'Default is to look on all connected devices.')
+  parser.add_argument('--adb-path', type=os.path.abspath,
+                      help='Absolute path to the adb binary to use.')
+  parser.add_argument('--upload-dir',
+                      help='Root path to upload files to GS. Recommend this '
+                           'be set to a combination of buildername and build '
+                           'number for bots.')
+
+  args = parser.parse_args()
+  devil_chromium.Initialize(adb_path=args.adb_path)
+  devices = device_utils.DeviceUtils.HealthyDevices(device_arg=args.devices)
+  ProcessRenderTestResults(
+      devices, args.render_results_dir, args.upload_dir, args.output_html_file)
+
+
+if __name__ == '__main__':
+  sys.exit(main())