Add script to process results from Android render tests.

build/android/render_tests/process_render_test_results.py

+#!/usr/bin/env python
+#
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import collections
+import logging
+import os
+import posixpath
+import re
+import shutil
+import sys
+import tempfile
+import zipfile
+from PIL import ImageChops
+from PIL import Image
+
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))
+import devil_chromium
+from devil.android import device_utils
+from devil.utils import cmd_helper
+from pylib.constants import host_paths
+
+sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'build'))
+import find_depot_tools  # pylint: disable=import-error
+
+sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'))
+import jinja2  # pylint: disable=import-error
+
+_RE_IMAGE_NAME = re.compile(
+    r'(?P<test_class>\w+)\.'
+    r'(?P<description>\w+)\.'
+    r'(?P<device_model>\w+)\.'
+    r'(?P<orientation>port|land)\.png')
+
+_RENDER_TEST_BASE_URL = 'https://storage.googleapis.com/chromium-render-tests/'
+_RENDER_TEST_BUCKET = 'gs://chromium-render-tests/'
+
+_JINJA_TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__))
+_JINJA_TEMPLATE_FILENAME = 'render_webpage.html.jinja2'
+
+
+def _UploadFiles(upload_dir, files):
+  """Upload files to the render tests GS bucket."""
+  google_storage_upload_dir = os.path.join(_RENDER_TEST_BUCKET, upload_dir)
+  cmd = [os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py'),
+         '-m', 'cp']
+  cmd.extend(files)
+  cmd.append(google_storage_upload_dir)
+  cmd_helper.RunCmd(cmd)
+
+
+def _GoogleStorageUrl(upload_dir, filename):
+  return os.path.join(
+      _RENDER_TEST_BASE_URL, upload_dir, os.path.basename(filename))
+
+
+def _ComputeImageDiff(failure_image, golden_image):
+  """Compute mask showing which pixels are different between two images."""
+  return (ImageChops.difference(failure_image, golden_image)
+      .convert('L')
+      .point(lambda i: 255 if i else 0))
+
+
+def ProcessRenderTestResults(devices, render_results_dir,
+                             upload_dir, html_file):
+  """Grabs render results from device and generates webpage displaying results.
+
+  Args:
+    devices: List of DeviceUtils objects to grab results from.
+    render_results_path: Path where render test results are storage.
+        Will look for failures render test results on the device in
+        /sdcard/chromium_tests_root/<render_results_path>/failures/
+        and will look for golden images at Chromium src/<render_results_path>/.
+    upload_dir: Directory to upload the render test results to.
+    html_file: File to write the test results to.
+  """
+  results_dict = collections.defaultdict(lambda: collections.defaultdict(list))
+
+  diff_upload_dir = os.path.join(upload_dir, 'diffs')
+  failure_upload_dir = os.path.join(upload_dir, 'failures')
+  golden_upload_dir = os.path.join(upload_dir, 'goldens')
+
+  diff_images = []
+  failure_images = []
+  golden_images = []
+
+  temp_dir = None
+  try:
+    temp_dir = tempfile.mkdtemp()
+
+    for device in devices:
+      failures_device_dir = posixpath.join(
+          device.GetExternalStoragePath(),
+          'chromium_tests_root', render_results_dir, 'failures')
+      device.PullFile(failures_device_dir, temp_dir)
+
+    for failure_filename in os.listdir(temp_dir):
+      m = _RE_IMAGE_NAME.match(failure_filename)
+      if not m:
+        logging.warning(
+            'Unexpected file in render test failures, %s', failure_filename)
+        continue
+      failure_file = os.path.join(temp_dir, failure_filename)
+
+      # Check to make sure we have golden image for this failure.
+      golden_file = os.path.join(
+          host_paths.DIR_SOURCE_ROOT, render_results_dir, failure_filename)
+      if not os.path.exists(golden_file):
+        logging.error('Cannot find golden image for %s', failure_filename)
+        continue
+
+      # Compute image diff between failure and golden.
+      diff_image = _ComputeImageDiff(
+          Image.open(failure_file), Image.open(golden_file))
+      diff_filename = '_diff'.join(
+          os.path.splitext(os.path.basename(failure_file)))
+      diff_file = os.path.join(temp_dir, diff_filename)
+      diff_image.save(diff_file)
+
+      diff_images.append(diff_file)
+      failure_images.append(failure_file)
+      golden_images.append(golden_file)
+
+      test_class = m.group('test_class')
+      device_model = m.group('device_model')
+      results_dict[test_class][device_model].append({
+          'description': m.group('description'),
+          'orientation': m.group('orientation'),
+          'diff_image': _GoogleStorageUrl(diff_upload_dir, diff_file),
+          'failure_image': _GoogleStorageUrl(failure_upload_dir, failure_file),
+          'golden_image': _GoogleStorageUrl(golden_upload_dir, golden_file),
+      })
+
+    _UploadFiles(diff_upload_dir, diff_images)
+    _UploadFiles(failure_upload_dir, failure_images)
+    _UploadFiles(golden_upload_dir, golden_images)
+
+    if failure_images:
+      failures_zipfile = os.path.join(temp_dir, 'failures.zip')
+      with zipfile.ZipFile(failures_zipfile, mode='w') as zf:
+        for failure_file in failure_images:
+          zf.write(failure_file, os.path.join(
+              render_results_dir, os.path.basename(failure_file)))
+        failure_zip_url = _GoogleStorageUrl(upload_dir, failures_zipfile)
+      _UploadFiles(upload_dir, [failures_zipfile])

[jbudorick, 2016/11/07 01:44:45]

Why are we uploading the failures images both zipped and unzipped?

[mikecase (-- gone --), 2016/11/07 18:51:44]

Because, I want to display the images on the generated html page (which I will
link from the bot results).

And I want to let devs download the failures zip if they want to update the
golden images.

+    else:
+      failure_zip_url = None
+
+    jinja2_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(_JINJA_TEMPLATE_DIR),
+        trim_blocks=True)
+    template = jinja2_env.get_template(_JINJA_TEMPLATE_FILENAME)
+    #  pylint: disable=no-member

[jbudorick, 2016/11/07 01:44:45]

?

[mikecase (-- gone --), 2016/11/07 18:51:44]

Getting this error...

E:157,32: Instance of 'str' has no 'render' member (no-member)

...which seems to be a pylint bug....

https://github.com/PyCQA/pylint/issues/490

+    processed_template_output = template.render(
+        full_results=dict(results_dict), failure_zip_url=failure_zip_url)
+    with open(html_file, 'wb') as f:
+      f.write(processed_template_output)
+  finally:
+    if temp_dir:
+      shutil.rmtree(temp_dir)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+
+  parser.add_argument('--render-results-dir',
+                      required=True,
+                      help='Path on device to look for render test images')
+  parser.add_argument('--output-html-file',
+                      required=True,
+                      help='File to output the results webpage.')
+  parser.add_argument('-d', '--device', dest='devices', action='append',
+                      default=[],
+                      help='Device to look for render test results on. '
+                           'Default is to look on all connected devices.')
+  parser.add_argument('--adb-path', type=os.path.abspath,
+                      help='Absolute path to the adb binary to use.')
+  parser.add_argument('--buildername', type=str, required=True,
+                      help='Bot buildername. Used to generate path to upload '
+                           'render test results')
+  parser.add_argument('--build-number', type=str, required=True,
+                      help='Bot build number. Used to generate path to upload '
+                           'render test results')
+
+  args = parser.parse_args()
+  devil_chromium.Initialize(adb_path=args.adb_path)
+  devices = device_utils.DeviceUtils.HealthyDevices(device_arg=args.devices)
+
+  upload_dir = os.path.join(args.buildername, args.build_number)
+  ProcessRenderTestResults(
+      devices, args.render_results_dir, upload_dir, args.output_html_file)
+
+
+if __name__ == '__main__':
+  sys.exit(main())