Add failure screenshots and render test images to results detail.

build/android/render_tests/process_render_test_results.py

-#!/usr/bin/env python
-#
-# Copyright 2016 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import argparse
-import collections
-import logging
-import os
-import posixpath
-import re
-import shutil
-import sys
-import tempfile
-import zipfile
-
-sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))
-import devil_chromium
-from devil.android import device_utils
-from devil.utils import cmd_helper
-from pylib.constants import host_paths
-
-sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'build'))
-import find_depot_tools  # pylint: disable=import-error
-
-sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'))
-import jinja2  # pylint: disable=import-error
-
-try:
-  from PIL import Image  # pylint: disable=import-error
-  from PIL import ImageChops  # pylint: disable=import-error
-  can_compute_diffs = True
-except ImportError:
-  can_compute_diffs = False
-  logging.exception('Error importing PIL library. Image diffs will not be '
-                    'displayed properly unless PIL module is installed.')
-
-_RE_IMAGE_NAME = re.compile(
-    r'(?P<test_class>\w+)\.'
-    r'(?P<description>\w+)\.'
-    r'(?P<device_model>\w+)\.'
-    r'(?P<orientation>port|land)\.png')
-
-_RENDER_TEST_BASE_URL = 'https://storage.googleapis.com/chromium-render-tests/'
-_RENDER_TEST_BUCKET = 'gs://chromium-render-tests/'
-
-_JINJA_TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__))
-_JINJA_TEMPLATE_FILENAME = 'render_webpage.html.jinja2'
-
-
-def _UploadFiles(upload_dir, files):
-  """Upload files to the render tests GS bucket."""
-  if files:
-    google_storage_upload_dir = os.path.join(_RENDER_TEST_BUCKET, upload_dir)
-    cmd = [os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py'),
-           '-m', 'cp']
-    cmd.extend(files)
-    cmd.append(google_storage_upload_dir)
-    cmd_helper.RunCmd(cmd)
-
-
-def _GoogleStorageUrl(upload_dir, filename):
-  return os.path.join(
-      _RENDER_TEST_BASE_URL, upload_dir, os.path.basename(filename))
-
-
-def _ComputeImageDiff(failure_image, golden_image):
-  """Compute mask showing which pixels are different between two images."""
-  return (ImageChops.difference(failure_image, golden_image)
-      .convert('L')
-      .point(lambda i: 255 if i else 0))
-
-
-def ProcessRenderTestResults(devices, render_results_dir,
-                             upload_dir, html_file):
-  """Grabs render results from device and generates webpage displaying results.
-
-  Args:
-    devices: List of DeviceUtils objects to grab results from.
-    render_results_path: Path where render test results are storage.
-        Will look for failures render test results on the device in
-        /sdcard/chromium_tests_root/<render_results_path>/failures/
-        and will look for golden images at Chromium src/<render_results_path>/.
-    upload_dir: Directory to upload the render test results to.
-    html_file: File to write the test results to.
-  """
-  results_dict = collections.defaultdict(lambda: collections.defaultdict(list))
-
-  diff_upload_dir = os.path.join(upload_dir, 'diffs')
-  failure_upload_dir = os.path.join(upload_dir, 'failures')
-  golden_upload_dir = os.path.join(upload_dir, 'goldens')
-
-  diff_images = []
-  failure_images = []
-  golden_images = []
-
-  temp_dir = None
-  try:
-    temp_dir = tempfile.mkdtemp()
-
-    for device in devices:
-      failures_device_dir = posixpath.join(
-          device.GetExternalStoragePath(),
-          'chromium_tests_root', render_results_dir, 'failures')
-      device.PullFile(failures_device_dir, temp_dir)
-
-    for failure_filename in os.listdir(os.path.join(temp_dir, 'failures')):
-      m = _RE_IMAGE_NAME.match(failure_filename)
-      if not m:
-        logging.warning(
-            'Unexpected file in render test failures, %s', failure_filename)
-        continue
-      failure_file = os.path.join(temp_dir, 'failures', failure_filename)
-
-      # Check to make sure we have golden image for this failure.
-      golden_file = os.path.join(
-          host_paths.DIR_SOURCE_ROOT, render_results_dir, failure_filename)
-      if not os.path.exists(golden_file):
-        logging.error('Cannot find golden image for %s', failure_filename)
-        continue
-
-      # Compute image diff between failure and golden.
-      if can_compute_diffs:
-        diff_image = _ComputeImageDiff(
-            Image.open(failure_file), Image.open(golden_file))
-        diff_filename = '_diff'.join(
-            os.path.splitext(os.path.basename(failure_file)))
-        diff_file = os.path.join(temp_dir, diff_filename)
-        diff_image.save(diff_file)
-        diff_images.append(diff_file)
-
-      failure_images.append(failure_file)
-      golden_images.append(golden_file)
-
-      test_class = m.group('test_class')
-      device_model = m.group('device_model')
-
-      results_entry = {
-          'description': m.group('description'),
-          'orientation': m.group('orientation'),
-          'failure_image': _GoogleStorageUrl(failure_upload_dir, failure_file),
-          'golden_image': _GoogleStorageUrl(golden_upload_dir, golden_file),
-      }
-      if can_compute_diffs:
-        results_entry.update(
-            {'diff_image': _GoogleStorageUrl(diff_upload_dir, diff_file)})
-      results_dict[test_class][device_model].append(results_entry)
-
-    if can_compute_diffs:
-      _UploadFiles(diff_upload_dir, diff_images)
-    _UploadFiles(failure_upload_dir, failure_images)
-    _UploadFiles(golden_upload_dir, golden_images)
-
-    if failure_images:
-      failures_zipfile = os.path.join(temp_dir, 'failures.zip')
-      with zipfile.ZipFile(failures_zipfile, mode='w') as zf:
-        for failure_file in failure_images:
-          zf.write(failure_file, os.path.join(
-              render_results_dir, os.path.basename(failure_file)))
-        failure_zip_url = _GoogleStorageUrl(upload_dir, failures_zipfile)
-      _UploadFiles(upload_dir, [failures_zipfile])
-    else:
-      failure_zip_url = None
-
-    jinja2_env = jinja2.Environment(
-        loader=jinja2.FileSystemLoader(_JINJA_TEMPLATE_DIR),
-        trim_blocks=True)
-    template = jinja2_env.get_template(_JINJA_TEMPLATE_FILENAME)
-    #  pylint: disable=no-member
-    processed_template_output = template.render(
-        full_results=dict(results_dict),
-        failure_zip_url=failure_zip_url, show_diffs=can_compute_diffs)
-    #  pylint: enable=no-member
-    with open(html_file, 'wb') as f:
-      f.write(processed_template_output)
-  finally:
-    if temp_dir:
-      shutil.rmtree(temp_dir)
-
-
-def main():
-  parser = argparse.ArgumentParser()
-
-  parser.add_argument('--render-results-dir',
-                      required=True,
-                      help='Path on device to look for render test images')
-  parser.add_argument('--output-html-file',
-                      required=True,
-                      help='File to output the results webpage.')
-  parser.add_argument('-d', '--device', dest='devices', action='append',
-                      default=[],
-                      help='Device to look for render test results on. '
-                           'Default is to look on all connected devices.')
-  parser.add_argument('--adb-path', type=os.path.abspath,
-                      help='Absolute path to the adb binary to use.')
-  parser.add_argument('--buildername', type=str, required=True,
-                      help='Bot buildername. Used to generate path to upload '
-                           'render test results')
-  parser.add_argument('--build-number', type=str, required=True,
-                      help='Bot build number. Used to generate path to upload '
-                           'render test results')
-
-  args = parser.parse_args()
-  devil_chromium.Initialize(adb_path=args.adb_path)
-  devices = device_utils.DeviceUtils.HealthyDevices(device_arg=args.devices)
-
-  upload_dir = os.path.join(args.buildername, args.build_number)
-  ProcessRenderTestResults(
-      devices, args.render_results_dir, upload_dir, args.output_html_file)
-
-
-if __name__ == '__main__':
-  sys.exit(main())