Add script to process results from Android render tests.

build/android/render_tests/process_render_test_results.py

Index: build/android/render_tests/process_render_test_results.py
diff --git a/build/android/render_tests/process_render_test_results.py b/build/android/render_tests/process_render_test_results.py
new file mode 100755
index 0000000000000000000000000000000000000000..e04d88594241f9150c513f3eb4c280e0755f8704
--- /dev/null
+++ b/build/android/render_tests/process_render_test_results.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import logging
+import os
+import re
+import shutil
+import sys
+import tempfile
+from PIL import ImageChops
+from PIL import Image
+
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))
+import devil_chromium
+from devil.android import device_utils
+from devil.utils import cmd_helper
+from pylib.constants import host_paths
+
+sys.path.append(os.path.abspath(
+    os.path.join(host_paths.DIR_SOURCE_ROOT, 'build')))
+import find_depot_tools  # pylint: disable=import-error,unused-import
+import download_from_google_storage
+
+sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'))
+import jinja2  # pylint: disable=F0401
+
+_RE_IMAGE_NAME = re.compile(
+    r'(?P<test_class>\w+)\.(?P<description>\w+)\.'
+    r'(?P<device_model>\w+)\.(?P<orientation>port|land)\.png')
+
+_RENDER_TEST_BASE_URL = 'https://storage.googleapis.com/chromium-render-tests/'
+_RENDER_TEST_BUCKET = 'gs://chromium-render-tests/'
+
+_JINJA_TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__))
+_JINJA_TEMPLATE_FILENAME = 'render_webpage.html.jinja2'
+
+
+def _UploadImage(upload_dir, image_file):
+  """Upload image to the render tests GS bucket.

[the real yoland, 2016/10/13 22:03:38]

Would it be better to have recipe control the name of the bucket?

+
+  Returns:
+    Web link to the uploaded image.
+  """
+  google_storage_path = os.path.join(
+      _RENDER_TEST_BUCKET, upload_dir, os.path.basename(image_file))
+  google_storage_url = os.path.join(
+      _RENDER_TEST_BASE_URL, upload_dir, os.path.basename(image_file))
+  cmd_helper.RunCmd(
+    [download_from_google_storage.GSUTIL_DEFAULT_PATH,
+     'cp', image_file, google_storage_path])
+  return google_storage_url
+
+
+def _ComputeImageDiff(failure_image, golden_image):
+  """Compute mask showing which pixels are different between two images."""
+  diff_image = ImageChops.difference(failure_image, golden_image)
+  diff_image = diff_image.convert('L')

[the real yoland, 2016/10/13 22:03:38]

nit: maybe just
```
return ImageChops.difference(failure_image, golden_image)
    .convert('L')
    .point(lambd i: 255 i else 0)
```
and change the method if the diff compute algo changes in the future

[mikecase (-- gone --), 2016/10/18 16:00:12]

Done

+  diff_image = diff_image.point(lambda i: 255 if i else 0)

[PEConn, 2016/10/14 09:08:35]

ImageMagick can do a really nice diff where you've got one of the original
images faint in the background, eg: http://i.stack.imgur.com/hKAXB.png

That'd probably be easy to implement as putting diff_image on top of the faded
golden image.

[mikecase (-- gone --), 2016/10/18 16:00:12]

I've never used ImageMagick. Is it available in Chromium?

[PEConn, 2016/10/19 10:34:54]

ImageMagick is a command line tool (it may be already installed if you have a
Linux machine). I wasn't suggesting that we use it, but that we create diff
images in a similar style to it.

+  return diff_image
+
+
+def ProcessRenderTestResults(devices, render_results_dir,
+                             upload_dir, html_file):
+  """Grabs render results from device and generates webpage displaying results.
+
+  Args:
+    devices: List of DeviceUtils objects to grab results from.
+    render_results_dir: Directory on the devices to look for results. Assumes
+        directory given contains the golden images for the tests and a directory
+        "failures" containing images for failed tests.
+    upload_dir: Directory to upload the render test results to.
+    html_file: File to write the test results to.
+  """
+  results_dict = {}
+
+  temp_dir = None
+  try:
+    temp_dir = tempfile.mkdtemp()
+    for device in devices:
+      device.adb.Pull(render_results_dir, temp_dir)
+
+    failure_dir = os.path.join(temp_dir, 'failures')
+
+    for failure_filename in os.listdir(failure_dir):
+      m = _RE_IMAGE_NAME.match(failure_filename)
+      if not m:
+        continue
+
+      # Check to make sure we have golden image for this failure.
+      golden_file = os.path.join(temp_dir, failure_filename)

[the real yoland, 2016/10/13 22:03:38]

hmm, out of the curiosity, when is the golden images pushed to devices and where
were they stored initially?

[PEConn, 2016/10/14 09:08:35]

The goldens are stored in src/chrome/test/data/android/render_tests and are
pushed before the instrumentation tests are run.

[the real yoland, 2016/10/14 18:37:37]

+peconn

Not sure how the entire of the process works, but I assume it's easier to just
store the goldens in a bucket directory and fetch them from there accordingly to
compare without getting device involved? the push and pull seems redundant.

But then maybe that somehow would limit the flexibility of creating new type of
rendering tests?

[PEConn, 2016/10/19 10:34:54]

I can't think of how it would reduce the flexibility at the moment, grabbing the
goldens from src/ should work fine.

+      if not os.path.exists(golden_file):
+        logging.error('Cannot find golden image for %s', failure_filename)
+        continue
+
+      failure_file = os.path.join(failure_dir, failure_filename)
+
+      # Compute image diff between failure and golden.
+      diff_image = _ComputeImageDiff(
+          Image.open(failure_file), Image.open(golden_file))
+      diff_filename = '_diff'.join(
+          os.path.splitext(os.path.basename(failure_file)))
+      diff_file = os.path.join(temp_dir, diff_filename)
+      diff_image.save(diff_file)
+
+      failure_image_url = _UploadImage(upload_dir, failure_file)
+      golden_image_url = _UploadImage(upload_dir, golden_file)
+      diff_image_url = _UploadImage(upload_dir, diff_file)
+
+      test_class = m.group('test_class')
+      device_model = m.group('device_model')
+
+      if test_class not in results_dict:

[the real yoland, 2016/10/13 22:03:38]

you can use collections.defaultdict here
results_dict = defaultdict(lambda: defaultdict(list))
which return results_dict[x] as a defaultdict for which the default return is a
list

then again, readability -> 0 lol 

[the real yoland, 2016/10/14 18:59:25]

you have to change it back to dict at the end though `dict(results_dict)` or
else anything jinja template refers to in the default_dict that wasn't supposed
to exist will return with an empty list, which would make bug hard to catch

[mikecase (-- gone --), 2016/10/18 16:00:13]

Done

+        results_dict[test_class] = {}
+
+      if device_model not in results_dict[test_class]:
+        results_dict[test_class][device_model] = []
+
+      results_dict[test_class][device_model].append({
+          'description': m.group('description'),
+          'orientation': m.group('orientation'),
+          'failure_image': failure_image_url,
+          'golden_image': golden_image_url,
+          'diff_image': diff_image_url,
+      })
+
+    jinja2_env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(_JINJA_TEMPLATE_DIR),
+        trim_blocks=True)
+    template = jinja2_env.get_template(_JINJA_TEMPLATE_FILENAME)
+    processed_template_output = template.render(full_results=results_dict)
+    with open(html_file, "wb") as f:
+      f.write(processed_template_output)
+  finally:
+    if temp_dir:
+      shutil.rmtree(temp_dir)
+
+def main():
+  parser = argparse.ArgumentParser()
+
+  parser.add_argument('--render-results-dir',
+                      required=True,
+                      help='Path on device to look for render test images')

[PEConn, 2016/10/11 12:20:07]

At the moment all the render results go in
'chrome/test/data/android/render_tests/' with the failures in '.../failures'. In
the future they may go in a tree based there, eg:

chrome/test/data/android/render_tests/
chrome/test/data/android/render_tests/new_tab_page/
chrome/test/data/android/render_tests/signin/

chrome/test/data/android/render_tests/failures/
chrome/test/data/android/render_tests/new_tab_page/failures/
chrome/test/data/android/render_tests/signin/failures/

What you're doing is great - just to keep that in mind for the future.

[mikecase (-- gone --), 2016/10/18 16:00:13]

Noted

+  parser.add_argument('--output-html-file',
+                      required=True,
+                      help='File to output the results webpage.')
+  parser.add_argument('-d', '--device', dest='devices', action='append',
+                      default=[],
+                      help='Device to look for render test results on. '
+                           'Default is to look on all connected devices.')
+  parser.add_argument('--adb-path', type=os.path.abspath,
+                      help='Absolute path to the adb binary to use.')
+  parser.add_argument('--upload-dir',
+                      help='Root path to upload files to GS. Recommend this '
+                           'be set to a combination of buildername and build '
+                           'number for bots.')
+
+  args = parser.parse_args()
+  devil_chromium.Initialize(adb_path=args.adb_path)
+  devices = device_utils.DeviceUtils.HealthyDevices(device_arg=args.devices)
+  ProcessRenderTestResults(
+      devices, args.render_results_dir, args.upload_dir, args.output_html_file)
+
+
+if __name__ == '__main__':
+  sys.exit(main())