Add render test results to the results_details webpage.

build/android/pylib/local/device/local_device_instrumentation_test_run.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import os
 import posixpath
 import re
+import sys
+import tempfile
 import time
 
 from devil.android import device_errors
 from devil.android import flag_changer
 from devil.android.sdk import shared_prefs
 from devil.utils import reraiser_thread
 from pylib import valgrind_tools
 from pylib.android import logdog_logcat_monitor
 from pylib.base import base_test_result
+from pylib.constants import host_paths
 from pylib.instrumentation import instrumentation_test_instance
 from pylib.local.device import local_device_environment
 from pylib.local.device import local_device_test_run
 from pylib.utils import google_storage_helper
 from pylib.utils import logdog_helper
 from py_trace_event import trace_event
 from py_utils import contextlib_ext
 from py_utils import tempfile_ext
 import tombstones
 
+sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'))
+import jinja2  # pylint: disable=import-error
+
+
+_JINJA_TEMPLATE_DIR = os.path.join(
+    host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'instrumentation')
+_JINJA_TEMPLATE_FILENAME = 'render_test.html.jinja'
+
 _TAG = 'test_runner_py'
 
 TIMEOUT_ANNOTATIONS = [
   ('Manual', 10 * 60 * 60),
   ('IntegrationTest', 30 * 60),
   ('External', 10 * 60),
   ('EnormousTest', 10 * 60),
   ('LargeTest', 5 * 60),
   ('MediumTest', 3 * 60),
   ('SmallTest', 1 * 60),
 ]
 
 LOGCAT_FILTERS = ['*:e', 'chromium:v', 'cr_*:v']
 
+FEATURE_ANNOTATION = 'Feature'
+RENDER_TEST_FEATURE_ANNOTATION = 'RenderTest'
+
+# This needs to be kept in sync with formatting in |RenderUtils.imageName|
+RE_RENDER_IMAGE_NAME = re.compile(
+      r'(?P<test_class>\w+)\.'
+      r'(?P<description>\w+)\.'
+      r'(?P<device_model>\w+)\.'
+      r'(?P<orientation>port|land)\.png')
 
 # TODO(jbudorick): Make this private once the instrumentation test_runner is
 # deprecated.
 def DidPackageCrashOnDevice(package_name, device):
   # Dismiss any error dialogs. Limit the number in case we have an error
   # loop or we are failing to dismiss.
   try:
     for _ in xrange(10):
       package = device.DismissCrashDialogIfNeeded()
       if not package:
@@ skipping 187 matching lines @@
     return tests
 
   #override
   def _GetUniqueTestName(self, test):
     return instrumentation_test_instance.GetUniqueTestName(test)
 
   #override
   def _RunTest(self, device, test):
     extras = {}
 
-    flags = None
+    flags_to_add = []
+    flags_to_remove = []
     test_timeout_scale = None
     if self._test_instance.coverage_directory:
       coverage_basename = '%s.ec' % ('%s_group' % test[0]['method']
           if isinstance(test, list) else test['method'])
       extras['coverage'] = 'true'
       coverage_directory = os.path.join(
           device.GetExternalStoragePath(), 'chrome', 'test', 'coverage')
       coverage_device_file = os.path.join(
           coverage_directory, coverage_basename)
       extras['coverageFile'] = coverage_device_file
@@ skipping 26 matching lines @@
       test_display_name = self._GetUniqueTestName(test)
       if test['is_junit4']:
         target = '%s/%s' % (
             self._test_instance.test_package,
             self._test_instance.test_runner_junit4)
       else:
         target = '%s/%s' % (
             self._test_instance.test_package, self._test_instance.test_runner)
       extras['class'] = test_name
       if 'flags' in test:
-        flags = test['flags']
+        flags_to_add.extend(test['flags'].add)
+        flags_to_remove.extend(test['flags'].remove)
       timeout = self._GetTimeoutFromAnnotations(
         test['annotations'], test_display_name)
 
       test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
           test['annotations'])
       if test_timeout_scale and test_timeout_scale != 1:
         valgrind_tools.SetChromeTimeoutScale(
             device, test_timeout_scale * self._test_instance.timeout_scale)
 
     logging.info('preparing to run %s: %s', test_display_name, test)
 
-    if flags:
+    render_tests_device_output_dir = None
+    if _IsRenderTest(test):
+      # TODO(mikecase): Add DeviceTempDirectory class and use that instead.

[jbudorick, 2017/05/10 15:10:05]

This would be a pretty nice addition to device_temp_file. :)

+      render_tests_device_output_dir = posixpath.join(
+          device.GetExternalStoragePath(),
+          'render_test_output_dir')
+      flags_to_add.append('--render-test-output-dir=%s' %
+                          render_tests_device_output_dir)
+
+    if flags_to_add or flags_to_remove:
       self._CreateFlagChangerIfNeeded(device)
       self._flag_changers[str(device)].PushFlags(
-        add=flags.add, remove=flags.remove)
+        add=flags_to_add, remove=flags_to_remove)
 
     try:
       device.RunShellCommand(
           ['log', '-p', 'i', '-t', _TAG, 'START %s' % test_name],
           check_return=True)
       time_ms = lambda: int(time.time() * 1e3)
       start_ms = time_ms()
 
       stream_name = 'logcat_%s_%s_%s' % (
           test_name.replace('#', '.'),
           time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()),
           device.serial)
       logmon = logdog_logcat_monitor.LogdogLogcatMonitor(
           device.adb, stream_name, filter_specs=LOGCAT_FILTERS)
 
       with contextlib_ext.Optional(
           logmon, self._test_instance.should_save_logcat):
         with contextlib_ext.Optional(
             trace_event.trace(test_name),
             self._env.trace_output):
           output = device.StartInstrumentation(
               target, raw=True, extras=extras, timeout=timeout, retries=0)
       logcat_url = logmon.GetLogcatURL()
     finally:
       device.RunShellCommand(
           ['log', '-p', 'i', '-t', _TAG, 'END %s' % test_name],
           check_return=True)
       duration_ms = time_ms() - start_ms
-      if flags:
+      if flags_to_add or flags_to_remove:
         self._flag_changers[str(device)].Restore()
       if test_timeout_scale:
         valgrind_tools.SetChromeTimeoutScale(
             device, self._test_instance.timeout_scale)
 
     # TODO(jbudorick): Make instrumentation tests output a JSON so this
     # doesn't have to parse the output.
     result_code, result_bundle, statuses = (
         self._test_instance.ParseAmInstrumentRawOutput(output))
     results = self._test_instance.GenerateTestResults(
         result_code, result_bundle, statuses, start_ms, duration_ms)
     for result in results:
       if logcat_url:
         result.SetLink('logcat', logcat_url)
 
+    if _IsRenderTest(test):
+      # Render tests do not cause test failure by default. So we have to check
+      # to see if any failure images were generated even if the test does not
+      # fail.
+      try:
+        self._ProcessRenderTestResults(
+            device, render_tests_device_output_dir, results)
+      finally:
+        device.RemovePath(render_tests_device_output_dir,
+                          recursive=True, force=True)
+
     # Update the result name if the test used flags.
-    if flags:
+    if flags_to_add or flags_to_remove:
       for r in results:
         if r.GetName() == test_name:
           r.SetName(test_display_name)
 
     # Add UNKNOWN results for any missing tests.
     iterable_test = test if isinstance(test, list) else [test]
     test_names = set(self._GetUniqueTestName(t) for t in iterable_test)
     results_names = set(r.GetName() for r in results)
     results.extend(
         base_test_result.BaseTestResult(u, base_test_result.ResultType.UNKNOWN)
@@ skipping 67 matching lines @@
                 include_stack_symbols=False,
                 wipe_tombstones=True)
             stream_name = 'tombstones_%s_%s' % (
                 time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()),
                 device.serial)
             tombstones_url = logdog_helper.text(
                 stream_name, '\n'.join(resolved_tombstones))
           result.SetLink('tombstones', tombstones_url)
     return results, None
 
+  def _ProcessRenderTestResults(
+      self, device, render_tests_device_output_dir, results):
+    # Will archive test images if we are given a GS bucket to store the results
+    # in and are given a results file to output the links to.
+    if not bool(self._test_instance.gs_results_bucket):
+      return
+
+    failure_images_device_dir = posixpath.join(
+        render_tests_device_output_dir, 'failures')
+
+    if not device.FileExists(failure_images_device_dir):
+      return
+
+    render_tests_bucket = (
+        self._test_instance.gs_results_bucket + '/render_tests')
+
+    diff_images_device_dir = posixpath.join(
+        render_tests_device_output_dir, 'diffs')
+
+    golden_images_device_dir = posixpath.join(
+        render_tests_device_output_dir, 'goldens')
+
+    with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
+      device.PullFile(failure_images_device_dir, temp_dir)
+
+      if device.FileExists(diff_images_device_dir):
+        device.PullFile(diff_images_device_dir, temp_dir)
+      else:
+        logging.error('Diff images not found on device.')
+
+      if device.FileExists(golden_images_device_dir):
+        device.PullFile(golden_images_device_dir, temp_dir)
+      else:
+        logging.error('Golden images not found on device.')
+
+      for failure_filename in os.listdir(os.path.join(temp_dir, 'failures')):
+
+        m = RE_RENDER_IMAGE_NAME.match(failure_filename)
+        if not m:
+          logging.warning('Unexpected file in render test failures: %s',
+                          failure_filename)
+          continue
+
+        failure_filepath = os.path.join(temp_dir, 'failures', failure_filename)
+        failure_link = google_storage_helper.upload(
+            google_storage_helper.unique_name(
+                'failure_%s' % failure_filename, device=device),
+            failure_filepath,
+            bucket=render_tests_bucket)
+
+        golden_filepath = os.path.join(temp_dir, 'goldens', failure_filename)
+        if os.path.exists(golden_filepath):
+          golden_link = google_storage_helper.upload(
+              google_storage_helper.unique_name(
+                  'golden_%s' % failure_filename, device=device),
+              golden_filepath,
+              bucket=render_tests_bucket)
+        else:
+          golden_link = ''
+
+        diff_filepath = os.path.join(temp_dir, 'diffs', failure_filename)
+        if os.path.exists(diff_filepath):
+          diff_link = google_storage_helper.upload(
+              google_storage_helper.unique_name(
+                  'diff_%s' % failure_filename, device=device),
+              diff_filepath,
+              bucket=render_tests_bucket)
+        else:
+          diff_link = ''
+
+        with tempfile.NamedTemporaryFile(suffix='.html') as temp_html:
+          jinja2_env = jinja2.Environment(
+              loader=jinja2.FileSystemLoader(_JINJA_TEMPLATE_DIR),
+              trim_blocks=True)
+          template = jinja2_env.get_template(_JINJA_TEMPLATE_FILENAME)
+          # pylint: disable=no-member
+          processed_template_output = template.render(
+              failure_link=failure_link,
+              golden_link=golden_link,
+              diff_link=diff_link)
+
+          temp_html.write(processed_template_output)
+          temp_html.flush()
+          html_results_link = google_storage_helper.upload(
+              google_storage_helper.unique_name('render_html', device=device),
+              temp_html.name,
+              bucket=render_tests_bucket,
+              content_type='text/html')
+          for result in results:
+            result.SetLink(failure_filename, html_results_link)
+
   #override
   def _ShouldRetry(self, test):
     if 'RetryOnFailure' in test.get('annotations', {}):
       return True
 
     # TODO(jbudorick): Remove this log message once @RetryOnFailure has been
     # enabled for a while. See crbug.com/619055 for more details.
     logging.error('Default retries are being phased out. crbug.com/619055')
     return False
 
@@ skipping 15 matching lines @@
       if k in annotations:
         timeout = v
         break
     else:
       logging.warning('Using default 1 minute timeout for %s', test_name)
       timeout = 60
 
     timeout *= cls._GetTimeoutScaleFromAnnotations(annotations)
 
     return timeout
+
+def _IsRenderTest(test):
+  """Determines if a test or list of tests has a RenderTest amongst them."""
+  if not isinstance(test, list):
+    test = [test]
+  return any([RENDER_TEST_FEATURE_ANNOTATION in t['annotations'].get(
+              FEATURE_ANNOTATION, ())] for t in test)