Add step to process Render Test results.

scripts/slave/recipe_modules/chromium_tests/steps.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import datetime
 import json
 import re
 import string
 
 
@@ skipping 497 matching lines @@
         use_swarming = True
         swarming_shards = swarming_spec.get('shards', 1)
         swarming_dimension_sets = swarming_spec.get('dimension_sets')
         swarming_priority = swarming_spec.get('priority_adjustment')
         swarming_expiration = swarming_spec.get('expiration')
     if use_swarming and swarming_dimension_sets:
       for dimensions in swarming_dimension_sets:
         # TODO(stip): Swarmify instrumentation tests
         pass
     else:
       yield AndroidInstrumentationTest(

[jbudorick, 2016/11/21 18:08:50]

Add a way for the JSON to set the render results dir here.

[mikecase (-- gone --), 2016/11/21 23:03:12]

Done. Just named the field 'render_results_dir' in the test dict.

           test_name,
           compile_targets=test.get('override_compile_targets'),
           timeout_scale=test.get('timeout_scale'),
           result_details=True,
           store_tombstones=True)
 
 
 def generate_junit_test(api, chromium_tests_api, mastername, buildername,
                         test_spec, bot_update_step, enable_swarming=False,
                         swarming_dimensions=None,
@@ skipping 950 matching lines @@
 
     Args:
       api: Caller's API.
       suffix: Suffix added to the test name.
       json_results_file: File to output the test results.
     """
     raise NotImplementedError()  # pragma: no cover
 
   def run(self, api, suffix, test_filter=None):
     assert api.chromium.c.TARGET_PLATFORM == 'android'
-    json_results_file = api.test_utils.gtest_results(add_json_log=False)
-    try:
-      self.run_tests(api, suffix, json_results_file)
-    finally:
-      step_result = api.step.active_result
-      self._test_runs[suffix] = {'valid': False}
-      if (hasattr(step_result, 'test_utils') and
-          hasattr(step_result.test_utils, 'gtest_results')):
-        gtest_results = step_result.test_utils.gtest_results
 
-        failures = gtest_results.failures
-        self._test_runs[suffix] = {'valid': True, 'failures': failures}
-        step_result.presentation.step_text += (
-            api.test_utils.format_step_text([['failures:', failures]]))
+    nested_step_name = '%s%s' % (self._name, ' (%s)' % suffix if suffix else '')
+    with api.step.nest(nested_step_name) as nested_step:
+      json_results_file = api.test_utils.gtest_results(add_json_log=False)
+      try:
+        step_result = self.run_tests(api, suffix, json_results_file)
+      except api.step.StepFailure as f:
+        step_result = f.result
+        raise
+      finally:
+        nested_step.presentation.status = step_result.presentation.status
+        self._test_runs[suffix] = {'valid': False}
+        if (hasattr(step_result, 'test_utils') and
+            hasattr(step_result.test_utils, 'gtest_results')):
+          gtest_results = step_result.test_utils.gtest_results
 
-        api.test_results.upload(
-            api.json.input(gtest_results.raw),
-            test_type=self.name,
-            chrome_revision=api.bot_update.last_returned_properties.get(
-                'got_revision_cp', 'x@{#0}'),
-            test_results_server='test-results.appspot.com')
+          failures = gtest_results.failures
+          self._test_runs[suffix] = {'valid': True, 'failures': failures}
+          nested_step.presentation.step_text += (
+              api.test_utils.format_step_text([['failures:', failures]]))
+
+          api.test_results.upload(
+              api.json.input(gtest_results.raw),
+              test_type=self.name,
+              chrome_revision=api.bot_update.last_returned_properties.get(
+                  'got_revision_cp', 'x@{#0}'),
+              test_results_server='test-results.appspot.com')
 
   def compile_targets(self, _):
     return self._compile_targets
 
   def has_valid_results(self, api, suffix):
     if suffix not in self._test_runs:
       return False  # pragma: no cover
     return self._test_runs[suffix]['valid']
 
   def failures(self, api, suffix):
     assert self.has_valid_results(api, suffix)
     return self._test_runs[suffix]['failures']
 
 
 class AndroidJunitTest(AndroidTest):
   def __init__(self, name):
     super(AndroidJunitTest, self).__init__(name, compile_targets=[name],
         isolate_file_path=None)
 
   @property
   def uses_local_devices(self):
     return False
 
   #override
   def run_tests(self, api, suffix, json_results_file):
-    api.chromium_android.run_java_unit_test_suite(
+    return api.chromium_android.run_java_unit_test_suite(
         self.name, verbose=True, suffix=suffix,
         json_results_file=json_results_file,
         step_test_data=lambda: api.test_utils.test_api.canned_gtest_output(False))
 
 
 class AndroidInstrumentationTest(AndroidTest):
   _DEFAULT_SUITES = {
     'AndroidWebViewTest': {
       'compile_target': 'android_webview_test_apk',
     },
@@ skipping 39 matching lines @@
     'remoting_test_apk': _DEFAULT_SUITES['ChromotingTest'],
     'system_webview_shell_layout_test_apk':
         _DEFAULT_SUITES['SystemWebViewShellLayoutTest'],
     'webview_ui_test_app_test_apk': _DEFAULT_SUITES['WebViewUiTest'],
   }
 
   def __init__(self, name, compile_targets=None, apk_under_test=None,
                test_apk=None, isolate_file_path=None, timeout_scale=None,
                annotation=None, except_annotation=None, screenshot=False,
                verbose=True, tool=None, additional_apks=None,
-               store_tombstones=False, result_details=False):
+               store_tombstones=False, result_details=False,
+               render_results_dir=None):
     suite_defaults = (
         AndroidInstrumentationTest._DEFAULT_SUITES.get(name)
         or AndroidInstrumentationTest._DEFAULT_SUITES_BY_TARGET.get(name)
         or {})
     if not compile_targets:
       compile_targets = [suite_defaults.get('compile_target', name)]
       compile_targets.extend(
           suite_defaults.get('additional_compile_targets', []))
 
     super(AndroidInstrumentationTest, self).__init__(
         name,
         compile_targets,
         isolate_file_path or suite_defaults.get('isolate_file_path'))
     self._additional_apks = (
         additional_apks or suite_defaults.get('additional_apks'))
     self._annotation = annotation
     self._apk_under_test = (
         apk_under_test or suite_defaults.get('apk_under_test'))
     self._except_annotation = except_annotation
     self._screenshot = screenshot
     self._test_apk = test_apk or suite_defaults.get('test_apk')
     self._timeout_scale = timeout_scale
     self._tool = tool
     self._verbose = verbose
     self._wrapper_script_suite_name = compile_targets[0]
     self._store_tombstones = store_tombstones
     self._result_details = result_details
+    self._render_results_dir = render_results_dir
 
   @property
   def uses_local_devices(self):
     return True
 
   #override
   def run_tests(self, api, suffix, json_results_file):
-    api.chromium_android.run_instrumentation_suite(
+    return api.chromium_android.run_instrumentation_suite(
         self.name,
         test_apk=api.chromium_android.apk_path(self._test_apk),
         apk_under_test=api.chromium_android.apk_path(self._apk_under_test),
         additional_apks=[
             api.chromium_android.apk_path(a)
             for a in self._additional_apks or []],
         suffix=suffix,
         isolate_file_path=self.isolate_file_path,
         annotation=self._annotation, except_annotation=self._except_annotation,
         screenshot=self._screenshot, verbose=self._verbose, tool=self._tool,
         json_results_file=json_results_file,
         timeout_scale=self._timeout_scale,
         result_details=self._result_details,
         store_tombstones=self._store_tombstones,
         wrapper_script_suite_name=self._wrapper_script_suite_name,
+        render_results_dir=self._render_results_dir,
         step_test_data=lambda: api.test_utils.test_api.canned_gtest_output(False))
 
 
 class BlinkTest(Test):
   # TODO(dpranke): This should be converted to a PythonBasedTest, although it
   # will need custom behavior because we archive the results as well.
   def __init__(self, extra_args=None):
     super(BlinkTest, self).__init__()
     self._extra_args = extra_args
 
@@ skipping 232 matching lines @@
           args=args)
       api.gsutil.upload(
           temp_output_dir.join(
               '%s-android-chrome.json' % timestamp_string),
           'chromium-annotated-tests', 'android')
 
 GOMA_TESTS = [
   GTestTest('base_unittests'),
   GTestTest('content_unittests'),
 ]