Support proper retrying of telemetry_unittests in recipes

scripts/slave/recipe_modules/chromium/steps.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 class Test(object):
   """
   Base class for tests that can be retried after deapplying a previously
   applied patch.
   """
@@ skipping 370 matching lines @@
     if not gtest_results.valid:  # pragma: no cover
       return False
     global_tags = gtest_results.raw.get('global_tags', [])
     return 'UNRELIABLE_RESULTS' not in global_tags
 
   def failures(self, api, suffix):
     assert self.has_valid_results(api, suffix)
     return self._results[suffix].failures
 
 
-class TelemetryUnitTests(Test):
+class PythonBasedTest(Test):
+  @staticmethod
+  def compile_targets(_):
+    return []
+
+  @classmethod
+  def path_to_harness(_cls, _api):
+    raise NotImplementedError
+
+  def run_step(self, api, suffix, cmd_args, **kwargs):
+    return api.python(self._step_name(suffix),
+                      self.path_to_harness(api),
+                      cmd_args, **kwargs)
+
+  def run(self, api, suffix):
+    args = ['--write-full-results-to',
+            api.json.test_results(add_json_log=False)]
+    if suffix == 'without patch':
+      args.extend(self.failures(api, 'with patch'))
+
+    def followup_fn(step_result):
+      r = step_result.json.test_results
+      p = step_result.presentation
+
+      p.step_text += api.test_utils.format_step_text([
+        ['unexpected_failures:', r.unexpected_failures.keys()],
+      ])
+
+    return self.run_step(
+        api, suffix, args, can_fail_build=(not suffix),
+        step_test_data=lambda: api.json.test_api.canned_test_output(
+            True), followup_fn=followup_fn)
+
+  def has_valid_results(self, api, suffix):
+    # TODO(dpranke): we should just return zero/nonzero for success/fail.
+    # crbug.com/357866
+    step = api.step_history[self._step_name(suffix)]
+    return (step.json.test_results.valid and
+            step.retcode <= step.json.test_results.MAX_FAILURES_EXIT_STATUS)
+
+  def failures(self, api, suffix):
+    sn = self._step_name(suffix)
+    return api.step_history[sn].json.test_results.unexpected_failures
+
+
+class MojoPythonTests(PythonBasedTest):  # pylint: disable=W0232
+  name = 'mojo_python_tests'
+
+  @staticmethod
+  def path_to_harness(api):
+    return api.path['checkout'].join('mojo', 'tools',
+                                     'run_mojo_python_tests.py')
+
+
+class TelemetryUnitTests(PythonBasedTest):  # pylint: disable=W0232
   name = 'telemetry_unittests'
 
-  def run(self, api, suffix):
-    # Until telemetry tests output JSON, need to fail on failure with patch.
-    # Otherwise, if the tests were failing on trunks and a cl introduces a
-    # new regression the cl would land since the failure text is hardcoded
-    # below. http://crbug.com/359521.
-    return api.chromium.run_telemetry_unittests(
-        suffix, always_run=True, can_fail_build=True)
+  @staticmethod
+  def path_to_harness(_api):
+      assert False, "This method should not be called."

[Paweł Hajdan Jr., 2014/07/24 15:00:34]

Can we remove path_to_harness then?

I prefer a slight duplication of the api.python call to an "irregular" API
surface.

[Dirk Pranke, 2014/07/24 16:02:44]

Yeah, that seems like an improvement to me as well, now that you've suggested
it. Will do.

+
+  @staticmethod
+  def compile_targets(_):
+      return ['chrome']
+
+  def run_step(self, api, suffix, cmd_args, **kwargs):
+    return api.chromium.run_telemetry_unittests(suffix, cmd_args, **kwargs)
+
+
+class TelemetryPerfUnitTests(PythonBasedTest):
+  name = 'telemetry_perf_unittests'
 
   @staticmethod
   def compile_targets(_):
     return ['chrome']
 
-  def has_valid_results(self, api, suffix):
-    return True
+  @staticmethod
+  def path_to_harness(_api):
+      assert False, "This method should not be called."
 
-  def failures(self, api, suffix):
-    # TODO(phajdan.jr): Make it possible to retry individual failing telemetry
-    # tests (add JSON).
-    if api.step_history[self._step_name(suffix)].retcode:
-      return ['telemetry_unittest']
-    return []
-
-class TelemetryPerfUnitTests(Test):
-  name = 'telemetry_perf_unittests'
-
-  def run(self, api, suffix):
-    # Until telemetry tests output JSON, need to fail on failure with patch.
-    # Otherwise, if the tests were failing on trunks and a cl introduces a
-    # new regression the cl would land since the failure text is hardcoded
-    # below. http://crbug.com/359521.
-    return api.chromium.run_telemetry_perf_unittests(
-        suffix, always_run=True, can_fail_build=True)
-
-  @staticmethod
-  def compile_targets(_):
-    return ['chrome']
-
-  def has_valid_results(self, api, suffix):
-    return True
-
-  def failures(self, api, suffix):
-    # TODO(phajdan.jr): Make it possible to retry individual failing telemetry
-    # tests (add JSON).
-    if api.step_history[self._step_name(suffix)].retcode:
-      return ['telemetry_perf_unittests']
-    return []
+  def run_step(self, api, suffix, cmd_args, **kwargs):
+    return api.chromium.run_telemetry_perf_unittests(suffix, cmd_args,
+                                                     **kwargs)
 
 
 class NaclIntegrationTest(Test):  # pylint: disable=W0232
   name = 'nacl_integration'
 
   @staticmethod
   def compile_targets(_):
     return ['chrome']
 
   def run(self, api, suffix):
@@ skipping 52 matching lines @@
           self.adb_install_apk[0], self.adb_install_apk[1])
     yield api.chromium_android.run_instrumentation_suite(
         self.name, test_data=self.test_data,
         flakiness_dashboard='test-results.appspot.com',
         verbose=True)
 
   def compile_targets(self, _):
     return [self.compile_target]
 
 
-class MojoPythonTests(Test):  # pylint: disable=W0232
-  name = 'mojo_python_tests'
+class BlinkTest(Test):
+  # TODO: This should be converted to a PythonBasedTest, although it

[Paweł Hajdan Jr., 2014/07/24 15:00:34]

nit: TODO(dpranke)?

[Dirk Pranke, 2014/07/24 16:02:44]

Sure, I can add that.

+  # will need custom behavior because we archive the results as well.
 
-  @staticmethod
-  def compile_targets(_):
-    return []
-
-  def run(self, api, suffix):
-    args = ['--write-full-results-to',
-            api.json.test_results(add_json_log=False)]
-    if suffix == 'without patch':
-      args.extend(self.failures(api, 'with patch'))
-
-    def followup_fn(step_result):
-      r = step_result.json.test_results
-      p = step_result.presentation
-
-      p.step_text += api.test_utils.format_step_text([
-        ['unexpected_failures:', r.unexpected_failures.keys()],
-      ])
-
-    return api.python(
-        self._step_name(suffix),
-        api.path['checkout'].join(
-            'mojo',
-            'tools',
-            'run_mojo_python_tests.py'),
-        args,
-        can_fail_build=(not suffix),
-        step_test_data=lambda: api.json.test_api.canned_test_output(
-            True), followup_fn=followup_fn)
-
-  def has_valid_results(self, api, suffix):
-    # TODO(dpranke): we should just return zero/nonzero for success/fail.
-    # crbug.com/357866
-    step = api.step_history[self._step_name(suffix)]
-    return (step.json.test_results.valid and
-            step.retcode <= step.json.test_results.MAX_FAILURES_EXIT_STATUS)
-
-  def failures(self, api, suffix):
-    sn = self._step_name(suffix)
-    return api.step_history[sn].json.test_results.unexpected_failures
-
-
-class BlinkTest(Test):
   name = 'webkit_tests'
 
   def __init__(self, api):
     self.results_dir = api.path['slave_build'].join('layout-test-results')
     self.layout_test_wrapper = api.path['build'].join(
         'scripts', 'slave', 'chromium', 'layout_test_wrapper.py')
 
   def run(self, api, suffix):
     args = ['--target', api.chromium.c.BUILD_CONFIG,
             '-o', self.results_dir,
@@ skipping 82 matching lines @@
   GTestTest('components_unittests'),
   GTestTest('crypto_unittests'),
   GTestTest('gfx_unittests'),
   GTestTest('url_unittests'),
   GTestTest('content_unittests'),
   GTestTest('net_unittests'),
   GTestTest('ui_unittests'),
   GTestTest('sync_unit_tests'),
   GTestTest('sql_unittests'),
 ]