Support proper retrying of telemetry_unittests in recipes

scripts/slave/recipe_modules/chromium/steps.py

Index: scripts/slave/recipe_modules/chromium/steps.py
diff --git a/scripts/slave/recipe_modules/chromium/steps.py b/scripts/slave/recipe_modules/chromium/steps.py
index 3f8ed8aaadcc516b0445eeebf183eadd679722d0..79ee3810c56a08086a9a7671030c2f41d2fdab23 100644
--- a/scripts/slave/recipe_modules/chromium/steps.py
+++ b/scripts/slave/recipe_modules/chromium/steps.py
@@ -388,55 +388,89 @@ class SwarmingGTestTest(Test):
     return self._results[suffix].failures
 
 
-class TelemetryUnitTests(Test):
-  name = 'telemetry_unittests'
+class PythonBasedTest(Test):
+  @staticmethod
+  def compile_targets(_):
+    return []
+
+  @classmethod
+  def path_to_harness(_cls, _api):
+    raise NotImplementedError
+
+  def run_step(self, api, suffix, cmd_args, **kwargs):
+    return api.python(self._step_name(suffix),
+                      self.path_to_harness(api),
+                      cmd_args, **kwargs)
 
   def run(self, api, suffix):
-    # Until telemetry tests output JSON, need to fail on failure with patch.
-    # Otherwise, if the tests were failing on trunks and a cl introduces a
-    # new regression the cl would land since the failure text is hardcoded
-    # below. http://crbug.com/359521.
-    return api.chromium.run_telemetry_unittests(
-        suffix, always_run=True, can_fail_build=True)
+    args = ['--write-full-results-to',
+            api.json.test_results(add_json_log=False)]
+    if suffix == 'without patch':
+      args.extend(self.failures(api, 'with patch'))
 
-  @staticmethod
-  def compile_targets(_):
-    return ['chrome']
+    def followup_fn(step_result):
+      r = step_result.json.test_results
+      p = step_result.presentation
+
+      p.step_text += api.test_utils.format_step_text([
+        ['unexpected_failures:', r.unexpected_failures.keys()],
+      ])
+
+    return self.run_step(
+        api, suffix, args, can_fail_build=(not suffix),
+        step_test_data=lambda: api.json.test_api.canned_test_output(
+            True), followup_fn=followup_fn)
 
   def has_valid_results(self, api, suffix):
-    return True
+    # TODO(dpranke): we should just return zero/nonzero for success/fail.
+    # crbug.com/357866
+    step = api.step_history[self._step_name(suffix)]
+    return (step.json.test_results.valid and
+            step.retcode <= step.json.test_results.MAX_FAILURES_EXIT_STATUS)
 
   def failures(self, api, suffix):
-    # TODO(phajdan.jr): Make it possible to retry individual failing telemetry
-    # tests (add JSON).
-    if api.step_history[self._step_name(suffix)].retcode:
-      return ['telemetry_unittest']
-    return []
+    sn = self._step_name(suffix)
+    return api.step_history[sn].json.test_results.unexpected_failures
 
-class TelemetryPerfUnitTests(Test):
-  name = 'telemetry_perf_unittests'
 
-  def run(self, api, suffix):
-    # Until telemetry tests output JSON, need to fail on failure with patch.
-    # Otherwise, if the tests were failing on trunks and a cl introduces a
-    # new regression the cl would land since the failure text is hardcoded
-    # below. http://crbug.com/359521.
-    return api.chromium.run_telemetry_perf_unittests(
-        suffix, always_run=True, can_fail_build=True)
+class MojoPythonTests(PythonBasedTest):  # pylint: disable=W0232
+  name = 'mojo_python_tests'
+
+  @staticmethod
+  def path_to_harness(api):
+    return api.path['checkout'].join('mojo', 'tools',
+                                     'run_mojo_python_tests.py')
+
+
+class TelemetryUnitTests(PythonBasedTest):  # pylint: disable=W0232
+  name = 'telemetry_unittests'
+
+  @staticmethod
+  def path_to_harness(_api):
+      assert False, "This method should not be called."

[Paweł Hajdan Jr., 2014/07/24 15:00:34]

Can we remove path_to_harness then?

I prefer a slight duplication of the api.python call to an "irregular" API
surface.

[Dirk Pranke, 2014/07/24 16:02:44]

Yeah, that seems like an improvement to me as well, now that you've suggested
it. Will do.

+
+  @staticmethod
+  def compile_targets(_):
+      return ['chrome']
+
+  def run_step(self, api, suffix, cmd_args, **kwargs):
+    return api.chromium.run_telemetry_unittests(suffix, cmd_args, **kwargs)
+
+
+class TelemetryPerfUnitTests(PythonBasedTest):
+  name = 'telemetry_perf_unittests'
 
   @staticmethod
   def compile_targets(_):
     return ['chrome']
 
-  def has_valid_results(self, api, suffix):
-    return True
+  @staticmethod
+  def path_to_harness(_api):
+      assert False, "This method should not be called."
 
-  def failures(self, api, suffix):
-    # TODO(phajdan.jr): Make it possible to retry individual failing telemetry
-    # tests (add JSON).
-    if api.step_history[self._step_name(suffix)].retcode:
-      return ['telemetry_perf_unittests']
-    return []
+  def run_step(self, api, suffix, cmd_args, **kwargs):
+    return api.chromium.run_telemetry_perf_unittests(suffix, cmd_args,
+                                                     **kwargs)
 
 
 class NaclIntegrationTest(Test):  # pylint: disable=W0232
@@ -509,51 +543,10 @@ class AndroidInstrumentationTest(Test):
     return [self.compile_target]
 
 
-class MojoPythonTests(Test):  # pylint: disable=W0232
-  name = 'mojo_python_tests'
-
-  @staticmethod
-  def compile_targets(_):
-    return []
-
-  def run(self, api, suffix):
-    args = ['--write-full-results-to',
-            api.json.test_results(add_json_log=False)]
-    if suffix == 'without patch':
-      args.extend(self.failures(api, 'with patch'))
-
-    def followup_fn(step_result):
-      r = step_result.json.test_results
-      p = step_result.presentation
-
-      p.step_text += api.test_utils.format_step_text([
-        ['unexpected_failures:', r.unexpected_failures.keys()],
-      ])
-
-    return api.python(
-        self._step_name(suffix),
-        api.path['checkout'].join(
-            'mojo',
-            'tools',
-            'run_mojo_python_tests.py'),
-        args,
-        can_fail_build=(not suffix),
-        step_test_data=lambda: api.json.test_api.canned_test_output(
-            True), followup_fn=followup_fn)
-
-  def has_valid_results(self, api, suffix):
-    # TODO(dpranke): we should just return zero/nonzero for success/fail.
-    # crbug.com/357866
-    step = api.step_history[self._step_name(suffix)]
-    return (step.json.test_results.valid and
-            step.retcode <= step.json.test_results.MAX_FAILURES_EXIT_STATUS)
-
-  def failures(self, api, suffix):
-    sn = self._step_name(suffix)
-    return api.step_history[sn].json.test_results.unexpected_failures
-
-
 class BlinkTest(Test):
+  # TODO: This should be converted to a PythonBasedTest, although it

[Paweł Hajdan Jr., 2014/07/24 15:00:34]

nit: TODO(dpranke)?

[Dirk Pranke, 2014/07/24 16:02:44]

Sure, I can add that.

+  # will need custom behavior because we archive the results as well.
+
   name = 'webkit_tests'
 
   def __init__(self, api):