Refactor stages 1, 2 and test_api overhaul.

scripts/slave/recipe_modules/auto_bisect/api.py

Index: scripts/slave/recipe_modules/auto_bisect/api.py
diff --git a/scripts/slave/recipe_modules/auto_bisect/api.py b/scripts/slave/recipe_modules/auto_bisect/api.py
index 72c9be581f3f4ba546e0a9ff713f044997134da6..816dc066d0e8dbea6ced063c4db4ea9523d900e8 100644
--- a/scripts/slave/recipe_modules/auto_bisect/api.py
+++ b/scripts/slave/recipe_modules/auto_bisect/api.py
@@ -57,9 +57,6 @@ class AutoBisectApi(recipe_api.RecipeApi):
      self._working_dir = self.m.chromium_checkout.get_checkout_dir({})
    return self._working_dir or self.m.path['slave_build']
 
-  def perform_bisect(self, **flags):
-    return local_bisect.perform_bisect(self, **flags)
-
   def create_bisector(self, bisect_config_dict, dummy_mode=False, **flags):
     """Passes the api and the config dictionary to the Bisector constructor.
 
@@ -104,11 +101,11 @@ class AutoBisectApi(recipe_api.RecipeApi):
     """Sets SVN repo url for triggering build jobs."""
     self.svn_repo_url = svn_repo_url
 
-  def gsutil_file_exists(self, path):
+  def gsutil_file_exists(self, path, **kwargs):
     """Returns True if a file exists at the given GS path."""
     try:
-      self.m.gsutil(['ls', path])
-    except self.m.step.StepFailure:  # pragma: no cover
+      self.m.gsutil( ['ls', path], **kwargs)
+    except self.m.step.StepFailure:
       # A step failure here simply means that the file does not exist, and
       # should not be treated as an error.
       self.m.step.active_result.presentation.status = self.m.step.SUCCESS
@@ -134,7 +131,9 @@ class AutoBisectApi(recipe_api.RecipeApi):
         step_name,
         self.resource('fetch_revision_info.py'),

[prasadv, 2016/09/09 18:52:30]

Can we replace this with gitiles recipe_module?

[RobertoCN, 2016/09/13 22:11:40]

Yes we can, but we don't want to do it on this CL because that would require
that we modify the gitiles module to take step_test_data, adding yet another
file to the CL

         [revision.commit_hash, revision.depot_name],
-        stdout=self.m.json.output())
+        stdout=self.m.json.output(),
+        step_test_data=lambda: self._test_data['cl_info'][revision.commit_hash],
+    )
     return result.stdout
 
   def _commit_info(self, commit_hash, url, step_name=None):  # pragma: no cover
@@ -169,7 +168,7 @@ class AutoBisectApi(recipe_api.RecipeApi):
      self.surface_result('BAD_REV')
      raise
 
-  def run_bisect_script(self, **kwargs):
+  def run_bisect_script(self, **kwargs):  # pragma: no cover
     """Executes src/tools/run-perf-bisect-regression.py to perform bisection."""
     self.m.python(
         'Preparing for Bisection',
@@ -182,10 +181,10 @@ class AutoBisectApi(recipe_api.RecipeApi):
 
     # TODO(prasadv): Remove this once bisect runs are no longer running
     # against revisions from February 2016 or earlier.
-    if self.internal_bisect:  # pragma: no cover
+    if self.internal_bisect:
       kwargs['env'] = {'CHROMIUM_OUTPUT_DIR': self.m.chromium.output_dir}
 
-    if kwargs.get('extra_src'):
+    if kwargs.get('extra_src'): 
       args = args + ['--extra_src', kwargs.pop('extra_src')]
     if kwargs.get('path_to_config'):
       args = args + ['--path_to_config', kwargs.pop('path_to_config')]
@@ -203,7 +202,7 @@ class AutoBisectApi(recipe_api.RecipeApi):
         xvfb=True, **kwargs)
 
   def run_local_test_run(self, test_config_params,
-                         skip_download=False):  # pragma: no cover
+                         skip_download=False, **kwargs):
     """Starts a test run on the same machine.
 
     This is for the merged director/tester flow.
@@ -211,13 +210,13 @@ class AutoBisectApi(recipe_api.RecipeApi):
     if self.m.platform.is_win:
       self.m.chromium.taskkill()
 
-    if skip_download:
+    if skip_download:  # pragma: no cover
       update_step = None
     else:
       update_step = self._SyncRevisionToTest(test_config_params)
-    self.start_test_run_for_bisect(update_step, self.bot_db,
-                                   test_config_params, run_locally=True,
-                                   skip_download=skip_download)
+    return self.start_test_run_for_bisect(
+        update_step, self.bot_db, test_config_params,
+        skip_download=skip_download, **kwargs)
 
   def ensure_checkout(self, *args, **kwargs):
     if self.working_dir:
@@ -225,11 +224,11 @@ class AutoBisectApi(recipe_api.RecipeApi):
 
     return self.m.bot_update.ensure_checkout(*args, **kwargs)
 
-  def _SyncRevisionToTest(self, test_config_params):  # pragma: no cover
+  def _SyncRevisionToTest(self, test_config_params):
     if not self.internal_bisect:
       return self.ensure_checkout(
           root_solution_revision=test_config_params['revision'])
-    else:
+    else:  # pragma: no cover
       return self._SyncRevisionsForAndroidChrome(
           test_config_params['revision_ladder'])
 
@@ -245,15 +244,12 @@ class AutoBisectApi(recipe_api.RecipeApi):
     self.m.gclient('sync %s' % '-'.join(revisions), params)
     return None
 
-  def start_test_run_for_bisect(self, update_step, bot_db,
-                                test_config_params, run_locally=False,
-                                skip_download=False):
+  def start_test_run_for_bisect(self, update_step, bot_db, test_config_params,
+                                skip_download=False, **kwargs):
     mastername = self.m.properties.get('mastername')
     buildername = self.m.properties.get('buildername')
     bot_config = bot_db.get_bot_config(mastername, buildername)
     build_archive_url = test_config_params['parent_build_archive_url']
-    if not run_locally:
-      self.m.bisect_tester.upload_job_url()
     if not skip_download:
       if self.m.chromium.c.TARGET_PLATFORM == 'android':
         # The best way to ensure the old build directory is not used is to
@@ -296,7 +292,9 @@ class AutoBisectApi(recipe_api.RecipeApi):
             build_revision=test_config_params['parent_got_revision'],
             override_bot_type='tester')
 
-    tests = [self.m.chromium_tests.steps.BisectTest(test_config_params)]
+    tests = [
+        self.m.chromium_tests.steps.BisectTest(
+            test_config_params, **kwargs)]
 
     if not tests:  # pragma: no cover
       return
@@ -326,6 +324,7 @@ class AutoBisectApi(recipe_api.RecipeApi):
         self.deploy_apk_on_device(
             self.full_deploy_script, deploy_apks, deploy_args)
       test_runner()
+      return tests[0].run_results
 
   def deploy_apk_on_device(self, deploy_script, deploy_apks, deploy_args):
     """Installs apk on the android device."""
@@ -375,8 +374,7 @@ class AutoBisectApi(recipe_api.RecipeApi):
       flags['do_not_nest_wait_for_revision'] = kwargs.pop(
           'do_not_nest_wait_for_revision')
     if bot_db is None:  # pragma: no cover
-      self.bot_db = api.chromium_tests.create_bot_db_from_master_dict(
-          '', None, None)
+      self.bot_db = api.chromium_tests.create_bot_db_from_master_dict('', None)
     else:
       self.bot_db = bot_db
 
@@ -403,7 +401,7 @@ class AutoBisectApi(recipe_api.RecipeApi):
           # test by checking for the presence of the good_revision key.
           if api.properties.get('bisect_config').get('good_revision'):
             api.step('***BISECT***', [])
-            local_bisect.perform_bisect(self, **flags)  # pragma: no cover
+            local_bisect.perform_bisect(self, **flags)
           else:
             api.step('***SINGLE TEST (deprecated)***', [])
             self.start_test_run_for_bisect(update_step, self.bot_db,
@@ -421,3 +419,32 @@ class AutoBisectApi(recipe_api.RecipeApi):
           else:
             self.ensure_checkout()
           api.chromium_android.common_tests_final_steps()
+
+  def stat_compare(self, values_a, values_b, metric,
+                   output_format='chartjson', **kwargs):
+    """Compares samples using catapult's statistics implementation.
+
+    Args:
+      values_a, values_b: lists of paths to the json files containing the values
+        produced by the test.
+      metric: the name of the metric as sent by dashboard.
+      output_format: either 'chartjson' or 'valueset'
+
+    Returns:
+      a dict containing 'result' which may be True, False or 'needMoreData', as
+      well as details about each sample ('debug_values', 'mean' and 'std_dev').
+
+    """
+    args = [','.join(map(str, values_a)),
+            ','.join(map(str, values_b)),
+            metric,
+            '--' + output_format]
+
+    script = self.m.path['catapult'].join(
+        'tracing', 'bin', 'compare_samples')
+    return self.m.python(
+        'Compare samples',
+        script=script,
+        args=args,
+        stdout=self.m.json.output(),
+        **kwargs).stdout