Refactor stages 1, 2 and test_api overhaul.

scripts/slave/recipes/bisection/android_bisect_staging.py

Index: scripts/slave/recipes/bisection/android_bisect_staging.py
diff --git a/scripts/slave/recipes/bisection/android_bisect_staging.py b/scripts/slave/recipes/bisection/android_bisect_staging.py
index ee24766a70f3e1d0143bd0308e95aca524aab286..b250269826319e1e05d60358a2e9038280bc7c90 100644
--- a/scripts/slave/recipes/bisection/android_bisect_staging.py
+++ b/scripts/slave/recipes/bisection/android_bisect_staging.py
@@ -118,15 +118,15 @@ def RunSteps(api, mastername, buildername):
   bot_db = api.chromium_tests.create_bot_db_from_master_dict(mastername,
                                                              master_dict)
 
-  api.auto_bisect_staging.start_try_job(
-      api, update_step=update_step, bot_db=bot_db,
-      do_not_nest_wait_for_revision=True)
+  api.auto_bisect_staging.start_try_job(api, update_step=update_step,
+                                        bot_db=bot_db,
+                                        do_not_nest_wait_for_revision=True)
 
 
 def GenTests(api):
   config_json_main = {
       'command': ('./tools/perf/run_benchmark -v --browser=android-chrome '
-                'sunspider'),
+                  '--output-format=chartjson sunspider'),
       'max_time_minutes': '25',
       'client_operation_id': '12345726327327',
       'repeat_count': '1',
@@ -189,17 +189,6 @@ results-without_patch
   for _, master_dict in BUILDERS.items():
     for buildername in master_dict.get('builders', {}):
       config_json = config_json_main.copy()
-      yield (api.test('basic_' + buildername) + api.properties.tryserver(
-        path_config='kitchen',
-        mastername='tryserver.chromium.perf',
-        buildername=buildername,
-        patch_storage='rietveld',
-        patchset='20001',
-        issue='12345',
-        is_test=True,
-        rietveld="https://codereview.chromium.org") + api.override_step_data(
-              'git diff to analyze patch',
-              api.raw_io.stream_output('tools/auto_bisect/bisect.cfg')))
 
       yield (
           api.test('basic_perf_tryjob_' + buildername) +
@@ -351,7 +340,8 @@ results-without_patch
           api.properties(requester='commit-bot@chromium.org') +
           api.override_step_data(
               'git diff to analyze patch',
-              api.raw_io.stream_output('tools/perf/benchmarks/blink_perf.py')) +
+              api.raw_io.stream_output(
+                  'tools/perf/benchmarks/blink_perf.py')) +
           api.step_data('buildbucket.put',
               stdout=api.json.output(buildbucket_put_response)) +
           api.step_data('buildbucket.get',
@@ -395,7 +385,8 @@ results-without_patch
       bisect_config = {
           'test_type': 'perf',
           'command': './tools/perf/run_benchmark -v '
-                     '--browser=android-chromium page_cycler.intl_ar_fa_he',
+                     '--browser=android-chromium --output-format=valueset '
+                     'page_cycler_v2.intl_ar_fa_he',
           'metric': 'warm_times/page_load_time',
           'repeat_count': '2',
           'max_time_minutes': '5',
@@ -410,12 +401,6 @@ results-without_patch
               path_config='kitchen',
               mastername='tryserver.chromium.perf',
               buildername=buildername) +
-          api.step_data(
-              'saving url to temp file',
-              stdout=api.raw_io.output('/tmp/dummy1')) +
-          api.step_data(
-              'saving json to temp file',
-              stdout=api.raw_io.output('/tmp/dummy2')) +
           api.properties(
                       bisect_config=bisect_config) + api.properties(
                           job_name='f7a7b4135624439cbd27fdd5133d74ec') +
@@ -426,7 +411,8 @@ results-without_patch
       local_bisect_config = {
           'test_type': 'perf',
           'command': './tools/perf/run_benchmark -v '
-                     '--browser=android-chromium page_cycler.intl_ar_fa_he',
+                     '--browser=android-chromium --output-format=valueset '
+                     'page_cycler_v2.intl_ar_fa_he',
           'metric': 'warm_times/page_load_time',
           'repeat_count': '2',
           'max_time_minutes': '5',
@@ -441,8 +427,6 @@ results-without_patch
       }
 
   buildername = 'android_one_perf_bisect'
-  good_revision_hash = 'e28dc0d49c331def2a3bbf3ddd0096eb51551155'
-  bad_revision_hash = 'fc6dfc7ff5b1073408499478969261b826441144'
   working_device = [
       {
         "battery": {
@@ -572,31 +556,25 @@ results-without_patch
         parent_got_revision='1111111',
         parent_build_archive_url='gs://test-domain/test-archive.zip') +
     api.bisect_tester_staging(tempfile='/tmp/dummy') +
-    api.step_data(
-        'Gathering reference values.saving json to temp file',
-        stdout=api.raw_io.output('/tmp/dummy3')) +
-    api.step_data(
-        'Gathering reference values.saving json to temp file (2)',
-        stdout=api.raw_io.output('/tmp/dummy4')) +
     api.override_step_data('device_status',
         api.json.output(two_devices)) +
     api.override_step_data('device_status (2)',
         api.json.output(two_devices)) +
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),
-        stdout=api.json.output(
-            {'git_sha': bad_revision_hash}))+
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config[
-            'good_revision']),
-        stdout=api.json.output(
-            {'git_sha': good_revision_hash}))+
-    api.step_data('Expanding revision range.for revisions %s:%s' % (
-        good_revision_hash, bad_revision_hash),
-        stdout=api.json.output([[bad_revision_hash, 'ignored'],
-            [good_revision_hash, 'ignored']])) +
     api.step_data('Post bisect results',
-        stdout=api.json.output({'status_code': 200})))
+        stdout=api.json.output({'status_code': 200})) +
+    api.auto_bisect_staging([
+          {
+              'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+              'commit_pos': '306475',
+              'parsed_values': [12, 13, 14, 15, 1],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          },
+          {
+              'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+              'commit_pos': '306476',
+              'parsed_values': [212, 213, 214, 215, 7],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          }]))
 
   # simulate the scenario when the no device is connected.
   yield (api.test('local_basic_recipe_no_device') +
@@ -611,7 +589,16 @@ results-without_patch
         parent_build_archive_url='gs://test-domain/test-archive.zip') +
     api.bisect_tester_staging(tempfile='/tmp/dummy') +
     api.override_step_data('device_status', api.json.output([])) +
-    api.override_step_data('device_status (2)', api.json.output([])))
+    api.override_step_data('device_status (2)', api.json.output([])) +
+    api.auto_bisect_staging([
+          {
+              'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+              'commit_pos': '306475',
+          },
+          {
+              'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+              'commit_pos': '306476',
+          }]))
 
   # simulate the scenario when tests fail not because of device
   # disconnection.
@@ -626,34 +613,28 @@ results-without_patch
         parent_got_revision='1111111',
         parent_build_archive_url='gs://test-domain/test-archive.zip') +
     api.bisect_tester_staging(tempfile='/tmp/dummy') +
-    api.step_data(
-        'Gathering reference values.saving json to temp file',
-        stdout=api.raw_io.output('/tmp/dummy3')) +
-    api.step_data(
-        'Gathering reference values.saving json to temp file (2)',
-        stdout=api.raw_io.output('/tmp/dummy4')) +
     api.override_step_data('device_status',
         api.json.output(working_device)) +
     api.override_step_data('device_status (2)',
         api.json.output(working_device)) +
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),
-        stdout=api.json.output(
-            {'git_sha': bad_revision_hash}))+
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config[
-            'good_revision']),
-          stdout=api.json.output(
-              {'git_sha': good_revision_hash}))+
-    api.step_data('Expanding revision range.for revisions %s:%s' % (
-        good_revision_hash, bad_revision_hash),
-        stdout=api.json.output([[bad_revision_hash, 'ignored'],
-            [good_revision_hash, 'ignored']])) +
     api.step_data('Debug Info', retcode=1) +
     api.step_data('Post bisect results',
         stdout=api.json.output({'status_code': 200})) +
     api.override_step_data('device_status (3)',
-        api.json.output(working_device)))
+        api.json.output(working_device)) +
+    api.auto_bisect_staging([
+          {
+              'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+              'commit_pos': '306475',
+              'parsed_values': [12, 13, 14, 15, 1],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          },
+          {
+              'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+              'commit_pos': '306476',
+              'parsed_values': [212, 213, 214, 215, 7],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          }]))
 
   # simulate the scenario when tests fail because of device disconnection.
   yield (api.test('local_basic_recipe_disconnected_device') +
@@ -661,49 +642,80 @@ results-without_patch
         mastername='tryserver.chromium.perf', buildername=buildername) +
     api.properties(
         path_config='kitchen',
-        bisect_config=local_bisect_config,
+        bisect_config={
+          'test_type': 'perf',
+          'command': './tools/perf/run_benchmark -v '
+                     '--browser=android-chromium --output-format=valueset '
+                     'page_cycler_v2.intl_ar_fa_he',
+          'metric': 'warm_times/page_load_time',
+          'bug_id': '425582',
+          'gs_bucket': 'chrome-perf',
+          'good_revision': '306474',
+          'bad_revision': '306476',
+      },
         job_name='f7a7b4135624439cbd27fdd5133d74ec',
         local_test=True,
         parent_got_revision='1111111',
         parent_build_archive_url='gs://test-domain/test-archive.zip') +
     api.bisect_tester_staging(tempfile='/tmp/dummy') +
-    api.step_data(
-        'Gathering reference values.saving json to temp file',
-        stdout=api.raw_io.output('/tmp/dummy3')) +
-    api.step_data(
-        'Gathering reference values.saving json to temp file (2)',
-        stdout=api.raw_io.output('/tmp/dummy4')) +
-    api.step_data(
-        'Gathering reference values.saving json to temp file (3)',
-        stdout=api.raw_io.output('/tmp/dummy5')) +
-    api.step_data(
-        'Gathering reference values.saving json to temp file (4)',
-        stdout=api.raw_io.output('/tmp/dummy6')) +
     api.override_step_data('device_status',
         api.json.output(two_devices)) +
     api.override_step_data('device_status (2)',
         api.json.output(two_devices)) +
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),
-        stdout=api.json.output(
-            {'git_sha': bad_revision_hash}))+
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config[
-            'good_revision']),
-        stdout=api.json.output(
-            {'git_sha': good_revision_hash}))+
-    api.step_data('Expanding revision range.for revisions %s:%s' % (
-            good_revision_hash, bad_revision_hash),
-        stdout=api.json.output([[bad_revision_hash, 'ignored'], [
-            good_revision_hash, 'ignored']])) +
     # Simulating disconnect by raising failure and changing the output of
     # multiple_device_status
     api.step_data('Debug Info', retcode=1) +
     api.override_step_data('device_status (3)',
         api.json.output(working_device)) +
-    api.step_data('Expanding revision range.for revisions %s:%s (2)' % (
-            good_revision_hash, bad_revision_hash),
-        stdout=api.json.output([[bad_revision_hash, 'ignored'], [
-            good_revision_hash, 'ignored']])) +
     api.step_data('Post bisect results',
-        stdout=api.json.output({'status_code': 200})))
+        stdout=api.json.output({'status_code': 200})) +
+    api.auto_bisect_staging([
+          {
+              'hash': '0000000000000000000000000000000000000000',
+              'commit_pos': '306474',
+              'parsed_values': [12, 13, 14, 15, 1],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          },
+          {
+              'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+              'commit_pos': '306475',
+              'parsed_values': [12, 13, 14, 15, 1],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 1}],
+          },
+          {
+              'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+              'commit_pos': '306476',
+              'parsed_values': [212, 213, 214, 215, 7],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          }]))
+
+  yield (api.test('failure_in_ref_range') +
+    api.properties.tryserver(
+        mastername='tryserver.chromium.perf', buildername=buildername) +
+    api.properties(
+        path_config='kitchen',
+        bisect_config=local_bisect_config,
+        job_name='f7a7b4135624439cbd27fdd5133d74ec',
+        local_test=True,
+        parent_got_revision='1111111',
+        parent_build_archive_url='gs://test-domain/test-archive.zip') +
+    api.bisect_tester_staging(tempfile='/tmp/dummy') +
+    api.override_step_data('device_status',
+        api.json.output(two_devices)) +
+    api.override_step_data('device_status (2)',
+        api.json.output(two_devices)) +
+    api.step_data('Post bisect results',
+        stdout=api.json.output({'status_code': 200})) +
+    api.auto_bisect_staging([
+          {
+              'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+              'commit_pos': '306475',
+              'parsed_values': [12, 13, 14, 15, 1],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          },
+          {
+              'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+              'commit_pos': '306476',
+              'parsed_values': [212, 213, 214, 215, 7],
+              'test_results': 5 * [{'stdout': 'benchmark fail', 'retcode': 1}],
+          }]))