Refactor stages 1, 2 and test_api overhaul.

scripts/slave/recipes/bisection/android_bisect.py

 # Copyright (c) 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from recipe_engine.types import freeze
 
 DEPS = [
     'auto_bisect',
     'bisect_tester',
     'depot_tools/bot_update',
@@ skipping 106 matching lines @@
   bot_db = api.chromium_tests.create_bot_db_from_master_dict(mastername,
                                                              master_dict)
 
   api.auto_bisect.start_try_job(api, update_step=update_step, bot_db=bot_db,
                                 do_not_nest_wait_for_revision=True)
 
 
 def GenTests(api):
   config_json_main = {
       'command': ('./tools/perf/run_benchmark -v --browser=android-chrome '
-                'sunspider'),
+                  '--output-format=chartjson sunspider'),
       'max_time_minutes': '25',
       'client_operation_id': '12345726327327',
       'repeat_count': '1',
       'truncate_percent': '25',
       'target_arch': 'ia32',
   }
 
   results_with_patch = """*RESULT dummy: dummy= [5.83,6.013,5.573]ms
 Avg dummy: 5.907711ms
 Sd  dummy: 0.255921ms
@@ skipping 248 matching lines @@
               'git diff to analyze patch',
               api.raw_io.stream_output('tools/perf/benchmarks/sunspider.py')) +
           api.step_data('buildbucket.put',
               stdout=api.json.output(buildbucket_put_response)) +
           api.step_data('buildbucket.get',
               stdout=api.json.output(buildbucket_get_response)))
 
       bisect_config = {
           'test_type': 'perf',
           'command': './tools/perf/run_benchmark -v '
-                     '--browser=android-chromium page_cycler.intl_ar_fa_he',
+                     '--browser=android-chromium --output-format=valueset '
+                     'page_cycler.intl_ar_fa_he',

[eakuefner, 2016/09/09 20:35:42]

strictly speaking, this should be page_cycler_v2 -- not only have the original
page_cycler benchmarks been deleted, they're not capable of producing
valueset-format output.

[RobertoCN, 2016/09/13 22:11:41]

Done.

           'metric': 'warm_times/page_load_time',
           'repeat_count': '2',
           'max_time_minutes': '5',
           'truncate_percent': '25',
           'bug_id': '425582',
           'gs_bucket': 'chrome-perf',
           'builder_host': 'master4.golo.chromium.org',
           'builder_port': '8341'
       }
       yield (api.test('basic_recipe_' + buildername) +
           api.properties.tryserver(
               path_config='kitchen',
               mastername='tryserver.chromium.perf',
               buildername=buildername) +
-          api.step_data(
-              'saving url to temp file',
-              stdout=api.raw_io.output('/tmp/dummy1')) +
-          api.step_data(
-              'saving json to temp file',
-              stdout=api.raw_io.output('/tmp/dummy2')) +
           api.properties(
                       bisect_config=bisect_config) + api.properties(
                           job_name='f7a7b4135624439cbd27fdd5133d74ec') +
              api.bisect_tester(tempfile='/tmp/dummy') +
              api.properties(parent_got_revision='1111111') + api.properties(
                  parent_build_archive_url='gs://test-domain/test-archive.zip'))
 
       local_bisect_config = {
           'test_type': 'perf',
           'command': './tools/perf/run_benchmark -v '
-                     '--browser=android-chromium page_cycler.intl_ar_fa_he',
+                     '--browser=android-chromium --output-format=valueset '
+                     'page_cycler.intl_ar_fa_he',

[eakuefner, 2016/09/09 20:35:42]

and here

[RobertoCN, 2016/09/13 22:11:41]

Done.

           'metric': 'warm_times/page_load_time',
           'repeat_count': '2',
           'max_time_minutes': '5',
           'truncate_percent': '25',
           'bug_id': '425582',
           'gs_bucket': 'chrome-perf',
           'builder_host': 'master4.golo.chromium.org',
           'builder_port': '8341',
           'good_revision': '306475',
           'bad_revision': '306476',
@@ skipping 125 matching lines @@
     api.properties.tryserver(
         mastername='tryserver.chromium.perf', buildername=buildername) +
     api.properties(
         path_config='kitchen',
         bisect_config=local_bisect_config,
         job_name='f7a7b4135624439cbd27fdd5133d74ec',
         local_test=True,
         parent_got_revision='1111111',
         parent_build_archive_url='gs://test-domain/test-archive.zip') +
     api.bisect_tester(tempfile='/tmp/dummy') +
-    api.step_data(
-        'Gathering reference values.saving json to temp file',
-        stdout=api.raw_io.output('/tmp/dummy3')) +
-    api.step_data(
-        'Gathering reference values.saving json to temp file (2)',
-        stdout=api.raw_io.output('/tmp/dummy4')) +
     api.override_step_data('device_status',
         api.json.output(two_devices)) +
     api.override_step_data('device_status (2)',
         api.json.output(two_devices)) +
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),
-        stdout=api.json.output(
-            {'git_sha': bad_revision_hash}))+
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config[
-            'good_revision']),
-        stdout=api.json.output(
-            {'git_sha': good_revision_hash}))+
-    api.step_data('Expanding revision range.for revisions %s:%s' % (
-        good_revision_hash, bad_revision_hash),
-        stdout=api.json.output([[bad_revision_hash, 'ignored'],
-            [good_revision_hash, 'ignored']])) +
     api.step_data('Post bisect results',
-        stdout=api.json.output({'status_code': 200})))
+        stdout=api.json.output({'status_code': 200})) +
+    api.auto_bisect([
+          {
+              'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+              'commit_pos': '306475',
+              'parsed_values': [12, 13, 14, 15, 16],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          },
+          {
+              'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+              'commit_pos': '306476',
+              'parsed_values': [212, 213, 214, 215, 216],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          }]))
 
   # simulate the scenario when the no device is connected.
   yield (api.test('local_basic_recipe_no_device') +
     api.properties.tryserver(
         mastername='tryserver.chromium.perf', buildername=buildername) +
     api.properties(
         path_config='kitchen',
         bisect_config=local_bisect_config,
         job_name='f7a7b4135624439cbd27fdd5133d74ec',
         local_test=True,
         parent_got_revision='1111111',
         parent_build_archive_url='gs://test-domain/test-archive.zip') +
     api.bisect_tester(tempfile='/tmp/dummy') +
     api.override_step_data('device_status', api.json.output([])) +
-    api.override_step_data('device_status (2)', api.json.output([])))
+    api.override_step_data('device_status (2)', api.json.output([])) +
+    api.auto_bisect([
+          {
+              'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+              'commit_pos': '306475',
+          },
+          {
+              'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+              'commit_pos': '306476',
+          }]))
 
   # simulate the scenario when tests fail not because of device
   # disconnection.
   yield (api.test('local_basic_recipe_failed_device') +
     api.properties.tryserver(
         mastername='tryserver.chromium.perf', buildername=buildername) +
     api.properties(
         path_config='kitchen',
         bisect_config=local_bisect_config,
         job_name='f7a7b4135624439cbd27fdd5133d74ec',
         local_test=True,
         parent_got_revision='1111111',
         parent_build_archive_url='gs://test-domain/test-archive.zip') +
     api.bisect_tester(tempfile='/tmp/dummy') +
-    api.step_data(
-        'Gathering reference values.saving json to temp file',
-        stdout=api.raw_io.output('/tmp/dummy3')) +
-    api.step_data(
-        'Gathering reference values.saving json to temp file (2)',
-        stdout=api.raw_io.output('/tmp/dummy4')) +
     api.override_step_data('device_status',
         api.json.output(working_device)) +
     api.override_step_data('device_status (2)',
         api.json.output(working_device)) +
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),
-        stdout=api.json.output(
-            {'git_sha': bad_revision_hash}))+
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config[
-            'good_revision']),
-          stdout=api.json.output(
-              {'git_sha': good_revision_hash}))+
-    api.step_data('Expanding revision range.for revisions %s:%s' % (
-        good_revision_hash, bad_revision_hash),
-        stdout=api.json.output([[bad_revision_hash, 'ignored'],
-            [good_revision_hash, 'ignored']])) +
     api.step_data('Debug Info', retcode=1) +
     api.step_data('Post bisect results',
         stdout=api.json.output({'status_code': 200})) +
     api.override_step_data('device_status (3)',
-        api.json.output(working_device)))
+        api.json.output(working_device)) +
+    api.auto_bisect([
+          {
+              'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+              'commit_pos': '306475',
+              'parsed_values': [12, 13, 14, 15, 16],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          },
+          {
+              'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+              'commit_pos': '306476',
+              'parsed_values': [212, 213, 214, 215, 216],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          }]))
 
   # simulate the scenario when tests fail because of device disconnection.
   yield (api.test('local_basic_recipe_disconnected_device') +
     api.properties.tryserver(
         mastername='tryserver.chromium.perf', buildername=buildername) +
     api.properties(
         path_config='kitchen',
+        bisect_config={
+          'test_type': 'perf',
+          'command': './tools/perf/run_benchmark -v '
+                     '--browser=android-chromium --output-format=valueset '
+                     'page_cycler.intl_ar_fa_he',

[eakuefner, 2016/09/09 20:35:42]

and here

[RobertoCN, 2016/09/13 22:11:41]

Done.

+          'metric': 'warm_times/page_load_time',
+          'bug_id': '425582',
+          'gs_bucket': 'chrome-perf',
+          'good_revision': '306474',
+          'bad_revision': '306476',
+      },
+        job_name='f7a7b4135624439cbd27fdd5133d74ec',
+        local_test=True,
+        parent_got_revision='1111111',
+        parent_build_archive_url='gs://test-domain/test-archive.zip') +
+    api.bisect_tester(tempfile='/tmp/dummy') +
+    api.override_step_data('device_status',
+        api.json.output(two_devices)) +
+    api.override_step_data('device_status (2)',
+        api.json.output(two_devices)) +
+    # Simulating disconnect by raising failure and changing the output of
+    # multiple_device_status
+    api.step_data('Debug Info', retcode=1) +
+    api.override_step_data('device_status (3)',
+        api.json.output(working_device)) +
+    api.step_data('Post bisect results',
+        stdout=api.json.output({'status_code': 200})) +
+    api.auto_bisect([
+          {
+              'hash': '0000000000000000000000000000000000000000',
+              'commit_pos': '306474',
+              'parsed_values': [12, 13, 14, 15, 16],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          },
+          {
+              'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+              'commit_pos': '306475',
+              'parsed_values': [12, 13, 14, 15, 16],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 1}],
+          },
+          {
+              'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+              'commit_pos': '306476',
+              'parsed_values': [212, 213, 214, 215, 216],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          }]))
+
+  yield (api.test('failure_in_ref_range') +
+    api.properties.tryserver(
+        mastername='tryserver.chromium.perf', buildername=buildername) +
+    api.properties(
+        path_config='kitchen',
         bisect_config=local_bisect_config,
         job_name='f7a7b4135624439cbd27fdd5133d74ec',
         local_test=True,
         parent_got_revision='1111111',
         parent_build_archive_url='gs://test-domain/test-archive.zip') +
     api.bisect_tester(tempfile='/tmp/dummy') +
-    api.step_data(
-        'Gathering reference values.saving json to temp file',
-        stdout=api.raw_io.output('/tmp/dummy3')) +
-    api.step_data(
-        'Gathering reference values.saving json to temp file (2)',
-        stdout=api.raw_io.output('/tmp/dummy4')) +
-    api.step_data(
-        'Gathering reference values.saving json to temp file (3)',
-        stdout=api.raw_io.output('/tmp/dummy5')) +
-    api.step_data(
-        'Gathering reference values.saving json to temp file (4)',
-        stdout=api.raw_io.output('/tmp/dummy6')) +
     api.override_step_data('device_status',
         api.json.output(two_devices)) +
     api.override_step_data('device_status (2)',
         api.json.output(two_devices)) +
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),
-        stdout=api.json.output(
-            {'git_sha': bad_revision_hash}))+
-    api.step_data('Resolving reference range.crrev get commit hash for ' +
-        ('refs/heads/master@{#%s}' % local_bisect_config[
-            'good_revision']),
-        stdout=api.json.output(
-            {'git_sha': good_revision_hash}))+
-    api.step_data('Expanding revision range.for revisions %s:%s' % (
-            good_revision_hash, bad_revision_hash),
-        stdout=api.json.output([[bad_revision_hash, 'ignored'], [
-            good_revision_hash, 'ignored']])) +
-    # Simulating disconnect by raising failure and changing the output of
-    # multiple_device_status
-    api.step_data('Debug Info', retcode=1) +
-    api.override_step_data('device_status (3)',
-        api.json.output(working_device)) +
-    api.step_data('Expanding revision range.for revisions %s:%s (2)' % (
-            good_revision_hash, bad_revision_hash),
-        stdout=api.json.output([[bad_revision_hash, 'ignored'], [
-            good_revision_hash, 'ignored']])) +
     api.step_data('Post bisect results',
-        stdout=api.json.output({'status_code': 200})))
+        stdout=api.json.output({'status_code': 200})) +
+    api.auto_bisect([
+          {
+              'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+              'commit_pos': '306475',
+              'parsed_values': [12, 13, 14, 15, 16],
+              'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+          },
+          {
+              'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+              'commit_pos': '306476',
+              'parsed_values': [212, 213, 214, 215, 216],
+              'test_results': 5 * [{'stdout': 'benchmark fail', 'retcode': 1}],
+          }]))