scripts/slave/recipes/bisection/android_bisect.py - Issue 2247373002: Refactor stages 1, 2 and test_api overhaul.

Unified Diff: scripts/slave/recipes/bisection/android_bisect.py

Issue 2247373002: Refactor stages 1, 2 and test_api overhaul. (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/build.git@master

Patch Set: Rebasing, addressing feedback. Created 4 years, 3 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Download patch

« scripts/slave/recipe_modules/auto_bisect/exceptions.py ('K') | « scripts/slave/recipes/bisect_test/example_bisect_results.expected/failed_test.json ('k') | scripts/slave/recipes/bisection/android_bisect.expected/basic_perf_tryjob_android_fyi_perf_bisect.json » ('j') | scripts/slave/recipes/bisection/desktop_bisect.py » ('J')
Expand Comments ('e') | Collapse Comments ('c') | Hide Comments ('s')

Index: scripts/slave/recipes/bisection/android_bisect.py

diff --git a/scripts/slave/recipes/bisection/android_bisect.py b/scripts/slave/recipes/bisection/android_bisect.py

index 701fe34fb5e11109a7b484bafa02ccfc8f08dc1b..d97bcdd445159887ffc9ec05797287f6b28108b7 100644

--- a/scripts/slave/recipes/bisection/android_bisect.py

+++ b/scripts/slave/recipes/bisection/android_bisect.py

@@ -124,7 +124,7 @@ def RunSteps(api, mastername, buildername):

def GenTests(api):

config_json_main = {

'command': ('./tools/perf/run_benchmark -v --browser=android-chrome '

- 'sunspider'),

+ '--output-format=chartjson sunspider'),

'max_time_minutes': '25',

'client_operation_id': '12345726327327',

'repeat_count': '1',

@@ -393,7 +393,8 @@ results-without_patch

bisect_config = {

'test_type': 'perf',

'command': './tools/perf/run_benchmark -v '

- '--browser=android-chromium page_cycler.intl_ar_fa_he',

+ '--browser=android-chromium --output-format=valueset '

+ 'page_cycler.intl_ar_fa_he',

eakuefner 2016/09/09 20:35:42 strictly speaking, this should be page_cycler_v2 -

RobertoCN 2016/09/13 22:11:41 Done.

'metric': 'warm_times/page_load_time',

'repeat_count': '2',

'max_time_minutes': '5',

@@ -408,12 +409,6 @@ results-without_patch

path_config='kitchen',

mastername='tryserver.chromium.perf',

buildername=buildername) +

- api.step_data(

- 'saving url to temp file',

- stdout=api.raw_io.output('/tmp/dummy1')) +

- api.step_data(

- 'saving json to temp file',

- stdout=api.raw_io.output('/tmp/dummy2')) +

api.properties(

bisect_config=bisect_config) + api.properties(

job_name='f7a7b4135624439cbd27fdd5133d74ec') +

@@ -424,7 +419,8 @@ results-without_patch

local_bisect_config = {

'test_type': 'perf',

'command': './tools/perf/run_benchmark -v '

- '--browser=android-chromium page_cycler.intl_ar_fa_he',

+ '--browser=android-chromium --output-format=valueset '

+ 'page_cycler.intl_ar_fa_he',

eakuefner 2016/09/09 20:35:42 and here

RobertoCN 2016/09/13 22:11:41 Done.

'metric': 'warm_times/page_load_time',

'repeat_count': '2',

'max_time_minutes': '5',

@@ -570,31 +566,25 @@ results-without_patch

parent_got_revision='1111111',

parent_build_archive_url='gs://test-domain/test-archive.zip') +

api.bisect_tester(tempfile='/tmp/dummy') +

- api.step_data(

- 'Gathering reference values.saving json to temp file',

- stdout=api.raw_io.output('/tmp/dummy3')) +

- api.step_data(

- 'Gathering reference values.saving json to temp file (2)',

- stdout=api.raw_io.output('/tmp/dummy4')) +

api.override_step_data('device_status',

api.json.output(two_devices)) +

api.override_step_data('device_status (2)',

api.json.output(two_devices)) +

- api.step_data('Resolving reference range.crrev get commit hash for ' +

- ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),

- stdout=api.json.output(

- {'git_sha': bad_revision_hash}))+

- api.step_data('Resolving reference range.crrev get commit hash for ' +

- ('refs/heads/master@{#%s}' % local_bisect_config[

- 'good_revision']),

- stdout=api.json.output(

- {'git_sha': good_revision_hash}))+

- api.step_data('Expanding revision range.for revisions %s:%s' % (

- good_revision_hash, bad_revision_hash),

- stdout=api.json.output([[bad_revision_hash, 'ignored'],

- [good_revision_hash, 'ignored']])) +

api.step_data('Post bisect results',

- stdout=api.json.output({'status_code': 200})))

+ stdout=api.json.output({'status_code': 200})) +

+ api.auto_bisect([

+ {

+ 'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',

+ 'commit_pos': '306475',

+ 'parsed_values': [12, 13, 14, 15, 16],

+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],

+ },

+ {

+ 'hash': 'fc6dfc7ff5b1073408499478969261b826441144',

+ 'commit_pos': '306476',

+ 'parsed_values': [212, 213, 214, 215, 216],

+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],

+ }]))

# simulate the scenario when the no device is connected.

yield (api.test('local_basic_recipe_no_device') +

@@ -609,7 +599,16 @@ results-without_patch

parent_build_archive_url='gs://test-domain/test-archive.zip') +

api.bisect_tester(tempfile='/tmp/dummy') +

api.override_step_data('device_status', api.json.output([])) +

- api.override_step_data('device_status (2)', api.json.output([])))

+ api.override_step_data('device_status (2)', api.json.output([])) +

+ api.auto_bisect([

+ {

+ 'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',

+ 'commit_pos': '306475',

+ },

+ {

+ 'hash': 'fc6dfc7ff5b1073408499478969261b826441144',

+ 'commit_pos': '306476',

+ }]))

# simulate the scenario when tests fail not because of device

# disconnection.

@@ -624,34 +623,28 @@ results-without_patch

parent_got_revision='1111111',

parent_build_archive_url='gs://test-domain/test-archive.zip') +

api.bisect_tester(tempfile='/tmp/dummy') +

- api.step_data(

- 'Gathering reference values.saving json to temp file',

- stdout=api.raw_io.output('/tmp/dummy3')) +

- api.step_data(

- 'Gathering reference values.saving json to temp file (2)',

- stdout=api.raw_io.output('/tmp/dummy4')) +

api.override_step_data('device_status',

api.json.output(working_device)) +

api.override_step_data('device_status (2)',

api.json.output(working_device)) +

- api.step_data('Resolving reference range.crrev get commit hash for ' +

- ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),

- stdout=api.json.output(

- {'git_sha': bad_revision_hash}))+

- api.step_data('Resolving reference range.crrev get commit hash for ' +

- ('refs/heads/master@{#%s}' % local_bisect_config[

- 'good_revision']),

- stdout=api.json.output(

- {'git_sha': good_revision_hash}))+

- api.step_data('Expanding revision range.for revisions %s:%s' % (

- good_revision_hash, bad_revision_hash),

- stdout=api.json.output([[bad_revision_hash, 'ignored'],

- [good_revision_hash, 'ignored']])) +

api.step_data('Debug Info', retcode=1) +

api.step_data('Post bisect results',

stdout=api.json.output({'status_code': 200})) +

api.override_step_data('device_status (3)',

- api.json.output(working_device)))

+ api.json.output(working_device)) +

+ api.auto_bisect([

+ {

+ 'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',

+ 'commit_pos': '306475',

+ 'parsed_values': [12, 13, 14, 15, 16],

+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],

+ },

+ {

+ 'hash': 'fc6dfc7ff5b1073408499478969261b826441144',

+ 'commit_pos': '306476',

+ 'parsed_values': [212, 213, 214, 215, 216],

+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],

+ }]))

# simulate the scenario when tests fail because of device disconnection.

yield (api.test('local_basic_recipe_disconnected_device') +

@@ -659,49 +652,80 @@ results-without_patch

mastername='tryserver.chromium.perf', buildername=buildername) +

api.properties(

path_config='kitchen',

- bisect_config=local_bisect_config,

+ bisect_config={

+ 'test_type': 'perf',

+ 'command': './tools/perf/run_benchmark -v '

+ '--browser=android-chromium --output-format=valueset '

+ 'page_cycler.intl_ar_fa_he',

eakuefner 2016/09/09 20:35:42 and here

RobertoCN 2016/09/13 22:11:41 Done.

+ 'metric': 'warm_times/page_load_time',

+ 'bug_id': '425582',

+ 'gs_bucket': 'chrome-perf',

+ 'good_revision': '306474',

+ 'bad_revision': '306476',

+ },

job_name='f7a7b4135624439cbd27fdd5133d74ec',

local_test=True,

parent_got_revision='1111111',

parent_build_archive_url='gs://test-domain/test-archive.zip') +

api.bisect_tester(tempfile='/tmp/dummy') +

- api.step_data(

- 'Gathering reference values.saving json to temp file',

- stdout=api.raw_io.output('/tmp/dummy3')) +

- api.step_data(

- 'Gathering reference values.saving json to temp file (2)',

- stdout=api.raw_io.output('/tmp/dummy4')) +

- api.step_data(

- 'Gathering reference values.saving json to temp file (3)',

- stdout=api.raw_io.output('/tmp/dummy5')) +

- api.step_data(

- 'Gathering reference values.saving json to temp file (4)',

- stdout=api.raw_io.output('/tmp/dummy6')) +

api.override_step_data('device_status',

api.json.output(two_devices)) +

api.override_step_data('device_status (2)',

api.json.output(two_devices)) +

- api.step_data('Resolving reference range.crrev get commit hash for ' +

- ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),

- stdout=api.json.output(

- {'git_sha': bad_revision_hash}))+

- api.step_data('Resolving reference range.crrev get commit hash for ' +

- ('refs/heads/master@{#%s}' % local_bisect_config[

- 'good_revision']),

- stdout=api.json.output(

- {'git_sha': good_revision_hash}))+

- api.step_data('Expanding revision range.for revisions %s:%s' % (

- good_revision_hash, bad_revision_hash),

- stdout=api.json.output([[bad_revision_hash, 'ignored'], [

- good_revision_hash, 'ignored']])) +

# Simulating disconnect by raising failure and changing the output of

# multiple_device_status

api.step_data('Debug Info', retcode=1) +

api.override_step_data('device_status (3)',

api.json.output(working_device)) +

- api.step_data('Expanding revision range.for revisions %s:%s (2)' % (

- good_revision_hash, bad_revision_hash),

- stdout=api.json.output([[bad_revision_hash, 'ignored'], [

- good_revision_hash, 'ignored']])) +

api.step_data('Post bisect results',

- stdout=api.json.output({'status_code': 200})))

+ stdout=api.json.output({'status_code': 200})) +

+ api.auto_bisect([

+ {

+ 'hash': '0000000000000000000000000000000000000000',

+ 'commit_pos': '306474',

+ 'parsed_values': [12, 13, 14, 15, 16],

+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],

+ },

+ {

+ 'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',

+ 'commit_pos': '306475',

+ 'parsed_values': [12, 13, 14, 15, 16],

+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 1}],

+ },

+ {

+ 'hash': 'fc6dfc7ff5b1073408499478969261b826441144',

+ 'commit_pos': '306476',

+ 'parsed_values': [212, 213, 214, 215, 216],

+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],

+ }]))

+ yield (api.test('failure_in_ref_range') +

+ api.properties.tryserver(

+ mastername='tryserver.chromium.perf', buildername=buildername) +

+ api.properties(

+ path_config='kitchen',

+ bisect_config=local_bisect_config,

+ job_name='f7a7b4135624439cbd27fdd5133d74ec',

+ local_test=True,

+ parent_got_revision='1111111',

+ parent_build_archive_url='gs://test-domain/test-archive.zip') +

+ api.bisect_tester(tempfile='/tmp/dummy') +

+ api.override_step_data('device_status',

+ api.json.output(two_devices)) +

+ api.override_step_data('device_status (2)',

+ api.json.output(two_devices)) +

+ api.step_data('Post bisect results',

+ stdout=api.json.output({'status_code': 200})) +

+ api.auto_bisect([

+ {

+ 'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',

+ 'commit_pos': '306475',

+ 'parsed_values': [12, 13, 14, 15, 16],

+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],

+ },

+ {

+ 'hash': 'fc6dfc7ff5b1073408499478969261b826441144',

+ 'commit_pos': '306476',

+ 'parsed_values': [212, 213, 214, 215, 216],

+ 'test_results': 5 * [{'stdout': 'benchmark fail', 'retcode': 1}],

+ }]))