Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(830)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: scripts/slave/recipes/bisection/android_bisect.py

Issue 2247373002: Refactor stages 1, 2 and test_api overhaul. (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/build.git@master
Patch Set: Addressing all early feedback. Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« scripts/slave/recipe_modules/auto_bisect/test_api.py ('K') | « scripts/slave/recipes/bisect_test/example_bisect_results.expected/failed_test.json ('k') | scripts/slave/recipes/bisection/android_bisect.expected/basic_android_fyi_perf_bisect.json » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: scripts/slave/recipes/bisection/android_bisect.py
diff --git a/scripts/slave/recipes/bisection/android_bisect.py b/scripts/slave/recipes/bisection/android_bisect.py
index 701fe34fb5e11109a7b484bafa02ccfc8f08dc1b..e8c7a920c3b9b5ec0171c4f013790b1cbe91583a 100644
--- a/scripts/slave/recipes/bisection/android_bisect.py
+++ b/scripts/slave/recipes/bisection/android_bisect.py
@@ -124,7 +124,7 @@ def RunSteps(api, mastername, buildername):
def GenTests(api):
config_json_main = {
'command': ('./tools/perf/run_benchmark -v --browser=android-chrome '
- 'sunspider'),
+ '--output-format=chartjson sunspider'),
'max_time_minutes': '25',
'client_operation_id': '12345726327327',
'repeat_count': '1',
@@ -187,17 +187,6 @@ results-without_patch
for _, master_dict in BUILDERS.items():
for buildername in master_dict.get('builders', {}):
config_json = config_json_main.copy()
- yield (api.test('basic_' + buildername) + api.properties.tryserver(
- path_config='kitchen',
- mastername='tryserver.chromium.perf',
- buildername=buildername,
- patch_storage='rietveld',
- patchset='20001',
- issue='12345',
- is_test=True,
- rietveld="https://codereview.chromium.org") + api.override_step_data(
- 'git diff to analyze patch',
- api.raw_io.stream_output('tools/auto_bisect/bisect.cfg')))
yield (
api.test('basic_perf_tryjob_' + buildername) +
@@ -393,7 +382,8 @@ results-without_patch
bisect_config = {
'test_type': 'perf',
'command': './tools/perf/run_benchmark -v '
- '--browser=android-chromium page_cycler.intl_ar_fa_he',
+ '--browser=android-chromium --output-format=valueset '
+ 'page_cycler_v2.intl_ar_fa_he',
'metric': 'warm_times/page_load_time',
'repeat_count': '2',
'max_time_minutes': '5',
@@ -408,12 +398,6 @@ results-without_patch
path_config='kitchen',
mastername='tryserver.chromium.perf',
buildername=buildername) +
- api.step_data(
- 'saving url to temp file',
- stdout=api.raw_io.output('/tmp/dummy1')) +
- api.step_data(
- 'saving json to temp file',
- stdout=api.raw_io.output('/tmp/dummy2')) +
api.properties(
bisect_config=bisect_config) + api.properties(
job_name='f7a7b4135624439cbd27fdd5133d74ec') +
@@ -424,7 +408,8 @@ results-without_patch
local_bisect_config = {
'test_type': 'perf',
'command': './tools/perf/run_benchmark -v '
- '--browser=android-chromium page_cycler.intl_ar_fa_he',
+ '--browser=android-chromium --output-format=valueset '
+ 'page_cycler_v2.intl_ar_fa_he',
'metric': 'warm_times/page_load_time',
'repeat_count': '2',
'max_time_minutes': '5',
@@ -570,31 +555,25 @@ results-without_patch
parent_got_revision='1111111',
parent_build_archive_url='gs://test-domain/test-archive.zip') +
api.bisect_tester(tempfile='/tmp/dummy') +
- api.step_data(
- 'Gathering reference values.saving json to temp file',
- stdout=api.raw_io.output('/tmp/dummy3')) +
- api.step_data(
- 'Gathering reference values.saving json to temp file (2)',
- stdout=api.raw_io.output('/tmp/dummy4')) +
api.override_step_data('device_status',
api.json.output(two_devices)) +
api.override_step_data('device_status (2)',
api.json.output(two_devices)) +
- api.step_data('Resolving reference range.crrev get commit hash for ' +
- ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),
- stdout=api.json.output(
- {'git_sha': bad_revision_hash}))+
- api.step_data('Resolving reference range.crrev get commit hash for ' +
- ('refs/heads/master@{#%s}' % local_bisect_config[
- 'good_revision']),
- stdout=api.json.output(
- {'git_sha': good_revision_hash}))+
- api.step_data('Expanding revision range.for revisions %s:%s' % (
- good_revision_hash, bad_revision_hash),
- stdout=api.json.output([[bad_revision_hash, 'ignored'],
- [good_revision_hash, 'ignored']])) +
api.step_data('Post bisect results',
- stdout=api.json.output({'status_code': 200})))
+ stdout=api.json.output({'status_code': 200})) +
+ api.auto_bisect([
+ {
+ 'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+ 'commit_pos': '306475',
+ 'parsed_values': [12, 13, 14, 15, 16],
+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+ },
+ {
+ 'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+ 'commit_pos': '306476',
+ 'parsed_values': [212, 213, 214, 215, 216],
+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+ }]))
# simulate the scenario when the no device is connected.
yield (api.test('local_basic_recipe_no_device') +
@@ -609,7 +588,16 @@ results-without_patch
parent_build_archive_url='gs://test-domain/test-archive.zip') +
api.bisect_tester(tempfile='/tmp/dummy') +
api.override_step_data('device_status', api.json.output([])) +
- api.override_step_data('device_status (2)', api.json.output([])))
+ api.override_step_data('device_status (2)', api.json.output([])) +
+ api.auto_bisect([
+ {
+ 'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+ 'commit_pos': '306475',
+ },
+ {
+ 'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+ 'commit_pos': '306476',
+ }]))
# simulate the scenario when tests fail not because of device
# disconnection.
@@ -624,34 +612,28 @@ results-without_patch
parent_got_revision='1111111',
parent_build_archive_url='gs://test-domain/test-archive.zip') +
api.bisect_tester(tempfile='/tmp/dummy') +
- api.step_data(
- 'Gathering reference values.saving json to temp file',
- stdout=api.raw_io.output('/tmp/dummy3')) +
- api.step_data(
- 'Gathering reference values.saving json to temp file (2)',
- stdout=api.raw_io.output('/tmp/dummy4')) +
api.override_step_data('device_status',
api.json.output(working_device)) +
api.override_step_data('device_status (2)',
api.json.output(working_device)) +
- api.step_data('Resolving reference range.crrev get commit hash for ' +
- ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),
- stdout=api.json.output(
- {'git_sha': bad_revision_hash}))+
- api.step_data('Resolving reference range.crrev get commit hash for ' +
- ('refs/heads/master@{#%s}' % local_bisect_config[
- 'good_revision']),
- stdout=api.json.output(
- {'git_sha': good_revision_hash}))+
- api.step_data('Expanding revision range.for revisions %s:%s' % (
- good_revision_hash, bad_revision_hash),
- stdout=api.json.output([[bad_revision_hash, 'ignored'],
- [good_revision_hash, 'ignored']])) +
api.step_data('Debug Info', retcode=1) +
api.step_data('Post bisect results',
stdout=api.json.output({'status_code': 200})) +
api.override_step_data('device_status (3)',
- api.json.output(working_device)))
+ api.json.output(working_device)) +
+ api.auto_bisect([
+ {
+ 'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+ 'commit_pos': '306475',
+ 'parsed_values': [12, 13, 14, 15, 16],
+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+ },
+ {
+ 'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+ 'commit_pos': '306476',
+ 'parsed_values': [212, 213, 214, 215, 216],
+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+ }]))
# simulate the scenario when tests fail because of device disconnection.
yield (api.test('local_basic_recipe_disconnected_device') +
@@ -659,49 +641,80 @@ results-without_patch
mastername='tryserver.chromium.perf', buildername=buildername) +
api.properties(
path_config='kitchen',
- bisect_config=local_bisect_config,
+ bisect_config={
+ 'test_type': 'perf',
+ 'command': './tools/perf/run_benchmark -v '
+ '--browser=android-chromium --output-format=valueset '
+ 'page_cycler_v2.intl_ar_fa_he',
+ 'metric': 'warm_times/page_load_time',
+ 'bug_id': '425582',
+ 'gs_bucket': 'chrome-perf',
+ 'good_revision': '306474',
+ 'bad_revision': '306476',
+ },
job_name='f7a7b4135624439cbd27fdd5133d74ec',
local_test=True,
parent_got_revision='1111111',
parent_build_archive_url='gs://test-domain/test-archive.zip') +
api.bisect_tester(tempfile='/tmp/dummy') +
- api.step_data(
- 'Gathering reference values.saving json to temp file',
- stdout=api.raw_io.output('/tmp/dummy3')) +
- api.step_data(
- 'Gathering reference values.saving json to temp file (2)',
- stdout=api.raw_io.output('/tmp/dummy4')) +
- api.step_data(
- 'Gathering reference values.saving json to temp file (3)',
- stdout=api.raw_io.output('/tmp/dummy5')) +
- api.step_data(
- 'Gathering reference values.saving json to temp file (4)',
- stdout=api.raw_io.output('/tmp/dummy6')) +
api.override_step_data('device_status',
api.json.output(two_devices)) +
api.override_step_data('device_status (2)',
api.json.output(two_devices)) +
- api.step_data('Resolving reference range.crrev get commit hash for ' +
- ('refs/heads/master@{#%s}' % local_bisect_config['bad_revision']),
- stdout=api.json.output(
- {'git_sha': bad_revision_hash}))+
- api.step_data('Resolving reference range.crrev get commit hash for ' +
- ('refs/heads/master@{#%s}' % local_bisect_config[
- 'good_revision']),
- stdout=api.json.output(
- {'git_sha': good_revision_hash}))+
- api.step_data('Expanding revision range.for revisions %s:%s' % (
- good_revision_hash, bad_revision_hash),
- stdout=api.json.output([[bad_revision_hash, 'ignored'], [
- good_revision_hash, 'ignored']])) +
# Simulating disconnect by raising failure and changing the output of
# multiple_device_status
api.step_data('Debug Info', retcode=1) +
api.override_step_data('device_status (3)',
api.json.output(working_device)) +
- api.step_data('Expanding revision range.for revisions %s:%s (2)' % (
- good_revision_hash, bad_revision_hash),
- stdout=api.json.output([[bad_revision_hash, 'ignored'], [
- good_revision_hash, 'ignored']])) +
api.step_data('Post bisect results',
- stdout=api.json.output({'status_code': 200})))
+ stdout=api.json.output({'status_code': 200})) +
+ api.auto_bisect([
+ {
+ 'hash': '0000000000000000000000000000000000000000',
+ 'commit_pos': '306474',
+ 'parsed_values': [12, 13, 14, 15, 16],
+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+ },
+ {
+ 'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+ 'commit_pos': '306475',
+ 'parsed_values': [12, 13, 14, 15, 16],
+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 1}],
+ },
+ {
+ 'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+ 'commit_pos': '306476',
+ 'parsed_values': [212, 213, 214, 215, 216],
+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+ }]))
+
+ yield (api.test('failure_in_ref_range') +
+ api.properties.tryserver(
+ mastername='tryserver.chromium.perf', buildername=buildername) +
+ api.properties(
+ path_config='kitchen',
+ bisect_config=local_bisect_config,
+ job_name='f7a7b4135624439cbd27fdd5133d74ec',
+ local_test=True,
+ parent_got_revision='1111111',
+ parent_build_archive_url='gs://test-domain/test-archive.zip') +
+ api.bisect_tester(tempfile='/tmp/dummy') +
+ api.override_step_data('device_status',
+ api.json.output(two_devices)) +
+ api.override_step_data('device_status (2)',
+ api.json.output(two_devices)) +
+ api.step_data('Post bisect results',
+ stdout=api.json.output({'status_code': 200})) +
+ api.auto_bisect([
+ {
+ 'hash': 'e28dc0d49c331def2a3bbf3ddd0096eb51551155',
+ 'commit_pos': '306475',
+ 'parsed_values': [12, 13, 14, 15, 16],
+ 'test_results': 5 * [{'stdout': 'benchmark text', 'retcode': 0}],
+ },
+ {
+ 'hash': 'fc6dfc7ff5b1073408499478969261b826441144',
+ 'commit_pos': '306476',
+ 'parsed_values': [212, 213, 214, 215, 216],
+ 'test_results': 5 * [{'stdout': 'benchmark fail', 'retcode': 1}],
+ }]))
« scripts/slave/recipe_modules/auto_bisect/test_api.py ('K') | « scripts/slave/recipes/bisect_test/example_bisect_results.expected/failed_test.json ('k') | scripts/slave/recipes/bisection/android_bisect.expected/basic_android_fyi_perf_bisect.json » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698