Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(2146)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: dashboard/dashboard/bisect_report_test.py

Issue 2588143002: Dashboard - Support build failures in bisect output. (Closed)
Patch Set: . Created 4 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « dashboard/dashboard/bisect_report.py ('k') | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: dashboard/dashboard/bisect_report_test.py
diff --git a/dashboard/dashboard/bisect_report_test.py b/dashboard/dashboard/bisect_report_test.py
index 92270750f52f98e90ae84b0ec381aa4d0365411d..fa5cda42dd7a96b5c1a28b2a1dbf067c505be29e 100644
--- a/dashboard/dashboard/bisect_report_test.py
+++ b/dashboard/dashboard/bisect_report_test.py
@@ -3,68 +3,63 @@
# found in the LICENSE file.
import copy
+import json
import unittest
from dashboard import bisect_report
from dashboard.common import testing_common
from dashboard.models import try_job
-_SAMPLE_BISECT_RESULTS_JSON = {
- 'try_job_id': None,
- 'bug_id': None,
- 'status': None,
- 'bisect_bot': 'linux',
- 'buildbot_log_url': 'http://build.chromium.org/513',
- 'command': ('tools/perf/run_benchmark -v '
- '--browser=release page_cycler'),
- 'metric': 'page_load_time',
- 'test_type': 'perf',
- 'issue_url': 'https://test-rietveld.appspot.com/200039',
- 'change': 10,
- 'good_revision': '306475',
- 'bad_revision': '306477',
- 'warnings': None,
- 'aborted_reason': None,
- 'culprit_data': {
- 'subject': 'subject',
- 'author': 'author',
- 'email': 'author@email.com',
- 'cl_date': '1/2/2015',
- 'commit_info': 'commit info',
- 'revisions_links': ['http://src.chromium.org/viewvc/chrome?view='
- 'revision&revision=306476'],
- 'cl': '306476abcdabcdfabcdfabcdfabcdfabcdfabcdf'
- },
- 'revision_data': [
- {
- 'depot_name': 'chromium',
- 'commit_hash': '306475abcdabcdfabcdfabcdfabcdfabcdfabcdf',
- 'revision_string': 'chromium@306475',
- 'mean_value': 70,
- 'std_dev': 0,
- 'values': [70, 70, 70],
- 'result': 'good'
- },
- {
- 'revision_string': 'chromium@306476',
- 'commit_hash': '306476abcdabcdfabcdfabcdfabcdfabcdfabcdf',
- 'depot_name': 'chromium',
- 'mean_value': 80,
- 'std_dev': 0,
- 'values': [80, 80, 80],
- 'result': 'bad'
- },
- {
- 'revision_string': 'chromium@306477',
- 'depot_name': 'chromium',
- 'commit_hash': '306477abcdabcdfabcdfabcdfabcdfabcdfabcdf',
- 'mean_value': 80.0,
- 'std_dev': 0.0,
- 'values': [80.0, 80.0, 80.0],
- 'result': 'bad'
- }
- ]
-}
+
+_SAMPLE_BISECT_RESULTS_JSON = json.loads("""
+ {
+ "issue_url": "https://test-rietveld.appspot.com/200039",
+ "aborted_reason": null,
+ "bad_revision": "",
+ "bisect_bot": "staging_android_nexus5X_perf_bisect",
+ "bug_id": 12345,
+ "buildbot_log_url": "http://build.chromium.org/513",
+ "change": "7.35%",
+ "command": "src/tools/perf/run_benchmark foo",
+ "culprit_data": null,
+ "good_revision": "",
+ "metric": "Total/Score",
+ "culprit_data": null,
+ "revision_data": [],
+ "secondary_regressions": [],
+ "status": "completed",
+ "test_type": "perf",
+ "try_job_id": 123456,
+ "warnings": []
+ }
+""")
+
+_SAMPLE_BISECT_REVISION_JSON = json.loads("""
+ {
+ "build_id": null,
+ "commit_hash": "",
+ "depot_name": "chromium",
+ "failed": false,
+ "failure_reason": null,
+ "mean_value": 100,
+ "n_observations": 1,
+ "result": "bad",
+ "revision_string": "",
+ "std_dev": 1.0
+ }
+""")
+
+_SAMPLE_BISECT_CULPRIT_JSON = json.loads("""
+ {
+ "author": "author",
+ "cl": "cl",
+ "cl_date": "Thu Dec 08 01:25:35 2016",
+ "commit_info": "commit_info",
+ "email": "email",
+ "revisions_links": [],
+ "subject": "subject"
+ }
+""")
class BisectReportTest(testing_common.TestCase):
@@ -77,16 +72,42 @@ class BisectReportTest(testing_common.TestCase):
job.put()
return job
- def _BisectResults(self, try_job_id, bug_id, status, **kwargs):
+ def _Revisions(self, revisions):
+ revision_data = []
+ for r in revisions:
+ data = copy.deepcopy(_SAMPLE_BISECT_REVISION_JSON)
+ data['commit_hash'] = r['commit']
+ data['failed'] = r.get('failed', False)
+ data['failure_reason'] = r.get('failure_reason', None)
+ data['mean_value'] = r.get('mean', 0)
+ data['std_dev'] = r.get('std_dev', 0)
+ data['n_observations'] = r.get('num', 0)
+ data['revision_string'] = r['commit']
+ data['result'] = r.get('result', 'unknown')
+ revision_data.append(data)
+ return revision_data
+
+ def _Culprit(self, **kwargs):
+ culprit = copy.deepcopy(_SAMPLE_BISECT_CULPRIT_JSON)
+ culprit.update(kwargs)
+ return culprit
+
+ def _BisectResults(self, **kwargs):
results = copy.deepcopy(_SAMPLE_BISECT_RESULTS_JSON)
- results['try_job_id'] = try_job_id
- results['bug_id'] = bug_id
- results['status'] = status
results.update(kwargs)
return results
def testGetReport_CompletedWithCulprit(self):
- results_data = self._BisectResults(6789, 12345, 'completed')
+ results_data = self._BisectResults(
+ revision_data=self._Revisions(
+ [
+ {'commit': 100, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 101, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 102, 'mean': 200, 'num': 10, 'result': 'bad'},
+ {'commit': 103, 'mean': 200, 'num': 10, 'result': 'bad'},
+ ]),
+ culprit_data=self._Culprit(cl=102),
+ good_revision=100, bad_revision=103)
job = self._AddTryJob(results_data)
log_with_culprit = r"""
@@ -98,23 +119,24 @@ Status: completed
Subject : subject
Author : author
Commit description:
- commit info
-Commit : 306476abcdabcdfabcdfabcdfabcdfabcdfabcdf
-Date : 1/2/2015
+ commit_info
+Commit : 102
+Date : Thu Dec 08 01:25:35 2016
===== TESTED REVISIONS =====
-Revision Mean Std Dev N Good?
-chromium@306475 70 0 3 good
-chromium@306476 80 0 3 bad <--
-chromium@306477 80.0 0.0 3 bad
+Revision Mean Std Dev N Good?
+100 100 0 10 good
+101 100 0 10 good
+102 200 0 10 bad <--
+103 200 0 10 bad
-Bisect job ran on: linux
+Bisect job ran on: staging_android_nexus5X_perf_bisect
Bug ID: 12345
-Test Command: tools/perf/run_benchmark -v --browser=release page_cycler
-Test Metric: page_load_time
-Relative Change: 10
+Test Command: src/tools/perf/run_benchmark foo
+Test Metric: Total/Score
+Relative Change: 7.35%
Buildbot stdio: http://build.chromium.org/513
Job details: https://test-rietveld.appspot.com/200039
@@ -126,8 +148,115 @@ Job details: https://test-rietveld.appspot.com/200039
self.assertEqual(log_with_culprit, bisect_report.GetReport(job))
def testGetReport_CompletedWithoutCulprit(self):
- results_data = self._BisectResults(6789, 12345, 'completed',
- culprit_data=None)
+ results_data = self._BisectResults(
+ revision_data=self._Revisions(
+ [
+ {'commit': 100, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 101, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 102, 'mean': 200, 'num': 10, 'result': 'bad'},
+ {'commit': 103, 'mean': 200, 'num': 10, 'result': 'bad'},
+ ]),
+ culprit_data=None,
+ good_revision=100, bad_revision=103)
+ job = self._AddTryJob(results_data)
+
+ log_without_culprit = r"""
+===== BISECT JOB RESULTS =====
+Status: completed
+
+
+===== TESTED REVISIONS =====
+Revision Mean Std Dev N Good?
+100 100 0 10 good
+101 100 0 10 good
+102 200 0 10 bad
+103 200 0 10 bad
+
+Bisect job ran on: staging_android_nexus5X_perf_bisect
+Bug ID: 12345
+
+Test Command: src/tools/perf/run_benchmark foo
+Test Metric: Total/Score
+Relative Change: 7.35%
+
+Buildbot stdio: http://build.chromium.org/513
+Job details: https://test-rietveld.appspot.com/200039
+
+
+| O O | Visit http://www.chromium.org/developers/speed-infra/perf-bug-faq
+| X | for more information addressing perf regression bugs. For feedback,
+| / \ | file a bug with component Tests>AutoBisect. Thank you!"""
+
+ self.assertEqual(log_without_culprit, bisect_report.GetReport(job))
+
+ def testGetReport_CompletedWithBuildFailures(self):
+ results_data = self._BisectResults(
+ revision_data=self._Revisions(
+ [
+ {'commit': 100, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 101, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 102, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 103, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 104, 'mean': 200, 'num': 10, 'result': 'bad'},
+ {'commit': 105, 'mean': 200, 'num': 10, 'result': 'bad'},
+ ]),
+ culprit_data=self._Culprit(cl=104),
+ good_revision=100, bad_revision=105)
+ job = self._AddTryJob(results_data)
+
+ log_without_culprit = r"""
+===== BISECT JOB RESULTS =====
+Status: completed
+
+
+===== SUSPECTED CL(s) =====
+Subject : subject
+Author : author
+Commit description:
+ commit_info
+Commit : 104
+Date : Thu Dec 08 01:25:35 2016
+
+
+===== TESTED REVISIONS =====
+Revision Mean Std Dev N Good?
+100 100 0 10 good
+102 100 0 10 good
+103 100 0 10 good
+104 200 0 10 bad <--
+105 200 0 10 bad
+
+Bisect job ran on: staging_android_nexus5X_perf_bisect
+Bug ID: 12345
+
+Test Command: src/tools/perf/run_benchmark foo
+Test Metric: Total/Score
+Relative Change: 7.35%
+
+Buildbot stdio: http://build.chromium.org/513
+Job details: https://test-rietveld.appspot.com/200039
+
+
+| O O | Visit http://www.chromium.org/developers/speed-infra/perf-bug-faq
+| X | for more information addressing perf regression bugs. For feedback,
+| / \ | file a bug with component Tests>AutoBisect. Thank you!"""
+
+ self.assertEqual(log_without_culprit, bisect_report.GetReport(job))
+
+ def testGetReport_CompletedCouldntNarrowCulprit(self):
+ results_data = self._BisectResults(
+ revision_data=self._Revisions(
+ [
+ {'commit': 100, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 101, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 102, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 103, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 104, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 105, 'mean': 200, 'num': 10, 'result': 'bad'},
+ {'commit': 106, 'mean': 200, 'num': 10, 'result': 'bad'},
+ ]),
+ culprit_data=None,
+ good_revision=100, bad_revision=106)
job = self._AddTryJob(results_data)
log_without_culprit = r"""
@@ -136,17 +265,78 @@ Status: completed
===== TESTED REVISIONS =====
-Revision Mean Std Dev N Good?
-chromium@306475 70 0 3 good
-chromium@306476 80 0 3 bad
-chromium@306477 80.0 0.0 3 bad
+Revision Mean Std Dev N Good?
+100 100 0 10 good
+102 100 0 10 good
+103 --- --- --- build failure
+104 --- --- --- build failure
+105 200 0 10 bad
+106 200 0 10 bad
+
+Bisect job ran on: staging_android_nexus5X_perf_bisect
+Bug ID: 12345
+
+Test Command: src/tools/perf/run_benchmark foo
+Test Metric: Total/Score
+Relative Change: 7.35%
+
+Buildbot stdio: http://build.chromium.org/513
+Job details: https://test-rietveld.appspot.com/200039
+
+
+| O O | Visit http://www.chromium.org/developers/speed-infra/perf-bug-faq
+| X | for more information addressing perf regression bugs. For feedback,
+| / \ | file a bug with component Tests>AutoBisect. Thank you!"""
+
+ self.assertEqual(log_without_culprit, bisect_report.GetReport(job))
+
+ def testGetReport_CompletedMoreThan10BuildFailures(self):
+ results_data = self._BisectResults(
+ revision_data=self._Revisions(
+ [
+ {'commit': 100, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 101, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 102, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 103, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 104, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 105, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 106, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 107, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 108, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 109, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 110, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 111, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 112, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 113, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 114, 'failed': True, 'failure_reason': 'reason'},
+ {'commit': 115, 'mean': 200, 'num': 10, 'result': 'bad'},
+ {'commit': 116, 'mean': 200, 'num': 10, 'result': 'bad'},
+ ]),
+ culprit_data=None,
+ good_revision=100, bad_revision=116)
+ job = self._AddTryJob(results_data)
+
+ log_without_culprit = r"""
+===== BISECT JOB RESULTS =====
+Status: completed
-Bisect job ran on: linux
+
+===== TESTED REVISIONS =====
+Revision Mean Std Dev N Good?
+100 100 0 10 good
+102 100 0 10 good
+103 --- --- --- build failure
+--- --- --- --- too many build failures to list
+114 --- --- --- build failure
+115 200 0 10 bad
+116 200 0 10 bad
+
+Bisect job ran on: staging_android_nexus5X_perf_bisect
Bug ID: 12345
-Test Command: tools/perf/run_benchmark -v --browser=release page_cycler
-Test Metric: page_load_time
-Relative Change: 10
+Test Command: src/tools/perf/run_benchmark foo
+Test Metric: Total/Score
+Relative Change: 7.35%
Buildbot stdio: http://build.chromium.org/513
Job details: https://test-rietveld.appspot.com/200039
@@ -159,9 +349,8 @@ Job details: https://test-rietveld.appspot.com/200039
self.assertEqual(log_without_culprit, bisect_report.GetReport(job))
def testGetReport_FailedBisect(self):
- results_data = self._BisectResults(6789, 12345, 'failed',
- culprit_data=None,
- revision_data=None)
+ results_data = self._BisectResults(
+ good_revision=100, bad_revision=110, status='failed')
job = self._AddTryJob(results_data)
log_failed_bisect = r"""
@@ -170,12 +359,12 @@ Status: failed
-Bisect job ran on: linux
+Bisect job ran on: staging_android_nexus5X_perf_bisect
Bug ID: 12345
-Test Command: tools/perf/run_benchmark -v --browser=release page_cycler
-Test Metric: page_load_time
-Relative Change: 10
+Test Command: src/tools/perf/run_benchmark foo
+Test Metric: Total/Score
+Relative Change: 7.35%
Buildbot stdio: http://build.chromium.org/513
Job details: https://test-rietveld.appspot.com/200039
@@ -188,10 +377,9 @@ Job details: https://test-rietveld.appspot.com/200039
self.assertEqual(log_failed_bisect, bisect_report.GetReport(job))
def testGetReport_BisectWithWarnings(self):
- results_data = self._BisectResults(6789, 12345, 'failed',
- culprit_data=None,
- revision_data=None,
- warnings=['A warning.'])
+ results_data = self._BisectResults(
+ status='failed', good_revision=100, bad_revision=103,
+ warnings=['A warning.'])
job = self._AddTryJob(results_data)
log_failed_bisect = r"""
@@ -205,12 +393,12 @@ The following warnings were raised by the bisect job:
* A warning.
-Bisect job ran on: linux
+Bisect job ran on: staging_android_nexus5X_perf_bisect
Bug ID: 12345
-Test Command: tools/perf/run_benchmark -v --browser=release page_cycler
-Test Metric: page_load_time
-Relative Change: 10
+Test Command: src/tools/perf/run_benchmark foo
+Test Metric: Total/Score
+Relative Change: 7.35%
Buildbot stdio: http://build.chromium.org/513
Job details: https://test-rietveld.appspot.com/200039
@@ -223,10 +411,16 @@ Job details: https://test-rietveld.appspot.com/200039
self.assertEqual(log_failed_bisect, bisect_report.GetReport(job))
def testGetReport_BisectWithAbortedReason(self):
- results_data = self._BisectResults(6789, 12345, 'aborted',
- culprit_data=None,
- revision_data=None,
- aborted_reason='invalid revisions.')
+ results_data = self._BisectResults(
+ revision_data=self._Revisions(
+ [
+ {'commit': 100, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 101, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 102, 'mean': 200, 'num': 10, 'result': 'bad'},
+ {'commit': 103, 'mean': 200, 'num': 10, 'result': 'bad'},
+ ]),
+ good_revision=100, bad_revision=103,
+ status='aborted', aborted_reason='Something terrible happened.')
job = self._AddTryJob(results_data)
log_failed_bisect = r"""
@@ -235,16 +429,22 @@ Status: aborted
=== Bisection aborted ===
-The bisect was aborted because invalid revisions.
+The bisect was aborted because Something terrible happened.
Please contact the the team (see below) if you believe this is in error.
+===== TESTED REVISIONS =====
+Revision Mean Std Dev N Good?
+100 100 0 10 good
+101 100 0 10 good
+102 200 0 10 bad
+103 200 0 10 bad
-Bisect job ran on: linux
+Bisect job ran on: staging_android_nexus5X_perf_bisect
Bug ID: 12345
-Test Command: tools/perf/run_benchmark -v --browser=release page_cycler
-Test Metric: page_load_time
-Relative Change: 10
+Test Command: src/tools/perf/run_benchmark foo
+Test Metric: Total/Score
+Relative Change: 7.35%
Buildbot stdio: http://build.chromium.org/513
Job details: https://test-rietveld.appspot.com/200039
@@ -256,9 +456,59 @@ Job details: https://test-rietveld.appspot.com/200039
self.assertEqual(log_failed_bisect, bisect_report.GetReport(job))
+ def testGetReport_StatusStarted(self):
+ results_data = self._BisectResults(
+ revision_data=self._Revisions(
+ [
+ {'commit': 100, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 101, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 102, 'mean': 200, 'num': 10, 'result': 'bad'},
+ {'commit': 103, 'mean': 200, 'num': 10, 'result': 'bad'},
+ ]),
+ good_revision=100, bad_revision=103,
+ status='started')
+ job = self._AddTryJob(results_data)
+
+ log_failed_bisect = r"""
+===== BISECT JOB RESULTS =====
+Status: started
+
+
+===== TESTED REVISIONS =====
+Revision Mean Std Dev N Good?
+100 100 0 10 good
+101 100 0 10 good
+102 200 0 10 bad
+103 200 0 10 bad
+
+Bisect job ran on: staging_android_nexus5X_perf_bisect
+Bug ID: 12345
+
+Test Command: src/tools/perf/run_benchmark foo
+Test Metric: Total/Score
+Relative Change: 7.35%
+
+Buildbot stdio: http://build.chromium.org/513
+Job details: https://test-rietveld.appspot.com/200039
+
+
+| O O | Visit http://www.chromium.org/developers/speed-infra/perf-bug-faq
+| X | for more information addressing perf regression bugs. For feedback,
+| / \ | file a bug with component Tests>AutoBisect. Thank you!"""
+
+ self.assertEqual(log_failed_bisect, bisect_report.GetReport(job))
+ # print bisect_report.GetReport(job)
+
def testGetReport_WithBugIdBadBisectFeedback(self):
- results_data = self._BisectResults(6789, 12345, 'completed',
- culprit_data=None)
+ results_data = self._BisectResults(
+ revision_data=self._Revisions(
+ [
+ {'commit': 100, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 101, 'mean': 100, 'num': 10, 'result': 'good'},
+ {'commit': 102, 'mean': 200, 'num': 10, 'result': 'bad'},
+ {'commit': 103, 'mean': 200, 'num': 10, 'result': 'bad'},
+ ]),
+ good_revision=100, bad_revision=103, bug_id=6789)
job = self._AddTryJob(results_data, bug_id=6789)
job_id = job.key.id()
@@ -268,17 +518,18 @@ Status: completed
===== TESTED REVISIONS =====
-Revision Mean Std Dev N Good?
-chromium@306475 70 0 3 good
-chromium@306476 80 0 3 bad
-chromium@306477 80.0 0.0 3 bad
+Revision Mean Std Dev N Good?
+100 100 0 10 good
+101 100 0 10 good
+102 200 0 10 bad
+103 200 0 10 bad
-Bisect job ran on: linux
-Bug ID: 12345
+Bisect job ran on: staging_android_nexus5X_perf_bisect
+Bug ID: 6789
-Test Command: tools/perf/run_benchmark -v --browser=release page_cycler
-Test Metric: page_load_time
-Relative Change: 10
+Test Command: src/tools/perf/run_benchmark foo
+Test Metric: Total/Score
+Relative Change: 7.35%%
Buildbot stdio: http://build.chromium.org/513
Job details: https://test-rietveld.appspot.com/200039
@@ -290,6 +541,7 @@ Not what you expected? We'll investigate and get back to you!
| O O | Visit http://www.chromium.org/developers/speed-infra/perf-bug-faq
| X | for more information addressing perf regression bugs. For feedback,
| / \ | file a bug with component Tests>AutoBisect. Thank you!""" % job_id
+
self.assertEqual(log_without_culprit, bisect_report.GetReport(job))
if __name__ == '__main__':
« no previous file with comments | « dashboard/dashboard/bisect_report.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698