Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(527)

Side by Side Diff: third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/views/buildbot_results.py

Issue 2069743002: Rebaseline the actual missing results only (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 4 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (C) 2012 Google Inc. All rights reserved. 1 # Copyright (C) 2012 Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 11 matching lines...) Expand all
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 28
29 29
30 from webkitpy.layout_tests.models import test_expectations 30 from webkitpy.layout_tests.models import test_expectations
31 31
32 from webkitpy.common.net import layouttestresults 32 from webkitpy.common.net.layouttestresults import LayoutTestResults
33 33
34 34
35 TestExpectations = test_expectations.TestExpectations 35 TestExpectations = test_expectations.TestExpectations
36 TestExpectationParser = test_expectations.TestExpectationParser 36 TestExpectationParser = test_expectations.TestExpectationParser
37 37
38 38
39 class BuildBotPrinter(object): 39 class BuildBotPrinter(object):
40 # This output is parsed by buildbots and must only be changed in coordinatio n with buildbot scripts (see webkit.org's 40 # This output is parsed by buildbots and must only be changed in coordinatio n with buildbot scripts (see webkit.org's
41 # Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg: RunWebKitTests ._parseNewRunWebKitTestsOutput 41 # Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg: RunWebKitTests ._parseNewRunWebKitTestsOutput
42 # and chromium.org's buildbot/master.chromium/scripts/master/log_parser/webk it_test_command.py). 42 # and chromium.org's buildbot/master.chromium/scripts/master/log_parser/webk it_test_command.py).
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after
88 self._print(" %5d %-24s (%4.1f%%)" % (len(results), desc, pct)) 88 self._print(" %5d %-24s (%4.1f%%)" % (len(results), desc, pct))
89 89
90 def print_unexpected_results(self, summarized_results, enabled_pixel_tests_i n_retry=False): 90 def print_unexpected_results(self, summarized_results, enabled_pixel_tests_i n_retry=False):
91 passes = {} 91 passes = {}
92 flaky = {} 92 flaky = {}
93 regressions = {} 93 regressions = {}
94 94
95 def add_to_dict_of_lists(dict, key, value): 95 def add_to_dict_of_lists(dict, key, value):
96 dict.setdefault(key, []).append(value) 96 dict.setdefault(key, []).append(value)
97 97
98 def add_result(test, results, passes=passes, flaky=flaky, regressions=re gressions): 98 def add_result(result):
99 actual = results['actual'].split(" ") 99 test = result.test_name()
100 expected = results['expected'].split(" ") 100 actual = result.actual_results().split(" ")
101 expected = result.expected_results().split(" ")
101 102
102 if 'is_unexpected' not in results or not results['is_unexpected']: 103 if result.did_run_as_expected():
103 # Don't print anything for tests that ran as expected. 104 # Don't print anything for tests that ran as expected.
104 return 105 return
105 106
106 if actual == ['PASS']: 107 if actual == ['PASS']:
107 if 'CRASH' in expected: 108 if 'CRASH' in expected:
108 add_to_dict_of_lists(passes, 'Expected to crash, but passed' , test) 109 add_to_dict_of_lists(passes, 'Expected to crash, but passed' , test)
109 elif 'TIMEOUT' in expected: 110 elif 'TIMEOUT' in expected:
110 add_to_dict_of_lists(passes, 'Expected to timeout, but passe d', test) 111 add_to_dict_of_lists(passes, 'Expected to timeout, but passe d', test)
111 else: 112 else:
112 add_to_dict_of_lists(passes, 'Expected to fail, but passed', test) 113 add_to_dict_of_lists(passes, 'Expected to fail, but passed', test)
113 elif enabled_pixel_tests_in_retry and actual == ['TEXT', 'IMAGE+TEXT ']: 114 elif enabled_pixel_tests_in_retry and actual == ['TEXT', 'IMAGE+TEXT ']:
114 add_to_dict_of_lists(regressions, actual[0], test) 115 add_to_dict_of_lists(regressions, actual[0], test)
115 elif len(actual) > 1 and bool(set(actual[1:]) & set(expected)): 116 elif len(actual) > 1 and bool(set(actual[1:]) & set(expected)):
116 # We group flaky tests by the first actual result we got. 117 # We group flaky tests by the first actual result we got.
117 add_to_dict_of_lists(flaky, actual[0], test) 118 add_to_dict_of_lists(flaky, actual[0], test)
118 else: 119 else:
119 add_to_dict_of_lists(regressions, actual[0], test) 120 add_to_dict_of_lists(regressions, actual[0], test)
120 121
121 layouttestresults.for_each_test(summarized_results['tests'], add_result) 122 test_results = LayoutTestResults(summarized_results)
123 test_results.for_each_test(add_result)
122 124
123 if len(passes) or len(flaky) or len(regressions): 125 if len(passes) or len(flaky) or len(regressions):
124 self._print("") 126 self._print("")
125 if len(passes): 127 if len(passes):
126 for key, tests in passes.iteritems(): 128 for key, tests in passes.iteritems():
127 self._print("%s: (%d)" % (key, len(tests))) 129 self._print("%s: (%d)" % (key, len(tests)))
128 tests.sort() 130 tests.sort()
129 for test in tests: 131 for test in tests:
130 self._print(" %s" % test) 132 self._print(" %s" % test)
131 self._print("") 133 self._print("")
132 self._print("") 134 self._print("")
133 135
134 if len(flaky): 136 if len(flaky):
135 descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS 137 descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS
136 for key, tests in flaky.iteritems(): 138 for key, tests in flaky.iteritems():
137 result_type = TestExpectations.EXPECTATIONS[key.lower()] 139 result_type = TestExpectations.EXPECTATIONS[key.lower()]
138 self._print("Unexpected flakiness: %s (%d)" % (descriptions[resu lt_type], len(tests))) 140 self._print("Unexpected flakiness: %s (%d)" % (descriptions[resu lt_type], len(tests)))
139 tests.sort() 141 tests.sort()
140 142
141 for test in tests: 143 for test in tests:
142 result = layouttestresults.result_for_test(summarized_result s['tests'], test) 144 result = test_results.result_for_test(test)
143 actual = result['actual'].split(" ") 145 actual = result.actual_results().split(" ")
144 expected = result['expected'].split(" ") 146 expected = result.expected_results().split(" ")
145 # FIXME: clean this up once the old syntax is gone 147 # FIXME: clean this up once the old syntax is gone
146 new_expectations_list = [TestExpectationParser._inverted_exp ectation_tokens[exp] 148 new_expectations_list = [TestExpectationParser._inverted_exp ectation_tokens[exp]
147 for exp in list(set(actual) | set(e xpected))] 149 for exp in list(set(actual) | set(e xpected))]
148 self._print(" %s [ %s ]" % (test, " ".join(new_expectations _list))) 150 self._print(" %s [ %s ]" % (test, " ".join(new_expectations _list)))
149 self._print("") 151 self._print("")
150 self._print("") 152 self._print("")
151 153
152 if len(regressions): 154 if len(regressions):
153 descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS 155 descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS
154 for key, tests in regressions.iteritems(): 156 for key, tests in regressions.iteritems():
155 result_type = TestExpectations.EXPECTATIONS[key.lower()] 157 result_type = TestExpectations.EXPECTATIONS[key.lower()]
156 self._print("Regressions: Unexpected %s (%d)" % (descriptions[re sult_type], len(tests))) 158 self._print("Regressions: Unexpected %s (%d)" % (descriptions[re sult_type], len(tests)))
157 tests.sort() 159 tests.sort()
158 for test in tests: 160 for test in tests:
159 result = layouttestresults.result_for_test(summarized_result s['tests'], test) 161 result = test_results.result_for_test(test)
160 actual = result['actual'].split(" ") 162 actual = result.actual_results().split(" ")
161 expected = result['expected'].split(" ") 163 expected = result.expected_results().split(" ")
162 new_expectations_list = [TestExpectationParser._inverted_exp ectation_tokens[exp] for exp in actual] 164 new_expectations_list = [TestExpectationParser._inverted_exp ectation_tokens[exp] for exp in actual]
163 self._print(" %s [ %s ]" % (test, " ".join(new_expectations _list))) 165 self._print(" %s [ %s ]" % (test, " ".join(new_expectations _list)))
164 self._print("") 166 self._print("")
165 167
166 if len(summarized_results['tests']) and self.debug_logging: 168 if len(summarized_results['tests']) and self.debug_logging:
167 self._print("%s" % ("-" * 78)) 169 self._print("%s" % ("-" * 78))
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698