| OLD | NEW |
| 1 # Copyright (C) 2012 Google Inc. All rights reserved. | 1 # Copyright (C) 2012 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 11 matching lines...) Expand all Loading... |
| 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 28 | 28 |
| 29 | 29 |
| 30 from webkitpy.layout_tests.models import test_expectations | 30 from webkitpy.layout_tests.models import test_expectations |
| 31 | 31 |
| 32 from webkitpy.common.net import layouttestresults | 32 from webkitpy.common.net.layouttestresults import LayoutTestResults |
| 33 | 33 |
| 34 | 34 |
| 35 TestExpectations = test_expectations.TestExpectations | 35 TestExpectations = test_expectations.TestExpectations |
| 36 TestExpectationParser = test_expectations.TestExpectationParser | 36 TestExpectationParser = test_expectations.TestExpectationParser |
| 37 | 37 |
| 38 | 38 |
| 39 class BuildBotPrinter(object): | 39 class BuildBotPrinter(object): |
| 40 # This output is parsed by buildbots and must only be changed in coordinatio
n with buildbot scripts (see webkit.org's | 40 # This output is parsed by buildbots and must only be changed in coordinatio
n with buildbot scripts (see webkit.org's |
| 41 # Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg: RunWebKitTests
._parseNewRunWebKitTestsOutput | 41 # Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg: RunWebKitTests
._parseNewRunWebKitTestsOutput |
| 42 # and chromium.org's buildbot/master.chromium/scripts/master/log_parser/webk
it_test_command.py). | 42 # and chromium.org's buildbot/master.chromium/scripts/master/log_parser/webk
it_test_command.py). |
| (...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 88 self._print(" %5d %-24s (%4.1f%%)" % (len(results), desc, pct)) | 88 self._print(" %5d %-24s (%4.1f%%)" % (len(results), desc, pct)) |
| 89 | 89 |
| 90 def print_unexpected_results(self, summarized_results, enabled_pixel_tests_i
n_retry=False): | 90 def print_unexpected_results(self, summarized_results, enabled_pixel_tests_i
n_retry=False): |
| 91 passes = {} | 91 passes = {} |
| 92 flaky = {} | 92 flaky = {} |
| 93 regressions = {} | 93 regressions = {} |
| 94 | 94 |
| 95 def add_to_dict_of_lists(dict, key, value): | 95 def add_to_dict_of_lists(dict, key, value): |
| 96 dict.setdefault(key, []).append(value) | 96 dict.setdefault(key, []).append(value) |
| 97 | 97 |
| 98 def add_result(test, results, passes=passes, flaky=flaky, regressions=re
gressions): | 98 def add_result(result): |
| 99 actual = results['actual'].split(" ") | 99 test = result.test_name() |
| 100 expected = results['expected'].split(" ") | 100 actual = result.actual_results().split(" ") |
| 101 expected = result.expected_results().split(" ") |
| 101 | 102 |
| 102 if 'is_unexpected' not in results or not results['is_unexpected']: | 103 if result.did_run_as_expected(): |
| 103 # Don't print anything for tests that ran as expected. | 104 # Don't print anything for tests that ran as expected. |
| 104 return | 105 return |
| 105 | 106 |
| 106 if actual == ['PASS']: | 107 if actual == ['PASS']: |
| 107 if 'CRASH' in expected: | 108 if 'CRASH' in expected: |
| 108 add_to_dict_of_lists(passes, 'Expected to crash, but passed'
, test) | 109 add_to_dict_of_lists(passes, 'Expected to crash, but passed'
, test) |
| 109 elif 'TIMEOUT' in expected: | 110 elif 'TIMEOUT' in expected: |
| 110 add_to_dict_of_lists(passes, 'Expected to timeout, but passe
d', test) | 111 add_to_dict_of_lists(passes, 'Expected to timeout, but passe
d', test) |
| 111 else: | 112 else: |
| 112 add_to_dict_of_lists(passes, 'Expected to fail, but passed',
test) | 113 add_to_dict_of_lists(passes, 'Expected to fail, but passed',
test) |
| 113 elif enabled_pixel_tests_in_retry and actual == ['TEXT', 'IMAGE+TEXT
']: | 114 elif enabled_pixel_tests_in_retry and actual == ['TEXT', 'IMAGE+TEXT
']: |
| 114 add_to_dict_of_lists(regressions, actual[0], test) | 115 add_to_dict_of_lists(regressions, actual[0], test) |
| 115 elif len(actual) > 1 and bool(set(actual[1:]) & set(expected)): | 116 elif len(actual) > 1 and bool(set(actual[1:]) & set(expected)): |
| 116 # We group flaky tests by the first actual result we got. | 117 # We group flaky tests by the first actual result we got. |
| 117 add_to_dict_of_lists(flaky, actual[0], test) | 118 add_to_dict_of_lists(flaky, actual[0], test) |
| 118 else: | 119 else: |
| 119 add_to_dict_of_lists(regressions, actual[0], test) | 120 add_to_dict_of_lists(regressions, actual[0], test) |
| 120 | 121 |
| 121 layouttestresults.for_each_test(summarized_results['tests'], add_result) | 122 test_results = LayoutTestResults(summarized_results) |
| 123 test_results.for_each_test(add_result) |
| 122 | 124 |
| 123 if len(passes) or len(flaky) or len(regressions): | 125 if len(passes) or len(flaky) or len(regressions): |
| 124 self._print("") | 126 self._print("") |
| 125 if len(passes): | 127 if len(passes): |
| 126 for key, tests in passes.iteritems(): | 128 for key, tests in passes.iteritems(): |
| 127 self._print("%s: (%d)" % (key, len(tests))) | 129 self._print("%s: (%d)" % (key, len(tests))) |
| 128 tests.sort() | 130 tests.sort() |
| 129 for test in tests: | 131 for test in tests: |
| 130 self._print(" %s" % test) | 132 self._print(" %s" % test) |
| 131 self._print("") | 133 self._print("") |
| 132 self._print("") | 134 self._print("") |
| 133 | 135 |
| 134 if len(flaky): | 136 if len(flaky): |
| 135 descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS | 137 descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS |
| 136 for key, tests in flaky.iteritems(): | 138 for key, tests in flaky.iteritems(): |
| 137 result_type = TestExpectations.EXPECTATIONS[key.lower()] | 139 result_type = TestExpectations.EXPECTATIONS[key.lower()] |
| 138 self._print("Unexpected flakiness: %s (%d)" % (descriptions[resu
lt_type], len(tests))) | 140 self._print("Unexpected flakiness: %s (%d)" % (descriptions[resu
lt_type], len(tests))) |
| 139 tests.sort() | 141 tests.sort() |
| 140 | 142 |
| 141 for test in tests: | 143 for test in tests: |
| 142 result = layouttestresults.result_for_test(summarized_result
s['tests'], test) | 144 result = test_results.result_for_test(test) |
| 143 actual = result['actual'].split(" ") | 145 actual = result.actual_results().split(" ") |
| 144 expected = result['expected'].split(" ") | 146 expected = result.expected_results().split(" ") |
| 145 # FIXME: clean this up once the old syntax is gone | 147 # FIXME: clean this up once the old syntax is gone |
| 146 new_expectations_list = [TestExpectationParser._inverted_exp
ectation_tokens[exp] | 148 new_expectations_list = [TestExpectationParser._inverted_exp
ectation_tokens[exp] |
| 147 for exp in list(set(actual) | set(e
xpected))] | 149 for exp in list(set(actual) | set(e
xpected))] |
| 148 self._print(" %s [ %s ]" % (test, " ".join(new_expectations
_list))) | 150 self._print(" %s [ %s ]" % (test, " ".join(new_expectations
_list))) |
| 149 self._print("") | 151 self._print("") |
| 150 self._print("") | 152 self._print("") |
| 151 | 153 |
| 152 if len(regressions): | 154 if len(regressions): |
| 153 descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS | 155 descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS |
| 154 for key, tests in regressions.iteritems(): | 156 for key, tests in regressions.iteritems(): |
| 155 result_type = TestExpectations.EXPECTATIONS[key.lower()] | 157 result_type = TestExpectations.EXPECTATIONS[key.lower()] |
| 156 self._print("Regressions: Unexpected %s (%d)" % (descriptions[re
sult_type], len(tests))) | 158 self._print("Regressions: Unexpected %s (%d)" % (descriptions[re
sult_type], len(tests))) |
| 157 tests.sort() | 159 tests.sort() |
| 158 for test in tests: | 160 for test in tests: |
| 159 result = layouttestresults.result_for_test(summarized_result
s['tests'], test) | 161 result = test_results.result_for_test(test) |
| 160 actual = result['actual'].split(" ") | 162 actual = result.actual_results().split(" ") |
| 161 expected = result['expected'].split(" ") | 163 expected = result.expected_results().split(" ") |
| 162 new_expectations_list = [TestExpectationParser._inverted_exp
ectation_tokens[exp] for exp in actual] | 164 new_expectations_list = [TestExpectationParser._inverted_exp
ectation_tokens[exp] for exp in actual] |
| 163 self._print(" %s [ %s ]" % (test, " ".join(new_expectations
_list))) | 165 self._print(" %s [ %s ]" % (test, " ".join(new_expectations
_list))) |
| 164 self._print("") | 166 self._print("") |
| 165 | 167 |
| 166 if len(summarized_results['tests']) and self.debug_logging: | 168 if len(summarized_results['tests']) and self.debug_logging: |
| 167 self._print("%s" % ("-" * 78)) | 169 self._print("%s" % ("-" * 78)) |
| OLD | NEW |