| OLD | NEW |
| 1 # Copyright (C) 2012 Google Inc. All rights reserved. | 1 # Copyright (C) 2012 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| 11 # in the documentation and/or other materials provided with the | 11 # in the documentation and/or other materials provided with the |
| 12 # distribution. | 12 # distribution. |
| 13 # * Neither the name of Google Inc. nor the names of its | 13 # * Neither the name of Google Inc. nor the names of its |
| 14 # contributors may be used to endorse or promote products derived from | 14 # contributors may be used to endorse or promote products derived from |
| 15 # this software without specific prior written permission. | 15 # this software without specific prior written permission. |
| 16 # | 16 # |
| 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 28 | 28 |
| 29 | |
| 30 from webkitpy.layout_tests.models import test_expectations | 29 from webkitpy.layout_tests.models import test_expectations |
| 31 | 30 |
| 32 from webkitpy.common.net import layouttestresults | 31 from webkitpy.common.net import layouttestresults |
| 33 | 32 |
| 34 | |
| 35 TestExpectations = test_expectations.TestExpectations | 33 TestExpectations = test_expectations.TestExpectations |
| 36 TestExpectationParser = test_expectations.TestExpectationParser | 34 TestExpectationParser = test_expectations.TestExpectationParser |
| 37 | 35 |
| 38 | 36 |
| 39 class BuildBotPrinter(object): | 37 class BuildBotPrinter(object): |
| 40 # This output is parsed by buildbots and must only be changed in coordinatio
n with buildbot scripts (see webkit.org's | 38 # This output is parsed by buildbots and must only be changed in coordinatio
n with buildbot scripts (see webkit.org's |
| 41 # Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg: RunWebKitTests
._parseNewRunWebKitTestsOutput | 39 # Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg: RunWebKitTests
._parseNewRunWebKitTestsOutput |
| 42 # and chromium.org's buildbot/master.chromium/scripts/master/log_parser/webk
it_test_command.py). | 40 # and chromium.org's buildbot/master.chromium/scripts/master/log_parser/webk
it_test_command.py). |
| 43 | 41 |
| 44 def __init__(self, stream, debug_logging): | 42 def __init__(self, stream, debug_logging): |
| (...skipping 16 matching lines...) Expand all Loading... |
| 61 if total > 0: | 59 if total > 0: |
| 62 percent_passed = float(passed) * 100 / total | 60 percent_passed = float(passed) * 100 / total |
| 63 | 61 |
| 64 self._print("=> Results: %d/%d tests passed (%.1f%%)" % (passed, total,
percent_passed)) | 62 self._print("=> Results: %d/%d tests passed (%.1f%%)" % (passed, total,
percent_passed)) |
| 65 self._print("") | 63 self._print("") |
| 66 self._print_run_results_entry(run_results, test_expectations.NOW, "Tests
to be fixed") | 64 self._print_run_results_entry(run_results, test_expectations.NOW, "Tests
to be fixed") |
| 67 | 65 |
| 68 self._print("") | 66 self._print("") |
| 69 # FIXME: We should be skipping anything marked WONTFIX, so we shouldn't
bother logging these stats. | 67 # FIXME: We should be skipping anything marked WONTFIX, so we shouldn't
bother logging these stats. |
| 70 self._print_run_results_entry(run_results, test_expectations.WONTFIX, | 68 self._print_run_results_entry(run_results, test_expectations.WONTFIX, |
| 71 "Tests that will only be fixed if they crash (WONTFIX)") | 69 "Tests that will only be fixed if they cra
sh (WONTFIX)") |
| 72 self._print("") | 70 self._print("") |
| 73 | 71 |
| 74 def _print_run_results_entry(self, run_results, timeline, heading): | 72 def _print_run_results_entry(self, run_results, timeline, heading): |
| 75 total = len(run_results.tests_by_timeline[timeline]) | 73 total = len(run_results.tests_by_timeline[timeline]) |
| 76 not_passing = (total - | 74 not_passing = ( |
| 77 len(run_results.tests_by_expectation[test_expectations.PASS] & | 75 total - len(run_results.tests_by_expectation[test_expectations.PASS]
& run_results.tests_by_timeline[timeline])) |
| 78 run_results.tests_by_timeline[timeline])) | |
| 79 self._print("=> %s (%d):" % (heading, not_passing)) | 76 self._print("=> %s (%d):" % (heading, not_passing)) |
| 80 | 77 |
| 81 for result in TestExpectations.EXPECTATION_DESCRIPTIONS.keys(): | 78 for result in TestExpectations.EXPECTATION_DESCRIPTIONS.keys(): |
| 82 if result in (test_expectations.PASS, test_expectations.SKIP): | 79 if result in (test_expectations.PASS, test_expectations.SKIP): |
| 83 continue | 80 continue |
| 84 results = (run_results.tests_by_expectation[result] & run_results.te
sts_by_timeline[timeline]) | 81 results = (run_results.tests_by_expectation[result] & run_results.te
sts_by_timeline[timeline]) |
| 85 desc = TestExpectations.EXPECTATION_DESCRIPTIONS[result] | 82 desc = TestExpectations.EXPECTATION_DESCRIPTIONS[result] |
| 86 if not_passing and len(results): | 83 if not_passing and len(results): |
| 87 pct = len(results) * 100.0 / not_passing | 84 pct = len(results) * 100.0 / not_passing |
| 88 self._print(" %5d %-24s (%4.1f%%)" % (len(results), desc, pct)) | 85 self._print(" %5d %-24s (%4.1f%%)" % (len(results), desc, pct)) |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 136 for key, tests in flaky.iteritems(): | 133 for key, tests in flaky.iteritems(): |
| 137 result_type = TestExpectations.EXPECTATIONS[key.lower()] | 134 result_type = TestExpectations.EXPECTATIONS[key.lower()] |
| 138 self._print("Unexpected flakiness: %s (%d)" % (descriptions[resu
lt_type], len(tests))) | 135 self._print("Unexpected flakiness: %s (%d)" % (descriptions[resu
lt_type], len(tests))) |
| 139 tests.sort() | 136 tests.sort() |
| 140 | 137 |
| 141 for test in tests: | 138 for test in tests: |
| 142 result = layouttestresults.result_for_test(summarized_result
s['tests'], test) | 139 result = layouttestresults.result_for_test(summarized_result
s['tests'], test) |
| 143 actual = result['actual'].split(" ") | 140 actual = result['actual'].split(" ") |
| 144 expected = result['expected'].split(" ") | 141 expected = result['expected'].split(" ") |
| 145 # FIXME: clean this up once the old syntax is gone | 142 # FIXME: clean this up once the old syntax is gone |
| 146 new_expectations_list = [TestExpectationParser._inverted_exp
ectation_tokens[exp] for exp in list(set(actual) | set(expected))] | 143 new_expectations_list = [TestExpectationParser._inverted_exp
ectation_tokens[exp] |
| 144 for exp in list(set(actual) | set(e
xpected))] |
| 147 self._print(" %s [ %s ]" % (test, " ".join(new_expectations
_list))) | 145 self._print(" %s [ %s ]" % (test, " ".join(new_expectations
_list))) |
| 148 self._print("") | 146 self._print("") |
| 149 self._print("") | 147 self._print("") |
| 150 | 148 |
| 151 if len(regressions): | 149 if len(regressions): |
| 152 descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS | 150 descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS |
| 153 for key, tests in regressions.iteritems(): | 151 for key, tests in regressions.iteritems(): |
| 154 result_type = TestExpectations.EXPECTATIONS[key.lower()] | 152 result_type = TestExpectations.EXPECTATIONS[key.lower()] |
| 155 self._print("Regressions: Unexpected %s (%d)" % (descriptions[re
sult_type], len(tests))) | 153 self._print("Regressions: Unexpected %s (%d)" % (descriptions[re
sult_type], len(tests))) |
| 156 tests.sort() | 154 tests.sort() |
| 157 for test in tests: | 155 for test in tests: |
| 158 result = layouttestresults.result_for_test(summarized_result
s['tests'], test) | 156 result = layouttestresults.result_for_test(summarized_result
s['tests'], test) |
| 159 actual = result['actual'].split(" ") | 157 actual = result['actual'].split(" ") |
| 160 expected = result['expected'].split(" ") | 158 expected = result['expected'].split(" ") |
| 161 new_expectations_list = [TestExpectationParser._inverted_exp
ectation_tokens[exp] for exp in actual] | 159 new_expectations_list = [TestExpectationParser._inverted_exp
ectation_tokens[exp] for exp in actual] |
| 162 self._print(" %s [ %s ]" % (test, " ".join(new_expectations
_list))) | 160 self._print(" %s [ %s ]" % (test, " ".join(new_expectations
_list))) |
| 163 self._print("") | 161 self._print("") |
| 164 | 162 |
| 165 if len(summarized_results['tests']) and self.debug_logging: | 163 if len(summarized_results['tests']) and self.debug_logging: |
| 166 self._print("%s" % ("-" * 78)) | 164 self._print("%s" % ("-" * 78)) |
| OLD | NEW |