| OLD | NEW |
| 1 # Copyright (C) 2013 Google Inc. All rights reserved. | 1 # Copyright (C) 2013 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 17 matching lines...) Expand all Loading... |
| 28 | 28 |
| 29 import webkitpy.thirdparty.unittest2 as unittest | 29 import webkitpy.thirdparty.unittest2 as unittest |
| 30 | 30 |
| 31 from webkitpy.layout_tests.layout_package import bot_test_expectations | 31 from webkitpy.layout_tests.layout_package import bot_test_expectations |
| 32 from webkitpy.layout_tests.models import test_expectations | 32 from webkitpy.layout_tests.models import test_expectations |
| 33 | 33 |
| 34 | 34 |
| 35 class BotTestExpectationsTest(unittest.TestCase): | 35 class BotTestExpectationsTest(unittest.TestCase): |
| 36 # FIXME: Find a way to import this map from Tools/TestResultServer/model/jso
nresults.py. | 36 # FIXME: Find a way to import this map from Tools/TestResultServer/model/jso
nresults.py. |
| 37 FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": "
MISSING", | 37 FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": "
MISSING", |
| 38 "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP",
"Z": "IMAGE+TEXT"} | 38 "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP",
"Z": "IMAGE+TEXT", "K": "LEAK"} |
| 39 | 39 |
| 40 # All result_string's in this file expect newest result | 40 # All result_string's in this file expect newest result |
| 41 # on left: "PFF", means it just passed after 2 failures. | 41 # on left: "PFF", means it just passed after 2 failures. |
| 42 | 42 |
| 43 def _assert_is_flaky(self, results_string, should_be_flaky): | 43 def _assert_is_flaky(self, results_string, should_be_flaky): |
| 44 results_json = self._results_json_from_test_data({}) | 44 results_json = self._results_json_from_test_data({}) |
| 45 expectations = bot_test_expectations.BotTestExpectations(results_json) | 45 expectations = bot_test_expectations.BotTestExpectations(results_json) |
| 46 length_encoded = self._results_from_string(results_string)['results'] | 46 length_encoded = self._results_from_string(results_string)['results'] |
| 47 num_actual_results = len(expectations._flaky_types_in_results(length_enc
oded, only_ignore_very_flaky=True)) | 47 num_actual_results = len(expectations._flaky_types_in_results(length_enc
oded, only_ignore_very_flaky=True)) |
| 48 if should_be_flaky: | 48 if should_be_flaky: |
| (...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 105 | 105 |
| 106 self._assert_expectations(test_data, { | 106 self._assert_expectations(test_data, { |
| 107 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), | 107 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), |
| 108 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]), | 108 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]), |
| 109 }, only_ignore_very_flaky=False) | 109 }, only_ignore_very_flaky=False) |
| 110 | 110 |
| 111 def test_all_failure_types(self): | 111 def test_all_failure_types(self): |
| 112 test_data = { | 112 test_data = { |
| 113 'tests': { | 113 'tests': { |
| 114 'foo': { | 114 'foo': { |
| 115 'allfailures.html': self._results_from_string('FPFPCNCNTXTXI
ZIZOCYOCY'), | 115 'allfailures.html': self._results_from_string('FPFPCNCNTXTXI
ZIZOCOCYKYK'), |
| 116 'imageplustextflake.html': self._results_from_string('ZPZPPP
PPPPPPPPPPPPPP'), | 116 'imageplustextflake.html': self._results_from_string('ZPZPPP
PPPPPPPPPPPPPP'), |
| 117 } | 117 } |
| 118 } | 118 } |
| 119 } | 119 } |
| 120 self._assert_expectations(test_data, { | 120 self._assert_expectations(test_data, { |
| 121 'foo/imageplustextflake.html': sorted(["IMAGE+TEXT", "PASS"]), | 121 'foo/imageplustextflake.html': sorted(["IMAGE+TEXT", "PASS"]), |
| 122 'foo/allfailures.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEO
UT", "CRASH", "IMAGE", "MISSING"]), | 122 'foo/allfailures.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEO
UT", "CRASH", "IMAGE", "MISSING", "LEAK"]), |
| 123 }, only_ignore_very_flaky=True) | 123 }, only_ignore_very_flaky=True) |
| 124 | 124 |
| 125 def test_unexpected_results_no_unexpected(self): | 125 def test_unexpected_results_no_unexpected(self): |
| 126 test_data = { | 126 test_data = { |
| 127 'tests': { | 127 'tests': { |
| 128 'foo': { | 128 'foo': { |
| 129 'pass1.html': {'results': [[4, 'P']]}, | 129 'pass1.html': {'results': [[4, 'P']]}, |
| 130 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL
'}, | 130 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL
'}, |
| 131 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': '
PASS FAIL'}, | 131 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': '
PASS FAIL'}, |
| 132 'not_run.html': {'results': []}, | 132 'not_run.html': {'results': []}, |
| 133 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected':
'CRASH FAIL WONTFIX'}, | 133 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected':
'CRASH FAIL WONTFIX'}, |
| 134 } | 134 } |
| 135 } | 135 } |
| 136 } | 136 } |
| 137 self._assert_unexpected_results(test_data, {}) | 137 self._assert_unexpected_results(test_data, {}) |
| 138 | 138 |
| 139 def test_unexpected_results_all_unexpected(self): | 139 def test_unexpected_results_all_unexpected(self): |
| 140 test_data = { | 140 test_data = { |
| 141 'tests': { | 141 'tests': { |
| 142 'foo': { | 142 'foo': { |
| 143 'pass1.html': {'results': [[4, 'P']], 'expected': 'FAIL'}, | 143 'pass1.html': {'results': [[4, 'P']], 'expected': 'FAIL'}, |
| 144 'pass2.html': {'results': [[2, 'P']], 'expected': 'IMAGE'}, | 144 'pass2.html': {'results': [[2, 'P']], 'expected': 'IMAGE'}, |
| 145 'fail.html': {'results': [[4, 'F']]}, | 145 'fail.html': {'results': [[4, 'F']]}, |
| 146 'f_p.html': {'results': [[1, 'F'], [2, 'P']]}, | 146 'f_p.html': {'results': [[1, 'F'], [2, 'P']]}, |
| 147 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected':
'WONTFIX'}, | 147 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected':
'WONTFIX'}, |
| 148 'image.html': {'results': [[2, 'F'], [1, 'I']], 'expected':
'CRASH FAIL'}, | 148 'image.html': {'results': [[2, 'F'], [1, 'I']], 'expected':
'CRASH FAIL'}, |
| 149 'i_f.html': {'results': [[1, 'F'], [5, 'I']], 'expected': 'P
ASS'}, | 149 'i_f.html': {'results': [[1, 'F'], [5, 'I']], 'expected': 'P
ASS'}, |
| 150 'all.html': self._results_from_string('FPFPCNCNTXTXIZIZOCYOC
Y'), | 150 'all.html': self._results_from_string('FPFPCNCNTXTXIZIZOCOCY
KYK'), |
| 151 } | 151 } |
| 152 } | 152 } |
| 153 } | 153 } |
| 154 self.maxDiff = None | 154 self.maxDiff = None |
| 155 self._assert_unexpected_results(test_data, { | 155 self._assert_unexpected_results(test_data, { |
| 156 'foo/pass1.html': sorted(["FAIL", "PASS"]), | 156 'foo/pass1.html': sorted(["FAIL", "PASS"]), |
| 157 'foo/pass2.html': sorted(["IMAGE", "PASS"]), | 157 'foo/pass2.html': sorted(["IMAGE", "PASS"]), |
| 158 'foo/fail.html': sorted(["TEXT", "PASS"]), | 158 'foo/fail.html': sorted(["TEXT", "PASS"]), |
| 159 'foo/f_p.html': sorted(["TEXT", "PASS"]), | 159 'foo/f_p.html': sorted(["TEXT", "PASS"]), |
| 160 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]), | 160 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]), |
| 161 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]), | 161 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]), |
| 162 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]), | 162 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]), |
| 163 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR
ASH", "IMAGE", "MISSING"]), | 163 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR
ASH", "IMAGE", "MISSING", "LEAK"]), |
| 164 }) | 164 }) |
| OLD | NEW |