| OLD | NEW |
| 1 # Copyright (C) 2013 Google Inc. All rights reserved. | 1 # Copyright (C) 2013 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 16 matching lines...) Expand all Loading... |
| 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 28 | 28 |
| 29 import unittest | 29 import unittest |
| 30 | 30 |
| 31 from webkitpy.layout_tests.layout_package import bot_test_expectations | 31 from webkitpy.layout_tests.layout_package import bot_test_expectations |
| 32 from webkitpy.layout_tests.models import test_expectations | 32 from webkitpy.layout_tests.models import test_expectations |
| 33 from webkitpy.layout_tests.port import builders | 33 from webkitpy.layout_tests.port import builders |
| 34 | 34 |
| 35 | 35 |
| 36 class BotTestExpectationsFactoryTest(unittest.TestCase): | 36 class BotTestExpectationsFactoryTest(unittest.TestCase): |
| 37 |
| 37 def fake_results_json_for_builder(self, builder): | 38 def fake_results_json_for_builder(self, builder): |
| 38 return bot_test_expectations.ResultsJSON(builder, 'Dummy content') | 39 return bot_test_expectations.ResultsJSON(builder, 'Dummy content') |
| 39 | 40 |
| 40 def test_expectations_for_builder(self): | 41 def test_expectations_for_builder(self): |
| 41 factory = bot_test_expectations.BotTestExpectationsFactory() | 42 factory = bot_test_expectations.BotTestExpectationsFactory() |
| 42 factory._results_json_for_builder = self.fake_results_json_for_builder | 43 factory._results_json_for_builder = self.fake_results_json_for_builder |
| 43 | 44 |
| 44 old_builders = builders._exact_matches | 45 old_builders = builders._exact_matches |
| 45 builders._exact_matches = { | 46 builders._exact_matches = { |
| 46 "Dummy builder name": {"port_name": "dummy-port", "specifiers": []}, | 47 'Dummy builder name': {'port_name': 'dummy-port', 'specifiers': []}, |
| 47 } | 48 } |
| 48 | 49 |
| 49 try: | 50 try: |
| 50 self.assertIsNotNone(factory.expectations_for_builder('Dummy builder
name')) | 51 self.assertIsNotNone(factory.expectations_for_builder('Dummy builder
name')) |
| 51 finally: | 52 finally: |
| 52 builders._exact_matches = old_builders | 53 builders._exact_matches = old_builders |
| 53 | 54 |
| 54 def test_expectations_for_port(self): | 55 def test_expectations_for_port(self): |
| 55 factory = bot_test_expectations.BotTestExpectationsFactory() | 56 factory = bot_test_expectations.BotTestExpectationsFactory() |
| 56 factory._results_json_for_builder = self.fake_results_json_for_builder | 57 factory._results_json_for_builder = self.fake_results_json_for_builder |
| 57 | 58 |
| 58 old_builders = builders._exact_matches | 59 old_builders = builders._exact_matches |
| 59 builders._exact_matches = { | 60 builders._exact_matches = { |
| 60 "Dummy builder name": {"port_name": "dummy-port", "specifiers": []}, | 61 'Dummy builder name': {'port_name': 'dummy-port', 'specifiers': []}, |
| 61 } | 62 } |
| 62 | 63 |
| 63 try: | 64 try: |
| 64 self.assertIsNotNone(factory.expectations_for_port('dummy-port')) | 65 self.assertIsNotNone(factory.expectations_for_port('dummy-port')) |
| 65 finally: | 66 finally: |
| 66 builders._exact_matches = old_builders | 67 builders._exact_matches = old_builders |
| 67 | 68 |
| 68 | 69 |
| 69 class BotTestExpectationsTest(unittest.TestCase): | 70 class BotTestExpectationsTest(unittest.TestCase): |
| 70 # FIXME: Find a way to import this map from Tools/TestResultServer/model/jso
nresults.py. | 71 # FIXME: Find a way to import this map from Tools/TestResultServer/model/jso
nresults.py. |
| 71 FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": "
MISSING", | 72 FAILURE_MAP = {'A': 'AUDIO', 'C': 'CRASH', 'F': 'TEXT', 'I': 'IMAGE', 'O': '
MISSING', |
| 72 "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP",
"Z": "IMAGE+TEXT", "K": "LEAK"} | 73 'N': 'NO DATA', 'P': 'PASS', 'T': 'TIMEOUT', 'Y': 'NOTRUN', '
X': 'SKIP', 'Z': 'IMAGE+TEXT', 'K': 'LEAK'} |
| 73 | 74 |
| 74 # All result_string's in this file expect newest result | 75 # All result_string's in this file expect newest result |
| 75 # on left: "PFF", means it just passed after 2 failures. | 76 # on left: "PFF", means it just passed after 2 failures. |
| 76 | 77 |
| 77 def _assert_is_flaky(self, results_string, should_be_flaky): | 78 def _assert_is_flaky(self, results_string, should_be_flaky): |
| 78 results_json = self._results_json_from_test_data({}) | 79 results_json = self._results_json_from_test_data({}) |
| 79 expectations = bot_test_expectations.BotTestExpectations(results_json, s
et('test')) | 80 expectations = bot_test_expectations.BotTestExpectations(results_json, s
et('test')) |
| 80 length_encoded = self._results_from_string(results_string)['results'] | 81 length_encoded = self._results_from_string(results_string)['results'] |
| 81 num_actual_results = len(expectations._flaky_types_in_results(length_enc
oded, only_ignore_very_flaky=True)) | 82 num_actual_results = len(expectations._flaky_types_in_results(length_enc
oded, only_ignore_very_flaky=True)) |
| 82 if should_be_flaky: | 83 if should_be_flaky: |
| (...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 127 'tests': { | 128 'tests': { |
| 128 'foo': { | 129 'foo': { |
| 129 'veryflaky.html': self._results_from_string('FPFP'), | 130 'veryflaky.html': self._results_from_string('FPFP'), |
| 130 'maybeflaky.html': self._results_from_string('PPFP'), | 131 'maybeflaky.html': self._results_from_string('PPFP'), |
| 131 'notflakypass.html': self._results_from_string('PPPP'), | 132 'notflakypass.html': self._results_from_string('PPPP'), |
| 132 'notflakyfail.html': self._results_from_string('FFFF'), | 133 'notflakyfail.html': self._results_from_string('FFFF'), |
| 133 } | 134 } |
| 134 } | 135 } |
| 135 } | 136 } |
| 136 self._assert_expectations(test_data, { | 137 self._assert_expectations(test_data, { |
| 137 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), | 138 'foo/veryflaky.html': sorted(['TEXT', 'PASS']), |
| 138 }, only_ignore_very_flaky=True) | 139 }, only_ignore_very_flaky=True) |
| 139 | 140 |
| 140 self._assert_expectations(test_data, { | 141 self._assert_expectations(test_data, { |
| 141 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), | 142 'foo/veryflaky.html': sorted(['TEXT', 'PASS']), |
| 142 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]), | 143 'foo/maybeflaky.html': sorted(['TEXT', 'PASS']), |
| 143 }, only_ignore_very_flaky=False) | 144 }, only_ignore_very_flaky=False) |
| 144 | 145 |
| 145 def test_all_failure_types(self): | 146 def test_all_failure_types(self): |
| 146 test_data = { | 147 test_data = { |
| 147 'tests': { | 148 'tests': { |
| 148 'foo': { | 149 'foo': { |
| 149 'allfailures.html': self._results_from_string('FPFPCNCNTXTXI
ZIZOCOCYKYK'), | 150 'allfailures.html': self._results_from_string('FPFPCNCNTXTXI
ZIZOCOCYKYK'), |
| 150 'imageplustextflake.html': self._results_from_string('ZPZPPP
PPPPPPPPPPPPPP'), | 151 'imageplustextflake.html': self._results_from_string('ZPZPPP
PPPPPPPPPPPPPP'), |
| 151 } | 152 } |
| 152 } | 153 } |
| 153 } | 154 } |
| 154 self._assert_expectations(test_data, { | 155 self._assert_expectations(test_data, { |
| 155 'foo/imageplustextflake.html': sorted(["IMAGE+TEXT", "PASS"]), | 156 'foo/imageplustextflake.html': sorted(['IMAGE+TEXT', 'PASS']), |
| 156 'foo/allfailures.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEO
UT", "CRASH", "IMAGE", "MISSING", "LEAK"]), | 157 'foo/allfailures.html': sorted(['TEXT', 'PASS', 'IMAGE+TEXT', 'TIMEO
UT', 'CRASH', 'IMAGE', 'MISSING', 'LEAK']), |
| 157 }, only_ignore_very_flaky=True) | 158 }, only_ignore_very_flaky=True) |
| 158 | 159 |
| 159 def test_unexpected_results_no_unexpected(self): | 160 def test_unexpected_results_no_unexpected(self): |
| 160 test_data = { | 161 test_data = { |
| 161 'tests': { | 162 'tests': { |
| 162 'foo': { | 163 'foo': { |
| 163 'pass1.html': {'results': [[4, 'P']]}, | 164 'pass1.html': {'results': [[4, 'P']]}, |
| 164 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL
'}, | 165 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL
'}, |
| 165 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': '
PASS FAIL'}, | 166 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': '
PASS FAIL'}, |
| 166 'not_run.html': {'results': []}, | 167 'not_run.html': {'results': []}, |
| (...skipping 13 matching lines...) Expand all Loading... |
| 180 'f_p.html': {'results': [[1, 'F'], [2, 'P']]}, | 181 'f_p.html': {'results': [[1, 'F'], [2, 'P']]}, |
| 181 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected':
'WONTFIX'}, | 182 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected':
'WONTFIX'}, |
| 182 'image.html': {'results': [[2, 'F'], [1, 'I']], 'expected':
'CRASH FAIL'}, | 183 'image.html': {'results': [[2, 'F'], [1, 'I']], 'expected':
'CRASH FAIL'}, |
| 183 'i_f.html': {'results': [[1, 'F'], [5, 'I']], 'expected': 'P
ASS'}, | 184 'i_f.html': {'results': [[1, 'F'], [5, 'I']], 'expected': 'P
ASS'}, |
| 184 'all.html': self._results_from_string('FPFPCNCNTXTXIZIZOCOCY
KYK'), | 185 'all.html': self._results_from_string('FPFPCNCNTXTXIZIZOCOCY
KYK'), |
| 185 } | 186 } |
| 186 } | 187 } |
| 187 } | 188 } |
| 188 self.maxDiff = None | 189 self.maxDiff = None |
| 189 self._assert_unexpected_results(test_data, { | 190 self._assert_unexpected_results(test_data, { |
| 190 'foo/pass1.html': sorted(["FAIL", "PASS"]), | 191 'foo/pass1.html': sorted(['FAIL', 'PASS']), |
| 191 'foo/pass2.html': sorted(["IMAGE", "PASS"]), | 192 'foo/pass2.html': sorted(['IMAGE', 'PASS']), |
| 192 'foo/fail.html': sorted(["TEXT", "PASS"]), | 193 'foo/fail.html': sorted(['TEXT', 'PASS']), |
| 193 'foo/f_p.html': sorted(["TEXT", "PASS"]), | 194 'foo/f_p.html': sorted(['TEXT', 'PASS']), |
| 194 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]), | 195 'foo/crash.html': sorted(['WONTFIX', 'CRASH', 'TEXT']), |
| 195 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]), | 196 'foo/image.html': sorted(['CRASH', 'FAIL', 'IMAGE']), |
| 196 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]), | 197 'foo/i_f.html': sorted(['PASS', 'IMAGE', 'TEXT']), |
| 197 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR
ASH", "IMAGE", "MISSING", "LEAK"]), | 198 'foo/all.html': sorted(['TEXT', 'PASS', 'IMAGE+TEXT', 'TIMEOUT', 'CR
ASH', 'IMAGE', 'MISSING', 'LEAK']), |
| 198 }) | 199 }) |
| OLD | NEW |