| Index: third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations_unittest.py
|
| diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations_unittest.py b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations_unittest.py
|
| index be2a4e915a521aef593f3c94a46d4d804f700fbe..6b4510117b2d77f44980d6a6a7e9f946464e31f4 100644
|
| --- a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations_unittest.py
|
| +++ b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations_unittest.py
|
| @@ -42,9 +42,7 @@ class BotTestExpectationsFactoryTest(unittest.TestCase):
|
| factory._results_json_for_builder = self.fake_results_json_for_builder
|
|
|
| old_builders = builders._exact_matches
|
| - builders._exact_matches = {
|
| - "Dummy builder name": {"port_name": "dummy-port", "specifiers": []},
|
| - }
|
| + builders._exact_matches = {"Dummy builder name": {"port_name": "dummy-port", "specifiers": []}, }
|
|
|
| try:
|
| self.assertIsNotNone(factory.expectations_for_builder('Dummy builder name'))
|
| @@ -56,9 +54,7 @@ class BotTestExpectationsFactoryTest(unittest.TestCase):
|
| factory._results_json_for_builder = self.fake_results_json_for_builder
|
|
|
| old_builders = builders._exact_matches
|
| - builders._exact_matches = {
|
| - "Dummy builder name": {"port_name": "dummy-port", "specifiers": []},
|
| - }
|
| + builders._exact_matches = {"Dummy builder name": {"port_name": "dummy-port", "specifiers": []}, }
|
|
|
| try:
|
| self.assertIsNotNone(factory.expectations_for_port('dummy-port'))
|
| @@ -68,8 +64,18 @@ class BotTestExpectationsFactoryTest(unittest.TestCase):
|
|
|
| class BotTestExpectationsTest(unittest.TestCase):
|
| # FIXME: Find a way to import this map from Tools/TestResultServer/model/jsonresults.py.
|
| - FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": "MISSING",
|
| - "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP", "Z": "IMAGE+TEXT", "K": "LEAK"}
|
| + FAILURE_MAP = {"A": "AUDIO",
|
| + "C": "CRASH",
|
| + "F": "TEXT",
|
| + "I": "IMAGE",
|
| + "O": "MISSING",
|
| + "N": "NO DATA",
|
| + "P": "PASS",
|
| + "T": "TIMEOUT",
|
| + "Y": "NOTRUN",
|
| + "X": "SKIP",
|
| + "Z": "IMAGE+TEXT",
|
| + "K": "LEAK"}
|
|
|
| # All result_string's in this file represent retries from a single run.
|
| # The left-most entry is the first try, the right-most is the last.
|
| @@ -109,9 +115,7 @@ class BotTestExpectationsTest(unittest.TestCase):
|
|
|
| def _results_json_from_test_data(self, test_data):
|
| test_data[bot_test_expectations.ResultsJSON.FAILURE_MAP_KEY] = self.FAILURE_MAP
|
| - json_dict = {
|
| - 'builder': test_data,
|
| - }
|
| + json_dict = {'builder': test_data, }
|
| return bot_test_expectations.ResultsJSON('builder', json_dict)
|
|
|
| def _results_from_string(self, results_string):
|
| @@ -140,29 +144,33 @@ class BotTestExpectationsTest(unittest.TestCase):
|
| # at the time of the given run.
|
| 'notverflakynoexpected.html': self._results_from_string('FT'),
|
| # If the test is flaky, but marked as such, it shouldn't get printed out.
|
| - 'notflakyexpected.html': {'results': [[2, 'FFFP']], 'expected': 'PASS FAIL'},
|
| + 'notflakyexpected.html': {'results': [[2, 'FFFP']],
|
| + 'expected': 'PASS FAIL'},
|
| }
|
| }
|
| }
|
| - self._assert_expectations(test_data, {
|
| - 'foo/veryflaky.html': sorted(["TEXT", "PASS"]),
|
| - }, only_ignore_very_flaky=True)
|
| + self._assert_expectations(test_data, {'foo/veryflaky.html': sorted(["TEXT", "PASS"]), }, only_ignore_very_flaky=True)
|
|
|
| - self._assert_expectations(test_data, {
|
| - 'foo/veryflaky.html': sorted(["TEXT", "PASS"]),
|
| - 'foo/notverflakynoexpected.html': ['TEXT', 'TIMEOUT'],
|
| - 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]),
|
| - }, only_ignore_very_flaky=False)
|
| + self._assert_expectations(test_data,
|
| + {
|
| + 'foo/veryflaky.html': sorted(["TEXT", "PASS"]),
|
| + 'foo/notverflakynoexpected.html': ['TEXT', 'TIMEOUT'],
|
| + 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]),
|
| + },
|
| + only_ignore_very_flaky=False)
|
|
|
| def test_unexpected_results_no_unexpected(self):
|
| test_data = {
|
| 'tests': {
|
| 'foo': {
|
| 'pass1.html': {'results': [[4, 'P']]},
|
| - 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL'},
|
| - 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': 'PASS FAIL'},
|
| + 'pass2.html': {'results': [[2, 'Z']],
|
| + 'expected': 'PASS FAIL'},
|
| + 'fail.html': {'results': [[2, 'P'], [1, 'F']],
|
| + 'expected': 'PASS FAIL'},
|
| 'not_run.html': {'results': []},
|
| - 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected': 'CRASH FAIL WONTFIX'},
|
| + 'crash.html': {'results': [[2, 'F'], [1, 'C']],
|
| + 'expected': 'CRASH FAIL WONTFIX'},
|
| }
|
| }
|
| }
|
| @@ -172,13 +180,18 @@ class BotTestExpectationsTest(unittest.TestCase):
|
| test_data = {
|
| 'tests': {
|
| 'foo': {
|
| - 'pass1.html': {'results': [[4, 'P']], 'expected': 'FAIL'},
|
| - 'pass2.html': {'results': [[2, 'P']], 'expected': 'IMAGE'},
|
| + 'pass1.html': {'results': [[4, 'P']],
|
| + 'expected': 'FAIL'},
|
| + 'pass2.html': {'results': [[2, 'P']],
|
| + 'expected': 'IMAGE'},
|
| 'fail.html': {'results': [[4, 'F']]},
|
| 'f_p.html': {'results': [[1, 'F'], [2, 'P']]},
|
| - 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected': 'WONTFIX'},
|
| - 'image.html': {'results': [[2, 'F'], [1, 'I']], 'expected': 'CRASH TEXT'},
|
| - 'i_f.html': {'results': [[1, 'F'], [5, 'I']], 'expected': 'PASS'},
|
| + 'crash.html': {'results': [[2, 'F'], [1, 'C']],
|
| + 'expected': 'WONTFIX'},
|
| + 'image.html': {'results': [[2, 'F'], [1, 'I']],
|
| + 'expected': 'CRASH TEXT'},
|
| + 'i_f.html': {'results': [[1, 'F'], [5, 'I']],
|
| + 'expected': 'PASS'},
|
| 'all.html': self._results_from_string('FPFPCNCNTXTXIZIZOCOCYKYK'),
|
| }
|
| }
|
|
|