Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(262)

Side by Side Diff: Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations_unittest.py

Issue 546133003: Reformat webkitpy.layout_tests w/ format-webkitpy. (Closed) Base URL: svn://svn.chromium.org/blink/trunk
Patch Set: Created 6 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright (C) 2013 Google Inc. All rights reserved. 1 # Copyright (C) 2013 Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 16 matching lines...) Expand all
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 28
29 import unittest 29 import unittest
30 30
31 from webkitpy.layout_tests.layout_package import bot_test_expectations 31 from webkitpy.layout_tests.layout_package import bot_test_expectations
32 from webkitpy.layout_tests.models import test_expectations 32 from webkitpy.layout_tests.models import test_expectations
33 from webkitpy.layout_tests.port import builders 33 from webkitpy.layout_tests.port import builders
34 34
35 35
36 class BotTestExpectationsFactoryTest(unittest.TestCase): 36 class BotTestExpectationsFactoryTest(unittest.TestCase):
37
37 def fake_results_json_for_builder(self, builder): 38 def fake_results_json_for_builder(self, builder):
38 return bot_test_expectations.ResultsJSON(builder, 'Dummy content') 39 return bot_test_expectations.ResultsJSON(builder, 'Dummy content')
39 40
40 def test_expectations_for_builder(self): 41 def test_expectations_for_builder(self):
41 factory = bot_test_expectations.BotTestExpectationsFactory() 42 factory = bot_test_expectations.BotTestExpectationsFactory()
42 factory._results_json_for_builder = self.fake_results_json_for_builder 43 factory._results_json_for_builder = self.fake_results_json_for_builder
43 44
44 old_builders = builders._exact_matches 45 old_builders = builders._exact_matches
45 builders._exact_matches = { 46 builders._exact_matches = {
46 "Dummy builder name": {"port_name": "dummy-port", "specifiers": []}, 47 'Dummy builder name': {'port_name': 'dummy-port', 'specifiers': []},
47 } 48 }
48 49
49 try: 50 try:
50 self.assertIsNotNone(factory.expectations_for_builder('Dummy builder name')) 51 self.assertIsNotNone(factory.expectations_for_builder('Dummy builder name'))
51 finally: 52 finally:
52 builders._exact_matches = old_builders 53 builders._exact_matches = old_builders
53 54
54 def test_expectations_for_port(self): 55 def test_expectations_for_port(self):
55 factory = bot_test_expectations.BotTestExpectationsFactory() 56 factory = bot_test_expectations.BotTestExpectationsFactory()
56 factory._results_json_for_builder = self.fake_results_json_for_builder 57 factory._results_json_for_builder = self.fake_results_json_for_builder
57 58
58 old_builders = builders._exact_matches 59 old_builders = builders._exact_matches
59 builders._exact_matches = { 60 builders._exact_matches = {
60 "Dummy builder name": {"port_name": "dummy-port", "specifiers": []}, 61 'Dummy builder name': {'port_name': 'dummy-port', 'specifiers': []},
61 } 62 }
62 63
63 try: 64 try:
64 self.assertIsNotNone(factory.expectations_for_port('dummy-port')) 65 self.assertIsNotNone(factory.expectations_for_port('dummy-port'))
65 finally: 66 finally:
66 builders._exact_matches = old_builders 67 builders._exact_matches = old_builders
67 68
68 69
69 class BotTestExpectationsTest(unittest.TestCase): 70 class BotTestExpectationsTest(unittest.TestCase):
70 # FIXME: Find a way to import this map from Tools/TestResultServer/model/jso nresults.py. 71 # FIXME: Find a way to import this map from Tools/TestResultServer/model/jso nresults.py.
71 FAILURE_MAP = {"A": "AUDIO", "C": "CRASH", "F": "TEXT", "I": "IMAGE", "O": " MISSING", 72 FAILURE_MAP = {'A': 'AUDIO', 'C': 'CRASH', 'F': 'TEXT', 'I': 'IMAGE', 'O': ' MISSING',
72 "N": "NO DATA", "P": "PASS", "T": "TIMEOUT", "Y": "NOTRUN", "X": "SKIP", "Z": "IMAGE+TEXT", "K": "LEAK"} 73 'N': 'NO DATA', 'P': 'PASS', 'T': 'TIMEOUT', 'Y': 'NOTRUN', ' X': 'SKIP', 'Z': 'IMAGE+TEXT', 'K': 'LEAK'}
73 74
74 # All result_string's in this file expect newest result 75 # All result_string's in this file expect newest result
75 # on left: "PFF", means it just passed after 2 failures. 76 # on left: "PFF", means it just passed after 2 failures.
76 77
77 def _assert_is_flaky(self, results_string, should_be_flaky): 78 def _assert_is_flaky(self, results_string, should_be_flaky):
78 results_json = self._results_json_from_test_data({}) 79 results_json = self._results_json_from_test_data({})
79 expectations = bot_test_expectations.BotTestExpectations(results_json, s et('test')) 80 expectations = bot_test_expectations.BotTestExpectations(results_json, s et('test'))
80 length_encoded = self._results_from_string(results_string)['results'] 81 length_encoded = self._results_from_string(results_string)['results']
81 num_actual_results = len(expectations._flaky_types_in_results(length_enc oded, only_ignore_very_flaky=True)) 82 num_actual_results = len(expectations._flaky_types_in_results(length_enc oded, only_ignore_very_flaky=True))
82 if should_be_flaky: 83 if should_be_flaky:
(...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after
127 'tests': { 128 'tests': {
128 'foo': { 129 'foo': {
129 'veryflaky.html': self._results_from_string('FPFP'), 130 'veryflaky.html': self._results_from_string('FPFP'),
130 'maybeflaky.html': self._results_from_string('PPFP'), 131 'maybeflaky.html': self._results_from_string('PPFP'),
131 'notflakypass.html': self._results_from_string('PPPP'), 132 'notflakypass.html': self._results_from_string('PPPP'),
132 'notflakyfail.html': self._results_from_string('FFFF'), 133 'notflakyfail.html': self._results_from_string('FFFF'),
133 } 134 }
134 } 135 }
135 } 136 }
136 self._assert_expectations(test_data, { 137 self._assert_expectations(test_data, {
137 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), 138 'foo/veryflaky.html': sorted(['TEXT', 'PASS']),
138 }, only_ignore_very_flaky=True) 139 }, only_ignore_very_flaky=True)
139 140
140 self._assert_expectations(test_data, { 141 self._assert_expectations(test_data, {
141 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), 142 'foo/veryflaky.html': sorted(['TEXT', 'PASS']),
142 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]), 143 'foo/maybeflaky.html': sorted(['TEXT', 'PASS']),
143 }, only_ignore_very_flaky=False) 144 }, only_ignore_very_flaky=False)
144 145
145 def test_all_failure_types(self): 146 def test_all_failure_types(self):
146 test_data = { 147 test_data = {
147 'tests': { 148 'tests': {
148 'foo': { 149 'foo': {
149 'allfailures.html': self._results_from_string('FPFPCNCNTXTXI ZIZOCOCYKYK'), 150 'allfailures.html': self._results_from_string('FPFPCNCNTXTXI ZIZOCOCYKYK'),
150 'imageplustextflake.html': self._results_from_string('ZPZPPP PPPPPPPPPPPPPP'), 151 'imageplustextflake.html': self._results_from_string('ZPZPPP PPPPPPPPPPPPPP'),
151 } 152 }
152 } 153 }
153 } 154 }
154 self._assert_expectations(test_data, { 155 self._assert_expectations(test_data, {
155 'foo/imageplustextflake.html': sorted(["IMAGE+TEXT", "PASS"]), 156 'foo/imageplustextflake.html': sorted(['IMAGE+TEXT', 'PASS']),
156 'foo/allfailures.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEO UT", "CRASH", "IMAGE", "MISSING", "LEAK"]), 157 'foo/allfailures.html': sorted(['TEXT', 'PASS', 'IMAGE+TEXT', 'TIMEO UT', 'CRASH', 'IMAGE', 'MISSING', 'LEAK']),
157 }, only_ignore_very_flaky=True) 158 }, only_ignore_very_flaky=True)
158 159
159 def test_unexpected_results_no_unexpected(self): 160 def test_unexpected_results_no_unexpected(self):
160 test_data = { 161 test_data = {
161 'tests': { 162 'tests': {
162 'foo': { 163 'foo': {
163 'pass1.html': {'results': [[4, 'P']]}, 164 'pass1.html': {'results': [[4, 'P']]},
164 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL '}, 165 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL '},
165 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': ' PASS FAIL'}, 166 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': ' PASS FAIL'},
166 'not_run.html': {'results': []}, 167 'not_run.html': {'results': []},
(...skipping 13 matching lines...) Expand all
180 'f_p.html': {'results': [[1, 'F'], [2, 'P']]}, 181 'f_p.html': {'results': [[1, 'F'], [2, 'P']]},
181 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected': 'WONTFIX'}, 182 'crash.html': {'results': [[2, 'F'], [1, 'C']], 'expected': 'WONTFIX'},
182 'image.html': {'results': [[2, 'F'], [1, 'I']], 'expected': 'CRASH FAIL'}, 183 'image.html': {'results': [[2, 'F'], [1, 'I']], 'expected': 'CRASH FAIL'},
183 'i_f.html': {'results': [[1, 'F'], [5, 'I']], 'expected': 'P ASS'}, 184 'i_f.html': {'results': [[1, 'F'], [5, 'I']], 'expected': 'P ASS'},
184 'all.html': self._results_from_string('FPFPCNCNTXTXIZIZOCOCY KYK'), 185 'all.html': self._results_from_string('FPFPCNCNTXTXIZIZOCOCY KYK'),
185 } 186 }
186 } 187 }
187 } 188 }
188 self.maxDiff = None 189 self.maxDiff = None
189 self._assert_unexpected_results(test_data, { 190 self._assert_unexpected_results(test_data, {
190 'foo/pass1.html': sorted(["FAIL", "PASS"]), 191 'foo/pass1.html': sorted(['FAIL', 'PASS']),
191 'foo/pass2.html': sorted(["IMAGE", "PASS"]), 192 'foo/pass2.html': sorted(['IMAGE', 'PASS']),
192 'foo/fail.html': sorted(["TEXT", "PASS"]), 193 'foo/fail.html': sorted(['TEXT', 'PASS']),
193 'foo/f_p.html': sorted(["TEXT", "PASS"]), 194 'foo/f_p.html': sorted(['TEXT', 'PASS']),
194 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]), 195 'foo/crash.html': sorted(['WONTFIX', 'CRASH', 'TEXT']),
195 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]), 196 'foo/image.html': sorted(['CRASH', 'FAIL', 'IMAGE']),
196 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]), 197 'foo/i_f.html': sorted(['PASS', 'IMAGE', 'TEXT']),
197 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR ASH", "IMAGE", "MISSING", "LEAK"]), 198 'foo/all.html': sorted(['TEXT', 'PASS', 'IMAGE+TEXT', 'TIMEOUT', 'CR ASH', 'IMAGE', 'MISSING', 'LEAK']),
198 }) 199 })
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698