OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
59 return [self.get_test('failures/expected/text.html'), | 59 return [self.get_test('failures/expected/text.html'), |
60 self.get_test('failures/expected/image_checksum.html'), | 60 self.get_test('failures/expected/image_checksum.html'), |
61 self.get_test('failures/expected/crash.html'), | 61 self.get_test('failures/expected/crash.html'), |
62 self.get_test('failures/expected/needsrebaseline.html'), | 62 self.get_test('failures/expected/needsrebaseline.html'), |
63 self.get_test('failures/expected/needsmanualrebaseline.html'), | 63 self.get_test('failures/expected/needsmanualrebaseline.html'), |
64 self.get_test('failures/expected/missing_text.html'), | 64 self.get_test('failures/expected/missing_text.html'), |
65 self.get_test('failures/expected/image.html'), | 65 self.get_test('failures/expected/image.html'), |
66 self.get_test('failures/expected/timeout.html'), | 66 self.get_test('failures/expected/timeout.html'), |
67 self.get_test('passes/text.html')] | 67 self.get_test('passes/text.html')] |
68 | 68 |
69 | |
70 def get_basic_expectations(self): | 69 def get_basic_expectations(self): |
71 return """ | 70 return """ |
72 Bug(test) failures/expected/text.html [ Failure ] | 71 Bug(test) failures/expected/text.html [ Failure ] |
73 Bug(test) failures/expected/crash.html [ WontFix ] | 72 Bug(test) failures/expected/crash.html [ WontFix ] |
74 Bug(test) failures/expected/needsrebaseline.html [ NeedsRebaseline ] | 73 Bug(test) failures/expected/needsrebaseline.html [ NeedsRebaseline ] |
75 Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ] | 74 Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ] |
76 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ] | 75 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ] |
77 Bug(test) failures/expected/image_checksum.html [ WontFix ] | 76 Bug(test) failures/expected/image_checksum.html [ WontFix ] |
78 Bug(test) failures/expected/image.html [ WontFix Mac ] | 77 Bug(test) failures/expected/image.html [ WontFix Mac ] |
79 """ | 78 """ |
80 | 79 |
81 def parse_exp(self, expectations, overrides=None, is_lint_mode=False): | 80 def parse_exp(self, expectations, overrides=None, is_lint_mode=False): |
82 expectations_dict = OrderedDict() | 81 expectations_dict = OrderedDict() |
83 expectations_dict['expectations'] = expectations | 82 expectations_dict['expectations'] = expectations |
84 if overrides: | 83 if overrides: |
85 expectations_dict['overrides'] = overrides | 84 expectations_dict['overrides'] = overrides |
86 self._port.expectations_dict = lambda: expectations_dict | 85 self._port.expectations_dict = lambda: expectations_dict |
87 expectations_to_lint = expectations_dict if is_lint_mode else None | 86 expectations_to_lint = expectations_dict if is_lint_mode else None |
88 self._exp = TestExpectations(self._port, self.get_basic_tests(), expecta
tions_dict=expectations_to_lint, is_lint_mode=is_lint_mode) | 87 self._exp = TestExpectations( |
| 88 self._port, |
| 89 self.get_basic_tests(), |
| 90 expectations_dict=expectations_to_lint, |
| 91 is_lint_mode=is_lint_mode) |
89 | 92 |
90 def assert_exp_list(self, test, results): | 93 def assert_exp_list(self, test, results): |
91 self.assertEqual(self._exp.get_expectations(self.get_test(test)), set(re
sults)) | 94 self.assertEqual(self._exp.get_expectations(self.get_test(test)), set(re
sults)) |
92 | 95 |
93 def assert_exp(self, test, result): | 96 def assert_exp(self, test, result): |
94 self.assert_exp_list(test, [result]) | 97 self.assert_exp_list(test, [result]) |
95 | 98 |
96 def assert_bad_expectations(self, expectations, overrides=None): | 99 def assert_bad_expectations(self, expectations, overrides=None): |
97 self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode
=True, overrides=overrides) | 100 self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode
=True, overrides=overrides) |
98 | 101 |
99 | 102 |
100 class BasicTests(Base): | 103 class BasicTests(Base): |
| 104 |
101 def test_basic(self): | 105 def test_basic(self): |
102 self.parse_exp(self.get_basic_expectations()) | 106 self.parse_exp(self.get_basic_expectations()) |
103 self.assert_exp('failures/expected/text.html', FAIL) | 107 self.assert_exp('failures/expected/text.html', FAIL) |
104 self.assert_exp_list('failures/expected/image_checksum.html', [WONTFIX,
SKIP]) | 108 self.assert_exp_list('failures/expected/image_checksum.html', [WONTFIX,
SKIP]) |
105 self.assert_exp('passes/text.html', PASS) | 109 self.assert_exp('passes/text.html', PASS) |
106 self.assert_exp('failures/expected/image.html', PASS) | 110 self.assert_exp('failures/expected/image.html', PASS) |
107 | 111 |
108 | 112 |
109 class MiscTests(Base): | 113 class MiscTests(Base): |
| 114 |
110 def test_multiple_results(self): | 115 def test_multiple_results(self): |
111 self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]') | 116 self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]') |
112 self.assertEqual(self._exp.get_expectations( | 117 self.assertEqual(self._exp.get_expectations( |
113 self.get_test('failures/expected/text.html')), | 118 self.get_test('failures/expected/text.html')), |
114 set([FAIL, CRASH])) | 119 set([FAIL, CRASH])) |
115 | 120 |
116 def test_result_was_expected(self): | 121 def test_result_was_expected(self): |
117 # test basics | 122 # test basics |
118 self.assertEqual(TestExpectations.result_was_expected(PASS, set([PASS]),
test_needs_rebaselining=False), True) | 123 self.assertEqual(TestExpectations.result_was_expected(PASS, set([PASS]),
test_needs_rebaselining=False), True) |
119 self.assertEqual(TestExpectations.result_was_expected(FAIL, set([PASS]),
test_needs_rebaselining=False), False) | 124 self.assertEqual(TestExpectations.result_was_expected(FAIL, set([PASS]),
test_needs_rebaselining=False), False) |
120 | 125 |
121 # test handling of SKIPped tests and results | 126 # test handling of SKIPped tests and results |
122 self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH])
, test_needs_rebaselining=False), True) | 127 self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH])
, test_needs_rebaselining=False), True) |
123 self.assertEqual(TestExpectations.result_was_expected(SKIP, set([LEAK]),
test_needs_rebaselining=False), True) | 128 self.assertEqual(TestExpectations.result_was_expected(SKIP, set([LEAK]),
test_needs_rebaselining=False), True) |
124 | 129 |
125 # test handling of MISSING results and the REBASELINE specifier | 130 # test handling of MISSING results and the REBASELINE specifier |
126 self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS
]), test_needs_rebaselining=True), True) | 131 self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS
]), test_needs_rebaselining=True), True) |
127 self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS
]), test_needs_rebaselining=False), False) | 132 self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS
]), test_needs_rebaselining=False), False) |
128 | 133 |
129 self.assertTrue(TestExpectations.result_was_expected(PASS, set([NEEDS_RE
BASELINE]), test_needs_rebaselining=False)) | 134 self.assertTrue(TestExpectations.result_was_expected(PASS, set([NEEDS_RE
BASELINE]), test_needs_rebaselining=False)) |
130 self.assertTrue(TestExpectations.result_was_expected(MISSING, set([NEEDS
_REBASELINE]), test_needs_rebaselining=False)) | 135 self.assertTrue(TestExpectations.result_was_expected(MISSING, set([NEEDS
_REBASELINE]), test_needs_rebaselining=False)) |
131 self.assertTrue(TestExpectations.result_was_expected(TEXT, set([NEEDS_RE
BASELINE]), test_needs_rebaselining=False)) | 136 self.assertTrue(TestExpectations.result_was_expected(TEXT, set([NEEDS_RE
BASELINE]), test_needs_rebaselining=False)) |
132 self.assertTrue(TestExpectations.result_was_expected(IMAGE, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) | 137 self.assertTrue(TestExpectations.result_was_expected(IMAGE, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) |
133 self.assertTrue(TestExpectations.result_was_expected(IMAGE_PLUS_TEXT, se
t([NEEDS_REBASELINE]), test_needs_rebaselining=False)) | 138 self.assertTrue( |
| 139 TestExpectations.result_was_expected( |
| 140 IMAGE_PLUS_TEXT, set( |
| 141 [NEEDS_REBASELINE]), test_needs_rebaselining=False)) |
134 self.assertTrue(TestExpectations.result_was_expected(AUDIO, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) | 142 self.assertTrue(TestExpectations.result_was_expected(AUDIO, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) |
135 self.assertFalse(TestExpectations.result_was_expected(TIMEOUT, set([NEED
S_REBASELINE]), test_needs_rebaselining=False)) | 143 self.assertFalse(TestExpectations.result_was_expected(TIMEOUT, set([NEED
S_REBASELINE]), test_needs_rebaselining=False)) |
136 self.assertFalse(TestExpectations.result_was_expected(CRASH, set([NEEDS_
REBASELINE]), test_needs_rebaselining=False)) | 144 self.assertFalse(TestExpectations.result_was_expected(CRASH, set([NEEDS_
REBASELINE]), test_needs_rebaselining=False)) |
137 self.assertFalse(TestExpectations.result_was_expected(LEAK, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) | 145 self.assertFalse(TestExpectations.result_was_expected(LEAK, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) |
138 | 146 |
139 def test_remove_pixel_failures(self): | 147 def test_remove_pixel_failures(self): |
140 self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), se
t([FAIL])) | 148 self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), se
t([FAIL])) |
141 self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS])), se
t([PASS])) | 149 self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS])), se
t([PASS])) |
142 self.assertEqual(TestExpectations.remove_pixel_failures(set([IMAGE])), s
et([PASS])) | 150 self.assertEqual(TestExpectations.remove_pixel_failures(set([IMAGE])), s
et([PASS])) |
143 self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), se
t([FAIL])) | 151 self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), se
t([FAIL])) |
(...skipping 13 matching lines...) Expand all Loading... |
157 self.parse_exp(exp_str) | 165 self.parse_exp(exp_str) |
158 test_name = 'failures/expected/unknown-test.html' | 166 test_name = 'failures/expected/unknown-test.html' |
159 unknown_test = self.get_test(test_name) | 167 unknown_test = self.get_test(test_name) |
160 self.assertRaises(KeyError, self._exp.get_expectations, | 168 self.assertRaises(KeyError, self._exp.get_expectations, |
161 unknown_test) | 169 unknown_test) |
162 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) | 170 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) |
163 | 171 |
164 def test_get_expectations_string(self): | 172 def test_get_expectations_string(self): |
165 self.parse_exp(self.get_basic_expectations()) | 173 self.parse_exp(self.get_basic_expectations()) |
166 self.assertEqual(self._exp.get_expectations_string( | 174 self.assertEqual(self._exp.get_expectations_string( |
167 self.get_test('failures/expected/text.html')), | 175 self.get_test('failures/expected/text.html')), |
168 'FAIL') | 176 'FAIL') |
169 | 177 |
170 def test_expectation_to_string(self): | 178 def test_expectation_to_string(self): |
171 # Normal cases are handled by other tests. | 179 # Normal cases are handled by other tests. |
172 self.parse_exp(self.get_basic_expectations()) | 180 self.parse_exp(self.get_basic_expectations()) |
173 self.assertRaises(ValueError, self._exp.expectation_to_string, | 181 self.assertRaises(ValueError, self._exp.expectation_to_string, |
174 -1) | 182 -1) |
175 | 183 |
176 def test_get_test_set(self): | 184 def test_get_test_set(self): |
177 # Handle some corner cases for this routine not covered by other tests. | 185 # Handle some corner cases for this routine not covered by other tests. |
178 self.parse_exp(self.get_basic_expectations()) | 186 self.parse_exp(self.get_basic_expectations()) |
179 s = self._exp.get_test_set(WONTFIX) | 187 s = self._exp.get_test_set(WONTFIX) |
180 self.assertEqual(s, | 188 self.assertEqual(s, |
181 set([self.get_test('failures/expected/crash.html'), | 189 set([self.get_test('failures/expected/crash.html'), |
182 self.get_test('failures/expected/image_checksum.html')])) | 190 self.get_test('failures/expected/image_checksum.ht
ml')])) |
183 | 191 |
184 def test_needs_rebaseline_reftest(self): | 192 def test_needs_rebaseline_reftest(self): |
185 try: | 193 try: |
186 filesystem = self._port.host.filesystem | 194 filesystem = self._port.host.filesystem |
187 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'failures/expected/needsrebaseline.html'), 'content') | 195 filesystem.write_text_file( |
188 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'failures/expected/needsrebaseline-expected.html'), 'content') | 196 filesystem.join( |
189 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'failures/expected/needsmanualrebaseline.html'), 'content') | 197 self._port.layout_tests_dir(), |
190 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'failures/expected/needsmanualrebaseline-expected.html'), 'content') | 198 'failures/expected/needsrebaseline.html'), |
| 199 'content') |
| 200 filesystem.write_text_file( |
| 201 filesystem.join( |
| 202 self._port.layout_tests_dir(), |
| 203 'failures/expected/needsrebaseline-expected.html'), |
| 204 'content') |
| 205 filesystem.write_text_file( |
| 206 filesystem.join( |
| 207 self._port.layout_tests_dir(), |
| 208 'failures/expected/needsmanualrebaseline.html'), |
| 209 'content') |
| 210 filesystem.write_text_file( |
| 211 filesystem.join( |
| 212 self._port.layout_tests_dir(), |
| 213 'failures/expected/needsmanualrebaseline-expected.html'), |
| 214 'content') |
191 self.parse_exp("""Bug(user) failures/expected/needsrebaseline.html [
NeedsRebaseline ] | 215 self.parse_exp("""Bug(user) failures/expected/needsrebaseline.html [
NeedsRebaseline ] |
192 Bug(user) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]
""", is_lint_mode=True) | 216 Bug(user) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]
""", is_lint_mode=True) |
193 self.assertFalse(True, "ParseError wasn't raised") | 217 self.assertFalse(True, "ParseError wasn't raised") |
194 except ParseError, e: | 218 except ParseError as e: |
195 warnings = """expectations:1 A reftest cannot be marked as NeedsReba
seline/NeedsManualRebaseline failures/expected/needsrebaseline.html | 219 warnings = """expectations:1 A reftest cannot be marked as NeedsReba
seline/NeedsManualRebaseline failures/expected/needsrebaseline.html |
196 expectations:2 A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseli
ne failures/expected/needsmanualrebaseline.html""" | 220 expectations:2 A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseli
ne failures/expected/needsmanualrebaseline.html""" |
197 self.assertEqual(str(e), warnings) | 221 self.assertEqual(str(e), warnings) |
198 | 222 |
199 def test_parse_warning(self): | 223 def test_parse_warning(self): |
200 try: | 224 try: |
201 filesystem = self._port.host.filesystem | 225 filesystem = self._port.host.filesystem |
202 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'disabled-test.html-disabled'), 'content') | 226 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'disabled-test.html-disabled'), 'content') |
203 self.get_test('disabled-test.html-disabled'), | 227 self.get_test('disabled-test.html-disabled'), |
204 self.parse_exp("Bug(user) [ FOO ] failures/expected/text.html [ Fail
ure ]\n" | 228 self.parse_exp('Bug(user) [ FOO ] failures/expected/text.html [ Fail
ure ]\n' |
205 "Bug(user) non-existent-test.html [ Failure ]\n" | 229 'Bug(user) non-existent-test.html [ Failure ]\n' |
206 "Bug(user) disabled-test.html-disabled [ ImageOnlyFailure ]", is
_lint_mode=True) | 230 'Bug(user) disabled-test.html-disabled [ ImageOnlyFai
lure ]', is_lint_mode=True) |
207 self.assertFalse(True, "ParseError wasn't raised") | 231 self.assertFalse(True, "ParseError wasn't raised") |
208 except ParseError, e: | 232 except ParseError as e: |
209 warnings = ("expectations:1 Unrecognized specifier 'foo' failures/ex
pected/text.html\n" | 233 warnings = ("expectations:1 Unrecognized specifier 'foo' failures/ex
pected/text.html\n" |
210 "expectations:2 Path does not exist. non-existent-test.h
tml") | 234 'expectations:2 Path does not exist. non-existent-test.h
tml') |
211 self.assertEqual(str(e), warnings) | 235 self.assertEqual(str(e), warnings) |
212 | 236 |
213 def test_parse_warnings_are_logged_if_not_in_lint_mode(self): | 237 def test_parse_warnings_are_logged_if_not_in_lint_mode(self): |
214 oc = OutputCapture() | 238 oc = OutputCapture() |
215 try: | 239 try: |
216 oc.capture_output() | 240 oc.capture_output() |
217 self.parse_exp('-- this should be a syntax error', is_lint_mode=Fals
e) | 241 self.parse_exp('-- this should be a syntax error', is_lint_mode=Fals
e) |
218 finally: | 242 finally: |
219 _, _, logs = oc.restore_output() | 243 _, _, logs = oc.restore_output() |
220 self.assertNotEquals(logs, '') | 244 self.assertNotEquals(logs, '') |
221 | 245 |
222 def test_error_on_different_platform(self): | 246 def test_error_on_different_platform(self): |
223 # parse_exp uses a Windows port. Assert errors on Mac show up in lint mo
de. | 247 # parse_exp uses a Windows port. Assert errors on Mac show up in lint mo
de. |
224 self.assertRaises(ParseError, self.parse_exp, | 248 self.assertRaises(ParseError, self.parse_exp, |
225 'Bug(test) [ Mac ] failures/expected/text.html [ Failure ]\nBug(test
) [ Mac ] failures/expected/text.html [ Failure ]', | 249 'Bug(test) [ Mac ] failures/expected/text.html [ Failu
re ]\nBug(test) [ Mac ] failures/expected/text.html [ Failure ]', |
226 is_lint_mode=True) | 250 is_lint_mode=True) |
227 | 251 |
228 def test_error_on_different_build_type(self): | 252 def test_error_on_different_build_type(self): |
229 # parse_exp uses a Release port. Assert errors on DEBUG show up in lint
mode. | 253 # parse_exp uses a Release port. Assert errors on DEBUG show up in lint
mode. |
230 self.assertRaises(ParseError, self.parse_exp, | 254 self.assertRaises(ParseError, self.parse_exp, |
231 'Bug(test) [ Debug ] failures/expected/text.html [ Failure ]\nBug(te
st) [ Debug ] failures/expected/text.html [ Failure ]', | 255 'Bug(test) [ Debug ] failures/expected/text.html [ Fai
lure ]\nBug(test) [ Debug ] failures/expected/text.html [ Failure ]', |
232 is_lint_mode=True) | 256 is_lint_mode=True) |
233 | 257 |
234 def test_overrides(self): | 258 def test_overrides(self): |
235 self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]", | 259 self.parse_exp('Bug(exp) failures/expected/text.html [ Failure ]', |
236 "Bug(override) failures/expected/text.html [ ImageOnlyFai
lure ]") | 260 'Bug(override) failures/expected/text.html [ ImageOnlyFai
lure ]') |
237 self.assert_exp_list('failures/expected/text.html', [FAIL, IMAGE]) | 261 self.assert_exp_list('failures/expected/text.html', [FAIL, IMAGE]) |
238 | 262 |
239 def test_overrides__directory(self): | 263 def test_overrides__directory(self): |
240 self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]", | 264 self.parse_exp('Bug(exp) failures/expected/text.html [ Failure ]', |
241 "Bug(override) failures/expected [ Crash ]") | 265 'Bug(override) failures/expected [ Crash ]') |
242 self.assert_exp_list('failures/expected/text.html', [FAIL, CRASH]) | 266 self.assert_exp_list('failures/expected/text.html', [FAIL, CRASH]) |
243 self.assert_exp_list('failures/expected/image.html', [CRASH]) | 267 self.assert_exp_list('failures/expected/image.html', [CRASH]) |
244 | 268 |
245 def test_overrides__duplicate(self): | 269 def test_overrides__duplicate(self): |
246 self.assert_bad_expectations("Bug(exp) failures/expected/text.html [ Fai
lure ]", | 270 self.assert_bad_expectations('Bug(exp) failures/expected/text.html [ Fai
lure ]', |
247 "Bug(override) failures/expected/text.html
[ ImageOnlyFailure ]\n" | 271 'Bug(override) failures/expected/text.html
[ ImageOnlyFailure ]\n' |
248 "Bug(override) failures/expected/text.html
[ Crash ]\n") | 272 'Bug(override) failures/expected/text.html
[ Crash ]\n') |
249 | 273 |
250 def test_pixel_tests_flag(self): | 274 def test_pixel_tests_flag(self): |
251 def match(test, result, pixel_tests_enabled): | 275 def match(test, result, pixel_tests_enabled): |
252 return self._exp.matches_an_expected_result( | 276 return self._exp.matches_an_expected_result( |
253 self.get_test(test), result, pixel_tests_enabled, sanitizer_is_e
nabled=False) | 277 self.get_test(test), result, pixel_tests_enabled, sanitizer_is_e
nabled=False) |
254 | 278 |
255 self.parse_exp(self.get_basic_expectations()) | 279 self.parse_exp(self.get_basic_expectations()) |
256 self.assertTrue(match('failures/expected/text.html', FAIL, True)) | 280 self.assertTrue(match('failures/expected/text.html', FAIL, True)) |
257 self.assertTrue(match('failures/expected/text.html', FAIL, False)) | 281 self.assertTrue(match('failures/expected/text.html', FAIL, False)) |
258 self.assertFalse(match('failures/expected/text.html', CRASH, True)) | 282 self.assertFalse(match('failures/expected/text.html', CRASH, True)) |
(...skipping 17 matching lines...) Expand all Loading... |
276 Bug(test) failures/expected/image.html [ ImageOnlyFailure ] | 300 Bug(test) failures/expected/image.html [ ImageOnlyFailure ] |
277 Bug(test) failures/expected/text.html [ Failure ] | 301 Bug(test) failures/expected/text.html [ Failure ] |
278 Bug(test) failures/expected/timeout.html [ Timeout ] | 302 Bug(test) failures/expected/timeout.html [ Timeout ] |
279 """) | 303 """) |
280 self.assertTrue(match('failures/expected/crash.html', CRASH)) | 304 self.assertTrue(match('failures/expected/crash.html', CRASH)) |
281 self.assertTrue(match('failures/expected/image.html', PASS)) | 305 self.assertTrue(match('failures/expected/image.html', PASS)) |
282 self.assertTrue(match('failures/expected/text.html', PASS)) | 306 self.assertTrue(match('failures/expected/text.html', PASS)) |
283 self.assertTrue(match('failures/expected/timeout.html', TIMEOUT)) | 307 self.assertTrue(match('failures/expected/timeout.html', TIMEOUT)) |
284 | 308 |
285 def test_more_specific_override_resets_skip(self): | 309 def test_more_specific_override_resets_skip(self): |
286 self.parse_exp("Bug(x) failures/expected [ Skip ]\n" | 310 self.parse_exp('Bug(x) failures/expected [ Skip ]\n' |
287 "Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\
n") | 311 'Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\
n') |
288 self.assert_exp('failures/expected/text.html', IMAGE) | 312 self.assert_exp('failures/expected/text.html', IMAGE) |
289 self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir
(), | 313 self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir
(), |
290 'failures/expected/text.htm
l') in | 314 'failures/expected/text.htm
l') in |
291 self._exp.get_tests_with_result_type(SKIP)) | 315 self._exp.get_tests_with_result_type(SKIP)) |
292 | 316 |
293 def test_bot_test_expectations(self): | 317 def test_bot_test_expectations(self): |
294 """Test that expectations are merged rather than overridden when using f
laky option 'unexpected'.""" | 318 """Test that expectations are merged rather than overridden when using f
laky option 'unexpected'.""" |
295 test_name1 = 'failures/expected/text.html' | 319 test_name1 = 'failures/expected/text.html' |
296 test_name2 = 'passes/text.html' | 320 test_name2 = 'passes/text.html' |
297 | 321 |
298 expectations_dict = OrderedDict() | 322 expectations_dict = OrderedDict() |
299 expectations_dict['expectations'] = "Bug(x) %s [ ImageOnlyFailure ]\nBug
(x) %s [ Slow ]\n" % (test_name1, test_name2) | 323 expectations_dict['expectations'] = 'Bug(x) %s [ ImageOnlyFailure ]\nBug
(x) %s [ Slow ]\n' % (test_name1, test_name2) |
300 self._port.expectations_dict = lambda: expectations_dict | 324 self._port.expectations_dict = lambda: expectations_dict |
301 | 325 |
302 expectations = TestExpectations(self._port, self.get_basic_tests()) | 326 expectations = TestExpectations(self._port, self.get_basic_tests()) |
303 self.assertEqual(expectations.get_expectations(self.get_test(test_name1)
), set([IMAGE])) | 327 self.assertEqual(expectations.get_expectations(self.get_test(test_name1)
), set([IMAGE])) |
304 self.assertEqual(expectations.get_expectations(self.get_test(test_name2)
), set([SLOW])) | 328 self.assertEqual(expectations.get_expectations(self.get_test(test_name2)
), set([SLOW])) |
305 | 329 |
306 def bot_expectations(): | 330 def bot_expectations(): |
307 return {test_name1: ['PASS', 'TIMEOUT'], test_name2: ['CRASH']} | 331 return {test_name1: ['PASS', 'TIMEOUT'], test_name2: ['CRASH']} |
308 self._port.bot_expectations = bot_expectations | 332 self._port.bot_expectations = bot_expectations |
309 self._port._options.ignore_flaky_tests = 'unexpected' | 333 self._port._options.ignore_flaky_tests = 'unexpected' |
310 | 334 |
311 expectations = TestExpectations(self._port, self.get_basic_tests()) | 335 expectations = TestExpectations(self._port, self.get_basic_tests()) |
312 self.assertEqual(expectations.get_expectations(self.get_test(test_name1)
), set([PASS, IMAGE, TIMEOUT])) | 336 self.assertEqual(expectations.get_expectations(self.get_test(test_name1)
), set([PASS, IMAGE, TIMEOUT])) |
313 self.assertEqual(expectations.get_expectations(self.get_test(test_name2)
), set([CRASH, SLOW])) | 337 self.assertEqual(expectations.get_expectations(self.get_test(test_name2)
), set([CRASH, SLOW])) |
314 | 338 |
| 339 |
315 class SkippedTests(Base): | 340 class SkippedTests(Base): |
| 341 |
316 def check(self, expectations, overrides, skips, lint=False, expected_results
=[WONTFIX, SKIP, FAIL]): | 342 def check(self, expectations, overrides, skips, lint=False, expected_results
=[WONTFIX, SKIP, FAIL]): |
317 port = MockHost().port_factory.get('test-win-xp') | 343 port = MockHost().port_factory.get('test-win-xp') |
318 port._filesystem.write_text_file(port._filesystem.join(port.layout_tests
_dir(), 'failures/expected/text.html'), 'foo') | 344 port._filesystem.write_text_file(port._filesystem.join(port.layout_tests
_dir(), 'failures/expected/text.html'), 'foo') |
319 expectations_dict = OrderedDict() | 345 expectations_dict = OrderedDict() |
320 expectations_dict['expectations'] = expectations | 346 expectations_dict['expectations'] = expectations |
321 if overrides: | 347 if overrides: |
322 expectations_dict['overrides'] = overrides | 348 expectations_dict['overrides'] = overrides |
323 port.expectations_dict = lambda: expectations_dict | 349 port.expectations_dict = lambda: expectations_dict |
324 port.skipped_layout_tests = lambda tests: set(skips) | 350 port.skipped_layout_tests = lambda tests: set(skips) |
325 expectations_to_lint = expectations_dict if lint else None | 351 expectations_to_lint = expectations_dict if lint else None |
326 exp = TestExpectations(port, ['failures/expected/text.html'], expectatio
ns_dict=expectations_to_lint, is_lint_mode=lint) | 352 exp = TestExpectations(port, ['failures/expected/text.html'], expectatio
ns_dict=expectations_to_lint, is_lint_mode=lint) |
327 self.assertEqual(exp.get_expectations('failures/expected/text.html'), se
t(expected_results)) | 353 self.assertEqual(exp.get_expectations('failures/expected/text.html'), se
t(expected_results)) |
328 | 354 |
329 def test_skipped_tests_work(self): | 355 def test_skipped_tests_work(self): |
330 self.check(expectations='', overrides=None, skips=['failures/expected/te
xt.html'], expected_results=[WONTFIX, SKIP]) | 356 self.check(expectations='', overrides=None, skips=['failures/expected/te
xt.html'], expected_results=[WONTFIX, SKIP]) |
331 | 357 |
332 def test_duplicate_skipped_test_fails_lint(self): | 358 def test_duplicate_skipped_test_fails_lint(self): |
333 self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/
expected/text.html [ Failure ]\n', | 359 self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/
expected/text.html [ Failure ]\n', |
334 overrides=None, skips=['failures/expected/text.html'], lint=True) | 360 overrides=None, skips=['failures/expected/text.html'],
lint=True) |
335 | 361 |
336 def test_skipped_file_overrides_expectations(self): | 362 def test_skipped_file_overrides_expectations(self): |
337 self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\
n', overrides=None, | 363 self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\
n', overrides=None, |
338 skips=['failures/expected/text.html']) | 364 skips=['failures/expected/text.html']) |
339 | 365 |
340 def test_skipped_dir_overrides_expectations(self): | 366 def test_skipped_dir_overrides_expectations(self): |
341 self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\
n', overrides=None, | 367 self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\
n', overrides=None, |
342 skips=['failures/expected']) | 368 skips=['failures/expected']) |
343 | 369 |
344 def test_skipped_file_overrides_overrides(self): | 370 def test_skipped_file_overrides_overrides(self): |
(...skipping 17 matching lines...) Expand all Loading... |
362 self.assertEqual('The following test foo/bar/baz.html from the Skipped l
ist doesn\'t exist\n', logs) | 388 self.assertEqual('The following test foo/bar/baz.html from the Skipped l
ist doesn\'t exist\n', logs) |
363 | 389 |
364 def test_expectations_string(self): | 390 def test_expectations_string(self): |
365 self.parse_exp(self.get_basic_expectations()) | 391 self.parse_exp(self.get_basic_expectations()) |
366 notrun = 'failures/expected/text.html' | 392 notrun = 'failures/expected/text.html' |
367 self._exp.add_extra_skipped_tests([notrun]) | 393 self._exp.add_extra_skipped_tests([notrun]) |
368 self.assertEqual('NOTRUN', self._exp.get_expectations_string(notrun)) | 394 self.assertEqual('NOTRUN', self._exp.get_expectations_string(notrun)) |
369 | 395 |
370 | 396 |
371 class ExpectationSyntaxTests(Base): | 397 class ExpectationSyntaxTests(Base): |
| 398 |
372 def test_unrecognized_expectation(self): | 399 def test_unrecognized_expectation(self): |
373 self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Un
known ]') | 400 self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Un
known ]') |
374 | 401 |
375 def test_macro(self): | 402 def test_macro(self): |
376 exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]' | 403 exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]' |
377 self.parse_exp(exp_str) | 404 self.parse_exp(exp_str) |
378 self.assert_exp('failures/expected/text.html', FAIL) | 405 self.assert_exp('failures/expected/text.html', FAIL) |
379 | 406 |
380 def assert_tokenize_exp(self, line, bugs=None, specifiers=None, expectations
=None, warnings=None, comment=None, name='foo.html'): | 407 def assert_tokenize_exp(self, line, bugs=None, specifiers=None, expectations
=None, |
| 408 warnings=None, comment=None, name='foo.html'): |
381 bugs = bugs or [] | 409 bugs = bugs or [] |
382 specifiers = specifiers or [] | 410 specifiers = specifiers or [] |
383 expectations = expectations or [] | 411 expectations = expectations or [] |
384 warnings = warnings or [] | 412 warnings = warnings or [] |
385 filename = 'TestExpectations' | 413 filename = 'TestExpectations' |
386 line_number = '1' | 414 line_number = '1' |
387 expectation_line = TestExpectationParser._tokenize_line(filename, line,
line_number) | 415 expectation_line = TestExpectationParser._tokenize_line(filename, line,
line_number) |
388 self.assertEqual(expectation_line.warnings, warnings) | 416 self.assertEqual(expectation_line.warnings, warnings) |
389 self.assertEqual(expectation_line.name, name) | 417 self.assertEqual(expectation_line.name, name) |
390 self.assertEqual(expectation_line.filename, filename) | 418 self.assertEqual(expectation_line.filename, filename) |
391 self.assertEqual(expectation_line.line_numbers, line_number) | 419 self.assertEqual(expectation_line.line_numbers, line_number) |
392 if not warnings: | 420 if not warnings: |
393 self.assertEqual(expectation_line.specifiers, specifiers) | 421 self.assertEqual(expectation_line.specifiers, specifiers) |
394 self.assertEqual(expectation_line.expectations, expectations) | 422 self.assertEqual(expectation_line.expectations, expectations) |
395 | 423 |
396 def test_comments(self): | 424 def test_comments(self): |
397 self.assert_tokenize_exp("# comment", name=None, comment="# comment") | 425 self.assert_tokenize_exp('# comment', name=None, comment='# comment') |
398 self.assert_tokenize_exp("foo.html [ Pass ] # comment", comment="# comme
nt", expectations=['PASS'], specifiers=[]) | 426 self.assert_tokenize_exp('foo.html [ Pass ] # comment', comment='# comme
nt', expectations=['PASS'], specifiers=[]) |
399 | 427 |
400 def test_config_specifiers(self): | 428 def test_config_specifiers(self): |
401 self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', specifiers=['M
AC'], expectations=['FAIL']) | 429 self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', specifiers=['M
AC'], expectations=['FAIL']) |
402 | 430 |
403 def test_unknown_config(self): | 431 def test_unknown_config(self): |
404 self.assert_tokenize_exp('[ Foo ] foo.html [ Pass ]', specifiers=['Foo']
, expectations=['PASS']) | 432 self.assert_tokenize_exp('[ Foo ] foo.html [ Pass ]', specifiers=['Foo']
, expectations=['PASS']) |
405 | 433 |
406 def test_unknown_expectation(self): | 434 def test_unknown_expectation(self): |
407 self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized e
xpectation "Audio"']) | 435 self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized e
xpectation "Audio"']) |
408 | 436 |
409 def test_skip(self): | 437 def test_skip(self): |
410 self.assert_tokenize_exp('foo.html [ Skip ]', specifiers=[], expectation
s=['SKIP']) | 438 self.assert_tokenize_exp('foo.html [ Skip ]', specifiers=[], expectation
s=['SKIP']) |
411 | 439 |
412 def test_slow(self): | 440 def test_slow(self): |
413 self.assert_tokenize_exp('foo.html [ Slow ]', specifiers=[], expectation
s=['SLOW']) | 441 self.assert_tokenize_exp('foo.html [ Slow ]', specifiers=[], expectation
s=['SLOW']) |
414 | 442 |
415 def test_wontfix(self): | 443 def test_wontfix(self): |
416 self.assert_tokenize_exp('foo.html [ WontFix ]', specifiers=[], expectat
ions=['WONTFIX', 'SKIP']) | 444 self.assert_tokenize_exp('foo.html [ WontFix ]', specifiers=[], expectat
ions=['WONTFIX', 'SKIP']) |
417 self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', specif
iers=[], expectations=['WONTFIX', 'SKIP'], | 445 self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', specif
iers=[], expectations=['WONTFIX', 'SKIP'], |
418 warnings=['A test marked Skip or WontFix must not have other expecta
tions.']) | 446 warnings=['A test marked Skip or WontFix must n
ot have other expectations.']) |
419 | 447 |
420 def test_blank_line(self): | 448 def test_blank_line(self): |
421 self.assert_tokenize_exp('', name=None) | 449 self.assert_tokenize_exp('', name=None) |
422 | 450 |
423 def test_warnings(self): | 451 def test_warnings(self): |
424 self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.
', 'Missing expectations.'], name=None) | 452 self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.
', 'Missing expectations.'], name=None) |
425 self.assert_tokenize_exp('[ [', warnings=['unexpected "["', 'Missing exp
ectations.'], name=None) | 453 self.assert_tokenize_exp('[ [', warnings=['unexpected "["', 'Missing exp
ectations.'], name=None) |
426 self.assert_tokenize_exp('crbug.com/12345 ]', warnings=['unexpected "]"'
, 'Missing expectations.'], name=None) | 454 self.assert_tokenize_exp('crbug.com/12345 ]', warnings=['unexpected "]"'
, 'Missing expectations.'], name=None) |
427 | 455 |
428 self.assert_tokenize_exp('foo.html crbug.com/12345 ]', warnings=['"crbug
.com/12345" is not at the start of the line.', 'Missing expectations.']) | 456 self.assert_tokenize_exp( |
| 457 'foo.html crbug.com/12345 ]', |
| 458 warnings=[ |
| 459 '"crbug.com/12345" is not at the start of the line.', |
| 460 'Missing expectations.']) |
429 self.assert_tokenize_exp('foo.html', warnings=['Missing expectations.']) | 461 self.assert_tokenize_exp('foo.html', warnings=['Missing expectations.']) |
430 | 462 |
431 | 463 |
432 class SemanticTests(Base): | 464 class SemanticTests(Base): |
| 465 |
433 def test_bug_format(self): | 466 def test_bug_format(self): |
434 self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected
/text.html [ Failure ]', is_lint_mode=True) | 467 self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected
/text.html [ Failure ]', is_lint_mode=True) |
435 | 468 |
436 def test_bad_bugid(self): | 469 def test_bad_bugid(self): |
437 try: | 470 try: |
438 self.parse_exp('crbug/1234 failures/expected/text.html [ Failure ]',
is_lint_mode=True) | 471 self.parse_exp('crbug/1234 failures/expected/text.html [ Failure ]',
is_lint_mode=True) |
439 self.fail('should have raised an error about a bad bug identifier') | 472 self.fail('should have raised an error about a bad bug identifier') |
440 except ParseError, exp: | 473 except ParseError as exp: |
441 self.assertEqual(len(exp.warnings), 3) | 474 self.assertEqual(len(exp.warnings), 3) |
442 | 475 |
443 def test_missing_bugid(self): | 476 def test_missing_bugid(self): |
444 self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mode=F
alse) | 477 self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mode=F
alse) |
445 self.assertFalse(self._exp.has_warnings()) | 478 self.assertFalse(self._exp.has_warnings()) |
446 | 479 |
447 try: | 480 try: |
448 self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mo
de=True) | 481 self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mo
de=True) |
449 except ParseError, exp: | 482 except ParseError as exp: |
450 self.assertEqual(exp.warnings, ['expectations:1 Test lacks BUG speci
fier. failures/expected/text.html']) | 483 self.assertEqual(exp.warnings, ['expectations:1 Test lacks BUG speci
fier. failures/expected/text.html']) |
451 | 484 |
452 def test_skip_and_wontfix(self): | 485 def test_skip_and_wontfix(self): |
453 # Skip is not allowed to have other expectations as well, because those | 486 # Skip is not allowed to have other expectations as well, because those |
454 # expectations won't be exercised and may become stale . | 487 # expectations won't be exercised and may become stale . |
455 self.parse_exp('failures/expected/text.html [ Failure Skip ]') | 488 self.parse_exp('failures/expected/text.html [ Failure Skip ]') |
456 self.assertTrue(self._exp.has_warnings()) | 489 self.assertTrue(self._exp.has_warnings()) |
457 | 490 |
458 self.parse_exp('failures/expected/text.html [ Crash WontFix ]') | 491 self.parse_exp('failures/expected/text.html [ Crash WontFix ]') |
459 self.assertTrue(self._exp.has_warnings()) | 492 self.assertTrue(self._exp.has_warnings()) |
460 | 493 |
461 self.parse_exp('failures/expected/text.html [ Pass WontFix ]') | 494 self.parse_exp('failures/expected/text.html [ Pass WontFix ]') |
462 self.assertTrue(self._exp.has_warnings()) | 495 self.assertTrue(self._exp.has_warnings()) |
463 | 496 |
464 def test_rebaseline(self): | 497 def test_rebaseline(self): |
465 # Can't lint a file w/ 'REBASELINE' in it. | 498 # Can't lint a file w/ 'REBASELINE' in it. |
466 self.assertRaises(ParseError, self.parse_exp, | 499 self.assertRaises(ParseError, self.parse_exp, |
467 'Bug(test) failures/expected/text.html [ Failure Rebaseline ]', | 500 'Bug(test) failures/expected/text.html [ Failure Rebas
eline ]', |
468 is_lint_mode=True) | 501 is_lint_mode=True) |
469 | 502 |
470 def test_duplicates(self): | 503 def test_duplicates(self): |
471 self.assertRaises(ParseError, self.parse_exp, """ | 504 self.assertRaises(ParseError, self.parse_exp, """ |
472 Bug(exp) failures/expected/text.html [ Failure ] | 505 Bug(exp) failures/expected/text.html [ Failure ] |
473 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True) | 506 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True) |
474 | 507 |
475 self.assertRaises(ParseError, self.parse_exp, | 508 self.assertRaises(ParseError, self.parse_exp, |
476 self.get_basic_expectations(), overrides=""" | 509 self.get_basic_expectations(), overrides=""" |
477 Bug(override) failures/expected/text.html [ Failure ] | 510 Bug(override) failures/expected/text.html [ Failure ] |
478 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=
True) | 511 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=
True) |
479 | 512 |
480 def test_duplicate_with_line_before_preceding_line(self): | 513 def test_duplicate_with_line_before_preceding_line(self): |
481 self.assert_bad_expectations("""Bug(exp) [ Debug ] failures/expected/tex
t.html [ Failure ] | 514 self.assert_bad_expectations("""Bug(exp) [ Debug ] failures/expected/tex
t.html [ Failure ] |
482 Bug(exp) [ Release ] failures/expected/text.html [ Failure ] | 515 Bug(exp) [ Release ] failures/expected/text.html [ Failure ] |
483 Bug(exp) [ Debug ] failures/expected/text.html [ Failure ] | 516 Bug(exp) [ Debug ] failures/expected/text.html [ Failure ] |
484 """) | 517 """) |
485 | 518 |
486 def test_missing_file(self): | 519 def test_missing_file(self): |
487 self.parse_exp('Bug(test) missing_file.html [ Failure ]') | 520 self.parse_exp('Bug(test) missing_file.html [ Failure ]') |
488 self.assertTrue(self._exp.has_warnings(), 1) | 521 self.assertTrue(self._exp.has_warnings(), 1) |
489 | 522 |
490 | 523 |
491 class PrecedenceTests(Base): | 524 class PrecedenceTests(Base): |
| 525 |
492 def test_file_over_directory(self): | 526 def test_file_over_directory(self): |
493 # This tests handling precedence of specific lines over directories | 527 # This tests handling precedence of specific lines over directories |
494 # and tests expectations covering entire directories. | 528 # and tests expectations covering entire directories. |
495 exp_str = """ | 529 exp_str = """ |
496 Bug(x) failures/expected/text.html [ Failure ] | 530 Bug(x) failures/expected/text.html [ Failure ] |
497 Bug(y) failures/expected [ WontFix ] | 531 Bug(y) failures/expected [ WontFix ] |
498 """ | 532 """ |
499 self.parse_exp(exp_str) | 533 self.parse_exp(exp_str) |
500 self.assert_exp('failures/expected/text.html', FAIL) | 534 self.assert_exp('failures/expected/text.html', FAIL) |
501 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) | 535 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) |
502 | 536 |
503 exp_str = """ | 537 exp_str = """ |
504 Bug(x) failures/expected [ WontFix ] | 538 Bug(x) failures/expected [ WontFix ] |
505 Bug(y) failures/expected/text.html [ Failure ] | 539 Bug(y) failures/expected/text.html [ Failure ] |
506 """ | 540 """ |
507 self.parse_exp(exp_str) | 541 self.parse_exp(exp_str) |
508 self.assert_exp('failures/expected/text.html', FAIL) | 542 self.assert_exp('failures/expected/text.html', FAIL) |
509 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) | 543 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) |
510 | 544 |
511 def test_ambiguous(self): | 545 def test_ambiguous(self): |
512 self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ P
ass ]\n" | 546 self.assert_bad_expectations('Bug(test) [ Release ] passes/text.html [ P
ass ]\n' |
513 "Bug(test) [ Win ] passes/text.html [ Failu
re ]\n") | 547 'Bug(test) [ Win ] passes/text.html [ Failu
re ]\n') |
514 | 548 |
515 def test_more_specifiers(self): | 549 def test_more_specifiers(self): |
516 self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ P
ass ]\n" | 550 self.assert_bad_expectations('Bug(test) [ Release ] passes/text.html [ P
ass ]\n' |
517 "Bug(test) [ Win Release ] passes/text.html
[ Failure ]\n") | 551 'Bug(test) [ Win Release ] passes/text.html
[ Failure ]\n') |
518 | 552 |
519 def test_order_in_file(self): | 553 def test_order_in_file(self): |
520 self.assert_bad_expectations("Bug(test) [ Win Release ] : passes/text.ht
ml [ Failure ]\n" | 554 self.assert_bad_expectations('Bug(test) [ Win Release ] : passes/text.ht
ml [ Failure ]\n' |
521 "Bug(test) [ Release ] : passes/text.html [
Pass ]\n") | 555 'Bug(test) [ Release ] : passes/text.html [
Pass ]\n') |
522 | 556 |
523 def test_macro_overrides(self): | 557 def test_macro_overrides(self): |
524 self.assert_bad_expectations("Bug(test) [ Win ] passes/text.html [ Pass
]\n" | 558 self.assert_bad_expectations('Bug(test) [ Win ] passes/text.html [ Pass
]\n' |
525 "Bug(test) [ XP ] passes/text.html [ Failur
e ]\n") | 559 'Bug(test) [ XP ] passes/text.html [ Failur
e ]\n') |
526 | 560 |
527 | 561 |
528 class RemoveConfigurationsTest(Base): | 562 class RemoveConfigurationsTest(Base): |
| 563 |
529 def test_remove(self): | 564 def test_remove(self): |
530 host = MockHost() | 565 host = MockHost() |
531 test_port = host.port_factory.get('test-win-xp', None) | 566 test_port = host.port_factory.get('test-win-xp', None) |
532 test_port.test_exists = lambda test: True | 567 test_port.test_exists = lambda test: True |
533 test_port.test_isfile = lambda test: True | 568 test_port.test_isfile = lambda test: True |
534 | 569 |
535 test_config = test_port.test_configuration() | 570 test_config = test_port.test_configuration() |
536 test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux
Win Release ] failures/expected/foo.html [ Failure ] | 571 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Linux
Win Release ] failures/expected/foo.html [ Failure ] |
537 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ] | 572 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ] |
538 """} | 573 """} |
539 expectations = TestExpectations(test_port, self.get_basic_tests()) | 574 expectations = TestExpectations(test_port, self.get_basic_tests()) |
540 | 575 |
541 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 576 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
542 | 577 |
543 self.assertEqual("""Bug(x) [ Linux Win7 Release ] failures/expected/foo.
html [ Failure ] | 578 self.assertEqual("""Bug(x) [ Linux Win7 Release ] failures/expected/foo.
html [ Failure ] |
544 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ] | 579 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ] |
545 """, actual_expectations) | 580 """, actual_expectations) |
546 | 581 |
547 def test_remove_needs_rebaseline(self): | 582 def test_remove_needs_rebaseline(self): |
548 host = MockHost() | 583 host = MockHost() |
549 test_port = host.port_factory.get('test-win-xp', None) | 584 test_port = host.port_factory.get('test-win-xp', None) |
550 test_port.test_exists = lambda test: True | 585 test_port.test_exists = lambda test: True |
551 test_port.test_isfile = lambda test: True | 586 test_port.test_isfile = lambda test: True |
552 | 587 |
553 test_config = test_port.test_configuration() | 588 test_config = test_port.test_configuration() |
554 test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Win ]
failures/expected/foo.html [ NeedsRebaseline ] | 589 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win ]
failures/expected/foo.html [ NeedsRebaseline ] |
555 """} | 590 """} |
556 expectations = TestExpectations(test_port, self.get_basic_tests()) | 591 expectations = TestExpectations(test_port, self.get_basic_tests()) |
557 | 592 |
558 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 593 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
559 | 594 |
560 self.assertEqual("""Bug(x) [ XP Debug ] failures/expected/foo.html [ Nee
dsRebaseline ] | 595 self.assertEqual("""Bug(x) [ XP Debug ] failures/expected/foo.html [ Nee
dsRebaseline ] |
561 Bug(x) [ Win7 ] failures/expected/foo.html [ NeedsRebaseline ] | 596 Bug(x) [ Win7 ] failures/expected/foo.html [ NeedsRebaseline ] |
562 """, actual_expectations) | 597 """, actual_expectations) |
563 | 598 |
564 def test_remove_multiple_configurations(self): | 599 def test_remove_multiple_configurations(self): |
(...skipping 24 matching lines...) Expand all Loading... |
589 | 624 |
590 test_config = test_port.test_configuration() | 625 test_config = test_port.test_configuration() |
591 test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win D
ebug ] failures/expected/foo.html [ Crash ] | 626 test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win D
ebug ] failures/expected/foo.html [ Crash ] |
592 | 627 |
593 # This comment line should get stripped. As should the preceding line. | 628 # This comment line should get stripped. As should the preceding line. |
594 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] | 629 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] |
595 """} | 630 """} |
596 expectations = TestExpectations(test_port) | 631 expectations = TestExpectations(test_port) |
597 | 632 |
598 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 633 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
599 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 634 actual_expectations = expectations.remove_configurations( |
| 635 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
600 | 636 |
601 self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Cr
ash ] | 637 self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Cr
ash ] |
602 """, actual_expectations) | 638 """, actual_expectations) |
603 | 639 |
604 def test_remove_line_with_comments_at_start(self): | 640 def test_remove_line_with_comments_at_start(self): |
605 host = MockHost() | 641 host = MockHost() |
606 test_port = host.port_factory.get('test-win-xp', None) | 642 test_port = host.port_factory.get('test-win-xp', None) |
607 test_port.test_exists = lambda test: True | 643 test_port.test_exists = lambda test: True |
608 test_port.test_isfile = lambda test: True | 644 test_port.test_isfile = lambda test: True |
609 | 645 |
610 test_config = test_port.test_configuration() | 646 test_config = test_port.test_configuration() |
611 test_port.expectations_dict = lambda: {'expectations': """ | 647 test_port.expectations_dict = lambda: {'expectations': """ |
612 # This comment line should get stripped. As should the preceding line. | 648 # This comment line should get stripped. As should the preceding line. |
613 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] | 649 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] |
614 | 650 |
615 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 651 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
616 """} | 652 """} |
617 expectations = TestExpectations(test_port) | 653 expectations = TestExpectations(test_port) |
618 | 654 |
619 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 655 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
620 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 656 actual_expectations = expectations.remove_configurations( |
| 657 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
621 | 658 |
622 self.assertEqual(""" | 659 self.assertEqual(""" |
623 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 660 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
624 """, actual_expectations) | 661 """, actual_expectations) |
625 | 662 |
626 def test_remove_line_with_comments_at_end_with_no_trailing_newline(self): | 663 def test_remove_line_with_comments_at_end_with_no_trailing_newline(self): |
627 host = MockHost() | 664 host = MockHost() |
628 test_port = host.port_factory.get('test-win-xp', None) | 665 test_port = host.port_factory.get('test-win-xp', None) |
629 test_port.test_exists = lambda test: True | 666 test_port.test_exists = lambda test: True |
630 test_port.test_isfile = lambda test: True | 667 test_port.test_isfile = lambda test: True |
631 | 668 |
632 test_config = test_port.test_configuration() | 669 test_config = test_port.test_configuration() |
633 test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win D
ebug ] failures/expected/foo.html [ Crash ] | 670 test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win D
ebug ] failures/expected/foo.html [ Crash ] |
634 | 671 |
635 # This comment line should get stripped. As should the preceding line. | 672 # This comment line should get stripped. As should the preceding line. |
636 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]"""} | 673 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]"""} |
637 expectations = TestExpectations(test_port) | 674 expectations = TestExpectations(test_port) |
638 | 675 |
639 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 676 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
640 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 677 actual_expectations = expectations.remove_configurations( |
| 678 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
641 | 679 |
642 self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Cr
ash ]""", actual_expectations) | 680 self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Cr
ash ]""", actual_expectations) |
643 | 681 |
644 def test_remove_line_leaves_comments_for_next_line(self): | 682 def test_remove_line_leaves_comments_for_next_line(self): |
645 host = MockHost() | 683 host = MockHost() |
646 test_port = host.port_factory.get('test-win-xp', None) | 684 test_port = host.port_factory.get('test-win-xp', None) |
647 test_port.test_exists = lambda test: True | 685 test_port.test_exists = lambda test: True |
648 test_port.test_isfile = lambda test: True | 686 test_port.test_isfile = lambda test: True |
649 | 687 |
650 test_config = test_port.test_configuration() | 688 test_config = test_port.test_configuration() |
651 test_port.expectations_dict = lambda: {'expectations': """ | 689 test_port.expectations_dict = lambda: {'expectations': """ |
652 # This comment line should not get stripped. | 690 # This comment line should not get stripped. |
653 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] | 691 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] |
654 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 692 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
655 """} | 693 """} |
656 expectations = TestExpectations(test_port) | 694 expectations = TestExpectations(test_port) |
657 | 695 |
658 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 696 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
659 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 697 actual_expectations = expectations.remove_configurations( |
| 698 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
660 | 699 |
661 self.assertEqual(""" | 700 self.assertEqual(""" |
662 # This comment line should not get stripped. | 701 # This comment line should not get stripped. |
663 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 702 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
664 """, actual_expectations) | 703 """, actual_expectations) |
665 | 704 |
666 def test_remove_line_no_whitespace_lines(self): | 705 def test_remove_line_no_whitespace_lines(self): |
667 host = MockHost() | 706 host = MockHost() |
668 test_port = host.port_factory.get('test-win-xp', None) | 707 test_port = host.port_factory.get('test-win-xp', None) |
669 test_port.test_exists = lambda test: True | 708 test_port.test_exists = lambda test: True |
670 test_port.test_isfile = lambda test: True | 709 test_port.test_isfile = lambda test: True |
671 | 710 |
672 test_config = test_port.test_configuration() | 711 test_config = test_port.test_configuration() |
673 test_port.expectations_dict = lambda: {'expectations': """ | 712 test_port.expectations_dict = lambda: {'expectations': """ |
674 # This comment line should get stripped. | 713 # This comment line should get stripped. |
675 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] | 714 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] |
676 # This comment line should not get stripped. | 715 # This comment line should not get stripped. |
677 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 716 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
678 """} | 717 """} |
679 expectations = TestExpectations(test_port) | 718 expectations = TestExpectations(test_port) |
680 | 719 |
681 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 720 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
682 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 721 actual_expectations = expectations.remove_configurations( |
| 722 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
683 | 723 |
684 self.assertEqual(""" # This comment line should not get stripped. | 724 self.assertEqual(""" # This comment line should not get stripped. |
685 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 725 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
686 """, actual_expectations) | 726 """, actual_expectations) |
687 | 727 |
688 def test_remove_first_line(self): | 728 def test_remove_first_line(self): |
689 host = MockHost() | 729 host = MockHost() |
690 test_port = host.port_factory.get('test-win-xp', None) | 730 test_port = host.port_factory.get('test-win-xp', None) |
691 test_port.test_exists = lambda test: True | 731 test_port.test_exists = lambda test: True |
692 test_port.test_isfile = lambda test: True | 732 test_port.test_isfile = lambda test: True |
693 | 733 |
694 test_config = test_port.test_configuration() | 734 test_config = test_port.test_configuration() |
695 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win R
elease ] failures/expected/foo.html [ Failure ] | 735 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win R
elease ] failures/expected/foo.html [ Failure ] |
696 # This comment line should not get stripped. | 736 # This comment line should not get stripped. |
697 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 737 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
698 """} | 738 """} |
699 expectations = TestExpectations(test_port) | 739 expectations = TestExpectations(test_port) |
700 | 740 |
701 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 741 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
702 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 742 actual_expectations = expectations.remove_configurations( |
| 743 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
703 | 744 |
704 self.assertEqual(""" # This comment line should not get stripped. | 745 self.assertEqual(""" # This comment line should not get stripped. |
705 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 746 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
706 """, actual_expectations) | 747 """, actual_expectations) |
707 | 748 |
708 def test_remove_flaky_line(self): | 749 def test_remove_flaky_line(self): |
709 host = MockHost() | 750 host = MockHost() |
710 test_port = host.port_factory.get('test-win-xp', None) | 751 test_port = host.port_factory.get('test-win-xp', None) |
711 test_port.test_exists = lambda test: True | 752 test_port.test_exists = lambda test: True |
712 test_port.test_isfile = lambda test: True | 753 test_port.test_isfile = lambda test: True |
713 | 754 |
714 test_config = test_port.test_configuration() | 755 test_config = test_port.test_configuration() |
715 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win ]
failures/expected/foo.html [ Failure Timeout ] | 756 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win ]
failures/expected/foo.html [ Failure Timeout ] |
716 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ] | 757 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ] |
717 """} | 758 """} |
718 expectations = TestExpectations(test_port) | 759 expectations = TestExpectations(test_port) |
719 | 760 |
720 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 761 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
721 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 762 actual_expectations = expectations.remove_configurations( |
| 763 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
722 | 764 |
723 self.assertEqual("""Bug(x) [ Win Debug ] failures/expected/foo.html [ Fa
ilure Timeout ] | 765 self.assertEqual("""Bug(x) [ Win Debug ] failures/expected/foo.html [ Fa
ilure Timeout ] |
724 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ] | 766 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ] |
725 """, actual_expectations) | 767 """, actual_expectations) |
726 | 768 |
727 | 769 |
728 class RebaseliningTest(Base): | 770 class RebaseliningTest(Base): |
| 771 |
729 def test_get_rebaselining_failures(self): | 772 def test_get_rebaselining_failures(self): |
730 # Make sure we find a test as needing a rebaseline even if it is not mar
ked as a failure. | 773 # Make sure we find a test as needing a rebaseline even if it is not mar
ked as a failure. |
731 self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n') | 774 self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n') |
732 self.assertEqual(len(self._exp.get_rebaselining_failures()), 1) | 775 self.assertEqual(len(self._exp.get_rebaselining_failures()), 1) |
733 | 776 |
734 self.parse_exp(self.get_basic_expectations()) | 777 self.parse_exp(self.get_basic_expectations()) |
735 self.assertEqual(len(self._exp.get_rebaselining_failures()), 0) | 778 self.assertEqual(len(self._exp.get_rebaselining_failures()), 0) |
736 | 779 |
737 | 780 |
738 class TestExpectationsParserTests(unittest.TestCase): | 781 class TestExpectationsParserTests(unittest.TestCase): |
| 782 |
739 def __init__(self, testFunc): | 783 def __init__(self, testFunc): |
740 host = MockHost() | 784 host = MockHost() |
741 test_port = host.port_factory.get('test-win-xp', None) | 785 test_port = host.port_factory.get('test-win-xp', None) |
742 self._converter = TestConfigurationConverter(test_port.all_test_configur
ations(), test_port.configuration_specifier_macros()) | 786 self._converter = TestConfigurationConverter( |
| 787 test_port.all_test_configurations(), |
| 788 test_port.configuration_specifier_macros()) |
743 unittest.TestCase.__init__(self, testFunc) | 789 unittest.TestCase.__init__(self, testFunc) |
744 self._parser = TestExpectationParser(host.port_factory.get('test-win-xp'
, None), [], is_lint_mode=False) | 790 self._parser = TestExpectationParser(host.port_factory.get('test-win-xp'
, None), [], is_lint_mode=False) |
745 | 791 |
746 def test_expectation_line_for_test(self): | 792 def test_expectation_line_for_test(self): |
747 # This is kind of a silly test, but it at least ensures that we don't th
row an error. | 793 # This is kind of a silly test, but it at least ensures that we don't th
row an error. |
748 test_name = 'foo/test.html' | 794 test_name = 'foo/test.html' |
749 expectations = set(["PASS", "IMAGE"]) | 795 expectations = set(['PASS', 'IMAGE']) |
750 | 796 |
751 expectation_line = TestExpectationLine() | 797 expectation_line = TestExpectationLine() |
752 expectation_line.original_string = test_name | 798 expectation_line.original_string = test_name |
753 expectation_line.name = test_name | 799 expectation_line.name = test_name |
754 expectation_line.filename = '<Bot TestExpectations>' | 800 expectation_line.filename = '<Bot TestExpectations>' |
755 expectation_line.line_numbers = '0' | 801 expectation_line.line_numbers = '0' |
756 expectation_line.expectations = expectations | 802 expectation_line.expectations = expectations |
757 self._parser._parse_line(expectation_line) | 803 self._parser._parse_line(expectation_line) |
758 | 804 |
759 self.assertEqual(self._parser.expectation_line_for_test(test_name, expec
tations), expectation_line) | 805 self.assertEqual(self._parser.expectation_line_for_test(test_name, expec
tations), expectation_line) |
760 | 806 |
761 | 807 |
762 class TestExpectationSerializationTests(unittest.TestCase): | 808 class TestExpectationSerializationTests(unittest.TestCase): |
| 809 |
763 def __init__(self, testFunc): | 810 def __init__(self, testFunc): |
764 host = MockHost() | 811 host = MockHost() |
765 test_port = host.port_factory.get('test-win-xp', None) | 812 test_port = host.port_factory.get('test-win-xp', None) |
766 self._converter = TestConfigurationConverter(test_port.all_test_configur
ations(), test_port.configuration_specifier_macros()) | 813 self._converter = TestConfigurationConverter( |
| 814 test_port.all_test_configurations(), |
| 815 test_port.configuration_specifier_macros()) |
767 unittest.TestCase.__init__(self, testFunc) | 816 unittest.TestCase.__init__(self, testFunc) |
768 | 817 |
769 def _tokenize(self, line): | 818 def _tokenize(self, line): |
770 return TestExpectationParser._tokenize_line('path', line, 0) | 819 return TestExpectationParser._tokenize_line('path', line, 0) |
771 | 820 |
772 def assert_round_trip(self, in_string, expected_string=None): | 821 def assert_round_trip(self, in_string, expected_string=None): |
773 expectation = self._tokenize(in_string) | 822 expectation = self._tokenize(in_string) |
774 if expected_string is None: | 823 if expected_string is None: |
775 expected_string = in_string | 824 expected_string = in_string |
776 self.assertEqual(expected_string, expectation.to_string(self._converter)
) | 825 self.assertEqual(expected_string, expectation.to_string(self._converter)
) |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
814 # FIXME: case should be preserved here but we can't until we drop the ol
d syntax. | 863 # FIXME: case should be preserved here but we can't until we drop the ol
d syntax. |
815 self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO
] bar [ BAZ1 BAZ2 ] #Qux.') | 864 self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO
] bar [ BAZ1 BAZ2 ] #Qux.') |
816 | 865 |
817 def test_parsed_to_string(self): | 866 def test_parsed_to_string(self): |
818 expectation_line = TestExpectationLine() | 867 expectation_line = TestExpectationLine() |
819 expectation_line.bugs = ['Bug(x)'] | 868 expectation_line.bugs = ['Bug(x)'] |
820 expectation_line.name = 'test/name/for/realz.html' | 869 expectation_line.name = 'test/name/for/realz.html' |
821 expectation_line.parsed_expectations = set([IMAGE]) | 870 expectation_line.parsed_expectations = set([IMAGE]) |
822 self.assertEqual(expectation_line.to_string(self._converter), None) | 871 self.assertEqual(expectation_line.to_string(self._converter), None) |
823 expectation_line.matching_configurations = set([TestConfiguration('xp',
'x86', 'release')]) | 872 expectation_line.matching_configurations = set([TestConfiguration('xp',
'x86', 'release')]) |
824 self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [
XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]') | 873 self.assertEqual( |
825 expectation_line.matching_configurations = set([TestConfiguration('xp',
'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]) | 874 expectation_line.to_string( |
| 875 self._converter), |
| 876 'Bug(x) [ XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]
') |
| 877 expectation_line.matching_configurations = set( |
| 878 [TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp',
'x86', 'debug')]) |
826 self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [
XP ] test/name/for/realz.html [ ImageOnlyFailure ]') | 879 self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [
XP ] test/name/for/realz.html [ ImageOnlyFailure ]') |
827 | 880 |
828 def test_serialize_parsed_expectations(self): | 881 def test_serialize_parsed_expectations(self): |
829 expectation_line = TestExpectationLine() | 882 expectation_line = TestExpectationLine() |
830 expectation_line.parsed_expectations = set([]) | 883 expectation_line.parsed_expectations = set([]) |
831 parsed_expectation_to_string = dict([[parsed_expectation, expectation_st
ring] for expectation_string, parsed_expectation in TestExpectations.EXPECTATION
S.items()]) | 884 parsed_expectation_to_string = dict([[parsed_expectation, expectation_st
ring] |
| 885 for expectation_string, parsed_expe
ctation in TestExpectations.EXPECTATIONS.items()]) |
832 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), '') | 886 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), '') |
833 expectation_line.parsed_expectations = set([FAIL]) | 887 expectation_line.parsed_expectations = set([FAIL]) |
834 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'fail') | 888 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'fail') |
835 expectation_line.parsed_expectations = set([PASS, IMAGE]) | 889 expectation_line.parsed_expectations = set([PASS, IMAGE]) |
836 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'image pass') | 890 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'image pass') |
837 expectation_line.parsed_expectations = set([FAIL, PASS]) | 891 expectation_line.parsed_expectations = set([FAIL, PASS]) |
838 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'pass fail') | 892 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'pass fail') |
839 | 893 |
840 def test_serialize_parsed_specifier_string(self): | 894 def test_serialize_parsed_specifier_string(self): |
841 expectation_line = TestExpectationLine() | 895 expectation_line = TestExpectationLine() |
842 expectation_line.bugs = ['garden-o-matic'] | 896 expectation_line.bugs = ['garden-o-matic'] |
843 expectation_line.parsed_specifiers = ['the', 'for'] | 897 expectation_line.parsed_specifiers = ['the', 'for'] |
844 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, []), 'for the') | 898 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, []), 'for the') |
845 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, ['win']), 'for the win') | 899 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, ['win']), 'for the win') |
846 expectation_line.bugs = [] | 900 expectation_line.bugs = [] |
847 expectation_line.parsed_specifiers = [] | 901 expectation_line.parsed_specifiers = [] |
848 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, []), '') | 902 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, []), '') |
849 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, ['win']), 'win') | 903 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, ['win']), 'win') |
850 | 904 |
851 def test_format_line(self): | 905 def test_format_line(self): |
852 self.assertEqual(TestExpectationLine._format_line([], ['MODIFIERS'], 'na
me', ['EXPECTATIONS'], 'comment'), '[ MODIFIERS ] name [ EXPECTATIONS ] #comment
') | 906 self.assertEqual( |
853 self.assertEqual(TestExpectationLine._format_line([], ['MODIFIERS'], 'na
me', ['EXPECTATIONS'], None), '[ MODIFIERS ] name [ EXPECTATIONS ]') | 907 TestExpectationLine._format_line( |
| 908 [], |
| 909 ['MODIFIERS'], |
| 910 'name', |
| 911 ['EXPECTATIONS'], |
| 912 'comment'), |
| 913 '[ MODIFIERS ] name [ EXPECTATIONS ] #comment') |
| 914 self.assertEqual( |
| 915 TestExpectationLine._format_line( |
| 916 [], |
| 917 ['MODIFIERS'], |
| 918 'name', |
| 919 ['EXPECTATIONS'], |
| 920 None), |
| 921 '[ MODIFIERS ] name [ EXPECTATIONS ]') |
854 | 922 |
855 def test_string_roundtrip(self): | 923 def test_string_roundtrip(self): |
856 self.assert_round_trip('') | 924 self.assert_round_trip('') |
857 self.assert_round_trip('[') | 925 self.assert_round_trip('[') |
858 self.assert_round_trip('FOO [') | 926 self.assert_round_trip('FOO [') |
859 self.assert_round_trip('FOO ] bar') | 927 self.assert_round_trip('FOO ] bar') |
860 self.assert_round_trip(' FOO [') | 928 self.assert_round_trip(' FOO [') |
861 self.assert_round_trip(' [ FOO ] ') | 929 self.assert_round_trip(' [ FOO ] ') |
862 self.assert_round_trip('[ FOO ] bar [ BAZ ]') | 930 self.assert_round_trip('[ FOO ] bar [ BAZ ]') |
863 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.') | 931 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.') |
(...skipping 18 matching lines...) Expand all Loading... |
882 self.assert_list_round_trip('bar') | 950 self.assert_list_round_trip('bar') |
883 self.assert_list_round_trip('bar\n# Qux.') | 951 self.assert_list_round_trip('bar\n# Qux.') |
884 self.assert_list_round_trip('bar\n# Qux.\n') | 952 self.assert_list_round_trip('bar\n# Qux.\n') |
885 | 953 |
886 def test_reconstitute_only_these(self): | 954 def test_reconstitute_only_these(self): |
887 lines = [] | 955 lines = [] |
888 reconstitute_only_these = [] | 956 reconstitute_only_these = [] |
889 | 957 |
890 def add_line(matching_configurations, reconstitute): | 958 def add_line(matching_configurations, reconstitute): |
891 expectation_line = TestExpectationLine() | 959 expectation_line = TestExpectationLine() |
892 expectation_line.original_string = "Nay" | 960 expectation_line.original_string = 'Nay' |
893 expectation_line.bugs = ['Bug(x)'] | 961 expectation_line.bugs = ['Bug(x)'] |
894 expectation_line.name = 'Yay' | 962 expectation_line.name = 'Yay' |
895 expectation_line.parsed_expectations = set([IMAGE]) | 963 expectation_line.parsed_expectations = set([IMAGE]) |
896 expectation_line.matching_configurations = matching_configurations | 964 expectation_line.matching_configurations = matching_configurations |
897 lines.append(expectation_line) | 965 lines.append(expectation_line) |
898 if reconstitute: | 966 if reconstitute: |
899 reconstitute_only_these.append(expectation_line) | 967 reconstitute_only_these.append(expectation_line) |
900 | 968 |
901 add_line(set([TestConfiguration('xp', 'x86', 'release')]), True) | 969 add_line(set([TestConfiguration('xp', 'x86', 'release')]), True) |
902 add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfigurati
on('xp', 'x86', 'debug')]), False) | 970 add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfigurati
on('xp', 'x86', 'debug')]), False) |
903 serialized = TestExpectations.list_to_string(lines, self._converter) | 971 serialized = TestExpectations.list_to_string(lines, self._converter) |
904 self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailu
re ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]") | 972 self.assertEqual(serialized, 'Bug(x) [ XP Release ] Yay [ ImageOnlyFailu
re ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]') |
905 serialized = TestExpectations.list_to_string(lines, self._converter, rec
onstitute_only_these=reconstitute_only_these) | 973 serialized = TestExpectations.list_to_string(lines, self._converter, rec
onstitute_only_these=reconstitute_only_these) |
906 self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailu
re ]\nNay") | 974 self.assertEqual(serialized, 'Bug(x) [ XP Release ] Yay [ ImageOnlyFailu
re ]\nNay') |
907 | 975 |
908 def disabled_test_string_whitespace_stripping(self): | 976 def disabled_test_string_whitespace_stripping(self): |
909 # FIXME: Re-enable this test once we rework the code to no longer suppor
t the old syntax. | 977 # FIXME: Re-enable this test once we rework the code to no longer suppor
t the old syntax. |
910 self.assert_round_trip('\n', '') | 978 self.assert_round_trip('\n', '') |
911 self.assert_round_trip(' [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]') | 979 self.assert_round_trip(' [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]') |
912 self.assert_round_trip('[ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]') | 980 self.assert_round_trip('[ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]') |
913 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] bar
[ BAZ ] # Qux.') | 981 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] bar
[ BAZ ] # Qux.') |
914 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] ba
r [ BAZ ] # Qux.') | 982 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] ba
r [ BAZ ] # Qux.') |
915 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ]
bar [ BAZ ] # Qux.') | 983 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ]
bar [ BAZ ] # Qux.') |
916 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FO
O ] bar [ BAZ ] # Qux.') | 984 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FO
O ] bar [ BAZ ] # Qux.') |
OLD | NEW |