| OLD | NEW |
| 1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 59 return [self.get_test('failures/expected/text.html'), | 59 return [self.get_test('failures/expected/text.html'), |
| 60 self.get_test('failures/expected/image_checksum.html'), | 60 self.get_test('failures/expected/image_checksum.html'), |
| 61 self.get_test('failures/expected/crash.html'), | 61 self.get_test('failures/expected/crash.html'), |
| 62 self.get_test('failures/expected/needsrebaseline.html'), | 62 self.get_test('failures/expected/needsrebaseline.html'), |
| 63 self.get_test('failures/expected/needsmanualrebaseline.html'), | 63 self.get_test('failures/expected/needsmanualrebaseline.html'), |
| 64 self.get_test('failures/expected/missing_text.html'), | 64 self.get_test('failures/expected/missing_text.html'), |
| 65 self.get_test('failures/expected/image.html'), | 65 self.get_test('failures/expected/image.html'), |
| 66 self.get_test('failures/expected/timeout.html'), | 66 self.get_test('failures/expected/timeout.html'), |
| 67 self.get_test('passes/text.html')] | 67 self.get_test('passes/text.html')] |
| 68 | 68 |
| 69 | |
| 70 def get_basic_expectations(self): | 69 def get_basic_expectations(self): |
| 71 return """ | 70 return """ |
| 72 Bug(test) failures/expected/text.html [ Failure ] | 71 Bug(test) failures/expected/text.html [ Failure ] |
| 73 Bug(test) failures/expected/crash.html [ WontFix ] | 72 Bug(test) failures/expected/crash.html [ WontFix ] |
| 74 Bug(test) failures/expected/needsrebaseline.html [ NeedsRebaseline ] | 73 Bug(test) failures/expected/needsrebaseline.html [ NeedsRebaseline ] |
| 75 Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ] | 74 Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ] |
| 76 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ] | 75 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ] |
| 77 Bug(test) failures/expected/image_checksum.html [ WontFix ] | 76 Bug(test) failures/expected/image_checksum.html [ WontFix ] |
| 78 Bug(test) failures/expected/image.html [ WontFix Mac ] | 77 Bug(test) failures/expected/image.html [ WontFix Mac ] |
| 79 """ | 78 """ |
| 80 | 79 |
| 81 def parse_exp(self, expectations, overrides=None, is_lint_mode=False): | 80 def parse_exp(self, expectations, overrides=None, is_lint_mode=False): |
| 82 expectations_dict = OrderedDict() | 81 expectations_dict = OrderedDict() |
| 83 expectations_dict['expectations'] = expectations | 82 expectations_dict['expectations'] = expectations |
| 84 if overrides: | 83 if overrides: |
| 85 expectations_dict['overrides'] = overrides | 84 expectations_dict['overrides'] = overrides |
| 86 self._port.expectations_dict = lambda: expectations_dict | 85 self._port.expectations_dict = lambda: expectations_dict |
| 87 expectations_to_lint = expectations_dict if is_lint_mode else None | 86 expectations_to_lint = expectations_dict if is_lint_mode else None |
| 88 self._exp = TestExpectations(self._port, self.get_basic_tests(), expecta
tions_dict=expectations_to_lint, is_lint_mode=is_lint_mode) | 87 self._exp = TestExpectations( |
| 88 self._port, |
| 89 self.get_basic_tests(), |
| 90 expectations_dict=expectations_to_lint, |
| 91 is_lint_mode=is_lint_mode) |
| 89 | 92 |
| 90 def assert_exp_list(self, test, results): | 93 def assert_exp_list(self, test, results): |
| 91 self.assertEqual(self._exp.get_expectations(self.get_test(test)), set(re
sults)) | 94 self.assertEqual(self._exp.get_expectations(self.get_test(test)), set(re
sults)) |
| 92 | 95 |
| 93 def assert_exp(self, test, result): | 96 def assert_exp(self, test, result): |
| 94 self.assert_exp_list(test, [result]) | 97 self.assert_exp_list(test, [result]) |
| 95 | 98 |
| 96 def assert_bad_expectations(self, expectations, overrides=None): | 99 def assert_bad_expectations(self, expectations, overrides=None): |
| 97 self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode
=True, overrides=overrides) | 100 self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode
=True, overrides=overrides) |
| 98 | 101 |
| 99 | 102 |
| 100 class BasicTests(Base): | 103 class BasicTests(Base): |
| 104 |
| 101 def test_basic(self): | 105 def test_basic(self): |
| 102 self.parse_exp(self.get_basic_expectations()) | 106 self.parse_exp(self.get_basic_expectations()) |
| 103 self.assert_exp('failures/expected/text.html', FAIL) | 107 self.assert_exp('failures/expected/text.html', FAIL) |
| 104 self.assert_exp_list('failures/expected/image_checksum.html', [WONTFIX,
SKIP]) | 108 self.assert_exp_list('failures/expected/image_checksum.html', [WONTFIX,
SKIP]) |
| 105 self.assert_exp('passes/text.html', PASS) | 109 self.assert_exp('passes/text.html', PASS) |
| 106 self.assert_exp('failures/expected/image.html', PASS) | 110 self.assert_exp('failures/expected/image.html', PASS) |
| 107 | 111 |
| 108 | 112 |
| 109 class MiscTests(Base): | 113 class MiscTests(Base): |
| 114 |
| 110 def test_multiple_results(self): | 115 def test_multiple_results(self): |
| 111 self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]') | 116 self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]') |
| 112 self.assertEqual(self._exp.get_expectations( | 117 self.assertEqual(self._exp.get_expectations( |
| 113 self.get_test('failures/expected/text.html')), | 118 self.get_test('failures/expected/text.html')), |
| 114 set([FAIL, CRASH])) | 119 set([FAIL, CRASH])) |
| 115 | 120 |
| 116 def test_result_was_expected(self): | 121 def test_result_was_expected(self): |
| 117 # test basics | 122 # test basics |
| 118 self.assertEqual(TestExpectations.result_was_expected(PASS, set([PASS]),
test_needs_rebaselining=False), True) | 123 self.assertEqual(TestExpectations.result_was_expected(PASS, set([PASS]),
test_needs_rebaselining=False), True) |
| 119 self.assertEqual(TestExpectations.result_was_expected(FAIL, set([PASS]),
test_needs_rebaselining=False), False) | 124 self.assertEqual(TestExpectations.result_was_expected(FAIL, set([PASS]),
test_needs_rebaselining=False), False) |
| 120 | 125 |
| 121 # test handling of SKIPped tests and results | 126 # test handling of SKIPped tests and results |
| 122 self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH])
, test_needs_rebaselining=False), True) | 127 self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH])
, test_needs_rebaselining=False), True) |
| 123 self.assertEqual(TestExpectations.result_was_expected(SKIP, set([LEAK]),
test_needs_rebaselining=False), True) | 128 self.assertEqual(TestExpectations.result_was_expected(SKIP, set([LEAK]),
test_needs_rebaselining=False), True) |
| 124 | 129 |
| 125 # test handling of MISSING results and the REBASELINE specifier | 130 # test handling of MISSING results and the REBASELINE specifier |
| 126 self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS
]), test_needs_rebaselining=True), True) | 131 self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS
]), test_needs_rebaselining=True), True) |
| 127 self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS
]), test_needs_rebaselining=False), False) | 132 self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS
]), test_needs_rebaselining=False), False) |
| 128 | 133 |
| 129 self.assertTrue(TestExpectations.result_was_expected(PASS, set([NEEDS_RE
BASELINE]), test_needs_rebaselining=False)) | 134 self.assertTrue(TestExpectations.result_was_expected(PASS, set([NEEDS_RE
BASELINE]), test_needs_rebaselining=False)) |
| 130 self.assertTrue(TestExpectations.result_was_expected(MISSING, set([NEEDS
_REBASELINE]), test_needs_rebaselining=False)) | 135 self.assertTrue(TestExpectations.result_was_expected(MISSING, set([NEEDS
_REBASELINE]), test_needs_rebaselining=False)) |
| 131 self.assertTrue(TestExpectations.result_was_expected(TEXT, set([NEEDS_RE
BASELINE]), test_needs_rebaselining=False)) | 136 self.assertTrue(TestExpectations.result_was_expected(TEXT, set([NEEDS_RE
BASELINE]), test_needs_rebaselining=False)) |
| 132 self.assertTrue(TestExpectations.result_was_expected(IMAGE, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) | 137 self.assertTrue(TestExpectations.result_was_expected(IMAGE, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) |
| 133 self.assertTrue(TestExpectations.result_was_expected(IMAGE_PLUS_TEXT, se
t([NEEDS_REBASELINE]), test_needs_rebaselining=False)) | 138 self.assertTrue( |
| 139 TestExpectations.result_was_expected( |
| 140 IMAGE_PLUS_TEXT, set( |
| 141 [NEEDS_REBASELINE]), test_needs_rebaselining=False)) |
| 134 self.assertTrue(TestExpectations.result_was_expected(AUDIO, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) | 142 self.assertTrue(TestExpectations.result_was_expected(AUDIO, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) |
| 135 self.assertFalse(TestExpectations.result_was_expected(TIMEOUT, set([NEED
S_REBASELINE]), test_needs_rebaselining=False)) | 143 self.assertFalse(TestExpectations.result_was_expected(TIMEOUT, set([NEED
S_REBASELINE]), test_needs_rebaselining=False)) |
| 136 self.assertFalse(TestExpectations.result_was_expected(CRASH, set([NEEDS_
REBASELINE]), test_needs_rebaselining=False)) | 144 self.assertFalse(TestExpectations.result_was_expected(CRASH, set([NEEDS_
REBASELINE]), test_needs_rebaselining=False)) |
| 137 self.assertFalse(TestExpectations.result_was_expected(LEAK, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) | 145 self.assertFalse(TestExpectations.result_was_expected(LEAK, set([NEEDS_R
EBASELINE]), test_needs_rebaselining=False)) |
| 138 | 146 |
| 139 def test_remove_pixel_failures(self): | 147 def test_remove_pixel_failures(self): |
| 140 self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), se
t([FAIL])) | 148 self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), se
t([FAIL])) |
| 141 self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS])), se
t([PASS])) | 149 self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS])), se
t([PASS])) |
| 142 self.assertEqual(TestExpectations.remove_pixel_failures(set([IMAGE])), s
et([PASS])) | 150 self.assertEqual(TestExpectations.remove_pixel_failures(set([IMAGE])), s
et([PASS])) |
| 143 self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), se
t([FAIL])) | 151 self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), se
t([FAIL])) |
| (...skipping 13 matching lines...) Expand all Loading... |
| 157 self.parse_exp(exp_str) | 165 self.parse_exp(exp_str) |
| 158 test_name = 'failures/expected/unknown-test.html' | 166 test_name = 'failures/expected/unknown-test.html' |
| 159 unknown_test = self.get_test(test_name) | 167 unknown_test = self.get_test(test_name) |
| 160 self.assertRaises(KeyError, self._exp.get_expectations, | 168 self.assertRaises(KeyError, self._exp.get_expectations, |
| 161 unknown_test) | 169 unknown_test) |
| 162 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) | 170 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) |
| 163 | 171 |
| 164 def test_get_expectations_string(self): | 172 def test_get_expectations_string(self): |
| 165 self.parse_exp(self.get_basic_expectations()) | 173 self.parse_exp(self.get_basic_expectations()) |
| 166 self.assertEqual(self._exp.get_expectations_string( | 174 self.assertEqual(self._exp.get_expectations_string( |
| 167 self.get_test('failures/expected/text.html')), | 175 self.get_test('failures/expected/text.html')), |
| 168 'FAIL') | 176 'FAIL') |
| 169 | 177 |
| 170 def test_expectation_to_string(self): | 178 def test_expectation_to_string(self): |
| 171 # Normal cases are handled by other tests. | 179 # Normal cases are handled by other tests. |
| 172 self.parse_exp(self.get_basic_expectations()) | 180 self.parse_exp(self.get_basic_expectations()) |
| 173 self.assertRaises(ValueError, self._exp.expectation_to_string, | 181 self.assertRaises(ValueError, self._exp.expectation_to_string, |
| 174 -1) | 182 -1) |
| 175 | 183 |
| 176 def test_get_test_set(self): | 184 def test_get_test_set(self): |
| 177 # Handle some corner cases for this routine not covered by other tests. | 185 # Handle some corner cases for this routine not covered by other tests. |
| 178 self.parse_exp(self.get_basic_expectations()) | 186 self.parse_exp(self.get_basic_expectations()) |
| 179 s = self._exp.get_test_set(WONTFIX) | 187 s = self._exp.get_test_set(WONTFIX) |
| 180 self.assertEqual(s, | 188 self.assertEqual(s, |
| 181 set([self.get_test('failures/expected/crash.html'), | 189 set([self.get_test('failures/expected/crash.html'), |
| 182 self.get_test('failures/expected/image_checksum.html')])) | 190 self.get_test('failures/expected/image_checksum.ht
ml')])) |
| 183 | 191 |
| 184 def test_needs_rebaseline_reftest(self): | 192 def test_needs_rebaseline_reftest(self): |
| 185 try: | 193 try: |
| 186 filesystem = self._port.host.filesystem | 194 filesystem = self._port.host.filesystem |
| 187 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'failures/expected/needsrebaseline.html'), 'content') | 195 filesystem.write_text_file( |
| 188 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'failures/expected/needsrebaseline-expected.html'), 'content') | 196 filesystem.join( |
| 189 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'failures/expected/needsmanualrebaseline.html'), 'content') | 197 self._port.layout_tests_dir(), |
| 190 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'failures/expected/needsmanualrebaseline-expected.html'), 'content') | 198 'failures/expected/needsrebaseline.html'), |
| 199 'content') |
| 200 filesystem.write_text_file( |
| 201 filesystem.join( |
| 202 self._port.layout_tests_dir(), |
| 203 'failures/expected/needsrebaseline-expected.html'), |
| 204 'content') |
| 205 filesystem.write_text_file( |
| 206 filesystem.join( |
| 207 self._port.layout_tests_dir(), |
| 208 'failures/expected/needsmanualrebaseline.html'), |
| 209 'content') |
| 210 filesystem.write_text_file( |
| 211 filesystem.join( |
| 212 self._port.layout_tests_dir(), |
| 213 'failures/expected/needsmanualrebaseline-expected.html'), |
| 214 'content') |
| 191 self.parse_exp("""Bug(user) failures/expected/needsrebaseline.html [
NeedsRebaseline ] | 215 self.parse_exp("""Bug(user) failures/expected/needsrebaseline.html [
NeedsRebaseline ] |
| 192 Bug(user) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]
""", is_lint_mode=True) | 216 Bug(user) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]
""", is_lint_mode=True) |
| 193 self.assertFalse(True, "ParseError wasn't raised") | 217 self.assertFalse(True, "ParseError wasn't raised") |
| 194 except ParseError, e: | 218 except ParseError as e: |
| 195 warnings = """expectations:1 A reftest cannot be marked as NeedsReba
seline/NeedsManualRebaseline failures/expected/needsrebaseline.html | 219 warnings = """expectations:1 A reftest cannot be marked as NeedsReba
seline/NeedsManualRebaseline failures/expected/needsrebaseline.html |
| 196 expectations:2 A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseli
ne failures/expected/needsmanualrebaseline.html""" | 220 expectations:2 A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseli
ne failures/expected/needsmanualrebaseline.html""" |
| 197 self.assertEqual(str(e), warnings) | 221 self.assertEqual(str(e), warnings) |
| 198 | 222 |
| 199 def test_parse_warning(self): | 223 def test_parse_warning(self): |
| 200 try: | 224 try: |
| 201 filesystem = self._port.host.filesystem | 225 filesystem = self._port.host.filesystem |
| 202 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'disabled-test.html-disabled'), 'content') | 226 filesystem.write_text_file(filesystem.join(self._port.layout_tests_d
ir(), 'disabled-test.html-disabled'), 'content') |
| 203 self.get_test('disabled-test.html-disabled'), | 227 self.get_test('disabled-test.html-disabled'), |
| 204 self.parse_exp("Bug(user) [ FOO ] failures/expected/text.html [ Fail
ure ]\n" | 228 self.parse_exp('Bug(user) [ FOO ] failures/expected/text.html [ Fail
ure ]\n' |
| 205 "Bug(user) non-existent-test.html [ Failure ]\n" | 229 'Bug(user) non-existent-test.html [ Failure ]\n' |
| 206 "Bug(user) disabled-test.html-disabled [ ImageOnlyFailure ]", is
_lint_mode=True) | 230 'Bug(user) disabled-test.html-disabled [ ImageOnlyFai
lure ]', is_lint_mode=True) |
| 207 self.assertFalse(True, "ParseError wasn't raised") | 231 self.assertFalse(True, "ParseError wasn't raised") |
| 208 except ParseError, e: | 232 except ParseError as e: |
| 209 warnings = ("expectations:1 Unrecognized specifier 'foo' failures/ex
pected/text.html\n" | 233 warnings = ("expectations:1 Unrecognized specifier 'foo' failures/ex
pected/text.html\n" |
| 210 "expectations:2 Path does not exist. non-existent-test.h
tml") | 234 'expectations:2 Path does not exist. non-existent-test.h
tml') |
| 211 self.assertEqual(str(e), warnings) | 235 self.assertEqual(str(e), warnings) |
| 212 | 236 |
| 213 def test_parse_warnings_are_logged_if_not_in_lint_mode(self): | 237 def test_parse_warnings_are_logged_if_not_in_lint_mode(self): |
| 214 oc = OutputCapture() | 238 oc = OutputCapture() |
| 215 try: | 239 try: |
| 216 oc.capture_output() | 240 oc.capture_output() |
| 217 self.parse_exp('-- this should be a syntax error', is_lint_mode=Fals
e) | 241 self.parse_exp('-- this should be a syntax error', is_lint_mode=Fals
e) |
| 218 finally: | 242 finally: |
| 219 _, _, logs = oc.restore_output() | 243 _, _, logs = oc.restore_output() |
| 220 self.assertNotEquals(logs, '') | 244 self.assertNotEquals(logs, '') |
| 221 | 245 |
| 222 def test_error_on_different_platform(self): | 246 def test_error_on_different_platform(self): |
| 223 # parse_exp uses a Windows port. Assert errors on Mac show up in lint mo
de. | 247 # parse_exp uses a Windows port. Assert errors on Mac show up in lint mo
de. |
| 224 self.assertRaises(ParseError, self.parse_exp, | 248 self.assertRaises(ParseError, self.parse_exp, |
| 225 'Bug(test) [ Mac ] failures/expected/text.html [ Failure ]\nBug(test
) [ Mac ] failures/expected/text.html [ Failure ]', | 249 'Bug(test) [ Mac ] failures/expected/text.html [ Failu
re ]\nBug(test) [ Mac ] failures/expected/text.html [ Failure ]', |
| 226 is_lint_mode=True) | 250 is_lint_mode=True) |
| 227 | 251 |
| 228 def test_error_on_different_build_type(self): | 252 def test_error_on_different_build_type(self): |
| 229 # parse_exp uses a Release port. Assert errors on DEBUG show up in lint
mode. | 253 # parse_exp uses a Release port. Assert errors on DEBUG show up in lint
mode. |
| 230 self.assertRaises(ParseError, self.parse_exp, | 254 self.assertRaises(ParseError, self.parse_exp, |
| 231 'Bug(test) [ Debug ] failures/expected/text.html [ Failure ]\nBug(te
st) [ Debug ] failures/expected/text.html [ Failure ]', | 255 'Bug(test) [ Debug ] failures/expected/text.html [ Fai
lure ]\nBug(test) [ Debug ] failures/expected/text.html [ Failure ]', |
| 232 is_lint_mode=True) | 256 is_lint_mode=True) |
| 233 | 257 |
| 234 def test_overrides(self): | 258 def test_overrides(self): |
| 235 self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]", | 259 self.parse_exp('Bug(exp) failures/expected/text.html [ Failure ]', |
| 236 "Bug(override) failures/expected/text.html [ ImageOnlyFai
lure ]") | 260 'Bug(override) failures/expected/text.html [ ImageOnlyFai
lure ]') |
| 237 self.assert_exp_list('failures/expected/text.html', [FAIL, IMAGE]) | 261 self.assert_exp_list('failures/expected/text.html', [FAIL, IMAGE]) |
| 238 | 262 |
| 239 def test_overrides__directory(self): | 263 def test_overrides__directory(self): |
| 240 self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]", | 264 self.parse_exp('Bug(exp) failures/expected/text.html [ Failure ]', |
| 241 "Bug(override) failures/expected [ Crash ]") | 265 'Bug(override) failures/expected [ Crash ]') |
| 242 self.assert_exp_list('failures/expected/text.html', [FAIL, CRASH]) | 266 self.assert_exp_list('failures/expected/text.html', [FAIL, CRASH]) |
| 243 self.assert_exp_list('failures/expected/image.html', [CRASH]) | 267 self.assert_exp_list('failures/expected/image.html', [CRASH]) |
| 244 | 268 |
| 245 def test_overrides__duplicate(self): | 269 def test_overrides__duplicate(self): |
| 246 self.assert_bad_expectations("Bug(exp) failures/expected/text.html [ Fai
lure ]", | 270 self.assert_bad_expectations('Bug(exp) failures/expected/text.html [ Fai
lure ]', |
| 247 "Bug(override) failures/expected/text.html
[ ImageOnlyFailure ]\n" | 271 'Bug(override) failures/expected/text.html
[ ImageOnlyFailure ]\n' |
| 248 "Bug(override) failures/expected/text.html
[ Crash ]\n") | 272 'Bug(override) failures/expected/text.html
[ Crash ]\n') |
| 249 | 273 |
| 250 def test_pixel_tests_flag(self): | 274 def test_pixel_tests_flag(self): |
| 251 def match(test, result, pixel_tests_enabled): | 275 def match(test, result, pixel_tests_enabled): |
| 252 return self._exp.matches_an_expected_result( | 276 return self._exp.matches_an_expected_result( |
| 253 self.get_test(test), result, pixel_tests_enabled, sanitizer_is_e
nabled=False) | 277 self.get_test(test), result, pixel_tests_enabled, sanitizer_is_e
nabled=False) |
| 254 | 278 |
| 255 self.parse_exp(self.get_basic_expectations()) | 279 self.parse_exp(self.get_basic_expectations()) |
| 256 self.assertTrue(match('failures/expected/text.html', FAIL, True)) | 280 self.assertTrue(match('failures/expected/text.html', FAIL, True)) |
| 257 self.assertTrue(match('failures/expected/text.html', FAIL, False)) | 281 self.assertTrue(match('failures/expected/text.html', FAIL, False)) |
| 258 self.assertFalse(match('failures/expected/text.html', CRASH, True)) | 282 self.assertFalse(match('failures/expected/text.html', CRASH, True)) |
| (...skipping 17 matching lines...) Expand all Loading... |
| 276 Bug(test) failures/expected/image.html [ ImageOnlyFailure ] | 300 Bug(test) failures/expected/image.html [ ImageOnlyFailure ] |
| 277 Bug(test) failures/expected/text.html [ Failure ] | 301 Bug(test) failures/expected/text.html [ Failure ] |
| 278 Bug(test) failures/expected/timeout.html [ Timeout ] | 302 Bug(test) failures/expected/timeout.html [ Timeout ] |
| 279 """) | 303 """) |
| 280 self.assertTrue(match('failures/expected/crash.html', CRASH)) | 304 self.assertTrue(match('failures/expected/crash.html', CRASH)) |
| 281 self.assertTrue(match('failures/expected/image.html', PASS)) | 305 self.assertTrue(match('failures/expected/image.html', PASS)) |
| 282 self.assertTrue(match('failures/expected/text.html', PASS)) | 306 self.assertTrue(match('failures/expected/text.html', PASS)) |
| 283 self.assertTrue(match('failures/expected/timeout.html', TIMEOUT)) | 307 self.assertTrue(match('failures/expected/timeout.html', TIMEOUT)) |
| 284 | 308 |
| 285 def test_more_specific_override_resets_skip(self): | 309 def test_more_specific_override_resets_skip(self): |
| 286 self.parse_exp("Bug(x) failures/expected [ Skip ]\n" | 310 self.parse_exp('Bug(x) failures/expected [ Skip ]\n' |
| 287 "Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\
n") | 311 'Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\
n') |
| 288 self.assert_exp('failures/expected/text.html', IMAGE) | 312 self.assert_exp('failures/expected/text.html', IMAGE) |
| 289 self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir
(), | 313 self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir
(), |
| 290 'failures/expected/text.htm
l') in | 314 'failures/expected/text.htm
l') in |
| 291 self._exp.get_tests_with_result_type(SKIP)) | 315 self._exp.get_tests_with_result_type(SKIP)) |
| 292 | 316 |
| 293 def test_bot_test_expectations(self): | 317 def test_bot_test_expectations(self): |
| 294 """Test that expectations are merged rather than overridden when using f
laky option 'unexpected'.""" | 318 """Test that expectations are merged rather than overridden when using f
laky option 'unexpected'.""" |
| 295 test_name1 = 'failures/expected/text.html' | 319 test_name1 = 'failures/expected/text.html' |
| 296 test_name2 = 'passes/text.html' | 320 test_name2 = 'passes/text.html' |
| 297 | 321 |
| 298 expectations_dict = OrderedDict() | 322 expectations_dict = OrderedDict() |
| 299 expectations_dict['expectations'] = "Bug(x) %s [ ImageOnlyFailure ]\nBug
(x) %s [ Slow ]\n" % (test_name1, test_name2) | 323 expectations_dict['expectations'] = 'Bug(x) %s [ ImageOnlyFailure ]\nBug
(x) %s [ Slow ]\n' % (test_name1, test_name2) |
| 300 self._port.expectations_dict = lambda: expectations_dict | 324 self._port.expectations_dict = lambda: expectations_dict |
| 301 | 325 |
| 302 expectations = TestExpectations(self._port, self.get_basic_tests()) | 326 expectations = TestExpectations(self._port, self.get_basic_tests()) |
| 303 self.assertEqual(expectations.get_expectations(self.get_test(test_name1)
), set([IMAGE])) | 327 self.assertEqual(expectations.get_expectations(self.get_test(test_name1)
), set([IMAGE])) |
| 304 self.assertEqual(expectations.get_expectations(self.get_test(test_name2)
), set([SLOW])) | 328 self.assertEqual(expectations.get_expectations(self.get_test(test_name2)
), set([SLOW])) |
| 305 | 329 |
| 306 def bot_expectations(): | 330 def bot_expectations(): |
| 307 return {test_name1: ['PASS', 'TIMEOUT'], test_name2: ['CRASH']} | 331 return {test_name1: ['PASS', 'TIMEOUT'], test_name2: ['CRASH']} |
| 308 self._port.bot_expectations = bot_expectations | 332 self._port.bot_expectations = bot_expectations |
| 309 self._port._options.ignore_flaky_tests = 'unexpected' | 333 self._port._options.ignore_flaky_tests = 'unexpected' |
| 310 | 334 |
| 311 expectations = TestExpectations(self._port, self.get_basic_tests()) | 335 expectations = TestExpectations(self._port, self.get_basic_tests()) |
| 312 self.assertEqual(expectations.get_expectations(self.get_test(test_name1)
), set([PASS, IMAGE, TIMEOUT])) | 336 self.assertEqual(expectations.get_expectations(self.get_test(test_name1)
), set([PASS, IMAGE, TIMEOUT])) |
| 313 self.assertEqual(expectations.get_expectations(self.get_test(test_name2)
), set([CRASH, SLOW])) | 337 self.assertEqual(expectations.get_expectations(self.get_test(test_name2)
), set([CRASH, SLOW])) |
| 314 | 338 |
| 339 |
| 315 class SkippedTests(Base): | 340 class SkippedTests(Base): |
| 341 |
| 316 def check(self, expectations, overrides, skips, lint=False, expected_results
=[WONTFIX, SKIP, FAIL]): | 342 def check(self, expectations, overrides, skips, lint=False, expected_results
=[WONTFIX, SKIP, FAIL]): |
| 317 port = MockHost().port_factory.get('test-win-xp') | 343 port = MockHost().port_factory.get('test-win-xp') |
| 318 port._filesystem.write_text_file(port._filesystem.join(port.layout_tests
_dir(), 'failures/expected/text.html'), 'foo') | 344 port._filesystem.write_text_file(port._filesystem.join(port.layout_tests
_dir(), 'failures/expected/text.html'), 'foo') |
| 319 expectations_dict = OrderedDict() | 345 expectations_dict = OrderedDict() |
| 320 expectations_dict['expectations'] = expectations | 346 expectations_dict['expectations'] = expectations |
| 321 if overrides: | 347 if overrides: |
| 322 expectations_dict['overrides'] = overrides | 348 expectations_dict['overrides'] = overrides |
| 323 port.expectations_dict = lambda: expectations_dict | 349 port.expectations_dict = lambda: expectations_dict |
| 324 port.skipped_layout_tests = lambda tests: set(skips) | 350 port.skipped_layout_tests = lambda tests: set(skips) |
| 325 expectations_to_lint = expectations_dict if lint else None | 351 expectations_to_lint = expectations_dict if lint else None |
| 326 exp = TestExpectations(port, ['failures/expected/text.html'], expectatio
ns_dict=expectations_to_lint, is_lint_mode=lint) | 352 exp = TestExpectations(port, ['failures/expected/text.html'], expectatio
ns_dict=expectations_to_lint, is_lint_mode=lint) |
| 327 self.assertEqual(exp.get_expectations('failures/expected/text.html'), se
t(expected_results)) | 353 self.assertEqual(exp.get_expectations('failures/expected/text.html'), se
t(expected_results)) |
| 328 | 354 |
| 329 def test_skipped_tests_work(self): | 355 def test_skipped_tests_work(self): |
| 330 self.check(expectations='', overrides=None, skips=['failures/expected/te
xt.html'], expected_results=[WONTFIX, SKIP]) | 356 self.check(expectations='', overrides=None, skips=['failures/expected/te
xt.html'], expected_results=[WONTFIX, SKIP]) |
| 331 | 357 |
| 332 def test_duplicate_skipped_test_fails_lint(self): | 358 def test_duplicate_skipped_test_fails_lint(self): |
| 333 self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/
expected/text.html [ Failure ]\n', | 359 self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/
expected/text.html [ Failure ]\n', |
| 334 overrides=None, skips=['failures/expected/text.html'], lint=True) | 360 overrides=None, skips=['failures/expected/text.html'],
lint=True) |
| 335 | 361 |
| 336 def test_skipped_file_overrides_expectations(self): | 362 def test_skipped_file_overrides_expectations(self): |
| 337 self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\
n', overrides=None, | 363 self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\
n', overrides=None, |
| 338 skips=['failures/expected/text.html']) | 364 skips=['failures/expected/text.html']) |
| 339 | 365 |
| 340 def test_skipped_dir_overrides_expectations(self): | 366 def test_skipped_dir_overrides_expectations(self): |
| 341 self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\
n', overrides=None, | 367 self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\
n', overrides=None, |
| 342 skips=['failures/expected']) | 368 skips=['failures/expected']) |
| 343 | 369 |
| 344 def test_skipped_file_overrides_overrides(self): | 370 def test_skipped_file_overrides_overrides(self): |
| (...skipping 17 matching lines...) Expand all Loading... |
| 362 self.assertEqual('The following test foo/bar/baz.html from the Skipped l
ist doesn\'t exist\n', logs) | 388 self.assertEqual('The following test foo/bar/baz.html from the Skipped l
ist doesn\'t exist\n', logs) |
| 363 | 389 |
| 364 def test_expectations_string(self): | 390 def test_expectations_string(self): |
| 365 self.parse_exp(self.get_basic_expectations()) | 391 self.parse_exp(self.get_basic_expectations()) |
| 366 notrun = 'failures/expected/text.html' | 392 notrun = 'failures/expected/text.html' |
| 367 self._exp.add_extra_skipped_tests([notrun]) | 393 self._exp.add_extra_skipped_tests([notrun]) |
| 368 self.assertEqual('NOTRUN', self._exp.get_expectations_string(notrun)) | 394 self.assertEqual('NOTRUN', self._exp.get_expectations_string(notrun)) |
| 369 | 395 |
| 370 | 396 |
| 371 class ExpectationSyntaxTests(Base): | 397 class ExpectationSyntaxTests(Base): |
| 398 |
| 372 def test_unrecognized_expectation(self): | 399 def test_unrecognized_expectation(self): |
| 373 self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Un
known ]') | 400 self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Un
known ]') |
| 374 | 401 |
| 375 def test_macro(self): | 402 def test_macro(self): |
| 376 exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]' | 403 exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]' |
| 377 self.parse_exp(exp_str) | 404 self.parse_exp(exp_str) |
| 378 self.assert_exp('failures/expected/text.html', FAIL) | 405 self.assert_exp('failures/expected/text.html', FAIL) |
| 379 | 406 |
| 380 def assert_tokenize_exp(self, line, bugs=None, specifiers=None, expectations
=None, warnings=None, comment=None, name='foo.html'): | 407 def assert_tokenize_exp(self, line, bugs=None, specifiers=None, expectations
=None, |
| 408 warnings=None, comment=None, name='foo.html'): |
| 381 bugs = bugs or [] | 409 bugs = bugs or [] |
| 382 specifiers = specifiers or [] | 410 specifiers = specifiers or [] |
| 383 expectations = expectations or [] | 411 expectations = expectations or [] |
| 384 warnings = warnings or [] | 412 warnings = warnings or [] |
| 385 filename = 'TestExpectations' | 413 filename = 'TestExpectations' |
| 386 line_number = '1' | 414 line_number = '1' |
| 387 expectation_line = TestExpectationParser._tokenize_line(filename, line,
line_number) | 415 expectation_line = TestExpectationParser._tokenize_line(filename, line,
line_number) |
| 388 self.assertEqual(expectation_line.warnings, warnings) | 416 self.assertEqual(expectation_line.warnings, warnings) |
| 389 self.assertEqual(expectation_line.name, name) | 417 self.assertEqual(expectation_line.name, name) |
| 390 self.assertEqual(expectation_line.filename, filename) | 418 self.assertEqual(expectation_line.filename, filename) |
| 391 self.assertEqual(expectation_line.line_numbers, line_number) | 419 self.assertEqual(expectation_line.line_numbers, line_number) |
| 392 if not warnings: | 420 if not warnings: |
| 393 self.assertEqual(expectation_line.specifiers, specifiers) | 421 self.assertEqual(expectation_line.specifiers, specifiers) |
| 394 self.assertEqual(expectation_line.expectations, expectations) | 422 self.assertEqual(expectation_line.expectations, expectations) |
| 395 | 423 |
| 396 def test_comments(self): | 424 def test_comments(self): |
| 397 self.assert_tokenize_exp("# comment", name=None, comment="# comment") | 425 self.assert_tokenize_exp('# comment', name=None, comment='# comment') |
| 398 self.assert_tokenize_exp("foo.html [ Pass ] # comment", comment="# comme
nt", expectations=['PASS'], specifiers=[]) | 426 self.assert_tokenize_exp('foo.html [ Pass ] # comment', comment='# comme
nt', expectations=['PASS'], specifiers=[]) |
| 399 | 427 |
| 400 def test_config_specifiers(self): | 428 def test_config_specifiers(self): |
| 401 self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', specifiers=['M
AC'], expectations=['FAIL']) | 429 self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', specifiers=['M
AC'], expectations=['FAIL']) |
| 402 | 430 |
| 403 def test_unknown_config(self): | 431 def test_unknown_config(self): |
| 404 self.assert_tokenize_exp('[ Foo ] foo.html [ Pass ]', specifiers=['Foo']
, expectations=['PASS']) | 432 self.assert_tokenize_exp('[ Foo ] foo.html [ Pass ]', specifiers=['Foo']
, expectations=['PASS']) |
| 405 | 433 |
| 406 def test_unknown_expectation(self): | 434 def test_unknown_expectation(self): |
| 407 self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized e
xpectation "Audio"']) | 435 self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized e
xpectation "Audio"']) |
| 408 | 436 |
| 409 def test_skip(self): | 437 def test_skip(self): |
| 410 self.assert_tokenize_exp('foo.html [ Skip ]', specifiers=[], expectation
s=['SKIP']) | 438 self.assert_tokenize_exp('foo.html [ Skip ]', specifiers=[], expectation
s=['SKIP']) |
| 411 | 439 |
| 412 def test_slow(self): | 440 def test_slow(self): |
| 413 self.assert_tokenize_exp('foo.html [ Slow ]', specifiers=[], expectation
s=['SLOW']) | 441 self.assert_tokenize_exp('foo.html [ Slow ]', specifiers=[], expectation
s=['SLOW']) |
| 414 | 442 |
| 415 def test_wontfix(self): | 443 def test_wontfix(self): |
| 416 self.assert_tokenize_exp('foo.html [ WontFix ]', specifiers=[], expectat
ions=['WONTFIX', 'SKIP']) | 444 self.assert_tokenize_exp('foo.html [ WontFix ]', specifiers=[], expectat
ions=['WONTFIX', 'SKIP']) |
| 417 self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', specif
iers=[], expectations=['WONTFIX', 'SKIP'], | 445 self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', specif
iers=[], expectations=['WONTFIX', 'SKIP'], |
| 418 warnings=['A test marked Skip or WontFix must not have other expecta
tions.']) | 446 warnings=['A test marked Skip or WontFix must n
ot have other expectations.']) |
| 419 | 447 |
| 420 def test_blank_line(self): | 448 def test_blank_line(self): |
| 421 self.assert_tokenize_exp('', name=None) | 449 self.assert_tokenize_exp('', name=None) |
| 422 | 450 |
| 423 def test_warnings(self): | 451 def test_warnings(self): |
| 424 self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.
', 'Missing expectations.'], name=None) | 452 self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.
', 'Missing expectations.'], name=None) |
| 425 self.assert_tokenize_exp('[ [', warnings=['unexpected "["', 'Missing exp
ectations.'], name=None) | 453 self.assert_tokenize_exp('[ [', warnings=['unexpected "["', 'Missing exp
ectations.'], name=None) |
| 426 self.assert_tokenize_exp('crbug.com/12345 ]', warnings=['unexpected "]"'
, 'Missing expectations.'], name=None) | 454 self.assert_tokenize_exp('crbug.com/12345 ]', warnings=['unexpected "]"'
, 'Missing expectations.'], name=None) |
| 427 | 455 |
| 428 self.assert_tokenize_exp('foo.html crbug.com/12345 ]', warnings=['"crbug
.com/12345" is not at the start of the line.', 'Missing expectations.']) | 456 self.assert_tokenize_exp( |
| 457 'foo.html crbug.com/12345 ]', |
| 458 warnings=[ |
| 459 '"crbug.com/12345" is not at the start of the line.', |
| 460 'Missing expectations.']) |
| 429 self.assert_tokenize_exp('foo.html', warnings=['Missing expectations.']) | 461 self.assert_tokenize_exp('foo.html', warnings=['Missing expectations.']) |
| 430 | 462 |
| 431 | 463 |
| 432 class SemanticTests(Base): | 464 class SemanticTests(Base): |
| 465 |
| 433 def test_bug_format(self): | 466 def test_bug_format(self): |
| 434 self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected
/text.html [ Failure ]', is_lint_mode=True) | 467 self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected
/text.html [ Failure ]', is_lint_mode=True) |
| 435 | 468 |
| 436 def test_bad_bugid(self): | 469 def test_bad_bugid(self): |
| 437 try: | 470 try: |
| 438 self.parse_exp('crbug/1234 failures/expected/text.html [ Failure ]',
is_lint_mode=True) | 471 self.parse_exp('crbug/1234 failures/expected/text.html [ Failure ]',
is_lint_mode=True) |
| 439 self.fail('should have raised an error about a bad bug identifier') | 472 self.fail('should have raised an error about a bad bug identifier') |
| 440 except ParseError, exp: | 473 except ParseError as exp: |
| 441 self.assertEqual(len(exp.warnings), 3) | 474 self.assertEqual(len(exp.warnings), 3) |
| 442 | 475 |
| 443 def test_missing_bugid(self): | 476 def test_missing_bugid(self): |
| 444 self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mode=F
alse) | 477 self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mode=F
alse) |
| 445 self.assertFalse(self._exp.has_warnings()) | 478 self.assertFalse(self._exp.has_warnings()) |
| 446 | 479 |
| 447 try: | 480 try: |
| 448 self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mo
de=True) | 481 self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mo
de=True) |
| 449 except ParseError, exp: | 482 except ParseError as exp: |
| 450 self.assertEqual(exp.warnings, ['expectations:1 Test lacks BUG speci
fier. failures/expected/text.html']) | 483 self.assertEqual(exp.warnings, ['expectations:1 Test lacks BUG speci
fier. failures/expected/text.html']) |
| 451 | 484 |
| 452 def test_skip_and_wontfix(self): | 485 def test_skip_and_wontfix(self): |
| 453 # Skip is not allowed to have other expectations as well, because those | 486 # Skip is not allowed to have other expectations as well, because those |
| 454 # expectations won't be exercised and may become stale . | 487 # expectations won't be exercised and may become stale . |
| 455 self.parse_exp('failures/expected/text.html [ Failure Skip ]') | 488 self.parse_exp('failures/expected/text.html [ Failure Skip ]') |
| 456 self.assertTrue(self._exp.has_warnings()) | 489 self.assertTrue(self._exp.has_warnings()) |
| 457 | 490 |
| 458 self.parse_exp('failures/expected/text.html [ Crash WontFix ]') | 491 self.parse_exp('failures/expected/text.html [ Crash WontFix ]') |
| 459 self.assertTrue(self._exp.has_warnings()) | 492 self.assertTrue(self._exp.has_warnings()) |
| 460 | 493 |
| 461 self.parse_exp('failures/expected/text.html [ Pass WontFix ]') | 494 self.parse_exp('failures/expected/text.html [ Pass WontFix ]') |
| 462 self.assertTrue(self._exp.has_warnings()) | 495 self.assertTrue(self._exp.has_warnings()) |
| 463 | 496 |
| 464 def test_rebaseline(self): | 497 def test_rebaseline(self): |
| 465 # Can't lint a file w/ 'REBASELINE' in it. | 498 # Can't lint a file w/ 'REBASELINE' in it. |
| 466 self.assertRaises(ParseError, self.parse_exp, | 499 self.assertRaises(ParseError, self.parse_exp, |
| 467 'Bug(test) failures/expected/text.html [ Failure Rebaseline ]', | 500 'Bug(test) failures/expected/text.html [ Failure Rebas
eline ]', |
| 468 is_lint_mode=True) | 501 is_lint_mode=True) |
| 469 | 502 |
| 470 def test_duplicates(self): | 503 def test_duplicates(self): |
| 471 self.assertRaises(ParseError, self.parse_exp, """ | 504 self.assertRaises(ParseError, self.parse_exp, """ |
| 472 Bug(exp) failures/expected/text.html [ Failure ] | 505 Bug(exp) failures/expected/text.html [ Failure ] |
| 473 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True) | 506 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True) |
| 474 | 507 |
| 475 self.assertRaises(ParseError, self.parse_exp, | 508 self.assertRaises(ParseError, self.parse_exp, |
| 476 self.get_basic_expectations(), overrides=""" | 509 self.get_basic_expectations(), overrides=""" |
| 477 Bug(override) failures/expected/text.html [ Failure ] | 510 Bug(override) failures/expected/text.html [ Failure ] |
| 478 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=
True) | 511 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=
True) |
| 479 | 512 |
| 480 def test_duplicate_with_line_before_preceding_line(self): | 513 def test_duplicate_with_line_before_preceding_line(self): |
| 481 self.assert_bad_expectations("""Bug(exp) [ Debug ] failures/expected/tex
t.html [ Failure ] | 514 self.assert_bad_expectations("""Bug(exp) [ Debug ] failures/expected/tex
t.html [ Failure ] |
| 482 Bug(exp) [ Release ] failures/expected/text.html [ Failure ] | 515 Bug(exp) [ Release ] failures/expected/text.html [ Failure ] |
| 483 Bug(exp) [ Debug ] failures/expected/text.html [ Failure ] | 516 Bug(exp) [ Debug ] failures/expected/text.html [ Failure ] |
| 484 """) | 517 """) |
| 485 | 518 |
| 486 def test_missing_file(self): | 519 def test_missing_file(self): |
| 487 self.parse_exp('Bug(test) missing_file.html [ Failure ]') | 520 self.parse_exp('Bug(test) missing_file.html [ Failure ]') |
| 488 self.assertTrue(self._exp.has_warnings(), 1) | 521 self.assertTrue(self._exp.has_warnings(), 1) |
| 489 | 522 |
| 490 | 523 |
| 491 class PrecedenceTests(Base): | 524 class PrecedenceTests(Base): |
| 525 |
| 492 def test_file_over_directory(self): | 526 def test_file_over_directory(self): |
| 493 # This tests handling precedence of specific lines over directories | 527 # This tests handling precedence of specific lines over directories |
| 494 # and tests expectations covering entire directories. | 528 # and tests expectations covering entire directories. |
| 495 exp_str = """ | 529 exp_str = """ |
| 496 Bug(x) failures/expected/text.html [ Failure ] | 530 Bug(x) failures/expected/text.html [ Failure ] |
| 497 Bug(y) failures/expected [ WontFix ] | 531 Bug(y) failures/expected [ WontFix ] |
| 498 """ | 532 """ |
| 499 self.parse_exp(exp_str) | 533 self.parse_exp(exp_str) |
| 500 self.assert_exp('failures/expected/text.html', FAIL) | 534 self.assert_exp('failures/expected/text.html', FAIL) |
| 501 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) | 535 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) |
| 502 | 536 |
| 503 exp_str = """ | 537 exp_str = """ |
| 504 Bug(x) failures/expected [ WontFix ] | 538 Bug(x) failures/expected [ WontFix ] |
| 505 Bug(y) failures/expected/text.html [ Failure ] | 539 Bug(y) failures/expected/text.html [ Failure ] |
| 506 """ | 540 """ |
| 507 self.parse_exp(exp_str) | 541 self.parse_exp(exp_str) |
| 508 self.assert_exp('failures/expected/text.html', FAIL) | 542 self.assert_exp('failures/expected/text.html', FAIL) |
| 509 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) | 543 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) |
| 510 | 544 |
| 511 def test_ambiguous(self): | 545 def test_ambiguous(self): |
| 512 self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ P
ass ]\n" | 546 self.assert_bad_expectations('Bug(test) [ Release ] passes/text.html [ P
ass ]\n' |
| 513 "Bug(test) [ Win ] passes/text.html [ Failu
re ]\n") | 547 'Bug(test) [ Win ] passes/text.html [ Failu
re ]\n') |
| 514 | 548 |
| 515 def test_more_specifiers(self): | 549 def test_more_specifiers(self): |
| 516 self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ P
ass ]\n" | 550 self.assert_bad_expectations('Bug(test) [ Release ] passes/text.html [ P
ass ]\n' |
| 517 "Bug(test) [ Win Release ] passes/text.html
[ Failure ]\n") | 551 'Bug(test) [ Win Release ] passes/text.html
[ Failure ]\n') |
| 518 | 552 |
| 519 def test_order_in_file(self): | 553 def test_order_in_file(self): |
| 520 self.assert_bad_expectations("Bug(test) [ Win Release ] : passes/text.ht
ml [ Failure ]\n" | 554 self.assert_bad_expectations('Bug(test) [ Win Release ] : passes/text.ht
ml [ Failure ]\n' |
| 521 "Bug(test) [ Release ] : passes/text.html [
Pass ]\n") | 555 'Bug(test) [ Release ] : passes/text.html [
Pass ]\n') |
| 522 | 556 |
| 523 def test_macro_overrides(self): | 557 def test_macro_overrides(self): |
| 524 self.assert_bad_expectations("Bug(test) [ Win ] passes/text.html [ Pass
]\n" | 558 self.assert_bad_expectations('Bug(test) [ Win ] passes/text.html [ Pass
]\n' |
| 525 "Bug(test) [ XP ] passes/text.html [ Failur
e ]\n") | 559 'Bug(test) [ XP ] passes/text.html [ Failur
e ]\n') |
| 526 | 560 |
| 527 | 561 |
| 528 class RemoveConfigurationsTest(Base): | 562 class RemoveConfigurationsTest(Base): |
| 563 |
| 529 def test_remove(self): | 564 def test_remove(self): |
| 530 host = MockHost() | 565 host = MockHost() |
| 531 test_port = host.port_factory.get('test-win-xp', None) | 566 test_port = host.port_factory.get('test-win-xp', None) |
| 532 test_port.test_exists = lambda test: True | 567 test_port.test_exists = lambda test: True |
| 533 test_port.test_isfile = lambda test: True | 568 test_port.test_isfile = lambda test: True |
| 534 | 569 |
| 535 test_config = test_port.test_configuration() | 570 test_config = test_port.test_configuration() |
| 536 test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux
Win Release ] failures/expected/foo.html [ Failure ] | 571 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Linux
Win Release ] failures/expected/foo.html [ Failure ] |
| 537 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ] | 572 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ] |
| 538 """} | 573 """} |
| 539 expectations = TestExpectations(test_port, self.get_basic_tests()) | 574 expectations = TestExpectations(test_port, self.get_basic_tests()) |
| 540 | 575 |
| 541 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 576 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
| 542 | 577 |
| 543 self.assertEqual("""Bug(x) [ Linux Win7 Release ] failures/expected/foo.
html [ Failure ] | 578 self.assertEqual("""Bug(x) [ Linux Win7 Release ] failures/expected/foo.
html [ Failure ] |
| 544 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ] | 579 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ] |
| 545 """, actual_expectations) | 580 """, actual_expectations) |
| 546 | 581 |
| 547 def test_remove_needs_rebaseline(self): | 582 def test_remove_needs_rebaseline(self): |
| 548 host = MockHost() | 583 host = MockHost() |
| 549 test_port = host.port_factory.get('test-win-xp', None) | 584 test_port = host.port_factory.get('test-win-xp', None) |
| 550 test_port.test_exists = lambda test: True | 585 test_port.test_exists = lambda test: True |
| 551 test_port.test_isfile = lambda test: True | 586 test_port.test_isfile = lambda test: True |
| 552 | 587 |
| 553 test_config = test_port.test_configuration() | 588 test_config = test_port.test_configuration() |
| 554 test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Win ]
failures/expected/foo.html [ NeedsRebaseline ] | 589 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win ]
failures/expected/foo.html [ NeedsRebaseline ] |
| 555 """} | 590 """} |
| 556 expectations = TestExpectations(test_port, self.get_basic_tests()) | 591 expectations = TestExpectations(test_port, self.get_basic_tests()) |
| 557 | 592 |
| 558 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 593 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
| 559 | 594 |
| 560 self.assertEqual("""Bug(x) [ XP Debug ] failures/expected/foo.html [ Nee
dsRebaseline ] | 595 self.assertEqual("""Bug(x) [ XP Debug ] failures/expected/foo.html [ Nee
dsRebaseline ] |
| 561 Bug(x) [ Win7 ] failures/expected/foo.html [ NeedsRebaseline ] | 596 Bug(x) [ Win7 ] failures/expected/foo.html [ NeedsRebaseline ] |
| 562 """, actual_expectations) | 597 """, actual_expectations) |
| 563 | 598 |
| 564 def test_remove_multiple_configurations(self): | 599 def test_remove_multiple_configurations(self): |
| (...skipping 24 matching lines...) Expand all Loading... |
| 589 | 624 |
| 590 test_config = test_port.test_configuration() | 625 test_config = test_port.test_configuration() |
| 591 test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win D
ebug ] failures/expected/foo.html [ Crash ] | 626 test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win D
ebug ] failures/expected/foo.html [ Crash ] |
| 592 | 627 |
| 593 # This comment line should get stripped. As should the preceding line. | 628 # This comment line should get stripped. As should the preceding line. |
| 594 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] | 629 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] |
| 595 """} | 630 """} |
| 596 expectations = TestExpectations(test_port) | 631 expectations = TestExpectations(test_port) |
| 597 | 632 |
| 598 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 633 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
| 599 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 634 actual_expectations = expectations.remove_configurations( |
| 635 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
| 600 | 636 |
| 601 self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Cr
ash ] | 637 self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Cr
ash ] |
| 602 """, actual_expectations) | 638 """, actual_expectations) |
| 603 | 639 |
| 604 def test_remove_line_with_comments_at_start(self): | 640 def test_remove_line_with_comments_at_start(self): |
| 605 host = MockHost() | 641 host = MockHost() |
| 606 test_port = host.port_factory.get('test-win-xp', None) | 642 test_port = host.port_factory.get('test-win-xp', None) |
| 607 test_port.test_exists = lambda test: True | 643 test_port.test_exists = lambda test: True |
| 608 test_port.test_isfile = lambda test: True | 644 test_port.test_isfile = lambda test: True |
| 609 | 645 |
| 610 test_config = test_port.test_configuration() | 646 test_config = test_port.test_configuration() |
| 611 test_port.expectations_dict = lambda: {'expectations': """ | 647 test_port.expectations_dict = lambda: {'expectations': """ |
| 612 # This comment line should get stripped. As should the preceding line. | 648 # This comment line should get stripped. As should the preceding line. |
| 613 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] | 649 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] |
| 614 | 650 |
| 615 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 651 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
| 616 """} | 652 """} |
| 617 expectations = TestExpectations(test_port) | 653 expectations = TestExpectations(test_port) |
| 618 | 654 |
| 619 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 655 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
| 620 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 656 actual_expectations = expectations.remove_configurations( |
| 657 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
| 621 | 658 |
| 622 self.assertEqual(""" | 659 self.assertEqual(""" |
| 623 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 660 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
| 624 """, actual_expectations) | 661 """, actual_expectations) |
| 625 | 662 |
| 626 def test_remove_line_with_comments_at_end_with_no_trailing_newline(self): | 663 def test_remove_line_with_comments_at_end_with_no_trailing_newline(self): |
| 627 host = MockHost() | 664 host = MockHost() |
| 628 test_port = host.port_factory.get('test-win-xp', None) | 665 test_port = host.port_factory.get('test-win-xp', None) |
| 629 test_port.test_exists = lambda test: True | 666 test_port.test_exists = lambda test: True |
| 630 test_port.test_isfile = lambda test: True | 667 test_port.test_isfile = lambda test: True |
| 631 | 668 |
| 632 test_config = test_port.test_configuration() | 669 test_config = test_port.test_configuration() |
| 633 test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win D
ebug ] failures/expected/foo.html [ Crash ] | 670 test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win D
ebug ] failures/expected/foo.html [ Crash ] |
| 634 | 671 |
| 635 # This comment line should get stripped. As should the preceding line. | 672 # This comment line should get stripped. As should the preceding line. |
| 636 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]"""} | 673 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]"""} |
| 637 expectations = TestExpectations(test_port) | 674 expectations = TestExpectations(test_port) |
| 638 | 675 |
| 639 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 676 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
| 640 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 677 actual_expectations = expectations.remove_configurations( |
| 678 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
| 641 | 679 |
| 642 self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Cr
ash ]""", actual_expectations) | 680 self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Cr
ash ]""", actual_expectations) |
| 643 | 681 |
| 644 def test_remove_line_leaves_comments_for_next_line(self): | 682 def test_remove_line_leaves_comments_for_next_line(self): |
| 645 host = MockHost() | 683 host = MockHost() |
| 646 test_port = host.port_factory.get('test-win-xp', None) | 684 test_port = host.port_factory.get('test-win-xp', None) |
| 647 test_port.test_exists = lambda test: True | 685 test_port.test_exists = lambda test: True |
| 648 test_port.test_isfile = lambda test: True | 686 test_port.test_isfile = lambda test: True |
| 649 | 687 |
| 650 test_config = test_port.test_configuration() | 688 test_config = test_port.test_configuration() |
| 651 test_port.expectations_dict = lambda: {'expectations': """ | 689 test_port.expectations_dict = lambda: {'expectations': """ |
| 652 # This comment line should not get stripped. | 690 # This comment line should not get stripped. |
| 653 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] | 691 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] |
| 654 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 692 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
| 655 """} | 693 """} |
| 656 expectations = TestExpectations(test_port) | 694 expectations = TestExpectations(test_port) |
| 657 | 695 |
| 658 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 696 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
| 659 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 697 actual_expectations = expectations.remove_configurations( |
| 698 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
| 660 | 699 |
| 661 self.assertEqual(""" | 700 self.assertEqual(""" |
| 662 # This comment line should not get stripped. | 701 # This comment line should not get stripped. |
| 663 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 702 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
| 664 """, actual_expectations) | 703 """, actual_expectations) |
| 665 | 704 |
| 666 def test_remove_line_no_whitespace_lines(self): | 705 def test_remove_line_no_whitespace_lines(self): |
| 667 host = MockHost() | 706 host = MockHost() |
| 668 test_port = host.port_factory.get('test-win-xp', None) | 707 test_port = host.port_factory.get('test-win-xp', None) |
| 669 test_port.test_exists = lambda test: True | 708 test_port.test_exists = lambda test: True |
| 670 test_port.test_isfile = lambda test: True | 709 test_port.test_isfile = lambda test: True |
| 671 | 710 |
| 672 test_config = test_port.test_configuration() | 711 test_config = test_port.test_configuration() |
| 673 test_port.expectations_dict = lambda: {'expectations': """ | 712 test_port.expectations_dict = lambda: {'expectations': """ |
| 674 # This comment line should get stripped. | 713 # This comment line should get stripped. |
| 675 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] | 714 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] |
| 676 # This comment line should not get stripped. | 715 # This comment line should not get stripped. |
| 677 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 716 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
| 678 """} | 717 """} |
| 679 expectations = TestExpectations(test_port) | 718 expectations = TestExpectations(test_port) |
| 680 | 719 |
| 681 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 720 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
| 682 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 721 actual_expectations = expectations.remove_configurations( |
| 722 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
| 683 | 723 |
| 684 self.assertEqual(""" # This comment line should not get stripped. | 724 self.assertEqual(""" # This comment line should not get stripped. |
| 685 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 725 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
| 686 """, actual_expectations) | 726 """, actual_expectations) |
| 687 | 727 |
| 688 def test_remove_first_line(self): | 728 def test_remove_first_line(self): |
| 689 host = MockHost() | 729 host = MockHost() |
| 690 test_port = host.port_factory.get('test-win-xp', None) | 730 test_port = host.port_factory.get('test-win-xp', None) |
| 691 test_port.test_exists = lambda test: True | 731 test_port.test_exists = lambda test: True |
| 692 test_port.test_isfile = lambda test: True | 732 test_port.test_isfile = lambda test: True |
| 693 | 733 |
| 694 test_config = test_port.test_configuration() | 734 test_config = test_port.test_configuration() |
| 695 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win R
elease ] failures/expected/foo.html [ Failure ] | 735 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win R
elease ] failures/expected/foo.html [ Failure ] |
| 696 # This comment line should not get stripped. | 736 # This comment line should not get stripped. |
| 697 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 737 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
| 698 """} | 738 """} |
| 699 expectations = TestExpectations(test_port) | 739 expectations = TestExpectations(test_port) |
| 700 | 740 |
| 701 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 741 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
| 702 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 742 actual_expectations = expectations.remove_configurations( |
| 743 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
| 703 | 744 |
| 704 self.assertEqual(""" # This comment line should not get stripped. | 745 self.assertEqual(""" # This comment line should not get stripped. |
| 705 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] | 746 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] |
| 706 """, actual_expectations) | 747 """, actual_expectations) |
| 707 | 748 |
| 708 def test_remove_flaky_line(self): | 749 def test_remove_flaky_line(self): |
| 709 host = MockHost() | 750 host = MockHost() |
| 710 test_port = host.port_factory.get('test-win-xp', None) | 751 test_port = host.port_factory.get('test-win-xp', None) |
| 711 test_port.test_exists = lambda test: True | 752 test_port.test_exists = lambda test: True |
| 712 test_port.test_isfile = lambda test: True | 753 test_port.test_isfile = lambda test: True |
| 713 | 754 |
| 714 test_config = test_port.test_configuration() | 755 test_config = test_port.test_configuration() |
| 715 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win ]
failures/expected/foo.html [ Failure Timeout ] | 756 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win ]
failures/expected/foo.html [ Failure Timeout ] |
| 716 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ] | 757 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ] |
| 717 """} | 758 """} |
| 718 expectations = TestExpectations(test_port) | 759 expectations = TestExpectations(test_port) |
| 719 | 760 |
| 720 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) | 761 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', test_config)]) |
| 721 actual_expectations = expectations.remove_configurations([('failures/exp
ected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration
())]) | 762 actual_expectations = expectations.remove_configurations( |
| 763 [('failures/expected/foo.html', host.port_factory.get('test-win-win7
', None).test_configuration())]) |
| 722 | 764 |
| 723 self.assertEqual("""Bug(x) [ Win Debug ] failures/expected/foo.html [ Fa
ilure Timeout ] | 765 self.assertEqual("""Bug(x) [ Win Debug ] failures/expected/foo.html [ Fa
ilure Timeout ] |
| 724 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ] | 766 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ] |
| 725 """, actual_expectations) | 767 """, actual_expectations) |
| 726 | 768 |
| 727 | 769 |
| 728 class RebaseliningTest(Base): | 770 class RebaseliningTest(Base): |
| 771 |
| 729 def test_get_rebaselining_failures(self): | 772 def test_get_rebaselining_failures(self): |
| 730 # Make sure we find a test as needing a rebaseline even if it is not mar
ked as a failure. | 773 # Make sure we find a test as needing a rebaseline even if it is not mar
ked as a failure. |
| 731 self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n') | 774 self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n') |
| 732 self.assertEqual(len(self._exp.get_rebaselining_failures()), 1) | 775 self.assertEqual(len(self._exp.get_rebaselining_failures()), 1) |
| 733 | 776 |
| 734 self.parse_exp(self.get_basic_expectations()) | 777 self.parse_exp(self.get_basic_expectations()) |
| 735 self.assertEqual(len(self._exp.get_rebaselining_failures()), 0) | 778 self.assertEqual(len(self._exp.get_rebaselining_failures()), 0) |
| 736 | 779 |
| 737 | 780 |
| 738 class TestExpectationsParserTests(unittest.TestCase): | 781 class TestExpectationsParserTests(unittest.TestCase): |
| 782 |
| 739 def __init__(self, testFunc): | 783 def __init__(self, testFunc): |
| 740 host = MockHost() | 784 host = MockHost() |
| 741 test_port = host.port_factory.get('test-win-xp', None) | 785 test_port = host.port_factory.get('test-win-xp', None) |
| 742 self._converter = TestConfigurationConverter(test_port.all_test_configur
ations(), test_port.configuration_specifier_macros()) | 786 self._converter = TestConfigurationConverter( |
| 787 test_port.all_test_configurations(), |
| 788 test_port.configuration_specifier_macros()) |
| 743 unittest.TestCase.__init__(self, testFunc) | 789 unittest.TestCase.__init__(self, testFunc) |
| 744 self._parser = TestExpectationParser(host.port_factory.get('test-win-xp'
, None), [], is_lint_mode=False) | 790 self._parser = TestExpectationParser(host.port_factory.get('test-win-xp'
, None), [], is_lint_mode=False) |
| 745 | 791 |
| 746 def test_expectation_line_for_test(self): | 792 def test_expectation_line_for_test(self): |
| 747 # This is kind of a silly test, but it at least ensures that we don't th
row an error. | 793 # This is kind of a silly test, but it at least ensures that we don't th
row an error. |
| 748 test_name = 'foo/test.html' | 794 test_name = 'foo/test.html' |
| 749 expectations = set(["PASS", "IMAGE"]) | 795 expectations = set(['PASS', 'IMAGE']) |
| 750 | 796 |
| 751 expectation_line = TestExpectationLine() | 797 expectation_line = TestExpectationLine() |
| 752 expectation_line.original_string = test_name | 798 expectation_line.original_string = test_name |
| 753 expectation_line.name = test_name | 799 expectation_line.name = test_name |
| 754 expectation_line.filename = '<Bot TestExpectations>' | 800 expectation_line.filename = '<Bot TestExpectations>' |
| 755 expectation_line.line_numbers = '0' | 801 expectation_line.line_numbers = '0' |
| 756 expectation_line.expectations = expectations | 802 expectation_line.expectations = expectations |
| 757 self._parser._parse_line(expectation_line) | 803 self._parser._parse_line(expectation_line) |
| 758 | 804 |
| 759 self.assertEqual(self._parser.expectation_line_for_test(test_name, expec
tations), expectation_line) | 805 self.assertEqual(self._parser.expectation_line_for_test(test_name, expec
tations), expectation_line) |
| 760 | 806 |
| 761 | 807 |
| 762 class TestExpectationSerializationTests(unittest.TestCase): | 808 class TestExpectationSerializationTests(unittest.TestCase): |
| 809 |
| 763 def __init__(self, testFunc): | 810 def __init__(self, testFunc): |
| 764 host = MockHost() | 811 host = MockHost() |
| 765 test_port = host.port_factory.get('test-win-xp', None) | 812 test_port = host.port_factory.get('test-win-xp', None) |
| 766 self._converter = TestConfigurationConverter(test_port.all_test_configur
ations(), test_port.configuration_specifier_macros()) | 813 self._converter = TestConfigurationConverter( |
| 814 test_port.all_test_configurations(), |
| 815 test_port.configuration_specifier_macros()) |
| 767 unittest.TestCase.__init__(self, testFunc) | 816 unittest.TestCase.__init__(self, testFunc) |
| 768 | 817 |
| 769 def _tokenize(self, line): | 818 def _tokenize(self, line): |
| 770 return TestExpectationParser._tokenize_line('path', line, 0) | 819 return TestExpectationParser._tokenize_line('path', line, 0) |
| 771 | 820 |
| 772 def assert_round_trip(self, in_string, expected_string=None): | 821 def assert_round_trip(self, in_string, expected_string=None): |
| 773 expectation = self._tokenize(in_string) | 822 expectation = self._tokenize(in_string) |
| 774 if expected_string is None: | 823 if expected_string is None: |
| 775 expected_string = in_string | 824 expected_string = in_string |
| 776 self.assertEqual(expected_string, expectation.to_string(self._converter)
) | 825 self.assertEqual(expected_string, expectation.to_string(self._converter)
) |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 814 # FIXME: case should be preserved here but we can't until we drop the ol
d syntax. | 863 # FIXME: case should be preserved here but we can't until we drop the ol
d syntax. |
| 815 self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO
] bar [ BAZ1 BAZ2 ] #Qux.') | 864 self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO
] bar [ BAZ1 BAZ2 ] #Qux.') |
| 816 | 865 |
| 817 def test_parsed_to_string(self): | 866 def test_parsed_to_string(self): |
| 818 expectation_line = TestExpectationLine() | 867 expectation_line = TestExpectationLine() |
| 819 expectation_line.bugs = ['Bug(x)'] | 868 expectation_line.bugs = ['Bug(x)'] |
| 820 expectation_line.name = 'test/name/for/realz.html' | 869 expectation_line.name = 'test/name/for/realz.html' |
| 821 expectation_line.parsed_expectations = set([IMAGE]) | 870 expectation_line.parsed_expectations = set([IMAGE]) |
| 822 self.assertEqual(expectation_line.to_string(self._converter), None) | 871 self.assertEqual(expectation_line.to_string(self._converter), None) |
| 823 expectation_line.matching_configurations = set([TestConfiguration('xp',
'x86', 'release')]) | 872 expectation_line.matching_configurations = set([TestConfiguration('xp',
'x86', 'release')]) |
| 824 self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [
XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]') | 873 self.assertEqual( |
| 825 expectation_line.matching_configurations = set([TestConfiguration('xp',
'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]) | 874 expectation_line.to_string( |
| 875 self._converter), |
| 876 'Bug(x) [ XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]
') |
| 877 expectation_line.matching_configurations = set( |
| 878 [TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp',
'x86', 'debug')]) |
| 826 self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [
XP ] test/name/for/realz.html [ ImageOnlyFailure ]') | 879 self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [
XP ] test/name/for/realz.html [ ImageOnlyFailure ]') |
| 827 | 880 |
| 828 def test_serialize_parsed_expectations(self): | 881 def test_serialize_parsed_expectations(self): |
| 829 expectation_line = TestExpectationLine() | 882 expectation_line = TestExpectationLine() |
| 830 expectation_line.parsed_expectations = set([]) | 883 expectation_line.parsed_expectations = set([]) |
| 831 parsed_expectation_to_string = dict([[parsed_expectation, expectation_st
ring] for expectation_string, parsed_expectation in TestExpectations.EXPECTATION
S.items()]) | 884 parsed_expectation_to_string = dict([[parsed_expectation, expectation_st
ring] |
| 885 for expectation_string, parsed_expe
ctation in TestExpectations.EXPECTATIONS.items()]) |
| 832 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), '') | 886 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), '') |
| 833 expectation_line.parsed_expectations = set([FAIL]) | 887 expectation_line.parsed_expectations = set([FAIL]) |
| 834 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'fail') | 888 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'fail') |
| 835 expectation_line.parsed_expectations = set([PASS, IMAGE]) | 889 expectation_line.parsed_expectations = set([PASS, IMAGE]) |
| 836 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'image pass') | 890 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'image pass') |
| 837 expectation_line.parsed_expectations = set([FAIL, PASS]) | 891 expectation_line.parsed_expectations = set([FAIL, PASS]) |
| 838 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'pass fail') | 892 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_
expectation_to_string), 'pass fail') |
| 839 | 893 |
| 840 def test_serialize_parsed_specifier_string(self): | 894 def test_serialize_parsed_specifier_string(self): |
| 841 expectation_line = TestExpectationLine() | 895 expectation_line = TestExpectationLine() |
| 842 expectation_line.bugs = ['garden-o-matic'] | 896 expectation_line.bugs = ['garden-o-matic'] |
| 843 expectation_line.parsed_specifiers = ['the', 'for'] | 897 expectation_line.parsed_specifiers = ['the', 'for'] |
| 844 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, []), 'for the') | 898 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, []), 'for the') |
| 845 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, ['win']), 'for the win') | 899 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, ['win']), 'for the win') |
| 846 expectation_line.bugs = [] | 900 expectation_line.bugs = [] |
| 847 expectation_line.parsed_specifiers = [] | 901 expectation_line.parsed_specifiers = [] |
| 848 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, []), '') | 902 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, []), '') |
| 849 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, ['win']), 'win') | 903 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._con
verter, ['win']), 'win') |
| 850 | 904 |
| 851 def test_format_line(self): | 905 def test_format_line(self): |
| 852 self.assertEqual(TestExpectationLine._format_line([], ['MODIFIERS'], 'na
me', ['EXPECTATIONS'], 'comment'), '[ MODIFIERS ] name [ EXPECTATIONS ] #comment
') | 906 self.assertEqual( |
| 853 self.assertEqual(TestExpectationLine._format_line([], ['MODIFIERS'], 'na
me', ['EXPECTATIONS'], None), '[ MODIFIERS ] name [ EXPECTATIONS ]') | 907 TestExpectationLine._format_line( |
| 908 [], |
| 909 ['MODIFIERS'], |
| 910 'name', |
| 911 ['EXPECTATIONS'], |
| 912 'comment'), |
| 913 '[ MODIFIERS ] name [ EXPECTATIONS ] #comment') |
| 914 self.assertEqual( |
| 915 TestExpectationLine._format_line( |
| 916 [], |
| 917 ['MODIFIERS'], |
| 918 'name', |
| 919 ['EXPECTATIONS'], |
| 920 None), |
| 921 '[ MODIFIERS ] name [ EXPECTATIONS ]') |
| 854 | 922 |
| 855 def test_string_roundtrip(self): | 923 def test_string_roundtrip(self): |
| 856 self.assert_round_trip('') | 924 self.assert_round_trip('') |
| 857 self.assert_round_trip('[') | 925 self.assert_round_trip('[') |
| 858 self.assert_round_trip('FOO [') | 926 self.assert_round_trip('FOO [') |
| 859 self.assert_round_trip('FOO ] bar') | 927 self.assert_round_trip('FOO ] bar') |
| 860 self.assert_round_trip(' FOO [') | 928 self.assert_round_trip(' FOO [') |
| 861 self.assert_round_trip(' [ FOO ] ') | 929 self.assert_round_trip(' [ FOO ] ') |
| 862 self.assert_round_trip('[ FOO ] bar [ BAZ ]') | 930 self.assert_round_trip('[ FOO ] bar [ BAZ ]') |
| 863 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.') | 931 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.') |
| (...skipping 18 matching lines...) Expand all Loading... |
| 882 self.assert_list_round_trip('bar') | 950 self.assert_list_round_trip('bar') |
| 883 self.assert_list_round_trip('bar\n# Qux.') | 951 self.assert_list_round_trip('bar\n# Qux.') |
| 884 self.assert_list_round_trip('bar\n# Qux.\n') | 952 self.assert_list_round_trip('bar\n# Qux.\n') |
| 885 | 953 |
| 886 def test_reconstitute_only_these(self): | 954 def test_reconstitute_only_these(self): |
| 887 lines = [] | 955 lines = [] |
| 888 reconstitute_only_these = [] | 956 reconstitute_only_these = [] |
| 889 | 957 |
| 890 def add_line(matching_configurations, reconstitute): | 958 def add_line(matching_configurations, reconstitute): |
| 891 expectation_line = TestExpectationLine() | 959 expectation_line = TestExpectationLine() |
| 892 expectation_line.original_string = "Nay" | 960 expectation_line.original_string = 'Nay' |
| 893 expectation_line.bugs = ['Bug(x)'] | 961 expectation_line.bugs = ['Bug(x)'] |
| 894 expectation_line.name = 'Yay' | 962 expectation_line.name = 'Yay' |
| 895 expectation_line.parsed_expectations = set([IMAGE]) | 963 expectation_line.parsed_expectations = set([IMAGE]) |
| 896 expectation_line.matching_configurations = matching_configurations | 964 expectation_line.matching_configurations = matching_configurations |
| 897 lines.append(expectation_line) | 965 lines.append(expectation_line) |
| 898 if reconstitute: | 966 if reconstitute: |
| 899 reconstitute_only_these.append(expectation_line) | 967 reconstitute_only_these.append(expectation_line) |
| 900 | 968 |
| 901 add_line(set([TestConfiguration('xp', 'x86', 'release')]), True) | 969 add_line(set([TestConfiguration('xp', 'x86', 'release')]), True) |
| 902 add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfigurati
on('xp', 'x86', 'debug')]), False) | 970 add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfigurati
on('xp', 'x86', 'debug')]), False) |
| 903 serialized = TestExpectations.list_to_string(lines, self._converter) | 971 serialized = TestExpectations.list_to_string(lines, self._converter) |
| 904 self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailu
re ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]") | 972 self.assertEqual(serialized, 'Bug(x) [ XP Release ] Yay [ ImageOnlyFailu
re ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]') |
| 905 serialized = TestExpectations.list_to_string(lines, self._converter, rec
onstitute_only_these=reconstitute_only_these) | 973 serialized = TestExpectations.list_to_string(lines, self._converter, rec
onstitute_only_these=reconstitute_only_these) |
| 906 self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailu
re ]\nNay") | 974 self.assertEqual(serialized, 'Bug(x) [ XP Release ] Yay [ ImageOnlyFailu
re ]\nNay') |
| 907 | 975 |
| 908 def disabled_test_string_whitespace_stripping(self): | 976 def disabled_test_string_whitespace_stripping(self): |
| 909 # FIXME: Re-enable this test once we rework the code to no longer suppor
t the old syntax. | 977 # FIXME: Re-enable this test once we rework the code to no longer suppor
t the old syntax. |
| 910 self.assert_round_trip('\n', '') | 978 self.assert_round_trip('\n', '') |
| 911 self.assert_round_trip(' [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]') | 979 self.assert_round_trip(' [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]') |
| 912 self.assert_round_trip('[ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]') | 980 self.assert_round_trip('[ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]') |
| 913 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] bar
[ BAZ ] # Qux.') | 981 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] bar
[ BAZ ] # Qux.') |
| 914 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] ba
r [ BAZ ] # Qux.') | 982 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] ba
r [ BAZ ] # Qux.') |
| 915 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ]
bar [ BAZ ] # Qux.') | 983 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ]
bar [ BAZ ] # Qux.') |
| 916 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FO
O ] bar [ BAZ ] # Qux.') | 984 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FO
O ] bar [ BAZ ] # Qux.') |
| OLD | NEW |