| OLD | NEW |
| 1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 168 if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectatio
ns: | 168 if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectatio
ns: |
| 169 expectation_line.warnings.append(self.MISSING_BUG_WARNING) | 169 expectation_line.warnings.append(self.MISSING_BUG_WARNING) |
| 170 if self.REBASELINE_MODIFIER in expectations: | 170 if self.REBASELINE_MODIFIER in expectations: |
| 171 expectation_line.warnings.append('REBASELINE should only be used for
running rebaseline.py. Cannot be checked in.') | 171 expectation_line.warnings.append('REBASELINE should only be used for
running rebaseline.py. Cannot be checked in.') |
| 172 | 172 |
| 173 if self.NEEDS_REBASELINE_MODIFIER in expectations or self.NEEDS_MANUAL_R
EBASELINE_MODIFIER in expectations: | 173 if self.NEEDS_REBASELINE_MODIFIER in expectations or self.NEEDS_MANUAL_R
EBASELINE_MODIFIER in expectations: |
| 174 for test in expectation_line.matching_tests: | 174 for test in expectation_line.matching_tests: |
| 175 if self._port.reference_files(test): | 175 if self._port.reference_files(test): |
| 176 text_expected_filename = self._port.expected_filename(test,
'.txt') | 176 text_expected_filename = self._port.expected_filename(test,
'.txt') |
| 177 if not self._port.host.filesystem.exists(text_expected_filen
ame): | 177 if not self._port.host.filesystem.exists(text_expected_filen
ame): |
| 178 expectation_line.warnings.append('A reftest without text
expectation cannot be marked as NeedsRebaseline/NeedsManualRebaseline') | 178 expectation_line.warnings.append( |
| 179 'A reftest without text expectation cannot be marked
as NeedsRebaseline/NeedsManualRebaseline') |
| 179 | 180 |
| 180 specifiers = [specifier.lower() for specifier in expectation_line.specif
iers] | 181 specifiers = [specifier.lower() for specifier in expectation_line.specif
iers] |
| 181 if (self.REBASELINE_MODIFIER in expectations or self.NEEDS_REBASELINE_MO
DIFIER in expectations) and ('debug' in specifiers or 'release' in specifiers): | 182 if (self.REBASELINE_MODIFIER in expectations or self.NEEDS_REBASELINE_MO
DIFIER in expectations) and ( |
| 183 'debug' in specifiers or 'release' in specifiers): |
| 182 expectation_line.warnings.append('A test cannot be rebaselined for D
ebug/Release.') | 184 expectation_line.warnings.append('A test cannot be rebaselined for D
ebug/Release.') |
| 183 | 185 |
| 184 def _parse_expectations(self, expectation_line): | 186 def _parse_expectations(self, expectation_line): |
| 185 result = set() | 187 result = set() |
| 186 for part in expectation_line.expectations: | 188 for part in expectation_line.expectations: |
| 187 expectation = TestExpectations.expectation_from_string(part) | 189 expectation = TestExpectations.expectation_from_string(part) |
| 188 if expectation is None: # Careful, PASS is currently 0. | 190 if expectation is None: # Careful, PASS is currently 0. |
| 189 expectation_line.warnings.append('Unsupported expectation: %s' %
part) | 191 expectation_line.warnings.append('Unsupported expectation: %s' %
part) |
| 190 continue | 192 continue |
| 191 result.add(expectation) | 193 result.add(expectation) |
| (...skipping 96 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 288 specifiers = [] | 290 specifiers = [] |
| 289 name = None | 291 name = None |
| 290 expectations = [] | 292 expectations = [] |
| 291 warnings = [] | 293 warnings = [] |
| 292 has_unrecognized_expectation = False | 294 has_unrecognized_expectation = False |
| 293 | 295 |
| 294 tokens = remaining_string.split() | 296 tokens = remaining_string.split() |
| 295 state = 'start' | 297 state = 'start' |
| 296 for token in tokens: | 298 for token in tokens: |
| 297 if (token.startswith(WEBKIT_BUG_PREFIX) or | 299 if (token.startswith(WEBKIT_BUG_PREFIX) or |
| 298 token.startswith(CHROMIUM_BUG_PREFIX) or | 300 token.startswith(CHROMIUM_BUG_PREFIX) or |
| 299 token.startswith(V8_BUG_PREFIX) or | 301 token.startswith(V8_BUG_PREFIX) or |
| 300 token.startswith(NAMED_BUG_PREFIX)): | 302 token.startswith(NAMED_BUG_PREFIX)): |
| 301 if state != 'start': | 303 if state != 'start': |
| 302 warnings.append('"%s" is not at the start of the line.' % to
ken) | 304 warnings.append('"%s" is not at the start of the line.' % to
ken) |
| 303 break | 305 break |
| 304 if token.startswith(WEBKIT_BUG_PREFIX): | 306 if token.startswith(WEBKIT_BUG_PREFIX): |
| 305 bugs.append(token) | 307 bugs.append(token) |
| 306 elif token.startswith(CHROMIUM_BUG_PREFIX): | 308 elif token.startswith(CHROMIUM_BUG_PREFIX): |
| 307 bugs.append(token) | 309 bugs.append(token) |
| 308 elif token.startswith(V8_BUG_PREFIX): | 310 elif token.startswith(V8_BUG_PREFIX): |
| 309 bugs.append(token) | 311 bugs.append(token) |
| 310 else: | 312 else: |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 357 if 'WONTFIX' in expectations and 'SKIP' not in expectations: | 359 if 'WONTFIX' in expectations and 'SKIP' not in expectations: |
| 358 expectations.append('SKIP') | 360 expectations.append('SKIP') |
| 359 | 361 |
| 360 if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(exp
ectations) - set(['SKIP', 'WONTFIX'])): | 362 if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(exp
ectations) - set(['SKIP', 'WONTFIX'])): |
| 361 warnings.append('A test marked Skip or WontFix must not have other e
xpectations.') | 363 warnings.append('A test marked Skip or WontFix must not have other e
xpectations.') |
| 362 | 364 |
| 363 if 'SLOW' in expectations and 'SlowTests' not in filename: | 365 if 'SLOW' in expectations and 'SlowTests' not in filename: |
| 364 warnings.append('SLOW tests should ony be added to SlowTests and not
to TestExpectations.') | 366 warnings.append('SLOW tests should ony be added to SlowTests and not
to TestExpectations.') |
| 365 | 367 |
| 366 if 'WONTFIX' in expectations and ('NeverFixTests' not in filename and 'S
taleTestExpectations' not in filename): | 368 if 'WONTFIX' in expectations and ('NeverFixTests' not in filename and 'S
taleTestExpectations' not in filename): |
| 367 warnings.append('WONTFIX tests should ony be added to NeverFixTests
or StaleTestExpectations and not to TestExpectations.') | 369 warnings.append( |
| 370 'WONTFIX tests should ony be added to NeverFixTests or StaleTest
Expectations and not to TestExpectations.') |
| 368 | 371 |
| 369 if 'NeverFixTests' in filename and expectations != ['WONTFIX', 'SKIP']: | 372 if 'NeverFixTests' in filename and expectations != ['WONTFIX', 'SKIP']: |
| 370 warnings.append('Only WONTFIX expectations are allowed in NeverFixTe
sts') | 373 warnings.append('Only WONTFIX expectations are allowed in NeverFixTe
sts') |
| 371 | 374 |
| 372 if 'SlowTests' in filename and expectations != ['SLOW']: | 375 if 'SlowTests' in filename and expectations != ['SLOW']: |
| 373 warnings.append('Only SLOW expectations are allowed in SlowTests') | 376 warnings.append('Only SLOW expectations are allowed in SlowTests') |
| 374 | 377 |
| 375 if not expectations and not has_unrecognized_expectation: | 378 if not expectations and not has_unrecognized_expectation: |
| 376 warnings.append('Missing expectations.') | 379 warnings.append('Missing expectations.') |
| 377 | 380 |
| (...skipping 25 matching lines...) Expand all Loading... |
| 403 self.parsed_specifiers = [] | 406 self.parsed_specifiers = [] |
| 404 self.matching_configurations = set() | 407 self.matching_configurations = set() |
| 405 self.expectations = [] | 408 self.expectations = [] |
| 406 self.parsed_expectations = set() | 409 self.parsed_expectations = set() |
| 407 self.comment = None | 410 self.comment = None |
| 408 self.matching_tests = [] | 411 self.matching_tests = [] |
| 409 self.warnings = [] | 412 self.warnings = [] |
| 410 self.is_skipped_outside_expectations_file = False | 413 self.is_skipped_outside_expectations_file = False |
| 411 | 414 |
| 412 def __str__(self): | 415 def __str__(self): |
| 413 return "TestExpectationLine{name=%s, matching_configurations=%s, origina
l_string=%s}" % (self.name, self.matching_configurations, self.original_string) | 416 return "TestExpectationLine{name=%s, matching_configurations=%s, origina
l_string=%s}" % ( |
| 417 self.name, self.matching_configurations, self.original_string) |
| 414 | 418 |
| 415 def __eq__(self, other): | 419 def __eq__(self, other): |
| 416 return (self.original_string == other.original_string | 420 return (self.original_string == other.original_string |
| 417 and self.filename == other.filename | 421 and self.filename == other.filename |
| 418 and self.line_numbers == other.line_numbers | 422 and self.line_numbers == other.line_numbers |
| 419 and self.name == other.name | 423 and self.name == other.name |
| 420 and self.path == other.path | 424 and self.path == other.path |
| 421 and self.bugs == other.bugs | 425 and self.bugs == other.bugs |
| 422 and self.specifiers == other.specifiers | 426 and self.specifiers == other.specifiers |
| 423 and self.parsed_specifiers == other.parsed_specifiers | 427 and self.parsed_specifiers == other.parsed_specifiers |
| (...skipping 96 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 520 | 524 |
| 521 @staticmethod | 525 @staticmethod |
| 522 def _filter_redundant_expectations(expectations): | 526 def _filter_redundant_expectations(expectations): |
| 523 if set(expectations) == set(['Pass', 'Skip']): | 527 if set(expectations) == set(['Pass', 'Skip']): |
| 524 return ['Skip'] | 528 return ['Skip'] |
| 525 if set(expectations) == set(['Pass', 'Slow']): | 529 if set(expectations) == set(['Pass', 'Slow']): |
| 526 return ['Slow'] | 530 return ['Slow'] |
| 527 return expectations | 531 return expectations |
| 528 | 532 |
| 529 @staticmethod | 533 @staticmethod |
| 530 def _format_line(bugs, specifiers, name, expectations, comment, include_spec
ifiers=True, include_expectations=True, include_comment=True): | 534 def _format_line(bugs, specifiers, name, expectations, comment, include_spec
ifiers=True, |
| 535 include_expectations=True, include_comment=True): |
| 531 new_specifiers = [] | 536 new_specifiers = [] |
| 532 new_expectations = [] | 537 new_expectations = [] |
| 533 for specifier in specifiers: | 538 for specifier in specifiers: |
| 534 # FIXME: Make this all work with the mixed-cased specifiers (e.g. Wo
ntFix, Slow, etc). | 539 # FIXME: Make this all work with the mixed-cased specifiers (e.g. Wo
ntFix, Slow, etc). |
| 535 specifier = specifier.upper() | 540 specifier = specifier.upper() |
| 536 new_specifiers.append(TestExpectationParser._inverted_configuration_
tokens.get(specifier, specifier)) | 541 new_specifiers.append(TestExpectationParser._inverted_configuration_
tokens.get(specifier, specifier)) |
| 537 | 542 |
| 538 for expectation in expectations: | 543 for expectation in expectations: |
| 539 expectation = expectation.upper() | 544 expectation = expectation.upper() |
| 540 new_expectations.append(TestExpectationParser._inverted_expectation_
tokens.get(expectation, expectation)) | 545 new_expectations.append(TestExpectationParser._inverted_expectation_
tokens.get(expectation, expectation)) |
| (...skipping 362 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 903 """Returns whether we got a result we were expecting. | 908 """Returns whether we got a result we were expecting. |
| 904 Args: | 909 Args: |
| 905 result: actual result of a test execution | 910 result: actual result of a test execution |
| 906 expected_results: set of results listed in test_expectations | 911 expected_results: set of results listed in test_expectations |
| 907 test_needs_rebaselining: whether test was marked as REBASELINE""" | 912 test_needs_rebaselining: whether test was marked as REBASELINE""" |
| 908 if not (set(expected_results) - (set(TestExpectations.NON_TEST_OUTCOME_E
XPECTATIONS))): | 913 if not (set(expected_results) - (set(TestExpectations.NON_TEST_OUTCOME_E
XPECTATIONS))): |
| 909 expected_results = set([PASS]) | 914 expected_results = set([PASS]) |
| 910 | 915 |
| 911 if result in expected_results: | 916 if result in expected_results: |
| 912 return True | 917 return True |
| 913 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and (N
EEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE in expected_resul
ts): | 918 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and ( |
| 919 NEEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE
in expected_results): |
| 914 return True | 920 return True |
| 915 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected
_results): | 921 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected
_results): |
| 916 return True | 922 return True |
| 917 if result == MISSING and test_needs_rebaselining: | 923 if result == MISSING and test_needs_rebaselining: |
| 918 return True | 924 return True |
| 919 if result == SKIP: | 925 if result == SKIP: |
| 920 return True | 926 return True |
| 921 return False | 927 return False |
| 922 | 928 |
| 923 @staticmethod | 929 @staticmethod |
| (...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 969 suffixes.add('wav') | 975 suffixes.add('wav') |
| 970 if 'MISSING' in expectations: | 976 if 'MISSING' in expectations: |
| 971 suffixes.add('txt') | 977 suffixes.add('txt') |
| 972 suffixes.add('png') | 978 suffixes.add('png') |
| 973 suffixes.add('wav') | 979 suffixes.add('wav') |
| 974 return suffixes | 980 return suffixes |
| 975 | 981 |
| 976 # FIXME: This constructor does too much work. We should move the actual pars
ing of | 982 # FIXME: This constructor does too much work. We should move the actual pars
ing of |
| 977 # the expectations into separate routines so that linting and handling overr
ides | 983 # the expectations into separate routines so that linting and handling overr
ides |
| 978 # can be controlled separately, and the constructor can be more of a no-op. | 984 # can be controlled separately, and the constructor can be more of a no-op. |
| 979 def __init__(self, port, tests=None, include_overrides=True, expectations_di
ct=None, model_all_expectations=False, is_lint_mode=False): | 985 def __init__(self, port, tests=None, include_overrides=True, expectations_di
ct=None, |
| 986 model_all_expectations=False, is_lint_mode=False): |
| 980 self._full_test_list = tests | 987 self._full_test_list = tests |
| 981 self._test_config = port.test_configuration() | 988 self._test_config = port.test_configuration() |
| 982 self._is_lint_mode = is_lint_mode | 989 self._is_lint_mode = is_lint_mode |
| 983 self._model_all_expectations = self._is_lint_mode or model_all_expectati
ons | 990 self._model_all_expectations = self._is_lint_mode or model_all_expectati
ons |
| 984 self._model = TestExpectationsModel(self._shorten_filename) | 991 self._model = TestExpectationsModel(self._shorten_filename) |
| 985 self._parser = TestExpectationParser(port, tests, self._is_lint_mode) | 992 self._parser = TestExpectationParser(port, tests, self._is_lint_mode) |
| 986 self._port = port | 993 self._port = port |
| 987 self._skipped_tests_warnings = [] | 994 self._skipped_tests_warnings = [] |
| 988 self._expectations = [] | 995 self._expectations = [] |
| 989 | 996 |
| (...skipping 176 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1166 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. | 1173 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. |
| 1167 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. | 1174 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. |
| 1168 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: | 1175 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: |
| 1169 return expectation_line.to_string(test_configuration_converter) | 1176 return expectation_line.to_string(test_configuration_converter) |
| 1170 return expectation_line.original_string | 1177 return expectation_line.original_string |
| 1171 | 1178 |
| 1172 def nones_out(expectation_line): | 1179 def nones_out(expectation_line): |
| 1173 return expectation_line is not None | 1180 return expectation_line is not None |
| 1174 | 1181 |
| 1175 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) | 1182 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) |
| OLD | NEW |