| OLD | NEW |
| 1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 189 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 200 if not self._port.test_exists(expectation_line.name) and not self._port.
test_exists(expectation_line.name + '-disabled'): | 200 if not self._port.test_exists(expectation_line.name) and not self._port.
test_exists(expectation_line.name + '-disabled'): |
| 201 # Log a warning here since you hit this case any | 201 # Log a warning here since you hit this case any |
| 202 # time you update TestExpectations without syncing | 202 # time you update TestExpectations without syncing |
| 203 # the LayoutTests directory | 203 # the LayoutTests directory |
| 204 expectation_line.warnings.append('Path does not exist.') | 204 expectation_line.warnings.append('Path does not exist.') |
| 205 return False | 205 return False |
| 206 return True | 206 return True |
| 207 | 207 |
| 208 def _collect_matching_tests(self, expectation_line): | 208 def _collect_matching_tests(self, expectation_line): |
| 209 """Convert the test specification to an absolute, normalized | 209 """Convert the test specification to an absolute, normalized |
| 210 path and make sure directories end with the OS path separator.""" | 210 path and make sure directories end with the OS path separator. |
| 211 """ |
| 211 if not self._all_tests: | 212 if not self._all_tests: |
| 212 expectation_line.matching_tests = [expectation_line.path] | 213 expectation_line.matching_tests = [expectation_line.path] |
| 213 return | 214 return |
| 214 | 215 |
| 215 if not expectation_line.is_file: | 216 if not expectation_line.is_file: |
| 216 # this is a test category, return all the tests of the category. | 217 # this is a test category, return all the tests of the category. |
| 217 expectation_line.matching_tests = [test for test in self._all_tests
if test.startswith(expectation_line.path)] | 218 expectation_line.matching_tests = [test for test in self._all_tests
if test.startswith(expectation_line.path)] |
| 218 return | 219 return |
| 219 | 220 |
| 220 # this is a test file, do a quick check if it's in the | 221 # this is a test file, do a quick check if it's in the |
| (...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 257 # FIXME: Seems like these should be classmethods on TestExpectationLine inst
ead of TestExpectationParser. | 258 # FIXME: Seems like these should be classmethods on TestExpectationLine inst
ead of TestExpectationParser. |
| 258 @classmethod | 259 @classmethod |
| 259 def _tokenize_line(cls, filename, expectation_string, line_number): | 260 def _tokenize_line(cls, filename, expectation_string, line_number): |
| 260 """Tokenizes a line from TestExpectations and returns an unparsed TestEx
pectationLine instance using the old format. | 261 """Tokenizes a line from TestExpectations and returns an unparsed TestEx
pectationLine instance using the old format. |
| 261 | 262 |
| 262 The new format for a test expectation line is: | 263 The new format for a test expectation line is: |
| 263 | 264 |
| 264 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations>
"]" ["#" <comment>] | 265 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations>
"]" ["#" <comment>] |
| 265 | 266 |
| 266 Any errant whitespace is not preserved. | 267 Any errant whitespace is not preserved. |
| 267 | |
| 268 """ | 268 """ |
| 269 expectation_line = TestExpectationLine() | 269 expectation_line = TestExpectationLine() |
| 270 expectation_line.original_string = expectation_string | 270 expectation_line.original_string = expectation_string |
| 271 expectation_line.filename = filename | 271 expectation_line.filename = filename |
| 272 expectation_line.line_numbers = str(line_number) | 272 expectation_line.line_numbers = str(line_number) |
| 273 | 273 |
| 274 comment_index = expectation_string.find("#") | 274 comment_index = expectation_string.find("#") |
| 275 if comment_index == -1: | 275 if comment_index == -1: |
| 276 comment_index = len(expectation_string) | 276 comment_index = len(expectation_string) |
| 277 else: | 277 else: |
| (...skipping 341 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 619 merged_line = other_line | 619 merged_line = other_line |
| 620 | 620 |
| 621 self._test_to_expectation_line[test] = merged_line | 621 self._test_to_expectation_line[test] = merged_line |
| 622 | 622 |
| 623 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_
to_tests) | 623 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_
to_tests) |
| 624 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes
ts) | 624 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes
ts) |
| 625 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_
to_tests) | 625 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_
to_tests) |
| 626 | 626 |
| 627 def _dict_of_sets(self, strings_to_constants): | 627 def _dict_of_sets(self, strings_to_constants): |
| 628 """Takes a dict of strings->constants and returns a dict mapping | 628 """Takes a dict of strings->constants and returns a dict mapping |
| 629 each constant to an empty set.""" | 629 each constant to an empty set. |
| 630 """ |
| 630 d = {} | 631 d = {} |
| 631 for c in strings_to_constants.values(): | 632 for c in strings_to_constants.values(): |
| 632 d[c] = set() | 633 d[c] = set() |
| 633 return d | 634 return d |
| 634 | 635 |
| 635 def get_test_set(self, expectation, include_skips=True): | 636 def get_test_set(self, expectation, include_skips=True): |
| 636 tests = self._expectation_to_tests[expectation] | 637 tests = self._expectation_to_tests[expectation] |
| 637 if not include_skips: | 638 if not include_skips: |
| 638 tests = tests - self.get_test_set(SKIP) | 639 tests = tests - self.get_test_set(SKIP) |
| 639 return tests | 640 return tests |
| (...skipping 19 matching lines...) Expand all Loading... |
| 659 return test in self._test_to_expectation_line | 660 return test in self._test_to_expectation_line |
| 660 | 661 |
| 661 def get_expectation_line(self, test): | 662 def get_expectation_line(self, test): |
| 662 return self._test_to_expectation_line.get(test) | 663 return self._test_to_expectation_line.get(test) |
| 663 | 664 |
| 664 def get_expectations(self, test): | 665 def get_expectations(self, test): |
| 665 return self._test_to_expectations[test] | 666 return self._test_to_expectations[test] |
| 666 | 667 |
| 667 def get_expectations_string(self, test): | 668 def get_expectations_string(self, test): |
| 668 """Returns the expectations for the given test as an uppercase string. | 669 """Returns the expectations for the given test as an uppercase string. |
| 669 If there are no expectations for the test, then "PASS" is returned.""" | 670 If there are no expectations for the test, then "PASS" is returned. |
| 671 """ |
| 670 if self.get_expectation_line(test).is_skipped_outside_expectations_file: | 672 if self.get_expectation_line(test).is_skipped_outside_expectations_file: |
| 671 return 'NOTRUN' | 673 return 'NOTRUN' |
| 672 | 674 |
| 673 expectations = self.get_expectations(test) | 675 expectations = self.get_expectations(test) |
| 674 retval = [] | 676 retval = [] |
| 675 | 677 |
| 676 # FIXME: WontFix should cause the test to get skipped without artificial
ly adding SKIP to the expectations list. | 678 # FIXME: WontFix should cause the test to get skipped without artificial
ly adding SKIP to the expectations list. |
| 677 if WONTFIX in expectations and SKIP in expectations: | 679 if WONTFIX in expectations and SKIP in expectations: |
| 678 expectations.remove(SKIP) | 680 expectations.remove(SKIP) |
| 679 | 681 |
| (...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 712 | 714 |
| 713 self._clear_expectations_for_test(test) | 715 self._clear_expectations_for_test(test) |
| 714 self._test_to_expectation_line[test] = expectation_line | 716 self._test_to_expectation_line[test] = expectation_line |
| 715 self._add_test(test, expectation_line) | 717 self._add_test(test, expectation_line) |
| 716 | 718 |
| 717 def _add_test(self, test, expectation_line): | 719 def _add_test(self, test, expectation_line): |
| 718 """Sets the expected state for a given test. | 720 """Sets the expected state for a given test. |
| 719 | 721 |
| 720 This routine assumes the test has not been added before. If it has, | 722 This routine assumes the test has not been added before. If it has, |
| 721 use _clear_expectations_for_test() to reset the state prior to | 723 use _clear_expectations_for_test() to reset the state prior to |
| 722 calling this.""" | 724 calling this. |
| 725 """ |
| 723 self._test_to_expectations[test] = expectation_line.parsed_expectations | 726 self._test_to_expectations[test] = expectation_line.parsed_expectations |
| 724 for expectation in expectation_line.parsed_expectations: | 727 for expectation in expectation_line.parsed_expectations: |
| 725 self._expectation_to_tests[expectation].add(test) | 728 self._expectation_to_tests[expectation].add(test) |
| 726 | 729 |
| 727 self._test_to_specifiers[test] = expectation_line.specifiers | 730 self._test_to_specifiers[test] = expectation_line.specifiers |
| 728 | 731 |
| 729 if WONTFIX in expectation_line.parsed_expectations: | 732 if WONTFIX in expectation_line.parsed_expectations: |
| 730 self._timeline_to_tests[WONTFIX].add(test) | 733 self._timeline_to_tests[WONTFIX].add(test) |
| 731 else: | 734 else: |
| 732 self._timeline_to_tests[NOW].add(test) | 735 self._timeline_to_tests[NOW].add(test) |
| (...skipping 17 matching lines...) Expand all Loading... |
| 750 self._test_to_expectations.pop(test, '') | 753 self._test_to_expectations.pop(test, '') |
| 751 self._remove_from_sets(test, self._expectation_to_tests) | 754 self._remove_from_sets(test, self._expectation_to_tests) |
| 752 self._remove_from_sets(test, self._timeline_to_tests) | 755 self._remove_from_sets(test, self._timeline_to_tests) |
| 753 self._remove_from_sets(test, self._result_type_to_tests) | 756 self._remove_from_sets(test, self._result_type_to_tests) |
| 754 | 757 |
| 755 def _remove_from_sets(self, test, dict_of_sets_of_tests): | 758 def _remove_from_sets(self, test, dict_of_sets_of_tests): |
| 756 """Removes the given test from the sets in the dictionary. | 759 """Removes the given test from the sets in the dictionary. |
| 757 | 760 |
| 758 Args: | 761 Args: |
| 759 test: test to look for | 762 test: test to look for |
| 760 dict: dict of sets of files""" | 763 dict: dict of sets of files |
| 764 """ |
| 761 for set_of_tests in dict_of_sets_of_tests.itervalues(): | 765 for set_of_tests in dict_of_sets_of_tests.itervalues(): |
| 762 if test in set_of_tests: | 766 if test in set_of_tests: |
| 763 set_of_tests.remove(test) | 767 set_of_tests.remove(test) |
| 764 | 768 |
| 765 def _already_seen_better_match(self, test, expectation_line): | 769 def _already_seen_better_match(self, test, expectation_line): |
| 766 """Returns whether we've seen a better match already in the file. | 770 """Returns whether we've seen a better match already in the file. |
| 767 | 771 |
| 768 Returns True if we've already seen a expectation_line.name that matches
more of the test | 772 Returns True if we've already seen a expectation_line.name that matches
more of the test |
| 769 than this path does | 773 than this path does |
| 770 """ | 774 """ |
| (...skipping 137 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 908 def expectation_from_string(cls, string): | 912 def expectation_from_string(cls, string): |
| 909 assert ' ' not in string # This only handles one expectation at a time. | 913 assert ' ' not in string # This only handles one expectation at a time. |
| 910 return cls.EXPECTATIONS.get(string.lower()) | 914 return cls.EXPECTATIONS.get(string.lower()) |
| 911 | 915 |
| 912 @staticmethod | 916 @staticmethod |
| 913 def result_was_expected(result, expected_results, test_needs_rebaselining): | 917 def result_was_expected(result, expected_results, test_needs_rebaselining): |
| 914 """Returns whether we got a result we were expecting. | 918 """Returns whether we got a result we were expecting. |
| 915 Args: | 919 Args: |
| 916 result: actual result of a test execution | 920 result: actual result of a test execution |
| 917 expected_results: set of results listed in test_expectations | 921 expected_results: set of results listed in test_expectations |
| 918 test_needs_rebaselining: whether test was marked as REBASELINE""" | 922 test_needs_rebaselining: whether test was marked as REBASELINE |
| 923 """ |
| 919 if not set(expected_results) - set(TestExpectations.NON_TEST_OUTCOME_EXP
ECTATIONS): | 924 if not set(expected_results) - set(TestExpectations.NON_TEST_OUTCOME_EXP
ECTATIONS): |
| 920 expected_results = set([PASS]) | 925 expected_results = set([PASS]) |
| 921 | 926 |
| 922 if result in expected_results: | 927 if result in expected_results: |
| 923 return True | 928 return True |
| 924 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and ( | 929 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and ( |
| 925 NEEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE
in expected_results): | 930 NEEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE
in expected_results): |
| 926 return True | 931 return True |
| 927 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected
_results): | 932 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected
_results): |
| 928 return True | 933 return True |
| 929 if result == MISSING and test_needs_rebaselining: | 934 if result == MISSING and test_needs_rebaselining: |
| 930 return True | 935 return True |
| 931 if result == SKIP: | 936 if result == SKIP: |
| 932 return True | 937 return True |
| 933 return False | 938 return False |
| 934 | 939 |
| 935 @staticmethod | 940 @staticmethod |
| 936 def remove_pixel_failures(expected_results): | 941 def remove_pixel_failures(expected_results): |
| 937 """Returns a copy of the expected results for a test, except that we | 942 """Returns a copy of the expected results for a test, except that we |
| 938 drop any pixel failures and return the remaining expectations. For examp
le, | 943 drop any pixel failures and return the remaining expectations. For examp
le, |
| 939 if we're not running pixel tests, then tests expected to fail as IMAGE | 944 if we're not running pixel tests, then tests expected to fail as IMAGE |
| 940 will PASS.""" | 945 will PASS. |
| 946 """ |
| 941 expected_results = expected_results.copy() | 947 expected_results = expected_results.copy() |
| 942 if IMAGE in expected_results: | 948 if IMAGE in expected_results: |
| 943 expected_results.remove(IMAGE) | 949 expected_results.remove(IMAGE) |
| 944 expected_results.add(PASS) | 950 expected_results.add(PASS) |
| 945 return expected_results | 951 return expected_results |
| 946 | 952 |
| 947 @staticmethod | 953 @staticmethod |
| 948 def remove_non_sanitizer_failures(expected_results): | 954 def remove_non_sanitizer_failures(expected_results): |
| 949 """Returns a copy of the expected results for a test, except that we | 955 """Returns a copy of the expected results for a test, except that we |
| 950 drop any failures that the sanitizers don't care about.""" | 956 drop any failures that the sanitizers don't care about. |
| 957 """ |
| 951 expected_results = expected_results.copy() | 958 expected_results = expected_results.copy() |
| 952 for result in (IMAGE, FAIL, IMAGE_PLUS_TEXT): | 959 for result in (IMAGE, FAIL, IMAGE_PLUS_TEXT): |
| 953 if result in expected_results: | 960 if result in expected_results: |
| 954 expected_results.remove(result) | 961 expected_results.remove(result) |
| 955 expected_results.add(PASS) | 962 expected_results.add(PASS) |
| 956 return expected_results | 963 return expected_results |
| 957 | 964 |
| 958 @staticmethod | 965 @staticmethod |
| 959 def has_pixel_failures(actual_results): | 966 def has_pixel_failures(actual_results): |
| 960 return IMAGE in actual_results or FAIL in actual_results | 967 return IMAGE in actual_results or FAIL in actual_results |
| (...skipping 228 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1189 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. | 1196 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. |
| 1190 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. | 1197 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. |
| 1191 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: | 1198 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: |
| 1192 return expectation_line.to_string(test_configuration_converter) | 1199 return expectation_line.to_string(test_configuration_converter) |
| 1193 return expectation_line.original_string | 1200 return expectation_line.original_string |
| 1194 | 1201 |
| 1195 def nones_out(expectation_line): | 1202 def nones_out(expectation_line): |
| 1196 return expectation_line is not None | 1203 return expectation_line is not None |
| 1197 | 1204 |
| 1198 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) | 1205 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) |
| OLD | NEW |