OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 189 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
200 if not self._port.test_exists(expectation_line.name) and not self._port.
test_exists(expectation_line.name + '-disabled'): | 200 if not self._port.test_exists(expectation_line.name) and not self._port.
test_exists(expectation_line.name + '-disabled'): |
201 # Log a warning here since you hit this case any | 201 # Log a warning here since you hit this case any |
202 # time you update TestExpectations without syncing | 202 # time you update TestExpectations without syncing |
203 # the LayoutTests directory | 203 # the LayoutTests directory |
204 expectation_line.warnings.append('Path does not exist.') | 204 expectation_line.warnings.append('Path does not exist.') |
205 return False | 205 return False |
206 return True | 206 return True |
207 | 207 |
208 def _collect_matching_tests(self, expectation_line): | 208 def _collect_matching_tests(self, expectation_line): |
209 """Convert the test specification to an absolute, normalized | 209 """Convert the test specification to an absolute, normalized |
210 path and make sure directories end with the OS path separator.""" | 210 path and make sure directories end with the OS path separator. |
| 211 """ |
211 if not self._all_tests: | 212 if not self._all_tests: |
212 expectation_line.matching_tests = [expectation_line.path] | 213 expectation_line.matching_tests = [expectation_line.path] |
213 return | 214 return |
214 | 215 |
215 if not expectation_line.is_file: | 216 if not expectation_line.is_file: |
216 # this is a test category, return all the tests of the category. | 217 # this is a test category, return all the tests of the category. |
217 expectation_line.matching_tests = [test for test in self._all_tests
if test.startswith(expectation_line.path)] | 218 expectation_line.matching_tests = [test for test in self._all_tests
if test.startswith(expectation_line.path)] |
218 return | 219 return |
219 | 220 |
220 # this is a test file, do a quick check if it's in the | 221 # this is a test file, do a quick check if it's in the |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
257 # FIXME: Seems like these should be classmethods on TestExpectationLine inst
ead of TestExpectationParser. | 258 # FIXME: Seems like these should be classmethods on TestExpectationLine inst
ead of TestExpectationParser. |
258 @classmethod | 259 @classmethod |
259 def _tokenize_line(cls, filename, expectation_string, line_number): | 260 def _tokenize_line(cls, filename, expectation_string, line_number): |
260 """Tokenizes a line from TestExpectations and returns an unparsed TestEx
pectationLine instance using the old format. | 261 """Tokenizes a line from TestExpectations and returns an unparsed TestEx
pectationLine instance using the old format. |
261 | 262 |
262 The new format for a test expectation line is: | 263 The new format for a test expectation line is: |
263 | 264 |
264 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations>
"]" ["#" <comment>] | 265 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations>
"]" ["#" <comment>] |
265 | 266 |
266 Any errant whitespace is not preserved. | 267 Any errant whitespace is not preserved. |
267 | |
268 """ | 268 """ |
269 expectation_line = TestExpectationLine() | 269 expectation_line = TestExpectationLine() |
270 expectation_line.original_string = expectation_string | 270 expectation_line.original_string = expectation_string |
271 expectation_line.filename = filename | 271 expectation_line.filename = filename |
272 expectation_line.line_numbers = str(line_number) | 272 expectation_line.line_numbers = str(line_number) |
273 | 273 |
274 comment_index = expectation_string.find("#") | 274 comment_index = expectation_string.find("#") |
275 if comment_index == -1: | 275 if comment_index == -1: |
276 comment_index = len(expectation_string) | 276 comment_index = len(expectation_string) |
277 else: | 277 else: |
(...skipping 341 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
619 merged_line = other_line | 619 merged_line = other_line |
620 | 620 |
621 self._test_to_expectation_line[test] = merged_line | 621 self._test_to_expectation_line[test] = merged_line |
622 | 622 |
623 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_
to_tests) | 623 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_
to_tests) |
624 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes
ts) | 624 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes
ts) |
625 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_
to_tests) | 625 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_
to_tests) |
626 | 626 |
627 def _dict_of_sets(self, strings_to_constants): | 627 def _dict_of_sets(self, strings_to_constants): |
628 """Takes a dict of strings->constants and returns a dict mapping | 628 """Takes a dict of strings->constants and returns a dict mapping |
629 each constant to an empty set.""" | 629 each constant to an empty set. |
| 630 """ |
630 d = {} | 631 d = {} |
631 for c in strings_to_constants.values(): | 632 for c in strings_to_constants.values(): |
632 d[c] = set() | 633 d[c] = set() |
633 return d | 634 return d |
634 | 635 |
635 def get_test_set(self, expectation, include_skips=True): | 636 def get_test_set(self, expectation, include_skips=True): |
636 tests = self._expectation_to_tests[expectation] | 637 tests = self._expectation_to_tests[expectation] |
637 if not include_skips: | 638 if not include_skips: |
638 tests = tests - self.get_test_set(SKIP) | 639 tests = tests - self.get_test_set(SKIP) |
639 return tests | 640 return tests |
(...skipping 19 matching lines...) Expand all Loading... |
659 return test in self._test_to_expectation_line | 660 return test in self._test_to_expectation_line |
660 | 661 |
661 def get_expectation_line(self, test): | 662 def get_expectation_line(self, test): |
662 return self._test_to_expectation_line.get(test) | 663 return self._test_to_expectation_line.get(test) |
663 | 664 |
664 def get_expectations(self, test): | 665 def get_expectations(self, test): |
665 return self._test_to_expectations[test] | 666 return self._test_to_expectations[test] |
666 | 667 |
667 def get_expectations_string(self, test): | 668 def get_expectations_string(self, test): |
668 """Returns the expectations for the given test as an uppercase string. | 669 """Returns the expectations for the given test as an uppercase string. |
669 If there are no expectations for the test, then "PASS" is returned.""" | 670 If there are no expectations for the test, then "PASS" is returned. |
| 671 """ |
670 if self.get_expectation_line(test).is_skipped_outside_expectations_file: | 672 if self.get_expectation_line(test).is_skipped_outside_expectations_file: |
671 return 'NOTRUN' | 673 return 'NOTRUN' |
672 | 674 |
673 expectations = self.get_expectations(test) | 675 expectations = self.get_expectations(test) |
674 retval = [] | 676 retval = [] |
675 | 677 |
676 # FIXME: WontFix should cause the test to get skipped without artificial
ly adding SKIP to the expectations list. | 678 # FIXME: WontFix should cause the test to get skipped without artificial
ly adding SKIP to the expectations list. |
677 if WONTFIX in expectations and SKIP in expectations: | 679 if WONTFIX in expectations and SKIP in expectations: |
678 expectations.remove(SKIP) | 680 expectations.remove(SKIP) |
679 | 681 |
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
712 | 714 |
713 self._clear_expectations_for_test(test) | 715 self._clear_expectations_for_test(test) |
714 self._test_to_expectation_line[test] = expectation_line | 716 self._test_to_expectation_line[test] = expectation_line |
715 self._add_test(test, expectation_line) | 717 self._add_test(test, expectation_line) |
716 | 718 |
717 def _add_test(self, test, expectation_line): | 719 def _add_test(self, test, expectation_line): |
718 """Sets the expected state for a given test. | 720 """Sets the expected state for a given test. |
719 | 721 |
720 This routine assumes the test has not been added before. If it has, | 722 This routine assumes the test has not been added before. If it has, |
721 use _clear_expectations_for_test() to reset the state prior to | 723 use _clear_expectations_for_test() to reset the state prior to |
722 calling this.""" | 724 calling this. |
| 725 """ |
723 self._test_to_expectations[test] = expectation_line.parsed_expectations | 726 self._test_to_expectations[test] = expectation_line.parsed_expectations |
724 for expectation in expectation_line.parsed_expectations: | 727 for expectation in expectation_line.parsed_expectations: |
725 self._expectation_to_tests[expectation].add(test) | 728 self._expectation_to_tests[expectation].add(test) |
726 | 729 |
727 self._test_to_specifiers[test] = expectation_line.specifiers | 730 self._test_to_specifiers[test] = expectation_line.specifiers |
728 | 731 |
729 if WONTFIX in expectation_line.parsed_expectations: | 732 if WONTFIX in expectation_line.parsed_expectations: |
730 self._timeline_to_tests[WONTFIX].add(test) | 733 self._timeline_to_tests[WONTFIX].add(test) |
731 else: | 734 else: |
732 self._timeline_to_tests[NOW].add(test) | 735 self._timeline_to_tests[NOW].add(test) |
(...skipping 17 matching lines...) Expand all Loading... |
750 self._test_to_expectations.pop(test, '') | 753 self._test_to_expectations.pop(test, '') |
751 self._remove_from_sets(test, self._expectation_to_tests) | 754 self._remove_from_sets(test, self._expectation_to_tests) |
752 self._remove_from_sets(test, self._timeline_to_tests) | 755 self._remove_from_sets(test, self._timeline_to_tests) |
753 self._remove_from_sets(test, self._result_type_to_tests) | 756 self._remove_from_sets(test, self._result_type_to_tests) |
754 | 757 |
755 def _remove_from_sets(self, test, dict_of_sets_of_tests): | 758 def _remove_from_sets(self, test, dict_of_sets_of_tests): |
756 """Removes the given test from the sets in the dictionary. | 759 """Removes the given test from the sets in the dictionary. |
757 | 760 |
758 Args: | 761 Args: |
759 test: test to look for | 762 test: test to look for |
760 dict: dict of sets of files""" | 763 dict: dict of sets of files |
| 764 """ |
761 for set_of_tests in dict_of_sets_of_tests.itervalues(): | 765 for set_of_tests in dict_of_sets_of_tests.itervalues(): |
762 if test in set_of_tests: | 766 if test in set_of_tests: |
763 set_of_tests.remove(test) | 767 set_of_tests.remove(test) |
764 | 768 |
765 def _already_seen_better_match(self, test, expectation_line): | 769 def _already_seen_better_match(self, test, expectation_line): |
766 """Returns whether we've seen a better match already in the file. | 770 """Returns whether we've seen a better match already in the file. |
767 | 771 |
768 Returns True if we've already seen a expectation_line.name that matches
more of the test | 772 Returns True if we've already seen a expectation_line.name that matches
more of the test |
769 than this path does | 773 than this path does |
770 """ | 774 """ |
(...skipping 137 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
908 def expectation_from_string(cls, string): | 912 def expectation_from_string(cls, string): |
909 assert ' ' not in string # This only handles one expectation at a time. | 913 assert ' ' not in string # This only handles one expectation at a time. |
910 return cls.EXPECTATIONS.get(string.lower()) | 914 return cls.EXPECTATIONS.get(string.lower()) |
911 | 915 |
912 @staticmethod | 916 @staticmethod |
913 def result_was_expected(result, expected_results, test_needs_rebaselining): | 917 def result_was_expected(result, expected_results, test_needs_rebaselining): |
914 """Returns whether we got a result we were expecting. | 918 """Returns whether we got a result we were expecting. |
915 Args: | 919 Args: |
916 result: actual result of a test execution | 920 result: actual result of a test execution |
917 expected_results: set of results listed in test_expectations | 921 expected_results: set of results listed in test_expectations |
918 test_needs_rebaselining: whether test was marked as REBASELINE""" | 922 test_needs_rebaselining: whether test was marked as REBASELINE |
| 923 """ |
919 if not set(expected_results) - set(TestExpectations.NON_TEST_OUTCOME_EXP
ECTATIONS): | 924 if not set(expected_results) - set(TestExpectations.NON_TEST_OUTCOME_EXP
ECTATIONS): |
920 expected_results = set([PASS]) | 925 expected_results = set([PASS]) |
921 | 926 |
922 if result in expected_results: | 927 if result in expected_results: |
923 return True | 928 return True |
924 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and ( | 929 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and ( |
925 NEEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE
in expected_results): | 930 NEEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE
in expected_results): |
926 return True | 931 return True |
927 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected
_results): | 932 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected
_results): |
928 return True | 933 return True |
929 if result == MISSING and test_needs_rebaselining: | 934 if result == MISSING and test_needs_rebaselining: |
930 return True | 935 return True |
931 if result == SKIP: | 936 if result == SKIP: |
932 return True | 937 return True |
933 return False | 938 return False |
934 | 939 |
935 @staticmethod | 940 @staticmethod |
936 def remove_pixel_failures(expected_results): | 941 def remove_pixel_failures(expected_results): |
937 """Returns a copy of the expected results for a test, except that we | 942 """Returns a copy of the expected results for a test, except that we |
938 drop any pixel failures and return the remaining expectations. For examp
le, | 943 drop any pixel failures and return the remaining expectations. For examp
le, |
939 if we're not running pixel tests, then tests expected to fail as IMAGE | 944 if we're not running pixel tests, then tests expected to fail as IMAGE |
940 will PASS.""" | 945 will PASS. |
| 946 """ |
941 expected_results = expected_results.copy() | 947 expected_results = expected_results.copy() |
942 if IMAGE in expected_results: | 948 if IMAGE in expected_results: |
943 expected_results.remove(IMAGE) | 949 expected_results.remove(IMAGE) |
944 expected_results.add(PASS) | 950 expected_results.add(PASS) |
945 return expected_results | 951 return expected_results |
946 | 952 |
947 @staticmethod | 953 @staticmethod |
948 def remove_non_sanitizer_failures(expected_results): | 954 def remove_non_sanitizer_failures(expected_results): |
949 """Returns a copy of the expected results for a test, except that we | 955 """Returns a copy of the expected results for a test, except that we |
950 drop any failures that the sanitizers don't care about.""" | 956 drop any failures that the sanitizers don't care about. |
| 957 """ |
951 expected_results = expected_results.copy() | 958 expected_results = expected_results.copy() |
952 for result in (IMAGE, FAIL, IMAGE_PLUS_TEXT): | 959 for result in (IMAGE, FAIL, IMAGE_PLUS_TEXT): |
953 if result in expected_results: | 960 if result in expected_results: |
954 expected_results.remove(result) | 961 expected_results.remove(result) |
955 expected_results.add(PASS) | 962 expected_results.add(PASS) |
956 return expected_results | 963 return expected_results |
957 | 964 |
958 @staticmethod | 965 @staticmethod |
959 def has_pixel_failures(actual_results): | 966 def has_pixel_failures(actual_results): |
960 return IMAGE in actual_results or FAIL in actual_results | 967 return IMAGE in actual_results or FAIL in actual_results |
(...skipping 228 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1189 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. | 1196 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. |
1190 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. | 1197 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. |
1191 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: | 1198 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: |
1192 return expectation_line.to_string(test_configuration_converter) | 1199 return expectation_line.to_string(test_configuration_converter) |
1193 return expectation_line.original_string | 1200 return expectation_line.original_string |
1194 | 1201 |
1195 def nones_out(expectation_line): | 1202 def nones_out(expectation_line): |
1196 return expectation_line is not None | 1203 return expectation_line is not None |
1197 | 1204 |
1198 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) | 1205 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) |
OLD | NEW |