OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
11 # in the documentation and/or other materials provided with the | 11 # in the documentation and/or other materials provided with the |
12 # distribution. | 12 # distribution. |
13 # * Neither the name of Google Inc. nor the names of its | 13 # * Neither the name of Google Inc. nor the names of its |
14 # contributors may be used to endorse or promote products derived from | 14 # contributors may be used to endorse or promote products derived from |
15 # this software without specific prior written permission. | 15 # this software without specific prior written permission. |
16 # | 16 # |
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
28 | |
29 """A helper class for reading in and dealing with tests expectations | 28 """A helper class for reading in and dealing with tests expectations |
30 for layout tests. | 29 for layout tests. |
31 """ | 30 """ |
32 | 31 |
33 from collections import defaultdict | 32 from collections import defaultdict |
34 | 33 |
35 import logging | 34 import logging |
36 import re | 35 import re |
37 | 36 |
38 from webkitpy.layout_tests.models.test_configuration import TestConfigurationCon
verter | 37 from webkitpy.layout_tests.models.test_configuration import TestConfigurationCon
verter |
39 | 38 |
40 _log = logging.getLogger(__name__) | 39 _log = logging.getLogger(__name__) |
41 | 40 |
42 | |
43 # Test expectation and specifier constants. | 41 # Test expectation and specifier constants. |
44 # | 42 # |
45 # FIXME: range() starts with 0 which makes if expectation checks harder | 43 # FIXME: range() starts with 0 which makes if expectation checks harder |
46 # as PASS is 0. | 44 # as PASS is 0. |
47 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, LEAK, SKIP, WO
NTFIX, | 45 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, LEAK, SKIP, WO
NTFIX, SLOW, REBASELINE, NEEDS_REBASELINE, |
48 SLOW, REBASELINE, NEEDS_REBASELINE, NEEDS_MANUAL_REBASELINE, MISSING, FLAKY, NO
W, NONE) = range(19) | 46 NEEDS_MANUAL_REBASELINE, MISSING, FLAKY, NOW, NONE) = range(19) |
49 | 47 |
50 # FIXME: Perhas these two routines should be part of the Port instead? | 48 # FIXME: Perhas these two routines should be part of the Port instead? |
51 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt') | 49 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt') |
52 | 50 |
53 WEBKIT_BUG_PREFIX = 'webkit.org/b/' | 51 WEBKIT_BUG_PREFIX = 'webkit.org/b/' |
54 CHROMIUM_BUG_PREFIX = 'crbug.com/' | 52 CHROMIUM_BUG_PREFIX = 'crbug.com/' |
55 V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id=' | 53 V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id=' |
56 NAMED_BUG_PREFIX = 'Bug(' | 54 NAMED_BUG_PREFIX = 'Bug(' |
57 | 55 |
58 MISSING_KEYWORD = 'Missing' | 56 MISSING_KEYWORD = 'Missing' |
59 NEEDS_REBASELINE_KEYWORD = 'NeedsRebaseline' | 57 NEEDS_REBASELINE_KEYWORD = 'NeedsRebaseline' |
60 NEEDS_MANUAL_REBASELINE_KEYWORD = 'NeedsManualRebaseline' | 58 NEEDS_MANUAL_REBASELINE_KEYWORD = 'NeedsManualRebaseline' |
61 | 59 |
62 | 60 |
63 class ParseError(Exception): | 61 class ParseError(Exception): |
64 def __init__(self, warnings): | 62 def __init__(self, warnings): |
65 super(ParseError, self).__init__() | 63 super(ParseError, self).__init__() |
66 self.warnings = warnings | 64 self.warnings = warnings |
67 | 65 |
68 def __str__(self): | 66 def __str__(self): |
69 return '\n'.join(map(str, self.warnings)) | 67 return '\n'.join(map(str, self.warnings)) |
70 | 68 |
71 def __repr__(self): | 69 def __repr__(self): |
72 return 'ParseError(warnings=%s)' % self.warnings | 70 return 'ParseError(warnings=%s)' % self.warnings |
73 | 71 |
74 | 72 |
75 class TestExpectationParser(object): | 73 class TestExpectationParser(object): |
76 """Provides parsing facilities for lines in the test_expectation.txt file.""
" | 74 """Provides parsing facilities for lines in the test_expectation.txt file.""
" |
77 | 75 |
78 # FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make the
case studdly-caps to match the actual file contents. | 76 # FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make |
| 77 # the case studdly-caps to match the actual file contents. |
79 REBASELINE_MODIFIER = 'rebaseline' | 78 REBASELINE_MODIFIER = 'rebaseline' |
80 NEEDS_REBASELINE_MODIFIER = 'needsrebaseline' | 79 NEEDS_REBASELINE_MODIFIER = 'needsrebaseline' |
81 NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline' | 80 NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline' |
82 PASS_EXPECTATION = 'pass' | 81 PASS_EXPECTATION = 'pass' |
83 SKIP_MODIFIER = 'skip' | 82 SKIP_MODIFIER = 'skip' |
84 SLOW_MODIFIER = 'slow' | 83 SLOW_MODIFIER = 'slow' |
85 WONTFIX_MODIFIER = 'wontfix' | 84 WONTFIX_MODIFIER = 'wontfix' |
86 | 85 |
87 TIMEOUT_EXPECTATION = 'timeout' | 86 TIMEOUT_EXPECTATION = 'timeout' |
88 | 87 |
89 MISSING_BUG_WARNING = 'Test lacks BUG specifier.' | 88 MISSING_BUG_WARNING = 'Test lacks BUG specifier.' |
90 | 89 |
91 def __init__(self, port, all_tests, is_lint_mode): | 90 def __init__(self, port, all_tests, is_lint_mode): |
92 self._port = port | 91 self._port = port |
93 self._test_configuration_converter = TestConfigurationConverter(set(port
.all_test_configurations()), port.configuration_specifier_macros()) | 92 self._test_configuration_converter = TestConfigurationConverter( |
| 93 set(port.all_test_configurations()), port.configuration_specifier_ma
cros()) |
94 | 94 |
95 if all_tests: | 95 if all_tests: |
96 self._all_tests = set(all_tests) | 96 self._all_tests = set(all_tests) |
97 else: | 97 else: |
98 self._all_tests = set() | 98 self._all_tests = set() |
99 | 99 |
100 self._is_lint_mode = is_lint_mode | 100 self._is_lint_mode = is_lint_mode |
101 | 101 |
102 def parse(self, filename, expectations_string): | 102 def parse(self, filename, expectations_string): |
103 expectation_lines = [] | 103 expectation_lines = [] |
(...skipping 11 matching lines...) Expand all Loading... |
115 expectation_line.name = test_name | 115 expectation_line.name = test_name |
116 expectation_line.filename = file_name | 116 expectation_line.filename = file_name |
117 expectation_line.expectations = expectations | 117 expectation_line.expectations = expectations |
118 return expectation_line | 118 return expectation_line |
119 | 119 |
120 def expectation_line_for_test(self, test_name, expectations): | 120 def expectation_line_for_test(self, test_name, expectations): |
121 expectation_line = self._create_expectation_line(test_name, expectations
, '<Bot TestExpectations>') | 121 expectation_line = self._create_expectation_line(test_name, expectations
, '<Bot TestExpectations>') |
122 self._parse_line(expectation_line) | 122 self._parse_line(expectation_line) |
123 return expectation_line | 123 return expectation_line |
124 | 124 |
125 | |
126 def expectation_for_skipped_test(self, test_name): | 125 def expectation_for_skipped_test(self, test_name): |
127 if not self._port.test_exists(test_name): | 126 if not self._port.test_exists(test_name): |
128 _log.warning('The following test %s from the Skipped list doesn\'t e
xist' % test_name) | 127 _log.warning('The following test %s from the Skipped list doesn\'t e
xist' % test_name) |
129 expectation_line = self._create_expectation_line(test_name, [TestExpecta
tionParser.PASS_EXPECTATION], '<Skipped file>') | 128 expectation_line = self._create_expectation_line(test_name, [TestExpecta
tionParser.PASS_EXPECTATION], '<Skipped file>') |
130 expectation_line.expectations = [TestExpectationParser.SKIP_MODIFIER, Te
stExpectationParser.WONTFIX_MODIFIER] | 129 expectation_line.expectations = [TestExpectationParser.SKIP_MODIFIER, Te
stExpectationParser.WONTFIX_MODIFIER] |
131 expectation_line.is_skipped_outside_expectations_file = True | 130 expectation_line.is_skipped_outside_expectations_file = True |
132 self._parse_line(expectation_line) | 131 self._parse_line(expectation_line) |
133 return expectation_line | 132 return expectation_line |
134 | 133 |
135 def _parse_line(self, expectation_line): | 134 def _parse_line(self, expectation_line): |
(...skipping 15 matching lines...) Expand all Loading... |
151 self._parse_expectations(expectation_line) | 150 self._parse_expectations(expectation_line) |
152 | 151 |
153 def _parse_specifier(self, specifier): | 152 def _parse_specifier(self, specifier): |
154 return specifier.lower() | 153 return specifier.lower() |
155 | 154 |
156 def _parse_specifiers(self, expectation_line): | 155 def _parse_specifiers(self, expectation_line): |
157 if self._is_lint_mode: | 156 if self._is_lint_mode: |
158 self._lint_line(expectation_line) | 157 self._lint_line(expectation_line) |
159 | 158 |
160 parsed_specifiers = set([self._parse_specifier(specifier) for specifier
in expectation_line.specifiers]) | 159 parsed_specifiers = set([self._parse_specifier(specifier) for specifier
in expectation_line.specifiers]) |
161 expectation_line.matching_configurations = self._test_configuration_conv
erter.to_config_set(parsed_specifiers, expectation_line.warnings) | 160 expectation_line.matching_configurations = self._test_configuration_conv
erter.to_config_set(parsed_specifiers, |
| 161
expectation_line.warnings) |
162 | 162 |
163 def _lint_line(self, expectation_line): | 163 def _lint_line(self, expectation_line): |
164 expectations = [expectation.lower() for expectation in expectation_line.
expectations] | 164 expectations = [expectation.lower() for expectation in expectation_line.
expectations] |
165 if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectatio
ns: | 165 if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectatio
ns: |
166 expectation_line.warnings.append(self.MISSING_BUG_WARNING) | 166 expectation_line.warnings.append(self.MISSING_BUG_WARNING) |
167 if self.REBASELINE_MODIFIER in expectations: | 167 if self.REBASELINE_MODIFIER in expectations: |
168 expectation_line.warnings.append('REBASELINE should only be used for
running rebaseline.py. Cannot be checked in.') | 168 expectation_line.warnings.append('REBASELINE should only be used for
running rebaseline.py. Cannot be checked in.') |
169 | 169 |
170 if self.NEEDS_REBASELINE_MODIFIER in expectations or self.NEEDS_MANUAL_R
EBASELINE_MODIFIER in expectations: | 170 if self.NEEDS_REBASELINE_MODIFIER in expectations or self.NEEDS_MANUAL_R
EBASELINE_MODIFIER in expectations: |
171 for test in expectation_line.matching_tests: | 171 for test in expectation_line.matching_tests: |
172 if self._port.reference_files(test): | 172 if self._port.reference_files(test): |
173 expectation_line.warnings.append('A reftest cannot be marked
as NeedsRebaseline/NeedsManualRebaseline') | 173 expectation_line.warnings.append('A reftest cannot be marked
as NeedsRebaseline/NeedsManualRebaseline') |
174 | 174 |
175 specifiers = [specifier.lower() for specifier in expectation_line.specif
iers] | 175 specifiers = [specifier.lower() for specifier in expectation_line.specif
iers] |
176 if (self.REBASELINE_MODIFIER in expectations or self.NEEDS_REBASELINE_MO
DIFIER in expectations) and ('debug' in specifiers or 'release' in specifiers): | 176 if (self.REBASELINE_MODIFIER in expectations or self.NEEDS_REBASELINE_MO
DIFIER in expectations) and ( |
| 177 'debug' in specifiers or 'release' in specifiers): |
177 expectation_line.warnings.append('A test cannot be rebaselined for D
ebug/Release.') | 178 expectation_line.warnings.append('A test cannot be rebaselined for D
ebug/Release.') |
178 | 179 |
179 def _parse_expectations(self, expectation_line): | 180 def _parse_expectations(self, expectation_line): |
180 result = set() | 181 result = set() |
181 for part in expectation_line.expectations: | 182 for part in expectation_line.expectations: |
182 expectation = TestExpectations.expectation_from_string(part) | 183 expectation = TestExpectations.expectation_from_string(part) |
183 if expectation is None: # Careful, PASS is currently 0. | 184 if expectation is None: # Careful, PASS is currently 0. |
184 expectation_line.warnings.append('Unsupported expectation: %s' %
part) | 185 expectation_line.warnings.append('Unsupported expectation: %s' %
part) |
185 continue | 186 continue |
186 result.add(expectation) | 187 result.add(expectation) |
(...skipping 23 matching lines...) Expand all Loading... |
210 expectation_line.matching_tests = [test for test in self._all_tests
if test.startswith(expectation_line.path)] | 211 expectation_line.matching_tests = [test for test in self._all_tests
if test.startswith(expectation_line.path)] |
211 return | 212 return |
212 | 213 |
213 # this is a test file, do a quick check if it's in the | 214 # this is a test file, do a quick check if it's in the |
214 # full test suite. | 215 # full test suite. |
215 if expectation_line.path in self._all_tests: | 216 if expectation_line.path in self._all_tests: |
216 expectation_line.matching_tests.append(expectation_line.path) | 217 expectation_line.matching_tests.append(expectation_line.path) |
217 | 218 |
218 # FIXME: Update the original specifiers and remove this once the old syntax
is gone. | 219 # FIXME: Update the original specifiers and remove this once the old syntax
is gone. |
219 _configuration_tokens_list = [ | 220 _configuration_tokens_list = [ |
220 'Mac', 'Mac10.9', 'Mac10.10', 'Mac10.11', 'Retina', | 221 'Mac', |
221 'Win', 'Win7', 'Win10', | 222 'Mac10.9', |
222 'Linux', 'Precise', 'Trusty', | 223 'Mac10.10', |
| 224 'Mac10.11', |
| 225 'Retina', |
| 226 'Win', |
| 227 'Win7', |
| 228 'Win10', |
| 229 'Linux', |
| 230 'Precise', |
| 231 'Trusty', |
223 'Android', | 232 'Android', |
224 'Release', | 233 'Release', |
225 'Debug', | 234 'Debug', |
226 ] | 235 ] |
227 | 236 |
228 _configuration_tokens = dict((token, token.upper()) for token in _configurat
ion_tokens_list) | 237 _configuration_tokens = dict((token, token.upper()) for token in _configurat
ion_tokens_list) |
229 _inverted_configuration_tokens = dict((value, name) for name, value in _conf
iguration_tokens.iteritems()) | 238 _inverted_configuration_tokens = dict((value, name) for name, value in _conf
iguration_tokens.iteritems()) |
230 | 239 |
231 # FIXME: Update the original specifiers list and remove this once the old sy
ntax is gone. | 240 # FIXME: Update the original specifiers list and remove this once the old sy
ntax is gone. |
232 _expectation_tokens = { | 241 _expectation_tokens = { |
233 'Crash': 'CRASH', | 242 'Crash': 'CRASH', |
234 'Leak': 'LEAK', | 243 'Leak': 'LEAK', |
235 'Failure': 'FAIL', | 244 'Failure': 'FAIL', |
236 MISSING_KEYWORD: 'MISSING', | 245 MISSING_KEYWORD: 'MISSING', |
237 'Pass': 'PASS', | 246 'Pass': 'PASS', |
238 'Rebaseline': 'REBASELINE', | 247 'Rebaseline': 'REBASELINE', |
239 NEEDS_REBASELINE_KEYWORD: 'NEEDSREBASELINE', | 248 NEEDS_REBASELINE_KEYWORD: 'NEEDSREBASELINE', |
240 NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE', | 249 NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE', |
241 'Skip': 'SKIP', | 250 'Skip': 'SKIP', |
242 'Slow': 'SLOW', | 251 'Slow': 'SLOW', |
243 'Timeout': 'TIMEOUT', | 252 'Timeout': 'TIMEOUT', |
244 'WontFix': 'WONTFIX', | 253 'WontFix': 'WONTFIX', |
245 } | 254 } |
246 | 255 |
247 _inverted_expectation_tokens = dict([(value, name) for name, value in _expec
tation_tokens.iteritems()] + | 256 _inverted_expectation_tokens = dict([(value, name) for name, value in _expec
tation_tokens.iteritems()] + [('TEXT', 'Failure'), ( |
248 [('TEXT', 'Failure'), ('IMAGE', 'Failure
'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')]) | 257 'IMAGE', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')]) |
249 | 258 |
250 # FIXME: Seems like these should be classmethods on TestExpectationLine inst
ead of TestExpectationParser. | 259 # FIXME: Seems like these should be classmethods on TestExpectationLine inst
ead of TestExpectationParser. |
251 @classmethod | 260 @classmethod |
252 def _tokenize_line(cls, filename, expectation_string, line_number): | 261 def _tokenize_line(cls, filename, expectation_string, line_number): |
253 """Tokenizes a line from TestExpectations and returns an unparsed TestEx
pectationLine instance using the old format. | 262 """Tokenizes a line from TestExpectations and returns an unparsed TestEx
pectationLine instance using the old format. |
254 | 263 |
255 The new format for a test expectation line is: | 264 The new format for a test expectation line is: |
256 | 265 |
257 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations>
"]" ["#" <comment>] | 266 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations>
"]" ["#" <comment>] |
258 | 267 |
(...skipping 23 matching lines...) Expand all Loading... |
282 bugs = [] | 291 bugs = [] |
283 specifiers = [] | 292 specifiers = [] |
284 name = None | 293 name = None |
285 expectations = [] | 294 expectations = [] |
286 warnings = [] | 295 warnings = [] |
287 has_unrecognized_expectation = False | 296 has_unrecognized_expectation = False |
288 | 297 |
289 tokens = remaining_string.split() | 298 tokens = remaining_string.split() |
290 state = 'start' | 299 state = 'start' |
291 for token in tokens: | 300 for token in tokens: |
292 if (token.startswith(WEBKIT_BUG_PREFIX) or | 301 if (token.startswith(WEBKIT_BUG_PREFIX) or token.startswith(CHROMIUM
_BUG_PREFIX) or token.startswith(V8_BUG_PREFIX) or |
293 token.startswith(CHROMIUM_BUG_PREFIX) or | 302 token.startswith(NAMED_BUG_PREFIX)): |
294 token.startswith(V8_BUG_PREFIX) or | |
295 token.startswith(NAMED_BUG_PREFIX)): | |
296 if state != 'start': | 303 if state != 'start': |
297 warnings.append('"%s" is not at the start of the line.' % to
ken) | 304 warnings.append('"%s" is not at the start of the line.' % to
ken) |
298 break | 305 break |
299 if token.startswith(WEBKIT_BUG_PREFIX): | 306 if token.startswith(WEBKIT_BUG_PREFIX): |
300 bugs.append(token) | 307 bugs.append(token) |
301 elif token.startswith(CHROMIUM_BUG_PREFIX): | 308 elif token.startswith(CHROMIUM_BUG_PREFIX): |
302 bugs.append(token) | 309 bugs.append(token) |
303 elif token.startswith(V8_BUG_PREFIX): | 310 elif token.startswith(V8_BUG_PREFIX): |
304 bugs.append(token) | 311 bugs.append(token) |
305 else: | 312 else: |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
352 if 'WONTFIX' in expectations and 'SKIP' not in expectations: | 359 if 'WONTFIX' in expectations and 'SKIP' not in expectations: |
353 expectations.append('SKIP') | 360 expectations.append('SKIP') |
354 | 361 |
355 if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(exp
ectations) - set(['SKIP', 'WONTFIX'])): | 362 if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(exp
ectations) - set(['SKIP', 'WONTFIX'])): |
356 warnings.append('A test marked Skip or WontFix must not have other e
xpectations.') | 363 warnings.append('A test marked Skip or WontFix must not have other e
xpectations.') |
357 | 364 |
358 if 'SLOW' in expectations and 'SlowTests' not in filename: | 365 if 'SLOW' in expectations and 'SlowTests' not in filename: |
359 warnings.append('SLOW tests should ony be added to SlowTests and not
to TestExpectations.') | 366 warnings.append('SLOW tests should ony be added to SlowTests and not
to TestExpectations.') |
360 | 367 |
361 if 'WONTFIX' in expectations and ('NeverFixTests' not in filename and 'S
taleTestExpectations' not in filename): | 368 if 'WONTFIX' in expectations and ('NeverFixTests' not in filename and 'S
taleTestExpectations' not in filename): |
362 warnings.append('WONTFIX tests should ony be added to NeverFixTests
or StaleTestExpectations and not to TestExpectations.') | 369 warnings.append( |
| 370 'WONTFIX tests should ony be added to NeverFixTests or StaleTest
Expectations and not to TestExpectations.') |
363 | 371 |
364 if 'NeverFixTests' in filename and expectations != ['WONTFIX', 'SKIP']: | 372 if 'NeverFixTests' in filename and expectations != ['WONTFIX', 'SKIP']: |
365 warnings.append('Only WONTFIX expectations are allowed in NeverFixTe
sts') | 373 warnings.append('Only WONTFIX expectations are allowed in NeverFixTe
sts') |
366 | 374 |
367 if 'SlowTests' in filename and expectations != ['SLOW']: | 375 if 'SlowTests' in filename and expectations != ['SLOW']: |
368 warnings.append('Only SLOW expectations are allowed in SlowTests') | 376 warnings.append('Only SLOW expectations are allowed in SlowTests') |
369 | 377 |
370 if not expectations and not has_unrecognized_expectation: | 378 if not expectations and not has_unrecognized_expectation: |
371 warnings.append('Missing expectations.') | 379 warnings.append('Missing expectations.') |
372 | 380 |
(...skipping 25 matching lines...) Expand all Loading... |
398 self.parsed_specifiers = [] | 406 self.parsed_specifiers = [] |
399 self.matching_configurations = set() | 407 self.matching_configurations = set() |
400 self.expectations = [] | 408 self.expectations = [] |
401 self.parsed_expectations = set() | 409 self.parsed_expectations = set() |
402 self.comment = None | 410 self.comment = None |
403 self.matching_tests = [] | 411 self.matching_tests = [] |
404 self.warnings = [] | 412 self.warnings = [] |
405 self.is_skipped_outside_expectations_file = False | 413 self.is_skipped_outside_expectations_file = False |
406 | 414 |
407 def __str__(self): | 415 def __str__(self): |
408 return "TestExpectationLine{name=%s, matching_configurations=%s, origina
l_string=%s}" % (self.name, self.matching_configurations, self.original_string) | 416 return "TestExpectationLine{name=%s, matching_configurations=%s, origina
l_string=%s}" % ( |
| 417 self.name, self.matching_configurations, self.original_string) |
409 | 418 |
410 def __eq__(self, other): | 419 def __eq__(self, other): |
411 return (self.original_string == other.original_string | 420 return (self.original_string == other.original_string and self.filename
== other.filename and |
412 and self.filename == other.filename | 421 self.line_numbers == other.line_numbers and self.name == other.n
ame and self.path == other.path and |
413 and self.line_numbers == other.line_numbers | 422 self.bugs == other.bugs and self.specifiers == other.specifiers
and |
414 and self.name == other.name | 423 self.parsed_specifiers == other.parsed_specifiers and |
415 and self.path == other.path | 424 self.matching_configurations == other.matching_configurations an
d self.expectations == other.expectations and |
416 and self.bugs == other.bugs | 425 self.parsed_expectations == other.parsed_expectations and self.c
omment == other.comment and |
417 and self.specifiers == other.specifiers | 426 self.matching_tests == other.matching_tests and self.warnings ==
other.warnings and |
418 and self.parsed_specifiers == other.parsed_specifiers | 427 self.is_skipped_outside_expectations_file == other.is_skipped_ou
tside_expectations_file) |
419 and self.matching_configurations == other.matching_configurations | |
420 and self.expectations == other.expectations | |
421 and self.parsed_expectations == other.parsed_expectations | |
422 and self.comment == other.comment | |
423 and self.matching_tests == other.matching_tests | |
424 and self.warnings == other.warnings | |
425 and self.is_skipped_outside_expectations_file == other.is_skipped_ou
tside_expectations_file) | |
426 | 428 |
427 def is_invalid(self): | 429 def is_invalid(self): |
428 return bool(self.warnings and self.warnings != [TestExpectationParser.MI
SSING_BUG_WARNING]) | 430 return bool(self.warnings and self.warnings != [TestExpectationParser.MI
SSING_BUG_WARNING]) |
429 | 431 |
430 def is_flaky(self): | 432 def is_flaky(self): |
431 return len(self.parsed_expectations) > 1 | 433 return len(self.parsed_expectations) > 1 |
432 | 434 |
433 def is_whitespace_or_comment(self): | 435 def is_whitespace_or_comment(self): |
434 return bool(re.match("^\s*$", self.original_string.split('#')[0])) | 436 return bool(re.match("^\s*$", self.original_string.split('#')[0])) |
435 | 437 |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
467 result.bugs = list(set(line1.bugs) | set(line2.bugs)) | 469 result.bugs = list(set(line1.bugs) | set(line2.bugs)) |
468 result.specifiers = list(set(line1.specifiers) | set(line2.specifiers)) | 470 result.specifiers = list(set(line1.specifiers) | set(line2.specifiers)) |
469 result.parsed_specifiers = list(set(line1.parsed_specifiers) | set(line2
.parsed_specifiers)) | 471 result.parsed_specifiers = list(set(line1.parsed_specifiers) | set(line2
.parsed_specifiers)) |
470 result.matching_configurations = set(line1.matching_configurations) | se
t(line2.matching_configurations) | 472 result.matching_configurations = set(line1.matching_configurations) | se
t(line2.matching_configurations) |
471 result.matching_tests = list(list(set(line1.matching_tests) | set(line2.
matching_tests))) | 473 result.matching_tests = list(list(set(line1.matching_tests) | set(line2.
matching_tests))) |
472 result.warnings = list(set(line1.warnings) | set(line2.warnings)) | 474 result.warnings = list(set(line1.warnings) | set(line2.warnings)) |
473 result.is_skipped_outside_expectations_file = line1.is_skipped_outside_e
xpectations_file or line2.is_skipped_outside_expectations_file | 475 result.is_skipped_outside_expectations_file = line1.is_skipped_outside_e
xpectations_file or line2.is_skipped_outside_expectations_file |
474 return result | 476 return result |
475 | 477 |
476 def to_string(self, test_configuration_converter, include_specifiers=True, i
nclude_expectations=True, include_comment=True): | 478 def to_string(self, test_configuration_converter, include_specifiers=True, i
nclude_expectations=True, include_comment=True): |
477 parsed_expectation_to_string = dict([[parsed_expectation, expectation_st
ring] for expectation_string, parsed_expectation in TestExpectations.EXPECTATION
S.items()]) | 479 parsed_expectation_to_string = dict([[parsed_expectation, expectation_st
ring] |
| 480 for expectation_string, parsed_expe
ctation in TestExpectations.EXPECTATIONS.items()]) |
478 | 481 |
479 if self.is_invalid(): | 482 if self.is_invalid(): |
480 return self.original_string or '' | 483 return self.original_string or '' |
481 | 484 |
482 if self.name is None: | 485 if self.name is None: |
483 return '' if self.comment is None else "#%s" % self.comment | 486 return '' if self.comment is None else "#%s" % self.comment |
484 | 487 |
485 if test_configuration_converter and self.bugs: | 488 if test_configuration_converter and self.bugs: |
486 specifiers_list = test_configuration_converter.to_specifiers_list(se
lf.matching_configurations) | 489 specifiers_list = test_configuration_converter.to_specifiers_list(se
lf.matching_configurations) |
487 result = [] | 490 result = [] |
488 for specifiers in specifiers_list: | 491 for specifiers in specifiers_list: |
489 # FIXME: this is silly that we join the specifiers and then imme
diately split them. | 492 # FIXME: this is silly that we join the specifiers and then imme
diately split them. |
490 specifiers = self._serialize_parsed_specifiers(test_configuratio
n_converter, specifiers).split() | 493 specifiers = self._serialize_parsed_specifiers(test_configuratio
n_converter, specifiers).split() |
491 expectations = self._serialize_parsed_expectations(parsed_expect
ation_to_string).split() | 494 expectations = self._serialize_parsed_expectations(parsed_expect
ation_to_string).split() |
492 result.append(self._format_line(self.bugs, specifiers, self.name
, expectations, self.comment)) | 495 result.append(self._format_line(self.bugs, specifiers, self.name
, expectations, self.comment)) |
493 return "\n".join(result) if result else None | 496 return "\n".join(result) if result else None |
494 | 497 |
495 return self._format_line(self.bugs, self.specifiers, self.name, self.exp
ectations, self.comment, | 498 return self._format_line(self.bugs, self.specifiers, self.name, self.exp
ectations, self.comment, include_specifiers, |
496 include_specifiers, include_expectations, include_comment) | 499 include_expectations, include_comment) |
497 | 500 |
498 def to_csv(self): | 501 def to_csv(self): |
499 # Note that this doesn't include the comments. | 502 # Note that this doesn't include the comments. |
500 return '%s,%s,%s,%s' % (self.name, ' '.join(self.bugs), ' '.join(self.sp
ecifiers), ' '.join(self.expectations)) | 503 return '%s,%s,%s,%s' % (self.name, ' '.join(self.bugs), ' '.join(self.sp
ecifiers), ' '.join(self.expectations)) |
501 | 504 |
502 def _serialize_parsed_expectations(self, parsed_expectation_to_string): | 505 def _serialize_parsed_expectations(self, parsed_expectation_to_string): |
503 result = [] | 506 result = [] |
504 for index in TestExpectations.EXPECTATIONS.values(): | 507 for index in TestExpectations.EXPECTATIONS.values(): |
505 if index in self.parsed_expectations: | 508 if index in self.parsed_expectations: |
506 result.append(parsed_expectation_to_string[index]) | 509 result.append(parsed_expectation_to_string[index]) |
507 return ' '.join(result) | 510 return ' '.join(result) |
508 | 511 |
509 def _serialize_parsed_specifiers(self, test_configuration_converter, specifi
ers): | 512 def _serialize_parsed_specifiers(self, test_configuration_converter, specifi
ers): |
510 result = [] | 513 result = [] |
511 result.extend(sorted(self.parsed_specifiers)) | 514 result.extend(sorted(self.parsed_specifiers)) |
512 result.extend(test_configuration_converter.specifier_sorter().sort_speci
fiers(specifiers)) | 515 result.extend(test_configuration_converter.specifier_sorter().sort_speci
fiers(specifiers)) |
513 return ' '.join(result) | 516 return ' '.join(result) |
514 | 517 |
515 @staticmethod | 518 @staticmethod |
516 def _filter_redundant_expectations(expectations): | 519 def _filter_redundant_expectations(expectations): |
517 if set(expectations) == set(['Pass', 'Skip']): | 520 if set(expectations) == set(['Pass', 'Skip']): |
518 return ['Skip'] | 521 return ['Skip'] |
519 if set(expectations) == set(['Pass', 'Slow']): | 522 if set(expectations) == set(['Pass', 'Slow']): |
520 return ['Slow'] | 523 return ['Slow'] |
521 return expectations | 524 return expectations |
522 | 525 |
523 @staticmethod | 526 @staticmethod |
524 def _format_line(bugs, specifiers, name, expectations, comment, include_spec
ifiers=True, include_expectations=True, include_comment=True): | 527 def _format_line(bugs, |
| 528 specifiers, |
| 529 name, |
| 530 expectations, |
| 531 comment, |
| 532 include_specifiers=True, |
| 533 include_expectations=True, |
| 534 include_comment=True): |
525 new_specifiers = [] | 535 new_specifiers = [] |
526 new_expectations = [] | 536 new_expectations = [] |
527 for specifier in specifiers: | 537 for specifier in specifiers: |
528 # FIXME: Make this all work with the mixed-cased specifiers (e.g. Wo
ntFix, Slow, etc). | 538 # FIXME: Make this all work with the mixed-cased specifiers (e.g. Wo
ntFix, Slow, etc). |
529 specifier = specifier.upper() | 539 specifier = specifier.upper() |
530 new_specifiers.append(TestExpectationParser._inverted_configuration_
tokens.get(specifier, specifier)) | 540 new_specifiers.append(TestExpectationParser._inverted_configuration_
tokens.get(specifier, specifier)) |
531 | 541 |
532 for expectation in expectations: | 542 for expectation in expectations: |
533 expectation = expectation.upper() | 543 expectation = expectation.upper() |
534 new_expectations.append(TestExpectationParser._inverted_expectation_
tokens.get(expectation, expectation)) | 544 new_expectations.append(TestExpectationParser._inverted_expectation_
tokens.get(expectation, expectation)) |
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
588 # so that we only call that n^2 in the number of *lines*. | 598 # so that we only call that n^2 in the number of *lines*. |
589 merge_lines_cache = defaultdict(dict) | 599 merge_lines_cache = defaultdict(dict) |
590 | 600 |
591 for test, other_line in other._test_to_expectation_line.items(): | 601 for test, other_line in other._test_to_expectation_line.items(): |
592 merged_line = None | 602 merged_line = None |
593 if test in self._test_to_expectation_line: | 603 if test in self._test_to_expectation_line: |
594 self_line = self._test_to_expectation_line[test] | 604 self_line = self._test_to_expectation_line[test] |
595 | 605 |
596 if other_line not in merge_lines_cache[self_line]: | 606 if other_line not in merge_lines_cache[self_line]: |
597 merge_lines_cache[self_line][other_line] = TestExpectationLi
ne.merge_expectation_lines( | 607 merge_lines_cache[self_line][other_line] = TestExpectationLi
ne.merge_expectation_lines( |
598 self_line, other_line, model_all_expectations=False) | 608 self_line, |
| 609 other_line, |
| 610 model_all_expectations=False) |
599 | 611 |
600 merged_line = merge_lines_cache[self_line][other_line] | 612 merged_line = merge_lines_cache[self_line][other_line] |
601 else: | 613 else: |
602 merged_line = other_line | 614 merged_line = other_line |
603 | 615 |
604 self._test_to_expectation_line[test] = merged_line | 616 self._test_to_expectation_line[test] = merged_line |
605 | 617 |
606 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_
to_tests) | 618 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_
to_tests) |
607 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes
ts) | 619 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes
ts) |
608 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_
to_tests) | 620 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_
to_tests) |
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
671 if item[1] == expectation: | 683 if item[1] == expectation: |
672 return item[0].upper() | 684 return item[0].upper() |
673 raise ValueError(expectation) | 685 raise ValueError(expectation) |
674 | 686 |
675 def remove_expectation_line(self, test): | 687 def remove_expectation_line(self, test): |
676 if not self.has_test(test): | 688 if not self.has_test(test): |
677 return | 689 return |
678 self._clear_expectations_for_test(test) | 690 self._clear_expectations_for_test(test) |
679 del self._test_to_expectation_line[test] | 691 del self._test_to_expectation_line[test] |
680 | 692 |
681 def add_expectation_line(self, expectation_line, | 693 def add_expectation_line(self, expectation_line, model_all_expectations=Fals
e): |
682 model_all_expectations=False): | |
683 """Returns a list of warnings encountered while matching specifiers.""" | 694 """Returns a list of warnings encountered while matching specifiers.""" |
684 | 695 |
685 if expectation_line.is_invalid(): | 696 if expectation_line.is_invalid(): |
686 return | 697 return |
687 | 698 |
688 for test in expectation_line.matching_tests: | 699 for test in expectation_line.matching_tests: |
689 if self._already_seen_better_match(test, expectation_line): | 700 if self._already_seen_better_match(test, expectation_line): |
690 continue | 701 continue |
691 | 702 |
692 if model_all_expectations: | 703 if model_all_expectations: |
693 expectation_line = TestExpectationLine.merge_expectation_lines(s
elf.get_expectation_line(test), expectation_line, model_all_expectations) | 704 expectation_line = TestExpectationLine.merge_expectation_lines( |
| 705 self.get_expectation_line(test), expectation_line, model_all
_expectations) |
694 | 706 |
695 self._clear_expectations_for_test(test) | 707 self._clear_expectations_for_test(test) |
696 self._test_to_expectation_line[test] = expectation_line | 708 self._test_to_expectation_line[test] = expectation_line |
697 self._add_test(test, expectation_line) | 709 self._add_test(test, expectation_line) |
698 | 710 |
699 def _add_test(self, test, expectation_line): | 711 def _add_test(self, test, expectation_line): |
700 """Sets the expected state for a given test. | 712 """Sets the expected state for a given test. |
701 | 713 |
702 This routine assumes the test has not been added before. If it has, | 714 This routine assumes the test has not been added before. If it has, |
703 use _clear_expectations_for_test() to reset the state prior to | 715 use _clear_expectations_for_test() to reset the state prior to |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
773 # base path, so we need to check the two sets of specifiers. | 785 # base path, so we need to check the two sets of specifiers. |
774 | 786 |
775 # FIXME: This code was originally designed to allow lines that matched | 787 # FIXME: This code was originally designed to allow lines that matched |
776 # more specifiers to override lines that matched fewer specifiers. | 788 # more specifiers to override lines that matched fewer specifiers. |
777 # However, we currently view these as errors. | 789 # However, we currently view these as errors. |
778 # | 790 # |
779 # To use the "more specifiers wins" policy, change the errors for overri
des | 791 # To use the "more specifiers wins" policy, change the errors for overri
des |
780 # to be warnings and return False". | 792 # to be warnings and return False". |
781 | 793 |
782 if prev_expectation_line.matching_configurations == expectation_line.mat
ching_configurations: | 794 if prev_expectation_line.matching_configurations == expectation_line.mat
ching_configurations: |
783 expectation_line.warnings.append('Duplicate or ambiguous entry lines
%s:%s and %s:%s.' % ( | 795 expectation_line.warnings.append('Duplicate or ambiguous entry lines
%s:%s and %s:%s.' % |
784 self._shorten_filename(prev_expectation_line.filename), prev_exp
ectation_line.line_numbers, | 796 (self._shorten_filename(prev_expect
ation_line.filename), |
785 self._shorten_filename(expectation_line.filename), expectation_l
ine.line_numbers)) | 797 prev_expectation_line.line_numbers
, self._shorten_filename(expectation_line.filename), |
| 798 expectation_line.line_numbers)) |
786 return True | 799 return True |
787 | 800 |
788 if prev_expectation_line.matching_configurations >= expectation_line.mat
ching_configurations: | 801 if prev_expectation_line.matching_configurations >= expectation_line.mat
ching_configurations: |
789 expectation_line.warnings.append('More specific entry for %s on line
%s:%s overrides line %s:%s.' % (expectation_line.name, | 802 expectation_line.warnings.append('More specific entry for %s on line
%s:%s overrides line %s:%s.' % |
790 self._shorten_filename(prev_expectation_line.filename), prev_exp
ectation_line.line_numbers, | 803 (expectation_line.name, self._short
en_filename(prev_expectation_line.filename), |
791 self._shorten_filename(expectation_line.filename), expectation_l
ine.line_numbers)) | 804 prev_expectation_line.line_numbers
, self._shorten_filename(expectation_line.filename), |
| 805 expectation_line.line_numbers)) |
792 # FIXME: return False if we want more specific to win. | 806 # FIXME: return False if we want more specific to win. |
793 return True | 807 return True |
794 | 808 |
795 if prev_expectation_line.matching_configurations <= expectation_line.mat
ching_configurations: | 809 if prev_expectation_line.matching_configurations <= expectation_line.mat
ching_configurations: |
796 expectation_line.warnings.append('More specific entry for %s on line
%s:%s overrides line %s:%s.' % (expectation_line.name, | 810 expectation_line.warnings.append('More specific entry for %s on line
%s:%s overrides line %s:%s.' % |
797 self._shorten_filename(expectation_line.filename), expectation_l
ine.line_numbers, | 811 (expectation_line.name, self._short
en_filename(expectation_line.filename), |
798 self._shorten_filename(prev_expectation_line.filename), prev_exp
ectation_line.line_numbers)) | 812 expectation_line.line_numbers, sel
f._shorten_filename(prev_expectation_line.filename), |
| 813 prev_expectation_line.line_numbers
)) |
799 return True | 814 return True |
800 | 815 |
801 if prev_expectation_line.matching_configurations & expectation_line.matc
hing_configurations: | 816 if prev_expectation_line.matching_configurations & expectation_line.matc
hing_configurations: |
802 expectation_line.warnings.append('Entries for %s on lines %s:%s and
%s:%s match overlapping sets of configurations.' % (expectation_line.name, | 817 expectation_line.warnings.append('Entries for %s on lines %s:%s and
%s:%s match overlapping sets of configurations.' % |
803 self._shorten_filename(prev_expectation_line.filename), prev_exp
ectation_line.line_numbers, | 818 (expectation_line.name, self._short
en_filename(prev_expectation_line.filename), |
804 self._shorten_filename(expectation_line.filename), expectation_l
ine.line_numbers)) | 819 prev_expectation_line.line_numbers
, self._shorten_filename(expectation_line.filename), |
| 820 expectation_line.line_numbers)) |
805 return True | 821 return True |
806 | 822 |
807 # Configuration sets are disjoint, then. | 823 # Configuration sets are disjoint, then. |
808 return False | 824 return False |
809 | 825 |
810 | 826 |
811 class TestExpectations(object): | 827 class TestExpectations(object): |
812 """Test expectations consist of lines with specifications of what | 828 """Test expectations consist of lines with specifications of what |
813 to expect from layout test cases. The test cases can be directories | 829 to expect from layout test cases. The test cases can be directories |
814 in which case the expectations apply to all test cases in that | 830 in which case the expectations apply to all test cases in that |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
846 'text': TEXT, | 862 'text': TEXT, |
847 'timeout': TIMEOUT, | 863 'timeout': TIMEOUT, |
848 'crash': CRASH, | 864 'crash': CRASH, |
849 'leak': LEAK, | 865 'leak': LEAK, |
850 'missing': MISSING, | 866 'missing': MISSING, |
851 TestExpectationParser.SKIP_MODIFIER: SKIP, | 867 TestExpectationParser.SKIP_MODIFIER: SKIP, |
852 TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBAS
ELINE, | 868 TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBAS
ELINE, |
853 TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEED
S_MANUAL_REBASELINE, | 869 TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEED
S_MANUAL_REBASELINE, |
854 TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, | 870 TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, |
855 TestExpectationParser.SLOW_MODIFIER: SLOW, | 871 TestExpectationParser.SLOW_MODIFIER: SLOW, |
856 TestExpectationParser.REBASELINE_MODIFIER: REBASELINE, | 872 TestExpectationParser.REBASELINE_MODIFIER: REBASELINE, } |
857 } | |
858 | 873 |
859 EXPECTATIONS_TO_STRING = dict((k, v) for (v, k) in EXPECTATIONS.iteritems()) | 874 EXPECTATIONS_TO_STRING = dict((k, v) for (v, k) in EXPECTATIONS.iteritems()) |
860 | 875 |
861 # (aggregated by category, pass/fail/skip, type) | 876 # (aggregated by category, pass/fail/skip, type) |
862 EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped', | 877 EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped', |
863 PASS: 'passes', | 878 PASS: 'passes', |
864 FAIL: 'failures', | 879 FAIL: 'failures', |
865 IMAGE: 'image-only failures', | 880 IMAGE: 'image-only failures', |
866 TEXT: 'text-only failures', | 881 TEXT: 'text-only failures', |
867 IMAGE_PLUS_TEXT: 'image and text failures', | 882 IMAGE_PLUS_TEXT: 'image and text failures', |
868 AUDIO: 'audio failures', | 883 AUDIO: 'audio failures', |
869 CRASH: 'crashes', | 884 CRASH: 'crashes', |
870 LEAK: 'leaks', | 885 LEAK: 'leaks', |
871 TIMEOUT: 'timeouts', | 886 TIMEOUT: 'timeouts', |
872 MISSING: 'missing results'} | 887 MISSING: 'missing results'} |
873 | 888 |
874 NON_TEST_OUTCOME_EXPECTATIONS = (REBASELINE, SKIP, SLOW, WONTFIX) | 889 NON_TEST_OUTCOME_EXPECTATIONS = (REBASELINE, SKIP, SLOW, WONTFIX) |
875 | 890 |
876 BUILD_TYPES = ('debug', 'release') | 891 BUILD_TYPES = ('debug', 'release') |
877 | 892 |
878 TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, | 893 TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, 'now': NOW} |
879 'now': NOW} | |
880 | 894 |
881 RESULT_TYPES = {'skip': SKIP, | 895 RESULT_TYPES = {'skip': SKIP, 'pass': PASS, 'fail': FAIL, 'flaky': FLAKY} |
882 'pass': PASS, | |
883 'fail': FAIL, | |
884 'flaky': FLAKY} | |
885 | 896 |
886 @classmethod | 897 @classmethod |
887 def expectation_from_string(cls, string): | 898 def expectation_from_string(cls, string): |
888 assert(' ' not in string) # This only handles one expectation at a time
. | 899 assert (' ' not in string) # This only handles one expectation at a tim
e. |
889 return cls.EXPECTATIONS.get(string.lower()) | 900 return cls.EXPECTATIONS.get(string.lower()) |
890 | 901 |
891 @staticmethod | 902 @staticmethod |
892 def result_was_expected(result, expected_results, test_needs_rebaselining): | 903 def result_was_expected(result, expected_results, test_needs_rebaselining): |
893 """Returns whether we got a result we were expecting. | 904 """Returns whether we got a result we were expecting. |
894 Args: | 905 Args: |
895 result: actual result of a test execution | 906 result: actual result of a test execution |
896 expected_results: set of results listed in test_expectations | 907 expected_results: set of results listed in test_expectations |
897 test_needs_rebaselining: whether test was marked as REBASELINE""" | 908 test_needs_rebaselining: whether test was marked as REBASELINE""" |
898 if not (set(expected_results) - (set(TestExpectations.NON_TEST_OUTCOME_E
XPECTATIONS))): | 909 if not (set(expected_results) - (set(TestExpectations.NON_TEST_OUTCOME_E
XPECTATIONS))): |
899 expected_results = set([PASS]) | 910 expected_results = set([PASS]) |
900 | 911 |
901 if result in expected_results: | 912 if result in expected_results: |
902 return True | 913 return True |
903 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and (N
EEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE in expected_resul
ts): | 914 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and (N
EEDS_REBASELINE in expected_results or |
| 915 N
EEDS_MANUAL_REBASELINE in expected_results): |
904 return True | 916 return True |
905 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected
_results): | 917 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected
_results): |
906 return True | 918 return True |
907 if result == MISSING and test_needs_rebaselining: | 919 if result == MISSING and test_needs_rebaselining: |
908 return True | 920 return True |
909 if result == SKIP: | 921 if result == SKIP: |
910 return True | 922 return True |
911 return False | 923 return False |
912 | 924 |
913 @staticmethod | 925 @staticmethod |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
959 suffixes.add('wav') | 971 suffixes.add('wav') |
960 if 'MISSING' in expectations: | 972 if 'MISSING' in expectations: |
961 suffixes.add('txt') | 973 suffixes.add('txt') |
962 suffixes.add('png') | 974 suffixes.add('png') |
963 suffixes.add('wav') | 975 suffixes.add('wav') |
964 return suffixes | 976 return suffixes |
965 | 977 |
966 # FIXME: This constructor does too much work. We should move the actual pars
ing of | 978 # FIXME: This constructor does too much work. We should move the actual pars
ing of |
967 # the expectations into separate routines so that linting and handling overr
ides | 979 # the expectations into separate routines so that linting and handling overr
ides |
968 # can be controlled separately, and the constructor can be more of a no-op. | 980 # can be controlled separately, and the constructor can be more of a no-op. |
969 def __init__(self, port, tests=None, include_overrides=True, expectations_di
ct=None, model_all_expectations=False, is_lint_mode=False): | 981 def __init__(self, |
| 982 port, |
| 983 tests=None, |
| 984 include_overrides=True, |
| 985 expectations_dict=None, |
| 986 model_all_expectations=False, |
| 987 is_lint_mode=False): |
970 self._full_test_list = tests | 988 self._full_test_list = tests |
971 self._test_config = port.test_configuration() | 989 self._test_config = port.test_configuration() |
972 self._is_lint_mode = is_lint_mode | 990 self._is_lint_mode = is_lint_mode |
973 self._model_all_expectations = self._is_lint_mode or model_all_expectati
ons | 991 self._model_all_expectations = self._is_lint_mode or model_all_expectati
ons |
974 self._model = TestExpectationsModel(self._shorten_filename) | 992 self._model = TestExpectationsModel(self._shorten_filename) |
975 self._parser = TestExpectationParser(port, tests, self._is_lint_mode) | 993 self._parser = TestExpectationParser(port, tests, self._is_lint_mode) |
976 self._port = port | 994 self._port = port |
977 self._skipped_tests_warnings = [] | 995 self._skipped_tests_warnings = [] |
978 self._expectations = [] | 996 self._expectations = [] |
979 | 997 |
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1050 | 1068 |
1051 def _shorten_filename(self, filename): | 1069 def _shorten_filename(self, filename): |
1052 if filename.startswith(self._port.path_from_webkit_base()): | 1070 if filename.startswith(self._port.path_from_webkit_base()): |
1053 return self._port.host.filesystem.relpath(filename, self._port.path_
from_webkit_base()) | 1071 return self._port.host.filesystem.relpath(filename, self._port.path_
from_webkit_base()) |
1054 return filename | 1072 return filename |
1055 | 1073 |
1056 def _report_warnings(self): | 1074 def _report_warnings(self): |
1057 warnings = [] | 1075 warnings = [] |
1058 for expectation in self._expectations: | 1076 for expectation in self._expectations: |
1059 for warning in expectation.warnings: | 1077 for warning in expectation.warnings: |
1060 warnings.append('%s:%s %s %s' % (self._shorten_filename(expectat
ion.filename), expectation.line_numbers, | 1078 warnings.append('%s:%s %s %s' % (self._shorten_filename(expectat
ion.filename), expectation.line_numbers, warning, |
1061 warning, expectation.name if expectation.expecta
tions else expectation.original_string)) | 1079 expectation.name if expectation
.expectations else expectation.original_string)) |
1062 | 1080 |
1063 if warnings: | 1081 if warnings: |
1064 self._has_warnings = True | 1082 self._has_warnings = True |
1065 if self._is_lint_mode: | 1083 if self._is_lint_mode: |
1066 raise ParseError(warnings) | 1084 raise ParseError(warnings) |
1067 _log.warning('--lint-test-files warnings:') | 1085 _log.warning('--lint-test-files warnings:') |
1068 for warning in warnings: | 1086 for warning in warnings: |
1069 _log.warning(warning) | 1087 _log.warning(warning) |
1070 _log.warning('') | 1088 _log.warning('') |
1071 | 1089 |
(...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1156 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. | 1174 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. |
1157 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. | 1175 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. |
1158 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: | 1176 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: |
1159 return expectation_line.to_string(test_configuration_converter) | 1177 return expectation_line.to_string(test_configuration_converter) |
1160 return expectation_line.original_string | 1178 return expectation_line.original_string |
1161 | 1179 |
1162 def nones_out(expectation_line): | 1180 def nones_out(expectation_line): |
1163 return expectation_line is not None | 1181 return expectation_line is not None |
1164 | 1182 |
1165 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) | 1183 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) |
OLD | NEW |