Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(354)

Side by Side Diff: third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py

Issue 1783073002: Run auto-formatter on files in webkitpy/layout_tests/. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Ran yapf -i --style '{based_on_style: pep8, column_limit: 132}' then did manual fix-up Created 4 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (C) 2010 Google Inc. All rights reserved. 1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the 11 # in the documentation and/or other materials provided with the
12 # distribution. 12 # distribution.
13 # * Neither the name of Google Inc. nor the names of its 13 # * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from 14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission. 15 # this software without specific prior written permission.
16 # 16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 """A helper class for reading in and dealing with tests expectations 28 """A helper class for reading in and dealing with tests expectations
30 for layout tests. 29 for layout tests.
31 """ 30 """
32 31
33 from collections import defaultdict 32 from collections import defaultdict
34 33
35 import logging 34 import logging
36 import re 35 import re
37 36
38 from webkitpy.layout_tests.models.test_configuration import TestConfigurationCon verter 37 from webkitpy.layout_tests.models.test_configuration import TestConfigurationCon verter
39 38
40 _log = logging.getLogger(__name__) 39 _log = logging.getLogger(__name__)
41 40
42
43 # Test expectation and specifier constants. 41 # Test expectation and specifier constants.
44 # 42 #
45 # FIXME: range() starts with 0 which makes if expectation checks harder 43 # FIXME: range() starts with 0 which makes if expectation checks harder
46 # as PASS is 0. 44 # as PASS is 0.
47 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, LEAK, SKIP, WO NTFIX, 45 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, LEAK, SKIP, WO NTFIX, SLOW, REBASELINE, NEEDS_REBASELINE,
48 SLOW, REBASELINE, NEEDS_REBASELINE, NEEDS_MANUAL_REBASELINE, MISSING, FLAKY, NO W, NONE) = range(19) 46 NEEDS_MANUAL_REBASELINE, MISSING, FLAKY, NOW, NONE) = range(19)
49 47
50 # FIXME: Perhas these two routines should be part of the Port instead? 48 # FIXME: Perhas these two routines should be part of the Port instead?
51 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt') 49 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt')
52 50
53 WEBKIT_BUG_PREFIX = 'webkit.org/b/' 51 WEBKIT_BUG_PREFIX = 'webkit.org/b/'
54 CHROMIUM_BUG_PREFIX = 'crbug.com/' 52 CHROMIUM_BUG_PREFIX = 'crbug.com/'
55 V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id=' 53 V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id='
56 NAMED_BUG_PREFIX = 'Bug(' 54 NAMED_BUG_PREFIX = 'Bug('
57 55
58 MISSING_KEYWORD = 'Missing' 56 MISSING_KEYWORD = 'Missing'
59 NEEDS_REBASELINE_KEYWORD = 'NeedsRebaseline' 57 NEEDS_REBASELINE_KEYWORD = 'NeedsRebaseline'
60 NEEDS_MANUAL_REBASELINE_KEYWORD = 'NeedsManualRebaseline' 58 NEEDS_MANUAL_REBASELINE_KEYWORD = 'NeedsManualRebaseline'
61 59
62 60
63 class ParseError(Exception): 61 class ParseError(Exception):
64 def __init__(self, warnings): 62 def __init__(self, warnings):
65 super(ParseError, self).__init__() 63 super(ParseError, self).__init__()
66 self.warnings = warnings 64 self.warnings = warnings
67 65
68 def __str__(self): 66 def __str__(self):
69 return '\n'.join(map(str, self.warnings)) 67 return '\n'.join(map(str, self.warnings))
70 68
71 def __repr__(self): 69 def __repr__(self):
72 return 'ParseError(warnings=%s)' % self.warnings 70 return 'ParseError(warnings=%s)' % self.warnings
73 71
74 72
75 class TestExpectationParser(object): 73 class TestExpectationParser(object):
76 """Provides parsing facilities for lines in the test_expectation.txt file."" " 74 """Provides parsing facilities for lines in the test_expectation.txt file."" "
77 75
78 # FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make the case studdly-caps to match the actual file contents. 76 # FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make
77 # the case studdly-caps to match the actual file contents.
79 REBASELINE_MODIFIER = 'rebaseline' 78 REBASELINE_MODIFIER = 'rebaseline'
80 NEEDS_REBASELINE_MODIFIER = 'needsrebaseline' 79 NEEDS_REBASELINE_MODIFIER = 'needsrebaseline'
81 NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline' 80 NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline'
82 PASS_EXPECTATION = 'pass' 81 PASS_EXPECTATION = 'pass'
83 SKIP_MODIFIER = 'skip' 82 SKIP_MODIFIER = 'skip'
84 SLOW_MODIFIER = 'slow' 83 SLOW_MODIFIER = 'slow'
85 WONTFIX_MODIFIER = 'wontfix' 84 WONTFIX_MODIFIER = 'wontfix'
86 85
87 TIMEOUT_EXPECTATION = 'timeout' 86 TIMEOUT_EXPECTATION = 'timeout'
88 87
89 MISSING_BUG_WARNING = 'Test lacks BUG specifier.' 88 MISSING_BUG_WARNING = 'Test lacks BUG specifier.'
90 89
91 def __init__(self, port, all_tests, is_lint_mode): 90 def __init__(self, port, all_tests, is_lint_mode):
92 self._port = port 91 self._port = port
93 self._test_configuration_converter = TestConfigurationConverter(set(port .all_test_configurations()), port.configuration_specifier_macros()) 92 self._test_configuration_converter = TestConfigurationConverter(
93 set(port.all_test_configurations()), port.configuration_specifier_ma cros())
94 94
95 if all_tests: 95 if all_tests:
96 self._all_tests = set(all_tests) 96 self._all_tests = set(all_tests)
97 else: 97 else:
98 self._all_tests = set() 98 self._all_tests = set()
99 99
100 self._is_lint_mode = is_lint_mode 100 self._is_lint_mode = is_lint_mode
101 101
102 def parse(self, filename, expectations_string): 102 def parse(self, filename, expectations_string):
103 expectation_lines = [] 103 expectation_lines = []
(...skipping 11 matching lines...) Expand all
115 expectation_line.name = test_name 115 expectation_line.name = test_name
116 expectation_line.filename = file_name 116 expectation_line.filename = file_name
117 expectation_line.expectations = expectations 117 expectation_line.expectations = expectations
118 return expectation_line 118 return expectation_line
119 119
120 def expectation_line_for_test(self, test_name, expectations): 120 def expectation_line_for_test(self, test_name, expectations):
121 expectation_line = self._create_expectation_line(test_name, expectations , '<Bot TestExpectations>') 121 expectation_line = self._create_expectation_line(test_name, expectations , '<Bot TestExpectations>')
122 self._parse_line(expectation_line) 122 self._parse_line(expectation_line)
123 return expectation_line 123 return expectation_line
124 124
125
126 def expectation_for_skipped_test(self, test_name): 125 def expectation_for_skipped_test(self, test_name):
127 if not self._port.test_exists(test_name): 126 if not self._port.test_exists(test_name):
128 _log.warning('The following test %s from the Skipped list doesn\'t e xist' % test_name) 127 _log.warning('The following test %s from the Skipped list doesn\'t e xist' % test_name)
129 expectation_line = self._create_expectation_line(test_name, [TestExpecta tionParser.PASS_EXPECTATION], '<Skipped file>') 128 expectation_line = self._create_expectation_line(test_name, [TestExpecta tionParser.PASS_EXPECTATION], '<Skipped file>')
130 expectation_line.expectations = [TestExpectationParser.SKIP_MODIFIER, Te stExpectationParser.WONTFIX_MODIFIER] 129 expectation_line.expectations = [TestExpectationParser.SKIP_MODIFIER, Te stExpectationParser.WONTFIX_MODIFIER]
131 expectation_line.is_skipped_outside_expectations_file = True 130 expectation_line.is_skipped_outside_expectations_file = True
132 self._parse_line(expectation_line) 131 self._parse_line(expectation_line)
133 return expectation_line 132 return expectation_line
134 133
135 def _parse_line(self, expectation_line): 134 def _parse_line(self, expectation_line):
(...skipping 15 matching lines...) Expand all
151 self._parse_expectations(expectation_line) 150 self._parse_expectations(expectation_line)
152 151
153 def _parse_specifier(self, specifier): 152 def _parse_specifier(self, specifier):
154 return specifier.lower() 153 return specifier.lower()
155 154
156 def _parse_specifiers(self, expectation_line): 155 def _parse_specifiers(self, expectation_line):
157 if self._is_lint_mode: 156 if self._is_lint_mode:
158 self._lint_line(expectation_line) 157 self._lint_line(expectation_line)
159 158
160 parsed_specifiers = set([self._parse_specifier(specifier) for specifier in expectation_line.specifiers]) 159 parsed_specifiers = set([self._parse_specifier(specifier) for specifier in expectation_line.specifiers])
161 expectation_line.matching_configurations = self._test_configuration_conv erter.to_config_set(parsed_specifiers, expectation_line.warnings) 160 expectation_line.matching_configurations = self._test_configuration_conv erter.to_config_set(parsed_specifiers,
161 expectation_line.warnings)
162 162
163 def _lint_line(self, expectation_line): 163 def _lint_line(self, expectation_line):
164 expectations = [expectation.lower() for expectation in expectation_line. expectations] 164 expectations = [expectation.lower() for expectation in expectation_line. expectations]
165 if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectatio ns: 165 if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectatio ns:
166 expectation_line.warnings.append(self.MISSING_BUG_WARNING) 166 expectation_line.warnings.append(self.MISSING_BUG_WARNING)
167 if self.REBASELINE_MODIFIER in expectations: 167 if self.REBASELINE_MODIFIER in expectations:
168 expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.') 168 expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
169 169
170 if self.NEEDS_REBASELINE_MODIFIER in expectations or self.NEEDS_MANUAL_R EBASELINE_MODIFIER in expectations: 170 if self.NEEDS_REBASELINE_MODIFIER in expectations or self.NEEDS_MANUAL_R EBASELINE_MODIFIER in expectations:
171 for test in expectation_line.matching_tests: 171 for test in expectation_line.matching_tests:
172 if self._port.reference_files(test): 172 if self._port.reference_files(test):
173 expectation_line.warnings.append('A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseline') 173 expectation_line.warnings.append('A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseline')
174 174
175 specifiers = [specifier.lower() for specifier in expectation_line.specif iers] 175 specifiers = [specifier.lower() for specifier in expectation_line.specif iers]
176 if (self.REBASELINE_MODIFIER in expectations or self.NEEDS_REBASELINE_MO DIFIER in expectations) and ('debug' in specifiers or 'release' in specifiers): 176 if (self.REBASELINE_MODIFIER in expectations or self.NEEDS_REBASELINE_MO DIFIER in expectations) and (
177 'debug' in specifiers or 'release' in specifiers):
177 expectation_line.warnings.append('A test cannot be rebaselined for D ebug/Release.') 178 expectation_line.warnings.append('A test cannot be rebaselined for D ebug/Release.')
178 179
179 def _parse_expectations(self, expectation_line): 180 def _parse_expectations(self, expectation_line):
180 result = set() 181 result = set()
181 for part in expectation_line.expectations: 182 for part in expectation_line.expectations:
182 expectation = TestExpectations.expectation_from_string(part) 183 expectation = TestExpectations.expectation_from_string(part)
183 if expectation is None: # Careful, PASS is currently 0. 184 if expectation is None: # Careful, PASS is currently 0.
184 expectation_line.warnings.append('Unsupported expectation: %s' % part) 185 expectation_line.warnings.append('Unsupported expectation: %s' % part)
185 continue 186 continue
186 result.add(expectation) 187 result.add(expectation)
(...skipping 23 matching lines...) Expand all
210 expectation_line.matching_tests = [test for test in self._all_tests if test.startswith(expectation_line.path)] 211 expectation_line.matching_tests = [test for test in self._all_tests if test.startswith(expectation_line.path)]
211 return 212 return
212 213
213 # this is a test file, do a quick check if it's in the 214 # this is a test file, do a quick check if it's in the
214 # full test suite. 215 # full test suite.
215 if expectation_line.path in self._all_tests: 216 if expectation_line.path in self._all_tests:
216 expectation_line.matching_tests.append(expectation_line.path) 217 expectation_line.matching_tests.append(expectation_line.path)
217 218
218 # FIXME: Update the original specifiers and remove this once the old syntax is gone. 219 # FIXME: Update the original specifiers and remove this once the old syntax is gone.
219 _configuration_tokens_list = [ 220 _configuration_tokens_list = [
220 'Mac', 'Mac10.9', 'Mac10.10', 'Mac10.11', 'Retina', 221 'Mac',
221 'Win', 'Win7', 'Win10', 222 'Mac10.9',
222 'Linux', 'Precise', 'Trusty', 223 'Mac10.10',
224 'Mac10.11',
225 'Retina',
226 'Win',
227 'Win7',
228 'Win10',
229 'Linux',
230 'Precise',
231 'Trusty',
223 'Android', 232 'Android',
224 'Release', 233 'Release',
225 'Debug', 234 'Debug',
226 ] 235 ]
227 236
228 _configuration_tokens = dict((token, token.upper()) for token in _configurat ion_tokens_list) 237 _configuration_tokens = dict((token, token.upper()) for token in _configurat ion_tokens_list)
229 _inverted_configuration_tokens = dict((value, name) for name, value in _conf iguration_tokens.iteritems()) 238 _inverted_configuration_tokens = dict((value, name) for name, value in _conf iguration_tokens.iteritems())
230 239
231 # FIXME: Update the original specifiers list and remove this once the old sy ntax is gone. 240 # FIXME: Update the original specifiers list and remove this once the old sy ntax is gone.
232 _expectation_tokens = { 241 _expectation_tokens = {
233 'Crash': 'CRASH', 242 'Crash': 'CRASH',
234 'Leak': 'LEAK', 243 'Leak': 'LEAK',
235 'Failure': 'FAIL', 244 'Failure': 'FAIL',
236 MISSING_KEYWORD: 'MISSING', 245 MISSING_KEYWORD: 'MISSING',
237 'Pass': 'PASS', 246 'Pass': 'PASS',
238 'Rebaseline': 'REBASELINE', 247 'Rebaseline': 'REBASELINE',
239 NEEDS_REBASELINE_KEYWORD: 'NEEDSREBASELINE', 248 NEEDS_REBASELINE_KEYWORD: 'NEEDSREBASELINE',
240 NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE', 249 NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE',
241 'Skip': 'SKIP', 250 'Skip': 'SKIP',
242 'Slow': 'SLOW', 251 'Slow': 'SLOW',
243 'Timeout': 'TIMEOUT', 252 'Timeout': 'TIMEOUT',
244 'WontFix': 'WONTFIX', 253 'WontFix': 'WONTFIX',
245 } 254 }
246 255
247 _inverted_expectation_tokens = dict([(value, name) for name, value in _expec tation_tokens.iteritems()] + 256 _inverted_expectation_tokens = dict([(value, name) for name, value in _expec tation_tokens.iteritems()] + [('TEXT', 'Failure'), (
248 [('TEXT', 'Failure'), ('IMAGE', 'Failure '), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')]) 257 'IMAGE', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
249 258
250 # FIXME: Seems like these should be classmethods on TestExpectationLine inst ead of TestExpectationParser. 259 # FIXME: Seems like these should be classmethods on TestExpectationLine inst ead of TestExpectationParser.
251 @classmethod 260 @classmethod
252 def _tokenize_line(cls, filename, expectation_string, line_number): 261 def _tokenize_line(cls, filename, expectation_string, line_number):
253 """Tokenizes a line from TestExpectations and returns an unparsed TestEx pectationLine instance using the old format. 262 """Tokenizes a line from TestExpectations and returns an unparsed TestEx pectationLine instance using the old format.
254 263
255 The new format for a test expectation line is: 264 The new format for a test expectation line is:
256 265
257 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>] 266 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
258 267
(...skipping 23 matching lines...) Expand all
282 bugs = [] 291 bugs = []
283 specifiers = [] 292 specifiers = []
284 name = None 293 name = None
285 expectations = [] 294 expectations = []
286 warnings = [] 295 warnings = []
287 has_unrecognized_expectation = False 296 has_unrecognized_expectation = False
288 297
289 tokens = remaining_string.split() 298 tokens = remaining_string.split()
290 state = 'start' 299 state = 'start'
291 for token in tokens: 300 for token in tokens:
292 if (token.startswith(WEBKIT_BUG_PREFIX) or 301 if (token.startswith(WEBKIT_BUG_PREFIX) or token.startswith(CHROMIUM _BUG_PREFIX) or token.startswith(V8_BUG_PREFIX) or
293 token.startswith(CHROMIUM_BUG_PREFIX) or 302 token.startswith(NAMED_BUG_PREFIX)):
294 token.startswith(V8_BUG_PREFIX) or
295 token.startswith(NAMED_BUG_PREFIX)):
296 if state != 'start': 303 if state != 'start':
297 warnings.append('"%s" is not at the start of the line.' % to ken) 304 warnings.append('"%s" is not at the start of the line.' % to ken)
298 break 305 break
299 if token.startswith(WEBKIT_BUG_PREFIX): 306 if token.startswith(WEBKIT_BUG_PREFIX):
300 bugs.append(token) 307 bugs.append(token)
301 elif token.startswith(CHROMIUM_BUG_PREFIX): 308 elif token.startswith(CHROMIUM_BUG_PREFIX):
302 bugs.append(token) 309 bugs.append(token)
303 elif token.startswith(V8_BUG_PREFIX): 310 elif token.startswith(V8_BUG_PREFIX):
304 bugs.append(token) 311 bugs.append(token)
305 else: 312 else:
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after
352 if 'WONTFIX' in expectations and 'SKIP' not in expectations: 359 if 'WONTFIX' in expectations and 'SKIP' not in expectations:
353 expectations.append('SKIP') 360 expectations.append('SKIP')
354 361
355 if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(exp ectations) - set(['SKIP', 'WONTFIX'])): 362 if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(exp ectations) - set(['SKIP', 'WONTFIX'])):
356 warnings.append('A test marked Skip or WontFix must not have other e xpectations.') 363 warnings.append('A test marked Skip or WontFix must not have other e xpectations.')
357 364
358 if 'SLOW' in expectations and 'SlowTests' not in filename: 365 if 'SLOW' in expectations and 'SlowTests' not in filename:
359 warnings.append('SLOW tests should ony be added to SlowTests and not to TestExpectations.') 366 warnings.append('SLOW tests should ony be added to SlowTests and not to TestExpectations.')
360 367
361 if 'WONTFIX' in expectations and ('NeverFixTests' not in filename and 'S taleTestExpectations' not in filename): 368 if 'WONTFIX' in expectations and ('NeverFixTests' not in filename and 'S taleTestExpectations' not in filename):
362 warnings.append('WONTFIX tests should ony be added to NeverFixTests or StaleTestExpectations and not to TestExpectations.') 369 warnings.append(
370 'WONTFIX tests should ony be added to NeverFixTests or StaleTest Expectations and not to TestExpectations.')
363 371
364 if 'NeverFixTests' in filename and expectations != ['WONTFIX', 'SKIP']: 372 if 'NeverFixTests' in filename and expectations != ['WONTFIX', 'SKIP']:
365 warnings.append('Only WONTFIX expectations are allowed in NeverFixTe sts') 373 warnings.append('Only WONTFIX expectations are allowed in NeverFixTe sts')
366 374
367 if 'SlowTests' in filename and expectations != ['SLOW']: 375 if 'SlowTests' in filename and expectations != ['SLOW']:
368 warnings.append('Only SLOW expectations are allowed in SlowTests') 376 warnings.append('Only SLOW expectations are allowed in SlowTests')
369 377
370 if not expectations and not has_unrecognized_expectation: 378 if not expectations and not has_unrecognized_expectation:
371 warnings.append('Missing expectations.') 379 warnings.append('Missing expectations.')
372 380
(...skipping 25 matching lines...) Expand all
398 self.parsed_specifiers = [] 406 self.parsed_specifiers = []
399 self.matching_configurations = set() 407 self.matching_configurations = set()
400 self.expectations = [] 408 self.expectations = []
401 self.parsed_expectations = set() 409 self.parsed_expectations = set()
402 self.comment = None 410 self.comment = None
403 self.matching_tests = [] 411 self.matching_tests = []
404 self.warnings = [] 412 self.warnings = []
405 self.is_skipped_outside_expectations_file = False 413 self.is_skipped_outside_expectations_file = False
406 414
407 def __str__(self): 415 def __str__(self):
408 return "TestExpectationLine{name=%s, matching_configurations=%s, origina l_string=%s}" % (self.name, self.matching_configurations, self.original_string) 416 return "TestExpectationLine{name=%s, matching_configurations=%s, origina l_string=%s}" % (
417 self.name, self.matching_configurations, self.original_string)
409 418
410 def __eq__(self, other): 419 def __eq__(self, other):
411 return (self.original_string == other.original_string 420 return (self.original_string == other.original_string and self.filename == other.filename and
412 and self.filename == other.filename 421 self.line_numbers == other.line_numbers and self.name == other.n ame and self.path == other.path and
413 and self.line_numbers == other.line_numbers 422 self.bugs == other.bugs and self.specifiers == other.specifiers and
414 and self.name == other.name 423 self.parsed_specifiers == other.parsed_specifiers and
415 and self.path == other.path 424 self.matching_configurations == other.matching_configurations an d self.expectations == other.expectations and
416 and self.bugs == other.bugs 425 self.parsed_expectations == other.parsed_expectations and self.c omment == other.comment and
417 and self.specifiers == other.specifiers 426 self.matching_tests == other.matching_tests and self.warnings == other.warnings and
418 and self.parsed_specifiers == other.parsed_specifiers 427 self.is_skipped_outside_expectations_file == other.is_skipped_ou tside_expectations_file)
419 and self.matching_configurations == other.matching_configurations
420 and self.expectations == other.expectations
421 and self.parsed_expectations == other.parsed_expectations
422 and self.comment == other.comment
423 and self.matching_tests == other.matching_tests
424 and self.warnings == other.warnings
425 and self.is_skipped_outside_expectations_file == other.is_skipped_ou tside_expectations_file)
426 428
427 def is_invalid(self): 429 def is_invalid(self):
428 return bool(self.warnings and self.warnings != [TestExpectationParser.MI SSING_BUG_WARNING]) 430 return bool(self.warnings and self.warnings != [TestExpectationParser.MI SSING_BUG_WARNING])
429 431
430 def is_flaky(self): 432 def is_flaky(self):
431 return len(self.parsed_expectations) > 1 433 return len(self.parsed_expectations) > 1
432 434
433 def is_whitespace_or_comment(self): 435 def is_whitespace_or_comment(self):
434 return bool(re.match("^\s*$", self.original_string.split('#')[0])) 436 return bool(re.match("^\s*$", self.original_string.split('#')[0]))
435 437
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
467 result.bugs = list(set(line1.bugs) | set(line2.bugs)) 469 result.bugs = list(set(line1.bugs) | set(line2.bugs))
468 result.specifiers = list(set(line1.specifiers) | set(line2.specifiers)) 470 result.specifiers = list(set(line1.specifiers) | set(line2.specifiers))
469 result.parsed_specifiers = list(set(line1.parsed_specifiers) | set(line2 .parsed_specifiers)) 471 result.parsed_specifiers = list(set(line1.parsed_specifiers) | set(line2 .parsed_specifiers))
470 result.matching_configurations = set(line1.matching_configurations) | se t(line2.matching_configurations) 472 result.matching_configurations = set(line1.matching_configurations) | se t(line2.matching_configurations)
471 result.matching_tests = list(list(set(line1.matching_tests) | set(line2. matching_tests))) 473 result.matching_tests = list(list(set(line1.matching_tests) | set(line2. matching_tests)))
472 result.warnings = list(set(line1.warnings) | set(line2.warnings)) 474 result.warnings = list(set(line1.warnings) | set(line2.warnings))
473 result.is_skipped_outside_expectations_file = line1.is_skipped_outside_e xpectations_file or line2.is_skipped_outside_expectations_file 475 result.is_skipped_outside_expectations_file = line1.is_skipped_outside_e xpectations_file or line2.is_skipped_outside_expectations_file
474 return result 476 return result
475 477
476 def to_string(self, test_configuration_converter, include_specifiers=True, i nclude_expectations=True, include_comment=True): 478 def to_string(self, test_configuration_converter, include_specifiers=True, i nclude_expectations=True, include_comment=True):
477 parsed_expectation_to_string = dict([[parsed_expectation, expectation_st ring] for expectation_string, parsed_expectation in TestExpectations.EXPECTATION S.items()]) 479 parsed_expectation_to_string = dict([[parsed_expectation, expectation_st ring]
480 for expectation_string, parsed_expe ctation in TestExpectations.EXPECTATIONS.items()])
478 481
479 if self.is_invalid(): 482 if self.is_invalid():
480 return self.original_string or '' 483 return self.original_string or ''
481 484
482 if self.name is None: 485 if self.name is None:
483 return '' if self.comment is None else "#%s" % self.comment 486 return '' if self.comment is None else "#%s" % self.comment
484 487
485 if test_configuration_converter and self.bugs: 488 if test_configuration_converter and self.bugs:
486 specifiers_list = test_configuration_converter.to_specifiers_list(se lf.matching_configurations) 489 specifiers_list = test_configuration_converter.to_specifiers_list(se lf.matching_configurations)
487 result = [] 490 result = []
488 for specifiers in specifiers_list: 491 for specifiers in specifiers_list:
489 # FIXME: this is silly that we join the specifiers and then imme diately split them. 492 # FIXME: this is silly that we join the specifiers and then imme diately split them.
490 specifiers = self._serialize_parsed_specifiers(test_configuratio n_converter, specifiers).split() 493 specifiers = self._serialize_parsed_specifiers(test_configuratio n_converter, specifiers).split()
491 expectations = self._serialize_parsed_expectations(parsed_expect ation_to_string).split() 494 expectations = self._serialize_parsed_expectations(parsed_expect ation_to_string).split()
492 result.append(self._format_line(self.bugs, specifiers, self.name , expectations, self.comment)) 495 result.append(self._format_line(self.bugs, specifiers, self.name , expectations, self.comment))
493 return "\n".join(result) if result else None 496 return "\n".join(result) if result else None
494 497
495 return self._format_line(self.bugs, self.specifiers, self.name, self.exp ectations, self.comment, 498 return self._format_line(self.bugs, self.specifiers, self.name, self.exp ectations, self.comment, include_specifiers,
496 include_specifiers, include_expectations, include_comment) 499 include_expectations, include_comment)
497 500
498 def to_csv(self): 501 def to_csv(self):
499 # Note that this doesn't include the comments. 502 # Note that this doesn't include the comments.
500 return '%s,%s,%s,%s' % (self.name, ' '.join(self.bugs), ' '.join(self.sp ecifiers), ' '.join(self.expectations)) 503 return '%s,%s,%s,%s' % (self.name, ' '.join(self.bugs), ' '.join(self.sp ecifiers), ' '.join(self.expectations))
501 504
502 def _serialize_parsed_expectations(self, parsed_expectation_to_string): 505 def _serialize_parsed_expectations(self, parsed_expectation_to_string):
503 result = [] 506 result = []
504 for index in TestExpectations.EXPECTATIONS.values(): 507 for index in TestExpectations.EXPECTATIONS.values():
505 if index in self.parsed_expectations: 508 if index in self.parsed_expectations:
506 result.append(parsed_expectation_to_string[index]) 509 result.append(parsed_expectation_to_string[index])
507 return ' '.join(result) 510 return ' '.join(result)
508 511
509 def _serialize_parsed_specifiers(self, test_configuration_converter, specifi ers): 512 def _serialize_parsed_specifiers(self, test_configuration_converter, specifi ers):
510 result = [] 513 result = []
511 result.extend(sorted(self.parsed_specifiers)) 514 result.extend(sorted(self.parsed_specifiers))
512 result.extend(test_configuration_converter.specifier_sorter().sort_speci fiers(specifiers)) 515 result.extend(test_configuration_converter.specifier_sorter().sort_speci fiers(specifiers))
513 return ' '.join(result) 516 return ' '.join(result)
514 517
515 @staticmethod 518 @staticmethod
516 def _filter_redundant_expectations(expectations): 519 def _filter_redundant_expectations(expectations):
517 if set(expectations) == set(['Pass', 'Skip']): 520 if set(expectations) == set(['Pass', 'Skip']):
518 return ['Skip'] 521 return ['Skip']
519 if set(expectations) == set(['Pass', 'Slow']): 522 if set(expectations) == set(['Pass', 'Slow']):
520 return ['Slow'] 523 return ['Slow']
521 return expectations 524 return expectations
522 525
523 @staticmethod 526 @staticmethod
524 def _format_line(bugs, specifiers, name, expectations, comment, include_spec ifiers=True, include_expectations=True, include_comment=True): 527 def _format_line(bugs,
528 specifiers,
529 name,
530 expectations,
531 comment,
532 include_specifiers=True,
533 include_expectations=True,
534 include_comment=True):
525 new_specifiers = [] 535 new_specifiers = []
526 new_expectations = [] 536 new_expectations = []
527 for specifier in specifiers: 537 for specifier in specifiers:
528 # FIXME: Make this all work with the mixed-cased specifiers (e.g. Wo ntFix, Slow, etc). 538 # FIXME: Make this all work with the mixed-cased specifiers (e.g. Wo ntFix, Slow, etc).
529 specifier = specifier.upper() 539 specifier = specifier.upper()
530 new_specifiers.append(TestExpectationParser._inverted_configuration_ tokens.get(specifier, specifier)) 540 new_specifiers.append(TestExpectationParser._inverted_configuration_ tokens.get(specifier, specifier))
531 541
532 for expectation in expectations: 542 for expectation in expectations:
533 expectation = expectation.upper() 543 expectation = expectation.upper()
534 new_expectations.append(TestExpectationParser._inverted_expectation_ tokens.get(expectation, expectation)) 544 new_expectations.append(TestExpectationParser._inverted_expectation_ tokens.get(expectation, expectation))
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after
588 # so that we only call that n^2 in the number of *lines*. 598 # so that we only call that n^2 in the number of *lines*.
589 merge_lines_cache = defaultdict(dict) 599 merge_lines_cache = defaultdict(dict)
590 600
591 for test, other_line in other._test_to_expectation_line.items(): 601 for test, other_line in other._test_to_expectation_line.items():
592 merged_line = None 602 merged_line = None
593 if test in self._test_to_expectation_line: 603 if test in self._test_to_expectation_line:
594 self_line = self._test_to_expectation_line[test] 604 self_line = self._test_to_expectation_line[test]
595 605
596 if other_line not in merge_lines_cache[self_line]: 606 if other_line not in merge_lines_cache[self_line]:
597 merge_lines_cache[self_line][other_line] = TestExpectationLi ne.merge_expectation_lines( 607 merge_lines_cache[self_line][other_line] = TestExpectationLi ne.merge_expectation_lines(
598 self_line, other_line, model_all_expectations=False) 608 self_line,
609 other_line,
610 model_all_expectations=False)
599 611
600 merged_line = merge_lines_cache[self_line][other_line] 612 merged_line = merge_lines_cache[self_line][other_line]
601 else: 613 else:
602 merged_line = other_line 614 merged_line = other_line
603 615
604 self._test_to_expectation_line[test] = merged_line 616 self._test_to_expectation_line[test] = merged_line
605 617
606 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_ to_tests) 618 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_ to_tests)
607 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes ts) 619 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes ts)
608 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_ to_tests) 620 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_ to_tests)
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after
671 if item[1] == expectation: 683 if item[1] == expectation:
672 return item[0].upper() 684 return item[0].upper()
673 raise ValueError(expectation) 685 raise ValueError(expectation)
674 686
675 def remove_expectation_line(self, test): 687 def remove_expectation_line(self, test):
676 if not self.has_test(test): 688 if not self.has_test(test):
677 return 689 return
678 self._clear_expectations_for_test(test) 690 self._clear_expectations_for_test(test)
679 del self._test_to_expectation_line[test] 691 del self._test_to_expectation_line[test]
680 692
681 def add_expectation_line(self, expectation_line, 693 def add_expectation_line(self, expectation_line, model_all_expectations=Fals e):
682 model_all_expectations=False):
683 """Returns a list of warnings encountered while matching specifiers.""" 694 """Returns a list of warnings encountered while matching specifiers."""
684 695
685 if expectation_line.is_invalid(): 696 if expectation_line.is_invalid():
686 return 697 return
687 698
688 for test in expectation_line.matching_tests: 699 for test in expectation_line.matching_tests:
689 if self._already_seen_better_match(test, expectation_line): 700 if self._already_seen_better_match(test, expectation_line):
690 continue 701 continue
691 702
692 if model_all_expectations: 703 if model_all_expectations:
693 expectation_line = TestExpectationLine.merge_expectation_lines(s elf.get_expectation_line(test), expectation_line, model_all_expectations) 704 expectation_line = TestExpectationLine.merge_expectation_lines(
705 self.get_expectation_line(test), expectation_line, model_all _expectations)
694 706
695 self._clear_expectations_for_test(test) 707 self._clear_expectations_for_test(test)
696 self._test_to_expectation_line[test] = expectation_line 708 self._test_to_expectation_line[test] = expectation_line
697 self._add_test(test, expectation_line) 709 self._add_test(test, expectation_line)
698 710
699 def _add_test(self, test, expectation_line): 711 def _add_test(self, test, expectation_line):
700 """Sets the expected state for a given test. 712 """Sets the expected state for a given test.
701 713
702 This routine assumes the test has not been added before. If it has, 714 This routine assumes the test has not been added before. If it has,
703 use _clear_expectations_for_test() to reset the state prior to 715 use _clear_expectations_for_test() to reset the state prior to
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after
773 # base path, so we need to check the two sets of specifiers. 785 # base path, so we need to check the two sets of specifiers.
774 786
775 # FIXME: This code was originally designed to allow lines that matched 787 # FIXME: This code was originally designed to allow lines that matched
776 # more specifiers to override lines that matched fewer specifiers. 788 # more specifiers to override lines that matched fewer specifiers.
777 # However, we currently view these as errors. 789 # However, we currently view these as errors.
778 # 790 #
779 # To use the "more specifiers wins" policy, change the errors for overri des 791 # To use the "more specifiers wins" policy, change the errors for overri des
780 # to be warnings and return False". 792 # to be warnings and return False".
781 793
782 if prev_expectation_line.matching_configurations == expectation_line.mat ching_configurations: 794 if prev_expectation_line.matching_configurations == expectation_line.mat ching_configurations:
783 expectation_line.warnings.append('Duplicate or ambiguous entry lines %s:%s and %s:%s.' % ( 795 expectation_line.warnings.append('Duplicate or ambiguous entry lines %s:%s and %s:%s.' %
784 self._shorten_filename(prev_expectation_line.filename), prev_exp ectation_line.line_numbers, 796 (self._shorten_filename(prev_expect ation_line.filename),
785 self._shorten_filename(expectation_line.filename), expectation_l ine.line_numbers)) 797 prev_expectation_line.line_numbers , self._shorten_filename(expectation_line.filename),
798 expectation_line.line_numbers))
786 return True 799 return True
787 800
788 if prev_expectation_line.matching_configurations >= expectation_line.mat ching_configurations: 801 if prev_expectation_line.matching_configurations >= expectation_line.mat ching_configurations:
789 expectation_line.warnings.append('More specific entry for %s on line %s:%s overrides line %s:%s.' % (expectation_line.name, 802 expectation_line.warnings.append('More specific entry for %s on line %s:%s overrides line %s:%s.' %
790 self._shorten_filename(prev_expectation_line.filename), prev_exp ectation_line.line_numbers, 803 (expectation_line.name, self._short en_filename(prev_expectation_line.filename),
791 self._shorten_filename(expectation_line.filename), expectation_l ine.line_numbers)) 804 prev_expectation_line.line_numbers , self._shorten_filename(expectation_line.filename),
805 expectation_line.line_numbers))
792 # FIXME: return False if we want more specific to win. 806 # FIXME: return False if we want more specific to win.
793 return True 807 return True
794 808
795 if prev_expectation_line.matching_configurations <= expectation_line.mat ching_configurations: 809 if prev_expectation_line.matching_configurations <= expectation_line.mat ching_configurations:
796 expectation_line.warnings.append('More specific entry for %s on line %s:%s overrides line %s:%s.' % (expectation_line.name, 810 expectation_line.warnings.append('More specific entry for %s on line %s:%s overrides line %s:%s.' %
797 self._shorten_filename(expectation_line.filename), expectation_l ine.line_numbers, 811 (expectation_line.name, self._short en_filename(expectation_line.filename),
798 self._shorten_filename(prev_expectation_line.filename), prev_exp ectation_line.line_numbers)) 812 expectation_line.line_numbers, sel f._shorten_filename(prev_expectation_line.filename),
813 prev_expectation_line.line_numbers ))
799 return True 814 return True
800 815
801 if prev_expectation_line.matching_configurations & expectation_line.matc hing_configurations: 816 if prev_expectation_line.matching_configurations & expectation_line.matc hing_configurations:
802 expectation_line.warnings.append('Entries for %s on lines %s:%s and %s:%s match overlapping sets of configurations.' % (expectation_line.name, 817 expectation_line.warnings.append('Entries for %s on lines %s:%s and %s:%s match overlapping sets of configurations.' %
803 self._shorten_filename(prev_expectation_line.filename), prev_exp ectation_line.line_numbers, 818 (expectation_line.name, self._short en_filename(prev_expectation_line.filename),
804 self._shorten_filename(expectation_line.filename), expectation_l ine.line_numbers)) 819 prev_expectation_line.line_numbers , self._shorten_filename(expectation_line.filename),
820 expectation_line.line_numbers))
805 return True 821 return True
806 822
807 # Configuration sets are disjoint, then. 823 # Configuration sets are disjoint, then.
808 return False 824 return False
809 825
810 826
811 class TestExpectations(object): 827 class TestExpectations(object):
812 """Test expectations consist of lines with specifications of what 828 """Test expectations consist of lines with specifications of what
813 to expect from layout test cases. The test cases can be directories 829 to expect from layout test cases. The test cases can be directories
814 in which case the expectations apply to all test cases in that 830 in which case the expectations apply to all test cases in that
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
846 'text': TEXT, 862 'text': TEXT,
847 'timeout': TIMEOUT, 863 'timeout': TIMEOUT,
848 'crash': CRASH, 864 'crash': CRASH,
849 'leak': LEAK, 865 'leak': LEAK,
850 'missing': MISSING, 866 'missing': MISSING,
851 TestExpectationParser.SKIP_MODIFIER: SKIP, 867 TestExpectationParser.SKIP_MODIFIER: SKIP,
852 TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBAS ELINE, 868 TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBAS ELINE,
853 TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEED S_MANUAL_REBASELINE, 869 TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEED S_MANUAL_REBASELINE,
854 TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, 870 TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
855 TestExpectationParser.SLOW_MODIFIER: SLOW, 871 TestExpectationParser.SLOW_MODIFIER: SLOW,
856 TestExpectationParser.REBASELINE_MODIFIER: REBASELINE, 872 TestExpectationParser.REBASELINE_MODIFIER: REBASELINE, }
857 }
858 873
859 EXPECTATIONS_TO_STRING = dict((k, v) for (v, k) in EXPECTATIONS.iteritems()) 874 EXPECTATIONS_TO_STRING = dict((k, v) for (v, k) in EXPECTATIONS.iteritems())
860 875
861 # (aggregated by category, pass/fail/skip, type) 876 # (aggregated by category, pass/fail/skip, type)
862 EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped', 877 EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped',
863 PASS: 'passes', 878 PASS: 'passes',
864 FAIL: 'failures', 879 FAIL: 'failures',
865 IMAGE: 'image-only failures', 880 IMAGE: 'image-only failures',
866 TEXT: 'text-only failures', 881 TEXT: 'text-only failures',
867 IMAGE_PLUS_TEXT: 'image and text failures', 882 IMAGE_PLUS_TEXT: 'image and text failures',
868 AUDIO: 'audio failures', 883 AUDIO: 'audio failures',
869 CRASH: 'crashes', 884 CRASH: 'crashes',
870 LEAK: 'leaks', 885 LEAK: 'leaks',
871 TIMEOUT: 'timeouts', 886 TIMEOUT: 'timeouts',
872 MISSING: 'missing results'} 887 MISSING: 'missing results'}
873 888
874 NON_TEST_OUTCOME_EXPECTATIONS = (REBASELINE, SKIP, SLOW, WONTFIX) 889 NON_TEST_OUTCOME_EXPECTATIONS = (REBASELINE, SKIP, SLOW, WONTFIX)
875 890
876 BUILD_TYPES = ('debug', 'release') 891 BUILD_TYPES = ('debug', 'release')
877 892
878 TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, 893 TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, 'now': NOW}
879 'now': NOW}
880 894
881 RESULT_TYPES = {'skip': SKIP, 895 RESULT_TYPES = {'skip': SKIP, 'pass': PASS, 'fail': FAIL, 'flaky': FLAKY}
882 'pass': PASS,
883 'fail': FAIL,
884 'flaky': FLAKY}
885 896
886 @classmethod 897 @classmethod
887 def expectation_from_string(cls, string): 898 def expectation_from_string(cls, string):
888 assert(' ' not in string) # This only handles one expectation at a time . 899 assert (' ' not in string) # This only handles one expectation at a tim e.
889 return cls.EXPECTATIONS.get(string.lower()) 900 return cls.EXPECTATIONS.get(string.lower())
890 901
891 @staticmethod 902 @staticmethod
892 def result_was_expected(result, expected_results, test_needs_rebaselining): 903 def result_was_expected(result, expected_results, test_needs_rebaselining):
893 """Returns whether we got a result we were expecting. 904 """Returns whether we got a result we were expecting.
894 Args: 905 Args:
895 result: actual result of a test execution 906 result: actual result of a test execution
896 expected_results: set of results listed in test_expectations 907 expected_results: set of results listed in test_expectations
897 test_needs_rebaselining: whether test was marked as REBASELINE""" 908 test_needs_rebaselining: whether test was marked as REBASELINE"""
898 if not (set(expected_results) - (set(TestExpectations.NON_TEST_OUTCOME_E XPECTATIONS))): 909 if not (set(expected_results) - (set(TestExpectations.NON_TEST_OUTCOME_E XPECTATIONS))):
899 expected_results = set([PASS]) 910 expected_results = set([PASS])
900 911
901 if result in expected_results: 912 if result in expected_results:
902 return True 913 return True
903 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and (N EEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE in expected_resul ts): 914 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and (N EEDS_REBASELINE in expected_results or
915 N EEDS_MANUAL_REBASELINE in expected_results):
904 return True 916 return True
905 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected _results): 917 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected _results):
906 return True 918 return True
907 if result == MISSING and test_needs_rebaselining: 919 if result == MISSING and test_needs_rebaselining:
908 return True 920 return True
909 if result == SKIP: 921 if result == SKIP:
910 return True 922 return True
911 return False 923 return False
912 924
913 @staticmethod 925 @staticmethod
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after
959 suffixes.add('wav') 971 suffixes.add('wav')
960 if 'MISSING' in expectations: 972 if 'MISSING' in expectations:
961 suffixes.add('txt') 973 suffixes.add('txt')
962 suffixes.add('png') 974 suffixes.add('png')
963 suffixes.add('wav') 975 suffixes.add('wav')
964 return suffixes 976 return suffixes
965 977
966 # FIXME: This constructor does too much work. We should move the actual pars ing of 978 # FIXME: This constructor does too much work. We should move the actual pars ing of
967 # the expectations into separate routines so that linting and handling overr ides 979 # the expectations into separate routines so that linting and handling overr ides
968 # can be controlled separately, and the constructor can be more of a no-op. 980 # can be controlled separately, and the constructor can be more of a no-op.
969 def __init__(self, port, tests=None, include_overrides=True, expectations_di ct=None, model_all_expectations=False, is_lint_mode=False): 981 def __init__(self,
982 port,
983 tests=None,
984 include_overrides=True,
985 expectations_dict=None,
986 model_all_expectations=False,
987 is_lint_mode=False):
970 self._full_test_list = tests 988 self._full_test_list = tests
971 self._test_config = port.test_configuration() 989 self._test_config = port.test_configuration()
972 self._is_lint_mode = is_lint_mode 990 self._is_lint_mode = is_lint_mode
973 self._model_all_expectations = self._is_lint_mode or model_all_expectati ons 991 self._model_all_expectations = self._is_lint_mode or model_all_expectati ons
974 self._model = TestExpectationsModel(self._shorten_filename) 992 self._model = TestExpectationsModel(self._shorten_filename)
975 self._parser = TestExpectationParser(port, tests, self._is_lint_mode) 993 self._parser = TestExpectationParser(port, tests, self._is_lint_mode)
976 self._port = port 994 self._port = port
977 self._skipped_tests_warnings = [] 995 self._skipped_tests_warnings = []
978 self._expectations = [] 996 self._expectations = []
979 997
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after
1050 1068
1051 def _shorten_filename(self, filename): 1069 def _shorten_filename(self, filename):
1052 if filename.startswith(self._port.path_from_webkit_base()): 1070 if filename.startswith(self._port.path_from_webkit_base()):
1053 return self._port.host.filesystem.relpath(filename, self._port.path_ from_webkit_base()) 1071 return self._port.host.filesystem.relpath(filename, self._port.path_ from_webkit_base())
1054 return filename 1072 return filename
1055 1073
1056 def _report_warnings(self): 1074 def _report_warnings(self):
1057 warnings = [] 1075 warnings = []
1058 for expectation in self._expectations: 1076 for expectation in self._expectations:
1059 for warning in expectation.warnings: 1077 for warning in expectation.warnings:
1060 warnings.append('%s:%s %s %s' % (self._shorten_filename(expectat ion.filename), expectation.line_numbers, 1078 warnings.append('%s:%s %s %s' % (self._shorten_filename(expectat ion.filename), expectation.line_numbers, warning,
1061 warning, expectation.name if expectation.expecta tions else expectation.original_string)) 1079 expectation.name if expectation .expectations else expectation.original_string))
1062 1080
1063 if warnings: 1081 if warnings:
1064 self._has_warnings = True 1082 self._has_warnings = True
1065 if self._is_lint_mode: 1083 if self._is_lint_mode:
1066 raise ParseError(warnings) 1084 raise ParseError(warnings)
1067 _log.warning('--lint-test-files warnings:') 1085 _log.warning('--lint-test-files warnings:')
1068 for warning in warnings: 1086 for warning in warnings:
1069 _log.warning(warning) 1087 _log.warning(warning)
1070 _log.warning('') 1088 _log.warning('')
1071 1089
(...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after
1156 # If reconstitute_only_these is an empty list, we want to return ori ginal_string. 1174 # If reconstitute_only_these is an empty list, we want to return ori ginal_string.
1157 # So we need to compare reconstitute_only_these to None, not just ch eck if it's falsey. 1175 # So we need to compare reconstitute_only_these to None, not just ch eck if it's falsey.
1158 if reconstitute_only_these is None or expectation_line in reconstitu te_only_these: 1176 if reconstitute_only_these is None or expectation_line in reconstitu te_only_these:
1159 return expectation_line.to_string(test_configuration_converter) 1177 return expectation_line.to_string(test_configuration_converter)
1160 return expectation_line.original_string 1178 return expectation_line.original_string
1161 1179
1162 def nones_out(expectation_line): 1180 def nones_out(expectation_line):
1163 return expectation_line is not None 1181 return expectation_line is not None
1164 1182
1165 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) 1183 return "\n".join(filter(nones_out, map(serialize, expectation_lines)))
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698