| OLD | NEW |
| 1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| 11 # in the documentation and/or other materials provided with the | 11 # in the documentation and/or other materials provided with the |
| 12 # distribution. | 12 # distribution. |
| 13 # * Neither the name of Google Inc. nor the names of its | 13 # * Neither the name of Google Inc. nor the names of its |
| 14 # contributors may be used to endorse or promote products derived from | 14 # contributors may be used to endorse or promote products derived from |
| 15 # this software without specific prior written permission. | 15 # this software without specific prior written permission. |
| 16 # | 16 # |
| 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 28 | |
| 29 """A helper class for reading in and dealing with tests expectations | 28 """A helper class for reading in and dealing with tests expectations |
| 30 for layout tests. | 29 for layout tests. |
| 31 """ | 30 """ |
| 32 | 31 |
| 33 from collections import defaultdict | 32 from collections import defaultdict |
| 34 | 33 |
| 35 import logging | 34 import logging |
| 36 import re | 35 import re |
| 37 | 36 |
| 38 from webkitpy.layout_tests.models.test_configuration import TestConfigurationCon
verter | 37 from webkitpy.layout_tests.models.test_configuration import TestConfigurationCon
verter |
| 39 | 38 |
| 40 _log = logging.getLogger(__name__) | 39 _log = logging.getLogger(__name__) |
| 41 | 40 |
| 42 | |
| 43 # Test expectation and specifier constants. | 41 # Test expectation and specifier constants. |
| 44 # | 42 # |
| 45 # FIXME: range() starts with 0 which makes if expectation checks harder | 43 # FIXME: range() starts with 0 which makes if expectation checks harder |
| 46 # as PASS is 0. | 44 # as PASS is 0. |
| 47 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, LEAK, SKIP, WO
NTFIX, | 45 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, LEAK, SKIP, WO
NTFIX, SLOW, REBASELINE, NEEDS_REBASELINE, |
| 48 SLOW, REBASELINE, NEEDS_REBASELINE, NEEDS_MANUAL_REBASELINE, MISSING, FLAKY, NO
W, NONE) = range(19) | 46 NEEDS_MANUAL_REBASELINE, MISSING, FLAKY, NOW, NONE) = range(19) |
| 49 | 47 |
| 50 # FIXME: Perhas these two routines should be part of the Port instead? | 48 # FIXME: Perhas these two routines should be part of the Port instead? |
| 51 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt') | 49 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt') |
| 52 | 50 |
| 53 WEBKIT_BUG_PREFIX = 'webkit.org/b/' | 51 WEBKIT_BUG_PREFIX = 'webkit.org/b/' |
| 54 CHROMIUM_BUG_PREFIX = 'crbug.com/' | 52 CHROMIUM_BUG_PREFIX = 'crbug.com/' |
| 55 V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id=' | 53 V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id=' |
| 56 NAMED_BUG_PREFIX = 'Bug(' | 54 NAMED_BUG_PREFIX = 'Bug(' |
| 57 | 55 |
| 58 MISSING_KEYWORD = 'Missing' | 56 MISSING_KEYWORD = 'Missing' |
| 59 NEEDS_REBASELINE_KEYWORD = 'NeedsRebaseline' | 57 NEEDS_REBASELINE_KEYWORD = 'NeedsRebaseline' |
| 60 NEEDS_MANUAL_REBASELINE_KEYWORD = 'NeedsManualRebaseline' | 58 NEEDS_MANUAL_REBASELINE_KEYWORD = 'NeedsManualRebaseline' |
| 61 | 59 |
| 62 | 60 |
| 63 class ParseError(Exception): | 61 class ParseError(Exception): |
| 64 def __init__(self, warnings): | 62 def __init__(self, warnings): |
| 65 super(ParseError, self).__init__() | 63 super(ParseError, self).__init__() |
| 66 self.warnings = warnings | 64 self.warnings = warnings |
| 67 | 65 |
| 68 def __str__(self): | 66 def __str__(self): |
| 69 return '\n'.join(map(str, self.warnings)) | 67 return '\n'.join(map(str, self.warnings)) |
| 70 | 68 |
| 71 def __repr__(self): | 69 def __repr__(self): |
| 72 return 'ParseError(warnings=%s)' % self.warnings | 70 return 'ParseError(warnings=%s)' % self.warnings |
| 73 | 71 |
| 74 | 72 |
| 75 class TestExpectationParser(object): | 73 class TestExpectationParser(object): |
| 76 """Provides parsing facilities for lines in the test_expectation.txt file.""
" | 74 """Provides parsing facilities for lines in the test_expectation.txt file.""
" |
| 77 | 75 |
| 78 # FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make the
case studdly-caps to match the actual file contents. | 76 # FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make |
| 77 # the case studdly-caps to match the actual file contents. |
| 79 REBASELINE_MODIFIER = 'rebaseline' | 78 REBASELINE_MODIFIER = 'rebaseline' |
| 80 NEEDS_REBASELINE_MODIFIER = 'needsrebaseline' | 79 NEEDS_REBASELINE_MODIFIER = 'needsrebaseline' |
| 81 NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline' | 80 NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline' |
| 82 PASS_EXPECTATION = 'pass' | 81 PASS_EXPECTATION = 'pass' |
| 83 SKIP_MODIFIER = 'skip' | 82 SKIP_MODIFIER = 'skip' |
| 84 SLOW_MODIFIER = 'slow' | 83 SLOW_MODIFIER = 'slow' |
| 85 WONTFIX_MODIFIER = 'wontfix' | 84 WONTFIX_MODIFIER = 'wontfix' |
| 86 | 85 |
| 87 TIMEOUT_EXPECTATION = 'timeout' | 86 TIMEOUT_EXPECTATION = 'timeout' |
| 88 | 87 |
| 89 MISSING_BUG_WARNING = 'Test lacks BUG specifier.' | 88 MISSING_BUG_WARNING = 'Test lacks BUG specifier.' |
| 90 | 89 |
| 91 def __init__(self, port, all_tests, is_lint_mode): | 90 def __init__(self, port, all_tests, is_lint_mode): |
| 92 self._port = port | 91 self._port = port |
| 93 self._test_configuration_converter = TestConfigurationConverter(set(port
.all_test_configurations()), port.configuration_specifier_macros()) | 92 self._test_configuration_converter = TestConfigurationConverter( |
| 93 set(port.all_test_configurations()), port.configuration_specifier_ma
cros()) |
| 94 | 94 |
| 95 if all_tests: | 95 if all_tests: |
| 96 self._all_tests = set(all_tests) | 96 self._all_tests = set(all_tests) |
| 97 else: | 97 else: |
| 98 self._all_tests = set() | 98 self._all_tests = set() |
| 99 | 99 |
| 100 self._is_lint_mode = is_lint_mode | 100 self._is_lint_mode = is_lint_mode |
| 101 | 101 |
| 102 def parse(self, filename, expectations_string): | 102 def parse(self, filename, expectations_string): |
| 103 expectation_lines = [] | 103 expectation_lines = [] |
| (...skipping 11 matching lines...) Expand all Loading... |
| 115 expectation_line.name = test_name | 115 expectation_line.name = test_name |
| 116 expectation_line.filename = file_name | 116 expectation_line.filename = file_name |
| 117 expectation_line.expectations = expectations | 117 expectation_line.expectations = expectations |
| 118 return expectation_line | 118 return expectation_line |
| 119 | 119 |
| 120 def expectation_line_for_test(self, test_name, expectations): | 120 def expectation_line_for_test(self, test_name, expectations): |
| 121 expectation_line = self._create_expectation_line(test_name, expectations
, '<Bot TestExpectations>') | 121 expectation_line = self._create_expectation_line(test_name, expectations
, '<Bot TestExpectations>') |
| 122 self._parse_line(expectation_line) | 122 self._parse_line(expectation_line) |
| 123 return expectation_line | 123 return expectation_line |
| 124 | 124 |
| 125 | |
| 126 def expectation_for_skipped_test(self, test_name): | 125 def expectation_for_skipped_test(self, test_name): |
| 127 if not self._port.test_exists(test_name): | 126 if not self._port.test_exists(test_name): |
| 128 _log.warning('The following test %s from the Skipped list doesn\'t e
xist' % test_name) | 127 _log.warning('The following test %s from the Skipped list doesn\'t e
xist' % test_name) |
| 129 expectation_line = self._create_expectation_line(test_name, [TestExpecta
tionParser.PASS_EXPECTATION], '<Skipped file>') | 128 expectation_line = self._create_expectation_line(test_name, [TestExpecta
tionParser.PASS_EXPECTATION], '<Skipped file>') |
| 130 expectation_line.expectations = [TestExpectationParser.SKIP_MODIFIER, Te
stExpectationParser.WONTFIX_MODIFIER] | 129 expectation_line.expectations = [TestExpectationParser.SKIP_MODIFIER, Te
stExpectationParser.WONTFIX_MODIFIER] |
| 131 expectation_line.is_skipped_outside_expectations_file = True | 130 expectation_line.is_skipped_outside_expectations_file = True |
| 132 self._parse_line(expectation_line) | 131 self._parse_line(expectation_line) |
| 133 return expectation_line | 132 return expectation_line |
| 134 | 133 |
| 135 def _parse_line(self, expectation_line): | 134 def _parse_line(self, expectation_line): |
| (...skipping 15 matching lines...) Expand all Loading... |
| 151 self._parse_expectations(expectation_line) | 150 self._parse_expectations(expectation_line) |
| 152 | 151 |
| 153 def _parse_specifier(self, specifier): | 152 def _parse_specifier(self, specifier): |
| 154 return specifier.lower() | 153 return specifier.lower() |
| 155 | 154 |
| 156 def _parse_specifiers(self, expectation_line): | 155 def _parse_specifiers(self, expectation_line): |
| 157 if self._is_lint_mode: | 156 if self._is_lint_mode: |
| 158 self._lint_line(expectation_line) | 157 self._lint_line(expectation_line) |
| 159 | 158 |
| 160 parsed_specifiers = set([self._parse_specifier(specifier) for specifier
in expectation_line.specifiers]) | 159 parsed_specifiers = set([self._parse_specifier(specifier) for specifier
in expectation_line.specifiers]) |
| 161 expectation_line.matching_configurations = self._test_configuration_conv
erter.to_config_set(parsed_specifiers, expectation_line.warnings) | 160 expectation_line.matching_configurations = self._test_configuration_conv
erter.to_config_set(parsed_specifiers, |
| 161
expectation_line.warnings) |
| 162 | 162 |
| 163 def _lint_line(self, expectation_line): | 163 def _lint_line(self, expectation_line): |
| 164 expectations = [expectation.lower() for expectation in expectation_line.
expectations] | 164 expectations = [expectation.lower() for expectation in expectation_line.
expectations] |
| 165 if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectatio
ns: | 165 if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectatio
ns: |
| 166 expectation_line.warnings.append(self.MISSING_BUG_WARNING) | 166 expectation_line.warnings.append(self.MISSING_BUG_WARNING) |
| 167 if self.REBASELINE_MODIFIER in expectations: | 167 if self.REBASELINE_MODIFIER in expectations: |
| 168 expectation_line.warnings.append('REBASELINE should only be used for
running rebaseline.py. Cannot be checked in.') | 168 expectation_line.warnings.append('REBASELINE should only be used for
running rebaseline.py. Cannot be checked in.') |
| 169 | 169 |
| 170 if self.NEEDS_REBASELINE_MODIFIER in expectations or self.NEEDS_MANUAL_R
EBASELINE_MODIFIER in expectations: | 170 if self.NEEDS_REBASELINE_MODIFIER in expectations or self.NEEDS_MANUAL_R
EBASELINE_MODIFIER in expectations: |
| 171 for test in expectation_line.matching_tests: | 171 for test in expectation_line.matching_tests: |
| 172 if self._port.reference_files(test): | 172 if self._port.reference_files(test): |
| 173 expectation_line.warnings.append('A reftest cannot be marked
as NeedsRebaseline/NeedsManualRebaseline') | 173 expectation_line.warnings.append('A reftest cannot be marked
as NeedsRebaseline/NeedsManualRebaseline') |
| 174 | 174 |
| 175 specifiers = [specifier.lower() for specifier in expectation_line.specif
iers] | 175 specifiers = [specifier.lower() for specifier in expectation_line.specif
iers] |
| 176 if (self.REBASELINE_MODIFIER in expectations or self.NEEDS_REBASELINE_MO
DIFIER in expectations) and ('debug' in specifiers or 'release' in specifiers): | 176 if (self.REBASELINE_MODIFIER in expectations or self.NEEDS_REBASELINE_MO
DIFIER in expectations) and ( |
| 177 'debug' in specifiers or 'release' in specifiers): |
| 177 expectation_line.warnings.append('A test cannot be rebaselined for D
ebug/Release.') | 178 expectation_line.warnings.append('A test cannot be rebaselined for D
ebug/Release.') |
| 178 | 179 |
| 179 def _parse_expectations(self, expectation_line): | 180 def _parse_expectations(self, expectation_line): |
| 180 result = set() | 181 result = set() |
| 181 for part in expectation_line.expectations: | 182 for part in expectation_line.expectations: |
| 182 expectation = TestExpectations.expectation_from_string(part) | 183 expectation = TestExpectations.expectation_from_string(part) |
| 183 if expectation is None: # Careful, PASS is currently 0. | 184 if expectation is None: # Careful, PASS is currently 0. |
| 184 expectation_line.warnings.append('Unsupported expectation: %s' %
part) | 185 expectation_line.warnings.append('Unsupported expectation: %s' %
part) |
| 185 continue | 186 continue |
| 186 result.add(expectation) | 187 result.add(expectation) |
| (...skipping 23 matching lines...) Expand all Loading... |
| 210 expectation_line.matching_tests = [test for test in self._all_tests
if test.startswith(expectation_line.path)] | 211 expectation_line.matching_tests = [test for test in self._all_tests
if test.startswith(expectation_line.path)] |
| 211 return | 212 return |
| 212 | 213 |
| 213 # this is a test file, do a quick check if it's in the | 214 # this is a test file, do a quick check if it's in the |
| 214 # full test suite. | 215 # full test suite. |
| 215 if expectation_line.path in self._all_tests: | 216 if expectation_line.path in self._all_tests: |
| 216 expectation_line.matching_tests.append(expectation_line.path) | 217 expectation_line.matching_tests.append(expectation_line.path) |
| 217 | 218 |
| 218 # FIXME: Update the original specifiers and remove this once the old syntax
is gone. | 219 # FIXME: Update the original specifiers and remove this once the old syntax
is gone. |
| 219 _configuration_tokens_list = [ | 220 _configuration_tokens_list = [ |
| 220 'Mac', 'Mac10.9', 'Mac10.10', 'Mac10.11', 'Retina', | 221 'Mac', |
| 221 'Win', 'Win7', 'Win10', | 222 'Mac10.9', |
| 222 'Linux', 'Precise', 'Trusty', | 223 'Mac10.10', |
| 224 'Mac10.11', |
| 225 'Retina', |
| 226 'Win', |
| 227 'Win7', |
| 228 'Win10', |
| 229 'Linux', |
| 230 'Precise', |
| 231 'Trusty', |
| 223 'Android', | 232 'Android', |
| 224 'Release', | 233 'Release', |
| 225 'Debug', | 234 'Debug', |
| 226 ] | 235 ] |
| 227 | 236 |
| 228 _configuration_tokens = dict((token, token.upper()) for token in _configurat
ion_tokens_list) | 237 _configuration_tokens = dict((token, token.upper()) for token in _configurat
ion_tokens_list) |
| 229 _inverted_configuration_tokens = dict((value, name) for name, value in _conf
iguration_tokens.iteritems()) | 238 _inverted_configuration_tokens = dict((value, name) for name, value in _conf
iguration_tokens.iteritems()) |
| 230 | 239 |
| 231 # FIXME: Update the original specifiers list and remove this once the old sy
ntax is gone. | 240 # FIXME: Update the original specifiers list and remove this once the old sy
ntax is gone. |
| 232 _expectation_tokens = { | 241 _expectation_tokens = { |
| 233 'Crash': 'CRASH', | 242 'Crash': 'CRASH', |
| 234 'Leak': 'LEAK', | 243 'Leak': 'LEAK', |
| 235 'Failure': 'FAIL', | 244 'Failure': 'FAIL', |
| 236 MISSING_KEYWORD: 'MISSING', | 245 MISSING_KEYWORD: 'MISSING', |
| 237 'Pass': 'PASS', | 246 'Pass': 'PASS', |
| 238 'Rebaseline': 'REBASELINE', | 247 'Rebaseline': 'REBASELINE', |
| 239 NEEDS_REBASELINE_KEYWORD: 'NEEDSREBASELINE', | 248 NEEDS_REBASELINE_KEYWORD: 'NEEDSREBASELINE', |
| 240 NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE', | 249 NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE', |
| 241 'Skip': 'SKIP', | 250 'Skip': 'SKIP', |
| 242 'Slow': 'SLOW', | 251 'Slow': 'SLOW', |
| 243 'Timeout': 'TIMEOUT', | 252 'Timeout': 'TIMEOUT', |
| 244 'WontFix': 'WONTFIX', | 253 'WontFix': 'WONTFIX', |
| 245 } | 254 } |
| 246 | 255 |
| 247 _inverted_expectation_tokens = dict([(value, name) for name, value in _expec
tation_tokens.iteritems()] + | 256 _inverted_expectation_tokens = dict([(value, name) for name, value in _expec
tation_tokens.iteritems()] + [('TEXT', 'Failure'), ( |
| 248 [('TEXT', 'Failure'), ('IMAGE', 'Failure
'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')]) | 257 'IMAGE', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')]) |
| 249 | 258 |
| 250 # FIXME: Seems like these should be classmethods on TestExpectationLine inst
ead of TestExpectationParser. | 259 # FIXME: Seems like these should be classmethods on TestExpectationLine inst
ead of TestExpectationParser. |
| 251 @classmethod | 260 @classmethod |
| 252 def _tokenize_line(cls, filename, expectation_string, line_number): | 261 def _tokenize_line(cls, filename, expectation_string, line_number): |
| 253 """Tokenizes a line from TestExpectations and returns an unparsed TestEx
pectationLine instance using the old format. | 262 """Tokenizes a line from TestExpectations and returns an unparsed TestEx
pectationLine instance using the old format. |
| 254 | 263 |
| 255 The new format for a test expectation line is: | 264 The new format for a test expectation line is: |
| 256 | 265 |
| 257 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations>
"]" ["#" <comment>] | 266 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations>
"]" ["#" <comment>] |
| 258 | 267 |
| (...skipping 23 matching lines...) Expand all Loading... |
| 282 bugs = [] | 291 bugs = [] |
| 283 specifiers = [] | 292 specifiers = [] |
| 284 name = None | 293 name = None |
| 285 expectations = [] | 294 expectations = [] |
| 286 warnings = [] | 295 warnings = [] |
| 287 has_unrecognized_expectation = False | 296 has_unrecognized_expectation = False |
| 288 | 297 |
| 289 tokens = remaining_string.split() | 298 tokens = remaining_string.split() |
| 290 state = 'start' | 299 state = 'start' |
| 291 for token in tokens: | 300 for token in tokens: |
| 292 if (token.startswith(WEBKIT_BUG_PREFIX) or | 301 if (token.startswith(WEBKIT_BUG_PREFIX) or token.startswith(CHROMIUM
_BUG_PREFIX) or token.startswith(V8_BUG_PREFIX) or |
| 293 token.startswith(CHROMIUM_BUG_PREFIX) or | 302 token.startswith(NAMED_BUG_PREFIX)): |
| 294 token.startswith(V8_BUG_PREFIX) or | |
| 295 token.startswith(NAMED_BUG_PREFIX)): | |
| 296 if state != 'start': | 303 if state != 'start': |
| 297 warnings.append('"%s" is not at the start of the line.' % to
ken) | 304 warnings.append('"%s" is not at the start of the line.' % to
ken) |
| 298 break | 305 break |
| 299 if token.startswith(WEBKIT_BUG_PREFIX): | 306 if token.startswith(WEBKIT_BUG_PREFIX): |
| 300 bugs.append(token) | 307 bugs.append(token) |
| 301 elif token.startswith(CHROMIUM_BUG_PREFIX): | 308 elif token.startswith(CHROMIUM_BUG_PREFIX): |
| 302 bugs.append(token) | 309 bugs.append(token) |
| 303 elif token.startswith(V8_BUG_PREFIX): | 310 elif token.startswith(V8_BUG_PREFIX): |
| 304 bugs.append(token) | 311 bugs.append(token) |
| 305 else: | 312 else: |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 352 if 'WONTFIX' in expectations and 'SKIP' not in expectations: | 359 if 'WONTFIX' in expectations and 'SKIP' not in expectations: |
| 353 expectations.append('SKIP') | 360 expectations.append('SKIP') |
| 354 | 361 |
| 355 if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(exp
ectations) - set(['SKIP', 'WONTFIX'])): | 362 if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(exp
ectations) - set(['SKIP', 'WONTFIX'])): |
| 356 warnings.append('A test marked Skip or WontFix must not have other e
xpectations.') | 363 warnings.append('A test marked Skip or WontFix must not have other e
xpectations.') |
| 357 | 364 |
| 358 if 'SLOW' in expectations and 'SlowTests' not in filename: | 365 if 'SLOW' in expectations and 'SlowTests' not in filename: |
| 359 warnings.append('SLOW tests should ony be added to SlowTests and not
to TestExpectations.') | 366 warnings.append('SLOW tests should ony be added to SlowTests and not
to TestExpectations.') |
| 360 | 367 |
| 361 if 'WONTFIX' in expectations and ('NeverFixTests' not in filename and 'S
taleTestExpectations' not in filename): | 368 if 'WONTFIX' in expectations and ('NeverFixTests' not in filename and 'S
taleTestExpectations' not in filename): |
| 362 warnings.append('WONTFIX tests should ony be added to NeverFixTests
or StaleTestExpectations and not to TestExpectations.') | 369 warnings.append( |
| 370 'WONTFIX tests should ony be added to NeverFixTests or StaleTest
Expectations and not to TestExpectations.') |
| 363 | 371 |
| 364 if 'NeverFixTests' in filename and expectations != ['WONTFIX', 'SKIP']: | 372 if 'NeverFixTests' in filename and expectations != ['WONTFIX', 'SKIP']: |
| 365 warnings.append('Only WONTFIX expectations are allowed in NeverFixTe
sts') | 373 warnings.append('Only WONTFIX expectations are allowed in NeverFixTe
sts') |
| 366 | 374 |
| 367 if 'SlowTests' in filename and expectations != ['SLOW']: | 375 if 'SlowTests' in filename and expectations != ['SLOW']: |
| 368 warnings.append('Only SLOW expectations are allowed in SlowTests') | 376 warnings.append('Only SLOW expectations are allowed in SlowTests') |
| 369 | 377 |
| 370 if not expectations and not has_unrecognized_expectation: | 378 if not expectations and not has_unrecognized_expectation: |
| 371 warnings.append('Missing expectations.') | 379 warnings.append('Missing expectations.') |
| 372 | 380 |
| (...skipping 25 matching lines...) Expand all Loading... |
| 398 self.parsed_specifiers = [] | 406 self.parsed_specifiers = [] |
| 399 self.matching_configurations = set() | 407 self.matching_configurations = set() |
| 400 self.expectations = [] | 408 self.expectations = [] |
| 401 self.parsed_expectations = set() | 409 self.parsed_expectations = set() |
| 402 self.comment = None | 410 self.comment = None |
| 403 self.matching_tests = [] | 411 self.matching_tests = [] |
| 404 self.warnings = [] | 412 self.warnings = [] |
| 405 self.is_skipped_outside_expectations_file = False | 413 self.is_skipped_outside_expectations_file = False |
| 406 | 414 |
| 407 def __str__(self): | 415 def __str__(self): |
| 408 return "TestExpectationLine{name=%s, matching_configurations=%s, origina
l_string=%s}" % (self.name, self.matching_configurations, self.original_string) | 416 return "TestExpectationLine{name=%s, matching_configurations=%s, origina
l_string=%s}" % ( |
| 417 self.name, self.matching_configurations, self.original_string) |
| 409 | 418 |
| 410 def __eq__(self, other): | 419 def __eq__(self, other): |
| 411 return (self.original_string == other.original_string | 420 return (self.original_string == other.original_string and self.filename
== other.filename and |
| 412 and self.filename == other.filename | 421 self.line_numbers == other.line_numbers and self.name == other.n
ame and self.path == other.path and |
| 413 and self.line_numbers == other.line_numbers | 422 self.bugs == other.bugs and self.specifiers == other.specifiers
and |
| 414 and self.name == other.name | 423 self.parsed_specifiers == other.parsed_specifiers and |
| 415 and self.path == other.path | 424 self.matching_configurations == other.matching_configurations an
d self.expectations == other.expectations and |
| 416 and self.bugs == other.bugs | 425 self.parsed_expectations == other.parsed_expectations and self.c
omment == other.comment and |
| 417 and self.specifiers == other.specifiers | 426 self.matching_tests == other.matching_tests and self.warnings ==
other.warnings and |
| 418 and self.parsed_specifiers == other.parsed_specifiers | 427 self.is_skipped_outside_expectations_file == other.is_skipped_ou
tside_expectations_file) |
| 419 and self.matching_configurations == other.matching_configurations | |
| 420 and self.expectations == other.expectations | |
| 421 and self.parsed_expectations == other.parsed_expectations | |
| 422 and self.comment == other.comment | |
| 423 and self.matching_tests == other.matching_tests | |
| 424 and self.warnings == other.warnings | |
| 425 and self.is_skipped_outside_expectations_file == other.is_skipped_ou
tside_expectations_file) | |
| 426 | 428 |
| 427 def is_invalid(self): | 429 def is_invalid(self): |
| 428 return bool(self.warnings and self.warnings != [TestExpectationParser.MI
SSING_BUG_WARNING]) | 430 return bool(self.warnings and self.warnings != [TestExpectationParser.MI
SSING_BUG_WARNING]) |
| 429 | 431 |
| 430 def is_flaky(self): | 432 def is_flaky(self): |
| 431 return len(self.parsed_expectations) > 1 | 433 return len(self.parsed_expectations) > 1 |
| 432 | 434 |
| 433 def is_whitespace_or_comment(self): | 435 def is_whitespace_or_comment(self): |
| 434 return bool(re.match("^\s*$", self.original_string.split('#')[0])) | 436 return bool(re.match("^\s*$", self.original_string.split('#')[0])) |
| 435 | 437 |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 467 result.bugs = list(set(line1.bugs) | set(line2.bugs)) | 469 result.bugs = list(set(line1.bugs) | set(line2.bugs)) |
| 468 result.specifiers = list(set(line1.specifiers) | set(line2.specifiers)) | 470 result.specifiers = list(set(line1.specifiers) | set(line2.specifiers)) |
| 469 result.parsed_specifiers = list(set(line1.parsed_specifiers) | set(line2
.parsed_specifiers)) | 471 result.parsed_specifiers = list(set(line1.parsed_specifiers) | set(line2
.parsed_specifiers)) |
| 470 result.matching_configurations = set(line1.matching_configurations) | se
t(line2.matching_configurations) | 472 result.matching_configurations = set(line1.matching_configurations) | se
t(line2.matching_configurations) |
| 471 result.matching_tests = list(list(set(line1.matching_tests) | set(line2.
matching_tests))) | 473 result.matching_tests = list(list(set(line1.matching_tests) | set(line2.
matching_tests))) |
| 472 result.warnings = list(set(line1.warnings) | set(line2.warnings)) | 474 result.warnings = list(set(line1.warnings) | set(line2.warnings)) |
| 473 result.is_skipped_outside_expectations_file = line1.is_skipped_outside_e
xpectations_file or line2.is_skipped_outside_expectations_file | 475 result.is_skipped_outside_expectations_file = line1.is_skipped_outside_e
xpectations_file or line2.is_skipped_outside_expectations_file |
| 474 return result | 476 return result |
| 475 | 477 |
| 476 def to_string(self, test_configuration_converter, include_specifiers=True, i
nclude_expectations=True, include_comment=True): | 478 def to_string(self, test_configuration_converter, include_specifiers=True, i
nclude_expectations=True, include_comment=True): |
| 477 parsed_expectation_to_string = dict([[parsed_expectation, expectation_st
ring] for expectation_string, parsed_expectation in TestExpectations.EXPECTATION
S.items()]) | 479 parsed_expectation_to_string = dict([[parsed_expectation, expectation_st
ring] |
| 480 for expectation_string, parsed_expe
ctation in TestExpectations.EXPECTATIONS.items()]) |
| 478 | 481 |
| 479 if self.is_invalid(): | 482 if self.is_invalid(): |
| 480 return self.original_string or '' | 483 return self.original_string or '' |
| 481 | 484 |
| 482 if self.name is None: | 485 if self.name is None: |
| 483 return '' if self.comment is None else "#%s" % self.comment | 486 return '' if self.comment is None else "#%s" % self.comment |
| 484 | 487 |
| 485 if test_configuration_converter and self.bugs: | 488 if test_configuration_converter and self.bugs: |
| 486 specifiers_list = test_configuration_converter.to_specifiers_list(se
lf.matching_configurations) | 489 specifiers_list = test_configuration_converter.to_specifiers_list(se
lf.matching_configurations) |
| 487 result = [] | 490 result = [] |
| 488 for specifiers in specifiers_list: | 491 for specifiers in specifiers_list: |
| 489 # FIXME: this is silly that we join the specifiers and then imme
diately split them. | 492 # FIXME: this is silly that we join the specifiers and then imme
diately split them. |
| 490 specifiers = self._serialize_parsed_specifiers(test_configuratio
n_converter, specifiers).split() | 493 specifiers = self._serialize_parsed_specifiers(test_configuratio
n_converter, specifiers).split() |
| 491 expectations = self._serialize_parsed_expectations(parsed_expect
ation_to_string).split() | 494 expectations = self._serialize_parsed_expectations(parsed_expect
ation_to_string).split() |
| 492 result.append(self._format_line(self.bugs, specifiers, self.name
, expectations, self.comment)) | 495 result.append(self._format_line(self.bugs, specifiers, self.name
, expectations, self.comment)) |
| 493 return "\n".join(result) if result else None | 496 return "\n".join(result) if result else None |
| 494 | 497 |
| 495 return self._format_line(self.bugs, self.specifiers, self.name, self.exp
ectations, self.comment, | 498 return self._format_line(self.bugs, self.specifiers, self.name, self.exp
ectations, self.comment, include_specifiers, |
| 496 include_specifiers, include_expectations, include_comment) | 499 include_expectations, include_comment) |
| 497 | 500 |
| 498 def to_csv(self): | 501 def to_csv(self): |
| 499 # Note that this doesn't include the comments. | 502 # Note that this doesn't include the comments. |
| 500 return '%s,%s,%s,%s' % (self.name, ' '.join(self.bugs), ' '.join(self.sp
ecifiers), ' '.join(self.expectations)) | 503 return '%s,%s,%s,%s' % (self.name, ' '.join(self.bugs), ' '.join(self.sp
ecifiers), ' '.join(self.expectations)) |
| 501 | 504 |
| 502 def _serialize_parsed_expectations(self, parsed_expectation_to_string): | 505 def _serialize_parsed_expectations(self, parsed_expectation_to_string): |
| 503 result = [] | 506 result = [] |
| 504 for index in TestExpectations.EXPECTATIONS.values(): | 507 for index in TestExpectations.EXPECTATIONS.values(): |
| 505 if index in self.parsed_expectations: | 508 if index in self.parsed_expectations: |
| 506 result.append(parsed_expectation_to_string[index]) | 509 result.append(parsed_expectation_to_string[index]) |
| 507 return ' '.join(result) | 510 return ' '.join(result) |
| 508 | 511 |
| 509 def _serialize_parsed_specifiers(self, test_configuration_converter, specifi
ers): | 512 def _serialize_parsed_specifiers(self, test_configuration_converter, specifi
ers): |
| 510 result = [] | 513 result = [] |
| 511 result.extend(sorted(self.parsed_specifiers)) | 514 result.extend(sorted(self.parsed_specifiers)) |
| 512 result.extend(test_configuration_converter.specifier_sorter().sort_speci
fiers(specifiers)) | 515 result.extend(test_configuration_converter.specifier_sorter().sort_speci
fiers(specifiers)) |
| 513 return ' '.join(result) | 516 return ' '.join(result) |
| 514 | 517 |
| 515 @staticmethod | 518 @staticmethod |
| 516 def _filter_redundant_expectations(expectations): | 519 def _filter_redundant_expectations(expectations): |
| 517 if set(expectations) == set(['Pass', 'Skip']): | 520 if set(expectations) == set(['Pass', 'Skip']): |
| 518 return ['Skip'] | 521 return ['Skip'] |
| 519 if set(expectations) == set(['Pass', 'Slow']): | 522 if set(expectations) == set(['Pass', 'Slow']): |
| 520 return ['Slow'] | 523 return ['Slow'] |
| 521 return expectations | 524 return expectations |
| 522 | 525 |
| 523 @staticmethod | 526 @staticmethod |
| 524 def _format_line(bugs, specifiers, name, expectations, comment, include_spec
ifiers=True, include_expectations=True, include_comment=True): | 527 def _format_line(bugs, |
| 528 specifiers, |
| 529 name, |
| 530 expectations, |
| 531 comment, |
| 532 include_specifiers=True, |
| 533 include_expectations=True, |
| 534 include_comment=True): |
| 525 new_specifiers = [] | 535 new_specifiers = [] |
| 526 new_expectations = [] | 536 new_expectations = [] |
| 527 for specifier in specifiers: | 537 for specifier in specifiers: |
| 528 # FIXME: Make this all work with the mixed-cased specifiers (e.g. Wo
ntFix, Slow, etc). | 538 # FIXME: Make this all work with the mixed-cased specifiers (e.g. Wo
ntFix, Slow, etc). |
| 529 specifier = specifier.upper() | 539 specifier = specifier.upper() |
| 530 new_specifiers.append(TestExpectationParser._inverted_configuration_
tokens.get(specifier, specifier)) | 540 new_specifiers.append(TestExpectationParser._inverted_configuration_
tokens.get(specifier, specifier)) |
| 531 | 541 |
| 532 for expectation in expectations: | 542 for expectation in expectations: |
| 533 expectation = expectation.upper() | 543 expectation = expectation.upper() |
| 534 new_expectations.append(TestExpectationParser._inverted_expectation_
tokens.get(expectation, expectation)) | 544 new_expectations.append(TestExpectationParser._inverted_expectation_
tokens.get(expectation, expectation)) |
| (...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 588 # so that we only call that n^2 in the number of *lines*. | 598 # so that we only call that n^2 in the number of *lines*. |
| 589 merge_lines_cache = defaultdict(dict) | 599 merge_lines_cache = defaultdict(dict) |
| 590 | 600 |
| 591 for test, other_line in other._test_to_expectation_line.items(): | 601 for test, other_line in other._test_to_expectation_line.items(): |
| 592 merged_line = None | 602 merged_line = None |
| 593 if test in self._test_to_expectation_line: | 603 if test in self._test_to_expectation_line: |
| 594 self_line = self._test_to_expectation_line[test] | 604 self_line = self._test_to_expectation_line[test] |
| 595 | 605 |
| 596 if other_line not in merge_lines_cache[self_line]: | 606 if other_line not in merge_lines_cache[self_line]: |
| 597 merge_lines_cache[self_line][other_line] = TestExpectationLi
ne.merge_expectation_lines( | 607 merge_lines_cache[self_line][other_line] = TestExpectationLi
ne.merge_expectation_lines( |
| 598 self_line, other_line, model_all_expectations=False) | 608 self_line, |
| 609 other_line, |
| 610 model_all_expectations=False) |
| 599 | 611 |
| 600 merged_line = merge_lines_cache[self_line][other_line] | 612 merged_line = merge_lines_cache[self_line][other_line] |
| 601 else: | 613 else: |
| 602 merged_line = other_line | 614 merged_line = other_line |
| 603 | 615 |
| 604 self._test_to_expectation_line[test] = merged_line | 616 self._test_to_expectation_line[test] = merged_line |
| 605 | 617 |
| 606 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_
to_tests) | 618 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_
to_tests) |
| 607 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes
ts) | 619 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes
ts) |
| 608 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_
to_tests) | 620 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_
to_tests) |
| (...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 671 if item[1] == expectation: | 683 if item[1] == expectation: |
| 672 return item[0].upper() | 684 return item[0].upper() |
| 673 raise ValueError(expectation) | 685 raise ValueError(expectation) |
| 674 | 686 |
| 675 def remove_expectation_line(self, test): | 687 def remove_expectation_line(self, test): |
| 676 if not self.has_test(test): | 688 if not self.has_test(test): |
| 677 return | 689 return |
| 678 self._clear_expectations_for_test(test) | 690 self._clear_expectations_for_test(test) |
| 679 del self._test_to_expectation_line[test] | 691 del self._test_to_expectation_line[test] |
| 680 | 692 |
| 681 def add_expectation_line(self, expectation_line, | 693 def add_expectation_line(self, expectation_line, model_all_expectations=Fals
e): |
| 682 model_all_expectations=False): | |
| 683 """Returns a list of warnings encountered while matching specifiers.""" | 694 """Returns a list of warnings encountered while matching specifiers.""" |
| 684 | 695 |
| 685 if expectation_line.is_invalid(): | 696 if expectation_line.is_invalid(): |
| 686 return | 697 return |
| 687 | 698 |
| 688 for test in expectation_line.matching_tests: | 699 for test in expectation_line.matching_tests: |
| 689 if self._already_seen_better_match(test, expectation_line): | 700 if self._already_seen_better_match(test, expectation_line): |
| 690 continue | 701 continue |
| 691 | 702 |
| 692 if model_all_expectations: | 703 if model_all_expectations: |
| 693 expectation_line = TestExpectationLine.merge_expectation_lines(s
elf.get_expectation_line(test), expectation_line, model_all_expectations) | 704 expectation_line = TestExpectationLine.merge_expectation_lines( |
| 705 self.get_expectation_line(test), expectation_line, model_all
_expectations) |
| 694 | 706 |
| 695 self._clear_expectations_for_test(test) | 707 self._clear_expectations_for_test(test) |
| 696 self._test_to_expectation_line[test] = expectation_line | 708 self._test_to_expectation_line[test] = expectation_line |
| 697 self._add_test(test, expectation_line) | 709 self._add_test(test, expectation_line) |
| 698 | 710 |
| 699 def _add_test(self, test, expectation_line): | 711 def _add_test(self, test, expectation_line): |
| 700 """Sets the expected state for a given test. | 712 """Sets the expected state for a given test. |
| 701 | 713 |
| 702 This routine assumes the test has not been added before. If it has, | 714 This routine assumes the test has not been added before. If it has, |
| 703 use _clear_expectations_for_test() to reset the state prior to | 715 use _clear_expectations_for_test() to reset the state prior to |
| (...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 773 # base path, so we need to check the two sets of specifiers. | 785 # base path, so we need to check the two sets of specifiers. |
| 774 | 786 |
| 775 # FIXME: This code was originally designed to allow lines that matched | 787 # FIXME: This code was originally designed to allow lines that matched |
| 776 # more specifiers to override lines that matched fewer specifiers. | 788 # more specifiers to override lines that matched fewer specifiers. |
| 777 # However, we currently view these as errors. | 789 # However, we currently view these as errors. |
| 778 # | 790 # |
| 779 # To use the "more specifiers wins" policy, change the errors for overri
des | 791 # To use the "more specifiers wins" policy, change the errors for overri
des |
| 780 # to be warnings and return False". | 792 # to be warnings and return False". |
| 781 | 793 |
| 782 if prev_expectation_line.matching_configurations == expectation_line.mat
ching_configurations: | 794 if prev_expectation_line.matching_configurations == expectation_line.mat
ching_configurations: |
| 783 expectation_line.warnings.append('Duplicate or ambiguous entry lines
%s:%s and %s:%s.' % ( | 795 expectation_line.warnings.append('Duplicate or ambiguous entry lines
%s:%s and %s:%s.' % |
| 784 self._shorten_filename(prev_expectation_line.filename), prev_exp
ectation_line.line_numbers, | 796 (self._shorten_filename(prev_expect
ation_line.filename), |
| 785 self._shorten_filename(expectation_line.filename), expectation_l
ine.line_numbers)) | 797 prev_expectation_line.line_numbers
, self._shorten_filename(expectation_line.filename), |
| 798 expectation_line.line_numbers)) |
| 786 return True | 799 return True |
| 787 | 800 |
| 788 if prev_expectation_line.matching_configurations >= expectation_line.mat
ching_configurations: | 801 if prev_expectation_line.matching_configurations >= expectation_line.mat
ching_configurations: |
| 789 expectation_line.warnings.append('More specific entry for %s on line
%s:%s overrides line %s:%s.' % (expectation_line.name, | 802 expectation_line.warnings.append('More specific entry for %s on line
%s:%s overrides line %s:%s.' % |
| 790 self._shorten_filename(prev_expectation_line.filename), prev_exp
ectation_line.line_numbers, | 803 (expectation_line.name, self._short
en_filename(prev_expectation_line.filename), |
| 791 self._shorten_filename(expectation_line.filename), expectation_l
ine.line_numbers)) | 804 prev_expectation_line.line_numbers
, self._shorten_filename(expectation_line.filename), |
| 805 expectation_line.line_numbers)) |
| 792 # FIXME: return False if we want more specific to win. | 806 # FIXME: return False if we want more specific to win. |
| 793 return True | 807 return True |
| 794 | 808 |
| 795 if prev_expectation_line.matching_configurations <= expectation_line.mat
ching_configurations: | 809 if prev_expectation_line.matching_configurations <= expectation_line.mat
ching_configurations: |
| 796 expectation_line.warnings.append('More specific entry for %s on line
%s:%s overrides line %s:%s.' % (expectation_line.name, | 810 expectation_line.warnings.append('More specific entry for %s on line
%s:%s overrides line %s:%s.' % |
| 797 self._shorten_filename(expectation_line.filename), expectation_l
ine.line_numbers, | 811 (expectation_line.name, self._short
en_filename(expectation_line.filename), |
| 798 self._shorten_filename(prev_expectation_line.filename), prev_exp
ectation_line.line_numbers)) | 812 expectation_line.line_numbers, sel
f._shorten_filename(prev_expectation_line.filename), |
| 813 prev_expectation_line.line_numbers
)) |
| 799 return True | 814 return True |
| 800 | 815 |
| 801 if prev_expectation_line.matching_configurations & expectation_line.matc
hing_configurations: | 816 if prev_expectation_line.matching_configurations & expectation_line.matc
hing_configurations: |
| 802 expectation_line.warnings.append('Entries for %s on lines %s:%s and
%s:%s match overlapping sets of configurations.' % (expectation_line.name, | 817 expectation_line.warnings.append('Entries for %s on lines %s:%s and
%s:%s match overlapping sets of configurations.' % |
| 803 self._shorten_filename(prev_expectation_line.filename), prev_exp
ectation_line.line_numbers, | 818 (expectation_line.name, self._short
en_filename(prev_expectation_line.filename), |
| 804 self._shorten_filename(expectation_line.filename), expectation_l
ine.line_numbers)) | 819 prev_expectation_line.line_numbers
, self._shorten_filename(expectation_line.filename), |
| 820 expectation_line.line_numbers)) |
| 805 return True | 821 return True |
| 806 | 822 |
| 807 # Configuration sets are disjoint, then. | 823 # Configuration sets are disjoint, then. |
| 808 return False | 824 return False |
| 809 | 825 |
| 810 | 826 |
| 811 class TestExpectations(object): | 827 class TestExpectations(object): |
| 812 """Test expectations consist of lines with specifications of what | 828 """Test expectations consist of lines with specifications of what |
| 813 to expect from layout test cases. The test cases can be directories | 829 to expect from layout test cases. The test cases can be directories |
| 814 in which case the expectations apply to all test cases in that | 830 in which case the expectations apply to all test cases in that |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 846 'text': TEXT, | 862 'text': TEXT, |
| 847 'timeout': TIMEOUT, | 863 'timeout': TIMEOUT, |
| 848 'crash': CRASH, | 864 'crash': CRASH, |
| 849 'leak': LEAK, | 865 'leak': LEAK, |
| 850 'missing': MISSING, | 866 'missing': MISSING, |
| 851 TestExpectationParser.SKIP_MODIFIER: SKIP, | 867 TestExpectationParser.SKIP_MODIFIER: SKIP, |
| 852 TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBAS
ELINE, | 868 TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBAS
ELINE, |
| 853 TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEED
S_MANUAL_REBASELINE, | 869 TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEED
S_MANUAL_REBASELINE, |
| 854 TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, | 870 TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, |
| 855 TestExpectationParser.SLOW_MODIFIER: SLOW, | 871 TestExpectationParser.SLOW_MODIFIER: SLOW, |
| 856 TestExpectationParser.REBASELINE_MODIFIER: REBASELINE, | 872 TestExpectationParser.REBASELINE_MODIFIER: REBASELINE, } |
| 857 } | |
| 858 | 873 |
| 859 EXPECTATIONS_TO_STRING = dict((k, v) for (v, k) in EXPECTATIONS.iteritems()) | 874 EXPECTATIONS_TO_STRING = dict((k, v) for (v, k) in EXPECTATIONS.iteritems()) |
| 860 | 875 |
| 861 # (aggregated by category, pass/fail/skip, type) | 876 # (aggregated by category, pass/fail/skip, type) |
| 862 EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped', | 877 EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped', |
| 863 PASS: 'passes', | 878 PASS: 'passes', |
| 864 FAIL: 'failures', | 879 FAIL: 'failures', |
| 865 IMAGE: 'image-only failures', | 880 IMAGE: 'image-only failures', |
| 866 TEXT: 'text-only failures', | 881 TEXT: 'text-only failures', |
| 867 IMAGE_PLUS_TEXT: 'image and text failures', | 882 IMAGE_PLUS_TEXT: 'image and text failures', |
| 868 AUDIO: 'audio failures', | 883 AUDIO: 'audio failures', |
| 869 CRASH: 'crashes', | 884 CRASH: 'crashes', |
| 870 LEAK: 'leaks', | 885 LEAK: 'leaks', |
| 871 TIMEOUT: 'timeouts', | 886 TIMEOUT: 'timeouts', |
| 872 MISSING: 'missing results'} | 887 MISSING: 'missing results'} |
| 873 | 888 |
| 874 NON_TEST_OUTCOME_EXPECTATIONS = (REBASELINE, SKIP, SLOW, WONTFIX) | 889 NON_TEST_OUTCOME_EXPECTATIONS = (REBASELINE, SKIP, SLOW, WONTFIX) |
| 875 | 890 |
| 876 BUILD_TYPES = ('debug', 'release') | 891 BUILD_TYPES = ('debug', 'release') |
| 877 | 892 |
| 878 TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, | 893 TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, 'now': NOW} |
| 879 'now': NOW} | |
| 880 | 894 |
| 881 RESULT_TYPES = {'skip': SKIP, | 895 RESULT_TYPES = {'skip': SKIP, 'pass': PASS, 'fail': FAIL, 'flaky': FLAKY} |
| 882 'pass': PASS, | |
| 883 'fail': FAIL, | |
| 884 'flaky': FLAKY} | |
| 885 | 896 |
| 886 @classmethod | 897 @classmethod |
| 887 def expectation_from_string(cls, string): | 898 def expectation_from_string(cls, string): |
| 888 assert(' ' not in string) # This only handles one expectation at a time
. | 899 assert (' ' not in string) # This only handles one expectation at a tim
e. |
| 889 return cls.EXPECTATIONS.get(string.lower()) | 900 return cls.EXPECTATIONS.get(string.lower()) |
| 890 | 901 |
| 891 @staticmethod | 902 @staticmethod |
| 892 def result_was_expected(result, expected_results, test_needs_rebaselining): | 903 def result_was_expected(result, expected_results, test_needs_rebaselining): |
| 893 """Returns whether we got a result we were expecting. | 904 """Returns whether we got a result we were expecting. |
| 894 Args: | 905 Args: |
| 895 result: actual result of a test execution | 906 result: actual result of a test execution |
| 896 expected_results: set of results listed in test_expectations | 907 expected_results: set of results listed in test_expectations |
| 897 test_needs_rebaselining: whether test was marked as REBASELINE""" | 908 test_needs_rebaselining: whether test was marked as REBASELINE""" |
| 898 if not (set(expected_results) - (set(TestExpectations.NON_TEST_OUTCOME_E
XPECTATIONS))): | 909 if not (set(expected_results) - (set(TestExpectations.NON_TEST_OUTCOME_E
XPECTATIONS))): |
| 899 expected_results = set([PASS]) | 910 expected_results = set([PASS]) |
| 900 | 911 |
| 901 if result in expected_results: | 912 if result in expected_results: |
| 902 return True | 913 return True |
| 903 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and (N
EEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE in expected_resul
ts): | 914 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and (N
EEDS_REBASELINE in expected_results or |
| 915 N
EEDS_MANUAL_REBASELINE in expected_results): |
| 904 return True | 916 return True |
| 905 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected
_results): | 917 if result in (TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected
_results): |
| 906 return True | 918 return True |
| 907 if result == MISSING and test_needs_rebaselining: | 919 if result == MISSING and test_needs_rebaselining: |
| 908 return True | 920 return True |
| 909 if result == SKIP: | 921 if result == SKIP: |
| 910 return True | 922 return True |
| 911 return False | 923 return False |
| 912 | 924 |
| 913 @staticmethod | 925 @staticmethod |
| (...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 959 suffixes.add('wav') | 971 suffixes.add('wav') |
| 960 if 'MISSING' in expectations: | 972 if 'MISSING' in expectations: |
| 961 suffixes.add('txt') | 973 suffixes.add('txt') |
| 962 suffixes.add('png') | 974 suffixes.add('png') |
| 963 suffixes.add('wav') | 975 suffixes.add('wav') |
| 964 return suffixes | 976 return suffixes |
| 965 | 977 |
| 966 # FIXME: This constructor does too much work. We should move the actual pars
ing of | 978 # FIXME: This constructor does too much work. We should move the actual pars
ing of |
| 967 # the expectations into separate routines so that linting and handling overr
ides | 979 # the expectations into separate routines so that linting and handling overr
ides |
| 968 # can be controlled separately, and the constructor can be more of a no-op. | 980 # can be controlled separately, and the constructor can be more of a no-op. |
| 969 def __init__(self, port, tests=None, include_overrides=True, expectations_di
ct=None, model_all_expectations=False, is_lint_mode=False): | 981 def __init__(self, |
| 982 port, |
| 983 tests=None, |
| 984 include_overrides=True, |
| 985 expectations_dict=None, |
| 986 model_all_expectations=False, |
| 987 is_lint_mode=False): |
| 970 self._full_test_list = tests | 988 self._full_test_list = tests |
| 971 self._test_config = port.test_configuration() | 989 self._test_config = port.test_configuration() |
| 972 self._is_lint_mode = is_lint_mode | 990 self._is_lint_mode = is_lint_mode |
| 973 self._model_all_expectations = self._is_lint_mode or model_all_expectati
ons | 991 self._model_all_expectations = self._is_lint_mode or model_all_expectati
ons |
| 974 self._model = TestExpectationsModel(self._shorten_filename) | 992 self._model = TestExpectationsModel(self._shorten_filename) |
| 975 self._parser = TestExpectationParser(port, tests, self._is_lint_mode) | 993 self._parser = TestExpectationParser(port, tests, self._is_lint_mode) |
| 976 self._port = port | 994 self._port = port |
| 977 self._skipped_tests_warnings = [] | 995 self._skipped_tests_warnings = [] |
| 978 self._expectations = [] | 996 self._expectations = [] |
| 979 | 997 |
| (...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1050 | 1068 |
| 1051 def _shorten_filename(self, filename): | 1069 def _shorten_filename(self, filename): |
| 1052 if filename.startswith(self._port.path_from_webkit_base()): | 1070 if filename.startswith(self._port.path_from_webkit_base()): |
| 1053 return self._port.host.filesystem.relpath(filename, self._port.path_
from_webkit_base()) | 1071 return self._port.host.filesystem.relpath(filename, self._port.path_
from_webkit_base()) |
| 1054 return filename | 1072 return filename |
| 1055 | 1073 |
| 1056 def _report_warnings(self): | 1074 def _report_warnings(self): |
| 1057 warnings = [] | 1075 warnings = [] |
| 1058 for expectation in self._expectations: | 1076 for expectation in self._expectations: |
| 1059 for warning in expectation.warnings: | 1077 for warning in expectation.warnings: |
| 1060 warnings.append('%s:%s %s %s' % (self._shorten_filename(expectat
ion.filename), expectation.line_numbers, | 1078 warnings.append('%s:%s %s %s' % (self._shorten_filename(expectat
ion.filename), expectation.line_numbers, warning, |
| 1061 warning, expectation.name if expectation.expecta
tions else expectation.original_string)) | 1079 expectation.name if expectation
.expectations else expectation.original_string)) |
| 1062 | 1080 |
| 1063 if warnings: | 1081 if warnings: |
| 1064 self._has_warnings = True | 1082 self._has_warnings = True |
| 1065 if self._is_lint_mode: | 1083 if self._is_lint_mode: |
| 1066 raise ParseError(warnings) | 1084 raise ParseError(warnings) |
| 1067 _log.warning('--lint-test-files warnings:') | 1085 _log.warning('--lint-test-files warnings:') |
| 1068 for warning in warnings: | 1086 for warning in warnings: |
| 1069 _log.warning(warning) | 1087 _log.warning(warning) |
| 1070 _log.warning('') | 1088 _log.warning('') |
| 1071 | 1089 |
| (...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1156 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. | 1174 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. |
| 1157 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. | 1175 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. |
| 1158 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: | 1176 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: |
| 1159 return expectation_line.to_string(test_configuration_converter) | 1177 return expectation_line.to_string(test_configuration_converter) |
| 1160 return expectation_line.original_string | 1178 return expectation_line.original_string |
| 1161 | 1179 |
| 1162 def nones_out(expectation_line): | 1180 def nones_out(expectation_line): |
| 1163 return expectation_line is not None | 1181 return expectation_line is not None |
| 1164 | 1182 |
| 1165 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) | 1183 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) |
| OLD | NEW |