| OLD | NEW |
| 1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 96 if all_tests: | 96 if all_tests: |
| 97 self._all_tests = set(all_tests) | 97 self._all_tests = set(all_tests) |
| 98 else: | 98 else: |
| 99 self._all_tests = set() | 99 self._all_tests = set() |
| 100 | 100 |
| 101 self._is_lint_mode = is_lint_mode | 101 self._is_lint_mode = is_lint_mode |
| 102 | 102 |
| 103 def parse(self, filename, expectations_string): | 103 def parse(self, filename, expectations_string): |
| 104 expectation_lines = [] | 104 expectation_lines = [] |
| 105 line_number = 0 | 105 line_number = 0 |
| 106 for line in expectations_string.split("\n"): | 106 for line in expectations_string.split('\n'): |
| 107 line_number += 1 | 107 line_number += 1 |
| 108 test_expectation = TestExpectationLine.tokenize_line(filename, line,
line_number) | 108 test_expectation = TestExpectationLine.tokenize_line(filename, line,
line_number) |
| 109 self._parse_line(test_expectation) | 109 self._parse_line(test_expectation) |
| 110 expectation_lines.append(test_expectation) | 110 expectation_lines.append(test_expectation) |
| 111 return expectation_lines | 111 return expectation_lines |
| 112 | 112 |
| 113 def _create_expectation_line(self, test_name, expectations, file_name): | 113 def _create_expectation_line(self, test_name, expectations, file_name): |
| 114 expectation_line = TestExpectationLine() | 114 expectation_line = TestExpectationLine() |
| 115 expectation_line.original_string = test_name | 115 expectation_line.original_string = test_name |
| 116 expectation_line.name = test_name | 116 expectation_line.name = test_name |
| (...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 222 expectation_line.matching_tests.append(expectation_line.path) | 222 expectation_line.matching_tests.append(expectation_line.path) |
| 223 | 223 |
| 224 | 224 |
| 225 class TestExpectationLine(object): | 225 class TestExpectationLine(object): |
| 226 """Represents a line in test expectations file.""" | 226 """Represents a line in test expectations file.""" |
| 227 | 227 |
| 228 def __init__(self): | 228 def __init__(self): |
| 229 """Initializes a blank-line equivalent of an expectation.""" | 229 """Initializes a blank-line equivalent of an expectation.""" |
| 230 self.original_string = None | 230 self.original_string = None |
| 231 self.filename = None # this is the path to the expectations file for th
is line | 231 self.filename = None # this is the path to the expectations file for th
is line |
| 232 self.line_numbers = "0" | 232 self.line_numbers = '0' |
| 233 self.name = None # this is the path in the line itself | 233 self.name = None # this is the path in the line itself |
| 234 self.path = None # this is the normpath of self.name | 234 self.path = None # this is the normpath of self.name |
| 235 self.bugs = [] | 235 self.bugs = [] |
| 236 self.specifiers = [] | 236 self.specifiers = [] |
| 237 self.parsed_specifiers = [] | 237 self.parsed_specifiers = [] |
| 238 self.matching_configurations = set() | 238 self.matching_configurations = set() |
| 239 self.expectations = [] | 239 self.expectations = [] |
| 240 self.parsed_expectations = set() | 240 self.parsed_expectations = set() |
| 241 self.comment = None | 241 self.comment = None |
| 242 self.matching_tests = [] | 242 self.matching_tests = [] |
| 243 self.warnings = [] | 243 self.warnings = [] |
| 244 self.is_extra_skipped_test = False | 244 self.is_extra_skipped_test = False |
| 245 | 245 |
| 246 def __str__(self): | 246 def __str__(self): |
| 247 return "TestExpectationLine{name=%s, matching_configurations=%s, origina
l_string=%s}" % ( | 247 return 'TestExpectationLine{name=%s, matching_configurations=%s, origina
l_string=%s}' % ( |
| 248 self.name, self.matching_configurations, self.original_string) | 248 self.name, self.matching_configurations, self.original_string) |
| 249 | 249 |
| 250 def __eq__(self, other): | 250 def __eq__(self, other): |
| 251 return (self.original_string == other.original_string | 251 return (self.original_string == other.original_string |
| 252 and self.filename == other.filename | 252 and self.filename == other.filename |
| 253 and self.line_numbers == other.line_numbers | 253 and self.line_numbers == other.line_numbers |
| 254 and self.name == other.name | 254 and self.name == other.name |
| 255 and self.path == other.path | 255 and self.path == other.path |
| 256 and self.bugs == other.bugs | 256 and self.bugs == other.bugs |
| 257 and self.specifiers == other.specifiers | 257 and self.specifiers == other.specifiers |
| (...skipping 11 matching lines...) Expand all Loading... |
| 269 | 269 |
| 270 def is_flaky(self): | 270 def is_flaky(self): |
| 271 return len(self.parsed_expectations) > 1 | 271 return len(self.parsed_expectations) > 1 |
| 272 | 272 |
| 273 def is_comment(self): | 273 def is_comment(self): |
| 274 return bool(re.match(r"^\s*#.*$", self.original_string)) | 274 return bool(re.match(r"^\s*#.*$", self.original_string)) |
| 275 | 275 |
| 276 def is_whitespace(self): | 276 def is_whitespace(self): |
| 277 return not self.original_string.strip() | 277 return not self.original_string.strip() |
| 278 | 278 |
| 279 | |
| 280 # FIXME: Update the original specifiers and remove this once the old syntax
is gone. | 279 # FIXME: Update the original specifiers and remove this once the old syntax
is gone. |
| 281 _configuration_tokens_list = [ | 280 _configuration_tokens_list = [ |
| 282 'Mac', 'Mac10.9', 'Mac10.10', 'Mac10.11', 'Retina', | 281 'Mac', 'Mac10.9', 'Mac10.10', 'Mac10.11', 'Retina', |
| 283 'Win', 'Win7', 'Win10', | 282 'Win', 'Win7', 'Win10', |
| 284 'Linux', 'Precise', 'Trusty', | 283 'Linux', 'Precise', 'Trusty', |
| 285 'Android', | 284 'Android', |
| 286 'Release', | 285 'Release', |
| 287 'Debug', | 286 'Debug', |
| 288 ] | 287 ] |
| 289 | 288 |
| (...skipping 28 matching lines...) Expand all Loading... |
| 318 | 317 |
| 319 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations>
"]" ["#" <comment>] | 318 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations>
"]" ["#" <comment>] |
| 320 | 319 |
| 321 Any errant whitespace is not preserved. | 320 Any errant whitespace is not preserved. |
| 322 """ | 321 """ |
| 323 expectation_line = TestExpectationLine() | 322 expectation_line = TestExpectationLine() |
| 324 expectation_line.original_string = expectation_string | 323 expectation_line.original_string = expectation_string |
| 325 expectation_line.filename = filename | 324 expectation_line.filename = filename |
| 326 expectation_line.line_numbers = str(line_number) | 325 expectation_line.line_numbers = str(line_number) |
| 327 | 326 |
| 328 comment_index = expectation_string.find("#") | 327 comment_index = expectation_string.find('#') |
| 329 if comment_index == -1: | 328 if comment_index == -1: |
| 330 comment_index = len(expectation_string) | 329 comment_index = len(expectation_string) |
| 331 else: | 330 else: |
| 332 expectation_line.comment = expectation_string[comment_index + 1:] | 331 expectation_line.comment = expectation_string[comment_index + 1:] |
| 333 | 332 |
| 334 remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index
].strip()) | 333 remaining_string = re.sub(r"\s+", ' ', expectation_string[:comment_index
].strip()) |
| 335 if len(remaining_string) == 0: | 334 if len(remaining_string) == 0: |
| 336 return expectation_line | 335 return expectation_line |
| 337 | 336 |
| 338 # special-case parsing this so that we fail immediately instead of treat
ing this as a test name | 337 # special-case parsing this so that we fail immediately instead of treat
ing this as a test name |
| 339 if remaining_string.startswith('//'): | 338 if remaining_string.startswith('//'): |
| 340 expectation_line.warnings = ['use "#" instead of "//" for comments'] | 339 expectation_line.warnings = ['use "#" instead of "//" for comments'] |
| 341 return expectation_line | 340 return expectation_line |
| 342 | 341 |
| 343 bugs = [] | 342 bugs = [] |
| 344 specifiers = [] | 343 specifiers = [] |
| (...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 413 elif state not in ('name_found', 'done'): | 412 elif state not in ('name_found', 'done'): |
| 414 warnings.append('Missing a "]"') | 413 warnings.append('Missing a "]"') |
| 415 | 414 |
| 416 if 'WONTFIX' in expectations and 'SKIP' not in expectations: | 415 if 'WONTFIX' in expectations and 'SKIP' not in expectations: |
| 417 expectations.append('SKIP') | 416 expectations.append('SKIP') |
| 418 | 417 |
| 419 if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(exp
ectations) - set(['SKIP', 'WONTFIX'])): | 418 if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(exp
ectations) - set(['SKIP', 'WONTFIX'])): |
| 420 warnings.append('A test marked Skip or WontFix must not have other e
xpectations.') | 419 warnings.append('A test marked Skip or WontFix must not have other e
xpectations.') |
| 421 | 420 |
| 422 if 'SLOW' in expectations and 'SlowTests' not in filename: | 421 if 'SLOW' in expectations and 'SlowTests' not in filename: |
| 423 warnings.append('SLOW tests should ony be added to SlowTests and not
to TestExpectations.') | 422 warnings.append('SLOW tests should only be added to SlowTests and no
t to TestExpectations.') |
| 424 | 423 |
| 425 if 'WONTFIX' in expectations and ('NeverFixTests' not in filename and 'S
taleTestExpectations' not in filename): | 424 if 'WONTFIX' in expectations and ('NeverFixTests' not in filename and 'S
taleTestExpectations' not in filename): |
| 426 warnings.append( | 425 warnings.append( |
| 427 'WONTFIX tests should ony be added to NeverFixTests or StaleTest
Expectations and not to TestExpectations.') | 426 'WONTFIX tests should only be added to NeverFixTests or StaleTes
tExpectations and not to TestExpectations.') |
| 428 | 427 |
| 429 if 'NeverFixTests' in filename and expectations != ['WONTFIX', 'SKIP']: | 428 if 'NeverFixTests' in filename and expectations != ['WONTFIX', 'SKIP']: |
| 430 warnings.append('Only WONTFIX expectations are allowed in NeverFixTe
sts') | 429 warnings.append('Only WONTFIX expectations are allowed in NeverFixTe
sts') |
| 431 | 430 |
| 432 if 'SlowTests' in filename and expectations != ['SLOW']: | 431 if 'SlowTests' in filename and expectations != ['SLOW']: |
| 433 warnings.append('Only SLOW expectations are allowed in SlowTests') | 432 warnings.append('Only SLOW expectations are allowed in SlowTests') |
| 434 | 433 |
| 435 if not expectations and not has_unrecognized_expectation: | 434 if not expectations and not has_unrecognized_expectation: |
| 436 warnings.append('Missing expectations.') | 435 warnings.append('Missing expectations.') |
| 437 | 436 |
| 438 if 'MISSING' in expectations: | 437 if 'MISSING' in expectations: |
| 439 warnings.append( | 438 warnings.append( |
| 440 '"Missing" expectations are not allowed; either download new bas
elines ' | 439 '"Missing" expectations are not allowed; either download new bas
elines ' |
| 441 '(see https://goo.gl/SHVYrZ) or use "NeedsRebaseline" expecatati
ons.') | 440 '(see https://goo.gl/SHVYrZ) or use "NeedsRebaseline" expectatio
ns.') |
| 442 | 441 |
| 443 expectation_line.bugs = bugs | 442 expectation_line.bugs = bugs |
| 444 expectation_line.specifiers = specifiers | 443 expectation_line.specifiers = specifiers |
| 445 expectation_line.expectations = expectations | 444 expectation_line.expectations = expectations |
| 446 expectation_line.name = name | 445 expectation_line.name = name |
| 447 expectation_line.warnings = warnings | 446 expectation_line.warnings = warnings |
| 448 return expectation_line | 447 return expectation_line |
| 449 | 448 |
| 450 @staticmethod | 449 @staticmethod |
| 451 def create_passing_expectation(test): | 450 def create_passing_expectation(test): |
| (...skipping 14 matching lines...) Expand all Loading... |
| 466 return line1 | 465 return line1 |
| 467 if model_all_expectations and line1.filename != line2.filename: | 466 if model_all_expectations and line1.filename != line2.filename: |
| 468 return line2 | 467 return line2 |
| 469 | 468 |
| 470 # Don't merge original_string or comment. | 469 # Don't merge original_string or comment. |
| 471 result = TestExpectationLine() | 470 result = TestExpectationLine() |
| 472 # We only care about filenames when we're linting, in which case the fil
enames are the same. | 471 # We only care about filenames when we're linting, in which case the fil
enames are the same. |
| 473 # Not clear that there's anything better to do when not linting and the
filenames are different. | 472 # Not clear that there's anything better to do when not linting and the
filenames are different. |
| 474 if model_all_expectations: | 473 if model_all_expectations: |
| 475 result.filename = line2.filename | 474 result.filename = line2.filename |
| 476 result.line_numbers = line1.line_numbers + "," + line2.line_numbers | 475 result.line_numbers = line1.line_numbers + ',' + line2.line_numbers |
| 477 result.name = line1.name | 476 result.name = line1.name |
| 478 result.path = line1.path | 477 result.path = line1.path |
| 479 result.parsed_expectations = set(line1.parsed_expectations) | set(line2.
parsed_expectations) | 478 result.parsed_expectations = set(line1.parsed_expectations) | set(line2.
parsed_expectations) |
| 480 result.expectations = list(set(line1.expectations) | set(line2.expectati
ons)) | 479 result.expectations = list(set(line1.expectations) | set(line2.expectati
ons)) |
| 481 result.bugs = list(set(line1.bugs) | set(line2.bugs)) | 480 result.bugs = list(set(line1.bugs) | set(line2.bugs)) |
| 482 result.specifiers = list(set(line1.specifiers) | set(line2.specifiers)) | 481 result.specifiers = list(set(line1.specifiers) | set(line2.specifiers)) |
| 483 result.parsed_specifiers = list(set(line1.parsed_specifiers) | set(line2
.parsed_specifiers)) | 482 result.parsed_specifiers = list(set(line1.parsed_specifiers) | set(line2
.parsed_specifiers)) |
| 484 result.matching_configurations = set(line1.matching_configurations) | se
t(line2.matching_configurations) | 483 result.matching_configurations = set(line1.matching_configurations) | se
t(line2.matching_configurations) |
| 485 result.matching_tests = list(list(set(line1.matching_tests) | set(line2.
matching_tests))) | 484 result.matching_tests = list(list(set(line1.matching_tests) | set(line2.
matching_tests))) |
| 486 result.warnings = list(set(line1.warnings) | set(line2.warnings)) | 485 result.warnings = list(set(line1.warnings) | set(line2.warnings)) |
| 487 result.is_extra_skipped_test = line1.is_extra_skipped_test or line2.is_e
xtra_skipped_test | 486 result.is_extra_skipped_test = line1.is_extra_skipped_test or line2.is_e
xtra_skipped_test |
| 488 return result | 487 return result |
| 489 | 488 |
| 490 def to_string(self, test_configuration_converter=None, include_specifiers=Tr
ue, | 489 def to_string(self, test_configuration_converter=None, include_specifiers=Tr
ue, |
| 491 include_expectations=True, include_comment=True): | 490 include_expectations=True, include_comment=True): |
| 492 parsed_expectation_to_string = dict( | 491 parsed_expectation_to_string = dict( |
| 493 [[parsed_expectation, expectation_string] | 492 [[parsed_expectation, expectation_string] |
| 494 for expectation_string, parsed_expectation in TestExpectations.EXPE
CTATIONS.items()]) | 493 for expectation_string, parsed_expectation in TestExpectations.EXPE
CTATIONS.items()]) |
| 495 | 494 |
| 496 if self.is_invalid(): | 495 if self.is_invalid(): |
| 497 return self.original_string or '' | 496 return self.original_string or '' |
| 498 | 497 |
| 499 if self.name is None: | 498 if self.name is None: |
| 500 return '' if self.comment is None else "#%s" % self.comment | 499 return '' if self.comment is None else '#%s' % self.comment |
| 501 | 500 |
| 502 if test_configuration_converter and self.bugs: | 501 if test_configuration_converter and self.bugs: |
| 503 specifiers_list = test_configuration_converter.to_specifiers_list(se
lf.matching_configurations) | 502 specifiers_list = test_configuration_converter.to_specifiers_list(se
lf.matching_configurations) |
| 504 result = [] | 503 result = [] |
| 505 for specifiers in specifiers_list: | 504 for specifiers in specifiers_list: |
| 506 # FIXME: this is silly that we join the specifiers and then imme
diately split them. | 505 # FIXME: this is silly that we join the specifiers and then imme
diately split them. |
| 507 specifiers = self._serialize_parsed_specifiers(test_configuratio
n_converter, specifiers).split() | 506 specifiers = self._serialize_parsed_specifiers(test_configuratio
n_converter, specifiers).split() |
| 508 expectations = self._serialize_parsed_expectations(parsed_expect
ation_to_string).split() | 507 expectations = self._serialize_parsed_expectations(parsed_expect
ation_to_string).split() |
| 509 result.append(self._format_line(self.bugs, specifiers, self.name
, expectations, self.comment)) | 508 result.append(self._format_line(self.bugs, specifiers, self.name
, expectations, self.comment)) |
| 510 return "\n".join(result) if result else None | 509 return '\n'.join(result) if result else None |
| 511 | 510 |
| 512 return self._format_line(self.bugs, self.specifiers, self.name, self.exp
ectations, self.comment, | 511 return self._format_line(self.bugs, self.specifiers, self.name, self.exp
ectations, self.comment, |
| 513 include_specifiers, include_expectations, inclu
de_comment) | 512 include_specifiers, include_expectations, inclu
de_comment) |
| 514 | 513 |
| 515 def to_csv(self): | 514 def to_csv(self): |
| 516 # Note that this doesn't include the comments. | 515 # Note that this doesn't include the comments. |
| 517 return '%s,%s,%s,%s' % (self.name, ' '.join(self.bugs), ' '.join(self.sp
ecifiers), ' '.join(self.expectations)) | 516 return '%s,%s,%s,%s' % (self.name, ' '.join(self.bugs), ' '.join(self.sp
ecifiers), ' '.join(self.expectations)) |
| 518 | 517 |
| 519 def _serialize_parsed_expectations(self, parsed_expectation_to_string): | 518 def _serialize_parsed_expectations(self, parsed_expectation_to_string): |
| 520 result = [] | 519 result = [] |
| (...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 555 if include_specifiers and (bugs or new_specifiers): | 554 if include_specifiers and (bugs or new_specifiers): |
| 556 if bugs: | 555 if bugs: |
| 557 result += ' '.join(bugs) + ' ' | 556 result += ' '.join(bugs) + ' ' |
| 558 if new_specifiers: | 557 if new_specifiers: |
| 559 result += '[ %s ] ' % ' '.join(new_specifiers) | 558 result += '[ %s ] ' % ' '.join(new_specifiers) |
| 560 result += name | 559 result += name |
| 561 if include_expectations and new_expectations: | 560 if include_expectations and new_expectations: |
| 562 new_expectations = TestExpectationLine._filter_redundant_expectation
s(new_expectations) | 561 new_expectations = TestExpectationLine._filter_redundant_expectation
s(new_expectations) |
| 563 result += ' [ %s ]' % ' '.join(sorted(set(new_expectations))) | 562 result += ' [ %s ]' % ' '.join(sorted(set(new_expectations))) |
| 564 if include_comment and comment is not None: | 563 if include_comment and comment is not None: |
| 565 result += " #%s" % comment | 564 result += ' #%s' % comment |
| 566 return result | 565 return result |
| 567 | 566 |
| 568 | 567 |
| 569 # FIXME: Refactor API to be a proper CRUD. | 568 # FIXME: Refactor API to be a proper CRUD. |
| 570 class TestExpectationsModel(object): | 569 class TestExpectationsModel(object): |
| 571 """Represents relational store of all expectations and provides CRUD semanti
cs to manage it.""" | 570 """Represents relational store of all expectations and provides CRUD semanti
cs to manage it.""" |
| 572 | 571 |
| 573 def __init__(self, shorten_filename=None): | 572 def __init__(self, shorten_filename=None): |
| 574 # Maps a test to its list of expectations. | 573 # Maps a test to its list of expectations. |
| 575 self._test_to_expectations = {} | 574 self._test_to_expectations = {} |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 619 else: | 618 else: |
| 620 merged_line = other_line | 619 merged_line = other_line |
| 621 | 620 |
| 622 self._test_to_expectation_line[test] = merged_line | 621 self._test_to_expectation_line[test] = merged_line |
| 623 | 622 |
| 624 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_
to_tests) | 623 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_
to_tests) |
| 625 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes
ts) | 624 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tes
ts) |
| 626 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_
to_tests) | 625 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_
to_tests) |
| 627 | 626 |
| 628 def _dict_of_sets(self, strings_to_constants): | 627 def _dict_of_sets(self, strings_to_constants): |
| 629 """Takes a dict of strings->constants and returns a dict mapping | 628 """Takes a dictionary of keys to values and returns a dict mapping each
value to an empty set.""" |
| 630 each constant to an empty set. | 629 result = {} |
| 631 """ | 630 for value in strings_to_constants.values(): |
| 632 d = {} | 631 result[value] = set() |
| 633 for c in strings_to_constants.values(): | 632 return result |
| 634 d[c] = set() | |
| 635 return d | |
| 636 | 633 |
| 637 def get_test_set(self, expectation, include_skips=True): | 634 def get_test_set(self, expectation, include_skips=True): |
| 638 tests = self._expectation_to_tests[expectation] | 635 tests = self._expectation_to_tests[expectation] |
| 639 if not include_skips: | 636 if not include_skips: |
| 640 tests = tests - self.get_test_set(SKIP) | 637 tests = tests - self.get_test_set(SKIP) |
| 641 return tests | 638 return tests |
| 642 | 639 |
| 643 def get_test_set_for_keyword(self, keyword): | 640 def get_test_set_for_keyword(self, keyword): |
| 644 expectation_enum = TestExpectations.EXPECTATIONS.get(keyword.lower(), No
ne) | 641 expectation_enum = TestExpectations.EXPECTATIONS.get(keyword.lower(), No
ne) |
| 645 if expectation_enum is not None: | 642 if expectation_enum is not None: |
| (...skipping 30 matching lines...) Expand all Loading... |
| 676 expectations = self.get_expectations(test) | 673 expectations = self.get_expectations(test) |
| 677 retval = [] | 674 retval = [] |
| 678 | 675 |
| 679 # FIXME: WontFix should cause the test to get skipped without artificial
ly adding SKIP to the expectations list. | 676 # FIXME: WontFix should cause the test to get skipped without artificial
ly adding SKIP to the expectations list. |
| 680 if WONTFIX in expectations and SKIP in expectations: | 677 if WONTFIX in expectations and SKIP in expectations: |
| 681 expectations.remove(SKIP) | 678 expectations.remove(SKIP) |
| 682 | 679 |
| 683 for expectation in expectations: | 680 for expectation in expectations: |
| 684 retval.append(self.expectation_to_string(expectation)) | 681 retval.append(self.expectation_to_string(expectation)) |
| 685 | 682 |
| 686 return " ".join(retval) | 683 return ' '.join(retval) |
| 687 | 684 |
| 688 def expectation_to_string(self, expectation): | 685 def expectation_to_string(self, expectation): |
| 689 """Return the uppercased string equivalent of a given expectation.""" | 686 """Return the uppercased string equivalent of a given expectation.""" |
| 690 for item in TestExpectations.EXPECTATIONS.items(): | 687 for item in TestExpectations.EXPECTATIONS.items(): |
| 691 if item[1] == expectation: | 688 if item[1] == expectation: |
| 692 return item[0].upper() | 689 return item[0].upper() |
| 693 raise ValueError(expectation) | 690 raise ValueError(expectation) |
| 694 | 691 |
| 695 def remove_expectation_line(self, test): | 692 def remove_expectation_line(self, test): |
| 696 if not self.has_test(test): | 693 if not self.has_test(test): |
| (...skipping 163 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 860 WontFix: For tests that we never intend to pass on a given platform (treated
like Skip). | 857 WontFix: For tests that we never intend to pass on a given platform (treated
like Skip). |
| 861 | 858 |
| 862 Notes: | 859 Notes: |
| 863 -A test cannot be both SLOW and TIMEOUT | 860 -A test cannot be both SLOW and TIMEOUT |
| 864 -A test can be included twice, but not via the same path. | 861 -A test can be included twice, but not via the same path. |
| 865 -If a test is included twice, then the more precise path wins. | 862 -If a test is included twice, then the more precise path wins. |
| 866 -CRASH tests cannot be WONTFIX | 863 -CRASH tests cannot be WONTFIX |
| 867 """ | 864 """ |
| 868 | 865 |
| 869 # FIXME: Update to new syntax once the old format is no longer supported. | 866 # FIXME: Update to new syntax once the old format is no longer supported. |
| 870 EXPECTATIONS = {'pass': PASS, | 867 EXPECTATIONS = { |
| 871 'audio': AUDIO, | 868 'pass': PASS, |
| 872 'fail': FAIL, | 869 'audio': AUDIO, |
| 873 'image': IMAGE, | 870 'fail': FAIL, |
| 874 'image+text': IMAGE_PLUS_TEXT, | 871 'image': IMAGE, |
| 875 'text': TEXT, | 872 'image+text': IMAGE_PLUS_TEXT, |
| 876 'timeout': TIMEOUT, | 873 'text': TEXT, |
| 877 'crash': CRASH, | 874 'timeout': TIMEOUT, |
| 878 'leak': LEAK, | 875 'crash': CRASH, |
| 879 'missing': MISSING, | 876 'leak': LEAK, |
| 880 TestExpectationParser.SKIP_MODIFIER: SKIP, | 877 'missing': MISSING, |
| 881 TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBAS
ELINE, | 878 TestExpectationParser.SKIP_MODIFIER: SKIP, |
| 882 TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEED
S_MANUAL_REBASELINE, | 879 TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBASELINE, |
| 883 TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, | 880 TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEEDS_MANUAL_REB
ASELINE, |
| 884 TestExpectationParser.SLOW_MODIFIER: SLOW, | 881 TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, |
| 885 TestExpectationParser.REBASELINE_MODIFIER: REBASELINE, | 882 TestExpectationParser.SLOW_MODIFIER: SLOW, |
| 886 } | 883 TestExpectationParser.REBASELINE_MODIFIER: REBASELINE, |
| 884 } |
| 887 | 885 |
| 888 EXPECTATIONS_TO_STRING = dict((k, v) for (v, k) in EXPECTATIONS.iteritems()) | 886 EXPECTATIONS_TO_STRING = dict((k, v) for (v, k) in EXPECTATIONS.iteritems()) |
| 889 | 887 |
| 890 # (aggregated by category, pass/fail/skip, type) | 888 # (aggregated by category, pass/fail/skip, type) |
| 891 EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped', | 889 EXPECTATION_DESCRIPTIONS = { |
| 892 PASS: 'passes', | 890 SKIP: 'skipped', |
| 893 FAIL: 'failures', | 891 PASS: 'passes', |
| 894 IMAGE: 'image-only failures', | 892 FAIL: 'failures', |
| 895 TEXT: 'text-only failures', | 893 IMAGE: 'image-only failures', |
| 896 IMAGE_PLUS_TEXT: 'image and text failures', | 894 TEXT: 'text-only failures', |
| 897 AUDIO: 'audio failures', | 895 IMAGE_PLUS_TEXT: 'image and text failures', |
| 898 CRASH: 'crashes', | 896 AUDIO: 'audio failures', |
| 899 LEAK: 'leaks', | 897 CRASH: 'crashes', |
| 900 TIMEOUT: 'timeouts', | 898 LEAK: 'leaks', |
| 901 MISSING: 'missing results'} | 899 TIMEOUT: 'timeouts', |
| 900 MISSING: 'missing results', |
| 901 } |
| 902 | 902 |
| 903 NON_TEST_OUTCOME_EXPECTATIONS = (REBASELINE, SKIP, SLOW, WONTFIX) | 903 NON_TEST_OUTCOME_EXPECTATIONS = (REBASELINE, SKIP, SLOW, WONTFIX) |
| 904 | 904 |
| 905 BUILD_TYPES = ('debug', 'release') | 905 BUILD_TYPES = ('debug', 'release') |
| 906 | 906 |
| 907 TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, | 907 TIMELINES = { |
| 908 'now': NOW} | 908 TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, |
| 909 'now': NOW, |
| 910 } |
| 909 | 911 |
| 910 RESULT_TYPES = {'skip': SKIP, | 912 RESULT_TYPES = { |
| 911 'pass': PASS, | 913 'skip': SKIP, |
| 912 'fail': FAIL, | 914 'pass': PASS, |
| 913 'flaky': FLAKY} | 915 'fail': FAIL, |
| 916 'flaky': FLAKY, |
| 917 } |
| 914 | 918 |
| 915 @classmethod | 919 @classmethod |
| 916 def expectation_from_string(cls, string): | 920 def expectation_from_string(cls, string): |
| 917 assert ' ' not in string # This only handles one expectation at a time. | 921 assert ' ' not in string # This only handles one expectation at a time. |
| 918 return cls.EXPECTATIONS.get(string.lower()) | 922 return cls.EXPECTATIONS.get(string.lower()) |
| 919 | 923 |
| 920 @staticmethod | 924 @staticmethod |
| 921 def result_was_expected(result, expected_results, test_needs_rebaselining): | 925 def result_was_expected(result, expected_results, test_needs_rebaselining): |
| 922 """Returns whether we got a result we were expecting. | 926 """Returns whether we got a result we were expecting. |
| 923 Args: | 927 Args: |
| (...skipping 277 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1201 def serialize(expectation_line): | 1205 def serialize(expectation_line): |
| 1202 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. | 1206 # If reconstitute_only_these is an empty list, we want to return ori
ginal_string. |
| 1203 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. | 1207 # So we need to compare reconstitute_only_these to None, not just ch
eck if it's falsey. |
| 1204 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: | 1208 if reconstitute_only_these is None or expectation_line in reconstitu
te_only_these: |
| 1205 return expectation_line.to_string(test_configuration_converter) | 1209 return expectation_line.to_string(test_configuration_converter) |
| 1206 return expectation_line.original_string | 1210 return expectation_line.original_string |
| 1207 | 1211 |
| 1208 def nones_out(expectation_line): | 1212 def nones_out(expectation_line): |
| 1209 return expectation_line is not None | 1213 return expectation_line is not None |
| 1210 | 1214 |
| 1211 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) | 1215 return '\n'.join(filter(nones_out, map(serialize, expectation_lines))) |
| OLD | NEW |