Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(321)

Side by Side Diff: third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/testharness_results.py

Issue 2668973002: Consider any testharness result with >= 1 PASS and no FAIL to be passing. (Closed)
Patch Set: Remove two lines from TestExpectations Created 3 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Utility module for checking testharness test output.""" 5 """Utility module for checking testharness test output."""
6 6
7 _TESTHARNESSREPORT_HEADER = 'This is a testharness.js-based test.' 7 _TESTHARNESSREPORT_HEADER = 'This is a testharness.js-based test.'
8 _TESTHARNESSREPORT_FOOTER = 'Harness: the test ran to completion.' 8 _TESTHARNESSREPORT_FOOTER = 'Harness: the test ran to completion.'
9 9
10 10
(...skipping 16 matching lines...) Expand all
27 for line in lines: 27 for line in lines:
28 if line == _TESTHARNESSREPORT_HEADER: 28 if line == _TESTHARNESSREPORT_HEADER:
29 found_header = True 29 found_header = True
30 elif line == _TESTHARNESSREPORT_FOOTER: 30 elif line == _TESTHARNESSREPORT_FOOTER:
31 found_footer = True 31 found_footer = True
32 32
33 return found_header and found_footer 33 return found_header and found_footer
34 34
35 35
36 def is_testharness_output_passing(content_text): 36 def is_testharness_output_passing(content_text):
37 """Returns whether |content_text| is a passing testharness output.""" 37 """Checks whether |content_text| is a passing testharness output.
38 38
39 # Leading and trailing white spaces are accepted. 39 Under a relatively loose/accepting definition of passing
40 testharness output, we consider any output with at least one
41 PASS result and no FAIL result (or TIMEOUT or NOTRUN).
42 """
43 # Leading and trailing whitespace are ignored.
40 lines = content_text.strip().splitlines() 44 lines = content_text.strip().splitlines()
41 lines = [line.strip() for line in lines] 45 lines = [line.strip() for line in lines]
42 46
43 # The check is very conservative and rejects any unexpected content in the o utput. 47 at_least_one_pass = False
44 previous_line_is_console_line = False 48
45 for line in lines: 49 for line in lines:
46 # There should be no empty lines, unless the empty line follows a consol e message. 50 if line.startswith('PASS'):
47 if len(line) == 0: 51 at_least_one_pass = True
48 if previous_line_is_console_line:
49 continue
50 else:
51 return False
52
53 # Those lines are expected to be exactly equivalent.
54 if line == _TESTHARNESSREPORT_HEADER or line == _TESTHARNESSREPORT_FOOTE R:
55 previous_line_is_console_line = False
56 continue 52 continue
57
58 # Those are expected passing output.
59 if line.startswith('CONSOLE'):
60 previous_line_is_console_line = True
61 continue
62
63 if line.startswith('PASS'):
64 previous_line_is_console_line = False
65 continue
66
67 # Those are expected failing output.
68 if (line.startswith('FAIL') or 53 if (line.startswith('FAIL') or
69 line.startswith('TIMEOUT') or 54 line.startswith('TIMEOUT') or
70 line.startswith('NOTRUN') or 55 line.startswith('NOTRUN') or
71 line.startswith('Harness Error. harness_status = ')): 56 line.startswith('Harness Error. harness_status = ')):
72 return False 57 return False
73 58
74 # Unexpected output should be considered as a failure. 59 return at_least_one_pass
75 return False
76
77 return True
78 60
79 61
80 def has_console_errors_or_warnings(content_text): 62 def has_console_errors_or_warnings(content_text):
81 """Returns whether |content_text| is has console errors or warnings.""" 63 """Returns whether |content_text| is has console errors or warnings."""
82 64
83 def is_warning_or_error(line): 65 def is_warning_or_error(line):
84 return line.startswith('CONSOLE ERROR:') or line.startswith('CONSOLE WAR NING:') 66 return line.startswith('CONSOLE ERROR:') or line.startswith('CONSOLE WAR NING:')
85 67
86 lines = content_text.strip().splitlines() 68 lines = content_text.strip().splitlines()
87 return any(is_warning_or_error(line) for line in lines) 69 return any(is_warning_or_error(line) for line in lines)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698