Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(777)

Side by Side Diff: webkit/tools/layout_tests/layout_package/compare_failures.py

Issue 356007: Add a way to mark a test as missing expectations. We avoided added this... (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src/
Patch Set: Created 11 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | webkit/tools/layout_tests/layout_package/test_expectations.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright (c) 2006-2008 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """A helper class for comparing the failures and crashes from a test run 5 """A helper class for comparing the failures and crashes from a test run
6 against what we expected to happen (as specified in test_expectations.txt).""" 6 against what we expected to happen (as specified in test_expectations.txt)."""
7 7
8 import errno 8 import errno
9 import os 9 import os
10 10
(...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after
111 PrintFilesFromSet(self._regressed_hangs, 111 PrintFilesFromSet(self._regressed_hangs,
112 "Regressions: Unexpected timeouts", 112 "Regressions: Unexpected timeouts",
113 output, 113 output,
114 'TIMEOUT') 114 'TIMEOUT')
115 PrintFilesFromSet(self._regressed_crashes, 115 PrintFilesFromSet(self._regressed_crashes,
116 "Regressions: Unexpected crashes", 116 "Regressions: Unexpected crashes",
117 output, 117 output,
118 'CRASH') 118 'CRASH')
119 PrintFilesFromSet(self._missing, 119 PrintFilesFromSet(self._missing,
120 "Missing expected results", 120 "Missing expected results",
121 output) 121 output,
122 'MISSING')
122 123
123 def _CalculateRegressions(self): 124 def _CalculateRegressions(self):
124 """Calculate regressions from this run through the layout tests.""" 125 """Calculate regressions from this run through the layout tests."""
125 worklist = self._test_files.copy() 126 worklist = self._test_files.copy()
126 127
127 passes = set() 128 passes = set()
128 crashes = set() 129 crashes = set()
129 hangs = set() 130 hangs = set()
130 missing = set() 131 missing = set()
131 image_failures = set() 132 image_failures = set()
(...skipping 17 matching lines...) Expand all
149 150
150 expectations = self._expectations.GetExpectations(test) 151 expectations = self._expectations.GetExpectations(test)
151 if is_crash: 152 if is_crash:
152 if not test_expectations.CRASH in expectations: crashes.add(test) 153 if not test_expectations.CRASH in expectations: crashes.add(test)
153 elif is_hang: 154 elif is_hang:
154 if not test_expectations.TIMEOUT in expectations: hangs.add(test) 155 if not test_expectations.TIMEOUT in expectations: hangs.add(test)
155 # Do not add to the missing list if a test is rebaselining and missing 156 # Do not add to the missing list if a test is rebaselining and missing
156 # expected files. 157 # expected files.
157 elif (is_missing and 158 elif (is_missing and
158 not self._expectations.IsRebaselining(test) and 159 not self._expectations.IsRebaselining(test) and
159 not self._expectations.IsIgnored(test)): 160 not self._expectations.IsIgnored(test) and
161 (test_expectations.MISSING not in
162 self._expectations.GetExpectations(test))):
160 missing.add(test) 163 missing.add(test)
161 elif is_image_failure and is_text_failure: 164 elif is_image_failure and is_text_failure:
162 if (not test_expectations.FAIL in expectations and 165 if (not test_expectations.FAIL in expectations and
163 not test_expectations.IMAGE_PLUS_TEXT in expectations): 166 not test_expectations.IMAGE_PLUS_TEXT in expectations):
164 image_plus_text_failures.add(test) 167 image_plus_text_failures.add(test)
165 elif is_image_failure: 168 elif is_image_failure:
166 if (not test_expectations.FAIL in expectations and 169 if (not test_expectations.FAIL in expectations and
167 not test_expectations.IMAGE in expectations): 170 not test_expectations.IMAGE in expectations):
168 image_failures.add(test) 171 image_failures.add(test)
169 elif is_text_failure: 172 elif is_text_failure:
(...skipping 19 matching lines...) Expand all
189 192
190 def GetRegressions(self): 193 def GetRegressions(self):
191 """Returns a set of regressions from the test expectations. This is 194 """Returns a set of regressions from the test expectations. This is
192 used to determine which tests to list in results.html and the 195 used to determine which tests to list in results.html and the
193 right script exit code for the build bots. The list does not 196 right script exit code for the build bots. The list does not
194 include the unexpected passes.""" 197 include the unexpected passes."""
195 return (self._regressed_text_failures | self._regressed_image_failures | 198 return (self._regressed_text_failures | self._regressed_image_failures |
196 self._regressed_image_plus_text_failures | self._regressed_hangs | 199 self._regressed_image_plus_text_failures | self._regressed_hangs |
197 self._regressed_crashes | self._missing) 200 self._regressed_crashes | self._missing)
198 201
OLDNEW
« no previous file with comments | « no previous file | webkit/tools/layout_tests/layout_package/test_expectations.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698