Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(189)

Side by Side Diff: Tools/Scripts/webkitpy/layout_tests/layout_package/bot_test_expectations_unittest.py

Issue 1289163002: Improve flakiness logic for print-flaky-tests. (Closed) Base URL: svn://svn.chromium.org/blink/trunk
Patch Set: Created 5 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright (C) 2013 Google Inc. All rights reserved. 1 # Copyright (C) 2013 Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after
128 self.assertEqual(expectations.unexpected_results_by_path(), expectations _string) 128 self.assertEqual(expectations.unexpected_results_by_path(), expectations _string)
129 129
130 def test_basic(self): 130 def test_basic(self):
131 test_data = { 131 test_data = {
132 'tests': { 132 'tests': {
133 'foo': { 133 'foo': {
134 'veryflaky.html': self._results_from_string('FFP'), 134 'veryflaky.html': self._results_from_string('FFP'),
135 'maybeflaky.html': self._results_from_string('FP'), 135 'maybeflaky.html': self._results_from_string('FP'),
136 'notflakypass.html': self._results_from_string('P'), 136 'notflakypass.html': self._results_from_string('P'),
137 'notflakyfail.html': self._results_from_string('F'), 137 'notflakyfail.html': self._results_from_string('F'),
138 # Even if there are no expected results, it's not very flaky if it didn't do mulitple retries.
139 # This accounts for the latest expectations not necessarily matching the expectations
140 # at the time of the given run.
141 'notverflakynoexpected.html': self._results_from_string('FT' ),
142 # If the test is flaky, but marked as such, it shouldn't get printed out.
143 'notflakyexpected.html': {'results': [[2, 'FFFP']], 'expecte d': 'PASS FAIL'},
138 } 144 }
139 } 145 }
140 } 146 }
141 self._assert_expectations(test_data, { 147 self._assert_expectations(test_data, {
142 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), 148 'foo/veryflaky.html': sorted(["TEXT", "PASS"]),
143 }, only_ignore_very_flaky=True) 149 }, only_ignore_very_flaky=True)
144 150
145 self._assert_expectations(test_data, { 151 self._assert_expectations(test_data, {
146 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), 152 'foo/veryflaky.html': sorted(["TEXT", "PASS"]),
153 'foo/notverflakynoexpected.html': ['TEXT', 'TIMEOUT'],
147 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]), 154 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]),
148 }, only_ignore_very_flaky=False) 155 }, only_ignore_very_flaky=False)
149 156
150 def test_unexpected_results_no_unexpected(self): 157 def test_unexpected_results_no_unexpected(self):
151 test_data = { 158 test_data = {
152 'tests': { 159 'tests': {
153 'foo': { 160 'foo': {
154 'pass1.html': {'results': [[4, 'P']]}, 161 'pass1.html': {'results': [[4, 'P']]},
155 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL '}, 162 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL '},
156 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': ' PASS FAIL'}, 163 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': ' PASS FAIL'},
(...skipping 23 matching lines...) Expand all
180 self._assert_unexpected_results(test_data, { 187 self._assert_unexpected_results(test_data, {
181 'foo/pass1.html': sorted(["FAIL", "PASS"]), 188 'foo/pass1.html': sorted(["FAIL", "PASS"]),
182 'foo/pass2.html': sorted(["IMAGE", "PASS"]), 189 'foo/pass2.html': sorted(["IMAGE", "PASS"]),
183 'foo/fail.html': sorted(["TEXT", "PASS"]), 190 'foo/fail.html': sorted(["TEXT", "PASS"]),
184 'foo/f_p.html': sorted(["TEXT", "PASS"]), 191 'foo/f_p.html': sorted(["TEXT", "PASS"]),
185 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]), 192 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]),
186 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]), 193 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]),
187 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]), 194 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]),
188 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR ASH", "IMAGE", "MISSING", "LEAK"]), 195 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR ASH", "IMAGE", "MISSING", "LEAK"]),
189 }) 196 })
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698