OLD | NEW |
1 # Copyright (C) 2013 Google Inc. All rights reserved. | 1 # Copyright (C) 2013 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
128 self.assertEqual(expectations.unexpected_results_by_path(), expectations
_string) | 128 self.assertEqual(expectations.unexpected_results_by_path(), expectations
_string) |
129 | 129 |
130 def test_basic(self): | 130 def test_basic(self): |
131 test_data = { | 131 test_data = { |
132 'tests': { | 132 'tests': { |
133 'foo': { | 133 'foo': { |
134 'veryflaky.html': self._results_from_string('FFP'), | 134 'veryflaky.html': self._results_from_string('FFP'), |
135 'maybeflaky.html': self._results_from_string('FP'), | 135 'maybeflaky.html': self._results_from_string('FP'), |
136 'notflakypass.html': self._results_from_string('P'), | 136 'notflakypass.html': self._results_from_string('P'), |
137 'notflakyfail.html': self._results_from_string('F'), | 137 'notflakyfail.html': self._results_from_string('F'), |
| 138 # Even if there are no expected results, it's not very flaky
if it didn't do mulitple retries. |
| 139 # This accounts for the latest expectations not necessarily
matching the expectations |
| 140 # at the time of the given run. |
| 141 'notverflakynoexpected.html': self._results_from_string('FT'
), |
| 142 # If the test is flaky, but marked as such, it shouldn't get
printed out. |
| 143 'notflakyexpected.html': {'results': [[2, 'FFFP']], 'expecte
d': 'PASS FAIL'}, |
138 } | 144 } |
139 } | 145 } |
140 } | 146 } |
141 self._assert_expectations(test_data, { | 147 self._assert_expectations(test_data, { |
142 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), | 148 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), |
143 }, only_ignore_very_flaky=True) | 149 }, only_ignore_very_flaky=True) |
144 | 150 |
145 self._assert_expectations(test_data, { | 151 self._assert_expectations(test_data, { |
146 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), | 152 'foo/veryflaky.html': sorted(["TEXT", "PASS"]), |
| 153 'foo/notverflakynoexpected.html': ['TEXT', 'TIMEOUT'], |
147 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]), | 154 'foo/maybeflaky.html': sorted(["TEXT", "PASS"]), |
148 }, only_ignore_very_flaky=False) | 155 }, only_ignore_very_flaky=False) |
149 | 156 |
150 def test_unexpected_results_no_unexpected(self): | 157 def test_unexpected_results_no_unexpected(self): |
151 test_data = { | 158 test_data = { |
152 'tests': { | 159 'tests': { |
153 'foo': { | 160 'foo': { |
154 'pass1.html': {'results': [[4, 'P']]}, | 161 'pass1.html': {'results': [[4, 'P']]}, |
155 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL
'}, | 162 'pass2.html': {'results': [[2, 'Z']], 'expected': 'PASS FAIL
'}, |
156 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': '
PASS FAIL'}, | 163 'fail.html': {'results': [[2, 'P'], [1, 'F']], 'expected': '
PASS FAIL'}, |
(...skipping 23 matching lines...) Expand all Loading... |
180 self._assert_unexpected_results(test_data, { | 187 self._assert_unexpected_results(test_data, { |
181 'foo/pass1.html': sorted(["FAIL", "PASS"]), | 188 'foo/pass1.html': sorted(["FAIL", "PASS"]), |
182 'foo/pass2.html': sorted(["IMAGE", "PASS"]), | 189 'foo/pass2.html': sorted(["IMAGE", "PASS"]), |
183 'foo/fail.html': sorted(["TEXT", "PASS"]), | 190 'foo/fail.html': sorted(["TEXT", "PASS"]), |
184 'foo/f_p.html': sorted(["TEXT", "PASS"]), | 191 'foo/f_p.html': sorted(["TEXT", "PASS"]), |
185 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]), | 192 'foo/crash.html': sorted(["WONTFIX", "CRASH", "TEXT"]), |
186 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]), | 193 'foo/image.html': sorted(["CRASH", "FAIL", "IMAGE"]), |
187 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]), | 194 'foo/i_f.html': sorted(["PASS", "IMAGE", "TEXT"]), |
188 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR
ASH", "IMAGE", "MISSING", "LEAK"]), | 195 'foo/all.html': sorted(["TEXT", "PASS", "IMAGE+TEXT", "TIMEOUT", "CR
ASH", "IMAGE", "MISSING", "LEAK"]), |
189 }) | 196 }) |
OLD | NEW |