Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(121)

Side by Side Diff: Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py

Issue 546133003: Reformat webkitpy.layout_tests w/ format-webkitpy. (Closed) Base URL: svn://svn.chromium.org/blink/trunk
Patch Set: Created 6 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright (C) 2010 Google Inc. All rights reserved. 1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after
72 skipped_result.type = test_expectations.SKIP 72 skipped_result.type = test_expectations.SKIP
73 initial_results.add(skipped_result, expected, test_is_slow) 73 initial_results.add(skipped_result, expected, test_is_slow)
74 74
75 initial_results.add(get_result('passes/text.html', run_time=1), expected , test_is_slow) 75 initial_results.add(get_result('passes/text.html', run_time=1), expected , test_is_slow)
76 initial_results.add(get_result('failures/expected/audio.html'), expected , test_is_slow) 76 initial_results.add(get_result('failures/expected/audio.html'), expected , test_is_slow)
77 initial_results.add(get_result('failures/expected/timeout.html'), expect ed, test_is_slow) 77 initial_results.add(get_result('failures/expected/timeout.html'), expect ed, test_is_slow)
78 initial_results.add(get_result('failures/expected/crash.html'), expected , test_is_slow) 78 initial_results.add(get_result('failures/expected/crash.html'), expected , test_is_slow)
79 initial_results.add(get_result('failures/expected/leak.html'), expected, test_is_slow) 79 initial_results.add(get_result('failures/expected/leak.html'), expected, test_is_slow)
80 else: 80 else:
81 initial_results.add(get_result('passes/text.html', test_expectations.TIM EOUT, run_time=1), expected, test_is_slow) 81 initial_results.add(get_result('passes/text.html', test_expectations.TIM EOUT, run_time=1), expected, test_is_slow)
82 initial_results.add(get_result('failures/expected/audio.html', test_expe ctations.AUDIO, run_time=0.049), expected, test_is_slow) 82 initial_results.add(
83 initial_results.add(get_result('failures/expected/timeout.html', test_ex pectations.CRASH, run_time=0.05), expected, test_is_slow) 83 get_result(
84 'failures/expected/audio.html',
85 test_expectations.AUDIO,
86 run_time=0.049),
87 expected,
88 test_is_slow)
89 initial_results.add(
90 get_result(
91 'failures/expected/timeout.html',
92 test_expectations.CRASH,
93 run_time=0.05),
94 expected,
95 test_is_slow)
84 initial_results.add(get_result('failures/expected/crash.html', test_expe ctations.TIMEOUT), expected, test_is_slow) 96 initial_results.add(get_result('failures/expected/crash.html', test_expe ctations.TIMEOUT), expected, test_is_slow)
85 initial_results.add(get_result('failures/expected/leak.html', test_expec tations.TIMEOUT), expected, test_is_slow) 97 initial_results.add(get_result('failures/expected/leak.html', test_expec tations.TIMEOUT), expected, test_is_slow)
86 98
87 # we only list keyboard.html here, since normally this is WontFix 99 # we only list keyboard.html here, since normally this is WontFix
88 initial_results.add(get_result('failures/expected/keyboard.html', test_e xpectations.SKIP), expected, test_is_slow) 100 initial_results.add(get_result('failures/expected/keyboard.html', test_e xpectations.SKIP), expected, test_is_slow)
89 101
90 if flaky: 102 if flaky:
91 retry_results = run_results(port, extra_skipped_tests) 103 retry_results = run_results(port, extra_skipped_tests)
92 retry_results.add(get_result('passes/text.html'), True, test_is_slow) 104 retry_results.add(get_result('passes/text.html'), True, test_is_slow)
93 retry_results.add(get_result('failures/expected/timeout.html'), True, te st_is_slow) 105 retry_results.add(get_result('failures/expected/timeout.html'), True, te st_is_slow)
94 retry_results.add(get_result('failures/expected/crash.html'), True, test _is_slow) 106 retry_results.add(get_result('failures/expected/crash.html'), True, test _is_slow)
95 retry_results.add(get_result('failures/expected/leak.html'), True, test_ is_slow) 107 retry_results.add(get_result('failures/expected/leak.html'), True, test_ is_slow)
96 else: 108 else:
97 retry_results = None 109 retry_results = None
98 110
99 return test_run_results.summarize_results(port, initial_results.expectations , initial_results, retry_results, enabled_pixel_tests_in_retry=False, only_inclu de_failing=only_include_failing) 111 return test_run_results.summarize_results(port, initial_results.expectations , initial_results,
112 retry_results, enabled_pixel_tests _in_retry=False, only_include_failing=only_include_failing)
100 113
101 114
102 class InterpretTestFailuresTest(unittest.TestCase): 115 class InterpretTestFailuresTest(unittest.TestCase):
116
103 def setUp(self): 117 def setUp(self):
104 host = MockHost() 118 host = MockHost()
105 self.port = host.port_factory.get(port_name='test') 119 self.port = host.port_factory.get(port_name='test')
106 120
107 def test_interpret_test_failures(self): 121 def test_interpret_test_failures(self):
108 test_dict = test_run_results._interpret_test_failures([test_failures.Fai lureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected- mismatch.html'))]) 122 test_dict = test_run_results._interpret_test_failures(
123 [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_f or_test('foo/reftest-expected-mismatch.html'))])
109 self.assertEqual(len(test_dict), 0) 124 self.assertEqual(len(test_dict), 0)
110 125
111 test_dict = test_run_results._interpret_test_failures([test_failures.Fai lureMissingAudio()]) 126 test_dict = test_run_results._interpret_test_failures([test_failures.Fai lureMissingAudio()])
112 self.assertIn('is_missing_audio', test_dict) 127 self.assertIn('is_missing_audio', test_dict)
113 128
114 test_dict = test_run_results._interpret_test_failures([test_failures.Fai lureMissingResult()]) 129 test_dict = test_run_results._interpret_test_failures([test_failures.Fai lureMissingResult()])
115 self.assertIn('is_missing_text', test_dict) 130 self.assertIn('is_missing_text', test_dict)
116 131
117 test_dict = test_run_results._interpret_test_failures([test_failures.Fai lureMissingImage()]) 132 test_dict = test_run_results._interpret_test_failures([test_failures.Fai lureMissingImage()])
118 self.assertIn('is_missing_image', test_dict) 133 self.assertIn('is_missing_image', test_dict)
119 134
120 test_dict = test_run_results._interpret_test_failures([test_failures.Fai lureMissingImageHash()]) 135 test_dict = test_run_results._interpret_test_failures([test_failures.Fai lureMissingImageHash()])
121 self.assertIn('is_missing_image', test_dict) 136 self.assertIn('is_missing_image', test_dict)
122 137
123 138
124 class SummarizedResultsTest(unittest.TestCase): 139 class SummarizedResultsTest(unittest.TestCase):
140
125 def setUp(self): 141 def setUp(self):
126 host = MockHost(initialize_scm_by_default=False) 142 host = MockHost(initialize_scm_by_default=False)
127 self.port = host.port_factory.get(port_name='test') 143 self.port = host.port_factory.get(port_name='test')
128 144
129 def test_no_svn_revision(self): 145 def test_no_svn_revision(self):
130 summary = summarized_results(self.port, expected=False, passing=False, f laky=False) 146 summary = summarized_results(self.port, expected=False, passing=False, f laky=False)
131 self.assertNotIn('revision', summary) 147 self.assertNotIn('revision', summary)
132 148
133 def test_num_failures_by_type(self): 149 def test_num_failures_by_type(self):
134 summary = summarized_results(self.port, expected=False, passing=False, f laky=False) 150 summary = summarized_results(self.port, expected=False, passing=False, f laky=False)
135 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING ': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, ' PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 3, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 1}) 151 self.assertEquals(summary['num_failures_by_type'],
152 {'CRASH': 1,
153 'MISSING': 0,
154 'TEXT': 0,
155 'IMAGE': 0,
156 'NEEDSREBASELINE': 0,
157 'NEEDSMANUALREBASELINE': 0,
158 'PASS': 0,
159 'REBASELINE': 0,
160 'SKIP': 0,
161 'SLOW': 0,
162 'TIMEOUT': 3,
163 'IMAGE+TEXT': 0,
164 'LEAK': 0,
165 'FAIL': 0,
166 'AUDIO': 1,
167 'WONTFIX': 1})
136 168
137 summary = summarized_results(self.port, expected=True, passing=False, fl aky=False) 169 summary = summarized_results(self.port, expected=True, passing=False, fl aky=False)
138 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING ': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, ' PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0, 'LEAK': 1, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0}) 170 self.assertEquals(summary['num_failures_by_type'],
171 {'CRASH': 1,
172 'MISSING': 0,
173 'TEXT': 0,
174 'IMAGE': 0,
175 'NEEDSREBASELINE': 0,
176 'NEEDSMANUALREBASELINE': 0,
177 'PASS': 1,
178 'REBASELINE': 0,
179 'SKIP': 0,
180 'SLOW': 0,
181 'TIMEOUT': 1,
182 'IMAGE+TEXT': 0,
183 'LEAK': 1,
184 'FAIL': 0,
185 'AUDIO': 1,
186 'WONTFIX': 0})
139 187
140 summary = summarized_results(self.port, expected=False, passing=True, fl aky=False) 188 summary = summarized_results(self.port, expected=False, passing=True, fl aky=False)
141 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING ': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, ' PASS': 5, 'REBASELINE': 0, 'SKIP': 1, 'SLOW': 0, 'TIMEOUT': 0, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0}) 189 self.assertEquals(summary['num_failures_by_type'],
190 {'CRASH': 0,
191 'MISSING': 0,
192 'TEXT': 0,
193 'IMAGE': 0,
194 'NEEDSREBASELINE': 0,
195 'NEEDSMANUALREBASELINE': 0,
196 'PASS': 5,
197 'REBASELINE': 0,
198 'SKIP': 1,
199 'SLOW': 0,
200 'TIMEOUT': 0,
201 'IMAGE+TEXT': 0,
202 'LEAK': 0,
203 'FAIL': 0,
204 'AUDIO': 0,
205 'WONTFIX': 0})
142 206
143 def test_svn_revision(self): 207 def test_svn_revision(self):
144 self.port._options.builder_name = 'dummy builder' 208 self.port._options.builder_name = 'dummy builder'
145 summary = summarized_results(self.port, expected=False, passing=False, f laky=False) 209 summary = summarized_results(self.port, expected=False, passing=False, f laky=False)
146 self.assertNotEquals(summary['blink_revision'], '') 210 self.assertNotEquals(summary['blink_revision'], '')
147 211
148 def test_bug_entry(self): 212 def test_bug_entry(self):
149 self.port._options.builder_name = 'dummy builder' 213 self.port._options.builder_name = 'dummy builder'
150 summary = summarized_results(self.port, expected=False, passing=True, fl aky=False) 214 summary = summarized_results(self.port, expected=False, passing=True, fl aky=False)
151 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bu gs'], ['Bug(test)']) 215 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bu gs'], ['Bug(test)'])
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after
194 self.assertTrue('time' not in summary['tests']['failures']['expected'][' crash.html']) 258 self.assertTrue('time' not in summary['tests']['failures']['expected'][' crash.html'])
195 self.assertTrue('time' not in summary['tests']['failures']['expected'][' leak.html']) 259 self.assertTrue('time' not in summary['tests']['failures']['expected'][' leak.html'])
196 260
197 def test_timeout_then_unexpected_pass(self): 261 def test_timeout_then_unexpected_pass(self):
198 tests = ['failures/expected/image.html'] 262 tests = ['failures/expected/image.html']
199 expectations = test_expectations.TestExpectations(self.port, tests) 263 expectations = test_expectations.TestExpectations(self.port, tests)
200 initial_results = test_run_results.TestRunResults(expectations, len(test s)) 264 initial_results = test_run_results.TestRunResults(expectations, len(test s))
201 initial_results.add(get_result('failures/expected/image.html', test_expe ctations.TIMEOUT, run_time=1), False, False) 265 initial_results.add(get_result('failures/expected/image.html', test_expe ctations.TIMEOUT, run_time=1), False, False)
202 retry_results = test_run_results.TestRunResults(expectations, len(tests) ) 266 retry_results = test_run_results.TestRunResults(expectations, len(tests) )
203 retry_results.add(get_result('failures/expected/image.html', test_expect ations.PASS, run_time=0.1), False, False) 267 retry_results.add(get_result('failures/expected/image.html', test_expect ations.PASS, run_time=0.1), False, False)
204 summary = test_run_results.summarize_results(self.port, expectations, in itial_results, retry_results, enabled_pixel_tests_in_retry=True, only_include_fa iling=True) 268 summary = test_run_results.summarize_results(
269 self.port,
270 expectations,
271 initial_results,
272 retry_results,
273 enabled_pixel_tests_in_retry=True,
274 only_include_failing=True)
205 self.assertEquals(summary['num_regressions'], 0) 275 self.assertEquals(summary['num_regressions'], 0)
206 self.assertEquals(summary['num_passes'], 1) 276 self.assertEquals(summary['num_passes'], 1)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698