OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
72 skipped_result.type = test_expectations.SKIP | 72 skipped_result.type = test_expectations.SKIP |
73 initial_results.add(skipped_result, expected, test_is_slow) | 73 initial_results.add(skipped_result, expected, test_is_slow) |
74 | 74 |
75 initial_results.add(get_result('passes/text.html', run_time=1), expected
, test_is_slow) | 75 initial_results.add(get_result('passes/text.html', run_time=1), expected
, test_is_slow) |
76 initial_results.add(get_result('failures/expected/audio.html'), expected
, test_is_slow) | 76 initial_results.add(get_result('failures/expected/audio.html'), expected
, test_is_slow) |
77 initial_results.add(get_result('failures/expected/timeout.html'), expect
ed, test_is_slow) | 77 initial_results.add(get_result('failures/expected/timeout.html'), expect
ed, test_is_slow) |
78 initial_results.add(get_result('failures/expected/crash.html'), expected
, test_is_slow) | 78 initial_results.add(get_result('failures/expected/crash.html'), expected
, test_is_slow) |
79 initial_results.add(get_result('failures/expected/leak.html'), expected,
test_is_slow) | 79 initial_results.add(get_result('failures/expected/leak.html'), expected,
test_is_slow) |
80 else: | 80 else: |
81 initial_results.add(get_result('passes/text.html', test_expectations.TIM
EOUT, run_time=1), expected, test_is_slow) | 81 initial_results.add(get_result('passes/text.html', test_expectations.TIM
EOUT, run_time=1), expected, test_is_slow) |
82 initial_results.add(get_result('failures/expected/audio.html', test_expe
ctations.AUDIO, run_time=0.049), expected, test_is_slow) | 82 initial_results.add( |
83 initial_results.add(get_result('failures/expected/timeout.html', test_ex
pectations.CRASH, run_time=0.05), expected, test_is_slow) | 83 get_result( |
| 84 'failures/expected/audio.html', |
| 85 test_expectations.AUDIO, |
| 86 run_time=0.049), |
| 87 expected, |
| 88 test_is_slow) |
| 89 initial_results.add( |
| 90 get_result( |
| 91 'failures/expected/timeout.html', |
| 92 test_expectations.CRASH, |
| 93 run_time=0.05), |
| 94 expected, |
| 95 test_is_slow) |
84 initial_results.add(get_result('failures/expected/crash.html', test_expe
ctations.TIMEOUT), expected, test_is_slow) | 96 initial_results.add(get_result('failures/expected/crash.html', test_expe
ctations.TIMEOUT), expected, test_is_slow) |
85 initial_results.add(get_result('failures/expected/leak.html', test_expec
tations.TIMEOUT), expected, test_is_slow) | 97 initial_results.add(get_result('failures/expected/leak.html', test_expec
tations.TIMEOUT), expected, test_is_slow) |
86 | 98 |
87 # we only list keyboard.html here, since normally this is WontFix | 99 # we only list keyboard.html here, since normally this is WontFix |
88 initial_results.add(get_result('failures/expected/keyboard.html', test_e
xpectations.SKIP), expected, test_is_slow) | 100 initial_results.add(get_result('failures/expected/keyboard.html', test_e
xpectations.SKIP), expected, test_is_slow) |
89 | 101 |
90 if flaky: | 102 if flaky: |
91 retry_results = run_results(port, extra_skipped_tests) | 103 retry_results = run_results(port, extra_skipped_tests) |
92 retry_results.add(get_result('passes/text.html'), True, test_is_slow) | 104 retry_results.add(get_result('passes/text.html'), True, test_is_slow) |
93 retry_results.add(get_result('failures/expected/timeout.html'), True, te
st_is_slow) | 105 retry_results.add(get_result('failures/expected/timeout.html'), True, te
st_is_slow) |
94 retry_results.add(get_result('failures/expected/crash.html'), True, test
_is_slow) | 106 retry_results.add(get_result('failures/expected/crash.html'), True, test
_is_slow) |
95 retry_results.add(get_result('failures/expected/leak.html'), True, test_
is_slow) | 107 retry_results.add(get_result('failures/expected/leak.html'), True, test_
is_slow) |
96 else: | 108 else: |
97 retry_results = None | 109 retry_results = None |
98 | 110 |
99 return test_run_results.summarize_results(port, initial_results.expectations
, initial_results, retry_results, enabled_pixel_tests_in_retry=False, only_inclu
de_failing=only_include_failing) | 111 return test_run_results.summarize_results(port, initial_results.expectations
, initial_results, |
| 112 retry_results, enabled_pixel_tests
_in_retry=False, only_include_failing=only_include_failing) |
100 | 113 |
101 | 114 |
102 class InterpretTestFailuresTest(unittest.TestCase): | 115 class InterpretTestFailuresTest(unittest.TestCase): |
| 116 |
103 def setUp(self): | 117 def setUp(self): |
104 host = MockHost() | 118 host = MockHost() |
105 self.port = host.port_factory.get(port_name='test') | 119 self.port = host.port_factory.get(port_name='test') |
106 | 120 |
107 def test_interpret_test_failures(self): | 121 def test_interpret_test_failures(self): |
108 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-
mismatch.html'))]) | 122 test_dict = test_run_results._interpret_test_failures( |
| 123 [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_f
or_test('foo/reftest-expected-mismatch.html'))]) |
109 self.assertEqual(len(test_dict), 0) | 124 self.assertEqual(len(test_dict), 0) |
110 | 125 |
111 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingAudio()]) | 126 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingAudio()]) |
112 self.assertIn('is_missing_audio', test_dict) | 127 self.assertIn('is_missing_audio', test_dict) |
113 | 128 |
114 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingResult()]) | 129 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingResult()]) |
115 self.assertIn('is_missing_text', test_dict) | 130 self.assertIn('is_missing_text', test_dict) |
116 | 131 |
117 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingImage()]) | 132 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingImage()]) |
118 self.assertIn('is_missing_image', test_dict) | 133 self.assertIn('is_missing_image', test_dict) |
119 | 134 |
120 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingImageHash()]) | 135 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingImageHash()]) |
121 self.assertIn('is_missing_image', test_dict) | 136 self.assertIn('is_missing_image', test_dict) |
122 | 137 |
123 | 138 |
124 class SummarizedResultsTest(unittest.TestCase): | 139 class SummarizedResultsTest(unittest.TestCase): |
| 140 |
125 def setUp(self): | 141 def setUp(self): |
126 host = MockHost(initialize_scm_by_default=False) | 142 host = MockHost(initialize_scm_by_default=False) |
127 self.port = host.port_factory.get(port_name='test') | 143 self.port = host.port_factory.get(port_name='test') |
128 | 144 |
129 def test_no_svn_revision(self): | 145 def test_no_svn_revision(self): |
130 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 146 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
131 self.assertNotIn('revision', summary) | 147 self.assertNotIn('revision', summary) |
132 | 148 |
133 def test_num_failures_by_type(self): | 149 def test_num_failures_by_type(self): |
134 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 150 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
135 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 3, 'IMAGE+TEXT': 0,
'LEAK': 0, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 1}) | 151 self.assertEquals(summary['num_failures_by_type'], |
| 152 {'CRASH': 1, |
| 153 'MISSING': 0, |
| 154 'TEXT': 0, |
| 155 'IMAGE': 0, |
| 156 'NEEDSREBASELINE': 0, |
| 157 'NEEDSMANUALREBASELINE': 0, |
| 158 'PASS': 0, |
| 159 'REBASELINE': 0, |
| 160 'SKIP': 0, |
| 161 'SLOW': 0, |
| 162 'TIMEOUT': 3, |
| 163 'IMAGE+TEXT': 0, |
| 164 'LEAK': 0, |
| 165 'FAIL': 0, |
| 166 'AUDIO': 1, |
| 167 'WONTFIX': 1}) |
136 | 168 |
137 summary = summarized_results(self.port, expected=True, passing=False, fl
aky=False) | 169 summary = summarized_results(self.port, expected=True, passing=False, fl
aky=False) |
138 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0,
'LEAK': 1, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0}) | 170 self.assertEquals(summary['num_failures_by_type'], |
| 171 {'CRASH': 1, |
| 172 'MISSING': 0, |
| 173 'TEXT': 0, |
| 174 'IMAGE': 0, |
| 175 'NEEDSREBASELINE': 0, |
| 176 'NEEDSMANUALREBASELINE': 0, |
| 177 'PASS': 1, |
| 178 'REBASELINE': 0, |
| 179 'SKIP': 0, |
| 180 'SLOW': 0, |
| 181 'TIMEOUT': 1, |
| 182 'IMAGE+TEXT': 0, |
| 183 'LEAK': 1, |
| 184 'FAIL': 0, |
| 185 'AUDIO': 1, |
| 186 'WONTFIX': 0}) |
139 | 187 |
140 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) | 188 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) |
141 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 5, 'REBASELINE': 0, 'SKIP': 1, 'SLOW': 0, 'TIMEOUT': 0, 'IMAGE+TEXT': 0,
'LEAK': 0, 'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0}) | 189 self.assertEquals(summary['num_failures_by_type'], |
| 190 {'CRASH': 0, |
| 191 'MISSING': 0, |
| 192 'TEXT': 0, |
| 193 'IMAGE': 0, |
| 194 'NEEDSREBASELINE': 0, |
| 195 'NEEDSMANUALREBASELINE': 0, |
| 196 'PASS': 5, |
| 197 'REBASELINE': 0, |
| 198 'SKIP': 1, |
| 199 'SLOW': 0, |
| 200 'TIMEOUT': 0, |
| 201 'IMAGE+TEXT': 0, |
| 202 'LEAK': 0, |
| 203 'FAIL': 0, |
| 204 'AUDIO': 0, |
| 205 'WONTFIX': 0}) |
142 | 206 |
143 def test_svn_revision(self): | 207 def test_svn_revision(self): |
144 self.port._options.builder_name = 'dummy builder' | 208 self.port._options.builder_name = 'dummy builder' |
145 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 209 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
146 self.assertNotEquals(summary['blink_revision'], '') | 210 self.assertNotEquals(summary['blink_revision'], '') |
147 | 211 |
148 def test_bug_entry(self): | 212 def test_bug_entry(self): |
149 self.port._options.builder_name = 'dummy builder' | 213 self.port._options.builder_name = 'dummy builder' |
150 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) | 214 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) |
151 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bu
gs'], ['Bug(test)']) | 215 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bu
gs'], ['Bug(test)']) |
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
194 self.assertTrue('time' not in summary['tests']['failures']['expected']['
crash.html']) | 258 self.assertTrue('time' not in summary['tests']['failures']['expected']['
crash.html']) |
195 self.assertTrue('time' not in summary['tests']['failures']['expected']['
leak.html']) | 259 self.assertTrue('time' not in summary['tests']['failures']['expected']['
leak.html']) |
196 | 260 |
197 def test_timeout_then_unexpected_pass(self): | 261 def test_timeout_then_unexpected_pass(self): |
198 tests = ['failures/expected/image.html'] | 262 tests = ['failures/expected/image.html'] |
199 expectations = test_expectations.TestExpectations(self.port, tests) | 263 expectations = test_expectations.TestExpectations(self.port, tests) |
200 initial_results = test_run_results.TestRunResults(expectations, len(test
s)) | 264 initial_results = test_run_results.TestRunResults(expectations, len(test
s)) |
201 initial_results.add(get_result('failures/expected/image.html', test_expe
ctations.TIMEOUT, run_time=1), False, False) | 265 initial_results.add(get_result('failures/expected/image.html', test_expe
ctations.TIMEOUT, run_time=1), False, False) |
202 retry_results = test_run_results.TestRunResults(expectations, len(tests)
) | 266 retry_results = test_run_results.TestRunResults(expectations, len(tests)
) |
203 retry_results.add(get_result('failures/expected/image.html', test_expect
ations.PASS, run_time=0.1), False, False) | 267 retry_results.add(get_result('failures/expected/image.html', test_expect
ations.PASS, run_time=0.1), False, False) |
204 summary = test_run_results.summarize_results(self.port, expectations, in
itial_results, retry_results, enabled_pixel_tests_in_retry=True, only_include_fa
iling=True) | 268 summary = test_run_results.summarize_results( |
| 269 self.port, |
| 270 expectations, |
| 271 initial_results, |
| 272 retry_results, |
| 273 enabled_pixel_tests_in_retry=True, |
| 274 only_include_failing=True) |
205 self.assertEquals(summary['num_regressions'], 0) | 275 self.assertEquals(summary['num_regressions'], 0) |
206 self.assertEquals(summary['num_passes'], 1) | 276 self.assertEquals(summary['num_passes'], 1) |
OLD | NEW |