| OLD | NEW |
| 1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 72 skipped_result.type = test_expectations.SKIP | 72 skipped_result.type = test_expectations.SKIP |
| 73 initial_results.add(skipped_result, expected, test_is_slow) | 73 initial_results.add(skipped_result, expected, test_is_slow) |
| 74 | 74 |
| 75 initial_results.add(get_result('passes/text.html', run_time=1), expected
, test_is_slow) | 75 initial_results.add(get_result('passes/text.html', run_time=1), expected
, test_is_slow) |
| 76 initial_results.add(get_result('failures/expected/audio.html'), expected
, test_is_slow) | 76 initial_results.add(get_result('failures/expected/audio.html'), expected
, test_is_slow) |
| 77 initial_results.add(get_result('failures/expected/timeout.html'), expect
ed, test_is_slow) | 77 initial_results.add(get_result('failures/expected/timeout.html'), expect
ed, test_is_slow) |
| 78 initial_results.add(get_result('failures/expected/crash.html'), expected
, test_is_slow) | 78 initial_results.add(get_result('failures/expected/crash.html'), expected
, test_is_slow) |
| 79 initial_results.add(get_result('failures/expected/leak.html'), expected,
test_is_slow) | 79 initial_results.add(get_result('failures/expected/leak.html'), expected,
test_is_slow) |
| 80 else: | 80 else: |
| 81 initial_results.add(get_result('passes/text.html', test_expectations.TIM
EOUT, run_time=1), expected, test_is_slow) | 81 initial_results.add(get_result('passes/text.html', test_expectations.TIM
EOUT, run_time=1), expected, test_is_slow) |
| 82 initial_results.add(get_result('failures/expected/audio.html', test_expe
ctations.AUDIO, run_time=0.049), expected, test_is_slow) | 82 initial_results.add( |
| 83 initial_results.add(get_result('failures/expected/timeout.html', test_ex
pectations.CRASH, run_time=0.05), expected, test_is_slow) | 83 get_result( |
| 84 'failures/expected/audio.html', |
| 85 test_expectations.AUDIO, |
| 86 run_time=0.049), |
| 87 expected, |
| 88 test_is_slow) |
| 89 initial_results.add( |
| 90 get_result( |
| 91 'failures/expected/timeout.html', |
| 92 test_expectations.CRASH, |
| 93 run_time=0.05), |
| 94 expected, |
| 95 test_is_slow) |
| 84 initial_results.add(get_result('failures/expected/crash.html', test_expe
ctations.TIMEOUT), expected, test_is_slow) | 96 initial_results.add(get_result('failures/expected/crash.html', test_expe
ctations.TIMEOUT), expected, test_is_slow) |
| 85 initial_results.add(get_result('failures/expected/leak.html', test_expec
tations.TIMEOUT), expected, test_is_slow) | 97 initial_results.add(get_result('failures/expected/leak.html', test_expec
tations.TIMEOUT), expected, test_is_slow) |
| 86 | 98 |
| 87 # we only list keyboard.html here, since normally this is WontFix | 99 # we only list keyboard.html here, since normally this is WontFix |
| 88 initial_results.add(get_result('failures/expected/keyboard.html', test_e
xpectations.SKIP), expected, test_is_slow) | 100 initial_results.add(get_result('failures/expected/keyboard.html', test_e
xpectations.SKIP), expected, test_is_slow) |
| 89 | 101 |
| 90 if flaky: | 102 if flaky: |
| 91 retry_results = run_results(port, extra_skipped_tests) | 103 retry_results = run_results(port, extra_skipped_tests) |
| 92 retry_results.add(get_result('passes/text.html'), True, test_is_slow) | 104 retry_results.add(get_result('passes/text.html'), True, test_is_slow) |
| 93 retry_results.add(get_result('failures/expected/timeout.html'), True, te
st_is_slow) | 105 retry_results.add(get_result('failures/expected/timeout.html'), True, te
st_is_slow) |
| 94 retry_results.add(get_result('failures/expected/crash.html'), True, test
_is_slow) | 106 retry_results.add(get_result('failures/expected/crash.html'), True, test
_is_slow) |
| 95 retry_results.add(get_result('failures/expected/leak.html'), True, test_
is_slow) | 107 retry_results.add(get_result('failures/expected/leak.html'), True, test_
is_slow) |
| 96 else: | 108 else: |
| 97 retry_results = None | 109 retry_results = None |
| 98 | 110 |
| 99 return test_run_results.summarize_results(port, initial_results.expectations
, initial_results, retry_results, enabled_pixel_tests_in_retry=False, only_inclu
de_failing=only_include_failing) | 111 return test_run_results.summarize_results(port, initial_results.expectations
, initial_results, |
| 112 retry_results, enabled_pixel_tests
_in_retry=False, only_include_failing=only_include_failing) |
| 100 | 113 |
| 101 | 114 |
| 102 class InterpretTestFailuresTest(unittest.TestCase): | 115 class InterpretTestFailuresTest(unittest.TestCase): |
| 116 |
| 103 def setUp(self): | 117 def setUp(self): |
| 104 host = MockHost() | 118 host = MockHost() |
| 105 self.port = host.port_factory.get(port_name='test') | 119 self.port = host.port_factory.get(port_name='test') |
| 106 | 120 |
| 107 def test_interpret_test_failures(self): | 121 def test_interpret_test_failures(self): |
| 108 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-
mismatch.html'))]) | 122 test_dict = test_run_results._interpret_test_failures( |
| 123 [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_f
or_test('foo/reftest-expected-mismatch.html'))]) |
| 109 self.assertEqual(len(test_dict), 0) | 124 self.assertEqual(len(test_dict), 0) |
| 110 | 125 |
| 111 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingAudio()]) | 126 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingAudio()]) |
| 112 self.assertIn('is_missing_audio', test_dict) | 127 self.assertIn('is_missing_audio', test_dict) |
| 113 | 128 |
| 114 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingResult()]) | 129 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingResult()]) |
| 115 self.assertIn('is_missing_text', test_dict) | 130 self.assertIn('is_missing_text', test_dict) |
| 116 | 131 |
| 117 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingImage()]) | 132 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingImage()]) |
| 118 self.assertIn('is_missing_image', test_dict) | 133 self.assertIn('is_missing_image', test_dict) |
| 119 | 134 |
| 120 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingImageHash()]) | 135 test_dict = test_run_results._interpret_test_failures([test_failures.Fai
lureMissingImageHash()]) |
| 121 self.assertIn('is_missing_image', test_dict) | 136 self.assertIn('is_missing_image', test_dict) |
| 122 | 137 |
| 123 | 138 |
| 124 class SummarizedResultsTest(unittest.TestCase): | 139 class SummarizedResultsTest(unittest.TestCase): |
| 140 |
| 125 def setUp(self): | 141 def setUp(self): |
| 126 host = MockHost(initialize_scm_by_default=False) | 142 host = MockHost(initialize_scm_by_default=False) |
| 127 self.port = host.port_factory.get(port_name='test') | 143 self.port = host.port_factory.get(port_name='test') |
| 128 | 144 |
| 129 def test_no_svn_revision(self): | 145 def test_no_svn_revision(self): |
| 130 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 146 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
| 131 self.assertNotIn('revision', summary) | 147 self.assertNotIn('revision', summary) |
| 132 | 148 |
| 133 def test_num_failures_by_type(self): | 149 def test_num_failures_by_type(self): |
| 134 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 150 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
| 135 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 3, 'IMAGE+TEXT': 0,
'LEAK': 0, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 1}) | 151 self.assertEquals(summary['num_failures_by_type'], |
| 152 {'CRASH': 1, |
| 153 'MISSING': 0, |
| 154 'TEXT': 0, |
| 155 'IMAGE': 0, |
| 156 'NEEDSREBASELINE': 0, |
| 157 'NEEDSMANUALREBASELINE': 0, |
| 158 'PASS': 0, |
| 159 'REBASELINE': 0, |
| 160 'SKIP': 0, |
| 161 'SLOW': 0, |
| 162 'TIMEOUT': 3, |
| 163 'IMAGE+TEXT': 0, |
| 164 'LEAK': 0, |
| 165 'FAIL': 0, |
| 166 'AUDIO': 1, |
| 167 'WONTFIX': 1}) |
| 136 | 168 |
| 137 summary = summarized_results(self.port, expected=True, passing=False, fl
aky=False) | 169 summary = summarized_results(self.port, expected=True, passing=False, fl
aky=False) |
| 138 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0,
'LEAK': 1, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0}) | 170 self.assertEquals(summary['num_failures_by_type'], |
| 171 {'CRASH': 1, |
| 172 'MISSING': 0, |
| 173 'TEXT': 0, |
| 174 'IMAGE': 0, |
| 175 'NEEDSREBASELINE': 0, |
| 176 'NEEDSMANUALREBASELINE': 0, |
| 177 'PASS': 1, |
| 178 'REBASELINE': 0, |
| 179 'SKIP': 0, |
| 180 'SLOW': 0, |
| 181 'TIMEOUT': 1, |
| 182 'IMAGE+TEXT': 0, |
| 183 'LEAK': 1, |
| 184 'FAIL': 0, |
| 185 'AUDIO': 1, |
| 186 'WONTFIX': 0}) |
| 139 | 187 |
| 140 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) | 188 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) |
| 141 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 5, 'REBASELINE': 0, 'SKIP': 1, 'SLOW': 0, 'TIMEOUT': 0, 'IMAGE+TEXT': 0,
'LEAK': 0, 'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0}) | 189 self.assertEquals(summary['num_failures_by_type'], |
| 190 {'CRASH': 0, |
| 191 'MISSING': 0, |
| 192 'TEXT': 0, |
| 193 'IMAGE': 0, |
| 194 'NEEDSREBASELINE': 0, |
| 195 'NEEDSMANUALREBASELINE': 0, |
| 196 'PASS': 5, |
| 197 'REBASELINE': 0, |
| 198 'SKIP': 1, |
| 199 'SLOW': 0, |
| 200 'TIMEOUT': 0, |
| 201 'IMAGE+TEXT': 0, |
| 202 'LEAK': 0, |
| 203 'FAIL': 0, |
| 204 'AUDIO': 0, |
| 205 'WONTFIX': 0}) |
| 142 | 206 |
| 143 def test_svn_revision(self): | 207 def test_svn_revision(self): |
| 144 self.port._options.builder_name = 'dummy builder' | 208 self.port._options.builder_name = 'dummy builder' |
| 145 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 209 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
| 146 self.assertNotEquals(summary['blink_revision'], '') | 210 self.assertNotEquals(summary['blink_revision'], '') |
| 147 | 211 |
| 148 def test_bug_entry(self): | 212 def test_bug_entry(self): |
| 149 self.port._options.builder_name = 'dummy builder' | 213 self.port._options.builder_name = 'dummy builder' |
| 150 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) | 214 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) |
| 151 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bu
gs'], ['Bug(test)']) | 215 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bu
gs'], ['Bug(test)']) |
| (...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 194 self.assertTrue('time' not in summary['tests']['failures']['expected']['
crash.html']) | 258 self.assertTrue('time' not in summary['tests']['failures']['expected']['
crash.html']) |
| 195 self.assertTrue('time' not in summary['tests']['failures']['expected']['
leak.html']) | 259 self.assertTrue('time' not in summary['tests']['failures']['expected']['
leak.html']) |
| 196 | 260 |
| 197 def test_timeout_then_unexpected_pass(self): | 261 def test_timeout_then_unexpected_pass(self): |
| 198 tests = ['failures/expected/image.html'] | 262 tests = ['failures/expected/image.html'] |
| 199 expectations = test_expectations.TestExpectations(self.port, tests) | 263 expectations = test_expectations.TestExpectations(self.port, tests) |
| 200 initial_results = test_run_results.TestRunResults(expectations, len(test
s)) | 264 initial_results = test_run_results.TestRunResults(expectations, len(test
s)) |
| 201 initial_results.add(get_result('failures/expected/image.html', test_expe
ctations.TIMEOUT, run_time=1), False, False) | 265 initial_results.add(get_result('failures/expected/image.html', test_expe
ctations.TIMEOUT, run_time=1), False, False) |
| 202 retry_results = test_run_results.TestRunResults(expectations, len(tests)
) | 266 retry_results = test_run_results.TestRunResults(expectations, len(tests)
) |
| 203 retry_results.add(get_result('failures/expected/image.html', test_expect
ations.PASS, run_time=0.1), False, False) | 267 retry_results.add(get_result('failures/expected/image.html', test_expect
ations.PASS, run_time=0.1), False, False) |
| 204 summary = test_run_results.summarize_results(self.port, expectations, in
itial_results, retry_results, enabled_pixel_tests_in_retry=True, only_include_fa
iling=True) | 268 summary = test_run_results.summarize_results( |
| 269 self.port, |
| 270 expectations, |
| 271 initial_results, |
| 272 retry_results, |
| 273 enabled_pixel_tests_in_retry=True, |
| 274 only_include_failing=True) |
| 205 self.assertEquals(summary['num_regressions'], 0) | 275 self.assertEquals(summary['num_regressions'], 0) |
| 206 self.assertEquals(summary['num_passes'], 1) | 276 self.assertEquals(summary['num_passes'], 1) |
| OLD | NEW |