| OLD | NEW |
| 1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 73 initial_results.add(get_result('failures/expected/audio.html'), expected
, test_is_slow) | 73 initial_results.add(get_result('failures/expected/audio.html'), expected
, test_is_slow) |
| 74 initial_results.add(get_result('failures/expected/timeout.html'), expect
ed, test_is_slow) | 74 initial_results.add(get_result('failures/expected/timeout.html'), expect
ed, test_is_slow) |
| 75 initial_results.add(get_result('failures/expected/crash.html'), expected
, test_is_slow) | 75 initial_results.add(get_result('failures/expected/crash.html'), expected
, test_is_slow) |
| 76 else: | 76 else: |
| 77 initial_results.add(get_result('passes/text.html', test_expectations.TIM
EOUT, run_time=1), expected, test_is_slow) | 77 initial_results.add(get_result('passes/text.html', test_expectations.TIM
EOUT, run_time=1), expected, test_is_slow) |
| 78 initial_results.add(get_result('failures/expected/audio.html', test_expe
ctations.AUDIO, run_time=0.049), expected, test_is_slow) | 78 initial_results.add(get_result('failures/expected/audio.html', test_expe
ctations.AUDIO, run_time=0.049), expected, test_is_slow) |
| 79 initial_results.add(get_result('failures/expected/timeout.html', test_ex
pectations.CRASH, run_time=0.05), expected, test_is_slow) | 79 initial_results.add(get_result('failures/expected/timeout.html', test_ex
pectations.CRASH, run_time=0.05), expected, test_is_slow) |
| 80 initial_results.add(get_result('failures/expected/crash.html', test_expe
ctations.TIMEOUT), expected, test_is_slow) | 80 initial_results.add(get_result('failures/expected/crash.html', test_expe
ctations.TIMEOUT), expected, test_is_slow) |
| 81 | 81 |
| 82 # we only list hang.html here, since normally this is WontFix | 82 # we only list hang.html here, since normally this is WontFix |
| 83 initial_results.add(get_result('failures/expected/hang.html', test_expec
tations.TIMEOUT), expected, test_is_slow) | 83 initial_results.add(get_result('failures/expected/hang.html', test_expec
tations.SKIP), expected, test_is_slow) |
| 84 | 84 |
| 85 if flaky: | 85 if flaky: |
| 86 retry_results = run_results(port, extra_skipped_tests) | 86 retry_results = run_results(port, extra_skipped_tests) |
| 87 retry_results.add(get_result('passes/text.html'), True, test_is_slow) | 87 retry_results.add(get_result('passes/text.html'), True, test_is_slow) |
| 88 retry_results.add(get_result('failures/expected/timeout.html'), True, te
st_is_slow) | 88 retry_results.add(get_result('failures/expected/timeout.html'), True, te
st_is_slow) |
| 89 retry_results.add(get_result('failures/expected/crash.html'), True, test
_is_slow) | 89 retry_results.add(get_result('failures/expected/crash.html'), True, test
_is_slow) |
| 90 else: | 90 else: |
| 91 retry_results = None | 91 retry_results = None |
| 92 | 92 |
| 93 return test_run_results.summarize_results(port, initial_results.expectations
, initial_results, retry_results, enabled_pixel_tests_in_retry=False, only_inclu
de_failing=only_include_failing) | 93 return test_run_results.summarize_results(port, initial_results.expectations
, initial_results, retry_results, enabled_pixel_tests_in_retry=False, only_inclu
de_failing=only_include_failing) |
| (...skipping 25 matching lines...) Expand all Loading... |
| 119 def setUp(self): | 119 def setUp(self): |
| 120 host = MockHost(initialize_scm_by_default=False) | 120 host = MockHost(initialize_scm_by_default=False) |
| 121 self.port = host.port_factory.get(port_name='test') | 121 self.port = host.port_factory.get(port_name='test') |
| 122 | 122 |
| 123 def test_no_svn_revision(self): | 123 def test_no_svn_revision(self): |
| 124 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 124 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
| 125 self.assertNotIn('revision', summary) | 125 self.assertNotIn('revision', summary) |
| 126 | 126 |
| 127 def test_num_failures_by_type(self): | 127 def test_num_failures_by_type(self): |
| 128 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 128 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
| 129 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 0, 'SKIP': 0, 'TIMEOUT': 2, 'IMAGE+TEXT': 0, 'FAIL': 0, 'AUDIO': 1}) | 129 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 2, 'IMAGE+TEXT': 0,
'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0}) |
| 130 | 130 |
| 131 summary = summarized_results(self.port, expected=True, passing=False, fl
aky=False) | 131 summary = summarized_results(self.port, expected=True, passing=False, fl
aky=False) |
| 132 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 1, 'SKIP': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0, 'FAIL': 0, 'AUDIO': 1}) | 132 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0,
'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0}) |
| 133 | 133 |
| 134 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) | 134 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) |
| 135 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 4, 'SKIP': 1, 'TIMEOUT': 0, 'IMAGE+TEXT': 0, 'FAIL': 0, 'AUDIO': 0}) | 135 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 4, 'REBASELINE': 0, 'SKIP': 1, 'SLOW': 0, 'TIMEOUT': 0, 'IMAGE+TEXT': 0,
'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0}) |
| 136 | 136 |
| 137 def test_svn_revision(self): | 137 def test_svn_revision(self): |
| 138 self.port._options.builder_name = 'dummy builder' | 138 self.port._options.builder_name = 'dummy builder' |
| 139 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 139 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
| 140 self.assertNotEquals(summary['blink_revision'], '') | 140 self.assertNotEquals(summary['blink_revision'], '') |
| 141 | 141 |
| 142 def test_bug_entry(self): | 142 def test_bug_entry(self): |
| 143 self.port._options.builder_name = 'dummy builder' | 143 self.port._options.builder_name = 'dummy builder' |
| 144 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) | 144 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) |
| 145 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bu
gs'], ['Bug(test)']) | 145 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bu
gs'], ['Bug(test)']) |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 179 self.port._options.builder_name = 'dummy builder' | 179 self.port._options.builder_name = 'dummy builder' |
| 180 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False, only_include_failing=True) | 180 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False, only_include_failing=True) |
| 181 self.assertTrue('passes' not in summary['tests']) | 181 self.assertTrue('passes' not in summary['tests']) |
| 182 | 182 |
| 183 def test_rounded_run_times(self): | 183 def test_rounded_run_times(self): |
| 184 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 184 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
| 185 self.assertEquals(summary['tests']['passes']['text.html']['time'], 1) | 185 self.assertEquals(summary['tests']['passes']['text.html']['time'], 1) |
| 186 self.assertTrue('time' not in summary['tests']['failures']['expected']['
audio.html']) | 186 self.assertTrue('time' not in summary['tests']['failures']['expected']['
audio.html']) |
| 187 self.assertEquals(summary['tests']['failures']['expected']['timeout.html
']['time'], 0.1) | 187 self.assertEquals(summary['tests']['failures']['expected']['timeout.html
']['time'], 0.1) |
| 188 self.assertTrue('time' not in summary['tests']['failures']['expected']['
crash.html']) | 188 self.assertTrue('time' not in summary['tests']['failures']['expected']['
crash.html']) |
| OLD | NEW |