OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
73 initial_results.add(get_result('failures/expected/audio.html'), expected
, test_is_slow) | 73 initial_results.add(get_result('failures/expected/audio.html'), expected
, test_is_slow) |
74 initial_results.add(get_result('failures/expected/timeout.html'), expect
ed, test_is_slow) | 74 initial_results.add(get_result('failures/expected/timeout.html'), expect
ed, test_is_slow) |
75 initial_results.add(get_result('failures/expected/crash.html'), expected
, test_is_slow) | 75 initial_results.add(get_result('failures/expected/crash.html'), expected
, test_is_slow) |
76 else: | 76 else: |
77 initial_results.add(get_result('passes/text.html', test_expectations.TIM
EOUT, run_time=1), expected, test_is_slow) | 77 initial_results.add(get_result('passes/text.html', test_expectations.TIM
EOUT, run_time=1), expected, test_is_slow) |
78 initial_results.add(get_result('failures/expected/audio.html', test_expe
ctations.AUDIO, run_time=0.049), expected, test_is_slow) | 78 initial_results.add(get_result('failures/expected/audio.html', test_expe
ctations.AUDIO, run_time=0.049), expected, test_is_slow) |
79 initial_results.add(get_result('failures/expected/timeout.html', test_ex
pectations.CRASH, run_time=0.05), expected, test_is_slow) | 79 initial_results.add(get_result('failures/expected/timeout.html', test_ex
pectations.CRASH, run_time=0.05), expected, test_is_slow) |
80 initial_results.add(get_result('failures/expected/crash.html', test_expe
ctations.TIMEOUT), expected, test_is_slow) | 80 initial_results.add(get_result('failures/expected/crash.html', test_expe
ctations.TIMEOUT), expected, test_is_slow) |
81 | 81 |
82 # we only list hang.html here, since normally this is WontFix | 82 # we only list hang.html here, since normally this is WontFix |
83 initial_results.add(get_result('failures/expected/hang.html', test_expec
tations.TIMEOUT), expected, test_is_slow) | 83 initial_results.add(get_result('failures/expected/hang.html', test_expec
tations.SKIP), expected, test_is_slow) |
84 | 84 |
85 if flaky: | 85 if flaky: |
86 retry_results = run_results(port, extra_skipped_tests) | 86 retry_results = run_results(port, extra_skipped_tests) |
87 retry_results.add(get_result('passes/text.html'), True, test_is_slow) | 87 retry_results.add(get_result('passes/text.html'), True, test_is_slow) |
88 retry_results.add(get_result('failures/expected/timeout.html'), True, te
st_is_slow) | 88 retry_results.add(get_result('failures/expected/timeout.html'), True, te
st_is_slow) |
89 retry_results.add(get_result('failures/expected/crash.html'), True, test
_is_slow) | 89 retry_results.add(get_result('failures/expected/crash.html'), True, test
_is_slow) |
90 else: | 90 else: |
91 retry_results = None | 91 retry_results = None |
92 | 92 |
93 return test_run_results.summarize_results(port, initial_results.expectations
, initial_results, retry_results, enabled_pixel_tests_in_retry=False, only_inclu
de_failing=only_include_failing) | 93 return test_run_results.summarize_results(port, initial_results.expectations
, initial_results, retry_results, enabled_pixel_tests_in_retry=False, only_inclu
de_failing=only_include_failing) |
(...skipping 25 matching lines...) Expand all Loading... |
119 def setUp(self): | 119 def setUp(self): |
120 host = MockHost(initialize_scm_by_default=False) | 120 host = MockHost(initialize_scm_by_default=False) |
121 self.port = host.port_factory.get(port_name='test') | 121 self.port = host.port_factory.get(port_name='test') |
122 | 122 |
123 def test_no_svn_revision(self): | 123 def test_no_svn_revision(self): |
124 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 124 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
125 self.assertNotIn('revision', summary) | 125 self.assertNotIn('revision', summary) |
126 | 126 |
127 def test_num_failures_by_type(self): | 127 def test_num_failures_by_type(self): |
128 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 128 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
129 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 0, 'SKIP': 0, 'TIMEOUT': 2, 'IMAGE+TEXT': 0, 'FAIL': 0, 'AUDIO': 1}) | 129 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 2, 'IMAGE+TEXT': 0,
'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0}) |
130 | 130 |
131 summary = summarized_results(self.port, expected=True, passing=False, fl
aky=False) | 131 summary = summarized_results(self.port, expected=True, passing=False, fl
aky=False) |
132 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 1, 'SKIP': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0, 'FAIL': 0, 'AUDIO': 1}) | 132 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0,
'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0}) |
133 | 133 |
134 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) | 134 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) |
135 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 4, 'SKIP': 1, 'TIMEOUT': 0, 'IMAGE+TEXT': 0, 'FAIL': 0, 'AUDIO': 0}) | 135 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING
': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, '
PASS': 4, 'REBASELINE': 0, 'SKIP': 1, 'SLOW': 0, 'TIMEOUT': 0, 'IMAGE+TEXT': 0,
'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0}) |
136 | 136 |
137 def test_svn_revision(self): | 137 def test_svn_revision(self): |
138 self.port._options.builder_name = 'dummy builder' | 138 self.port._options.builder_name = 'dummy builder' |
139 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 139 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
140 self.assertNotEquals(summary['blink_revision'], '') | 140 self.assertNotEquals(summary['blink_revision'], '') |
141 | 141 |
142 def test_bug_entry(self): | 142 def test_bug_entry(self): |
143 self.port._options.builder_name = 'dummy builder' | 143 self.port._options.builder_name = 'dummy builder' |
144 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) | 144 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False) |
145 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bu
gs'], ['Bug(test)']) | 145 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bu
gs'], ['Bug(test)']) |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
179 self.port._options.builder_name = 'dummy builder' | 179 self.port._options.builder_name = 'dummy builder' |
180 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False, only_include_failing=True) | 180 summary = summarized_results(self.port, expected=False, passing=True, fl
aky=False, only_include_failing=True) |
181 self.assertTrue('passes' not in summary['tests']) | 181 self.assertTrue('passes' not in summary['tests']) |
182 | 182 |
183 def test_rounded_run_times(self): | 183 def test_rounded_run_times(self): |
184 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) | 184 summary = summarized_results(self.port, expected=False, passing=False, f
laky=False) |
185 self.assertEquals(summary['tests']['passes']['text.html']['time'], 1) | 185 self.assertEquals(summary['tests']['passes']['text.html']['time'], 1) |
186 self.assertTrue('time' not in summary['tests']['failures']['expected']['
audio.html']) | 186 self.assertTrue('time' not in summary['tests']['failures']['expected']['
audio.html']) |
187 self.assertEquals(summary['tests']['failures']['expected']['timeout.html
']['time'], 0.1) | 187 self.assertEquals(summary['tests']['failures']['expected']['timeout.html
']['time'], 0.1) |
188 self.assertTrue('time' not in summary['tests']['failures']['expected']['
crash.html']) | 188 self.assertTrue('time' not in summary['tests']['failures']['expected']['
crash.html']) |
OLD | NEW |