OLD | NEW |
---|---|
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """ | 5 """ |
6 This recipe can be used by components like v8 to verify blink tests with a | 6 This recipe can be used by components like v8 to verify blink tests with a |
7 low false positive rate. Similar to a trybot, this recipe compares test | 7 low false positive rate. Similar to a trybot, this recipe compares test |
8 failures from a build with a current component revision with test failures | 8 failures from a build with a current component revision with test failures |
9 from a build with a pinned component revision. | 9 from a build with a pinned component revision. |
10 | 10 |
(...skipping 113 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
124 api.chromium.apply_config(c) | 124 api.chromium.apply_config(c) |
125 | 125 |
126 # Sync component to current component revision. | 126 # Sync component to current component revision. |
127 component_revision = api.properties.get('revision') or 'HEAD' | 127 component_revision = api.properties.get('revision') or 'HEAD' |
128 api.gclient.c.revisions[bot_config['component']['path']] = ( | 128 api.gclient.c.revisions[bot_config['component']['path']] = ( |
129 bot_config['component']['revision'] % component_revision) | 129 bot_config['component']['revision'] % component_revision) |
130 | 130 |
131 # Ensure we remember the chromium revision. | 131 # Ensure we remember the chromium revision. |
132 api.gclient.c.got_revision_mapping['src'] = 'got_cr_revision' | 132 api.gclient.c.got_revision_mapping['src'] = 'got_cr_revision' |
133 | 133 |
134 step_result = api.bot_update.ensure_checkout(force=True) | 134 context = {} |
135 checkout_dir = api.chromium_tests.get_checkout_dir(bot_config) | |
136 if checkout_dir: | |
137 context['cwd'] = checkout_dir | |
135 | 138 |
136 api.chromium.c.project_generator.tool = 'mb' | 139 with api.step.context(context): |
Sergiy Byelozyorov
2016/07/21 12:00:24
please explain in description why this context is
Paweł Hajdan Jr.
2016/07/21 13:53:05
Done.
| |
137 api.chromium.runhooks() | 140 step_result = api.bot_update.ensure_checkout(force=True) |
138 | 141 |
139 api.chromium_tests.run_mb_and_compile( | 142 api.chromium.c.project_generator.tool = 'mb' |
140 ['blink_tests'], [], | 143 api.chromium.runhooks() |
141 name_suffix=' (with patch)', | |
142 ) | |
143 | |
144 api.chromium.runtest('webkit_unit_tests', xvfb=True) | |
145 | |
146 def component_pinned_fn(_failing_steps): | |
147 bot_update_json = step_result.json.output | |
148 api.gclient.c.revisions['src'] = str( | |
149 bot_update_json['properties']['got_cr_revision']) | |
150 # Reset component revision to the pinned revision from chromium's DEPS | |
151 # for comparison. | |
152 del api.gclient.c.revisions[bot_config['component']['path']] | |
153 # Update without changing got_revision. The first sync is the revision | |
154 # that is tested. The second is just for comparison. Setting got_revision | |
155 # again confuses the waterfall's console view. | |
156 api.bot_update.ensure_checkout(force=True, update_presentation=False) | |
157 | 144 |
158 api.chromium_tests.run_mb_and_compile( | 145 api.chromium_tests.run_mb_and_compile( |
159 ['blink_tests'], [], | 146 ['blink_tests'], [], |
160 name_suffix=' (without patch)', | 147 name_suffix=' (with patch)', |
161 ) | 148 ) |
162 | 149 |
163 extra_args = list(bot_config.get('test_args', [])) | 150 api.chromium.runtest('webkit_unit_tests', xvfb=True) |
164 if bot_config.get('additional_expectations'): | |
165 extra_args.extend([ | |
166 '--additional-expectations', | |
167 api.path['checkout'].join(*bot_config['additional_expectations']), | |
168 ]) | |
169 | 151 |
170 tests = [ | 152 def component_pinned_fn(_failing_steps): |
171 api.chromium_tests.steps.BlinkTest(extra_args=extra_args), | 153 bot_update_json = step_result.json.output |
172 ] | 154 api.gclient.c.revisions['src'] = str( |
155 bot_update_json['properties']['got_cr_revision']) | |
156 # Reset component revision to the pinned revision from chromium's DEPS | |
157 # for comparison. | |
158 del api.gclient.c.revisions[bot_config['component']['path']] | |
159 # Update without changing got_revision. The first sync is the revision | |
160 # that is tested. The second is just for comparison. Setting got_revision | |
161 # again confuses the waterfall's console view. | |
162 api.bot_update.ensure_checkout(force=True, update_presentation=False) | |
173 | 163 |
174 if 'ignition' in buildername: | 164 api.chromium_tests.run_mb_and_compile( |
175 determine_new_ignition_failures(api, extra_args) | 165 ['blink_tests'], [], |
176 else: | 166 name_suffix=' (without patch)', |
177 api.test_utils.determine_new_failures(api, tests, component_pinned_fn) | 167 ) |
168 | |
169 extra_args = list(bot_config.get('test_args', [])) | |
170 if bot_config.get('additional_expectations'): | |
171 extra_args.extend([ | |
172 '--additional-expectations', | |
173 api.path['checkout'].join(*bot_config['additional_expectations']), | |
174 ]) | |
175 | |
176 tests = [ | |
177 api.chromium_tests.steps.BlinkTest(extra_args=extra_args), | |
178 ] | |
179 | |
180 if 'ignition' in buildername: | |
181 determine_new_ignition_failures(api, extra_args) | |
182 else: | |
183 api.test_utils.determine_new_failures(api, tests, component_pinned_fn) | |
178 | 184 |
179 | 185 |
180 def _sanitize_nonalpha(text): | 186 def _sanitize_nonalpha(text): |
181 return ''.join(c if c.isalnum() else '_' for c in text) | 187 return ''.join(c if c.isalnum() else '_' for c in text) |
182 | 188 |
183 | 189 |
184 def GenTests(api): | 190 def GenTests(api): |
185 canned_test = api.test_utils.canned_test_output | 191 canned_test = api.test_utils.canned_test_output |
186 with_patch = 'webkit_tests (with patch)' | 192 with_patch = 'webkit_tests (with patch)' |
187 without_patch = 'webkit_tests (without patch)' | 193 without_patch = 'webkit_tests (without patch)' |
188 | 194 |
189 def properties(mastername, buildername): | 195 def properties(mastername, buildername): |
190 return ( | 196 return ( |
191 api.properties.generic(mastername=mastername, | 197 api.properties.generic(mastername=mastername, |
192 buildername=buildername, | 198 buildername=buildername, |
193 revision='20123') | 199 revision='20123', |
200 path_config='kitchen') | |
194 ) | 201 ) |
195 | 202 |
196 for mastername, master_config in BUILDERS.iteritems(): | 203 for mastername, master_config in BUILDERS.iteritems(): |
197 for buildername, bot_config in master_config['builders'].iteritems(): | 204 for buildername, bot_config in master_config['builders'].iteritems(): |
198 test_name = 'full_%s_%s' % (_sanitize_nonalpha(mastername), | 205 test_name = 'full_%s_%s' % (_sanitize_nonalpha(mastername), |
199 _sanitize_nonalpha(buildername)) | 206 _sanitize_nonalpha(buildername)) |
200 tests = [] | 207 tests = [] |
201 for (pass_first, suffix) in ((True, '_pass'), (False, '_fail')): | 208 for (pass_first, suffix) in ((True, '_pass'), (False, '_fail')): |
202 test = ( | 209 test = ( |
203 properties(mastername, buildername) + | 210 properties(mastername, buildername) + |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
255 # and compare the lists of failing tests). | 262 # and compare the lists of failing tests). |
256 yield ( | 263 yield ( |
257 api.test('too_many_failures_for_retcode') + | 264 api.test('too_many_failures_for_retcode') + |
258 properties('client.v8.fyi', 'V8-Blink Linux 64') + | 265 properties('client.v8.fyi', 'V8-Blink Linux 64') + |
259 api.override_step_data(with_patch, | 266 api.override_step_data(with_patch, |
260 canned_test(passing=False, | 267 canned_test(passing=False, |
261 num_additional_failures=125)) + | 268 num_additional_failures=125)) + |
262 api.override_step_data(without_patch, | 269 api.override_step_data(without_patch, |
263 canned_test(passing=True, minimal=True)) | 270 canned_test(passing=True, minimal=True)) |
264 ) | 271 ) |
OLD | NEW |