| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """ | 5 """ |
| 6 This recipe can be used by components like v8 to verify blink tests with a | 6 This recipe can be used by components like v8 to verify blink tests with a |
| 7 low false positive rate. Similar to a trybot, this recipe compares test | 7 low false positive rate. Similar to a trybot, this recipe compares test |
| 8 failures from a build with a current component revision with test failures | 8 failures from a build with a current component revision with test failures |
| 9 from a build with a pinned component revision. | 9 from a build with a pinned component revision. |
| 10 | 10 |
| (...skipping 113 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 124 api.chromium.apply_config(c) | 124 api.chromium.apply_config(c) |
| 125 | 125 |
| 126 # Sync component to current component revision. | 126 # Sync component to current component revision. |
| 127 component_revision = api.properties.get('revision') or 'HEAD' | 127 component_revision = api.properties.get('revision') or 'HEAD' |
| 128 api.gclient.c.revisions[bot_config['component']['path']] = ( | 128 api.gclient.c.revisions[bot_config['component']['path']] = ( |
| 129 bot_config['component']['revision'] % component_revision) | 129 bot_config['component']['revision'] % component_revision) |
| 130 | 130 |
| 131 # Ensure we remember the chromium revision. | 131 # Ensure we remember the chromium revision. |
| 132 api.gclient.c.got_revision_mapping['src'] = 'got_cr_revision' | 132 api.gclient.c.got_revision_mapping['src'] = 'got_cr_revision' |
| 133 | 133 |
| 134 step_result = api.bot_update.ensure_checkout(force=True) | 134 context = {} |
| 135 checkout_dir = api.chromium_tests.get_checkout_dir(bot_config) |
| 136 if checkout_dir: |
| 137 context['cwd'] = checkout_dir |
| 135 | 138 |
| 136 api.chromium.c.project_generator.tool = 'mb' | 139 # Run all steps in the checkout dir (consistent with chromium_tests). |
| 137 api.chromium.runhooks() | 140 with api.step.context(context): |
| 141 # TODO(phajdan.jr): remove redundant **context below once we fix things |
| 142 # to behave the same without it. |
| 143 step_result = api.bot_update.ensure_checkout(force=True, **context) |
| 138 | 144 |
| 139 api.chromium_tests.run_mb_and_compile( | 145 api.chromium.ensure_goma() |
| 140 ['blink_tests'], [], | |
| 141 name_suffix=' (with patch)', | |
| 142 ) | |
| 143 | 146 |
| 144 api.chromium.runtest('webkit_unit_tests', xvfb=True) | 147 api.chromium.c.project_generator.tool = 'mb' |
| 145 | 148 api.chromium.runhooks() |
| 146 def component_pinned_fn(_failing_steps): | |
| 147 bot_update_json = step_result.json.output | |
| 148 api.gclient.c.revisions['src'] = str( | |
| 149 bot_update_json['properties']['got_cr_revision']) | |
| 150 # Reset component revision to the pinned revision from chromium's DEPS | |
| 151 # for comparison. | |
| 152 del api.gclient.c.revisions[bot_config['component']['path']] | |
| 153 # Update without changing got_revision. The first sync is the revision | |
| 154 # that is tested. The second is just for comparison. Setting got_revision | |
| 155 # again confuses the waterfall's console view. | |
| 156 api.bot_update.ensure_checkout(force=True, update_presentation=False) | |
| 157 | 149 |
| 158 api.chromium_tests.run_mb_and_compile( | 150 api.chromium_tests.run_mb_and_compile( |
| 159 ['blink_tests'], [], | 151 ['blink_tests'], [], |
| 160 name_suffix=' (without patch)', | 152 name_suffix=' (with patch)', |
| 161 ) | 153 ) |
| 162 | 154 |
| 163 extra_args = list(bot_config.get('test_args', [])) | 155 api.chromium.runtest('webkit_unit_tests', xvfb=True) |
| 164 if bot_config.get('additional_expectations'): | |
| 165 extra_args.extend([ | |
| 166 '--additional-expectations', | |
| 167 api.path['checkout'].join(*bot_config['additional_expectations']), | |
| 168 ]) | |
| 169 | 156 |
| 170 tests = [ | 157 def component_pinned_fn(_failing_steps): |
| 171 api.chromium_tests.steps.BlinkTest(extra_args=extra_args), | 158 bot_update_json = step_result.json.output |
| 172 ] | 159 api.gclient.c.revisions['src'] = str( |
| 160 bot_update_json['properties']['got_cr_revision']) |
| 161 # Reset component revision to the pinned revision from chromium's DEPS |
| 162 # for comparison. |
| 163 del api.gclient.c.revisions[bot_config['component']['path']] |
| 164 # Update without changing got_revision. The first sync is the revision |
| 165 # that is tested. The second is just for comparison. Setting got_revision |
| 166 # again confuses the waterfall's console view. |
| 167 api.bot_update.ensure_checkout(force=True, update_presentation=False) |
| 173 | 168 |
| 174 if 'ignition' in buildername: | 169 api.chromium_tests.run_mb_and_compile( |
| 175 determine_new_ignition_failures(api, extra_args) | 170 ['blink_tests'], [], |
| 176 else: | 171 name_suffix=' (without patch)', |
| 177 api.test_utils.determine_new_failures(api, tests, component_pinned_fn) | 172 ) |
| 173 |
| 174 extra_args = list(bot_config.get('test_args', [])) |
| 175 if bot_config.get('additional_expectations'): |
| 176 extra_args.extend([ |
| 177 '--additional-expectations', |
| 178 api.path['checkout'].join(*bot_config['additional_expectations']), |
| 179 ]) |
| 180 |
| 181 tests = [ |
| 182 api.chromium_tests.steps.BlinkTest(extra_args=extra_args), |
| 183 ] |
| 184 |
| 185 if 'ignition' in buildername: |
| 186 determine_new_ignition_failures(api, extra_args) |
| 187 else: |
| 188 api.test_utils.determine_new_failures(api, tests, component_pinned_fn) |
| 178 | 189 |
| 179 | 190 |
| 180 def _sanitize_nonalpha(text): | 191 def _sanitize_nonalpha(text): |
| 181 return ''.join(c if c.isalnum() else '_' for c in text) | 192 return ''.join(c if c.isalnum() else '_' for c in text) |
| 182 | 193 |
| 183 | 194 |
| 184 def GenTests(api): | 195 def GenTests(api): |
| 185 canned_test = api.test_utils.canned_test_output | 196 canned_test = api.test_utils.canned_test_output |
| 186 with_patch = 'webkit_tests (with patch)' | 197 with_patch = 'webkit_tests (with patch)' |
| 187 without_patch = 'webkit_tests (without patch)' | 198 without_patch = 'webkit_tests (without patch)' |
| 188 | 199 |
| 189 def properties(mastername, buildername): | 200 def properties(mastername, buildername): |
| 190 return ( | 201 return ( |
| 191 api.properties.generic(mastername=mastername, | 202 api.properties.generic(mastername=mastername, |
| 192 buildername=buildername, | 203 buildername=buildername, |
| 193 revision='20123') | 204 revision='20123', |
| 205 path_config='kitchen') |
| 194 ) | 206 ) |
| 195 | 207 |
| 196 for mastername, master_config in BUILDERS.iteritems(): | 208 for mastername, master_config in BUILDERS.iteritems(): |
| 197 for buildername, bot_config in master_config['builders'].iteritems(): | 209 for buildername, bot_config in master_config['builders'].iteritems(): |
| 198 test_name = 'full_%s_%s' % (_sanitize_nonalpha(mastername), | 210 test_name = 'full_%s_%s' % (_sanitize_nonalpha(mastername), |
| 199 _sanitize_nonalpha(buildername)) | 211 _sanitize_nonalpha(buildername)) |
| 200 tests = [] | 212 tests = [] |
| 201 for (pass_first, suffix) in ((True, '_pass'), (False, '_fail')): | 213 for (pass_first, suffix) in ((True, '_pass'), (False, '_fail')): |
| 202 test = ( | 214 test = ( |
| 203 properties(mastername, buildername) + | 215 properties(mastername, buildername) + |
| (...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 255 # and compare the lists of failing tests). | 267 # and compare the lists of failing tests). |
| 256 yield ( | 268 yield ( |
| 257 api.test('too_many_failures_for_retcode') + | 269 api.test('too_many_failures_for_retcode') + |
| 258 properties('client.v8.fyi', 'V8-Blink Linux 64') + | 270 properties('client.v8.fyi', 'V8-Blink Linux 64') + |
| 259 api.override_step_data(with_patch, | 271 api.override_step_data(with_patch, |
| 260 canned_test(passing=False, | 272 canned_test(passing=False, |
| 261 num_additional_failures=125)) + | 273 num_additional_failures=125)) + |
| 262 api.override_step_data(without_patch, | 274 api.override_step_data(without_patch, |
| 263 canned_test(passing=True, minimal=True)) | 275 canned_test(passing=True, minimal=True)) |
| 264 ) | 276 ) |
| OLD | NEW |