Index: infra/recipes/blink_downstream.py |
diff --git a/infra/recipes/blink_downstream.py b/infra/recipes/blink_downstream.py |
new file mode 100644 |
index 0000000000000000000000000000000000000000..07ec622b623f00d0d6a316d8387a37d40d3e11ee |
--- /dev/null |
+++ b/infra/recipes/blink_downstream.py |
@@ -0,0 +1,277 @@ |
+# Copyright 2014 The Chromium Authors. All rights reserved. |
+# Use of this source code is governed by a BSD-style license that can be |
+# found in the LICENSE file. |
+ |
+""" |
+This recipe can be used by components like v8 to verify blink tests with a |
+low false positive rate. Similar to a trybot, this recipe compares test |
+failures from a build with a current component revision with test failures |
+from a build with a pinned component revision. |
+ |
+Summary of the recipe flow: |
+1. Sync chromium to HEAD |
+2. Sync blink to HEAD |
+3. Sync component X to revision Y |
+4. Run blink tests |
+-> In case of failures: |
+5. Sync chromium to same revision as 1 |
+6. Sync blink to same revision as 2 |
+7. Sync component X to pinned revision from DEPS file |
+8. Run blink tests |
+-> If failures in 4 don't happen in 8, then revision Y reveals a problem not |
+ present in the pinned revision |
+ |
+Revision Y will be the revision property as provided by buildbot or HEAD (i.e. |
+in a forced build with no revision provided). |
+""" |
+ |
+from recipe_engine.types import freeze |
+ |
+DEPS = [ |
+ 'build/chromium', |
+ 'build/chromium_checkout', |
+ 'build/chromium_tests', |
+ 'build/test_utils', |
+ 'depot_tools/bot_update', |
+ 'depot_tools/gclient', |
+ 'recipe_engine/json', |
+ 'recipe_engine/path', |
+ 'recipe_engine/platform', |
+ 'recipe_engine/properties', |
+ 'recipe_engine/python', |
+ 'recipe_engine/raw_io', |
+ 'recipe_engine/step', |
+] |
+ |
+ |
+def V8Builder(config, bits, platform): |
+ chromium_configs = [] |
+ if config == 'Debug': |
+ chromium_configs.append('v8_optimize_medium') |
+ chromium_configs.append('v8_slow_dchecks') |
+ return { |
+ 'gclient_apply_config': ['show_v8_revision'], |
+ 'chromium_apply_config': chromium_configs, |
+ 'chromium_config_kwargs': { |
+ 'BUILD_CONFIG': config, |
+ 'TARGET_BITS': bits, |
+ }, |
+ 'test_args': ['--no-pixel-tests'], |
+ 'additional_expectations': [ |
+ 'v8', 'tools', 'blink_tests', 'TestExpectations', |
+ ], |
+ 'component': {'path': 'src/v8', 'revision': '%s'}, |
+ 'testing': {'platform': platform}, |
+ } |
+ |
+ |
+BUILDERS = freeze({ |
+ 'client.v8.fyi': { |
+ 'builders': { |
+ 'V8-Blink Win': V8Builder('Release', 32, 'win'), |
+ 'V8-Blink Mac': V8Builder('Release', 64, 'mac'), |
+ 'V8-Blink Linux 64': V8Builder('Release', 64, 'linux'), |
+ 'V8-Blink Linux 64 - ignition': V8Builder('Release', 64, 'linux'), |
+ 'V8-Blink Linux 64 (dbg)': V8Builder('Debug', 64, 'linux'), |
+ }, |
+ }, |
+}) |
+ |
+ |
+def determine_new_ignition_failures(caller_api, extra_args): |
+ tests = [ |
+ caller_api.chromium_tests.steps.BlinkTest( |
+ extra_args=extra_args + [ |
+ '--additional-expectations', |
+ caller_api.path['checkout'].join( |
+ 'v8', 'tools', 'blink_tests', 'TestExpectationsIgnition'), |
+ '--additional-driver-flag', |
+ '--js-flags=--ignition', |
+ ], |
+ ), |
+ ] |
+ |
+ failing_tests = caller_api.test_utils.run_tests_with_patch(caller_api, tests) |
+ if not failing_tests: |
+ return |
+ |
+ try: |
+ # HACK(machenbach): Blink tests store state about failing tests. In order |
+ # to rerun without ignition, we need to remove the extra args from the |
+ # existing test object. TODO(machenbach): Remove this once ignition ships. |
+ failing_tests[0]._extra_args = extra_args |
+ caller_api.test_utils.run_tests(caller_api, failing_tests, 'without patch') |
+ finally: |
+ with caller_api.step.defer_results(): |
+ for t in failing_tests: |
+ caller_api.test_utils._summarize_retried_test(caller_api, t) |
+ |
+ |
+def RunSteps(api): |
+ mastername = api.properties.get('mastername') |
+ buildername = api.properties.get('buildername') |
+ master_dict = BUILDERS.get(mastername, {}) |
+ bot_config = master_dict.get('builders', {}).get(buildername) |
+ |
+ # Sync chromium to HEAD. |
+ api.gclient.set_config('chromium', GIT_MODE=True) |
+ api.gclient.c.revisions['src'] = 'HEAD' |
+ api.chromium.set_config('blink', |
+ **bot_config.get('chromium_config_kwargs', {})) |
+ |
+ for c in bot_config.get('gclient_apply_config', []): |
+ api.gclient.apply_config(c) |
+ for c in bot_config.get('chromium_apply_config', []): |
+ api.chromium.apply_config(c) |
+ |
+ # Sync component to current component revision. |
+ component_revision = api.properties.get('revision') or 'HEAD' |
+ api.gclient.c.revisions[bot_config['component']['path']] = ( |
+ bot_config['component']['revision'] % component_revision) |
+ |
+ # Ensure we remember the chromium revision. |
+ api.gclient.c.got_revision_mapping['src'] = 'got_cr_revision' |
+ |
+ context = {} |
+ checkout_dir = api.chromium_checkout.get_checkout_dir(bot_config) |
+ if checkout_dir: |
+ context['cwd'] = checkout_dir |
+ |
+ # Run all steps in the checkout dir (consistent with chromium_tests). |
+ with api.step.context(context): |
+ # TODO(phajdan.jr): remove redundant **context below once we fix things |
+ # to behave the same without it. |
+ step_result = api.bot_update.ensure_checkout(force=True, **context) |
+ |
+ api.chromium.ensure_goma() |
+ |
+ api.chromium.c.project_generator.tool = 'mb' |
+ api.chromium.runhooks() |
+ |
+ api.chromium_tests.run_mb_and_compile( |
+ ['blink_tests'], [], |
+ name_suffix=' (with patch)', |
+ ) |
+ |
+ api.chromium.runtest('webkit_unit_tests', xvfb=True) |
+ |
+ def component_pinned_fn(_failing_steps): |
+ bot_update_json = step_result.json.output |
+ api.gclient.c.revisions['src'] = str( |
+ bot_update_json['properties']['got_cr_revision']) |
+ # Reset component revision to the pinned revision from chromium's DEPS |
+ # for comparison. |
+ del api.gclient.c.revisions[bot_config['component']['path']] |
+ # Update without changing got_revision. The first sync is the revision |
+ # that is tested. The second is just for comparison. Setting got_revision |
+ # again confuses the waterfall's console view. |
+ api.bot_update.ensure_checkout(force=True, update_presentation=False) |
+ |
+ api.chromium_tests.run_mb_and_compile( |
+ ['blink_tests'], [], |
+ name_suffix=' (without patch)', |
+ ) |
+ |
+ extra_args = list(bot_config.get('test_args', [])) |
+ if bot_config.get('additional_expectations'): |
+ extra_args.extend([ |
+ '--additional-expectations', |
+ api.path['checkout'].join(*bot_config['additional_expectations']), |
+ ]) |
+ |
+ tests = [ |
+ api.chromium_tests.steps.BlinkTest(extra_args=extra_args), |
+ ] |
+ |
+ if 'ignition' in buildername: |
+ determine_new_ignition_failures(api, extra_args) |
+ else: |
+ api.test_utils.determine_new_failures(api, tests, component_pinned_fn) |
+ |
+ |
+def _sanitize_nonalpha(text): |
+ return ''.join(c if c.isalnum() else '_' for c in text) |
+ |
+ |
+def GenTests(api): |
+ canned_test = api.test_utils.canned_test_output |
+ with_patch = 'webkit_tests (with patch)' |
+ without_patch = 'webkit_tests (without patch)' |
+ |
+ def properties(mastername, buildername): |
+ return ( |
+ api.properties.generic(mastername=mastername, |
+ buildername=buildername, |
+ revision='20123', |
+ path_config='kitchen') |
+ ) |
+ |
+ for mastername, master_config in BUILDERS.iteritems(): |
+ for buildername, bot_config in master_config['builders'].iteritems(): |
+ test_name = 'full_%s_%s' % (_sanitize_nonalpha(mastername), |
+ _sanitize_nonalpha(buildername)) |
+ tests = [] |
+ for (pass_first, suffix) in ((True, '_pass'), (False, '_fail')): |
+ test = ( |
+ properties(mastername, buildername) + |
+ api.platform( |
+ bot_config['testing']['platform'], |
+ bot_config.get( |
+ 'chromium_config_kwargs', {}).get('TARGET_BITS', 64)) + |
+ api.test(test_name + suffix) + |
+ api.override_step_data(with_patch, canned_test(passing=pass_first)) |
+ ) |
+ if not pass_first: |
+ test += api.override_step_data( |
+ without_patch, canned_test(passing=False, minimal=True)) |
+ tests.append(test) |
+ |
+ for test in tests: |
+ yield test |
+ |
+ # This tests that if the first fails, but the second pass succeeds |
+ # that we fail the whole build. |
+ yield ( |
+ api.test('minimal_pass_continues') + |
+ properties('client.v8.fyi', 'V8-Blink Linux 64') + |
+ api.override_step_data(with_patch, canned_test(passing=False)) + |
+ api.override_step_data(without_patch, |
+ canned_test(passing=True, minimal=True)) |
+ ) |
+ |
+ |
+ # This tests what happens if something goes horribly wrong in |
+ # run-webkit-tests and we return an internal error; the step should |
+ # be considered a hard failure and we shouldn't try to compare the |
+ # lists of failing tests. |
+ # 255 == test_run_results.UNEXPECTED_ERROR_EXIT_STATUS in run-webkit-tests. |
+ yield ( |
+ api.test('webkit_tests_unexpected_error') + |
+ properties('client.v8.fyi', 'V8-Blink Linux 64') + |
+ api.override_step_data(with_patch, canned_test(passing=False, retcode=255)) |
+ ) |
+ |
+ # TODO(dpranke): crbug.com/357866 . This tests what happens if we exceed the |
+ # number of failures specified with --exit-after-n-crashes-or-times or |
+ # --exit-after-n-failures; the step should be considered a hard failure and |
+ # we shouldn't try to compare the lists of failing tests. |
+ # 130 == test_run_results.INTERRUPTED_EXIT_STATUS in run-webkit-tests. |
+ yield ( |
+ api.test('webkit_tests_interrupted') + |
+ properties('client.v8.fyi', 'V8-Blink Linux 64') + |
+ api.override_step_data(with_patch, canned_test(passing=False, retcode=130)) |
+ ) |
+ |
+ # This tests what happens if we don't trip the thresholds listed |
+ # above, but fail more tests than we can safely fit in a return code. |
+ # (this should be a soft failure and we can still retry w/o the patch |
+ # and compare the lists of failing tests). |
+ yield ( |
+ api.test('too_many_failures_for_retcode') + |
+ properties('client.v8.fyi', 'V8-Blink Linux 64') + |
+ api.override_step_data(with_patch, |
+ canned_test(passing=False, |
+ num_additional_failures=125)) + |
+ api.override_step_data(without_patch, |
+ canned_test(passing=True, minimal=True)) |
+ ) |