Add IsolatedScriptTest

scripts/slave/recipe_modules/chromium_tests/steps.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import re
 import string
 
 
 class Test(object):
   """
@@ skipping 787 matching lines @@
   def run_test_locally(self, api, suffix):
     isolate_file_path = (api.path['checkout'].join(self._android_isolate_path)
                          if self._android_isolate_path else None)
     AndroidInstrumentationTest(
         name=self.name,
         compile_target=self._compile_target,
         adb_install_apk=self._apk_under_test,
         isolate_file_path=isolate_file_path).run(api, suffix)
 
 
+class IsolatedScriptTest(Test):
+  def __init__(self, name, args=None, target_name=None, revision=None,
+               webkit_revision=None, override_compile_targets=None,
+               **runtest_kwargs):
+    """Constructs an instance of IsolatedScriptTest.
+
+    An IsolatedScriptTest knows how to invoke an isolate which obeys a certain
+    contract. The isolate's main target must be a wrapper script which must
+    interpret certain command line arguments as follows:
+
+      --test-launcher-summary-output [FILENAME]
+
+    The wrapper script must write the simplified json output that the recipes
+    consume (similar to GTestTest and ScriptTest) into |FILENAME|.
+
+    The contract may be expanded later to support functionality like sharding
+    and retries of specific failed tests. Currently the examples of such wrapper
+    scripts live in src/testing/scripts/ in the Chromium workspace.
+
+    Args:
+      name: Displayed name of the test. May be modified by suffixes.
+      args: Arguments to be passed to the test.
+      target_name: Actual name of the test. Defaults to name.
+      revision: Revision of the Chrome checkout.
+      webkit_revision: Revision of the WebKit checkout.
+      override_compile_targets: List of compile targets for this test
+          (for tests that don't follow target naming conventions).
+      runtest_kwargs: Additional keyword args forwarded to the runtest.
+    """
+    super(IsolatedScriptTest, self).__init__()
+    self._name = name
+    self._args = args or []
+    self._target_name = target_name
+    self._revision = revision
+    self._webkit_revision = webkit_revision
+    self._override_compile_targets = override_compile_targets
+    self._runtest_kwargs = runtest_kwargs
+
+  @property
+  def name(self):
+    return self._name
+
+  @property
+  def target_name(self):
+    return self._target_name or self._name
+
+  @property
+  def isolate_target(self):

[Ken Russell (switch to Gerrit), 2015/09/15 01:41:16]

I think we need to override the "uses_swarming" property too, and make it return
true.

+    return self.target_name  # pragma: no cover
+
+  # TODO(nednguyen, kbr): figure out what to do with Android.
+  def compile_targets(self, api):
+    if self._override_compile_targets:
+      return self._override_compile_targets
+
+    if api.chromium.c.TARGET_PLATFORM == 'android':
+      return [self.target_name + '_apk']
+
+    return [self.target_name]
+
+  def run(self, api, suffix):
+    # Copy the list because run can be invoked multiple times and we modify
+    # the local copy.
+    args = self._args[:]
+    is_android = api.chromium.c.TARGET_PLATFORM == 'android'
+
+    # TODO(nednguyen, kbr): figure out what to do with Android.
+    if is_android:
+      raise UnimplementedError
+
+    kwargs = {}
+    # TODO(nednguyen, kbr): define contract with the wrapper script to rerun
+    # a subset of the tests.
+    # if suffix == 'without patch':
+    # failures = self.failures(api, 'with patch')
+    # if is_android:
+    #   kwargs['gtest_filter'] = ':'.join(failures)  # pragma: no cover
+    # else:
+    #   args.append(api.chromium.test_launcher_filter(failures))
+
+    gtest_results_file = api.test_utils.gtest_results(add_json_log=False)

[Ken Russell (switch to Gerrit), 2015/09/17 01:19:31]

We should use api.json.output() here and a similar step_test_data to what
ScriptTest uses. I didn't realize this, but the use of placeholders like
api.json.output() causes properties to be added to the step's active_result
corresponding to where the placeholder came from. So using api.json.output()
results in api.step.active_result.json.output being synthesized. The mismatch
between the use of api.test_utils.gtest_results and api.step.active_result.json
is what's breaking this code.

+    step_test_data = lambda: api.test_utils.test_api.canned_gtest_output(True)
+
+    kwargs['name'] = self._step_name(suffix)
+    kwargs['args'] = args
+    kwargs['step_test_data'] = step_test_data
+    kwargs['xvfb'] = True
+    kwargs['test_type'] = self.name
+    kwargs['annotate'] = 'gtest'
+    kwargs['test_launcher_summary_output'] = gtest_results_file
+    kwargs.update(self._runtest_kwargs)
+
+    try:
+      api.isolate.runtest(self.target_name, self._revision,
+                          self._webkit_revision, **kwargs)
+    finally:
+      self._test_runs[suffix] = api.step.active_result

[iannucci, 2015/09/17 00:49:48]

talked to kbr. I think what you want to do is switch to api.json.output() and
mimic what ScriptTest does in its processing of the api.step.active_result.

+      if self.has_valid_results(api, suffix):
+        self._test_runs[suffix].presentation.step_text += (
+            api.test_utils.format_step_text([
+              ['failures:', self.failures(api, suffix)]
+            ]))
+    return self._test_runs[suffix]
+
+  def has_valid_results(self, api, suffix):
+    try:
+      # Make sure the JSON includes all necessary data.
+      self.failures(api, suffix)
+
+      return self._test_runs[suffix].json.output['valid']
+    except Exception:  # pragma: no cover
+      return False
+
+  def failures(self, api, suffix):
+    return self._test_runs[suffix].json.output['failures']
+
+
+def generate_isolated_script(api, mastername, buildername, test_spec,
+                             enable_swarming=False,
+                             scripts_compile_targets=None):
+  for spec in test_spec.get(buildername, {}).get(
+      'isolated_scripts', []):
+    # TODO(kbr, nednguyen): figure out where revision & webkit_revision come
+    # from.
+    yield IsolatedScriptTest(
+        str(spec['name']),
+        args=spec.get('args', []),
+        target_name=spec['isolate_name'])
+
+
 class GTestTest(Test):
   def __init__(self, name, args=None, target_name=None, enable_swarming=False,
                swarming_shards=1, swarming_dimensions=None, swarming_tags=None,
                swarming_extra_suffix=None, **runtest_kwargs):
     super(GTestTest, self).__init__()
     if enable_swarming:
       self._test = SwarmingGTestTest(
           name, args, target_name, swarming_shards, swarming_dimensions,
           swarming_tags, swarming_extra_suffix)
     else:
@@ skipping 641 matching lines @@
   def run(self, api, suffix):
     api.chromium_android.coverage_report(upload=False)
     api.chromium_android.get_changed_lines_for_revision()
     api.chromium_android.incremental_coverage_report()
 
 
 GOMA_TESTS = [
   GTestTest('base_unittests'),
   GTestTest('content_unittests'),
 ]