Add helper script for running CTS APKs stored in the GS buckets.

android_webview/tools/run_cts.py

+#!/usr/bin/env python
+#
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Runs the CTS test APKs stored in GS."""
+
+import argparse
+import json
+import os
+import shutil
+import sys
+import tempfile
+
+# pylint: disable=import-error
+sys.path.append(os.path.join(
+    os.path.dirname(__file__), os.pardir, os.pardir, 'build', 'android'))
+import devil_chromium
+from devil.utils import cmd_helper
+
+sys.path.append(os.path.join(
+    os.path.dirname(__file__), os.pardir, os.pardir, 'build'))
+import find_depot_tools
+# pylint: enable=import-error
+
+_CTS_BUCKET = 'gs://chromium-cts'
+
+_GSUTIL_PATH = os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py')
+_TEST_RUNNER_PATH = os.path.join(
+    os.path.dirname(__file__), os.pardir, os.pardir,
+    'build', 'android', 'test_runner.py')
+
+_EXPECTED_FAILURES_FILE = os.path.join(
+    os.path.dirname(__file__), 'cts_config', 'expected_failure_on_bot.json')

[the real yoland, 2016/12/09 20:56:36]

maybe use DIR_SOURCE_ROOT instead of relative path from this file's location?

not sure though

[mikecase (-- gone --), 2016/12/12 17:31:43]

Yeah, in order to do that I would have to import something from pylib. I could
do this, since I already have to add build/android to PYTHONPATH for something
else. I dont think it would make the code cleaner though.

Leaving as is for now

+_WEBVIEW_CTS_GCS_PATH_FILE = os.path.join(
+    os.path.dirname(__file__), 'cts_config', 'webview_cts_gcs_path.json')
+
+
+def GetCtsPath(arch, platform):
+  """Gets relative path CTS APK is stored."""
+  with open(_WEBVIEW_CTS_GCS_PATH_FILE) as f:
+    cts_gcs_path_info = json.load(f)
+  try:
+    return cts_gcs_path_info[arch][platform]['apk']
+  except KeyError:
+    raise Exception('No CTS test available for arch:%s, android:%s' %
+                    (arch, platform))
+
+
+def GetExpectedFailures():
+  """Gets list of tests expected to fail."""
+  with open(_EXPECTED_FAILURES_FILE) as f:
+    expected_failures_info = json.load(f)
+  expected_failures = []
+  for class_name, methods in expected_failures_info.iteritems():
+    expected_failures.extend(['%s#%s' % (class_name, m['name'])
+                              for m in methods])
+  return expected_failures
+
+
+def DownloadAndRunCTS(args, test_runner_args):
+  base_cts_dir = None
+  delete_cts_dir = False
+  try:
+    relative_cts_path = GetCtsPath(args.arch, args.platform)
+
+    if args.apk_dir:
+      base_cts_dir = args.apk_dir
+    else:
+      base_cts_dir = tempfile.mkdtemp()
+      delete_cts_dir = True
+
+    local_cts_path = os.path.join(base_cts_dir, relative_cts_path)
+    google_storage_cts_path = '%s/%s' % (_CTS_BUCKET, relative_cts_path)
+
+    # Download CTS APK if needed.
+    if not os.path.exists(local_cts_path):
+      if cmd_helper.RunCmd(
+          [_GSUTIL_PATH, 'cp', google_storage_cts_path, local_cts_path]):

[the real yoland, 2016/12/09 20:56:36]

would using tools/depot_tools/download_from_google_storage.py be easier?

[mikecase (-- gone --), 2016/12/12 17:31:43]

IIRC, download_from_google_storage.py only downloads files using SHA1 as
filenames. I don't think it will work for this case.

[the real yoland, 2016/12/12 18:31:09]

right right, I remember now

+        raise Exception('Error downloading CTS from Google Storage.')
+
+    test_runner_args += ['--test-apk', local_cts_path]
+    # TODO(mikecase): This doesn't work at all with the
+    # --gtest-filter test runner option currently. The
+    # filter options will just override eachother.
+    if args.skip_expected_failures:
+      test_runner_args += ['-f=-%s' % ':'.join(GetExpectedFailures())]
+    return cmd_helper.RunCmd(
+        [_TEST_RUNNER_PATH] + ['instrumentation'] + test_runner_args)
+  finally:
+    if delete_cts_dir and base_cts_dir:
+      shutil.rmtree(base_cts_dir)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--arch',
+      choices=['arm_64'],
+      required=True,

[the real yoland, 2016/12/09 20:56:36]

maybe default this if there is only one option now and in the near future

[mikecase (-- gone --), 2016/12/12 17:31:43]

agreed, Done

+      help='Arch for CTS tests.')
+  parser.add_argument(
+      '--platform',
+      choices=['L', 'M', 'N'],
+      required=True,
+      help='Android platform version for CTS tests.')
+  parser.add_argument(
+      '--skip-expected-failures',

[the real yoland, 2016/12/09 20:56:36]

default this to true?

[mikecase (-- gone --), 2016/12/12 17:31:43]

hmm, leaving as False so people aren't confused as to why some tests aren't
running

+      action='store_true',
+      help='Option to skip all tests that are expected to fail.')
+  parser.add_argument(
+      '--apk-dir',
+      help='Directory to load/save CTS APKs. Will try to load CTS APK '
+           'from this directory before downloading from Google Storage '
+           'and will then cache APK here.')
+
+  args, test_runner_args = parser.parse_known_args()

[the real yoland, 2016/12/09 20:56:36]

TIL :O
I assume these args aren't needed in test_runner.py (at least for CTS) then?

[mikecase (-- gone --), 2016/12/12 17:31:43]

Yeah, this is to allow forwarding some args to the test runner that people might
want to use.
Like -f to filter tests, -v for verbose, or --results-json-file to get results
in recipes.

I guess just have to make sure no args added to this script that share a name
with a test_runner.py arg :/

+  devil_chromium.Initialize()
+
+  return DownloadAndRunCTS(args, test_runner_args)
+
+
+if __name__ == '__main__':
+  sys.exit(main())