Add option to push files to device using isolate for instrumentation tests.

build/android/pylib/instrumentation/setup.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generates test runner factory and tests for instrumentation tests."""
 
 import logging
 import os
 
+from pylib import constants
+from pylib import valgrind_tools
+
+from pylib.device import device_utils
 from pylib.instrumentation import test_package
 from pylib.instrumentation import test_runner
+from pylib.utils import isolator
+
+_DEVICE_DATA_DIR = 'chrome/test/data'
+
+_ISOLATE_FILE_PATHS = {
+    'AndroidWebViewTest': 'android_webview/android_webview_test_apk.isolate',
+    'ChromeShellTest': 'chrome/chrome_shell_test_apk.isolate',
+    'ContentShellTest': 'content/content_shell_test_apk.isolate',
+    'MojoTest': 'mojo/mojo_test_apk.isolate',
+}
+
+def _GenerateDepsDirUsingIsolate(suite_name, isolate_file_path=None):
+  """Generate the dependency dir for the test suite using isolate.
+
+  Args:
+    suite_name: Name of the test suite (e.g. base_unittests).
+    isolate_file_path: .isolate file path to use. If there is a default .isolate
+                       file path for the suite_name, this will override it.
+  """
+  if isolate_file_path:
+    if os.path.isabs(isolate_file_path):
+      isolate_abs_path = isolate_file_path
+    else:
+      isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT,
+                                      isolate_file_path)
+  else:
+    isolate_rel_path = _ISOLATE_FILE_PATHS.get(suite_name)
+    if not isolate_rel_path:
+      logging.info('Did not find an isolate file for the test suite.')
+      return
+    isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_rel_path)
+
+  isolated_abs_path = os.path.join(
+      constants.GetOutDirectory(), '%s.isolated' % suite_name)
+  assert os.path.exists(isolate_abs_path), 'Cannot find %s' % isolate_abs_path
+
+  i = isolator.Isolator(constants.ISOLATE_DEPS_DIR)
+  i.Clear()
+  i.Remap(isolate_abs_path, isolated_abs_path)
+  # We're relying on the fact that timestamps are preserved
+  # by the remap command (hardlinked). Otherwise, all the data
+  # will be pushed to the device once we move to using time diff
+  # instead of md5sum. Perform a sanity check here.
+  i.VerifyHardlinks()
+  i.MoveOutputDeps()
 
 

[mikecase (-- gone --), 2014/11/01 06:11:06]

Wondering if you have a good solution to this. I would prefer to only have 1
function that does any data pushing. The reason I have _PushExtraSuiteDataDeps
is because for example with the ContentShellTests...

src/content/test/data/android/device_files/ 
to end up in
sdcard/CHROME/TEST/DATA/content/test/data/android/device_files

BUT

src/net/data/ssl/certificates/
to end up in 
sdcard/net/data/sll/certificates/

This means that if we used isolate for everything, we would have to push half of
ISOLATE_DEPS_DIR to sdcard/chrome/test/data/ and the other half of
ISOLATE_DEPS_DIR to sdcard/

Three solution I can think of but none are clean.

1. Leave _PushExtraSuiteDataDeps as is and don't use isolate to push
src/net/data/sll/certificates/

2. After running isolate remap, go into the ISOLATE_DEPS_DIR and change its
structure prepending chrome/test/data/ to some file paths.

3. Have everything pushed to the base sdcard/ directory. This would include a
good number of changes to individual tests that hardcode where they grab the
dependencies from. But mostly would require lots of LGTMs. 

-def Setup(test_options):
+def _PushExtraSuiteDataDeps(device, test_apk):
+  """Pushes some extra data files/dirs needed by some test suite.
+
+  Args:
+    test_apk: The test suite basename for which to return file paths.
+  """
+  if test_apk in ['ChromeTest', 'ContentShellTest']:
+    test_files = 'net/data/ssl/certificates'
+    host_device_file_tuple = [
+        (os.path.join(constants.DIR_SOURCE_ROOT, test_files),
+         os.path.join(device.GetExternalStoragePath(), test_files))]
+    device.PushChangedFiles(host_device_file_tuple)
+
+
+def _PushDataDepsWithIsolate(device, test_options):
+  valgrind_tools.PushFilesForTool(test_options.tool, device)
+  _PushExtraSuiteDataDeps(device, test_options.test_apk)
+
+  if os.path.exists(constants.ISOLATE_DEPS_DIR):
+    device_dir = os.path.join(device.GetExternalStoragePath(),
+        _DEVICE_DATA_DIR)
+    device.PushChangedFiles([
+        (os.path.join(constants.ISOLATE_DEPS_DIR, p),
+         os.path.join(device_dir, p))
+        for p in os.listdir(constants.ISOLATE_DEPS_DIR)])
+
+
+# TODO(mikecase): Remove this function once everything uses
+# PushDataDepsWithIsolate instead.
+def _PushDataDeps(device, test_options):
+  valgrind_tools.PushFilesForTool(test_options.tool, device)
+  _PushExtraSuiteDataDeps(device, test_options.test_apk)
+
+  host_device_file_tuples = []
+  for dest_host_pair in test_options.test_data:
+    dst_src = dest_host_pair.split(':', 1)
+    dst_layer = dst_src[0]
+    host_src = dst_src[1]
+    host_test_files_path = os.path.join(constants.DIR_SOURCE_ROOT, host_src)
+    if os.path.exists(host_test_files_path):
+      host_device_file_tuples += [(
+          host_test_files_path,
+          '%s/%s/%s' % (
+              device.GetExternalStoragePath(),
+              _DEVICE_DATA_DIR,
+              dst_layer))]
+  if host_device_file_tuples:
+    device.PushChangedFiles(host_device_file_tuples)
+
+
+def Setup(test_options, devices):
   """Create and return the test runner factory and tests.
 
   Args:
     test_options: An InstrumentationOptions object.
 
   Returns:
     A tuple of (TestRunnerFactory, tests).
   """
   if (test_options.coverage_dir and not
       os.path.exists(test_options.coverage_dir)):
     os.makedirs(test_options.coverage_dir)
 
   test_pkg = test_package.TestPackage(test_options.test_apk_path,
                                       test_options.test_apk_jar_path,
                                       test_options.test_support_apk_path)
   tests = test_pkg.GetAllMatchingTests(
       test_options.annotations,
       test_options.exclude_annotations,
       test_options.test_filter)
   if not tests:
     logging.error('No instrumentation tests to run with current args.')
 
+  if test_options.test_data:
+    device_utils.DeviceUtils.parallel(devices).pMap(
+        _PushDataDeps, test_options)
+  else:
+    _GenerateDepsDirUsingIsolate(test_options.test_apk,
+                                 test_options.isolate_file_path)
+    device_utils.DeviceUtils.parallel(devices).pMap(
+        _PushDataDepsWithIsolate, test_options)
+
   def TestRunnerFactory(device, shard_index):
     return test_runner.TestRunner(test_options, device, shard_index,
                                   test_pkg)
 
   return (TestRunnerFactory, tests)