[Android] Use isolate remap instead of check.

build/android/pylib/gtest/dispatch.py

 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import copy
 import fnmatch
+import glob
 import logging
 import os
+import shutil
 
 from pylib import android_commands
 from pylib import cmd_helper
 from pylib import constants
 from pylib import ports
 from pylib.base import shard
 from pylib.utils import emulator
 from pylib.utils import report_results
 from pylib.utils import xvfb
 
 import gtest_config
 import test_runner
 
 
+# TODO(frankf): Enable the use of the remaining isolate files
+# after making sure we don't blow up the dependency size (and
+# the world).
+_ISOLATE_FILE_PATHS = {
+  'base_unittests': 'base/base_unittests.isolate',
+  'unit_tests': 'chrome/unit_tests.isolate',
+  #'net_unittests': 'net/net_unittests.isolate',

[M-A Ruel, 2013/07/08 17:56:02]

I'd recommend to keep the keys sorted even if the lines are commented out.

It makes adding and removing entries easier afterward. I don't think it's useful
to keep the commented out entries now.

[frankf, 2013/07/09 01:09:35]

Done.

+  #'content_browsertests': 'content/content_browsertests.isolate',
+  #'content_unittests': 'content/content_unittests.isolate',
+  }
+# Used for filtering large data deps at a finer grain than what's allowed in

[M-A Ruel, 2013/07/08 17:56:02]

Add at least one whitespace between a variable and the next comment.

You could add a note to make it clear the list is using wildcards. I don't
really mind in fact.

[frankf, 2013/07/09 01:09:35]

Done.

+# isolate files since pushing deps to devices is expensive.
+_DEPS_EXCLUSION_LIST = [
+  'chrome/test/data/extensions/api_test',
+  'chrome/test/data/extensions/secure_shell',
+  'chrome/test/data/firefox*',
+  'chrome/test/data/image_decoding',
+  'chrome/test/data/import',
+  'chrome/test/data/page_cycler',
+  'chrome/test/data/perf',
+  'chrome/test/data/pyauto_private',
+  'chrome/test/data/safari_import',
+  'chrome/test/data/scroll',
+  'chrome/test/data/third_party',
+  'third_party/hunspell_dictionaries/*.dic',
+  ]
+_ISOLATE_SCRIPT = os.path.join(
+    constants.DIR_SOURCE_ROOT, 'tools', 'swarm_client', 'isolate.py')
+
+
+def _GenerateDepsDirUsingIsolate(test_suite, build_type):
+  """Generate the dependency dir for the test suite using isolate.
+
+  Args:
+    test_suite: The test suite basename (e.g. base_unittests).
+    build_type: Release/Debug
+
+  Returns:
+    If an isolate file exists, returns path to dependency dir on the host.
+    Otherwise, returns False.
+  """
+  product_dir = os.path.join(cmd_helper.OutDirectory.get(), build_type)
+  assert os.path.isabs(product_dir)
+  isolate_rel_path = _ISOLATE_FILE_PATHS.get(test_suite)
+  if not isolate_rel_path:
+    return False
+
+  isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_rel_path)
+  isolated_abs_path = os.path.join(
+      product_dir, '%s.isolated' % test_suite)
+  assert os.path.exists(isolate_abs_path)
+  deps_dir = os.path.join(product_dir, 'isolate_deps_dir')
+  if os.path.isdir(deps_dir):
+    shutil.rmtree(deps_dir)
+  # We're relying on the fact that timestamps are preserved
+  # by the remap command (hardlinked). Otherwise, all the data

[csharp, 2013/07/04 13:23:16]

I'm a bit concerned about this code relying on remap to work a certain way.
Would it be worth adding a quick sanity check to ensure that timestamps are
preserved in the destination directory? So if something does change we easily
spot it instead of seeing problems at the pushing step.

[M-A Ruel, 2013/07/08 17:56:02]

Or add a unit test for remap to ensure this assumption. I'm fine with that too.

[frankf, 2013/07/09 01:09:35]

Done. I'm checking that the first file we hit during a walk has the same inode
as the original.

On 2013/07/08 17:56:02, Marc-Antoine Ruel wrote:

+  # will be pushed to the device once we move to using time diff
+  # instead of md5sum.
+  isolate_cmd = [
+    'python', _ISOLATE_SCRIPT,
+    'remap',
+    '--isolate=%s' % isolate_abs_path,

[M-A Ruel, 2013/07/08 17:56:02]

'--isolate', isolate_abs_path,

same for --isolated and --outdir.

[frankf, 2013/07/09 01:09:35]

Done.

+    '--isolated=%s' % isolated_abs_path,
+    '-V', 'PRODUCT_DIR=%s' % product_dir,
+    '-V', 'OS=android',
+    '--outdir=%s' % deps_dir,
+    ]
+  assert not cmd_helper.RunCmd(isolate_cmd)
+
+  # Delete excluded files as defined by _DEPS_EXCLUSION_LIST.
+  old_cwd = os.getcwd()
+  os.chdir(deps_dir)

[M-A Ruel, 2013/07/08 17:56:02]

try/finally

[frankf, 2013/07/09 01:09:35]

Done.

+  excluded_paths = [x for y in _DEPS_EXCLUSION_LIST for x in glob.glob(y)]
+  if excluded_paths:
+    logging.warning('Excluding the following from dependency list: %s',
+                    excluded_paths)
+  for p in excluded_paths:
+    if os.path.isdir(p):
+      shutil.rmtree(p)
+    else:
+      os.remove(p)
+  os.chdir(old_cwd)
+
+  # On Android, all pak files need to be in the top-level 'paks' directory.
+  paks_dir = os.path.join(deps_dir, 'paks')
+  os.mkdir(paks_dir)
+  for root, _, filenames in os.walk(os.path.join(deps_dir, 'out')):
+    for filename in fnmatch.filter(filenames, '*.pak'):
+      shutil.move(os.path.join(root, filename), paks_dir)
+
+  # Move everything in PRODUCT_DIR to top level.
+  deps_product_dir = os.path.join(deps_dir, 'out', build_type)
+  if os.path.isdir(deps_product_dir):
+    for p in os.listdir(deps_product_dir):
+      shutil.move(os.path.join(deps_product_dir, p), deps_dir)
+
+  return deps_dir
+
+
 def _FullyQualifiedTestSuites(exe, option_test_suite, build_type):
   """Get a list of absolute paths to test suite targets.
 
   Args:
     exe: if True, use the executable-based test runner.
     option_test_suite: the test_suite specified as an option.
     build_type: 'Release' or 'Debug'.
 
   Returns:
     A list of tuples containing the suite and absolute path.
@@ skipping 98 matching lines @@
     attached_devices = android_commands.GetAttachedDevices()
 
   if not attached_devices:
     raise Exception('A device must be attached and online.')
 
   # Reset the test port allocation. It's important to do it before starting
   # to dispatch any tests.
   if not ports.ResetTestServerPortAllocation():
     raise Exception('Failed to reset test server port.')
 
+  deps_dir = _GenerateDepsDirUsingIsolate(suite_name, options.build_type)
+
   # Constructs a new TestRunner with the current options.
   def RunnerFactory(device, shard_index):
     return test_runner.TestRunner(
         device,
         options.test_suite,
         options.test_arguments,
         options.timeout,
         options.cleanup_test_files,
         options.tool,
         options.build_type,
         options.webkit,
         options.push_deps,
         constants.GTEST_TEST_PACKAGE_NAME,
         constants.GTEST_TEST_ACTIVITY_NAME,
-        constants.GTEST_COMMAND_LINE_FILE)
+        constants.GTEST_COMMAND_LINE_FILE,
+        deps_dir=deps_dir)
 
   # Get tests and split them up based on the number of devices.
   if options.gtest_filter:
     all_tests = [t for t in options.gtest_filter.split(':') if t]
   else:
     all_tests = GetAllEnabledTests(RunnerFactory, attached_devices)
   num_devices = len(attached_devices)
   tests = [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)]
   tests = [t for t in tests if t]
 
@@ skipping 48 matching lines @@
   failures = 0
   for suite_name, suite_path in all_test_suites:
     # Give each test suite its own copy of options.
     test_options = copy.deepcopy(options)
     test_options.test_suite = suite_path
     failures += _RunATestSuite(test_options, suite_name)
 
   if options.use_xvfb:
     framebuffer.Stop()
   return failures