Reland of Include isolate.py in data for Android unit tests

PRESUBMIT.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Top-level presubmit script for Chromium.
 
 See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
 for more details about the presubmit API built into depot_tools.
 """
 
@@ skipping 280 matching lines @@
     'OS_NACL_NONSFI',
     'OS_NACL_SFI',
     'OS_OPENBSD',
     'OS_POSIX',
     'OS_QNX',
     'OS_SOLARIS',
     'OS_WIN',
 )
 
 
+_ANDROID_SPECIFIC_PYDEPS_FILES = [
+    'build/android/test_runner.pydeps',
+]
+
+_GENERIC_PYDEPS_FILES = [
+    'build/secondary/tools/swarming_client/isolate.pydeps',
+]
+
+_ALL_PYDEPS_FILES = _ANDROID_SPECIFIC_PYDEPS_FILES + _GENERIC_PYDEPS_FILES
+
+
 def _CheckNoProductionCodeUsingTestOnlyFunctions(input_api, output_api):
   """Attempts to prevent use of functions intended only for testing in
   non-testing code. For now this is just a best-effort implementation
   that ignores header files and may have some false positives. A
   better implementation would probably need a proper C++ parser.
   """
   # We only scan .cc files and the like, as the declaration of
   # for-testing functions in header files are hard to distinguish from
   # calls to such functions without a proper C++ parser.
   file_inclusion_pattern = r'.+%s' % _IMPLEMENTATION_EXTENSIONS
@@ skipping 1180 matching lines @@
   results = []
   if errors:
     results.append(output_api.PresubmitError(
         'MDPI assets should be placed in /res/drawable-mdpi/ or '
         '/res/drawable-ldrtl-mdpi/\ninstead of /res/drawable/ and'
         '/res/drawable-ldrtl/.\n'
         'Contact newt@chromium.org if you have questions.', errors))
   return results
 
 
+class PydepsChecker(object):
+  def __init__(self, input_api, pydeps_files):
+    self._file_cache = {}
+    self._input_api = input_api
+    self._pydeps_files = pydeps_files
+
+  def _LoadFile(self, path):
+    """Returns the list of paths within a .pydeps file relative to //."""
+    if path not in self._file_cache:
+      with open(path) as f:
+        self._file_cache[path] = f.read()
+    return self._file_cache[path]
+
+  def _ComputeNormalizedPydepsEntries(self, pydeps_path):
+    """Returns an interable of paths within the .pydep, relativized to //."""
+    os_path = self._input_api.os_path
+    pydeps_dir = os_path.dirname(pydeps_path)
+    entries = (l.rstrip() for l in self._LoadFile(pydeps_path).splitlines()
+               if not l.startswith('*'))
+    return (os_path.normpath(os_path.join(pydeps_dir, e)) for e in entries)
+
+  def _CreateFilesToPydepsMap(self):
+    """Returns a map of local_path -> list_of_pydeps."""
+    ret = {}
+    for pydep_local_path in self._pydeps_files:
+      for path in self._ComputeNormalizedPydepsEntries(pydep_local_path):
+        ret.setdefault(path, []).append(pydep_local_path)
+    return ret
+
+  def ComputeAffectedPydeps(self):
+    """Returns an iterable of .pydeps files that might need regenerating."""
+    affected_pydeps = set()
+    file_to_pydeps_map = None
+    for f in self._input_api.AffectedFiles(include_deletes=True):
+      local_path = f.LocalPath()
+      if local_path  == 'DEPS':
+        return self._pydeps_files
+      elif local_path.endswith('.pydeps'):
+        if local_path in self._pydeps_files:
+          affected_pydeps.add(local_path)
+      elif local_path.endswith('.py'):
+        if file_to_pydeps_map is None:
+          file_to_pydeps_map = self._CreateFilesToPydepsMap()
+        affected_pydeps.update(file_to_pydeps_map.get(local_path, ()))
+    return affected_pydeps
+
+  def DetermineIfStale(self, pydeps_path):
+    """Runs print_python_deps.py to see if the files is stale."""
+    old_pydeps_data = self._LoadFile(pydeps_path)
+
+    m = self._input_api.re.search(r'# target: //(.*)', old_pydeps_data)
+    if not m:
+      return ['COULD NOT FIND .pydeps TARGET']
+    target = m.group(1)
+    m = self._input_api.re.search(r'# root: //(.*)', old_pydeps_data)
+    if not m:
+      return ['COULD NOT FIND .pydeps ROOT']
+    root = m.group(1) or '.'
+
+    cmd = ['build/print_python_deps.py', '--root', root, target]
+    new_pydeps_data = self._input_api.subprocess.check_output(cmd)
+    if old_pydeps_data != new_pydeps_data:
+      return cmd[:-1] + ['--output', pydeps_path] + cmd[-1:]
+
+
+def _CheckPydepsNeedsUpdating(input_api, output_api, checker_for_tests=None):
+  """Checks if a .pydeps file needs to be regenerated."""
+  # TODO(agrieve): Update when there's a better way to detect this: crbug/570091
+  is_android = input_api.os_path.exists('third_party/android_tools')
+  pydeps_files = _ALL_PYDEPS_FILES if is_android else _GENERIC_PYDEPS_FILES
+  results = []
+  # First, check for new / deleted .pydeps.
+  for f in input_api.AffectedFiles(include_deletes=True):
+    if f.LocalPath().endswith('.pydeps'):
+      if f.Action() == 'D' and f.LocalPath() in _ALL_PYDEPS_FILES:
+        results.append(output_api.PresubmitError(
+            'Please update _ALL_PYDEPS_FILES within //PRESUBMIT.py to '
+            'remove %s' % f.LocalPath()))
+      elif f.Action() != 'D' and f.LocalPath() not in _ALL_PYDEPS_FILES:
+        results.append(output_api.PresubmitError(
+            'Please update _ALL_PYDEPS_FILES within //PRESUBMIT.py to '
+            'include %s' % f.LocalPath()))
+
+  if results:
+    return results
+
+  checker = checker_for_tests or PydepsChecker(input_api, pydeps_files)
+
+  for pydep_path in checker.ComputeAffectedPydeps():
+    try:
+      cmd = checker.DetermineIfStale(pydep_path)
+      if cmd:
+        results.append(output_api.PresubmitError(
+            'File is stale: %s\nTo regenerate, run:\n%s' % (pydep_path,
+                                                            ' '.join(cmd))))
+    except input_api.subprocess.CalledProcessError as error:
+      return [output_api.PresubmitError('Error running ' + ' '.join(error.cmd),
+          long_text=error.output)]
+
+  return results
+
+
 def _CheckForCopyrightedCode(input_api, output_api):
   """Verifies that newly added code doesn't contain copyrighted material
   and is properly licensed under the standard Chromium license.
 
   As there can be false positives, we maintain a whitelist file. This check
   also verifies that the whitelist file is up to date.
   """
   import sys
   original_sys_path = sys.path
   try:
@@ skipping 133 matching lines @@
               (fpath.LocalPath(), lnum, deprecated, replacement)))
   return results
 
 
 def _AndroidSpecificOnUploadChecks(input_api, output_api):
   """Groups checks that target android code."""
   results = []
   results.extend(_CheckAndroidCrLogUsage(input_api, output_api))
   results.extend(_CheckAndroidNewMdpiAssetLocation(input_api, output_api))
   results.extend(_CheckAndroidToastUsage(input_api, output_api))
+  results.extend(_CheckPydepsNeedsUpdating(input_api, output_api))
   return results
 
 
 def _CommonChecks(input_api, output_api):
   """Checks common to both upload and commit."""
   results = []
   results.extend(input_api.canned_checks.PanProjectChecks(
       input_api, output_api,
       excluded_paths=_EXCLUDED_PATHS + _TESTRUNNER_PATHS))
   results.extend(_CheckAuthorizedAuthor(input_api, output_api))
@@ skipping 311 matching lines @@
   results.extend(input_api.canned_checks.CheckTreeIsOpen(
       input_api,
       output_api,
       json_url='http://chromium-status.appspot.com/current?format=json'))
 
   results.extend(input_api.canned_checks.CheckChangeHasBugField(
       input_api, output_api))
   results.extend(input_api.canned_checks.CheckChangeHasDescription(
       input_api, output_api))
   return results