Android: Replace symlinks with gradle filters

build/android/gradle/generate_gradle.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generates an Android Studio project from a GN target."""
 
 import argparse
 import codecs
+import glob
 import logging
 import os
 import re
 import shutil
 import subprocess
 import sys
 import zipfile
 
 _BUILD_ANDROID = os.path.join(os.path.dirname(__file__), os.pardir)
 sys.path.append(_BUILD_ANDROID)
 import devil_chromium
 from devil.utils import run_tests_helper
 from pylib import constants
 from pylib.constants import host_paths
 
 sys.path.append(os.path.join(_BUILD_ANDROID, 'gyp'))
 import jinja_template
 from util import build_utils
 
 
 _DEFAULT_ANDROID_MANIFEST_PATH = os.path.join(
     host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'AndroidManifest.xml')
 _FILE_DIR = os.path.dirname(__file__)
-_JAVA_SUBDIR = 'symlinked-java'
 _SRCJARS_SUBDIR = 'extracted-srcjars'
 _JNI_LIBS_SUBDIR = 'symlinked-libs'
 _ARMEABI_SUBDIR = 'armeabi'
 
 _DEFAULT_TARGETS = [
     # TODO(agrieve): Requires alternate android.jar to compile.
     # '//android_webview:system_webview_apk',
     '//android_webview/test:android_webview_apk',
     '//android_webview/test:android_webview_test_apk',
     '//base:base_junit_tests',
@@ skipping 153 matching lines @@
     native_section = entry.BuildConfig().get('native')
     if native_section:
       jni_libs = _CreateJniLibsDir(
           constants.GetOutDirectory(), self.EntryOutputDir(entry),
           native_section.get('libraries'))
     else:
       jni_libs = []
     return jni_libs
 
   def _GenJavaDirs(self, entry):
-    java_dirs = _CreateJavaSourceDir(
-        constants.GetOutDirectory(), self.EntryOutputDir(entry),
-        entry.JavaFiles())
+    java_dirs, excludes = _CreateJavaSourceDir(
+        constants.GetOutDirectory(), entry.JavaFiles())
     if self.Srcjars(entry):
       java_dirs.append(
           os.path.join(self.EntryOutputDir(entry), _SRCJARS_SUBDIR))
-    return java_dirs
+    return java_dirs, excludes
 
   def _Relativize(self, entry, paths):
     return _RebasePath(paths, self.EntryOutputDir(entry))
 
   def EntryOutputDir(self, entry):
     return os.path.join(self.project_dir, entry.GradleSubdir())
 
   def Srcjars(self, entry):
     srcjars = _RebasePath(entry.Gradle().get('bundled_srcjars', []))
     if not self.use_gradle_process_resources:
       srcjars += _RebasePath(entry.BuildConfig()['javac']['srcjars'])
     return srcjars
 
   def GeneratedInputs(self, entry):
     generated_inputs = []
     generated_inputs.extend(self.Srcjars(entry))
     generated_inputs.extend(
         p for p in entry.JavaFiles() if not p.startswith('..'))
     generated_inputs.extend(entry.Gradle()['dependent_prebuilt_jars'])
     return generated_inputs
 
   def Generate(self, entry):
     variables = {}
     android_test_manifest = entry.Gradle().get(
         'android_manifest', _DEFAULT_ANDROID_MANIFEST_PATH)
     variables['android_manifest'] = self._Relativize(
         entry, android_test_manifest)
-    variables['java_dirs'] = self._Relativize(entry, self._GenJavaDirs(entry))
+    java_dirs, excludes = self._GenJavaDirs(entry)
+    variables['java_dirs'] = self._Relativize(entry, java_dirs)
+    variables['java_excludes'] = excludes
     variables['jni_libs'] = self._Relativize(entry, self._GenJniLibs(entry))
     deps = [_ProjectEntry.FromBuildConfigPath(p)
             for p in entry.Gradle()['dependent_android_projects']]
     variables['android_project_deps'] = [d.ProjectName() for d in deps]
     # TODO(agrieve): Add an option to use interface jars and see if that speeds
     # things up at all.
     variables['prebuilts'] = self._Relativize(
         entry, entry.Gradle()['dependent_prebuilt_jars'])
     deps = [_ProjectEntry.FromBuildConfigPath(p)
             for p in entry.Gradle()['dependent_java_projects']]
     variables['java_project_deps'] = [d.ProjectName() for d in deps]
     return variables
 
 
 def _ComputeJavaSourceDirs(java_files):
-  """Returns the list of source directories for the given files."""
-  found_roots = set()
+  """Returns a dictionary of source dirs with each given files in one."""
+  found_roots = {}
   for path in java_files:
     path_root = path
     # Recognize these tokens as top-level.
     while True:
       path_root = os.path.dirname(path_root)
       basename = os.path.basename(path_root)
       assert basename, 'Failed to find source dir for ' + path
       if basename in ('java', 'src'):
         break
       if basename in ('javax', 'org', 'com'):
         path_root = os.path.dirname(path_root)
         break
-    found_roots.add(path_root)
-  return list(found_roots)
+    if path_root not in found_roots:
+      found_roots[path_root] = []
+    found_roots[path_root].append(path)
+  return found_roots
 
 
 def _CreateRelativeSymlink(target_path, link_path):
   link_dir = os.path.dirname(link_path)
   relpath = os.path.relpath(target_path, link_dir)
   logging.debug('Creating symlink %s -> %s', link_path, relpath)
   os.symlink(relpath, link_path)
 
 
 def _CreateSymlinkTree(entry_output_dir, symlink_dir, desired_files,
                        parent_dirs):
   """Creates a directory tree of symlinks to the given files.
 
   The idea here is to replicate a directory tree while leaving out files within
   it not listed by |desired_files|.
   """
   assert _IsSubpathOf(symlink_dir, entry_output_dir)
 
   for target_path in desired_files:
     prefix = next(d for d in parent_dirs if target_path.startswith(d))
     subpath = os.path.relpath(target_path, prefix)
     symlinked_path = os.path.join(symlink_dir, subpath)
     symlinked_dir = os.path.dirname(symlinked_path)
     if not os.path.exists(symlinked_dir):
       os.makedirs(symlinked_dir)
     _CreateRelativeSymlink(target_path, symlinked_path)
 
 
-def _CreateJavaSourceDir(output_dir, entry_output_dir, java_files):
-  """Computes and constructs when necessary the list of java source directories.
+def _CreateJavaSourceDir(output_dir, java_files):
+  """Computes the list of java source directories and exclude patterns.
 
   1. Computes the root java source directories from the list of files.
-  2. Determines whether there are any .java files in them that are not included
-     in |java_files|.
-  3. If not, returns the list of java source directories. If so, constructs a
-     tree of symlinks within |entry_output_dir| of all files in |java_files|.
+  2. Compute exclude patterns that exclude all extra files only.
+  3. Returns the list of java source directories and exclude patterns.
   """
   java_dirs = []
+  excludes = []
   if java_files:
     java_files = _RebasePath(java_files)
-    java_dirs = _ComputeJavaSourceDirs(java_files)
+    computed_dirs = _ComputeJavaSourceDirs(java_files)
+    java_dirs = computed_dirs.keys()
+    all_found_java_files = set()
 
-    found_java_files = build_utils.FindInDirectories(java_dirs, '*.java')
-    unwanted_java_files = set(found_java_files) - set(java_files)
-    missing_java_files = set(java_files) - set(found_java_files)
+    for directory, files in computed_dirs.iteritems():
+      found_java_files = build_utils.FindInDirectory(directory, '*.java')
+      all_found_java_files.update(found_java_files)
+      unwanted_java_files = set(found_java_files) - set(files)
+
+      if unwanted_java_files:
+        logging.debug('Target requires .java symlinks: %s, %s',

[agrieve, 2017/02/09 10:00:38]

nit: update log message to not say symlinks

[Peter Wen, 2017/02/09 14:38:28]

Done.

+            directory, files)
+        # Shorten exclude list by globbing if possible.

[agrieve, 2017/02/09 10:00:38]

Might be worth pulling this out into a helper function. I found it a bit hard to
follow with it nested here and doing very similar looking things.

[Peter Wen, 2017/02/09 14:38:28]

Done.

+        while unwanted_java_files:
+          unwanted_file = unwanted_java_files.pop()
+          target_exclude = os.path.join(
+              os.path.dirname(unwanted_file), '*.java')
+          found_files = glob.glob(target_exclude)
+          valid_files = set(found_files) & set(files)
+          if valid_files:
+            excludes.append(os.path.relpath(unwanted_file, directory))
+          else:
+            excludes.append(os.path.relpath(target_exclude, directory))
+            unwanted_java_files -= set(found_files)

[agrieve, 2017/02/09 10:00:38]

you convert found_files to a set twice. Maybe just declare it as one in the
glob() line?

[Peter Wen, 2017/02/09 14:38:28]

Done.

+
+    missing_java_files = set(java_files) - set(all_found_java_files)
     # Warn only about non-generated files that are missing.
     missing_java_files = [p for p in missing_java_files
                           if not p.startswith(output_dir)]
+    if missing_java_files:
+      logging.warning(
+          'Some java files were not found: %s', missing_java_files)
 
-    symlink_dir = os.path.join(entry_output_dir, _JAVA_SUBDIR)
-    shutil.rmtree(symlink_dir, True)
-
-    if unwanted_java_files:
-      logging.debug('Target requires .java symlinks: %s', entry_output_dir)
-      _CreateSymlinkTree(entry_output_dir, symlink_dir, java_files, java_dirs)
-      java_dirs = [symlink_dir]
-
-    if missing_java_files:
-      logging.warning('Some java files were not found: %s', missing_java_files)
-
-  return java_dirs
+  return java_dirs, excludes
 
 
 def _CreateJniLibsDir(output_dir, entry_output_dir, so_files):
   """Creates directory with symlinked .so files if necessary.
 
   Returns list of JNI libs directories."""
 
   if so_files:
     symlink_dir = os.path.join(entry_output_dir, _JNI_LIBS_SUBDIR)
     shutil.rmtree(symlink_dir, True)
@@ skipping 246 matching lines @@
     _ExtractSrcjars(generator.project_dir, srcjar_tuples)
 
   logging.warning('Project created! (%d subprojects)', len(project_entries))
   logging.warning('Generated projects work best with Android Studio 2.2')
   logging.warning('For more tips: https://chromium.googlesource.com/chromium'
                   '/src.git/+/master/docs/android_studio.md')
 
 
 if __name__ == '__main__':
   main()