Android: Cache android studio project entries

build/android/gradle/generate_gradle.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generates an Android Studio project from a GN target."""
 
 import argparse
 import codecs
 import glob
@@ skipping 99 matching lines @@
   ret = []
   SUFFIX_LEN = len('__build_config')
   for line in ninja_output.splitlines():
     ninja_target = line.rsplit(':', 1)[0]
     # Ignore root aliases by ensure a : exists.
     if ':' in ninja_target and ninja_target.endswith('__build_config'):
       ret.append('//' + ninja_target[:-SUFFIX_LEN])
   return ret
 
 
+def _Dedup(entries):

[agrieve, 2017/02/17 03:07:01]

nit: this doesn't save any lines over just calling _DedupAgainst() directly. I'd
remove it.

[Peter Wen, 2017/02/17 16:37:26]

Done.

+  """Return unique entries in iterable in the same order."""
+  return _DedupAgainst(entries, [])
+
+
+def _DedupAgainst(entries, existing_entries):
+  """Return unique entries in iterable given existing entries."""

[agrieve, 2017/02/17 03:07:01]

You return a list, so I'd say that rather than iterable.

I think the point of this is that it maintains the order of entries (or else you
could just do set(A) - set(B). So that's worth saying as well.

[Peter Wen, 2017/02/17 16:37:26]

Done.

+  seen = set(existing_entries)
+  unique_entries = []
+  for entry in entries:
+    if entry not in seen:
+      unique_entries.append(entry)
+      seen.add(entry)
+  return unique_entries
+
+
 class _ProjectEntry(object):
   """Helper class for project entries."""
+
+  _cached_entries = {}
+
   def __init__(self, gn_target):
+    # Use _ProjectEntry.FromGnTarget instead for caching.
+    self._gn_target = gn_target
+    self._build_config = None
+    self._java_files = None
+    self._all_entries = None
+    self.android_test_entry = None
+
+  @classmethod
+  def FromGnTarget(cls, gn_target):
     assert gn_target.startswith('//'), gn_target
     if ':' not in gn_target:
       gn_target = '%s:%s' % (gn_target, os.path.basename(gn_target))
-    self._gn_target = gn_target
-    self._build_config = None
-    self._java_files = None
-    self.android_test_entry = None
+    if gn_target not in cls._cached_entries:
+      cls._cached_entries[gn_target] = cls(gn_target)
+    return cls._cached_entries[gn_target]
 
   @classmethod
   def FromBuildConfigPath(cls, path):
     prefix = 'gen/'
     suffix = '.build_config'
     assert path.startswith(prefix) and path.endswith(suffix), path
     subdir = path[len(prefix):-len(suffix)]
-    return cls('//%s:%s' % (os.path.split(subdir)))
+    gn_target = '//%s:%s' % (os.path.split(subdir))
+    return cls.FromGnTarget(gn_target)
 
   def __hash__(self):
-    return hash(self._gn_target)
+    return hash(self.GnTarget())

[agrieve, 2017/02/17 03:07:01]

nit: I think it's generally better to leave these how they were. I don't see
what it buys us to go through a getter

[Peter Wen, 2017/02/17 16:37:25]

Done.

We don't do any checks/calculations in the getter, reverted.

 
   def __eq__(self, other):
-    return self._gn_target == other.GnTarget()
+    return self.GnTarget() == other.GnTarget()
 
   def GnTarget(self):
     return self._gn_target
 
   def NinjaTarget(self):
-    return self._gn_target[2:]
+    return self.GnTarget()[2:]
 
   def GnBuildConfigTarget(self):
-    return '%s__build_config' % self._gn_target
+    return '%s__build_config' % self.GnTarget()
 
   def NinjaBuildConfigTarget(self):
     return '%s__build_config' % self.NinjaTarget()
 
   def GradleSubdir(self):
     """Returns the output subdirectory."""
     return self.NinjaTarget().replace(':', os.path.sep)
 
   def ProjectName(self):
     """Returns the Gradle project name."""
@@ skipping 25 matching lines @@
   def JavaFiles(self):
     if self._java_files is None:
       java_sources_file = self.Gradle().get('java_sources_file')
       java_files = []
       if java_sources_file:
         java_sources_file = _RebasePath(java_sources_file)
         java_files = build_utils.ReadSourcesList(java_sources_file)
       self._java_files = java_files
     return self._java_files
 
+  def SubdirJavaFiles(self):

[agrieve, 2017/02/17 03:07:01]

Is this the same as saying generated java files? If so, I think that would be
clearer.

[Peter Wen, 2017/02/17 16:37:25]

Done.

+    return [p for p in self.JavaFiles() if not p.startswith('..')]
+
+  def PrebuiltJars(self):
+    return self.Gradle()['dependent_prebuilt_jars']
+
+  def AllEntries(self):

[agrieve, 2017/02/17 03:07:01]

This is worth adding pydoc for.

[Peter Wen, 2017/02/17 16:37:26]

Done.

+    if self._all_entries is None:
+      logging.debug('Generating entries for %s', self.GnTarget())
+      deps = [_ProjectEntry.FromBuildConfigPath(p)
+          for p in self.Gradle()['dependent_android_projects']]
+      deps.extend([_ProjectEntry.FromBuildConfigPath(p)

[agrieve, 2017/02/17 03:07:01]

nit: shouldn't need the [].

[Peter Wen, 2017/02/17 16:37:26]

Done.

+          for p in self.Gradle()['dependent_java_projects']])
+      all_entries = []
+      for dep in deps:
+        all_entries += dep.AllEntries()

[agrieve, 2017/02/17 03:07:01]

I think it might be easier to follow if you de-dupe as you go rather than adding
all and then removing again. E.g.:

for dep in deps:
  for e in dep.AllEntries():
    if e not in all_entries:
      all_entries.append(e)

Note: Doing an O(n) lookup in the list is fine, as n will be small.

[Peter Wen, 2017/02/17 16:37:25]

Switched to using sets instead, it's better to sort everything after
calculations for the generated build.gradle file to be nice (in-order dedup
doesn't work quite as nicely as I originally hoped).

+      all_entries.append(self)
+      self._all_entries = _Dedup(all_entries)
+    return self._all_entries
+
 
 class _ProjectContextGenerator(object):
   """Helper class to generate gradle build files"""
   def __init__(self, project_dir, build_vars, use_gradle_process_resources,
       jinja_processor):
     self.project_dir = project_dir
     self.build_vars = build_vars
     self.use_gradle_process_resources = use_gradle_process_resources
     self.jinja_processor = jinja_processor
 
   def _GenJniLibs(self, entry):
     native_section = entry.BuildConfig().get('native')
     if native_section:
       jni_libs = _CreateJniLibsDir(
           constants.GetOutDirectory(), self.EntryOutputDir(entry),
           native_section.get('libraries'))
     else:
       jni_libs = []
     return jni_libs
 
   def _GenJavaDirs(self, entry):
-    java_dirs, excludes = _CreateJavaSourceDir(
+    java_dirs, excludes = _ComputeJavaSourceDirsAndExcludes(
         constants.GetOutDirectory(), entry.JavaFiles())
     if self.Srcjars(entry):
       java_dirs.append(
           os.path.join(self.EntryOutputDir(entry), _SRCJARS_SUBDIR))
     return java_dirs, excludes
 
   def _GenResDirs(self, entry):
     res_dirs = list(entry.DepsInfo().get('owned_resources_dirs', []))
     if entry.ResZips():
       res_dirs.append(os.path.join(self.EntryOutputDir(entry), _RES_SUBDIR))
     return res_dirs
 
   def _GenCustomManifest(self, entry):
-    """Returns the path to the generated AndroidManifest.xml."""
-    javac = entry.Javac()
-    resource_packages = javac['resource_packages']
-    output_file = os.path.join(
-        self.EntryOutputDir(entry), 'AndroidManifest.xml')
+    """Returns the path to the generated AndroidManifest.xml.
 
+    Gradle uses package id from manifest when generating R.class. So, we need
+    to generate a custom manifest if we let gradle process resources. We cannot
+    simply set android.defaultConfig.applicationId because it is not supported
+    for library targets."""
+    resource_packages = entry.Javac().get('resource_packages')
     if not resource_packages:
-      logging.error('Target ' + entry.GnTarget() + ' includes resources from '
+      logging.debug('Target ' + entry.GnTarget() + ' includes resources from '
           'unknown package. Unable to process with gradle.')
       return _DEFAULT_ANDROID_MANIFEST_PATH
     elif len(resource_packages) > 1:
-      logging.error('Target ' + entry.GnTarget() + ' includes resources from '
+      logging.debug('Target ' + entry.GnTarget() + ' includes resources from '
           'multiple packages. Unable to process with gradle.')
       return _DEFAULT_ANDROID_MANIFEST_PATH
 
     variables = {}
     variables['compile_sdk_version'] = self.build_vars['android_sdk_version']
     variables['package'] = resource_packages[0]
 
+    output_file = os.path.join(
+        self.EntryOutputDir(entry), 'AndroidManifest.xml')
     data = self.jinja_processor.Render(_TemplatePath('manifest'), variables)
     _WriteFile(output_file, data)
 
     return output_file
 
   def _Relativize(self, entry, paths):
     return _RebasePath(paths, self.EntryOutputDir(entry))
 
   def EntryOutputDir(self, entry):
     return os.path.join(self.project_dir, entry.GradleSubdir())
 
   def Srcjars(self, entry):
     srcjars = _RebasePath(entry.Gradle().get('bundled_srcjars', []))
     if not self.use_gradle_process_resources:
       srcjars += _RebasePath(entry.BuildConfig()['javac']['srcjars'])
     return srcjars
 
   def GeneratedInputs(self, entry):
     generated_inputs = []
     generated_inputs.extend(self.Srcjars(entry))
     generated_inputs.extend(_RebasePath(entry.ResZips()))
-    generated_inputs.extend(
-        p for p in entry.JavaFiles() if not p.startswith('..'))
-    generated_inputs.extend(entry.Gradle()['dependent_prebuilt_jars'])
+    generated_inputs.extend(entry.SubdirJavaFiles())
+    generated_inputs.extend(entry.PrebuiltJars())
     return generated_inputs
 
   def Generate(self, entry):
     variables = {}
     java_dirs, excludes = self._GenJavaDirs(entry)
     variables['java_dirs'] = self._Relativize(entry, java_dirs)
     variables['java_excludes'] = excludes
     variables['jni_libs'] = self._Relativize(entry, self._GenJniLibs(entry))
     variables['res_dirs'] = self._Relativize(entry, self._GenResDirs(entry))
     android_manifest = entry.Gradle().get('android_manifest')
     if not android_manifest:
-      # Gradle uses package id from manifest when generating R.class. So, we
-      # need to generate a custom manifest if we let gradle process resources.
-      # We cannot simply set android.defaultConfig.applicationId because it is
-      # not supported for library targets.
-      if variables['res_dirs']:
-        android_manifest = self._GenCustomManifest(entry)
-      else:
-        android_manifest = _DEFAULT_ANDROID_MANIFEST_PATH
+      android_manifest = self._GenCustomManifest(entry)
     variables['android_manifest'] = self._Relativize(entry, android_manifest)
+    # TODO(agrieve): Add an option to use interface jars and see if that speeds
+    # things up at all.
+    variables['prebuilts'] = self._Relativize(entry, entry.PrebuiltJars())
     deps = [_ProjectEntry.FromBuildConfigPath(p)
             for p in entry.Gradle()['dependent_android_projects']]
     variables['android_project_deps'] = [d.ProjectName() for d in deps]
-    # TODO(agrieve): Add an option to use interface jars and see if that speeds
-    # things up at all.
-    variables['prebuilts'] = self._Relativize(
-        entry, entry.Gradle()['dependent_prebuilt_jars'])
     deps = [_ProjectEntry.FromBuildConfigPath(p)
             for p in entry.Gradle()['dependent_java_projects']]
     variables['java_project_deps'] = [d.ProjectName() for d in deps]
     return variables
 
 
 def _ComputeJavaSourceDirs(java_files):
   """Returns a dictionary of source dirs with each given files in one."""
   found_roots = {}
   for path in java_files:
@@ skipping 30 matching lines @@
     found_files = set(glob.glob(target_exclude))
     valid_files = found_files & files_to_include
     if valid_files:
       excludes.append(os.path.relpath(unwanted_file, parent_dir))
     else:
       excludes.append(os.path.relpath(target_exclude, parent_dir))
       files_to_exclude -= found_files
   return excludes
 
 
-def _CreateJavaSourceDir(output_dir, java_files):
+def _ComputeJavaSourceDirsAndExcludes(output_dir, java_files):
   """Computes the list of java source directories and exclude patterns.
 
   1. Computes the root java source directories from the list of files.
   2. Compute exclude patterns that exclude all extra files only.
   3. Returns the list of java source directories and exclude patterns.
   """
   java_dirs = []
   excludes = []
   if java_files:
     java_files = _RebasePath(java_files)
@@ skipping 90 matching lines @@
   variables['template_type'] = target_type
   variables['use_gradle_process_resources'] = (
       generator.use_gradle_process_resources)
   variables['build_tools_version'] = (
       build_vars['android_sdk_build_tools_version'])
   variables['compile_sdk_version'] = build_vars['android_sdk_version']
   variables['main'] = generator.Generate(entry)
   if entry.android_test_entry:
     variables['android_test'] = generator.Generate(
         entry.android_test_entry)
+    for key, value in variables['android_test'].iteritems():
+      if isinstance(value, list):
+        variables['android_test'][key] = _DedupAgainst(
+            value, variables['main'][key])
 
   return jinja_processor.Render(
       _TemplatePath(target_type.split('_')[0]), variables)
 
 
 def _GenerateRootGradle(jinja_processor):
   """Returns the data for the root project's build.gradle."""
   return jinja_processor.Render(_TemplatePath('root'))
 
 
@@ skipping 119 matching lines @@
     _RunNinja(constants.GetOutDirectory(), ['build.ninja'])
     # Query ninja for all __build_config targets.
     targets = _QueryForAllGnTargets(output_dir)
   else:
     targets = args.targets or _DEFAULT_TARGETS
     targets = [re.sub(r'_test_apk$', '_test_apk__apk', t) for t in targets]
     # TODO(wnwen): Utilize Gradle's test constructs for our junit tests?
     targets = [re.sub(r'_junit_tests$', '_junit_tests__java_binary', t)
                for t in targets]
 
-  main_entries = [_ProjectEntry(t) for t in targets]
+  main_entries = [_ProjectEntry.FromGnTarget(t) for t in targets]
 
   logging.warning('Building .build_config files...')
   _RunNinja(output_dir, [e.NinjaBuildConfigTarget() for e in main_entries])
 
   # There are many unused libraries, so restrict to those that are actually used
   # when using --all.
   if args.all:
     main_entries = [e for e in main_entries if e.GetType() == 'android_apk']
 
   all_entries = _FindAllProjectEntries(main_entries)
@@ skipping 42 matching lines @@
     _ExtractZips(generator.project_dir, zip_tuples)
 
   logging.warning('Project created! (%d subprojects)', len(project_entries))
   logging.warning('Generated projects work best with Android Studio 2.2')
   logging.warning('For more tips: https://chromium.googlesource.com/chromium'
                   '/src.git/+/master/docs/android_studio.md')
 
 
 if __name__ == '__main__':
   main()