Android: Android studio single module per target

build/android/gradle/generate_gradle.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generates an Android Studio project from a GN target."""
 
 import argparse
 import codecs
+import collections
 import glob
 import logging
 import os
 import re
 import shutil
 import subprocess
 import sys
 import zipfile
 
 _BUILD_ANDROID = os.path.join(os.path.dirname(__file__), os.pardir)
@@ skipping 170 matching lines @@
     return self.BuildConfig()['gradle']
 
   def Javac(self):
     return self.BuildConfig()['javac']
 
   def GetType(self):
     """Returns the target type from its .build_config."""
     return self.DepsInfo()['type']
 
   def ResZips(self):
-    return self.DepsInfo().get('owned_resources_zips')
+    return self.DepsInfo().get('owned_resources_zips', [])
+
+  def ResDirs(self):
+    return self.DepsInfo().get('owned_resources_dirs', [])
 
   def JavaFiles(self):
     if self._java_files is None:
       java_sources_file = self.Gradle().get('java_sources_file')
       java_files = []
       if java_sources_file:
         java_sources_file = _RebasePath(java_sources_file)
         java_files = build_utils.ReadSourcesList(java_sources_file)
       self._java_files = java_files
     return self._java_files
 
   def GeneratedJavaFiles(self):
     return [p for p in self.JavaFiles() if not p.startswith('..')]
 
   def PrebuiltJars(self):
-    return self.Gradle()['dependent_prebuilt_jars']
+    return self.Gradle().get('dependent_prebuilt_jars', [])
 
   def AllEntries(self):
     """Returns a list of all entries that the current entry depends on.
 
     This includes the entry itself to make iterating simpler."""
     if self._all_entries is None:
       logging.debug('Generating entries for %s', self.GnTarget())
       deps = [_ProjectEntry.FromBuildConfigPath(p)
           for p in self.Gradle()['dependent_android_projects']]
       deps.extend(_ProjectEntry.FromBuildConfigPath(p)
           for p in self.Gradle()['dependent_java_projects'])
       all_entries = set()
       for dep in deps:
         all_entries.update(dep.AllEntries())
       all_entries.add(self)
       self._all_entries = list(all_entries)
     return self._all_entries
 
 
 class _ProjectContextGenerator(object):
   """Helper class to generate gradle build files"""
   def __init__(self, project_dir, build_vars, use_gradle_process_resources,
-      jinja_processor):
+      jinja_processor, split_projects):
     self.project_dir = project_dir
     self.build_vars = build_vars
     self.use_gradle_process_resources = use_gradle_process_resources
     self.jinja_processor = jinja_processor
+    self.split_projects = split_projects
 
-  def _GenJniLibs(self, entry):
-    native_section = entry.BuildConfig().get('native')
-    if native_section:
-      jni_libs = _CreateJniLibsDir(
-          constants.GetOutDirectory(), self.EntryOutputDir(entry),
-          native_section.get('libraries'))
-    else:
-      jni_libs = []
-    return jni_libs
+  def _GenJniLibs(self, root_entry):
+    libraries = []
+    for entry in self._GetEntries(root_entry):
+      libraries += entry.BuildConfig().get('native', [])
+    if libraries:
+      return _CreateJniLibsDir(constants.GetOutDirectory(),
+          self.EntryOutputDir(root_entry), libraries)
+    return []
 
-  def _GenJavaDirs(self, entry):
+  def _GenJavaDirs(self, root_entry):
+    java_files = []
+    for entry in self._GetEntries(root_entry):
+      java_files += entry.JavaFiles()
     java_dirs, excludes = _ComputeJavaSourceDirsAndExcludes(
-        constants.GetOutDirectory(), entry.JavaFiles())
-    if self.Srcjars(entry):
-      java_dirs.append(
-          os.path.join(self.EntryOutputDir(entry), _SRCJARS_SUBDIR))
+        constants.GetOutDirectory(), java_files)
     return java_dirs, excludes
 
-  def _GenResDirs(self, entry):
-    res_dirs = list(entry.DepsInfo().get('owned_resources_dirs', []))
-    if entry.ResZips():
-      res_dirs.append(os.path.join(self.EntryOutputDir(entry), _RES_SUBDIR))
-    return res_dirs
-
   def _GenCustomManifest(self, entry):
     """Returns the path to the generated AndroidManifest.xml.
 
     Gradle uses package id from manifest when generating R.class. So, we need
     to generate a custom manifest if we let gradle process resources. We cannot
     simply set android.defaultConfig.applicationId because it is not supported
     for library targets."""
     resource_packages = entry.Javac().get('resource_packages')
     if not resource_packages:
       logging.debug('Target ' + entry.GnTarget() + ' includes resources from '
@@ skipping 11 matching lines @@
     output_file = os.path.join(
         self.EntryOutputDir(entry), 'AndroidManifest.xml')
     data = self.jinja_processor.Render(_TemplatePath('manifest'), variables)
     _WriteFile(output_file, data)
 
     return output_file
 
   def _Relativize(self, entry, paths):
     return _RebasePath(paths, self.EntryOutputDir(entry))
 
-  def EntryOutputDir(self, entry):
-    return os.path.join(self.project_dir, entry.GradleSubdir())
-
-  def Srcjars(self, entry):
+  def _Srcjars(self, entry):
     srcjars = _RebasePath(entry.Gradle().get('bundled_srcjars', []))
     if not self.use_gradle_process_resources:
       srcjars += _RebasePath(entry.BuildConfig()['javac']['srcjars'])
     return srcjars
 
-  def GeneratedInputs(self, entry):
-    generated_inputs = []
-    generated_inputs.extend(self.Srcjars(entry))
-    generated_inputs.extend(_RebasePath(entry.ResZips()))
-    generated_inputs.extend(entry.GeneratedJavaFiles())
-    generated_inputs.extend(entry.PrebuiltJars())
-    return generated_inputs
+  def _GetEntries(self, entry):
+    if self.split_projects:
+      return [entry]
+    return entry.AllEntries()
 
-  def Generate(self, entry):
-    variables = {}
-    java_dirs, excludes = self._GenJavaDirs(entry)
-    variables['java_dirs'] = self._Relativize(entry, java_dirs)
-    variables['java_excludes'] = excludes
-    variables['jni_libs'] = self._Relativize(entry, self._GenJniLibs(entry))
-    variables['res_dirs'] = self._Relativize(entry, self._GenResDirs(entry))
-    android_manifest = entry.Gradle().get('android_manifest')
-    if not android_manifest:
-      android_manifest = self._GenCustomManifest(entry)
-    variables['android_manifest'] = self._Relativize(entry, android_manifest)
+  def EntryOutputDir(self, entry):
+    return os.path.join(self.project_dir, entry.GradleSubdir())
+
+  def AllSrcjars(self, root_entry):
+    srcjars = []
+    for entry in self._GetEntries(root_entry):
+      srcjars += self._Srcjars(entry)
+    return set(srcjars)
+
+  def AllResZips(self, root_entry):
+    res_zips = []
+    for entry in self._GetEntries(root_entry):
+      res_zips += entry.ResZips()
+    return set(_RebasePath(res_zips))
+
+  def GeneratedInputs(self, root_entry):
+    generated_inputs = set(self.AllResZips(root_entry))
+    generated_inputs.update(self.AllSrcjars(root_entry))
+    for entry in self._GetEntries(root_entry):
+      generated_inputs.update(entry.GeneratedJavaFiles())
+      generated_inputs.update(entry.PrebuiltJars())
+    return set(generated_inputs)
+
+  def Generate(self, root_entry):
     # TODO(agrieve): Add an option to use interface jars and see if that speeds
     # things up at all.
-    variables['prebuilts'] = self._Relativize(entry, entry.PrebuiltJars())
-    deps = [_ProjectEntry.FromBuildConfigPath(p)
-            for p in entry.Gradle()['dependent_android_projects']]
-    variables['android_project_deps'] = [d.ProjectName() for d in deps]
-    deps = [_ProjectEntry.FromBuildConfigPath(p)
-            for p in entry.Gradle()['dependent_java_projects']]
-    variables['java_project_deps'] = [d.ProjectName() for d in deps]
+    variables = collections.defaultdict(list)

[agrieve, 2017/02/21 17:52:05]

nit: some variables can be non-lists, so using a defaultdict doesn't make a lot
of sense to me. Seems you could change most of the +=s to just =s anyways?

+    java_dirs, excludes = self._GenJavaDirs(root_entry)
+    java_dirs.sort()
+    variables['java_dirs'] += self._Relativize(root_entry, java_dirs)
+    variables['java_dirs'].append(_SRCJARS_SUBDIR)
+    variables['java_excludes'] += excludes
+    variables['jni_libs'] += self._Relativize(
+        root_entry, set(self._GenJniLibs(root_entry)))
+    for entry in self._GetEntries(root_entry):
+      variables['prebuilts'] += entry.PrebuiltJars()
+      variables['res_dirs'] += entry.ResDirs()
+    variables['prebuilts'] = self._Relativize(
+        root_entry, set(variables['prebuilts']))
+    variables['res_dirs'] = self._Relativize(
+        root_entry, set(variables['res_dirs']))
+    variables['res_dirs'].append(_RES_SUBDIR)
+    android_manifest = root_entry.Gradle().get('android_manifest')
+    if not android_manifest:
+      android_manifest = self._GenCustomManifest(root_entry)
+    variables['android_manifest'] = self._Relativize(
+        root_entry, android_manifest)
+    if self.split_projects:
+      deps = [_ProjectEntry.FromBuildConfigPath(p)
+              for p in root_entry.Gradle()['dependent_android_projects']]
+      variables['android_project_deps'] = [d.ProjectName() for d in deps]
+      deps = [_ProjectEntry.FromBuildConfigPath(p)
+              for p in root_entry.Gradle()['dependent_java_projects']]
+      variables['java_project_deps'] = [d.ProjectName() for d in deps]
     return variables
 
 
 def _ComputeJavaSourceDirs(java_files):
   """Returns a dictionary of source dirs with each given files in one."""
   found_roots = {}
   for path in java_files:
     path_root = path
     # Recognize these tokens as top-level.
     while True:
@@ skipping 254 matching lines @@
                       dest='targets',
                       action='append',
                       help='GN target to generate project for. '
                            'May be repeated.')
   parser.add_argument('--project-dir',
                       help='Root of the output project.',
                       default=os.path.join('$CHROMIUM_OUTPUT_DIR', 'gradle'))
   parser.add_argument('--all',
                       action='store_true',
                       help='Generate all java targets (slows down IDE)')
   parser.add_argument('--use-gradle-process-resources',

[agrieve, 2017/02/21 17:52:05]

I'm guessing that non-split projects will currently break with
--use-gradle-process-resources because of the issue of resource package names.
If so, maybe add an assert that --split-projects is always used when
--use-gradle-process-resources is used (and maybe a TODO to fix it up)

                       action='store_true',
                       help='Have gradle generate R.java rather than ninja')
+  parser.add_argument('--split-projects',
+                      action='store_true',
+                      help='Split projects by their gn deps rather than '
+                           'combining all the dependencies of each target')
   args = parser.parse_args()
   if args.output_directory:
     constants.SetOutputDirectory(args.output_directory)
   constants.CheckOutputDirectory()
   output_dir = constants.GetOutDirectory()
   devil_chromium.Initialize(output_directory=output_dir)
   run_tests_helper.SetLogLevel(args.verbose_count)
 
   _gradle_output_dir = os.path.abspath(
       args.project_dir.replace('$CHROMIUM_OUTPUT_DIR', output_dir))
   jinja_processor = jinja_template.JinjaProcessor(_FILE_DIR)
   build_vars = _ReadBuildVars(output_dir)
   generator = _ProjectContextGenerator(_gradle_output_dir, build_vars,
-      args.use_gradle_process_resources, jinja_processor)
+      args.use_gradle_process_resources, jinja_processor, args.split_projects)
   logging.warning('Creating project at: %s', generator.project_dir)
 
   if args.all:
     # Run GN gen if necessary (faster than running "gn gen" in the no-op case).
     _RunNinja(constants.GetOutDirectory(), ['build.ninja'])
     # Query ninja for all __build_config targets.
     targets = _QueryForAllGnTargets(output_dir)
   else:
     targets = args.targets or _DEFAULT_TARGETS
     targets = [re.sub(r'_test_apk$', '_test_apk__apk', t) for t in targets]
     # TODO(wnwen): Utilize Gradle's test constructs for our junit tests?
     targets = [re.sub(r'_junit_tests$', '_junit_tests__java_binary', t)
                for t in targets]
 
   main_entries = [_ProjectEntry.FromGnTarget(t) for t in targets]
 
   logging.warning('Building .build_config files...')
   _RunNinja(output_dir, [e.NinjaBuildConfigTarget() for e in main_entries])
 
   # There are many unused libraries, so restrict to those that are actually used
   # when using --all.
   if args.all:
     main_entries = [e for e in main_entries if e.GetType() == 'android_apk']
 
-  all_entries = _FindAllProjectEntries(main_entries)
-  logging.info('Found %d dependent build_config targets.', len(all_entries))
-  entries = _CombineTestEntries(all_entries)
+  if args.split_projects:
+    main_entries = _FindAllProjectEntries(main_entries)
+    logging.info('Found %d dependent build_config targets.', len(main_entries))
+  entries = _CombineTestEntries(main_entries)
   logging.info('Creating %d projects for targets.', len(entries))
 
   logging.warning('Writing .gradle files...')
   project_entries = []
   zip_tuples = []
   generated_inputs = []
   for entry in entries:
     if entry.GetType() not in ('android_apk', 'java_library', 'java_binary'):
       continue
 
     data = _GenerateGradleFile(entry, generator, build_vars, jinja_processor)
     if data:
       project_entries.append(entry)
       # Build all paths references by .gradle that exist within output_dir.
       generated_inputs.extend(generator.GeneratedInputs(entry))
       zip_tuples.extend(
           (s, os.path.join(generator.EntryOutputDir(entry), _SRCJARS_SUBDIR))
-          for s in generator.Srcjars(entry))
+          for s in generator.AllSrcjars(entry))
       zip_tuples.extend(
           (s, os.path.join(generator.EntryOutputDir(entry), _RES_SUBDIR))
-          for s in _RebasePath(entry.ResZips()))
+          for s in generator.AllResZips(entry))
       _WriteFile(
           os.path.join(generator.EntryOutputDir(entry), 'build.gradle'), data)
 
   _WriteFile(os.path.join(generator.project_dir, 'build.gradle'),
              _GenerateRootGradle(jinja_processor))
 
   _WriteFile(os.path.join(generator.project_dir, 'settings.gradle'),
              _GenerateSettingsGradle(project_entries))
 
   sdk_path = _RebasePath(build_vars['android_sdk_root'])
   _WriteFile(os.path.join(generator.project_dir, 'local.properties'),
              _GenerateLocalProperties(sdk_path))
 
   if generated_inputs:
     logging.warning('Building generated source files...')
     targets = _RebasePath(generated_inputs, output_dir)
     _RunNinja(output_dir, targets)
 
   if zip_tuples:
     _ExtractZips(generator.project_dir, zip_tuples)
 
   logging.warning('Project created! (%d subprojects)', len(project_entries))
   logging.warning('Generated projects work best with Android Studio 2.2')
   logging.warning('For more tips: https://chromium.googlesource.com/chromium'
                   '/src.git/+/master/docs/android_studio.md')
 
 
 if __name__ == '__main__':
   main()