generate_gradle.py: Allow multiple --target and add --all

build/android/gradle/generate_gradle.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generates an Android Studio project from a GN target."""
 
 import argparse
 import codecs
 import logging
 import os
+import re
 import shutil
 import subprocess
 import sys
 import zipfile
 
 _BUILD_ANDROID = os.path.join(os.path.dirname(__file__), os.pardir)
 sys.path.append(_BUILD_ANDROID)
 import devil_chromium
 from devil.utils import run_tests_helper
 from pylib import constants
 from pylib.constants import host_paths
 
 sys.path.append(os.path.join(_BUILD_ANDROID, 'gyp'))
 import jinja_template
 from util import build_utils
 
 
 _DEFAULT_ANDROID_MANIFEST_PATH = os.path.join(
     host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'AndroidManifest.xml')
 _JINJA_TEMPLATE_PATH = os.path.join(
     os.path.dirname(__file__), 'build.gradle.jinja')
 
 _JAVA_SUBDIR = 'symlinked-java'
 _SRCJARS_SUBDIR = 'extracted-srcjars'
 
+_DEFAULT_TARGETS = [
+    '//android_webview:system_webview_apk',
+    '//android_webview/test:android_webview_apk',
+    '//android_webview/test:android_webview_test_apk',
+    '//chrome/android:chrome_public_apk',
+    '//chrome/android:chrome_public_test_apk',
+    '//chrome/android:chrome_sync_shell_apk',
+    '//chrome/android:chrome_sync_shell_test_apk',
+]
 
 def _RebasePath(path_or_list, new_cwd=None, old_cwd=None):
   """Makes the given path(s) relative to new_cwd, or absolute if not specified.
 
   If new_cwd is not specified, absolute paths are returned.
   If old_cwd is not specified, constants.GetOutDirectory() is assumed.
   """
   if not isinstance(path_or_list, basestring):
     return [_RebasePath(p, new_cwd, old_cwd) for p in path_or_list]
   if old_cwd is None:
@@ skipping 13 matching lines @@
 def _WriteFile(path, data):
   """Writes |data| to |path|, constucting parent directories if necessary."""
   logging.info('Writing %s', path)
   dirname = os.path.dirname(path)
   if not os.path.exists(dirname):
     os.makedirs(dirname)
   with codecs.open(path, 'w', 'utf-8') as output_file:
     output_file.write(data)
 
 
-def _RunNinja(output_dir, ninja_targets):
+def _RunNinja(output_dir, args):
   cmd = ['ninja', '-C', output_dir, '-j50']
-  cmd.extend(ninja_targets)
+  cmd.extend(args)
   logging.info('Running: %r', cmd)
   subprocess.check_call(cmd)
 
 
+def _QueryForAllGnTargets(output_dir):
+  # Query ninja rather than GN since it's faster.
+  cmd = ['ninja', '-C', output_dir, '-t', 'targets']
+  logging.info('Running: %r', cmd)
+  ninja_output = build_utils.CheckOutput(cmd)
+  ret = []
+  SUFFIX_LEN = len('__build_config')
+  for line in ninja_output.splitlines():
+    ninja_target = line.rsplit(':', 1)[0]
+    # Ignore root aliases by ensure a : exists.
+    if ':' in ninja_target and ninja_target.endswith('__build_config'):
+      ret.append('//' + ninja_target[:-SUFFIX_LEN])
+  return ret
+
+
 class _ProjectEntry(object):
   """Helper class for various path transformations."""
   def __init__(self, gn_target):
     assert gn_target.startswith('//'), gn_target
     if ':' not in gn_target:
       gn_target = '%s:%s' % (gn_target, os.path.basename(gn_target))
     self._gn_target = gn_target
     self._build_config = None
 
   @classmethod
@@ skipping 70 matching lines @@
     subpath = os.path.relpath(target_path, prefix)
     symlinked_path = os.path.join(symlink_dir, subpath)
     symlinked_dir = os.path.dirname(symlinked_path)
     if not os.path.exists(symlinked_dir):
       os.makedirs(symlinked_dir)
     relpath = os.path.relpath(target_path, symlinked_dir)
     logging.debug('Creating symlink %s -> %s', symlinked_path, relpath)
     os.symlink(relpath, symlinked_path)
 
 
-def _CreateJavaSourceDir(entry_output_dir, java_sources_file):
+def _CreateJavaSourceDir(output_dir, entry_output_dir, java_sources_file):
   """Computes and constructs when necessary the list of java source directories.
 
   1. Computes the root java source directories from the list of files.
   2. Determines whether there are any .java files in them that are not included
      in |java_sources_file|.
   3. If not, returns the list of java source directories. If so, constructs a
      tree of symlinks within |entry_output_dir| of all files in
      |java_sources_file|.
   """
   java_dirs = []
   if java_sources_file:
     java_files = _RebasePath(build_utils.ReadSourcesList(java_sources_file))
     java_dirs = _ComputeJavaSourceDirs(java_files)
 
     found_java_files = build_utils.FindInDirectories(java_dirs, '*.java')
     unwanted_java_files = set(found_java_files) - set(java_files)
     missing_java_files = set(java_files) - set(found_java_files)
+    # Warn only about non-generated files that are missing.
+    missing_java_files = [p for p in missing_java_files
+                          if not p.startswith(output_dir)]
     if unwanted_java_files:
       logging.debug('Target requires .java symlinks: %s', entry_output_dir)
       symlink_dir = os.path.join(entry_output_dir, _JAVA_SUBDIR)
       _CreateSymlinkTree(entry_output_dir, symlink_dir, java_files, java_dirs)
       java_dirs = [symlink_dir]
+
     if missing_java_files:
       logging.warning('Some java files were not found: %s', missing_java_files)
 
   return java_dirs
 
 
 def _GenerateLocalProperties(sdk_dir):
   """Returns the data for project.properties as a string."""
   return '\n'.join([
       '# Generated by //build/android/gradle/generate_gradle.py',
       'sdk.dir=%s' % sdk_dir,
       ''])
 
 
 def _GenerateGradleFile(build_config, config_json, java_dirs, relativize,
-                        use_gradle_process_resources):
+                        use_gradle_process_resources, jinja_processor):
   """Returns the data for a project's build.gradle."""
   deps_info = build_config['deps_info']
   gradle = build_config['gradle']
 
   if deps_info['type'] == 'android_apk':
     target_type = 'android_apk'
   elif deps_info['type'] == 'java_library' and not deps_info['is_prebuilt']:
     if deps_info['requires_android']:
       target_type = 'android_library'
     else:
@@ skipping 12 matching lines @@
   variables['java_dirs'] = relativize(java_dirs)
   variables['prebuilts'] = relativize(gradle['dependent_prebuilt_jars'])
   deps = [_ProjectEntry.FromBuildConfigPath(p)
           for p in gradle['dependent_android_projects']]
 
   variables['android_project_deps'] = [d.ProjectName() for d in deps]
   deps = [_ProjectEntry.FromBuildConfigPath(p)
           for p in gradle['dependent_java_projects']]
   variables['java_project_deps'] = [d.ProjectName() for d in deps]
 
-  processor = jinja_template.JinjaProcessor(host_paths.DIR_SOURCE_ROOT)
-  return processor.Render(_JINJA_TEMPLATE_PATH, variables)
+  return jinja_processor.Render(_JINJA_TEMPLATE_PATH, variables)
 
 
-def _GenerateRootGradle():
+def _GenerateRootGradle(jinja_processor):
   """Returns the data for the root project's build.gradle."""
   variables = {'template_type': 'root'}
-  processor = jinja_template.JinjaProcessor(host_paths.DIR_SOURCE_ROOT)
-  return processor.Render(_JINJA_TEMPLATE_PATH, variables)
+  return jinja_processor.Render(_JINJA_TEMPLATE_PATH, variables)
 
 
 def _GenerateSettingsGradle(project_entries):
   """Returns the data for settings.gradle."""
   project_name = os.path.basename(os.path.dirname(host_paths.DIR_SOURCE_ROOT))
   lines = []
   lines.append('// Generated by //build/android/gradle/generate_gradle.py')
   lines.append('rootProject.name = "%s"' % project_name)
   lines.append('rootProject.projectDir = settingsDir')
   lines.append('')
@@ skipping 13 matching lines @@
     assert _IsSubpathOf(extracted_path, entry_output_dir)
     if os.path.exists(extracted_path):
       shutil.rmtree(extracted_path)
 
   for srcjar_path, extracted_path in srcjar_tuples:
     logging.info('Extracting %s to %s', srcjar_path, extracted_path)
     with zipfile.ZipFile(srcjar_path) as z:
       z.extractall(extracted_path)
 
 
-def _FindAllProjectEntries(main_entry):
+def _FindAllProjectEntries(main_entries):
   """Returns the list of all _ProjectEntry instances given the root project."""
   found = set()
-  to_scan = [main_entry]
+  to_scan = list(main_entries)
   while to_scan:
     cur_entry = to_scan.pop()
     if cur_entry in found:
       continue
     found.add(cur_entry)
     build_config = cur_entry.BuildConfig()
     sub_config_paths = build_config['deps_info']['deps_configs']
     to_scan.extend(
         _ProjectEntry.FromBuildConfigPath(p) for p in sub_config_paths)
   return list(found)
 
 
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('--output-directory',
                       help='Path to the root build directory.')
   parser.add_argument('-v',
                       '--verbose',
                       dest='verbose_count',
                       default=0,
                       action='count',
                       help='Verbose level')
   parser.add_argument('--target',
-                      help='GN target to generate project for.',
-                      default='//chrome/android:chrome_public_test_apk')
+                      dest='targets',
+                      action='append',
+                      help='GN target to generate project for. '
+                           'May be repeated.')
   parser.add_argument('--project-dir',
                       help='Root of the output project.',
                       default=os.path.join('$CHROMIUM_OUTPUT_DIR', 'gradle'))
+  parser.add_argument('--all',
+                      action='store_true',
+                      help='Generate all java targets (slows down IDE)')
   parser.add_argument('--use-gradle-process-resources',
                       action='store_true',
                       help='Have gradle generate R.java rather than ninja')
   args = parser.parse_args()
   if args.output_directory:
     constants.SetOutputDirectory(args.output_directory)
   constants.CheckOutputDirectory()
   output_dir = constants.GetOutDirectory()
   devil_chromium.Initialize(output_directory=output_dir)
   run_tests_helper.SetLogLevel(args.verbose_count)
 
   gradle_output_dir = os.path.abspath(
       args.project_dir.replace('$CHROMIUM_OUTPUT_DIR', output_dir))
   logging.warning('Creating project at: %s', gradle_output_dir)
 
-  target = args.target
-  # TODO(agrieve): See if it makes sense to utilize Gradle's test constructs for
-  # our instrumentation tests.
-  if target.endswith('_test_apk'):
-    target += '__apk'
-  main_entry = _ProjectEntry(target)
+  if args.all:
+    # Run GN gen if necessary (faster than running "gn gen" in the no-op case).
+    _RunNinja(output_dir, ['build.ninja'])
+    # Query ninja for all __build_config targets.
+    targets = _QueryForAllGnTargets(output_dir)
+  else:
+    targets = args.targets or _DEFAULT_TARGETS
+    # TODO(agrieve): See if it makes sense to utilize Gradle's test constructs
+    # for our instrumentation tests.
+    targets = [re.sub(r'_test_apk$', '_test_apk__apk', t) for t in targets]

[jbudorick, 2016/09/22 15:57:44]

just curious -- why the __apk target?

[agrieve, 2016/09/22 16:58:15]

That's the name of the android_apk() target forinstrumentation_test_apk()s:
https://cs.chromium.org/chromium/src/build/config/android/rules.gni?q=file:ru...

It'd actually probably make more sense to just change the template so that the
android_apk() wasn't a sub-target. Maybe in a follow-up :P

+
+  main_entries = [_ProjectEntry(t) for t in targets]
   logging.warning('Building .build_config files...')
-  _RunNinja(output_dir, [main_entry.NinjaBuildConfigTarget()])
+  _RunNinja(output_dir, [e.NinjaBuildConfigTarget() for e in main_entries])
 
-  all_entries = _FindAllProjectEntries(main_entry)
+  all_entries = _FindAllProjectEntries(main_entries)
   logging.info('Found %d dependent build_config targets.', len(all_entries))
 
+  logging.warning('Writing .gradle files...')
+  jinja_processor = jinja_template.JinjaProcessor(host_paths.DIR_SOURCE_ROOT)
   config_json = build_utils.ReadJson(
       os.path.join(output_dir, 'gradle', 'config.json'))
   project_entries = []
   srcjar_tuples = []
   for entry in all_entries:
     build_config = entry.BuildConfig()
     if build_config['deps_info']['type'] not in ('android_apk', 'java_library'):
       continue
 
     entry_output_dir = os.path.join(gradle_output_dir, entry.GradleSubdir())
     relativize = lambda x, d=entry_output_dir: _RebasePath(x, d)
 
     srcjars = _RebasePath(build_config['gradle'].get('bundled_srcjars', []))
     if not args.use_gradle_process_resources:
       srcjars += _RebasePath(build_config['javac']['srcjars'])
 
     java_sources_file = build_config['gradle'].get('java_sources_file')
     if java_sources_file:
       java_sources_file = _RebasePath(java_sources_file)
 
-    java_dirs = _CreateJavaSourceDir(entry_output_dir, java_sources_file)
+    java_dirs = _CreateJavaSourceDir(output_dir, entry_output_dir,
+                                     java_sources_file)
     if srcjars:
       java_dirs.append(os.path.join(entry_output_dir, _SRCJARS_SUBDIR))
 
     data = _GenerateGradleFile(build_config, config_json, java_dirs, relativize,
-                               args.use_gradle_process_resources)
+                               args.use_gradle_process_resources,
+                               jinja_processor)
     if data:
       project_entries.append(entry)
       srcjar_tuples.extend(
           (s, os.path.join(entry_output_dir, _SRCJARS_SUBDIR)) for s in srcjars)
       _WriteFile(os.path.join(entry_output_dir, 'build.gradle'), data)
 
   _WriteFile(os.path.join(gradle_output_dir, 'build.gradle'),
-             _GenerateRootGradle())
+             _GenerateRootGradle(jinja_processor))
 
   _WriteFile(os.path.join(gradle_output_dir, 'settings.gradle'),
              _GenerateSettingsGradle(project_entries))
 
   sdk_path = _RebasePath(config_json['android_sdk_root'])
   _WriteFile(os.path.join(gradle_output_dir, 'local.properties'),
              _GenerateLocalProperties(sdk_path))
 
   if srcjar_tuples:
     logging.warning('Building all .srcjar files...')
     targets = _RebasePath([s[0] for s in srcjar_tuples], output_dir)
     _RunNinja(output_dir, targets)
     _ExtractSrcjars(gradle_output_dir, srcjar_tuples)
   logging.warning('Project created successfully!')
   logging.warning('Generated projects work best with Android Studio 2.2')
   logging.warning('For more tips: https://chromium.googlesource.com/chromium'
                   '/src.git/+/master/docs/android_studio.md')
 
 
 if __name__ == '__main__':
   main()