generate_gradle.py: Allow multiple --target and add --all

build/android/gradle/generate_gradle.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generates an Android Studio project from a GN target."""
 
 import argparse
 import codecs
 import logging
@@ skipping 49 matching lines @@
 def _WriteFile(path, data):
   """Writes |data| to |path|, constucting parent directories if necessary."""
   logging.info('Writing %s', path)
   dirname = os.path.dirname(path)
   if not os.path.exists(dirname):
     os.makedirs(dirname)
   with codecs.open(path, 'w', 'utf-8') as output_file:
     output_file.write(data)
 
 
-def _RunNinja(output_dir, ninja_targets):
+def _RunNinja(output_dir, args):
   cmd = ['ninja', '-C', output_dir, '-j50']
-  cmd.extend(ninja_targets)
+  cmd.extend(args)
+  if len(args) < 10:

[jbudorick, 2016/09/22 02:34:46]

Why < 10? What about only logging part of cmd if it's too big?

[agrieve, 2016/09/22 15:51:25]

meh, removed the check. It shows only when running with -v, and it's not *that*
bad (~250 args)

+    logging.info('Running: %r', cmd)
+  subprocess.check_call(cmd)
+
+
+def _QueryForAllGnTargets(output_dir):
+  cmd = ['ninja', '-C', output_dir, '-t', 'targets']
   logging.info('Running: %r', cmd)
-  subprocess.check_call(cmd)
+  ninja_output = build_utils.CheckOutput(cmd)
+  ret = []
+  SUFFIX_LEN = len('__build_config')
+  for line in ninja_output.splitlines():
+    ninja_target = line.rsplit(':', 1)[0]
+    # Ignore root aliases by ensure a : exists.
+    if ':' in ninja_target and ninja_target.endswith('__build_config'):

[jbudorick, 2016/09/22 02:34:46]

Why is this checking for ':'? We'd never hit a __build_config that doesn't have
one, and it seems like it takes more time to check than it'd take for
endswith...

[agrieve, 2016/09/22 15:51:25]

It's because we're querying for ninja rather than GN for the list of targets
(added a comment to the top that we do this because it's faster). GN creates
non-fully-qualified aliases for unique target names (which is why you can "ninja
chrome_apk". Ensuring there's a : in it ensures we're skipping over these
aliases.

[jbudorick, 2016/09/22 15:57:43]

Ah, yeah, I was thinking about the GN targets.

+      ret.append('//' + ninja_target[:-SUFFIX_LEN])
+  return ret
 
 
 class _ProjectEntry(object):
   """Helper class for various path transformations."""
   def __init__(self, gn_target):
     assert gn_target.startswith('//'), gn_target
     if ':' not in gn_target:
       gn_target = '%s:%s' % (gn_target, os.path.basename(gn_target))
     self._gn_target = gn_target
     self._build_config = None
@@ skipping 72 matching lines @@
     subpath = os.path.relpath(target_path, prefix)
     symlinked_path = os.path.join(symlink_dir, subpath)
     symlinked_dir = os.path.dirname(symlinked_path)
     if not os.path.exists(symlinked_dir):
       os.makedirs(symlinked_dir)
     relpath = os.path.relpath(target_path, symlinked_dir)
     logging.debug('Creating symlink %s -> %s', symlinked_path, relpath)
     os.symlink(relpath, symlinked_path)
 
 
-def _CreateJavaSourceDir(entry_output_dir, java_sources_file):
+def _CreateJavaSourceDir(output_dir, entry_output_dir, java_sources_file):
   """Computes and constructs when necessary the list of java source directories.
 
   1. Computes the root java source directories from the list of files.
   2. Determines whether there are any .java files in them that are not included
      in |java_sources_file|.
   3. If not, returns the list of java source directories. If so, constructs a
      tree of symlinks within |entry_output_dir| of all files in
      |java_sources_file|.
   """
   java_dirs = []
   if java_sources_file:
     java_files = _RebasePath(build_utils.ReadSourcesList(java_sources_file))
     java_dirs = _ComputeJavaSourceDirs(java_files)
 
     found_java_files = build_utils.FindInDirectories(java_dirs, '*.java')
     unwanted_java_files = set(found_java_files) - set(java_files)
     missing_java_files = set(java_files) - set(found_java_files)
+    # Warn only about non-generated files that are missing.
+    missing_java_files = [p for p in missing_java_files
+                          if not p.startswith(output_dir)]
     if unwanted_java_files:
       logging.debug('Target requires .java symlinks: %s', entry_output_dir)
       symlink_dir = os.path.join(entry_output_dir, _JAVA_SUBDIR)
       _CreateSymlinkTree(entry_output_dir, symlink_dir, java_files, java_dirs)
       java_dirs = [symlink_dir]
+
     if missing_java_files:
       logging.warning('Some java files were not found: %s', missing_java_files)
 
   return java_dirs
 
 
 def _GenerateLocalProperties(sdk_dir):
   """Returns the data for project.properties as a string."""
   return '\n'.join([
       '# Generated by //build/android/gradle/generate_gradle.py',
       'sdk.dir=%s' % sdk_dir,
       ''])
 
 
 def _GenerateGradleFile(build_config, config_json, java_dirs, relativize,
-                        use_gradle_process_resources):
+                        use_gradle_process_resources, jinja_processor):
   """Returns the data for a project's build.gradle."""
   deps_info = build_config['deps_info']
   gradle = build_config['gradle']
 
   if deps_info['type'] == 'android_apk':
     target_type = 'android_apk'
   elif deps_info['type'] == 'java_library' and not deps_info['is_prebuilt']:
     if deps_info['requires_android']:
       target_type = 'android_library'
     else:
@@ skipping 12 matching lines @@
   variables['java_dirs'] = relativize(java_dirs)
   variables['prebuilts'] = relativize(gradle['dependent_prebuilt_jars'])
   deps = [_ProjectEntry.FromBuildConfigPath(p)
           for p in gradle['dependent_android_projects']]
 
   variables['android_project_deps'] = [d.ProjectName() for d in deps]
   deps = [_ProjectEntry.FromBuildConfigPath(p)
           for p in gradle['dependent_java_projects']]
   variables['java_project_deps'] = [d.ProjectName() for d in deps]
 
-  processor = jinja_template.JinjaProcessor(host_paths.DIR_SOURCE_ROOT)
-  return processor.Render(_JINJA_TEMPLATE_PATH, variables)
+  return jinja_processor.Render(_JINJA_TEMPLATE_PATH, variables)
 
 
-def _GenerateRootGradle():
+def _GenerateRootGradle(jinja_processor):
   """Returns the data for the root project's build.gradle."""
   variables = {'template_type': 'root'}
-  processor = jinja_template.JinjaProcessor(host_paths.DIR_SOURCE_ROOT)
-  return processor.Render(_JINJA_TEMPLATE_PATH, variables)
+  return jinja_processor.Render(_JINJA_TEMPLATE_PATH, variables)
 
 
 def _GenerateSettingsGradle(project_entries):
   """Returns the data for settings.gradle."""
   project_name = os.path.basename(os.path.dirname(host_paths.DIR_SOURCE_ROOT))
   lines = []
   lines.append('// Generated by //build/android/gradle/generate_gradle.py')
   lines.append('rootProject.name = "%s"' % project_name)
   lines.append('rootProject.projectDir = settingsDir')
   lines.append('')
@@ skipping 13 matching lines @@
     assert _IsSubpathOf(extracted_path, entry_output_dir)
     if os.path.exists(extracted_path):
       shutil.rmtree(extracted_path)
 
   for srcjar_path, extracted_path in srcjar_tuples:
     logging.info('Extracting %s to %s', srcjar_path, extracted_path)
     with zipfile.ZipFile(srcjar_path) as z:
       z.extractall(extracted_path)
 
 
-def _FindAllProjectEntries(main_entry):
+def _FindAllProjectEntries(main_entries):
   """Returns the list of all _ProjectEntry instances given the root project."""
   found = set()
-  to_scan = [main_entry]
+  to_scan = list(main_entries)
   while to_scan:
     cur_entry = to_scan.pop()
     if cur_entry in found:
       continue
     found.add(cur_entry)
     build_config = cur_entry.BuildConfig()
     sub_config_paths = build_config['deps_info']['deps_configs']
     to_scan.extend(
         _ProjectEntry.FromBuildConfigPath(p) for p in sub_config_paths)
   return list(found)
 
 
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('--output-directory',
                       help='Path to the root build directory.')
   parser.add_argument('-v',
                       '--verbose',
                       dest='verbose_count',
                       default=0,
                       action='count',
                       help='Verbose level')
   parser.add_argument('--target',
-                      help='GN target to generate project for.',
-                      default='//chrome/android:chrome_public_apk')
+                      dest='targets',
+                      action='append',
+                      help='GN target to generate project for. '
+                           'May be repeated.',
+                      default=['//chrome/android:chrome_public_test_apk'])
   parser.add_argument('--project-dir',
                       help='Root of the output project.',
                       default=os.path.join('$CHROMIUM_OUTPUT_DIR', 'gradle'))
+  parser.add_argument('--all',
+                      action='store_true',
+                      help='Generate all java targets (slows down IDE)')
   parser.add_argument('--use-gradle-process-resources',
                       action='store_true',
                       help='Have gradle generate R.java rather than ninja')
   args = parser.parse_args()
   if args.output_directory:
     constants.SetOutputDirectory(args.output_directory)
   constants.CheckOutputDirectory()
   output_dir = constants.GetOutDirectory()
   devil_chromium.Initialize(output_directory=output_dir)
   run_tests_helper.SetLogLevel(args.verbose_count)
 
   gradle_output_dir = os.path.abspath(
       args.project_dir.replace('$CHROMIUM_OUTPUT_DIR', output_dir))
   logging.warning('Creating project at: %s', gradle_output_dir)
 
-  main_entry = _ProjectEntry(args.target)
+  if args.all:
+    # Run GN gen if necessary.
+    _RunNinja(output_dir, ['build.ninja'])

[jbudorick, 2016/09/22 02:34:46]

This is ... building build.ninja? I take it that's a no-op?

If you want to run gn, why not just run gn directly?

[agrieve, 2016/09/22 15:51:25]

Added a comment - it's faster than running gn gen in the no-op case. 

[jbudorick, 2016/09/22 15:57:43]

ah, sgtm

+    # Query ninja for all __build_config targets.
+    targets = _QueryForAllGnTargets(output_dir)
+  else:
+    targets = args.targets
+
+  main_entries = [_ProjectEntry(t) for t in targets]
   logging.warning('Building .build_config files...')
-  _RunNinja(output_dir, [main_entry.NinjaBuildConfigTarget()])
+  _RunNinja(output_dir, [e.NinjaBuildConfigTarget() for e in main_entries])
 
-  all_entries = _FindAllProjectEntries(main_entry)
+  all_entries = _FindAllProjectEntries(main_entries)
   logging.info('Found %d dependent build_config targets.', len(all_entries))
 
+  jinja_processor = jinja_template.JinjaProcessor(host_paths.DIR_SOURCE_ROOT)
   config_json = build_utils.ReadJson(
       os.path.join(output_dir, 'gradle', 'config.json'))
   project_entries = []
   srcjar_tuples = []
   for entry in all_entries:
     build_config = entry.BuildConfig()
     if build_config['deps_info']['type'] not in ('android_apk', 'java_library'):
       continue
 
     entry_output_dir = os.path.join(gradle_output_dir, entry.GradleSubdir())
     relativize = lambda x, d=entry_output_dir: _RebasePath(x, d)
 
     srcjars = _RebasePath(build_config['gradle'].get('bundled_srcjars', []))
     if not args.use_gradle_process_resources:
       srcjars += _RebasePath(build_config['javac']['srcjars'])
 
     java_sources_file = build_config['gradle'].get('java_sources_file')
     if java_sources_file:
       java_sources_file = _RebasePath(java_sources_file)
 
-    java_dirs = _CreateJavaSourceDir(entry_output_dir, java_sources_file)
+    java_dirs = _CreateJavaSourceDir(output_dir, entry_output_dir,
+                                     java_sources_file)
     if srcjars:
       java_dirs.append(os.path.join(entry_output_dir, _SRCJARS_SUBDIR))
 
     data = _GenerateGradleFile(build_config, config_json, java_dirs, relativize,
-                               args.use_gradle_process_resources)
+                               args.use_gradle_process_resources,
+                               jinja_processor)
     if data:
       project_entries.append(entry)
       srcjar_tuples.extend(
           (s, os.path.join(entry_output_dir, _SRCJARS_SUBDIR)) for s in srcjars)
       _WriteFile(os.path.join(entry_output_dir, 'build.gradle'), data)
 
   _WriteFile(os.path.join(gradle_output_dir, 'build.gradle'),
-             _GenerateRootGradle())
+             _GenerateRootGradle(jinja_processor))
 
   _WriteFile(os.path.join(gradle_output_dir, 'settings.gradle'),
              _GenerateSettingsGradle(project_entries))
 
   sdk_path = _RebasePath(config_json['android_sdk_root'])
   _WriteFile(os.path.join(gradle_output_dir, 'local.properties'),
              _GenerateLocalProperties(sdk_path))
 
   if srcjar_tuples:
     logging.warning('Building all .srcjar files...')
     targets = _RebasePath([s[0] for s in srcjar_tuples], output_dir)
     _RunNinja(output_dir, targets)
     _ExtractSrcjars(gradle_output_dir, srcjar_tuples)
   logging.warning('Project created successfully!')
   logging.warning('Generated projects work best with Android Studio 2.2')
   logging.warning('For more tips: https://chromium.googlesource.com/chromium'
                   '/src.git/+/master/docs/android_studio.md')
 
 
 if __name__ == '__main__':
   main()