🎧 Initial version of Android Studio project generation

build/android/generate_gradle.py

+#!/usr/bin/env python
+# Copyright 2016 The Chromium Authors. All rights reserved.

[jbudorick, 2016/07/07 22:32:29]

Again, //build/android/gradle/?

[agrieve, 2016/07/08 19:07:28]

Done.

+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Generates an Android Studio project from a GN target."""
+
+import argparse
+import codecs
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import zipfile
+
+from pylib import constants
+from pylib.constants import host_paths
+from pylib.utils import run_tests_helper
+
+sys.path.append(os.path.join(os.path.dirname(__file__), 'gyp'))
+import jinja_template
+from util import build_utils
+
+
+_JINJA_TEMPLATE_PATH = os.path.join(

[jbudorick, 2016/07/07 22:32:29]

nit: sort

[agrieve, 2016/07/08 19:07:28]

Done.

+    os.path.dirname(__file__), 'build.gradle.jinja')
+_DEFAULT_ANDROID_MANIFEST_PATH = os.path.join(
+    host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'AndroidManifest.xml')
+_SRCJARS_SUBDIR = 'extracted-srcjars'
+_JAVA_SUBDIR = 'symlinked-java'
+
+
+def _WriteFile(path, data):
+  logging.info('Writing %s', path)
+  dirname = os.path.dirname(path)
+  if not os.path.exists(dirname):
+    os.makedirs(dirname)
+  with codecs.open(path, 'w', 'utf-8') as output_file:
+    output_file.write(data)
+
+
+def _RunNinja(output_dir, ninja_targets):
+  cmd = ['ninja', '-C', output_dir, '-j50']
+  cmd.extend(ninja_targets)
+  logging.info('Running: %r', cmd)
+  subprocess.check_call(cmd)
+
+
+class _ProjectEntry(object):
+  """Helper class for various path transformations."""
+  def __init__(self, gn_target):
+    assert gn_target.startswith('//') and ':' in gn_target, gn_target
+    self._gn_target = gn_target
+
+  @classmethod
+  def FromBuildConfigPath(cls, path):
+    prefix = 'gen/'
+    suffix = '.build_config'
+    assert path.startswith(prefix) and path.endswith(suffix), path
+    subdir = path[len(prefix):-len(suffix)]
+    return cls('//%s:%s' % (os.path.split(subdir)))
+
+  def GnTarget(self):
+    return self._gn_target
+
+  def NinjaTarget(self):
+    return self._gn_target[2:]
+
+  def GnBuildConfigTarget(self):
+    return '%s__build_config' % self._gn_target
+
+  def NinjaBuildConfigTarget(self):
+    return '%s__build_config' % self.NinjaTarget()
+
+  def GradleSubdir(self):
+    """Returns the output subdirectory."""
+    return self.NinjaTarget().replace(':', os.path.sep)
+
+  def BuildConfigPath(self):
+    """Returns the relative path to the .build_config"""
+    return os.path.join('gen', self.GradleSubdir() + '.build_config')
+
+  def ProjectName(self):
+    """Returns the Gradle project name."""
+    return self.GradleSubdir().replace(os.path.sep, '\\$')
+
+
+def _ComputeJavaSourceDirs(java_files):
+  """Returns the list of source directories for the given files."""
+  found_roots = set()
+  for path in java_files:
+    path_root = path
+    # Recognize these tokens as top-level.
+    while os.path.basename(path_root) not in ('javax', 'org', 'com'):

[jbudorick, 2016/07/07 22:32:30]

nit: I think this is a bit simpler than while/break/else:

  while os.path.basename(path_root) not in ('com', 'javax', 'org', 'src'):
    assert path_root
    path_root = os.path.dirname(path_root)
  if os.path.basename(path_root) != 'src':
    path_root = os.path.dirname(path_root)

  

[agrieve, 2016/07/08 19:07:28]

Done.

+      assert path_root, 'Failed to find source dir for ' + path
+      # Assume that if we've hit "src", the we're at the root.
+      if os.path.basename(path_root) == 'src':
+        break
+      path_root = os.path.dirname(path_root)
+    else:
+      path_root = os.path.dirname(path_root)
+    found_roots.add(path_root)
+  return list(found_roots)
+
+
+def _CreateSymlinkTree(output_dir, symlink_dir, desired_files, parent_dirs):
+  """Creates a directory tree of symlinks to the given files.
+
+  The idea here is to replicate a directory tree while leaving out files within
+  it not listed by |desired_files|.
+  """
+  if os.path.exists(symlink_dir):
+    shutil.rmtree(symlink_dir)

[jbudorick, 2016/07/07 22:32:30]

I would expect this case to be an error. We could possibly nuke directories
behind a flag, but I don't think that should be the default behavior.

[agrieve, 2016/07/08 19:07:28]

It happens when regenerating. There are .idea files in the top-level project dir
once you get going that Android Studio doesn't like being removed, so we can't
just delete the entire output directory. It can also cause excessive re-sync
times if you delete all of the build/ directories.

Added an assert to at least make sure we're in the project directory.

+
+  for target_path in desired_files:

[jbudorick, 2016/07/07 22:32:29]

Are the target_paths relative or absolute?

[agrieve, 2016/07/08 19:07:28]

Reworked this to be less brain-hurting.

+    prefix = next(d for d in parent_dirs if target_path.startswith(d))
+    subpath = target_path[len(prefix) + 1:]

[jbudorick, 2016/07/07 22:32:29]

os.path.relpath(target_path, prefix)

?

[agrieve, 2016/07/08 19:07:28]

Done.

+    symlinked_path = os.path.join(symlink_dir, subpath)
+    symlinked_dir = os.path.dirname(symlinked_path)
+    if not os.path.exists(symlinked_dir):
+      os.makedirs(symlinked_dir)
+    relpath = os.path.relpath(os.path.join(output_dir, target_path),

[jbudorick, 2016/07/07 22:32:29]

... if the target paths are absolute, this seems odd.

[agrieve, 2016/07/08 19:07:28]

Done.

+                              symlinked_dir)
+    logging.debug('Creating symlink %s -> %s', symlinked_path, relpath)
+    os.symlink(relpath, symlinked_path)
+
+
+def _CreateJavaSourceDir(output_dir, gradle_dir, java_sources_file):
+  """Computes and constructs when necessary the list of java source directories.
+
+  1. Computes the root java source directories from the list of files.
+  2. Determines whether there are any .java files in them that are not included
+     in |java_sources_file|.
+  3. If not, returns the list of java source directories. If so, constructs a
+     tree of symlinks within |gradle_dir| of all files in |java_sources_file|.
+  """
+  java_dirs = []
+  if java_sources_file:
+    with open(java_sources_file) as f:
+      java_files = [l.strip() for l in f]
+    java_dirs = _ComputeJavaSourceDirs(java_files)
+    abs_java_dirs = [os.path.join(output_dir, d) for d in java_dirs]
+
+    found_java_files = build_utils.FindInDirectories(abs_java_dirs, '*.java')
+    found_java_files = [os.path.relpath(p, output_dir)
+                        for p in found_java_files]
+
+    unwanted_java_files = set(found_java_files) - set(java_files)
+    if unwanted_java_files:
+      logging.debug('Target requires .java symlinks: %s', gradle_dir)
+      symlink_dir = os.path.join(gradle_dir, _JAVA_SUBDIR)
+      _CreateSymlinkTree(output_dir, symlink_dir, java_files, java_dirs)
+      java_dirs = [symlink_dir]
+
+  return java_dirs
+
+
+def _GenerateLocalProperties(sdk_dir):
+  """Returns the data for project.properties as a string."""
+  return '\n'.join([
+      '# Generated by //build/android/generate_gradle.py',
+      'sdk.dir=%s' % sdk_dir,
+      ''])
+
+
+def _GenerateGradleFile(build_config, config_json, java_dirs, relativize,
+                        use_gradle_process_resources):
+  """Returns the data for a project's build.gradle."""
+  deps_info = build_config['deps_info']
+  gradle = build_config['gradle']
+
+  if deps_info['type'] == 'android_apk':
+    target_type = 'android_apk'
+  elif deps_info['type'] == 'java_library' and not deps_info['is_prebuilt']:
+    if deps_info['requires_android']:

[jbudorick, 2016/07/07 22:32:29]

I'm a bit surprised that we write both of these as 'java_library' now.

[agrieve, 2016/07/08 19:07:28]

Meh, in rules.gni, android_library is just a shortcut for
java_library{requires_android=true}

+      target_type = 'android_library'
+    else:
+      target_type = 'java_library'
+  else:
+    return None
+
+  variables = {}
+  variables['template_type'] = target_type
+  variables['use_gradle_process_resources'] = use_gradle_process_resources
+  variables['build_tools_version'] = config_json['build_tools_version']
+  variables['compile_sdk_version'] = config_json['compile_sdk_version']
+  android_manifest = gradle.get('android_manifest',
+                                _DEFAULT_ANDROID_MANIFEST_PATH)
+  variables['android_manifest'] = relativize(android_manifest)
+
+  variables['java_dirs'] = relativize(java_dirs)
+  variables['prebuilts'] = relativize(gradle['dependent_prebuilt_jars'])
+  deps = [_ProjectEntry.FromBuildConfigPath(p)
+          for p in gradle['dependent_android_projects']]
+
+  variables['android_project_deps'] = [d.ProjectName() for d in deps]
+  deps = [_ProjectEntry.FromBuildConfigPath(p)
+          for p in gradle['dependent_java_projects']]
+  variables['java_project_deps'] = [d.ProjectName() for d in deps]
+
+  processor = jinja_template.JinjaProcessor(host_paths.DIR_SOURCE_ROOT)
+  return processor.Render(_JINJA_TEMPLATE_PATH, variables)
+
+
+def _GenerateRootGradle():
+  """Returns the data for the root project's build.gradle."""
+  variables = {'template_type': 'root'}
+  processor = jinja_template.JinjaProcessor(host_paths.DIR_SOURCE_ROOT)
+  return processor.Render(_JINJA_TEMPLATE_PATH, variables)
+
+
+def _GenerateSettingsGradle(project_entries):
+  """Returns the data for settings.gradle."""
+  project_name = os.path.basename(os.path.dirname(host_paths.DIR_SOURCE_ROOT))
+  lines = []
+  lines.append('// Generated by //build/android/generate_gradle.py')
+  lines.append('rootProject.name = "%s"' % project_name)
+  lines.append('rootProject.projectDir = settingsDir')
+  lines.append('')
+
+  for entry in project_entries:
+    # Example target: android_webview:android_webview_java__build_config
+    lines.append('include ":%s"' % entry.ProjectName())
+    lines.append('project(":%s").projectDir = new File(settingsDir, "%s")' %
+                 (entry.ProjectName(), entry.GradleSubdir()))
+  return '\n'.join(lines)
+
+
+def _ExtractSrcjars(output_dir, srcjar_tuples):
+  """Extracts all srcjars to the directory given by the tuples."""
+  extracted_paths = set(s[1] for s in srcjar_tuples)
+  for extracted_path in extracted_paths:
+    if os.path.exists(extracted_path):
+      shutil.rmtree(extracted_path)

[jbudorick, 2016/07/07 22:32:29]

Same comment as before about making destruction non-default.

[agrieve, 2016/07/08 19:07:28]

Done.

+
+  for srcjar_path, extracted_path in srcjar_tuples:
+    logging.info('Extracting %s to %s', srcjar_path, extracted_path)
+    with zipfile.ZipFile(os.path.join(output_dir, srcjar_path)) as z:
+      z.extractall(extracted_path)
+
+
+def _Relativizer(old_cwd, new_cwd):
+  """Returns a function that rebases a path from |old_cwd| to |new_cwd|."""
+  old_cwd = os.path.abspath(old_cwd)
+  new_cwd = os.path.abspath(new_cwd)
+  def rebase_path(path_or_list):
+    if isinstance(path_or_list, basestring):
+      return os.path.relpath(os.path.join(old_cwd, path_or_list), new_cwd)
+    return [rebase_path(p) for p in path_or_list]
+  return rebase_path
+
+
+def _FindAllProjectEntries(output_dir, main_entry):
+  """Returns the list of all _ProjectEntry instances given the root project."""
+  found = set()
+  to_scan = [main_entry.BuildConfigPath()]
+  while to_scan:
+    build_config_path = to_scan.pop()
+    if build_config_path in found:
+      continue
+    found.add(build_config_path)
+    build_config = build_utils.ReadJson(
+        os.path.join(output_dir, build_config_path))
+    to_scan.extend(build_config['deps_info']['deps_configs'])
+  return [_ProjectEntry.FromBuildConfigPath(p) for p in found]
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--output-directory',
+                      help='Path to the root build directory.')
+  parser.add_argument('-v',
+                      '--verbose',
+                      dest='verbose_count',
+                      default=0,
+                      action='count',
+                      help='Verbose level')
+  parser.add_argument('--target',
+                      help='GN target to generate project for.',
+                      default='//chrome/android:chrome_public_apk')
+  parser.add_argument('--project-dir',
+                      help='Root of the output project.',
+                      default=os.path.join('$CHROMIUM_OUTPUT_DIR', 'gradle'))
+  parser.add_argument('--use-gradle-process-resources',
+                      action='store_true',
+                      help='Have gradle generate R.java rather than ninja')
+  args = parser.parse_args()
+  run_tests_helper.SetLogLevel(args.verbose_count)
+  if args.output_directory:
+    constants.SetOutputDirectory(args.output_directory)
+  constants.CheckOutputDirectory()
+  output_dir = constants.GetOutDirectory()
+
+  gradle_output_dir = os.path.abspath(
+      args.project_dir.replace('$CHROMIUM_OUTPUT_DIR', output_dir))
+  logging.warning('Creating project at: %s', gradle_output_dir)
+
+  main_entry = _ProjectEntry(args.target)
+  logging.warning('Building .build_config files...')
+  _RunNinja(output_dir, [main_entry.NinjaBuildConfigTarget()])
+
+  all_entries = _FindAllProjectEntries(output_dir, main_entry)
+  logging.info('Found %d dependent build_config targets.', len(all_entries))
+
+  config_json = build_utils.ReadJson(
+      os.path.join(output_dir, 'gradle', 'config.json'))
+  project_entries = []
+  srcjar_tuples = []
+  for entry in all_entries:
+    build_config = build_utils.ReadJson(
+        os.path.join(output_dir, entry.BuildConfigPath()))
+    if build_config['deps_info']['type'] not in ('android_apk', 'java_library'):
+      continue
+
+    gradle_dir = os.path.join(gradle_output_dir, entry.GradleSubdir())

[jbudorick, 2016/07/07 22:32:29]

optional nit: maybe gradle_subdir instead of gradle_dir to avoid confusion w/
gradle_output_dir?

[agrieve, 2016/07/08 19:07:28]

changed to entry_output_dir

+    relativize = _Relativizer(output_dir, gradle_dir)
+
+    srcjars = build_config['gradle'].get('bundled_srcjars', [])
+    if not args.use_gradle_process_resources:
+      srcjars += build_config['javac']['srcjars']
+
+    java_sources_file = build_config['gradle'].get('java_sources_file')
+    if java_sources_file:
+      java_sources_file = os.path.join(output_dir, java_sources_file)
+
+    java_dirs = _CreateJavaSourceDir(output_dir, gradle_dir, java_sources_file)
+    if srcjars:
+      java_dirs.append(os.path.join(gradle_dir, _SRCJARS_SUBDIR))
+
+    data = _GenerateGradleFile(build_config, config_json, java_dirs, relativize,
+                               args.use_gradle_process_resources)
+    if data:
+      project_entries.append(entry)
+      srcjar_tuples.extend(
+          (s, os.path.join(gradle_dir, _SRCJARS_SUBDIR)) for s in srcjars)
+      _WriteFile(os.path.join(gradle_dir, 'build.gradle'), data)
+
+  _WriteFile(os.path.join(gradle_output_dir, 'build.gradle'),
+             _GenerateRootGradle())
+
+  _WriteFile(os.path.join(gradle_output_dir, 'settings.gradle'),
+             _GenerateSettingsGradle(project_entries))
+
+  sdk_path = os.path.abspath(os.path.join(output_dir,
+                                          config_json['android_sdk_root']))
+  _WriteFile(os.path.join(gradle_output_dir, 'local.properties'),
+             _GenerateLocalProperties(sdk_path))
+
+  if srcjar_tuples:
+    logging.warning('Building all .srcjar files...')
+    _RunNinja(output_dir, [s[0] for s in srcjar_tuples])
+    _ExtractSrcjars(output_dir, srcjar_tuples)
+  logging.warning('Project created successfully!')
+
+
+if __name__ == '__main__':
+  main()