Android: Instrumentation tests in gradle

build/android/gradle/generate_gradle.py

 #!/usr/bin/env python
 # Copyright 2016 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generates an Android Studio project from a GN target."""
 
 import argparse
 import codecs
 import logging
@@ skipping 97 matching lines @@
   SUFFIX_LEN = len('__build_config')
   for line in ninja_output.splitlines():
     ninja_target = line.rsplit(':', 1)[0]
     # Ignore root aliases by ensure a : exists.
     if ':' in ninja_target and ninja_target.endswith('__build_config'):
       ret.append('//' + ninja_target[:-SUFFIX_LEN])
   return ret
 
 
 class _ProjectEntry(object):
-  """Helper class for various path transformations."""
+  """Helper class for project entries."""
   def __init__(self, gn_target):
     assert gn_target.startswith('//'), gn_target
     if ':' not in gn_target:
       gn_target = '%s:%s' % (gn_target, os.path.basename(gn_target))
     self._gn_target = gn_target
     self._build_config = None
+    self.android_test_entry = None
 
   @classmethod
   def FromBuildConfigPath(cls, path):
     prefix = 'gen/'
     suffix = '.build_config'
     assert path.startswith(prefix) and path.endswith(suffix), path
     subdir = path[len(prefix):-len(suffix)]
     return cls('//%s:%s' % (os.path.split(subdir)))
 
   def __hash__(self):
@@ skipping 22 matching lines @@
     """Returns the Gradle project name."""
     return self.GradleSubdir().replace(os.path.sep, '>')
 
   def BuildConfig(self):
     """Reads and returns the project's .build_config JSON."""
     if not self._build_config:
       path = os.path.join('gen', self.GradleSubdir() + '.build_config')
       self._build_config = build_utils.ReadJson(_RebasePath(path))
     return self._build_config
 
+  def DepsInfo(self):
+    return self.BuildConfig()['deps_info']
+
+  def Gradle(self):
+    return self.BuildConfig()['gradle']
+
   def GetType(self):
     """Returns the target type from its .build_config."""
-    return self.BuildConfig()['deps_info']['type']
+    return self.DepsInfo()['type']
+
+  def JavaFiles(self):
+    java_sources_file = self.Gradle().get('java_sources_file')
+    java_files = []
+    if java_sources_file:
+      java_sources_file = _RebasePath(java_sources_file)
+      java_files = build_utils.ReadSourcesList(java_sources_file)

[agrieve, 2017/01/18 21:58:10]

nit: cache this value.

[Peter Wen, 2017/01/19 16:43:07]

Done.

+    return java_files
+
+
+class _GradleGenerator(object):

[agrieve, 2017/01/18 21:58:10]

nit: this doesn't actuall generate gradle files. How about
_ProjectVariablesGenerator or somesuch

[Peter Wen, 2017/01/19 16:43:07]

Done.

+  """Helper class to generate gradle build files"""
+  def __init__(self, project_dir, use_gradle_process_resources):
+    self.project_dir = project_dir
+    self.use_gradle_process_resources = use_gradle_process_resources
+
+  def _GenJniLibs(self, entry):
+    native_section = entry.BuildConfig().get('native')
+    if native_section:
+      jni_libs = _CreateJniLibsDir(
+          constants.GetOutDirectory(), self.OutputDir(entry),
+          native_section.get('libraries'))
+    else:
+      jni_libs = []
+    return jni_libs
+
+  def _GenJavaDirs(self, entry):
+    java_dirs = _CreateJavaSourceDir(
+        constants.GetOutDirectory(), self.OutputDir(entry),
+        entry.JavaFiles())
+    if self.GenerateSrcjars(entry):
+      java_dirs.append(os.path.join(self.OutputDir(entry), _SRCJARS_SUBDIR))
+    return java_dirs
+
+  def _Relativize(self, entry, paths):
+    return _RebasePath(paths, self.OutputDir(entry))
+
+  def OutputDir(self, entry):

[agrieve, 2017/01/18 21:58:10]

nit: conflicts with other output_dir. How about "EntryOutputDir"

[Peter Wen, 2017/01/19 16:43:07]

Done.

+    return os.path.join(self.project_dir, entry.GradleSubdir())
+
+  def GenerateSrcjars(self, entry):

[agrieve, 2017/01/18 21:58:10]

this doesn't generate anything. How about jjust SrcJars()

[Peter Wen, 2017/01/19 16:43:07]

Done.

+    srcjars = _RebasePath(entry.Gradle().get('bundled_srcjars', []))
+    if not self.use_gradle_process_resources:
+      srcjars += _RebasePath(entry.BuildConfig()['javac']['srcjars'])
+    return srcjars
+
+  def Generate(self, entry):
+    variables = {}
+    android_test_manifest = entry.Gradle().get(
+        'android_manifest', _DEFAULT_ANDROID_MANIFEST_PATH)
+    variables['android_manifest'] = self._Relativize(
+        entry, android_test_manifest)
+    variables['java_dirs'] = self._Relativize(entry, self._GenJavaDirs(entry))
+    variables['jni_libs'] = self._Relativize(entry, self._GenJniLibs(entry))
+    deps = [_ProjectEntry.FromBuildConfigPath(p)
+            for p in entry.Gradle()['dependent_android_projects']]
+    variables['android_project_deps'] = [d.ProjectName() for d in deps]
+    # TODO(agrieve): Add an option to use interface jars and see if that speeds
+    # things up at all.
+    variables['prebuilts'] = self._Relativize(
+        entry, entry.Gradle()['dependent_prebuilt_jars'])
+    deps = [_ProjectEntry.FromBuildConfigPath(p)
+            for p in entry.Gradle()['dependent_java_projects']]
+    variables['java_project_deps'] = [d.ProjectName() for d in deps]
+    return variables
+
+  def GeneratedInputs(self, entry):

[agrieve, 2017/01/18 21:58:10]

nit: might make more sense to have this right beside GenerateSrcjars

[Peter Wen, 2017/01/19 16:43:07]

Done.

+    generated_inputs = []
+    generated_inputs.extend(self.GenerateSrcjars(entry))
+    generated_inputs.extend(
+        p for p in entry.JavaFiles() if not p.startswith('..'))
+    generated_inputs.extend(entry.Gradle()['dependent_prebuilt_jars'])
+    return generated_inputs
 
 
 def _ComputeJavaSourceDirs(java_files):
   """Returns the list of source directories for the given files."""
   found_roots = set()
   for path in java_files:
     path_root = path
     # Recognize these tokens as top-level.
     while True:
       path_root = os.path.dirname(path_root)
@@ skipping 91 matching lines @@
 
 
 def _GenerateLocalProperties(sdk_dir):
   """Returns the data for project.properties as a string."""
   return '\n'.join([
       '# Generated by //build/android/gradle/generate_gradle.py',
       'sdk.dir=%s' % sdk_dir,
       ''])
 
 
-def _GenerateGradleFile(build_config, build_vars, java_dirs, jni_libs,
-                        relativize, use_gradle_process_resources,
-                        jinja_processor):
+def _GenerateGradleFile(entry, generator, build_vars, jinja_processor):
   """Returns the data for a project's build.gradle."""
-  deps_info = build_config['deps_info']
-  gradle = build_config['gradle']
+  deps_info = entry.DepsInfo()
+  gradle = entry.Gradle()
 
   variables = {
       'sourceSetName': 'main',
       'depCompileName': 'compile',
   }
   if deps_info['type'] == 'android_apk':
     target_type = 'android_apk'
   elif deps_info['type'] == 'java_library':
     if deps_info['is_prebuilt'] or deps_info['gradle_treat_as_prebuilt']:
       return None
-
-    if deps_info['requires_android']:
+    elif deps_info['requires_android']:
       target_type = 'android_library'
     else:
       target_type = 'java_library'
   elif deps_info['type'] == 'java_binary':
     if gradle['main_class'] == 'org.chromium.testing.local.JunitTestMain':
       target_type = 'android_junit'
       variables['sourceSetName'] = 'test'
       variables['depCompileName'] = 'testCompile'
     else:
       target_type = 'java_binary'
       variables['main_class'] = gradle['main_class']
   else:
     return None
 
   variables['target_name'] = os.path.splitext(deps_info['name'])[0]
   variables['template_type'] = target_type
-  variables['use_gradle_process_resources'] = use_gradle_process_resources
+  variables['use_gradle_process_resources'] = (
+      generator.use_gradle_process_resources)
   variables['build_tools_version'] = (
       build_vars['android_sdk_build_tools_version'])
   variables['compile_sdk_version'] = build_vars['android_sdk_version']
-  android_manifest = gradle.get('android_manifest',
-                                _DEFAULT_ANDROID_MANIFEST_PATH)
-  variables['android_manifest'] = relativize(android_manifest)
-  variables['java_dirs'] = relativize(java_dirs)
-  variables['jni_libs'] = relativize(jni_libs)
-  # TODO(agrieve): Add an option to use interface jars and see if that speeds
-  # things up at all.
-  variables['prebuilts'] = relativize(gradle['dependent_prebuilt_jars'])
-  deps = [_ProjectEntry.FromBuildConfigPath(p)
-          for p in gradle['dependent_android_projects']]
-
-  variables['android_project_deps'] = [d.ProjectName() for d in deps]
-  deps = [_ProjectEntry.FromBuildConfigPath(p)
-          for p in gradle['dependent_java_projects']]
-  variables['java_project_deps'] = [d.ProjectName() for d in deps]
+  variables['main'] = generator.Generate(entry)
+  if entry.android_test_entry:
+    variables['android_test'] = generator.Generate(
+        entry.android_test_entry)
 
   return jinja_processor.Render(
       _TemplatePath(target_type.split('_')[0]), variables)
 
 
 def _GenerateRootGradle(jinja_processor):
   """Returns the data for the root project's build.gradle."""
   return jinja_processor.Render(_TemplatePath('root'))
 
 
@@ skipping 29 matching lines @@
 
 def _FindAllProjectEntries(main_entries):
   """Returns the list of all _ProjectEntry instances given the root project."""
   found = set()
   to_scan = list(main_entries)
   while to_scan:
     cur_entry = to_scan.pop()
     if cur_entry in found:
       continue
     found.add(cur_entry)
-    build_config = cur_entry.BuildConfig()
-    sub_config_paths = build_config['deps_info']['deps_configs']
+    sub_config_paths = cur_entry.DepsInfo()['deps_configs']
     to_scan.extend(
         _ProjectEntry.FromBuildConfigPath(p) for p in sub_config_paths)
   return list(found)
 
 
+def _CombineTestEntries(entries):

[agrieve, 2017/01/18 21:58:10]

It would be great to add some comments to this method saying why we do this:
- minimize gradle files
- adds under-test as a dependency
- does not work for junit

[Peter Wen, 2017/01/19 16:43:07]

Done.

+  combined_entries = []
+  android_test_entries = {}
+  for entry in entries:
+    target_name = entry.GnTarget()
+    if (target_name.endswith('_test_apk__apk') and
+        'apk_under_test' in entry.Gradle()):
+      apk_name = entry.Gradle()['apk_under_test']
+      android_test_entries[apk_name] = entry
+    else:
+      combined_entries.append(entry)
+  for entry in combined_entries:
+    target_name = entry.DepsInfo()['name']
+    if target_name in android_test_entries:
+      entry.android_test_entry = android_test_entries[target_name]
+      del android_test_entries[target_name]
+  # Add unmatched test entries as individual targets.
+  combined_entries.extend(android_test_entries.values())
+  return combined_entries
+
+
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('--output-directory',
+                      default='out/Debug',
                       help='Path to the root build directory.')
   parser.add_argument('-v',
                       '--verbose',
                       dest='verbose_count',
                       default=0,
                       action='count',
                       help='Verbose level')
   parser.add_argument('--target',
                       dest='targets',
                       action='append',
                       help='GN target to generate project for. '
                            'May be repeated.')
   parser.add_argument('--project-dir',
                       help='Root of the output project.',
                       default=os.path.join('$CHROMIUM_OUTPUT_DIR', 'gradle'))
   parser.add_argument('--all',
                       action='store_true',
                       help='Generate all java targets (slows down IDE)')
   parser.add_argument('--use-gradle-process-resources',
                       action='store_true',
                       help='Have gradle generate R.java rather than ninja')
   args = parser.parse_args()
   if args.output_directory:
     constants.SetOutputDirectory(args.output_directory)
   constants.CheckOutputDirectory()
   output_dir = constants.GetOutDirectory()
   devil_chromium.Initialize(output_directory=output_dir)
   run_tests_helper.SetLogLevel(args.verbose_count)
 
-  gradle_output_dir = os.path.abspath(
+  _gradle_output_dir = os.path.abspath(
       args.project_dir.replace('$CHROMIUM_OUTPUT_DIR', output_dir))
-  logging.warning('Creating project at: %s', gradle_output_dir)
+  generator = _GradleGenerator(
+      _gradle_output_dir, args.use_gradle_process_resources)
+  logging.warning('Creating project at: %s', generator.project_dir)
 
   if args.all:
     # Run GN gen if necessary (faster than running "gn gen" in the no-op case).
-    _RunNinja(output_dir, ['build.ninja'])
+    _RunNinja(constants.GetOutDirectory(), ['build.ninja'])
     # Query ninja for all __build_config targets.
     targets = _QueryForAllGnTargets(output_dir)
   else:
     targets = args.targets or _DEFAULT_TARGETS
-    # TODO(agrieve): See if it makes sense to utilize Gradle's test constructs
-    # for our instrumentation tests.
     targets = [re.sub(r'_test_apk$', '_test_apk__apk', t) for t in targets]
+    # TODO(wnwen): Utilize Gradle's test constructs for our junit tests?
     targets = [re.sub(r'_junit_tests$', '_junit_tests__java_binary', t)
                for t in targets]
 
   main_entries = [_ProjectEntry(t) for t in targets]
 
   logging.warning('Building .build_config files...')
   _RunNinja(output_dir, [e.NinjaBuildConfigTarget() for e in main_entries])
 
   # There are many unused libraries, so restrict to those that are actually used
   # when using --all.
   if args.all:
     main_entries = [e for e in main_entries if e.GetType() == 'android_apk']
 
   all_entries = _FindAllProjectEntries(main_entries)
   logging.info('Found %d dependent build_config targets.', len(all_entries))
+  entries = _CombineTestEntries(all_entries)
+  logging.info('Creating %d projects for targets.', len(entries))
 
   logging.warning('Writing .gradle files...')
   jinja_processor = jinja_template.JinjaProcessor(_FILE_DIR)
   build_vars = _ReadBuildVars(output_dir)
   project_entries = []
   srcjar_tuples = []
   generated_inputs = []
-  for entry in all_entries:
+  for entry in entries:
     if entry.GetType() not in ('android_apk', 'java_library', 'java_binary'):
       continue
 
-    entry_output_dir = os.path.join(gradle_output_dir, entry.GradleSubdir())
-    relativize = lambda x, d=entry_output_dir: _RebasePath(x, d)
-    build_config = entry.BuildConfig()
-
-    srcjars = _RebasePath(build_config['gradle'].get('bundled_srcjars', []))
-    if not args.use_gradle_process_resources:
-      srcjars += _RebasePath(build_config['javac']['srcjars'])
-
-    java_sources_file = build_config['gradle'].get('java_sources_file')
-    java_files = []
-    if java_sources_file:
-      java_sources_file = _RebasePath(java_sources_file)
-      java_files = build_utils.ReadSourcesList(java_sources_file)
-
-    java_dirs = _CreateJavaSourceDir(output_dir, entry_output_dir, java_files)
-    if srcjars:
-      java_dirs.append(os.path.join(entry_output_dir, _SRCJARS_SUBDIR))
-
-    native_section = build_config.get('native')
-    if native_section:
-      jni_libs = _CreateJniLibsDir(
-          output_dir, entry_output_dir, native_section.get('libraries'))
-    else:
-      jni_libs = []
-
-    data = _GenerateGradleFile(build_config, build_vars, java_dirs, jni_libs,
-                               relativize, args.use_gradle_process_resources,
-                               jinja_processor)
+    data = _GenerateGradleFile(entry, generator, build_vars, jinja_processor)
     if data:
       project_entries.append(entry)
       # Build all paths references by .gradle that exist within output_dir.
-      generated_inputs.extend(srcjars)
-      generated_inputs.extend(p for p in java_files if not p.startswith('..'))
-      generated_inputs.extend(build_config['gradle']['dependent_prebuilt_jars'])
+      generated_inputs.extend(generator.GeneratedInputs(entry))
+      srcjar_tuples.extend(
+          (s, os.path.join(generator.OutputDir(entry), _SRCJARS_SUBDIR))
+          for s in generator.GenerateSrcjars(entry))
+      _WriteFile(os.path.join(generator.OutputDir(entry), 'build.gradle'), data)
 
-      srcjar_tuples.extend(
-          (s, os.path.join(entry_output_dir, _SRCJARS_SUBDIR)) for s in srcjars)
-      _WriteFile(os.path.join(entry_output_dir, 'build.gradle'), data)
-
-  _WriteFile(os.path.join(gradle_output_dir, 'build.gradle'),
+  _WriteFile(os.path.join(generator.project_dir, 'build.gradle'),
              _GenerateRootGradle(jinja_processor))
 
-  _WriteFile(os.path.join(gradle_output_dir, 'settings.gradle'),
+  _WriteFile(os.path.join(generator.project_dir, 'settings.gradle'),
              _GenerateSettingsGradle(project_entries))
 
   sdk_path = _RebasePath(build_vars['android_sdk_root'])
-  _WriteFile(os.path.join(gradle_output_dir, 'local.properties'),
+  _WriteFile(os.path.join(generator.project_dir, 'local.properties'),
              _GenerateLocalProperties(sdk_path))
 
   if generated_inputs:
     logging.warning('Building generated source files...')
     targets = _RebasePath(generated_inputs, output_dir)
     _RunNinja(output_dir, targets)
 
   if srcjar_tuples:
-    _ExtractSrcjars(gradle_output_dir, srcjar_tuples)
+    _ExtractSrcjars(generator.project_dir, srcjar_tuples)
 
   logging.warning('Project created! (%d subprojects)', len(project_entries))
   logging.warning('Generated projects work best with Android Studio 2.2')
   logging.warning('For more tips: https://chromium.googlesource.com/chromium'
                   '/src.git/+/master/docs/android_studio.md')
 
 
 if __name__ == '__main__':
   main()