[Android] Add gyp support for multidex.

build/android/gyp/dex.py

 #!/usr/bin/env python
 #
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import optparse
 import os
 import sys
+import tempfile
+import zipfile
 
 from util import build_utils
 from util import md5_check
 
 
+def _GenerateMainDexListFile(main_dex_list_file, paths, options):
+  proguard_binary = os.path.abspath(
+      os.path.join(options.android_sdk_tools, os.pardir, os.pardir, 'tools',
+                   'proguard', 'bin', 'proguard.sh'))
+  shrinked_android_jar = os.path.join(
+      options.android_sdk_tools, 'lib', 'shrinkedAndroid.jar')
+  dx_jar = os.path.join(
+      options.android_sdk_tools, 'lib', 'dx.jar')
+  paths_arg = ':'.join(paths)
+  rules_file = os.path.abspath(os.path.join(
+      options.android_sdk_tools, 'mainDexClasses.rules'))
+
+  with tempfile.NamedTemporaryFile(suffix='.jar') as temp_jar:
+    proguard_cmd = [

[Yaron, 2015/08/10 20:15:40]

I hate running proguard in debug mode. :(
What's the performance like for this call?

[jbudorick, 2015/08/10 22:00:49]

I tried nuking *.dex* in the chrome_public_apk intermediate directory both with
& witout multidex enabled.

multidex not enabled:

jbudorick@jbudorick:/usr/local/google/code/clankium-split/2/src$ time ninja -C
out/Debug -j500 -l15 chrome_public_apk
ninja: Entering directory `out/Debug'
[2/4] ACTION Packaging ChromePublic
[echo] multidex disabled

[4/4] STAMP obj/chrome/android/chrome_public_apk.actions_rules_copies.stamp

real	0m29.538s
user	0m34.090s
sys	0m1.706s


multidex enabled:

jbudorick@jbudorick:/usr/local/google/code/clankium-split/2/src$ time ninja -C
out/Debug -j500 -l15 chrome_public_apk
ninja: Entering directory `out/Debug'
[1/4] ACTION Creating dex file: chrome_public_apk/classes.dex.zip
      You should check if you need to specify additional program jars.
... //// Warnings omitted
[2/4] ACTION Packaging ChromePublic
[echo] multidex enabled

[4/4] STAMP obj/chrome/android/chrome_public_apk.actions_rules_copies.stamp

real	0m42.297s
user	1m3.343s
sys	0m3.408s


I suppose we could instead list the classes explicitly, but that seems much,
much more brittle to me.

[Yaron, 2015/08/11 13:38:36]

So.. many changes which touch java files are loads slower. :(
Can't say I'm thrilled unless we could do a managed-install type thing for java
files. agrieve was interested in building that on top of multidex

[jbudorick, 2015/08/11 13:42:21]

A what? I'm not sure I follow...

I'm also not thrilled about raising the build time.

+      proguard_binary,
+      '-forceprocessing',
+      '-dontwarn', '-dontoptimize', '-dontobfuscate', '-dontpreverify',
+      '-injars', paths_arg,
+      '-outjars', temp_jar.name,
+      '-libraryjars', shrinked_android_jar,
+      '-include', rules_file,
+    ]
+    for m in options.main_dex_rules_files:
+      proguard_cmd.extend(['-include', m])
+    build_utils.CheckOutput(proguard_cmd)
+
+    java_cmd = ['java', '-cp', dx_jar,
+                'com.android.multidex.MainDexListBuilder',
+                temp_jar.name, paths_arg]
+    main_dex_list = build_utils.CheckOutput(java_cmd)
+    main_dex_list_file.write(main_dex_list)
+    main_dex_list_file.flush()
+
+
 def DoDex(options, paths):
   dx_binary = os.path.join(options.android_sdk_tools, 'dx')
   # See http://crbug.com/272064 for context on --force-jumbo.
-  dex_cmd = [dx_binary, '--dex', '--force-jumbo', '--output', options.dex_path]
+  dex_cmd = [dx_binary, '--dex', '--force-jumbo']
   if options.no_locals != '0':
     dex_cmd.append('--no-locals')
+  with tempfile.NamedTemporaryFile() as main_dex_list_file:
+    if options.multi_dex:
+      _GenerateMainDexListFile(main_dex_list_file, paths, options)
 
-  dex_cmd += paths
+      dex_cmd.append('--multi-dex')
+      dex_cmd.append('--minimal-main-dex')
+      dex_cmd.append('--main-dex-list=%s' % main_dex_list_file.name)
 
-  record_path = '%s.md5.stamp' % options.dex_path
-  md5_check.CallAndRecordIfStale(
-      lambda: build_utils.CheckOutput(dex_cmd, print_stderr=False),
-      record_path=record_path,
-      input_paths=paths,
-      input_strings=dex_cmd,
-      force=not os.path.exists(options.dex_path))
-  build_utils.WriteJson(
-      [os.path.relpath(p, options.output_directory) for p in paths],
-      options.dex_path + '.inputs')
+    dex_cmd += ['--output', options.dex_path]
+    dex_cmd += paths
+
+    record_path = '%s.md5.stamp' % options.dex_path
+    md5_check.CallAndRecordIfStale(
+        lambda: build_utils.CheckOutput(dex_cmd, print_stderr=False),
+        record_path=record_path,
+        input_paths=paths,
+        input_strings=dex_cmd,
+        force=not os.path.exists(options.dex_path))
+    build_utils.WriteJson(
+        [os.path.relpath(p, options.output_directory) for p in paths],
+        options.dex_path + '.inputs')
+
+    if options.multi_dex and options.dex_path.endswith('.zip'):
+      iz = zipfile.ZipFile(options.dex_path, 'r')
+      tmp_dex_path = '%s.tmp.zip' % options.dex_path
+      oz = zipfile.ZipFile(tmp_dex_path, 'w', zipfile.ZIP_DEFLATED)
+      for i in iz.namelist():
+        if i.endswith('.dex'):
+          oz.writestr(i, iz.read(i))
+      os.remove(options.dex_path)
+      os.rename(tmp_dex_path, options.dex_path)
 
 
 def main():
   args = build_utils.ExpandFileArgs(sys.argv[1:])
 
   parser = optparse.OptionParser()
   build_utils.AddDepfileOption(parser)
 
   parser.add_option('--android-sdk-tools',
                     help='Android sdk build tools directory.')
   parser.add_option('--output-directory',
                     default=os.getcwd(),
                     help='Path to the output build directory.')
   parser.add_option('--dex-path', help='Dex output path.')
   parser.add_option('--configuration-name',
                     help='The build CONFIGURATION_NAME.')
   parser.add_option('--proguard-enabled',
                     help='"true" if proguard is enabled.')
   parser.add_option('--proguard-enabled-input-path',
                     help=('Path to dex in Release mode when proguard '
                           'is enabled.'))
   parser.add_option('--no-locals',
                     help='Exclude locals list from the dex file.')
+  parser.add_option('--multi-dex', default=False, action='store_true',
+                    help='Create multiple dex files.')
   parser.add_option('--inputs', help='A list of additional input paths.')
   parser.add_option('--excluded-paths',
                     help='A list of paths to exclude from the dex file.')
+  parser.add_option('--main-dex-rules-file',
+                    action='append',
+                    default=[],
+                    dest='main_dex_rules_files',
+                    help='A file containing a list of proguard rules to use '
+                         'in determining the classes to include in the '
+                         'main dex.')
 
   options, paths = parser.parse_args(args)
 
   required_options = ('android_sdk_tools',)
   build_utils.CheckOptions(options, parser, required=required_options)
 
   if (options.proguard_enabled == 'true'
       and options.configuration_name == 'Release'):
     paths = [options.proguard_enabled_input_path]
 
@@ skipping 10 matching lines @@
 
   if options.depfile:
     build_utils.WriteDepfile(
         options.depfile,
         paths + build_utils.GetPythonDependencies())
 
 
 
 if __name__ == '__main__':
   sys.exit(main())