Make javac invocations incremental when possible

build/android/gyp/javac.py

 #!/usr/bin/env python
 #
 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import optparse
 import os
 import shutil
 import re
@@ skipping 90 matching lines @@
   wrapper = textwrap.TextWrapper(break_long_words=True,
                                  drop_whitespace=False,
                                  subsequent_indent=' ',
                                  width=_MAX_MANIFEST_LINE_LEN - 2)
   output = '\r\n'.join(w for l in output for w in wrapper.wrap(l))
 
   with open(manifest_path, 'w') as f:
     f.write(output)
 
 
+def _ExtractClassFiles(jar_path, dest_dir, java_files):
+  """Extracts all .class files not corresponding to |java_files|."""
+  def extract_predicate(path):
+    if not path.endswith('.class'):
+      return False
+    path_without_suffix = re.sub(r'(?:\$|\.class)[^/]*$', '', path)

[jbudorick, 2015/09/23 00:26:20]

path[:-len('.class')]

or something along those lines? We already know that path ends with '.class' by
this point.

[agrieve, 2015/09/23 02:07:55]

Hmm, yeah, sorely lacking a comment here. 
Added:
  # Two challenges exist here:
  # 1. |java_files| have prefixes that are not represented in the the jar paths.
  # 2. A single .java file results in multiple .class files when it contains
  #    nested classes.
  # Here's an example:
  #   source path: ../../base/android/java/src/org/chromium/Foo.java
  #   jar paths: org/chromium/Foo.class, org/chromium/Foo$Inner.class
  # To extract only .class files not related to the given .java files, we strip
  # off ".class" and "$*.class" and use a substring match against java_files.

+    ret = not any(path_without_suffix in p for p in java_files)
+    return ret

[jbudorick, 2015/09/23 00:26:20]

nit: just

  return not any(path_without_suffix in p for p in java_files)

[agrieve, 2015/09/23 02:07:55]

Done.

+
+  build_utils.ExtractAll(jar_path, path=dest_dir, predicate=extract_predicate)
+
+
+def _OnStaleMd5(changes, options, javac_cmd, java_files, runtime_classpath):
+  with build_utils.TempDir() as temp_dir:
+    srcjars = options.java_srcjars
+    # The .excluded.jar contains .class files excluded from the main jar.
+    # It is used for incremental compiles.
+    excluded_jar_path = options.jar_path.replace('.jar', '.excluded.jar')
+
+    classes_dir = os.path.join(temp_dir, 'classes')
+    os.makedirs(classes_dir)
+
+    changed_paths = None
+    if changes.AddedOrModifiedOnly():
+      changed_paths = set(changes.IterModifiedPaths())
+      srcjars = [p for p in srcjars if p in changed_paths]
+
+    if srcjars:
+      java_dir = os.path.join(temp_dir, 'java')
+      os.makedirs(java_dir)
+      for srcjar in options.java_srcjars:
+        build_utils.ExtractAll(srcjar, path=java_dir, pattern='*.java')
+      jar_srcs = build_utils.FindInDirectory(java_dir, '*.java')
+      java_files.extend(_FilterJavaFiles(jar_srcs, options.javac_includes))
+
+    if java_files:
+      if changed_paths:
+        # When no files have been added / removed and the output jar already
+        # exists, reuse .class files from the existing jar.
+        for path in srcjars:
+          changed_paths.update(changes.IterChangedSubpaths(path))
+
+        java_files = [p for p in java_files if p in changed_paths]
+
+        _ExtractClassFiles(options.jar_path, classes_dir, java_files)
+        _ExtractClassFiles(excluded_jar_path, classes_dir, java_files)
+        # Add the extracted files to the classpath.
+        classpath_idx = javac_cmd.index('-classpath')
+        javac_cmd[classpath_idx + 1] += ':' + classes_dir
+
+      # Don't include the output directory in the initial set of args since it
+      # being in a temp dir makes it unstable (breaks md5 stamping).
+      cmd = javac_cmd + ['-d', classes_dir] + java_files
+
+      build_utils.CheckOutput(
+          cmd,
+          print_stdout=options.chromium_code,
+          stderr_filter=ColorJavacOutput)
+
+    if options.main_class or options.manifest_entry:
+      entries = []
+      if options.manifest_entry:
+        entries = [e.split(':') for e in options.manifest_entry]
+      manifest_file = os.path.join(temp_dir, 'manifest')
+      _CreateManifest(manifest_file, runtime_classpath, options.main_class,
+                      entries)
+    else:
+      manifest_file = None
+
+    glob = options.jar_excluded_classes
+    inclusion_predicate = lambda f: not build_utils.MatchesGlob(f, glob)
+    exclusion_predicate = lambda f: not inclusion_predicate(f)
+
+    jar.JarDirectory(classes_dir,
+                     options.jar_path,
+                     manifest_file=manifest_file,
+                     predicate=inclusion_predicate)
+    jar.JarDirectory(classes_dir,
+                     excluded_jar_path,
+                     predicate=exclusion_predicate)
+
+
 def _ParseOptions(argv):
   parser = optparse.OptionParser()
   build_utils.AddDepfileOption(parser)
 
   parser.add_option(
       '--src-gendirs',
       help='Directories containing generated java files.')
   parser.add_option(
       '--java-srcjars',
       action='append',
@@ skipping 121 matching lines @@
   # Compute the list of paths that when changed, we need to rebuild.
   input_paths = options.bootclasspath + options.java_srcjars + java_files
   # TODO(agrieve): Remove this .TOC heuristic once GYP is no more.
   if not options.use_ijars:
     for path in compile_classpath:
       if os.path.exists(path + '.TOC'):
         input_paths.append(path + '.TOC')
       else:
         input_paths.append(path)
 
-  def on_stale_md5():
-    with build_utils.TempDir() as temp_dir:
-      if options.java_srcjars:
-        java_dir = os.path.join(temp_dir, 'java')
-        os.makedirs(java_dir)
-        for srcjar in options.java_srcjars:
-          build_utils.ExtractAll(srcjar, path=java_dir, pattern='*.java')
-        jar_srcs = build_utils.FindInDirectory(java_dir, '*.java')
-        java_files.extend(_FilterJavaFiles(jar_srcs, options.javac_includes))
+  output_paths = [
+      options.jar_path,
+      options.jar_path.replace('.jar', '.excluded.jar'),
+  ]
 
-      classes_dir = os.path.join(temp_dir, 'classes')
-      os.makedirs(classes_dir)
-
-      if java_files:
-        # Don't include the output directory in the initial set of args since it
-        # being in a temp dir makes it unstable (breaks md5 stamping).
-        cmd = javac_cmd + ['-d', classes_dir] + java_files
-
-        build_utils.CheckOutput(
-            cmd,
-            print_stdout=options.chromium_code,
-            stderr_filter=ColorJavacOutput)
-
-      if options.main_class or options.manifest_entry:
-        entries = []
-        if options.manifest_entry:
-          entries = [e.split(':') for e in options.manifest_entry]
-        manifest_file = os.path.join(temp_dir, 'manifest')
-        _CreateManifest(manifest_file, runtime_classpath, options.main_class,
-                        entries)
-      else:
-        manifest_file = None
-      jar.JarDirectory(classes_dir,
-                       options.jar_excluded_classes,
-                       options.jar_path,
-                       manifest_file=manifest_file)
-
+  # An escape hatch to be able to check if incremental compiles are causing
+  # problems.
+  force = int(os.environ.get('DISABLE_INCREMENTAL_JAVAC', 0))
 
   # List python deps in input_strings rather than input_paths since the contents
   # of them does not change what gets written to the depsfile.
   build_utils.CallAndRecordIfStale(
-      on_stale_md5,
+      lambda changes: _OnStaleMd5(changes, options, javac_cmd, java_files,
+                                  runtime_classpath),
       options,
       input_paths=input_paths,
       input_strings=javac_cmd,
-      output_paths=[options.jar_path])
+      output_paths=output_paths,
+      force=force,
+      pass_changes=True)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))