[Android WebView] Run license scanning in parallel

android_webview/tools/webview_licenses.py

 #!/usr/bin/python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Checks third-party licenses for the purposes of the Android WebView build.
 
 The Android tree includes a snapshot of Chromium in order to power the system
 WebView.  This tool checks that all code uses open-source licenses compatible
 with Android, and that we meet the requirements of those licenses. It can also
 be used to generate an Android NOTICE file for the third-party code.
 
 It makes use of src/tools/licenses.py and the README.chromium files on which
 it depends. It also makes use of a data file, third_party_files_whitelist.txt,
 which whitelists indicidual files which contain third-party code but which
 aren't in a third-party directory with a README.chromium file.
 """
 
 import glob
 import imp
+import multiprocessing
 import optparse
 import os
 import re
 import subprocess
 import sys
 import textwrap
 
 
 REPOSITORY_ROOT = os.path.abspath(os.path.join(
     os.path.dirname(__file__), '..', '..'))
@@ skipping 57 matching lines @@
                  ' skipping.' % exclude)
         exclude = exclude_dirname
       known_incompatible.append(os.path.normpath(os.path.join(path, exclude)))
   known_incompatible = frozenset(known_incompatible)
   return incompatible_directories.difference(known_incompatible)
 
 
 class ScanResult(object):
   Ok, Warnings, Errors = range(3)
 
+# Needs to be a top-level function for multiprocessing
+def _FindCopyrights(files_to_scan):
+  args = [os.path.join('android_webview', 'tools', 'find_copyrights.pl')]
+  p = subprocess.Popen(
+    args=args, cwd=REPOSITORY_ROOT,
+    stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+  lines = p.communicate(files_to_scan)[0].splitlines()
+
+  offending_files = []
+  allowed_copyrights = '^(?:\*No copyright\*' \
+      '|20[0-9][0-9](?:-20[0-9][0-9])? The Chromium Authors\. ' \
+      'All rights reserved.*)$'
+  allowed_copyrights_re = re.compile(allowed_copyrights)
+  for l in lines:
+    entries = l.split('\t')
+    if entries[1] == "GENERATED FILE":
+      continue
+    copyrights = entries[1].split(' / ')
+    for c in copyrights:
+      if c and not allowed_copyrights_re.match(c):
+        offending_files.append(os.path.normpath(entries[0]))
+        break
+  return offending_files
+
+def _ShardString(s, delimiter, shard_len):
+  result = []
+  index = 0
+  last_pos = 0
+  for m in re.finditer(delimiter, s):
+    index += 1
+    if index % shard_len == 0:
+      result.append(s[last_pos:m.end()])
+      last_pos = m.end()
+  if not index % shard_len == 0:
+    result.append(s[last_pos:])
+  return result
+
 def _CheckLicenseHeaders(excluded_dirs_list, whitelisted_files):
   """Checks that all files which are not in a listed third-party directory,
   and which do not use the standard Chromium license, are whitelisted.
   Args:
     excluded_dirs_list: The list of directories to exclude from scanning.
     whitelisted_files: The whitelist of files.
   Returns:
     ScanResult.Ok if all files with non-standard license headers are whitelisted
     and the whitelist contains no stale entries;
     ScanResult.Warnings if there are stale entries;
@@ skipping 32 matching lines @@
   excluded_dirs_list.append('tools/histograms')
   # Swarming tools, doesn't exist in the snapshot
   excluded_dirs_list.append('tools/swarming_client')
   # Arm sysroot tools, doesn't exist in the snapshot
   excluded_dirs_list.append('arm-sysroot')
   # Data is not part of open source chromium, but are included on some bots.
   excluded_dirs_list.append('data')
   # This is not part of open source chromium, but are included on some bots.
   excluded_dirs_list.append('skia/tools/clusterfuzz-data')
 
-  args = ['android_webview/tools/find_copyrights.pl',
+  args = [os.path.join('android_webview', 'tools', 'find_files.pl'),
           '.'
           ] + excluded_dirs_list
   p = subprocess.Popen(args=args, cwd=REPOSITORY_ROOT, stdout=subprocess.PIPE)
-  lines = p.communicate()[0].splitlines()
+  files_to_scan = p.communicate()[0]
 
-  offending_files = []
-  allowed_copyrights = '^(?:\*No copyright\*' \
-      '|20[0-9][0-9](?:-20[0-9][0-9])? The Chromium Authors\. ' \
-      'All rights reserved.*)$'
-  allowed_copyrights_re = re.compile(allowed_copyrights)
-  for l in lines:
-    entries = l.split('\t')
-    if entries[1] == "GENERATED FILE":
-      continue
-    copyrights = entries[1].split(' / ')
-    for c in copyrights:
-      if c and not allowed_copyrights_re.match(c):
-        offending_files.append(os.path.normpath(entries[0]))
-        break
+  sharded_files_to_scan = _ShardString(files_to_scan, '\n', 2000)
+  pool = multiprocessing.Pool()
+  offending_files_chunks = pool.map_async(
+      _FindCopyrights, sharded_files_to_scan).get(999999)
+  pool.close()
+  pool.join()
+  # Flatten out the result
+  offending_files = \
+    [item for sublist in offending_files_chunks for item in sublist]
 
   unknown = set(offending_files) - set(whitelisted_files)
   if unknown:
     print 'The following files contain a third-party license but are not in ' \
           'a listed third-party directory and are not whitelisted. You must ' \
           'add the following files to the whitelist.\n%s' % \
           '\n'.join(sorted(unknown))
 
   stale = set(whitelisted_files) - set(offending_files)
   if stale:
@@ skipping 157 matching lines @@
     return ScanResult.Ok
   elif args[0] == 'incompatible_directories':
     return _ProcessIncompatibleResult(GetUnknownIncompatibleDirectories())
   elif args[0] == 'all_incompatible_directories':
     return _ProcessIncompatibleResult(GetIncompatibleDirectories())
   parser.print_help()
   return ScanResult.Errors
 
 if __name__ == '__main__':
   sys.exit(main())