[Android WebView] Rewrite copyrights scanner in Python

android_webview/tools/webview_licenses.py

 #!/usr/bin/python
-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Checks third-party licenses for the purposes of the Android WebView build.
 
 The Android tree includes a snapshot of Chromium in order to power the system
 WebView.  This tool checks that all code uses open-source licenses compatible
 with Android, and that we meet the requirements of those licenses. It can also
 be used to generate an Android NOTICE file for the third-party code.
 
 It makes use of src/tools/licenses.py and the README.chromium files on which
 it depends. It also makes use of a data file, third_party_files_whitelist.txt,
 which whitelists indicidual files which contain third-party code but which
 aren't in a third-party directory with a README.chromium file.
 """
 
 import glob
 import imp
 import multiprocessing
 import optparse
 import os
 import re
-import subprocess
 import sys
 import textwrap
 
 
 REPOSITORY_ROOT = os.path.abspath(os.path.join(
     os.path.dirname(__file__), '..', '..'))
 
 # Import third_party/PRESUBMIT.py via imp to avoid importing a random
 # PRESUBMIT.py from $PATH, also make sure we don't generate a .pyc file.
 sys.dont_write_bytecode = True
 third_party = \
   imp.load_source('PRESUBMIT', \
                   os.path.join(REPOSITORY_ROOT, 'third_party', 'PRESUBMIT.py'))
 
 sys.path.append(os.path.join(REPOSITORY_ROOT, 'tools'))
 import licenses
 
+import copyright_scanner
 import known_issues
 
 class InputApi(object):
   def __init__(self):
     self.re = re
 
 def GetIncompatibleDirectories():
   """Gets a list of third-party directories which use licenses incompatible
   with Android. This is used by the snapshot tool.
   Returns:
@@ skipping 37 matching lines @@
         exclude = exclude_dirname
       known_incompatible.append(os.path.normpath(os.path.join(path, exclude)))
   known_incompatible = frozenset(known_incompatible)
   return incompatible_directories.difference(known_incompatible)
 
 
 class ScanResult(object):
   Ok, Warnings, Errors = range(3)
 
 # Needs to be a top-level function for multiprocessing
-def _FindCopyrights(files_to_scan):
-  args = [os.path.join('android_webview', 'tools', 'find_copyrights.pl')]
-  p = subprocess.Popen(
-    args=args, cwd=REPOSITORY_ROOT,
-    stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-  lines = p.communicate(files_to_scan)[0].splitlines()
+def _FindCopyrightViolations(files_to_scan_as_string):
+  return copyright_scanner.FindCopyrightViolations(
+    REPOSITORY_ROOT, files_to_scan_as_string)
 
-  offending_files = []
-  allowed_copyrights = '^(?:\*No copyright\*' \
-      '|20[0-9][0-9](?:-20[0-9][0-9])? The Chromium Authors\. ' \
-      'All rights reserved.*)$'
-  allowed_copyrights_re = re.compile(allowed_copyrights)
-  for l in lines:
-    entries = l.split('\t')
-    if entries[1] == "GENERATED FILE":
-      continue
-    copyrights = entries[1].split(' / ')
-    for c in copyrights:
-      if c and not allowed_copyrights_re.match(c):
-        offending_files.append(os.path.normpath(entries[0]))
-        break
-  return offending_files
-
-def _ShardString(s, delimiter, shard_len):
-  result = []
-  index = 0
-  last_pos = 0
-  for m in re.finditer(delimiter, s):
-    index += 1
-    if index % shard_len == 0:
-      result.append(s[last_pos:m.end()])
-      last_pos = m.end()
-  if not index % shard_len == 0:
-    result.append(s[last_pos:])
-  return result
+def _ShardList(l, shard_len):
+  return [l[i:i + shard_len] for i in range(0, len(l), shard_len)]
 
 def _CheckLicenseHeaders(excluded_dirs_list, whitelisted_files):
   """Checks that all files which are not in a listed third-party directory,
   and which do not use the standard Chromium license, are whitelisted.
   Args:
     excluded_dirs_list: The list of directories to exclude from scanning.
     whitelisted_files: The whitelist of files.
   Returns:
     ScanResult.Ok if all files with non-standard license headers are whitelisted
     and the whitelist contains no stale entries;
@@ skipping 33 matching lines @@
   excluded_dirs_list.append('tools/histograms')
   # Swarming tools, doesn't exist in the snapshot
   excluded_dirs_list.append('tools/swarming_client')
   # Arm sysroot tools, doesn't exist in the snapshot
   excluded_dirs_list.append('arm-sysroot')
   # Data is not part of open source chromium, but are included on some bots.
   excluded_dirs_list.append('data')
   # This is not part of open source chromium, but are included on some bots.
   excluded_dirs_list.append('skia/tools/clusterfuzz-data')
 
-  args = [os.path.join('android_webview', 'tools', 'find_files.pl'),
-          '.'
-          ] + excluded_dirs_list
-  p = subprocess.Popen(args=args, cwd=REPOSITORY_ROOT, stdout=subprocess.PIPE)
-  files_to_scan = p.communicate()[0]
-
-  sharded_files_to_scan = _ShardString(files_to_scan, '\n', 2000)
+  files_to_scan = copyright_scanner.FindFiles(
+    REPOSITORY_ROOT, ['.'], excluded_dirs_list)
+  sharded_files_to_scan = _ShardList(files_to_scan, 2000)
   pool = multiprocessing.Pool()
   offending_files_chunks = pool.map_async(
-      _FindCopyrights, sharded_files_to_scan).get(999999)
+      _FindCopyrightViolations, sharded_files_to_scan).get(999999)
   pool.close()
   pool.join()
   # Flatten out the result
   offending_files = \
     [item for sublist in offending_files_chunks for item in sublist]
 
   unknown = set(offending_files) - set(whitelisted_files)
   if unknown:
     print 'The following files contain a third-party license but are not in ' \
           'a listed third-party directory and are not whitelisted. You must ' \
@@ skipping 19 matching lines @@
 
 
 def _ReadFile(path):
   """Reads a file from disk.
   Args:
     path: The path of the file to read, relative to the root of the repository.
   Returns:
     The contents of the file as a string.
   """
 
-  return open(os.path.join(REPOSITORY_ROOT, path), 'rb').read()
+  with open(os.path.join(REPOSITORY_ROOT, path), 'rb') as f:
+    return f.read()
 
 
 def _FindThirdPartyDirs():
   """Gets the list of third-party directories.
   Returns:
     The list of third-party directories.
   """
 
   # Please don't add here paths that have problems with license files,
   # as they will end up included in Android WebView snapshot.
@@ skipping 92 matching lines @@
 def main():
   class FormatterWithNewLines(optparse.IndentedHelpFormatter):
     def format_description(self, description):
       paras = description.split('\n')
       formatted_paras = [textwrap.fill(para, self.width) for para in paras]
       return '\n'.join(formatted_paras) + '\n'
 
   parser = optparse.OptionParser(formatter=FormatterWithNewLines(),
                                  usage='%prog [options]')
   parser.description = (__doc__ +
-                       '\nCommands:\n' \
-                       '  scan Check licenses.\n' \
-                       '  notice Generate Android NOTICE file on stdout.\n' \
-                       '  incompatible_directories Scan for incompatibly'
-                       ' licensed directories.\n'
-                       '  all_incompatible_directories Scan for incompatibly'
-                       ' licensed directories (even those in'
-                       ' known_issues.py).\n')
+                        '\nCommands:\n'
+                        '  scan Check licenses.\n'
+                        '  notice Generate Android NOTICE file on stdout.\n'
+                        '  incompatible_directories Scan for incompatibly'
+                        ' licensed directories.\n'
+                        '  all_incompatible_directories Scan for incompatibly'
+                        ' licensed directories (even those in'
+                        ' known_issues.py).\n'
+                        '  display_copyrights Display autorship on the files'
+                        ' using names provided via stdin.\n')
   (_, args) = parser.parse_args()
   if len(args) != 1:
     parser.print_help()
     return ScanResult.Errors
 
   if args[0] == 'scan':
     scan_result = _Scan()
     if scan_result == ScanResult.Ok:
       print 'OK!'
     return scan_result
   elif args[0] == 'notice':
     print GenerateNoticeFile()
     return ScanResult.Ok
   elif args[0] == 'incompatible_directories':
     return _ProcessIncompatibleResult(GetUnknownIncompatibleDirectories())
   elif args[0] == 'all_incompatible_directories':
     return _ProcessIncompatibleResult(GetIncompatibleDirectories())
+  elif args[0] == 'display_copyrights':
+    files = sys.stdin.read().splitlines()
+    for f, c in zip(files, copyright_scanner.FindCopyrights('.', files)):
+      print f, '\t', ' / '.join(sorted(c))
+    return ScanResult.Ok
   parser.print_help()
   return ScanResult.Errors
 
 if __name__ == '__main__':
   sys.exit(main())