Implement without patch retries and src-side script for webview_licenses.py

android_webview/tools/webview_licenses.py

 #!/usr/bin/python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Checks third-party licenses for the purposes of the Android WebView build.
 
 The Android tree includes a snapshot of Chromium in order to power the system
 WebView.  This tool checks that all code uses open-source licenses compatible
 with Android, and that we meet the requirements of those licenses. It can also
 be used to generate an Android NOTICE file for the third-party code.
 
 It makes use of src/tools/licenses.py and the README.chromium files on which
 it depends. It also makes use of a data file, third_party_files_whitelist.txt,
 which whitelists indicidual files which contain third-party code but which
 aren't in a third-party directory with a README.chromium file.
 """
 
 import glob
 import imp
+import json
 import multiprocessing
 import optparse
 import os
 import re
 import sys
 import textwrap
 
 
 REPOSITORY_ROOT = os.path.abspath(os.path.join(
     os.path.dirname(__file__), '..', '..'))
@@ skipping 123 matching lines @@
           '\n'.join(sorted(unknown))
   if missing:
     print 'The following files are whitelisted, but do not exist.\n%s' % \
         '\n'.join(sorted(missing))
   if stale:
     print 'The following files are whitelisted unnecessarily. You must ' \
           'remove the following files from the whitelist.\n%s' % \
           '\n'.join(sorted(stale))
 
   if unknown:
-    return ScanResult.Errors
+    code = ScanResult.Errors
   elif stale or missing:
-    return ScanResult.Warnings
+    code = ScanResult.Warnings
   else:
-    return ScanResult.Ok
+    code = ScanResult.Ok
+
+  problem_paths = sorted(set(unknown + missing + stale))

[mnaganov (inactive), 2015/02/17 11:27:09]

How exactly will this list be used? Will any human be looking at it? If so, I
don't think it is OK to mash all those problems together. Just a list paths with
no indication of what problems they have might be confusing. 

E.g. consider a deleted file which hasn't been removed from the whitelist file.
The path does not exist anymore, so for a human it will not be immediately
obvious what is wrong with it.

[mnaganov (inactive), 2015/02/17 11:28:35]

Looked at the bug -- if it is only for scripts that check the list of problems
with and without the patch, then it is certainly OK. Please confirm that.

[Paweł Hajdan Jr., 2015/02/17 14:09:36]

Yes, JSON is for machine-readable processing.

Content intended to be read by people should be just printed to stdout, and the
script already does it.

+  return (code, problem_paths)
 
 
 def _ReadFile(full_path, mode='rU'):
   """Reads a file from disk. This emulates presubmit InputApi.ReadFile func.
   Args:
     full_path: The path of the file to read.
   Returns:
     The contents of the file as a string.
   """
 
@@ skipping 55 matching lines @@
      that all non third-party code doesn't contain external copyrighted code.
   Returns:
     ScanResult.Ok if everything is in order;
     ScanResult.Warnings if there are non-fatal problems (e.g. stale whitelist
       entries)
     ScanResult.Errors otherwise.
   """
 
   third_party_dirs = _FindThirdPartyDirs()
 
+  problem_paths = []
+
   # First, check designated third-party directories using src/tools/licenses.py.
   all_licenses_valid = True
   for path in sorted(third_party_dirs):
     try:
       licenses.ParseDir(path, REPOSITORY_ROOT)
     except licenses.LicenseError, e:
       if not (path in known_issues.KNOWN_ISSUES):
         print 'Got LicenseError "%s" while scanning %s' % (e, path)
+        problem_paths.append(path)
         all_licenses_valid = False
 
   # Second, check for non-standard license text.
   whitelisted_files = copyright_scanner.LoadWhitelistedFilesList(InputApi())
-  licenses_check = _CheckLicenseHeaders(third_party_dirs, whitelisted_files)
+  licenses_check, more_problem_paths = _CheckLicenseHeaders(
+      third_party_dirs, whitelisted_files)
 
-  return licenses_check if all_licenses_valid else ScanResult.Errors
+  problem_paths.extend(more_problem_paths)
+
+  return (licenses_check if all_licenses_valid else ScanResult.Errors,
+          problem_paths)
 
 
 class TemplateEntryGenerator(object):
   def __init__(self):
     self._generate_licenses_file_list_only = False
     self._toc_index = 0
 
   def SetGenerateLicensesFileListOnly(self, generate_licenses_file_list_only):
     self._generate_licenses_file_list_only = generate_licenses_file_list_only
 
@@ skipping 74 matching lines @@
 
 def main():
   class FormatterWithNewLines(optparse.IndentedHelpFormatter):
     def format_description(self, description):
       paras = description.split('\n')
       formatted_paras = [textwrap.fill(para, self.width) for para in paras]
       return '\n'.join(formatted_paras) + '\n'
 
   parser = optparse.OptionParser(formatter=FormatterWithNewLines(),
                                  usage='%prog [options]')
+  parser.add_option('--json', help='Path to JSON output file')
   parser.description = (__doc__ +
                         '\nCommands:\n'
                         '  scan Check licenses.\n'
                         '  notice_deps Generate the list of dependencies for '
                         'Android NOTICE file.\n'
                         '  notice [file] Generate Android NOTICE file on '
                         'stdout or into |file|.\n'
                         '  incompatible_directories Scan for incompatibly'
                         ' licensed directories.\n'
                         '  all_incompatible_directories Scan for incompatibly'
                         ' licensed directories (even those in'
                         ' known_issues.py).\n'
                         '  display_copyrights Display autorship on the files'
                         ' using names provided via stdin.\n')
-  (_, args) = parser.parse_args()
+  (options, args) = parser.parse_args()
   if len(args) < 1:
     parser.print_help()
     return ScanResult.Errors
 
   if args[0] == 'scan':
-    scan_result = _Scan()
+    scan_result, problem_paths = _Scan()
     if scan_result == ScanResult.Ok:
       print 'OK!'
+    if options.json:
+      with open(options.json, 'w') as f:
+        json.dump(problem_paths, f)
     return scan_result
   elif args[0] == 'notice_deps':
     # 'set' is used to eliminate duplicate references to the same license file.
     print ' '.join(
       sorted(set(GenerateNoticeFile(generate_licenses_file_list_only=True))))
     return ScanResult.Ok
   elif args[0] == 'notice':
     notice_file_contents = GenerateNoticeFile()
     if len(args) == 1:
       print notice_file_contents
     else:
       with open(args[1], 'w') as output_file:
         output_file.write(notice_file_contents)
     return ScanResult.Ok
   elif args[0] == 'incompatible_directories':
     return _ProcessIncompatibleResult(GetUnknownIncompatibleDirectories())
   elif args[0] == 'all_incompatible_directories':
     return _ProcessIncompatibleResult(GetIncompatibleDirectories())
   elif args[0] == 'display_copyrights':
     files = sys.stdin.read().splitlines()
     for f, c in \
         zip(files, copyright_scanner.FindCopyrights(InputApi(), '.', files)):
       print f, '\t', ' / '.join(sorted(c))
     return ScanResult.Ok
   parser.print_help()
   return ScanResult.Errors
 
 if __name__ == '__main__':
   sys.exit(main())