Add a tool to check license compatibility with Android

android_webview/tools/webview_licenses.py

+#!/usr/bin/python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Checks third-party licenses for the purposes of the Android WebView build.
+
+The Android tree includes a snapshot of Chromium in order to power the system
+WebView.  This tool checks that all code uses open-source licenses compatible
+with Android, and that we meet the requirements of those licenses. It can also
+be used to generate an Android NOTICE file for the third-party code.
+
+It makes use of src/tools/licenses.py and the README.chromium files on which
+it depends. It also makes use of a data file, third_party_files_whitelist.txt,
+which whitelists indicidual files which contain third-party code but which
+aren't in a third-party directory with a README.chromium file.
+"""
+
+import optparse
+import os
+import re
+import subprocess
+import sys
+import textwrap
+
+
+REPOSITORY_ROOT = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '..', '..'))
+
+sys.path.append(os.path.join(REPOSITORY_ROOT, 'tools'))
+import licenses
+
+
+def _CheckLicenseHeaders(directory_list, whitelisted_files):
+  """Checks that all files which are not in a listed third-party directory,
+  and which do not use the standard Chromium license, are listed.

[Nico, 2012/08/09 15:02:29]

s/are listed/are whitelisted/?

[Steve Block, 2012/08/09 17:24:59]

Done.

+  Args:
+    directory_list: The list of directories.
+    whitelisted_files: The list of files.
+  Returns:
+    True if all files with non-standard license headers are listed and the
+    file list contains no stale entries, otherwise false.
+  """
+
+  # Matches one of ...
+  # - '[Cc]opyright', but not when followed by
+  #   ' (c) 20[0-9][0-9] The Chromium Authors.', with optional date range
+  # - '([Cc]) (19|20)[0-9][0-9]', but not when preceeded by the word copyright,
+  #   as this is handled above
+  regex = '[Cc]opyright(?!( \(c\))? 20[0-9][0-9](-20[01][0189])? ' \

[Nico, 2012/08/09 15:02:29]

Why not 20[0-9][0-9] for the second year in a range too? Since we changed
policies, new files shouldn't have ranges, but we've changes this policy often
enough that I wouldn't be surprised if we change it again.

[Steve Block, 2012/08/09 17:24:59]

Done.

+          'The Chromium Authors\. All rights reserved\.)' \
+          '|' \
+          '(?<!(pyright |opyright))\([Cc]\) (19|20)[0-9][0-9]'
+
+  args = ['grep',
+          '-rPlI',
+          '--exclude-dir', 'third_party',
+          '--exclude-dir', 'out',
+          '--exclude-dir', '.git',
+          regex,
+          '.']

[Nico, 2012/08/09 15:02:29]

Do you set / check the cwd anywhere?

[Steve Block, 2012/08/09 17:24:59]

I set the cwd on the line below. But it turns out that the existing license tool
assumes it's run from the repository root, so I've added a check for that too.

+  p = subprocess.Popen(args=args, cwd=REPOSITORY_ROOT, stdout=subprocess.PIPE)
+  files = p.communicate()[0].splitlines()
+
+  # Exclude files under listed directories and some known offendors.

[Nico, 2012/08/09 15:02:29]

"offenders" I think

[Steve Block, 2012/08/09 17:24:59]

Done.

+  offending_files = []
+  for x in files:
+    x = os.path.normpath(x)
+    is_in_listed_directory = False
+    for y in directory_list:
+      if x.startswith(y):
+        is_in_listed_directory = True
+        break
+    if (not is_in_listed_directory
+        # Ignore these tools.
+        and not x.startswith('android_webview/tools/')
+        # This is a build intermediate directory.
+        and not x.startswith('chrome/app/theme/google_chrome/')
+        # This is a test output directory.
+        and not x.startswith('data/page_cycler/')
+        # 'Copyright' appears in strings.
+        and not x.startswith('chrome/app/resources/')):

[Nico, 2012/08/09 15:02:29]

nit: Could you instead say

  directory_list = directory_list + ['android_webview/tools/', ...]

at the top of the function / above the loop? That's a bit more table-driven and
a bit easier to understand.

[Steve Block, 2012/08/09 17:24:59]

Done.

+      offending_files += [x]

[Nico, 2012/08/09 15:02:29]

.append

[Steve Block, 2012/08/09 17:24:59]

Done.

+
+  result = True

[Nico, 2012/08/09 15:02:29]

s/result/all_files_valid/

[Steve Block, 2012/08/09 17:24:59]

Done.

+  unknown = set(offending_files) - set(whitelisted_files)
+  if unknown:
+    print 'The following files contain a third-party license but are not in ' \
+          'a listed third-party directory and are not whitelisted. You must ' \
+          'add the following files to the whitelist.\n%s' % \
+          '\n'.join(sorted(unknown))
+    result = False
+
+  stale = set(whitelisted_files) - set(offending_files)
+  if stale:
+    print 'The following files are whitelisted unnecessarily. You must ' \
+          ' remove the following files from the whitelist list.\n%s' % \

[Nico, 2012/08/09 15:02:29]

s/whitelist list/whitelist/

[Steve Block, 2012/08/09 17:24:59]

Done.

+          '\n'.join(sorted(stale))
+    result = False
+
+  return result
+
+
+def _ReadFile(path):
+  """Reads a file from disk.
+  Args:
+    path: The path of the file to read, relative to the root of the repository.
+  Returns:
+    The contents of the file as a string.
+  """
+
+  with file(os.path.join(REPOSITORY_ROOT, path), 'r') as f:
+    lines = f.read()

[Nico, 2012/08/09 15:02:29]

fyi: if you write

  return open(...path...).read()

python will clean up the file at the end of the statement too. Maybe a bit
shorter.

[Steve Block, 2012/08/09 17:24:59]

Done.

+  return lines
+
+
+def _Scan():
+  """Checks that license meta-data is present for all third-party code.
+  Returns:
+    Whether the check succeeded.
+  """
+
+  third_party_dirs = _FindThirdPartyDirs()
+
+  # First, check designated third-party directories using src/tools/licenses.py.
+  result = True

[Nico, 2012/08/09 15:02:29]

s/result/all_licenses_valid/

[Steve Block, 2012/08/09 17:24:59]

Done.

+  for path in sorted(third_party_dirs):
+    try:
+      licenses.ParseDir(path)
+    except licenses.LicenseError, e:
+      print 'Got LicenseError "%s" while scanning %s' % (e, path)
+      result = False
+
+  # Second, check for non-standard license text.
+  files_data = _ReadFile(os.path.join('android_webview', 'tools',
+                                      'third_party_files_whitelist.txt'))
+  whitelisted_files = []
+  for line in files_data.splitlines():
+    match = re.match(r'([^#\s]*)', line)
+    if match and not len(match.group(1)) == 0:

[Nico, 2012/08/09 15:02:29]

if you say + instead of *, you don't need the second part of the conditional

[Steve Block, 2012/08/09 17:24:59]

Done.

+      whitelisted_files += [match.group(1)]

[Nico, 2012/08/09 15:02:29]

append

[Steve Block, 2012/08/09 17:24:59]

Done.

+  return _CheckLicenseHeaders(third_party_dirs, whitelisted_files) and result
+
+
+def _FindThirdPartyDirs():

[Nico, 2012/08/09 15:02:29]

Move above _Scan

[Steve Block, 2012/08/09 17:24:59]

Done.

+  """Gets the list of third-party directories.
+  Returns:
+    The list of third-party directories.
+  """
+
+  prune_paths = [
+    # Placeholder directory, no third-party code.
+    os.path.join('third_party', 'adobe'),
+    # Apache 2.0 license. See
+    # https://code.google.com/p/chromium/issues/detail?id=140478.
+    os.path.join('third_party', 'bidichecker'),
+  ]
+  return licenses.FindThirdPartyDirs(prune_paths)
+
+
+def _GenerateNoticeFile(print_warnings):
+  """Generates the contents of an Android NOTICE file for the third-party code.
+  Args:
+    print_warnings: Whether to print warnings.
+  Returns:
+    The contents of the NOTICE file.
+  """
+
+  third_party_dirs = _FindThirdPartyDirs()
+
+  # Don't forget Chromium's LICENSE file
+  content = [_ReadFile('LICENSE')]
+
+  # We provide attribution for all third-party directories.
+  # TODO(steveblock): Limit this to only code used by the WebView binary.
+  for directory in third_party_dirs:
+    content += [_ReadFile(licenses.ParseDir(directory)['License File'])]
+
+  return '\n'.join(content)
+
+
+def main():
+  class IndentedHelpFormatterWithNL(optparse.IndentedHelpFormatter):
+    def format_description(self, description):
+      if not description: return ""
+      desc_width = self.width - self.current_indent
+      indent = " "*self.current_indent
+      bits = description.split('\n')
+      formatted_bits = [
+        textwrap.fill(bit,
+          desc_width,
+          initial_indent=indent,
+          subsequent_indent=indent)
+        for bit in bits]
+      result = '\n'.join(formatted_bits) + '\n'
+      return result

[Nico, 2012/08/09 15:02:29]

Do you need this? If so, adjust coding style (it kinda looks like third-party
code, maybe it should be in a third_party subdirectory with a LICENSE file :-)
). But we can probably just live without it.

[Steve Block, 2012/08/09 17:24:59]

I found the snippet on a forum. I meant to clean it up - now done.

+
+  parser = optparse.OptionParser(formatter=IndentedHelpFormatterWithNL(),
+                                 usage='%prog [options]')
+  parser.description = (__doc__ +
+                       '\nCommands:\n' \
+                       '  scan  Check licenses.\n' \
+                       '  notice Generate Android NOTICE file on stdout')
+  (options, args) = parser.parse_args()
+  if len(args) != 1:
+    parser.print_help()
+    return 1
+
+  if args[0] == 'scan':
+    if _Scan():
+      print 'OK!'
+      return 0
+    else:
+      return 1
+  elif args[0] == 'notice':
+    print _GenerateNoticeFile(print_warnings=False)
+    return 0
+
+  parser.print_help()
+  return 1
+
+if __name__ == '__main__':
+  sys.exit(main())