Add a tool to check license compatibility with Android

android_webview/tools/webview_licenses.py

+#!/usr/bin/python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This tool checks third-party licenses for the purposes of the Android WebView
+# build. See the output of '--help' for details.
+
+
+import optparse
+import os
+import re
+import subprocess
+import sys
+import textwrap
+
+
+REPOSITORY_ROOT = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '..', '..'))
+
+sys.path.append(os.path.join(REPOSITORY_ROOT, 'tools'))
+import licenses
+
+
+def _CheckDirectories(directory_list):
+  """Checks that all top-level directories under directories named 'third_party'
+  are listed.
+  Args:
+    directory_list: The list of directories.
+  Returns:
+    True if all directories are listed and the list contains no stale entries,
+    otherwise false.
+  """
+
+  cwd = os.getcwd()
+  os.chdir(REPOSITORY_ROOT)
+  unlisted_directories = []
+  parent_listed_directory = None
+  for root, _, _ in os.walk('.'):
+    root = os.path.normpath(root)
+    if root in directory_list:
+      parent_listed_directory = root
+    is_listed = (parent_listed_directory and
+                 root.startswith(parent_listed_directory))
+    if (not is_listed
+        and not root.startswith('out/')
+        and os.path.dirname(root).endswith('third_party')):
+      unlisted_directories += [root]
+  stale = [x for x in directory_list if not os.path.exists(x)]
+  os.chdir(cwd)
+
+  if unlisted_directories:
+    print 'Some third-party directories are not listed. You must add the ' \
+          'following directories to the list.\n%s' % \
+          '\n'.join(unlisted_directories)
+    return False
+
+  if stale:
+    print 'Some third-party directories are listed but not present. You must ' \
+          'remove the following directories from the list.\n%s' % \
+          '\n'.join(stale)
+    return False
+
+  return True
+
+
+def _GetCmdOutput(args):
+  p = subprocess.Popen(args=args, cwd=REPOSITORY_ROOT, stdout=subprocess.PIPE)
+  ret = p.communicate()[0]
+  return ret
+
+
+def _CheckLicenseHeaders(directory_list, file_list):
+  """Checks that all files which are not in a listed third-party directory,
+  and which do not use the standard Chromium license, are listed.
+  Args:
+    directory_list: The list of directories.
+    file_list: The list of files.
+  Returns:
+    True if all files with non-standard license headers are listed and the
+    file list contains no stale entries, otherwise false.
+  """
+  # Matches one of ...
+  # - '[Cc]opyright' but not when followed by
+  #   ' 20[0-9][0-9] [Tt]he Chromium Authors' or
+  #   ' 20[0-9][0-9]-20[0-9][0-9] [Tt]he [Cc]hromium [Aa]uthors', with an
+  #   optional '([Cc])'
+  # - '([Cc]) 20[0-9][0-9] but not when preceeded by '[Cc]opyright' or
+  #   'opyright '
+  regex = '[Cc]opyright(?!( \([Cc]\))? 20[0-9][0-9](-20[0-9][0-9])? ' \
+          '[Tt]he [Cc]hromium [Aa]uthors)' \
+          '|' \
+          '(?<!([Cc]opyright|opyright ))\([Cc]\) (19|20)[0-9][0-9]'

[Evan Martin, 2012/07/24 19:27:55]

Why not fix the case on the code rather than this complicated regex?

It looks like any instance of "the chromium authors" in the Chrome tree (with
that casing) is a typo.

+
+  args = ['grep',
+          '-rPlI',
+          '--exclude-dir', 'third_party',
+          '--exclude-dir', 'out',
+          '--exclude-dir', '.git',
+          regex,
+          '.']
+  files = _GetCmdOutput(args).splitlines()
+
+  # Exclude files under listed directories and some known offendors.
+  offending_files = []
+  for x in files:
+    x = os.path.normpath(x)
+    is_in_listed_directory = False
+    for y in directory_list:
+      if x.startswith(y):
+        is_in_listed_directory = True
+        break
+    if (not is_in_listed_directory
+        # Exists in Android tree.
+        and not x == 'ThirdPartyProject.prop'
+        # Ignore these tools.
+        and not x.startswith('android_webview/tools/')
+        # This is a build intermediate directory.
+        and not x.startswith('chrome/app/theme/google_chrome/')
+        # This is a test output directory.
+        and not x.startswith('data/page_cycler/')
+        # 'Copyright' appears in strings.
+        and not x.startswith('chrome/app/resources/')):
+      offending_files += [x]
+
+  unknown = set(offending_files) - set(file_list)
+  if unknown:
+    print 'The following files contain a third-party license but are not in ' \
+          'a listed third-party directory and are not themselves listed. You ' \
+          'must add the following files to the list.\n%s' % '\n'.join(unknown)
+    return False
+
+  stale = set(file_list) - set(offending_files)
+  if stale:
+    print 'The following third-party files are listed unnecessarily. You ' \
+          'must remove the following files from the list.\n%s' % \
+          '\n'.join(stale)
+    return False
+
+  return True
+
+
+def _GetEntriesWithAnnotation(entries, annotation):
+  """Gets a list of all entries with the specified annotation.
+  Args:
+    entries: The list of entries.
+    annotation: The annotation.
+  Returns:
+    A list of entries.
+  """
+
+  result = []
+  for line in entries.splitlines():
+    match = re.match(r'([^#\s]*)\s+' + annotation + r'\s+', line)
+    if match:
+      result += [match.group(1)]
+  return result
+
+
+def _GetLicenseFile(directory):
+  """Gets the path to the license file for the specified directory. Uses the
+  licenses tool from scripts/'.
+  Args:
+    directory: The directory to consider, relative to the root of the
+               repository.
+  Returns:
+    The absolute path to the license file.
+  """
+
+  return licenses.ParseDir(directory, False)['License File']
+
+
+def _CheckLicenseFiles(directories):
+  """Checks that all directories annotated with REQUIRES_ATTRIBUTION have a
+  license file.
+  Args:
+    directories: The list of directories.
+  Returns:
+    Whether the check succeeded.
+  """
+
+  offending_directories = []
+  for directory in directories:
+    if not os.path.exists(_GetLicenseFile(directory)):
+      offending_directories += [license_file]
+
+  if offending_directories:
+    print 'Some license files are missing. You must provide license files in ' \
+          'the following directories.\n%s' % '\n'.join(offending_directories)
+    return False
+
+  return True
+
+
+def _ReadFile(path):
+  """Reads a file from disk.
+  Args:
+    path: The path of the file to read, relative to the root of the repository.
+  Returns:
+    The contents of the file as a string.
+  """
+
+  with file(os.path.join(REPOSITORY_ROOT, path), 'r') as f:
+    lines = f.read()
+  return lines

[Evan Martin, 2012/07/24 19:41:28]

"lines" is a confusing name, as it is a single string of the contents of the
file.  (You can just "return f.read()" in the with statement to eliminate this
anywya.)

+
+
+def _GetEntriesWithoutAnnotation(entries, annotation):
+  """Gets a list of all entries without the specified annotation.
+  Args:
+    entries: The list of entries.
+    annotation: The annotation.
+  Returns:
+    A list of entries.
+  """
+
+  result = []
+  for line in entries.splitlines():
+    match = re.match(r'([^#\s]*)((?!' + annotation + r').)*$', line)
+    if match and not len(match.group(1)) == 0:
+      result += [match.group(1)]
+  return result
+
+
+def _Check(directories_data, files_data):
+  """Checks that all third-party code in projects used by the WebView either
+  uses a license compatible with Android or is exlcuded from the snapshot. Also
+  checks that license text is present for third-party code requiring
+  attribution.
+  Args:
+    directories_data: The contents of the directories data file.
+    files_data: The contents of the files data file.
+  Returns:
+    Whether the check succeeded.
+  """
+
+
+  # We use two signals to find third-party code. First, directories named
+  # 'third-party' and second, non-standard license text.
+  directories = _GetEntriesWithoutAnnotation(directories_data,
+                                             'INCOMPATIBLE_AND_UNUSED')
+  files = _GetEntriesWithoutAnnotation(files_data, 'INCOMPATIBLE_AND_UNUSED')
+  result = _CheckDirectories(directories)
+  result = _CheckLicenseHeaders(directories, files) and result
+
+  # Also check that all directories annotated with REQUIRES_ATTRIBUTION have a
+  # license file.
+  directories = _GetEntriesWithAnnotation(directories_data,
+                                          'REQUIRES_ATTRIBUTION')
+  return _CheckLicenseFiles(directories) and result
+
+
+def _GenerateNoticeFile(directories_data, print_warnings):
+  """Generates the contents of an Android NOTICE file for the third-party code.
+  Args:
+    directories_data: The contents of the directories data file.
+    print_warnings: Whether to print warnings.
+  Returns:
+    The contents of the NOTICE file.
+  """
+
+  # Don't forget Chromium's LICENSE file
+  content = [_ReadFile('LICENSE')]
+
+  for directory in _GetEntriesWithAnnotation(directories_data,
+                                            'REQUIRES_ATTRIBUTION'):
+    content += [_ReadFile(_GetLicenseFile(directory))]
+
+  return '\n'.join(content)
+
+
+def main():
+  class IndentedHelpFormatterWithNL(optparse.IndentedHelpFormatter):
+    def format_description(self, description):
+      if not description: return ""
+      desc_width = self.width - self.current_indent
+      indent = " "*self.current_indent
+      bits = description.split('\n')
+      formatted_bits = [
+        textwrap.fill(bit,
+          desc_width,
+          initial_indent=indent,
+          subsequent_indent=indent)
+        for bit in bits]
+      result = '\n'.join(formatted_bits) + '\n'
+      return result
+
+  parser = optparse.OptionParser(formatter=IndentedHelpFormatterWithNL(),
+                                 usage='%prog [options]')
+  parser.description = 'Checks third-party licenses for the purposes of the ' \
+                       'Android WebView build.\n\n' \
+                       'The Android tree includes a snapshot of Chromium in ' \
+                       'order to power the system WebView. The snapshot '\
+                       'includes only the third-party DEPS projects required ' \
+                       'for the WebView. This tool is intended to be run in ' \
+                       'the snapshot and checks that all code uses ' \
+                       'open-source licenses compatible with Android, and ' \
+                       'that we meet the requirements of those licenses. It ' \
+                       'can also be used to generate an Android NOTICE file ' \
+                       'for the third-party code.\n\n' \
+                       'It makes use of two data files, ' \
+                       'third_party_files.txt and ' \
+                       'third_party_directories.txt. These record the ' \
+                       'license status of all third-party code in the main ' \
+                       'Chromium repository and in the third-party DEPS ' \
+                       'projects used in the snapshot. This status includes ' \
+                       'why the code\'s license is compatible with Android, ' \
+                       'or why the code must be excluded from the ' \
+                       'snapshot.\n\n' \

[Evan Martin, 2012/07/24 19:27:55]

I think this long string should be the docstring of the module, which you can
retrieve with __doc__.  I.e.:

parser.description = (__doc__ +
   'Commands:\n'
   '  check ... [as before]')

+                       'Commands:\n' \
+                       '  check  Check licenses.\n' \
+                       '  notice Generate Android NOTICE file on stdout'
+  (options, args) = parser.parse_args()
+  if len(args) != 1:
+    parser.print_help()
+    return 1
+
+  tools_directory = os.path.join('android_webview', 'tools')
+  directories_data = _ReadFile(os.path.join(tools_directory,
+                                            'third_party_directories.txt'))
+  files_data = _ReadFile(os.path.join(tools_directory, 'third_party_files.txt'))
+
+  if args[0] == 'check':
+    if _Check(directories_data, files_data):
+      print 'OK!'
+      return 0
+    else:
+      return 1
+  elif args[0] == 'notice':
+    print _GenerateNoticeFile(directories_data, False)
+    return 0
+
+  parser.print_help()
+  return 1
+
+if __name__ == '__main__':
+  sys.exit(main())