Use a depfile to know when to regenerate notice files

android_webview/tools/webview_licenses.py

 #!/usr/bin/python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Checks third-party licenses for the purposes of the Android WebView build.
 
 The Android tree includes a snapshot of Chromium in order to power the system
 WebView.  This tool checks that all code uses open-source licenses compatible
 with Android, and that we meet the requirements of those licenses. It can also
@@ skipping 18 matching lines @@
 REPOSITORY_ROOT = os.path.abspath(os.path.join(
     os.path.dirname(__file__), '..', '..'))
 
 # Import third_party/PRESUBMIT.py via imp to avoid importing a random
 # PRESUBMIT.py from $PATH, also make sure we don't generate a .pyc file.
 sys.dont_write_bytecode = True
 third_party = \
   imp.load_source('PRESUBMIT', \
                   os.path.join(REPOSITORY_ROOT, 'third_party', 'PRESUBMIT.py'))
 
+sys.path.append(os.path.join(REPOSITORY_ROOT, 'build/android/gyp/util'))
+import build_utils
 sys.path.append(os.path.join(REPOSITORY_ROOT, 'third_party'))
 import jinja2
 sys.path.append(os.path.join(REPOSITORY_ROOT, 'tools'))
 from copyright_scanner import copyright_scanner
 import licenses
 
 
 class InputApi(object):
   def __init__(self):
     self.os_path = os.path
@@ skipping 114 matching lines @@
       third_party_dirs, whitelisted_files)
 
   problem_paths.extend(more_problem_paths)
 
   return (licenses_check if all_licenses_valid else ScanResult.Errors,
           problem_paths)
 
 
 class TemplateEntryGenerator(object):
   def __init__(self):
-    self._generate_licenses_file_list_only = False
     self._toc_index = 0
 
-  def SetGenerateLicensesFileListOnly(self, generate_licenses_file_list_only):
-    self._generate_licenses_file_list_only = generate_licenses_file_list_only
-
   def _ReadFileGuessEncoding(self, name):
-    if self._generate_licenses_file_list_only:
-      return ''
     contents = ''
     with open(name, 'rb') as input_file:
       contents = input_file.read()
     try:
       return contents.decode('utf8')
     except UnicodeDecodeError:
       pass
     # If it's not UTF-8, it must be CP-1252. Fail otherwise.
     return contents.decode('cp1252')
 
   def MetadataToTemplateEntry(self, metadata):
     self._toc_index += 1
     return {
       'name': metadata['Name'],
       'url': metadata['URL'],
       'license_file': metadata['License File'],
       'license': self._ReadFileGuessEncoding(metadata['License File']),
       'toc_href': 'entry' + str(self._toc_index),
     }
 
 
-def GenerateNoticeFile(generate_licenses_file_list_only=False):
+def GenerateNoticeFile():
   """Generates the contents of an Android NOTICE file for the third-party code.
   This is used by the snapshot tool.
   Returns:
-    The contents of the NOTICE file.
+    A tuple of (input paths, contents of the NOTICE file).
   """
-
   generator = TemplateEntryGenerator()
-  generator.SetGenerateLicensesFileListOnly(generate_licenses_file_list_only)
   # Start from Chromium's LICENSE file
   entries = [generator.MetadataToTemplateEntry({
     'Name': 'The Chromium Project',
     'URL': 'http://www.chromium.org',
     'License File': os.path.join(REPOSITORY_ROOT, 'LICENSE') })
   ]
 
   third_party_dirs = licenses.FindThirdPartyDirsWithFiles(REPOSITORY_ROOT)
   # We provide attribution for all third-party directories.
   # TODO(mnaganov): Limit this to only code used by the WebView binary.
   for directory in sorted(third_party_dirs):
     try:
       metadata = licenses.ParseDir(directory, REPOSITORY_ROOT,
                                    require_license_file=False)
     except licenses.LicenseError:
       # Since this code is called during project files generation,
       # we don't want to break the it. But we assume that release
       # WebView apks are built using checkouts that pass
       # 'webview_licenses.py scan' check, thus they don't contain
       # projects with non-compatible licenses.
       continue
     license_file = metadata['License File']
     if license_file and license_file != licenses.NOT_SHIPPED:
       entries.append(generator.MetadataToTemplateEntry(metadata))
 
-  if generate_licenses_file_list_only:
-    return [entry['license_file'] for entry in entries]
-  else:
-    env = jinja2.Environment(
-      loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
-      extensions=['jinja2.ext.autoescape'])
-    template = env.get_template('licenses_notice.tmpl')
-    return template.render({ 'entries': entries }).encode('utf8')
+  entries.sort(key=lambda entry: entry['name'])
+
+  license_file_list = sorted(set([entry['license_file'] for entry in entries]))
+  license_file_list = [os.path.relpath(p) for p in license_file_list]
+  env = jinja2.Environment(
+    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
+    extensions=['jinja2.ext.autoescape'])
+  template = env.get_template('licenses_notice.tmpl')
+  notice_file_contents = template.render({'entries': entries}).encode('utf8')
+  return (license_file_list, notice_file_contents)
 
 
 def main():
   class FormatterWithNewLines(optparse.IndentedHelpFormatter):
     def format_description(self, description):
       paras = description.split('\n')
       formatted_paras = [textwrap.fill(para, self.width) for para in paras]
       return '\n'.join(formatted_paras) + '\n'
 
   parser = optparse.OptionParser(formatter=FormatterWithNewLines(),
                                  usage='%prog [options]')
   parser.add_option('--json', help='Path to JSON output file')
+  build_utils.AddDepfileOption(parser)
   parser.description = (__doc__ +
                         '\nCommands:\n'
                         '  scan Check licenses.\n'
-                        '  notice_deps Generate the list of dependencies for '
                         'Android NOTICE file.\n'
                         '  notice [file] Generate Android NOTICE file on '
                         'stdout or into |file|.\n'
                         '  display_copyrights Display autorship on the files'
                         ' using names provided via stdin.\n')
-  (options, args) = parser.parse_args()
+  options, args = parser.parse_args()
   if len(args) < 1:
     parser.print_help()
     return ScanResult.Errors
 
   if args[0] == 'scan':
     scan_result, problem_paths = _Scan()
     if scan_result == ScanResult.Ok:
       print 'OK!'
     if options.json:
       with open(options.json, 'w') as f:
         json.dump(problem_paths, f)
     return scan_result
-  elif args[0] == 'notice_deps':
-    # 'set' is used to eliminate duplicate references to the same license file.
-    print ' '.join(
-      sorted(set(GenerateNoticeFile(generate_licenses_file_list_only=True))))
-    return ScanResult.Ok
-  elif args[0] == 'gn_notice_deps':
-    # generate list for gn.
-    # 'set' is used to eliminate duplicate references to the same license file.
-    gn_file_list = ['"' + f + '"' for f in
-        sorted(set(GenerateNoticeFile(generate_licenses_file_list_only=True)))]
-    print '[%s] ' % ','.join(gn_file_list)
-    return ScanResult.Ok
   elif args[0] == 'notice':
-    notice_file_contents = GenerateNoticeFile()
+    license_file_list, notice_file_contents = GenerateNoticeFile()
     if len(args) == 1:
       print notice_file_contents
     else:
       with open(args[1], 'w') as output_file:
         output_file.write(notice_file_contents)
+    if options.depfile:
+      assert args[1]
+      # Add in build.ninja so that the target will be considered dirty whenever
+      # gn gen is run. Otherwise, it will fail to notice new files being added.
+      # This is still no perfect, as it will fail if no build files are changed,
+      # but a new README.chromium / LICENSE is added. This shouldn't happen in
+      # practice however.
+      build_utils.WriteDepfile(options.depfile, args[1],
+                               license_file_list + ['build.ninja'])
+

[Dirk Pranke, 2017/02/10 00:15:59]

I don't understand this block. Why do you want to re-run this whenever `gn gen`
is run? What are you worried about becoming out of date? 

[agrieve, 2017/02/10 00:32:35]

Before this change, it was scanning the filesystem during "gn gen", and that
would cause it to tell ninja of any new LICENSE files that popped up.

With this change, it's not scanning during gn gen, so any new third_party
subdirectories that get pulled in will not be noticed by incremental builds. So,
the assumption here is that a new third_party library being added will result in
a "gn gen", which will in turn tell the license scraper to re-scan.

     return ScanResult.Ok
   elif args[0] == 'display_copyrights':
     files = sys.stdin.read().splitlines()
     for f, c in \
         zip(files, copyright_scanner.FindCopyrights(InputApi(), '.', files)):
       print f, '\t', ' / '.join(sorted(c))
     return ScanResult.Ok
   parser.print_help()
   return ScanResult.Errors
 
 if __name__ == '__main__':
   sys.exit(main())