Add check_gn_headers.py to find missing headers in GN

build/check_gn_headers.py

+#!/usr/bin/env python
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Find header files missing in GN.
+
+This script gets all the header files from ninja_deps, which is from the true
+dependency generated by the compiler, and report if they don't exist in GN.
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+
+def get_d_headers(out_dir):

[Dirk Pranke, 2017/01/09 06:47:49]

Chromium Python style is (sadly) to use InitialCaps rather than
unix_hacker_style hames.

[wychen, 2017/01/09 11:26:38]

Done.

+  """Return all the header files from ninja_deps"""
+  ninja_out = subprocess.check_output(['ninja', '-C', out_dir, '-t', 'deps'])
+  return extract_d_headers(ninja_out)
+
+
+def extract_d_headers(ninja_out):

[Dirk Pranke, 2017/01/09 06:47:49]

extract_d_headers is a weird function name. It's not really "extracting"
anything, and it's unclear what "d" refers to (same thing with the
"get_d_headers" above).

I'd call this something like ParseNinjaDepsOutput().

[wychen, 2017/01/09 11:26:38]

Done.

+  """Extract header files from ninja output"""
+  all_headers = set()
+
+  prefix = '..' + os.sep + '..' + os.sep
+
+  is_valid = False
+  for line in ninja_out.split('\n'):
+    if line.startswith('    '):
+      if not is_valid:
+        continue
+      if line.endswith('.h') or line.endswith('.hh'):
+        f = line.strip()
+        if f.startswith(prefix):
+          f = f[6:]  # Remove the '../../' prefix
+          # build/ only contains build-specific files like build_config.h
+          # and buildflag.h, and system header files, so they should be
+          # skipped.
+          if not f.startswith('build'):
+            all_headers.add(f)
+    else:
+      is_valid = line.endswith('(VALID)')
+
+  return all_headers
+
+
+def get_gn_headers(out_dir):
+  """Return all the header files from GN"""
+  subprocess.check_call(['gn', 'gen', out_dir, '--ide=json', '-q'])
+  gn_json = json.load(open(os.path.join(out_dir, 'project.json')))
+  return extract_gn_headers(gn_json)
+
+
+def extract_gn_headers(gn):

[Dirk Pranke, 2017/01/09 06:47:49]

I'd call this something more like "GetHeadersFromGNProjectJSON"

[wychen, 2017/01/09 11:26:38]

I use ParseGNProjectJSON() so it's parallel to ParseNinjaDepsOutput().

+  """Extract header files from GN output"""
+  all_headers = set()
+
+  for target, properties in gn['targets'].iteritems():
+    for f in properties.get('sources', []):
+      if f.endswith('.h') or f.endswith('.hh'):
+        if f.startswith('//'):
+          f = f[2:]  # Strip the '//' prefix.
+          all_headers.add(f)
+
+  return all_headers
+
+
+def get_deps_prefixes(deps_file):
+  """Return all the folders controlled by DEPS file"""
+  return extract_deps_prefixes(open(deps_file).read())
+
+
+def extract_deps_prefixes(deps):
+  var_def = 'def Var(a): return ""\n'
+
+  exec(var_def + deps)

[Dirk Pranke, 2017/01/09 06:47:49]

Please add a comment about what lines 78-80 are doing (loading the DEPS file and
stubbing out the Var() function).

[wychen, 2017/01/09 11:26:38]

Done.

+  all_prefixes = set()
+  for f in deps.keys():
+    if f.startswith('src/'):
+      f = f[4:]
+      all_prefixes.add(f)
+  for i in deps_os.keys():
+    for f in deps_os[i].keys():
+      if f.startswith('src/'):
+        f = f[4:]
+        all_prefixes.add(f)
+  return all_prefixes
+
+
+def starts_with_any(prefixes, string):
+  for p in prefixes:
+    if string.startswith(p):
+      return True
+  return False
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--out-dir', default='out/Release')
+  parser.add_argument('--deps-file', default='DEPS')
+  parser.add_argument('--json')
+  parser.add_argument('--whitelist')
+  parser.add_argument('args', nargs=argparse.REMAINDER)
+
+  args, _ = parser.parse_known_args()
+
+  d = get_d_headers(args.out_dir)
+  gn = get_gn_headers(args.out_dir)
+  missing = d - gn
+
+  deps = get_deps_prefixes(args.deps_file)
+  missing = set(filter(lambda x: not starts_with_any(deps, x), missing))
+
+  if args.whitelist:
+    whitelist = set(open(args.whitelist).read().split('\n'))

[Dirk Pranke, 2017/01/09 06:47:49]

Please change this to be able to handle a file format that has comments in it (I
suggest stripping blank lines and "# comment"s); you'll want to annotate the
whitelist sooner or later.

[wychen, 2017/01/09 11:26:38]

Great idea!

+    missing -= whitelist
+
+  missing = sorted(missing)
+
+  if args.json:
+    with open(args.json, 'w') as f:
+      json.dump(missing, f)
+
+  if len(missing) == 0:
+    return 0
+
+  print 'The following files should be included in gn files:'
+  for i in missing:
+    print i
+  return 1
+
+
+if __name__ == '__main__':
+  sys.exit(main())