Support checking and fixing non-existing header files in GN

build/check_gn_headers.py

 #!/usr/bin/env python
 # Copyright 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Find header files missing in GN.
 
 This script gets all the header files from ninja_deps, which is from the true
 dependency generated by the compiler, and report if they don't exist in GN.
 """
@@ skipping 89 matching lines @@
 
 def ParseWhiteList(whitelist):
   out = set()
   for line in whitelist.split('\n'):
     line = re.sub(r'#.*', '', line).strip()
     if line:
       out.add(line)
   return out
 
 
+def FilterOutDepsedRepo(files, deps):

[Nico, 2017/04/28 20:31:19]

a) isn't this slow? i would've thought that deps has many elements, and this is
in O(len(files) * len(deps))

b) why is it needed?

[wychen, 2017/04/28 22:04:43]

This is asymptotically slow, but only takes <0.1sec. The bottleneck is at ninja
deps, so I'm not too worried about the speed here.
Originally this script was intended for CQ and waterfall. The problem of having
missing GN header file is that "gn analyze" step would wrongly skip some
targets. However, for header changes due to DEPS updates, "gn analyze" would
bail to build all, so it's less bad.

The design doc discussion for reference:
https://docs.google.com/a/google.com/document/d/1TZtquZp1uv1ENe2Okk8zyyeJP03h...

We could fix these as well, but the priority should be on headers that could
lead to practical problems.

+  return {f for f in files if not any(f.startswith(d) for d in deps)}
+
+
+def GetNonExistingFiles(lst):
+  out = set()
+  for f in lst:
+    if not os.path.isfile(f):
+      out.add(f)
+  return out
+
+
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('--out-dir', default='out/Release')
   parser.add_argument('--json')
   parser.add_argument('--whitelist')
   parser.add_argument('args', nargs=argparse.REMAINDER)
 
   args, _extras = parser.parse_known_args()
 
   d_q = Queue()
   d_p = Process(target=GetHeadersFromNinja, args=(args.out_dir, d_q,))
   d_p.start()
 
   gn_q = Queue()
   gn_p = Process(target=GetHeadersFromGN, args=(args.out_dir, gn_q,))
   gn_p.start()
 
   deps_q = Queue()
   deps_p = Process(target=GetDepsPrefixes, args=(deps_q,))
   deps_p.start()
 
   d = d_q.get()
+  assert len(GetNonExistingFiles(d)) == 0, \
+      'Found non-existing files in ninja deps'
   gn = gn_q.get()
   missing = d - gn
+  extra = GetNonExistingFiles(gn)
 
   deps = deps_q.get()
-  missing = {m for m in missing if not any(m.startswith(d) for d in deps)}
+  missing = FilterOutDepsedRepo(missing, deps)
+  extra = FilterOutDepsedRepo(extra, deps)
 
   d_p.join()
   gn_p.join()
   deps_p.join()
 
   if args.whitelist:
     whitelist = ParseWhiteList(open(args.whitelist).read())
     missing -= whitelist
 
   missing = sorted(missing)
+  extra = sorted(extra)
 
   if args.json:
     with open(args.json, 'w') as f:
       json.dump(missing, f)
 
-  if len(missing) == 0:
+  if len(missing) == 0 and len(extra) == 0:
     return 0
 
-  print 'The following files should be included in gn files:'
-  for i in missing:
-    print i
+  if len(missing) > 0:
+    print '\nThe following files should be included in gn files:'
+    for i in missing:
+      print i
+
+  if len(extra) > 0:
+    print '\nThe following files should be removed from gn files:'
+    for i in extra:
+      print i
+
   return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())