Support checking and fixing non-existing header files in GN

build/check_gn_headers.py

Index: build/check_gn_headers.py
diff --git a/build/check_gn_headers.py b/build/check_gn_headers.py
index be1e797a77e381623735d75de356268279721fa9..71f394805117c83eb83446858b0e87dcda8e4d56 100755
--- a/build/check_gn_headers.py
+++ b/build/check_gn_headers.py
@@ -107,6 +107,18 @@ def ParseWhiteList(whitelist):
   return out
 
 
+def FilterOutDepsedRepo(files, deps):

[Nico, 2017/04/28 20:31:19]

a) isn't this slow? i would've thought that deps has many elements, and this is
in O(len(files) * len(deps))

b) why is it needed?

[wychen, 2017/04/28 22:04:43]

This is asymptotically slow, but only takes <0.1sec. The bottleneck is at ninja
deps, so I'm not too worried about the speed here.
Originally this script was intended for CQ and waterfall. The problem of having
missing GN header file is that "gn analyze" step would wrongly skip some
targets. However, for header changes due to DEPS updates, "gn analyze" would
bail to build all, so it's less bad.

The design doc discussion for reference:
https://docs.google.com/a/google.com/document/d/1TZtquZp1uv1ENe2Okk8zyyeJP03h...

We could fix these as well, but the priority should be on headers that could
lead to practical problems.

+  return {f for f in files if not any(f.startswith(d) for d in deps)}
+
+
+def GetNonExistingFiles(lst):
+  out = set()
+  for f in lst:
+    if not os.path.isfile(f):
+      out.add(f)
+  return out
+
+
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('--out-dir', default='out/Release')
@@ -129,11 +141,15 @@ def main():
   deps_p.start()
 
   d = d_q.get()
+  assert len(GetNonExistingFiles(d)) == 0, \
+      'Found non-existing files in ninja deps'
   gn = gn_q.get()
   missing = d - gn
+  extra = GetNonExistingFiles(gn)
 
   deps = deps_q.get()
-  missing = {m for m in missing if not any(m.startswith(d) for d in deps)}
+  missing = FilterOutDepsedRepo(missing, deps)
+  extra = FilterOutDepsedRepo(extra, deps)
 
   d_p.join()
   gn_p.join()
@@ -144,17 +160,25 @@ def main():
     missing -= whitelist
 
   missing = sorted(missing)
+  extra = sorted(extra)
 
   if args.json:
     with open(args.json, 'w') as f:
       json.dump(missing, f)
 
-  if len(missing) == 0:
+  if len(missing) == 0 and len(extra) == 0:
     return 0
 
-  print 'The following files should be included in gn files:'
-  for i in missing:
-    print i
+  if len(missing) > 0:
+    print '\nThe following files should be included in gn files:'
+    for i in missing:
+      print i
+
+  if len(extra) > 0:
+    print '\nThe following files should be removed from gn files:'
+    for i in extra:
+      print i
+
   return 1