Print detailed info in check_gn_headers.py

build/check_gn_headers.py

 #!/usr/bin/env python
 # Copyright 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Find header files missing in GN.
 
 This script gets all the header files from ninja_deps, which is from the true
 dependency generated by the compiler, and report if they don't exist in GN.
 """
 
 import argparse
 import json
 import os
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 from multiprocessing import Process, Queue
 
 SRC_DIR = os.path.abspath(
     os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir))
 DEPOT_TOOLS_DIR = os.path.join(SRC_DIR, 'third_party', 'depot_tools')
 
 
-def GetHeadersFromNinja(out_dir, q):
+def GetHeadersFromNinja(out_dir, skip_obj, q):
   """Return all the header files from ninja_deps"""
 
   def NinjaSource():
     cmd = [os.path.join(DEPOT_TOOLS_DIR, 'ninja'), '-C', out_dir, '-t', 'deps']
     # A negative bufsize means to use the system default, which usually
     # means fully buffered.
     popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=-1)
     for line in iter(popen.stdout.readline, ''):
       yield line.rstrip()
 
     popen.stdout.close()
     return_code = popen.wait()
     if return_code:
       raise subprocess.CalledProcessError(return_code, cmd)
 
   ans, err = set(), None
   try:
-    ans = ParseNinjaDepsOutput(NinjaSource(), out_dir)
+    ans = ParseNinjaDepsOutput(NinjaSource(), out_dir, skip_obj)
   except Exception as e:
     err = str(e)
   q.put((ans, err))
 
 
-def ParseNinjaDepsOutput(ninja_out, out_dir):
+def ParseNinjaDepsOutput(ninja_out, out_dir, skip_obj):
   """Parse ninja output and get the header files"""
-  all_headers = set()
+  all_headers = {}
 
   # Ninja always uses "/", even on Windows.
   prefix = '../../'
 
   is_valid = False
+  obj_file = ''
   for line in ninja_out:
     if line.startswith('    '):
       if not is_valid:
         continue
       if line.endswith('.h') or line.endswith('.hh'):
         f = line.strip()
         if f.startswith(prefix):
           f = f[6:]  # Remove the '../../' prefix
           # build/ only contains build-specific files like build_config.h
           # and buildflag.h, and system header files, so they should be
           # skipped.
           if f.startswith(out_dir) or f.startswith('out'):
             continue
           if not f.startswith('build'):
-            all_headers.add(f)
+            all_headers.setdefault(f, [])
+            if not skip_obj:
+              all_headers[f].append(obj_file)
     else:
       is_valid = line.endswith('(VALID)')
+      obj_file = line.split(':')[0]
 
   return all_headers
 
 
 def GetHeadersFromGN(out_dir, q):
   """Return all the header files from GN"""
 
   tmp = None
   ans, err = set(), None
   try:
     # Argument |dir| is needed to make sure it's on the same drive on Windows.
     # dir='' means dir='.', but doesn't introduce an unneeded prefix.
     tmp = tempfile.mkdtemp(dir='')
     shutil.copy2(os.path.join(out_dir, 'args.gn'),
                  os.path.join(tmp, 'args.gn'))
     # Do "gn gen" in a temp dir to prevent dirtying |out_dir|.
     gn_exe = 'gn.bat' if sys.platform == 'win32' else 'gn'
     subprocess.check_call([
-      os.path.join(DEPOT_TOOLS_DIR, gn_exe), 'gen', tmp, '--ide=json', '-q'])
+        os.path.join(DEPOT_TOOLS_DIR, gn_exe), 'gen', tmp, '--ide=json', '-q'])
     gn_json = json.load(open(os.path.join(tmp, 'project.json')))
     ans = ParseGNProjectJSON(gn_json, out_dir, tmp)
   except Exception as e:
     err = str(e)
   finally:
     if tmp:
       shutil.rmtree(tmp)
   q.put((ans, err))
 
 
@@ skipping 78 matching lines @@
   parser = argparse.ArgumentParser(description='''
       NOTE: Use ninja to build all targets in OUT_DIR before running
       this script.''')
   parser.add_argument('--out-dir', metavar='OUT_DIR', default='out/Release',
                       help='output directory of the build')
   parser.add_argument('--json',
                       help='JSON output filename for missing headers')
   parser.add_argument('--whitelist', help='file containing whitelist')
   parser.add_argument('--skip-dirty-check', action='store_true',
                       help='skip checking whether the build is dirty')
+  parser.add_argument('--verbose', action='store_true',
+                      help='print more diagnostic info')
 
   args, _extras = parser.parse_known_args()
 
   if not os.path.isdir(args.out_dir):
     parser.error('OUT_DIR "%s" does not exist.' % args.out_dir)
 
   if not args.skip_dirty_check and not IsBuildClean(args.out_dir):
     dirty_msg = 'OUT_DIR looks dirty. You need to build all there.'
     if args.json:
       # Assume running on the bots. Silently skip this step.
       # This is possible because "analyze" step can be wrong due to
       # underspecified header files. See crbug.com/725877
       print dirty_msg
       DumpJson([])
       return 0
     else:
       # Assume running interactively.
       parser.error(dirty_msg)
 
   d_q = Queue()
-  d_p = Process(target=GetHeadersFromNinja, args=(args.out_dir, d_q,))
+  d_p = Process(target=GetHeadersFromNinja, args=(args.out_dir, True, d_q,))
   d_p.start()
 
   gn_q = Queue()
   gn_p = Process(target=GetHeadersFromGN, args=(args.out_dir, gn_q,))
   gn_p.start()
 
   deps_q = Queue()
   deps_p = Process(target=GetDepsPrefixes, args=(deps_q,))
   deps_p.start()
 
   d, d_err = d_q.get()
   gn, gn_err = gn_q.get()
-  missing = d - gn
+  missing = set(d.keys()) - gn
   nonexisting = GetNonExistingFiles(gn)
 
   deps, deps_err = deps_q.get()
   missing = FilterOutDepsedRepo(missing, deps)
   nonexisting = FilterOutDepsedRepo(nonexisting, deps)
 
   d_p.join()
   gn_p.join()
   deps_p.join()
 
@@ skipping 28 matching lines @@
   if len(missing) > 0:
     print '\nThe following files should be included in gn files:'
     for i in missing:
       print i
 
   if len(nonexisting) > 0:
     print '\nThe following non-existing files should be removed from gn files:'
     for i in nonexisting:
       print i
 
+  if args.verbose:
+    # Only get detailed obj dependency here since it is slower.
+    GetHeadersFromNinja(args.out_dir, False, d_q)
+    d, d_err = d_q.get()
+    print '\nDetailed dependency info:'
+    for f in missing:
+      print f
+      for cc in d[f]:
+        print '  ', cc
+
+    print '\nMissing headers sorted by number of affected object files:'
+    count = {k: len(v) for (k, v) in d.iteritems()}
+    for f in sorted(count, key=count.get, reverse=True):
+      if f in missing:
+        print count[f], f
+
   return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())