check_gn_headers: use pinned depot_tools

build/check_gn_headers.py

 #!/usr/bin/env python
 # Copyright 2017 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Find header files missing in GN.
 
 This script gets all the header files from ninja_deps, which is from the true
 dependency generated by the compiler, and report if they don't exist in GN.
 """
 
 import argparse
 import json
 import os
 import re
 import shutil
 import subprocess
 import sys
 import tempfile
 from multiprocessing import Process, Queue
 
+SRC_DIR = os.path.abspath(
+    os.path.join(os.path.abspath(os.path.dirname(__file__)), os.path.pardir))
+DEPOT_TOOLS_DIR = os.path.join(SRC_DIR, 'depot_tools')

[Dirk Pranke, 2017/06/07 01:29:53]

This needs to be os.path.join(SRC_DIR, 'third_party', 'depot_tools').

+
 
 def GetHeadersFromNinja(out_dir, q):
   """Return all the header files from ninja_deps"""
 
   def NinjaSource():
-    cmd = ['ninja', '-C', out_dir, '-t', 'deps']
+    cmd = [os.path.join(DEPOT_TOOLS_DIR, 'ninja'), '-C', out_dir, '-t', 'deps']

[wychen, 2017/06/07 01:27:03]

I don't have src/depot_tools in my checkout. Is there a pending CL adding that?
Would it be safer to prepend the pinned depot_tools to PATH? If we are sure
src/depot_tools would be available on all bots and dev environments, then this
is fine.

[Dirk Pranke, 2017/06/07 01:29:53]

Good catch, see comment above.

[wychen, 2017/06/07 01:32:20]

Ah. I haven't updated to https://chromium-review.googlesource.com/c/522802/.

     # A negative bufsize means to use the system default, which usually
     # means fully buffered.
     popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=-1)
     for line in iter(popen.stdout.readline, ''):
       yield line.rstrip()
 
     popen.stdout.close()
     return_code = popen.wait()
     if return_code:
       raise subprocess.CalledProcessError(return_code, cmd)
@@ skipping 37 matching lines @@
 def GetHeadersFromGN(out_dir, q):
   """Return all the header files from GN"""
 
   tmp = None
   ans, err = set(), None
   try:
     tmp = tempfile.mkdtemp()
     shutil.copy2(os.path.join(out_dir, 'args.gn'),
                  os.path.join(tmp, 'args.gn'))
     # Do "gn gen" in a temp dir to prevent dirtying |out_dir|.
-    subprocess.check_call(['gn', 'gen', tmp, '--ide=json', '-q'])
+    subprocess.check_call([
+      os.path.join(DEPOT_TOOLS_DIR, 'gn'), 'gen', tmp, '--ide=json', '-q'])
     gn_json = json.load(open(os.path.join(tmp, 'project.json')))
     ans = ParseGNProjectJSON(gn_json, out_dir, tmp)
   except Exception as e:
     err = str(e)
   finally:
     if tmp:
       shutil.rmtree(tmp)
   q.put((ans, err))
 
 
@@ skipping 15 matching lines @@
             f = out_dir + f[len(tmp_out):]
           all_headers.add(f)
 
   return all_headers
 
 
 def GetDepsPrefixes(q):
   """Return all the folders controlled by DEPS file"""
   prefixes, err = set(), None
   try:
-    gclient_out = subprocess.check_output(
-        ['gclient', 'recurse', '--no-progress', '-j1',
-         'python', '-c', 'import os;print os.environ["GCLIENT_DEP_PATH"]'])
+    gclient_out = subprocess.check_output([
+      os.path.join(DEPOT_TOOLS_DIR, 'gclient'),
+      'recurse', '--no-progress', '-j1',
+      'python', '-c', 'import os;print os.environ["GCLIENT_DEP_PATH"]'])
     for i in gclient_out.split('\n'):
       if i.startswith('src/'):
         i = i[4:]
         prefixes.add(i)
   except Exception as e:
     err = str(e)
   q.put((prefixes, err))
 
 
 def IsBuildClean(out_dir):
-  cmd = ['ninja', '-C', out_dir, '-n']
+  cmd = [os.path.join(DEPOT_TOOLS_DIR, 'ninja'), '-C', out_dir, '-n']
   out = subprocess.check_output(cmd)
   return 'no work to do.' in out
 
 
 def ParseWhiteList(whitelist):
   out = set()
   for line in whitelist.split('\n'):
     line = re.sub(r'#.*', '', line).strip()
     if line:
       out.add(line)
@@ skipping 113 matching lines @@
   if len(nonexisting) > 0:
     print '\nThe following non-existing files should be removed from gn files:'
     for i in nonexisting:
       print i
 
   return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())