Use git ls-tree to collect source files instead of walking the fs

tools/verify_source_deps.py

Index: tools/verify_source_deps.py
diff --git a/tools/verify_source_deps.py b/tools/verify_source_deps.py
index 941943bf17a44547b2ba969bb9606e27936267f4..23b7492f1799d5c2d36ca6c4dbfe2aa5f1eff97a 100755
--- a/tools/verify_source_deps.py
+++ b/tools/verify_source_deps.py
@@ -15,6 +15,7 @@ is referenced from a gyp/gn file, we won't necessarily detect it.
 import itertools
 import re
 import os
+import subprocess
 import sys
 
 
@@ -28,14 +29,37 @@ GYP_FILES = [
   os.path.join(V8_BASE, 'src', 'v8.gyp'),
   os.path.join(V8_BASE, 'src', 'inspector', 'inspector.gypi'),
   os.path.join(V8_BASE, 'src', 'third_party', 'vtune', 'v8vtune.gyp'),
+  os.path.join(V8_BASE, 'samples', 'samples.gyp'),
   os.path.join(V8_BASE, 'test', 'cctest', 'cctest.gyp'),
   os.path.join(V8_BASE, 'test', 'fuzzer', 'fuzzer.gyp'),
   os.path.join(V8_BASE, 'test', 'unittests', 'unittests.gyp'),
+  os.path.join(V8_BASE, 'testing', 'gmock.gyp'),

[Michael Achenbach, 2016/09/19 15:08:21]

Why do we need to list third_party gyp files?

[jochen (gone - plz use gerrit), 2016/09/19 15:11:46]

it's not third_party, but checked into v8 

[Michael Achenbach, 2016/09/19 15:16:30]

Acknowledged.

+  os.path.join(V8_BASE, 'testing', 'gtest.gyp'),
   os.path.join(V8_BASE, 'tools', 'parser-shell.gyp'),
 ]
 
+ALL_GYP_PREFIXES = [
+  '..',
+  'common',
+  os.path.join('src', 'third_party', 'vtune'),
+  'src',
+  'samples',
+  'testing',
+  'tools',
+  os.path.join('test', 'cctest'),
+  os.path.join('test', 'common'),
+  os.path.join('test', 'fuzzer'),
+  os.path.join('test', 'unittests'),
+]
+
+GYP_UNSUPPORTED_FEATURES = [
+  'gcmole',
+]
+
 GN_FILES = [
   os.path.join(V8_BASE, 'BUILD.gn'),
+  os.path.join(V8_BASE, 'build', 'secondary', 'testing', 'gmock', 'BUILD.gn'),
+  os.path.join(V8_BASE, 'build', 'secondary', 'testing', 'gtest', 'BUILD.gn'),
   os.path.join(V8_BASE, 'src', 'inspector', 'BUILD.gn'),
   os.path.join(V8_BASE, 'test', 'cctest', 'BUILD.gn'),
   os.path.join(V8_BASE, 'test', 'unittests', 'BUILD.gn'),
@@ -46,10 +70,12 @@ GN_UNSUPPORTED_FEATURES = [
   'aix',
   'cygwin',
   'freebsd',
+  'gcmole',
   'openbsd',
   'ppc',
   'qnx',
   'solaris',
+  'testing',
   'vtune',
   'x87',
 ]
@@ -62,17 +88,6 @@ ALL_GN_PREFIXES = [
   os.path.join('test', 'unittests'),
 ]
 
-ALL_GYP_PREFIXES = [
-  '..',
-  'common',
-  os.path.join('src', 'third_party', 'vtune'),
-  'src',
-  os.path.join('test', 'cctest'),
-  os.path.join('test', 'common'),
-  os.path.join('test', 'fuzzer'),
-  os.path.join('test', 'unittests'),
-]
-
 def pathsplit(path):
   return re.split('[/\\\\]', path)
 
@@ -83,13 +98,12 @@ def path_no_prefix(path, prefixes):
   return path
 
 
-def isources(directory, prefixes):
-  for root, dirs, files in os.walk(directory):
-    for f in files:
-      if not (f.endswith('.h') or f.endswith('.cc')):
-        continue
-      yield path_no_prefix(
-          os.path.relpath(os.path.join(root, f), V8_BASE), prefixes)
+def isources(prefixes):
+  cmd = ['git', 'ls-tree', '-r', 'HEAD', '--full-name', '--name-only']

[Michael Achenbach, 2016/09/19 15:12:48]

This might now work on windows (not sure if that's a goal). Might require
git.bat

[Michael Achenbach, 2016/09/19 15:13:16]

And with now I mean not :)

[jochen (gone - plz use gerrit), 2016/09/19 15:14:39]

Well, I developed this CL on windows... :)

but I'm using mingw, I can try with cmd as well

[Michael Achenbach, 2016/09/19 15:16:30]

Don't bother.

+  for f in subprocess.check_output(cmd).split('\n'):

[Michael Achenbach, 2016/09/19 15:12:48]

Unlikely that this fails after it first works, but if it does, I guess we won't
see the output.

[jochen (gone - plz use gerrit), 2016/09/19 15:14:39]

ah, I guess I forgot to add universal_newlines=True

+    if not (f.endswith('.h') or f.endswith('.cc')):
+      continue
+    yield path_no_prefix(os.path.join(*pathsplit(f)), prefixes)
 
 
 def iflatten(obj):
@@ -127,10 +141,8 @@ def iflatten_gn_file(gn_file):
             os.path.join(*pathsplit(match.group(1))), ALL_GN_PREFIXES)
 
 
-def icheck_values(values, prefixes, *source_dirs):
-  for source_file in itertools.chain(
-      *[isources(source_dir, prefixes) for source_dir in source_dirs]
-    ):
+def icheck_values(values, prefixes):
+  for source_file in isources(prefixes):
     if source_file not in values:
       yield source_file
 
@@ -139,8 +151,9 @@ def missing_gyp_files():
   gyp_values = set(itertools.chain(
     *[iflatten_gyp_file(gyp_file) for gyp_file in GYP_FILES]
     ))
-  return sorted(icheck_values(
-      gyp_values, ALL_GYP_PREFIXES, V8_SRC_BASE, V8_INCLUDE_BASE, V8_TEST_BASE))
+  gyp_files = sorted(icheck_values(gyp_values, ALL_GYP_PREFIXES))
+  return filter(
+      lambda x: not any(i in x for i in GYP_UNSUPPORTED_FEATURES), gyp_files)
 
 
 def missing_gn_files():
@@ -148,8 +161,7 @@ def missing_gn_files():
     *[iflatten_gn_file(gn_file) for gn_file in GN_FILES]
     ))
 
-  gn_files = sorted(icheck_values(
-      gn_values, ALL_GN_PREFIXES, V8_SRC_BASE, V8_INCLUDE_BASE, V8_TEST_BASE))
+  gn_files = sorted(icheck_values(gn_values, ALL_GN_PREFIXES))
   return filter(
       lambda x: not any(i in x for i in GN_UNSUPPORTED_FEATURES), gn_files)