Fix BUILD.gn files and add presubmit step

tools/verify_source_deps.py

Index: tools/verify_source_deps.py
diff --git a/tools/verify_source_deps.py b/tools/verify_source_deps.py
index 7c147c06e038a317b0c476e7d131f2e9eed37db3..b121dd12db10e4d4a7b6bcf60b835693ba5112fe 100755
--- a/tools/verify_source_deps.py
+++ b/tools/verify_source_deps.py
@@ -16,18 +16,21 @@ tool won't be accurate for other sources in other directories (e.g. cctest).
 import itertools
 import re
 import os
+import sys
 
 
 V8_BASE = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 V8_SRC_BASE = os.path.join(V8_BASE, 'src')
+V8_TEST_BASE = os.path.join(V8_BASE, 'test')
 V8_INCLUDE_BASE = os.path.join(V8_BASE, 'include')
 
 GYP_FILES = [
   os.path.join(V8_BASE, 'src', 'd8.gyp'),
   os.path.join(V8_BASE, 'src', 'v8.gyp'),
-  os.path.join(V8_BASE, 'src', 'inspector', 'inspector.gyp'),
+  os.path.join(V8_BASE, 'src', 'inspector', 'inspector.gypi'),
   os.path.join(V8_BASE, 'src', 'third_party', 'vtune', 'v8vtune.gyp'),
   os.path.join(V8_BASE, 'test', 'cctest', 'cctest.gyp'),
+  os.path.join(V8_BASE, 'test', 'fuzzer', 'fuzzer.gyp'),
   os.path.join(V8_BASE, 'test', 'unittests', 'unittests.gyp'),
   os.path.join(V8_BASE, 'tools', 'parser-shell.gyp'),
 ]
@@ -48,25 +51,43 @@ GN_UNSUPPORTED_FEATURES = [
   'ppc',
   'qnx',
   'solaris',
-  'valgrind',
   'vtune',
   'x87',
 ]
 
+ALL_GN_PREFIXES = [
+  '..',
+  os.path.join('src', 'inspector'),
+  'src',
+  os.path.join('test', 'cctest'),
+  os.path.join('test', 'unittests'),
+]
+
+ALL_GYP_PREFIXES = [
+  '..',
+  'common',
+  os.path.join('src', 'third_party', 'vtune'),
+  'src',
+  os.path.join('test', 'cctest'),
+  os.path.join('test', 'common'),
+  os.path.join('test', 'fuzzer'),
+  os.path.join('test', 'unittests'),
+]
 
-def path_no_prefix(path):
-  for prefix in ['../', 'src/inspector/', 'src/']:
-    if path.startswith(prefix):
-      return path_no_prefix(path[len(prefix):])
+def path_no_prefix(path, prefixes):
+  for prefix in prefixes:
+    if path.startswith(prefix + os.sep):
+      return path_no_prefix(path[len(prefix) + 1:], prefixes)
   return path
 
 
-def isources(directory):
+def isources(directory, prefixes):
   for root, dirs, files in os.walk(directory):
     for f in files:
       if not (f.endswith('.h') or f.endswith('.cc')):
         continue
-      yield path_no_prefix(os.path.relpath(os.path.join(root, f), V8_BASE))
+      yield path_no_prefix(
+          os.path.relpath(os.path.join(root, f), V8_BASE), prefixes)
 
 
 def iflatten(obj):
@@ -79,7 +100,7 @@ def iflatten(obj):
       for i in iflatten(value):
         yield i
   elif isinstance(obj, basestring):
-    yield path_no_prefix(obj)
+    yield path_no_prefix(os.path.join(*os.path.split(obj)), ALL_GYP_PREFIXES)

[Michael Achenbach, 2016/09/19 09:35:33]

Should this not do a obj.split('/') instead of os.path.split (also below) in
order to transform gyp/gn slashes into os.sep?

I'd assume os.path.join(*os.path.split(X)) == X in any case, not? 

[jochen (gone - plz use gerrit), 2016/09/19 10:54:38]

I fixed that in the latest patchset.

os.path.split() splits on both / and \, but only on one, not all, so I added a
function pathsplit that splits any OS-style path into its components

 
 
 def iflatten_gyp_file(gyp_file):
@@ -100,30 +121,46 @@ def iflatten_gn_file(gn_file):
     for line in f.read().splitlines():
       match = re.match(r'.*"([^"]*)".*', line)
       if match:
-        yield path_no_prefix(match.group(1))
+        yield path_no_prefix(
+            os.path.join(*os.path.split(match.group(1))), ALL_GN_PREFIXES)
 
 
-def icheck_values(values, *source_dirs):
+def icheck_values(values, prefixes, *source_dirs):
   for source_file in itertools.chain(
-      *[isources(source_dir) for source_dir in source_dirs]
+      *[isources(source_dir, prefixes) for source_dir in source_dirs]
     ):
     if source_file not in values:
       yield source_file
 
 
-gyp_values = set(itertools.chain(
-  *[iflatten_gyp_file(gyp_file) for gyp_file in GYP_FILES]
-  ))
+def missing_gyp_files():
+  gyp_values = set(itertools.chain(
+    *[iflatten_gyp_file(gyp_file) for gyp_file in GYP_FILES]
+    ))
+  return sorted(icheck_values(
+      gyp_values, ALL_GYP_PREFIXES, V8_SRC_BASE, V8_INCLUDE_BASE, V8_TEST_BASE))
+
 
-print "----------- Files not in gyp: ------------"
-for i in sorted(icheck_values(gyp_values, V8_SRC_BASE, V8_INCLUDE_BASE)):
-  print i
+def missing_gn_files():
+  gn_values = set(itertools.chain(
+    *[iflatten_gn_file(gn_file) for gn_file in GN_FILES]
+    ))
 
-gn_values = set(itertools.chain(
-  *[iflatten_gn_file(gn_file) for gn_file in GN_FILES]
-  ))
+  gn_files = sorted(icheck_values(
+      gn_values, ALL_GN_PREFIXES, V8_SRC_BASE, V8_INCLUDE_BASE, V8_TEST_BASE))
+  return filter(
+      lambda x: not any(i in x for i in GN_UNSUPPORTED_FEATURES), gn_files)
 
-print "\n----------- Files not in gn: -------------"
-for i in sorted(icheck_values(gn_values, V8_SRC_BASE, V8_INCLUDE_BASE)):
-  if not any(f in i for f in GN_UNSUPPORTED_FEATURES):
+
+def main():
+  print "----------- Files not in gyp: ------------"
+  for i in missing_gyp_files():
+    print i
+
+  print "\n----------- Files not in gn: -------------"
+  for i in missing_gn_files():
     print i
+  return 0
+
+if '__main__' == __name__:
+  sys.exit(main())