Clean net_test_support_data_sources and net_unittests_data_source.

net/data/update_net_gypi.py

Index: net/data/update_net_gypi.py
diff --git a/net/data/update_net_gypi.py b/net/data/update_net_gypi.py
new file mode 100644
index 0000000000000000000000000000000000000000..fccebaed19cb5cabf5f499b565878ad1df225bdf
--- /dev/null
+++ b/net/data/update_net_gypi.py
@@ -0,0 +1,145 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import re
+import sys
+
+try:
+  import cStringIO as StringIO

[eroman, 2016/04/27 16:23:14]

I would remove this and just import StringIO -- I don't think this level of
optimization for this kind of script is necessary.

[sdefresne, 2016/04/30 12:31:40]

Done.

+except ImportError:
+  import StringIO
+
+
+VARIABLE_PATTERN = re.compile("^(?P<indentation>\s*)'(?P<name>[^']*)':\s*\[$")
+EXCLUSION_PATTERN = re.compile("^(?:README|OWNERS|.*\.(pyc?|sh))$")

[eroman, 2016/04/27 16:23:14]

Can also exclude editor swap files:

*.swp
*~

When I ran the script in the past turned out I had some open files and they got
added too :)

[sdefresne, 2016/04/30 12:31:40]

Done.

+
+DATA_SOURCES_PATH_FOR_VARIABLES = {
+  "net_test_support_data_sources": [
+    "net/data/ssl/certificates",
+  ],
+  "net_unittests_data_sources": [
+    "net/data/certificate_policies_unittest",
+    "net/data/name_constraints_unittest",
+    "net/data/parse_certificate_unittest",
+    "net/data/parse_ocsp_unittest",
+    "net/data/test.html",
+    "net/data/url_request_unittest",
+    "net/data/verify_certificate_chain_unittest",
+    "net/data/verify_name_match_unittest/names",
+    "net/data/verify_signed_data_unittest",
+    "net/third_party/nist-pkits/certs",
+    "net/third_party/nist-pkits/crls",
+  ],
+}
+
+
+def RelativePath(dir, path):

[eroman, 2016/04/27 16:23:14]

style: functions_are_like_this() --
https://google.github.io/styleguide/pyguide.html?showone=Naming#Naming

This applies throughout.

[sdefresne, 2016/04/30 12:31:41]

Done.

+  """Converts |path| to be relative to |dir|.

[eroman, 2016/04/27 16:23:14]

Can you use os.path.relpath() ?

Or alternately use relative paths throughout, and only join the paths when
needed for file operations.

[sdefresne, 2016/04/30 12:31:40]

Done.

+
+  Args:
+    dir: string, a directory path
+    path: string, path that should be converted to relative path.
+  """
+  if not dir or dir == os.path.curdir or os.path.isabs(path):
+    return path
+  if dir == os.path.pardir:
+    return os.path.join(dir, path)
+  if dir == path:
+    return os.path.curdir
+  if path.startswith(dir) and path[len(dir)] in (os.path.sep, os.path.altsep):
+    return path[len(dir) + 1:]
+  return os.path.join(os.path.pardir, RelativePath(os.path.dirname(dir), path))
+
+
+def ListDataSources(root, paths, exclusion):
+  """Returns the list of data source found in |paths|.
+
+  Args:
+    root: string, path to the repository root
+    paths: list of string, paths relative to repository root
+    exclusion: compiled regular expression, filename matching this pattern
+        will be excluded from the result
+  """
+  data_sources = []
+  for path in paths:
+    fullpath = os.path.normpath(os.path.join(root, path))
+    if os.path.isfile(fullpath):
+      if not exclusion.match(os.path.basename(path)):
+        data_sources.append(path)
+      continue
+
+    for dirpath, dirnames, filenames in os.walk(fullpath):
+      for filename in filenames:
+        if not exclusion.match(filename):
+          data_sources.append(os.path.normpath(os.path.join(dirpath, filename)))
+  return data_sources
+
+
+def FormatDataSources(name, dir, data_sources, indentation):
+  """Converts |data_sources| to a gyp variable assignment.
+
+  Args:
+    name: string, name of the variable
+    dir: string, path to the directory containing the gyp file
+    data_sources: list of filenames
+    indentation: string
+  """
+  buffer = StringIO.StringIO()
+  buffer.write("%s'%s': [\n" % (indentation, name))
+  for data_source in sorted(data_sources):
+    buffer.write("  %s'%s',\n" % (indentation, RelativePath(dir, data_source)))
+  buffer.write("%s],\n" % (indentation,))
+  return buffer.getvalue()
+
+
+def EditFile(path, root, data_sources_for_variables):
+  """Updates file at |path| by rewriting variables values.
+
+  Args:
+    path: string, path of the file to edit
+    root: string, path to the repository root
+    data_sources_for_variables: dictionary mapping variable names to
+        the list of data sources to use
+  """
+  dir = RelativePath(root, os.path.dirname(path))
+  buffer = StringIO.StringIO()
+  with open(path, 'r') as file:
+    indentation = ""
+    current_var = None
+    for line in file:
+      if not current_var:
+        match = VARIABLE_PATTERN.match(line)
+        if not match:
+          buffer.write(line)
+          continue
+        variable = match.group('name')
+        if variable not in data_sources_for_variables:
+          buffer.write(line)
+          continue
+        current_var = variable
+        indentation = match.group('indentation')
+        buffer.write(FormatDataSources(
+            variable, dir, data_sources_for_variables[variable], indentation))
+      else:
+        if line == indentation + '],\n':
+          current_var = None
+  with open(path, 'w') as file:
+    file.write(buffer.getvalue())

[eroman, 2016/04/27 16:23:14]

nit: I suggest including an output message from the script indicating what file
was written to.

[sdefresne, 2016/04/30 12:31:41]

Done.

+
+
+def Main(args):
+  root_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), "../.."))

[eroman, 2016/04/27 16:23:14]

nit: rather than "../../" would be more general to call os.path.dirname() twice.
Or join with os.pardir.

[sdefresne, 2016/04/30 12:31:41]

Done.

+  net_gypi = os.path.normpath(os.path.join(root_dir, "net/net.gypi"))
+
+  data_sources_for_variables = {}
+  for variable in DATA_SOURCES_PATH_FOR_VARIABLES:
+    data_sources_for_variables[variable] = ListDataSources(
+        root_dir, DATA_SOURCES_PATH_FOR_VARIABLES[variable], EXCLUSION_PATTERN)
+
+  EditFile(net_gypi, root_dir, data_sources_for_variables)
+
+
+if __name__ == '__main__':
+  sys.exit(Main(sys.argv[1:]))