Script to extract components from OWNERS files throughout the repo.

tools/checkteamtags/extract_components.py

Index: tools/checkteamtags/extract_components.py
diff --git a/tools/checkteamtags/extract_components.py b/tools/checkteamtags/extract_components.py
new file mode 100755
index 0000000000000000000000000000000000000000..143aac6c5320e01ddfa14ebd6f5536d8282a5032
--- /dev/null
+++ b/tools/checkteamtags/extract_components.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+# Copyright (c) 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Parses OWNERS recursively and generates a machine readable component mapping.
+
+OWNERS files are expected to contain a well-formatted pair of tags as shown
+below.  A presubmit check exists that validates this.
+
+This script finds lines in the OWNERS files such as:
+  `# TEAM: team@chromium.org` and
+  `# COMPONENT: Tools>Test>Findit`
+and dumps this information into a json file.
+
+Refer to crbug.com/667952
+"""
+
+import json
+import optparse
+import os
+import re
+import sys
+
+from collections import defaultdict
+
+
+_DEFAULT_SRC_LOCATION = os.path.join(
+    os.path.dirname(__file__), os.pardir, os.pardir)
+
+_README = """
+This file is generated by src/tools/checkteamtags/extract_components.py
+by parsing the contents of OWNERS files throughout the chromium source code and
+extracting `# TEAM:` and `# COMPONENT:` tags.
+
+Manual edits of this file will be overwritten by an automated process.
+""".splitlines()
+
+def parse(filename):
+  """Search the file for lines that start with `# TEAM:` or `# COMPONENT:`.
+
+  Args:
+    filename (str): path to the file to parse.
+  Returns:
+    (team (str), component(str)): The team and component found in the file, the
+        last one of each if multiple, None if missing.
+  """
+  team = None
+  component = None
+  team_regex = re.compile('\s*#\s*TEAM\s*:\s*(\S+)')
+  component_regex = re.compile('\s*#\s*COMPONENT\s*:\s*(\S+)')
+  with open(filename) as f:
+    for line in f:
+      team_matches = team_regex.match(line)
+      if team_matches:
+        team = team_matches.group(1)
+      component_matches = component_regex.match(line)
+      if component_matches:
+        component = component_matches.group(1)
+  return team, component
+
+
+def traverse(root):
+  """Traverse the given dir and parse OWNERS files for team and component tags.
+
+  Args:
+    root (str): the path to the src directory.
+
+  Returns:
+    A pair (data, warnings) where data is a dict of the form
+      {'component-to-team': {'Component1': 'team1@chr...', ...},
+       'dir-to-component': {'/path/to/1': 'Component1', ...}}
+      and warnings is a list of strings.
+  """
+  warnings = []
+  component_to_team = defaultdict(set)
+  dir_to_component = {}
+  for dirname, _, files in os.walk(root):
+    # Proofing against windows casing oddities.
+    owners_file_names = [f for f in files if f.upper() == 'OWNERS']
+    if owners_file_names:
+      team, component = parse(os.path.join(dirname, owners_file_names[0]))
+      if component:
+        dir_to_component[os.path.relpath(dirname, root)] = component
+        if team:
+          component_to_team[component].add(team)
+      else:
+        warnings.append('%s has OWNERS but no COMPONENT' % dirname)
+  # Since the keys are sorted alphabetically, prepending AAA to the readme
+  # field.
+  return ({'AAA-README': _README,
+           'component-to-team': component_to_team,
+           'dir-to-component': dir_to_component},
+          warnings)
+
+
+def validate_mappings(m):

[stgao, 2017/01/09 19:31:58]

Should we do the validation in the post-submit check instead?
In that case, we might want the traverse/parse above to be in a separate module
to be shared.

[RobertoCN, 2017/01/09 23:58:58]

Done.

+  """Validate that each component is associated with at most 1 team."""
+  errors = []
+  # TODO(robertocn): Validate the component names somehow.
+  component_to_team = m['component-to-team']
+  for c in component_to_team:
+    if len(component_to_team[c]) > 1:
+      errors.append('Component %s has more than one team assigned to it: %s' % (
+          c, ', '.join(list(component_to_team[c]))))
+  return errors
+
+
+def unwrap(mappings):
+  """Remove the set() wrapper around values in component-to-team mapping."""
+  for c in mappings['component-to-team']:
+    mappings['component-to-team'][c] = mappings['component-to-team'][c].pop()
+  return mappings
+
+
+def write_results(filename, data):
+  """Write data to the named file, or the default location."""
+  if not filename:
+    filename = os.path.join(_DEFAULT_SRC_LOCATION,
+                            'component_map.json')
+  with open(filename, 'w') as f:
+    f.write(data)
+
+
+def main(argv):
+  usage = """Usage: python %prog [options] [<root_dir>]
+  root_dir  specifies the topmost directory to traverse looking for OWNERS
+            files, defaults to two levels up from this file's directory.
+            i.e. where src/ is expected to be.
+
+Examples:
+  python %prog
+  python %prog /b/build/src
+  python %prog -v /b/build/src
+  python %prog -w /b/build/src
+  python %prog -o ~/components.json /b/build/src
+  """
+  parser = optparse.OptionParser(usage=usage)
+  parser.add_option('-w', '--write', action='store_true',
+                    help='If no errors occur, write the mappings to disk.')
+  parser.add_option('-v', '--verbose', action='store_true',
+                    help='Print warnings.')
+  parser.add_option('-o', '--output_file', help='Specify file to write the '
+                    'mappings to instead of the default: src/components.json '
+                    '(implies -w)')
+  options, args = parser.parse_args(argv[1:])
+  if args:
+    root = args[0]
+  else:
+    root = _DEFAULT_SRC_LOCATION
+
+  mappings, warnings = traverse(root)
+  if options.verbose:
+    for w in warnings:
+      print w
+
+  errors = validate_mappings(mappings)
+  for e in errors:
+    print e
+
+  results = json.dumps(unwrap(mappings), sort_keys=True, indent=2)
+  if options.write or options.output_file:
+    if errors:
+      print 'Not writing to file due to errors'
+      print results
+    else:
+      write_results(options.output_file, results)
+  else:
+    print results
+
+  return len(errors)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))