Move DEPS file docs into a README.md

checkdeps/README.md

Index: checkdeps/README.md
diff --git a/checkdeps/builddeps.py b/checkdeps/README.md
old mode 100755
new mode 100644
similarity index 5%
copy from checkdeps/builddeps.py
copy to checkdeps/README.md
index eb3eb1e211c30c978f955a2c4b22a9037715338a..ff51884ba77990163d7be6aa590ab500d33f27cf
--- a/checkdeps/builddeps.py
+++ b/checkdeps/README.md
@@ -1,62 +1,65 @@
-#!/usr/bin/env python
-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
+# DEPS Files
 
-"""Traverses the source tree, parses all found DEPS files, and constructs
-a dependency rule table to be used by subclasses.
+DEPS files specify which files the sources in a directory tree may include.
 
-The format of the deps file:
+## File format
 
 First you have the normal module-level deps. These are the ones used by
 gclient. An example would be:
 
-  deps = {
-    "base":"http://foo.bar/trunk/base"
-  }
-
-DEPS files not in the top-level of a module won't need this. Then you
-have any additional include rules. You can add (using "+") or subtract
-(using "-") from the previously specified rules (including
-module-level deps). You can also specify a path that is allowed for
-now but that we intend to remove, using "!"; this is treated the same
-as "+" when check_deps is run by our bots, but a presubmit step will
-show a warning if you add a new include of a file that is only allowed
-by "!".
-
-Note that for .java files, there is currently no difference between
-"+" and "!", even in the presubmit step.
-
-  include_rules = [
-    # Code should be able to use base (it's specified in the module-level
-    # deps above), but nothing in "base/evil" because it's evil.
-    "-base/evil",
-
-    # But this one subdirectory of evil is OK.
-    "+base/evil/not",
-
-    # And it can include files from this other directory even though there is
-    # no deps rule for it.
-    "+tools/crime_fighter",
-
-    # This dependency is allowed for now but work is ongoing to remove it,
-    # so you shouldn't add further dependencies on it.
-    "!base/evil/ok_for_now.h",
-  ]
-
-If you have certain include rules that should only be applied for some
-files within this directory and subdirectories, you can write a
-section named specific_include_rules that is a hash map of regular
-expressions to the list of rules that should apply to files matching
-them.  Note that such rules will always be applied before the rules
-from 'include_rules' have been applied, but the order in which rules
-associated with different regular expressions is applied is arbitrary.
-
-  specific_include_rules = {
-    ".*_(unit|browser|api)test\.cc": [
-      "+libraries/testsupport",
-    ],
-  }
+```
+deps = {
+  "base":"http://foo.bar/trunk/base"
+}
+```
+
+DEPS files not in the top-level of a module won't need this. Then you have any
+additional include rules. You can add (using `+`) or subtract (using `-`) from
+the previously specified rules (including module-level deps). You can also
+specify a path that is allowed for now but that we intend to remove, using `!`;
+this is treated the same as `+` when `check_deps` is run by our bots, but a
+presubmit step will show a warning if you add a new include of a file that is
+only allowed by `!`.
+
+Note that for .java files, there is currently no difference between `+` and
+`!`, even in the presubmit step.
+
+```
+include_rules = [
+  # Code should be able to use base (it's specified in the module-level
+  # deps above), but nothing in "base/evil" because it's evil.
+  "-base/evil",
+
+  # But this one subdirectory of evil is OK.
+  "+base/evil/not",
+
+  # And it can include files from this other directory even though there is
+  # no deps rule for it.
+  "+tools/crime_fighter",
+
+  # This dependency is allowed for now but work is ongoing to remove it,
+  # so you shouldn't add further dependencies on it.
+  "!base/evil/ok_for_now.h",
+]
+```
+
+If you have certain include rules that should only be applied for some files
+within this directory and subdirectories, you can write a section named
+`specific_include_rules` that is a hash map of regular expressions to the list
+of rules that should apply to files matching them. Note that such rules will
+always be applied before the rules from `include_rules` have been applied, but
+the order in which rules associated with different regular expressions is
+applied is arbitrary.
+
+```
+specific_include_rules = {
+  ".*_(unit|browser|api)test\.cc": [
+    "+libraries/testsupport",
+  ],
+}
+```
+
+# Directory structure
 
 DEPS files may be placed anywhere in the tree. Each one applies to all
 subdirectories, where there may be more DEPS files that provide additions or
@@ -64,370 +67,12 @@ subtractions for their own sub-trees.
 
 There is an implicit rule for the current directory (where the DEPS file lives)
 and all of its subdirectories. This prevents you from having to explicitly
-allow the current directory everywhere.  This implicit rule is applied first,
-so you can modify or remove it using the normal include rules.
+allow the current directory everywhere. This implicit rule is applied first, so
+you can modify or remove it using the normal include rules.
 
 The rules are processed in order. This means you can explicitly allow a higher
 directory and then take away permissions from sub-parts, or the reverse.
 
-Note that all directory separators must be slashes (Unix-style) and not
+Note that all directory separators must be `/` slashes (Unix-style) and not
 backslashes. All directories should be relative to the source root and use
 only lowercase.
-"""
-
-import copy
-import os.path
-import posixpath
-import subprocess
-
-from rules import Rule, Rules
-
-
-# Variable name used in the DEPS file to add or subtract include files from
-# the module-level deps.
-INCLUDE_RULES_VAR_NAME = 'include_rules'
-
-# Variable name used in the DEPS file to add or subtract include files
-# from module-level deps specific to files whose basename (last
-# component of path) matches a given regular expression.
-SPECIFIC_INCLUDE_RULES_VAR_NAME = 'specific_include_rules'
-
-# Optionally present in the DEPS file to list subdirectories which should not
-# be checked. This allows us to skip third party code, for example.
-SKIP_SUBDIRS_VAR_NAME = 'skip_child_includes'
-
-
-class DepsBuilderError(Exception):
-    """Base class for exceptions in this module."""
-    pass
-
-
-def NormalizePath(path):
-  """Returns a path normalized to how we write DEPS rules and compare paths."""
-  return os.path.normcase(path).replace(os.path.sep, posixpath.sep)
-
-
-def _GitSourceDirectories(base_directory):
-  """Returns set of normalized paths to subdirectories containing sources
-  managed by git."""
-  base_dir_norm = NormalizePath(base_directory)
-  git_source_directories = set([base_dir_norm])
-
-  git_cmd = 'git.bat' if os.name == 'nt' else 'git'
-  git_ls_files_cmd = [git_cmd, 'ls-files']
-  # FIXME: Use a context manager in Python 3.2+
-  popen = subprocess.Popen(git_ls_files_cmd,
-                           stdout=subprocess.PIPE,
-                           bufsize=1,  # line buffering, since read by line
-                           cwd=base_directory)
-  try:
-    try:
-      for line in popen.stdout:
-        dir_path = os.path.join(base_directory, os.path.dirname(line))
-        dir_path_norm = NormalizePath(dir_path)
-        # Add the directory as well as all the parent directories,
-        # stopping once we reach an already-listed directory.
-        while dir_path_norm not in git_source_directories:
-          git_source_directories.add(dir_path_norm)
-          dir_path_norm = posixpath.dirname(dir_path_norm)
-    finally:
-      popen.stdout.close()
-  finally:
-    popen.wait()
-
-  return git_source_directories
-
-
-class DepsBuilder(object):
-  """Parses include_rules from DEPS files."""
-
-  def __init__(self,
-               base_directory=None,
-               extra_repos=[],
-               verbose=False,
-               being_tested=False,
-               ignore_temp_rules=False,
-               ignore_specific_rules=False):
-    """Creates a new DepsBuilder.
-
-    Args:
-      base_directory: local path to root of checkout, e.g. C:\chr\src.
-      verbose: Set to True for debug output.
-      being_tested: Set to True to ignore the DEPS file at tools/checkdeps/DEPS.
-      ignore_temp_rules: Ignore rules that start with Rule.TEMP_ALLOW ("!").
-    """
-    base_directory = (base_directory or
-                      os.path.join(os.path.dirname(__file__),
-                      os.path.pardir, os.path.pardir))
-    self.base_directory = os.path.abspath(base_directory)  # Local absolute path
-    self.extra_repos = extra_repos
-    self.verbose = verbose
-    self._under_test = being_tested
-    self._ignore_temp_rules = ignore_temp_rules
-    self._ignore_specific_rules = ignore_specific_rules
-    self._git_source_directories = None
-
-    if os.path.exists(os.path.join(base_directory, '.git')):
-      self.is_git = True
-    elif os.path.exists(os.path.join(base_directory, '.svn')):
-      self.is_git = False
-    else:
-      raise DepsBuilderError("%s is not a repository root" % base_directory)
-
-    # Map of normalized directory paths to rules to use for those
-    # directories, or None for directories that should be skipped.
-    # Normalized is: absolute, lowercase, / for separator.
-    self.directory_rules = {}
-    self._ApplyDirectoryRulesAndSkipSubdirs(Rules(), self.base_directory)
-
-  def _ApplyRules(self, existing_rules, includes, specific_includes,
-                  cur_dir_norm):
-    """Applies the given include rules, returning the new rules.
-
-    Args:
-      existing_rules: A set of existing rules that will be combined.
-      include: The list of rules from the "include_rules" section of DEPS.
-      specific_includes: E.g. {'.*_unittest\.cc': ['+foo', '-blat']} rules
-                         from the "specific_include_rules" section of DEPS.
-      cur_dir_norm: The current directory, normalized path. We will create an
-                    implicit rule that allows inclusion from this directory.
-
-    Returns: A new set of rules combining the existing_rules with the other
-             arguments.
-    """
-    rules = copy.deepcopy(existing_rules)
-
-    # First apply the implicit "allow" rule for the current directory.
-    base_dir_norm = NormalizePath(self.base_directory)
-    if not cur_dir_norm.startswith(base_dir_norm):
-      raise Exception(
-          'Internal error: base directory is not at the beginning for\n'
-          '  %s and base dir\n'
-          '  %s' % (cur_dir_norm, base_dir_norm))
-    relative_dir = posixpath.relpath(cur_dir_norm, base_dir_norm)
-
-    # Make the help string a little more meaningful.
-    source = relative_dir or 'top level'
-    rules.AddRule('+' + relative_dir,
-                  relative_dir,
-                  'Default rule for ' + source)
-
-    def ApplyOneRule(rule_str, dependee_regexp=None):
-      """Deduces a sensible description for the rule being added, and
-      adds the rule with its description to |rules|.
-
-      If we are ignoring temporary rules, this function does nothing
-      for rules beginning with the Rule.TEMP_ALLOW character.
-      """
-      if self._ignore_temp_rules and rule_str.startswith(Rule.TEMP_ALLOW):
-        return
-
-      rule_block_name = 'include_rules'
-      if dependee_regexp:
-        rule_block_name = 'specific_include_rules'
-      if relative_dir:
-        rule_description = relative_dir + "'s %s" % rule_block_name
-      else:
-        rule_description = 'the top level %s' % rule_block_name
-      rules.AddRule(rule_str, relative_dir, rule_description, dependee_regexp)
-
-    # Apply the additional explicit rules.
-    for rule_str in includes:
-      ApplyOneRule(rule_str)
-
-    # Finally, apply the specific rules.
-    if self._ignore_specific_rules:
-      return rules
-
-    for regexp, specific_rules in specific_includes.iteritems():
-      for rule_str in specific_rules:
-        ApplyOneRule(rule_str, regexp)
-
-    return rules
-
-  def _ApplyDirectoryRules(self, existing_rules, dir_path_local_abs):
-    """Combines rules from the existing rules and the new directory.
-
-    Any directory can contain a DEPS file. Top-level DEPS files can contain
-    module dependencies which are used by gclient. We use these, along with
-    additional include rules and implicit rules for the given directory, to
-    come up with a combined set of rules to apply for the directory.
-
-    Args:
-      existing_rules: The rules for the parent directory. We'll add-on to these.
-      dir_path_local_abs: The directory path that the DEPS file may live in (if
-                          it exists). This will also be used to generate the
-                          implicit rules. This is a local path.
-
-    Returns: A 2-tuple of:
-      (1) the combined set of rules to apply to the sub-tree,
-      (2) a list of all subdirectories that should NOT be checked, as specified
-          in the DEPS file (if any).
-          Subdirectories are single words, hence no OS dependence.
-    """
-    dir_path_norm = NormalizePath(dir_path_local_abs)
-
-    # Check the DEPS file in this directory.
-    if self.verbose:
-      print 'Applying rules from', dir_path_local_abs
-    def FromImpl(*_):
-      pass  # NOP function so "From" doesn't fail.
-
-    def FileImpl(_):
-      pass  # NOP function so "File" doesn't fail.
-
-    class _VarImpl:
-      def __init__(self, local_scope):
-        self._local_scope = local_scope
-
-      def Lookup(self, var_name):
-        """Implements the Var syntax."""
-        try:
-          return self._local_scope['vars'][var_name]
-        except KeyError:
-          raise Exception('Var is not defined: %s' % var_name)
-
-    local_scope = {}
-    global_scope = {
-      'File': FileImpl,
-      'From': FromImpl,
-      'Var': _VarImpl(local_scope).Lookup,
-    }
-    deps_file_path = os.path.join(dir_path_local_abs, 'DEPS')
-
-    # The second conditional here is to disregard the
-    # tools/checkdeps/DEPS file while running tests.  This DEPS file
-    # has a skip_child_includes for 'testdata' which is necessary for
-    # running production tests, since there are intentional DEPS
-    # violations under the testdata directory.  On the other hand when
-    # running tests, we absolutely need to verify the contents of that
-    # directory to trigger those intended violations and see that they
-    # are handled correctly.
-    if os.path.isfile(deps_file_path) and not (
-        self._under_test and
-        os.path.basename(dir_path_local_abs) == 'checkdeps'):
-      execfile(deps_file_path, global_scope, local_scope)
-    elif self.verbose:
-      print '  No deps file found in', dir_path_local_abs
-
-    # Even if a DEPS file does not exist we still invoke ApplyRules
-    # to apply the implicit "allow" rule for the current directory
-    include_rules = local_scope.get(INCLUDE_RULES_VAR_NAME, [])
-    specific_include_rules = local_scope.get(SPECIFIC_INCLUDE_RULES_VAR_NAME,
-                                             {})
-    skip_subdirs = local_scope.get(SKIP_SUBDIRS_VAR_NAME, [])
-
-    return (self._ApplyRules(existing_rules, include_rules,
-                             specific_include_rules, dir_path_norm),
-            skip_subdirs)
-
-  def _ApplyDirectoryRulesAndSkipSubdirs(self, parent_rules,
-                                         dir_path_local_abs):
-    """Given |parent_rules| and a subdirectory |dir_path_local_abs| of the
-    directory that owns the |parent_rules|, add |dir_path_local_abs|'s rules to
-    |self.directory_rules|, and add None entries for any of its
-    subdirectories that should be skipped.
-    """
-    directory_rules, excluded_subdirs = self._ApplyDirectoryRules(
-        parent_rules, dir_path_local_abs)
-    dir_path_norm = NormalizePath(dir_path_local_abs)
-    self.directory_rules[dir_path_norm] = directory_rules
-    for subdir in excluded_subdirs:
-      subdir_path_norm = posixpath.join(dir_path_norm, subdir)
-      self.directory_rules[subdir_path_norm] = None
-
-  def GetAllRulesAndFiles(self, dir_name=None):
-    """Yields (rules, filenames) for each repository directory with DEPS rules.
-
-    This walks the directory tree while staying in the repository. Specify
-    |dir_name| to walk just one directory and its children; omit |dir_name| to
-    walk the entire repository.
-
-    Yields:
-      Two-element (rules, filenames) tuples. |rules| is a rules.Rules object
-      for a directory, and |filenames| is a list of the absolute local paths
-      of all files in that directory.
-    """
-    if self.is_git and self._git_source_directories is None:
-      self._git_source_directories = _GitSourceDirectories(self.base_directory)
-      for repo in self.extra_repos:
-        repo_path = os.path.join(self.base_directory, repo)
-        self._git_source_directories.update(_GitSourceDirectories(repo_path))
-
-    # Collect a list of all files and directories to check.
-    files_to_check = []
-    if dir_name and not os.path.isabs(dir_name):
-      dir_name = os.path.join(self.base_directory, dir_name)
-    dirs_to_check = [dir_name or self.base_directory]
-    while dirs_to_check:
-      current_dir = dirs_to_check.pop()
-
-      # Check that this directory is part of the source repository. This
-      # prevents us from descending into third-party code or directories
-      # generated by the build system.
-      if self.is_git:
-        if NormalizePath(current_dir) not in self._git_source_directories:
-          continue
-      elif not os.path.exists(os.path.join(current_dir, '.svn')):
-        continue
-
-      current_dir_rules = self.GetDirectoryRules(current_dir)
-
-      if not current_dir_rules:
-        continue  # Handle the 'skip_child_includes' case.
-
-      current_dir_contents = sorted(os.listdir(current_dir))
-      file_names = []
-      sub_dirs = []
-      for file_name in current_dir_contents:
-        full_name = os.path.join(current_dir, file_name)
-        if os.path.isdir(full_name):
-          sub_dirs.append(full_name)
-        else:
-          file_names.append(full_name)
-      dirs_to_check.extend(reversed(sub_dirs))
-
-      yield (current_dir_rules, file_names)
-
-  def GetDirectoryRules(self, dir_path_local):
-    """Returns a Rules object to use for the given directory, or None
-    if the given directory should be skipped.
-
-    Also modifies |self.directory_rules| to store the Rules.
-    This takes care of first building rules for parent directories (up to
-    |self.base_directory|) if needed, which may add rules for skipped
-    subdirectories.
-
-    Args:
-      dir_path_local: A local path to the directory you want rules for.
-        Can be relative and unnormalized. It is the caller's responsibility
-        to ensure that this is part of the repository rooted at
-        |self.base_directory|.
-    """
-    if os.path.isabs(dir_path_local):
-      dir_path_local_abs = dir_path_local
-    else:
-      dir_path_local_abs = os.path.join(self.base_directory, dir_path_local)
-    dir_path_norm = NormalizePath(dir_path_local_abs)
-
-    if dir_path_norm in self.directory_rules:
-      return self.directory_rules[dir_path_norm]
-
-    parent_dir_local_abs = os.path.dirname(dir_path_local_abs)
-    parent_rules = self.GetDirectoryRules(parent_dir_local_abs)
-    # We need to check for an entry for our dir_path again, since
-    # GetDirectoryRules can modify entries for subdirectories, namely setting
-    # to None if they should be skipped, via _ApplyDirectoryRulesAndSkipSubdirs.
-    # For example, if dir_path == 'A/B/C' and A/B/DEPS specifies that the C
-    # subdirectory be skipped, GetDirectoryRules('A/B') will fill in the entry
-    # for 'A/B/C' as None.
-    if dir_path_norm in self.directory_rules:
-      return self.directory_rules[dir_path_norm]
-
-    if parent_rules:
-      self._ApplyDirectoryRulesAndSkipSubdirs(parent_rules, dir_path_local_abs)
-    else:
-      # If the parent directory should be skipped, then the current
-      # directory should also be skipped.
-      self.directory_rules[dir_path_norm] = None
-    return self.directory_rules[dir_path_norm]