Implement ability to specify temporarily-allowed dependencies in DEPS

tools/checkdeps/checkdeps.py

Index: tools/checkdeps/checkdeps.py
diff --git a/tools/checkdeps/checkdeps.py b/tools/checkdeps/checkdeps.py
index 9624812086a0d732db98a5a0e5825d4bad09b739..c08f0414542c061b256fc29ce4edc5d537f0cb95 100755
--- a/tools/checkdeps/checkdeps.py
+++ b/tools/checkdeps/checkdeps.py
@@ -18,9 +18,17 @@ gclient. An example would be:
     "base":"http://foo.bar/trunk/base"
   }
 
-DEPS files not in the top-level of a module won't need this. Then you have
-any additional include rules. You can add (using "+") or subtract (using "-")
-from the previously specified rules (including module-level deps).
+DEPS files not in the top-level of a module won't need this. Then you
+have any additional include rules. You can add (using "+") or subtract
+(using "-") from the previously specified rules (including
+module-level deps). You can also specify a path that is allowed for
+now but that we intend to remove, using "!"; this is treated the same
+as "+" when check_deps is run by our bots, but a presubmit step will
+show a warning if you add a new include of a file that is only allowed
+by "!".
+
+Note that for .java files, there is currently no difference between
+"+" and "!", even in the presubmit step.
 
   include_rules = {
     # Code should be able to use base (it's specified in the module-level
@@ -32,7 +40,11 @@ from the previously specified rules (including module-level deps).
 
     # And it can include files from this other directory even though there is
     # no deps rule for it.
-    "+tools/crime_fighter"
+    "+tools/crime_fighter",
+
+    # This dependency is allowed for now but work is ongoing to remove it,
+    # so you shouldn't add further dependencies on it.
+    "!base/evil/ok_for_now.h",
   }
 
 DEPS files may be placed anywhere in the tree. Each one applies to all
@@ -54,12 +66,13 @@ only lowercase.
 
 import os
 import optparse
-import pipes
+import subprocess
 import sys
 import copy
 
 import cpp_checker
 import java_checker
+from rules import Rule, Rules
 
 
 # Variable name used in the DEPS file to add or subtract include files from
@@ -73,92 +86,14 @@ SKIP_SUBDIRS_VAR_NAME = "skip_child_includes"
 # Set to true for more output. This is set by the command line options.
 VERBOSE = False
 
-# In lowercase, using forward slashes as directory separators, ending in a
-# forward slash. Set by the command line options.
+# OS-compatible path for base directory (e.g. C:\chrome\src). Set
+# implicitly or by command-line option.
 BASE_DIRECTORY = ""
 
 # The directories which contain the sources managed by git.
 GIT_SOURCE_DIRECTORY = set()
 
 
-# Specifies a single rule for an include, which can be either allow or disallow.
-class Rule(object):
-  def __init__(self, allow, directory, source):
-    self.allow = allow
-    self._dir = directory
-    self._source = source
-
-  def __str__(self):
-    if (self.allow):
-      return '"+%s" from %s.' % (self._dir, self._source)
-    return '"-%s" from %s.' % (self._dir, self._source)
-
-  def ParentOrMatch(self, other):
-    """Returns true if the input string is an exact match or is a parent
-    of the current rule. For example, the input "foo" would match "foo/bar"."""
-    return self._dir == other or self._dir.startswith(other + "/")
-
-  def ChildOrMatch(self, other):
-    """Returns true if the input string would be covered by this rule. For
-    example, the input "foo/bar" would match the rule "foo"."""
-    return self._dir == other or other.startswith(self._dir + "/")
-
-
-def ParseRuleString(rule_string, source):
-  """Returns a tuple of a boolean indicating whether the directory is an allow
-  rule, and a string holding the directory name.
-  """
-  if len(rule_string) < 1:
-    raise Exception('The rule string "%s" is too short\nin %s' %
-                    (rule_string, source))
-
-  if rule_string[0] == "+":
-    return (True, rule_string[1:])
-  if rule_string[0] == "-":
-    return (False, rule_string[1:])
-  raise Exception('The rule string "%s" does not begin with a "+" or a "-"' %
-                  rule_string)
-
-
-class Rules:
-  def __init__(self):
-    """Initializes the current rules with an empty rule list."""
-    self._rules = []
-
-  def __str__(self):
-    ret = "Rules = [\n"
-    ret += "\n".join([" %s" % x for x in self._rules])
-    ret += "]\n"
-    return ret
-
-  def AddRule(self, rule_string, source):
-    """Adds a rule for the given rule string.
-
-    Args:
-      rule_string: The include_rule string read from the DEPS file to apply.
-      source: A string representing the location of that string (filename, etc.)
-              so that we can give meaningful errors.
-    """
-    (add_rule, rule_dir) = ParseRuleString(rule_string, source)
-    # Remove any existing rules or sub-rules that apply. For example, if we're
-    # passed "foo", we should remove "foo", "foo/bar", but not "foobar".
-    self._rules = [x for x in self._rules if not x.ParentOrMatch(rule_dir)]
-    self._rules.insert(0, Rule(add_rule, rule_dir, source))
-
-  def DirAllowed(self, allowed_dir):
-    """Returns a tuple (success, message), where success indicates if the given
-    directory is allowed given the current set of rules, and the message tells
-    why if the comparison failed."""
-    for rule in self._rules:
-      if rule.ChildOrMatch(allowed_dir):
-        # This rule applies.
-        if rule.allow:
-          return (True, "")
-        return (False, rule.__str__())
-    # No rules apply, fail.
-    return (False, "no rule applying")
-
-
 def ApplyRules(existing_rules, includes, cur_dir):
   """Applies the given include rules, returning the new rules.
 
@@ -174,10 +109,9 @@ def ApplyRules(existing_rules, includes, cur_dir):
   rules = copy.copy(existing_rules)
 
   # First apply the implicit "allow" rule for the current directory.
-  if cur_dir.lower().startswith(BASE_DIRECTORY):
+  if os.path.normpath(cur_dir).lower().startswith(

[M-A Ruel, 2012/07/26 13:50:51]

BTW, is cur_dir an absolute path?

FYI only: I like to sprinkle asserts and enforce absolute paths. But that means
realpath() must be called consistently otherwise symlinks will get in the way.

[Jói, 2012/07/26 17:05:52]

It's an absolute path.

+      os.path.normpath(BASE_DIRECTORY).lower()):
     relative_dir = cur_dir[len(BASE_DIRECTORY) + 1:]
-    # Normalize path separators to slashes.
-    relative_dir = relative_dir.replace("\\", "/")
     source = relative_dir
     if len(source) == 0:
       source = "top level"  # Make the help string a little more meaningful.
@@ -297,6 +231,75 @@ def CheckDirectory(parent_rules, checkers, dir_name):
   return success
 
 
+def CheckAddedIncludes(added_includes):
+  """This is used from PRESUBMIT.py to check new #include statements added in
+  the change being presubmit checked.
+
+  Args:
+    added_includes: ((file_path, (include_line, include_line, ...), ...)
+
+  Return:
+    A list of tuples, (bad_file_path, rule_type, rule_description)
+    where rule_type is one of Rule.DISALLOW or Rule.TEMP_ALLOW and
+    rule_description is human-readable. Empty if no problems.
+  """
+  CommonSetup()
+
+  # Map of directory paths to rules to use for those directories, or
+  # None for directories that should be skipped.
+  directory_rules = {}
+
+  def ApplyDirectoryRulesAndSkipSubdirs(parent_rules, dir_path):
+    rules_tuple = ApplyDirectoryRules(parent_rules, dir_path)
+    directory_rules[dir_path] = rules_tuple[0]
+    for subdir in rules_tuple[1]:
+      directory_rules[os.path.join(dir_path, subdir)] = None
+
+  ApplyDirectoryRulesAndSkipSubdirs(Rules(), BASE_DIRECTORY)
+
+  def GetDirectoryRules(dir_path):
+    """Returns a Rules object to use for the given directory, or None
+    if the given directory should be skipped.
+    """
+    if not dir_path.startswith(BASE_DIRECTORY):
+      dir_path = os.path.join(BASE_DIRECTORY, dir_path)
+
+    parent_dir = os.path.dirname(dir_path)
+    parent_rules = None
+    if not dir_path in directory_rules:
+      parent_rules = GetDirectoryRules(parent_dir)
+
+    # We need to check for an entry for our dir_path again, in case we
+    # are at a path e.g. A/B/C where A/B/DEPS specifies the C
+    # subdirectory to be skipped; in this case, the invocation to
+    # GetDirectoryRules(parent_dir) has already filled in an entry for
+    # A/B/C.
+    if not dir_path in directory_rules:
+      if not parent_rules:
+        # If the parent directory should be skipped, then the current
+        # directory should also be skipped.
+        directory_rules[dir_path] = None
+      else:
+        ApplyDirectoryRulesAndSkipSubdirs(parent_rules, dir_path)
+    return directory_rules[dir_path]
+
+  cpp = cpp_checker.CppChecker(VERBOSE)
+
+  problems = []
+  for file_path, include_lines in added_includes:
+    # TODO(joi): Make this cover Java as well.
+    if not os.path.splitext(file_path)[1] in cpp.EXTENSIONS:
+      pass
+    rules_for_file = GetDirectoryRules(os.path.dirname(file_path))
+    if rules_for_file:
+      for line in include_lines:
+        is_include, line_status, rule_type = cpp.CheckLine(
+            rules_for_file, line, True)
+        if rule_type != Rule.ALLOW:
+          problems.append((file_path, rule_type, line_status))
+  return problems
+
+
 def GetGitSourceDirectory(root):
   """Returns a set of the directories to be checked.
 
@@ -308,17 +311,34 @@ def GetGitSourceDirectory(root):
   """
   git_source_directory = set()
   popen_out = os.popen("cd %s && git ls-files --full-name ." %
-                       pipes.quote(root))
+                       subprocess.list2cmdline([root]))
   for line in popen_out.readlines():
     dir_name = os.path.join(root, os.path.dirname(line))
     # Add the directory as well as all the parent directories.
     while dir_name != root:
-      git_source_directory.add(dir_name)
+      git_source_directory.add(dir_name.lower())
       dir_name = os.path.dirname(dir_name)
-  git_source_directory.add(root)
+  git_source_directory.add(root.lower())
   return git_source_directory
 
 
+def CommonSetup(base_dir=None):
+  """Setup that is in common between the main() entrypoint to this
+  script and the CheckAddedIncludes (presubmit check) entrypoint.
+  """
+  global BASE_DIRECTORY
+  global GIT_SOURCE_DIRECTORY
+
+  if base_dir:
+    BASE_DIRECTORY = base_dir
+  else:
+    BASE_DIRECTORY = os.path.abspath(

[M-A Ruel, 2012/07/26 13:50:51]

Did you mean normpath?

[Jói, 2012/07/26 17:05:52]

No, needs to be an absolute path.

+        os.path.join(os.path.abspath(os.path.dirname(__file__)), "../.."))

[M-A Ruel, 2012/07/26 13:50:51]

'..', '..' instead of '../..'

[Jói, 2012/07/26 17:05:52]

Done.

+
+  if os.path.exists(os.path.join(BASE_DIRECTORY, ".git")):
+    GIT_SOURCE_DIRECTORY = GetGitSourceDirectory(BASE_DIRECTORY)
+
+
 def PrintUsage():
   print """Usage: python checkdeps.py [--root <root>] [tocheck]
   --root   Specifies the repository root. This defaults to "../../.." relative
@@ -335,17 +355,16 @@ Examples:
 
 
 def checkdeps(options, args):
+  global BASE_DIRECTORY
   global VERBOSE
   if options.verbose:
     VERBOSE = True
 
   # Optional base directory of the repository.
-  global BASE_DIRECTORY
   if not options.base_directory:
-    BASE_DIRECTORY = os.path.abspath(
-        os.path.join(os.path.abspath(os.path.dirname(__file__)), "../.."))
+    CommonSetup()
   else:
-    BASE_DIRECTORY = os.path.abspath(options.base_directory)
+    CommonSetup(os.path.abspath(options.base_directory))
 
   # Figure out which directory we have to check.
   if len(args) == 0:
@@ -364,19 +383,6 @@ def checkdeps(options, args):
   print "Checking:", start_dir
 
   base_rules = Rules()
-
-  # The base directory should be lower case from here on since it will be used
-  # for substring matching on the includes, and we compile on case-insensitive
-  # systems. Plus, we always use slashes here since the include parsing code
-  # will also normalize to slashes.
-  BASE_DIRECTORY = BASE_DIRECTORY.lower()
-  BASE_DIRECTORY = BASE_DIRECTORY.replace("\\", "/")
-  start_dir = start_dir.replace("\\", "/")
-
-  if os.path.exists(os.path.join(BASE_DIRECTORY, ".git")):
-    global GIT_SOURCE_DIRECTORY
-    GIT_SOURCE_DIRECTORY = GetGitSourceDirectory(BASE_DIRECTORY)
-
   java = java_checker.JavaChecker(BASE_DIRECTORY, VERBOSE)
   cpp = cpp_checker.CppChecker(VERBOSE)
   checkers = dict(