Make git-freeze bail out if the user has too much untracked data.

git_common.py

Index: git_common.py
diff --git a/git_common.py b/git_common.py
index b7626e8a3ed4d81434443ef15e427c2cae2bf89a..ec18f41b2ee2f7ea967555f72ad5a9835fb870f6 100644
--- a/git_common.py
+++ b/git_common.py
@@ -32,8 +32,10 @@ import threading
 
 import subprocess2
 
-ROOT = os.path.abspath(os.path.dirname(__file__))
+from cStringIO import StringIO
+
 
+ROOT = os.path.abspath(os.path.dirname(__file__))
 IS_WIN = sys.platform == 'win32'
 GIT_EXE = ROOT+'\\git.bat' if IS_WIN else 'git'
 TEST_MODE = False
@@ -281,6 +283,10 @@ def once(function):
 
 ## Git functions
 
+def die(message, *args):
+  print >> sys.stderr, textwrap.dedent(message % args)
+  sys.exit(1)
+
 
 def blame(filename, revision=None, porcelain=False, *_args):
   command = ['blame']
@@ -293,7 +299,7 @@ def blame(filename, revision=None, porcelain=False, *_args):
 
 
 def branch_config(branch, option, default=None):
-  return config('branch.%s.%s' % (branch, option), default=default)
+  return get_config('branch.%s.%s' % (branch, option), default=default)
 
 
 def config_regexp(pattern):
@@ -319,23 +325,19 @@ def branches(*args):
   NO_BRANCH = ('* (no branch', '* (detached', '* (HEAD detached')
 
   key = 'depot-tools.branch-limit'
-  limit = 20
-  try:
-    limit = int(config(key, limit))
-  except ValueError:
-    pass
+  limit = get_config_int(key, 20)
 
   raw_branches = run('branch', *args).splitlines()
 
   num = len(raw_branches)
+
   if num > limit:
-    print >> sys.stderr, textwrap.dedent("""\
-    Your git repo has too many branches (%d/%d) for this tool to work well.
+    die("""\
+      Your git repo has too many branches (%d/%d) for this tool to work well.
 
-    You may adjust this limit by running:
+      You may adjust this limit by running:
       git config %s <new_limit>
-    """ % (num, limit, key))
-    sys.exit(1)
+      """, num, limit, key)
 
   for line in raw_branches:
     if line.startswith(NO_BRANCH):
@@ -343,14 +345,22 @@ def branches(*args):
     yield line.split()[-1]
 
 
-def config(option, default=None):
+def get_config(option, default=None):

[iannucci, 2016/06/16 00:24:20]

would you mind terribly doing the config and die refactors in another CL first?

[agable, 2016/06/16 12:47:32]

Done: https://codereview.chromium.org/2075603002

   try:
     return run('config', '--get', option) or default
   except subprocess2.CalledProcessError:
     return default
 
 
-def config_list(option):
+def get_config_int(option, default=0):
+  assert isinstance(default, int)
+  try:
+    return int(get_config(option, default))
+  except ValueError:
+    return default
+
+
+def get_config_list(option):
   try:
     return run('config', '--get-all', option).split()
   except subprocess2.CalledProcessError:
@@ -382,6 +392,38 @@ def diff(oldrev, newrev, *args):
 def freeze():
   took_action = False
 
+  stat = status()
+  if any(is_unmerged(s) for s in stat.itervalues()):
+    die("Cannot freeze unmerged changes!")
+
+  key = 'depot-tools.freeze-size-limit'
+  MB = 2**20
+  limit_mb = get_config_int(key, 100)
+  if limit_mb > 0:
+    untracked_size = sum(

[iannucci, 2016/06/16 00:24:20]

I would consider doing this as a proper loop: if the user has many many files,
this will end up doing a whole bunch of stat's that aren't needed.

[agable, 2016/06/16 15:26:55]

Done.

+        os.stat(f).st_size
+        for f, s in stat.iteritems() if s.lstat == '?'
+    ) / MB
+    if untracked_size > limit_mb:
+      die("""\
+        You appear to have too much untracked+unignored data in your git
+        checkout: %d/%dMB.
+
+        Run `git status` to see what it is.
+
+        In addition to making many git commands slower, this will prevent
+        depot_tools from freezing your in-progress changes.
+
+        You should add untracked data that you want to ignore to your repo's
+          .git/info/excludes
+        file. See `git help ignore` for the format of this file.
+
+        If this data is indended as part of your commit, you may adjust the
+        freeze limit by running:
+          git config %s <new_limit>
+        Where <new_limit> is an integer threshold in megabytes.""",
+        untracked_size, limit_mb, key)
+
   try:
     run('commit', '--no-verify', '-m', FREEZE + '.indexed')
     took_action = True
@@ -491,6 +533,13 @@ def is_dormant(branch):
   return branch_config(branch, 'dormant', 'false') != 'false'
 
 
+def is_unmerged(stat_value):
+  return (
+      'U' in (stat_value.lstat, stat_value.rstat) or
+      ((stat_value.lstat == stat_value.rstat) and stat_value.lstat in 'AD')
+  )
+
+
 def manual_merge_base(branch, base, parent):
   set_branch_config(branch, 'base', base)
   set_branch_config(branch, 'base-upstream', parent)
@@ -567,7 +616,7 @@ def repo_root():
 
 
 def root():
-  return config('depot-tools.upstream', 'origin/master')
+  return get_config('depot-tools.upstream', 'origin/master')
 
 
 @contextlib.contextmanager
@@ -700,6 +749,49 @@ def is_dirty_git_tree(cmd):
   return False
 
 
+def status():
+  """Returns a parsed version of git-status.
+
+  Returns a dictionary of {current_name: (lstat, rstat, src)} where:
+    * lstat is the left status code letter from git-status
+    * rstat is the left status code letter from git-status
+    * src is the current name of the file, or the original name of the file
+      if lstat == 'R'
+  """
+  stat_entry = collections.namedtuple('stat_entry', 'lstat rstat src')
+
+  def tokenizer(stream):
+    acc = StringIO()
+    c = None
+    while c != '':
+      c = stream.read(1)
+      if c in (None, '', '\0'):
+        s = acc.getvalue()
+        acc = StringIO()
+        if s:
+          yield s
+      else:
+        acc.write(c)
+
+  def parser(tokens):
+    END = object()
+    tok = lambda: next(tokens, END)
+    while True:
+      status_dest = tok()
+      if status_dest is END:
+        return
+      stat, dest = status_dest[:2], status_dest[3:]
+      lstat, rstat = stat
+      if lstat == 'R':
+        src = tok()
+        assert src is not END
+      else:
+        src = dest
+      yield (dest, stat_entry(lstat, rstat, src))
+
+  return dict(parser(tokenizer(run_stream('status', '-z', bufsize=-1))))

[iannucci, 2016/06/16 00:24:20]

why dict-ify this if you're just going to stream it anyway? may as well yield a
series of tuples from this function (and if someone wants a dict they can just
wrap this with dict())

[agable, 2016/06/16 15:26:55]

Done. Resulted in some more efficient code in status() as well.

+
+
 def squash_current_branch(header=None, merge_base=None):
   header = header or 'git squash commit.'
   merge_base = merge_base or get_or_create_merge_base(current_branch())