Make git-freeze bail out if the user has too much untracked data.

git_common.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Monkeypatch IMapIterator so that Ctrl-C can kill everything properly.
 # Derived from https://gist.github.com/aljungberg/626518
 import multiprocessing.pool
 from multiprocessing.pool import IMapIterator
 def wrapper(func):
   def wrap(self, timeout=None):
@@ skipping 12 matching lines @@
 import os
 import re
 import signal
 import sys
 import tempfile
 import textwrap
 import threading
 
 import subprocess2
 
+from cStringIO import StringIO
+
 
 GIT_EXE = 'git.bat' if sys.platform.startswith('win') else 'git'
 TEST_MODE = False
 
 FREEZE = 'FREEZE'
 FREEZE_SECTIONS = {
   'indexed': 'soft',
   'unindexed': 'mixed'
 }
 FREEZE_MATCHER = re.compile(r'%s.(%s)' % (FREEZE, '|'.join(FREEZE_SECTIONS)))
@@ skipping 173 matching lines @@
   how many times the decorated |function| is called."""
   def _inner_gen():
     yield function()
     while True:
       yield
   return _inner_gen().next
 
 
 ## Git functions
 
+def die_if(cond, message, *args):

[agable, 2014/06/11 21:30:16]

I'd prefer just die(message, *args), and leave the if: control flow in the
caller.

[iannucci, 2014/06/28 18:08:55]

Done.

+  if cond:
+    print >> sys.stderr, textwrap.dedent(message % args)
+    sys.exit(1)
+
 
 def branch_config(branch, option, default=None):
   return config('branch.%s.%s' % (branch, option), default=default)
 
 
 def branch_config_map(option):
   """Return {branch: <|option| value>} for all branches."""
   try:
     reg = re.compile(r'^branch\.(.*)\.%s$' % option)
     lines = run('config', '--get-regexp', reg.pattern).splitlines()
     return {reg.match(k).group(1): v for k, v in (l.split() for l in lines)}
   except subprocess2.CalledProcessError:
     return {}
 
 
 def branches(*args):
   NO_BRANCH = ('* (no branch', '* (detached from ')
 
   key = 'depot-tools.branch-limit'
-  limit = 20
-  try:
-    limit = int(config(key, limit))
-  except ValueError:
-    pass
+  limit = config_int(key, 20)

[agable, 2014/06/11 21:30:16]

See comment below. This is super weird -- it looks like you're trying to run
"git config depot-tools.branch-limit 20".

[iannucci, 2014/06/28 18:08:55]

Done.

 
   raw_branches = run('branch', *args).splitlines()
 
   num = len(raw_branches)
-  if num > limit:
-    print >> sys.stderr, textwrap.dedent("""\
+
+  die_if(num > limit, """\
     Your git repo has too many branches (%d/%d) for this tool to work well.
 
     You may adjust this limit by running:
       git config %s <new_limit>
-    """ % (num, limit, key))
-    sys.exit(1)
+    """, num, limit, key)
 
   for line in raw_branches:
     if line.startswith(NO_BRANCH):
       continue
     yield line.split()[-1]
 
 
 def run_with_retcode(*cmd, **kwargs):
   """Run a command but only return the status code."""
   try:
     run(*cmd, **kwargs)
     return 0
   except subprocess2.CalledProcessError as cpe:
     return cpe.returncode
 
 
 def config(option, default=None):

[agable, 2014/06/11 21:30:17]

I find this API weird. "git config key value" *sets* key to that value. Here,
config(key, value) just gets, and covers errors by return value instead. I
suppose it makes sense when you're used to get/getattr, but is weird when you're
used to git config.

[iannucci, 2014/06/28 18:08:55]

Done.

   try:
     return run('config', '--get', option) or default
   except subprocess2.CalledProcessError:
     return default
 
 
+def config_int(option, default=0):
+  assert isinstance(default, int)
+  try:
+    return int(config(option, default))
+  except ValueError:
+    return default
+
+
 def config_list(option):
   try:
     return run('config', '--get-all', option).split()
   except subprocess2.CalledProcessError:
     return []
 
 
 def current_branch():
   try:
     return run('rev-parse', '--abbrev-ref', 'HEAD')
   except subprocess2.CalledProcessError:
     return None
 
 
 def del_branch_config(branch, option, scope='local'):
   del_config('branch.%s.%s' % (branch, option), scope=scope)
 
 
 def del_config(option, scope='local'):
   try:
     run('config', '--' + scope, '--unset', option)
   except subprocess2.CalledProcessError:
     pass
 
 
 def freeze():
   took_action = False
 
+  stat = status()
+  die_if(any('U' in (s.lstat, s.rstat) for s in stat.itervalues()),
+         "Cannot freeze unmerged changes!")
+
+  key = 'depot-tools.freeze-size-limit'
+  MB = 2**20
+  limit_mb = config_int(key, 100)
+  if limit_mb > 0:
+    untracked_size = sum(
+        os.stat(f).st_size
+        for f, s in stat.iteritems() if s.lstat == '?'
+    ) / MB
+    die_if(untracked_size > limit_mb, """\
+      You have too much untracked+unignored data in your git checkout: %d/%dMB.
+      Run `git status` to see what it is.
+
+      In addition to making many git commands slower, this will prevent
+      depot_tools from freezing your in-progress changes.
+
+      You should add untracked data that you want to ignore to your repo's
+        .git/info/excludes
+      file. See `git help ignore` for the format of this file.
+
+      If this data is indended as part of your commit, you may adjust the freeze
+      limit by running:
+        git config %s <new_limit>
+      Where <new_limit> is an integer threshold in megabytes.""",
+      untracked_size, limit_mb, key)
+
   try:
     run('commit', '-m', FREEZE + '.indexed')
     took_action = True
   except subprocess2.CalledProcessError:
     pass
 
   try:
     run('add', '-A')
     run('commit', '-m', FREEZE + '.unindexed')
     took_action = True
@@ skipping 214 matching lines @@
   return ret, err
 
 
 def set_branch_config(branch, option, value, scope='local'):
   set_config('branch.%s.%s' % (branch, option), value, scope=scope)
 
 
 def set_config(option, value, scope='local'):
   run('config', '--' + scope, option, value)
 
+
+def status():
+  """Returns a parsed version of git-status.
+
+  Returns a dictionary of {current_name: (lstat, rstat, src)} where:
+    * lstat is the left status code letter from git-status
+    * rstat is the left status code letter from git-status
+    * src is the 'original' name of the file if lstat == 'R', or

[agable, 2014/06/11 21:30:16]

src is the current name of the file, or the 'original' name if lstat == 'R'.

[iannucci, 2014/06/28 18:08:55]

Done.

+      the current_name.
+  """
+  stat_entry = collections.namedtuple('stat_entry', 'lstat rstat src')
+
+  def tokenizer(stream):
+    acc = StringIO()
+    c = None
+    while c != '':
+      c = stream.read(1)
+      if c in (None, '', '\0'):
+        s = acc.getvalue()
+        acc = StringIO()
+        if s:
+          yield s
+      else:
+        acc.write(c)
+
+  def parser(tokens):
+    END = object()
+    tok = lambda: next(tokens, END)
+    while True:
+      status_dest = tok()
+      if status_dest is END:
+        return
+      stat, dest = status_dest[:2], status_dest[3:]
+      lstat, rstat = stat
+      if lstat == 'R':
+        src = tok()
+        assert src is not END
+      else:
+        src = dest
+      yield (dest, stat_entry(lstat, rstat, src))
+
+  return dict(parser(tokenizer(run_stream('status', '-z', bufsize=-1))))

[agable, 2014/06/11 21:30:16]

Why not use --porcelain and then iterate over readlines()? That's easier than
having to tokenize on \0 yourself.

[iannucci, 2014/06/28 18:08:55]

Then I have to tokenzie quotes. --porcelain is way worse than splitting on \0.

+
+
 def squash_current_branch(header=None, merge_base=None):
   header = header or 'git squash commit.'
   merge_base = merge_base or get_or_create_merge_base(current_branch())
   log_msg = header + '\n'
   if log_msg:
     log_msg += '\n'
   log_msg += run('log', '--reverse', '--format=%H%n%B', '%s..HEAD' % merge_base)
   run('reset', '--soft', merge_base)
   run('commit', '-a', '-F', '-', indata=log_msg)
 
@@ skipping 102 matching lines @@
     return None
   return ret
 
 
 def upstream(branch):
   try:
     return run('rev-parse', '--abbrev-ref', '--symbolic-full-name',
                branch+'@{upstream}')
   except subprocess2.CalledProcessError:
     return None