Add git-reup and friends

git_common.py

-# Copyright 2013 The Chromium Authors. All rights reserved.
+# Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Monkeypatch IMapIterator so that Ctrl-C can kill everything properly.
 # Derived from https://gist.github.com/aljungberg/626518
 import multiprocessing.pool
 from multiprocessing.pool import IMapIterator
 def wrapper(func):
   def wrap(self, timeout=None):
     return func(self, timeout=timeout or 1e100)
   return wrap
 IMapIterator.next = wrapper(IMapIterator.next)
 IMapIterator.__next__ = IMapIterator.next
 # TODO(iannucci): Monkeypatch all other 'wait' methods too.
 
 
 import binascii
+import collections
 import contextlib
 import functools
 import logging
 import os
 import signal
 import sys
 import tempfile
 import threading
 
 import subprocess2
 
 
+MERGE_BASE_FMT = 'branch.%s.base'
 GIT_EXE = 'git.bat' if sys.platform.startswith('win') else 'git'
 
 
 class BadCommitRefException(Exception):
   def __init__(self, refs):
     msg = ('one of %s does not seem to be a valid commitref.' %
            str(refs))
     super(BadCommitRefException, self).__init__(msg)
 
 
@@ skipping 152 matching lines @@
     return self.inc
 
   def __exit__(self, _exc_type, _exc_value, _traceback):
     self._dead = True
     with self._dead_cond:
       self._dead_cond.notifyAll()
     self._thread.join()
     del self._thread
 
 
+def once(function):
+  """@Decorates |function| so that it only performs its action once, no matter
+  how many times the decorated |function| is called."""
+  def _inner_gen():
+    yield function()
+    while True:
+      yield
+  return _inner_gen().next
+
+
+## Git functions
+
 def branches(*args):
-  NO_BRANCH = ('* (no branch)', '* (detached from ')
+  NO_BRANCH = ('* (no branch', '* (detached from ')
   for line in run('branch', *args).splitlines():
     if line.startswith(NO_BRANCH):
       continue
     yield line.split()[-1]
 
 
+def config(option, default=None):
+  try:
+    return run('config', '--get', option) or default
+  except subprocess2.CalledProcessError:
+    return default
+
+
 def config_list(option):
   try:
     return run('config', '--get-all', option).split()
   except subprocess2.CalledProcessError:
     return []
 
 
 def current_branch():
   return run('rev-parse', '--abbrev-ref', 'HEAD')
 
 
+def del_config(option, scope='local'):
+  run('config', '--' + scope, '--unset', option)
+
+
+def get_branch_tree():
+  """Get the dictionary of {branch: parent}, compatible with topo_iter.
+
+  Returns a tuple of (skipped, <branch_tree dict>) where skipped is a list of
+  branches without upstream branches defined.
+  """
+  skipped = []

[agable, 2014/03/21 01:14:21]

set?

[iannucci, 2014/03/22 04:17:35]

Done.

+  branch_tree = {}
+
+  for branch in branches():
+    parent = upstream(branch)
+    if not parent:
+      skipped.append(branch)
+      continue
+    branch_tree[branch] = parent
+
+  return skipped, branch_tree
+
+
 def parse_commitrefs(*commitrefs):
   """Returns binary encoded commit hashes for one or more commitrefs.
 
   A commitref is anything which can resolve to a commit. Popular examples:
     * 'HEAD'
     * 'origin/master'
     * 'cool_branch~2'
   """
   try:
     return map(binascii.unhexlify, hash_multi(*commitrefs))
   except subprocess2.CalledProcessError:
     raise BadCommitRefException(commitrefs)
 
 
+def root():
+  return config('depot-tools.upstream', 'origin/master')
+
+
 def run(*cmd, **kwargs):
-  """Runs a git command. Returns stdout as a string.
+  """The same as run_both, except it only returns stdout."""
+  return run_both(*cmd, **kwargs)[0]
 
-  If logging is DEBUG, we'll print the command before we run it.
+
+def run_both(*cmd, **kwargs):

[agable, 2014/03/21 01:14:21]

run_both? really? least descriptive name evar.

[iannucci, 2014/03/22 04:17:35]

Done.

+  """Runs a git command.
+
+  Returns (stdout, stderr) as a pair of strings.
 
   kwargs
     autostrip (bool) - Strip the output. Defaults to True.
+    indata (str) - Specifies stdin data for the process.
   """
+  kwargs.setdefault('stdin', subprocess2.PIPE)
+  kwargs.setdefault('stdout', subprocess2.PIPE)
+  kwargs.setdefault('stderr', subprocess2.PIPE)
   autostrip = kwargs.pop('autostrip', True)
+  indata = kwargs.pop('indata', None)
 
-  retstream, proc = stream_proc(*cmd, **kwargs)
-  ret = retstream.read()
+  cmd = (GIT_EXE,) + cmd
+  proc = subprocess2.Popen(cmd, **kwargs)
+  ret, err = proc.communicate(indata)
   retcode = proc.wait()
   if retcode != 0:
-    raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret, None)
+    raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret, err)
 
   if autostrip:
     ret = (ret or '').strip()
-  return ret
+    err = (err or '').strip()
 
-
-def stream_proc(*cmd, **kwargs):
-  """Runs a git command. Returns stdout as a file.
-
-  If logging is DEBUG, we'll print the command before we run it.
-  """
-  cmd = (GIT_EXE,) + cmd
-  logging.debug('Running %s', ' '.join(repr(tok) for tok in cmd))
-  proc = subprocess2.Popen(cmd, stderr=subprocess2.VOID,
-                           stdout=subprocess2.PIPE, **kwargs)
-  return proc.stdout, proc
+  return ret, err
 
 
 def stream(*cmd, **kwargs):

[agable, 2014/03/21 01:14:21]

Also poorly named. Since it's executing a command too, not just streaming
something somewhere. Also doesn't mention that it drops stderr on the floor.

[iannucci, 2014/03/22 04:17:35]

Done.

-  return stream_proc(*cmd, **kwargs)[0]
+  """Runs a git command. Returns stdout as a file."""
+  kwargs.setdefault('stderr', subprocess2.VOID)
+  kwargs.setdefault('stdout', subprocess2.PIPE)
+  cmd = (GIT_EXE,) + cmd
+  proc = subprocess2.Popen(cmd, **kwargs)
+  return proc.stdout
+
+
+def set_config(option, value, scope='local'):
+  run('config', '--' + scope, option, value)
+
+
+def get_or_create_merge_base(branch, parent=None):
+  """Finds the configured merge base for branch.
+
+  If parent is supplied, it's used instead of calling upstream(branch).
+  """
+  option = MERGE_BASE_FMT % branch
+  base = config(option)
+  if base:
+    try:
+      run('merge-base', '--is-ancestor', base, branch)
+      logging.debug('Found pre-set merge-base for %s: %s', branch, base)
+    except subprocess2.CalledProcessError:
+      logging.debug('Found WRONG pre-set merge-base for %s: %s', branch, base)
+      base = None
+
+  if not base:
+    base = run('merge-base', parent or upstream(branch), branch)
+    set_config(option, base)
+
+  return base
 
 
 def hash_one(reflike):
   return run('rev-parse', reflike)
 
 
 def hash_multi(*reflike):
   return run('rev-parse', *reflike).splitlines()
 
 
 def intern_f(f, kind='blob'):
   """Interns a file object into the git object store.
 
   Args:
     f (file-like object) - The file-like object to intern
     kind (git object type) - One of 'blob', 'commit', 'tree', 'tag'.
 
   Returns the git hash of the interned object (hex encoded).
   """
   ret = run('hash-object', '-t', kind, '-w', '--stdin', stdin=f)
   f.close()
   return ret
 
 
+def in_rebase():
+  git_dir = run('rev-parse', '--git-dir')
+  return (
+    os.path.exists(os.path.join(git_dir, 'rebase-merge')) or
+    os.path.exists(os.path.join(git_dir, 'rebase-apply')))
+
+
+def manual_merge_base(branch, base):
+  set_config(MERGE_BASE_FMT % branch, base)
+
+
+def mktree(treedict):
+  """Makes a git tree object and returns its hash.
+
+  See |tree()| for the values of mode, type, and ref.
+
+  Args:
+    treedict - { name: (mode, type, ref) }
+  """
+  with tempfile.TemporaryFile() as f:
+    for name, (mode, typ, ref) in treedict.iteritems():
+      f.write('%s %s %s\t%s\0' % (mode, typ, ref, name))
+    f.seek(0)
+    return run('mktree', '-z', stdin=f)
+
+
+def remove_merge_base(branch):
+  del_config(MERGE_BASE_FMT % branch)
+
+
+RebaseRet = collections.namedtuple('RebaseRet', 'success message')
+
+
+def rebase(parent, start, branch, abort=False, ignore_date=False):
+  """Rebases |start|..|branch| onto the branch |parent|.

[agable, 2014/03/21 01:14:21]

start..branch doesn't actually include start, in git syntax. This method does
include start in the set of commits that get rebased.

[iannucci, 2014/03/22 04:17:35]

Er... no it doesn't include start, which is why I used A..B :).

I updated git_common_test:....testRebase to more clearly illustrate this.

+
+  Args:
+    parent - The new parent ref for the rebased commits.
+    start  - The commit to start from
+    branch - The branch to rebase
+    abort  - If True, will call git-rebase --abort in the event that the rebase
+             doesn't complete successfully.
+    ignore_date - If True, will cause commit timestamps to match the timestamps
+                  of the original commits. Mostly used for getting deterministic
+                  timestamps in tests.
+
+  Returns a namedtuple with fields:
+    success - a boolean indicating that the rebase command completed
+              successfully.
+    message - if the rebase failed, this contains the stdout of the failed
+              rebase.
+  """
+  try:
+    args = ['--committer-date-is-author-date'] if ignore_date else []
+    args.extend(('--onto', parent, start, branch))
+    run('rebase', *args)
+    return RebaseRet(True, '')
+  except subprocess2.CalledProcessError as cpe:
+    if abort:
+      run('rebase', '--abort')
+    return RebaseRet(False, cpe.output)
+
+
+def squash_current_branch(header=None, merge_base=None):
+  header = header or 'git squash commit.'
+  merge_base = merge_base or get_or_create_merge_base(current_branch())
+  log_msg = header
+  if log_msg:
+    log_msg += '\n'
+  log_msg += run('log', '--reverse', '--format=%H%n%B', '%s..HEAD' % merge_base)
+  run('reset', '--soft', merge_base)
+  run('commit', '-a', '-F', '-', indata=log_msg)
+
+
 def tags(*args):
   return run('tag', *args).splitlines()
 
 
+def topo_iter(branch_tree, top_down=True):
+  """Generates (branch, parent) in topographical order for a branch tree.
+
+  Given a tree:
+
+            A1
+        B1      B2
+      C1  C2    C3
+                D1
+
+  branch_tree would look like: {
+    'D1': 'C3',
+    'C3': 'B2',
+    'B2': 'A1',
+    'C1': 'B1',
+    'C2': 'B1',
+    'B1': 'A1',
+  }
+
+  It is OK to have multiple 'root' nodes in your graph.
+
+  if top_down is True, items are yielded from A->D. Otherwise they're yielded
+  from D->A. There is no specified ordering within a layer.
+  """
+  branch_tree = branch_tree.copy()
+
+  # TODO(iannucci): There is probably a more efficient way to do these.
+  if top_down:
+    while branch_tree:
+      this_pass = [(b, p) for b, p in branch_tree.iteritems()
+                   if p not in branch_tree]
+      assert this_pass, "Branch tree has cycles: %r" % branch_tree
+      for branch, parent in this_pass:
+        yield branch, parent
+        del branch_tree[branch]
+  else:
+    parent_to_branches = collections.defaultdict(set)
+    for branch, parent in branch_tree.iteritems():
+      parent_to_branches[parent].add(branch)
+
+    while branch_tree:
+      this_pass = [(b, p) for b, p in branch_tree.iteritems()
+                   if not parent_to_branches[b]]
+      assert this_pass, "Branch tree has cycles: %r" % branch_tree
+      for branch, parent in this_pass:
+        yield branch, parent
+        parent_to_branches[parent].discard(branch)
+        del branch_tree[branch]
+
+
 def tree(treeref, recurse=False):
   """Returns a dict representation of a git tree object.
 
   Args:
     treeref (str) - a git ref which resolves to a tree (commits count as trees).
     recurse (bool) - include all of the tree's decendants too. File names will
       take the form of 'some/path/to/file'.
 
   Return format:
     { 'file_name': (mode, type, ref) }
@@ skipping 23 matching lines @@
     return None
   return ret
 
 
 def upstream(branch):
   try:
     return run('rev-parse', '--abbrev-ref', '--symbolic-full-name',
                branch+'@{upstream}')
   except subprocess2.CalledProcessError:
     return None
-
-
-def mktree(treedict):
-  """Makes a git tree object and returns its hash.
-
-  See |tree()| for the values of mode, type, and ref.
-
-  Args:
-    treedict - { name: (mode, type, ref) }
-  """
-  with tempfile.TemporaryFile() as f:
-    for name, (mode, typ, ref) in treedict.iteritems():
-      f.write('%s %s %s\t%s\0' % (mode, typ, ref, name))
-    f.seek(0)
-    return run('mktree', '-z', stdin=f)