Add retry feature for push-to-trunk script.

tools/push-to-trunk/common_includes.py

 #!/usr/bin/env python
 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
@@ skipping 13 matching lines @@
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import os
 import re
 import subprocess
 import sys
 import textwrap
+import time
 import urllib2
 
 PERSISTFILE_BASENAME = "PERSISTFILE_BASENAME"
 TEMP_BRANCH = "TEMP_BRANCH"
 BRANCHNAME = "BRANCHNAME"
 DOT_GIT_LOCATION = "DOT_GIT_LOCATION"
 VERSION_FILE = "VERSION_FILE"
 CHANGELOG_FILE = "CHANGELOG_FILE"
 CHANGELOG_ENTRY_FILE = "CHANGELOG_ENTRY_FILE"
 COMMITMSG_FILE = "COMMITMSG_FILE"
@@ skipping 122 matching lines @@
 
   if len(bug_groups) > 0:
     return "(%s)" % ", ".join(bug_groups)
   else:
     return ""
 
 
 # Some commands don't like the pipe, e.g. calling vi from within the script or
 # from subscripts like git cl upload.
 def Command(cmd, args="", prefix="", pipe=True):
+  # TODO(machenbach): Use timeout.
   cmd_line = "%s %s %s" % (prefix, cmd, args)
   print "Command: %s" % cmd_line
   try:
     if pipe:
       return subprocess.check_output(cmd_line, shell=True)
     else:
       return subprocess.check_call(cmd_line, shell=True)
   except subprocess.CalledProcessError:
     return None
 
 
 # Wrapper for side effects.
 class SideEffectHandler(object):
   def Command(self, cmd, args="", prefix="", pipe=True):
     return Command(cmd, args, prefix, pipe)
 
   def ReadLine(self):
     return sys.stdin.readline().strip()
 
   def ReadURL(self, url):
     # pylint: disable=E1121
     url_fh = urllib2.urlopen(url, None, 60)
     try:
       return url_fh.read()
     finally:
       url_fh.close()
 
+  def Sleep(seconds):
+    time.sleep(seconds)
+
 DEFAULT_SIDE_EFFECT_HANDLER = SideEffectHandler()
 
 
 class Step(object):
   def __init__(self, text, requires, number, config, state, options, handler):
     self._text = text
     self._requires = requires
     self._number = number
     self._config = config
     self._state = state
@@ skipping 11 matching lines @@
     if self._requires:
       self.RestoreIfUnset(self._requires)
       if not self._state[self._requires]:
         return
     print ">>> Step %d: %s" % (self._number, self._text)
     self.RunStep()
 
   def RunStep(self):
     raise NotImplementedError
 
+  def Retry(self, cb, retry_on=None, wait_plan=None):
+    """ Retry a function.
+    Params:
+      cb: The function to retry.
+      retry_on: A callback that takes the result of the function and returns True

[Jakob Kummerow, 2013/11/28 13:47:27]

nit: 80col

[Michael Achenbach, 2013/11/28 14:28:17]

Done.

+                if the function should be retried. A function throwing an
+                exception is always retried.
+      wait_plan: A list of waiting delays between retries in seconds. The maximum

[Jakob Kummerow, 2013/11/28 13:47:27]

nit: 80col

[Michael Achenbach, 2013/11/28 14:28:17]

Done.

+                 number of retries is len(wait_plan).
+    """
+    retry_on = retry_on or (lambda x: False)
+    wait_plan = list(wait_plan or [])

[Jakob Kummerow, 2013/11/28 13:47:27]

I don't think the explicit list() conversion is needed.

[Michael Achenbach, 2013/11/28 14:28:17]

I'd rather keep it, since the method has side effects on the parameter. If
somebody ever saved a wait plan in a variable on the call side for reuse...

+    # Sentinel for the first try.

[Jakob Kummerow, 2013/11/28 13:47:27]

s/first/last/! This is seriously misleading.

Here's a slight refactoring that doesn't need the sentinel:

wait_plan.reverse()
while True:
  try:
    result = cb()
  except Exception:
    got_exception = True
  if got_exception or retry_on(result):
    if not wait_plan:
      raise Exception("Retried too often. Giving up.")
    wait_time = wait_plan.pop()
    print "Waiting for %f seconds." % wait_time
    self._side_effect_handler.Sleep(wait_time)
    print "Retrying..."
  else:
    return result

[Michael Achenbach, 2013/11/28 14:28:17]

I take it :) Just added an initialization of got_exception.

+    wait_plan.append(0)
+    wait_plan.reverse()
+    while wait_plan:
+      got_exception = False
+      try:
+        result = cb()
+      except Exception:
+        got_exception = True
+      if got_exception or retry_on(result):
+        wait_time = wait_plan.pop()
+        print "Waiting for %f seconds." % wait_time
+        self._side_effect_handler.Sleep(wait_time)
+        print "Retrying..."
+      else:
+        return result
+    raise Exception("Retried too often. Giving up.")
+
   def ReadLine(self, default=None):
     # Don't prompt in forced mode.
     if self._options and self._options.f and default is not None:
       print "%s (forced)" % default
       return default
     else:
       return self._side_effect_handler.ReadLine()
 
-  def Git(self, args="", prefix="", pipe=True):
-    return self._side_effect_handler.Command("git", args, prefix, pipe)
+  def Git(self, args="", prefix="", pipe=True, retry_on=None):
+    cmd = lambda: self._side_effect_handler.Command("git", args, prefix, pipe)
+    return self.Retry(cmd, retry_on, [5, 30])
 
   def Editor(self, args):
     return self._side_effect_handler.Command(os.environ["EDITOR"], args,
                                              pipe=False)
 
-  def ReadURL(self, url):
-    return self._side_effect_handler.ReadURL(url)
+  def ReadURL(self, url, retry_on=None, wait_plan=None):
+    wait_plan = wait_plan or [3, 10, 30]

[Jakob Kummerow, 2013/11/28 13:47:27]

I'd give the server much more time. Maybe [3, 60, 600].

[Michael Achenbach, 2013/11/28 14:28:17]

Done.

+    cmd = lambda: self._side_effect_handler.ReadURL(url)
+    return self.Retry(cmd, retry_on, wait_plan)
 
   def Die(self, msg=""):
     if msg != "":
       print "Error: %s" % msg
     print "Exiting"
     raise Exception(msg)
 
   def DieInForcedMode(self, msg=""):
     if self._options and self._options.f:
       msg = msg or "Not implemented in forced mode."
@@ skipping 179 matching lines @@
               options,
               side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
   state = {}
   steps = []
   for (number, step_class) in enumerate(step_classes):
     steps.append(MakeStep(step_class, number, state, config,
                           options, side_effect_handler))
 
   for step in steps[options.s:]:
     step.Run()