More strategic retries in fetch.

recipe_engine/util.py

 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import contextlib
+import datetime
 import functools
+import logging
 import os
 import sys
 import traceback
+import time
 import urllib
 
 from cStringIO import StringIO
 
 class RecipeAbort(Exception):
   pass
 
 
 class ModuleInjectionError(AttributeError):
   pass
@@ skipping 163 matching lines @@
         self.lines[-1].write(s)
         break
       self.lines[-1].write(s[:i])
       self.lines[-1] = self.lines[-1].getvalue()
       self.lines.append(StringIO())
       s = s[i+1:]
 
   def close(self):
     if not isinstance(self.lines[-1], basestring):
       self.lines[-1] = self.lines[-1].getvalue()
+
+
+class exponential_retry(object):
+  """Decorator which retries the function if an exception is encountered."""
+
+  def __init__(self, retries=None, delay=None, condition=None):
+    """Creates a new exponential retry decorator.
+
+    Args:
+      retries (int): Maximum number of retries before giving up.
+      delay (datetime.timedelta): Amount of time to wait before retrying. This
+          will double every retry attempt (exponential).
+      condition (func): If not None, a function that will be passed the
+          exception as its one argument. Retries will only happen if this
+          function returns True. If None, retries will always happen.
+    """
+    self.retries = retries or 5
+    self.delay = delay or datetime.timedelta(seconds=1)
+    self.condition = condition or (lambda e: True)
+
+  def __call__(self, f):
+    @functools.wraps(f)
+    def wrapper(*args, **kwargs):
+      retry_delay = self.delay
+      for i in xrange(self.retries):
+        try:
+          return f(*args, **kwargs)
+        except Exception as e:
+          if (i+1) >= self.retries or not self.condition(e):
+            raise
+          logging.exception('Exception encountered, retrying in %s',
+                            retry_delay)
+          time.sleep(retry_delay.total_seconds())
+          retry_delay *= 2
+    return wrapper