More strategic retries in fetch.

recipe_engine/util.py

Index: recipe_engine/util.py
diff --git a/recipe_engine/util.py b/recipe_engine/util.py
index cb3257b37f1f98027aae40c9b4b699c15d88c26a..91c5192c8c7ccad7d5da3e1b04b327b3a796e8df 100644
--- a/recipe_engine/util.py
+++ b/recipe_engine/util.py
@@ -3,10 +3,13 @@
 # that can be found in the LICENSE file.
 
 import contextlib
+import datetime
 import functools
+import logging
 import os
 import sys
 import traceback
+import time
 import urllib
 
 from cStringIO import StringIO
@@ -190,3 +193,38 @@ class StringListIO(object):
   def close(self):
     if not isinstance(self.lines[-1], basestring):
       self.lines[-1] = self.lines[-1].getvalue()
+
+
+class exponential_retry(object):
+  """Decorator which retries the function if an exception is encountered."""
+
+  def __init__(self, retries=None, delay=None, condition=None):
+    """Creates a new exponential retry decorator.
+
+    Args:
+      retries (int): Maximum number of retries before giving up.
+      delay (datetime.timedelta): Amount of time to wait before retrying. This
+          will double every retry attempt (exponential).
+      condition (func): If not None, a function that will be passed the
+          exception as its one argument. Retries will only happen if this
+          function returns True. If None, retries will always happen.
+    """
+    self.retries = retries or 5
+    self.delay = delay or datetime.timedelta(seconds=1)
+    self.condition = condition or (lambda e: True)
+
+  def __call__(self, f):
+    @functools.wraps(f)
+    def wrapper(*args, **kwargs):
+      retry_delay = self.delay
+      for i in xrange(self.retries):
+        try:
+          return f(*args, **kwargs)
+        except Exception as e:
+          if (i+1) >= self.retries or not self.condition(e):
+            raise
+          logging.exception('Exception encountered, retrying in %s',
+                            retry_delay)
+          time.sleep(retry_delay.total_seconds())
+          retry_delay *= 2
+    return wrapper