Add logging to fetch, retry if gitiles gives us weird unicode

recipe_engine/fetch.py

Index: recipe_engine/fetch.py
diff --git a/recipe_engine/fetch.py b/recipe_engine/fetch.py
index a221155cba1a24c5897e017a9cc0738a19ce7f6f..c2034ad443cc685f00cca41a70d2f236248e4036 100644
--- a/recipe_engine/fetch.py
+++ b/recipe_engine/fetch.py
@@ -24,6 +24,12 @@ from google.protobuf import text_format
 
 from . import package_pb2
 
+class RetryFetch(Exception):
+  """Raised to indicate that the fetch operation should be retried"""
+  def __init__(self, e):
+    super(RetryFetch, self).__init__(
+        'Retrying fetch because of exception: %r' % e)
+    self.exc = e
 
 class FetchError(Exception):
   pass
@@ -59,9 +65,10 @@ def _retry(f):
       try:
         return f(*args, **kwargs)
       except (requests.exceptions.RequestException,
-              subprocess42.CalledProcessError):
+              subprocess42.CalledProcessError,
+              RetryFetch) as e:
         # Only retry specific errors that may be transient.
-        logging.exception('retrying')
+        logging.warning('retrying because of %s', e)
         time.sleep(delay)
         delay *= 2
     return f(*args, **kwargs)
@@ -190,7 +197,15 @@ class GitilesBackend(Backend):
         repo, requests.utils.quote(revision))
     logging.info('fetching %s' % recipes_cfg_url)
     recipes_cfg_request = requests.get(recipes_cfg_url)
-    recipes_cfg_text = base64.b64decode(recipes_cfg_request.text)
+    try:
+      recipes_cfg_text = base64.b64decode(recipes_cfg_request.text)
+    except UnicodeEncodeError as e:
+      # UnicodeEncodeError can happen sometimes, if gitiles gives us some weird
+      # response. Just retry the whole thing.
+      logging.exception(
+          "got strange response from gitiles: %s", recipes_cfg_request.text)
+      raise RetryFetch(e)
+
     recipes_cfg_proto = package_pb2.Package()
     text_format.Merge(recipes_cfg_text, recipes_cfg_proto)
     recipes_path_rel = recipes_cfg_proto.recipes_path
@@ -277,7 +292,12 @@ class GitilesBackend(Backend):
   def _fetch_gitiles_json(self, url):
     """Fetches JSON from Gitiles and returns parsed result."""
     logging.info('fetching %s', url)
-    raw = requests.get(url).text
+    resp = requests.get(url)
+    raw = resp.text
     if not raw.startswith(')]}\'\n'):
-      raise FetchError('Unexpected gitiles response: %s' % raw)
+      logging.warning("Unexpected gitiles response: %s", raw)
+      raise FetchError(

[martiniss, 2016/09/23 00:42:58]

A (maybe unfounded) hypothesis is that the exception printing logic in python
doesn't like printing out gigantic text, which raw might be. So, I'm printing a
truncated text in the exception, and logging the full thing above.

+          'Unexpected gitiles response (code %d): %s... (truncated, see log for'
+          'full details)' % (resp.status_code, raw[:100]))
+
     return json.loads(raw.split('\n', 1)[1])