More strategic retries in fetch.

recipe_engine/unittests/fetch_test.py

 #!/usr/bin/env python
 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import base64
 import io
 import json
 import os
 import sys
 import unittest
+import time
 
 import test_env
 
 import mock
 import subprocess42
 
 from recipe_engine import fetch
 from recipe_engine import requests_ssl
 
 
@@ skipping 129 matching lines @@
   @mock.patch('os.makedirs')
   @mock.patch('tarfile.open')
   @mock.patch('requests.get')
   def test_checkout(self, requests_get, tarfile_open, makedirs, rmtree):
     proto_text = u"""
 api_version: 1
 project_id: "foo"
 recipes_path: "path/to/recipes"
 """.lstrip()
     requests_get.side_effect = [
-        mock.Mock(text=u')]}\'\n{ "commit": "abc123" }'),
-        mock.Mock(text=base64.b64encode(proto_text)),
-        mock.Mock(content=''),
+        mock.Mock(text=u')]}\'\n{ "commit": "abc123" }', status_code=200),
+        mock.Mock(text=base64.b64encode(proto_text), status_code=200),
+        mock.Mock(content='', status_code=200),
     ]
 
     fetch.GitilesBackend().checkout(
         'repo', 'revision', 'dir', allow_fetch=True)
 
     requests_get.assert_has_calls([
         mock.call('repo/+/revision?format=JSON'),
         mock.call('repo/+/abc123/infra/config/recipes.cfg?format=TEXT'),
         mock.call('repo/+archive/abc123/path/to/recipes.tar.gz'),
     ])
@@ skipping 36 matching lines @@
                     },
                     {
                         'old_path': 'path2/foo',
                         'new_path': '/dev/null',
                     },
                 ],
             },
         ],
     }
     requests_get.side_effect = [
-        mock.Mock(text=u')]}\'\n{ "commit": "sha_a" }'),
-        mock.Mock(text=u')]}\'\n{ "commit": "sha_b" }'),
-        mock.Mock(text=u')]}\'\n%s' % json.dumps(log_json)),
+        mock.Mock(text=u')]}\'\n{ "commit": "sha_a" }', status_code=200),
+        mock.Mock(text=u')]}\'\n{ "commit": "sha_b" }', status_code=200),
+        mock.Mock(text=u')]}\'\n%s' % json.dumps(log_json), status_code=200),
     ]
 
     self.assertEqual(
         ['ghi789', 'abc123'],
         fetch.GitilesBackend().updates(
             'repo', 'reva', 'dir', True, 'revb',
             ['path1', 'path2']))
 
     requests_get.assert_has_calls([
         mock.call('repo/+/reva?format=JSON'),
         mock.call('repo/+/revb?format=JSON'),
         mock.call('repo/+log/sha_a..sha_b?name-status=1&format=JSON'),
     ])
 
   @mock.patch('requests.get')
   def test_commit_metadata(self, requests_get):
     revision_json = {
       'author': {'email': 'author'},
       'message': 'message',
     }
     requests_get.side_effect = [
-        mock.Mock(text=u')]}\'\n%s' % json.dumps(revision_json)),
+        mock.Mock(text=u')]}\'\n%s' % json.dumps(revision_json),
+                  status_code=200),
     ]
     result = fetch.GitilesBackend().commit_metadata(
         'repo', 'revision', 'dir', allow_fetch=True)
     self.assertEqual(result, {
       'author': 'author',
       'message': 'message',
     })
     requests_get.assert_has_calls([
       mock.call('repo/+/revision?format=JSON'),
     ])
 
+  @mock.patch('requests.get')
+  def test_non_transient_error(self, requests_get):
+    requests_get.side_effect = [
+        mock.Mock(text='Not permitted.', status_code=403),
+    ]
+    with self.assertRaises(fetch.GitilesFetchError):
+      fetch.GitilesBackend().commit_metadata(
+          'repo', 'revision', 'dir', allow_fetch=True)
+    requests_get.assert_has_calls([
+      mock.call('repo/+/revision?format=JSON'),
+    ])
+
+  @mock.patch('requests.get')
+  @mock.patch('time.sleep')
+  @mock.patch('logging.exception')
+  def test_transient_retry(self, logging_exception, time_sleep, requests_get):
+    counts = {
+        'sleeps': 0,
+        'fails': 0,
+    }
+
+    def count_sleep(delay):
+      counts['sleeps'] += 1
+    time_sleep.side_effect = count_sleep
+
+    revision_json = {
+      'author': {'email': 'author'},
+      'message': 'message',
+    }
+
+    # Fail transiently 4 times, but succeed on the 5th.
+    def transient_side_effect(*args, **kwargs):
+      if counts['fails'] < 4:
+        counts['fails'] += 1
+        return mock.Mock(text=u'Not permitted (%(fails)d).' % counts,
+                         status_code=500)
+      return mock.Mock(text=u')]}\'\n%s' % json.dumps(revision_json),
+                       status_code=200)
+    requests_get.side_effect = transient_side_effect
+
+    result = fetch.GitilesBackend().commit_metadata(
+        'repo', 'revision', 'dir', allow_fetch=True)
+    self.assertEqual(result, {
+      'author': 'author',
+      'message': 'message',
+    })
+    self.assertEqual(counts['sleeps'], 4)
+    requests_get.assert_has_calls([
+      mock.call('repo/+/revision?format=JSON'),
+    ] * 5)
 
 if __name__ == '__main__':
   unittest.main()