Changed backoff to be exponential. This will help to reduce apply_issue flakiness.

rietveld.py

 # coding: utf-8
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Defines class Rietveld to easily access a rietveld instance.
 
 Security implications:
 
 The following hypothesis are made:
 - Rietveld enforces:
@@ skipping 391 matching lines @@
       def trap_http_500(msg):
         """Converts an incorrect ErrorExit() call into a HTTPError exception."""
         m = re.search(r'(50\d) Server Error', msg)
         if m:
           # Fake an HTTPError exception. Cheezy. :(
           raise urllib2.HTTPError(
               request_path, int(m.group(1)), msg, None, None)
         old_error_exit(msg)
       upload.ErrorExit = trap_http_500
 
-      maxtries = 5
+      maxtries = 40
       for retry in xrange(maxtries):
         try:
           logging.debug('%s' % request_path)
           result = self.rpc_server.Send(request_path, **kwargs)
           # Sometimes GAE returns a HTTP 200 but with HTTP 500 as the content.
           # How nice.
           return result
         except urllib2.HTTPError, e:
           if retry >= (maxtries - 1):
             raise
           if e.code not in (500, 502, 503):
             raise
         except urllib2.URLError, e:
           if retry >= (maxtries - 1):
             raise
           if (not 'Name or service not known' in e.reason and
               not 'EOF occurred in violation of protocol' in e.reason):
             # Usually internal GAE flakiness.
             raise
         except ssl.SSLError, e:
           if retry >= (maxtries - 1):
             raise
           if not 'timed out' in str(e):
             raise
         # If reaching this line, loop again. Uses a small backoff.
-        time.sleep(1+maxtries*2)
+        time.sleep(10)

[M-A Ruel, 2014/06/13 15:42:55]

Please replace with:

time.sleep(min(10, 1+maxtries*2))

[Sergiy Byelozyorov, 2014/06/13 17:02:16]

Done.

     finally:
       upload.ErrorExit = old_error_exit
 
   # DEPRECATED.
   Send = get
 
 
 class OAuthRpcServer(object):
   def __init__(self,
                host,
@@ skipping 268 matching lines @@
   def trigger_try_jobs(  # pylint:disable=R0201
       self, issue, patchset, reason, clobber, revision, builders_and_tests,
       master=None):
     logging.info('ReadOnlyRietveld: triggering try jobs %r for issue %d' %
         (builders_and_tests, issue))
 
   def trigger_distributed_try_jobs(  # pylint:disable=R0201
       self, issue, patchset, reason, clobber, revision, masters):
     logging.info('ReadOnlyRietveld: triggering try jobs %r for issue %d' %
         (masters, issue))