CQ: catch more exceptions when looking for new pending changes.

commit-queue/pending_manager.py

 # coding=utf8
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Commit queue manager class.
 
 Security implications:
 
 The following hypothesis are made:
 - Commit queue:
   - Impersonate the same svn credentials that the patchset owner.
   - Can't impersonate a non committer.
   - SVN will check the committer write access.
 """
 
+import errno
 import logging
 import os
+import socket
+import ssl
 import time
 import traceback
 import urllib2
 
 import find_depot_tools  # pylint: disable=W0611
 import checkout
 import git_cl
 import patch
-import ssl
 import subprocess2
 
 import errors
 import model
 from verification import base
 
 
 class PendingCommit(base.Verified):
   """Represents a pending commit that is being processed."""
   # Important since they tell if we need to revalidate and send try jobs
@@ skipping 149 matching lines @@
             pending,
             { 'verification': 'abort',
               'payload': {
                 'output': 'CQ bit was unchecked on CL. Ignoring.' }})
         pending.get_state = lambda: base.IGNORED
         self._discard_pending(pending, None)
 
     # Find new issues.
     for issue_id in new_issues:
       if str(issue_id) not in self.queue.pending_commits:
-        issue_data = self.context.rietveld.get_issue_properties(
-            issue_id, True)
+        try:
+          issue_data = self.context.rietveld.get_issue_properties(
+              issue_id, True)
+        except urllib2.HTTPError as e:
+          if e.code in (500, 502, 503):
+            # Temporary AppEngine hiccup. Just log it and continue.
+            logging.warning(str(e))

[Ryan Tseng, 2013/11/16 01:38:37]

logging.warning('%s while accessing %s.  Ignoring error.' % (str(e), e.url))

+            continue
+          raise

[Ryan Tseng, 2013/11/16 01:38:37]

"raise e" so we can propagate this downstream.

Also 212, 218, and 224

+        except urllib2.URLError as e:
+          # Temporary AppEngine hiccup. Just log it and continue.
+          if 'timed out' in e.reason:
+            logging.warning(str(e))

[Ryan Tseng, 2013/11/16 01:38:37]

Make this more verbose.
'%s while accessing rietveld issue %d.  Ignoring error.' % (str(e), issue_id)

Same with 216, 222

+            continue
+          raise
+        except socket.error as e:
+          # Temporary AppEngine hiccup. Just log it and continue.
+          if e.errno == errno.ECONNRESET:
+            logging.warning(str(e))
+            continue
+          raise
+        except IOError as e:
+          # Temporary AppEngine hiccup. Just log it and continue.
+          if e.errno == 'socket error':
+            logging.warning(str(e))
+            continue
+          raise
         # This assumption needs to hold.
         assert issue_id == issue_data['issue']
         if issue_data['patchsets'] and issue_data['commit']:
           logging.info('Found new issue %d' % issue_id)
           self.queue.add(
               PendingCommit(
                   issue=issue_id,
                   owner=issue_data['owner_email'],
                   reviewers=issue_data['reviewers'],
                   patchset=issue_data['patchsets'][-1],
@@ skipping 319 matching lines @@
     """Loads the commit queue state from a JSON file."""
     self.queue = model.load_from_json_file(filename)
 
   def save(self, filename):
     """Save the commit queue state in a simple JSON file."""
     model.save_to_json_file(filename, self.queue)
 
   def close(self):
     """Close all the active pending manager items."""
     self.context.status.close()