Fixes incremental verification when the commit queue crashes.

pending_manager.py

 # coding=utf8
 # Copyright (c) 2010 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Commit queue manager class.
 
 Security implications:
 
 The following hypothesis are made:
 - Commit queue:
@@ skipping 61 matching lines @@
 
     It makes it possible to regenerate the try_jobs array if ever needed."""
     return '%d-%d' % (self.issue, self.patchset)
 
   def verify(self, revision, verifiers):
     """Runs verifiers on a pending change.
 
     Returns True if all Verifiers were run.
     """
     for verifier in verifiers:
-      assert verifier.name not in self.verifications
+      if verifier.name in self.verifications:
+        logging.warning(
+            'Re-running verififer %s for issue %s' % (
+                verifier.name, self.issue))
       verifier.verify(self, revision)
       assert verifier.name in self.verifications
       if self.get_state() == base.IGNORED:
         assert self.verifications[verifier.name].get_state() == base.IGNORED
         # Remove all the other verifiers since we need to keep it in the
         # 'datastore' to not retry this issue constantly.
         for key in self.verifications.keys():
           if key != verifier.name:
             del self.verifications[key]
         return False
@@ skipping 61 matching lines @@
       send_stack(e)
 
   def _fetch_pending_issues(self):
     """Returns the list of issue number for reviews on Rietveld with their last
     patchset with commit+ flag set.
     """
     return self.rietveld.get_pending_issues()
 
   def process_new_pending_commit(self):
     """Starts verification on newly found pending commits."""
+    expected = set(i.name for i in self.all_verifiers)
     for pending in self.queue.pending_commits[:]:
       try:
-        if (len(pending.verifications) == len(self.all_verifiers) or
-            pending.get_state() != base.PROCESSING):
+        # Take in account the case where a verifier was removed.
+        done = set(pending.verifications.keys())
+        missing = expected - done
+        if (not missing or pending.get_state() != base.PROCESSING):
           continue
-        logging.info('Processing issue %s' % pending.issue)
+        logging.info(
+            'Processing issue %s (%s, %d)' % (
+                pending.issue, missing, pending.get_state()))
 
         revision = self.checkout.prepare()
         if pending.verify(revision, self.pre_patch_verifiers):
           self._apply_patch(pending, revision)
           previous_cwd = os.getcwd()
           try:
             os.chdir(self.checkout.project_path)
             pending.verify(revision, self.verifiers)
           finally:
             os.chdir(previous_cwd)
@@ skipping 154 matching lines @@
       raise base.DiscardPending(pending, out)
 
   def load(self, filename):
     """Loads the commit queue state from a JSON file."""
     self.queue = model.load_from_json_file(filename)
     self.queue.pending_commits = self.queue.pending_commits or []
 
   def save(self, filename):
     """Save the commit queue state in a simple JSON file."""
     model.save_to_json_file(filename, self.queue)