CQ: add landmines to delete database after unclean shutdown

commit-queue/commit_queue.py

Index: commit-queue/commit_queue.py
===================================================================
--- commit-queue/commit_queue.py	(revision 225230)
+++ commit-queue/commit_queue.py	(working copy)
@@ -12,8 +12,10 @@
 import logging.handlers
 import optparse
 import os
+import shutil
 import signal
 import sys
+import tempfile
 import time
 
 import find_depot_tools  # pylint: disable=W0611
@@ -123,6 +125,26 @@
   logging.getLogger().addHandler(logging_rotating_file)
 
 
+class SignalInterrupt(Exception):
+  """Exception that indicates being interrupted by a caught signal."""
+  
+  def __init__(self, signal_set=None, *args, **kwargs):
+    super(SignalInterrupt, self).__init__(*args, **kwargs)
+    self.signal_set = signal_set
+
+
+def SaveDatabaseCopyForDebugging(db_path):
+  """Saves database file for debugging. Returns name of the saved file."""
+  with tempfile.NamedTemporaryFile(
+      dir=os.path.dirname(db_path),
+      prefix='db.debug.',
+      suffix='.json',
+      delete=False) as tmp_file:
+    with open(db_path) as db_file:
+      shutil.copyfileobj(db_file, tmp_file)
+    return tmp_file.name
+
+
 def main():
   parser = optparse.OptionParser(
       description=sys.modules['__main__'].__doc__)
@@ -235,13 +257,31 @@
       print 'Using read-only chromium-status interface'
       pc.context.status = async_push.AsyncPushStore()
 
+    landmine_path = os.path.join(work_dir,
+                                 pc.context.checkout.project_name + '.landmine')
     db_path = os.path.join(work_dir, pc.context.checkout.project_name + '.json')
     if os.path.isfile(db_path):
-      try:
-        pc.load(db_path)
-      except ValueError:
+      if os.path.isfile(landmine_path):
+        debugging_path = SaveDatabaseCopyForDebugging(db_path)
         os.remove(db_path)
+        logging.warning(('Deleting database because previous shutdown '
+                        'was unclean. The copy of the database is saved '
+                        'as %s.') % debugging_path)
+      else:
+        try:
+          pc.load(db_path)
+        except ValueError as e:
+          debugging_path = SaveDatabaseCopyForDebugging(db_path)
+          os.remove(db_path)
+          logging.warning(('Failed to parse database (%r), deleting it. '
+                          'The copy of the database is saved as %s.') %
+                          (e, debugging_path))
+          raise e
 
+    # Create a file to indicate unclean shutdown.
+    with open(landmine_path, 'w'):
+      pass
+
     sig_handler.installHandlers(
         signal.SIGINT,
         signal.SIGHUP
@@ -254,6 +294,7 @@
         pc.look_for_new_pending_commit()
         pc.update_status()
         print(str(pc.queue))
+        os.remove(landmine_path)
         return 0
 
       now = time.time()
@@ -268,7 +309,7 @@
         pc.update_status()
         pc.scan_results()
         if sig_handler.getTriggeredSignals():
-          raise KeyboardInterrupt()
+          raise SignalInterrupt(signal_set=sig_handler.getTriggeredSignals())
         # Save the db at each loop. The db can easily be in the 1mb range so
         # it's slowing down the CQ a tad but it in the 100ms range even for that
         # size.
@@ -293,7 +334,7 @@
         while True:
           # Abort if any signals are set
           if sig_handler.getTriggeredSignals():
-            raise KeyboardInterrupt()
+            raise SignalInterrupt(signal_set=sig_handler.getTriggeredSignals())
           delay = next_loop - now
           if delay <= 0:
             break
@@ -314,13 +355,32 @@
       pc.save(db_path)
       pc.close()
       logging.warning('db save successful.')
-  except KeyboardInterrupt as e:
-    print 'Bye bye'
+  except SignalInterrupt:
+    # This is considered a clean shutdown: we only throw this exception
+    # from selected places in the code where the database should be
+    # in a known and consistent state.
+    os.remove(landmine_path)
+
+    print 'Bye bye (SignalInterrupt)'
     # 23 is an arbitrary value to signal loop.sh that it must stop looping.
     return 23
+  except KeyboardInterrupt:
+    # This is actually an unclean shutdown. Do not remove the landmine file.
+    # One example of this is user hitting ctrl-c twice at an arbitrary point
+    # inside the CQ loop. There are no guarantees about consistent state
+    # of the database then.
+
+    print 'Bye bye (KeyboardInterrupt - this is considered unclean shutdown)'
+    # 23 is an arbitrary value to signal loop.sh that it must stop looping.
+    return 23
   except errors.ConfigurationError as e:
     parser.error(str(e))
     return 1
+
+  # CQ generally doesn't exit by itself, but if we ever get here, it looks
+  # like a clean shutdown so remove the landmine file.
+  # TODO(phajdan.jr): Do we ever get here?
+  os.remove(landmine_path)
   return 0