[Android] Print the stack of deadlocked threads when sharding.

build/android/pylib/utils/reraiser_thread.py

Index: build/android/pylib/utils/reraiser_thread.py
diff --git a/build/android/pylib/utils/reraiser_thread.py b/build/android/pylib/utils/reraiser_thread.py
index 95c3f74d163e512f734d740cc701409a89912086..01ec18592d8b00e9378d810e51ff015cb7ca74c6 100644
--- a/build/android/pylib/utils/reraiser_thread.py
+++ b/build/android/pylib/utils/reraiser_thread.py
@@ -4,12 +4,23 @@
 
 """Thread and ThreadGroup that reraise exceptions on the main thread."""
 
+import logging
 import sys
 import threading
+import time
+import traceback
+
+import watchdog_timer
+
+
+class TimeoutError(Exception):
+  """Module-specific timeout exception."""
+  pass
 
 
 class ReraiserThread(threading.Thread):
   """Thread class that can reraise exceptions."""
+
   def __init__(self, func, args=[], kwargs={}):
     super(ReraiserThread, self).__init__()
     self.daemon = True
@@ -35,6 +46,7 @@ class ReraiserThread(threading.Thread):
 
 class ReraiserThreadGroup(object):
   """A group of ReraiserThread objects."""
+
   def __init__(self, threads=[]):
     """Initialize thread group.
 
@@ -56,15 +68,21 @@ class ReraiserThreadGroup(object):
     for thread in self._threads:
       thread.start()
 
-  def JoinAll(self):
-    """Join all threads.
+  def _JoinAll(self, watcher=watchdog_timer.WatchdogTimer(None)):
+    """Join all threads without stack dumps.
 
-    Reraises exceptions raised by the child threads and supports
-    breaking immediately on exceptions raised on the main thread.
+    Reraises exceptions raised by the child threads and supports breaking
+    immediately on exceptions raised on the main thread.
+
+    Args:
+      watcher: Watchdog object providing timeout, by default waits forever.
     """
     alive_threads = self._threads[:]
     while alive_threads:
       for thread in alive_threads[:]:
+        if watcher.IsTimedOut():
+          raise TimeoutError('Timed out waiting for %d of %d threads.' %
+                             (len(alive_threads), len(self._threads)))
         # Allow the main thread to periodically check for interrupts.
         thread.join(0.1)
         if not thread.isAlive():
@@ -72,3 +90,27 @@ class ReraiserThreadGroup(object):
     # All threads are allowed to complete before reraising exceptions.
     for thread in self._threads:
       thread.ReraiseIfException()
+
+  def JoinAll(self, watcher=watchdog_timer.WatchdogTimer(None)):
+    """Join all threads.
+
+    Reraises exceptions raised by the child threads and supports breaking
+    immediately on exceptions raised on the main thread. Unfinished threads'
+    stacks will be logged on watchdog timeout.
+
+    Args:
+      watcher: Watchdog object providing timeout, by default waits forever.
+    """
+    try:
+      self._JoinAll(watcher)
+    except TimeoutError:
+      alive_thread_ids = (t.ident for t in self._threads if t.isAlive())
+      for thread_id in alive_thread_ids:
+        stack = sys._current_frames()[thread_id]
+        logging.critical('STACK DUMP FOR TIMED OUT THREAD ID = %s', thread_id)

[frankf, 2013/04/03 23:17:48]

'*' * 80
Blah
'*' * 80

+        for filename, lineno, name, line in traceback.extract_stack(stack):
+          logging.critical('File: "%s", line %d, in %s', filename, lineno, name)
+          if line:
+            logging.critical('  %s', line.strip())
+        logging.critical('')
+      raise