Make TimeoutRetryThread's stoppable

build/android/pylib/utils/timeout_retry.py

Index: build/android/pylib/utils/timeout_retry.py
diff --git a/build/android/pylib/utils/timeout_retry.py b/build/android/pylib/utils/timeout_retry.py
index bae8b16cf7ffd5b16b32e849ffb1af26bf290ead..7f05a4c3073678b4a2efa2194cba40f9ad5981cc 100644
--- a/build/android/pylib/utils/timeout_retry.py
+++ b/build/android/pylib/utils/timeout_retry.py
@@ -6,13 +6,27 @@
 # pylint: disable=W0702
 
 import threading
+import time
 
 from pylib.utils import reraiser_thread
 from pylib.utils import watchdog_timer
 
 
 class TimeoutRetryThread(reraiser_thread.ReraiserThread):
-  pass
+  def __init__(self, *args, **kwargs):
+    super(TimeoutRetryThread, self).__init__(*args, **kwargs)
+    self._timeout_expired = False
+    self._start = time.time()
+
+  def ElapsedTime(self):
+    return time.time() - self._start
+
+  def SetTimeoutAsExpired(self):
+    self._timeout_expired = True

[perezju, 2014/10/28 14:31:18]

Have tried several names now, maybe RequestThreadTimeout would work better?

[jbudorick, 2014/10/28 17:28:46]

Yeah, I like that better. That implies that we're asking the thread to do
something, not telling it something, which seems more accurate.

+
+  def CheckTimeout(self):
+    if self._timeout_expired:
+      raise reraiser_thread.TimeoutError('Thread timed out')
 
 
 def Run(func, timeout, retries, args=None, kwargs=None):
@@ -40,15 +54,19 @@ def Run(func, timeout, retries, args=None, kwargs=None):
   def RunOnTimeoutThread():
     ret[0] = func(*args, **kwargs)
 
+  num_try = 1
   while True:
+    child_thread = TimeoutRetryThread(
+      RunOnTimeoutThread,
+      name='TimeoutThread-%d-for-%s' % (num_try,
+                                        threading.current_thread().name))
     try:
-      name = 'TimeoutThread-for-%s' % threading.current_thread().name
-      thread_group = reraiser_thread.ReraiserThreadGroup(
-          [TimeoutRetryThread(RunOnTimeoutThread, name=name)])
+      thread_group = reraiser_thread.ReraiserThreadGroup([child_thread])
       thread_group.StartAll()
       thread_group.JoinAll(watchdog_timer.WatchdogTimer(timeout))
       return ret[0]
     except:
-      if retries <= 0:
+      child_thread.SetTimeoutAsExpired()

[jbudorick, 2014/10/28 17:28:46]

I think we only want to do this if we catch a timeout exception.

+      if num_try > retries:
         raise
-      retries -= 1
+      num_try += 1