| Index: build/android/pylib/utils/timeout_retry.py
|
| diff --git a/build/android/pylib/utils/timeout_retry.py b/build/android/pylib/utils/timeout_retry.py
|
| new file mode 100644
|
| index 0000000000000000000000000000000000000000..61f7c70baa91396bed71ecdb64a7695d3e74f11f
|
| --- /dev/null
|
| +++ b/build/android/pylib/utils/timeout_retry.py
|
| @@ -0,0 +1,167 @@
|
| +# Copyright 2013 The Chromium Authors. All rights reserved.
|
| +# Use of this source code is governed by a BSD-style license that can be
|
| +# found in the LICENSE file.
|
| +
|
| +"""A utility to run functions with timeouts and retries."""
|
| +# pylint: disable=W0702
|
| +
|
| +import logging
|
| +import threading
|
| +import time
|
| +import traceback
|
| +
|
| +from pylib.utils import reraiser_thread
|
| +from pylib.utils import watchdog_timer
|
| +
|
| +
|
| +class TimeoutRetryThread(reraiser_thread.ReraiserThread):
|
| + def __init__(self, func, timeout, name):
|
| + super(TimeoutRetryThread, self).__init__(func, name=name)
|
| + self._watcher = watchdog_timer.WatchdogTimer(timeout)
|
| + self._expired = False
|
| +
|
| + def GetWatcher(self):
|
| + """Returns the watchdog keeping track of this thread's time."""
|
| + return self._watcher
|
| +
|
| + def GetElapsedTime(self):
|
| + return self._watcher.GetElapsed()
|
| +
|
| + def GetRemainingTime(self, required=0, msg=None):
|
| + """Get the remaining time before the thread times out.
|
| +
|
| + Useful to send as the |timeout| parameter of async IO operations.
|
| +
|
| + Args:
|
| + required: minimum amount of time that will be required to complete, e.g.,
|
| + some sleep or IO operation.
|
| + msg: error message to show if timing out.
|
| +
|
| + Returns:
|
| + The number of seconds remaining before the thread times out, or None
|
| + if the thread never times out.
|
| +
|
| + Raises:
|
| + reraiser_thread.TimeoutError if the remaining time is less than the
|
| + required time.
|
| + """
|
| + remaining = self._watcher.GetRemaining()
|
| + if remaining is not None and remaining < required:
|
| + if msg is None:
|
| + msg = 'Timeout expired'
|
| + if remaining > 0:
|
| + msg += (', wait of %.1f secs required but only %.1f secs left'
|
| + % (required, remaining))
|
| + self._expired = True
|
| + raise reraiser_thread.TimeoutError(msg)
|
| + return remaining
|
| +
|
| + def LogTimeoutException(self):
|
| + """Log the exception that terminated this thread."""
|
| + if not self._expired:
|
| + return
|
| + logging.critical('*' * 80)
|
| + logging.critical('%s on thread %r', self._exc_info[0].__name__, self.name)
|
| + logging.critical('*' * 80)
|
| + fmt_exc = ''.join(traceback.format_exception(*self._exc_info))
|
| + for line in fmt_exc.splitlines():
|
| + logging.critical(line.rstrip())
|
| + logging.critical('*' * 80)
|
| +
|
| +
|
| +def CurrentTimeoutThread():
|
| + """Get the current thread if it is a TimeoutRetryThread.
|
| +
|
| + Returns:
|
| + The current thread if it is a TimeoutRetryThread, otherwise None.
|
| + """
|
| + current_thread = threading.current_thread()
|
| + if isinstance(current_thread, TimeoutRetryThread):
|
| + return current_thread
|
| + else:
|
| + return None
|
| +
|
| +
|
| +def WaitFor(condition, wait_period=5, max_tries=None):
|
| + """Wait for a condition to become true.
|
| +
|
| + Repeadly call the function condition(), with no arguments, until it returns
|
| + a true value.
|
| +
|
| + If called within a TimeoutRetryThread, it cooperates nicely with it.
|
| +
|
| + Args:
|
| + condition: function with the condition to check
|
| + wait_period: number of seconds to wait before retrying to check the
|
| + condition
|
| + max_tries: maximum number of checks to make, the default tries forever
|
| + or until the TimeoutRetryThread expires.
|
| +
|
| + Returns:
|
| + The true value returned by the condition, or None if the condition was
|
| + not met after max_tries.
|
| +
|
| + Raises:
|
| + reraiser_thread.TimeoutError if the current thread is a TimeoutRetryThread
|
| + and the timeout expires.
|
| + """
|
| + condition_name = condition.__name__
|
| + timeout_thread = CurrentTimeoutThread()
|
| + while max_tries is None or max_tries > 0:
|
| + result = condition()
|
| + if max_tries is not None:
|
| + max_tries -= 1
|
| + msg = ['condition', repr(condition_name), 'met' if result else 'not met']
|
| + if timeout_thread:
|
| + msg.append('(%.1fs)' % timeout_thread.GetElapsedTime())
|
| + logging.info(' '.join(msg))
|
| + if result:
|
| + return result
|
| + if timeout_thread:
|
| + timeout_thread.GetRemainingTime(wait_period,
|
| + msg='Timed out waiting for %r' % condition_name)
|
| + time.sleep(wait_period)
|
| + return None
|
| +
|
| +
|
| +def Run(func, timeout, retries, args=None, kwargs=None):
|
| + """Runs the passed function in a separate thread with timeouts and retries.
|
| +
|
| + Args:
|
| + func: the function to be wrapped.
|
| + timeout: the timeout in seconds for each try.
|
| + retries: the number of retries.
|
| + args: list of positional args to pass to |func|.
|
| + kwargs: dictionary of keyword args to pass to |func|.
|
| +
|
| + Returns:
|
| + The return value of func(*args, **kwargs).
|
| + """
|
| + if not args:
|
| + args = []
|
| + if not kwargs:
|
| + kwargs = {}
|
| +
|
| + # The return value uses a list because Python variables are references, not
|
| + # values. Closures make a copy of the reference, so updating the closure's
|
| + # reference wouldn't update where the original reference pointed.
|
| + ret = [None]
|
| + def RunOnTimeoutThread():
|
| + ret[0] = func(*args, **kwargs)
|
| +
|
| + num_try = 1
|
| + while True:
|
| + child_thread = TimeoutRetryThread(
|
| + RunOnTimeoutThread, timeout,
|
| + name='TimeoutThread-%d-for-%s' % (num_try,
|
| + threading.current_thread().name))
|
| + try:
|
| + thread_group = reraiser_thread.ReraiserThreadGroup([child_thread])
|
| + thread_group.StartAll()
|
| + thread_group.JoinAll(child_thread.GetWatcher())
|
| + return ret[0]
|
| + except:
|
| + child_thread.LogTimeoutException()
|
| + if num_try > retries:
|
| + raise
|
| + num_try += 1
|
|
|