Index: build/android/devil/utils/timeout_retry.py |
diff --git a/build/android/devil/utils/timeout_retry.py b/build/android/devil/utils/timeout_retry.py |
deleted file mode 100644 |
index e950e1160a82cd95f19eee2e981ff8eed8eadf67..0000000000000000000000000000000000000000 |
--- a/build/android/devil/utils/timeout_retry.py |
+++ /dev/null |
@@ -1,181 +0,0 @@ |
-# Copyright 2013 The Chromium Authors. All rights reserved. |
-# Use of this source code is governed by a BSD-style license that can be |
-# found in the LICENSE file. |
- |
-"""A utility to run functions with timeouts and retries.""" |
-# pylint: disable=W0702 |
- |
-import logging |
-import threading |
-import time |
-import traceback |
- |
-from devil.utils import reraiser_thread |
-from devil.utils import watchdog_timer |
- |
- |
- |
-class TimeoutRetryThreadGroup(reraiser_thread.ReraiserThreadGroup): |
- def __init__(self, timeout, threads=None): |
- super(TimeoutRetryThreadGroup, self).__init__(threads) |
- self._watcher = watchdog_timer.WatchdogTimer(timeout) |
- |
- def GetWatcher(self): |
- """Returns the watchdog keeping track of this thread's time.""" |
- return self._watcher |
- |
- def GetElapsedTime(self): |
- return self._watcher.GetElapsed() |
- |
- def GetRemainingTime(self, required=0, msg=None): |
- """Get the remaining time before the thread times out. |
- |
- Useful to send as the |timeout| parameter of async IO operations. |
- |
- Args: |
- required: minimum amount of time that will be required to complete, e.g., |
- some sleep or IO operation. |
- msg: error message to show if timing out. |
- |
- Returns: |
- The number of seconds remaining before the thread times out, or None |
- if the thread never times out. |
- |
- Raises: |
- reraiser_thread.TimeoutError if the remaining time is less than the |
- required time. |
- """ |
- remaining = self._watcher.GetRemaining() |
- if remaining is not None and remaining < required: |
- if msg is None: |
- msg = 'Timeout expired' |
- if remaining > 0: |
- msg += (', wait of %.1f secs required but only %.1f secs left' |
- % (required, remaining)) |
- raise reraiser_thread.TimeoutError(msg) |
- return remaining |
- |
- |
-def CurrentTimeoutThreadGroup(): |
- """Returns the thread group that owns or is blocked on the active thread. |
- |
- Returns: |
- Returns None if no TimeoutRetryThreadGroup is tracking the current thread. |
- """ |
- thread_group = reraiser_thread.CurrentThreadGroup() |
- while thread_group: |
- if isinstance(thread_group, TimeoutRetryThreadGroup): |
- return thread_group |
- thread_group = thread_group.blocked_parent_thread_group |
- return None |
- |
- |
-def WaitFor(condition, wait_period=5, max_tries=None): |
- """Wait for a condition to become true. |
- |
- Repeatedly call the function condition(), with no arguments, until it returns |
- a true value. |
- |
- If called within a TimeoutRetryThreadGroup, it cooperates nicely with it. |
- |
- Args: |
- condition: function with the condition to check |
- wait_period: number of seconds to wait before retrying to check the |
- condition |
- max_tries: maximum number of checks to make, the default tries forever |
- or until the TimeoutRetryThreadGroup expires. |
- |
- Returns: |
- The true value returned by the condition, or None if the condition was |
- not met after max_tries. |
- |
- Raises: |
- reraiser_thread.TimeoutError: if the current thread is a |
- TimeoutRetryThreadGroup and the timeout expires. |
- """ |
- condition_name = condition.__name__ |
- timeout_thread_group = CurrentTimeoutThreadGroup() |
- while max_tries is None or max_tries > 0: |
- result = condition() |
- if max_tries is not None: |
- max_tries -= 1 |
- msg = ['condition', repr(condition_name), 'met' if result else 'not met'] |
- if timeout_thread_group: |
- # pylint: disable=no-member |
- msg.append('(%.1fs)' % timeout_thread_group.GetElapsedTime()) |
- logging.info(' '.join(msg)) |
- if result: |
- return result |
- if timeout_thread_group: |
- # pylint: disable=no-member |
- timeout_thread_group.GetRemainingTime(wait_period, |
- msg='Timed out waiting for %r' % condition_name) |
- time.sleep(wait_period) |
- return None |
- |
- |
-def _LogLastException(thread_name, attempt, max_attempts, log_func): |
- log_func('*' * 80) |
- log_func('Exception on thread %s (attempt %d of %d)', thread_name, |
- attempt, max_attempts) |
- log_func('*' * 80) |
- fmt_exc = ''.join(traceback.format_exc()) |
- for line in fmt_exc.splitlines(): |
- log_func(line.rstrip()) |
- log_func('*' * 80) |
- |
- |
-def AlwaysRetry(_exception): |
- return True |
- |
- |
-def Run(func, timeout, retries, args=None, kwargs=None, desc=None, |
- error_log_func=logging.critical, retry_if_func=AlwaysRetry): |
- """Runs the passed function in a separate thread with timeouts and retries. |
- |
- Args: |
- func: the function to be wrapped. |
- timeout: the timeout in seconds for each try. |
- retries: the number of retries. |
- args: list of positional args to pass to |func|. |
- kwargs: dictionary of keyword args to pass to |func|. |
- desc: An optional description of |func| used in logging. If omitted, |
- |func.__name__| will be used. |
- error_log_func: Logging function when logging errors. |
- retry_if_func: Unary callable that takes an exception and returns |
- whether |func| should be retried. Defaults to always retrying. |
- |
- Returns: |
- The return value of func(*args, **kwargs). |
- """ |
- if not args: |
- args = [] |
- if not kwargs: |
- kwargs = {} |
- |
- num_try = 1 |
- while True: |
- thread_name = 'TimeoutThread-%d-for-%s' % (num_try, |
- threading.current_thread().name) |
- child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), |
- name=thread_name) |
- try: |
- thread_group = TimeoutRetryThreadGroup(timeout, threads=[child_thread]) |
- thread_group.StartAll(will_block=True) |
- while True: |
- thread_group.JoinAll(watcher=thread_group.GetWatcher(), timeout=60, |
- error_log_func=error_log_func) |
- if thread_group.IsAlive(): |
- logging.info('Still working on %s', desc if desc else func.__name__) |
- else: |
- return thread_group.GetAllReturnValues()[0] |
- except reraiser_thread.TimeoutError as e: |
- # Timeouts already get their stacks logged. |
- if num_try > retries or not retry_if_func(e): |
- raise |
- # Do not catch KeyboardInterrupt. |
- except Exception as e: # pylint: disable=broad-except |
- if num_try > retries or not retry_if_func(e): |
- raise |
- _LogLastException(thread_name, num_try, retries + 1, error_log_func) |
- num_try += 1 |