OLD | NEW |
1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """A utility to run functions with timeouts and retries.""" | 5 # pylint: disable=unused-wildcard-import |
6 # pylint: disable=W0702 | 6 # pylint: disable=wildcard-import |
7 | 7 |
8 import logging | 8 from devil.utils.timeout_retry import * |
9 import threading | |
10 import time | |
11 import traceback | |
12 | |
13 from pylib.utils import reraiser_thread | |
14 from pylib.utils import watchdog_timer | |
15 | |
16 | |
17 class TimeoutRetryThread(reraiser_thread.ReraiserThread): | |
18 def __init__(self, func, timeout, name): | |
19 super(TimeoutRetryThread, self).__init__(func, name=name) | |
20 self._watcher = watchdog_timer.WatchdogTimer(timeout) | |
21 self._expired = False | |
22 | |
23 def GetWatcher(self): | |
24 """Returns the watchdog keeping track of this thread's time.""" | |
25 return self._watcher | |
26 | |
27 def GetElapsedTime(self): | |
28 return self._watcher.GetElapsed() | |
29 | |
30 def GetRemainingTime(self, required=0, msg=None): | |
31 """Get the remaining time before the thread times out. | |
32 | |
33 Useful to send as the |timeout| parameter of async IO operations. | |
34 | |
35 Args: | |
36 required: minimum amount of time that will be required to complete, e.g., | |
37 some sleep or IO operation. | |
38 msg: error message to show if timing out. | |
39 | |
40 Returns: | |
41 The number of seconds remaining before the thread times out, or None | |
42 if the thread never times out. | |
43 | |
44 Raises: | |
45 reraiser_thread.TimeoutError if the remaining time is less than the | |
46 required time. | |
47 """ | |
48 remaining = self._watcher.GetRemaining() | |
49 if remaining is not None and remaining < required: | |
50 if msg is None: | |
51 msg = 'Timeout expired' | |
52 if remaining > 0: | |
53 msg += (', wait of %.1f secs required but only %.1f secs left' | |
54 % (required, remaining)) | |
55 self._expired = True | |
56 raise reraiser_thread.TimeoutError(msg) | |
57 return remaining | |
58 | |
59 def LogTimeoutException(self): | |
60 """Log the exception that terminated this thread.""" | |
61 if not self._expired: | |
62 return | |
63 logging.critical('*' * 80) | |
64 logging.critical('%s on thread %r', self._exc_info[0].__name__, self.name) | |
65 logging.critical('*' * 80) | |
66 fmt_exc = ''.join(traceback.format_exception(*self._exc_info)) | |
67 for line in fmt_exc.splitlines(): | |
68 logging.critical(line.rstrip()) | |
69 logging.critical('*' * 80) | |
70 | |
71 | |
72 def CurrentTimeoutThread(): | |
73 """Get the current thread if it is a TimeoutRetryThread. | |
74 | |
75 Returns: | |
76 The current thread if it is a TimeoutRetryThread, otherwise None. | |
77 """ | |
78 current_thread = threading.current_thread() | |
79 if isinstance(current_thread, TimeoutRetryThread): | |
80 return current_thread | |
81 else: | |
82 return None | |
83 | |
84 | |
85 def WaitFor(condition, wait_period=5, max_tries=None): | |
86 """Wait for a condition to become true. | |
87 | |
88 Repeadly call the function condition(), with no arguments, until it returns | |
89 a true value. | |
90 | |
91 If called within a TimeoutRetryThread, it cooperates nicely with it. | |
92 | |
93 Args: | |
94 condition: function with the condition to check | |
95 wait_period: number of seconds to wait before retrying to check the | |
96 condition | |
97 max_tries: maximum number of checks to make, the default tries forever | |
98 or until the TimeoutRetryThread expires. | |
99 | |
100 Returns: | |
101 The true value returned by the condition, or None if the condition was | |
102 not met after max_tries. | |
103 | |
104 Raises: | |
105 reraiser_thread.TimeoutError if the current thread is a TimeoutRetryThread | |
106 and the timeout expires. | |
107 """ | |
108 condition_name = condition.__name__ | |
109 timeout_thread = CurrentTimeoutThread() | |
110 while max_tries is None or max_tries > 0: | |
111 result = condition() | |
112 if max_tries is not None: | |
113 max_tries -= 1 | |
114 msg = ['condition', repr(condition_name), 'met' if result else 'not met'] | |
115 if timeout_thread: | |
116 msg.append('(%.1fs)' % timeout_thread.GetElapsedTime()) | |
117 logging.info(' '.join(msg)) | |
118 if result: | |
119 return result | |
120 if timeout_thread: | |
121 timeout_thread.GetRemainingTime(wait_period, | |
122 msg='Timed out waiting for %r' % condition_name) | |
123 time.sleep(wait_period) | |
124 return None | |
125 | |
126 | |
127 def Run(func, timeout, retries, args=None, kwargs=None): | |
128 """Runs the passed function in a separate thread with timeouts and retries. | |
129 | |
130 Args: | |
131 func: the function to be wrapped. | |
132 timeout: the timeout in seconds for each try. | |
133 retries: the number of retries. | |
134 args: list of positional args to pass to |func|. | |
135 kwargs: dictionary of keyword args to pass to |func|. | |
136 | |
137 Returns: | |
138 The return value of func(*args, **kwargs). | |
139 """ | |
140 if not args: | |
141 args = [] | |
142 if not kwargs: | |
143 kwargs = {} | |
144 | |
145 # The return value uses a list because Python variables are references, not | |
146 # values. Closures make a copy of the reference, so updating the closure's | |
147 # reference wouldn't update where the original reference pointed. | |
148 ret = [None] | |
149 def RunOnTimeoutThread(): | |
150 ret[0] = func(*args, **kwargs) | |
151 | |
152 num_try = 1 | |
153 while True: | |
154 child_thread = TimeoutRetryThread( | |
155 RunOnTimeoutThread, timeout, | |
156 name='TimeoutThread-%d-for-%s' % (num_try, | |
157 threading.current_thread().name)) | |
158 try: | |
159 thread_group = reraiser_thread.ReraiserThreadGroup([child_thread]) | |
160 thread_group.StartAll() | |
161 thread_group.JoinAll(child_thread.GetWatcher()) | |
162 return ret[0] | |
163 except: | |
164 child_thread.LogTimeoutException() | |
165 if num_try > retries: | |
166 raise | |
167 num_try += 1 | |
OLD | NEW |