Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(177)

Side by Side Diff: build/android/pylib/base/shard.py

Issue 18323020: Updates the test runner script exit codes (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Changes exit_code tracking to a singleton paradigm Created 7 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Implements test sharding logic.""" 5 """Implements test sharding logic."""
6 6
7 import logging 7 import logging
8 import threading 8 import threading
9 9
10 from pylib import android_commands 10 from pylib import android_commands
11 from pylib import constants
12 from pylib import exit_code
11 from pylib import forwarder 13 from pylib import forwarder
12 from pylib.utils import reraiser_thread 14 from pylib.utils import reraiser_thread
13 from pylib.utils import watchdog_timer 15 from pylib.utils import watchdog_timer
14 16
15 import base_test_result 17 import base_test_result
16 18
17 19
18 DEFAULT_TIMEOUT = 7 * 60 # seven minutes 20 DEFAULT_TIMEOUT = 7 * 60 # seven minutes
19 21
20 22
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after
85 try: 87 try:
86 return self._tests.pop(0) 88 return self._tests.pop(0)
87 except IndexError: 89 except IndexError:
88 # Another thread beat us to the avaliable test, wait again. 90 # Another thread beat us to the avaliable test, wait again.
89 self._item_avaliable_or_all_done.clear() 91 self._item_avaliable_or_all_done.clear()
90 92
91 def add(self, test): 93 def add(self, test):
92 """Add an test to the collection. 94 """Add an test to the collection.
93 95
94 Args: 96 Args:
95 item: A test to add. 97 test: A test to add.
96 """ 98 """
97 with self._lock: 99 with self._lock:
98 self._tests.append(test) 100 self._tests.append(test)
99 self._item_avaliable_or_all_done.set() 101 self._item_avaliable_or_all_done.set()
100 self._tests_in_progress += 1 102 self._tests_in_progress += 1
101 103
102 def test_completed(self): 104 def test_completed(self):
103 """Indicate that a test has been fully handled.""" 105 """Indicate that a test has been fully handled."""
104 with self._lock: 106 with self._lock:
105 self._tests_in_progress -= 1 107 self._tests_in_progress -= 1
106 if self._tests_in_progress == 0: 108 if self._tests_in_progress == 0:
107 # All tests have been handled, signal all waiting threads. 109 # All tests have been handled, signal all waiting threads.
108 self._item_avaliable_or_all_done.set() 110 self._item_avaliable_or_all_done.set()
109 111
110 def __iter__(self): 112 def __iter__(self):
111 """Iterate through tests in the collection until all have been handled.""" 113 """Iterate through tests in the collection until all have been handled."""
112 while True: 114 while True:
113 r = self._pop() 115 r = self._pop()
114 if r is None: 116 if r is None:
115 break 117 break
116 yield r 118 yield r
117 119
118 120
119 def _RunTestsFromQueue(runner, test_collection, out_results, watcher, 121 def _RunTestsFromQueue(runner, test_collection, out_results, watcher,
120 num_retries): 122 num_retries):
121 """Runs tests from the test_collection until empty using the given runner. 123 """Runs tests from the test_collection until empty using the given runner.
122 124
123 Adds TestRunResults objects to the out_results list and may add tests to the 125 Adds TestRunResults objects to the out_results list and may add tests to the
124 out_retry list. 126 out_retry list.
125 127
126 Args: 128 Args:
127 runner: A TestRunner object used to run the tests. 129 runner: A TestRunner object used to run the tests.
128 test_collection: A _TestCollection from which to get _Test objects to run. 130 test_collection: A _TestCollection from which to get _Test objects to run.
129 out_results: A list to add TestRunResults to. 131 out_results: A list to add TestRunResults to.
130 watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout. 132 watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout.
(...skipping 12 matching lines...) Expand all
143 if retry and test.tries <= num_retries: 145 if retry and test.tries <= num_retries:
144 # Retry non-passing results, only record passing results. 146 # Retry non-passing results, only record passing results.
145 pass_results = base_test_result.TestRunResults() 147 pass_results = base_test_result.TestRunResults()
146 pass_results.AddResults(result.GetPass()) 148 pass_results.AddResults(result.GetPass())
147 out_results.append(pass_results) 149 out_results.append(pass_results)
148 logging.warning('Will retry test, try #%s.' % test.tries) 150 logging.warning('Will retry test, try #%s.' % test.tries)
149 test_collection.add(_Test(test=retry, tries=test.tries)) 151 test_collection.add(_Test(test=retry, tries=test.tries))
150 else: 152 else:
151 # All tests passed or retry limit reached. Either way, record results. 153 # All tests passed or retry limit reached. Either way, record results.
152 out_results.append(result) 154 out_results.append(result)
153 except android_commands.errors.DeviceUnresponsiveError:
154 # Device is unresponsive, stop handling tests on this device and ensure
155 # current test gets runs by another device. Don't reraise this exception
156 # on the main thread.
157 test_collection.add(test)
158 return
159 except: 155 except:
160 # An unhandleable exception, ensure tests get run by another device and 156 # An unhandleable exception, ensure tests get run by another device and
161 # reraise this exception on the main thread. 157 # reraise this exception on the main thread.
162 test_collection.add(test) 158 test_collection.add(test)
163 raise 159 raise
164 finally: 160 finally:
165 # Retries count as separate tasks so always mark the popped test as done. 161 # Retries count as separate tasks so always mark the popped test as done.
166 test_collection.test_completed() 162 test_collection.test_completed()
167 163
168 164
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after
205 (len(tests), len(runners))) 201 (len(tests), len(runners)))
206 tests_collection = _TestCollection([_Test(t) for t in tests]) 202 tests_collection = _TestCollection([_Test(t) for t in tests])
207 results = [] 203 results = []
208 watcher = watchdog_timer.WatchdogTimer(timeout) 204 watcher = watchdog_timer.WatchdogTimer(timeout)
209 workers = reraiser_thread.ReraiserThreadGroup( 205 workers = reraiser_thread.ReraiserThreadGroup(
210 [reraiser_thread.ReraiserThread( 206 [reraiser_thread.ReraiserThread(
211 _RunTestsFromQueue, 207 _RunTestsFromQueue,
212 [r, tests_collection, results, watcher, num_retries], 208 [r, tests_collection, results, watcher, num_retries],
213 name=r.device[-4:]) 209 name=r.device[-4:])
214 for r in runners]) 210 for r in runners])
211 run_results = base_test_result.TestRunResults()
215 workers.StartAll() 212 workers.StartAll()
216 workers.JoinAll(watcher) 213
217 run_results = base_test_result.TestRunResults() 214 # Catch DeviceUnresponsiveErrors and set a warning exit code
215 try:
216 workers.JoinAll(watcher)
217 except android_commands.errors.DeviceUnresponsiveError as e:
218 logging.error(e)
219 exit_code.UpdateExitCode(constants.WARNING_EXIT_CODE)
220
218 for r in results: 221 for r in results:
219 run_results.AddTestRunResults(r) 222 run_results.AddTestRunResults(r)
220 return run_results 223 return run_results
221 224
222 225
223 def _CreateRunners(runner_factory, devices, timeout=None): 226 def _CreateRunners(runner_factory, devices, timeout=None):
224 """Creates a test runner for each device and calls SetUp() in parallel. 227 """Creates a test runner for each device and calls SetUp() in parallel.
225 228
226 Note: if a device is unresponsive the corresponding TestRunner will not be 229 Note: if a device is unresponsive the corresponding TestRunner will not be
227 included in the returned list. 230 included in the returned list.
(...skipping 15 matching lines...) Expand all
243 [runner_factory, d, runners, counter], 246 [runner_factory, d, runners, counter],
244 name=d[-4:]) 247 name=d[-4:])
245 for d in devices]) 248 for d in devices])
246 threads.StartAll() 249 threads.StartAll()
247 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) 250 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout))
248 return runners 251 return runners
249 252
250 253
251 def _TearDownRunners(runners, timeout=None): 254 def _TearDownRunners(runners, timeout=None):
252 """Calls TearDown() for each test runner in parallel. 255 """Calls TearDown() for each test runner in parallel.
256
253 Args: 257 Args:
254 runners: a list of TestRunner objects. 258 runners: a list of TestRunner objects.
255 timeout: watchdog timeout in seconds, defaults to the default timeout. 259 timeout: watchdog timeout in seconds, defaults to the default timeout.
256 """ 260 """
257 threads = reraiser_thread.ReraiserThreadGroup( 261 threads = reraiser_thread.ReraiserThreadGroup(
258 [reraiser_thread.ReraiserThread(r.TearDown, name=r.device[-4:]) 262 [reraiser_thread.ReraiserThread(r.TearDown, name=r.device[-4:])
259 for r in runners]) 263 for r in runners])
260 threads.StartAll() 264 threads.StartAll()
261 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) 265 threads.JoinAll(watchdog_timer.WatchdogTimer(timeout))
262 266
(...skipping 28 matching lines...) Expand all
291 runners = _CreateRunners(runner_factory, devices, setup_timeout) 295 runners = _CreateRunners(runner_factory, devices, setup_timeout)
292 try: 296 try:
293 return _RunAllTests(runners, tests, num_retries, test_timeout) 297 return _RunAllTests(runners, tests, num_retries, test_timeout)
294 finally: 298 finally:
295 try: 299 try:
296 _TearDownRunners(runners, setup_timeout) 300 _TearDownRunners(runners, setup_timeout)
297 except android_commands.errors.DeviceUnresponsiveError as e: 301 except android_commands.errors.DeviceUnresponsiveError as e:
298 logging.warning('Device unresponsive during TearDown: [%s]', e) 302 logging.warning('Device unresponsive during TearDown: [%s]', e)
299 finally: 303 finally:
300 forwarder.Forwarder.KillHost(build_type) 304 forwarder.Forwarder.KillHost(build_type)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698