OLD | NEW |
---|---|
(Empty) | |
1 # Copyright 2016 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 import io | |
jbudorick
2016/06/07 15:29:27
?
rnephew (Reviews Here)
2016/06/09 22:06:39
io is used during the archiving of the output dir.
| |
6 import json | |
7 import logging | |
8 import os | |
9 import pickle | |
jbudorick
2016/06/07 15:29:27
uh oh
rnephew (Reviews Here)
2016/06/09 22:06:38
Talked offline.
| |
10 import shutil | |
11 import sys | |
12 import tempfile | |
13 import time | |
14 import zipfile | |
15 | |
16 from devil.android import battery_utils | |
17 from devil.android import device_errors | |
18 from devil.android import device_list | |
19 from devil.android import device_utils | |
20 from devil.android import forwarder | |
21 from devil.utils import cmd_helper | |
22 from devil.utils import reraiser_thread | |
23 from devil.utils import watchdog_timer | |
24 from pylib import constants | |
25 from pylib.base import base_test_result | |
26 from pylib.constants import host_paths | |
27 from pylib.local.device import local_device_test_run | |
28 | |
29 | |
30 _BOOTUP_TIMEOUT = 60 * 2 | |
jbudorick
2016/06/07 15:29:28
?
rnephew (Reviews Here)
2016/06/09 22:06:37
self._device.WaitUntilFullyBooted(timeout=_BOOTUP_
| |
31 | |
32 | |
33 class TestShard(object): | |
34 def __init__(self, test_instance, device, index, tests, results, watcher=None, | |
35 retries=3, timeout=None): | |
36 logging.info('Create shard %s for device %s to run the following tests:', | |
37 index, device) | |
38 for t in tests: | |
39 logging.info(' %s', t) | |
40 self._battery = battery_utils.BatteryUtils(device) | |
41 self._device = device | |
42 self._index = index | |
43 self._output_dir = None | |
44 self._results = results | |
45 self._retries = retries | |
46 self._test_instance = test_instance | |
47 self._tests = tests | |
48 self._timeout = timeout | |
49 self._watcher = watcher | |
50 | |
51 def _TestSetUp(self): | |
52 self._ResetWatcher() | |
53 self._BatteryLevelCheck() | |
54 self._BatteryTempCheck() | |
55 self._ScreenCheck() | |
56 if not self._device.IsOnline(): | |
57 msg = 'Device %s is unresponsive.' % str(self._device) | |
58 raise device_errors.DeviceUnreachableError(msg) | |
59 | |
60 def _TestTearDown(self): | |
61 try: | |
62 logging.info('Unmapping device ports.') | |
63 forwarder.Forwarder.UnmapAllDevicePorts(self._device) | |
64 except Exception: # pylint: disable=broad-except | |
65 logging.exception('Exception when resetting ports.') | |
66 | |
67 def _CleanupOutputDirectory(self): | |
jbudorick
2016/06/07 15:29:27
This either shouldn't be its own function or it sh
rnephew (Reviews Here)
2016/06/09 22:06:38
Done.
| |
68 if self._output_dir: | |
69 shutil.rmtree(self._output_dir, ignore_errors=True) | |
70 self._output_dir = None | |
71 | |
72 def _CreateCmd(self, test): | |
73 cmd = '%s --device %s' % (self._tests[test]['cmd'], str(self._device)) | |
74 if (self._test_instance.collect_chartjson_data | |
75 or self._tests[test].get('archive_output_dir')): | |
76 self._output_dir = tempfile.mkdtemp() | |
jbudorick
2016/06/07 15:29:27
This seems like the wrong place for this.
Also, w
rnephew (Reviews Here)
2016/06/09 22:06:38
Done.
| |
77 cmd = cmd + ' --output-dir=%s' % self._output_dir | |
78 if self._test_instance.dry_run: | |
79 cmd = 'echo %s' % cmd | |
80 return cmd | |
81 | |
82 def _RunSingleTest(self, test): | |
83 | |
84 logging.info('Running %s on shard %s', test, self._index) | |
85 timeout = ( | |
86 None if self._test_instance.no_timeout | |
jbudorick
2016/06/07 15:29:26
When would we want no timeout?
rnephew (Reviews Here)
2016/06/09 22:06:38
probably never, but it was a command line option i
| |
87 else self._tests[test].get('timeout', self._timeout)) | |
88 logging.info('Timeout for %s test: %s', test, timeout) | |
89 | |
90 logfile = sys.stdout | |
jbudorick
2016/06/07 15:29:28
Why is this set here and not in the GetCmdStatusAn
rnephew (Reviews Here)
2016/06/09 22:06:37
Artifact of previous changes, its done at the corr
| |
91 cmd = self._CreateCmd(test) | |
92 self._test_instance.WriteBuildBotJson(self._output_dir) | |
93 cwd = os.path.abspath(host_paths.DIR_SOURCE_ROOT) | |
94 | |
95 try: | |
96 logging.debug('Running test with command \'%s\'', cmd) | |
97 exit_code, output = cmd_helper.GetCmdStatusAndOutputWithTimeout( | |
jbudorick
2016/06/07 15:29:28
I'm somewhat worried about muxed output here with
rnephew (Reviews Here)
2016/06/09 22:06:38
Yeah, switched to not passing logfile.
| |
98 cmd, timeout, cwd=cwd, shell=True, logfile=logfile) | |
99 json_output = self._test_instance.ReadChartjsonOutput(self._output_dir) | |
100 except cmd_helper.TimeoutError as e: | |
101 exit_code = -1 | |
102 output = e.output | |
103 json_output = '' | |
104 return cmd, exit_code, output, json_output | |
105 | |
106 def _ProcessTestResult( | |
jbudorick
2016/06/07 15:29:27
Why doesn't _RunSingleTest handle both running the
rnephew (Reviews Here)
2016/06/09 22:06:38
No reason it cant, just didnt do it that way; but
| |
107 self, test, cmd, start_time, end_time, exit_code, output, json_output): | |
108 if exit_code is None: | |
109 exit_code = -1 | |
110 logging.info('%s : exit_code=%d in %d secs on device %s', | |
111 test, exit_code, end_time - start_time, | |
112 str(self._device)) | |
113 if exit_code == 0: | |
114 result_type = base_test_result.ResultType.PASS | |
115 else: | |
116 result_type = base_test_result.ResultType.FAIL | |
117 # TODO(rnephew): Improve device recovery logic. | |
118 try: | |
119 self._device.WaitUntilFullyBooted(timeout=_BOOTUP_TIMEOUT) | |
120 self._device.RestartAdbd() | |
121 except device_errors.CommandTimeoutError: | |
122 logging.exception('Device failed to return after %s.', test) | |
123 actual_exit_code = exit_code | |
124 if (self._test_instance.flaky_steps | |
125 and test in self._test_instance.flaky_steps): | |
126 exit_code = 0 | |
127 archive_bytes = (self._ArchiveOutputDir() | |
128 if self._tests[test].get('archive_output_dir') | |
129 else None) | |
130 persisted_result = { | |
131 'name': test, | |
132 'output': [output], | |
133 'chartjson': json_output, | |
134 'archive_bytes': archive_bytes, | |
135 'exit_code': exit_code, | |
136 'actual_exit_code': actual_exit_code, | |
137 'result_type': result_type, | |
138 'start_time': start_time, | |
139 'end_time': end_time, | |
140 'total_time': end_time - start_time, | |
141 'device': str(self._device), | |
142 'cmd': cmd, | |
143 } | |
144 self._SaveResult(persisted_result) | |
145 return result_type | |
146 | |
147 def RunTestsOnShard(self): | |
148 for test in self._tests: | |
149 try: | |
150 exit_code = None | |
151 tries_left = self._retries | |
152 | |
153 while exit_code != 0 and tries_left > 0: | |
jbudorick
2016/06/07 15:29:28
Internal retry handling, interesting.
rnephew (Reviews Here)
2016/06/09 22:06:37
Acknowledged.
| |
154 self._TestSetUp() | |
155 self._ResetWatcher() | |
jbudorick
2016/06/07 15:29:26
This already happens in _TestSetUp?
rnephew (Reviews Here)
2016/06/09 22:06:37
I did it afterwards so that there is actually two
| |
156 tries_left = tries_left - 1 | |
157 start_time = time.time() | |
158 cmd, exit_code, output, json_output = self._RunSingleTest(test) | |
jbudorick
2016/06/07 15:29:28
It looks like most of the logic below this should
rnephew (Reviews Here)
2016/06/09 22:06:39
Done.
| |
159 end_time = time.time() | |
160 result_type = self._ProcessTestResult( | |
161 test, cmd, start_time, end_time, exit_code, output, json_output) | |
162 self._TestTearDown() | |
163 | |
164 result = base_test_result.TestRunResults() | |
165 result.AddResult(base_test_result.BaseTestResult(test, result_type)) | |
166 self._results.append(result) | |
167 finally: | |
jbudorick
2016/06/07 15:29:27
Do we really want a shard to die on exception, esp
rnephew (Reviews Here)
2016/06/09 22:06:38
Fixed.
| |
168 self._CleanupOutputDirectory() | |
169 | |
170 @staticmethod | |
171 def _SaveResult(result): | |
172 pickled = os.path.join(constants.PERF_OUTPUT_DIR, result['name']) | |
173 if os.path.exists(pickled): | |
174 with file(pickled, 'r') as f: | |
175 previous = pickle.loads(f.read()) | |
176 result['output'] = previous['output'] + result['output'] | |
177 with file(pickled, 'w') as f: | |
178 f.write(pickle.dumps(result)) | |
179 | |
180 def _ArchiveOutputDir(self): | |
181 """Archive all files in the output dir, and return as compressed bytes.""" | |
182 with io.BytesIO() as archive: | |
183 with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED) as contents: | |
184 num_files = 0 | |
185 for absdir, _, files in os.walk(self._output_dir): | |
186 reldir = os.path.relpath(absdir, self._output_dir) | |
187 for filename in files: | |
188 src_path = os.path.join(absdir, filename) | |
189 # We use normpath to turn './file.txt' into just 'file.txt'. | |
190 dst_path = os.path.normpath(os.path.join(reldir, filename)) | |
191 contents.write(src_path, dst_path) | |
192 num_files += 1 | |
193 if num_files: | |
194 logging.info('%d files in the output dir were archived.', num_files) | |
195 else: | |
196 logging.warning('No files in the output dir. Archive is empty.') | |
197 return archive.getvalue() | |
198 | |
199 def _ResetWatcher(self): | |
jbudorick
2016/06/07 15:29:28
This doesn't seem like it needs to be a separate f
rnephew (Reviews Here)
2016/06/09 22:06:37
Done.
| |
200 if self._watcher: | |
201 self._watcher.Reset() | |
202 | |
203 def _BatteryLevelCheck(self): | |
204 logging.info('Charge level: %s%%', | |
205 str(self._battery.GetBatteryInfo().get('level'))) | |
206 if self._test_instance.min_battery_level: | |
207 self._battery.ChargeDeviceToLevel(self._test_instance.min_battery_level) | |
208 | |
209 def _ScreenCheck(self): | |
jbudorick
2016/06/07 15:29:28
Again, doesn't need to be a separate function.
rnephew (Reviews Here)
2016/06/09 22:06:37
Done.
| |
210 if not self._device.IsScreenOn(): | |
211 self._device.SetScreen(True) | |
212 | |
213 def _BatteryTempCheck(self): | |
jbudorick
2016/06/07 15:29:28
Leaning toward the two battery checks not needing
rnephew (Reviews Here)
2016/06/09 22:06:37
I felt liking doing it made the setup easier to un
| |
214 logging.info('temperature: %s (0.1 C)', | |
215 str(self._battery.GetBatteryInfo().get('temperature'))) | |
216 if self._test_instance.max_battery_temp: | |
217 self._battery.LetBatteryCoolToTemperature( | |
218 self._test_instance.max_battery_temp) | |
219 | |
220 | |
221 class LocalDevicePerfTestRun(local_device_test_run.LocalDeviceTestRun): | |
222 def __init__(self, env, test_instance): | |
223 super(LocalDevicePerfTestRun, self).__init__(env, test_instance) | |
224 self._test_instance = test_instance | |
225 self._env = env | |
226 self._timeout = None if test_instance.no_timeout else 60 * 60 | |
227 self._devices = None | |
228 self._test_buckets = [] | |
229 self._watcher = None | |
230 | |
231 def SetUp(self): | |
232 self._devices = self._GetAllDevices(self._env.devices, | |
233 self._test_instance.known_devices_file) | |
234 self._watcher = watchdog_timer.WatchdogTimer(self._timeout) | |
235 | |
236 if (not (self._test_instance.print_step | |
237 or self._test_instance.output_json_list)): | |
238 if os.path.exists(constants.PERF_OUTPUT_DIR): | |
239 shutil.rmtree(constants.PERF_OUTPUT_DIR) | |
240 os.makedirs(constants.PERF_OUTPUT_DIR) | |
241 | |
242 def TearDown(self): | |
243 pass | |
244 | |
245 def _GetStepsFromDict(self): | |
246 if self._test_instance.single_step: | |
247 return { | |
248 'version': 1, | |
249 'steps': { | |
250 'single_step': { | |
251 'device_affinity': 0, | |
252 'cmd': self._test_instance.single_step | |
253 }, | |
254 } | |
255 } | |
256 if self._test_instance.steps: | |
jbudorick
2016/06/07 15:29:28
What happens if neither of these is set?
rnephew (Reviews Here)
2016/06/09 22:06:38
It cannot get here without at least one of those s
| |
257 with file(self._test_instance.steps, 'r') as f: | |
258 steps = json.load(f) | |
259 assert steps['version'] == 1 | |
jbudorick
2016/06/07 15:29:26
This should be an exception, but not an assertion.
rnephew (Reviews Here)
2016/06/09 22:06:38
Done.
| |
260 return steps | |
261 | |
262 def _SplitTestsByAffinity(self): | |
263 test_dict = self._GetStepsFromDict() | |
264 for test in test_dict['steps']: | |
jbudorick
2016/06/07 15:29:28
for test, test_config in test_dict['steps'].iterit
rnephew (Reviews Here)
2016/06/09 22:06:38
Done.
| |
265 affinity = test_dict['steps'][test]['device_affinity'] | |
266 if len(self._test_buckets) < affinity + 1: | |
267 while len(self._test_buckets) != affinity + 1: | |
268 self._test_buckets.append({}) | |
269 self._test_buckets[affinity][test] = test_dict['steps'][test] | |
270 return self._test_buckets | |
271 | |
272 @staticmethod | |
273 def _GetAllDevices(active_devices, devices_path): | |
274 try: | |
275 if devices_path: | |
276 devices = [device_utils.DeviceUtils(s) | |
277 for s in device_list.GetPersistentDeviceList(devices_path)] | |
278 else: | |
279 logging.warning('Known devices file path not being passed. For device ' | |
280 'affinity to work properly, it must be passed.') | |
281 devices = active_devices | |
282 except IOError as e: | |
283 logging.error('Unable to find %s [%s]', devices_path, e) | |
284 devices = active_devices | |
285 return sorted(devices) | |
286 | |
jbudorick
2016/06/07 15:29:28
nit: one too many empty lines
rnephew (Reviews Here)
2016/06/09 22:06:38
Done.
| |
287 | |
288 def _RunOutputJsonList(self): | |
jbudorick
2016/06/07 15:29:28
This seems like it could be in the test instance..
rnephew (Reviews Here)
2016/06/09 22:06:38
Done.
| |
289 if self._test_instance.OutputJsonList() == 0: | |
290 result_type = base_test_result.ResultType.PASS | |
291 else: | |
292 result_type = base_test_result.ResultType.FAIL | |
293 result = base_test_result.TestRunResults() | |
294 result.AddResult( | |
295 base_test_result.BaseTestResult('OutputJsonList', result_type)) | |
296 return [result] | |
297 | |
298 def _RunPrintStep(self): | |
jbudorick
2016/06/07 15:29:28
... same.
rnephew (Reviews Here)
2016/06/09 22:06:37
I had already moved over a few things, just hadn't
| |
299 if self._test_instance.PrintTestOutput() == 0: | |
300 result_type = base_test_result.ResultType.PASS | |
301 else: | |
302 result_type = base_test_result.ResultType.FAIL | |
303 result = base_test_result.TestRunResults() | |
304 result.AddResult( | |
305 base_test_result.BaseTestResult('PrintStep', result_type)) | |
306 return [result] | |
307 | |
308 def RunTests(self): | |
309 # Option selected for saving a json file with a list of test names. | |
310 if self._test_instance.output_json_list: | |
311 return self._RunOutputJsonList() | |
312 | |
313 # Just print the results from a single previously executed step. | |
314 if self._test_instance.print_step: | |
315 return self._RunPrintStep() | |
316 | |
317 # Affinitize the tests. | |
318 test_buckets = self._SplitTestsByAffinity() | |
319 if not test_buckets: | |
320 local_device_test_run.NoTestsError() | |
jbudorick
2016/06/07 15:29:26
this needs to raise the exception it's creating
rnephew (Reviews Here)
2016/06/09 22:06:38
Done.
| |
321 threads = [] | |
322 results = [] | |
323 for x in xrange(min(len(self._devices), len(test_buckets))): | |
324 new_shard = TestShard(self._test_instance, self._devices[x], x, | |
325 test_buckets[x], results, watcher=self._watcher, | |
326 retries=self._env.max_tries, timeout=self._timeout) | |
327 threads.append(reraiser_thread.ReraiserThread(new_shard.RunTestsOnShard)) | |
jbudorick
2016/06/07 15:29:27
Thinking about the parallelizer here.
rnephew (Reviews Here)
2016/06/09 22:06:38
Acknowledged.
| |
328 | |
329 workers = reraiser_thread.ReraiserThreadGroup(threads) | |
330 workers.StartAll() | |
331 | |
332 try: | |
jbudorick
2016/06/07 15:29:26
These should be handled on each thread, not on the
rnephew (Reviews Here)
2016/06/09 22:06:38
Done.
| |
333 workers.JoinAll(self._watcher) | |
334 except device_errors.CommandFailedError: | |
335 logging.exception('Command failed on device.') | |
336 except device_errors.CommandTimeoutError: | |
337 logging.exception('Command timed out on device.') | |
338 except device_errors.DeviceUnreachableError: | |
339 logging.exception('Device became unreachable.') | |
340 return results | |
341 | |
342 # override | |
343 def TestPackage(self): | |
344 return 'perf' | |
345 | |
346 # override | |
347 def _CreateShards(self, _tests): | |
348 raise NotImplementedError | |
349 | |
350 # override | |
351 def _GetTests(self): | |
352 return self._test_buckets | |
353 | |
354 # override | |
355 def _RunTest(self, _device, _test): | |
356 raise NotImplementedError | |
357 | |
358 # override | |
359 def _ShouldShard(self): | |
360 return False | |
OLD | NEW |