OLD | NEW |
| (Empty) |
1 # Copyright 2013 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 """Runs perf tests. | |
6 | |
7 Our buildbot infrastructure requires each slave to run steps serially. | |
8 This is sub-optimal for android, where these steps can run independently on | |
9 multiple connected devices. | |
10 | |
11 The buildbots will run this script multiple times per cycle: | |
12 - First: all steps listed in --steps in will be executed in parallel using all | |
13 connected devices. Step results will be pickled to disk. Each step has a unique | |
14 name. The result code will be ignored if the step name is listed in | |
15 --flaky-steps. | |
16 The buildbot will treat this step as a regular step, and will not process any | |
17 graph data. | |
18 | |
19 - Then, with -print-step STEP_NAME: at this stage, we'll simply print the file | |
20 with the step results previously saved. The buildbot will then process the graph | |
21 data accordingly. | |
22 | |
23 The JSON steps file contains a dictionary in the format: | |
24 { "version": int, | |
25 "steps": { | |
26 "foo": { | |
27 "device_affinity": int, | |
28 "cmd": "script_to_execute foo" | |
29 }, | |
30 "bar": { | |
31 "device_affinity": int, | |
32 "cmd": "script_to_execute bar" | |
33 } | |
34 } | |
35 } | |
36 | |
37 The JSON flaky steps file contains a list with step names which results should | |
38 be ignored: | |
39 [ | |
40 "step_name_foo", | |
41 "step_name_bar" | |
42 ] | |
43 | |
44 Note that script_to_execute necessarily have to take at least the following | |
45 option: | |
46 --device: the serial number to be passed to all adb commands. | |
47 """ | |
48 | |
49 import collections | |
50 import datetime | |
51 import json | |
52 import logging | |
53 import os | |
54 import pickle | |
55 import shutil | |
56 import sys | |
57 import tempfile | |
58 import threading | |
59 import time | |
60 | |
61 from pylib import cmd_helper | |
62 from pylib import constants | |
63 from pylib import forwarder | |
64 from pylib.base import base_test_result | |
65 from pylib.base import base_test_runner | |
66 from pylib.device import battery_utils | |
67 from pylib.device import device_errors | |
68 | |
69 | |
70 def GetPersistedResult(test_name): | |
71 file_name = os.path.join(constants.PERF_OUTPUT_DIR, test_name) | |
72 if not os.path.exists(file_name): | |
73 logging.error('File not found %s', file_name) | |
74 return None | |
75 | |
76 with file(file_name, 'r') as f: | |
77 return pickle.loads(f.read()) | |
78 | |
79 | |
80 def OutputJsonList(json_input, json_output): | |
81 with file(json_input, 'r') as i: | |
82 all_steps = json.load(i) | |
83 | |
84 step_values = [] | |
85 for k, v in all_steps['steps'].iteritems(): | |
86 data = {'test': k, 'device_affinity': v['device_affinity']} | |
87 | |
88 persisted_result = GetPersistedResult(k) | |
89 if persisted_result: | |
90 data['total_time'] = persisted_result['total_time'] | |
91 step_values.append(data) | |
92 | |
93 with file(json_output, 'w') as o: | |
94 o.write(json.dumps(step_values)) | |
95 return 0 | |
96 | |
97 | |
98 def PrintTestOutput(test_name, json_file_name=None): | |
99 """Helper method to print the output of previously executed test_name. | |
100 | |
101 Args: | |
102 test_name: name of the test that has been previously executed. | |
103 json_file_name: name of the file to output chartjson data to. | |
104 | |
105 Returns: | |
106 exit code generated by the test step. | |
107 """ | |
108 persisted_result = GetPersistedResult(test_name) | |
109 if not persisted_result: | |
110 return 1 | |
111 logging.info('*' * 80) | |
112 logging.info('Output from:') | |
113 logging.info(persisted_result['cmd']) | |
114 logging.info('*' * 80) | |
115 print persisted_result['output'] | |
116 | |
117 if json_file_name: | |
118 with file(json_file_name, 'w') as f: | |
119 f.write(persisted_result['chartjson']) | |
120 | |
121 return persisted_result['exit_code'] | |
122 | |
123 | |
124 def PrintSummary(test_names): | |
125 logging.info('*' * 80) | |
126 logging.info('Sharding summary') | |
127 device_total_time = collections.defaultdict(int) | |
128 for test_name in test_names: | |
129 file_name = os.path.join(constants.PERF_OUTPUT_DIR, test_name) | |
130 if not os.path.exists(file_name): | |
131 logging.info('%s : No status file found', test_name) | |
132 continue | |
133 with file(file_name, 'r') as f: | |
134 result = pickle.loads(f.read()) | |
135 logging.info('%s : exit_code=%d in %d secs at %s', | |
136 result['name'], result['exit_code'], result['total_time'], | |
137 result['device']) | |
138 device_total_time[result['device']] += result['total_time'] | |
139 for device, device_time in device_total_time.iteritems(): | |
140 logging.info('Total for device %s : %d secs', device, device_time) | |
141 logging.info('Total steps time: %d secs', sum(device_total_time.values())) | |
142 | |
143 | |
144 class _HeartBeatLogger(object): | |
145 # How often to print the heartbeat on flush(). | |
146 _PRINT_INTERVAL = 30.0 | |
147 | |
148 def __init__(self): | |
149 """A file-like class for keeping the buildbot alive.""" | |
150 self._len = 0 | |
151 self._tick = time.time() | |
152 self._stopped = threading.Event() | |
153 self._timer = threading.Thread(target=self._runner) | |
154 self._timer.start() | |
155 | |
156 def _runner(self): | |
157 while not self._stopped.is_set(): | |
158 self.flush() | |
159 self._stopped.wait(_HeartBeatLogger._PRINT_INTERVAL) | |
160 | |
161 def write(self, data): | |
162 self._len += len(data) | |
163 | |
164 def flush(self): | |
165 now = time.time() | |
166 if now - self._tick >= _HeartBeatLogger._PRINT_INTERVAL: | |
167 self._tick = now | |
168 print '--single-step output length %d' % self._len | |
169 sys.stdout.flush() | |
170 | |
171 def stop(self): | |
172 self._stopped.set() | |
173 | |
174 | |
175 class TestRunner(base_test_runner.BaseTestRunner): | |
176 def __init__(self, test_options, device, shard_index, max_shard, tests, | |
177 flaky_tests): | |
178 """A TestRunner instance runs a perf test on a single device. | |
179 | |
180 Args: | |
181 test_options: A PerfOptions object. | |
182 device: Device to run the tests. | |
183 shard_index: the index of this device. | |
184 max_shards: the maximum shard index. | |
185 tests: a dict mapping test_name to command. | |
186 flaky_tests: a list of flaky test_name. | |
187 """ | |
188 super(TestRunner, self).__init__(device, None) | |
189 self._options = test_options | |
190 self._shard_index = shard_index | |
191 self._max_shard = max_shard | |
192 self._tests = tests | |
193 self._flaky_tests = flaky_tests | |
194 self._output_dir = None | |
195 self._device_battery = battery_utils.BatteryUtils(self.device) | |
196 | |
197 @staticmethod | |
198 def _IsBetter(result): | |
199 if result['actual_exit_code'] == 0: | |
200 return True | |
201 pickled = os.path.join(constants.PERF_OUTPUT_DIR, | |
202 result['name']) | |
203 if not os.path.exists(pickled): | |
204 return True | |
205 with file(pickled, 'r') as f: | |
206 previous = pickle.loads(f.read()) | |
207 return result['actual_exit_code'] < previous['actual_exit_code'] | |
208 | |
209 @staticmethod | |
210 def _SaveResult(result): | |
211 if TestRunner._IsBetter(result): | |
212 with file(os.path.join(constants.PERF_OUTPUT_DIR, | |
213 result['name']), 'w') as f: | |
214 f.write(pickle.dumps(result)) | |
215 | |
216 def _CheckDeviceAffinity(self, test_name): | |
217 """Returns True if test_name has affinity for this shard.""" | |
218 affinity = (self._tests['steps'][test_name]['device_affinity'] % | |
219 self._max_shard) | |
220 if self._shard_index == affinity: | |
221 return True | |
222 logging.info('Skipping %s on %s (affinity is %s, device is %s)', | |
223 test_name, self.device_serial, affinity, self._shard_index) | |
224 return False | |
225 | |
226 def _CleanupOutputDirectory(self): | |
227 if self._output_dir: | |
228 shutil.rmtree(self._output_dir, ignore_errors=True) | |
229 self._output_dir = None | |
230 | |
231 def _ReadChartjsonOutput(self): | |
232 if not self._output_dir: | |
233 return '' | |
234 | |
235 json_output_path = os.path.join(self._output_dir, 'results-chart.json') | |
236 try: | |
237 with open(json_output_path) as f: | |
238 return f.read() | |
239 except IOError: | |
240 logging.exception('Exception when reading chartjson.') | |
241 logging.error('This usually means that telemetry did not run, so it could' | |
242 ' not generate the file. Please check the device running' | |
243 ' the test.') | |
244 return '' | |
245 | |
246 def _LaunchPerfTest(self, test_name): | |
247 """Runs a perf test. | |
248 | |
249 Args: | |
250 test_name: the name of the test to be executed. | |
251 | |
252 Returns: | |
253 A tuple containing (Output, base_test_result.ResultType) | |
254 """ | |
255 if not self._CheckDeviceAffinity(test_name): | |
256 return '', base_test_result.ResultType.PASS | |
257 | |
258 try: | |
259 logging.warning('Unmapping device ports') | |
260 forwarder.Forwarder.UnmapAllDevicePorts(self.device) | |
261 self.device.old_interface.RestartAdbdOnDevice() | |
262 except Exception as e: | |
263 logging.error('Exception when tearing down device %s', e) | |
264 | |
265 cmd = ('%s --device %s' % | |
266 (self._tests['steps'][test_name]['cmd'], | |
267 self.device_serial)) | |
268 | |
269 if self._options.collect_chartjson_data: | |
270 self._output_dir = tempfile.mkdtemp() | |
271 cmd = cmd + ' --output-dir=%s' % self._output_dir | |
272 | |
273 logging.info( | |
274 'temperature: %s (0.1 C)', | |
275 str(self._device_battery.GetBatteryInfo().get('temperature'))) | |
276 if self._options.max_battery_temp: | |
277 self._device_battery.LetBatteryCoolToTemperature( | |
278 self._options.max_battery_temp) | |
279 | |
280 logging.info('Charge level: %s%%', | |
281 str(self._device_battery.GetBatteryInfo().get('level'))) | |
282 if self._options.min_battery_level: | |
283 self._device_battery.ChargeDeviceToLevel( | |
284 self._options.min_battery_level) | |
285 | |
286 logging.info('%s : %s', test_name, cmd) | |
287 start_time = datetime.datetime.now() | |
288 | |
289 timeout = self._tests['steps'][test_name].get('timeout', 5400) | |
290 if self._options.no_timeout: | |
291 timeout = None | |
292 logging.info('Timeout for %s test: %s', test_name, timeout) | |
293 full_cmd = cmd | |
294 if self._options.dry_run: | |
295 full_cmd = 'echo %s' % cmd | |
296 | |
297 logfile = sys.stdout | |
298 if self._options.single_step: | |
299 # Just print a heart-beat so that the outer buildbot scripts won't timeout | |
300 # without response. | |
301 logfile = _HeartBeatLogger() | |
302 cwd = os.path.abspath(constants.DIR_SOURCE_ROOT) | |
303 if full_cmd.startswith('src/'): | |
304 cwd = os.path.abspath(os.path.join(constants.DIR_SOURCE_ROOT, os.pardir)) | |
305 try: | |
306 exit_code, output = cmd_helper.GetCmdStatusAndOutputWithTimeout( | |
307 full_cmd, timeout, cwd=cwd, shell=True, logfile=logfile) | |
308 json_output = self._ReadChartjsonOutput() | |
309 except cmd_helper.TimeoutError as e: | |
310 exit_code = -1 | |
311 output = str(e) | |
312 json_output = '' | |
313 finally: | |
314 self._CleanupOutputDirectory() | |
315 if self._options.single_step: | |
316 logfile.stop() | |
317 end_time = datetime.datetime.now() | |
318 if exit_code is None: | |
319 exit_code = -1 | |
320 logging.info('%s : exit_code=%d in %d secs at %s', | |
321 test_name, exit_code, (end_time - start_time).seconds, | |
322 self.device_serial) | |
323 | |
324 if exit_code == 0: | |
325 result_type = base_test_result.ResultType.PASS | |
326 else: | |
327 result_type = base_test_result.ResultType.FAIL | |
328 # Since perf tests use device affinity, give the device a chance to | |
329 # recover if it is offline after a failure. Otherwise, the master sharder | |
330 # will remove it from the pool and future tests on this device will fail. | |
331 try: | |
332 self.device.WaitUntilFullyBooted(timeout=120) | |
333 except device_errors.CommandTimeoutError as e: | |
334 logging.error('Device failed to return after %s: %s' % (test_name, e)) | |
335 | |
336 actual_exit_code = exit_code | |
337 if test_name in self._flaky_tests: | |
338 # The exit_code is used at the second stage when printing the | |
339 # test output. If the test is flaky, force to "0" to get that step green | |
340 # whilst still gathering data to the perf dashboards. | |
341 # The result_type is used by the test_dispatcher to retry the test. | |
342 exit_code = 0 | |
343 | |
344 persisted_result = { | |
345 'name': test_name, | |
346 'output': output, | |
347 'chartjson': json_output, | |
348 'exit_code': exit_code, | |
349 'actual_exit_code': actual_exit_code, | |
350 'result_type': result_type, | |
351 'total_time': (end_time - start_time).seconds, | |
352 'device': self.device_serial, | |
353 'cmd': cmd, | |
354 } | |
355 self._SaveResult(persisted_result) | |
356 | |
357 return (output, result_type) | |
358 | |
359 def RunTest(self, test_name): | |
360 """Run a perf test on the device. | |
361 | |
362 Args: | |
363 test_name: String to use for logging the test result. | |
364 | |
365 Returns: | |
366 A tuple of (TestRunResults, retry). | |
367 """ | |
368 _, result_type = self._LaunchPerfTest(test_name) | |
369 results = base_test_result.TestRunResults() | |
370 results.AddResult(base_test_result.BaseTestResult(test_name, result_type)) | |
371 retry = None | |
372 if not results.DidRunPass(): | |
373 retry = test_name | |
374 return results, retry | |
OLD | NEW |