Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(98)

Side by Side Diff: build/android/pylib/perf/test_runner.py

Issue 2101243005: Add a snapshot of flutter/engine/src/build to our sdk (Closed) Base URL: git@github.com:dart-lang/sdk.git@master
Patch Set: add README.dart Created 4 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « build/android/pylib/perf/test_options.py ('k') | build/android/pylib/perf/thermal_throttle.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 # Copyright 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
4
5 """Runs perf tests.
6
7 Our buildbot infrastructure requires each slave to run steps serially.
8 This is sub-optimal for android, where these steps can run independently on
9 multiple connected devices.
10
11 The buildbots will run this script multiple times per cycle:
12 - First: all steps listed in --steps in will be executed in parallel using all
13 connected devices. Step results will be pickled to disk. Each step has a unique
14 name. The result code will be ignored if the step name is listed in
15 --flaky-steps.
16 The buildbot will treat this step as a regular step, and will not process any
17 graph data.
18
19 - Then, with -print-step STEP_NAME: at this stage, we'll simply print the file
20 with the step results previously saved. The buildbot will then process the graph
21 data accordingly.
22
23 The JSON steps file contains a dictionary in the format:
24 { "version": int,
25 "steps": {
26 "foo": {
27 "device_affinity": int,
28 "cmd": "script_to_execute foo"
29 },
30 "bar": {
31 "device_affinity": int,
32 "cmd": "script_to_execute bar"
33 }
34 }
35 }
36
37 The JSON flaky steps file contains a list with step names which results should
38 be ignored:
39 [
40 "step_name_foo",
41 "step_name_bar"
42 ]
43
44 Note that script_to_execute necessarily have to take at least the following
45 option:
46 --device: the serial number to be passed to all adb commands.
47 """
48
49 import collections
50 import datetime
51 import json
52 import logging
53 import os
54 import pickle
55 import shutil
56 import sys
57 import tempfile
58 import threading
59 import time
60
61 from pylib import cmd_helper
62 from pylib import constants
63 from pylib import forwarder
64 from pylib.base import base_test_result
65 from pylib.base import base_test_runner
66 from pylib.device import battery_utils
67 from pylib.device import device_errors
68
69
70 def GetPersistedResult(test_name):
71 file_name = os.path.join(constants.PERF_OUTPUT_DIR, test_name)
72 if not os.path.exists(file_name):
73 logging.error('File not found %s', file_name)
74 return None
75
76 with file(file_name, 'r') as f:
77 return pickle.loads(f.read())
78
79
80 def OutputJsonList(json_input, json_output):
81 with file(json_input, 'r') as i:
82 all_steps = json.load(i)
83
84 step_values = []
85 for k, v in all_steps['steps'].iteritems():
86 data = {'test': k, 'device_affinity': v['device_affinity']}
87
88 persisted_result = GetPersistedResult(k)
89 if persisted_result:
90 data['total_time'] = persisted_result['total_time']
91 step_values.append(data)
92
93 with file(json_output, 'w') as o:
94 o.write(json.dumps(step_values))
95 return 0
96
97
98 def PrintTestOutput(test_name, json_file_name=None):
99 """Helper method to print the output of previously executed test_name.
100
101 Args:
102 test_name: name of the test that has been previously executed.
103 json_file_name: name of the file to output chartjson data to.
104
105 Returns:
106 exit code generated by the test step.
107 """
108 persisted_result = GetPersistedResult(test_name)
109 if not persisted_result:
110 return 1
111 logging.info('*' * 80)
112 logging.info('Output from:')
113 logging.info(persisted_result['cmd'])
114 logging.info('*' * 80)
115 print persisted_result['output']
116
117 if json_file_name:
118 with file(json_file_name, 'w') as f:
119 f.write(persisted_result['chartjson'])
120
121 return persisted_result['exit_code']
122
123
124 def PrintSummary(test_names):
125 logging.info('*' * 80)
126 logging.info('Sharding summary')
127 device_total_time = collections.defaultdict(int)
128 for test_name in test_names:
129 file_name = os.path.join(constants.PERF_OUTPUT_DIR, test_name)
130 if not os.path.exists(file_name):
131 logging.info('%s : No status file found', test_name)
132 continue
133 with file(file_name, 'r') as f:
134 result = pickle.loads(f.read())
135 logging.info('%s : exit_code=%d in %d secs at %s',
136 result['name'], result['exit_code'], result['total_time'],
137 result['device'])
138 device_total_time[result['device']] += result['total_time']
139 for device, device_time in device_total_time.iteritems():
140 logging.info('Total for device %s : %d secs', device, device_time)
141 logging.info('Total steps time: %d secs', sum(device_total_time.values()))
142
143
144 class _HeartBeatLogger(object):
145 # How often to print the heartbeat on flush().
146 _PRINT_INTERVAL = 30.0
147
148 def __init__(self):
149 """A file-like class for keeping the buildbot alive."""
150 self._len = 0
151 self._tick = time.time()
152 self._stopped = threading.Event()
153 self._timer = threading.Thread(target=self._runner)
154 self._timer.start()
155
156 def _runner(self):
157 while not self._stopped.is_set():
158 self.flush()
159 self._stopped.wait(_HeartBeatLogger._PRINT_INTERVAL)
160
161 def write(self, data):
162 self._len += len(data)
163
164 def flush(self):
165 now = time.time()
166 if now - self._tick >= _HeartBeatLogger._PRINT_INTERVAL:
167 self._tick = now
168 print '--single-step output length %d' % self._len
169 sys.stdout.flush()
170
171 def stop(self):
172 self._stopped.set()
173
174
175 class TestRunner(base_test_runner.BaseTestRunner):
176 def __init__(self, test_options, device, shard_index, max_shard, tests,
177 flaky_tests):
178 """A TestRunner instance runs a perf test on a single device.
179
180 Args:
181 test_options: A PerfOptions object.
182 device: Device to run the tests.
183 shard_index: the index of this device.
184 max_shards: the maximum shard index.
185 tests: a dict mapping test_name to command.
186 flaky_tests: a list of flaky test_name.
187 """
188 super(TestRunner, self).__init__(device, None)
189 self._options = test_options
190 self._shard_index = shard_index
191 self._max_shard = max_shard
192 self._tests = tests
193 self._flaky_tests = flaky_tests
194 self._output_dir = None
195 self._device_battery = battery_utils.BatteryUtils(self.device)
196
197 @staticmethod
198 def _IsBetter(result):
199 if result['actual_exit_code'] == 0:
200 return True
201 pickled = os.path.join(constants.PERF_OUTPUT_DIR,
202 result['name'])
203 if not os.path.exists(pickled):
204 return True
205 with file(pickled, 'r') as f:
206 previous = pickle.loads(f.read())
207 return result['actual_exit_code'] < previous['actual_exit_code']
208
209 @staticmethod
210 def _SaveResult(result):
211 if TestRunner._IsBetter(result):
212 with file(os.path.join(constants.PERF_OUTPUT_DIR,
213 result['name']), 'w') as f:
214 f.write(pickle.dumps(result))
215
216 def _CheckDeviceAffinity(self, test_name):
217 """Returns True if test_name has affinity for this shard."""
218 affinity = (self._tests['steps'][test_name]['device_affinity'] %
219 self._max_shard)
220 if self._shard_index == affinity:
221 return True
222 logging.info('Skipping %s on %s (affinity is %s, device is %s)',
223 test_name, self.device_serial, affinity, self._shard_index)
224 return False
225
226 def _CleanupOutputDirectory(self):
227 if self._output_dir:
228 shutil.rmtree(self._output_dir, ignore_errors=True)
229 self._output_dir = None
230
231 def _ReadChartjsonOutput(self):
232 if not self._output_dir:
233 return ''
234
235 json_output_path = os.path.join(self._output_dir, 'results-chart.json')
236 try:
237 with open(json_output_path) as f:
238 return f.read()
239 except IOError:
240 logging.exception('Exception when reading chartjson.')
241 logging.error('This usually means that telemetry did not run, so it could'
242 ' not generate the file. Please check the device running'
243 ' the test.')
244 return ''
245
246 def _LaunchPerfTest(self, test_name):
247 """Runs a perf test.
248
249 Args:
250 test_name: the name of the test to be executed.
251
252 Returns:
253 A tuple containing (Output, base_test_result.ResultType)
254 """
255 if not self._CheckDeviceAffinity(test_name):
256 return '', base_test_result.ResultType.PASS
257
258 try:
259 logging.warning('Unmapping device ports')
260 forwarder.Forwarder.UnmapAllDevicePorts(self.device)
261 self.device.old_interface.RestartAdbdOnDevice()
262 except Exception as e:
263 logging.error('Exception when tearing down device %s', e)
264
265 cmd = ('%s --device %s' %
266 (self._tests['steps'][test_name]['cmd'],
267 self.device_serial))
268
269 if self._options.collect_chartjson_data:
270 self._output_dir = tempfile.mkdtemp()
271 cmd = cmd + ' --output-dir=%s' % self._output_dir
272
273 logging.info(
274 'temperature: %s (0.1 C)',
275 str(self._device_battery.GetBatteryInfo().get('temperature')))
276 if self._options.max_battery_temp:
277 self._device_battery.LetBatteryCoolToTemperature(
278 self._options.max_battery_temp)
279
280 logging.info('Charge level: %s%%',
281 str(self._device_battery.GetBatteryInfo().get('level')))
282 if self._options.min_battery_level:
283 self._device_battery.ChargeDeviceToLevel(
284 self._options.min_battery_level)
285
286 logging.info('%s : %s', test_name, cmd)
287 start_time = datetime.datetime.now()
288
289 timeout = self._tests['steps'][test_name].get('timeout', 5400)
290 if self._options.no_timeout:
291 timeout = None
292 logging.info('Timeout for %s test: %s', test_name, timeout)
293 full_cmd = cmd
294 if self._options.dry_run:
295 full_cmd = 'echo %s' % cmd
296
297 logfile = sys.stdout
298 if self._options.single_step:
299 # Just print a heart-beat so that the outer buildbot scripts won't timeout
300 # without response.
301 logfile = _HeartBeatLogger()
302 cwd = os.path.abspath(constants.DIR_SOURCE_ROOT)
303 if full_cmd.startswith('src/'):
304 cwd = os.path.abspath(os.path.join(constants.DIR_SOURCE_ROOT, os.pardir))
305 try:
306 exit_code, output = cmd_helper.GetCmdStatusAndOutputWithTimeout(
307 full_cmd, timeout, cwd=cwd, shell=True, logfile=logfile)
308 json_output = self._ReadChartjsonOutput()
309 except cmd_helper.TimeoutError as e:
310 exit_code = -1
311 output = str(e)
312 json_output = ''
313 finally:
314 self._CleanupOutputDirectory()
315 if self._options.single_step:
316 logfile.stop()
317 end_time = datetime.datetime.now()
318 if exit_code is None:
319 exit_code = -1
320 logging.info('%s : exit_code=%d in %d secs at %s',
321 test_name, exit_code, (end_time - start_time).seconds,
322 self.device_serial)
323
324 if exit_code == 0:
325 result_type = base_test_result.ResultType.PASS
326 else:
327 result_type = base_test_result.ResultType.FAIL
328 # Since perf tests use device affinity, give the device a chance to
329 # recover if it is offline after a failure. Otherwise, the master sharder
330 # will remove it from the pool and future tests on this device will fail.
331 try:
332 self.device.WaitUntilFullyBooted(timeout=120)
333 except device_errors.CommandTimeoutError as e:
334 logging.error('Device failed to return after %s: %s' % (test_name, e))
335
336 actual_exit_code = exit_code
337 if test_name in self._flaky_tests:
338 # The exit_code is used at the second stage when printing the
339 # test output. If the test is flaky, force to "0" to get that step green
340 # whilst still gathering data to the perf dashboards.
341 # The result_type is used by the test_dispatcher to retry the test.
342 exit_code = 0
343
344 persisted_result = {
345 'name': test_name,
346 'output': output,
347 'chartjson': json_output,
348 'exit_code': exit_code,
349 'actual_exit_code': actual_exit_code,
350 'result_type': result_type,
351 'total_time': (end_time - start_time).seconds,
352 'device': self.device_serial,
353 'cmd': cmd,
354 }
355 self._SaveResult(persisted_result)
356
357 return (output, result_type)
358
359 def RunTest(self, test_name):
360 """Run a perf test on the device.
361
362 Args:
363 test_name: String to use for logging the test result.
364
365 Returns:
366 A tuple of (TestRunResults, retry).
367 """
368 _, result_type = self._LaunchPerfTest(test_name)
369 results = base_test_result.TestRunResults()
370 results.AddResult(base_test_result.BaseTestResult(test_name, result_type))
371 retry = None
372 if not results.DidRunPass():
373 retry = test_name
374 return results, retry
OLDNEW
« no previous file with comments | « build/android/pylib/perf/test_options.py ('k') | build/android/pylib/perf/thermal_throttle.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698