Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(60)

Side by Side Diff: build/android/pylib/local/device/local_device_perf_test_run.py

Issue 2012323002: [Android] Implement perf tests to platform mode. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: address comments in ps5 Created 4 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 # Copyright 2016 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
4
5 import io
6 import json
7 import logging
8 import os
9 import pickle
10 import re
11 import shutil
12 import sys
13 import tempfile
14 import threading
15 import time
16 import zipfile
17
18 from devil.android import battery_utils
19 from devil.android import device_errors
20 from devil.android import device_list
21 from devil.android import device_utils
22 from devil.android import forwarder
23 from devil.utils import cmd_helper
24 from devil.utils import reraiser_thread
25 from devil.utils import watchdog_timer
26 from pylib import constants
27 from pylib.base import base_test_result
28 from pylib.constants import host_paths
29 from pylib.local.device import local_device_test_run
30
31
32 # Regex for the master branch commit position.
33 _GIT_CR_POS_RE = re.compile(r'^Cr-Commit-Position: refs/heads/master@{#(\d+)}$')
34 _BOOTUP_TIMEOUT = 60 * 2
35
36
37 class _HeartBeatLogger(object):
38 # How often in seconds to print the heartbeat on flush().
39 _PRINT_INTERVAL = 30.0
40
41 def __init__(self):
42 """A file-like class for keeping the buildbot alive."""
43 self._len = 0
44 self._tick = time.time()
45 self._stopped = threading.Event()
46 self._timer = threading.Thread(target=self._runner)
47 self._timer.start()
48
49 def _runner(self):
50 while not self._stopped.is_set():
51 self.flush()
52 self._stopped.wait(_HeartBeatLogger._PRINT_INTERVAL)
53
54 def write(self, data):
55 self._len += len(data)
56
57 def flush(self):
58 now = time.time()
59 if now - self._tick >= _HeartBeatLogger._PRINT_INTERVAL:
60 self._tick = now
61 logging.info('--single-step output length %d', self._len)
62 sys.stdout.flush()
63
64 def stop(self):
65 self._stopped.set()
66
67
68 def _GetChromiumRevision():
69 # pylint: disable=line-too-long
70 """Get the git hash and commit position of the chromium master branch.
71
72 See: https://chromium.googlesource.com/chromium/tools/build/+/master/scripts/s lave/runtest.py#212
73
74 Returns:
75 A dictionary with 'revision' and 'commit_pos' keys.
76 """
77 # pylint: enable=line-too-long
78 status, output = cmd_helper.GetCmdStatusAndOutput(
79 ['git', 'log', '-n', '1', '--pretty=format:%H%n%B', 'HEAD'],
80 cwd=host_paths.DIR_SOURCE_ROOT)
81 revision = None
82 commit_pos = None
83 if not status:
84 lines = output.splitlines()
85 revision = lines[0]
86 for line in reversed(lines):
87 m = _GIT_CR_POS_RE.match(line.strip())
88 if m:
89 commit_pos = int(m.group(1))
90 break
91 return {'revision': revision, 'commit_pos': commit_pos}
92
93
94 class TestShard(object):
95 def __init__(self, test_instance, device, index, tests, results, watcher=None,
96 retries=3, timeout=None):
97 logging.info('Create shard %s for device %s to run the following tests:',
98 index, device)
99 for t in tests:
100 logging.info(' %s', t)
101 self._battery = battery_utils.BatteryUtils(device)
102 self._device = device
103 self._index = index
104 self._output_dir = None
105 self._results = results
106 self._retries = retries
107 self._test_instance = test_instance
108 self._tests = tests
109 self._timeout = timeout
110 self._watcher = watcher
111
112 def _WriteBuildBotJson(self):
113 """Write metadata about the buildbot environment to the output dir."""
114 if not self._output_dir:
115 return
116 data = {
117 'chromium': _GetChromiumRevision(),
118 'environment': dict(os.environ)
119 }
120 with open(os.path.join(self._output_dir, 'buildbot.json'), 'w') as f:
121 json.dump(data, f, sort_keys=True, indent=2, separators=(',', ': '))
122
123 def _TestSetUp(self):
124 self._ResetWatcher()
125 try:
126 logging.info('Unmapping device ports.')
127 forwarder.Forwarder.UnmapAllDevicePorts(self._device)
128 except Exception: # pylint: disable=broad-except
129 logging.exception('Exception when resetting ports.')
130 try:
131 self._device.RestartAdbd()
132 except Exception: # pylint: disable=broad-except
133 logging.exception('Exception when restarting adbd')
134
135 self._BatteryLevelCheck()
136 self._BatteryTempCheck()
137 self._ScreenCheck()
138
139 if not self._device.IsOnline():
140 msg = 'Device %s is unresponsive.' % str(self._device)
141 logging.warning(msg)
142 raise device_errors.DeviceUnreachableError(msg)
143
144 def _CleanupOutputDirectory(self):
145 if self._output_dir:
146 shutil.rmtree(self._output_dir, ignore_errors=True)
147 self._output_dir = None
148
149 def _CreateCmd(self, test):
150 cmd = '%s --device %s' % (self._tests[test]['cmd'], str(self._device))
151 if (self._test_instance.collect_chartjson_data
152 or self._tests[test].get('archive_output_dir')):
153 self._output_dir = tempfile.mkdtemp()
154 cmd = cmd + ' --output-dir=%s' % self._output_dir
155 if self._test_instance.dry_run:
156 cmd = 'echo %s' % cmd
157 return cmd
158
159 def _RunSingleTest(self, test):
160
161 logging.info('Running %s on shard %s', test, self._index)
162 timeout = (
163 None if self._test_instance.no_timeout
164 else self._tests[test].get('timeout', self._timeout))
165 logging.info('Timeout for %s test: %s', test, timeout)
166
167 logfile = sys.stdout
168 if self._test_instance.single_step:
169 logfile = _HeartBeatLogger()
170 cmd = self._CreateCmd(test)
171 self._WriteBuildBotJson()
172 cwd = os.path.abspath(host_paths.DIR_SOURCE_ROOT)
173 if cmd.startswith('src/'):
174 logging.critical('Path to cmd should be relative to src.')
175 cwd = os.path.abspath(os.path.join(host_paths.DIR_SOURCE_ROOT, os.pardir))
176
177 try:
178 logging.debug('Running test with command \'%s\'', cmd)
179 exit_code, output = cmd_helper.GetCmdStatusAndOutputWithTimeout(
180 cmd, timeout, cwd=cwd, shell=True, logfile=logfile)
181 json_output = self._test_instance.ReadChartjsonOutput(self._output_dir)
182 except cmd_helper.TimeoutError as e:
183 exit_code = -1
184 output = e.output
185 json_output = ''
186 finally:
187 if self._test_instance.single_step:
188 logfile.stop()
189 return cmd, exit_code, output, json_output
190
191 def _ProcessTestResult(
192 self, test, cmd, start_time, end_time, exit_code, output, json_output):
193 if exit_code is None:
194 exit_code = -1
195 logging.info('%s : exit_code=%d in %d secs on device %s',
196 test, exit_code, end_time - start_time,
197 str(self._device))
198 if exit_code == 0:
199 result_type = base_test_result.ResultType.PASS
200 else:
201 result_type = base_test_result.ResultType.FAIL
202 # TODO(rnephew): Improve device recovery logic.
203 try:
204 self._device.WaitUntilFullyBooted(timeout=_BOOTUP_TIMEOUT)
205 except device_errors.CommandTimeoutError:
206 logging.exception('Device failed to return after %s.', test)
207 actual_exit_code = exit_code
208 if (self._test_instance.flaky_steps
209 and test in self._test_instance.flaky_steps):
210 exit_code = 0
211 archive_bytes = (self._ArchiveOutputDir()
212 if self._tests[test].get('archive_output_dir')
213 else None)
214 persisted_result = {
215 'name': test,
216 'output': [output],
217 'chartjson': json_output,
218 'archive_bytes': archive_bytes,
219 'exit_code': exit_code,
220 'actual_exit_code': actual_exit_code,
221 'result_type': result_type,
222 'start_time': start_time,
223 'end_time': end_time,
224 'total_time': end_time - start_time,
225 'device': str(self._device),
226 'cmd': cmd,
227 }
228 self._SaveResult(persisted_result)
229 return result_type
230
231 def RunTestsOnShard(self):
232 for test in self._tests:
233 self._TestSetUp()
234
235 try:
236 exit_code = None
237 tries_left = self._retries
238
239 while exit_code != 0 and tries_left > 0:
240 self._ResetWatcher()
241 tries_left = tries_left - 1
242 start_time = time.time()
243 cmd, exit_code, output, json_output = self._RunSingleTest(test)
244 end_time = time.time()
245 result_type = self._ProcessTestResult(
246 test, cmd, start_time, end_time, exit_code, output, json_output)
247
248 result = base_test_result.TestRunResults()
249 result.AddResult(base_test_result.BaseTestResult(test, result_type))
250 self._results.append(result)
251 finally:
252 self._CleanupOutputDirectory()
253
254 @staticmethod
255 def _SaveResult(result):
256 pickled = os.path.join(constants.PERF_OUTPUT_DIR, result['name'])
257 if os.path.exists(pickled):
258 with file(pickled, 'r') as f:
259 previous = pickle.loads(f.read())
260 result['output'] = previous['output'] + result['output']
261 with file(pickled, 'w') as f:
262 f.write(pickle.dumps(result))
263
264 def _ArchiveOutputDir(self):
265 """Archive all files in the output dir, and return as compressed bytes."""
266 with io.BytesIO() as archive:
267 with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED) as contents:
268 num_files = 0
269 for absdir, _, files in os.walk(self._output_dir):
270 reldir = os.path.relpath(absdir, self._output_dir)
271 for filename in files:
272 src_path = os.path.join(absdir, filename)
273 # We use normpath to turn './file.txt' into just 'file.txt'.
274 dst_path = os.path.normpath(os.path.join(reldir, filename))
275 contents.write(src_path, dst_path)
276 num_files += 1
277 if num_files:
278 logging.info('%d files in the output dir were archived.', num_files)
279 else:
280 logging.warning('No files in the output dir. Archive is empty.')
281 return archive.getvalue()
282
283 def _ResetWatcher(self):
284 if self._watcher:
285 self._watcher.Reset()
286
287 def _BatteryLevelCheck(self):
288 logging.info('Charge level: %s%%',
289 str(self._battery.GetBatteryInfo().get('level')))
290 if self._test_instance.min_battery_level:
291 self._battery.ChargeDeviceToLevel(self._test_instance.min_battery_level)
292
293 def _ScreenCheck(self):
294 if not self._device.IsScreenOn():
295 self._device.SetScreen(True)
296
297 def _BatteryTempCheck(self):
298 logging.info('temperature: %s (0.1 C)',
299 str(self._battery.GetBatteryInfo().get('temperature')))
300 if self._test_instance.max_battery_temp:
301 self._battery.LetBatteryCoolToTemperature(
302 self._test_instance.max_battery_temp)
303
304
305 class LocalDevicePerfTestRun(local_device_test_run.LocalDeviceTestRun):
306 def __init__(self, env, test_instance):
307 super(LocalDevicePerfTestRun, self).__init__(env, test_instance)
308 self._test_instance = test_instance
309 self._env = env
310 self._timeout = None if test_instance.no_timeout else 60 * 60
311 self._devices = None
312 self._test_buckets = []
313 self._watcher = None
314
315 def SetUp(self):
316 self._devices = self._GetAllDevices(self._env.devices,
317 self._test_instance.known_devices_file)
318 self._watcher = watchdog_timer.WatchdogTimer(self._timeout)
319 if os.path.exists(constants.PERF_OUTPUT_DIR):
320 shutil.rmtree(constants.PERF_OUTPUT_DIR)
321 os.makedirs(constants.PERF_OUTPUT_DIR)
322
323 def TearDown(self):
324 pass
325
326 def _GetStepsFromDict(self):
327 if self._test_instance.single_step:
328 return {
329 'version': 1,
330 'steps': {
331 'single_step': {
332 'device_affinity': 0,
333 'cmd': self._test_instance.single_step
334 },
335 }
336 }
337 if self._test_instance.steps:
338 with file(self._test_instance.steps, 'r') as f:
339 steps = json.load(f)
340 assert steps['version'] == 1
341 return steps
342
343 def _SplitTestsByAffinity(self):
344 test_dict = self._GetStepsFromDict()
345 for test in test_dict['steps']:
346 affinity = test_dict['steps'][test]['device_affinity']
347 if len(self._test_buckets) < affinity + 1:
348 while len(self._test_buckets) != affinity + 1:
349 self._test_buckets.append({})
350 self._test_buckets[affinity][test] = test_dict['steps'][test]
351 return self._test_buckets
352
353 @staticmethod
354 def _GetAllDevices(active_devices, devices_path):
355 if not devices_path:
356 logging.warning('Known devices file path not being passed. For device '
357 'affinity to work properly, it must be passed.')
358 try:
359 if devices_path:
360 devices = [device_utils.DeviceUtils(s)
361 for s in device_list.GetPersistentDeviceList(devices_path)]
362 else:
363 logging.warning('Known devices file path not being passed. For device '
364 'affinity to work properly, it must be passed.')
365 devices = active_devices
366 except IOError as e:
367 logging.error('Unable to find %s [%s]', devices_path, e)
368 devices = active_devices
369 return sorted(devices)
370
371
372 def RunTests(self):
373 # Option selected for saving a json file with a list of test names.
374 if self._test_instance.output_json_list:
375 return self._test_instance.OutputJsonList()
376
377 # Just print the results from a single previously executed step.
378 if self._test_instance.print_step:
379 return self._test_instance.PrintTestOutput()
380
381 # Affinitize the tests.
382 test_buckets = self._SplitTestsByAffinity()
383 if not test_buckets:
384 raise NotImplementedError('No tests found!')
385
386 threads = []
387 results = []
388 for x in xrange(min(len(self._devices), len(test_buckets))):
389 new_shard = TestShard(self._test_instance, self._devices[x], x,
390 test_buckets[x], results, watcher=self._watcher,
391 retries=self._env.max_tries, timeout=self._timeout)
392 threads.append(reraiser_thread.ReraiserThread(new_shard.RunTestsOnShard))
393
394 workers = reraiser_thread.ReraiserThreadGroup(threads)
395 workers.StartAll()
396
397 try:
398 workers.JoinAll(self._watcher)
399 except device_errors.CommandFailedError:
400 logging.exception('Command failed on device.')
401 except device_errors.CommandTimeoutError:
402 logging.exception('Command timed out on device.')
403 except device_errors.DeviceUnreachableError:
404 logging.exception('Device became unreachable.')
405 return results
406
407 # override
408 def TestPackage(self):
409 return 'Perf'
410
411 # override
412 def _CreateShards(self, _tests):
413 raise NotImplementedError
414
415 # override
416 def _GetTests(self):
417 return self._test_buckets
418
419 # override
420 def _RunTest(self, _device, _test):
421 raise NotImplementedError
422
423 # override
424 def _ShouldShard(self):
425 return False
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698