OLD | NEW |
(Empty) | |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. |
| 4 |
| 5 import io |
| 6 import json |
| 7 import logging |
| 8 import os |
| 9 import pickle |
| 10 import re |
| 11 import shutil |
| 12 import sys |
| 13 import tempfile |
| 14 import threading |
| 15 import time |
| 16 import zipfile |
| 17 |
| 18 from devil.android import battery_utils |
| 19 from devil.android import device_errors |
| 20 from devil.android import device_list |
| 21 from devil.android import device_utils |
| 22 from devil.android import forwarder |
| 23 from devil.utils import cmd_helper |
| 24 from devil.utils import reraiser_thread |
| 25 from devil.utils import watchdog_timer |
| 26 from pylib import constants |
| 27 from pylib.base import base_test_result |
| 28 from pylib.constants import host_paths |
| 29 from pylib.local.device import local_device_test_run |
| 30 |
| 31 |
| 32 # Regex for the master branch commit position. |
| 33 _GIT_CR_POS_RE = re.compile(r'^Cr-Commit-Position: refs/heads/master@{#(\d+)}$') |
| 34 _BOOTUP_TIMEOUT = 60 * 2 |
| 35 |
| 36 |
| 37 class _HeartBeatLogger(object): |
| 38 # How often in seconds to print the heartbeat on flush(). |
| 39 _PRINT_INTERVAL = 30.0 |
| 40 |
| 41 def __init__(self): |
| 42 """A file-like class for keeping the buildbot alive.""" |
| 43 self._len = 0 |
| 44 self._tick = time.time() |
| 45 self._stopped = threading.Event() |
| 46 self._timer = threading.Thread(target=self._runner) |
| 47 self._timer.start() |
| 48 |
| 49 def _runner(self): |
| 50 while not self._stopped.is_set(): |
| 51 self.flush() |
| 52 self._stopped.wait(_HeartBeatLogger._PRINT_INTERVAL) |
| 53 |
| 54 def write(self, data): |
| 55 self._len += len(data) |
| 56 |
| 57 def flush(self): |
| 58 now = time.time() |
| 59 if now - self._tick >= _HeartBeatLogger._PRINT_INTERVAL: |
| 60 self._tick = now |
| 61 logging.info('--single-step output length %d', self._len) |
| 62 sys.stdout.flush() |
| 63 |
| 64 def stop(self): |
| 65 self._stopped.set() |
| 66 |
| 67 |
| 68 def _GetChromiumRevision(): |
| 69 # pylint: disable=line-too-long |
| 70 """Get the git hash and commit position of the chromium master branch. |
| 71 |
| 72 See: https://chromium.googlesource.com/chromium/tools/build/+/master/scripts/s
lave/runtest.py#212 |
| 73 |
| 74 Returns: |
| 75 A dictionary with 'revision' and 'commit_pos' keys. |
| 76 """ |
| 77 # pylint: enable=line-too-long |
| 78 status, output = cmd_helper.GetCmdStatusAndOutput( |
| 79 ['git', 'log', '-n', '1', '--pretty=format:%H%n%B', 'HEAD'], |
| 80 cwd=host_paths.DIR_SOURCE_ROOT) |
| 81 revision = None |
| 82 commit_pos = None |
| 83 if not status: |
| 84 lines = output.splitlines() |
| 85 revision = lines[0] |
| 86 for line in reversed(lines): |
| 87 m = _GIT_CR_POS_RE.match(line.strip()) |
| 88 if m: |
| 89 commit_pos = int(m.group(1)) |
| 90 break |
| 91 return {'revision': revision, 'commit_pos': commit_pos} |
| 92 |
| 93 |
| 94 class TestShard(object): |
| 95 def __init__(self, test_instance, device, index, tests, results, watcher=None, |
| 96 retries=3, timeout=None): |
| 97 logging.info('Create shard %s for device %s to run the following tests:', |
| 98 index, device) |
| 99 for t in tests: |
| 100 logging.info(' %s', t) |
| 101 self._battery = battery_utils.BatteryUtils(device) |
| 102 self._device = device |
| 103 self._index = index |
| 104 self._output_dir = None |
| 105 self._results = results |
| 106 self._retries = retries |
| 107 self._test_instance = test_instance |
| 108 self._tests = tests |
| 109 self._timeout = timeout |
| 110 self._watcher = watcher |
| 111 |
| 112 def _WriteBuildBotJson(self): |
| 113 """Write metadata about the buildbot environment to the output dir.""" |
| 114 if not self._output_dir: |
| 115 return |
| 116 data = { |
| 117 'chromium': _GetChromiumRevision(), |
| 118 'environment': dict(os.environ) |
| 119 } |
| 120 with open(os.path.join(self._output_dir, 'buildbot.json'), 'w') as f: |
| 121 json.dump(data, f, sort_keys=True, indent=2, separators=(',', ': ')) |
| 122 |
| 123 def _TestSetUp(self): |
| 124 self._ResetWatcher() |
| 125 try: |
| 126 logging.info('Unmapping device ports.') |
| 127 forwarder.Forwarder.UnmapAllDevicePorts(self._device) |
| 128 except Exception: # pylint: disable=broad-except |
| 129 logging.exception('Exception when resetting ports.') |
| 130 try: |
| 131 self._device.RestartAdbd() |
| 132 except Exception: # pylint: disable=broad-except |
| 133 logging.exception('Exception when restarting adbd') |
| 134 |
| 135 self._BatteryLevelCheck() |
| 136 self._BatteryTempCheck() |
| 137 self._ScreenCheck() |
| 138 |
| 139 if not self._device.IsOnline(): |
| 140 msg = 'Device %s is unresponsive.' % str(self._device) |
| 141 logging.warning(msg) |
| 142 raise device_errors.DeviceUnreachableError(msg) |
| 143 |
| 144 def _CleanupOutputDirectory(self): |
| 145 if self._output_dir: |
| 146 shutil.rmtree(self._output_dir, ignore_errors=True) |
| 147 self._output_dir = None |
| 148 |
| 149 def _CreateCmd(self, test): |
| 150 cmd = '%s --device %s' % (self._tests[test]['cmd'], str(self._device)) |
| 151 if (self._test_instance.collect_chartjson_data |
| 152 or self._tests[test].get('archive_output_dir')): |
| 153 self._output_dir = tempfile.mkdtemp() |
| 154 cmd = cmd + ' --output-dir=%s' % self._output_dir |
| 155 if self._test_instance.dry_run: |
| 156 cmd = 'echo %s' % cmd |
| 157 return cmd |
| 158 |
| 159 def _RunSingleTest(self, test): |
| 160 |
| 161 logging.info('Running %s on shard %s', test, self._index) |
| 162 timeout = ( |
| 163 None if self._test_instance.no_timeout |
| 164 else self._tests[test].get('timeout', self._timeout)) |
| 165 logging.info('Timeout for %s test: %s', test, timeout) |
| 166 |
| 167 logfile = sys.stdout |
| 168 if self._test_instance.single_step: |
| 169 logfile = _HeartBeatLogger() |
| 170 cmd = self._CreateCmd(test) |
| 171 self._WriteBuildBotJson() |
| 172 cwd = os.path.abspath(host_paths.DIR_SOURCE_ROOT) |
| 173 if cmd.startswith('src/'): |
| 174 logging.critical('Path to cmd should be relative to src.') |
| 175 cwd = os.path.abspath(os.path.join(host_paths.DIR_SOURCE_ROOT, os.pardir)) |
| 176 |
| 177 try: |
| 178 logging.debug('Running test with command \'%s\'', cmd) |
| 179 exit_code, output = cmd_helper.GetCmdStatusAndOutputWithTimeout( |
| 180 cmd, timeout, cwd=cwd, shell=True, logfile=logfile) |
| 181 json_output = self._test_instance.ReadChartjsonOutput(self._output_dir) |
| 182 except cmd_helper.TimeoutError as e: |
| 183 exit_code = -1 |
| 184 output = e.output |
| 185 json_output = '' |
| 186 finally: |
| 187 if self._test_instance.single_step: |
| 188 logfile.stop() |
| 189 return cmd, exit_code, output, json_output |
| 190 |
| 191 def _ProcessTestResult( |
| 192 self, test, cmd, start_time, end_time, exit_code, output, json_output): |
| 193 if exit_code is None: |
| 194 exit_code = -1 |
| 195 logging.info('%s : exit_code=%d in %d secs on device %s', |
| 196 test, exit_code, end_time - start_time, |
| 197 str(self._device)) |
| 198 if exit_code == 0: |
| 199 result_type = base_test_result.ResultType.PASS |
| 200 else: |
| 201 result_type = base_test_result.ResultType.FAIL |
| 202 # TODO(rnephew): Improve device recovery logic. |
| 203 try: |
| 204 self._device.WaitUntilFullyBooted(timeout=_BOOTUP_TIMEOUT) |
| 205 except device_errors.CommandTimeoutError: |
| 206 logging.exception('Device failed to return after %s.', test) |
| 207 actual_exit_code = exit_code |
| 208 if (self._test_instance.flaky_steps |
| 209 and test in self._test_instance.flaky_steps): |
| 210 exit_code = 0 |
| 211 archive_bytes = (self._ArchiveOutputDir() |
| 212 if self._tests[test].get('archive_output_dir') |
| 213 else None) |
| 214 persisted_result = { |
| 215 'name': test, |
| 216 'output': [output], |
| 217 'chartjson': json_output, |
| 218 'archive_bytes': archive_bytes, |
| 219 'exit_code': exit_code, |
| 220 'actual_exit_code': actual_exit_code, |
| 221 'result_type': result_type, |
| 222 'start_time': start_time, |
| 223 'end_time': end_time, |
| 224 'total_time': end_time - start_time, |
| 225 'device': str(self._device), |
| 226 'cmd': cmd, |
| 227 } |
| 228 self._SaveResult(persisted_result) |
| 229 return result_type |
| 230 |
| 231 def RunTestsOnShard(self): |
| 232 for test in self._tests: |
| 233 self._TestSetUp() |
| 234 |
| 235 try: |
| 236 exit_code = None |
| 237 tries_left = self._retries |
| 238 |
| 239 while exit_code != 0 and tries_left > 0: |
| 240 self._ResetWatcher() |
| 241 tries_left = tries_left - 1 |
| 242 start_time = time.time() |
| 243 cmd, exit_code, output, json_output = self._RunSingleTest(test) |
| 244 end_time = time.time() |
| 245 result_type = self._ProcessTestResult( |
| 246 test, cmd, start_time, end_time, exit_code, output, json_output) |
| 247 |
| 248 result = base_test_result.TestRunResults() |
| 249 result.AddResult(base_test_result.BaseTestResult(test, result_type)) |
| 250 self._results.append(result) |
| 251 finally: |
| 252 self._CleanupOutputDirectory() |
| 253 |
| 254 @staticmethod |
| 255 def _SaveResult(result): |
| 256 pickled = os.path.join(constants.PERF_OUTPUT_DIR, result['name']) |
| 257 if os.path.exists(pickled): |
| 258 with file(pickled, 'r') as f: |
| 259 previous = pickle.loads(f.read()) |
| 260 result['output'] = previous['output'] + result['output'] |
| 261 with file(pickled, 'w') as f: |
| 262 f.write(pickle.dumps(result)) |
| 263 |
| 264 def _ArchiveOutputDir(self): |
| 265 """Archive all files in the output dir, and return as compressed bytes.""" |
| 266 with io.BytesIO() as archive: |
| 267 with zipfile.ZipFile(archive, 'w', zipfile.ZIP_DEFLATED) as contents: |
| 268 num_files = 0 |
| 269 for absdir, _, files in os.walk(self._output_dir): |
| 270 reldir = os.path.relpath(absdir, self._output_dir) |
| 271 for filename in files: |
| 272 src_path = os.path.join(absdir, filename) |
| 273 # We use normpath to turn './file.txt' into just 'file.txt'. |
| 274 dst_path = os.path.normpath(os.path.join(reldir, filename)) |
| 275 contents.write(src_path, dst_path) |
| 276 num_files += 1 |
| 277 if num_files: |
| 278 logging.info('%d files in the output dir were archived.', num_files) |
| 279 else: |
| 280 logging.warning('No files in the output dir. Archive is empty.') |
| 281 return archive.getvalue() |
| 282 |
| 283 def _ResetWatcher(self): |
| 284 if self._watcher: |
| 285 self._watcher.Reset() |
| 286 |
| 287 def _BatteryLevelCheck(self): |
| 288 logging.info('Charge level: %s%%', |
| 289 str(self._battery.GetBatteryInfo().get('level'))) |
| 290 if self._test_instance.min_battery_level: |
| 291 self._battery.ChargeDeviceToLevel(self._test_instance.min_battery_level) |
| 292 |
| 293 def _ScreenCheck(self): |
| 294 if not self._device.IsScreenOn(): |
| 295 self._device.SetScreen(True) |
| 296 |
| 297 def _BatteryTempCheck(self): |
| 298 logging.info('temperature: %s (0.1 C)', |
| 299 str(self._battery.GetBatteryInfo().get('temperature'))) |
| 300 if self._test_instance.max_battery_temp: |
| 301 self._battery.LetBatteryCoolToTemperature( |
| 302 self._test_instance.max_battery_temp) |
| 303 |
| 304 |
| 305 class LocalDevicePerfTestRun(local_device_test_run.LocalDeviceTestRun): |
| 306 def __init__(self, env, test_instance): |
| 307 super(LocalDevicePerfTestRun, self).__init__(env, test_instance) |
| 308 self._test_instance = test_instance |
| 309 self._env = env |
| 310 self._timeout = None if test_instance.no_timeout else 60 * 60 |
| 311 self._devices = None |
| 312 self._test_buckets = [] |
| 313 self._watcher = None |
| 314 |
| 315 def SetUp(self): |
| 316 self._devices = self._GetAllDevices(self._env.devices, |
| 317 self._test_instance.known_devices_file) |
| 318 self._watcher = watchdog_timer.WatchdogTimer(self._timeout) |
| 319 if os.path.exists(constants.PERF_OUTPUT_DIR): |
| 320 shutil.rmtree(constants.PERF_OUTPUT_DIR) |
| 321 os.makedirs(constants.PERF_OUTPUT_DIR) |
| 322 |
| 323 def TearDown(self): |
| 324 pass |
| 325 |
| 326 def _GetStepsFromDict(self): |
| 327 if self._test_instance.single_step: |
| 328 return { |
| 329 'version': 1, |
| 330 'steps': { |
| 331 'single_step': { |
| 332 'device_affinity': 0, |
| 333 'cmd': self._test_instance.single_step |
| 334 }, |
| 335 } |
| 336 } |
| 337 if self._test_instance.steps: |
| 338 with file(self._test_instance.steps, 'r') as f: |
| 339 steps = json.load(f) |
| 340 assert steps['version'] == 1 |
| 341 return steps |
| 342 |
| 343 def _SplitTestsByAffinity(self): |
| 344 test_dict = self._GetStepsFromDict() |
| 345 for test in test_dict['steps']: |
| 346 affinity = test_dict['steps'][test]['device_affinity'] |
| 347 if len(self._test_buckets) < affinity + 1: |
| 348 while len(self._test_buckets) != affinity + 1: |
| 349 self._test_buckets.append({}) |
| 350 self._test_buckets[affinity][test] = test_dict['steps'][test] |
| 351 return self._test_buckets |
| 352 |
| 353 @staticmethod |
| 354 def _GetAllDevices(active_devices, devices_path): |
| 355 if not devices_path: |
| 356 logging.warning('Known devices file path not being passed. For device ' |
| 357 'affinity to work properly, it must be passed.') |
| 358 try: |
| 359 if devices_path: |
| 360 devices = [device_utils.DeviceUtils(s) |
| 361 for s in device_list.GetPersistentDeviceList(devices_path)] |
| 362 else: |
| 363 logging.warning('Known devices file path not being passed. For device ' |
| 364 'affinity to work properly, it must be passed.') |
| 365 devices = active_devices |
| 366 except IOError as e: |
| 367 logging.error('Unable to find %s [%s]', devices_path, e) |
| 368 devices = active_devices |
| 369 return sorted(devices) |
| 370 |
| 371 |
| 372 def RunTests(self): |
| 373 # Option selected for saving a json file with a list of test names. |
| 374 if self._test_instance.output_json_list: |
| 375 return self._test_instance.OutputJsonList() |
| 376 |
| 377 # Just print the results from a single previously executed step. |
| 378 if self._test_instance.print_step: |
| 379 return self._test_instance.PrintTestOutput() |
| 380 |
| 381 # Affinitize the tests. |
| 382 test_buckets = self._SplitTestsByAffinity() |
| 383 if not test_buckets: |
| 384 raise NotImplementedError('No tests found!') |
| 385 |
| 386 threads = [] |
| 387 results = [] |
| 388 for x in xrange(min(len(self._devices), len(test_buckets))): |
| 389 new_shard = TestShard(self._test_instance, self._devices[x], x, |
| 390 test_buckets[x], results, watcher=self._watcher, |
| 391 retries=self._env.max_tries, timeout=self._timeout) |
| 392 threads.append(reraiser_thread.ReraiserThread(new_shard.RunTestsOnShard)) |
| 393 |
| 394 workers = reraiser_thread.ReraiserThreadGroup(threads) |
| 395 workers.StartAll() |
| 396 |
| 397 try: |
| 398 workers.JoinAll(self._watcher) |
| 399 except device_errors.CommandFailedError: |
| 400 logging.exception('Command failed on device.') |
| 401 except device_errors.CommandTimeoutError: |
| 402 logging.exception('Command timed out on device.') |
| 403 except device_errors.DeviceUnreachableError: |
| 404 logging.exception('Device became unreachable.') |
| 405 return results |
| 406 |
| 407 # override |
| 408 def TestPackage(self): |
| 409 return 'Perf' |
| 410 |
| 411 # override |
| 412 def _CreateShards(self, _tests): |
| 413 raise NotImplementedError |
| 414 |
| 415 # override |
| 416 def _GetTests(self): |
| 417 return self._test_buckets |
| 418 |
| 419 # override |
| 420 def _RunTest(self, _device, _test): |
| 421 raise NotImplementedError |
| 422 |
| 423 # override |
| 424 def _ShouldShard(self): |
| 425 return False |
OLD | NEW |