| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 """Run specific test on specific environment.""" | |
| 6 | |
| 7 import json | |
| 8 import logging | |
| 9 import os | |
| 10 import re | |
| 11 import shutil | |
| 12 import string | |
| 13 import tempfile | |
| 14 import time | |
| 15 import zipfile | |
| 16 | |
| 17 from devil.utils import zip_utils | |
| 18 from pylib.base import base_test_result | |
| 19 from pylib.base import test_run | |
| 20 from pylib.remote.device import appurify_constants | |
| 21 from pylib.remote.device import appurify_sanitized | |
| 22 from pylib.remote.device import remote_device_helper | |
| 23 | |
| 24 _DEVICE_OFFLINE_RE = re.compile('error: device not found') | |
| 25 _LONG_MSG_RE = re.compile('longMsg=(.*)$') | |
| 26 _SHORT_MSG_RE = re.compile('shortMsg=(.*)$') | |
| 27 | |
| 28 class RemoteDeviceTestRun(test_run.TestRun): | |
| 29 """Run tests on a remote device.""" | |
| 30 | |
| 31 _TEST_RUN_KEY = 'test_run' | |
| 32 _TEST_RUN_ID_KEY = 'test_run_id' | |
| 33 | |
| 34 WAIT_TIME = 5 | |
| 35 COMPLETE = 'complete' | |
| 36 HEARTBEAT_INTERVAL = 300 | |
| 37 | |
| 38 def __init__(self, env, test_instance): | |
| 39 """Constructor. | |
| 40 | |
| 41 Args: | |
| 42 env: Environment the tests will run in. | |
| 43 test_instance: The test that will be run. | |
| 44 """ | |
| 45 super(RemoteDeviceTestRun, self).__init__(env, test_instance) | |
| 46 self._env = env | |
| 47 self._test_instance = test_instance | |
| 48 self._app_id = '' | |
| 49 self._test_id = '' | |
| 50 self._results = '' | |
| 51 self._test_run_id = '' | |
| 52 self._results_temp_dir = None | |
| 53 | |
| 54 #override | |
| 55 def SetUp(self): | |
| 56 """Set up a test run.""" | |
| 57 if self._env.trigger: | |
| 58 self._TriggerSetUp() | |
| 59 elif self._env.collect: | |
| 60 assert isinstance(self._env.collect, basestring), ( | |
| 61 'File for storing test_run_id must be a string.') | |
| 62 with open(self._env.collect, 'r') as persisted_data_file: | |
| 63 persisted_data = json.loads(persisted_data_file.read()) | |
| 64 self._env.LoadFrom(persisted_data) | |
| 65 self.LoadFrom(persisted_data) | |
| 66 | |
| 67 def _TriggerSetUp(self): | |
| 68 """Set up the triggering of a test run.""" | |
| 69 raise NotImplementedError | |
| 70 | |
| 71 #override | |
| 72 def RunTests(self): | |
| 73 """Run the test.""" | |
| 74 if self._env.trigger: | |
| 75 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
| 76 logging.WARNING): | |
| 77 test_start_res = appurify_sanitized.api.tests_run( | |
| 78 self._env.token, self._env.device_type_id, self._app_id, | |
| 79 self._test_id) | |
| 80 remote_device_helper.TestHttpResponse( | |
| 81 test_start_res, 'Unable to run test.') | |
| 82 self._test_run_id = test_start_res.json()['response']['test_run_id'] | |
| 83 logging.info('Test run id: %s', self._test_run_id) | |
| 84 | |
| 85 if self._env.collect: | |
| 86 current_status = '' | |
| 87 timeout_counter = 0 | |
| 88 heartbeat_counter = 0 | |
| 89 while self._GetTestStatus(self._test_run_id) != self.COMPLETE: | |
| 90 if self._results['detailed_status'] != current_status: | |
| 91 logging.info('Test status: %s', self._results['detailed_status']) | |
| 92 current_status = self._results['detailed_status'] | |
| 93 timeout_counter = 0 | |
| 94 heartbeat_counter = 0 | |
| 95 if heartbeat_counter > self.HEARTBEAT_INTERVAL: | |
| 96 logging.info('Test status: %s', self._results['detailed_status']) | |
| 97 heartbeat_counter = 0 | |
| 98 | |
| 99 timeout = self._env.timeouts.get( | |
| 100 current_status, self._env.timeouts['unknown']) | |
| 101 if timeout_counter > timeout: | |
| 102 raise remote_device_helper.RemoteDeviceError( | |
| 103 'Timeout while in %s state for %s seconds' | |
| 104 % (current_status, timeout), | |
| 105 is_infra_error=True) | |
| 106 time.sleep(self.WAIT_TIME) | |
| 107 timeout_counter += self.WAIT_TIME | |
| 108 heartbeat_counter += self.WAIT_TIME | |
| 109 self._DownloadTestResults(self._env.results_path) | |
| 110 | |
| 111 if self._results['results']['exception']: | |
| 112 raise remote_device_helper.RemoteDeviceError( | |
| 113 self._results['results']['exception'], is_infra_error=True) | |
| 114 | |
| 115 return [self._ParseTestResults()] | |
| 116 | |
| 117 #override | |
| 118 def TearDown(self): | |
| 119 """Tear down the test run.""" | |
| 120 if self._env.collect: | |
| 121 self._CollectTearDown() | |
| 122 elif self._env.trigger: | |
| 123 assert isinstance(self._env.trigger, basestring), ( | |
| 124 'File for storing test_run_id must be a string.') | |
| 125 with open(self._env.trigger, 'w') as persisted_data_file: | |
| 126 persisted_data = {} | |
| 127 self.DumpTo(persisted_data) | |
| 128 self._env.DumpTo(persisted_data) | |
| 129 persisted_data_file.write(json.dumps(persisted_data)) | |
| 130 | |
| 131 def _CollectTearDown(self): | |
| 132 if self._GetTestStatus(self._test_run_id) != self.COMPLETE: | |
| 133 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
| 134 logging.WARNING): | |
| 135 test_abort_res = appurify_sanitized.api.tests_abort( | |
| 136 self._env.token, self._test_run_id, reason='Test runner exiting.') | |
| 137 remote_device_helper.TestHttpResponse(test_abort_res, | |
| 138 'Unable to abort test.') | |
| 139 if self._results_temp_dir: | |
| 140 shutil.rmtree(self._results_temp_dir) | |
| 141 | |
| 142 def __enter__(self): | |
| 143 """Set up the test run when used as a context manager.""" | |
| 144 self.SetUp() | |
| 145 return self | |
| 146 | |
| 147 def __exit__(self, exc_type, exc_val, exc_tb): | |
| 148 """Tear down the test run when used as a context manager.""" | |
| 149 self.TearDown() | |
| 150 | |
| 151 def DumpTo(self, persisted_data): | |
| 152 test_run_data = { | |
| 153 self._TEST_RUN_ID_KEY: self._test_run_id, | |
| 154 } | |
| 155 persisted_data[self._TEST_RUN_KEY] = test_run_data | |
| 156 | |
| 157 def LoadFrom(self, persisted_data): | |
| 158 test_run_data = persisted_data[self._TEST_RUN_KEY] | |
| 159 self._test_run_id = test_run_data[self._TEST_RUN_ID_KEY] | |
| 160 | |
| 161 def _ParseTestResults(self): | |
| 162 raise NotImplementedError | |
| 163 | |
| 164 def _GetTestByName(self, test_name): | |
| 165 """Gets test_id for specific test. | |
| 166 | |
| 167 Args: | |
| 168 test_name: Test to find the ID of. | |
| 169 """ | |
| 170 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
| 171 logging.WARNING): | |
| 172 test_list_res = appurify_sanitized.api.tests_list(self._env.token) | |
| 173 remote_device_helper.TestHttpResponse(test_list_res, | |
| 174 'Unable to get tests list.') | |
| 175 for test in test_list_res.json()['response']: | |
| 176 if test['test_type'] == test_name: | |
| 177 return test['test_id'] | |
| 178 raise remote_device_helper.RemoteDeviceError( | |
| 179 'No test found with name %s' % (test_name)) | |
| 180 | |
| 181 def _DownloadTestResults(self, results_path): | |
| 182 """Download the test results from remote device service. | |
| 183 | |
| 184 Downloads results in temporary location, and then copys results | |
| 185 to results_path if results_path is not set to None. | |
| 186 | |
| 187 Args: | |
| 188 results_path: Path to download appurify results zipfile. | |
| 189 | |
| 190 Returns: | |
| 191 Path to downloaded file. | |
| 192 """ | |
| 193 | |
| 194 if self._results_temp_dir is None: | |
| 195 self._results_temp_dir = tempfile.mkdtemp() | |
| 196 logging.info('Downloading results to %s.', self._results_temp_dir) | |
| 197 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
| 198 logging.WARNING): | |
| 199 appurify_sanitized.utils.wget(self._results['results']['url'], | |
| 200 self._results_temp_dir + '/results') | |
| 201 if results_path: | |
| 202 logging.info('Copying results to %s', results_path) | |
| 203 if not os.path.exists(os.path.dirname(results_path)): | |
| 204 os.makedirs(os.path.dirname(results_path)) | |
| 205 shutil.copy(self._results_temp_dir + '/results', results_path) | |
| 206 return self._results_temp_dir + '/results' | |
| 207 | |
| 208 def _GetTestStatus(self, test_run_id): | |
| 209 """Checks the state of the test, and sets self._results | |
| 210 | |
| 211 Args: | |
| 212 test_run_id: Id of test on on remote service. | |
| 213 """ | |
| 214 | |
| 215 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
| 216 logging.WARNING): | |
| 217 test_check_res = appurify_sanitized.api.tests_check_result( | |
| 218 self._env.token, test_run_id) | |
| 219 remote_device_helper.TestHttpResponse(test_check_res, | |
| 220 'Unable to get test status.') | |
| 221 self._results = test_check_res.json()['response'] | |
| 222 return self._results['status'] | |
| 223 | |
| 224 def _AmInstrumentTestSetup(self, app_path, test_path, runner_package, | |
| 225 environment_variables, extra_apks=None): | |
| 226 config = {'runner': runner_package} | |
| 227 if environment_variables: | |
| 228 config['environment_vars'] = ','.join( | |
| 229 '%s=%s' % (k, v) for k, v in environment_variables.iteritems()) | |
| 230 | |
| 231 self._app_id = self._UploadAppToDevice(app_path) | |
| 232 | |
| 233 # TODO(agrieve): If AMP is ever ressurected, this needs to be changed to put | |
| 234 # test files under /sdcard/gtestdata. http://crbug.com/607169 | |
| 235 data_deps = self._test_instance.GetDataDependencies() | |
| 236 if data_deps: | |
| 237 with tempfile.NamedTemporaryFile(suffix='.zip') as test_with_deps: | |
| 238 sdcard_files = [] | |
| 239 additional_apks = [] | |
| 240 host_test = os.path.basename(test_path) | |
| 241 with zipfile.ZipFile(test_with_deps.name, 'w') as zip_file: | |
| 242 zip_file.write(test_path, host_test, zipfile.ZIP_DEFLATED) | |
| 243 for h, _ in data_deps: | |
| 244 if os.path.isdir(h): | |
| 245 zip_utils.WriteToZipFile(zip_file, h, '.') | |
| 246 sdcard_files.extend(os.listdir(h)) | |
| 247 else: | |
| 248 zip_utils.WriteToZipFile(zip_file, h, os.path.basename(h)) | |
| 249 sdcard_files.append(os.path.basename(h)) | |
| 250 for a in extra_apks or (): | |
| 251 zip_utils.WriteToZipFile(zip_file, a, os.path.basename(a)) | |
| 252 additional_apks.append(os.path.basename(a)) | |
| 253 | |
| 254 config['sdcard_files'] = ','.join(sdcard_files) | |
| 255 config['host_test'] = host_test | |
| 256 if additional_apks: | |
| 257 config['additional_apks'] = ','.join(additional_apks) | |
| 258 self._test_id = self._UploadTestToDevice( | |
| 259 'robotium', test_with_deps.name, app_id=self._app_id) | |
| 260 else: | |
| 261 self._test_id = self._UploadTestToDevice('robotium', test_path) | |
| 262 | |
| 263 logging.info('Setting config: %s', config) | |
| 264 appurify_configs = {} | |
| 265 if self._env.network_config: | |
| 266 appurify_configs['network'] = self._env.network_config | |
| 267 self._SetTestConfig('robotium', config, **appurify_configs) | |
| 268 | |
| 269 def _UploadAppToDevice(self, app_path): | |
| 270 """Upload app to device.""" | |
| 271 logging.info('Uploading %s to remote service as %s.', app_path, | |
| 272 self._test_instance.suite) | |
| 273 with open(app_path, 'rb') as apk_src: | |
| 274 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
| 275 logging.WARNING): | |
| 276 upload_results = appurify_sanitized.api.apps_upload( | |
| 277 self._env.token, apk_src, 'raw', name=self._test_instance.suite) | |
| 278 remote_device_helper.TestHttpResponse( | |
| 279 upload_results, 'Unable to upload %s.' % app_path) | |
| 280 return upload_results.json()['response']['app_id'] | |
| 281 | |
| 282 def _UploadTestToDevice(self, test_type, test_path, app_id=None): | |
| 283 """Upload test to device | |
| 284 Args: | |
| 285 test_type: Type of test that is being uploaded. Ex. uirobot, gtest.. | |
| 286 """ | |
| 287 logging.info('Uploading %s to remote service.', test_path) | |
| 288 with open(test_path, 'rb') as test_src: | |
| 289 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
| 290 logging.WARNING): | |
| 291 upload_results = appurify_sanitized.api.tests_upload( | |
| 292 self._env.token, test_src, 'raw', test_type, app_id=app_id) | |
| 293 remote_device_helper.TestHttpResponse(upload_results, | |
| 294 'Unable to upload %s.' % test_path) | |
| 295 return upload_results.json()['response']['test_id'] | |
| 296 | |
| 297 def _SetTestConfig(self, runner_type, runner_configs, | |
| 298 network=appurify_constants.NETWORK.WIFI_1_BAR, | |
| 299 pcap=0, profiler=0, videocapture=0): | |
| 300 """Generates and uploads config file for test. | |
| 301 Args: | |
| 302 runner_configs: Configs specific to the runner you are using. | |
| 303 network: Config to specify the network environment the devices running | |
| 304 the tests will be in. | |
| 305 pcap: Option to set the recording the of network traffic from the device. | |
| 306 profiler: Option to set the recording of CPU, memory, and network | |
| 307 transfer usage in the tests. | |
| 308 videocapture: Option to set video capture during the tests. | |
| 309 | |
| 310 """ | |
| 311 logging.info('Generating config file for test.') | |
| 312 with tempfile.TemporaryFile() as config: | |
| 313 config_data = [ | |
| 314 '[appurify]', | |
| 315 'network=%s' % network, | |
| 316 'pcap=%s' % pcap, | |
| 317 'profiler=%s' % profiler, | |
| 318 'videocapture=%s' % videocapture, | |
| 319 '[%s]' % runner_type | |
| 320 ] | |
| 321 config_data.extend( | |
| 322 '%s=%s' % (k, v) for k, v in runner_configs.iteritems()) | |
| 323 config.write(''.join('%s\n' % l for l in config_data)) | |
| 324 config.flush() | |
| 325 config.seek(0) | |
| 326 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
| 327 logging.WARNING): | |
| 328 config_response = appurify_sanitized.api.config_upload( | |
| 329 self._env.token, config, self._test_id) | |
| 330 remote_device_helper.TestHttpResponse( | |
| 331 config_response, 'Unable to upload test config.') | |
| 332 | |
| 333 def _LogLogcat(self, level=logging.CRITICAL): | |
| 334 """Prints out logcat downloaded from remote service. | |
| 335 Args: | |
| 336 level: logging level to print at. | |
| 337 | |
| 338 Raises: | |
| 339 KeyError: If appurify_results/logcat.txt file cannot be found in | |
| 340 downloaded zip. | |
| 341 """ | |
| 342 zip_file = self._DownloadTestResults(None) | |
| 343 with zipfile.ZipFile(zip_file) as z: | |
| 344 try: | |
| 345 logcat = z.read('appurify_results/logcat.txt') | |
| 346 printable_logcat = ''.join(c for c in logcat if c in string.printable) | |
| 347 for line in printable_logcat.splitlines(): | |
| 348 logging.log(level, line) | |
| 349 except KeyError: | |
| 350 logging.error('No logcat found.') | |
| 351 | |
| 352 def _LogAdbTraceLog(self): | |
| 353 zip_file = self._DownloadTestResults(None) | |
| 354 with zipfile.ZipFile(zip_file) as z: | |
| 355 adb_trace_log = z.read('adb_trace.log') | |
| 356 for line in adb_trace_log.splitlines(): | |
| 357 logging.critical(line) | |
| 358 | |
| 359 def _DidDeviceGoOffline(self): | |
| 360 zip_file = self._DownloadTestResults(None) | |
| 361 with zipfile.ZipFile(zip_file) as z: | |
| 362 adb_trace_log = z.read('adb_trace.log') | |
| 363 if any(_DEVICE_OFFLINE_RE.search(l) for l in adb_trace_log.splitlines()): | |
| 364 return True | |
| 365 return False | |
| 366 | |
| 367 def _DetectPlatformErrors(self, results): | |
| 368 if not self._results['results']['pass']: | |
| 369 crash_msg = None | |
| 370 for line in self._results['results']['output'].splitlines(): | |
| 371 m = _LONG_MSG_RE.search(line) | |
| 372 if m: | |
| 373 crash_msg = m.group(1) | |
| 374 break | |
| 375 m = _SHORT_MSG_RE.search(line) | |
| 376 if m: | |
| 377 crash_msg = m.group(1) | |
| 378 if crash_msg: | |
| 379 self._LogLogcat() | |
| 380 results.AddResult(base_test_result.BaseTestResult( | |
| 381 crash_msg, base_test_result.ResultType.CRASH)) | |
| 382 elif self._DidDeviceGoOffline(): | |
| 383 self._LogLogcat() | |
| 384 self._LogAdbTraceLog() | |
| 385 raise remote_device_helper.RemoteDeviceError( | |
| 386 'Remote service unable to reach device.', is_infra_error=True) | |
| 387 else: | |
| 388 # Remote service is reporting a failure, but no failure in results obj. | |
| 389 if results.DidRunPass(): | |
| 390 results.AddResult(base_test_result.BaseTestResult( | |
| 391 'Remote service detected error.', | |
| 392 base_test_result.ResultType.UNKNOWN)) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2488453002:
[android] Remove pylib/{remote,uirobot}. (Closed)
