| OLD | NEW |
| 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Class for running instrumentation tests on a single device.""" | 5 """Class for running instrumentation tests on a single device.""" |
| 6 | 6 |
| 7 import logging | 7 import logging |
| 8 import os | 8 import os |
| 9 import re | 9 import re |
| 10 import shutil | |
| 11 import sys | |
| 12 import time | 10 import time |
| 13 | 11 |
| 14 from pylib import android_commands | 12 from pylib import android_commands |
| 15 from pylib import cmd_helper | |
| 16 from pylib import constants | 13 from pylib import constants |
| 17 from pylib import json_perf_parser | 14 from pylib import json_perf_parser |
| 18 from pylib import perf_tests_helper | 15 from pylib import perf_tests_helper |
| 19 from pylib import valgrind_tools | 16 from pylib import valgrind_tools |
| 20 from pylib.base import base_test_result | 17 from pylib.base import base_test_result |
| 21 from pylib.base import base_test_runner | 18 from pylib.base import base_test_runner |
| 22 | 19 |
| 23 import test_result | 20 import test_result |
| 24 | 21 |
| 25 | 22 |
| (...skipping 19 matching lines...) Expand all Loading... |
| 45 | 42 |
| 46 class TestRunner(base_test_runner.BaseTestRunner): | 43 class TestRunner(base_test_runner.BaseTestRunner): |
| 47 """Responsible for running a series of tests connected to a single device.""" | 44 """Responsible for running a series of tests connected to a single device.""" |
| 48 | 45 |
| 49 _DEVICE_DATA_DIR = 'chrome/test/data' | 46 _DEVICE_DATA_DIR = 'chrome/test/data' |
| 50 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' | 47 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' |
| 51 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + | 48 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + |
| 52 '/chrome-profile*') | 49 '/chrome-profile*') |
| 53 _DEVICE_HAS_TEST_FILES = {} | 50 _DEVICE_HAS_TEST_FILES = {} |
| 54 | 51 |
| 55 def __init__(self, build_type, test_data, install_apk, save_perf_json, | 52 def __init__(self, test_options, device, shard_index, test_pkg, |
| 56 screenshot_failures, tool, wait_for_debugger, disable_assertions, | |
| 57 push_deps, cleanup_test_files, device, shard_index, test_pkg, | |
| 58 ports_to_forward): | 53 ports_to_forward): |
| 59 """Create a new TestRunner. | 54 """Create a new TestRunner. |
| 60 | 55 |
| 61 Args: | 56 Args: |
| 62 build_type: 'Release' or 'Debug'. | 57 test_options: An InstrumentationOptions object containing the set of |
| 63 test_data: Location of the test data. | 58 options relevant to running instrumentation tests. |
| 64 install_apk: Re-installs the apk if opted. | |
| 65 save_perf_json: Whether or not to save the JSON file from UI perf tests. | |
| 66 screenshot_failures: Take a screenshot for a test failure | |
| 67 tool: Name of the Valgrind tool. | |
| 68 wait_for_debugger: Blocks until the debugger is connected. | |
| 69 disable_assertions: Whether to disable java assertions on the device. | |
| 70 push_deps: If True, push all dependencies to the device. | |
| 71 cleanup_test_files: Whether or not to cleanup test files on device. | |
| 72 device: Attached android device. | 59 device: Attached android device. |
| 73 shard_index: Shard index. | 60 shard_index: Shard index. |
| 74 test_pkg: A TestPackage object. | 61 test_pkg: A TestPackage object. |
| 75 ports_to_forward: A list of port numbers for which to set up forwarders. | 62 ports_to_forward: A list of port numbers for which to set up forwarders. |
| 76 Can be optionally requested by a test case. | 63 Can be optionally requested by a test case. |
| 77 """ | 64 """ |
| 78 super(TestRunner, self).__init__(device, tool, build_type, push_deps, | 65 super(TestRunner, self).__init__(device, test_options.tool, |
| 79 cleanup_test_files) | 66 test_options.build_type, |
| 67 test_options.push_deps, |
| 68 test_options.cleanup_test_files) |
| 80 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index | 69 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index |
| 81 | 70 |
| 82 self.build_type = build_type | 71 self.options = test_options |
| 83 self.test_data = test_data | |
| 84 self.save_perf_json = save_perf_json | |
| 85 self.screenshot_failures = screenshot_failures | |
| 86 self.wait_for_debugger = wait_for_debugger | |
| 87 self.disable_assertions = disable_assertions | |
| 88 self.test_pkg = test_pkg | 72 self.test_pkg = test_pkg |
| 89 self.ports_to_forward = ports_to_forward | 73 self.ports_to_forward = ports_to_forward |
| 90 self.install_apk = install_apk | |
| 91 | 74 |
| 92 #override | 75 #override |
| 93 def InstallTestPackage(self): | 76 def InstallTestPackage(self): |
| 94 if self.install_apk: | 77 if self.options.install_apk: |
| 95 self.test_pkg.Install(self.adb) | 78 self.test_pkg.Install(self.adb) |
| 96 | 79 |
| 97 #override | 80 #override |
| 98 def PushDataDeps(self): | 81 def PushDataDeps(self): |
| 99 # TODO(frankf): Implement a general approach for copying/installing | 82 # TODO(frankf): Implement a general approach for copying/installing |
| 100 # once across test runners. | 83 # once across test runners. |
| 101 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): | 84 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): |
| 102 logging.warning('Already copied test files to device %s, skipping.', | 85 logging.warning('Already copied test files to device %s, skipping.', |
| 103 self.device) | 86 self.device) |
| 104 return | 87 return |
| 105 | 88 |
| 106 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName()) | 89 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName()) |
| 107 if test_data: | 90 if test_data: |
| 108 # Make sure SD card is ready. | 91 # Make sure SD card is ready. |
| 109 self.adb.WaitForSdCardReady(20) | 92 self.adb.WaitForSdCardReady(20) |
| 110 for p in test_data: | 93 for p in test_data: |
| 111 self.adb.PushIfNeeded( | 94 self.adb.PushIfNeeded( |
| 112 os.path.join(constants.DIR_SOURCE_ROOT, p), | 95 os.path.join(constants.DIR_SOURCE_ROOT, p), |
| 113 os.path.join(self.adb.GetExternalStorage(), p)) | 96 os.path.join(self.adb.GetExternalStorage(), p)) |
| 114 | 97 |
| 115 # TODO(frankf): Specify test data in this file as opposed to passing | 98 # TODO(frankf): Specify test data in this file as opposed to passing |
| 116 # as command-line. | 99 # as command-line. |
| 117 for dest_host_pair in self.test_data: | 100 for dest_host_pair in self.options.test_data: |
| 118 dst_src = dest_host_pair.split(':',1) | 101 dst_src = dest_host_pair.split(':',1) |
| 119 dst_layer = dst_src[0] | 102 dst_layer = dst_src[0] |
| 120 host_src = dst_src[1] | 103 host_src = dst_src[1] |
| 121 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src | 104 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src |
| 122 if os.path.exists(host_test_files_path): | 105 if os.path.exists(host_test_files_path): |
| 123 self.adb.PushIfNeeded(host_test_files_path, | 106 self.adb.PushIfNeeded(host_test_files_path, |
| 124 self.adb.GetExternalStorage() + '/' + | 107 self.adb.GetExternalStorage() + '/' + |
| 125 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) | 108 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) |
| 126 self.tool.CopyFiles() | 109 self.tool.CopyFiles() |
| 127 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True | 110 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True |
| 128 | 111 |
| 129 def _GetInstrumentationArgs(self): | 112 def _GetInstrumentationArgs(self): |
| 130 ret = {} | 113 ret = {} |
| 131 if self.wait_for_debugger: | 114 if self.options.wait_for_debugger: |
| 132 ret['debug'] = 'true' | 115 ret['debug'] = 'true' |
| 133 return ret | 116 return ret |
| 134 | 117 |
| 135 def _TakeScreenshot(self, test): | 118 def _TakeScreenshot(self, test): |
| 136 """Takes a screenshot from the device.""" | 119 """Takes a screenshot from the device.""" |
| 137 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') | 120 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') |
| 138 logging.info('Taking screenshot named %s', screenshot_name) | 121 logging.info('Taking screenshot named %s', screenshot_name) |
| 139 self.adb.TakeScreenshot(screenshot_name) | 122 self.adb.TakeScreenshot(screenshot_name) |
| 140 | 123 |
| 141 def SetUp(self): | 124 def SetUp(self): |
| 142 """Sets up the test harness and device before all tests are run.""" | 125 """Sets up the test harness and device before all tests are run.""" |
| 143 super(TestRunner, self).SetUp() | 126 super(TestRunner, self).SetUp() |
| 144 if not self.adb.IsRootEnabled(): | 127 if not self.adb.IsRootEnabled(): |
| 145 logging.warning('Unable to enable java asserts for %s, non rooted device', | 128 logging.warning('Unable to enable java asserts for %s, non rooted device', |
| 146 self.device) | 129 self.device) |
| 147 else: | 130 else: |
| 148 if self.adb.SetJavaAssertsEnabled(enable=not self.disable_assertions): | 131 if self.adb.SetJavaAssertsEnabled( |
| 132 enable=not self.options.disable_assertions): |
| 149 self.adb.Reboot(full_reboot=False) | 133 self.adb.Reboot(full_reboot=False) |
| 150 | 134 |
| 151 # We give different default value to launch HTTP server based on shard index | 135 # We give different default value to launch HTTP server based on shard index |
| 152 # because it may have race condition when multiple processes are trying to | 136 # because it may have race condition when multiple processes are trying to |
| 153 # launch lighttpd with same port at same time. | 137 # launch lighttpd with same port at same time. |
| 154 http_server_ports = self.LaunchTestHttpServer( | 138 http_server_ports = self.LaunchTestHttpServer( |
| 155 os.path.join(constants.DIR_SOURCE_ROOT), self._lighttp_port) | 139 os.path.join(constants.DIR_SOURCE_ROOT), self._lighttp_port) |
| 156 if self.ports_to_forward: | 140 if self.ports_to_forward: |
| 157 self.StartForwarder([(port, port) for port in self.ports_to_forward]) | 141 self.StartForwarder([(port, port) for port in self.ports_to_forward]) |
| 158 self.flags.AddFlags(['--enable-test-intents']) | 142 self.flags.AddFlags(['--enable-test-intents']) |
| (...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 244 # Obtain the relevant perf data. The data is dumped to a | 228 # Obtain the relevant perf data. The data is dumped to a |
| 245 # JSON formatted file. | 229 # JSON formatted file. |
| 246 json_string = self.adb.GetProtectedFileContents( | 230 json_string = self.adb.GetProtectedFileContents( |
| 247 '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt') | 231 '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt') |
| 248 | 232 |
| 249 if json_string: | 233 if json_string: |
| 250 json_string = '\n'.join(json_string) | 234 json_string = '\n'.join(json_string) |
| 251 else: | 235 else: |
| 252 raise Exception('Perf file does not exist or is empty') | 236 raise Exception('Perf file does not exist or is empty') |
| 253 | 237 |
| 254 if self.save_perf_json: | 238 if self.options.save_perf_json: |
| 255 json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name | 239 json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name |
| 256 with open(json_local_file, 'w') as f: | 240 with open(json_local_file, 'w') as f: |
| 257 f.write(json_string) | 241 f.write(json_string) |
| 258 logging.info('Saving Perf UI JSON from test ' + | 242 logging.info('Saving Perf UI JSON from test ' + |
| 259 test + ' to ' + json_local_file) | 243 test + ' to ' + json_local_file) |
| 260 | 244 |
| 261 raw_perf_data = regex.group(1).split(';') | 245 raw_perf_data = regex.group(1).split(';') |
| 262 | 246 |
| 263 for raw_perf_set in raw_perf_data: | 247 for raw_perf_set in raw_perf_data: |
| 264 if raw_perf_set: | 248 if raw_perf_set: |
| (...skipping 15 matching lines...) Expand all Loading... |
| 280 | 264 |
| 281 def _GetIndividualTestTimeoutScale(self, test): | 265 def _GetIndividualTestTimeoutScale(self, test): |
| 282 """Returns the timeout scale for the given |test|.""" | 266 """Returns the timeout scale for the given |test|.""" |
| 283 annotations = self.test_pkg.GetTestAnnotations(test) | 267 annotations = self.test_pkg.GetTestAnnotations(test) |
| 284 timeout_scale = 1 | 268 timeout_scale = 1 |
| 285 if 'TimeoutScale' in annotations: | 269 if 'TimeoutScale' in annotations: |
| 286 for annotation in annotations: | 270 for annotation in annotations: |
| 287 scale_match = re.match('TimeoutScale:([0-9]+)', annotation) | 271 scale_match = re.match('TimeoutScale:([0-9]+)', annotation) |
| 288 if scale_match: | 272 if scale_match: |
| 289 timeout_scale = int(scale_match.group(1)) | 273 timeout_scale = int(scale_match.group(1)) |
| 290 if self.wait_for_debugger: | 274 if self.options.wait_for_debugger: |
| 291 timeout_scale *= 100 | 275 timeout_scale *= 100 |
| 292 return timeout_scale | 276 return timeout_scale |
| 293 | 277 |
| 294 def _GetIndividualTestTimeoutSecs(self, test): | 278 def _GetIndividualTestTimeoutSecs(self, test): |
| 295 """Returns the timeout in seconds for the given |test|.""" | 279 """Returns the timeout in seconds for the given |test|.""" |
| 296 annotations = self.test_pkg.GetTestAnnotations(test) | 280 annotations = self.test_pkg.GetTestAnnotations(test) |
| 297 if 'Manual' in annotations: | 281 if 'Manual' in annotations: |
| 298 return 600 * 60 | 282 return 600 * 60 |
| 299 if 'External' in annotations: | 283 if 'External' in annotations: |
| 300 return 10 * 60 | 284 return 10 * 60 |
| (...skipping 23 matching lines...) Expand all Loading... |
| 324 try: | 308 try: |
| 325 self.TestSetup(test) | 309 self.TestSetup(test) |
| 326 start_date_ms = int(time.time()) * 1000 | 310 start_date_ms = int(time.time()) * 1000 |
| 327 raw_result = self._RunTest(test, timeout) | 311 raw_result = self._RunTest(test, timeout) |
| 328 duration_ms = int(time.time()) * 1000 - start_date_ms | 312 duration_ms = int(time.time()) * 1000 - start_date_ms |
| 329 status_code = raw_result.GetStatusCode() | 313 status_code = raw_result.GetStatusCode() |
| 330 if status_code: | 314 if status_code: |
| 331 log = raw_result.GetFailureReason() | 315 log = raw_result.GetFailureReason() |
| 332 if not log: | 316 if not log: |
| 333 log = 'No information.' | 317 log = 'No information.' |
| 334 if self.screenshot_failures or log.find('INJECT_EVENTS perm') >= 0: | 318 if (self.options.screenshot_failures or |
| 319 log.find('INJECT_EVENTS perm') >= 0): |
| 335 self._TakeScreenshot(test) | 320 self._TakeScreenshot(test) |
| 336 result = test_result.InstrumentationTestResult( | 321 result = test_result.InstrumentationTestResult( |
| 337 test, base_test_result.ResultType.FAIL, start_date_ms, duration_ms, | 322 test, base_test_result.ResultType.FAIL, start_date_ms, duration_ms, |
| 338 log=log) | 323 log=log) |
| 339 else: | 324 else: |
| 340 result = test_result.InstrumentationTestResult( | 325 result = test_result.InstrumentationTestResult( |
| 341 test, base_test_result.ResultType.PASS, start_date_ms, duration_ms) | 326 test, base_test_result.ResultType.PASS, start_date_ms, duration_ms) |
| 342 results.AddResult(result) | 327 results.AddResult(result) |
| 343 # Catch exceptions thrown by StartInstrumentation(). | 328 # Catch exceptions thrown by StartInstrumentation(). |
| 344 # See ../../third_party/android/testrunner/adb_interface.py | 329 # See ../../third_party/android/testrunner/adb_interface.py |
| 345 except (android_commands.errors.WaitForResponseTimedOutError, | 330 except (android_commands.errors.WaitForResponseTimedOutError, |
| 346 android_commands.errors.DeviceUnresponsiveError, | 331 android_commands.errors.DeviceUnresponsiveError, |
| 347 android_commands.errors.InstrumentationError), e: | 332 android_commands.errors.InstrumentationError), e: |
| 348 if start_date_ms: | 333 if start_date_ms: |
| 349 duration_ms = int(time.time()) * 1000 - start_date_ms | 334 duration_ms = int(time.time()) * 1000 - start_date_ms |
| 350 else: | 335 else: |
| 351 start_date_ms = int(time.time()) * 1000 | 336 start_date_ms = int(time.time()) * 1000 |
| 352 duration_ms = 0 | 337 duration_ms = 0 |
| 353 message = str(e) | 338 message = str(e) |
| 354 if not message: | 339 if not message: |
| 355 message = 'No information.' | 340 message = 'No information.' |
| 356 results.AddResult(test_result.InstrumentationTestResult( | 341 results.AddResult(test_result.InstrumentationTestResult( |
| 357 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, | 342 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, |
| 358 log=message)) | 343 log=message)) |
| 359 raw_result = None | 344 raw_result = None |
| 360 self.TestTeardown(test, raw_result) | 345 self.TestTeardown(test, raw_result) |
| 361 return (results, None if results.DidRunPass() else test) | 346 return (results, None if results.DidRunPass() else test) |
| OLD | NEW |