OLD | NEW |
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Runs the Java tests. See more information on run_instrumentation_tests.py.""" | 5 """Class for running instrumentation tests on a single device.""" |
6 | 6 |
7 import logging | 7 import logging |
8 import os | 8 import os |
9 import re | 9 import re |
10 import shutil | 10 import shutil |
11 import sys | 11 import sys |
12 import time | 12 import time |
13 | 13 |
14 from pylib import android_commands | 14 from pylib import android_commands |
15 from pylib import cmd_helper | 15 from pylib import cmd_helper |
(...skipping 30 matching lines...) Expand all Loading... |
46 | 46 |
47 class TestRunner(base_test_runner.BaseTestRunner): | 47 class TestRunner(base_test_runner.BaseTestRunner): |
48 """Responsible for running a series of tests connected to a single device.""" | 48 """Responsible for running a series of tests connected to a single device.""" |
49 | 49 |
50 _DEVICE_DATA_DIR = 'chrome/test/data' | 50 _DEVICE_DATA_DIR = 'chrome/test/data' |
51 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' | 51 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' |
52 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + | 52 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + |
53 '/chrome-profile*') | 53 '/chrome-profile*') |
54 _DEVICE_HAS_TEST_FILES = {} | 54 _DEVICE_HAS_TEST_FILES = {} |
55 | 55 |
56 def __init__(self, options, device, shard_index, test_pkg, | 56 def __init__(self, options, device, shard_index, test_pkg, ports_to_forward): |
57 ports_to_forward, is_uiautomator_test=False): | |
58 """Create a new TestRunner. | 57 """Create a new TestRunner. |
59 | 58 |
60 Args: | 59 Args: |
61 options: An options object with the following required attributes: | 60 options: An options object with the following required attributes: |
62 - build_type: 'Release' or 'Debug'. | 61 - build_type: 'Release' or 'Debug'. |
63 - install_apk: Re-installs the apk if opted. | 62 - install_apk: Re-installs the apk if opted. |
64 - save_perf_json: Whether or not to save the JSON file from UI perf | 63 - save_perf_json: Whether or not to save the JSON file from UI perf |
65 tests. | 64 tests. |
66 - screenshot_failures: Take a screenshot for a test failure | 65 - screenshot_failures: Take a screenshot for a test failure |
67 - tool: Name of the Valgrind tool. | 66 - tool: Name of the Valgrind tool. |
68 - wait_for_debugger: blocks until the debugger is connected. | 67 - wait_for_debugger: blocks until the debugger is connected. |
69 - disable_assertions: Whether to disable java assertions on the device. | 68 - disable_assertions: Whether to disable java assertions on the device. |
70 device: Attached android device. | 69 device: Attached android device. |
71 shard_index: Shard index. | 70 shard_index: Shard index. |
72 test_pkg: A TestPackage object. | 71 test_pkg: A TestPackage object. |
73 ports_to_forward: A list of port numbers for which to set up forwarders. | 72 ports_to_forward: A list of port numbers for which to set up forwarders. |
74 Can be optionally requested by a test case. | 73 Can be optionally requested by a test case. |
75 is_uiautomator_test: Whether this is a uiautomator test. | |
76 """ | 74 """ |
77 super(TestRunner, self).__init__(device, options.tool, options.build_type) | 75 super(TestRunner, self).__init__(device, options.tool, options.build_type) |
78 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index | 76 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index |
79 | 77 |
80 self.build_type = options.build_type | 78 self.build_type = options.build_type |
81 self.test_data = options.test_data | 79 self.test_data = options.test_data |
82 self.save_perf_json = options.save_perf_json | 80 self.save_perf_json = options.save_perf_json |
83 self.screenshot_failures = options.screenshot_failures | 81 self.screenshot_failures = options.screenshot_failures |
84 self.wait_for_debugger = options.wait_for_debugger | 82 self.wait_for_debugger = options.wait_for_debugger |
85 self.disable_assertions = options.disable_assertions | 83 self.disable_assertions = options.disable_assertions |
86 self.test_pkg = test_pkg | 84 self.test_pkg = test_pkg |
87 self.ports_to_forward = ports_to_forward | 85 self.ports_to_forward = ports_to_forward |
88 self.is_uiautomator_test = is_uiautomator_test | 86 self.install_apk = options.install_apk |
89 if self.is_uiautomator_test: | |
90 self.package_name = options.package_name | |
91 else: | |
92 self.install_apk = options.install_apk | |
93 | |
94 self.forwarder = None | 87 self.forwarder = None |
95 | 88 |
96 #override. | 89 #override. |
97 def PushDependencies(self): | 90 def PushDependencies(self): |
98 # TODO(frankf): Implement a general approach for copying/installing | 91 # TODO(frankf): Implement a general approach for copying/installing |
99 # once across test runners. | 92 # once across test runners. |
100 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): | 93 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): |
101 logging.warning('Already copied test files to device %s, skipping.', | 94 logging.warning('Already copied test files to device %s, skipping.', |
102 self.device) | 95 self.device) |
103 return | 96 return |
104 | 97 |
105 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName()) | 98 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName()) |
106 if test_data: | 99 if test_data: |
107 # Make sure SD card is ready. | 100 # Make sure SD card is ready. |
108 self.adb.WaitForSdCardReady(20) | 101 self.adb.WaitForSdCardReady(20) |
109 for data in test_data: | 102 for data in test_data: |
110 self.CopyTestData([data], self.adb.GetExternalStorage()) | 103 self.CopyTestData([data], self.adb.GetExternalStorage()) |
111 | 104 |
112 # TODO(frankf): Specify test data in this file as opposed to passing | 105 # TODO(frankf): Specify test data in this file as opposed to passing |
113 # as command-line. | 106 # as command-line. |
114 for dest_host_pair in self.test_data: | 107 for dest_host_pair in self.test_data: |
115 dst_src = dest_host_pair.split(':',1) | 108 dst_src = dest_host_pair.split(':',1) |
116 dst_layer = dst_src[0] | 109 dst_layer = dst_src[0] |
117 host_src = dst_src[1] | 110 host_src = dst_src[1] |
118 host_test_files_path = constants.CHROME_DIR + '/' + host_src | 111 host_test_files_path = constants.CHROME_DIR + '/' + host_src |
119 if os.path.exists(host_test_files_path): | 112 if os.path.exists(host_test_files_path): |
120 self.adb.PushIfNeeded(host_test_files_path, | 113 self.adb.PushIfNeeded(host_test_files_path, |
121 self.adb.GetExternalStorage() + '/' + | 114 self.adb.GetExternalStorage() + '/' + |
122 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) | 115 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) |
123 if self.is_uiautomator_test or self.install_apk: | 116 if self.install_apk: |
124 self.test_pkg.Install(self.adb) | 117 self.test_pkg.Install(self.adb) |
125 self.tool.CopyFiles() | 118 self.tool.CopyFiles() |
126 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True | 119 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True |
127 | 120 |
128 def _GetInstrumentationArgs(self): | 121 def _GetInstrumentationArgs(self): |
129 ret = {} | 122 ret = {} |
130 if self.wait_for_debugger: | 123 if self.wait_for_debugger: |
131 ret['debug'] = 'true' | 124 ret['debug'] = 'true' |
132 return ret | 125 return ret |
133 | 126 |
(...skipping 169 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
303 if 'Manual' in annotations: | 296 if 'Manual' in annotations: |
304 return 600 * 60 | 297 return 600 * 60 |
305 if 'External' in annotations: | 298 if 'External' in annotations: |
306 return 10 * 60 | 299 return 10 * 60 |
307 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations: | 300 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations: |
308 return 5 * 60 | 301 return 5 * 60 |
309 if 'MediumTest' in annotations: | 302 if 'MediumTest' in annotations: |
310 return 3 * 60 | 303 return 3 * 60 |
311 return 1 * 60 | 304 return 1 * 60 |
312 | 305 |
313 def _RunUIAutomatorTest(self, test, timeout): | 306 def _RunTest(self, test, timeout): |
314 """Runs a single uiautomator test. | 307 return self.adb.RunInstrumentationTest( |
315 | 308 test, self.test_pkg.GetPackageName(), |
316 Args: | 309 self._GetInstrumentationArgs(), timeout) |
317 test: Test class/method. | |
318 timeout: Timeout time in seconds. | |
319 | |
320 Returns: | |
321 An instance of am_instrument_parser.TestResult object. | |
322 """ | |
323 self.adb.ClearApplicationState(self.package_name) | |
324 if 'Feature:FirstRunExperience' in self.test_pkg.GetTestAnnotations(test): | |
325 self.flags.RemoveFlags(['--disable-fre']) | |
326 else: | |
327 self.flags.AddFlags(['--disable-fre']) | |
328 return self.adb.RunUIAutomatorTest( | |
329 test, self.test_pkg.GetPackageName(), timeout) | |
330 | 310 |
331 #override. | 311 #override. |
332 def RunTest(self, test): | 312 def RunTest(self, test): |
333 raw_result = None | 313 raw_result = None |
334 start_date_ms = None | 314 start_date_ms = None |
335 results = base_test_result.TestRunResults() | 315 results = base_test_result.TestRunResults() |
336 timeout=(self._GetIndividualTestTimeoutSecs(test) * | 316 timeout=(self._GetIndividualTestTimeoutSecs(test) * |
337 self._GetIndividualTestTimeoutScale(test) * | 317 self._GetIndividualTestTimeoutScale(test) * |
338 self.tool.GetTimeoutScale()) | 318 self.tool.GetTimeoutScale()) |
339 try: | 319 try: |
340 self.TestSetup(test) | 320 self.TestSetup(test) |
341 start_date_ms = int(time.time()) * 1000 | 321 start_date_ms = int(time.time()) * 1000 |
342 | 322 raw_result = self._RunTest(test, timeout) |
343 if self.is_uiautomator_test: | |
344 raw_result = self._RunUIAutomatorTest(test, timeout) | |
345 else: | |
346 raw_result = self.adb.RunInstrumentationTest( | |
347 test, self.test_pkg.GetPackageName(), | |
348 self._GetInstrumentationArgs(), timeout) | |
349 | |
350 duration_ms = int(time.time()) * 1000 - start_date_ms | 323 duration_ms = int(time.time()) * 1000 - start_date_ms |
351 status_code = raw_result.GetStatusCode() | 324 status_code = raw_result.GetStatusCode() |
352 if status_code: | 325 if status_code: |
353 log = raw_result.GetFailureReason() | 326 log = raw_result.GetFailureReason() |
354 if not log: | 327 if not log: |
355 log = 'No information.' | 328 log = 'No information.' |
356 if self.screenshot_failures or log.find('INJECT_EVENTS perm') >= 0: | 329 if self.screenshot_failures or log.find('INJECT_EVENTS perm') >= 0: |
357 self._TakeScreenshot(test) | 330 self._TakeScreenshot(test) |
358 result = test_result.InstrumentationTestResult( | 331 result = test_result.InstrumentationTestResult( |
359 test, base_test_result.ResultType.FAIL, start_date_ms, duration_ms, | 332 test, base_test_result.ResultType.FAIL, start_date_ms, duration_ms, |
(...skipping 14 matching lines...) Expand all Loading... |
374 duration_ms = 0 | 347 duration_ms = 0 |
375 message = str(e) | 348 message = str(e) |
376 if not message: | 349 if not message: |
377 message = 'No information.' | 350 message = 'No information.' |
378 results.AddResult(test_result.InstrumentationTestResult( | 351 results.AddResult(test_result.InstrumentationTestResult( |
379 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, | 352 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, |
380 log=message)) | 353 log=message)) |
381 raw_result = None | 354 raw_result = None |
382 self.TestTeardown(test, raw_result) | 355 self.TestTeardown(test, raw_result) |
383 return (results, None if results.DidRunPass() else test) | 356 return (results, None if results.DidRunPass() else test) |
OLD | NEW |