OLD | NEW |
---|---|
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Class for running instrumentation tests on a single device.""" | 5 """Class for running instrumentation tests on a single device.""" |
6 | 6 |
7 import logging | 7 import logging |
8 import os | 8 import os |
9 import re | 9 import re |
10 import shutil | 10 import shutil |
(...skipping 29 matching lines...) Expand all Loading... | |
40 test_files += [ | 40 test_files += [ |
41 'net/data/ssl/certificates/', | 41 'net/data/ssl/certificates/', |
42 ] | 42 ] |
43 return test_files | 43 return test_files |
44 | 44 |
45 | 45 |
46 class TestRunner(base_test_runner.BaseTestRunner): | 46 class TestRunner(base_test_runner.BaseTestRunner): |
47 """Responsible for running a series of tests connected to a single device.""" | 47 """Responsible for running a series of tests connected to a single device.""" |
48 | 48 |
49 _DEVICE_DATA_DIR = 'chrome/test/data' | 49 _DEVICE_DATA_DIR = 'chrome/test/data' |
50 _DEVICE_COVERAGE_DIR = 'chrome/test/coverage' | |
50 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' | 51 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' |
51 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + | 52 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + |
52 '/chrome-profile*') | 53 '/chrome-profile*') |
53 _DEVICE_HAS_TEST_FILES = {} | 54 _DEVICE_HAS_TEST_FILES = {} |
54 | 55 |
55 def __init__(self, build_type, test_data, install_apk, save_perf_json, | 56 def __init__(self, build_type, test_data, install_apk, save_perf_json, |
56 screenshot_failures, tool, wait_for_debugger, disable_assertions, | 57 screenshot_failures, tool, wait_for_debugger, disable_assertions, |
57 push_deps, cleanup_test_files, device, shard_index, test_pkg, | 58 push_deps, cleanup_test_files, device, shard_index, test_pkg, |
58 ports_to_forward): | 59 ports_to_forward, coverage_dir): |
59 """Create a new TestRunner. | 60 """Create a new TestRunner. |
60 | 61 |
61 Args: | 62 Args: |
62 build_type: 'Release' or 'Debug'. | 63 build_type: 'Release' or 'Debug'. |
63 test_data: Location of the test data. | 64 test_data: Location of the test data. |
64 install_apk: Re-installs the apk if opted. | 65 install_apk: Re-installs the apk if opted. |
65 save_perf_json: Whether or not to save the JSON file from UI perf tests. | 66 save_perf_json: Whether or not to save the JSON file from UI perf tests. |
66 screenshot_failures: Take a screenshot for a test failure | 67 screenshot_failures: Take a screenshot for a test failure |
67 tool: Name of the Valgrind tool. | 68 tool: Name of the Valgrind tool. |
68 wait_for_debugger: Blocks until the debugger is connected. | 69 wait_for_debugger: Blocks until the debugger is connected. |
69 disable_assertions: Whether to disable java assertions on the device. | 70 disable_assertions: Whether to disable java assertions on the device. |
70 push_deps: If True, push all dependencies to the device. | 71 push_deps: If True, push all dependencies to the device. |
71 cleanup_test_files: Whether or not to cleanup test files on device. | 72 cleanup_test_files: Whether or not to cleanup test files on device. |
72 device: Attached android device. | 73 device: Attached android device. |
73 shard_index: Shard index. | 74 shard_index: Shard index. |
74 test_pkg: A TestPackage object. | 75 test_pkg: A TestPackage object. |
75 ports_to_forward: A list of port numbers for which to set up forwarders. | 76 ports_to_forward: A list of port numbers for which to set up forwarders. |
76 Can be optionally requested by a test case. | 77 Can be optionally requested by a test case. |
78 coverage_dir: Directory to pull all EMMA coverage files into, or None if | |
79 coverage is not being used. | |
77 """ | 80 """ |
78 super(TestRunner, self).__init__(device, tool, build_type, push_deps, | 81 super(TestRunner, self).__init__(device, tool, build_type, push_deps, |
79 cleanup_test_files) | 82 cleanup_test_files) |
80 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index | 83 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index |
81 | 84 |
82 self.build_type = build_type | 85 self.build_type = build_type |
83 self.test_data = test_data | 86 self.test_data = test_data |
84 self.save_perf_json = save_perf_json | 87 self.save_perf_json = save_perf_json |
85 self.screenshot_failures = screenshot_failures | 88 self.screenshot_failures = screenshot_failures |
86 self.wait_for_debugger = wait_for_debugger | 89 self.wait_for_debugger = wait_for_debugger |
87 self.disable_assertions = disable_assertions | 90 self.disable_assertions = disable_assertions |
88 self.test_pkg = test_pkg | 91 self.test_pkg = test_pkg |
89 self.ports_to_forward = ports_to_forward | 92 self.ports_to_forward = ports_to_forward |
90 self.install_apk = install_apk | 93 self.install_apk = install_apk |
94 self.coverage_dir = coverage_dir | |
91 | 95 |
92 #override | 96 #override |
93 def InstallTestPackage(self): | 97 def InstallTestPackage(self): |
94 if self.install_apk: | 98 if self.install_apk: |
95 self.test_pkg.Install(self.adb) | 99 self.test_pkg.Install(self.adb) |
96 | 100 |
97 #override | 101 #override |
98 def PushDataDeps(self): | 102 def PushDataDeps(self): |
99 # TODO(frankf): Implement a general approach for copying/installing | 103 # TODO(frankf): Implement a general approach for copying/installing |
100 # once across test runners. | 104 # once across test runners. |
(...skipping 10 matching lines...) Expand all Loading... | |
111 self.adb.PushIfNeeded( | 115 self.adb.PushIfNeeded( |
112 os.path.join(constants.DIR_SOURCE_ROOT, p), | 116 os.path.join(constants.DIR_SOURCE_ROOT, p), |
113 os.path.join(self.adb.GetExternalStorage(), p)) | 117 os.path.join(self.adb.GetExternalStorage(), p)) |
114 | 118 |
115 # TODO(frankf): Specify test data in this file as opposed to passing | 119 # TODO(frankf): Specify test data in this file as opposed to passing |
116 # as command-line. | 120 # as command-line. |
117 for dest_host_pair in self.test_data: | 121 for dest_host_pair in self.test_data: |
118 dst_src = dest_host_pair.split(':',1) | 122 dst_src = dest_host_pair.split(':',1) |
119 dst_layer = dst_src[0] | 123 dst_layer = dst_src[0] |
120 host_src = dst_src[1] | 124 host_src = dst_src[1] |
121 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src | 125 host_test_files_path = '%s/%s' % (constants.DIR_SOURCE_ROOT, host_src) |
122 if os.path.exists(host_test_files_path): | 126 if os.path.exists(host_test_files_path): |
123 self.adb.PushIfNeeded(host_test_files_path, | 127 self.adb.PushIfNeeded(host_test_files_path, '%s/%s/%s' % ( |
124 self.adb.GetExternalStorage() + '/' + | 128 self.adb.GetExternalStorage(), TestRunner._DEVICE_DATA_DIR, |
125 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) | 129 dst_layer) |
126 self.tool.CopyFiles() | 130 self.tool.CopyFiles() |
127 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True | 131 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True |
128 | 132 |
129 def _GetInstrumentationArgs(self): | 133 def _GetInstrumentationArgs(self): |
130 ret = {} | 134 ret = {} |
131 if self.wait_for_debugger: | 135 if self.wait_for_debugger: |
132 ret['debug'] = 'true' | 136 ret['debug'] = 'true' |
137 if self.coverage_dir: | |
138 ret['coverage'] = 'true' | |
139 ret['coverageFile'] = self.coverage_device_file | |
140 | |
133 return ret | 141 return ret |
134 | 142 |
135 def _TakeScreenshot(self, test): | 143 def _TakeScreenshot(self, test): |
136 """Takes a screenshot from the device.""" | 144 """Takes a screenshot from the device.""" |
137 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') | 145 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, '%s.png' % (test)) |
138 logging.info('Taking screenshot named %s', screenshot_name) | 146 logging.info('Taking screenshot named %s', screenshot_name) |
139 self.adb.TakeScreenshot(screenshot_name) | 147 self.adb.TakeScreenshot(screenshot_name) |
140 | 148 |
141 def SetUp(self): | 149 def SetUp(self): |
142 """Sets up the test harness and device before all tests are run.""" | 150 """Sets up the test harness and device before all tests are run.""" |
143 super(TestRunner, self).SetUp() | 151 super(TestRunner, self).SetUp() |
144 if not self.adb.IsRootEnabled(): | 152 if not self.adb.IsRootEnabled(): |
145 logging.warning('Unable to enable java asserts for %s, non rooted device', | 153 logging.warning('Unable to enable java asserts for %s, non rooted device', |
146 self.device) | 154 self.device) |
147 else: | 155 else: |
(...skipping 19 matching lines...) Expand all Loading... | |
167 Args: | 175 Args: |
168 test: The name of the test that will be run. | 176 test: The name of the test that will be run. |
169 """ | 177 """ |
170 self.SetupPerfMonitoringIfNeeded(test) | 178 self.SetupPerfMonitoringIfNeeded(test) |
171 self._SetupIndividualTestTimeoutScale(test) | 179 self._SetupIndividualTestTimeoutScale(test) |
172 self.tool.SetupEnvironment() | 180 self.tool.SetupEnvironment() |
173 | 181 |
174 # Make sure the forwarder is still running. | 182 # Make sure the forwarder is still running. |
175 self.RestartHttpServerForwarderIfNecessary() | 183 self.RestartHttpServerForwarderIfNecessary() |
176 | 184 |
185 if self.coverage_dir: | |
186 self.coverage_basename = '%s.ec' % (test) | |
frankf
2013/07/25 20:28:19
No need for ()
frankf
2013/07/25 20:28:19
This var is local now
gkanwar1
2013/08/01 02:14:35
Done.
gkanwar1
2013/08/01 02:14:35
Done.
| |
187 self.coverage_device_file = '%s/%s/%s' % (self.adb.GetExternalStorage(), | |
188 TestRunner._DEVICE_COVERAGE_DIR, | |
189 self.coverage_basename) | |
190 self.coverage_host_file = os.path.join( | |
191 self.coverage_dir, self.coverage_basename) | |
192 | |
177 def _IsPerfTest(self, test): | 193 def _IsPerfTest(self, test): |
178 """Determines whether a test is a performance test. | 194 """Determines whether a test is a performance test. |
179 | 195 |
180 Args: | 196 Args: |
181 test: The name of the test to be checked. | 197 test: The name of the test to be checked. |
182 | 198 |
183 Returns: | 199 Returns: |
184 Whether the test is annotated as a performance test. | 200 Whether the test is annotated as a performance test. |
185 """ | 201 """ |
186 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test) | 202 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test) |
(...skipping 22 matching lines...) Expand all Loading... | |
209 """ | 225 """ |
210 | 226 |
211 self.tool.CleanUpEnvironment() | 227 self.tool.CleanUpEnvironment() |
212 | 228 |
213 # The logic below relies on the test passing. | 229 # The logic below relies on the test passing. |
214 if not raw_result or raw_result.GetStatusCode(): | 230 if not raw_result or raw_result.GetStatusCode(): |
215 return | 231 return |
216 | 232 |
217 self.TearDownPerfMonitoring(test) | 233 self.TearDownPerfMonitoring(test) |
218 | 234 |
235 if self.coverage_dir: | |
236 self.adb.Adb().Pull(self.coverage_device_file, self.coverage_host_file) | |
237 self.adb.RunShellCommand('rm %s' % self.coverage_device_file) | |
238 | |
219 def TearDownPerfMonitoring(self, test): | 239 def TearDownPerfMonitoring(self, test): |
220 """Cleans up performance monitoring if the specified test required it. | 240 """Cleans up performance monitoring if the specified test required it. |
221 | 241 |
222 Args: | 242 Args: |
223 test: The name of the test that was just run. | 243 test: The name of the test that was just run. |
224 Raises: | 244 Raises: |
225 Exception: if there's anything wrong with the perf data. | 245 Exception: if there's anything wrong with the perf data. |
226 """ | 246 """ |
227 if not self._IsPerfTest(test): | 247 if not self._IsPerfTest(test): |
228 return | 248 return |
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
301 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations: | 321 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations: |
302 return 5 * 60 | 322 return 5 * 60 |
303 if 'MediumTest' in annotations: | 323 if 'MediumTest' in annotations: |
304 return 3 * 60 | 324 return 3 * 60 |
305 return 1 * 60 | 325 return 1 * 60 |
306 | 326 |
307 def _RunTest(self, test, timeout): | 327 def _RunTest(self, test, timeout): |
308 try: | 328 try: |
309 return self.adb.RunInstrumentationTest( | 329 return self.adb.RunInstrumentationTest( |
310 test, self.test_pkg.GetPackageName(), | 330 test, self.test_pkg.GetPackageName(), |
311 self._GetInstrumentationArgs(), timeout) | 331 self._GetInstrumentationArgs(test), timeout) |
312 except android_commands.errors.WaitForResponseTimedOutError: | 332 except android_commands.errors.WaitForResponseTimedOutError: |
313 logging.info('Ran the test with timeout of %ds.' % timeout) | 333 logging.info('Ran the test with timeout of %ds.' % timeout) |
314 raise | 334 raise |
315 | 335 |
316 #override | 336 #override |
317 def RunTest(self, test): | 337 def RunTest(self, test): |
318 raw_result = None | 338 raw_result = None |
319 start_date_ms = None | 339 start_date_ms = None |
320 results = base_test_result.TestRunResults() | 340 results = base_test_result.TestRunResults() |
321 timeout=(self._GetIndividualTestTimeoutSecs(test) * | 341 timeout=(self._GetIndividualTestTimeoutSecs(test) * |
(...skipping 30 matching lines...) Expand all Loading... | |
352 duration_ms = 0 | 372 duration_ms = 0 |
353 message = str(e) | 373 message = str(e) |
354 if not message: | 374 if not message: |
355 message = 'No information.' | 375 message = 'No information.' |
356 results.AddResult(test_result.InstrumentationTestResult( | 376 results.AddResult(test_result.InstrumentationTestResult( |
357 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, | 377 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, |
358 log=message)) | 378 log=message)) |
359 raw_result = None | 379 raw_result = None |
360 self.TestTeardown(test, raw_result) | 380 self.TestTeardown(test, raw_result) |
361 return (results, None if results.DidRunPass() else test) | 381 return (results, None if results.DidRunPass() else test) |
OLD | NEW |