OLD | NEW |
---|---|
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Class for running instrumentation tests on a single device.""" | 5 """Class for running instrumentation tests on a single device.""" |
6 | 6 |
7 import logging | 7 import logging |
8 import os | 8 import os |
9 import re | 9 import re |
10 import shutil | 10 import shutil |
(...skipping 29 matching lines...) Expand all Loading... | |
40 test_files += [ | 40 test_files += [ |
41 'net/data/ssl/certificates/', | 41 'net/data/ssl/certificates/', |
42 ] | 42 ] |
43 return test_files | 43 return test_files |
44 | 44 |
45 | 45 |
46 class TestRunner(base_test_runner.BaseTestRunner): | 46 class TestRunner(base_test_runner.BaseTestRunner): |
47 """Responsible for running a series of tests connected to a single device.""" | 47 """Responsible for running a series of tests connected to a single device.""" |
48 | 48 |
49 _DEVICE_DATA_DIR = 'chrome/test/data' | 49 _DEVICE_DATA_DIR = 'chrome/test/data' |
50 _DEVICE_COVERAGE_DIR = 'chrome/test/coverage' | |
50 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' | 51 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' |
51 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + | 52 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + |
52 '/chrome-profile*') | 53 '/chrome-profile*') |
53 _DEVICE_HAS_TEST_FILES = {} | 54 _DEVICE_HAS_TEST_FILES = {} |
54 | 55 |
55 def __init__(self, build_type, test_data, install_apk, save_perf_json, | 56 def __init__(self, build_type, test_data, install_apk, save_perf_json, |
56 screenshot_failures, tool, wait_for_debugger, disable_assertions, | 57 screenshot_failures, tool, wait_for_debugger, disable_assertions, |
57 push_deps, cleanup_test_files, device, shard_index, test_pkg, | 58 push_deps, cleanup_test_files, device, shard_index, test_pkg, |
58 ports_to_forward): | 59 ports_to_forward, coverage, coverage_dir): |
59 """Create a new TestRunner. | 60 """Create a new TestRunner. |
60 | 61 |
61 Args: | 62 Args: |
62 build_type: 'Release' or 'Debug'. | 63 build_type: 'Release' or 'Debug'. |
63 test_data: Location of the test data. | 64 test_data: Location of the test data. |
64 install_apk: Re-installs the apk if opted. | 65 install_apk: Re-installs the apk if opted. |
65 save_perf_json: Whether or not to save the JSON file from UI perf tests. | 66 save_perf_json: Whether or not to save the JSON file from UI perf tests. |
66 screenshot_failures: Take a screenshot for a test failure | 67 screenshot_failures: Take a screenshot for a test failure |
67 tool: Name of the Valgrind tool. | 68 tool: Name of the Valgrind tool. |
68 wait_for_debugger: Blocks until the debugger is connected. | 69 wait_for_debugger: Blocks until the debugger is connected. |
69 disable_assertions: Whether to disable java assertions on the device. | 70 disable_assertions: Whether to disable java assertions on the device. |
70 push_deps: If True, push all dependencies to the device. | 71 push_deps: If True, push all dependencies to the device. |
71 cleanup_test_files: Whether or not to cleanup test files on device. | 72 cleanup_test_files: Whether or not to cleanup test files on device. |
72 device: Attached android device. | 73 device: Attached android device. |
73 shard_index: Shard index. | 74 shard_index: Shard index. |
74 test_pkg: A TestPackage object. | 75 test_pkg: A TestPackage object. |
75 ports_to_forward: A list of port numbers for which to set up forwarders. | 76 ports_to_forward: A list of port numbers for which to set up forwarders. |
76 Can be optionally requested by a test case. | 77 Can be optionally requested by a test case. |
78 coverage: If True, instrument with EMMA coverage tool. | |
79 coverage_dir: Directory to pull all EMMA coverage files into, or None if | |
80 coverage is not being used. | |
77 """ | 81 """ |
78 super(TestRunner, self).__init__(device, tool, build_type, push_deps, | 82 super(TestRunner, self).__init__(device, tool, build_type, push_deps, |
79 cleanup_test_files) | 83 cleanup_test_files) |
80 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index | 84 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index |
81 | 85 |
82 self.build_type = build_type | 86 self.build_type = build_type |
83 self.test_data = test_data | 87 self.test_data = test_data |
84 self.save_perf_json = save_perf_json | 88 self.save_perf_json = save_perf_json |
85 self.screenshot_failures = screenshot_failures | 89 self.screenshot_failures = screenshot_failures |
86 self.wait_for_debugger = wait_for_debugger | 90 self.wait_for_debugger = wait_for_debugger |
87 self.disable_assertions = disable_assertions | 91 self.disable_assertions = disable_assertions |
88 self.test_pkg = test_pkg | 92 self.test_pkg = test_pkg |
89 self.ports_to_forward = ports_to_forward | 93 self.ports_to_forward = ports_to_forward |
90 self.install_apk = install_apk | 94 self.install_apk = install_apk |
95 self.coverage = coverage | |
96 self.coverage_dir = coverage_dir | |
91 | 97 |
92 #override | 98 #override |
93 def InstallTestPackage(self): | 99 def InstallTestPackage(self): |
94 if self.install_apk: | 100 if self.install_apk: |
95 self.test_pkg.Install(self.adb) | 101 self.test_pkg.Install(self.adb) |
96 | 102 |
97 #override | 103 #override |
98 def PushDataDeps(self): | 104 def PushDataDeps(self): |
99 # TODO(frankf): Implement a general approach for copying/installing | 105 # TODO(frankf): Implement a general approach for copying/installing |
100 # once across test runners. | 106 # once across test runners. |
(...skipping 18 matching lines...) Expand all Loading... | |
119 dst_layer = dst_src[0] | 125 dst_layer = dst_src[0] |
120 host_src = dst_src[1] | 126 host_src = dst_src[1] |
121 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src | 127 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src |
122 if os.path.exists(host_test_files_path): | 128 if os.path.exists(host_test_files_path): |
123 self.adb.PushIfNeeded(host_test_files_path, | 129 self.adb.PushIfNeeded(host_test_files_path, |
124 self.adb.GetExternalStorage() + '/' + | 130 self.adb.GetExternalStorage() + '/' + |
125 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) | 131 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) |
126 self.tool.CopyFiles() | 132 self.tool.CopyFiles() |
127 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True | 133 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True |
128 | 134 |
129 def _GetInstrumentationArgs(self): | 135 def _GetInstrumentationArgs(self, test): |
frankf
2013/07/25 18:53:44
where test used?
gkanwar1
2013/07/25 19:06:38
Oops, this a remnant of something else I was doing
| |
130 ret = {} | 136 ret = {} |
131 if self.wait_for_debugger: | 137 if self.wait_for_debugger: |
132 ret['debug'] = 'true' | 138 ret['debug'] = 'true' |
139 if self.coverage: | |
140 ret['coverage'] = 'true' | |
141 ret['coverageFile'] = self.coverage_file | |
142 | |
133 return ret | 143 return ret |
134 | 144 |
135 def _TakeScreenshot(self, test): | 145 def _TakeScreenshot(self, test): |
136 """Takes a screenshot from the device.""" | 146 """Takes a screenshot from the device.""" |
137 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') | 147 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') |
138 logging.info('Taking screenshot named %s', screenshot_name) | 148 logging.info('Taking screenshot named %s', screenshot_name) |
139 self.adb.TakeScreenshot(screenshot_name) | 149 self.adb.TakeScreenshot(screenshot_name) |
140 | 150 |
141 def SetUp(self): | 151 def SetUp(self): |
142 """Sets up the test harness and device before all tests are run.""" | 152 """Sets up the test harness and device before all tests are run.""" |
(...skipping 24 matching lines...) Expand all Loading... | |
167 Args: | 177 Args: |
168 test: The name of the test that will be run. | 178 test: The name of the test that will be run. |
169 """ | 179 """ |
170 self.SetupPerfMonitoringIfNeeded(test) | 180 self.SetupPerfMonitoringIfNeeded(test) |
171 self._SetupIndividualTestTimeoutScale(test) | 181 self._SetupIndividualTestTimeoutScale(test) |
172 self.tool.SetupEnvironment() | 182 self.tool.SetupEnvironment() |
173 | 183 |
174 # Make sure the forwarder is still running. | 184 # Make sure the forwarder is still running. |
175 self.RestartHttpServerForwarderIfNecessary() | 185 self.RestartHttpServerForwarderIfNecessary() |
176 | 186 |
187 if self.coverage: | |
188 self.coverage_basename = test + '.ec' | |
frankf
2013/07/25 18:53:44
Use string formatter: '%s.ec' % test
also below
gkanwar1
2013/07/25 19:06:38
Done. Fixed in several other places in this file a
| |
189 self.coverage_file = (self.adb.GetExternalStorage() + '/' + | |
frankf
2013/07/25 18:53:44
-> coverage_device_file
gkanwar1
2013/07/25 19:06:38
Done.
| |
190 TestRunner._DEVICE_COVERAGE_DIR + '/' + | |
191 self.coverage_basename) | |
frankf
2013/07/25 18:53:44
define coverage_host_file here
gkanwar1
2013/07/25 19:06:38
Done.
| |
192 | |
177 def _IsPerfTest(self, test): | 193 def _IsPerfTest(self, test): |
178 """Determines whether a test is a performance test. | 194 """Determines whether a test is a performance test. |
179 | 195 |
180 Args: | 196 Args: |
181 test: The name of the test to be checked. | 197 test: The name of the test to be checked. |
182 | 198 |
183 Returns: | 199 Returns: |
184 Whether the test is annotated as a performance test. | 200 Whether the test is annotated as a performance test. |
185 """ | 201 """ |
186 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test) | 202 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test) |
(...skipping 22 matching lines...) Expand all Loading... | |
209 """ | 225 """ |
210 | 226 |
211 self.tool.CleanUpEnvironment() | 227 self.tool.CleanUpEnvironment() |
212 | 228 |
213 # The logic below relies on the test passing. | 229 # The logic below relies on the test passing. |
214 if not raw_result or raw_result.GetStatusCode(): | 230 if not raw_result or raw_result.GetStatusCode(): |
215 return | 231 return |
216 | 232 |
217 self.TearDownPerfMonitoring(test) | 233 self.TearDownPerfMonitoring(test) |
218 | 234 |
235 if self.coverage: | |
236 self.adb.Adb().Pull(self.coverage_file, os.path.join( | |
237 self.coverage_dir, self.coverage_basename)) | |
238 self.adb.RunShellCommand('rm %s' % self.coverage_file) | |
239 | |
219 def TearDownPerfMonitoring(self, test): | 240 def TearDownPerfMonitoring(self, test): |
220 """Cleans up performance monitoring if the specified test required it. | 241 """Cleans up performance monitoring if the specified test required it. |
221 | 242 |
222 Args: | 243 Args: |
223 test: The name of the test that was just run. | 244 test: The name of the test that was just run. |
224 Raises: | 245 Raises: |
225 Exception: if there's anything wrong with the perf data. | 246 Exception: if there's anything wrong with the perf data. |
226 """ | 247 """ |
227 if not self._IsPerfTest(test): | 248 if not self._IsPerfTest(test): |
228 return | 249 return |
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
301 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations: | 322 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations: |
302 return 5 * 60 | 323 return 5 * 60 |
303 if 'MediumTest' in annotations: | 324 if 'MediumTest' in annotations: |
304 return 3 * 60 | 325 return 3 * 60 |
305 return 1 * 60 | 326 return 1 * 60 |
306 | 327 |
307 def _RunTest(self, test, timeout): | 328 def _RunTest(self, test, timeout): |
308 try: | 329 try: |
309 return self.adb.RunInstrumentationTest( | 330 return self.adb.RunInstrumentationTest( |
310 test, self.test_pkg.GetPackageName(), | 331 test, self.test_pkg.GetPackageName(), |
311 self._GetInstrumentationArgs(), timeout) | 332 self._GetInstrumentationArgs(test), timeout) |
312 except android_commands.errors.WaitForResponseTimedOutError: | 333 except android_commands.errors.WaitForResponseTimedOutError: |
313 logging.info('Ran the test with timeout of %ds.' % timeout) | 334 logging.info('Ran the test with timeout of %ds.' % timeout) |
314 raise | 335 raise |
315 | 336 |
316 #override | 337 #override |
317 def RunTest(self, test): | 338 def RunTest(self, test): |
318 raw_result = None | 339 raw_result = None |
319 start_date_ms = None | 340 start_date_ms = None |
320 results = base_test_result.TestRunResults() | 341 results = base_test_result.TestRunResults() |
321 timeout=(self._GetIndividualTestTimeoutSecs(test) * | 342 timeout=(self._GetIndividualTestTimeoutSecs(test) * |
(...skipping 30 matching lines...) Expand all Loading... | |
352 duration_ms = 0 | 373 duration_ms = 0 |
353 message = str(e) | 374 message = str(e) |
354 if not message: | 375 if not message: |
355 message = 'No information.' | 376 message = 'No information.' |
356 results.AddResult(test_result.InstrumentationTestResult( | 377 results.AddResult(test_result.InstrumentationTestResult( |
357 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, | 378 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, |
358 log=message)) | 379 log=message)) |
359 raw_result = None | 380 raw_result = None |
360 self.TestTeardown(test, raw_result) | 381 self.TestTeardown(test, raw_result) |
361 return (results, None if results.DidRunPass() else test) | 382 return (results, None if results.DidRunPass() else test) |
OLD | NEW |