OLD | NEW |
---|---|
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Class for running instrumentation tests on a single device.""" | 5 """Class for running instrumentation tests on a single device.""" |
6 | 6 |
7 import logging | 7 import logging |
8 import os | 8 import os |
9 import re | 9 import re |
10 import shutil | 10 import shutil |
(...skipping 29 matching lines...) Expand all Loading... | |
40 test_files += [ | 40 test_files += [ |
41 'net/data/ssl/certificates/', | 41 'net/data/ssl/certificates/', |
42 ] | 42 ] |
43 return test_files | 43 return test_files |
44 | 44 |
45 | 45 |
46 class TestRunner(base_test_runner.BaseTestRunner): | 46 class TestRunner(base_test_runner.BaseTestRunner): |
47 """Responsible for running a series of tests connected to a single device.""" | 47 """Responsible for running a series of tests connected to a single device.""" |
48 | 48 |
49 _DEVICE_DATA_DIR = 'chrome/test/data' | 49 _DEVICE_DATA_DIR = 'chrome/test/data' |
50 _DEVICE_COVERAGE_DIR = 'chrome/test/coverage' | |
50 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' | 51 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' |
51 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + | 52 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + |
52 '/chrome-profile*') | 53 '/chrome-profile*') |
53 _DEVICE_HAS_TEST_FILES = {} | 54 _DEVICE_HAS_TEST_FILES = {} |
54 | 55 |
55 def __init__(self, build_type, test_data, save_perf_json, screenshot_failures, | 56 def __init__(self, build_type, test_data, save_perf_json, screenshot_failures, |
56 tool, wait_for_debugger, disable_assertions, push_deps, | 57 tool, wait_for_debugger, disable_assertions, push_deps, |
57 cleanup_test_files, device, shard_index, test_pkg, | 58 cleanup_test_files, device, shard_index, test_pkg, |
58 ports_to_forward): | 59 ports_to_forward, coverage_dir): |
59 """Create a new TestRunner. | 60 """Create a new TestRunner. |
60 | 61 |
61 Args: | 62 Args: |
62 build_type: 'Release' or 'Debug'. | 63 build_type: 'Release' or 'Debug'. |
63 test_data: Location of the test data. | 64 test_data: Location of the test data. |
64 save_perf_json: Whether or not to save the JSON file from UI perf tests. | 65 save_perf_json: Whether or not to save the JSON file from UI perf tests. |
65 screenshot_failures: Take a screenshot for a test failure | 66 screenshot_failures: Take a screenshot for a test failure |
66 tool: Name of the Valgrind tool. | 67 tool: Name of the Valgrind tool. |
67 wait_for_debugger: Blocks until the debugger is connected. | 68 wait_for_debugger: Blocks until the debugger is connected. |
68 disable_assertions: Whether to disable java assertions on the device. | 69 disable_assertions: Whether to disable java assertions on the device. |
69 push_deps: If True, push all dependencies to the device. | 70 push_deps: If True, push all dependencies to the device. |
70 cleanup_test_files: Whether or not to cleanup test files on device. | 71 cleanup_test_files: Whether or not to cleanup test files on device. |
71 device: Attached android device. | 72 device: Attached android device. |
72 shard_index: Shard index. | 73 shard_index: Shard index. |
73 test_pkg: A TestPackage object. | 74 test_pkg: A TestPackage object. |
74 ports_to_forward: A list of port numbers for which to set up forwarders. | 75 ports_to_forward: A list of port numbers for which to set up forwarders. |
75 Can be optionally requested by a test case. | 76 Can be optionally requested by a test case. |
77 coverage_dir: Directory to pull all EMMA coverage files into, or None if | |
78 coverage is not being used. | |
76 """ | 79 """ |
77 super(TestRunner, self).__init__(device, tool, build_type, push_deps, | 80 super(TestRunner, self).__init__(device, tool, build_type, push_deps, |
78 cleanup_test_files) | 81 cleanup_test_files) |
79 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index | 82 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index |
80 | 83 |
81 self.build_type = build_type | 84 self.build_type = build_type |
82 self.test_data = test_data | 85 self.test_data = test_data |
83 self.save_perf_json = save_perf_json | 86 self.save_perf_json = save_perf_json |
84 self.screenshot_failures = screenshot_failures | 87 self.screenshot_failures = screenshot_failures |
85 self.wait_for_debugger = wait_for_debugger | 88 self.wait_for_debugger = wait_for_debugger |
86 self.disable_assertions = disable_assertions | 89 self.disable_assertions = disable_assertions |
87 self.test_pkg = test_pkg | 90 self.test_pkg = test_pkg |
88 self.ports_to_forward = ports_to_forward | 91 self.ports_to_forward = ports_to_forward |
92 self.coverage_dir = coverage_dir | |
89 | 93 |
90 #override | 94 #override |
91 def InstallTestPackage(self): | 95 def InstallTestPackage(self): |
92 self.test_pkg.Install(self.adb) | 96 self.test_pkg.Install(self.adb) |
93 | 97 |
94 #override | 98 #override |
95 def PushDataDeps(self): | 99 def PushDataDeps(self): |
96 # TODO(frankf): Implement a general approach for copying/installing | 100 # TODO(frankf): Implement a general approach for copying/installing |
97 # once across test runners. | 101 # once across test runners. |
98 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): | 102 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): |
99 logging.warning('Already copied test files to device %s, skipping.', | 103 logging.warning('Already copied test files to device %s, skipping.', |
100 self.device) | 104 self.device) |
101 return | 105 return |
102 | 106 |
103 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName()) | 107 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName()) |
104 if test_data: | 108 if test_data: |
105 # Make sure SD card is ready. | 109 # Make sure SD card is ready. |
106 self.adb.WaitForSdCardReady(20) | 110 self.adb.WaitForSdCardReady(20) |
107 for p in test_data: | 111 for p in test_data: |
108 self.adb.PushIfNeeded( | 112 self.adb.PushIfNeeded( |
109 os.path.join(constants.DIR_SOURCE_ROOT, p), | 113 os.path.join(constants.DIR_SOURCE_ROOT, p), |
110 os.path.join(self.adb.GetExternalStorage(), p)) | 114 os.path.join(self.adb.GetExternalStorage(), p)) |
111 | 115 |
112 # TODO(frankf): Specify test data in this file as opposed to passing | 116 # TODO(frankf): Specify test data in this file as opposed to passing |
113 # as command-line. | 117 # as command-line. |
114 for dest_host_pair in self.test_data: | 118 for dest_host_pair in self.test_data: |
115 dst_src = dest_host_pair.split(':',1) | 119 dst_src = dest_host_pair.split(':',1) |
116 dst_layer = dst_src[0] | 120 dst_layer = dst_src[0] |
117 host_src = dst_src[1] | 121 host_src = dst_src[1] |
118 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src | 122 host_test_files_path = '%s/%s' % (constants.DIR_SOURCE_ROOT, host_src) |
119 if os.path.exists(host_test_files_path): | 123 if os.path.exists(host_test_files_path): |
120 self.adb.PushIfNeeded(host_test_files_path, | 124 self.adb.PushIfNeeded(host_test_files_path, '%s/%s/%s' % ( |
121 self.adb.GetExternalStorage() + '/' + | 125 self.adb.GetExternalStorage(), TestRunner._DEVICE_DATA_DIR, |
122 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) | 126 dst_layer)) |
123 self.tool.CopyFiles() | 127 self.tool.CopyFiles() |
124 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True | 128 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True |
125 | 129 |
126 def _GetInstrumentationArgs(self): | 130 def _GetInstrumentationArgs(self): |
127 ret = {} | 131 ret = {} |
128 if self.wait_for_debugger: | 132 if self.wait_for_debugger: |
129 ret['debug'] = 'true' | 133 ret['debug'] = 'true' |
134 if self.coverage_dir: | |
135 ret['coverage'] = 'true' | |
136 ret['coverageFile'] = self.coverage_device_file | |
137 | |
130 return ret | 138 return ret |
131 | 139 |
132 def _TakeScreenshot(self, test): | 140 def _TakeScreenshot(self, test): |
133 """Takes a screenshot from the device.""" | 141 """Takes a screenshot from the device.""" |
134 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') | 142 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, '%s.png' % test) |
135 logging.info('Taking screenshot named %s', screenshot_name) | 143 logging.info('Taking screenshot named %s', screenshot_name) |
136 self.adb.TakeScreenshot(screenshot_name) | 144 self.adb.TakeScreenshot(screenshot_name) |
137 | 145 |
138 def SetUp(self): | 146 def SetUp(self): |
139 """Sets up the test harness and device before all tests are run.""" | 147 """Sets up the test harness and device before all tests are run.""" |
140 super(TestRunner, self).SetUp() | 148 super(TestRunner, self).SetUp() |
141 if not self.adb.IsRootEnabled(): | 149 if not self.adb.IsRootEnabled(): |
142 logging.warning('Unable to enable java asserts for %s, non rooted device', | 150 logging.warning('Unable to enable java asserts for %s, non rooted device', |
143 self.device) | 151 self.device) |
144 else: | 152 else: |
(...skipping 19 matching lines...) Expand all Loading... | |
164 Args: | 172 Args: |
165 test: The name of the test that will be run. | 173 test: The name of the test that will be run. |
166 """ | 174 """ |
167 self.SetupPerfMonitoringIfNeeded(test) | 175 self.SetupPerfMonitoringIfNeeded(test) |
168 self._SetupIndividualTestTimeoutScale(test) | 176 self._SetupIndividualTestTimeoutScale(test) |
169 self.tool.SetupEnvironment() | 177 self.tool.SetupEnvironment() |
170 | 178 |
171 # Make sure the forwarder is still running. | 179 # Make sure the forwarder is still running. |
172 self.RestartHttpServerForwarderIfNecessary() | 180 self.RestartHttpServerForwarderIfNecessary() |
173 | 181 |
182 if self.coverage_dir: | |
183 coverage_basename = '%s.ec' % test | |
184 self.coverage_device_file = '%s/%s/%s' % (self.adb.GetExternalStorage(), | |
185 TestRunner._DEVICE_COVERAGE_DIR, | |
186 coverage_basename) | |
187 self.coverage_host_file = os.path.join( | |
188 self.coverage_dir, coverage_basename) | |
189 | |
174 def _IsPerfTest(self, test): | 190 def _IsPerfTest(self, test): |
175 """Determines whether a test is a performance test. | 191 """Determines whether a test is a performance test. |
176 | 192 |
177 Args: | 193 Args: |
178 test: The name of the test to be checked. | 194 test: The name of the test to be checked. |
179 | 195 |
180 Returns: | 196 Returns: |
181 Whether the test is annotated as a performance test. | 197 Whether the test is annotated as a performance test. |
182 """ | 198 """ |
183 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test) | 199 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test) |
(...skipping 22 matching lines...) Expand all Loading... | |
206 """ | 222 """ |
207 | 223 |
208 self.tool.CleanUpEnvironment() | 224 self.tool.CleanUpEnvironment() |
209 | 225 |
210 # The logic below relies on the test passing. | 226 # The logic below relies on the test passing. |
211 if not raw_result or raw_result.GetStatusCode(): | 227 if not raw_result or raw_result.GetStatusCode(): |
212 return | 228 return |
213 | 229 |
214 self.TearDownPerfMonitoring(test) | 230 self.TearDownPerfMonitoring(test) |
215 | 231 |
232 if self.coverage_dir: | |
233 self.adb.Adb().Pull(self.coverage_device_file, self.coverage_host_file) | |
234 self.adb.RunShellCommand('rm %s' % self.coverage_device_file) | |
frankf
2013/08/01 19:57:41
rm -f
gkanwar1
2013/08/07 19:24:56
Done.
| |
235 | |
216 def TearDownPerfMonitoring(self, test): | 236 def TearDownPerfMonitoring(self, test): |
217 """Cleans up performance monitoring if the specified test required it. | 237 """Cleans up performance monitoring if the specified test required it. |
218 | 238 |
219 Args: | 239 Args: |
220 test: The name of the test that was just run. | 240 test: The name of the test that was just run. |
221 Raises: | 241 Raises: |
222 Exception: if there's anything wrong with the perf data. | 242 Exception: if there's anything wrong with the perf data. |
223 """ | 243 """ |
224 if not self._IsPerfTest(test): | 244 if not self._IsPerfTest(test): |
225 return | 245 return |
(...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
349 duration_ms = 0 | 369 duration_ms = 0 |
350 message = str(e) | 370 message = str(e) |
351 if not message: | 371 if not message: |
352 message = 'No information.' | 372 message = 'No information.' |
353 results.AddResult(test_result.InstrumentationTestResult( | 373 results.AddResult(test_result.InstrumentationTestResult( |
354 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, | 374 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, |
355 log=message)) | 375 log=message)) |
356 raw_result = None | 376 raw_result = None |
357 self.TestTeardown(test, raw_result) | 377 self.TestTeardown(test, raw_result) |
358 return (results, None if results.DidRunPass() else test) | 378 return (results, None if results.DidRunPass() else test) |
OLD | NEW |