Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(95)

Side by Side Diff: build/android/pylib/instrumentation/test_runner.py

Issue 13604006: [Android] Remove emma coverage logic for instrumentation tests. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: Created 7 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « build/android/pylib/instrumentation/dispatch.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Runs the Java tests. See more information on run_instrumentation_tests.py.""" 5 """Runs the Java tests. See more information on run_instrumentation_tests.py."""
6 6
7 import logging 7 import logging
8 import os 8 import os
9 import re 9 import re
10 import shutil 10 import shutil
(...skipping 30 matching lines...) Expand all
41 test_files += [ 41 test_files += [
42 'net/data/ssl/certificates/', 42 'net/data/ssl/certificates/',
43 ] 43 ]
44 return test_files 44 return test_files
45 45
46 46
47 class TestRunner(base_test_runner.BaseTestRunner): 47 class TestRunner(base_test_runner.BaseTestRunner):
48 """Responsible for running a series of tests connected to a single device.""" 48 """Responsible for running a series of tests connected to a single device."""
49 49
50 _DEVICE_DATA_DIR = 'chrome/test/data' 50 _DEVICE_DATA_DIR = 'chrome/test/data'
51 _EMMA_JAR = os.path.join(os.environ.get('ANDROID_BUILD_TOP', ''),
52 'external/emma/lib/emma.jar')
53 _COVERAGE_MERGED_FILENAME = 'unittest_coverage.es'
54 _COVERAGE_WEB_ROOT_DIR = os.environ.get('EMMA_WEB_ROOTDIR')
55 _COVERAGE_FILENAME = 'coverage.ec'
56 _COVERAGE_RESULT_PATH = ('/data/data/com.google.android.apps.chrome/files/' +
57 _COVERAGE_FILENAME)
58 _COVERAGE_META_INFO_PATH = os.path.join(os.environ.get('ANDROID_BUILD_TOP',
59 ''),
60 'out/target/common/obj/APPS',
61 'Chrome_intermediates/coverage.em')
62 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' 51 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile'
63 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + 52 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR +
64 '/chrome-profile*') 53 '/chrome-profile*')
65 _DEVICE_HAS_TEST_FILES = {} 54 _DEVICE_HAS_TEST_FILES = {}
66 55
67 def __init__(self, options, device, shard_index, coverage, test_pkg, 56 def __init__(self, options, device, shard_index, test_pkg,
68 ports_to_forward, is_uiautomator_test=False): 57 ports_to_forward, is_uiautomator_test=False):
69 """Create a new TestRunner. 58 """Create a new TestRunner.
70 59
71 Args: 60 Args:
72 options: An options object with the following required attributes: 61 options: An options object with the following required attributes:
73 - build_type: 'Release' or 'Debug'. 62 - build_type: 'Release' or 'Debug'.
74 - install_apk: Re-installs the apk if opted. 63 - install_apk: Re-installs the apk if opted.
75 - save_perf_json: Whether or not to save the JSON file from UI perf 64 - save_perf_json: Whether or not to save the JSON file from UI perf
76 tests. 65 tests.
77 - screenshot_failures: Take a screenshot for a test failure 66 - screenshot_failures: Take a screenshot for a test failure
78 - tool: Name of the Valgrind tool. 67 - tool: Name of the Valgrind tool.
79 - wait_for_debugger: blocks until the debugger is connected. 68 - wait_for_debugger: blocks until the debugger is connected.
80 - disable_assertions: Whether to disable java assertions on the device. 69 - disable_assertions: Whether to disable java assertions on the device.
81 device: Attached android device. 70 device: Attached android device.
82 shard_index: Shard index. 71 shard_index: Shard index.
83 coverage: Collects coverage information if opted.
84 test_pkg: A TestPackage object. 72 test_pkg: A TestPackage object.
85 ports_to_forward: A list of port numbers for which to set up forwarders. 73 ports_to_forward: A list of port numbers for which to set up forwarders.
86 Can be optionally requested by a test case. 74 Can be optionally requested by a test case.
87 is_uiautomator_test: Whether this is a uiautomator test. 75 is_uiautomator_test: Whether this is a uiautomator test.
88 Raises:
89 Exception: if coverage metadata is not available.
90 """ 76 """
91 super(TestRunner, self).__init__(device, options.tool, options.build_type) 77 super(TestRunner, self).__init__(device, options.tool, options.build_type)
92 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index 78 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index
93 79
94 self.build_type = options.build_type 80 self.build_type = options.build_type
95 self.test_data = options.test_data 81 self.test_data = options.test_data
96 self.save_perf_json = options.save_perf_json 82 self.save_perf_json = options.save_perf_json
97 self.screenshot_failures = options.screenshot_failures 83 self.screenshot_failures = options.screenshot_failures
98 self.wait_for_debugger = options.wait_for_debugger 84 self.wait_for_debugger = options.wait_for_debugger
99 self.disable_assertions = options.disable_assertions 85 self.disable_assertions = options.disable_assertions
100 self.coverage = coverage
101 self.test_pkg = test_pkg 86 self.test_pkg = test_pkg
102 self.ports_to_forward = ports_to_forward 87 self.ports_to_forward = ports_to_forward
103 self.is_uiautomator_test = is_uiautomator_test 88 self.is_uiautomator_test = is_uiautomator_test
104 if self.is_uiautomator_test: 89 if self.is_uiautomator_test:
105 self.package_name = options.package_name 90 self.package_name = options.package_name
106 else: 91 else:
107 self.install_apk = options.install_apk 92 self.install_apk = options.install_apk
108 93
109 self.forwarder = None 94 self.forwarder = None
110 95
111 if self.coverage:
112 if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME):
113 os.remove(TestRunner._COVERAGE_MERGED_FILENAME)
114 if not os.path.exists(TestRunner._COVERAGE_META_INFO_PATH):
115 raise Exception('FATAL ERROR in ' + sys.argv[0] +
116 ' : Coverage meta info [' +
117 TestRunner._COVERAGE_META_INFO_PATH +
118 '] does not exist.')
119 if (not TestRunner._COVERAGE_WEB_ROOT_DIR or
120 not os.path.exists(TestRunner._COVERAGE_WEB_ROOT_DIR)):
121 raise Exception('FATAL ERROR in ' + sys.argv[0] +
122 ' : Path specified in $EMMA_WEB_ROOTDIR [' +
123 TestRunner._COVERAGE_WEB_ROOT_DIR +
124 '] does not exist.')
125
126 #override. 96 #override.
127 def PushDependencies(self): 97 def PushDependencies(self):
128 # TODO(frankf): Implement a general approach for copying/installing 98 # TODO(frankf): Implement a general approach for copying/installing
129 # once across test runners. 99 # once across test runners.
130 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): 100 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False):
131 logging.warning('Already copied test files to device %s, skipping.', 101 logging.warning('Already copied test files to device %s, skipping.',
132 self.device) 102 self.device)
133 return 103 return
134 104
135 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName()) 105 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName())
(...skipping 12 matching lines...) Expand all
148 host_test_files_path = constants.CHROME_DIR + '/' + host_src 118 host_test_files_path = constants.CHROME_DIR + '/' + host_src
149 if os.path.exists(host_test_files_path): 119 if os.path.exists(host_test_files_path):
150 self.adb.PushIfNeeded(host_test_files_path, 120 self.adb.PushIfNeeded(host_test_files_path,
151 self.adb.GetExternalStorage() + '/' + 121 self.adb.GetExternalStorage() + '/' +
152 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) 122 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer)
153 if self.is_uiautomator_test or self.install_apk: 123 if self.is_uiautomator_test or self.install_apk:
154 self.test_pkg.Install(self.adb) 124 self.test_pkg.Install(self.adb)
155 self.tool.CopyFiles() 125 self.tool.CopyFiles()
156 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True 126 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True
157 127
158 def SaveCoverageData(self, test):
159 """Saves the Emma coverage data before it's overwritten by the next test.
160
161 Args:
162 test: the test whose coverage data is collected.
163 """
164 if not self.coverage:
165 return
166 if not self.adb.Adb().Pull(TestRunner._COVERAGE_RESULT_PATH,
167 constants.CHROME_DIR):
168 logging.error('ERROR: Unable to find file ' +
169 TestRunner._COVERAGE_RESULT_PATH +
170 ' on the device for test ' + test)
171 pulled_coverage_file = os.path.join(constants.CHROME_DIR,
172 TestRunner._COVERAGE_FILENAME)
173 if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME):
174 cmd = ['java', '-classpath', TestRunner._EMMA_JAR, 'emma', 'merge',
175 '-in', pulled_coverage_file,
176 '-in', TestRunner._COVERAGE_MERGED_FILENAME,
177 '-out', TestRunner._COVERAGE_MERGED_FILENAME]
178 cmd_helper.RunCmd(cmd)
179 else:
180 shutil.copy(pulled_coverage_file,
181 TestRunner._COVERAGE_MERGED_FILENAME)
182 os.remove(pulled_coverage_file)
183
184 def GenerateCoverageReportIfNeeded(self):
185 """Uses the Emma to generate a coverage report and a html page."""
186 if not self.coverage:
187 return
188 cmd = ['java', '-classpath', TestRunner._EMMA_JAR,
189 'emma', 'report', '-r', 'html',
190 '-in', TestRunner._COVERAGE_MERGED_FILENAME,
191 '-in', TestRunner._COVERAGE_META_INFO_PATH]
192 cmd_helper.RunCmd(cmd)
193 new_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR,
194 time.strftime('Coverage_for_%Y_%m_%d_%a_%H:%M'))
195 shutil.copytree('coverage', new_dir)
196
197 latest_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR,
198 'Latest_Coverage_Run')
199 if os.path.exists(latest_dir):
200 shutil.rmtree(latest_dir)
201 os.mkdir(latest_dir)
202 webserver_new_index = os.path.join(new_dir, 'index.html')
203 webserver_new_files = os.path.join(new_dir, '_files')
204 webserver_latest_index = os.path.join(latest_dir, 'index.html')
205 webserver_latest_files = os.path.join(latest_dir, '_files')
206 # Setup new softlinks to last result.
207 os.symlink(webserver_new_index, webserver_latest_index)
208 os.symlink(webserver_new_files, webserver_latest_files)
209 cmd_helper.RunCmd(['chmod', '755', '-R', latest_dir, new_dir])
210
211 def _GetInstrumentationArgs(self): 128 def _GetInstrumentationArgs(self):
212 ret = {} 129 ret = {}
213 if self.coverage:
214 ret['coverage'] = 'true'
215 if self.wait_for_debugger: 130 if self.wait_for_debugger:
216 ret['debug'] = 'true' 131 ret['debug'] = 'true'
217 return ret 132 return ret
218 133
219 def _TakeScreenshot(self, test): 134 def _TakeScreenshot(self, test):
220 """Takes a screenshot from the device.""" 135 """Takes a screenshot from the device."""
221 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') 136 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png')
222 logging.info('Taking screenshot named %s', screenshot_name) 137 logging.info('Taking screenshot named %s', screenshot_name)
223 self.adb.TakeScreenshot(screenshot_name) 138 self.adb.TakeScreenshot(screenshot_name)
224 139
(...skipping 18 matching lines...) Expand all
243 # forwarder will stomp on them otherwise. 158 # forwarder will stomp on them otherwise.
244 port_pairs.append(http_server_ports) 159 port_pairs.append(http_server_ports)
245 self.forwarder = forwarder.Forwarder(self.adb, self.build_type) 160 self.forwarder = forwarder.Forwarder(self.adb, self.build_type)
246 self.forwarder.Run(port_pairs, self.tool, '127.0.0.1') 161 self.forwarder.Run(port_pairs, self.tool, '127.0.0.1')
247 self.flags.AddFlags(['--enable-test-intents']) 162 self.flags.AddFlags(['--enable-test-intents'])
248 163
249 def TearDown(self): 164 def TearDown(self):
250 """Cleans up the test harness and saves outstanding data from test run.""" 165 """Cleans up the test harness and saves outstanding data from test run."""
251 if self.forwarder: 166 if self.forwarder:
252 self.forwarder.Close() 167 self.forwarder.Close()
253 self.GenerateCoverageReportIfNeeded()
254 super(TestRunner, self).TearDown() 168 super(TestRunner, self).TearDown()
255 169
256 def TestSetup(self, test): 170 def TestSetup(self, test):
257 """Sets up the test harness for running a particular test. 171 """Sets up the test harness for running a particular test.
258 172
259 Args: 173 Args:
260 test: The name of the test that will be run. 174 test: The name of the test that will be run.
261 """ 175 """
262 self.SetupPerfMonitoringIfNeeded(test) 176 self.SetupPerfMonitoringIfNeeded(test)
263 self._SetupIndividualTestTimeoutScale(test) 177 self._SetupIndividualTestTimeoutScale(test)
(...skipping 21 matching lines...) Expand all
285 """ 199 """
286 if not self._IsPerfTest(test): 200 if not self._IsPerfTest(test):
287 return 201 return
288 self.adb.Adb().SendCommand('shell rm ' + 202 self.adb.Adb().SendCommand('shell rm ' +
289 TestRunner._DEVICE_PERF_OUTPUT_SEARCH_PREFIX) 203 TestRunner._DEVICE_PERF_OUTPUT_SEARCH_PREFIX)
290 self.adb.StartMonitoringLogcat() 204 self.adb.StartMonitoringLogcat()
291 205
292 def TestTeardown(self, test, raw_result): 206 def TestTeardown(self, test, raw_result):
293 """Cleans up the test harness after running a particular test. 207 """Cleans up the test harness after running a particular test.
294 208
295 Depending on the options of this TestRunner this might handle coverage 209 Depending on the options of this TestRunner this might handle performance
296 tracking or performance tracking. This method will only be called if the 210 tracking. This method will only be called if the test passed.
297 test passed.
298 211
299 Args: 212 Args:
300 test: The name of the test that was just run. 213 test: The name of the test that was just run.
301 raw_result: result for this test. 214 raw_result: result for this test.
302 """ 215 """
303 216
304 self.tool.CleanUpEnvironment() 217 self.tool.CleanUpEnvironment()
305 218
306 # The logic below relies on the test passing. 219 # The logic below relies on the test passing.
307 if not raw_result or raw_result.GetStatusCode(): 220 if not raw_result or raw_result.GetStatusCode():
308 return 221 return
309 222
310 self.TearDownPerfMonitoring(test) 223 self.TearDownPerfMonitoring(test)
311 self.SaveCoverageData(test)
312 224
313 def TearDownPerfMonitoring(self, test): 225 def TearDownPerfMonitoring(self, test):
314 """Cleans up performance monitoring if the specified test required it. 226 """Cleans up performance monitoring if the specified test required it.
315 227
316 Args: 228 Args:
317 test: The name of the test that was just run. 229 test: The name of the test that was just run.
318 Raises: 230 Raises:
319 Exception: if there's anything wrong with the perf data. 231 Exception: if there's anything wrong with the perf data.
320 """ 232 """
321 if not self._IsPerfTest(test): 233 if not self._IsPerfTest(test):
(...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after
462 duration_ms = 0 374 duration_ms = 0
463 message = str(e) 375 message = str(e)
464 if not message: 376 if not message:
465 message = 'No information.' 377 message = 'No information.'
466 results.AddResult(test_result.InstrumentationTestResult( 378 results.AddResult(test_result.InstrumentationTestResult(
467 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, 379 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms,
468 log=message)) 380 log=message))
469 raw_result = None 381 raw_result = None
470 self.TestTeardown(test, raw_result) 382 self.TestTeardown(test, raw_result)
471 return (results, None if results.DidRunPass() else test) 383 return (results, None if results.DidRunPass() else test)
OLDNEW
« no previous file with comments | « build/android/pylib/instrumentation/dispatch.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698