Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(59)

Side by Side Diff: build/android/pylib/instrumentation/test_runner.py

Issue 20210002: [Android] Sets up a coverage system for java using EMMA (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 7 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Class for running instrumentation tests on a single device.""" 5 """Class for running instrumentation tests on a single device."""
6 6
7 import logging 7 import logging
8 import os 8 import os
9 import re 9 import re
10 import shutil 10 import shutil
(...skipping 29 matching lines...) Expand all
40 test_files += [ 40 test_files += [
41 'net/data/ssl/certificates/', 41 'net/data/ssl/certificates/',
42 ] 42 ]
43 return test_files 43 return test_files
44 44
45 45
46 class TestRunner(base_test_runner.BaseTestRunner): 46 class TestRunner(base_test_runner.BaseTestRunner):
47 """Responsible for running a series of tests connected to a single device.""" 47 """Responsible for running a series of tests connected to a single device."""
48 48
49 _DEVICE_DATA_DIR = 'chrome/test/data' 49 _DEVICE_DATA_DIR = 'chrome/test/data'
50 _DEVICE_COVERAGE_DIR = 'chrome/test/coverage'
50 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' 51 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile'
51 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + 52 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR +
52 '/chrome-profile*') 53 '/chrome-profile*')
53 _DEVICE_HAS_TEST_FILES = {} 54 _DEVICE_HAS_TEST_FILES = {}
54 55
55 def __init__(self, build_type, test_data, install_apk, save_perf_json, 56 def __init__(self, build_type, test_data, install_apk, save_perf_json,
56 screenshot_failures, tool, wait_for_debugger, disable_assertions, 57 screenshot_failures, tool, wait_for_debugger, disable_assertions,
57 push_deps, cleanup_test_files, device, shard_index, test_pkg, 58 push_deps, cleanup_test_files, device, shard_index, test_pkg,
58 ports_to_forward): 59 ports_to_forward, coverage, coverage_dir):
59 """Create a new TestRunner. 60 """Create a new TestRunner.
60 61
61 Args: 62 Args:
62 build_type: 'Release' or 'Debug'. 63 build_type: 'Release' or 'Debug'.
63 test_data: Location of the test data. 64 test_data: Location of the test data.
64 install_apk: Re-installs the apk if opted. 65 install_apk: Re-installs the apk if opted.
65 save_perf_json: Whether or not to save the JSON file from UI perf tests. 66 save_perf_json: Whether or not to save the JSON file from UI perf tests.
66 screenshot_failures: Take a screenshot for a test failure 67 screenshot_failures: Take a screenshot for a test failure
67 tool: Name of the Valgrind tool. 68 tool: Name of the Valgrind tool.
68 wait_for_debugger: Blocks until the debugger is connected. 69 wait_for_debugger: Blocks until the debugger is connected.
69 disable_assertions: Whether to disable java assertions on the device. 70 disable_assertions: Whether to disable java assertions on the device.
70 push_deps: If True, push all dependencies to the device. 71 push_deps: If True, push all dependencies to the device.
71 cleanup_test_files: Whether or not to cleanup test files on device. 72 cleanup_test_files: Whether or not to cleanup test files on device.
72 device: Attached android device. 73 device: Attached android device.
73 shard_index: Shard index. 74 shard_index: Shard index.
74 test_pkg: A TestPackage object. 75 test_pkg: A TestPackage object.
75 ports_to_forward: A list of port numbers for which to set up forwarders. 76 ports_to_forward: A list of port numbers for which to set up forwarders.
76 Can be optionally requested by a test case. 77 Can be optionally requested by a test case.
78 coverage: If True, instrument with EMMA coverage tool.
79 coverage_dir: Directory to pull all EMMA coverage files into, or None if
80 coverage is not being used.
77 """ 81 """
78 super(TestRunner, self).__init__(device, tool, build_type, push_deps, 82 super(TestRunner, self).__init__(device, tool, build_type, push_deps,
79 cleanup_test_files) 83 cleanup_test_files)
80 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index 84 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index
81 85
82 self.build_type = build_type 86 self.build_type = build_type
83 self.test_data = test_data 87 self.test_data = test_data
84 self.save_perf_json = save_perf_json 88 self.save_perf_json = save_perf_json
85 self.screenshot_failures = screenshot_failures 89 self.screenshot_failures = screenshot_failures
86 self.wait_for_debugger = wait_for_debugger 90 self.wait_for_debugger = wait_for_debugger
87 self.disable_assertions = disable_assertions 91 self.disable_assertions = disable_assertions
88 self.test_pkg = test_pkg 92 self.test_pkg = test_pkg
89 self.ports_to_forward = ports_to_forward 93 self.ports_to_forward = ports_to_forward
90 self.install_apk = install_apk 94 self.install_apk = install_apk
95 self.coverage = coverage
96 self.coverage_dir = coverage_dir
91 97
92 #override 98 #override
93 def InstallTestPackage(self): 99 def InstallTestPackage(self):
94 if self.install_apk: 100 if self.install_apk:
95 self.test_pkg.Install(self.adb) 101 self.test_pkg.Install(self.adb)
96 102
97 #override 103 #override
98 def PushDataDeps(self): 104 def PushDataDeps(self):
99 # TODO(frankf): Implement a general approach for copying/installing 105 # TODO(frankf): Implement a general approach for copying/installing
100 # once across test runners. 106 # once across test runners.
(...skipping 18 matching lines...) Expand all
119 dst_layer = dst_src[0] 125 dst_layer = dst_src[0]
120 host_src = dst_src[1] 126 host_src = dst_src[1]
121 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src 127 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src
122 if os.path.exists(host_test_files_path): 128 if os.path.exists(host_test_files_path):
123 self.adb.PushIfNeeded(host_test_files_path, 129 self.adb.PushIfNeeded(host_test_files_path,
124 self.adb.GetExternalStorage() + '/' + 130 self.adb.GetExternalStorage() + '/' +
125 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) 131 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer)
126 self.tool.CopyFiles() 132 self.tool.CopyFiles()
127 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True 133 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True
128 134
129 def _GetInstrumentationArgs(self): 135 def _GetInstrumentationArgs(self, test):
frankf 2013/07/25 18:53:44 where test used?
gkanwar1 2013/07/25 19:06:38 Oops, this a remnant of something else I was doing
130 ret = {} 136 ret = {}
131 if self.wait_for_debugger: 137 if self.wait_for_debugger:
132 ret['debug'] = 'true' 138 ret['debug'] = 'true'
139 if self.coverage:
140 ret['coverage'] = 'true'
141 ret['coverageFile'] = self.coverage_file
142
133 return ret 143 return ret
134 144
135 def _TakeScreenshot(self, test): 145 def _TakeScreenshot(self, test):
136 """Takes a screenshot from the device.""" 146 """Takes a screenshot from the device."""
137 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') 147 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png')
138 logging.info('Taking screenshot named %s', screenshot_name) 148 logging.info('Taking screenshot named %s', screenshot_name)
139 self.adb.TakeScreenshot(screenshot_name) 149 self.adb.TakeScreenshot(screenshot_name)
140 150
141 def SetUp(self): 151 def SetUp(self):
142 """Sets up the test harness and device before all tests are run.""" 152 """Sets up the test harness and device before all tests are run."""
(...skipping 24 matching lines...) Expand all
167 Args: 177 Args:
168 test: The name of the test that will be run. 178 test: The name of the test that will be run.
169 """ 179 """
170 self.SetupPerfMonitoringIfNeeded(test) 180 self.SetupPerfMonitoringIfNeeded(test)
171 self._SetupIndividualTestTimeoutScale(test) 181 self._SetupIndividualTestTimeoutScale(test)
172 self.tool.SetupEnvironment() 182 self.tool.SetupEnvironment()
173 183
174 # Make sure the forwarder is still running. 184 # Make sure the forwarder is still running.
175 self.RestartHttpServerForwarderIfNecessary() 185 self.RestartHttpServerForwarderIfNecessary()
176 186
187 if self.coverage:
188 self.coverage_basename = test + '.ec'
frankf 2013/07/25 18:53:44 Use string formatter: '%s.ec' % test also below
gkanwar1 2013/07/25 19:06:38 Done. Fixed in several other places in this file a
189 self.coverage_file = (self.adb.GetExternalStorage() + '/' +
frankf 2013/07/25 18:53:44 -> coverage_device_file
gkanwar1 2013/07/25 19:06:38 Done.
190 TestRunner._DEVICE_COVERAGE_DIR + '/' +
191 self.coverage_basename)
frankf 2013/07/25 18:53:44 define coverage_host_file here
gkanwar1 2013/07/25 19:06:38 Done.
192
177 def _IsPerfTest(self, test): 193 def _IsPerfTest(self, test):
178 """Determines whether a test is a performance test. 194 """Determines whether a test is a performance test.
179 195
180 Args: 196 Args:
181 test: The name of the test to be checked. 197 test: The name of the test to be checked.
182 198
183 Returns: 199 Returns:
184 Whether the test is annotated as a performance test. 200 Whether the test is annotated as a performance test.
185 """ 201 """
186 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test) 202 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test)
(...skipping 22 matching lines...) Expand all
209 """ 225 """
210 226
211 self.tool.CleanUpEnvironment() 227 self.tool.CleanUpEnvironment()
212 228
213 # The logic below relies on the test passing. 229 # The logic below relies on the test passing.
214 if not raw_result or raw_result.GetStatusCode(): 230 if not raw_result or raw_result.GetStatusCode():
215 return 231 return
216 232
217 self.TearDownPerfMonitoring(test) 233 self.TearDownPerfMonitoring(test)
218 234
235 if self.coverage:
236 self.adb.Adb().Pull(self.coverage_file, os.path.join(
237 self.coverage_dir, self.coverage_basename))
238 self.adb.RunShellCommand('rm %s' % self.coverage_file)
239
219 def TearDownPerfMonitoring(self, test): 240 def TearDownPerfMonitoring(self, test):
220 """Cleans up performance monitoring if the specified test required it. 241 """Cleans up performance monitoring if the specified test required it.
221 242
222 Args: 243 Args:
223 test: The name of the test that was just run. 244 test: The name of the test that was just run.
224 Raises: 245 Raises:
225 Exception: if there's anything wrong with the perf data. 246 Exception: if there's anything wrong with the perf data.
226 """ 247 """
227 if not self._IsPerfTest(test): 248 if not self._IsPerfTest(test):
228 return 249 return
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after
301 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations: 322 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations:
302 return 5 * 60 323 return 5 * 60
303 if 'MediumTest' in annotations: 324 if 'MediumTest' in annotations:
304 return 3 * 60 325 return 3 * 60
305 return 1 * 60 326 return 1 * 60
306 327
307 def _RunTest(self, test, timeout): 328 def _RunTest(self, test, timeout):
308 try: 329 try:
309 return self.adb.RunInstrumentationTest( 330 return self.adb.RunInstrumentationTest(
310 test, self.test_pkg.GetPackageName(), 331 test, self.test_pkg.GetPackageName(),
311 self._GetInstrumentationArgs(), timeout) 332 self._GetInstrumentationArgs(test), timeout)
312 except android_commands.errors.WaitForResponseTimedOutError: 333 except android_commands.errors.WaitForResponseTimedOutError:
313 logging.info('Ran the test with timeout of %ds.' % timeout) 334 logging.info('Ran the test with timeout of %ds.' % timeout)
314 raise 335 raise
315 336
316 #override 337 #override
317 def RunTest(self, test): 338 def RunTest(self, test):
318 raw_result = None 339 raw_result = None
319 start_date_ms = None 340 start_date_ms = None
320 results = base_test_result.TestRunResults() 341 results = base_test_result.TestRunResults()
321 timeout=(self._GetIndividualTestTimeoutSecs(test) * 342 timeout=(self._GetIndividualTestTimeoutSecs(test) *
(...skipping 30 matching lines...) Expand all
352 duration_ms = 0 373 duration_ms = 0
353 message = str(e) 374 message = str(e)
354 if not message: 375 if not message:
355 message = 'No information.' 376 message = 'No information.'
356 results.AddResult(test_result.InstrumentationTestResult( 377 results.AddResult(test_result.InstrumentationTestResult(
357 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, 378 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms,
358 log=message)) 379 log=message))
359 raw_result = None 380 raw_result = None
360 self.TestTeardown(test, raw_result) 381 self.TestTeardown(test, raw_result)
361 return (results, None if results.DidRunPass() else test) 382 return (results, None if results.DidRunPass() else test)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698