Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1573)

Side by Side Diff: build/android/pylib/instrumentation/test_runner.py

Issue 20210002: [Android] Sets up a coverage system for java using EMMA (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Removes unnecessary option, cleans up some string formatting Created 7 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Class for running instrumentation tests on a single device.""" 5 """Class for running instrumentation tests on a single device."""
6 6
7 import logging 7 import logging
8 import os 8 import os
9 import re 9 import re
10 import shutil 10 import shutil
(...skipping 29 matching lines...) Expand all
40 test_files += [ 40 test_files += [
41 'net/data/ssl/certificates/', 41 'net/data/ssl/certificates/',
42 ] 42 ]
43 return test_files 43 return test_files
44 44
45 45
46 class TestRunner(base_test_runner.BaseTestRunner): 46 class TestRunner(base_test_runner.BaseTestRunner):
47 """Responsible for running a series of tests connected to a single device.""" 47 """Responsible for running a series of tests connected to a single device."""
48 48
49 _DEVICE_DATA_DIR = 'chrome/test/data' 49 _DEVICE_DATA_DIR = 'chrome/test/data'
50 _DEVICE_COVERAGE_DIR = 'chrome/test/coverage'
50 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' 51 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile'
51 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + 52 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR +
52 '/chrome-profile*') 53 '/chrome-profile*')
53 _DEVICE_HAS_TEST_FILES = {} 54 _DEVICE_HAS_TEST_FILES = {}
54 55
55 def __init__(self, build_type, test_data, install_apk, save_perf_json, 56 def __init__(self, build_type, test_data, install_apk, save_perf_json,
56 screenshot_failures, tool, wait_for_debugger, disable_assertions, 57 screenshot_failures, tool, wait_for_debugger, disable_assertions,
57 push_deps, cleanup_test_files, device, shard_index, test_pkg, 58 push_deps, cleanup_test_files, device, shard_index, test_pkg,
58 ports_to_forward): 59 ports_to_forward, coverage_dir):
59 """Create a new TestRunner. 60 """Create a new TestRunner.
60 61
61 Args: 62 Args:
62 build_type: 'Release' or 'Debug'. 63 build_type: 'Release' or 'Debug'.
63 test_data: Location of the test data. 64 test_data: Location of the test data.
64 install_apk: Re-installs the apk if opted. 65 install_apk: Re-installs the apk if opted.
65 save_perf_json: Whether or not to save the JSON file from UI perf tests. 66 save_perf_json: Whether or not to save the JSON file from UI perf tests.
66 screenshot_failures: Take a screenshot for a test failure 67 screenshot_failures: Take a screenshot for a test failure
67 tool: Name of the Valgrind tool. 68 tool: Name of the Valgrind tool.
68 wait_for_debugger: Blocks until the debugger is connected. 69 wait_for_debugger: Blocks until the debugger is connected.
69 disable_assertions: Whether to disable java assertions on the device. 70 disable_assertions: Whether to disable java assertions on the device.
70 push_deps: If True, push all dependencies to the device. 71 push_deps: If True, push all dependencies to the device.
71 cleanup_test_files: Whether or not to cleanup test files on device. 72 cleanup_test_files: Whether or not to cleanup test files on device.
72 device: Attached android device. 73 device: Attached android device.
73 shard_index: Shard index. 74 shard_index: Shard index.
74 test_pkg: A TestPackage object. 75 test_pkg: A TestPackage object.
75 ports_to_forward: A list of port numbers for which to set up forwarders. 76 ports_to_forward: A list of port numbers for which to set up forwarders.
76 Can be optionally requested by a test case. 77 Can be optionally requested by a test case.
78 coverage_dir: Directory to pull all EMMA coverage files into, or None if
79 coverage is not being used.
77 """ 80 """
78 super(TestRunner, self).__init__(device, tool, build_type, push_deps, 81 super(TestRunner, self).__init__(device, tool, build_type, push_deps,
79 cleanup_test_files) 82 cleanup_test_files)
80 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index 83 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index
81 84
82 self.build_type = build_type 85 self.build_type = build_type
83 self.test_data = test_data 86 self.test_data = test_data
84 self.save_perf_json = save_perf_json 87 self.save_perf_json = save_perf_json
85 self.screenshot_failures = screenshot_failures 88 self.screenshot_failures = screenshot_failures
86 self.wait_for_debugger = wait_for_debugger 89 self.wait_for_debugger = wait_for_debugger
87 self.disable_assertions = disable_assertions 90 self.disable_assertions = disable_assertions
88 self.test_pkg = test_pkg 91 self.test_pkg = test_pkg
89 self.ports_to_forward = ports_to_forward 92 self.ports_to_forward = ports_to_forward
90 self.install_apk = install_apk 93 self.install_apk = install_apk
94 self.coverage_dir = coverage_dir
91 95
92 #override 96 #override
93 def InstallTestPackage(self): 97 def InstallTestPackage(self):
94 if self.install_apk: 98 if self.install_apk:
95 self.test_pkg.Install(self.adb) 99 self.test_pkg.Install(self.adb)
96 100
97 #override 101 #override
98 def PushDataDeps(self): 102 def PushDataDeps(self):
99 # TODO(frankf): Implement a general approach for copying/installing 103 # TODO(frankf): Implement a general approach for copying/installing
100 # once across test runners. 104 # once across test runners.
(...skipping 10 matching lines...) Expand all
111 self.adb.PushIfNeeded( 115 self.adb.PushIfNeeded(
112 os.path.join(constants.DIR_SOURCE_ROOT, p), 116 os.path.join(constants.DIR_SOURCE_ROOT, p),
113 os.path.join(self.adb.GetExternalStorage(), p)) 117 os.path.join(self.adb.GetExternalStorage(), p))
114 118
115 # TODO(frankf): Specify test data in this file as opposed to passing 119 # TODO(frankf): Specify test data in this file as opposed to passing
116 # as command-line. 120 # as command-line.
117 for dest_host_pair in self.test_data: 121 for dest_host_pair in self.test_data:
118 dst_src = dest_host_pair.split(':',1) 122 dst_src = dest_host_pair.split(':',1)
119 dst_layer = dst_src[0] 123 dst_layer = dst_src[0]
120 host_src = dst_src[1] 124 host_src = dst_src[1]
121 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src 125 host_test_files_path = '%s/%s' % (constants.DIR_SOURCE_ROOT, host_src)
122 if os.path.exists(host_test_files_path): 126 if os.path.exists(host_test_files_path):
123 self.adb.PushIfNeeded(host_test_files_path, 127 self.adb.PushIfNeeded(host_test_files_path, '%s/%s/%s' % (
124 self.adb.GetExternalStorage() + '/' + 128 self.adb.GetExternalStorage(), TestRunner._DEVICE_DATA_DIR,
125 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) 129 dst_layer)
126 self.tool.CopyFiles() 130 self.tool.CopyFiles()
127 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True 131 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True
128 132
129 def _GetInstrumentationArgs(self): 133 def _GetInstrumentationArgs(self):
130 ret = {} 134 ret = {}
131 if self.wait_for_debugger: 135 if self.wait_for_debugger:
132 ret['debug'] = 'true' 136 ret['debug'] = 'true'
137 if self.coverage_dir:
138 ret['coverage'] = 'true'
139 ret['coverageFile'] = self.coverage_device_file
140
133 return ret 141 return ret
134 142
135 def _TakeScreenshot(self, test): 143 def _TakeScreenshot(self, test):
136 """Takes a screenshot from the device.""" 144 """Takes a screenshot from the device."""
137 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') 145 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, '%s.png' % (test))
138 logging.info('Taking screenshot named %s', screenshot_name) 146 logging.info('Taking screenshot named %s', screenshot_name)
139 self.adb.TakeScreenshot(screenshot_name) 147 self.adb.TakeScreenshot(screenshot_name)
140 148
141 def SetUp(self): 149 def SetUp(self):
142 """Sets up the test harness and device before all tests are run.""" 150 """Sets up the test harness and device before all tests are run."""
143 super(TestRunner, self).SetUp() 151 super(TestRunner, self).SetUp()
144 if not self.adb.IsRootEnabled(): 152 if not self.adb.IsRootEnabled():
145 logging.warning('Unable to enable java asserts for %s, non rooted device', 153 logging.warning('Unable to enable java asserts for %s, non rooted device',
146 self.device) 154 self.device)
147 else: 155 else:
(...skipping 19 matching lines...) Expand all
167 Args: 175 Args:
168 test: The name of the test that will be run. 176 test: The name of the test that will be run.
169 """ 177 """
170 self.SetupPerfMonitoringIfNeeded(test) 178 self.SetupPerfMonitoringIfNeeded(test)
171 self._SetupIndividualTestTimeoutScale(test) 179 self._SetupIndividualTestTimeoutScale(test)
172 self.tool.SetupEnvironment() 180 self.tool.SetupEnvironment()
173 181
174 # Make sure the forwarder is still running. 182 # Make sure the forwarder is still running.
175 self.RestartHttpServerForwarderIfNecessary() 183 self.RestartHttpServerForwarderIfNecessary()
176 184
185 if self.coverage_dir:
186 self.coverage_basename = '%s.ec' % (test)
frankf 2013/07/25 20:28:19 No need for ()
frankf 2013/07/25 20:28:19 This var is local now
gkanwar1 2013/08/01 02:14:35 Done.
gkanwar1 2013/08/01 02:14:35 Done.
187 self.coverage_device_file = '%s/%s/%s' % (self.adb.GetExternalStorage(),
188 TestRunner._DEVICE_COVERAGE_DIR,
189 self.coverage_basename)
190 self.coverage_host_file = os.path.join(
191 self.coverage_dir, self.coverage_basename)
192
177 def _IsPerfTest(self, test): 193 def _IsPerfTest(self, test):
178 """Determines whether a test is a performance test. 194 """Determines whether a test is a performance test.
179 195
180 Args: 196 Args:
181 test: The name of the test to be checked. 197 test: The name of the test to be checked.
182 198
183 Returns: 199 Returns:
184 Whether the test is annotated as a performance test. 200 Whether the test is annotated as a performance test.
185 """ 201 """
186 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test) 202 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test)
(...skipping 22 matching lines...) Expand all
209 """ 225 """
210 226
211 self.tool.CleanUpEnvironment() 227 self.tool.CleanUpEnvironment()
212 228
213 # The logic below relies on the test passing. 229 # The logic below relies on the test passing.
214 if not raw_result or raw_result.GetStatusCode(): 230 if not raw_result or raw_result.GetStatusCode():
215 return 231 return
216 232
217 self.TearDownPerfMonitoring(test) 233 self.TearDownPerfMonitoring(test)
218 234
235 if self.coverage_dir:
236 self.adb.Adb().Pull(self.coverage_device_file, self.coverage_host_file)
237 self.adb.RunShellCommand('rm %s' % self.coverage_device_file)
238
219 def TearDownPerfMonitoring(self, test): 239 def TearDownPerfMonitoring(self, test):
220 """Cleans up performance monitoring if the specified test required it. 240 """Cleans up performance monitoring if the specified test required it.
221 241
222 Args: 242 Args:
223 test: The name of the test that was just run. 243 test: The name of the test that was just run.
224 Raises: 244 Raises:
225 Exception: if there's anything wrong with the perf data. 245 Exception: if there's anything wrong with the perf data.
226 """ 246 """
227 if not self._IsPerfTest(test): 247 if not self._IsPerfTest(test):
228 return 248 return
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after
301 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations: 321 if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations:
302 return 5 * 60 322 return 5 * 60
303 if 'MediumTest' in annotations: 323 if 'MediumTest' in annotations:
304 return 3 * 60 324 return 3 * 60
305 return 1 * 60 325 return 1 * 60
306 326
307 def _RunTest(self, test, timeout): 327 def _RunTest(self, test, timeout):
308 try: 328 try:
309 return self.adb.RunInstrumentationTest( 329 return self.adb.RunInstrumentationTest(
310 test, self.test_pkg.GetPackageName(), 330 test, self.test_pkg.GetPackageName(),
311 self._GetInstrumentationArgs(), timeout) 331 self._GetInstrumentationArgs(test), timeout)
312 except android_commands.errors.WaitForResponseTimedOutError: 332 except android_commands.errors.WaitForResponseTimedOutError:
313 logging.info('Ran the test with timeout of %ds.' % timeout) 333 logging.info('Ran the test with timeout of %ds.' % timeout)
314 raise 334 raise
315 335
316 #override 336 #override
317 def RunTest(self, test): 337 def RunTest(self, test):
318 raw_result = None 338 raw_result = None
319 start_date_ms = None 339 start_date_ms = None
320 results = base_test_result.TestRunResults() 340 results = base_test_result.TestRunResults()
321 timeout=(self._GetIndividualTestTimeoutSecs(test) * 341 timeout=(self._GetIndividualTestTimeoutSecs(test) *
(...skipping 30 matching lines...) Expand all
352 duration_ms = 0 372 duration_ms = 0
353 message = str(e) 373 message = str(e)
354 if not message: 374 if not message:
355 message = 'No information.' 375 message = 'No information.'
356 results.AddResult(test_result.InstrumentationTestResult( 376 results.AddResult(test_result.InstrumentationTestResult(
357 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, 377 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms,
358 log=message)) 378 log=message))
359 raw_result = None 379 raw_result = None
360 self.TestTeardown(test, raw_result) 380 self.TestTeardown(test, raw_result)
361 return (results, None if results.DidRunPass() else test) 381 return (results, None if results.DidRunPass() else test)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698