Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(699)

Side by Side Diff: build/android/pylib/instrumentation/test_runner.py

Issue 20210002: [Android] Sets up a coverage system for java using EMMA (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Gets code coverage for non-apk java working Created 7 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Class for running instrumentation tests on a single device.""" 5 """Class for running instrumentation tests on a single device."""
6 6
7 import logging 7 import logging
8 import os 8 import os
9 import re 9 import re
10 import shutil 10 import shutil
(...skipping 29 matching lines...) Expand all
40 test_files += [ 40 test_files += [
41 'net/data/ssl/certificates/', 41 'net/data/ssl/certificates/',
42 ] 42 ]
43 return test_files 43 return test_files
44 44
45 45
46 class TestRunner(base_test_runner.BaseTestRunner): 46 class TestRunner(base_test_runner.BaseTestRunner):
47 """Responsible for running a series of tests connected to a single device.""" 47 """Responsible for running a series of tests connected to a single device."""
48 48
49 _DEVICE_DATA_DIR = 'chrome/test/data' 49 _DEVICE_DATA_DIR = 'chrome/test/data'
50 _DEVICE_COVERAGE_DIR = 'chrome/test/coverage'
50 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' 51 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile'
51 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + 52 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR +
52 '/chrome-profile*') 53 '/chrome-profile*')
53 _DEVICE_HAS_TEST_FILES = {} 54 _DEVICE_HAS_TEST_FILES = {}
54 55
55 def __init__(self, build_type, test_data, save_perf_json, screenshot_failures, 56 def __init__(self, build_type, test_data, save_perf_json, screenshot_failures,
56 tool, wait_for_debugger, disable_assertions, push_deps, 57 tool, wait_for_debugger, disable_assertions, push_deps,
57 cleanup_test_files, device, shard_index, test_pkg, 58 cleanup_test_files, device, shard_index, test_pkg,
58 ports_to_forward): 59 ports_to_forward, coverage_dir):
59 """Create a new TestRunner. 60 """Create a new TestRunner.
60 61
61 Args: 62 Args:
62 build_type: 'Release' or 'Debug'. 63 build_type: 'Release' or 'Debug'.
63 test_data: Location of the test data. 64 test_data: Location of the test data.
64 save_perf_json: Whether or not to save the JSON file from UI perf tests. 65 save_perf_json: Whether or not to save the JSON file from UI perf tests.
65 screenshot_failures: Take a screenshot for a test failure 66 screenshot_failures: Take a screenshot for a test failure
66 tool: Name of the Valgrind tool. 67 tool: Name of the Valgrind tool.
67 wait_for_debugger: Blocks until the debugger is connected. 68 wait_for_debugger: Blocks until the debugger is connected.
68 disable_assertions: Whether to disable java assertions on the device. 69 disable_assertions: Whether to disable java assertions on the device.
69 push_deps: If True, push all dependencies to the device. 70 push_deps: If True, push all dependencies to the device.
70 cleanup_test_files: Whether or not to cleanup test files on device. 71 cleanup_test_files: Whether or not to cleanup test files on device.
71 device: Attached android device. 72 device: Attached android device.
72 shard_index: Shard index. 73 shard_index: Shard index.
73 test_pkg: A TestPackage object. 74 test_pkg: A TestPackage object.
74 ports_to_forward: A list of port numbers for which to set up forwarders. 75 ports_to_forward: A list of port numbers for which to set up forwarders.
75 Can be optionally requested by a test case. 76 Can be optionally requested by a test case.
77 coverage_dir: Directory to pull all EMMA coverage files into, or None if
78 coverage is not being used.
76 """ 79 """
77 super(TestRunner, self).__init__(device, tool, build_type, push_deps, 80 super(TestRunner, self).__init__(device, tool, build_type, push_deps,
78 cleanup_test_files) 81 cleanup_test_files)
79 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index 82 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index
80 83
81 self.build_type = build_type 84 self.build_type = build_type
82 self.test_data = test_data 85 self.test_data = test_data
83 self.save_perf_json = save_perf_json 86 self.save_perf_json = save_perf_json
84 self.screenshot_failures = screenshot_failures 87 self.screenshot_failures = screenshot_failures
85 self.wait_for_debugger = wait_for_debugger 88 self.wait_for_debugger = wait_for_debugger
86 self.disable_assertions = disable_assertions 89 self.disable_assertions = disable_assertions
87 self.test_pkg = test_pkg 90 self.test_pkg = test_pkg
88 self.ports_to_forward = ports_to_forward 91 self.ports_to_forward = ports_to_forward
92 self.coverage_dir = coverage_dir
89 93
90 #override 94 #override
91 def InstallTestPackage(self): 95 def InstallTestPackage(self):
92 self.test_pkg.Install(self.adb) 96 self.test_pkg.Install(self.adb)
93 97
94 #override 98 #override
95 def PushDataDeps(self): 99 def PushDataDeps(self):
96 # TODO(frankf): Implement a general approach for copying/installing 100 # TODO(frankf): Implement a general approach for copying/installing
97 # once across test runners. 101 # once across test runners.
98 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): 102 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False):
99 logging.warning('Already copied test files to device %s, skipping.', 103 logging.warning('Already copied test files to device %s, skipping.',
100 self.device) 104 self.device)
101 return 105 return
102 106
103 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName()) 107 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName())
104 if test_data: 108 if test_data:
105 # Make sure SD card is ready. 109 # Make sure SD card is ready.
106 self.adb.WaitForSdCardReady(20) 110 self.adb.WaitForSdCardReady(20)
107 for p in test_data: 111 for p in test_data:
108 self.adb.PushIfNeeded( 112 self.adb.PushIfNeeded(
109 os.path.join(constants.DIR_SOURCE_ROOT, p), 113 os.path.join(constants.DIR_SOURCE_ROOT, p),
110 os.path.join(self.adb.GetExternalStorage(), p)) 114 os.path.join(self.adb.GetExternalStorage(), p))
111 115
112 # TODO(frankf): Specify test data in this file as opposed to passing 116 # TODO(frankf): Specify test data in this file as opposed to passing
113 # as command-line. 117 # as command-line.
114 for dest_host_pair in self.test_data: 118 for dest_host_pair in self.test_data:
115 dst_src = dest_host_pair.split(':',1) 119 dst_src = dest_host_pair.split(':',1)
116 dst_layer = dst_src[0] 120 dst_layer = dst_src[0]
117 host_src = dst_src[1] 121 host_src = dst_src[1]
118 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src 122 host_test_files_path = '%s/%s' % (constants.DIR_SOURCE_ROOT, host_src)
119 if os.path.exists(host_test_files_path): 123 if os.path.exists(host_test_files_path):
120 self.adb.PushIfNeeded(host_test_files_path, 124 self.adb.PushIfNeeded(host_test_files_path, '%s/%s/%s' % (
121 self.adb.GetExternalStorage() + '/' + 125 self.adb.GetExternalStorage(), TestRunner._DEVICE_DATA_DIR,
122 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) 126 dst_layer))
123 self.tool.CopyFiles() 127 self.tool.CopyFiles()
124 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True 128 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True
125 129
126 def _GetInstrumentationArgs(self): 130 def _GetInstrumentationArgs(self):
127 ret = {} 131 ret = {}
128 if self.wait_for_debugger: 132 if self.wait_for_debugger:
129 ret['debug'] = 'true' 133 ret['debug'] = 'true'
134 if self.coverage_dir:
135 ret['coverage'] = 'true'
136 ret['coverageFile'] = self.coverage_device_file
137
130 return ret 138 return ret
131 139
132 def _TakeScreenshot(self, test): 140 def _TakeScreenshot(self, test):
133 """Takes a screenshot from the device.""" 141 """Takes a screenshot from the device."""
134 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') 142 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, '%s.png' % test)
135 logging.info('Taking screenshot named %s', screenshot_name) 143 logging.info('Taking screenshot named %s', screenshot_name)
136 self.adb.TakeScreenshot(screenshot_name) 144 self.adb.TakeScreenshot(screenshot_name)
137 145
138 def SetUp(self): 146 def SetUp(self):
139 """Sets up the test harness and device before all tests are run.""" 147 """Sets up the test harness and device before all tests are run."""
140 super(TestRunner, self).SetUp() 148 super(TestRunner, self).SetUp()
141 if not self.adb.IsRootEnabled(): 149 if not self.adb.IsRootEnabled():
142 logging.warning('Unable to enable java asserts for %s, non rooted device', 150 logging.warning('Unable to enable java asserts for %s, non rooted device',
143 self.device) 151 self.device)
144 else: 152 else:
(...skipping 19 matching lines...) Expand all
164 Args: 172 Args:
165 test: The name of the test that will be run. 173 test: The name of the test that will be run.
166 """ 174 """
167 self.SetupPerfMonitoringIfNeeded(test) 175 self.SetupPerfMonitoringIfNeeded(test)
168 self._SetupIndividualTestTimeoutScale(test) 176 self._SetupIndividualTestTimeoutScale(test)
169 self.tool.SetupEnvironment() 177 self.tool.SetupEnvironment()
170 178
171 # Make sure the forwarder is still running. 179 # Make sure the forwarder is still running.
172 self.RestartHttpServerForwarderIfNecessary() 180 self.RestartHttpServerForwarderIfNecessary()
173 181
182 if self.coverage_dir:
183 coverage_basename = '%s.ec' % test
184 self.coverage_device_file = '%s/%s/%s' % (self.adb.GetExternalStorage(),
185 TestRunner._DEVICE_COVERAGE_DIR,
186 coverage_basename)
187 self.coverage_host_file = os.path.join(
188 self.coverage_dir, coverage_basename)
189
174 def _IsPerfTest(self, test): 190 def _IsPerfTest(self, test):
175 """Determines whether a test is a performance test. 191 """Determines whether a test is a performance test.
176 192
177 Args: 193 Args:
178 test: The name of the test to be checked. 194 test: The name of the test to be checked.
179 195
180 Returns: 196 Returns:
181 Whether the test is annotated as a performance test. 197 Whether the test is annotated as a performance test.
182 """ 198 """
183 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test) 199 return _PERF_TEST_ANNOTATION in self.test_pkg.GetTestAnnotations(test)
(...skipping 22 matching lines...) Expand all
206 """ 222 """
207 223
208 self.tool.CleanUpEnvironment() 224 self.tool.CleanUpEnvironment()
209 225
210 # The logic below relies on the test passing. 226 # The logic below relies on the test passing.
211 if not raw_result or raw_result.GetStatusCode(): 227 if not raw_result or raw_result.GetStatusCode():
212 return 228 return
213 229
214 self.TearDownPerfMonitoring(test) 230 self.TearDownPerfMonitoring(test)
215 231
232 if self.coverage_dir:
233 self.adb.Adb().Pull(self.coverage_device_file, self.coverage_host_file)
234 self.adb.RunShellCommand('rm %s' % self.coverage_device_file)
frankf 2013/08/01 19:57:41 rm -f
gkanwar1 2013/08/07 19:24:56 Done.
235
216 def TearDownPerfMonitoring(self, test): 236 def TearDownPerfMonitoring(self, test):
217 """Cleans up performance monitoring if the specified test required it. 237 """Cleans up performance monitoring if the specified test required it.
218 238
219 Args: 239 Args:
220 test: The name of the test that was just run. 240 test: The name of the test that was just run.
221 Raises: 241 Raises:
222 Exception: if there's anything wrong with the perf data. 242 Exception: if there's anything wrong with the perf data.
223 """ 243 """
224 if not self._IsPerfTest(test): 244 if not self._IsPerfTest(test):
225 return 245 return
(...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after
349 duration_ms = 0 369 duration_ms = 0
350 message = str(e) 370 message = str(e)
351 if not message: 371 if not message:
352 message = 'No information.' 372 message = 'No information.'
353 results.AddResult(test_result.InstrumentationTestResult( 373 results.AddResult(test_result.InstrumentationTestResult(
354 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, 374 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms,
355 log=message)) 375 log=message))
356 raw_result = None 376 raw_result = None
357 self.TestTeardown(test, raw_result) 377 self.TestTeardown(test, raw_result)
358 return (results, None if results.DidRunPass() else test) 378 return (results, None if results.DidRunPass() else test)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698