Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(779)

Side by Side Diff: build/android/pylib/instrumentation/test_runner.py

Issue 21008004: Changes argument passing to use options objects (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 7 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Class for running instrumentation tests on a single device.""" 5 """Class for running instrumentation tests on a single device."""
6 6
7 import logging 7 import logging
8 import os 8 import os
9 import re 9 import re
10 import shutil
11 import sys
12 import time 10 import time
13 11
14 from pylib import android_commands 12 from pylib import android_commands
15 from pylib import cmd_helper
16 from pylib import constants 13 from pylib import constants
17 from pylib import json_perf_parser 14 from pylib import json_perf_parser
18 from pylib import perf_tests_helper 15 from pylib import perf_tests_helper
19 from pylib import valgrind_tools 16 from pylib import valgrind_tools
20 from pylib.base import base_test_result 17 from pylib.base import base_test_result
21 from pylib.base import base_test_runner 18 from pylib.base import base_test_runner
22 19
23 import test_result 20 import test_result
24 21
25 22
(...skipping 19 matching lines...) Expand all
45 42
46 class TestRunner(base_test_runner.BaseTestRunner): 43 class TestRunner(base_test_runner.BaseTestRunner):
47 """Responsible for running a series of tests connected to a single device.""" 44 """Responsible for running a series of tests connected to a single device."""
48 45
49 _DEVICE_DATA_DIR = 'chrome/test/data' 46 _DEVICE_DATA_DIR = 'chrome/test/data'
50 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' 47 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile'
51 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + 48 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR +
52 '/chrome-profile*') 49 '/chrome-profile*')
53 _DEVICE_HAS_TEST_FILES = {} 50 _DEVICE_HAS_TEST_FILES = {}
54 51
55 def __init__(self, build_type, test_data, install_apk, save_perf_json, 52 def __init__(self, test_options, device, shard_index, test_pkg,
56 screenshot_failures, tool, wait_for_debugger, disable_assertions,
57 push_deps, cleanup_test_files, device, shard_index, test_pkg,
58 ports_to_forward): 53 ports_to_forward):
59 """Create a new TestRunner. 54 """Create a new TestRunner.
60 55
61 Args: 56 Args:
62 build_type: 'Release' or 'Debug'. 57 test_options: An InstrumentationOptions object containing the set of
63 test_data: Location of the test data. 58 options relevant to running instrumentation tests.
64 install_apk: Re-installs the apk if opted.
65 save_perf_json: Whether or not to save the JSON file from UI perf tests.
66 screenshot_failures: Take a screenshot for a test failure
67 tool: Name of the Valgrind tool.
68 wait_for_debugger: Blocks until the debugger is connected.
69 disable_assertions: Whether to disable java assertions on the device.
70 push_deps: If True, push all dependencies to the device.
71 cleanup_test_files: Whether or not to cleanup test files on device.
72 device: Attached android device. 59 device: Attached android device.
73 shard_index: Shard index. 60 shard_index: Shard index.
74 test_pkg: A TestPackage object. 61 test_pkg: A TestPackage object.
75 ports_to_forward: A list of port numbers for which to set up forwarders. 62 ports_to_forward: A list of port numbers for which to set up forwarders.
76 Can be optionally requested by a test case. 63 Can be optionally requested by a test case.
77 """ 64 """
78 super(TestRunner, self).__init__(device, tool, build_type, push_deps, 65 super(TestRunner, self).__init__(device, test_options.tool,
79 cleanup_test_files) 66 test_options.build_type,
67 test_options.push_deps,
68 test_options.cleanup_test_files)
80 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index 69 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index
81 70
82 self.build_type = build_type 71 self.options = test_options
83 self.test_data = test_data
84 self.save_perf_json = save_perf_json
85 self.screenshot_failures = screenshot_failures
86 self.wait_for_debugger = wait_for_debugger
87 self.disable_assertions = disable_assertions
88 self.test_pkg = test_pkg 72 self.test_pkg = test_pkg
89 self.ports_to_forward = ports_to_forward 73 self.ports_to_forward = ports_to_forward
90 self.install_apk = install_apk
91 74
92 #override 75 #override
93 def InstallTestPackage(self): 76 def InstallTestPackage(self):
94 if self.install_apk: 77 if self.options.install_apk:
95 self.test_pkg.Install(self.adb) 78 self.test_pkg.Install(self.adb)
96 79
97 #override 80 #override
98 def PushDataDeps(self): 81 def PushDataDeps(self):
99 # TODO(frankf): Implement a general approach for copying/installing 82 # TODO(frankf): Implement a general approach for copying/installing
100 # once across test runners. 83 # once across test runners.
101 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): 84 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False):
102 logging.warning('Already copied test files to device %s, skipping.', 85 logging.warning('Already copied test files to device %s, skipping.',
103 self.device) 86 self.device)
104 return 87 return
105 88
106 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName()) 89 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName())
107 if test_data: 90 if test_data:
108 # Make sure SD card is ready. 91 # Make sure SD card is ready.
109 self.adb.WaitForSdCardReady(20) 92 self.adb.WaitForSdCardReady(20)
110 for p in test_data: 93 for p in test_data:
111 self.adb.PushIfNeeded( 94 self.adb.PushIfNeeded(
112 os.path.join(constants.DIR_SOURCE_ROOT, p), 95 os.path.join(constants.DIR_SOURCE_ROOT, p),
113 os.path.join(self.adb.GetExternalStorage(), p)) 96 os.path.join(self.adb.GetExternalStorage(), p))
114 97
115 # TODO(frankf): Specify test data in this file as opposed to passing 98 # TODO(frankf): Specify test data in this file as opposed to passing
116 # as command-line. 99 # as command-line.
117 for dest_host_pair in self.test_data: 100 for dest_host_pair in self.options.test_data:
118 dst_src = dest_host_pair.split(':',1) 101 dst_src = dest_host_pair.split(':',1)
119 dst_layer = dst_src[0] 102 dst_layer = dst_src[0]
120 host_src = dst_src[1] 103 host_src = dst_src[1]
121 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src 104 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src
122 if os.path.exists(host_test_files_path): 105 if os.path.exists(host_test_files_path):
123 self.adb.PushIfNeeded(host_test_files_path, 106 self.adb.PushIfNeeded(host_test_files_path,
124 self.adb.GetExternalStorage() + '/' + 107 self.adb.GetExternalStorage() + '/' +
125 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) 108 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer)
126 self.tool.CopyFiles() 109 self.tool.CopyFiles()
127 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True 110 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True
128 111
129 def _GetInstrumentationArgs(self): 112 def _GetInstrumentationArgs(self):
130 ret = {} 113 ret = {}
131 if self.wait_for_debugger: 114 if self.options.wait_for_debugger:
132 ret['debug'] = 'true' 115 ret['debug'] = 'true'
133 return ret 116 return ret
134 117
135 def _TakeScreenshot(self, test): 118 def _TakeScreenshot(self, test):
136 """Takes a screenshot from the device.""" 119 """Takes a screenshot from the device."""
137 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') 120 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png')
138 logging.info('Taking screenshot named %s', screenshot_name) 121 logging.info('Taking screenshot named %s', screenshot_name)
139 self.adb.TakeScreenshot(screenshot_name) 122 self.adb.TakeScreenshot(screenshot_name)
140 123
141 def SetUp(self): 124 def SetUp(self):
142 """Sets up the test harness and device before all tests are run.""" 125 """Sets up the test harness and device before all tests are run."""
143 super(TestRunner, self).SetUp() 126 super(TestRunner, self).SetUp()
144 if not self.adb.IsRootEnabled(): 127 if not self.adb.IsRootEnabled():
145 logging.warning('Unable to enable java asserts for %s, non rooted device', 128 logging.warning('Unable to enable java asserts for %s, non rooted device',
146 self.device) 129 self.device)
147 else: 130 else:
148 if self.adb.SetJavaAssertsEnabled(enable=not self.disable_assertions): 131 if self.adb.SetJavaAssertsEnabled(
132 enable=not self.options.disable_assertions):
149 self.adb.Reboot(full_reboot=False) 133 self.adb.Reboot(full_reboot=False)
150 134
151 # We give different default value to launch HTTP server based on shard index 135 # We give different default value to launch HTTP server based on shard index
152 # because it may have race condition when multiple processes are trying to 136 # because it may have race condition when multiple processes are trying to
153 # launch lighttpd with same port at same time. 137 # launch lighttpd with same port at same time.
154 http_server_ports = self.LaunchTestHttpServer( 138 http_server_ports = self.LaunchTestHttpServer(
155 os.path.join(constants.DIR_SOURCE_ROOT), self._lighttp_port) 139 os.path.join(constants.DIR_SOURCE_ROOT), self._lighttp_port)
156 if self.ports_to_forward: 140 if self.ports_to_forward:
157 self.StartForwarder([(port, port) for port in self.ports_to_forward]) 141 self.StartForwarder([(port, port) for port in self.ports_to_forward])
158 self.flags.AddFlags(['--enable-test-intents']) 142 self.flags.AddFlags(['--enable-test-intents'])
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after
244 # Obtain the relevant perf data. The data is dumped to a 228 # Obtain the relevant perf data. The data is dumped to a
245 # JSON formatted file. 229 # JSON formatted file.
246 json_string = self.adb.GetProtectedFileContents( 230 json_string = self.adb.GetProtectedFileContents(
247 '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt') 231 '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt')
248 232
249 if json_string: 233 if json_string:
250 json_string = '\n'.join(json_string) 234 json_string = '\n'.join(json_string)
251 else: 235 else:
252 raise Exception('Perf file does not exist or is empty') 236 raise Exception('Perf file does not exist or is empty')
253 237
254 if self.save_perf_json: 238 if self.options.save_perf_json:
255 json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name 239 json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name
256 with open(json_local_file, 'w') as f: 240 with open(json_local_file, 'w') as f:
257 f.write(json_string) 241 f.write(json_string)
258 logging.info('Saving Perf UI JSON from test ' + 242 logging.info('Saving Perf UI JSON from test ' +
259 test + ' to ' + json_local_file) 243 test + ' to ' + json_local_file)
260 244
261 raw_perf_data = regex.group(1).split(';') 245 raw_perf_data = regex.group(1).split(';')
262 246
263 for raw_perf_set in raw_perf_data: 247 for raw_perf_set in raw_perf_data:
264 if raw_perf_set: 248 if raw_perf_set:
(...skipping 15 matching lines...) Expand all
280 264
281 def _GetIndividualTestTimeoutScale(self, test): 265 def _GetIndividualTestTimeoutScale(self, test):
282 """Returns the timeout scale for the given |test|.""" 266 """Returns the timeout scale for the given |test|."""
283 annotations = self.test_pkg.GetTestAnnotations(test) 267 annotations = self.test_pkg.GetTestAnnotations(test)
284 timeout_scale = 1 268 timeout_scale = 1
285 if 'TimeoutScale' in annotations: 269 if 'TimeoutScale' in annotations:
286 for annotation in annotations: 270 for annotation in annotations:
287 scale_match = re.match('TimeoutScale:([0-9]+)', annotation) 271 scale_match = re.match('TimeoutScale:([0-9]+)', annotation)
288 if scale_match: 272 if scale_match:
289 timeout_scale = int(scale_match.group(1)) 273 timeout_scale = int(scale_match.group(1))
290 if self.wait_for_debugger: 274 if self.options.wait_for_debugger:
291 timeout_scale *= 100 275 timeout_scale *= 100
292 return timeout_scale 276 return timeout_scale
293 277
294 def _GetIndividualTestTimeoutSecs(self, test): 278 def _GetIndividualTestTimeoutSecs(self, test):
295 """Returns the timeout in seconds for the given |test|.""" 279 """Returns the timeout in seconds for the given |test|."""
296 annotations = self.test_pkg.GetTestAnnotations(test) 280 annotations = self.test_pkg.GetTestAnnotations(test)
297 if 'Manual' in annotations: 281 if 'Manual' in annotations:
298 return 600 * 60 282 return 600 * 60
299 if 'External' in annotations: 283 if 'External' in annotations:
300 return 10 * 60 284 return 10 * 60
(...skipping 23 matching lines...) Expand all
324 try: 308 try:
325 self.TestSetup(test) 309 self.TestSetup(test)
326 start_date_ms = int(time.time()) * 1000 310 start_date_ms = int(time.time()) * 1000
327 raw_result = self._RunTest(test, timeout) 311 raw_result = self._RunTest(test, timeout)
328 duration_ms = int(time.time()) * 1000 - start_date_ms 312 duration_ms = int(time.time()) * 1000 - start_date_ms
329 status_code = raw_result.GetStatusCode() 313 status_code = raw_result.GetStatusCode()
330 if status_code: 314 if status_code:
331 log = raw_result.GetFailureReason() 315 log = raw_result.GetFailureReason()
332 if not log: 316 if not log:
333 log = 'No information.' 317 log = 'No information.'
334 if self.screenshot_failures or log.find('INJECT_EVENTS perm') >= 0: 318 if (self.options.screenshot_failures or
319 log.find('INJECT_EVENTS perm') >= 0):
335 self._TakeScreenshot(test) 320 self._TakeScreenshot(test)
336 result = test_result.InstrumentationTestResult( 321 result = test_result.InstrumentationTestResult(
337 test, base_test_result.ResultType.FAIL, start_date_ms, duration_ms, 322 test, base_test_result.ResultType.FAIL, start_date_ms, duration_ms,
338 log=log) 323 log=log)
339 else: 324 else:
340 result = test_result.InstrumentationTestResult( 325 result = test_result.InstrumentationTestResult(
341 test, base_test_result.ResultType.PASS, start_date_ms, duration_ms) 326 test, base_test_result.ResultType.PASS, start_date_ms, duration_ms)
342 results.AddResult(result) 327 results.AddResult(result)
343 # Catch exceptions thrown by StartInstrumentation(). 328 # Catch exceptions thrown by StartInstrumentation().
344 # See ../../third_party/android/testrunner/adb_interface.py 329 # See ../../third_party/android/testrunner/adb_interface.py
345 except (android_commands.errors.WaitForResponseTimedOutError, 330 except (android_commands.errors.WaitForResponseTimedOutError,
346 android_commands.errors.DeviceUnresponsiveError, 331 android_commands.errors.DeviceUnresponsiveError,
347 android_commands.errors.InstrumentationError), e: 332 android_commands.errors.InstrumentationError), e:
348 if start_date_ms: 333 if start_date_ms:
349 duration_ms = int(time.time()) * 1000 - start_date_ms 334 duration_ms = int(time.time()) * 1000 - start_date_ms
350 else: 335 else:
351 start_date_ms = int(time.time()) * 1000 336 start_date_ms = int(time.time()) * 1000
352 duration_ms = 0 337 duration_ms = 0
353 message = str(e) 338 message = str(e)
354 if not message: 339 if not message:
355 message = 'No information.' 340 message = 'No information.'
356 results.AddResult(test_result.InstrumentationTestResult( 341 results.AddResult(test_result.InstrumentationTestResult(
357 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, 342 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms,
358 log=message)) 343 log=message))
359 raw_result = None 344 raw_result = None
360 self.TestTeardown(test, raw_result) 345 self.TestTeardown(test, raw_result)
361 return (results, None if results.DidRunPass() else test) 346 return (results, None if results.DidRunPass() else test)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698