Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(418)

Side by Side Diff: build/android/pylib/instrumentation/test_runner.py

Issue 21008004: Changes argument passing to use options objects (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Rebases and fixes some issues Created 7 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Class for running instrumentation tests on a single device.""" 5 """Class for running instrumentation tests on a single device."""
6 6
7 import logging 7 import logging
8 import os 8 import os
9 import re 9 import re
10 import shutil
11 import sys
12 import time 10 import time
13 11
14 from pylib import android_commands 12 from pylib import android_commands
15 from pylib import cmd_helper
16 from pylib import constants 13 from pylib import constants
17 from pylib import json_perf_parser 14 from pylib import json_perf_parser
18 from pylib import perf_tests_helper 15 from pylib import perf_tests_helper
19 from pylib import valgrind_tools 16 from pylib import valgrind_tools
20 from pylib.base import base_test_result 17 from pylib.base import base_test_result
21 from pylib.base import base_test_runner 18 from pylib.base import base_test_runner
22 19
23 import test_result 20 import test_result
24 21
25 22
(...skipping 19 matching lines...) Expand all
45 42
46 class TestRunner(base_test_runner.BaseTestRunner): 43 class TestRunner(base_test_runner.BaseTestRunner):
47 """Responsible for running a series of tests connected to a single device.""" 44 """Responsible for running a series of tests connected to a single device."""
48 45
49 _DEVICE_DATA_DIR = 'chrome/test/data' 46 _DEVICE_DATA_DIR = 'chrome/test/data'
50 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile' 47 _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile'
51 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR + 48 _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR +
52 '/chrome-profile*') 49 '/chrome-profile*')
53 _DEVICE_HAS_TEST_FILES = {} 50 _DEVICE_HAS_TEST_FILES = {}
54 51
55 def __init__(self, build_type, test_data, save_perf_json, screenshot_failures, 52 def __init__(self, test_options, device, shard_index, test_pkg,
56 tool, wait_for_debugger, disable_assertions, push_deps,
57 cleanup_test_files, device, shard_index, test_pkg,
58 ports_to_forward): 53 ports_to_forward):
59 """Create a new TestRunner. 54 """Create a new TestRunner.
60 55
61 Args: 56 Args:
62 build_type: 'Release' or 'Debug'. 57 test_options: An InstrumentationOptions object containing the set of
63 test_data: Location of the test data. 58 options relevant to running instrumentation tests.
frankf 2013/07/31 19:03:56 too verbose. test_options: An InstrumentationOpti
gkanwar1 2013/07/31 19:09:30 Done.
64 save_perf_json: Whether or not to save the JSON file from UI perf tests.
65 screenshot_failures: Take a screenshot for a test failure
66 tool: Name of the Valgrind tool.
67 wait_for_debugger: Blocks until the debugger is connected.
68 disable_assertions: Whether to disable java assertions on the device.
69 push_deps: If True, push all dependencies to the device.
70 cleanup_test_files: Whether or not to cleanup test files on device.
71 device: Attached android device. 59 device: Attached android device.
72 shard_index: Shard index. 60 shard_index: Shard index.
73 test_pkg: A TestPackage object. 61 test_pkg: A TestPackage object.
74 ports_to_forward: A list of port numbers for which to set up forwarders. 62 ports_to_forward: A list of port numbers for which to set up forwarders.
75 Can be optionally requested by a test case. 63 Can be optionally requested by a test case.
76 """ 64 """
77 super(TestRunner, self).__init__(device, tool, build_type, push_deps, 65 super(TestRunner, self).__init__(device, test_options.tool,
78 cleanup_test_files) 66 test_options.build_type,
67 test_options.push_deps,
68 test_options.cleanup_test_files)
79 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index 69 self._lighttp_port = constants.LIGHTTPD_RANDOM_PORT_FIRST + shard_index
80 70
81 self.build_type = build_type 71 self.options = test_options
82 self.test_data = test_data
83 self.save_perf_json = save_perf_json
84 self.screenshot_failures = screenshot_failures
85 self.wait_for_debugger = wait_for_debugger
86 self.disable_assertions = disable_assertions
87 self.test_pkg = test_pkg 72 self.test_pkg = test_pkg
88 self.ports_to_forward = ports_to_forward 73 self.ports_to_forward = ports_to_forward
89 74
90 #override 75 #override
91 def InstallTestPackage(self): 76 def InstallTestPackage(self):
92 self.test_pkg.Install(self.adb) 77 self.test_pkg.Install(self.adb)
93 78
94 #override 79 #override
95 def PushDataDeps(self): 80 def PushDataDeps(self):
96 # TODO(frankf): Implement a general approach for copying/installing 81 # TODO(frankf): Implement a general approach for copying/installing
97 # once across test runners. 82 # once across test runners.
98 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False): 83 if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False):
99 logging.warning('Already copied test files to device %s, skipping.', 84 logging.warning('Already copied test files to device %s, skipping.',
100 self.device) 85 self.device)
101 return 86 return
102 87
103 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName()) 88 test_data = _GetDataFilesForTestSuite(self.test_pkg.GetApkName())
104 if test_data: 89 if test_data:
105 # Make sure SD card is ready. 90 # Make sure SD card is ready.
106 self.adb.WaitForSdCardReady(20) 91 self.adb.WaitForSdCardReady(20)
107 for p in test_data: 92 for p in test_data:
108 self.adb.PushIfNeeded( 93 self.adb.PushIfNeeded(
109 os.path.join(constants.DIR_SOURCE_ROOT, p), 94 os.path.join(constants.DIR_SOURCE_ROOT, p),
110 os.path.join(self.adb.GetExternalStorage(), p)) 95 os.path.join(self.adb.GetExternalStorage(), p))
111 96
112 # TODO(frankf): Specify test data in this file as opposed to passing 97 # TODO(frankf): Specify test data in this file as opposed to passing
113 # as command-line. 98 # as command-line.
114 for dest_host_pair in self.test_data: 99 for dest_host_pair in self.options.test_data:
115 dst_src = dest_host_pair.split(':',1) 100 dst_src = dest_host_pair.split(':',1)
116 dst_layer = dst_src[0] 101 dst_layer = dst_src[0]
117 host_src = dst_src[1] 102 host_src = dst_src[1]
118 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src 103 host_test_files_path = constants.DIR_SOURCE_ROOT + '/' + host_src
119 if os.path.exists(host_test_files_path): 104 if os.path.exists(host_test_files_path):
120 self.adb.PushIfNeeded(host_test_files_path, 105 self.adb.PushIfNeeded(host_test_files_path,
121 self.adb.GetExternalStorage() + '/' + 106 self.adb.GetExternalStorage() + '/' +
122 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer) 107 TestRunner._DEVICE_DATA_DIR + '/' + dst_layer)
123 self.tool.CopyFiles() 108 self.tool.CopyFiles()
124 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True 109 TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True
125 110
126 def _GetInstrumentationArgs(self): 111 def _GetInstrumentationArgs(self):
127 ret = {} 112 ret = {}
128 if self.wait_for_debugger: 113 if self.options.wait_for_debugger:
129 ret['debug'] = 'true' 114 ret['debug'] = 'true'
130 return ret 115 return ret
131 116
132 def _TakeScreenshot(self, test): 117 def _TakeScreenshot(self, test):
133 """Takes a screenshot from the device.""" 118 """Takes a screenshot from the device."""
134 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png') 119 screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png')
135 logging.info('Taking screenshot named %s', screenshot_name) 120 logging.info('Taking screenshot named %s', screenshot_name)
136 self.adb.TakeScreenshot(screenshot_name) 121 self.adb.TakeScreenshot(screenshot_name)
137 122
138 def SetUp(self): 123 def SetUp(self):
139 """Sets up the test harness and device before all tests are run.""" 124 """Sets up the test harness and device before all tests are run."""
140 super(TestRunner, self).SetUp() 125 super(TestRunner, self).SetUp()
141 if not self.adb.IsRootEnabled(): 126 if not self.adb.IsRootEnabled():
142 logging.warning('Unable to enable java asserts for %s, non rooted device', 127 logging.warning('Unable to enable java asserts for %s, non rooted device',
143 self.device) 128 self.device)
144 else: 129 else:
145 if self.adb.SetJavaAssertsEnabled(enable=not self.disable_assertions): 130 if self.adb.SetJavaAssertsEnabled(
131 enable=not self.options.disable_assertions):
146 self.adb.Reboot(full_reboot=False) 132 self.adb.Reboot(full_reboot=False)
147 133
148 # We give different default value to launch HTTP server based on shard index 134 # We give different default value to launch HTTP server based on shard index
149 # because it may have race condition when multiple processes are trying to 135 # because it may have race condition when multiple processes are trying to
150 # launch lighttpd with same port at same time. 136 # launch lighttpd with same port at same time.
151 http_server_ports = self.LaunchTestHttpServer( 137 http_server_ports = self.LaunchTestHttpServer(
152 os.path.join(constants.DIR_SOURCE_ROOT), self._lighttp_port) 138 os.path.join(constants.DIR_SOURCE_ROOT), self._lighttp_port)
153 if self.ports_to_forward: 139 if self.ports_to_forward:
154 self.StartForwarder([(port, port) for port in self.ports_to_forward]) 140 self.StartForwarder([(port, port) for port in self.ports_to_forward])
155 self.flags.AddFlags(['--enable-test-intents']) 141 self.flags.AddFlags(['--enable-test-intents'])
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after
241 # Obtain the relevant perf data. The data is dumped to a 227 # Obtain the relevant perf data. The data is dumped to a
242 # JSON formatted file. 228 # JSON formatted file.
243 json_string = self.adb.GetProtectedFileContents( 229 json_string = self.adb.GetProtectedFileContents(
244 '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt') 230 '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt')
245 231
246 if json_string: 232 if json_string:
247 json_string = '\n'.join(json_string) 233 json_string = '\n'.join(json_string)
248 else: 234 else:
249 raise Exception('Perf file does not exist or is empty') 235 raise Exception('Perf file does not exist or is empty')
250 236
251 if self.save_perf_json: 237 if self.options.save_perf_json:
252 json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name 238 json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name
253 with open(json_local_file, 'w') as f: 239 with open(json_local_file, 'w') as f:
254 f.write(json_string) 240 f.write(json_string)
255 logging.info('Saving Perf UI JSON from test ' + 241 logging.info('Saving Perf UI JSON from test ' +
256 test + ' to ' + json_local_file) 242 test + ' to ' + json_local_file)
257 243
258 raw_perf_data = regex.group(1).split(';') 244 raw_perf_data = regex.group(1).split(';')
259 245
260 for raw_perf_set in raw_perf_data: 246 for raw_perf_set in raw_perf_data:
261 if raw_perf_set: 247 if raw_perf_set:
(...skipping 15 matching lines...) Expand all
277 263
278 def _GetIndividualTestTimeoutScale(self, test): 264 def _GetIndividualTestTimeoutScale(self, test):
279 """Returns the timeout scale for the given |test|.""" 265 """Returns the timeout scale for the given |test|."""
280 annotations = self.test_pkg.GetTestAnnotations(test) 266 annotations = self.test_pkg.GetTestAnnotations(test)
281 timeout_scale = 1 267 timeout_scale = 1
282 if 'TimeoutScale' in annotations: 268 if 'TimeoutScale' in annotations:
283 for annotation in annotations: 269 for annotation in annotations:
284 scale_match = re.match('TimeoutScale:([0-9]+)', annotation) 270 scale_match = re.match('TimeoutScale:([0-9]+)', annotation)
285 if scale_match: 271 if scale_match:
286 timeout_scale = int(scale_match.group(1)) 272 timeout_scale = int(scale_match.group(1))
287 if self.wait_for_debugger: 273 if self.options.wait_for_debugger:
288 timeout_scale *= 100 274 timeout_scale *= 100
289 return timeout_scale 275 return timeout_scale
290 276
291 def _GetIndividualTestTimeoutSecs(self, test): 277 def _GetIndividualTestTimeoutSecs(self, test):
292 """Returns the timeout in seconds for the given |test|.""" 278 """Returns the timeout in seconds for the given |test|."""
293 annotations = self.test_pkg.GetTestAnnotations(test) 279 annotations = self.test_pkg.GetTestAnnotations(test)
294 if 'Manual' in annotations: 280 if 'Manual' in annotations:
295 return 600 * 60 281 return 600 * 60
296 if 'External' in annotations: 282 if 'External' in annotations:
297 return 10 * 60 283 return 10 * 60
(...skipping 23 matching lines...) Expand all
321 try: 307 try:
322 self.TestSetup(test) 308 self.TestSetup(test)
323 start_date_ms = int(time.time()) * 1000 309 start_date_ms = int(time.time()) * 1000
324 raw_result = self._RunTest(test, timeout) 310 raw_result = self._RunTest(test, timeout)
325 duration_ms = int(time.time()) * 1000 - start_date_ms 311 duration_ms = int(time.time()) * 1000 - start_date_ms
326 status_code = raw_result.GetStatusCode() 312 status_code = raw_result.GetStatusCode()
327 if status_code: 313 if status_code:
328 log = raw_result.GetFailureReason() 314 log = raw_result.GetFailureReason()
329 if not log: 315 if not log:
330 log = 'No information.' 316 log = 'No information.'
331 if self.screenshot_failures or log.find('INJECT_EVENTS perm') >= 0: 317 if (self.options.screenshot_failures or
318 log.find('INJECT_EVENTS perm') >= 0):
332 self._TakeScreenshot(test) 319 self._TakeScreenshot(test)
333 result = test_result.InstrumentationTestResult( 320 result = test_result.InstrumentationTestResult(
334 test, base_test_result.ResultType.FAIL, start_date_ms, duration_ms, 321 test, base_test_result.ResultType.FAIL, start_date_ms, duration_ms,
335 log=log) 322 log=log)
336 else: 323 else:
337 result = test_result.InstrumentationTestResult( 324 result = test_result.InstrumentationTestResult(
338 test, base_test_result.ResultType.PASS, start_date_ms, duration_ms) 325 test, base_test_result.ResultType.PASS, start_date_ms, duration_ms)
339 results.AddResult(result) 326 results.AddResult(result)
340 # Catch exceptions thrown by StartInstrumentation(). 327 # Catch exceptions thrown by StartInstrumentation().
341 # See ../../third_party/android/testrunner/adb_interface.py 328 # See ../../third_party/android/testrunner/adb_interface.py
342 except (android_commands.errors.WaitForResponseTimedOutError, 329 except (android_commands.errors.WaitForResponseTimedOutError,
343 android_commands.errors.DeviceUnresponsiveError, 330 android_commands.errors.DeviceUnresponsiveError,
344 android_commands.errors.InstrumentationError), e: 331 android_commands.errors.InstrumentationError), e:
345 if start_date_ms: 332 if start_date_ms:
346 duration_ms = int(time.time()) * 1000 - start_date_ms 333 duration_ms = int(time.time()) * 1000 - start_date_ms
347 else: 334 else:
348 start_date_ms = int(time.time()) * 1000 335 start_date_ms = int(time.time()) * 1000
349 duration_ms = 0 336 duration_ms = 0
350 message = str(e) 337 message = str(e)
351 if not message: 338 if not message:
352 message = 'No information.' 339 message = 'No information.'
353 results.AddResult(test_result.InstrumentationTestResult( 340 results.AddResult(test_result.InstrumentationTestResult(
354 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, 341 test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms,
355 log=message)) 342 log=message))
356 raw_result = None 343 raw_result = None
357 self.TestTeardown(test, raw_result) 344 self.TestTeardown(test, raw_result)
358 return (results, None if results.DidRunPass() else test) 345 return (results, None if results.DidRunPass() else test)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698