OLD | NEW |
| (Empty) |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 """Run specific test on specific environment.""" | |
6 | |
7 import json | |
8 import logging | |
9 import os | |
10 import sys | |
11 import tempfile | |
12 import time | |
13 import zipfile | |
14 | |
15 from pylib import constants | |
16 from pylib.base import test_run | |
17 from pylib.remote.device import appurify_constants | |
18 from pylib.remote.device import appurify_sanitized | |
19 from pylib.remote.device import remote_device_helper | |
20 from pylib.utils import zip_utils | |
21 | |
22 class RemoteDeviceTestRun(test_run.TestRun): | |
23 """Run tests on a remote device.""" | |
24 | |
25 _TEST_RUN_KEY = 'test_run' | |
26 _TEST_RUN_ID_KEY = 'test_run_id' | |
27 | |
28 WAIT_TIME = 5 | |
29 COMPLETE = 'complete' | |
30 HEARTBEAT_INTERVAL = 300 | |
31 | |
32 def __init__(self, env, test_instance): | |
33 """Constructor. | |
34 | |
35 Args: | |
36 env: Environment the tests will run in. | |
37 test_instance: The test that will be run. | |
38 """ | |
39 super(RemoteDeviceTestRun, self).__init__(env, test_instance) | |
40 self._env = env | |
41 self._test_instance = test_instance | |
42 self._app_id = '' | |
43 self._test_id = '' | |
44 self._results = '' | |
45 self._test_run_id = '' | |
46 | |
47 #override | |
48 def SetUp(self): | |
49 """Set up a test run.""" | |
50 if self._env.trigger: | |
51 self._TriggerSetUp() | |
52 elif self._env.collect: | |
53 assert isinstance(self._env.collect, basestring), ( | |
54 'File for storing test_run_id must be a string.') | |
55 with open(self._env.collect, 'r') as persisted_data_file: | |
56 persisted_data = json.loads(persisted_data_file.read()) | |
57 self._env.LoadFrom(persisted_data) | |
58 self.LoadFrom(persisted_data) | |
59 | |
60 def _TriggerSetUp(self): | |
61 """Set up the triggering of a test run.""" | |
62 raise NotImplementedError | |
63 | |
64 #override | |
65 def RunTests(self): | |
66 """Run the test.""" | |
67 if self._env.trigger: | |
68 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
69 logging.WARNING): | |
70 test_start_res = appurify_sanitized.api.tests_run( | |
71 self._env.token, self._env.device_type_id, self._app_id, | |
72 self._test_id) | |
73 remote_device_helper.TestHttpResponse( | |
74 test_start_res, 'Unable to run test.') | |
75 self._test_run_id = test_start_res.json()['response']['test_run_id'] | |
76 logging.info('Test run id: %s' % self._test_run_id) | |
77 | |
78 if self._env.collect: | |
79 current_status = '' | |
80 timeout_counter = 0 | |
81 heartbeat_counter = 0 | |
82 while self._GetTestStatus(self._test_run_id) != self.COMPLETE: | |
83 if self._results['detailed_status'] != current_status: | |
84 logging.info('Test status: %s', self._results['detailed_status']) | |
85 current_status = self._results['detailed_status'] | |
86 timeout_counter = 0 | |
87 heartbeat_counter = 0 | |
88 if heartbeat_counter > self.HEARTBEAT_INTERVAL: | |
89 logging.info('Test status: %s', self._results['detailed_status']) | |
90 heartbeat_counter = 0 | |
91 | |
92 timeout = self._env.timeouts.get( | |
93 current_status, self._env.timeouts['unknown']) | |
94 if timeout_counter > timeout: | |
95 raise remote_device_helper.RemoteDeviceError( | |
96 'Timeout while in %s state for %s seconds' | |
97 % (current_status, timeout), | |
98 is_infra_error=True) | |
99 time.sleep(self.WAIT_TIME) | |
100 timeout_counter += self.WAIT_TIME | |
101 heartbeat_counter += self.WAIT_TIME | |
102 self._DownloadTestResults(self._env.results_path) | |
103 | |
104 if self._results['results']['exception']: | |
105 raise remote_device_helper.RemoteDeviceError( | |
106 self._results['results']['exception'], is_infra_error=True) | |
107 | |
108 return self._ParseTestResults() | |
109 | |
110 #override | |
111 def TearDown(self): | |
112 """Tear down the test run.""" | |
113 if self._env.collect: | |
114 self._CollectTearDown() | |
115 elif self._env.trigger: | |
116 assert isinstance(self._env.trigger, basestring), ( | |
117 'File for storing test_run_id must be a string.') | |
118 with open(self._env.trigger, 'w') as persisted_data_file: | |
119 persisted_data = {} | |
120 self.DumpTo(persisted_data) | |
121 self._env.DumpTo(persisted_data) | |
122 persisted_data_file.write(json.dumps(persisted_data)) | |
123 | |
124 def _CollectTearDown(self): | |
125 if self._GetTestStatus(self._test_run_id) != self.COMPLETE: | |
126 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
127 logging.WARNING): | |
128 test_abort_res = appurify_sanitized.api.tests_abort( | |
129 self._env.token, self._test_run_id, reason='Test runner exiting.') | |
130 remote_device_helper.TestHttpResponse(test_abort_res, | |
131 'Unable to abort test.') | |
132 | |
133 def __enter__(self): | |
134 """Set up the test run when used as a context manager.""" | |
135 self.SetUp() | |
136 return self | |
137 | |
138 def __exit__(self, exc_type, exc_val, exc_tb): | |
139 """Tear down the test run when used as a context manager.""" | |
140 self.TearDown() | |
141 | |
142 def DumpTo(self, persisted_data): | |
143 test_run_data = { | |
144 self._TEST_RUN_ID_KEY: self._test_run_id, | |
145 } | |
146 persisted_data[self._TEST_RUN_KEY] = test_run_data | |
147 | |
148 def LoadFrom(self, persisted_data): | |
149 test_run_data = persisted_data[self._TEST_RUN_KEY] | |
150 self._test_run_id = test_run_data[self._TEST_RUN_ID_KEY] | |
151 | |
152 def _ParseTestResults(self): | |
153 raise NotImplementedError | |
154 | |
155 def _GetTestByName(self, test_name): | |
156 """Gets test_id for specific test. | |
157 | |
158 Args: | |
159 test_name: Test to find the ID of. | |
160 """ | |
161 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
162 logging.WARNING): | |
163 test_list_res = appurify_sanitized.api.tests_list(self._env.token) | |
164 remote_device_helper.TestHttpResponse(test_list_res, | |
165 'Unable to get tests list.') | |
166 for test in test_list_res.json()['response']: | |
167 if test['test_type'] == test_name: | |
168 return test['test_id'] | |
169 raise remote_device_helper.RemoteDeviceError( | |
170 'No test found with name %s' % (test_name)) | |
171 | |
172 def _DownloadTestResults(self, results_path): | |
173 """Download the test results from remote device service. | |
174 | |
175 Args: | |
176 results_path: Path to download appurify results zipfile. | |
177 """ | |
178 if results_path: | |
179 logging.info('Downloading results to %s.' % results_path) | |
180 if not os.path.exists(os.path.dirname(results_path)): | |
181 os.makedirs(os.path.dirname(results_path)) | |
182 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
183 logging.WARNING): | |
184 appurify_sanitized.utils.wget(self._results['results']['url'], | |
185 results_path) | |
186 | |
187 def _GetTestStatus(self, test_run_id): | |
188 """Checks the state of the test, and sets self._results | |
189 | |
190 Args: | |
191 test_run_id: Id of test on on remote service. | |
192 """ | |
193 | |
194 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
195 logging.WARNING): | |
196 test_check_res = appurify_sanitized.api.tests_check_result( | |
197 self._env.token, test_run_id) | |
198 remote_device_helper.TestHttpResponse(test_check_res, | |
199 'Unable to get test status.') | |
200 self._results = test_check_res.json()['response'] | |
201 return self._results['status'] | |
202 | |
203 def _AmInstrumentTestSetup(self, app_path, test_path, runner_package, | |
204 environment_variables, extra_apks=None): | |
205 config = {'runner': runner_package} | |
206 if environment_variables: | |
207 config['environment_vars'] = ','.join( | |
208 '%s=%s' % (k, v) for k, v in environment_variables.iteritems()) | |
209 | |
210 self._app_id = self._UploadAppToDevice(app_path) | |
211 | |
212 data_deps = self._test_instance.GetDataDependencies() | |
213 if data_deps: | |
214 with tempfile.NamedTemporaryFile(suffix='.zip') as test_with_deps: | |
215 sdcard_files = [] | |
216 additional_apks = [] | |
217 host_test = os.path.basename(test_path) | |
218 with zipfile.ZipFile(test_with_deps.name, 'w') as zip_file: | |
219 zip_file.write(test_path, host_test, zipfile.ZIP_DEFLATED) | |
220 for h, _ in data_deps: | |
221 if os.path.isdir(h): | |
222 zip_utils.WriteToZipFile(zip_file, h, '.') | |
223 sdcard_files.extend(os.listdir(h)) | |
224 else: | |
225 zip_utils.WriteToZipFile(zip_file, h, os.path.basename(h)) | |
226 sdcard_files.append(os.path.basename(h)) | |
227 for a in extra_apks or (): | |
228 zip_utils.WriteToZipFile(zip_file, a, os.path.basename(a)); | |
229 additional_apks.append(os.path.basename(a)) | |
230 | |
231 config['sdcard_files'] = ','.join(sdcard_files) | |
232 config['host_test'] = host_test | |
233 if additional_apks: | |
234 config['additional_apks'] = ','.join(additional_apks) | |
235 self._test_id = self._UploadTestToDevice( | |
236 'robotium', test_with_deps.name, app_id=self._app_id) | |
237 else: | |
238 self._test_id = self._UploadTestToDevice('robotium', test_path) | |
239 | |
240 logging.info('Setting config: %s' % config) | |
241 appurify_configs = {} | |
242 if self._env.network_config: | |
243 appurify_configs['network'] = self._env.network_config | |
244 self._SetTestConfig('robotium', config, **appurify_configs) | |
245 | |
246 def _UploadAppToDevice(self, app_path): | |
247 """Upload app to device.""" | |
248 logging.info('Uploading %s to remote service as %s.', app_path, | |
249 self._test_instance.suite) | |
250 with open(app_path, 'rb') as apk_src: | |
251 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
252 logging.WARNING): | |
253 upload_results = appurify_sanitized.api.apps_upload( | |
254 self._env.token, apk_src, 'raw', name=self._test_instance.suite) | |
255 remote_device_helper.TestHttpResponse( | |
256 upload_results, 'Unable to upload %s.' % app_path) | |
257 return upload_results.json()['response']['app_id'] | |
258 | |
259 def _UploadTestToDevice(self, test_type, test_path, app_id=None): | |
260 """Upload test to device | |
261 Args: | |
262 test_type: Type of test that is being uploaded. Ex. uirobot, gtest.. | |
263 """ | |
264 logging.info('Uploading %s to remote service.' % test_path) | |
265 with open(test_path, 'rb') as test_src: | |
266 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
267 logging.WARNING): | |
268 upload_results = appurify_sanitized.api.tests_upload( | |
269 self._env.token, test_src, 'raw', test_type, app_id=app_id) | |
270 remote_device_helper.TestHttpResponse(upload_results, | |
271 'Unable to upload %s.' % test_path) | |
272 return upload_results.json()['response']['test_id'] | |
273 | |
274 def _SetTestConfig(self, runner_type, runner_configs, | |
275 network=appurify_constants.NETWORK.WIFI_1_BAR, | |
276 pcap=0, profiler=0, videocapture=0): | |
277 """Generates and uploads config file for test. | |
278 Args: | |
279 runner_configs: Configs specific to the runner you are using. | |
280 network: Config to specify the network environment the devices running | |
281 the tests will be in. | |
282 pcap: Option to set the recording the of network traffic from the device. | |
283 profiler: Option to set the recording of CPU, memory, and network | |
284 transfer usage in the tests. | |
285 videocapture: Option to set video capture during the tests. | |
286 | |
287 """ | |
288 logging.info('Generating config file for test.') | |
289 with tempfile.TemporaryFile() as config: | |
290 config_data = [ | |
291 '[appurify]', | |
292 'network=%s' % network, | |
293 'pcap=%s' % pcap, | |
294 'profiler=%s' % profiler, | |
295 'videocapture=%s' % videocapture, | |
296 '[%s]' % runner_type | |
297 ] | |
298 config_data.extend( | |
299 '%s=%s' % (k, v) for k, v in runner_configs.iteritems()) | |
300 config.write(''.join('%s\n' % l for l in config_data)) | |
301 config.flush() | |
302 config.seek(0) | |
303 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, | |
304 logging.WARNING): | |
305 config_response = appurify_sanitized.api.config_upload( | |
306 self._env.token, config, self._test_id) | |
307 remote_device_helper.TestHttpResponse( | |
308 config_response, 'Unable to upload test config.') | |
OLD | NEW |