OLD | NEW |
| (Empty) |
1 # Copyright 2015 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 import logging | |
6 import os | |
7 import pickle | |
8 import re | |
9 import sys | |
10 | |
11 from pylib import cmd_helper | |
12 from pylib import constants | |
13 from pylib import flag_changer | |
14 from pylib.base import base_test_result | |
15 from pylib.base import test_instance | |
16 from pylib.instrumentation import test_result | |
17 from pylib.instrumentation import instrumentation_parser | |
18 from pylib.utils import apk_helper | |
19 from pylib.utils import md5sum | |
20 from pylib.utils import proguard | |
21 | |
22 sys.path.append( | |
23 os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib', 'common')) | |
24 import unittest_util | |
25 | |
26 # Ref: http://developer.android.com/reference/android/app/Activity.html | |
27 _ACTIVITY_RESULT_CANCELED = 0 | |
28 _ACTIVITY_RESULT_OK = -1 | |
29 | |
30 _DEFAULT_ANNOTATIONS = [ | |
31 'Smoke', 'SmallTest', 'MediumTest', 'LargeTest', | |
32 'EnormousTest', 'IntegrationTest'] | |
33 _EXTRA_ENABLE_HTTP_SERVER = ( | |
34 'org.chromium.chrome.test.ChromeInstrumentationTestRunner.' | |
35 + 'EnableTestHttpServer') | |
36 _EXTRA_DRIVER_TEST_LIST = ( | |
37 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestList') | |
38 _EXTRA_DRIVER_TEST_LIST_FILE = ( | |
39 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestListFile') | |
40 _EXTRA_DRIVER_TARGET_PACKAGE = ( | |
41 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetPackage') | |
42 _EXTRA_DRIVER_TARGET_CLASS = ( | |
43 'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetClass') | |
44 _NATIVE_CRASH_RE = re.compile('native crash', re.IGNORECASE) | |
45 _PICKLE_FORMAT_VERSION = 10 | |
46 | |
47 | |
48 # TODO(jbudorick): Make these private class methods of | |
49 # InstrumentationTestInstance once the instrumentation test_runner is | |
50 # deprecated. | |
51 def ParseAmInstrumentRawOutput(raw_output): | |
52 """Parses the output of an |am instrument -r| call. | |
53 | |
54 Args: | |
55 raw_output: the output of an |am instrument -r| call as a list of lines | |
56 Returns: | |
57 A 3-tuple containing: | |
58 - the instrumentation code as an integer | |
59 - the instrumentation result as a list of lines | |
60 - the instrumentation statuses received as a list of 2-tuples | |
61 containing: | |
62 - the status code as an integer | |
63 - the bundle dump as a dict mapping string keys to a list of | |
64 strings, one for each line. | |
65 """ | |
66 parser = instrumentation_parser.InstrumentationParser(raw_output) | |
67 statuses = list(parser.IterStatus()) | |
68 code, bundle = parser.GetResult() | |
69 return (code, bundle, statuses) | |
70 | |
71 | |
72 def GenerateTestResults( | |
73 result_code, result_bundle, statuses, start_ms, duration_ms): | |
74 """Generate test results from |statuses|. | |
75 | |
76 Args: | |
77 result_code: The overall status code as an integer. | |
78 result_bundle: The summary bundle dump as a dict. | |
79 statuses: A list of 2-tuples containing: | |
80 - the status code as an integer | |
81 - the bundle dump as a dict mapping string keys to string values | |
82 Note that this is the same as the third item in the 3-tuple returned by | |
83 |_ParseAmInstrumentRawOutput|. | |
84 start_ms: The start time of the test in milliseconds. | |
85 duration_ms: The duration of the test in milliseconds. | |
86 | |
87 Returns: | |
88 A list containing an instance of InstrumentationTestResult for each test | |
89 parsed. | |
90 """ | |
91 | |
92 results = [] | |
93 | |
94 current_result = None | |
95 | |
96 for status_code, bundle in statuses: | |
97 test_class = bundle.get('class', '') | |
98 test_method = bundle.get('test', '') | |
99 if test_class and test_method: | |
100 test_name = '%s#%s' % (test_class, test_method) | |
101 else: | |
102 continue | |
103 | |
104 if status_code == instrumentation_parser.STATUS_CODE_START: | |
105 if current_result: | |
106 results.append(current_result) | |
107 current_result = test_result.InstrumentationTestResult( | |
108 test_name, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms) | |
109 else: | |
110 if status_code == instrumentation_parser.STATUS_CODE_OK: | |
111 if bundle.get('test_skipped', '').lower() in ('true', '1', 'yes'): | |
112 current_result.SetType(base_test_result.ResultType.SKIP) | |
113 elif current_result.GetType() == base_test_result.ResultType.UNKNOWN: | |
114 current_result.SetType(base_test_result.ResultType.PASS) | |
115 else: | |
116 if status_code not in (instrumentation_parser.STATUS_CODE_ERROR, | |
117 instrumentation_parser.STATUS_CODE_FAILURE): | |
118 logging.error('Unrecognized status code %d. Handling as an error.', | |
119 status_code) | |
120 current_result.SetType(base_test_result.ResultType.FAIL) | |
121 if 'stack' in bundle: | |
122 current_result.SetLog(bundle['stack']) | |
123 | |
124 if current_result: | |
125 if current_result.GetType() == base_test_result.ResultType.UNKNOWN: | |
126 crashed = (result_code == _ACTIVITY_RESULT_CANCELED | |
127 and any(_NATIVE_CRASH_RE.search(l) | |
128 for l in result_bundle.itervalues())) | |
129 if crashed: | |
130 current_result.SetType(base_test_result.ResultType.CRASH) | |
131 | |
132 results.append(current_result) | |
133 | |
134 return results | |
135 | |
136 | |
137 class InstrumentationTestInstance(test_instance.TestInstance): | |
138 | |
139 def __init__(self, args, isolate_delegate, error_func): | |
140 super(InstrumentationTestInstance, self).__init__() | |
141 | |
142 self._apk_under_test = None | |
143 self._package_info = None | |
144 self._suite = None | |
145 self._test_apk = None | |
146 self._test_jar = None | |
147 self._test_package = None | |
148 self._test_runner = None | |
149 self._test_support_apk = None | |
150 self._initializeApkAttributes(args, error_func) | |
151 | |
152 self._data_deps = None | |
153 self._isolate_abs_path = None | |
154 self._isolate_delegate = None | |
155 self._isolated_abs_path = None | |
156 self._test_data = None | |
157 self._initializeDataDependencyAttributes(args, isolate_delegate) | |
158 | |
159 self._annotations = None | |
160 self._excluded_annotations = None | |
161 self._test_filter = None | |
162 self._initializeTestFilterAttributes(args) | |
163 | |
164 self._flags = None | |
165 self._initializeFlagAttributes(args) | |
166 | |
167 self._driver_apk = None | |
168 self._driver_package = None | |
169 self._driver_name = None | |
170 self._initializeDriverAttributes() | |
171 | |
172 def _initializeApkAttributes(self, args, error_func): | |
173 if args.apk_under_test.endswith('.apk'): | |
174 self._apk_under_test = args.apk_under_test | |
175 else: | |
176 self._apk_under_test = os.path.join( | |
177 constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR, | |
178 '%s.apk' % args.apk_under_test) | |
179 | |
180 if not os.path.exists(self._apk_under_test): | |
181 error_func('Unable to find APK under test: %s' % self._apk_under_test) | |
182 | |
183 if args.test_apk.endswith('.apk'): | |
184 self._suite = os.path.splitext(os.path.basename(args.test_apk))[0] | |
185 self._test_apk = args.test_apk | |
186 else: | |
187 self._suite = args.test_apk | |
188 self._test_apk = os.path.join( | |
189 constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR, | |
190 '%s.apk' % args.test_apk) | |
191 | |
192 self._test_jar = os.path.join( | |
193 constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR, | |
194 '%s.jar' % self._suite) | |
195 self._test_support_apk = os.path.join( | |
196 constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR, | |
197 '%sSupport.apk' % self._suite) | |
198 | |
199 if not os.path.exists(self._test_apk): | |
200 error_func('Unable to find test APK: %s' % self._test_apk) | |
201 if not os.path.exists(self._test_jar): | |
202 error_func('Unable to find test JAR: %s' % self._test_jar) | |
203 | |
204 apk = apk_helper.ApkHelper(self.test_apk) | |
205 self._test_package = apk.GetPackageName() | |
206 self._test_runner = apk.GetInstrumentationName() | |
207 | |
208 self._package_info = None | |
209 for package_info in constants.PACKAGE_INFO.itervalues(): | |
210 if self._test_package == package_info.test_package: | |
211 self._package_info = package_info | |
212 if not self._package_info: | |
213 logging.warning('Unable to find package info for %s', self._test_package) | |
214 | |
215 def _initializeDataDependencyAttributes(self, args, isolate_delegate): | |
216 self._data_deps = [] | |
217 if args.isolate_file_path: | |
218 self._isolate_abs_path = os.path.abspath(args.isolate_file_path) | |
219 self._isolate_delegate = isolate_delegate | |
220 self._isolated_abs_path = os.path.join( | |
221 constants.GetOutDirectory(), '%s.isolated' % self._test_package) | |
222 else: | |
223 self._isolate_delegate = None | |
224 | |
225 # TODO(jbudorick): Deprecate and remove --test-data once data dependencies | |
226 # are fully converted to isolate. | |
227 if args.test_data: | |
228 logging.info('Data dependencies specified via --test-data') | |
229 self._test_data = args.test_data | |
230 else: | |
231 self._test_data = None | |
232 | |
233 if not self._isolate_delegate and not self._test_data: | |
234 logging.warning('No data dependencies will be pushed.') | |
235 | |
236 def _initializeTestFilterAttributes(self, args): | |
237 self._test_filter = args.test_filter | |
238 | |
239 def annotation_dict_element(a): | |
240 a = a.split('=') | |
241 return (a[0], a[1] if len(a) == 2 else None) | |
242 | |
243 if args.annotation_str: | |
244 self._annotations = dict( | |
245 annotation_dict_element(a) | |
246 for a in args.annotation_str.split(',')) | |
247 elif not self._test_filter: | |
248 self._annotations = dict( | |
249 annotation_dict_element(a) | |
250 for a in _DEFAULT_ANNOTATIONS) | |
251 else: | |
252 self._annotations = {} | |
253 | |
254 if args.exclude_annotation_str: | |
255 self._excluded_annotations = dict( | |
256 annotation_dict_element(a) | |
257 for a in args.exclude_annotation_str.split(',')) | |
258 else: | |
259 self._excluded_annotations = {} | |
260 | |
261 def _initializeFlagAttributes(self, args): | |
262 self._flags = ['--disable-fre', '--enable-test-intents'] | |
263 # TODO(jbudorick): Transition "--device-flags" to "--device-flags-file" | |
264 if hasattr(args, 'device_flags') and args.device_flags: | |
265 with open(args.device_flags) as device_flags_file: | |
266 stripped_lines = (l.strip() for l in device_flags_file) | |
267 self._flags.extend([flag for flag in stripped_lines if flag]) | |
268 if hasattr(args, 'device_flags_file') and args.device_flags_file: | |
269 with open(args.device_flags_file) as device_flags_file: | |
270 stripped_lines = (l.strip() for l in device_flags_file) | |
271 self._flags.extend([flag for flag in stripped_lines if flag]) | |
272 | |
273 def _initializeDriverAttributes(self): | |
274 self._driver_apk = os.path.join( | |
275 constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR, | |
276 'OnDeviceInstrumentationDriver.apk') | |
277 if os.path.exists(self._driver_apk): | |
278 driver_apk = apk_helper.ApkHelper(self._driver_apk) | |
279 self._driver_package = driver_apk.GetPackageName() | |
280 self._driver_name = driver_apk.GetInstrumentationName() | |
281 else: | |
282 self._driver_apk = None | |
283 | |
284 @property | |
285 def apk_under_test(self): | |
286 return self._apk_under_test | |
287 | |
288 @property | |
289 def flags(self): | |
290 return self._flags | |
291 | |
292 @property | |
293 def driver_apk(self): | |
294 return self._driver_apk | |
295 | |
296 @property | |
297 def driver_package(self): | |
298 return self._driver_package | |
299 | |
300 @property | |
301 def driver_name(self): | |
302 return self._driver_name | |
303 | |
304 @property | |
305 def package_info(self): | |
306 return self._package_info | |
307 | |
308 @property | |
309 def suite(self): | |
310 return self._suite | |
311 | |
312 @property | |
313 def test_apk(self): | |
314 return self._test_apk | |
315 | |
316 @property | |
317 def test_jar(self): | |
318 return self._test_jar | |
319 | |
320 @property | |
321 def test_support_apk(self): | |
322 return self._test_support_apk | |
323 | |
324 @property | |
325 def test_package(self): | |
326 return self._test_package | |
327 | |
328 @property | |
329 def test_runner(self): | |
330 return self._test_runner | |
331 | |
332 #override | |
333 def TestType(self): | |
334 return 'instrumentation' | |
335 | |
336 #override | |
337 def SetUp(self): | |
338 if self._isolate_delegate: | |
339 self._isolate_delegate.Remap( | |
340 self._isolate_abs_path, self._isolated_abs_path) | |
341 self._isolate_delegate.MoveOutputDeps() | |
342 self._data_deps.extend([(constants.ISOLATE_DEPS_DIR, None)]) | |
343 | |
344 # TODO(jbudorick): Convert existing tests that depend on the --test-data | |
345 # mechanism to isolate, then remove this. | |
346 if self._test_data: | |
347 for t in self._test_data: | |
348 device_rel_path, host_rel_path = t.split(':') | |
349 host_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, host_rel_path) | |
350 self._data_deps.extend( | |
351 [(host_abs_path, | |
352 [None, 'chrome', 'test', 'data', device_rel_path])]) | |
353 | |
354 def GetDataDependencies(self): | |
355 return self._data_deps | |
356 | |
357 def GetTests(self): | |
358 pickle_path = '%s-proguard.pickle' % self.test_jar | |
359 try: | |
360 tests = self._GetTestsFromPickle(pickle_path, self.test_jar) | |
361 except self.ProguardPickleException as e: | |
362 logging.info('Getting tests from JAR via proguard. (%s)' % str(e)) | |
363 tests = self._GetTestsFromProguard(self.test_jar) | |
364 self._SaveTestsToPickle(pickle_path, self.test_jar, tests) | |
365 return self._InflateTests(self._FilterTests(tests)) | |
366 | |
367 class ProguardPickleException(Exception): | |
368 pass | |
369 | |
370 def _GetTestsFromPickle(self, pickle_path, jar_path): | |
371 if not os.path.exists(pickle_path): | |
372 raise self.ProguardPickleException('%s does not exist.' % pickle_path) | |
373 if os.path.getmtime(pickle_path) <= os.path.getmtime(jar_path): | |
374 raise self.ProguardPickleException( | |
375 '%s newer than %s.' % (jar_path, pickle_path)) | |
376 | |
377 with open(pickle_path, 'r') as pickle_file: | |
378 pickle_data = pickle.loads(pickle_file.read()) | |
379 jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path] | |
380 | |
381 try: | |
382 if pickle_data['VERSION'] != _PICKLE_FORMAT_VERSION: | |
383 raise self.ProguardPickleException('PICKLE_FORMAT_VERSION has changed.') | |
384 if pickle_data['JAR_MD5SUM'] != jar_md5: | |
385 raise self.ProguardPickleException('JAR file MD5 sum differs.') | |
386 return pickle_data['TEST_METHODS'] | |
387 except TypeError as e: | |
388 logging.error(pickle_data) | |
389 raise self.ProguardPickleException(str(e)) | |
390 | |
391 def _GetTestsFromProguard(self, jar_path): | |
392 p = proguard.Dump(jar_path) | |
393 | |
394 def is_test_class(c): | |
395 return c['class'].endswith('Test') | |
396 | |
397 def is_test_method(m): | |
398 return m['method'].startswith('test') | |
399 | |
400 class_lookup = dict((c['class'], c) for c in p['classes']) | |
401 def recursive_get_class_annotations(c): | |
402 s = c['superclass'] | |
403 if s in class_lookup: | |
404 a = recursive_get_class_annotations(class_lookup[s]) | |
405 else: | |
406 a = {} | |
407 a.update(c['annotations']) | |
408 return a | |
409 | |
410 def stripped_test_class(c): | |
411 return { | |
412 'class': c['class'], | |
413 'annotations': recursive_get_class_annotations(c), | |
414 'methods': [m for m in c['methods'] if is_test_method(m)], | |
415 } | |
416 | |
417 return [stripped_test_class(c) for c in p['classes'] | |
418 if is_test_class(c)] | |
419 | |
420 def _SaveTestsToPickle(self, pickle_path, jar_path, tests): | |
421 jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path] | |
422 pickle_data = { | |
423 'VERSION': _PICKLE_FORMAT_VERSION, | |
424 'JAR_MD5SUM': jar_md5, | |
425 'TEST_METHODS': tests, | |
426 } | |
427 with open(pickle_path, 'w') as pickle_file: | |
428 pickle.dump(pickle_data, pickle_file) | |
429 | |
430 def _FilterTests(self, tests): | |
431 | |
432 def gtest_filter(c, m): | |
433 t = ['%s.%s' % (c['class'].split('.')[-1], m['method'])] | |
434 return (not self._test_filter | |
435 or unittest_util.FilterTestNames(t, self._test_filter)) | |
436 | |
437 def annotation_filter(all_annotations): | |
438 if not self._annotations: | |
439 return True | |
440 return any_annotation_matches(self._annotations, all_annotations) | |
441 | |
442 def excluded_annotation_filter(all_annotations): | |
443 if not self._excluded_annotations: | |
444 return True | |
445 return not any_annotation_matches(self._excluded_annotations, | |
446 all_annotations) | |
447 | |
448 def any_annotation_matches(annotations, all_annotations): | |
449 return any( | |
450 ak in all_annotations and (av is None or av == all_annotations[ak]) | |
451 for ak, av in annotations.iteritems()) | |
452 | |
453 filtered_classes = [] | |
454 for c in tests: | |
455 filtered_methods = [] | |
456 for m in c['methods']: | |
457 # Gtest filtering | |
458 if not gtest_filter(c, m): | |
459 continue | |
460 | |
461 all_annotations = dict(c['annotations']) | |
462 all_annotations.update(m['annotations']) | |
463 if (not annotation_filter(all_annotations) | |
464 or not excluded_annotation_filter(all_annotations)): | |
465 continue | |
466 | |
467 filtered_methods.append(m) | |
468 | |
469 if filtered_methods: | |
470 filtered_class = dict(c) | |
471 filtered_class['methods'] = filtered_methods | |
472 filtered_classes.append(filtered_class) | |
473 | |
474 return filtered_classes | |
475 | |
476 def _InflateTests(self, tests): | |
477 inflated_tests = [] | |
478 for c in tests: | |
479 for m in c['methods']: | |
480 a = dict(c['annotations']) | |
481 a.update(m['annotations']) | |
482 inflated_tests.append({ | |
483 'class': c['class'], | |
484 'method': m['method'], | |
485 'annotations': a, | |
486 }) | |
487 return inflated_tests | |
488 | |
489 @staticmethod | |
490 def GetHttpServerEnvironmentVars(): | |
491 return { | |
492 _EXTRA_ENABLE_HTTP_SERVER: None, | |
493 } | |
494 | |
495 def GetDriverEnvironmentVars( | |
496 self, test_list=None, test_list_file_path=None): | |
497 env = { | |
498 _EXTRA_DRIVER_TARGET_PACKAGE: self.test_package, | |
499 _EXTRA_DRIVER_TARGET_CLASS: self.test_runner, | |
500 } | |
501 | |
502 if test_list: | |
503 env[_EXTRA_DRIVER_TEST_LIST] = ','.join(test_list) | |
504 | |
505 if test_list_file_path: | |
506 env[_EXTRA_DRIVER_TEST_LIST_FILE] = ( | |
507 os.path.basename(test_list_file_path)) | |
508 | |
509 return env | |
510 | |
511 @staticmethod | |
512 def ParseAmInstrumentRawOutput(raw_output): | |
513 return ParseAmInstrumentRawOutput(raw_output) | |
514 | |
515 @staticmethod | |
516 def GenerateTestResults( | |
517 result_code, result_bundle, statuses, start_ms, duration_ms): | |
518 return GenerateTestResults(result_code, result_bundle, statuses, | |
519 start_ms, duration_ms) | |
520 | |
521 #override | |
522 def TearDown(self): | |
523 if self._isolate_delegate: | |
524 self._isolate_delegate.Clear() | |
525 | |
OLD | NEW |