OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import json | |
6 import re | 5 import re |
7 | 6 |
8 | 7 |
9 class Test(object): | 8 class Test(object): |
10 """ | 9 """ |
11 Base class for tests that can be retried after deapplying a previously | 10 Base class for tests that can be retried after deapplying a previously |
12 applied patch. | 11 applied patch. |
13 """ | 12 """ |
14 | 13 |
15 def __init__(self): | 14 def __init__(self): |
(...skipping 247 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
263 elif self._use_isolate: | 262 elif self._use_isolate: |
264 api.isolate.runtest(self.target_name, self._revision, | 263 api.isolate.runtest(self.target_name, self._revision, |
265 self._webkit_revision, **kwargs) | 264 self._webkit_revision, **kwargs) |
266 else: | 265 else: |
267 api.chromium.runtest(self.target_name, revision=self._revision, | 266 api.chromium.runtest(self.target_name, revision=self._revision, |
268 webkit_revision=self._webkit_revision, **kwargs) | 267 webkit_revision=self._webkit_revision, **kwargs) |
269 finally: | 268 finally: |
270 step_result = api.step.active_result | 269 step_result = api.step.active_result |
271 self._test_runs[suffix] = step_result | 270 self._test_runs[suffix] = step_result |
272 | 271 |
273 r = step_result.test_utils.gtest_results | 272 if hasattr(step_result, 'test_utils'): |
274 p = step_result.presentation | 273 r = step_result.test_utils.gtest_results |
| 274 p = step_result.presentation |
275 | 275 |
276 if r.valid: | 276 if r.valid: |
277 p.step_text += api.test_utils.format_step_text([ | 277 p.step_text += api.test_utils.format_step_text([ |
278 ['failures:', r.failures] | 278 ['failures:', r.failures] |
279 ]) | 279 ]) |
280 | 280 |
281 return step_result | 281 return step_result |
282 | 282 |
283 def has_valid_results(self, api, suffix): | 283 def has_valid_results(self, api, suffix): |
284 if suffix not in self._test_runs: | 284 if suffix not in self._test_runs: |
285 return False | 285 return False |
| 286 if not hasattr(self._test_runs[suffix], 'test_utils'): |
| 287 return False |
286 gtest_results = self._test_runs[suffix].test_utils.gtest_results | 288 gtest_results = self._test_runs[suffix].test_utils.gtest_results |
287 if not gtest_results.valid: # pragma: no cover | 289 if not gtest_results.valid: # pragma: no cover |
288 return False | 290 return False |
289 global_tags = gtest_results.raw.get('global_tags', []) | 291 global_tags = gtest_results.raw.get('global_tags', []) |
290 return 'UNRELIABLE_RESULTS' not in global_tags | 292 return 'UNRELIABLE_RESULTS' not in global_tags |
291 | 293 |
292 def failures(self, api, suffix): | 294 def failures(self, api, suffix): |
293 return self._test_runs[suffix].test_utils.gtest_results.failures | 295 return self._test_runs[suffix].test_utils.gtest_results.failures |
294 | 296 |
295 | 297 |
(...skipping 270 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
566 args.append('--gtest_filter=%s' % ':'.join(failed_tests)) | 568 args.append('--gtest_filter=%s' % ':'.join(failed_tests)) |
567 | 569 |
568 return api.swarming.gtest_task( | 570 return api.swarming.gtest_task( |
569 title=self._step_name(suffix), | 571 title=self._step_name(suffix), |
570 isolated_hash=isolated_hash, | 572 isolated_hash=isolated_hash, |
571 shards=self._shards, | 573 shards=self._shards, |
572 test_launcher_summary_output=api.test_utils.gtest_results(add_json_log=F
alse), | 574 test_launcher_summary_output=api.test_utils.gtest_results(add_json_log=F
alse), |
573 extra_args=args) | 575 extra_args=args) |
574 | 576 |
575 def validate_task_results(self, api, step_result): | 577 def validate_task_results(self, api, step_result): |
| 578 if not hasattr(step_result, 'test_utils'): |
| 579 return False, None |
| 580 |
576 gtest_results = step_result.test_utils.gtest_results | 581 gtest_results = step_result.test_utils.gtest_results |
577 if not gtest_results: | 582 if not gtest_results: |
578 return False, None | 583 return False, None |
579 | 584 |
580 global_tags = gtest_results.raw.get('global_tags', []) | 585 global_tags = gtest_results.raw.get('global_tags', []) |
581 if 'UNRELIABLE_RESULTS' in global_tags: | 586 if 'UNRELIABLE_RESULTS' in global_tags: |
582 return False, None | 587 return False, None |
583 | 588 |
584 return True, gtest_results.failures | 589 return True, gtest_results.failures |
585 | 590 |
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
656 cmd_args.extend(self.failures(api, 'with patch')) | 661 cmd_args.extend(self.failures(api, 'with patch')) |
657 | 662 |
658 try: | 663 try: |
659 self.run_step( | 664 self.run_step( |
660 api, | 665 api, |
661 suffix, | 666 suffix, |
662 cmd_args, | 667 cmd_args, |
663 step_test_data=lambda: api.test_utils.test_api.canned_test_output(True
)) | 668 step_test_data=lambda: api.test_utils.test_api.canned_test_output(True
)) |
664 finally: | 669 finally: |
665 step_result = api.step.active_result | 670 step_result = api.step.active_result |
666 r = step_result.test_utils.test_results | |
667 p = step_result.presentation | |
668 p.step_text += api.test_utils.format_step_text([ | |
669 ['unexpected_failures:', r.unexpected_failures.keys()], | |
670 ]) | |
671 self._test_runs[suffix] = step_result | 671 self._test_runs[suffix] = step_result |
672 | 672 |
| 673 if hasattr(step_result, 'test_utils'): |
| 674 r = step_result.test_utils.test_results |
| 675 p = step_result.presentation |
| 676 p.step_text += api.test_utils.format_step_text([ |
| 677 ['unexpected_failures:', r.unexpected_failures.keys()], |
| 678 ]) |
| 679 |
673 return step_result | 680 return step_result |
674 | 681 |
675 def has_valid_results(self, api, suffix): | 682 def has_valid_results(self, api, suffix): |
676 # TODO(dpranke): we should just return zero/nonzero for success/fail. | 683 # TODO(dpranke): we should just return zero/nonzero for success/fail. |
677 # crbug.com/357866 | 684 # crbug.com/357866 |
678 step = self._test_runs[suffix] | 685 step = self._test_runs[suffix] |
| 686 if not hasattr(step, 'test_utils'): |
| 687 return False |
679 return (step.test_utils.test_results.valid and | 688 return (step.test_utils.test_results.valid and |
680 step.retcode <= step.test_utils.test_results.MAX_FAILURES_EXIT_STATU
S and | 689 step.retcode <= step.test_utils.test_results.MAX_FAILURES_EXIT_STATU
S and |
681 (step.retcode == 0) or self.failures(api, suffix)) | 690 (step.retcode == 0) or self.failures(api, suffix)) |
682 | 691 |
683 def failures(self, api, suffix): | 692 def failures(self, api, suffix): |
684 return self._test_runs[suffix].test_utils.test_results.unexpected_failures | 693 return self._test_runs[suffix].test_utils.test_results.unexpected_failures |
685 | 694 |
686 | 695 |
687 class PrintPreviewTests(PythonBasedTest): # pylint: disable=W032 | 696 class PrintPreviewTests(PythonBasedTest): # pylint: disable=W032 |
688 name = 'print_preview_tests' | 697 name = 'print_preview_tests' |
(...skipping 164 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
853 step_result = api.step.active_result | 862 step_result = api.step.active_result |
854 self._test_runs[suffix] = step_result | 863 self._test_runs[suffix] = step_result |
855 | 864 |
856 try: | 865 try: |
857 res = api.json.loads(step_result.raw_io.output_dir['results.json']) | 866 res = api.json.loads(step_result.raw_io.output_dir['results.json']) |
858 self._failures[suffix] = [res['pages'][str(value['page_id'])]['name'] | 867 self._failures[suffix] = [res['pages'][str(value['page_id'])]['name'] |
859 for value in res['per_page_values'] | 868 for value in res['per_page_values'] |
860 if value['type'] == 'failure'] | 869 if value['type'] == 'failure'] |
861 | 870 |
862 self._valid[suffix] = True | 871 self._valid[suffix] = True |
863 except (ValueError, KeyError): | 872 except (ValueError, KeyError, AttributeError): |
864 self._valid[suffix] = False | 873 self._valid[suffix] = False |
865 | 874 |
866 if self._valid[suffix]: | 875 if self._valid[suffix]: |
867 step_result.presentation.step_text += api.test_utils.format_step_text([ | 876 step_result.presentation.step_text += api.test_utils.format_step_text([ |
868 ['failures:', self._failures[suffix]] | 877 ['failures:', self._failures[suffix]] |
869 ]) | 878 ]) |
870 | 879 |
871 def has_valid_results(self, api, suffix): | 880 def has_valid_results(self, api, suffix): |
872 return suffix in self._valid and self._valid[suffix] | 881 return suffix in self._valid and self._valid[suffix] |
873 | 882 |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
937 self.compile_target = compile_target | 946 self.compile_target = compile_target |
938 | 947 |
939 self.test_data = test_data | 948 self.test_data = test_data |
940 self.adb_install_apk = adb_install_apk | 949 self.adb_install_apk = adb_install_apk |
941 self.isolate_file_path = isolate_file_path | 950 self.isolate_file_path = isolate_file_path |
942 | 951 |
943 @property | 952 @property |
944 def name(self): | 953 def name(self): |
945 return self._name | 954 return self._name |
946 | 955 |
947 def _get_failing_tests(self, json_results): | 956 def _get_failing_tests(self, step_result): |
948 """Parses test results and returns a list of failed tests. | 957 """Parses test results and returns a list of failed tests. |
949 | 958 |
950 Args: | 959 Args: |
951 json_results: Parsed JSON output file returned from the test runner. | 960 step_result: Result returned from the test. |
952 | 961 |
953 Returns: | 962 Returns: |
954 None if results are invalid, a list of failures otherwise (may be empty). | 963 None if results are invalid, a list of failures otherwise (may be empty). |
955 """ | 964 """ |
956 try: | 965 try: |
957 # Extract test results. | 966 # Extract test results. |
| 967 json_results = step_result.json.output |
958 test_results = {test_name: test_data[0]['status'] | 968 test_results = {test_name: test_data[0]['status'] |
959 for result_dict in json_results['per_iteration_data'] | 969 for result_dict in json_results['per_iteration_data'] |
960 for test_name, test_data in result_dict.iteritems()} | 970 for test_name, test_data in result_dict.iteritems()} |
961 | 971 |
962 # TODO(sergiyb): Figure out how to handle status UNKNOWN. | 972 # TODO(sergiyb): Figure out how to handle status UNKNOWN. |
963 return [test_name for test_name, test_status in test_results.iteritems() | 973 return [test_name for test_name, test_status in test_results.iteritems() |
964 if test_status not in ['SUCCESS', 'SKIPPED']] | 974 if test_status not in ['SUCCESS', 'SKIPPED']] |
965 except (KeyError, IndexError, TypeError): | 975 except (KeyError, IndexError, TypeError, AttributeError): |
966 return None | 976 return None |
967 | 977 |
968 def run(self, api, suffix): | 978 def run(self, api, suffix): |
969 assert api.chromium.c.TARGET_PLATFORM == 'android' | 979 assert api.chromium.c.TARGET_PLATFORM == 'android' |
970 if self.adb_install_apk: | 980 if self.adb_install_apk: |
971 api.chromium_android.adb_install_apk( | 981 api.chromium_android.adb_install_apk( |
972 self.adb_install_apk[0], self.adb_install_apk[1]) | 982 self.adb_install_apk[0], self.adb_install_apk[1]) |
973 | 983 |
974 mock_test_results = { | 984 mock_test_results = { |
975 'per_iteration_data': [{'TestA': [{'status': 'SUCCESS'}]}, | 985 'per_iteration_data': [{'TestA': [{'status': 'SUCCESS'}]}, |
976 {'TestB': [{'status': 'FAILURE'}]}] | 986 {'TestB': [{'status': 'FAILURE'}]}] |
977 } | 987 } |
978 | 988 |
979 try: | 989 try: |
980 api.chromium_android.run_instrumentation_suite( | 990 api.chromium_android.run_instrumentation_suite( |
981 self.name, test_data=self.test_data, suffix=suffix, | 991 self.name, test_data=self.test_data, suffix=suffix, |
982 flakiness_dashboard='http://test-results.appspot.com', | 992 flakiness_dashboard='http://test-results.appspot.com', |
983 verbose=True, isolate_file_path=self.isolate_file_path, | 993 verbose=True, isolate_file_path=self.isolate_file_path, |
984 json_results_file=api.json.output(add_json_log=False), | 994 json_results_file=api.json.output(add_json_log=False), |
985 step_test_data=lambda: api.json.test_api.output(mock_test_results)) | 995 step_test_data=lambda: api.json.test_api.output(mock_test_results)) |
986 finally: | 996 finally: |
987 step_result = api.step.active_result | 997 step_result = api.step.active_result |
988 failures = self._get_failing_tests(step_result.json.output) | 998 failures = self._get_failing_tests(step_result) |
| 999 |
989 if failures is None: | 1000 if failures is None: |
990 self._test_runs[suffix] = {'valid': False} | 1001 self._test_runs[suffix] = {'valid': False} |
991 else: | 1002 else: |
992 self._test_runs[suffix] = {'valid': True, 'failures': failures} | 1003 self._test_runs[suffix] = {'valid': True, 'failures': failures} |
993 | 1004 |
994 step_result.presentation.step_text += api.test_utils.format_step_text([ | 1005 step_result.presentation.step_text += api.test_utils.format_step_text([ |
995 ['failures:', failures] | 1006 ['failures:', failures] |
996 ]) | 1007 ]) |
997 | 1008 |
998 def compile_targets(self, _): | 1009 def compile_targets(self, _): |
(...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1144 GTestTest('crypto_unittests'), | 1155 GTestTest('crypto_unittests'), |
1145 GTestTest('gfx_unittests'), | 1156 GTestTest('gfx_unittests'), |
1146 GTestTest('url_unittests'), | 1157 GTestTest('url_unittests'), |
1147 GTestTest('content_unittests'), | 1158 GTestTest('content_unittests'), |
1148 GTestTest('net_unittests'), | 1159 GTestTest('net_unittests'), |
1149 GTestTest('ui_base_unittests'), | 1160 GTestTest('ui_base_unittests'), |
1150 GTestTest('ui_ios_unittests'), | 1161 GTestTest('ui_ios_unittests'), |
1151 GTestTest('sync_unit_tests'), | 1162 GTestTest('sync_unit_tests'), |
1152 GTestTest('sql_unittests'), | 1163 GTestTest('sql_unittests'), |
1153 ] | 1164 ] |
OLD | NEW |