| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import json | |
| 6 import re | 5 import re |
| 7 | 6 |
| 8 | 7 |
| 9 class Test(object): | 8 class Test(object): |
| 10 """ | 9 """ |
| 11 Base class for tests that can be retried after deapplying a previously | 10 Base class for tests that can be retried after deapplying a previously |
| 12 applied patch. | 11 applied patch. |
| 13 """ | 12 """ |
| 14 | 13 |
| 15 def __init__(self): | 14 def __init__(self): |
| (...skipping 247 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 263 elif self._use_isolate: | 262 elif self._use_isolate: |
| 264 api.isolate.runtest(self.target_name, self._revision, | 263 api.isolate.runtest(self.target_name, self._revision, |
| 265 self._webkit_revision, **kwargs) | 264 self._webkit_revision, **kwargs) |
| 266 else: | 265 else: |
| 267 api.chromium.runtest(self.target_name, revision=self._revision, | 266 api.chromium.runtest(self.target_name, revision=self._revision, |
| 268 webkit_revision=self._webkit_revision, **kwargs) | 267 webkit_revision=self._webkit_revision, **kwargs) |
| 269 finally: | 268 finally: |
| 270 step_result = api.step.active_result | 269 step_result = api.step.active_result |
| 271 self._test_runs[suffix] = step_result | 270 self._test_runs[suffix] = step_result |
| 272 | 271 |
| 273 r = step_result.test_utils.gtest_results | 272 if hasattr(step_result, 'test_utils'): |
| 274 p = step_result.presentation | 273 r = step_result.test_utils.gtest_results |
| 274 p = step_result.presentation |
| 275 | 275 |
| 276 if r.valid: | 276 if r.valid: |
| 277 p.step_text += api.test_utils.format_step_text([ | 277 p.step_text += api.test_utils.format_step_text([ |
| 278 ['failures:', r.failures] | 278 ['failures:', r.failures] |
| 279 ]) | 279 ]) |
| 280 | 280 |
| 281 return step_result | 281 return step_result |
| 282 | 282 |
| 283 def has_valid_results(self, api, suffix): | 283 def has_valid_results(self, api, suffix): |
| 284 if suffix not in self._test_runs: | 284 if suffix not in self._test_runs: |
| 285 return False | 285 return False |
| 286 if not hasattr(self._test_runs[suffix], 'test_utils'): |
| 287 return False |
| 286 gtest_results = self._test_runs[suffix].test_utils.gtest_results | 288 gtest_results = self._test_runs[suffix].test_utils.gtest_results |
| 287 if not gtest_results.valid: # pragma: no cover | 289 if not gtest_results.valid: # pragma: no cover |
| 288 return False | 290 return False |
| 289 global_tags = gtest_results.raw.get('global_tags', []) | 291 global_tags = gtest_results.raw.get('global_tags', []) |
| 290 return 'UNRELIABLE_RESULTS' not in global_tags | 292 return 'UNRELIABLE_RESULTS' not in global_tags |
| 291 | 293 |
| 292 def failures(self, api, suffix): | 294 def failures(self, api, suffix): |
| 293 return self._test_runs[suffix].test_utils.gtest_results.failures | 295 return self._test_runs[suffix].test_utils.gtest_results.failures |
| 294 | 296 |
| 295 | 297 |
| (...skipping 270 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 566 args.append('--gtest_filter=%s' % ':'.join(failed_tests)) | 568 args.append('--gtest_filter=%s' % ':'.join(failed_tests)) |
| 567 | 569 |
| 568 return api.swarming.gtest_task( | 570 return api.swarming.gtest_task( |
| 569 title=self._step_name(suffix), | 571 title=self._step_name(suffix), |
| 570 isolated_hash=isolated_hash, | 572 isolated_hash=isolated_hash, |
| 571 shards=self._shards, | 573 shards=self._shards, |
| 572 test_launcher_summary_output=api.test_utils.gtest_results(add_json_log=F
alse), | 574 test_launcher_summary_output=api.test_utils.gtest_results(add_json_log=F
alse), |
| 573 extra_args=args) | 575 extra_args=args) |
| 574 | 576 |
| 575 def validate_task_results(self, api, step_result): | 577 def validate_task_results(self, api, step_result): |
| 578 if not hasattr(step_result, 'test_utils'): |
| 579 return False, None |
| 580 |
| 576 gtest_results = step_result.test_utils.gtest_results | 581 gtest_results = step_result.test_utils.gtest_results |
| 577 if not gtest_results: | 582 if not gtest_results: |
| 578 return False, None | 583 return False, None |
| 579 | 584 |
| 580 global_tags = gtest_results.raw.get('global_tags', []) | 585 global_tags = gtest_results.raw.get('global_tags', []) |
| 581 if 'UNRELIABLE_RESULTS' in global_tags: | 586 if 'UNRELIABLE_RESULTS' in global_tags: |
| 582 return False, None | 587 return False, None |
| 583 | 588 |
| 584 return True, gtest_results.failures | 589 return True, gtest_results.failures |
| 585 | 590 |
| (...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 656 cmd_args.extend(self.failures(api, 'with patch')) | 661 cmd_args.extend(self.failures(api, 'with patch')) |
| 657 | 662 |
| 658 try: | 663 try: |
| 659 self.run_step( | 664 self.run_step( |
| 660 api, | 665 api, |
| 661 suffix, | 666 suffix, |
| 662 cmd_args, | 667 cmd_args, |
| 663 step_test_data=lambda: api.test_utils.test_api.canned_test_output(True
)) | 668 step_test_data=lambda: api.test_utils.test_api.canned_test_output(True
)) |
| 664 finally: | 669 finally: |
| 665 step_result = api.step.active_result | 670 step_result = api.step.active_result |
| 666 r = step_result.test_utils.test_results | |
| 667 p = step_result.presentation | |
| 668 p.step_text += api.test_utils.format_step_text([ | |
| 669 ['unexpected_failures:', r.unexpected_failures.keys()], | |
| 670 ]) | |
| 671 self._test_runs[suffix] = step_result | 671 self._test_runs[suffix] = step_result |
| 672 | 672 |
| 673 if hasattr(step_result, 'test_utils'): |
| 674 r = step_result.test_utils.test_results |
| 675 p = step_result.presentation |
| 676 p.step_text += api.test_utils.format_step_text([ |
| 677 ['unexpected_failures:', r.unexpected_failures.keys()], |
| 678 ]) |
| 679 |
| 673 return step_result | 680 return step_result |
| 674 | 681 |
| 675 def has_valid_results(self, api, suffix): | 682 def has_valid_results(self, api, suffix): |
| 676 # TODO(dpranke): we should just return zero/nonzero for success/fail. | 683 # TODO(dpranke): we should just return zero/nonzero for success/fail. |
| 677 # crbug.com/357866 | 684 # crbug.com/357866 |
| 678 step = self._test_runs[suffix] | 685 step = self._test_runs[suffix] |
| 686 if not hasattr(step, 'test_utils'): |
| 687 return False |
| 679 return (step.test_utils.test_results.valid and | 688 return (step.test_utils.test_results.valid and |
| 680 step.retcode <= step.test_utils.test_results.MAX_FAILURES_EXIT_STATU
S and | 689 step.retcode <= step.test_utils.test_results.MAX_FAILURES_EXIT_STATU
S and |
| 681 (step.retcode == 0) or self.failures(api, suffix)) | 690 (step.retcode == 0) or self.failures(api, suffix)) |
| 682 | 691 |
| 683 def failures(self, api, suffix): | 692 def failures(self, api, suffix): |
| 684 return self._test_runs[suffix].test_utils.test_results.unexpected_failures | 693 return self._test_runs[suffix].test_utils.test_results.unexpected_failures |
| 685 | 694 |
| 686 | 695 |
| 687 class PrintPreviewTests(PythonBasedTest): # pylint: disable=W032 | 696 class PrintPreviewTests(PythonBasedTest): # pylint: disable=W032 |
| 688 name = 'print_preview_tests' | 697 name = 'print_preview_tests' |
| (...skipping 164 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 853 step_result = api.step.active_result | 862 step_result = api.step.active_result |
| 854 self._test_runs[suffix] = step_result | 863 self._test_runs[suffix] = step_result |
| 855 | 864 |
| 856 try: | 865 try: |
| 857 res = api.json.loads(step_result.raw_io.output_dir['results.json']) | 866 res = api.json.loads(step_result.raw_io.output_dir['results.json']) |
| 858 self._failures[suffix] = [res['pages'][str(value['page_id'])]['name'] | 867 self._failures[suffix] = [res['pages'][str(value['page_id'])]['name'] |
| 859 for value in res['per_page_values'] | 868 for value in res['per_page_values'] |
| 860 if value['type'] == 'failure'] | 869 if value['type'] == 'failure'] |
| 861 | 870 |
| 862 self._valid[suffix] = True | 871 self._valid[suffix] = True |
| 863 except (ValueError, KeyError): | 872 except (ValueError, KeyError, AttributeError): |
| 864 self._valid[suffix] = False | 873 self._valid[suffix] = False |
| 865 | 874 |
| 866 if self._valid[suffix]: | 875 if self._valid[suffix]: |
| 867 step_result.presentation.step_text += api.test_utils.format_step_text([ | 876 step_result.presentation.step_text += api.test_utils.format_step_text([ |
| 868 ['failures:', self._failures[suffix]] | 877 ['failures:', self._failures[suffix]] |
| 869 ]) | 878 ]) |
| 870 | 879 |
| 871 def has_valid_results(self, api, suffix): | 880 def has_valid_results(self, api, suffix): |
| 872 return suffix in self._valid and self._valid[suffix] | 881 return suffix in self._valid and self._valid[suffix] |
| 873 | 882 |
| (...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 937 self.compile_target = compile_target | 946 self.compile_target = compile_target |
| 938 | 947 |
| 939 self.test_data = test_data | 948 self.test_data = test_data |
| 940 self.adb_install_apk = adb_install_apk | 949 self.adb_install_apk = adb_install_apk |
| 941 self.isolate_file_path = isolate_file_path | 950 self.isolate_file_path = isolate_file_path |
| 942 | 951 |
| 943 @property | 952 @property |
| 944 def name(self): | 953 def name(self): |
| 945 return self._name | 954 return self._name |
| 946 | 955 |
| 947 def _get_failing_tests(self, json_results): | 956 def _get_failing_tests(self, step_result): |
| 948 """Parses test results and returns a list of failed tests. | 957 """Parses test results and returns a list of failed tests. |
| 949 | 958 |
| 950 Args: | 959 Args: |
| 951 json_results: Parsed JSON output file returned from the test runner. | 960 step_result: Result returned from the test. |
| 952 | 961 |
| 953 Returns: | 962 Returns: |
| 954 None if results are invalid, a list of failures otherwise (may be empty). | 963 None if results are invalid, a list of failures otherwise (may be empty). |
| 955 """ | 964 """ |
| 956 try: | 965 try: |
| 957 # Extract test results. | 966 # Extract test results. |
| 967 json_results = step_result.json.output |
| 958 test_results = {test_name: test_data[0]['status'] | 968 test_results = {test_name: test_data[0]['status'] |
| 959 for result_dict in json_results['per_iteration_data'] | 969 for result_dict in json_results['per_iteration_data'] |
| 960 for test_name, test_data in result_dict.iteritems()} | 970 for test_name, test_data in result_dict.iteritems()} |
| 961 | 971 |
| 962 # TODO(sergiyb): Figure out how to handle status UNKNOWN. | 972 # TODO(sergiyb): Figure out how to handle status UNKNOWN. |
| 963 return [test_name for test_name, test_status in test_results.iteritems() | 973 return [test_name for test_name, test_status in test_results.iteritems() |
| 964 if test_status not in ['SUCCESS', 'SKIPPED']] | 974 if test_status not in ['SUCCESS', 'SKIPPED']] |
| 965 except (KeyError, IndexError, TypeError): | 975 except (KeyError, IndexError, TypeError, AttributeError): |
| 966 return None | 976 return None |
| 967 | 977 |
| 968 def run(self, api, suffix): | 978 def run(self, api, suffix): |
| 969 assert api.chromium.c.TARGET_PLATFORM == 'android' | 979 assert api.chromium.c.TARGET_PLATFORM == 'android' |
| 970 if self.adb_install_apk: | 980 if self.adb_install_apk: |
| 971 api.chromium_android.adb_install_apk( | 981 api.chromium_android.adb_install_apk( |
| 972 self.adb_install_apk[0], self.adb_install_apk[1]) | 982 self.adb_install_apk[0], self.adb_install_apk[1]) |
| 973 | 983 |
| 974 mock_test_results = { | 984 mock_test_results = { |
| 975 'per_iteration_data': [{'TestA': [{'status': 'SUCCESS'}]}, | 985 'per_iteration_data': [{'TestA': [{'status': 'SUCCESS'}]}, |
| 976 {'TestB': [{'status': 'FAILURE'}]}] | 986 {'TestB': [{'status': 'FAILURE'}]}] |
| 977 } | 987 } |
| 978 | 988 |
| 979 try: | 989 try: |
| 980 api.chromium_android.run_instrumentation_suite( | 990 api.chromium_android.run_instrumentation_suite( |
| 981 self.name, test_data=self.test_data, suffix=suffix, | 991 self.name, test_data=self.test_data, suffix=suffix, |
| 982 flakiness_dashboard='http://test-results.appspot.com', | 992 flakiness_dashboard='http://test-results.appspot.com', |
| 983 verbose=True, isolate_file_path=self.isolate_file_path, | 993 verbose=True, isolate_file_path=self.isolate_file_path, |
| 984 json_results_file=api.json.output(add_json_log=False), | 994 json_results_file=api.json.output(add_json_log=False), |
| 985 step_test_data=lambda: api.json.test_api.output(mock_test_results)) | 995 step_test_data=lambda: api.json.test_api.output(mock_test_results)) |
| 986 finally: | 996 finally: |
| 987 step_result = api.step.active_result | 997 step_result = api.step.active_result |
| 988 failures = self._get_failing_tests(step_result.json.output) | 998 failures = self._get_failing_tests(step_result) |
| 999 |
| 989 if failures is None: | 1000 if failures is None: |
| 990 self._test_runs[suffix] = {'valid': False} | 1001 self._test_runs[suffix] = {'valid': False} |
| 991 else: | 1002 else: |
| 992 self._test_runs[suffix] = {'valid': True, 'failures': failures} | 1003 self._test_runs[suffix] = {'valid': True, 'failures': failures} |
| 993 | 1004 |
| 994 step_result.presentation.step_text += api.test_utils.format_step_text([ | 1005 step_result.presentation.step_text += api.test_utils.format_step_text([ |
| 995 ['failures:', failures] | 1006 ['failures:', failures] |
| 996 ]) | 1007 ]) |
| 997 | 1008 |
| 998 def compile_targets(self, _): | 1009 def compile_targets(self, _): |
| (...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1144 GTestTest('crypto_unittests'), | 1155 GTestTest('crypto_unittests'), |
| 1145 GTestTest('gfx_unittests'), | 1156 GTestTest('gfx_unittests'), |
| 1146 GTestTest('url_unittests'), | 1157 GTestTest('url_unittests'), |
| 1147 GTestTest('content_unittests'), | 1158 GTestTest('content_unittests'), |
| 1148 GTestTest('net_unittests'), | 1159 GTestTest('net_unittests'), |
| 1149 GTestTest('ui_base_unittests'), | 1160 GTestTest('ui_base_unittests'), |
| 1150 GTestTest('ui_ios_unittests'), | 1161 GTestTest('ui_ios_unittests'), |
| 1151 GTestTest('sync_unit_tests'), | 1162 GTestTest('sync_unit_tests'), |
| 1152 GTestTest('sql_unittests'), | 1163 GTestTest('sql_unittests'), |
| 1153 ] | 1164 ] |
| OLD | NEW |