OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import datetime | 5 import datetime |
6 import re | 6 import re |
7 import string | 7 import string |
8 | 8 |
9 | 9 |
10 class Test(object): | 10 class Test(object): |
(...skipping 1064 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1075 | 1075 |
1076 # TODO(nednguyen): only rerun the tests that failed for the "without patch" | 1076 # TODO(nednguyen): only rerun the tests that failed for the "without patch" |
1077 # suffix. | 1077 # suffix. |
1078 | 1078 |
1079 # For the time being, we assume all isolated_script_test are not idempotent | 1079 # For the time being, we assume all isolated_script_test are not idempotent |
1080 # TODO(nednguyen): make this configurable in isolated_scripts's spec. | 1080 # TODO(nednguyen): make this configurable in isolated_scripts's spec. |
1081 return api.swarming.isolated_script_task( | 1081 return api.swarming.isolated_script_task( |
1082 title=self._step_name(suffix), isolated_hash=isolated_hash, | 1082 title=self._step_name(suffix), isolated_hash=isolated_hash, |
1083 shards=self._shards, idempotent=False, extra_args=args) | 1083 shards=self._shards, idempotent=False, extra_args=args) |
1084 | 1084 |
| 1085 def validate_simplified_results(self, results): |
| 1086 return results['valid'], results['failures'] |
| 1087 |
| 1088 def validate_json_test_results(self, api, results): |
| 1089 test_results = api.test_utils.create_results_from_json(results) |
| 1090 tests = test_results.tests |
| 1091 failures = [] |
| 1092 for t in tests: |
| 1093 expected_results = set(tests[t]['expected'].split()) |
| 1094 actual_results = set(tests[t]['actual'].split()) |
| 1095 if expected_results != actual_results: |
| 1096 failures.append(t) |
| 1097 return True, failures |
| 1098 |
1085 def validate_task_results(self, api, step_result): | 1099 def validate_task_results(self, api, step_result): |
1086 results = getattr(step_result, 'isolated_script_results', None) or {} | 1100 results = getattr(step_result, 'isolated_script_results', None) or {} |
1087 | 1101 valid = True |
| 1102 failures = [] |
1088 try: | 1103 try: |
1089 failures = results['failures'] | 1104 if results.get('version', 0) == 3: |
1090 valid = results['valid'] | 1105 valid, failures = self.validate_json_test_results(api, results) |
1091 if not failures and step_result.retcode != 0: | 1106 else: |
1092 failures = ['%s (entire test suite)' % self.name] | 1107 valid, failures = self.validate_simplified_results(results) |
1093 valid = False | |
1094 except (ValueError, KeyError) as e: | 1108 except (ValueError, KeyError) as e: |
1095 step_result.presentation.logs['invalid_results_exc'] = [str(e)] | 1109 step_result.presentation.logs['invalid_results_exc'] = [str(e)] |
1096 valid = False | 1110 valid = False |
1097 failures = None | 1111 failures = None |
| 1112 if not failures and step_result.retcode != 0: |
| 1113 failures = ['%s (entire test suite)' % self.name] |
| 1114 valid = False |
1098 if valid: | 1115 if valid: |
1099 step_result.presentation.step_text += api.test_utils.format_step_text([ | 1116 step_result.presentation.step_text += api.test_utils.format_step_text([ |
1100 ['failures:', failures] | 1117 ['failures:', failures] |
1101 ]) | 1118 ]) |
1102 return valid, failures | 1119 return valid, failures |
1103 | 1120 |
1104 | 1121 |
1105 def generate_isolated_script(api, chromium_tests_api, mastername, buildername, | 1122 def generate_isolated_script(api, chromium_tests_api, mastername, buildername, |
1106 test_spec, bot_update_step, enable_swarming=False, | 1123 test_spec, bot_update_step, enable_swarming=False, |
1107 swarming_dimensions=None, | 1124 swarming_dimensions=None, |
(...skipping 646 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1754 args=args) | 1771 args=args) |
1755 api.gsutil.upload( | 1772 api.gsutil.upload( |
1756 temp_output_dir.join( | 1773 temp_output_dir.join( |
1757 '%s-android-chrome.json' % timestamp_string), | 1774 '%s-android-chrome.json' % timestamp_string), |
1758 'chromium-annotated-tests', 'android') | 1775 'chromium-annotated-tests', 'android') |
1759 | 1776 |
1760 GOMA_TESTS = [ | 1777 GOMA_TESTS = [ |
1761 GTestTest('base_unittests'), | 1778 GTestTest('base_unittests'), |
1762 GTestTest('content_unittests'), | 1779 GTestTest('content_unittests'), |
1763 ] | 1780 ] |
OLD | NEW |