OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import datetime | 5 import datetime |
6 import json | 6 import json |
7 import re | 7 import re |
8 import string | 8 import string |
9 | 9 |
10 | 10 |
(...skipping 1065 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1076 | 1076 |
1077 # TODO(nednguyen): only rerun the tests that failed for the "without patch" | 1077 # TODO(nednguyen): only rerun the tests that failed for the "without patch" |
1078 # suffix. | 1078 # suffix. |
1079 | 1079 |
1080 # For the time being, we assume all isolated_script_test are not idempotent | 1080 # For the time being, we assume all isolated_script_test are not idempotent |
1081 # TODO(nednguyen): make this configurable in isolated_scripts's spec. | 1081 # TODO(nednguyen): make this configurable in isolated_scripts's spec. |
1082 return api.swarming.isolated_script_task( | 1082 return api.swarming.isolated_script_task( |
1083 title=self._step_name(suffix), isolated_hash=isolated_hash, | 1083 title=self._step_name(suffix), isolated_hash=isolated_hash, |
1084 shards=self._shards, idempotent=False, extra_args=args) | 1084 shards=self._shards, idempotent=False, extra_args=args) |
1085 | 1085 |
| 1086 def validate_simplified_results(self, results): |
| 1087 return results['valid'], results['failures'] |
| 1088 |
| 1089 def validate_json_test_results(self, api, results): |
| 1090 test_results = api.test_utils.create_results_from_json(results) |
| 1091 tests = test_results.tests |
| 1092 failures = list( |
| 1093 t for t in tests |
| 1094 if all(res not in tests[t]['expected'].split() |
| 1095 for res in tests[t]['actual'].split())) |
| 1096 return True, failures |
| 1097 |
1086 def validate_task_results(self, api, step_result): | 1098 def validate_task_results(self, api, step_result): |
1087 results = getattr(step_result, 'isolated_script_results', None) or {} | 1099 results = getattr(step_result, 'isolated_script_results', None) or {} |
1088 | 1100 valid = True |
| 1101 failures = [] |
1089 try: | 1102 try: |
1090 failures = results['failures'] | 1103 if results.get('version', 0) == 3: |
1091 valid = results['valid'] | 1104 valid, failures = self.validate_json_test_results(api, results) |
1092 if not failures and step_result.retcode != 0: | 1105 else: |
1093 failures = ['%s (entire test suite)' % self.name] | 1106 valid, failures = self.validate_simplified_results(results) |
1094 valid = False | |
1095 | |
1096 except (ValueError, KeyError) as e: | 1107 except (ValueError, KeyError) as e: |
1097 step_result.presentation.logs['invalid_results_exc'] = [str(e)] | 1108 step_result.presentation.logs['invalid_results_exc'] = [repr(e)] |
1098 valid = False | 1109 valid = False |
1099 failures = None | 1110 failures = None |
| 1111 if not failures and step_result.retcode != 0: |
| 1112 failures = ['%s (entire test suite)' % self.name] |
| 1113 valid = False |
1100 if valid: | 1114 if valid: |
1101 step_result.presentation.step_text += api.test_utils.format_step_text([ | 1115 step_result.presentation.step_text += api.test_utils.format_step_text([ |
1102 ['failures:', failures] | 1116 ['failures:', failures] |
1103 ]) | 1117 ]) |
1104 # Check for chartjson results and upload to results dashboard if present. | 1118 # Check for chartjson results and upload to results dashboard if present. |
1105 self._output_chartjson_results_if_present(api, step_result) | 1119 self._output_chartjson_results_if_present(api, step_result) |
1106 return valid, failures | 1120 return valid, failures |
1107 | 1121 |
1108 def _output_chartjson_results_if_present(self, api, step_result): | 1122 def _output_chartjson_results_if_present(self, api, step_result): |
1109 results = \ | 1123 results = \ |
(...skipping 684 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1794 args=args) | 1808 args=args) |
1795 api.gsutil.upload( | 1809 api.gsutil.upload( |
1796 temp_output_dir.join( | 1810 temp_output_dir.join( |
1797 '%s-android-chrome.json' % timestamp_string), | 1811 '%s-android-chrome.json' % timestamp_string), |
1798 'chromium-annotated-tests', 'android') | 1812 'chromium-annotated-tests', 'android') |
1799 | 1813 |
1800 GOMA_TESTS = [ | 1814 GOMA_TESTS = [ |
1801 GTestTest('base_unittests'), | 1815 GTestTest('base_unittests'), |
1802 GTestTest('content_unittests'), | 1816 GTestTest('content_unittests'), |
1803 ] | 1817 ] |
OLD | NEW |