| OLD | NEW |
| 1 # Copyright (C) 2013 Google Inc. All rights reserved. | 1 # Copyright (C) 2013 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 128 results_json = self._results_json_for_port(port_name, builder_category) | 128 results_json = self._results_json_for_port(port_name, builder_category) |
| 129 if not results_json: | 129 if not results_json: |
| 130 return None | 130 return None |
| 131 return BotTestExpectations(results_json) | 131 return BotTestExpectations(results_json) |
| 132 | 132 |
| 133 | 133 |
| 134 class BotTestExpectations(object): | 134 class BotTestExpectations(object): |
| 135 # FIXME: Get this from the json instead of hard-coding it. | 135 # FIXME: Get this from the json instead of hard-coding it. |
| 136 RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y'] | 136 RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y'] |
| 137 | 137 |
| 138 def __init__(self, results_json): | 138 # specifiers arg is used in unittests to avoid the static dependency on buil
ders. |
| 139 def __init__(self, results_json, specifiers=None): |
| 139 self.results_json = results_json | 140 self.results_json = results_json |
| 141 self.specifiers = specifiers or set(builders.specifiers_for_builder(resu
lts_json.builder_name)) |
| 140 | 142 |
| 141 def _line_from_test_and_flaky_types_and_bug_urls(self, test_path, flaky_type
s, bug_urls): | 143 def _line_from_test_and_flaky_types_and_bug_urls(self, test_path, flaky_type
s, bug_urls): |
| 142 line = TestExpectationLine() | 144 line = TestExpectationLine() |
| 143 line.original_string = test_path | 145 line.original_string = test_path |
| 144 line.name = test_path | 146 line.name = test_path |
| 145 line.filename = test_path | 147 line.filename = test_path |
| 146 line.specifiers = bug_urls if bug_urls else "" | 148 line.path = test_path # FIXME: Should this be normpath? |
| 149 line.matching_tests = [test_path] |
| 150 line.bugs = bug_urls if bug_urls else ["Bug(gardener)"] |
| 147 line.expectations = sorted(map(self.results_json.expectation_for_type, f
laky_types)) | 151 line.expectations = sorted(map(self.results_json.expectation_for_type, f
laky_types)) |
| 152 line.specifiers = self.specifiers |
| 148 return line | 153 return line |
| 149 | 154 |
| 150 def flakes_by_path(self, only_ignore_very_flaky): | 155 def flakes_by_path(self, only_ignore_very_flaky): |
| 151 """Sets test expectations to bot results if there are at least two disti
nct results.""" | 156 """Sets test expectations to bot results if there are at least two disti
nct results.""" |
| 152 flakes_by_path = {} | 157 flakes_by_path = {} |
| 153 for test_path, entry in self.results_json.walk_results(): | 158 for test_path, entry in self.results_json.walk_results(): |
| 154 results_dict = entry[self.results_json.RESULTS_KEY] | 159 results_dict = entry[self.results_json.RESULTS_KEY] |
| 155 flaky_types = self._flaky_types_in_results(results_dict, only_ignore
_very_flaky) | 160 flaky_types = self._flaky_types_in_results(results_dict, only_ignore
_very_flaky) |
| 156 if len(flaky_types) <= 1: | 161 if len(flaky_types) <= 1: |
| 157 continue | 162 continue |
| (...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 196 additional_expectations = set(e for e in result_exp if not expected(
e)) | 201 additional_expectations = set(e for e in result_exp if not expected(
e)) |
| 197 | 202 |
| 198 # Test did not have unexpected results. | 203 # Test did not have unexpected results. |
| 199 if not additional_expectations: | 204 if not additional_expectations: |
| 200 continue | 205 continue |
| 201 | 206 |
| 202 expectations.update(additional_expectations) | 207 expectations.update(additional_expectations) |
| 203 unexpected_results_by_path[test_path] = sorted(map(exp_to_string, ex
pectations)) | 208 unexpected_results_by_path[test_path] = sorted(map(exp_to_string, ex
pectations)) |
| 204 return unexpected_results_by_path | 209 return unexpected_results_by_path |
| 205 | 210 |
| 206 def expectation_lines(self): | 211 def expectation_lines(self, only_ignore_very_flaky=False): |
| 207 lines = [] | 212 lines = [] |
| 208 for test_path, entry in self.results_json.walk_results(): | 213 for test_path, entry in self.results_json.walk_results(): |
| 209 results_array = entry[self.results_json.RESULTS_KEY] | 214 results_array = entry[self.results_json.RESULTS_KEY] |
| 210 flaky_types = self._flaky_types_in_results(results_array, False) | 215 flaky_types = self._flaky_types_in_results(results_array, only_ignor
e_very_flaky) |
| 211 if len(flaky_types) > 1: | 216 if len(flaky_types) > 1: |
| 212 bug_urls = entry.get(self.results_json.BUGS_KEY) | 217 bug_urls = entry.get(self.results_json.BUGS_KEY) |
| 213 line = self._line_from_test_and_flaky_types_and_bug_urls(test_pa
th, flaky_types, bug_urls) | 218 line = self._line_from_test_and_flaky_types_and_bug_urls(test_pa
th, flaky_types, bug_urls) |
| 214 lines.append(line) | 219 lines.append(line) |
| 215 return lines | 220 return lines |
| 216 | 221 |
| 217 def _flaky_types_in_results(self, run_length_encoded_results, only_ignore_ve
ry_flaky=False): | 222 def _flaky_types_in_results(self, run_length_encoded_results, only_ignore_ve
ry_flaky=False): |
| 218 results_map = {} | 223 results_map = {} |
| 219 seen_results = {} | 224 seen_results = {} |
| 220 | 225 |
| 221 for result_item in run_length_encoded_results: | 226 for result_item in run_length_encoded_results: |
| 222 _, result_type = self.results_json.occurances_and_type_from_result_i
tem(result_item) | 227 _, result_type = self.results_json.occurances_and_type_from_result_i
tem(result_item) |
| 223 if result_type in self.RESULT_TYPES_TO_IGNORE: | 228 if result_type in self.RESULT_TYPES_TO_IGNORE: |
| 224 continue | 229 continue |
| 225 | 230 |
| 226 if only_ignore_very_flaky and result_type not in seen_results: | 231 if only_ignore_very_flaky and result_type not in seen_results: |
| 227 # Only consider a short-lived result if we've seen it more than
once. | 232 # Only consider a short-lived result if we've seen it more than
once. |
| 228 # Otherwise, we include lots of false-positives due to tests tha
t fail | 233 # Otherwise, we include lots of false-positives due to tests tha
t fail |
| 229 # for a couple runs and then start passing. | 234 # for a couple runs and then start passing. |
| 230 # FIXME: Maybe we should make this more liberal and consider it
a flake | 235 # FIXME: Maybe we should make this more liberal and consider it
a flake |
| 231 # even if we only see that failure once. | 236 # even if we only see that failure once. |
| 232 seen_results[result_type] = True | 237 seen_results[result_type] = True |
| 233 continue | 238 continue |
| 234 | 239 |
| 235 results_map[result_type] = True | 240 results_map[result_type] = True |
| 236 | 241 |
| 237 return results_map.keys() | 242 return results_map.keys() |
| OLD | NEW |