| OLD | NEW |
| 1 # Copyright (C) 2013 Google Inc. All rights reserved. | 1 # Copyright (C) 2013 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 42 | 42 |
| 43 _log = logging.getLogger(__name__) | 43 _log = logging.getLogger(__name__) |
| 44 | 44 |
| 45 | 45 |
| 46 # results.json v4 format: | 46 # results.json v4 format: |
| 47 # { | 47 # { |
| 48 # 'version': 4, | 48 # 'version': 4, |
| 49 # 'builder name' : { | 49 # 'builder name' : { |
| 50 # 'blinkRevision': [], | 50 # 'blinkRevision': [], |
| 51 # 'tests': { | 51 # 'tests': { |
| 52 # 'directory' { # Each path component is a dictionary. | 52 # 'directory' { # Each path component is a dictionary. |
| 53 # 'testname.html': { | 53 # 'testname.html': { |
| 54 # 'expected' : 'FAIL', # expectation name | 54 # 'expected' : 'FAIL', # expectation name |
| 55 # 'results': [], # Run-length encoded result. | 55 # 'results': [], # Run-length encoded result. |
| 56 # 'times': [], | 56 # 'times': [], |
| 57 # 'bugs': [], # bug urls | 57 # 'bugs': [], # bug urls |
| 58 # } | 58 # } |
| 59 # } | 59 # } |
| 60 # } | 60 # } |
| 61 # 'buildNumbers': [], | 61 # 'buildNumbers': [], |
| 62 # 'secondsSinceEpoch': [], | 62 # 'secondsSinceEpoch': [], |
| 63 # 'chromeRevision': [], | 63 # 'chromeRevision': [], |
| 64 # 'failure_map': { } # Map from letter code to expectation name. | 64 # 'failure_map': { } # Map from letter code to expectation name. |
| 65 # }, | 65 # }, |
| 66 class ResultsJSON(object): | 66 class ResultsJSON(object): |
| 67 TESTS_KEY = 'tests' | 67 TESTS_KEY = 'tests' |
| 68 FAILURE_MAP_KEY = 'failure_map' | 68 FAILURE_MAP_KEY = 'failure_map' |
| 69 RESULTS_KEY = 'results' | 69 RESULTS_KEY = 'results' |
| 70 EXPECTATIONS_KEY = 'expected' | 70 EXPECTATIONS_KEY = 'expected' |
| 71 BUGS_KEY = 'bugs' | 71 BUGS_KEY = 'bugs' |
| 72 RLE_LENGTH = 0 | 72 RLE_LENGTH = 0 |
| 73 RLE_VALUE = 1 | 73 RLE_VALUE = 1 |
| 74 | 74 |
| (...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 137 if not results_json: | 137 if not results_json: |
| 138 return None | 138 return None |
| 139 return BotTestExpectations(results_json) | 139 return BotTestExpectations(results_json) |
| 140 | 140 |
| 141 def expectations_for_builder(self, builder): | 141 def expectations_for_builder(self, builder): |
| 142 results_json = self._results_json_for_builder(builder) | 142 results_json = self._results_json_for_builder(builder) |
| 143 if not results_json: | 143 if not results_json: |
| 144 return None | 144 return None |
| 145 return BotTestExpectations(results_json) | 145 return BotTestExpectations(results_json) |
| 146 | 146 |
| 147 |
| 147 class BotTestExpectations(object): | 148 class BotTestExpectations(object): |
| 148 # FIXME: Get this from the json instead of hard-coding it. | 149 # FIXME: Get this from the json instead of hard-coding it. |
| 149 RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y'] | 150 RESULT_TYPES_TO_IGNORE = ['N', 'X', 'Y'] |
| 150 | 151 |
| 151 # specifiers arg is used in unittests to avoid the static dependency on buil
ders. | 152 # specifiers arg is used in unittests to avoid the static dependency on buil
ders. |
| 152 def __init__(self, results_json, specifiers=None): | 153 def __init__(self, results_json, specifiers=None): |
| 153 self.results_json = results_json | 154 self.results_json = results_json |
| 154 self.specifiers = specifiers or set(builders.specifiers_for_builder(resu
lts_json.builder_name)) | 155 self.specifiers = specifiers or set(builders.specifiers_for_builder(resu
lts_json.builder_name)) |
| 155 | 156 |
| 156 def _line_from_test_and_flaky_types_and_bug_urls(self, test_path, flaky_type
s, bug_urls): | 157 def _line_from_test_and_flaky_types_and_bug_urls(self, test_path, flaky_type
s, bug_urls): |
| 157 line = TestExpectationLine() | 158 line = TestExpectationLine() |
| 158 line.original_string = test_path | 159 line.original_string = test_path |
| 159 line.name = test_path | 160 line.name = test_path |
| 160 line.filename = test_path | 161 line.filename = test_path |
| 161 line.path = test_path # FIXME: Should this be normpath? | 162 line.path = test_path # FIXME: Should this be normpath? |
| 162 line.matching_tests = [test_path] | 163 line.matching_tests = [test_path] |
| 163 line.bugs = bug_urls if bug_urls else ["Bug(gardener)"] | 164 line.bugs = bug_urls if bug_urls else ['Bug(gardener)'] |
| 164 line.expectations = sorted(map(self.results_json.expectation_for_type, f
laky_types)) | 165 line.expectations = sorted(map(self.results_json.expectation_for_type, f
laky_types)) |
| 165 line.specifiers = self.specifiers | 166 line.specifiers = self.specifiers |
| 166 return line | 167 return line |
| 167 | 168 |
| 168 def flakes_by_path(self, only_ignore_very_flaky): | 169 def flakes_by_path(self, only_ignore_very_flaky): |
| 169 """Sets test expectations to bot results if there are at least two disti
nct results.""" | 170 """Sets test expectations to bot results if there are at least two disti
nct results.""" |
| 170 flakes_by_path = {} | 171 flakes_by_path = {} |
| 171 for test_path, entry in self.results_json.walk_results(): | 172 for test_path, entry in self.results_json.walk_results(): |
| 172 results_dict = entry[self.results_json.RESULTS_KEY] | 173 results_dict = entry[self.results_json.RESULTS_KEY] |
| 173 flaky_types = self._flaky_types_in_results(results_dict, only_ignore
_very_flaky) | 174 flaky_types = self._flaky_types_in_results(results_dict, only_ignore
_very_flaky) |
| (...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 246 # Otherwise, we include lots of false-positives due to tests tha
t fail | 247 # Otherwise, we include lots of false-positives due to tests tha
t fail |
| 247 # for a couple runs and then start passing. | 248 # for a couple runs and then start passing. |
| 248 # FIXME: Maybe we should make this more liberal and consider it
a flake | 249 # FIXME: Maybe we should make this more liberal and consider it
a flake |
| 249 # even if we only see that failure once. | 250 # even if we only see that failure once. |
| 250 seen_results[result_type] = True | 251 seen_results[result_type] = True |
| 251 continue | 252 continue |
| 252 | 253 |
| 253 results_map[result_type] = True | 254 results_map[result_type] = True |
| 254 | 255 |
| 255 return results_map.keys() | 256 return results_map.keys() |
| OLD | NEW |