OLD | NEW |
1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Updates TestExpectations based on results in builder bots. | 5 """Updates TestExpectations based on results in builder bots. |
6 | 6 |
7 Scans the TestExpectations file and uses results from actual builder bots runs | 7 Scans the TestExpectations file and uses results from actual builder bots runs |
8 to remove tests that are marked as flaky but don't fail in the specified way. | 8 to remove tests that are marked as flaky but don't fail in the specified way. |
9 | 9 |
10 E.g. If a test has this expectation: | 10 E.g. If a test has this expectation: |
(...skipping 23 matching lines...) Expand all Loading... |
34 def main(host, bot_test_expectations_factory, argv): | 34 def main(host, bot_test_expectations_factory, argv): |
35 parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.Ra
wTextHelpFormatter) | 35 parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.Ra
wTextHelpFormatter) |
36 parser.parse_args(argv) | 36 parser.parse_args(argv) |
37 | 37 |
38 port = host.port_factory.get() | 38 port = host.port_factory.get() |
39 | 39 |
40 logging.basicConfig(level=logging.INFO, format="%(message)s") | 40 logging.basicConfig(level=logging.INFO, format="%(message)s") |
41 | 41 |
42 expectations_file = port.path_to_generic_test_expectations_file() | 42 expectations_file = port.path_to_generic_test_expectations_file() |
43 if not host.filesystem.isfile(expectations_file): | 43 if not host.filesystem.isfile(expectations_file): |
44 _log.warn("Didn't find generic expectations file at: " + expectations_fi
le) | 44 _log.warning("Didn't find generic expectations file at: " + expectations
_file) |
45 return 1 | 45 return 1 |
46 | 46 |
47 remove_flakes_o_matic = RemoveFlakesOMatic(host, | 47 remove_flakes_o_matic = RemoveFlakesOMatic(host, |
48 port, | 48 port, |
49 bot_test_expectations_factory) | 49 bot_test_expectations_factory) |
50 | 50 |
51 test_expectations = remove_flakes_o_matic.get_updated_test_expectations() | 51 test_expectations = remove_flakes_o_matic.get_updated_test_expectations() |
52 | 52 |
53 remove_flakes_o_matic.write_test_expectations(test_expectations, | 53 remove_flakes_o_matic.write_test_expectations(test_expectations, |
54 expectations_file) | 54 expectations_file) |
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
97 | 97 |
98 if not builder_name: | 98 if not builder_name: |
99 _log.error('Failed to get builder for config [%s, %s, %s]', | 99 _log.error('Failed to get builder for config [%s, %s, %s]', |
100 config.version, config.architecture, config.build_typ
e) | 100 config.version, config.architecture, config.build_typ
e) |
101 # TODO(bokan): Matching configurations often give us bots that d
on't have a | 101 # TODO(bokan): Matching configurations often give us bots that d
on't have a |
102 # builder in builders.py's exact_matches. Should we ignore those
or be conservative | 102 # builder in builders.py's exact_matches. Should we ignore those
or be conservative |
103 # and assume we need these expectations to make a decision? | 103 # and assume we need these expectations to make a decision? |
104 return False | 104 return False |
105 | 105 |
106 if builder_name not in self._builder_results_by_path.keys(): | 106 if builder_name not in self._builder_results_by_path.keys(): |
107 _log.error('Failed to find results for builder "%s"' % builder_n
ame) | 107 _log.error('Failed to find results for builder "%s"', builder_na
me) |
108 return False | 108 return False |
109 | 109 |
110 results_by_path = self._builder_results_by_path[builder_name] | 110 results_by_path = self._builder_results_by_path[builder_name] |
111 | 111 |
112 # No results means the tests were all skipped or all results are pas
sing. | 112 # No results means the tests were all skipped or all results are pas
sing. |
113 if test_expectation_line.path not in results_by_path.keys(): | 113 if test_expectation_line.path not in results_by_path.keys(): |
114 continue | 114 continue |
115 | 115 |
116 results_for_single_test = results_by_path[test_expectation_line.path
] | 116 results_for_single_test = results_by_path[test_expectation_line.path
] |
117 | 117 |
(...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
189 builder_results_by_path = {} | 189 builder_results_by_path = {} |
190 for builder_name in self._host.builders.all_builder_names(): | 190 for builder_name in self._host.builders.all_builder_names(): |
191 expectations_for_builder = ( | 191 expectations_for_builder = ( |
192 self._expectations_factory.expectations_for_builder(builder_name
) | 192 self._expectations_factory.expectations_for_builder(builder_name
) |
193 ) | 193 ) |
194 | 194 |
195 if not expectations_for_builder: | 195 if not expectations_for_builder: |
196 # This is not fatal since we may not need to check these | 196 # This is not fatal since we may not need to check these |
197 # results. If we do need these results we'll log an error later | 197 # results. If we do need these results we'll log an error later |
198 # when trying to check against them. | 198 # when trying to check against them. |
199 _log.warn('Downloaded results are missing results for builder "%
s"' % builder_name) | 199 _log.warning('Downloaded results are missing results for builder
"%s"', builder_name) |
200 continue | 200 continue |
201 | 201 |
202 builder_results_by_path[builder_name] = ( | 202 builder_results_by_path[builder_name] = ( |
203 expectations_for_builder.all_results_by_path() | 203 expectations_for_builder.all_results_by_path() |
204 ) | 204 ) |
205 return builder_results_by_path | 205 return builder_results_by_path |
206 | 206 |
207 def _remove_associated_comments_and_whitespace(self, expectations, removed_i
ndex): | 207 def _remove_associated_comments_and_whitespace(self, expectations, removed_i
ndex): |
208 """Removes comments and whitespace from an empty expectation block. | 208 """Removes comments and whitespace from an empty expectation block. |
209 | 209 |
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
282 """Writes the given TestExpectations object to the filesystem. | 282 """Writes the given TestExpectations object to the filesystem. |
283 | 283 |
284 Args: | 284 Args: |
285 test_expectations: The TestExpectations object to write. | 285 test_expectations: The TestExpectations object to write. |
286 test_expectations_file: The full file path of the Blink | 286 test_expectations_file: The full file path of the Blink |
287 TestExpectations file. This file will be overwritten. | 287 TestExpectations file. This file will be overwritten. |
288 """ | 288 """ |
289 self._host.filesystem.write_text_file( | 289 self._host.filesystem.write_text_file( |
290 test_expectations_file, | 290 test_expectations_file, |
291 TestExpectations.list_to_string(test_expectations, reconstitute_only
_these=[])) | 291 TestExpectations.list_to_string(test_expectations, reconstitute_only
_these=[])) |
OLD | NEW |