| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Updates TestExpectations based on results in builder bots. | 5 """Updates TestExpectations based on results in builder bots. |
| 6 | 6 |
| 7 Scans the TestExpectations file and uses results from actual builder bots runs | 7 Scans the TestExpectations file and uses results from actual builder bots runs |
| 8 to remove tests that are marked as flaky but don't fail in the specified way. | 8 to remove tests that are marked as flaky but don't fail in the specified way. |
| 9 | 9 |
| 10 E.g. If a test has this expectation: | 10 E.g. If a test has this expectation: |
| (...skipping 23 matching lines...) Expand all Loading... |
| 34 def main(host, bot_test_expectations_factory, argv): | 34 def main(host, bot_test_expectations_factory, argv): |
| 35 parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.Ra
wTextHelpFormatter) | 35 parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.Ra
wTextHelpFormatter) |
| 36 parser.parse_args(argv) | 36 parser.parse_args(argv) |
| 37 | 37 |
| 38 port = host.port_factory.get() | 38 port = host.port_factory.get() |
| 39 | 39 |
| 40 logging.basicConfig(level=logging.INFO, format="%(message)s") | 40 logging.basicConfig(level=logging.INFO, format="%(message)s") |
| 41 | 41 |
| 42 expectations_file = port.path_to_generic_test_expectations_file() | 42 expectations_file = port.path_to_generic_test_expectations_file() |
| 43 if not host.filesystem.isfile(expectations_file): | 43 if not host.filesystem.isfile(expectations_file): |
| 44 _log.warn("Didn't find generic expectations file at: " + expectations_fi
le) | 44 _log.warning("Didn't find generic expectations file at: " + expectations
_file) |
| 45 return 1 | 45 return 1 |
| 46 | 46 |
| 47 remove_flakes_o_matic = RemoveFlakesOMatic(host, | 47 remove_flakes_o_matic = RemoveFlakesOMatic(host, |
| 48 port, | 48 port, |
| 49 bot_test_expectations_factory) | 49 bot_test_expectations_factory) |
| 50 | 50 |
| 51 test_expectations = remove_flakes_o_matic.get_updated_test_expectations() | 51 test_expectations = remove_flakes_o_matic.get_updated_test_expectations() |
| 52 | 52 |
| 53 remove_flakes_o_matic.write_test_expectations(test_expectations, | 53 remove_flakes_o_matic.write_test_expectations(test_expectations, |
| 54 expectations_file) | 54 expectations_file) |
| (...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 97 | 97 |
| 98 if not builder_name: | 98 if not builder_name: |
| 99 _log.error('Failed to get builder for config [%s, %s, %s]', | 99 _log.error('Failed to get builder for config [%s, %s, %s]', |
| 100 config.version, config.architecture, config.build_typ
e) | 100 config.version, config.architecture, config.build_typ
e) |
| 101 # TODO(bokan): Matching configurations often give us bots that d
on't have a | 101 # TODO(bokan): Matching configurations often give us bots that d
on't have a |
| 102 # builder in builders.py's exact_matches. Should we ignore those
or be conservative | 102 # builder in builders.py's exact_matches. Should we ignore those
or be conservative |
| 103 # and assume we need these expectations to make a decision? | 103 # and assume we need these expectations to make a decision? |
| 104 return False | 104 return False |
| 105 | 105 |
| 106 if builder_name not in self._builder_results_by_path.keys(): | 106 if builder_name not in self._builder_results_by_path.keys(): |
| 107 _log.error('Failed to find results for builder "%s"' % builder_n
ame) | 107 _log.error('Failed to find results for builder "%s"', builder_na
me) |
| 108 return False | 108 return False |
| 109 | 109 |
| 110 results_by_path = self._builder_results_by_path[builder_name] | 110 results_by_path = self._builder_results_by_path[builder_name] |
| 111 | 111 |
| 112 # No results means the tests were all skipped or all results are pas
sing. | 112 # No results means the tests were all skipped or all results are pas
sing. |
| 113 if test_expectation_line.path not in results_by_path.keys(): | 113 if test_expectation_line.path not in results_by_path.keys(): |
| 114 continue | 114 continue |
| 115 | 115 |
| 116 results_for_single_test = results_by_path[test_expectation_line.path
] | 116 results_for_single_test = results_by_path[test_expectation_line.path
] |
| 117 | 117 |
| (...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 189 builder_results_by_path = {} | 189 builder_results_by_path = {} |
| 190 for builder_name in self._host.builders.all_builder_names(): | 190 for builder_name in self._host.builders.all_builder_names(): |
| 191 expectations_for_builder = ( | 191 expectations_for_builder = ( |
| 192 self._expectations_factory.expectations_for_builder(builder_name
) | 192 self._expectations_factory.expectations_for_builder(builder_name
) |
| 193 ) | 193 ) |
| 194 | 194 |
| 195 if not expectations_for_builder: | 195 if not expectations_for_builder: |
| 196 # This is not fatal since we may not need to check these | 196 # This is not fatal since we may not need to check these |
| 197 # results. If we do need these results we'll log an error later | 197 # results. If we do need these results we'll log an error later |
| 198 # when trying to check against them. | 198 # when trying to check against them. |
| 199 _log.warn('Downloaded results are missing results for builder "%
s"' % builder_name) | 199 _log.warning('Downloaded results are missing results for builder
"%s"', builder_name) |
| 200 continue | 200 continue |
| 201 | 201 |
| 202 builder_results_by_path[builder_name] = ( | 202 builder_results_by_path[builder_name] = ( |
| 203 expectations_for_builder.all_results_by_path() | 203 expectations_for_builder.all_results_by_path() |
| 204 ) | 204 ) |
| 205 return builder_results_by_path | 205 return builder_results_by_path |
| 206 | 206 |
| 207 def _remove_associated_comments_and_whitespace(self, expectations, removed_i
ndex): | 207 def _remove_associated_comments_and_whitespace(self, expectations, removed_i
ndex): |
| 208 """Removes comments and whitespace from an empty expectation block. | 208 """Removes comments and whitespace from an empty expectation block. |
| 209 | 209 |
| (...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 282 """Writes the given TestExpectations object to the filesystem. | 282 """Writes the given TestExpectations object to the filesystem. |
| 283 | 283 |
| 284 Args: | 284 Args: |
| 285 test_expectations: The TestExpectations object to write. | 285 test_expectations: The TestExpectations object to write. |
| 286 test_expectations_file: The full file path of the Blink | 286 test_expectations_file: The full file path of the Blink |
| 287 TestExpectations file. This file will be overwritten. | 287 TestExpectations file. This file will be overwritten. |
| 288 """ | 288 """ |
| 289 self._host.filesystem.write_text_file( | 289 self._host.filesystem.write_text_file( |
| 290 test_expectations_file, | 290 test_expectations_file, |
| 291 TestExpectations.list_to_string(test_expectations, reconstitute_only
_these=[])) | 291 TestExpectations.list_to_string(test_expectations, reconstitute_only
_these=[])) |
| OLD | NEW |