Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Updates TestExpectations based on results in builder bots. | 5 """Updates TestExpectations based on results in builder bots. |
| 6 | 6 |
| 7 Scans the TestExpectations file and uses results from actual builder bots runs | 7 Scans the TestExpectations file and uses results from actual builder bots runs |
| 8 to remove tests that are marked as flaky but don't fail in the specified way. | 8 to remove tests that are marked as flaky but don't fail in the specified way. |
| 9 | 9 |
| 10 E.g. If a test has this expectation: | 10 E.g. If a test has this expectation: |
| (...skipping 88 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 99 # Don't check lines that have expectations like NeedsRebaseline or Skip. | 99 # Don't check lines that have expectations like NeedsRebaseline or Skip. |
| 100 if self._has_unstrippable_expectations(expectations): | 100 if self._has_unstrippable_expectations(expectations): |
| 101 return False | 101 return False |
| 102 | 102 |
| 103 # Don't check lines unless they're flaky. i.e. At least one expectation is a PASS. | 103 # Don't check lines unless they're flaky. i.e. At least one expectation is a PASS. |
| 104 if not self._has_pass_expectation(expectations): | 104 if not self._has_pass_expectation(expectations): |
| 105 return False | 105 return False |
| 106 | 106 |
| 107 # Don't check lines that have expectations for directories, since | 107 # Don't check lines that have expectations for directories, since |
| 108 # the flakiness of all sub-tests isn't as easy to check. | 108 # the flakiness of all sub-tests isn't as easy to check. |
| 109 if self._is_directory(test_expectation_line.path): | 109 if self._port.test_isdir(test_expectation_line.name): |
|
qyearsley
2017/02/08 18:30:56
I'm not really sure whether it matters whether tes
bokan
2017/02/08 19:12:01
Looks to me like the only difference is that path
| |
| 110 return False | 110 return False |
| 111 | 111 |
| 112 # The line can be deleted if the only expectation on the line that appea rs in the actual | 112 # The line can be deleted if the only expectation on the line that appea rs in the actual |
| 113 # results is the PASS expectation. | 113 # results is the PASS expectation. |
| 114 builders_checked = [] | 114 builders_checked = [] |
| 115 for config in test_expectation_line.matching_configurations: | 115 for config in test_expectation_line.matching_configurations: |
| 116 builder_name = self._host.builders.builder_name_for_specifiers(confi g.version, config.build_type) | 116 builder_name = self._host.builders.builder_name_for_specifiers(confi g.version, config.build_type) |
| 117 | 117 |
| 118 if not builder_name: | 118 if not builder_name: |
| 119 _log.debug('No builder with config %s', config) | 119 _log.debug('No builder with config %s', config) |
| (...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 189 | 189 |
| 190 Returns: | 190 Returns: |
| 191 True if at least one of the expectations is unstrippable. False | 191 True if at least one of the expectations is unstrippable. False |
| 192 otherwise. | 192 otherwise. |
| 193 """ | 193 """ |
| 194 unstrippable_expectations = ('REBASELINE', 'NEEDSREBASELINE', | 194 unstrippable_expectations = ('REBASELINE', 'NEEDSREBASELINE', |
| 195 'NEEDSMANUALREBASELINE', 'SLOW', | 195 'NEEDSMANUALREBASELINE', 'SLOW', |
| 196 'SKIP') | 196 'SKIP') |
| 197 return any(s in expectations for s in unstrippable_expectations) | 197 return any(s in expectations for s in unstrippable_expectations) |
| 198 | 198 |
| 199 def _is_directory(self, path): | |
| 200 """Checks whether a path relative to the layout tests directory is a dir ectory.""" | |
| 201 filesystem = self._host.filesystem | |
| 202 abs_path = filesystem.join(self._port.layout_tests_dir(), path) | |
| 203 return filesystem.isdir(abs_path) | |
| 204 | |
| 205 def _get_builder_results_by_path(self): | 199 def _get_builder_results_by_path(self): |
| 206 """Returns a dictionary of results for each builder. | 200 """Returns a dictionary of results for each builder. |
| 207 | 201 |
| 208 Returns a dictionary where each key is a builder and value is a dictiona ry containing | 202 Returns a dictionary where each key is a builder and value is a dictiona ry containing |
| 209 the distinct results for each test. E.g. | 203 the distinct results for each test. E.g. |
| 210 | 204 |
| 211 { | 205 { |
| 212 'WebKit Linux Precise': { | 206 'WebKit Linux Precise': { |
| 213 'test1.html': ['PASS', 'IMAGE'], | 207 'test1.html': ['PASS', 'IMAGE'], |
| 214 'test2.html': ['PASS'], | 208 'test2.html': ['PASS'], |
| (...skipping 126 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 341 """Writes the given TestExpectations object to the filesystem. | 335 """Writes the given TestExpectations object to the filesystem. |
| 342 | 336 |
| 343 Args: | 337 Args: |
| 344 test_expectations: The TestExpectations object to write. | 338 test_expectations: The TestExpectations object to write. |
| 345 test_expectations_file: The full file path of the Blink | 339 test_expectations_file: The full file path of the Blink |
| 346 TestExpectations file. This file will be overwritten. | 340 TestExpectations file. This file will be overwritten. |
| 347 """ | 341 """ |
| 348 self._host.filesystem.write_text_file( | 342 self._host.filesystem.write_text_file( |
| 349 test_expectations_file, | 343 test_expectations_file, |
| 350 TestExpectations.list_to_string(test_expectations, reconstitute_only _these=[])) | 344 TestExpectations.list_to_string(test_expectations, reconstitute_only _these=[])) |
| OLD | NEW |