| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """A command to fetch new baselines from try jobs for the current CL.""" | 5 """A command to fetch new baselines from try jobs for the current CL.""" |
| 6 | 6 |
| 7 import json | 7 import json |
| 8 import logging | 8 import logging |
| 9 import optparse | 9 import optparse |
| 10 | 10 |
| 11 from webkitpy.common.net.git_cl import GitCL | 11 from webkitpy.common.net.git_cl import GitCL |
| 12 from webkitpy.tool.commands.rebaseline import AbstractParallelRebaselineCommand | 12 from webkitpy.tool.commands.rebaseline import AbstractParallelRebaselineCommand,
TestBaselineSet |
| 13 from webkitpy.w3c.wpt_manifest import WPTManifest | 13 from webkitpy.w3c.wpt_manifest import WPTManifest |
| 14 | 14 |
| 15 _log = logging.getLogger(__name__) | 15 _log = logging.getLogger(__name__) |
| 16 | 16 |
| 17 | 17 |
| 18 class RebaselineCL(AbstractParallelRebaselineCommand): | 18 class RebaselineCL(AbstractParallelRebaselineCommand): |
| 19 name = 'rebaseline-cl' | 19 name = 'rebaseline-cl' |
| 20 help_text = 'Fetches new baselines for a CL from test runs on try bots.' | 20 help_text = 'Fetches new baselines for a CL from test runs on try bots.' |
| 21 long_help = ('By default, this command will check the latest try job results
' | 21 long_help = ('By default, this command will check the latest try job results
' |
| 22 'for all platforms, and start try jobs for platforms with no ' | 22 'for all platforms, and start try jobs for platforms with no ' |
| (...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 79 _log.error('The following builders have no results:') | 79 _log.error('The following builders have no results:') |
| 80 for builder in builders_with_no_results: | 80 for builder in builders_with_no_results: |
| 81 _log.error(' %s', builder) | 81 _log.error(' %s', builder) |
| 82 return 1 | 82 return 1 |
| 83 | 83 |
| 84 _log.debug('Getting results for issue %d.', issue_number) | 84 _log.debug('Getting results for issue %d.', issue_number) |
| 85 builds_to_results = self._fetch_results(builds) | 85 builds_to_results = self._fetch_results(builds) |
| 86 if builds_to_results is None: | 86 if builds_to_results is None: |
| 87 return 1 | 87 return 1 |
| 88 | 88 |
| 89 test_prefix_list = {} | 89 test_baseline_set = TestBaselineSet(tool) |
| 90 if args: | 90 if args: |
| 91 for test in args: | 91 for test in args: |
| 92 test_prefix_list[test] = builds | 92 for build in builds: |
| 93 test_baseline_set.add(test, build) |
| 93 else: | 94 else: |
| 94 test_prefix_list = self._test_prefix_list( | 95 test_baseline_set = self._make_test_baseline_set( |
| 95 builds_to_results, | 96 builds_to_results, |
| 96 only_changed_tests=options.only_changed_tests) | 97 only_changed_tests=options.only_changed_tests) |
| 97 | 98 |
| 98 self._log_test_prefix_list(test_prefix_list) | 99 _log.debug('Rebaselining: %s', test_baseline_set) |
| 99 | 100 |
| 100 if not options.dry_run: | 101 if not options.dry_run: |
| 101 self.rebaseline(options, test_prefix_list) | 102 self.rebaseline(options, test_baseline_set) |
| 102 return 0 | 103 return 0 |
| 103 | 104 |
| 104 def _get_issue_number(self): | 105 def _get_issue_number(self): |
| 105 """Returns the current CL issue number, or None.""" | 106 """Returns the current CL issue number, or None.""" |
| 106 issue = self.git_cl().get_issue_number() | 107 issue = self.git_cl().get_issue_number() |
| 107 if not issue.isdigit(): | 108 if not issue.isdigit(): |
| 108 return None | 109 return None |
| 109 return int(issue) | 110 return int(issue) |
| 110 | 111 |
| 111 def git_cl(self): | 112 def git_cl(self): |
| (...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 153 results_url = buildbot.results_url(build.builder_name, build.build_n
umber) | 154 results_url = buildbot.results_url(build.builder_name, build.build_n
umber) |
| 154 layout_test_results = buildbot.fetch_results(build) | 155 layout_test_results = buildbot.fetch_results(build) |
| 155 if layout_test_results is None: | 156 if layout_test_results is None: |
| 156 _log.error('Failed to fetch results for: %s', build) | 157 _log.error('Failed to fetch results for: %s', build) |
| 157 _log.error('Results were expected to exist at:\n%s/results.html'
, results_url) | 158 _log.error('Results were expected to exist at:\n%s/results.html'
, results_url) |
| 158 _log.error('If the job failed, you could retry by running:\ngit
cl try -b %s', build.builder_name) | 159 _log.error('If the job failed, you could retry by running:\ngit
cl try -b %s', build.builder_name) |
| 159 return None | 160 return None |
| 160 results[build] = layout_test_results | 161 results[build] = layout_test_results |
| 161 return results | 162 return results |
| 162 | 163 |
| 163 def _test_prefix_list(self, builds_to_results, only_changed_tests): | 164 def _make_test_baseline_set(self, builds_to_results, only_changed_tests): |
| 164 """Returns a dict which lists the set of baselines to fetch. | 165 """Returns a dict which lists the set of baselines to fetch. |
| 165 | 166 |
| 166 The dict that is returned is a dict of tests to Build objects | 167 The dict that is returned is a dict of tests to Build objects |
| 167 to baseline file extensions. | 168 to baseline file extensions. |
| 168 | 169 |
| 169 Args: | 170 Args: |
| 170 builds_to_results: A dict mapping Builds to LayoutTestResults. | 171 builds_to_results: A dict mapping Builds to LayoutTestResults. |
| 171 only_changed_tests: Whether to only include baselines for tests that | 172 only_changed_tests: Whether to only include baselines for tests that |
| 172 are changed in this CL. If False, all new baselines for failing | 173 are changed in this CL. If False, all new baselines for failing |
| 173 tests will be downloaded, even for tests that were not modified. | 174 tests will be downloaded, even for tests that were not modified. |
| 174 | 175 |
| 175 Returns: | 176 Returns: |
| 176 A dict containing information about which new baselines to download. | 177 A dict containing information about which new baselines to download. |
| 177 """ | 178 """ |
| 178 builds_to_tests = {} | 179 builds_to_tests = {} |
| 179 for build, results in builds_to_results.iteritems(): | 180 for build, results in builds_to_results.iteritems(): |
| 180 builds_to_tests[build] = self._tests_to_rebaseline(build, results) | 181 builds_to_tests[build] = self._tests_to_rebaseline(build, results) |
| 181 if only_changed_tests: | 182 if only_changed_tests: |
| 182 files_in_cl = self._tool.git().changed_files(diff_filter='AM') | 183 files_in_cl = self._tool.git().changed_files(diff_filter='AM') |
| 183 # In the changed files list from Git, paths always use "/" as | 184 # In the changed files list from Git, paths always use "/" as |
| 184 # the path separator, and they're always relative to repo root. | 185 # the path separator, and they're always relative to repo root. |
| 185 # TODO(qyearsley): Do this without using a hard-coded constant. | 186 # TODO(qyearsley): Do this without using a hard-coded constant. |
| 186 test_base = 'third_party/WebKit/LayoutTests/' | 187 test_base = 'third_party/WebKit/LayoutTests/' |
| 187 tests_in_cl = [f[len(test_base):] for f in files_in_cl if f.startswi
th(test_base)] | 188 tests_in_cl = [f[len(test_base):] for f in files_in_cl if f.startswi
th(test_base)] |
| 188 result = {} | 189 |
| 190 test_baseline_set = TestBaselineSet(self._tool) |
| 189 for build, tests in builds_to_tests.iteritems(): | 191 for build, tests in builds_to_tests.iteritems(): |
| 190 for test in tests: | 192 for test in tests: |
| 191 if only_changed_tests and test not in tests_in_cl: | 193 if only_changed_tests and test not in tests_in_cl: |
| 192 continue | 194 continue |
| 193 if test not in result: | 195 test_baseline_set.add(test, build) |
| 194 result[test] = [] | 196 return test_baseline_set |
| 195 result[test].append(build) | |
| 196 return result | |
| 197 | 197 |
| 198 def _tests_to_rebaseline(self, build, layout_test_results): | 198 def _tests_to_rebaseline(self, build, layout_test_results): |
| 199 """Fetches a list of tests that should be rebaselined for some build.""" | 199 """Fetches a list of tests that should be rebaselined for some build.""" |
| 200 unexpected_results = layout_test_results.didnt_run_as_expected_results() | 200 unexpected_results = layout_test_results.didnt_run_as_expected_results() |
| 201 tests = sorted(r.test_name() for r in unexpected_results | 201 tests = sorted(r.test_name() for r in unexpected_results |
| 202 if r.is_missing_baseline() or r.has_mismatch_result()) | 202 if r.is_missing_baseline() or r.has_mismatch_result()) |
| 203 | 203 |
| 204 new_failures = self._fetch_tests_with_new_failures(build) | 204 new_failures = self._fetch_tests_with_new_failures(build) |
| 205 if new_failures is None: | 205 if new_failures is None: |
| 206 _log.warning('No retry summary available for build %s.', build) | 206 _log.warning('No retry summary available for build %s.', build) |
| 207 else: | 207 else: |
| 208 tests = [t for t in tests if t in new_failures] | 208 tests = [t for t in tests if t in new_failures] |
| 209 return tests | 209 return tests |
| 210 | 210 |
| 211 def _fetch_tests_with_new_failures(self, build): | 211 def _fetch_tests_with_new_failures(self, build): |
| 212 """For a given try job, lists tests that only occurred with the patch.""
" | 212 """For a given try job, lists tests that only occurred with the patch.""
" |
| 213 buildbot = self._tool.buildbot | 213 buildbot = self._tool.buildbot |
| 214 content = buildbot.fetch_retry_summary_json(build) | 214 content = buildbot.fetch_retry_summary_json(build) |
| 215 if content is None: | 215 if content is None: |
| 216 return None | 216 return None |
| 217 try: | 217 try: |
| 218 retry_summary = json.loads(content) | 218 retry_summary = json.loads(content) |
| 219 return retry_summary['failures'] | 219 return retry_summary['failures'] |
| 220 except (ValueError, KeyError): | 220 except (ValueError, KeyError): |
| 221 _log.warning('Unexpected retry summary content:\n%s', content) | 221 _log.warning('Unexpected retry summary content:\n%s', content) |
| 222 return None | 222 return None |
| 223 | |
| 224 @staticmethod | |
| 225 def _log_test_prefix_list(test_prefix_list): | |
| 226 """Logs the tests to download new baselines for.""" | |
| 227 if not test_prefix_list: | |
| 228 _log.info('No tests to rebaseline; exiting.') | |
| 229 return | |
| 230 _log.debug('Tests to rebaseline:') | |
| 231 for test, builds in test_prefix_list.iteritems(): | |
| 232 _log.debug(' %s:', test) | |
| 233 for build in sorted(builds): | |
| 234 _log.debug(' %s', build) | |
| OLD | NEW |