OLD | NEW |
1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """A command to fetch new baselines from try jobs for the current CL.""" | 5 """A command to fetch new baselines from try jobs for the current CL.""" |
6 | 6 |
7 import json | 7 import json |
8 import logging | 8 import logging |
9 import optparse | 9 import optparse |
10 | 10 |
11 from webkitpy.common.net.git_cl import GitCL | 11 from webkitpy.common.net.git_cl import GitCL |
12 from webkitpy.tool.commands.rebaseline import AbstractParallelRebaselineCommand | 12 from webkitpy.tool.commands.rebaseline import AbstractParallelRebaselineCommand,
TestBaselineSet |
13 from webkitpy.w3c.wpt_manifest import WPTManifest | 13 from webkitpy.w3c.wpt_manifest import WPTManifest |
14 | 14 |
15 _log = logging.getLogger(__name__) | 15 _log = logging.getLogger(__name__) |
16 | 16 |
17 | 17 |
18 class RebaselineCL(AbstractParallelRebaselineCommand): | 18 class RebaselineCL(AbstractParallelRebaselineCommand): |
19 name = 'rebaseline-cl' | 19 name = 'rebaseline-cl' |
20 help_text = 'Fetches new baselines for a CL from test runs on try bots.' | 20 help_text = 'Fetches new baselines for a CL from test runs on try bots.' |
21 long_help = ('By default, this command will check the latest try job results
' | 21 long_help = ('By default, this command will check the latest try job results
' |
22 'for all platforms, and start try jobs for platforms with no ' | 22 'for all platforms, and start try jobs for platforms with no ' |
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
79 _log.error('The following builders have no results:') | 79 _log.error('The following builders have no results:') |
80 for builder in builders_with_no_results: | 80 for builder in builders_with_no_results: |
81 _log.error(' %s', builder) | 81 _log.error(' %s', builder) |
82 return 1 | 82 return 1 |
83 | 83 |
84 _log.debug('Getting results for issue %d.', issue_number) | 84 _log.debug('Getting results for issue %d.', issue_number) |
85 builds_to_results = self._fetch_results(builds) | 85 builds_to_results = self._fetch_results(builds) |
86 if builds_to_results is None: | 86 if builds_to_results is None: |
87 return 1 | 87 return 1 |
88 | 88 |
89 test_prefix_list = {} | 89 test_baseline_set = TestBaselineSet(tool) |
90 if args: | 90 if args: |
91 for test in args: | 91 for test in args: |
92 test_prefix_list[test] = builds | 92 for build in builds: |
| 93 test_baseline_set.add(test, build) |
93 else: | 94 else: |
94 test_prefix_list = self._test_prefix_list( | 95 test_baseline_set = self._make_test_baseline_set( |
95 builds_to_results, | 96 builds_to_results, |
96 only_changed_tests=options.only_changed_tests) | 97 only_changed_tests=options.only_changed_tests) |
97 | 98 |
98 self._log_test_prefix_list(test_prefix_list) | 99 _log.debug('Rebaselining: %s', test_baseline_set) |
99 | 100 |
100 if not options.dry_run: | 101 if not options.dry_run: |
101 self.rebaseline(options, test_prefix_list) | 102 self.rebaseline(options, test_baseline_set) |
102 return 0 | 103 return 0 |
103 | 104 |
104 def _get_issue_number(self): | 105 def _get_issue_number(self): |
105 """Returns the current CL issue number, or None.""" | 106 """Returns the current CL issue number, or None.""" |
106 issue = self.git_cl().get_issue_number() | 107 issue = self.git_cl().get_issue_number() |
107 if not issue.isdigit(): | 108 if not issue.isdigit(): |
108 return None | 109 return None |
109 return int(issue) | 110 return int(issue) |
110 | 111 |
111 def git_cl(self): | 112 def git_cl(self): |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
153 results_url = buildbot.results_url(build.builder_name, build.build_n
umber) | 154 results_url = buildbot.results_url(build.builder_name, build.build_n
umber) |
154 layout_test_results = buildbot.fetch_results(build) | 155 layout_test_results = buildbot.fetch_results(build) |
155 if layout_test_results is None: | 156 if layout_test_results is None: |
156 _log.error('Failed to fetch results for: %s', build) | 157 _log.error('Failed to fetch results for: %s', build) |
157 _log.error('Results were expected to exist at:\n%s/results.html'
, results_url) | 158 _log.error('Results were expected to exist at:\n%s/results.html'
, results_url) |
158 _log.error('If the job failed, you could retry by running:\ngit
cl try -b %s', build.builder_name) | 159 _log.error('If the job failed, you could retry by running:\ngit
cl try -b %s', build.builder_name) |
159 return None | 160 return None |
160 results[build] = layout_test_results | 161 results[build] = layout_test_results |
161 return results | 162 return results |
162 | 163 |
163 def _test_prefix_list(self, builds_to_results, only_changed_tests): | 164 def _make_test_baseline_set(self, builds_to_results, only_changed_tests): |
164 """Returns a dict which lists the set of baselines to fetch. | 165 """Returns a dict which lists the set of baselines to fetch. |
165 | 166 |
166 The dict that is returned is a dict of tests to Build objects | 167 The dict that is returned is a dict of tests to Build objects |
167 to baseline file extensions. | 168 to baseline file extensions. |
168 | 169 |
169 Args: | 170 Args: |
170 builds_to_results: A dict mapping Builds to LayoutTestResults. | 171 builds_to_results: A dict mapping Builds to LayoutTestResults. |
171 only_changed_tests: Whether to only include baselines for tests that | 172 only_changed_tests: Whether to only include baselines for tests that |
172 are changed in this CL. If False, all new baselines for failing | 173 are changed in this CL. If False, all new baselines for failing |
173 tests will be downloaded, even for tests that were not modified. | 174 tests will be downloaded, even for tests that were not modified. |
174 | 175 |
175 Returns: | 176 Returns: |
176 A dict containing information about which new baselines to download. | 177 A dict containing information about which new baselines to download. |
177 """ | 178 """ |
178 builds_to_tests = {} | 179 builds_to_tests = {} |
179 for build, results in builds_to_results.iteritems(): | 180 for build, results in builds_to_results.iteritems(): |
180 builds_to_tests[build] = self._tests_to_rebaseline(build, results) | 181 builds_to_tests[build] = self._tests_to_rebaseline(build, results) |
181 if only_changed_tests: | 182 if only_changed_tests: |
182 files_in_cl = self._tool.git().changed_files(diff_filter='AM') | 183 files_in_cl = self._tool.git().changed_files(diff_filter='AM') |
183 # In the changed files list from Git, paths always use "/" as | 184 # In the changed files list from Git, paths always use "/" as |
184 # the path separator, and they're always relative to repo root. | 185 # the path separator, and they're always relative to repo root. |
185 # TODO(qyearsley): Do this without using a hard-coded constant. | 186 # TODO(qyearsley): Do this without using a hard-coded constant. |
186 test_base = 'third_party/WebKit/LayoutTests/' | 187 test_base = 'third_party/WebKit/LayoutTests/' |
187 tests_in_cl = [f[len(test_base):] for f in files_in_cl if f.startswi
th(test_base)] | 188 tests_in_cl = [f[len(test_base):] for f in files_in_cl if f.startswi
th(test_base)] |
188 result = {} | 189 |
| 190 test_baseline_set = TestBaselineSet(self._tool) |
189 for build, tests in builds_to_tests.iteritems(): | 191 for build, tests in builds_to_tests.iteritems(): |
190 for test in tests: | 192 for test in tests: |
191 if only_changed_tests and test not in tests_in_cl: | 193 if only_changed_tests and test not in tests_in_cl: |
192 continue | 194 continue |
193 if test not in result: | 195 test_baseline_set.add(test, build) |
194 result[test] = [] | 196 return test_baseline_set |
195 result[test].append(build) | |
196 return result | |
197 | 197 |
198 def _tests_to_rebaseline(self, build, layout_test_results): | 198 def _tests_to_rebaseline(self, build, layout_test_results): |
199 """Fetches a list of tests that should be rebaselined for some build.""" | 199 """Fetches a list of tests that should be rebaselined for some build.""" |
200 unexpected_results = layout_test_results.didnt_run_as_expected_results() | 200 unexpected_results = layout_test_results.didnt_run_as_expected_results() |
201 tests = sorted(r.test_name() for r in unexpected_results | 201 tests = sorted(r.test_name() for r in unexpected_results |
202 if r.is_missing_baseline() or r.has_mismatch_result()) | 202 if r.is_missing_baseline() or r.has_mismatch_result()) |
203 | 203 |
204 new_failures = self._fetch_tests_with_new_failures(build) | 204 new_failures = self._fetch_tests_with_new_failures(build) |
205 if new_failures is None: | 205 if new_failures is None: |
206 _log.warning('No retry summary available for build %s.', build) | 206 _log.warning('No retry summary available for build %s.', build) |
207 else: | 207 else: |
208 tests = [t for t in tests if t in new_failures] | 208 tests = [t for t in tests if t in new_failures] |
209 return tests | 209 return tests |
210 | 210 |
211 def _fetch_tests_with_new_failures(self, build): | 211 def _fetch_tests_with_new_failures(self, build): |
212 """For a given try job, lists tests that only occurred with the patch.""
" | 212 """For a given try job, lists tests that only occurred with the patch.""
" |
213 buildbot = self._tool.buildbot | 213 buildbot = self._tool.buildbot |
214 content = buildbot.fetch_retry_summary_json(build) | 214 content = buildbot.fetch_retry_summary_json(build) |
215 if content is None: | 215 if content is None: |
216 return None | 216 return None |
217 try: | 217 try: |
218 retry_summary = json.loads(content) | 218 retry_summary = json.loads(content) |
219 return retry_summary['failures'] | 219 return retry_summary['failures'] |
220 except (ValueError, KeyError): | 220 except (ValueError, KeyError): |
221 _log.warning('Unexpected retry summary content:\n%s', content) | 221 _log.warning('Unexpected retry summary content:\n%s', content) |
222 return None | 222 return None |
223 | |
224 @staticmethod | |
225 def _log_test_prefix_list(test_prefix_list): | |
226 """Logs the tests to download new baselines for.""" | |
227 if not test_prefix_list: | |
228 _log.info('No tests to rebaseline; exiting.') | |
229 return | |
230 _log.debug('Tests to rebaseline:') | |
231 for test, builds in test_prefix_list.iteritems(): | |
232 _log.debug(' %s:', test) | |
233 for build in sorted(builds): | |
234 _log.debug(' %s', build) | |
OLD | NEW |