| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """A command to download new baselines for NeedsRebaseline tests. | 5 """A command to download new baselines for NeedsRebaseline tests. |
| 6 | 6 |
| 7 This command checks the list of tests with NeedsRebaseline expectations, | 7 This command checks the list of tests with NeedsRebaseline expectations, |
| 8 and downloads the latest baselines for those tests from the results archived | 8 and downloads the latest baselines for those tests from the results archived |
| 9 by the continuous builders. | 9 by the continuous builders. |
| 10 """ | 10 """ |
| (...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 55 ( | 55 ( |
| 56 [^@]+ # Username preceding @ | 56 [^@]+ # Username preceding @ |
| 57 @ | 57 @ |
| 58 [^@>]+ # Domain terminated by @ or >, some lines have an ad
ditional @ fragment after the email. | 58 [^@>]+ # Domain terminated by @ or >, some lines have an ad
ditional @ fragment after the email. |
| 59 ) | 59 ) |
| 60 .*?([^ ]*) # Test file name | 60 .*?([^ ]*) # Test file name |
| 61 \ \[ # Single space followed by opening [ for expectation
specifier | 61 \ \[ # Single space followed by opening [ for expectation
specifier |
| 62 [^[]*$ # Prevents matching previous [ for version specifier
s instead of expectation specifiers | 62 [^[]*$ # Prevents matching previous [ for version specifier
s instead of expectation specifiers |
| 63 ''', re.VERBOSE) | 63 ''', re.VERBOSE) |
| 64 | 64 |
| 65 def bot_revision_data(self, scm): | 65 def bot_revision_data(self, git): |
| 66 revisions = [] | 66 revisions = [] |
| 67 for builder_name in self._release_builders(): | 67 for builder_name in self._release_builders(): |
| 68 result = self._tool.buildbot.fetch_results(Build(builder_name)) | 68 result = self._tool.buildbot.fetch_results(Build(builder_name)) |
| 69 if result.run_was_interrupted(): | 69 if result.run_was_interrupted(): |
| 70 _log.error("Can't rebaseline because the latest run on %s exited
early.", result.builder_name()) | 70 _log.error("Can't rebaseline because the latest run on %s exited
early.", result.builder_name()) |
| 71 return [] | 71 return [] |
| 72 revisions.append({ | 72 revisions.append({ |
| 73 "builder": result.builder_name(), | 73 "builder": result.builder_name(), |
| 74 "revision": result.chromium_revision(scm), | 74 "revision": result.chromium_revision(git), |
| 75 }) | 75 }) |
| 76 return revisions | 76 return revisions |
| 77 | 77 |
| 78 @staticmethod | 78 @staticmethod |
| 79 def _strip_comments(line): | 79 def _strip_comments(line): |
| 80 comment_index = line.find("#") | 80 comment_index = line.find("#") |
| 81 if comment_index == -1: | 81 if comment_index == -1: |
| 82 comment_index = len(line) | 82 comment_index = len(line) |
| 83 return re.sub(r"\s+", " ", line[:comment_index].strip()) | 83 return re.sub(r"\s+", " ", line[:comment_index].strip()) |
| 84 | 84 |
| 85 def tests_to_rebaseline(self, tool, min_revision, print_revisions): | 85 def tests_to_rebaseline(self, tool, min_revision, print_revisions): |
| 86 port = tool.port_factory.get() | 86 port = tool.port_factory.get() |
| 87 expectations_file_path = port.path_to_generic_test_expectations_file() | 87 expectations_file_path = port.path_to_generic_test_expectations_file() |
| 88 | 88 |
| 89 tests = set() | 89 tests = set() |
| 90 revision = None | 90 revision = None |
| 91 commit = None | 91 commit = None |
| 92 author = None | 92 author = None |
| 93 bugs = set() | 93 bugs = set() |
| 94 has_any_needs_rebaseline_lines = False | 94 has_any_needs_rebaseline_lines = False |
| 95 | 95 |
| 96 for line in tool.scm().blame(expectations_file_path).split("\n"): | 96 for line in tool.git().blame(expectations_file_path).split("\n"): |
| 97 line = self._strip_comments(line) | 97 line = self._strip_comments(line) |
| 98 if "NeedsRebaseline" not in line: | 98 if "NeedsRebaseline" not in line: |
| 99 continue | 99 continue |
| 100 | 100 |
| 101 has_any_needs_rebaseline_lines = True | 101 has_any_needs_rebaseline_lines = True |
| 102 | 102 |
| 103 parsed_line = self._blame_regex.match(line) | 103 parsed_line = self._blame_regex.match(line) |
| 104 if not parsed_line: | 104 if not parsed_line: |
| 105 # Deal gracefully with inability to parse blame info for a line
in TestExpectations. | 105 # Deal gracefully with inability to parse blame info for a line
in TestExpectations. |
| 106 # Parsing could fail if for example during local debugging the d
eveloper modifies | 106 # Parsing could fail if for example during local debugging the d
eveloper modifies |
| 107 # TestExpectations and does not commit. | 107 # TestExpectations and does not commit. |
| 108 _log.info("Couldn't find blame info for expectations line, skipp
ing [line=%s].", line) | 108 _log.info("Couldn't find blame info for expectations line, skipp
ing [line=%s].", line) |
| 109 continue | 109 continue |
| 110 | 110 |
| 111 commit_hash = parsed_line.group(1) | 111 commit_hash = parsed_line.group(1) |
| 112 commit_position = tool.scm().commit_position_from_git_commit(commit_
hash) | 112 commit_position = tool.git().commit_position_from_git_commit(commit_
hash) |
| 113 | 113 |
| 114 test = parsed_line.group(3) | 114 test = parsed_line.group(3) |
| 115 if print_revisions: | 115 if print_revisions: |
| 116 _log.info("%s is waiting for r%s", test, commit_position) | 116 _log.info("%s is waiting for r%s", test, commit_position) |
| 117 | 117 |
| 118 if not commit_position or commit_position > min_revision: | 118 if not commit_position or commit_position > min_revision: |
| 119 continue | 119 continue |
| 120 | 120 |
| 121 if revision and commit_position != revision: | 121 if revision and commit_position != revision: |
| 122 continue | 122 continue |
| (...skipping 80 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 203 blink_tree_status_url = "http://chromium-status.appspot.com/status" | 203 blink_tree_status_url = "http://chromium-status.appspot.com/status" |
| 204 status = urllib2.urlopen(blink_tree_status_url).read().lower() | 204 status = urllib2.urlopen(blink_tree_status_url).read().lower() |
| 205 if 'closed' in status or status == "0": | 205 if 'closed' in status or status == "0": |
| 206 return 'closed' | 206 return 'closed' |
| 207 elif 'open' in status or status == "1": | 207 elif 'open' in status or status == "1": |
| 208 return 'open' | 208 return 'open' |
| 209 return 'unknown' | 209 return 'unknown' |
| 210 | 210 |
| 211 def execute(self, options, args, tool): | 211 def execute(self, options, args, tool): |
| 212 self._tool = tool | 212 self._tool = tool |
| 213 if tool.scm().executable_name == "svn": | 213 if tool.git().executable_name == "svn": |
| 214 _log.error("Auto rebaseline only works with a git checkout.") | 214 _log.error("Auto rebaseline only works with a git checkout.") |
| 215 return | 215 return |
| 216 | 216 |
| 217 if not options.dry_run and tool.scm().has_working_directory_changes(): | 217 if not options.dry_run and tool.git().has_working_directory_changes(): |
| 218 _log.error("Cannot proceed with working directory changes. Clean wor
king directory first.") | 218 _log.error("Cannot proceed with working directory changes. Clean wor
king directory first.") |
| 219 return | 219 return |
| 220 | 220 |
| 221 revision_data = self.bot_revision_data(tool.scm()) | 221 revision_data = self.bot_revision_data(tool.git()) |
| 222 if not revision_data: | 222 if not revision_data: |
| 223 return | 223 return |
| 224 | 224 |
| 225 min_revision = int(min([item["revision"] for item in revision_data])) | 225 min_revision = int(min([item["revision"] for item in revision_data])) |
| 226 tests, revision, commit, author, bugs, _ = self.tests_to_rebaseline( | 226 tests, revision, commit, author, bugs, _ = self.tests_to_rebaseline( |
| 227 tool, min_revision, print_revisions=options.verbose) | 227 tool, min_revision, print_revisions=options.verbose) |
| 228 | 228 |
| 229 if options.verbose: | 229 if options.verbose: |
| 230 _log.info("Min revision across all bots is %s.", min_revision) | 230 _log.info("Min revision across all bots is %s.", min_revision) |
| 231 for item in revision_data: | 231 for item in revision_data: |
| (...skipping 10 matching lines...) Expand all Loading... |
| 242 _log.info('Rebaselining %s for r%s by %s.', list(tests), revision, autho
r) | 242 _log.info('Rebaselining %s for r%s by %s.', list(tests), revision, autho
r) |
| 243 | 243 |
| 244 test_prefix_list, _ = self.get_test_prefix_list(tests) | 244 test_prefix_list, _ = self.get_test_prefix_list(tests) |
| 245 | 245 |
| 246 did_switch_branches = False | 246 did_switch_branches = False |
| 247 did_finish = False | 247 did_finish = False |
| 248 old_branch_name_or_ref = '' | 248 old_branch_name_or_ref = '' |
| 249 rebaseline_branch_name = self.AUTO_REBASELINE_BRANCH_NAME | 249 rebaseline_branch_name = self.AUTO_REBASELINE_BRANCH_NAME |
| 250 try: | 250 try: |
| 251 # Save the current branch name and check out a clean branch for the
patch. | 251 # Save the current branch name and check out a clean branch for the
patch. |
| 252 old_branch_name_or_ref = tool.scm().current_branch_or_ref() | 252 old_branch_name_or_ref = tool.git().current_branch_or_ref() |
| 253 if old_branch_name_or_ref == self.AUTO_REBASELINE_BRANCH_NAME: | 253 if old_branch_name_or_ref == self.AUTO_REBASELINE_BRANCH_NAME: |
| 254 rebaseline_branch_name = self.AUTO_REBASELINE_ALT_BRANCH_NAME | 254 rebaseline_branch_name = self.AUTO_REBASELINE_ALT_BRANCH_NAME |
| 255 if not options.dry_run: | 255 if not options.dry_run: |
| 256 tool.scm().delete_branch(rebaseline_branch_name) | 256 tool.git().delete_branch(rebaseline_branch_name) |
| 257 tool.scm().create_clean_branch(rebaseline_branch_name) | 257 tool.git().create_clean_branch(rebaseline_branch_name) |
| 258 did_switch_branches = True | 258 did_switch_branches = True |
| 259 | 259 |
| 260 if test_prefix_list: | 260 if test_prefix_list: |
| 261 self.rebaseline(options, test_prefix_list) | 261 self.rebaseline(options, test_prefix_list) |
| 262 | 262 |
| 263 if options.dry_run: | 263 if options.dry_run: |
| 264 return | 264 return |
| 265 | 265 |
| 266 tool.scm().commit_locally_with_message( | 266 tool.git().commit_locally_with_message( |
| 267 self.commit_message(author, revision, commit, bugs)) | 267 self.commit_message(author, revision, commit, bugs)) |
| 268 | 268 |
| 269 # FIXME: It would be nice if we could dcommit the patch without uplo
ading, but still | 269 # FIXME: It would be nice if we could dcommit the patch without uplo
ading, but still |
| 270 # go through all the precommit hooks. For rebaselines with lots of f
iles, uploading | 270 # go through all the precommit hooks. For rebaselines with lots of f
iles, uploading |
| 271 # takes a long time and sometimes fails, but we don't want to commit
if, e.g. the | 271 # takes a long time and sometimes fails, but we don't want to commit
if, e.g. the |
| 272 # tree is closed. | 272 # tree is closed. |
| 273 did_finish = self._run_git_cl_command(options, ['upload', '-f']) | 273 did_finish = self._run_git_cl_command(options, ['upload', '-f']) |
| 274 | 274 |
| 275 if did_finish: | 275 if did_finish: |
| 276 # Uploading can take a very long time. Do another pull to make s
ure TestExpectations is up to date, | 276 # Uploading can take a very long time. Do another pull to make s
ure TestExpectations is up to date, |
| 277 # so the dcommit can go through. | 277 # so the dcommit can go through. |
| 278 # FIXME: Log the pull and dcommit stdout/stderr to the log-serve
r. | 278 # FIXME: Log the pull and dcommit stdout/stderr to the log-serve
r. |
| 279 tool.executive.run_command(['git', 'pull']) | 279 tool.executive.run_command(['git', 'pull']) |
| 280 | 280 |
| 281 self._run_git_cl_command(options, ['land', '-f', '-v']) | 281 self._run_git_cl_command(options, ['land', '-f', '-v']) |
| 282 except OSError: | 282 except OSError: |
| 283 traceback.print_exc(file=sys.stderr) | 283 traceback.print_exc(file=sys.stderr) |
| 284 finally: | 284 finally: |
| 285 if did_switch_branches: | 285 if did_switch_branches: |
| 286 if did_finish: | 286 if did_finish: |
| 287 # Close the issue if dcommit failed. | 287 # Close the issue if dcommit failed. |
| 288 issue_already_closed = tool.executive.run_command( | 288 issue_already_closed = tool.executive.run_command( |
| 289 ['git', 'config', 'branch.%s.rietveldissue' % rebaseline
_branch_name], | 289 ['git', 'config', 'branch.%s.rietveldissue' % rebaseline
_branch_name], |
| 290 return_exit_code=True) | 290 return_exit_code=True) |
| 291 if not issue_already_closed: | 291 if not issue_already_closed: |
| 292 self._run_git_cl_command(options, ['set_close']) | 292 self._run_git_cl_command(options, ['set_close']) |
| 293 | 293 |
| 294 tool.scm().ensure_cleanly_tracking_remote_master() | 294 tool.git().ensure_cleanly_tracking_remote_master() |
| 295 if old_branch_name_or_ref: | 295 if old_branch_name_or_ref: |
| 296 tool.scm().checkout_branch(old_branch_name_or_ref) | 296 tool.git().checkout_branch(old_branch_name_or_ref) |
| 297 tool.scm().delete_branch(rebaseline_branch_name) | 297 tool.git().delete_branch(rebaseline_branch_name) |
| OLD | NEW |