Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(943)

Side by Side Diff: third_party/WebKit/Tools/Scripts/webkitpy/tool/commands/rebaseline.py

Issue 1379493002: Use chromium revision instead of blink svn revision when rebaselining layout tests. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: remove multiple repo support Created 5 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2010 Google Inc. All rights reserved. 1 # Copyright (c) 2010 Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 317 matching lines...) Expand 10 before | Expand all | Expand 10 after
328 self._builder_data = {} 328 self._builder_data = {}
329 329
330 def builder_data(self): 330 def builder_data(self):
331 if not self._builder_data: 331 if not self._builder_data:
332 for builder_name in self._release_builders(): 332 for builder_name in self._release_builders():
333 builder = self._tool.buildbot.builder_with_name(builder_name) 333 builder = self._tool.buildbot.builder_with_name(builder_name)
334 builder_results = builder.latest_layout_test_results() 334 builder_results = builder.latest_layout_test_results()
335 if builder_results: 335 if builder_results:
336 self._builder_data[builder_name] = builder_results 336 self._builder_data[builder_name] = builder_results
337 else: 337 else:
338 _log.warning("No result for builder '%s'" % builder_name) 338 raise Exception("No result for builder %s." % builder_name)
339 return self._builder_data 339 return self._builder_data
340 340
341 # The release builders cycle much faster than the debug ones and cover all t he platforms. 341 # The release builders cycle much faster than the debug ones and cover all t he platforms.
342 def _release_builders(self): 342 def _release_builders(self):
343 release_builders = [] 343 release_builders = []
344 for builder_name in builders.all_builder_names(): 344 for builder_name in builders.all_builder_names():
345 if builder_name.find('ASAN') != -1: 345 if builder_name.find('ASAN') != -1:
346 continue 346 continue
347 port = self._tool.port_factory.get_from_builder_name(builder_name) 347 port = self._tool.port_factory.get_from_builder_name(builder_name)
348 if port.test_configuration().build_type == 'release': 348 if port.test_configuration().build_type == 'release':
(...skipping 332 matching lines...) Expand 10 before | Expand all | Expand 10 after
681 ]) 681 ])
682 682
683 def bot_revision_data(self): 683 def bot_revision_data(self):
684 revisions = [] 684 revisions = []
685 for result in self.builder_data().values(): 685 for result in self.builder_data().values():
686 if result.run_was_interrupted(): 686 if result.run_was_interrupted():
687 _log.error("Can't rebaseline because the latest run on %s exited early." % result.builder_name()) 687 _log.error("Can't rebaseline because the latest run on %s exited early." % result.builder_name())
688 return [] 688 return []
689 revisions.append({ 689 revisions.append({
690 "builder": result.builder_name(), 690 "builder": result.builder_name(),
691 "revision": result.blink_revision(), 691 "revision": result.chromium_revision(),
692 }) 692 })
693 return revisions 693 return revisions
694 694
695 def tests_to_rebaseline(self, tool, min_revision, print_revisions): 695 def tests_to_rebaseline(self, tool, min_revision, print_revisions):
696 port = tool.port_factory.get() 696 port = tool.port_factory.get()
697 expectations_file_path = port.path_to_generic_test_expectations_file() 697 expectations_file_path = port.path_to_generic_test_expectations_file()
698 698
699 tests = set() 699 tests = set()
700 revision = None 700 revision = None
701 commit = None
701 author = None 702 author = None
702 bugs = set() 703 bugs = set()
703 has_any_needs_rebaseline_lines = False 704 has_any_needs_rebaseline_lines = False
704 705
705 for line in tool.scm().blame(expectations_file_path).split("\n"): 706 for line in tool.scm().blame(expectations_file_path).split("\n"):
706 comment_index = line.find("#") 707 comment_index = line.find("#")
707 if comment_index == -1: 708 if comment_index == -1:
708 comment_index = len(line) 709 comment_index = len(line)
709 line_without_comments = re.sub(r"\s+", " ", line[:comment_index].str ip()) 710 line_without_comments = re.sub(r"\s+", " ", line[:comment_index].str ip())
710 711
711 if "NeedsRebaseline" not in line_without_comments: 712 if "NeedsRebaseline" not in line_without_comments:
712 continue 713 continue
713 714
714 has_any_needs_rebaseline_lines = True 715 has_any_needs_rebaseline_lines = True
715 716
716 parsed_line = re.match("^(\S*)[^(]*\((\S*).*?([^ ]*)\ \[[^[]*$", lin e_without_comments) 717 parsed_line = re.match("^(\S*)[^(]*\((\S*).*?([^ ]*)\ \[[^[]*$", lin e_without_comments)
717 718
718 commit_hash = parsed_line.group(1) 719 commit_hash = parsed_line.group(1)
719 svn_revision = tool.scm().svn_revision_from_git_commit(commit_hash) 720 commit_position = tool.scm().commit_position_from_git_commit(commit_ hash)
720 721
721 test = parsed_line.group(3) 722 test = parsed_line.group(3)
722 if print_revisions: 723 if print_revisions:
723 _log.info("%s is waiting for r%s" % (test, svn_revision)) 724 _log.info("%s is waiting for r%s" % (test, commit_position))
724 725
725 if not svn_revision or svn_revision > min_revision: 726 if not commit_position or commit_position > min_revision:
726 continue 727 continue
727 728
728 if revision and svn_revision != revision: 729 if revision and commit_position != revision:
729 continue 730 continue
730 731
731 if not revision: 732 if not revision:
732 revision = svn_revision 733 revision = commit_position
734 commit = commit_hash
733 author = parsed_line.group(2) 735 author = parsed_line.group(2)
734 736
735 bugs.update(re.findall("crbug\.com\/(\d+)", line_without_comments)) 737 bugs.update(re.findall("crbug\.com\/(\d+)", line_without_comments))
736 tests.add(test) 738 tests.add(test)
737 739
738 if len(tests) >= self.MAX_LINES_TO_REBASELINE: 740 if len(tests) >= self.MAX_LINES_TO_REBASELINE:
739 _log.info("Too many tests to rebaseline in one patch. Doing the first %d." % self.MAX_LINES_TO_REBASELINE) 741 _log.info("Too many tests to rebaseline in one patch. Doing the first %d." % self.MAX_LINES_TO_REBASELINE)
740 break 742 break
741 743
742 return tests, revision, author, bugs, has_any_needs_rebaseline_lines 744 return tests, revision, commit, author, bugs, has_any_needs_rebaseline_l ines
743 745
744 def link_to_patch(self, revision): 746 def link_to_patch(self, commit):
745 return "http://src.chromium.org/viewvc/blink?view=revision&revision=" + str(revision) 747 return "https://chromium.googlesource.com/chromium/src/+/" + commit
746 748
747 def commit_message(self, author, revision, bugs): 749 def commit_message(self, author, revision, commit, bugs):
748 bug_string = "" 750 bug_string = ""
749 if bugs: 751 if bugs:
750 bug_string = "BUG=%s\n" % ",".join(bugs) 752 bug_string = "BUG=%s\n" % ",".join(bugs)
751 753
752 return """Auto-rebaseline for r%s 754 return """Auto-rebaseline for r%s
753 755
754 %s 756 %s
755 757
756 %sTBR=%s 758 %sTBR=%s
757 """ % (revision, self.link_to_patch(revision), bug_string, author) 759 """ % (revision, self.link_to_patch(commit), bug_string, author)
758 760
759 def get_test_prefix_list(self, tests): 761 def get_test_prefix_list(self, tests):
760 test_prefix_list = {} 762 test_prefix_list = {}
761 lines_to_remove = {} 763 lines_to_remove = {}
762 764
763 for builder_name in self._release_builders(): 765 for builder_name in self._release_builders():
764 port_name = builders.port_name_for_builder_name(builder_name) 766 port_name = builders.port_name_for_builder_name(builder_name)
765 port = self._tool.port_factory.get(port_name) 767 port = self._tool.port_factory.get(port_name)
766 expectations = TestExpectations(port, include_overrides=True) 768 expectations = TestExpectations(port, include_overrides=True)
767 for test in expectations.get_needs_rebaseline_failures(): 769 for test in expectations.get_needs_rebaseline_failures():
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after
816 818
817 if tool.scm().has_working_directory_changes(): 819 if tool.scm().has_working_directory_changes():
818 _log.error("Cannot proceed with working directory changes. Clean wor king directory first.") 820 _log.error("Cannot proceed with working directory changes. Clean wor king directory first.")
819 return 821 return
820 822
821 revision_data = self.bot_revision_data() 823 revision_data = self.bot_revision_data()
822 if not revision_data: 824 if not revision_data:
823 return 825 return
824 826
825 min_revision = int(min([item["revision"] for item in revision_data])) 827 min_revision = int(min([item["revision"] for item in revision_data]))
826 tests, revision, author, bugs, has_any_needs_rebaseline_lines = self.tes ts_to_rebaseline(tool, min_revision, print_revisions=options.verbose) 828 tests, revision, commit, author, bugs, has_any_needs_rebaseline_lines = self.tests_to_rebaseline(tool, min_revision, print_revisions=options.verbose)
827 829
828 if options.verbose: 830 if options.verbose:
829 _log.info("Min revision across all bots is %s." % min_revision) 831 _log.info("Min revision across all bots is %s." % min_revision)
830 for item in revision_data: 832 for item in revision_data:
831 _log.info("%s: r%s" % (item["builder"], item["revision"])) 833 _log.info("%s: r%s" % (item["builder"], item["revision"]))
832 834
833 if not tests: 835 if not tests:
834 _log.debug('No tests to rebaseline.') 836 _log.debug('No tests to rebaseline.')
835 return 837 return
836 838
837 if self.tree_status() == 'closed': 839 if self.tree_status() == 'closed':
838 _log.info('Cannot proceed. Tree is closed.') 840 _log.info('Cannot proceed. Tree is closed.')
839 return 841 return
840 842
841 _log.info('Rebaselining %s for r%s by %s.' % (list(tests), revision, aut hor)) 843 _log.info('Rebaselining %s for r%s by %s.' % (list(tests), revision, aut hor))
842 844
843 test_prefix_list, lines_to_remove = self.get_test_prefix_list(tests) 845 test_prefix_list, lines_to_remove = self.get_test_prefix_list(tests)
844 846
845 did_finish = False 847 did_finish = False
846 old_branch_name_or_ref = '' 848 old_branch_name_or_ref = ''
847 rebaseline_branch_name = self.AUTO_REBASELINE_BRANCH_NAME 849 rebaseline_branch_name = self.AUTO_REBASELINE_BRANCH_NAME
848 try: 850 try:
849 # Setup git-svn for dcommit if necessary.
850 if tool.executive.run_command(
851 ['git', 'config', '--local', '--get-regexp', r'^svn-remote\. '],
852 return_exit_code=True):
853 tool.executive.run_command(['git', 'auto-svn'])
854
855 # Save the current branch name and checkout a clean branch for the p atch. 851 # Save the current branch name and checkout a clean branch for the p atch.
856 old_branch_name_or_ref = _get_branch_name_or_ref(tool) 852 old_branch_name_or_ref = _get_branch_name_or_ref(tool)
857 if old_branch_name_or_ref == self.AUTO_REBASELINE_BRANCH_NAME: 853 if old_branch_name_or_ref == self.AUTO_REBASELINE_BRANCH_NAME:
858 rebaseline_branch_name = self.AUTO_REBASELINE_ALT_BRANCH_NAME 854 rebaseline_branch_name = self.AUTO_REBASELINE_ALT_BRANCH_NAME
859 tool.scm().delete_branch(rebaseline_branch_name) 855 tool.scm().delete_branch(rebaseline_branch_name)
860 tool.scm().create_clean_branch(rebaseline_branch_name) 856 tool.scm().create_clean_branch(rebaseline_branch_name)
861 857
862 # If the tests are passing everywhere, then this list will be empty. We don't need 858 # If the tests are passing everywhere, then this list will be empty. We don't need
863 # to rebaseline, but we'll still need to update TestExpectations. 859 # to rebaseline, but we'll still need to update TestExpectations.
864 if test_prefix_list: 860 if test_prefix_list:
865 self._rebaseline(options, test_prefix_list) 861 self._rebaseline(options, test_prefix_list)
866 862
867 tool.scm().commit_locally_with_message(self.commit_message(author, r evision, bugs)) 863 tool.scm().commit_locally_with_message(self.commit_message(author, r evision, commit, bugs))
868 864
869 # FIXME: It would be nice if we could dcommit the patch without uplo ading, but still 865 # FIXME: It would be nice if we could dcommit the patch without uplo ading, but still
870 # go through all the precommit hooks. For rebaselines with lots of f iles, uploading 866 # go through all the precommit hooks. For rebaselines with lots of f iles, uploading
871 # takes a long time and sometimes fails, but we don't want to commit if, e.g. the 867 # takes a long time and sometimes fails, but we don't want to commit if, e.g. the
872 # tree is closed. 868 # tree is closed.
873 did_finish = self._run_git_cl_command(options, ['upload', '-f']) 869 did_finish = self._run_git_cl_command(options, ['upload', '-f'])
874 870
875 if did_finish: 871 if did_finish:
876 # Uploading can take a very long time. Do another pull to make s ure TestExpectations is up to date, 872 # Uploading can take a very long time. Do another pull to make s ure TestExpectations is up to date,
877 # so the dcommit can go through. 873 # so the dcommit can go through.
(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after
969 self._tool.scm().checkout_branch(old_branch_name_or_ref) 965 self._tool.scm().checkout_branch(old_branch_name_or_ref)
970 else: 966 else:
971 self._log_queue.put(self.QUIT_LOG) 967 self._log_queue.put(self.QUIT_LOG)
972 log_thread.join() 968 log_thread.join()
973 969
974 def execute(self, options, args, tool): 970 def execute(self, options, args, tool):
975 self._verbose = options.verbose 971 self._verbose = options.verbose
976 while True: 972 while True:
977 self._do_one_rebaseline() 973 self._do_one_rebaseline()
978 time.sleep(self.SLEEP_TIME_IN_SECONDS) 974 time.sleep(self.SLEEP_TIME_IN_SECONDS)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698