OLD | NEW |
1 # Copyright (c) 2010 Google Inc. All rights reserved. | 1 # Copyright (c) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 317 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
328 self._builder_data = {} | 328 self._builder_data = {} |
329 | 329 |
330 def builder_data(self): | 330 def builder_data(self): |
331 if not self._builder_data: | 331 if not self._builder_data: |
332 for builder_name in self._release_builders(): | 332 for builder_name in self._release_builders(): |
333 builder = self._tool.buildbot.builder_with_name(builder_name) | 333 builder = self._tool.buildbot.builder_with_name(builder_name) |
334 builder_results = builder.latest_layout_test_results() | 334 builder_results = builder.latest_layout_test_results() |
335 if builder_results: | 335 if builder_results: |
336 self._builder_data[builder_name] = builder_results | 336 self._builder_data[builder_name] = builder_results |
337 else: | 337 else: |
338 _log.warning("No result for builder '%s'" % builder_name) | 338 raise Exception("No result for builder %s." % builder_name) |
339 return self._builder_data | 339 return self._builder_data |
340 | 340 |
341 # The release builders cycle much faster than the debug ones and cover all t
he platforms. | 341 # The release builders cycle much faster than the debug ones and cover all t
he platforms. |
342 def _release_builders(self): | 342 def _release_builders(self): |
343 release_builders = [] | 343 release_builders = [] |
344 for builder_name in builders.all_builder_names(): | 344 for builder_name in builders.all_builder_names(): |
345 if builder_name.find('ASAN') != -1: | 345 if builder_name.find('ASAN') != -1: |
346 continue | 346 continue |
347 port = self._tool.port_factory.get_from_builder_name(builder_name) | 347 port = self._tool.port_factory.get_from_builder_name(builder_name) |
348 if port.test_configuration().build_type == 'release': | 348 if port.test_configuration().build_type == 'release': |
(...skipping 332 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
681 ]) | 681 ]) |
682 | 682 |
683 def bot_revision_data(self): | 683 def bot_revision_data(self): |
684 revisions = [] | 684 revisions = [] |
685 for result in self.builder_data().values(): | 685 for result in self.builder_data().values(): |
686 if result.run_was_interrupted(): | 686 if result.run_was_interrupted(): |
687 _log.error("Can't rebaseline because the latest run on %s exited
early." % result.builder_name()) | 687 _log.error("Can't rebaseline because the latest run on %s exited
early." % result.builder_name()) |
688 return [] | 688 return [] |
689 revisions.append({ | 689 revisions.append({ |
690 "builder": result.builder_name(), | 690 "builder": result.builder_name(), |
691 "revision": result.blink_revision(), | 691 "revision": result.chromium_revision(), |
692 }) | 692 }) |
693 return revisions | 693 return revisions |
694 | 694 |
695 def tests_to_rebaseline(self, tool, min_revision, print_revisions): | 695 def tests_to_rebaseline(self, tool, min_revision, print_revisions): |
696 port = tool.port_factory.get() | 696 port = tool.port_factory.get() |
697 expectations_file_path = port.path_to_generic_test_expectations_file() | 697 expectations_file_path = port.path_to_generic_test_expectations_file() |
698 | 698 |
699 tests = set() | 699 tests = set() |
700 revision = None | 700 revision = None |
| 701 commit = None |
701 author = None | 702 author = None |
702 bugs = set() | 703 bugs = set() |
703 has_any_needs_rebaseline_lines = False | 704 has_any_needs_rebaseline_lines = False |
704 | 705 |
705 for line in tool.scm().blame(expectations_file_path).split("\n"): | 706 for line in tool.scm().blame(expectations_file_path).split("\n"): |
706 comment_index = line.find("#") | 707 comment_index = line.find("#") |
707 if comment_index == -1: | 708 if comment_index == -1: |
708 comment_index = len(line) | 709 comment_index = len(line) |
709 line_without_comments = re.sub(r"\s+", " ", line[:comment_index].str
ip()) | 710 line_without_comments = re.sub(r"\s+", " ", line[:comment_index].str
ip()) |
710 | 711 |
711 if "NeedsRebaseline" not in line_without_comments: | 712 if "NeedsRebaseline" not in line_without_comments: |
712 continue | 713 continue |
713 | 714 |
714 has_any_needs_rebaseline_lines = True | 715 has_any_needs_rebaseline_lines = True |
715 | 716 |
716 parsed_line = re.match("^(\S*)[^(]*\((\S*).*?([^ ]*)\ \[[^[]*$", lin
e_without_comments) | 717 parsed_line = re.match("^(\S*)[^(]*\((\S*).*?([^ ]*)\ \[[^[]*$", lin
e_without_comments) |
717 | 718 |
718 commit_hash = parsed_line.group(1) | 719 commit_hash = parsed_line.group(1) |
719 svn_revision = tool.scm().svn_revision_from_git_commit(commit_hash) | 720 commit_position = tool.scm().commit_position_from_git_commit(commit_
hash) |
720 | 721 |
721 test = parsed_line.group(3) | 722 test = parsed_line.group(3) |
722 if print_revisions: | 723 if print_revisions: |
723 _log.info("%s is waiting for r%s" % (test, svn_revision)) | 724 _log.info("%s is waiting for r%s" % (test, commit_position)) |
724 | 725 |
725 if not svn_revision or svn_revision > min_revision: | 726 if not commit_position or commit_position > min_revision: |
726 continue | 727 continue |
727 | 728 |
728 if revision and svn_revision != revision: | 729 if revision and commit_position != revision: |
729 continue | 730 continue |
730 | 731 |
731 if not revision: | 732 if not revision: |
732 revision = svn_revision | 733 revision = commit_position |
| 734 commit = commit_hash |
733 author = parsed_line.group(2) | 735 author = parsed_line.group(2) |
734 | 736 |
735 bugs.update(re.findall("crbug\.com\/(\d+)", line_without_comments)) | 737 bugs.update(re.findall("crbug\.com\/(\d+)", line_without_comments)) |
736 tests.add(test) | 738 tests.add(test) |
737 | 739 |
738 if len(tests) >= self.MAX_LINES_TO_REBASELINE: | 740 if len(tests) >= self.MAX_LINES_TO_REBASELINE: |
739 _log.info("Too many tests to rebaseline in one patch. Doing the
first %d." % self.MAX_LINES_TO_REBASELINE) | 741 _log.info("Too many tests to rebaseline in one patch. Doing the
first %d." % self.MAX_LINES_TO_REBASELINE) |
740 break | 742 break |
741 | 743 |
742 return tests, revision, author, bugs, has_any_needs_rebaseline_lines | 744 return tests, revision, commit, author, bugs, has_any_needs_rebaseline_l
ines |
743 | 745 |
744 def link_to_patch(self, revision): | 746 def link_to_patch(self, commit): |
745 return "http://src.chromium.org/viewvc/blink?view=revision&revision=" +
str(revision) | 747 return "https://chromium.googlesource.com/chromium/src/+/" + commit |
746 | 748 |
747 def commit_message(self, author, revision, bugs): | 749 def commit_message(self, author, revision, commit, bugs): |
748 bug_string = "" | 750 bug_string = "" |
749 if bugs: | 751 if bugs: |
750 bug_string = "BUG=%s\n" % ",".join(bugs) | 752 bug_string = "BUG=%s\n" % ",".join(bugs) |
751 | 753 |
752 return """Auto-rebaseline for r%s | 754 return """Auto-rebaseline for r%s |
753 | 755 |
754 %s | 756 %s |
755 | 757 |
756 %sTBR=%s | 758 %sTBR=%s |
757 """ % (revision, self.link_to_patch(revision), bug_string, author) | 759 """ % (revision, self.link_to_patch(commit), bug_string, author) |
758 | 760 |
759 def get_test_prefix_list(self, tests): | 761 def get_test_prefix_list(self, tests): |
760 test_prefix_list = {} | 762 test_prefix_list = {} |
761 lines_to_remove = {} | 763 lines_to_remove = {} |
762 | 764 |
763 for builder_name in self._release_builders(): | 765 for builder_name in self._release_builders(): |
764 port_name = builders.port_name_for_builder_name(builder_name) | 766 port_name = builders.port_name_for_builder_name(builder_name) |
765 port = self._tool.port_factory.get(port_name) | 767 port = self._tool.port_factory.get(port_name) |
766 expectations = TestExpectations(port, include_overrides=True) | 768 expectations = TestExpectations(port, include_overrides=True) |
767 for test in expectations.get_needs_rebaseline_failures(): | 769 for test in expectations.get_needs_rebaseline_failures(): |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
816 | 818 |
817 if tool.scm().has_working_directory_changes(): | 819 if tool.scm().has_working_directory_changes(): |
818 _log.error("Cannot proceed with working directory changes. Clean wor
king directory first.") | 820 _log.error("Cannot proceed with working directory changes. Clean wor
king directory first.") |
819 return | 821 return |
820 | 822 |
821 revision_data = self.bot_revision_data() | 823 revision_data = self.bot_revision_data() |
822 if not revision_data: | 824 if not revision_data: |
823 return | 825 return |
824 | 826 |
825 min_revision = int(min([item["revision"] for item in revision_data])) | 827 min_revision = int(min([item["revision"] for item in revision_data])) |
826 tests, revision, author, bugs, has_any_needs_rebaseline_lines = self.tes
ts_to_rebaseline(tool, min_revision, print_revisions=options.verbose) | 828 tests, revision, commit, author, bugs, has_any_needs_rebaseline_lines =
self.tests_to_rebaseline(tool, min_revision, print_revisions=options.verbose) |
827 | 829 |
828 if options.verbose: | 830 if options.verbose: |
829 _log.info("Min revision across all bots is %s." % min_revision) | 831 _log.info("Min revision across all bots is %s." % min_revision) |
830 for item in revision_data: | 832 for item in revision_data: |
831 _log.info("%s: r%s" % (item["builder"], item["revision"])) | 833 _log.info("%s: r%s" % (item["builder"], item["revision"])) |
832 | 834 |
833 if not tests: | 835 if not tests: |
834 _log.debug('No tests to rebaseline.') | 836 _log.debug('No tests to rebaseline.') |
835 return | 837 return |
836 | 838 |
837 if self.tree_status() == 'closed': | 839 if self.tree_status() == 'closed': |
838 _log.info('Cannot proceed. Tree is closed.') | 840 _log.info('Cannot proceed. Tree is closed.') |
839 return | 841 return |
840 | 842 |
841 _log.info('Rebaselining %s for r%s by %s.' % (list(tests), revision, aut
hor)) | 843 _log.info('Rebaselining %s for r%s by %s.' % (list(tests), revision, aut
hor)) |
842 | 844 |
843 test_prefix_list, lines_to_remove = self.get_test_prefix_list(tests) | 845 test_prefix_list, lines_to_remove = self.get_test_prefix_list(tests) |
844 | 846 |
845 did_finish = False | 847 did_finish = False |
846 old_branch_name_or_ref = '' | 848 old_branch_name_or_ref = '' |
847 rebaseline_branch_name = self.AUTO_REBASELINE_BRANCH_NAME | 849 rebaseline_branch_name = self.AUTO_REBASELINE_BRANCH_NAME |
848 try: | 850 try: |
849 # Setup git-svn for dcommit if necessary. | |
850 if tool.executive.run_command( | |
851 ['git', 'config', '--local', '--get-regexp', r'^svn-remote\.
'], | |
852 return_exit_code=True): | |
853 tool.executive.run_command(['git', 'auto-svn']) | |
854 | |
855 # Save the current branch name and checkout a clean branch for the p
atch. | 851 # Save the current branch name and checkout a clean branch for the p
atch. |
856 old_branch_name_or_ref = _get_branch_name_or_ref(tool) | 852 old_branch_name_or_ref = _get_branch_name_or_ref(tool) |
857 if old_branch_name_or_ref == self.AUTO_REBASELINE_BRANCH_NAME: | 853 if old_branch_name_or_ref == self.AUTO_REBASELINE_BRANCH_NAME: |
858 rebaseline_branch_name = self.AUTO_REBASELINE_ALT_BRANCH_NAME | 854 rebaseline_branch_name = self.AUTO_REBASELINE_ALT_BRANCH_NAME |
859 tool.scm().delete_branch(rebaseline_branch_name) | 855 tool.scm().delete_branch(rebaseline_branch_name) |
860 tool.scm().create_clean_branch(rebaseline_branch_name) | 856 tool.scm().create_clean_branch(rebaseline_branch_name) |
861 | 857 |
862 # If the tests are passing everywhere, then this list will be empty.
We don't need | 858 # If the tests are passing everywhere, then this list will be empty.
We don't need |
863 # to rebaseline, but we'll still need to update TestExpectations. | 859 # to rebaseline, but we'll still need to update TestExpectations. |
864 if test_prefix_list: | 860 if test_prefix_list: |
865 self._rebaseline(options, test_prefix_list) | 861 self._rebaseline(options, test_prefix_list) |
866 | 862 |
867 tool.scm().commit_locally_with_message(self.commit_message(author, r
evision, bugs)) | 863 tool.scm().commit_locally_with_message(self.commit_message(author, r
evision, commit, bugs)) |
868 | 864 |
869 # FIXME: It would be nice if we could dcommit the patch without uplo
ading, but still | 865 # FIXME: It would be nice if we could dcommit the patch without uplo
ading, but still |
870 # go through all the precommit hooks. For rebaselines with lots of f
iles, uploading | 866 # go through all the precommit hooks. For rebaselines with lots of f
iles, uploading |
871 # takes a long time and sometimes fails, but we don't want to commit
if, e.g. the | 867 # takes a long time and sometimes fails, but we don't want to commit
if, e.g. the |
872 # tree is closed. | 868 # tree is closed. |
873 did_finish = self._run_git_cl_command(options, ['upload', '-f']) | 869 did_finish = self._run_git_cl_command(options, ['upload', '-f']) |
874 | 870 |
875 if did_finish: | 871 if did_finish: |
876 # Uploading can take a very long time. Do another pull to make s
ure TestExpectations is up to date, | 872 # Uploading can take a very long time. Do another pull to make s
ure TestExpectations is up to date, |
877 # so the dcommit can go through. | 873 # so the dcommit can go through. |
(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
969 self._tool.scm().checkout_branch(old_branch_name_or_ref) | 965 self._tool.scm().checkout_branch(old_branch_name_or_ref) |
970 else: | 966 else: |
971 self._log_queue.put(self.QUIT_LOG) | 967 self._log_queue.put(self.QUIT_LOG) |
972 log_thread.join() | 968 log_thread.join() |
973 | 969 |
974 def execute(self, options, args, tool): | 970 def execute(self, options, args, tool): |
975 self._verbose = options.verbose | 971 self._verbose = options.verbose |
976 while True: | 972 while True: |
977 self._do_one_rebaseline() | 973 self._do_one_rebaseline() |
978 time.sleep(self.SLEEP_TIME_IN_SECONDS) | 974 time.sleep(self.SLEEP_TIME_IN_SECONDS) |
OLD | NEW |