| OLD | NEW |
| 1 # Copyright (c) 2011 Google Inc. All rights reserved. | 1 # Copyright (c) 2011 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 13 matching lines...) Expand all Loading... |
| 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 28 | 28 |
| 29 import os | 29 import os |
| 30 import optparse | 30 import optparse |
| 31 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand | 31 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand |
| 32 from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestEx
pectationsFactory | 32 from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestEx
pectationsFactory |
| 33 from webkitpy.layout_tests.models.test_expectations import TestExpectationParser
, TestExpectationsModel, TestExpectations | 33 from webkitpy.layout_tests.models.test_expectations import TestExpectationParser
, TestExpectationsModel, TestExpectations |
| 34 from webkitpy.layout_tests.port import builders |
| 34 from webkitpy.common.net import sheriff_calendar | 35 from webkitpy.common.net import sheriff_calendar |
| 35 | 36 |
| 36 | 37 |
| 37 class FlakyTests(AbstractDeclarativeCommand): | 38 class FlakyTests(AbstractDeclarativeCommand): |
| 38 name = "update-flaky-tests" | 39 name = "update-flaky-tests" |
| 39 help_text = "Update FlakyTests file from the flakiness dashboard" | 40 help_text = "Update FlakyTests file from the flakiness dashboard" |
| 40 show_in_main_help = True | 41 show_in_main_help = True |
| 41 | 42 |
| 42 ALWAYS_CC = [ | 43 ALWAYS_CC = [ |
| 43 'ojan@chromium.org', | 44 'ojan@chromium.org', |
| 44 'dpranke@chromium.org', | 45 'dpranke@chromium.org', |
| 45 'eseidel@chromium.org', | 46 'eseidel@chromium.org', |
| 46 ] | 47 ] |
| 47 | 48 |
| 48 def __init__(self): | 49 def __init__(self): |
| 49 options = [ | 50 options = [ |
| 50 optparse.make_option('--upload', action='store_true', | 51 optparse.make_option('--upload', action='store_true', |
| 51 help='upload the changed FlakyTest file for review'), | 52 help='upload the changed FlakyTest file for review'), |
| 52 optparse.make_option('--reviewers', action='store', | 53 optparse.make_option('--reviewers', action='store', |
| 53 help='comma-separated list of reviewers, defaults to blink garde
ners'), | 54 help='comma-separated list of reviewers, defaults to blink garde
ners'), |
| 54 ] | 55 ] |
| 55 AbstractDeclarativeCommand.__init__(self, options=options) | 56 AbstractDeclarativeCommand.__init__(self, options=options) |
| 56 # This is sorta silly, but allows for unit testing: | 57 # This is sorta silly, but allows for unit testing: |
| 57 self.expectations_factory = BotTestExpectationsFactory | 58 self.expectations_factory = BotTestExpectationsFactory |
| 58 | 59 |
| 59 def _collect_expectation_lines(self, port_names, factory): | 60 def _collect_expectation_lines(self, builder_names, factory): |
| 60 model = TestExpectationsModel() | 61 model = TestExpectationsModel() |
| 61 for port_name in port_names: | 62 for builder_name in builder_names: |
| 62 expectations = factory.expectations_for_port(port_name) | 63 expectations = factory.expectations_for_builder(builder_name) |
| 63 for line in expectations.expectation_lines(only_ignore_very_flaky=Tr
ue): | 64 for line in expectations.expectation_lines(only_ignore_very_flaky=Tr
ue): |
| 64 model.add_expectation_line(line) | 65 model.add_expectation_line(line) |
| 65 # FIXME: We need an official API to get all the test names or all test l
ines. | 66 # FIXME: We need an official API to get all the test names or all test l
ines. |
| 66 return model._test_to_expectation_line.values() | 67 return model._test_to_expectation_line.values() |
| 67 | 68 |
| 68 def _commit_and_upload(self, tool, options): | 69 def _commit_and_upload(self, tool, options): |
| 69 files = tool.scm().changed_files() | 70 files = tool.scm().changed_files() |
| 70 flaky_tests_path = 'LayoutTests/FlakyTests' | 71 flaky_tests_path = 'LayoutTests/FlakyTests' |
| 71 if flaky_tests_path not in files: | 72 if flaky_tests_path not in files: |
| 72 print "%s is not changed, not uploading." % flaky_tests_path | 73 print "%s is not changed, not uploading." % flaky_tests_path |
| (...skipping 22 matching lines...) Expand all Loading... |
| 95 | 96 |
| 96 # FIXME: There must be a cleaner way to avoid the editor! | 97 # FIXME: There must be a cleaner way to avoid the editor! |
| 97 # Silence the editor. | 98 # Silence the editor. |
| 98 os.environ['EDITOR'] = 'true' | 99 os.environ['EDITOR'] = 'true' |
| 99 | 100 |
| 100 git_cmd = ['git', 'cl', 'upload', '--send-mail', | 101 git_cmd = ['git', 'cl', 'upload', '--send-mail', |
| 101 '--cc', ','.join(self.ALWAYS_CC)] | 102 '--cc', ','.join(self.ALWAYS_CC)] |
| 102 tool.executive.run_and_throw_if_fail(git_cmd) | 103 tool.executive.run_and_throw_if_fail(git_cmd) |
| 103 | 104 |
| 104 def execute(self, options, args, tool): | 105 def execute(self, options, args, tool): |
| 106 factory = self.expectations_factory() |
| 107 lines = self._collect_expectation_lines(builders.all_builder_names(), fa
ctory) |
| 108 lines.sort(key=lambda line: line.path) |
| 109 |
| 105 port = tool.port_factory.get() | 110 port = tool.port_factory.get() |
| 106 port_names = tool.port_factory.all_port_names() | |
| 107 factory = self.expectations_factory() | |
| 108 lines = self._collect_expectation_lines(port_names, factory) | |
| 109 lines.sort(key=lambda line: line.path) | |
| 110 # Skip any tests which are mentioned in the dashboard but not in our che
ckout: | 111 # Skip any tests which are mentioned in the dashboard but not in our che
ckout: |
| 111 fs = tool.filesystem | 112 fs = tool.filesystem |
| 112 lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), l
ine.path)), lines) | 113 lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), l
ine.path)), lines) |
| 113 | 114 |
| 114 # Note: This includes all flaky tests from the dashboard, even ones ment
ioned | 115 # Note: This includes all flaky tests from the dashboard, even ones ment
ioned |
| 115 # in existing TestExpectations. We could certainly load existing TestExp
ecations | 116 # in existing TestExpectations. We could certainly load existing TestExp
ecations |
| 116 # and filter accordingly, or update existing TestExpectations instead of
FlakyTests. | 117 # and filter accordingly, or update existing TestExpectations instead of
FlakyTests. |
| 117 flaky_tests_path = fs.join(port.layout_tests_dir(), 'FlakyTests') | 118 flaky_tests_path = fs.join(port.layout_tests_dir(), 'FlakyTests') |
| 118 fs.write_text_file(flaky_tests_path, TestExpectations.list_to_string(lin
es)) | 119 fs.write_text_file(flaky_tests_path, TestExpectations.list_to_string(lin
es)) |
| 119 print "Updated %s" % flaky_tests_path | 120 print "Updated %s" % flaky_tests_path |
| 120 | 121 |
| 121 if options.upload: | 122 if options.upload: |
| 122 return self._commit_and_upload(tool, options) | 123 return self._commit_and_upload(tool, options) |
| 123 | 124 |
| OLD | NEW |