| Index: third_party/WebKit/Tools/Scripts/webkitpy/tool/commands/flakytests.py
|
| diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/tool/commands/flakytests.py b/third_party/WebKit/Tools/Scripts/webkitpy/tool/commands/flakytests.py
|
| deleted file mode 100644
|
| index 71469ce09c9304452536d543f055eab8209ad75b..0000000000000000000000000000000000000000
|
| --- a/third_party/WebKit/Tools/Scripts/webkitpy/tool/commands/flakytests.py
|
| +++ /dev/null
|
| @@ -1,119 +0,0 @@
|
| -# Copyright (c) 2011 Google Inc. All rights reserved.
|
| -#
|
| -# Redistribution and use in source and binary forms, with or without
|
| -# modification, are permitted provided that the following conditions are
|
| -# met:
|
| -#
|
| -# * Redistributions of source code must retain the above copyright
|
| -# notice, this list of conditions and the following disclaimer.
|
| -# * Redistributions in binary form must reproduce the above
|
| -# copyright notice, this list of conditions and the following disclaimer
|
| -# in the documentation and/or other materials provided with the
|
| -# distribution.
|
| -# * Neither the name of Google Inc. nor the names of its
|
| -# contributors may be used to endorse or promote products derived from
|
| -# this software without specific prior written permission.
|
| -#
|
| -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
| -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
| -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
| -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
| -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
| -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
| -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
| -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
| -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
| -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
| -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
| -
|
| -import logging
|
| -
|
| -from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
|
| -from webkitpy.layout_tests.models.test_expectations import TestExpectations
|
| -from webkitpy.layout_tests.models.test_expectations import TestExpectationsModel
|
| -from webkitpy.tool.commands.command import Command
|
| -
|
| -
|
| -_log = logging.getLogger(__name__)
|
| -
|
| -
|
| -class FlakyTests(Command):
|
| - name = "print-flaky-tests"
|
| - help_text = "Print out flaky lines from the flakiness dashboard"
|
| - show_in_main_help = True
|
| -
|
| - FLAKINESS_DASHBOARD_URL = 'https://test-results.appspot.com/dashboards/flakiness_dashboard.html#tests=%s'
|
| -
|
| - BUG_TEMPLATE = 'https://code.google.com/p/chromium/issues/entry?owner=FILL_ME_IN&status=Assigned&labels=Pri-1,Cr-Blink,FlakyLayoutTest&summary=XXXXXXX%20is%20flaky&comment=XXXXXXX%20is%20flaky.%0A%0AIt%20failed%20twice%20and%20then%20passed%20on%20the%203rd%20or%204th%20retry.%20This%20is%20too%20flaky.%20The%20test%20will%20be%20skipped%20until%20it%27s%20fixed.%20If%20not%20fixed%20in%203%20months,%20it%20will%20be%20deleted%20or%20perma-skipped.%0A%0AIn%20the%20flakiness%20dashboard,%20the%20turquoise%20boxes%20are%20runs%20where%20the%20test%20failed%20and%20then%20passed%20on%20retry.%0A%0Ahttp://test-results.appspot.com/dashboards/flakiness_dashboard.html%23tests=XXXXXXX'
|
| -
|
| - HEADER = '''Manually add bug numbers for these and then put the lines in LayoutTests/TestExpectations.
|
| -Look up the test in the flakiness dashboard first to see if the the platform
|
| -specifiers should be made more general.
|
| -
|
| -Bug template:
|
| -%s
|
| -''' % BUG_TEMPLATE
|
| -
|
| - OUTPUT = '''%s
|
| -%s
|
| -
|
| -Flakiness dashboard: %s
|
| -'''
|
| -
|
| - def __init__(self):
|
| - super(FlakyTests, self).__init__()
|
| - # This is sorta silly, but allows for unit testing:
|
| - self.expectations_factory = BotTestExpectationsFactory
|
| -
|
| - def _filter_build_type_specifiers(self, specifiers):
|
| - filtered = []
|
| - for specifier in specifiers:
|
| - if specifier.lower() not in TestExpectations.BUILD_TYPES:
|
| - filtered.append(specifier)
|
| - return filtered
|
| -
|
| - def _collect_expectation_lines(self, builder_names, factory):
|
| - models = []
|
| - for builder_name in builder_names:
|
| - model = TestExpectationsModel()
|
| - models.append(model)
|
| -
|
| - expectations = factory.expectations_for_builder(builder_name)
|
| -
|
| - # TODO(ojan): We should also skip bots that haven't uploaded recently,
|
| - # e.g. if they're >24h stale.
|
| - if not expectations:
|
| - _log.error("Can't load flakiness data for builder: %s" % builder_name)
|
| - continue
|
| -
|
| - for line in expectations.expectation_lines(only_ignore_very_flaky=True):
|
| - # TODO(ojan): Find a way to merge specifiers instead of removing build types.
|
| - # We can't just union because some specifiers will change the meaning of others.
|
| - # For example, it's not clear how to merge [ Mac Release ] with [ Linux Debug ].
|
| - # But, in theory we should be able to merge [ Mac Release ] and [ Mac Debug ].
|
| - line.specifiers = self._filter_build_type_specifiers(line.specifiers)
|
| - model.add_expectation_line(line)
|
| -
|
| - final_model = None
|
| - for model in models:
|
| - if final_model:
|
| - final_model.merge_model(model)
|
| - else:
|
| - final_model = model
|
| - return final_model._test_to_expectation_line.values()
|
| -
|
| - def execute(self, options, args, tool):
|
| - factory = self.expectations_factory(tool.builders)
|
| - lines = self._collect_expectation_lines(tool.builders.all_builder_names(), factory)
|
| - lines.sort(key=lambda line: line.path)
|
| -
|
| - port = tool.port_factory.get()
|
| - # Skip any tests which are mentioned in the dashboard but not in our checkout:
|
| - fs = tool.filesystem
|
| - lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), line.path)), lines)
|
| -
|
| - test_names = [line.name for line in lines]
|
| - flakiness_dashboard_url = self.FLAKINESS_DASHBOARD_URL % ','.join(test_names)
|
| - expectations_string = TestExpectations.list_to_string(lines)
|
| -
|
| - print self.OUTPUT % (self.HEADER, expectations_string, flakiness_dashboard_url)
|
|
|