Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(34)

Side by Side Diff: Tools/Scripts/webkitpy/tool/commands/flakytests.py

Issue 1289163002: Improve flakiness logic for print-flaky-tests. (Closed) Base URL: svn://svn.chromium.org/blink/trunk
Patch Set: Created 5 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 # Copyright (c) 2011 Google Inc. All rights reserved. 1 # Copyright (c) 2011 Google Inc. All rights reserved.
2 # 2 #
3 # Redistribution and use in source and binary forms, with or without 3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are 4 # modification, are permitted provided that the following conditions are
5 # met: 5 # met:
6 # 6 #
7 # * Redistributions of source code must retain the above copyright 7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer. 8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above 9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer 10 # copyright notice, this list of conditions and the following disclaimer
(...skipping 22 matching lines...) Expand all
33 from webkitpy.layout_tests.models.test_expectations import TestExpectationParser , TestExpectationsModel, TestExpectations 33 from webkitpy.layout_tests.models.test_expectations import TestExpectationParser , TestExpectationsModel, TestExpectations
34 from webkitpy.layout_tests.port import builders 34 from webkitpy.layout_tests.port import builders
35 from webkitpy.common.net import sheriff_calendar 35 from webkitpy.common.net import sheriff_calendar
36 36
37 37
38 class FlakyTests(AbstractDeclarativeCommand): 38 class FlakyTests(AbstractDeclarativeCommand):
39 name = "print-flaky-tests" 39 name = "print-flaky-tests"
40 help_text = "Print out flaky lines from the flakiness dashboard" 40 help_text = "Print out flaky lines from the flakiness dashboard"
41 show_in_main_help = True 41 show_in_main_help = True
42 42
43 FLAKY_TEST_CONTENTS = '''%s 43 FLAKINESS_DASHBOARD_URL = 'https://test-results.appspot.com/dashboards/flaki ness_dashboard.html#tests=%s'
44 44
45 Manually add bug numbers for these and then put the lines in LayoutTests/TestExp ectations. 45 BUG_TEMPLATE = 'https://code.google.com/p/chromium/issues/entry?owner=FILL_M E_IN&status=Assigned&labels=Pri-1,Cr-Blink,FlakyLayoutTest&summary=XXXXXXX%20is% 20flaky&comment=XXXXXXX%20is%20flaky.%0A%0AIt%20failed%20twice%20and%20then%20pa ssed%20on%20the%203rd%20or%204th%20retry.%20This%20is%20too%20flaky.%20The%20tes t%20will%20be%20skipped%20until%20it%27s%20fixed.%20If%20not%20fixed%20in%203%20 months,%20it%20will%20be%20deleted%20or%20perma-skipped.%0A%0AIn%20the%20flakine ss%20dashboard,%20the%20turquoise%20boxes%20are%20runs%20where%20the%20test%20fa iled%20and%20then%20passed%20on%20retry.%0A%0Ahttp://test-results.appspot.com/da shboards/flakiness_dashboard.html%23tests=XXXXXXX'
46 TODO(ojan): Write a script to file/assign the bugs then create a bot to do this automatically. 46
47 HEADER = '''Manually add bug numbers for these and then put the lines in Lay outTests/TestExpectations.
48 Look up the test in the flakiness dashboard first to see if the the platform
49 specifiers should be made more general.
50
51 Bug template:
52 %s
53 ''' % BUG_TEMPLATE
54
55 OUTPUT = '''%s
56 %s
57
58 Flakiness dashboard: %s
47 ''' 59 '''
48 60
49 def __init__(self): 61 def __init__(self):
50 AbstractDeclarativeCommand.__init__(self) 62 AbstractDeclarativeCommand.__init__(self)
51 # This is sorta silly, but allows for unit testing: 63 # This is sorta silly, but allows for unit testing:
52 self.expectations_factory = BotTestExpectationsFactory 64 self.expectations_factory = BotTestExpectationsFactory
53 65
54 def _filter_build_type_specifiers(self, specifiers): 66 def _filter_build_type_specifiers(self, specifiers):
55 filtered = [] 67 filtered = []
56 for specifier in specifiers: 68 for specifier in specifiers:
(...skipping 29 matching lines...) Expand all
86 def execute(self, options, args, tool): 98 def execute(self, options, args, tool):
87 factory = self.expectations_factory() 99 factory = self.expectations_factory()
88 lines = self._collect_expectation_lines(builders.all_builder_names(), fa ctory) 100 lines = self._collect_expectation_lines(builders.all_builder_names(), fa ctory)
89 lines.sort(key=lambda line: line.path) 101 lines.sort(key=lambda line: line.path)
90 102
91 port = tool.port_factory.get() 103 port = tool.port_factory.get()
92 # Skip any tests which are mentioned in the dashboard but not in our che ckout: 104 # Skip any tests which are mentioned in the dashboard but not in our che ckout:
93 fs = tool.filesystem 105 fs = tool.filesystem
94 lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), l ine.path)), lines) 106 lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), l ine.path)), lines)
95 107
96 print self.FLAKY_TEST_CONTENTS % TestExpectations.list_to_string(lines) # pylint: disable=E1601 108 test_names = [line.name for line in lines]
109 flakiness_dashbord_url = self.FLAKINESS_DASHBOARD_URL % ','.join(test_na mes)
110 expectations_string = TestExpectations.list_to_string(lines)
97 111
112 # pylint: disable=E1601
113 print self.OUTPUT % (self.HEADER, expectations_string, flakiness_dashbor d_url)
114
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698