OLD | NEW |
1 # Copyright (c) 2011 Google Inc. All rights reserved. | 1 # Copyright (c) 2011 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 22 matching lines...) Expand all Loading... |
33 from webkitpy.layout_tests.models.test_expectations import TestExpectationParser
, TestExpectationsModel, TestExpectations | 33 from webkitpy.layout_tests.models.test_expectations import TestExpectationParser
, TestExpectationsModel, TestExpectations |
34 from webkitpy.layout_tests.port import builders | 34 from webkitpy.layout_tests.port import builders |
35 from webkitpy.common.net import sheriff_calendar | 35 from webkitpy.common.net import sheriff_calendar |
36 | 36 |
37 | 37 |
38 class FlakyTests(AbstractDeclarativeCommand): | 38 class FlakyTests(AbstractDeclarativeCommand): |
39 name = "print-flaky-tests" | 39 name = "print-flaky-tests" |
40 help_text = "Print out flaky lines from the flakiness dashboard" | 40 help_text = "Print out flaky lines from the flakiness dashboard" |
41 show_in_main_help = True | 41 show_in_main_help = True |
42 | 42 |
43 FLAKY_TEST_CONTENTS = '''%s | 43 FLAKINESS_DASHBOARD_URL = 'https://test-results.appspot.com/dashboards/flaki
ness_dashboard.html#tests=%s' |
44 | 44 |
45 Manually add bug numbers for these and then put the lines in LayoutTests/TestExp
ectations. | 45 BUG_TEMPLATE = 'https://code.google.com/p/chromium/issues/entry?owner=FILL_M
E_IN&status=Assigned&labels=Pri-1,Cr-Blink,FlakyLayoutTest&summary=XXXXXXX%20is%
20flaky&comment=XXXXXXX%20is%20flaky.%0A%0AIt%20failed%20twice%20and%20then%20pa
ssed%20on%20the%203rd%20or%204th%20retry.%20This%20is%20too%20flaky.%20The%20tes
t%20will%20be%20skipped%20until%20it%27s%20fixed.%20If%20not%20fixed%20in%203%20
months,%20it%20will%20be%20deleted%20or%20perma-skipped.%0A%0AIn%20the%20flakine
ss%20dashboard,%20the%20turquoise%20boxes%20are%20runs%20where%20the%20test%20fa
iled%20and%20then%20passed%20on%20retry.%0A%0Ahttp://test-results.appspot.com/da
shboards/flakiness_dashboard.html%23tests=XXXXXXX' |
46 TODO(ojan): Write a script to file/assign the bugs then create a bot to do this
automatically. | 46 |
| 47 HEADER = '''Manually add bug numbers for these and then put the lines in Lay
outTests/TestExpectations. |
| 48 Look up the test in the flakiness dashboard first to see if the the platform |
| 49 specifiers should be made more general. |
| 50 |
| 51 Bug template: |
| 52 %s |
| 53 ''' % BUG_TEMPLATE |
| 54 |
| 55 OUTPUT = '''%s |
| 56 %s |
| 57 |
| 58 Flakiness dashboard: %s |
47 ''' | 59 ''' |
48 | 60 |
49 def __init__(self): | 61 def __init__(self): |
50 AbstractDeclarativeCommand.__init__(self) | 62 AbstractDeclarativeCommand.__init__(self) |
51 # This is sorta silly, but allows for unit testing: | 63 # This is sorta silly, but allows for unit testing: |
52 self.expectations_factory = BotTestExpectationsFactory | 64 self.expectations_factory = BotTestExpectationsFactory |
53 | 65 |
54 def _filter_build_type_specifiers(self, specifiers): | 66 def _filter_build_type_specifiers(self, specifiers): |
55 filtered = [] | 67 filtered = [] |
56 for specifier in specifiers: | 68 for specifier in specifiers: |
(...skipping 29 matching lines...) Expand all Loading... |
86 def execute(self, options, args, tool): | 98 def execute(self, options, args, tool): |
87 factory = self.expectations_factory() | 99 factory = self.expectations_factory() |
88 lines = self._collect_expectation_lines(builders.all_builder_names(), fa
ctory) | 100 lines = self._collect_expectation_lines(builders.all_builder_names(), fa
ctory) |
89 lines.sort(key=lambda line: line.path) | 101 lines.sort(key=lambda line: line.path) |
90 | 102 |
91 port = tool.port_factory.get() | 103 port = tool.port_factory.get() |
92 # Skip any tests which are mentioned in the dashboard but not in our che
ckout: | 104 # Skip any tests which are mentioned in the dashboard but not in our che
ckout: |
93 fs = tool.filesystem | 105 fs = tool.filesystem |
94 lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), l
ine.path)), lines) | 106 lines = filter(lambda line: fs.exists(fs.join(port.layout_tests_dir(), l
ine.path)), lines) |
95 | 107 |
96 print self.FLAKY_TEST_CONTENTS % TestExpectations.list_to_string(lines)
# pylint: disable=E1601 | 108 test_names = [line.name for line in lines] |
| 109 flakiness_dashbord_url = self.FLAKINESS_DASHBOARD_URL % ','.join(test_na
mes) |
| 110 expectations_string = TestExpectations.list_to_string(lines) |
97 | 111 |
| 112 # pylint: disable=E1601 |
| 113 print self.OUTPUT % (self.HEADER, expectations_string, flakiness_dashbor
d_url) |
| 114 |
OLD | NEW |