| OLD | NEW |
| 1 # Copyright (c) 2011 Google Inc. All rights reserved. | 1 # Copyright (c) 2011 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 26 matching lines...) Expand all Loading... |
| 37 _log = logging.getLogger(__name__) | 37 _log = logging.getLogger(__name__) |
| 38 | 38 |
| 39 | 39 |
| 40 class FlakyTests(Command): | 40 class FlakyTests(Command): |
| 41 name = "print-flaky-tests" | 41 name = "print-flaky-tests" |
| 42 help_text = "Print out flaky lines from the flakiness dashboard" | 42 help_text = "Print out flaky lines from the flakiness dashboard" |
| 43 show_in_main_help = True | 43 show_in_main_help = True |
| 44 | 44 |
| 45 FLAKINESS_DASHBOARD_URL = 'https://test-results.appspot.com/dashboards/flaki
ness_dashboard.html#tests=%s' | 45 FLAKINESS_DASHBOARD_URL = 'https://test-results.appspot.com/dashboards/flaki
ness_dashboard.html#tests=%s' |
| 46 | 46 |
| 47 BUG_TEMPLATE = 'https://code.google.com/p/chromium/issues/entry?owner=FILL_M
E_IN&status=Assigned&labels=Pri-1,Cr-Blink,FlakyLayoutTest&summary=XXXXXXX%20is%
20flaky&comment=XXXXXXX%20is%20flaky.%0A%0AIt%20failed%20twice%20and%20then%20pa
ssed%20on%20the%203rd%20or%204th%20retry.%20This%20is%20too%20flaky.%20The%20tes
t%20will%20be%20skipped%20until%20it%27s%20fixed.%20If%20not%20fixed%20in%203%20
months,%20it%20will%20be%20deleted%20or%20perma-skipped.%0A%0AIn%20the%20flakine
ss%20dashboard,%20the%20turquoise%20boxes%20are%20runs%20where%20the%20test%20fa
iled%20and%20then%20passed%20on%20retry.%0A%0Ahttp://test-results.appspot.com/da
shboards/flakiness_dashboard.html%23tests=XXXXXXX' | 47 BUG_TEMPLATE = ('https://code.google.com/p/chromium/issues/entry?owner=FILL_
ME_IN&status=Assigned&' |
| 48 'labels=Pri-1,Cr-Blink,FlakyLayoutTest&summary=XXXXXXX%20is%
20flaky&' |
| 49 'comment=XXXXXXX%20is%20flaky.%0A%0AIt%20failed%20twice%20an
d%20then' |
| 50 '%20passed%20on%20the%203rd%20or%204th%20retry.%20This%20is%
20too%20' |
| 51 'flaky.%20The%20test%20will%20be%20skipped%20until%20it%27s%
20fixed.' |
| 52 '%20If%20not%20fixed%20in%203%20months,%20it%20will%20be%20d
eleted%20' |
| 53 'or%20perma-skipped.%0A%0AIn%20the%20flakiness%20dashboard,%
20the%20' |
| 54 'turquoise%20boxes%20are%20runs%20where%20the%20test%20faile
d%20and%20' |
| 55 'then%20passed%20on%20retry.%0A%0Ahttp://test-results.appspo
t.com' |
| 56 '/dashboards/flakiness_dashboard.html%23tests=XXXXXXX') |
| 48 | 57 |
| 49 HEADER = '''Manually add bug numbers for these and then put the lines in Lay
outTests/TestExpectations. | 58 HEADER = '''Manually add bug numbers for these and then put the lines in Lay
outTests/TestExpectations. |
| 50 Look up the test in the flakiness dashboard first to see if the the platform | 59 Look up the test in the flakiness dashboard first to see if the the platform |
| 51 specifiers should be made more general. | 60 specifiers should be made more general. |
| 52 | 61 |
| 53 Bug template: | 62 Bug template: |
| 54 %s | 63 %s |
| 55 ''' % BUG_TEMPLATE | 64 ''' % BUG_TEMPLATE |
| 56 | 65 |
| 57 OUTPUT = '''%s | 66 OUTPUT = '''%s |
| (...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 110 port = tool.port_factory.get() | 119 port = tool.port_factory.get() |
| 111 # Skip any tests which are mentioned in the dashboard but not in our che
ckout: | 120 # Skip any tests which are mentioned in the dashboard but not in our che
ckout: |
| 112 fs = tool.filesystem | 121 fs = tool.filesystem |
| 113 lines = [line for line in lines if fs.exists(fs.join(port.layout_tests_d
ir(), line.path))] | 122 lines = [line for line in lines if fs.exists(fs.join(port.layout_tests_d
ir(), line.path))] |
| 114 | 123 |
| 115 test_names = [line.name for line in lines] | 124 test_names = [line.name for line in lines] |
| 116 flakiness_dashboard_url = self.FLAKINESS_DASHBOARD_URL % ','.join(test_n
ames) | 125 flakiness_dashboard_url = self.FLAKINESS_DASHBOARD_URL % ','.join(test_n
ames) |
| 117 expectations_string = TestExpectations.list_to_string(lines) | 126 expectations_string = TestExpectations.list_to_string(lines) |
| 118 | 127 |
| 119 print self.OUTPUT % (self.HEADER, expectations_string, flakiness_dashboa
rd_url) | 128 print self.OUTPUT % (self.HEADER, expectations_string, flakiness_dashboa
rd_url) |
| OLD | NEW |