| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Run the first page of one benchmark for every module. | 5 """Run the first page of one benchmark for every module. |
| 6 | 6 |
| 7 Only benchmarks that have a composable measurement are included. | 7 Only benchmarks that have a composable measurement are included. |
| 8 Ideally this test would be comprehensive, however, running one page | 8 Ideally this test would be comprehensive, however, running one page |
| 9 of every benchmark would run impractically long. | 9 of every benchmark would run impractically long. |
| 10 """ | 10 """ |
| (...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 77 self.skipTest('Benchmark %s has ShouldDisable return True' % | 77 self.skipTest('Benchmark %s has ShouldDisable return True' % |
| 78 SinglePageBenchmark.Name()) | 78 SinglePageBenchmark.Name()) |
| 79 | 79 |
| 80 self.assertEqual(0, SinglePageBenchmark().Run(options), | 80 self.assertEqual(0, SinglePageBenchmark().Run(options), |
| 81 msg='Failed: %s' % benchmark) | 81 msg='Failed: %s' % benchmark) |
| 82 | 82 |
| 83 return BenchmarkSmokeTest | 83 return BenchmarkSmokeTest |
| 84 | 84 |
| 85 | 85 |
| 86 # The list of benchmark modules to be excluded from our smoke tests. | 86 # The list of benchmark modules to be excluded from our smoke tests. |
| 87 # TODO(nednguyen): remove black listing by module & change this to black listing |
| 88 # by specific benchmark. |
| 87 _BLACK_LIST_TEST_MODULES = { | 89 _BLACK_LIST_TEST_MODULES = { |
| 88 image_decoding, # Always fails on Mac10.9 Tests builder. | 90 image_decoding, # Always fails on Mac10.9 Tests builder. |
| 89 indexeddb_perf, # Always fails on Win7 & Android Tests builder. | 91 indexeddb_perf, # Always fails on Win7 & Android Tests builder. |
| 90 octane, # Often fails & take long time to timeout on cq bot. | 92 octane, # Often fails & take long time to timeout on cq bot. |
| 91 rasterize_and_record_micro, # Always fails on cq bot. | 93 rasterize_and_record_micro, # Always fails on cq bot. |
| 92 repaint, # Often fails & takes long time to timeout on cq bot. | 94 repaint, # Often fails & takes long time to timeout on cq bot. |
| 93 spaceport, # Takes 451 seconds. | 95 spaceport, # Takes 451 seconds. |
| 94 speedometer, # Takes 101 seconds. | 96 speedometer, # Takes 101 seconds. |
| 95 jetstream, # Take 206 seconds. | 97 jetstream, # Take 206 seconds. |
| 96 text_selection, # Always fails on cq bot. | 98 text_selection, # Always fails on cq bot. |
| 97 memory # Flaky on bots, crbug.com/513767. | 99 memory # Flaky on bots, crbug.com/513767. |
| 98 } | 100 } |
| 99 | 101 |
| 100 # Some smoke benchmark tests that run quickly on desktop platform can be very | 102 # Some smoke benchmark tests that run quickly on desktop platform can be very |
| 101 # slow on Android. So we create a separate set of black list only for Android. | 103 # slow on Android. So we create a separate set of black list only for Android. |
| 102 _ANDROID_BLACK_LIST_MODULES = { | 104 _ANDROID_BLACK_LIST_MODULES = { |
| 103 kraken, # Takes 275 seconds on Android. | 105 kraken, # Takes 275 seconds on Android. |
| 104 sunspider, # Takes 163 seconds on Android. | 106 sunspider, # Takes 163 seconds on Android. |
| 105 tracing, # Failed on Android bot, crbug.com/600989. | 107 tracing, # Failed on Android bot, crbug.com/600989. |
| 106 } | 108 } |
| 107 | 109 |
| 108 | 110 |
| 109 def load_tests(loader, standard_tests, pattern): | 111 def load_tests(loader, standard_tests, pattern): |
| 110 del loader, standard_tests, pattern # unused | 112 del loader, standard_tests, pattern # unused |
| 111 suite = progress_reporter.TestSuite() | 113 suite = progress_reporter.TestSuite() |
| 112 | 114 |
| 113 benchmarks_dir = os.path.dirname(__file__) | 115 benchmarks_dir = os.path.dirname(__file__) |
| 114 top_level_dir = os.path.dirname(benchmarks_dir) | 116 top_level_dir = os.path.dirname(benchmarks_dir) |
| 115 | 117 |
| 116 # Using the default of |index_by_class_name=False| means that if a module | |
| 117 # has multiple benchmarks, only the last one is returned. | |
| 118 all_benchmarks = discover.DiscoverClasses( | 118 all_benchmarks = discover.DiscoverClasses( |
| 119 benchmarks_dir, top_level_dir, benchmark_module.Benchmark, | 119 benchmarks_dir, top_level_dir, benchmark_module.Benchmark).values() |
| 120 index_by_class_name=False).values() | |
| 121 for benchmark in all_benchmarks: | 120 for benchmark in all_benchmarks: |
| 122 if sys.modules[benchmark.__module__] in _BLACK_LIST_TEST_MODULES: | 121 if sys.modules[benchmark.__module__] in _BLACK_LIST_TEST_MODULES: |
| 123 continue | 122 continue |
| 124 # TODO(tonyg): Smoke doesn't work with session_restore yet. | |
| 125 if (benchmark.Name().startswith('session_restore') or | |
| 126 benchmark.Name().startswith('skpicture_printer')): | |
| 127 continue | |
| 128 | 123 |
| 129 if hasattr(benchmark, 'generated_profile_archive'): | 124 if hasattr(benchmark, 'generated_profile_archive'): |
| 130 # We'd like to test these, but don't know how yet. | 125 # We'd like to test these, but don't know how yet. |
| 131 continue | 126 continue |
| 132 | 127 |
| 133 class BenchmarkSmokeTest(unittest.TestCase): | 128 class BenchmarkSmokeTest(unittest.TestCase): |
| 134 pass | 129 pass |
| 135 | 130 |
| 136 method = SmokeTestGenerator(benchmark) | 131 method = SmokeTestGenerator(benchmark) |
| 137 | 132 |
| (...skipping 18 matching lines...) Expand all Loading... |
| 156 | 151 |
| 157 # TODO(bashi): Remove once crrev.com/1266833004 is landed. | 152 # TODO(bashi): Remove once crrev.com/1266833004 is landed. |
| 158 if benchmark.Name() == 'memory.blink_memory_mobile': | 153 if benchmark.Name() == 'memory.blink_memory_mobile': |
| 159 method._disabled_strings.add('android') | 154 method._disabled_strings.add('android') |
| 160 | 155 |
| 161 setattr(BenchmarkSmokeTest, benchmark.Name(), method) | 156 setattr(BenchmarkSmokeTest, benchmark.Name(), method) |
| 162 | 157 |
| 163 suite.addTest(BenchmarkSmokeTest(benchmark.Name())) | 158 suite.addTest(BenchmarkSmokeTest(benchmark.Name())) |
| 164 | 159 |
| 165 return suite | 160 return suite |
| OLD | NEW |