Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(154)

Side by Side Diff: tools/perf/benchmarks/benchmark_smoke_unittest.py

Issue 2081403003: [tools/perf] Enable smoke coverage for all telemetry benchmarks Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Add path to pywebsocket lib Created 4 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Run the first page of one benchmark for every module. 5 """Run the first page of one benchmark for every module.
6 6
7 Only benchmarks that have a composable measurement are included. 7 Only benchmarks that have a composable measurement are included.
8 Ideally this test would be comprehensive, however, running one page 8 Ideally this test would be comprehensive, however, running one page
9 of every benchmark would run impractically long. 9 of every benchmark would run impractically long.
10 """ 10 """
(...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after
77 self.skipTest('Benchmark %s has ShouldDisable return True' % 77 self.skipTest('Benchmark %s has ShouldDisable return True' %
78 SinglePageBenchmark.Name()) 78 SinglePageBenchmark.Name())
79 79
80 self.assertEqual(0, SinglePageBenchmark().Run(options), 80 self.assertEqual(0, SinglePageBenchmark().Run(options),
81 msg='Failed: %s' % benchmark) 81 msg='Failed: %s' % benchmark)
82 82
83 return BenchmarkSmokeTest 83 return BenchmarkSmokeTest
84 84
85 85
86 # The list of benchmark modules to be excluded from our smoke tests. 86 # The list of benchmark modules to be excluded from our smoke tests.
87 # TODO(nednguyen): remove black listing by module & change this to black listing
88 # by specific benchmark.
87 _BLACK_LIST_TEST_MODULES = { 89 _BLACK_LIST_TEST_MODULES = {
88 image_decoding, # Always fails on Mac10.9 Tests builder. 90 image_decoding, # Always fails on Mac10.9 Tests builder.
89 indexeddb_perf, # Always fails on Win7 & Android Tests builder. 91 indexeddb_perf, # Always fails on Win7 & Android Tests builder.
90 octane, # Often fails & take long time to timeout on cq bot. 92 octane, # Often fails & take long time to timeout on cq bot.
91 rasterize_and_record_micro, # Always fails on cq bot. 93 rasterize_and_record_micro, # Always fails on cq bot.
92 repaint, # Often fails & takes long time to timeout on cq bot. 94 repaint, # Often fails & takes long time to timeout on cq bot.
93 spaceport, # Takes 451 seconds. 95 spaceport, # Takes 451 seconds.
94 speedometer, # Takes 101 seconds. 96 speedometer, # Takes 101 seconds.
95 jetstream, # Take 206 seconds. 97 jetstream, # Take 206 seconds.
96 text_selection, # Always fails on cq bot. 98 text_selection, # Always fails on cq bot.
97 memory # Flaky on bots, crbug.com/513767. 99 memory # Flaky on bots, crbug.com/513767.
98 } 100 }
99 101
100 # Some smoke benchmark tests that run quickly on desktop platform can be very 102 # Some smoke benchmark tests that run quickly on desktop platform can be very
101 # slow on Android. So we create a separate set of black list only for Android. 103 # slow on Android. So we create a separate set of black list only for Android.
102 _ANDROID_BLACK_LIST_MODULES = { 104 _ANDROID_BLACK_LIST_MODULES = {
103 kraken, # Takes 275 seconds on Android. 105 kraken, # Takes 275 seconds on Android.
104 sunspider, # Takes 163 seconds on Android. 106 sunspider, # Takes 163 seconds on Android.
105 tracing, # Failed on Android bot, crbug.com/600989. 107 tracing, # Failed on Android bot, crbug.com/600989.
106 } 108 }
107 109
108 110
109 def load_tests(loader, standard_tests, pattern): 111 def load_tests(loader, standard_tests, pattern):
110 del loader, standard_tests, pattern # unused 112 del loader, standard_tests, pattern # unused
111 suite = progress_reporter.TestSuite() 113 suite = progress_reporter.TestSuite()
112 114
113 benchmarks_dir = os.path.dirname(__file__) 115 benchmarks_dir = os.path.dirname(__file__)
114 top_level_dir = os.path.dirname(benchmarks_dir) 116 top_level_dir = os.path.dirname(benchmarks_dir)
115 117
116 # Using the default of |index_by_class_name=False| means that if a module
117 # has multiple benchmarks, only the last one is returned.
118 all_benchmarks = discover.DiscoverClasses( 118 all_benchmarks = discover.DiscoverClasses(
119 benchmarks_dir, top_level_dir, benchmark_module.Benchmark, 119 benchmarks_dir, top_level_dir, benchmark_module.Benchmark).values()
120 index_by_class_name=False).values()
121 for benchmark in all_benchmarks: 120 for benchmark in all_benchmarks:
122 if sys.modules[benchmark.__module__] in _BLACK_LIST_TEST_MODULES: 121 if sys.modules[benchmark.__module__] in _BLACK_LIST_TEST_MODULES:
123 continue 122 continue
124 # TODO(tonyg): Smoke doesn't work with session_restore yet.
125 if (benchmark.Name().startswith('session_restore') or
126 benchmark.Name().startswith('skpicture_printer')):
127 continue
128 123
129 if hasattr(benchmark, 'generated_profile_archive'): 124 if hasattr(benchmark, 'generated_profile_archive'):
130 # We'd like to test these, but don't know how yet. 125 # We'd like to test these, but don't know how yet.
131 continue 126 continue
132 127
133 class BenchmarkSmokeTest(unittest.TestCase): 128 class BenchmarkSmokeTest(unittest.TestCase):
134 pass 129 pass
135 130
136 method = SmokeTestGenerator(benchmark) 131 method = SmokeTestGenerator(benchmark)
137 132
(...skipping 18 matching lines...) Expand all
156 151
157 # TODO(bashi): Remove once crrev.com/1266833004 is landed. 152 # TODO(bashi): Remove once crrev.com/1266833004 is landed.
158 if benchmark.Name() == 'memory.blink_memory_mobile': 153 if benchmark.Name() == 'memory.blink_memory_mobile':
159 method._disabled_strings.add('android') 154 method._disabled_strings.add('android')
160 155
161 setattr(BenchmarkSmokeTest, benchmark.Name(), method) 156 setattr(BenchmarkSmokeTest, benchmark.Name(), method)
162 157
163 suite.addTest(BenchmarkSmokeTest(benchmark.Name())) 158 suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
164 159
165 return suite 160 return suite
OLDNEW
« tools/perf/BUILD.gn ('K') | « tools/perf/BUILD.gn ('k') | tools/perf/perf.isolate » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698