Fix tab_switching measurement bug.

tools/perf/benchmarks/benchmark_smoke_unittest.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Run the first page of one benchmark for every module.
 
 Only benchmarks that have a composable measurement are included.
 Ideally this test would be comprehensive, however, running one page
 of every benchmark would run impractically long.
 """
@@ skipping 14 matching lines @@
 from benchmarks import octane
 from benchmarks import kraken
 from benchmarks import sunspider
 from benchmarks import rasterize_and_record_micro
 from benchmarks import repaint
 from benchmarks import spaceport
 from benchmarks import speedometer
 from benchmarks import jetstream
 
 
-def SmokeTestGenerator(benchmark):
+def SmokeTestGenerator(benchmark, num_pages=1):
+  """Generates a benchmark that includes first N pages from pageset."""
   # NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST.
   #
   # This smoke test dynamically tests all benchmarks. So disabling it for one
   # failing or flaky benchmark would disable a much wider swath of coverage
   # than is usally intended. Instead, if a particular benchmark is failing,
   # disable it in tools/perf/benchmarks/*.
   @benchmark_module.Disabled('chromeos')  # crbug.com/351114
   def BenchmarkSmokeTest(self):
     # Only measure a single page so that this test cycles reasonably quickly.
     benchmark.options['pageset_repeat'] = 1
     benchmark.options['page_repeat'] = 1
 
     class SinglePageBenchmark(benchmark):  # pylint: disable=W0232
       def CreatePageSet(self, options):
         # pylint: disable=E1002
         ps = super(SinglePageBenchmark, self).CreatePageSet(options)
-        for p in ps.pages:
+        user_stories = []
+        for p in ps.pages[:num_pages]:
           p.skip_waits = True
-          ps.user_stories = [p]
-          break
+          user_stories.append(p)
+        ps.user_stories = user_stories
         return ps
 
     # Set the benchmark's default arguments.
     options = options_for_unittests.GetCopy()
     options.output_format = 'none'
     options.suppress_gtest_report = True
     parser = options.CreateParser()
 
     benchmark.AddCommandLineArgs(parser)
     benchmark_module.AddCommandLineArgs(parser)
@@ skipping 55 matching lines @@
         benchmark.Name().startswith('skpicture_printer')):
       continue
 
     if hasattr(benchmark, 'generated_profile_archive'):
       # We'd like to test these, but don't know how yet.
       continue
 
     class BenchmarkSmokeTest(unittest.TestCase):
       pass
 
-    method = SmokeTestGenerator(benchmark)
+    # It is wrong to give tab_switching only one page to switch.
+    if 'tab_switching' in benchmark.Name():
+      method = SmokeTestGenerator(benchmark, num_pages=2)

[nednguyen, 2015/06/24 14:53:10]

This fix is reasonable. But please check the cycle time of
telemetry_perf_unittest in mac_chromium_rel_ng, win_chromium_rel_ng,
win_chromium_x64_rel_ng, linux_android_rel_ng to make sure that they don't
exceed 3 minutes.

+    else:
+      method = SmokeTestGenerator(benchmark)
 
     # Make sure any decorators are propagated from the original declaration.
     # (access to protected members) pylint: disable=W0212
     # TODO(dpranke): Since we only pick the first test from every class
     # (above), if that test is disabled, we'll end up not running *any*
     # test from the class. We should probably discover all of the tests
     # in a class, and then throw the ones we don't need away instead.
 
     # Merge decorators.
     for attribute in ['_enabled_strings', '_disabled_strings']:
@@ skipping 10 matching lines @@
 
     # Disable some tests on android platform only.
     if sys.modules[benchmark.__module__] in _ANDROID_BLACK_LIST_MODULES:
       method._disabled_strings.append('android')
 
     setattr(BenchmarkSmokeTest, benchmark.Name(), method)
 
     suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
 
   return suite