Revert of [tools/perf] Add system_health_smoke_test that run all system health stories

tools/perf/benchmarks/system_health_smoke_test.py

-# Copyright 2016 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Run all system health stories used by system health benchmarks.
-
-Only memory benchmarks are used when running these stories to make the total
-cycle time manageable. Other system health benchmarks should be using the same
-stories as memory ones, only with fewer actions (no memory dumping).
-"""
-
-import unittest
-
-from core import perf_benchmark
-
-from telemetry import benchmark as benchmark_module
-from telemetry import decorators
-from telemetry.core import discover
-from telemetry.internal.browser import browser_finder
-from telemetry.testing import options_for_unittests
-from telemetry.testing import progress_reporter
-
-from benchmarks import system_health
-
-
-# We only cover memory system health
-_SH_BENCHMARKS_TO_SMOKE_TEST = [
-  system_health.DesktopMemorySystemHealth,
-  system_health.MobileMemorySystemHealth,
-]
-
-
-def GetSystemHealthBenchmarksToSmokeTest():
-  sh_benchmark_classes = discover.DiscoverClassesInModule(
-      system_health, perf_benchmark.PerfBenchmark,
-      index_by_class_name=True).values()
-  return list(b for b in sh_benchmark_classes if
-              b.Name().startswith('system_health.memory'))
-
-
-_DISABLED_TESTS = [
-  # crbug.com/624474
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop.load:tools:dropbox',  # pylint: disable=line-too-long
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop.load:tools:docs',  # pylint: disable=line-too-long
-  # crbug.com/624587
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop.load:search:ebay',  # pylint: disable=line-too-long
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop.load:news:cnn',  # pylint: disable=line-too-long
-  # crbug.com/624607
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop.load:media:dailymotion',  # pylint: disable=line-too-long
-  # crbug.com/624701
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_mobile.load:games:bubbles',  # pylint: disable=line-too-long
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_mobile.load:games:spychase',  # pylint: disable=line-too-long
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_mobile.load:news:cnn',  # pylint: disable=line-too-long
-]
-
-
-def _GenerateSmokeTestCase(benchmark_class, story_to_smoke_test):
-
-  # NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST.
-  #
-  # This smoke test dynamically tests all system health user stories. So
-  # disabling it for one failing or flaky benchmark would disable a much
-  # wider swath of coverage  than is usally intended. Instead, if a test is
-  # failing, disable it by putting it into the _DISABLED_TESTS list above.
-  @benchmark_module.Disabled('chromeos')  # crbug.com/351114
-  def RunTest(self):
-
-    class SinglePageBenchmark(benchmark_class):  # pylint: disable=no-init
-      def CreateStorySet(self, options):
-        # pylint: disable=super-on-old-class
-        story_set = super(SinglePageBenchmark, self).CreateStorySet(options)
-        assert story_to_smoke_test in story_set.stories
-        story_set.stories = [story_to_smoke_test]
-        return story_set
-
-    options = GenerateBenchmarkOptions(benchmark_class)
-    possible_browser = browser_finder.FindBrowser(options)
-    if possible_browser is None:
-      self.skipTest('Cannot find the browser to run the test.')
-    if (SinglePageBenchmark.ShouldDisable(possible_browser) or
-        not decorators.IsEnabled(benchmark_class, possible_browser)[0]):
-      self.skipTest('Benchmark %s is disabled' % SinglePageBenchmark.Name())
-
-    if self.id() in _DISABLED_TESTS:
-      self.skipTest('Test is explictly disabled')
-
-    self.assertEqual(0, SinglePageBenchmark().Run(options),
-                     msg='Failed: %s' % benchmark_class)
-
-  # We attach the test method to SystemHealthBenchmarkSmokeTest dynamically
-  # so that we can set the test method name to include
-  # '<benchmark class name>.<story display name>'.
-  test_method_name = '%s.%s' % (
-      benchmark_class.Name(), story_to_smoke_test.display_name)
-
-  class SystemHealthBenchmarkSmokeTest(unittest.TestCase):
-    pass
-
-  setattr(SystemHealthBenchmarkSmokeTest, test_method_name, RunTest)
-
-  return SystemHealthBenchmarkSmokeTest(methodName=test_method_name)
-
-
-def GenerateBenchmarkOptions(benchmark_class):
-  # Set the benchmark's default arguments.
-  options = options_for_unittests.GetCopy()
-  options.output_format = 'none'
-  parser = options.CreateParser()
-
-  # TODO(nednguyen): probably this logic of setting up the benchmark options
-  # parser & processing the options should be sharable with telemetry's
-  # core.
-  benchmark_class.AddCommandLineArgs(parser)
-  benchmark_module.AddCommandLineArgs(parser)
-  benchmark_class.SetArgumentDefaults(parser)
-  options.MergeDefaultValues(parser.get_default_values())
-
-  benchmark_class.ProcessCommandLineArgs(None, options)
-  benchmark_module.ProcessCommandLineArgs(None, options)
-  # Only measure a single story so that this test cycles reasonably quickly.
-  options.pageset_repeat = 1
-  options.page_repeat = 1
-  return options
-
-
-def load_tests(loader, standard_tests, pattern):
-  del loader, standard_tests, pattern  # unused
-  suite = progress_reporter.TestSuite()
-  benchmark_classes = GetSystemHealthBenchmarksToSmokeTest()
-  assert benchmark_classes, 'This list should never be empty'
-  for benchmark_class in benchmark_classes:
-
-    # HACK: these options should be derived from options_for_unittests which are
-    # the resolved options from run_tests' arguments. However, options is only
-    # parsed during test time which happens after load_tests are called.
-    # Since none of our system health benchmarks creates stories based on
-    # command line options, it should be ok to pass options=None to
-    # CreateStorySet.
-    for story_to_smoke_test in (
-        benchmark_class().CreateStorySet(options=None).stories):
-      suite.addTest(
-          _GenerateSmokeTestCase(benchmark_class, story_to_smoke_test))
-
-  return suite