[Telemetry] Make `run_benchmark list` support SUPPORTED_PLATFORMS and StoryExpectations.

telemetry/telemetry/benchmark_runner_unittest.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import unittest
 
 from telemetry import benchmark
 from telemetry import benchmark_runner
+from telemetry.story import expectations
 from telemetry.testing import stream
 import mock
 
 
-class BenchmarkFoo(benchmark.Benchmark):
-  """ Benchmark Foo for testing."""
+class DisabledExpectation(expectations.StoryExpectations):
+  def SetExpectations(self):
+    self.DisableBenchmark([expectations.ALL], 'crbug.com/123')
+
+
+class BenchmarkEnabled(benchmark.Benchmark):
+  """ Enabled benchmark for testing."""
 
   @classmethod
   def Name(cls):
-    return 'FooBenchmark'
+    return 'EnabledBench'
 
 
-class BenchmarkBar(benchmark.Benchmark):
-  """ Benchmark Bar for testing long description line."""
+class BenchmarkEnabledTwo(benchmark.Benchmark):
+  """ Second enabled benchmark for testing."""
 
   @classmethod
   def Name(cls):
-    return 'BarBenchmarkkkkk'
+    return 'EnabledBench2'
 
 
-class UnusualBenchmark(benchmark.Benchmark):
+class BenchmarkDisabled(benchmark.Benchmark):
+  """ Disabled benchmark for testing."""
 
   @classmethod
   def Name(cls):
-    return 'I have a very unusual name'
+    return 'DisabledBench'
+
+  def GetExpectations(self):
+    return DisabledExpectation()
 
 
 class BenchmarkRunnerUnittest(unittest.TestCase):
 
   def setUp(self):
     self._stream = stream.TestOutputStream()
     self._mock_possible_browser = mock.MagicMock()
     self._mock_possible_browser.browser_type = 'TestBrowser'
 
   def testPrintBenchmarkListWithNoDisabledBenchmark(self):
     expected_printed_stream = (
         'Available benchmarks for TestBrowser are:\n'
-        '  BarBenchmarkkkkk  Benchmark Bar for testing long description line.\n'
-        '  FooBenchmark      Benchmark Foo for testing.\n'
+        '  EnabledBench  Enabled benchmark for testing.\n'
+        '  EnabledBench  Enabled benchmark for testing.\n'
         'Pass --browser to list benchmarks for another browser.\n\n')
-    with mock.patch('telemetry.benchmark_runner.decorators') as mock_module:
-      mock_module.IsEnabled.return_value = (True, None)
-      benchmark_runner.PrintBenchmarkList([BenchmarkFoo, BenchmarkBar],
-                                          self._mock_possible_browser,
-                                          self._stream)
-      self.assertEquals(expected_printed_stream, self._stream.output_data)
+    benchmark_runner.PrintBenchmarkList([BenchmarkEnabled, BenchmarkEnabled],
+                                        self._mock_possible_browser,
+                                        self._stream)
+    self.assertEquals(expected_printed_stream, self._stream.output_data)
 
   def testPrintBenchmarkListWithOneDisabledBenchmark(self):
     expected_printed_stream = (
         'Available benchmarks for TestBrowser are:\n'
-        '  FooBenchmark      Benchmark Foo for testing.\n'
+        '  EnabledBench   Enabled benchmark for testing.\n'
         '\n'
         'Disabled benchmarks for TestBrowser are (force run with -d):\n'
-        '  BarBenchmarkkkkk  Benchmark Bar for testing long description line.\n'
+        '  DisabledBench  Disabled benchmark for testing.\n'
         'Pass --browser to list benchmarks for another browser.\n\n')
-    with mock.patch('telemetry.benchmark_runner.decorators') as mock_module:
 
-      def FakeIsEnabled(benchmark_class, _):
-        if benchmark_class is BenchmarkFoo:
-          return True
-        else:
-          return False
-
-      mock_module.IsBenchmarkEnabled = FakeIsEnabled
-      benchmark_runner.PrintBenchmarkList([BenchmarkFoo, BenchmarkBar],
-                                          self._mock_possible_browser,
-                                          self._stream)
-      self.assertEquals(expected_printed_stream, self._stream.output_data)
+    benchmark_runner.PrintBenchmarkList([BenchmarkEnabled, BenchmarkDisabled],
+                                        self._mock_possible_browser,
+                                        self._stream)
+    self.assertEquals(expected_printed_stream, self._stream.output_data)
 
   def testShouldDisable(self):
     """Ensure that overridden ShouldDisable class methods are respected."""
     expected_printed_stream = (
         'Available benchmarks for TestBrowser are:\n'
-        '  BarBenchmarkkkkk  Benchmark Bar for testing long description line.\n'
+        '  EnabledBench   Enabled benchmark for testing.\n'
         '\n'
         'Disabled benchmarks for TestBrowser are (force run with -d):\n'
-        '  FooBenchmark      Benchmark Foo for testing.\n'
+        '  DisabledBench  Disabled benchmark for testing.\n'
         'Pass --browser to list benchmarks for another browser.\n\n')
 
     @classmethod
     def FakeShouldDisable(cls, possible_browser):
       del possible_browser  # unused
-      return cls is BenchmarkFoo
+      return cls is BenchmarkDisabled
 
-    BenchmarkFoo.ShouldDisable = FakeShouldDisable
-    BenchmarkBar.ShouldDisable = FakeShouldDisable
+    BenchmarkDisabled.ShouldDisable = FakeShouldDisable

[nednguyen, 2017/09/26 15:17:30]

Are we also gonna remove these ShouldDisable methods?

[rnephew (Reviews Here), 2017/09/26 15:55:02]

For now we cannot. The BattOr checks use it; but once I move those elsewhere I
will re-evaluate removing ShouldDisable completely. 

+    BenchmarkEnabled.ShouldDisable = FakeShouldDisable
     benchmark_runner.PrintBenchmarkList(
-        [BenchmarkFoo, BenchmarkBar], self._mock_possible_browser, self._stream)
+        [BenchmarkDisabled, BenchmarkEnabled], self._mock_possible_browser,
+        self._stream)
     self.assertEquals(expected_printed_stream, self._stream.output_data)
 
   def testShouldDisableComplex(self):
     """Ensure that browser-dependent ShouldDisable overrides are respected."""
     expected_printed_stream = (
         # Expected output for 'TestBrowser':
         'Available benchmarks for TestBrowser are:\n'
-        '  FooBenchmark      Benchmark Foo for testing.\n'
+        '  EnabledBench   Enabled benchmark for testing.\n'
         '\n'
         'Disabled benchmarks for TestBrowser are (force run with -d):\n'
-        '  BarBenchmarkkkkk  Benchmark Bar for testing long description line.\n'
+        '  EnabledBench2  Second enabled benchmark for testing.\n'
         'Pass --browser to list benchmarks for another browser.\n\n'
         # Expected output for 'MockBrowser':
         'Available benchmarks for MockBrowser are:\n'
-        '  BarBenchmarkkkkk  Benchmark Bar for testing long description line.\n'
-        '  FooBenchmark      Benchmark Foo for testing.\n'
+        '  EnabledBench   Enabled benchmark for testing.\n'
+        '  EnabledBench2  Second enabled benchmark for testing.\n'
         'Pass --browser to list benchmarks for another browser.\n\n')
 
     @classmethod
     def FakeShouldDisable(cls, possible_browser):
-      return cls is BenchmarkBar and not 'Mock' in possible_browser.browser_type
+      return (
+          cls is BenchmarkEnabledTwo and
+          not 'Mock' in possible_browser.browser_type)
 
-    BenchmarkFoo.ShouldDisable = FakeShouldDisable
-    BenchmarkBar.ShouldDisable = FakeShouldDisable
+    BenchmarkEnabled.ShouldDisable = FakeShouldDisable
+    BenchmarkEnabledTwo.ShouldDisable = FakeShouldDisable
     benchmark_runner.PrintBenchmarkList(
-        [BenchmarkFoo, BenchmarkBar], self._mock_possible_browser, self._stream)
+        [BenchmarkEnabled, BenchmarkEnabledTwo], self._mock_possible_browser,
+        self._stream)
     self._mock_possible_browser.browser_type = 'MockBrowser'
     benchmark_runner.PrintBenchmarkList(
-        [BenchmarkFoo, BenchmarkBar], self._mock_possible_browser, self._stream)
+        [BenchmarkEnabled, BenchmarkEnabledTwo], self._mock_possible_browser,
+        self._stream)
     self.assertEquals(expected_printed_stream, self._stream.output_data)