[Telemetry] Add Expectation module that enables disabling benchmarks/stories

telemetry/telemetry/story/expectations_unittest.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+from telemetry.story import expectations
+
+
+class MockState(object):
+
+  class Platform(object):

[nednguyen, 2017/05/10 21:07:58]

You can use
https://github.com/catapult-project/catapult/blob/master/telemetry/telemetry/...
 & expand FakePlatform class if needed instead

[rnephew (Reviews Here), 2017/05/10 21:58:47]

Done.

+    def __init__(self):
+      self._os_name = 'win'
+      self._battor = False
+
+    def GetOSName(self):
+      return self._os_name
+
+    def SetOSName(self, os):
+      self._os_name = os
+
+    def SetBattOrDetected(self, b):
+      assert isinstance(b, bool)
+      self._battor = b
+
+    def HasBattOrConnected(self):
+      return self._battor
+
+  def __init__(self):
+    self.platform = self.Platform()
+
+
+class MockStory(object):
+  def __init__(self, name):
+    self._name = name
+
+  @property
+  def display_name(self):
+    return self._name
+
+
+class StoryExpectationsTest(unittest.TestCase):
+  def setUp(self):
+    self._state = MockState()
+
+  def testInitialization(self):
+    e = expectations.StoryExpectations()
+    self.assertTrue(e._frozen)
+    self.assertFalse(e._expectations, {})
+    self.assertFalse(e._disabled_platforms)
+
+  def testCantDisableAfterInit(self):
+    e = expectations.StoryExpectations()
+    with self.assertRaises(AssertionError):
+      e.DisableBenchmark(['test'], 'test')
+    with self.assertRaises(AssertionError):
+      e.DisableStory('story', ['platform'], 'reason')
+
+  def testDisableBenchmarkOnPlatform(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableBenchmark(
+            [expectations.ALL_WIN,
+             expectations.ALL_MAC],
+            'crbug/123')
+
+    e = FooExpectations()
+
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+
+    self._state.platform.SetOSName('android')
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisabledBenchmarkNoBattOr(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableBenchmark(
+            [expectations.NO_BATTOR], 'BATTOR')
+
+    e = FooExpectations()
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'BATTOR')
+
+    self._state.platform.SetBattOrDetected(True)
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+
+  def testDisableBenchmarkDesktop(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableBenchmark(
+            [expectations.ALL_DESKTOP], 'crbug/123')
+
+    e = FooExpectations()
+
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+
+    self._state.platform.SetOSName('android')
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisableBenchmarkMobile(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableBenchmark(
+            [expectations.ALL_MOBILE], 'crbug/123')
+
+    e = FooExpectations()
+
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+    self._state.platform.SetOSName('android')
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+
+  def testDisableBenchmarkAll(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableBenchmark(
+            [expectations.ALL_MAC], 'Mac')
+        self.DisableBenchmark(
+            [expectations.ALL], 'ALL')
+
+    e = FooExpectations()
+
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'ALL')
+
+  def testDisableStoryNoBattOrWithoutBattOr(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'battor', [expectations.NO_BATTOR], 'crbug/123')
+
+    e = FooExpectations()
+
+    self._state.platform.SetBattOrDetected(False)
+    is_disabled, reason = e.IsStoryDisabled(MockStory('battor'), self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+
+  def testDisableStoryNoBattOrWithBattOr(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'battor', [expectations.NO_BATTOR], 'crbug/123')
+
+    e = FooExpectations()
+
+    self._state.platform.SetBattOrDetected(True)
+    is_disabled, reason = e.IsStoryDisabled(MockStory('battor'), self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisableStoryMultiplePlatforms(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'multi', [expectations.ALL_WIN], 'crbug/123')
+        self.DisableStory(
+            'multi', [expectations.ALL_MAC], 'crbug/456')
+
+    e = FooExpectations()
+
+    self._state.platform.SetOSName('mac')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('multi'), self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/456')
+
+    self._state.platform.SetOSName('win')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('multi'), self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+
+    self._state.platform.SetOSName('android')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('multi'), self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisableStoryAllPlatforms(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory('all', [expectations.ALL], 'crbug/123')
+
+    e = FooExpectations()
+
+    self._state.platform.SetOSName('win')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('all'), self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+
+    self._state.platform.SetOSName('mac')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('all'), self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+
+    self._state.platform.SetOSName('android')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('all'), self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+
+    self._state.platform.SetOSName('JUNK_PLATFORM_NAME')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('all'), self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+
+  def testDisableStoryOnePlatform(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'disable', [expectations.ALL_WIN], 'crbug/123')
+
+    e = FooExpectations()
+
+    # Test disabling on one platform.
+    self._state.platform.SetOSName('win')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('disable'), self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+    self._state.platform.SetOSName('mac')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('disbled'), self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisableStoryDesktop(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'desktop', [expectations.ALL_DESKTOP], 'crbug/123')
+
+    e = FooExpectations()
+
+    self._state.platform.SetOSName('win')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('desktop'), self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+    self._state.platform.SetOSName('android')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('desktop'), self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisableStoryMobile(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'mobile', [expectations.ALL_MOBILE], 'crbug/123')
+
+    e = FooExpectations()
+
+    # Test disabling on mobile platforms.
+    self._state.platform.SetOSName('android')
+    is_disabled, reason = e.IsStoryDisabled(MockStory('mobile'), self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'crbug/123')
+    self._state.platform.SetOSName(expectations.ALL_LINUX)
+    is_disabled, reason = e.IsStoryDisabled(MockStory('mobile'), self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)