[Telemetry] Add Expectation module that enables disabling benchmarks/stories

telemetry/telemetry/story/expectations_unittest.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+from telemetry.story import expectations
+
+
+class MockState(object):
+
+  class Platform(object):
+    def __init__(self):
+      self._os_name = expectations.TestConditions.WIN
+      self._battor = False
+
+    def GetOSName(self):
+      return self._os_name
+
+    def SetOSName(self, os):
+      self._os_name = os
+
+    def SetBattOrDetected(self, b):
+      assert isinstance(b, bool)
+      self._battor = b
+
+    def HasBattOrConnected(self):
+      return self._battor
+
+  def __init__(self):
+    self.platform = self.Platform()
+

[perezju, 2017/05/02 09:19:52]

nit: extra blank line here

[rnephew (Reviews Here), 2017/05/10 19:45:13]

Done.

+class StoryExpectationsTest(unittest.TestCase):
+  def setUp(self):
+    self._state = MockState()
+
+  def testInitialization(self):
+    e = expectations.StoryExpectations()
+    self.assertTrue(e._frozen)
+    self.assertDictEqual(e._expectations, {'disabled_platforms': []})
+
+  def testCantDisableAfterInit(self):
+    e = expectations.StoryExpectations()
+    with self.assertRaises(AssertionError):
+      e.DisableBenchmark(['test'], 'test')
+    with self.assertRaises(AssertionError):
+      e.DisableStory('story', ['platform'], 'reason')
+
+  def testDisableBenchmarkOnPlatform(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableBenchmark(
+            [expectations.TestConditions.WIN,
+             expectations.TestConditions.MAC],
+            'TEST')

[perezju, 2017/05/02 09:19:52]

nit: use 'crbug/123' (or something like that) rather than "TEST" to set the tone
of what we expect to be usually set as reason.

[rnephew (Reviews Here), 2017/05/10 19:45:13]

Done.

+
+    e = FooExpectations()
+
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TEST')
+
+    self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisabledBenchmarkNoBattOr(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableBenchmark(
+            [expectations.TestConditions.NO_BATTOR], 'BATTOR')
+
+    e = FooExpectations()
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'BATTOR')
+
+    self._state.platform.SetBattOrDetected(True)
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+
+  def testDisableBenchmarkDesktop(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableBenchmark(
+            [expectations.TestConditions.DESKTOP], 'TEST')
+
+    e = FooExpectations()
+
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TEST')
+
+    self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisableBenchmarkMobile(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableBenchmark(
+            [expectations.TestConditions.MOBILE], 'TEST')
+
+    e = FooExpectations()
+
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+    self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TEST')
+
+  def testDisableBenchmarkAll(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableBenchmark(
+            [expectations.TestConditions.MAC], 'Mac')
+        self.DisableBenchmark(
+            [expectations.TestConditions.ALL], 'ALL')
+
+    e = FooExpectations()
+
+    is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'ALL')
+
+  def testDisableStoryNoBattOrWithoutBattOr(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'battor', [expectations.TestConditions.NO_BATTOR], 'TEST')
+
+    e = FooExpectations()
+
+    self._state.platform.SetBattOrDetected(False)
+    is_disabled, reason = e.IsStoryDisabled('battor', self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TEST')
+
+  def testDisableStoryNoBattOrWithBattOr(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'battor', [expectations.TestConditions.NO_BATTOR], 'TEST')
+
+    e = FooExpectations()
+
+    self._state.platform.SetBattOrDetected(True)
+    is_disabled, reason = e.IsStoryDisabled('battor', self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisableStoryMultiplePlatforms(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'multi-platform', [expectations.TestConditions.WIN], 'TESTa')
+        self.DisableStory(
+            'multi-platform', [expectations.TestConditions.MAC], 'TESTb')
+
+    e = FooExpectations()
+
+    self._state.platform.SetOSName(expectations.TestConditions.MAC)
+    is_disabled, reason = e.IsStoryDisabled('multi-platform', self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TESTb')
+
+    self._state.platform.SetOSName(expectations.TestConditions.WIN)
+    is_disabled, reason = e.IsStoryDisabled('multi-platform', self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TESTa')
+
+    self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+    is_disabled, reason = e.IsStoryDisabled('multi-platform', self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisableStoryAllPlatforms(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'disable-all', [expectations.TestConditions.ALL], 'TEST')
+
+    e = FooExpectations()
+
+    self._state.platform.SetOSName(expectations.TestConditions.WIN)
+    is_disabled, reason = e.IsStoryDisabled('disable-all', self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TEST')
+
+    self._state.platform.SetOSName(expectations.TestConditions.MAC)
+    is_disabled, reason = e.IsStoryDisabled('disable-all', self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TEST')
+
+    self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+    is_disabled, reason = e.IsStoryDisabled('disable-all', self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TEST')
+
+    self._state.platform.SetOSName('JUNK_PLATFORM_NAME')
+    is_disabled, reason = e.IsStoryDisabled('disable-all', self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TEST')
+
+  def testDisableStoryOnePlatform(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'disable-win', [expectations.TestConditions.WIN], 'TEST')
+
+    e = FooExpectations()
+
+    # Test disabling on one platform.
+    self._state.platform.SetOSName(expectations.TestConditions.WIN)
+    is_disabled, reason = e.IsStoryDisabled('disable-win', self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TEST')
+    self._state.platform.SetOSName(expectations.TestConditions.MAC)
+    is_disabled, reason = e.IsStoryDisabled('disbled-win', self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisableStoryDesktop(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'disable-desktop', [expectations.TestConditions.DESKTOP], 'TEST')
+
+    e = FooExpectations()
+
+    self._state.platform.SetOSName(expectations.TestConditions.WIN)
+    is_disabled, reason = e.IsStoryDisabled('disable-desktop', self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TEST')
+    self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+    is_disabled, reason = e.IsStoryDisabled('disable-desktop', self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)
+
+  def testDisableStoryMobile(self):
+    class FooExpectations(expectations.StoryExpectations):
+      def SetExpectations(self):
+        self.DisableStory(
+            'disable-mobile', [expectations.TestConditions.MOBILE], 'TEST')
+
+    e = FooExpectations()
+
+    # Test disabling on mobile platforms.
+    self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+    is_disabled, reason = e.IsStoryDisabled('disable-mobile', self._state)
+    self.assertTrue(is_disabled)
+    self.assertEqual(reason, 'TEST')
+    self._state.platform.SetOSName(expectations.TestConditions.LINUX)
+    is_disabled, reason = e.IsStoryDisabled('disable-mobile', self._state)
+    self.assertFalse(is_disabled)
+    self.assertIsNone(reason)