Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(398)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: telemetry/telemetry/story/expectations_unittest.py

Issue 2843403005: [Telemetry] Add Expectation module that enables disabling benchmarks/stories (Closed)
Patch Set: [Telemetry] Have benchmarks have story expectations. Created 3 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« telemetry/telemetry/story/expectations.py ('K') | « telemetry/telemetry/story/expectations.py ('k') | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: telemetry/telemetry/story/expectations_unittest.py
diff --git a/telemetry/telemetry/story/expectations_unittest.py b/telemetry/telemetry/story/expectations_unittest.py
new file mode 100644
index 0000000000000000000000000000000000000000..119264225349ec33c950c220119628d681dd4415
--- /dev/null
+++ b/telemetry/telemetry/story/expectations_unittest.py
@@ -0,0 +1,273 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+from telemetry.story import expectations
+
+
+class MockState(object):
+
+ class Platform(object):
nednguyen 2017/05/10 21:07:58 You can use https://github.com/catapult-project/ca
rnephew (Reviews Here) 2017/05/10 21:58:47 Done.
+ def __init__(self):
+ self._os_name = 'win'
+ self._battor = False
+
+ def GetOSName(self):
+ return self._os_name
+
+ def SetOSName(self, os):
+ self._os_name = os
+
+ def SetBattOrDetected(self, b):
+ assert isinstance(b, bool)
+ self._battor = b
+
+ def HasBattOrConnected(self):
+ return self._battor
+
+ def __init__(self):
+ self.platform = self.Platform()
+
+
+class MockStory(object):
+ def __init__(self, name):
+ self._name = name
+
+ @property
+ def display_name(self):
+ return self._name
+
+
+class StoryExpectationsTest(unittest.TestCase):
+ def setUp(self):
+ self._state = MockState()
+
+ def testInitialization(self):
+ e = expectations.StoryExpectations()
+ self.assertTrue(e._frozen)
+ self.assertFalse(e._expectations, {})
+ self.assertFalse(e._disabled_platforms)
+
+ def testCantDisableAfterInit(self):
+ e = expectations.StoryExpectations()
+ with self.assertRaises(AssertionError):
+ e.DisableBenchmark(['test'], 'test')
+ with self.assertRaises(AssertionError):
+ e.DisableStory('story', ['platform'], 'reason')
+
+ def testDisableBenchmarkOnPlatform(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableBenchmark(
+ [expectations.ALL_WIN,
+ expectations.ALL_MAC],
+ 'crbug/123')
+
+ e = FooExpectations()
+
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+
+ self._state.platform.SetOSName('android')
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisabledBenchmarkNoBattOr(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableBenchmark(
+ [expectations.NO_BATTOR], 'BATTOR')
+
+ e = FooExpectations()
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'BATTOR')
+
+ self._state.platform.SetBattOrDetected(True)
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+
+ def testDisableBenchmarkDesktop(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableBenchmark(
+ [expectations.ALL_DESKTOP], 'crbug/123')
+
+ e = FooExpectations()
+
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+
+ self._state.platform.SetOSName('android')
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisableBenchmarkMobile(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableBenchmark(
+ [expectations.ALL_MOBILE], 'crbug/123')
+
+ e = FooExpectations()
+
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ self._state.platform.SetOSName('android')
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+
+ def testDisableBenchmarkAll(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableBenchmark(
+ [expectations.ALL_MAC], 'Mac')
+ self.DisableBenchmark(
+ [expectations.ALL], 'ALL')
+
+ e = FooExpectations()
+
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'ALL')
+
+ def testDisableStoryNoBattOrWithoutBattOr(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'battor', [expectations.NO_BATTOR], 'crbug/123')
+
+ e = FooExpectations()
+
+ self._state.platform.SetBattOrDetected(False)
+ is_disabled, reason = e.IsStoryDisabled(MockStory('battor'), self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+
+ def testDisableStoryNoBattOrWithBattOr(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'battor', [expectations.NO_BATTOR], 'crbug/123')
+
+ e = FooExpectations()
+
+ self._state.platform.SetBattOrDetected(True)
+ is_disabled, reason = e.IsStoryDisabled(MockStory('battor'), self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisableStoryMultiplePlatforms(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'multi', [expectations.ALL_WIN], 'crbug/123')
+ self.DisableStory(
+ 'multi', [expectations.ALL_MAC], 'crbug/456')
+
+ e = FooExpectations()
+
+ self._state.platform.SetOSName('mac')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('multi'), self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/456')
+
+ self._state.platform.SetOSName('win')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('multi'), self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+
+ self._state.platform.SetOSName('android')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('multi'), self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisableStoryAllPlatforms(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory('all', [expectations.ALL], 'crbug/123')
+
+ e = FooExpectations()
+
+ self._state.platform.SetOSName('win')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('all'), self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+
+ self._state.platform.SetOSName('mac')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('all'), self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+
+ self._state.platform.SetOSName('android')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('all'), self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+
+ self._state.platform.SetOSName('JUNK_PLATFORM_NAME')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('all'), self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+
+ def testDisableStoryOnePlatform(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'disable', [expectations.ALL_WIN], 'crbug/123')
+
+ e = FooExpectations()
+
+ # Test disabling on one platform.
+ self._state.platform.SetOSName('win')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('disable'), self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+ self._state.platform.SetOSName('mac')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('disbled'), self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisableStoryDesktop(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'desktop', [expectations.ALL_DESKTOP], 'crbug/123')
+
+ e = FooExpectations()
+
+ self._state.platform.SetOSName('win')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('desktop'), self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+ self._state.platform.SetOSName('android')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('desktop'), self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisableStoryMobile(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'mobile', [expectations.ALL_MOBILE], 'crbug/123')
+
+ e = FooExpectations()
+
+ # Test disabling on mobile platforms.
+ self._state.platform.SetOSName('android')
+ is_disabled, reason = e.IsStoryDisabled(MockStory('mobile'), self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'crbug/123')
+ self._state.platform.SetOSName(expectations.ALL_LINUX)
+ is_disabled, reason = e.IsStoryDisabled(MockStory('mobile'), self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
« telemetry/telemetry/story/expectations.py ('K') | « telemetry/telemetry/story/expectations.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698