Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1238)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: telemetry/telemetry/story/expectations_unittest.py

Issue 2843403005: [Telemetry] Add Expectation module that enables disabling benchmarks/stories (Closed)
Patch Set: [Telemetry] Have benchmarks have story expectations. Created 3 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« telemetry/telemetry/story/expectations.py ('K') | « telemetry/telemetry/story/expectations.py ('k') | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: telemetry/telemetry/story/expectations_unittest.py
diff --git a/telemetry/telemetry/story/expectations_unittest.py b/telemetry/telemetry/story/expectations_unittest.py
new file mode 100644
index 0000000000000000000000000000000000000000..222e4b1ce7a6d55b40e55ffea90a0b6bb868e6b9
--- /dev/null
+++ b/telemetry/telemetry/story/expectations_unittest.py
@@ -0,0 +1,263 @@
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+from telemetry.story import expectations
+
+
+class MockState(object):
+
+ class Platform(object):
+ def __init__(self):
+ self._os_name = expectations.TestConditions.WIN
+ self._battor = False
+
+ def GetOSName(self):
+ return self._os_name
+
+ def SetOSName(self, os):
+ self._os_name = os
+
+ def SetBattOrDetected(self, b):
+ assert isinstance(b, bool)
+ self._battor = b
+
+ def HasBattOrConnected(self):
+ return self._battor
+
+ def __init__(self):
+ self.platform = self.Platform()
+
perezju 2017/05/02 09:19:52 nit: extra blank line here
rnephew (Reviews Here) 2017/05/10 19:45:13 Done.
+class StoryExpectationsTest(unittest.TestCase):
+ def setUp(self):
+ self._state = MockState()
+
+ def testInitialization(self):
+ e = expectations.StoryExpectations()
+ self.assertTrue(e._frozen)
+ self.assertDictEqual(e._expectations, {'disabled_platforms': []})
+
+ def testCantDisableAfterInit(self):
+ e = expectations.StoryExpectations()
+ with self.assertRaises(AssertionError):
+ e.DisableBenchmark(['test'], 'test')
+ with self.assertRaises(AssertionError):
+ e.DisableStory('story', ['platform'], 'reason')
+
+ def testDisableBenchmarkOnPlatform(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableBenchmark(
+ [expectations.TestConditions.WIN,
+ expectations.TestConditions.MAC],
+ 'TEST')
perezju 2017/05/02 09:19:52 nit: use 'crbug/123' (or something like that) rath
rnephew (Reviews Here) 2017/05/10 19:45:13 Done.
+
+ e = FooExpectations()
+
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TEST')
+
+ self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisabledBenchmarkNoBattOr(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableBenchmark(
+ [expectations.TestConditions.NO_BATTOR], 'BATTOR')
+
+ e = FooExpectations()
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'BATTOR')
+
+ self._state.platform.SetBattOrDetected(True)
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+
+ def testDisableBenchmarkDesktop(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableBenchmark(
+ [expectations.TestConditions.DESKTOP], 'TEST')
+
+ e = FooExpectations()
+
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TEST')
+
+ self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisableBenchmarkMobile(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableBenchmark(
+ [expectations.TestConditions.MOBILE], 'TEST')
+
+ e = FooExpectations()
+
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TEST')
+
+ def testDisableBenchmarkAll(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableBenchmark(
+ [expectations.TestConditions.MAC], 'Mac')
+ self.DisableBenchmark(
+ [expectations.TestConditions.ALL], 'ALL')
+
+ e = FooExpectations()
+
+ is_disabled, reason = e.IsBenchmarkDisabled(self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'ALL')
+
+ def testDisableStoryNoBattOrWithoutBattOr(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'battor', [expectations.TestConditions.NO_BATTOR], 'TEST')
+
+ e = FooExpectations()
+
+ self._state.platform.SetBattOrDetected(False)
+ is_disabled, reason = e.IsStoryDisabled('battor', self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TEST')
+
+ def testDisableStoryNoBattOrWithBattOr(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'battor', [expectations.TestConditions.NO_BATTOR], 'TEST')
+
+ e = FooExpectations()
+
+ self._state.platform.SetBattOrDetected(True)
+ is_disabled, reason = e.IsStoryDisabled('battor', self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisableStoryMultiplePlatforms(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'multi-platform', [expectations.TestConditions.WIN], 'TESTa')
+ self.DisableStory(
+ 'multi-platform', [expectations.TestConditions.MAC], 'TESTb')
+
+ e = FooExpectations()
+
+ self._state.platform.SetOSName(expectations.TestConditions.MAC)
+ is_disabled, reason = e.IsStoryDisabled('multi-platform', self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TESTb')
+
+ self._state.platform.SetOSName(expectations.TestConditions.WIN)
+ is_disabled, reason = e.IsStoryDisabled('multi-platform', self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TESTa')
+
+ self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+ is_disabled, reason = e.IsStoryDisabled('multi-platform', self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisableStoryAllPlatforms(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'disable-all', [expectations.TestConditions.ALL], 'TEST')
+
+ e = FooExpectations()
+
+ self._state.platform.SetOSName(expectations.TestConditions.WIN)
+ is_disabled, reason = e.IsStoryDisabled('disable-all', self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TEST')
+
+ self._state.platform.SetOSName(expectations.TestConditions.MAC)
+ is_disabled, reason = e.IsStoryDisabled('disable-all', self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TEST')
+
+ self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+ is_disabled, reason = e.IsStoryDisabled('disable-all', self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TEST')
+
+ self._state.platform.SetOSName('JUNK_PLATFORM_NAME')
+ is_disabled, reason = e.IsStoryDisabled('disable-all', self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TEST')
+
+ def testDisableStoryOnePlatform(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'disable-win', [expectations.TestConditions.WIN], 'TEST')
+
+ e = FooExpectations()
+
+ # Test disabling on one platform.
+ self._state.platform.SetOSName(expectations.TestConditions.WIN)
+ is_disabled, reason = e.IsStoryDisabled('disable-win', self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TEST')
+ self._state.platform.SetOSName(expectations.TestConditions.MAC)
+ is_disabled, reason = e.IsStoryDisabled('disbled-win', self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisableStoryDesktop(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'disable-desktop', [expectations.TestConditions.DESKTOP], 'TEST')
+
+ e = FooExpectations()
+
+ self._state.platform.SetOSName(expectations.TestConditions.WIN)
+ is_disabled, reason = e.IsStoryDisabled('disable-desktop', self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TEST')
+ self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+ is_disabled, reason = e.IsStoryDisabled('disable-desktop', self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
+
+ def testDisableStoryMobile(self):
+ class FooExpectations(expectations.StoryExpectations):
+ def SetExpectations(self):
+ self.DisableStory(
+ 'disable-mobile', [expectations.TestConditions.MOBILE], 'TEST')
+
+ e = FooExpectations()
+
+ # Test disabling on mobile platforms.
+ self._state.platform.SetOSName(expectations.TestConditions.ANDROID)
+ is_disabled, reason = e.IsStoryDisabled('disable-mobile', self._state)
+ self.assertTrue(is_disabled)
+ self.assertEqual(reason, 'TEST')
+ self._state.platform.SetOSName(expectations.TestConditions.LINUX)
+ is_disabled, reason = e.IsStoryDisabled('disable-mobile', self._state)
+ self.assertFalse(is_disabled)
+ self.assertIsNone(reason)
« telemetry/telemetry/story/expectations.py ('K') | « telemetry/telemetry/story/expectations.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698