tools/bisect-perf-regression_test.py - Issue 564663002: Move bisect-perf-regression.py into auto_bisect directory.

Unified Diff: tools/bisect-perf-regression_test.py

Issue 564663002: Move bisect-perf-regression.py into auto_bisect directory. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master

Patch Set: Upload a config for doing a test run. Created 6 years, 3 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Download patch

Index: tools/bisect-perf-regression_test.py

diff --git a/tools/bisect-perf-regression_test.py b/tools/bisect-perf-regression_test.py

deleted file mode 100644

index b91179216293505e5d893d6fa4c0c8a238324a94..0000000000000000000000000000000000000000

--- a/tools/bisect-perf-regression_test.py

+++ /dev/null

@@ -1,305 +0,0 @@

-# Use of this source code is governed by a BSD-style license that can be

-# found in the LICENSE file.

-import os

-import re

-import unittest

-from auto_bisect import source_control as source_control_module

-# Special import necessary because filename contains dash characters.

-bisect_perf_module = __import__('bisect-perf-regression')

-def _GetBisectPerformanceMetricsInstance():

- """Returns an instance of the BisectPerformanceMetrics class."""

- options_dict = {

- 'debug_ignore_build': True,

- 'debug_ignore_sync': True,

- 'debug_ignore_perf_test': True,

- 'command': 'fake_command',

- 'metric': 'fake/metric',

- 'good_revision': 280000,

- 'bad_revision': 280005,

- }

- bisect_options = bisect_perf_module.BisectOptions.FromDict(options_dict)

- source_control = source_control_module.DetermineAndCreateSourceControl(

- bisect_options)

- bisect_instance = bisect_perf_module.BisectPerformanceMetrics(

- source_control, bisect_options)

- bisect_instance.src_cwd = os.path.abspath(

- os.path.join(os.path.dirname(__file__), os.path.pardir))

- return bisect_instance

-class BisectPerfRegressionTest(unittest.TestCase):

- """Test case for other functions and classes in bisect-perf-regression.py."""

- def _AssertConfidence(self, score, bad_values, good_values):

- """Checks whether the given sets of values have a given confidence score.

- The score represents our confidence that the two sets of values wouldn't

- be as different as they are just by chance; that is, that some real change

- occurred between the two sets of values.

- Args:

- score: Expected confidence score.

- bad_values: First list of numbers.

- good_values: Second list of numbers.

- """

- # ConfidenceScore takes a list of lists but these lists are flattened

- # inside the function.

- confidence = bisect_perf_module.ConfidenceScore(

- [[v] for v in bad_values],

- [[v] for v in good_values])

- self.assertEqual(score, confidence)

- def testConfidenceScore_ZeroConfidence(self):

- # The good and bad sets contain the same values, so the confidence that

- # they're different should be zero.

- self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4])

- def testConfidenceScore_MediumConfidence(self):

- self._AssertConfidence(80.0, [0, 1, 1, 1, 2, 2], [1, 1, 1, 3, 3, 4])

- def testConfidenceScore_HighConfidence(self):

- self._AssertConfidence(95.0, [0, 1, 1, 1, 2, 2], [1, 2, 2, 3, 3, 4])

- def testConfidenceScore_VeryHighConfidence(self):

- # Confidence is high if the two sets of values have no internal variance.

- self._AssertConfidence(99.9, [1, 1, 1, 1], [1.2, 1.2, 1.2, 1.2])

- self._AssertConfidence(99.9, [1, 1, 1, 1], [1.01, 1.01, 1.01, 1.01])

- def testConfidenceScore_UnbalancedSampleSize(self):

- # The second set of numbers only contains one number, so confidence is 0.

- self._AssertConfidence(0.0, [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2], [1.4])

- def testConfidenceScore_EmptySample(self):

- # Confidence is zero if either or both samples are empty.

- self._AssertConfidence(0.0, [], [])

- self._AssertConfidence(0.0, [], [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2, 1.3])

- self._AssertConfidence(0.0, [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2, 1.3], [])

- def testConfidenceScore_FunctionalTestResults(self):

- self._AssertConfidence(80.0, [1, 1, 0, 1, 1, 1, 0, 1], [0, 0, 1, 0, 1, 0])

- self._AssertConfidence(99.9, [1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0])

- def testConfidenceScore_RealWorldCases(self):

- """This method contains a set of data from actual bisect results.

- The confidence scores asserted below were all copied from the actual

- results, so the purpose of this test method is mainly to show what the

- results for real cases are, and compare when we change the confidence

- score function in the future.

- """

- self._AssertConfidence(80, [133, 130, 132, 132, 130, 129], [129, 129, 125])

- self._AssertConfidence(99.5, [668, 667], [498, 498, 499])

- self._AssertConfidence(80, [67, 68], [65, 65, 67])

- self._AssertConfidence(0, [514], [514])

- self._AssertConfidence(90, [616, 613, 607, 615], [617, 619, 619, 617])

- self._AssertConfidence(0, [3.5, 5.8, 4.7, 3.5, 3.6], [2.8])

- self._AssertConfidence(90, [3, 3, 3], [2, 2, 2, 3])

- self._AssertConfidence(0, [1999004, 1999627], [223355])

- self._AssertConfidence(90, [1040, 934, 961], [876, 875, 789])

- self._AssertConfidence(90, [309, 305, 304], [302, 302, 299, 303, 298])

- def testParseDEPSStringManually(self):

- """Tests DEPS parsing."""

- deps_file_contents = """

- vars = {

- 'ffmpeg_hash':

- '@ac4a9f31fe2610bd146857bbd55d7a260003a888',

- 'webkit_url':

- 'https://chromium.googlesource.com/chromium/blink.git',

- 'git_url':

- 'https://chromium.googlesource.com',

- 'webkit_rev':

- '@e01ac0a267d1017288bc67fa3c366b10469d8a24',

- 'angle_revision':

- '74697cf2064c0a2c0d7e1b1b28db439286766a05'

- }"""

- # Should only expect SVN/git revisions to come through, and URLs should be

- # filtered out.

- expected_vars_dict = {

- 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888',

- 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24',

- 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05'

- }

- # Testing private function.

- # pylint: disable=W0212

- vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually(

- deps_file_contents)

- self.assertEqual(vars_dict, expected_vars_dict)

- def _AssertParseResult(self, expected_values, result_string):

- """Asserts some values are parsed from a RESULT line."""

- results_template = ('RESULT other_chart: other_trace= 123 count\n'

- 'RESULT my_chart: my_trace= %(value)s\n')

- results = results_template % {'value': result_string}

- metric = ['my_chart', 'my_trace']

- # Testing private function.

- # pylint: disable=W0212

- values = bisect_perf_module._TryParseResultValuesFromOutput(metric, results)

- self.assertEqual(expected_values, values)

- def testTryParseResultValuesFromOutput_WithSingleValue(self):

- """Tests result pattern <*>RESULT <graph>: <trace>= <value>"""

- self._AssertParseResult([66.88], '66.88 kb')

- self._AssertParseResult([66.88], '66.88 ')

- self._AssertParseResult([-66.88], '-66.88 kb')

- self._AssertParseResult([66], '66 kb')

- self._AssertParseResult([0.66], '.66 kb')

- self._AssertParseResult([], '. kb')

- self._AssertParseResult([], 'aaa kb')

- def testTryParseResultValuesFromOutput_WithMultiValue(self):

- """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]"""

- self._AssertParseResult([66.88], '[66.88] kb')

- self._AssertParseResult([66.88, 99.44], '[66.88, 99.44]kb')

- self._AssertParseResult([66.88, 99.44], '[ 66.88, 99.44 ]')

- self._AssertParseResult([-66.88, 99.44], '[-66.88, 99.44] kb')

- self._AssertParseResult([-66, 99], '[-66,99] kb')

- self._AssertParseResult([-66, 99], '[-66,99,] kb')

- self._AssertParseResult([-66, 0.99], '[-66,.99] kb')

- self._AssertParseResult([], '[] kb')

- self._AssertParseResult([], '[-66,abc] kb')

- def testTryParseResultValuesFromOutputWithMeanStd(self):

- """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}"""

- self._AssertParseResult([33.22], '{33.22, 3.6} kb')

- self._AssertParseResult([33.22], '{33.22,3.6}kb')

- self._AssertParseResult([33.22], '{33.22,3.6} kb')

- self._AssertParseResult([33.22], '{ 33.22,3.6 }kb')

- self._AssertParseResult([-33.22], '{-33.22,3.6}kb')

- self._AssertParseResult([22], '{22,6}kb')

- self._AssertParseResult([.22], '{.22,6}kb')

- self._AssertParseResult([], '{.22,6, 44}kb')

- self._AssertParseResult([], '{}kb')

- self._AssertParseResult([], '{XYZ}kb')

- def _AssertCompatibleCommand(

- self, expected_command, original_command, revision, target_platform):

- """Tests the modification of the command that might be done.

- This modification to the command is done in order to get a Telemetry

- command that works; before some revisions, the browser name that Telemetry

- expects is different in some cases, but we want it to work anyway.

- Specifically, only for android:

- After r276628, only android-chrome-shell works.

- Prior to r274857, only android-chromium-testshell works.

- In the range [274857, 276628], both work.

- """

- bisect_options = bisect_perf_module.BisectOptions()

- bisect_options.output_buildbot_annotations = None

- source_control = source_control_module.DetermineAndCreateSourceControl(

- bisect_options)

- bisect_instance = bisect_perf_module.BisectPerformanceMetrics(

- source_control, bisect_options)

- bisect_instance.opts.target_platform = target_platform

- git_revision = bisect_instance.source_control.ResolveToRevision(

- revision, 'chromium', bisect_perf_module.DEPOT_DEPS_NAME, 100)

- depot = 'chromium'

- command = bisect_instance.GetCompatibleCommand(

- original_command, git_revision, depot)

- self.assertEqual(expected_command, command)

- def testGetCompatibleCommand_ChangeToTestShell(self):

- # For revisions <= r274857, only android-chromium-testshell is used.

- self._AssertCompatibleCommand(

- 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',

- 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',

- 274857, 'android')

- def testGetCompatibleCommand_ChangeToShell(self):

- # For revisions >= r276728, only android-chrome-shell can be used.

- self._AssertCompatibleCommand(

- 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',

- 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',

- 276628, 'android')

- def testGetCompatibleCommand_NoChange(self):

- # For revisions < r276728, android-chromium-testshell can be used.

- self._AssertCompatibleCommand(

- 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',

- 274858, 'android')

- # For revisions > r274857, android-chrome-shell can be used.

- self._AssertCompatibleCommand(

- 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',

- 274858, 'android')

- def testGetCompatibleCommand_NonAndroidPlatform(self):

- # In most cases, there's no need to change Telemetry command.

- # For revisions >= r276728, only android-chrome-shell can be used.

- self._AssertCompatibleCommand(

- 'tools/perf/run_benchmark -v --browser=release foo',

- 276628, 'chromium')

- # This method doesn't reference self; it fails if an error is thrown.

- # pylint: disable=R0201

- def testDryRun(self):

- """Does a dry run of the bisect script.

- This serves as a smoke test to catch errors in the basic execution of the

- script.

- """

- bisect_instance = _GetBisectPerformanceMetricsInstance()

- results = bisect_instance.Run(bisect_instance.opts.command,

- bisect_instance.opts.bad_revision,

- bisect_instance.opts.good_revision,

- bisect_instance.opts.metric)

- bisect_instance.FormatAndPrintResults(results)

- def testGetCommitPosition(self):

- bisect_instance = _GetBisectPerformanceMetricsInstance()

- cp_git_rev = '7017a81991de983e12ab50dfc071c70e06979531'

- self.assertEqual(

- 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev))

- svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58'

- self.assertEqual(

- 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev))

- def testGetCommitPositionForV8(self):

- bisect_instance = _GetBisectPerformanceMetricsInstance()

- v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb'

- depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'v8')

- self.assertEqual(

- 23634,

- bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path))

- def testGetCommitPositionForWebKit(self):

- bisect_instance = _GetBisectPerformanceMetricsInstance()

- wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'

- depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'third_party',

- 'WebKit')

- self.assertEqual(

- 181660,

- bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path))

- def testUpdateDepsContent(self):

- bisect_instance = _GetBisectPerformanceMetricsInstance()

- deps_file = 'DEPS'

- # We are intentionally reading DEPS file contents instead of string literal

- # with few lines from DEPS because to check if the format we are expecting

- # to search is not changed in DEPS content.

- # TODO (prasadv): Add a separate test to validate the DEPS contents with the

- # format that bisect script expects.

- deps_contents = bisect_perf_module.ReadStringFromFile(deps_file)

- deps_key = 'v8_revision'

- depot = 'v8'

- git_revision = 'a12345789a23456789a123456789a123456789'

- updated_content = bisect_instance.UpdateDepsContents(

- deps_contents, depot, git_revision, deps_key)

- self.assertIsNotNone(updated_content)

- ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision))

- self.assertIsNotNone(re.search(ss, updated_content))

-if __name__ == '__main__':

- unittest.main()

« tools/auto_bisect/bisect_perf_regression.py ('K') | « tools/bisect-perf-regression.py ('k') | tools/run-bisect-manual-test.py » ('j') | no next file with comments »