Refactored bisect results dicts into a separate class

tools/auto_bisect/bisect_perf_regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 18 matching lines @@
   "out/Release/performance_ui_tests --gtest_filter=ShutdownTest.SimpleUserQuit"\
   -g 1f6e67861535121c5c819c16a666f2436c207e7b\
   -b b732f23b4f81c382db0b23b9035f3dadc7d925bb\
   -m shutdown/simple-user-quit
 """
 
 import copy
 import datetime
 import errno
 import hashlib
-import math
 import optparse
 import os
 import re
 import shlex
 import shutil
 import StringIO
 import sys
 import time
 import zipfile
 
 sys.path.append(os.path.join(
     os.path.dirname(__file__), os.path.pardir, 'telemetry'))
 
+from bisect_results import BisectResults
 import bisect_utils
 import builder
 import math_utils
 import request_build
 import source_control as source_control_module
-import ttest
 from telemetry.util import cloud_storage
 
 # Below is the map of "depot" names to information about each depot. Each depot
 # is a repository, and in the process of bisecting, revision ranges in these
 # repositories may also be bisected.
 #
 # Each depot information dictionary may contain:
 #   src: Path to the working directory.
 #   recurse: True if this repository will get bisected.
 #   depends: A list of other repositories that are actually part of the same
@@ skipping 200 matching lines @@
 
 
 def _AddAdditionalDepotInfo(depot_info):
   """Adds additional depot info to the global depot variables."""
   global DEPOT_DEPS_NAME
   global DEPOT_NAMES
   DEPOT_DEPS_NAME = dict(DEPOT_DEPS_NAME.items() + depot_info.items())
   DEPOT_NAMES = DEPOT_DEPS_NAME.keys()
 
 
-def ConfidenceScore(good_results_lists, bad_results_lists):
-  """Calculates a confidence score.
-
-  This score is a percentage which represents our degree of confidence in the
-  proposition that the good results and bad results are distinct groups, and
-  their differences aren't due to chance alone.
-
-
-  Args:
-    good_results_lists: A list of lists of "good" result numbers.
-    bad_results_lists: A list of lists of "bad" result numbers.
-
-  Returns:
-    A number in the range [0, 100].
-  """
-  # If there's only one item in either list, this means only one revision was
-  # classified good or bad; this isn't good enough evidence to make a decision.
-  # If an empty list was passed, that also implies zero confidence.
-  if len(good_results_lists) <= 1 or len(bad_results_lists) <= 1:
-    return 0.0
-
-  # Flatten the lists of results lists.
-  sample1 = sum(good_results_lists, [])
-  sample2 = sum(bad_results_lists, [])
-
-  # If there were only empty lists in either of the lists (this is unexpected
-  # and normally shouldn't happen), then we also want to return 0.
-  if not sample1 or not sample2:
-    return 0.0
-
-  # The p-value is approximately the probability of obtaining the given set
-  # of good and bad values just by chance.
-  _, _, p_value = ttest.WelchsTTest(sample1, sample2)
-  return 100.0 * (1.0 - p_value)
-
-
 def GetSHA1HexDigest(contents):
   """Returns SHA1 hex digest of the given string."""
   return hashlib.sha1(contents).hexdigest()
 
 
 def GetZipFileName(build_revision=None, target_arch='ia32', patch_sha=None):
   """Gets the archive file name for the given revision."""
   def PlatformName():
     """Return a string to be used in paths for the platform."""
     if bisect_utils.IsWindowsHost():
@@ skipping 536 matching lines @@
   if step_count:
     step_perf_time_avg = step_perf_time_avg / step_count
     step_build_time_avg = step_build_time_avg / step_count
   print
   print 'Average build time : %s' % datetime.timedelta(
       seconds=int(step_build_time_avg))
   print 'Average test time  : %s' % datetime.timedelta(
       seconds=int(step_perf_time_avg))
 
 
-def _FindOtherRegressions(revision_data_sorted, bad_greater_than_good):
-  """Compiles a list of other possible regressions from the revision data.
+class DepotDirectoryRegistry(object):
+  """Manages depots (code repositories) and their directories."""

[ojan, 2014/09/26 17:34:30]

IMO, this comment doesn't say anything the name of the class already does.

 
-  Args:
-    revision_data_sorted: Sorted list of (revision, revision data) pairs.
-    bad_greater_than_good: Whether the result value at the "bad" revision is
-        numerically greater than the result value at the "good" revision.
+  def __init__(self):
+    self.depot_cwd = {}
+    self.cros_cwd = os.path.join(SRC_DIR, 'tools', 'cros')
+    for depot in DEPOT_NAMES:
+      # The working directory of each depot is just the path to the depot, but
+      # since we're already in 'src', we can skip that part.
+      self.depot_cwd[depot] = os.path.join(
+          SRC_DIR, DEPOT_DEPS_NAME[depot]['src'][4:])
 
-  Returns:
-    A list of [current_rev, previous_rev, confidence] for other places where
-    there may have been a regression.
-  """
-  other_regressions = []
-  previous_values = []
-  previous_id = None
-  for current_id, current_data in revision_data_sorted:
-    current_values = current_data['value']
-    if current_values:
-      current_values = current_values['values']
-      if previous_values:
-        confidence = ConfidenceScore(previous_values, [current_values])
-        mean_of_prev_runs = math_utils.Mean(sum(previous_values, []))
-        mean_of_current_runs = math_utils.Mean(current_values)
+  def AddDepot(self, depot_name, depot_dir):
+    self.depot_cwd[depot_name] = depot_dir
 
-        # Check that the potential regression is in the same direction as
-        # the overall regression. If the mean of the previous runs < the
-        # mean of the current runs, this local regression is in same
-        # direction.
-        prev_less_than_current = mean_of_prev_runs < mean_of_current_runs
-        is_same_direction = (prev_less_than_current if
-            bad_greater_than_good else not prev_less_than_current)
+  def GetDepotDir(self, depot_name):
+    if depot_name == 'chromium':
+      return SRC_DIR
+    elif depot_name == 'cros':
+      return self.cros_cwd
+    elif depot_name in DEPOT_NAMES:
+      return self.depot_cwd[depot_name]
+    else:
+      assert False, ('Unknown depot [ %s ] encountered. Possibly a new one '
+                     'was added without proper support?' % depot_name)
 
-        # Only report potential regressions with high confidence.
-        if is_same_direction and confidence > 50:
-          other_regressions.append([current_id, previous_id, confidence])
-      previous_values.append(current_values)
-      previous_id = current_id
-  return other_regressions
+  def ChangeToDepotDir(self, depot_name):
+    """Given a depot, changes to the appropriate working directory.
+
+    Args:
+      depot_name: The name of the depot (see DEPOT_NAMES).
+    """
+    os.chdir(self.GetDepotDir(depot_name))
 
 
 class BisectPerformanceMetrics(object):
   """This class contains functionality to perform a bisection of a range of
   revisions to narrow down where performance regressions may have occurred.
 
   The main entry-point is the Run method.
   """
 
   def __init__(self, source_control, opts):
     super(BisectPerformanceMetrics, self).__init__()
 
     self.opts = opts
     self.source_control = source_control
-    self.cros_cwd = os.path.join(SRC_DIR, 'tools', 'cros')
-    self.depot_cwd = {}
+    self.depot_manager = DepotDirectoryRegistry()
     self.cleanup_commands = []
     self.warnings = []
     self.builder = builder.Builder.FromOpts(opts)
 
-    for depot in DEPOT_NAMES:
-      # The working directory of each depot is just the path to the depot, but
-      # since we're already in 'src', we can skip that part.
-      self.depot_cwd[depot] = os.path.join(
-          SRC_DIR, DEPOT_DEPS_NAME[depot]['src'][4:])
 
   def PerformCleanup(self):
     """Performs cleanup when script is finished."""
     os.chdir(SRC_DIR)
     for c in self.cleanup_commands:
       if c[0] == 'mv':
         shutil.move(c[1], c[2])
       else:
         assert False, 'Invalid cleanup command.'
 
   def GetRevisionList(self, depot, bad_revision, good_revision):
     """Retrieves a list of all the commits between the bad revision and
     last known good revision."""
 
     revision_work_list = []
 
     if depot == 'cros':
       revision_range_start = good_revision
       revision_range_end = bad_revision
 
       cwd = os.getcwd()
-      self.ChangeToDepotWorkingDirectory('cros')
+      self.depot_manager.ChangeToDepotDir('cros')
 
       # Print the commit timestamps for every commit in the revision time
       # range. We'll sort them and bisect by that. There is a remote chance that
       # 2 (or more) commits will share the exact same timestamp, but it's
       # probably safe to ignore that case.
       cmd = ['repo', 'forall', '-c',
           'git log --format=%%ct --before=%d --after=%d' % (
           revision_range_end, revision_range_start)]
       output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd)
 
       assert not return_code, ('An error occurred while running '
                                '"%s"' % ' '.join(cmd))
 
       os.chdir(cwd)
 
       revision_work_list = list(set(
           [int(o) for o in output.split('\n') if bisect_utils.IsStringInt(o)]))
       revision_work_list = sorted(revision_work_list, reverse=True)
     else:
-      cwd = self._GetDepotDirectory(depot)
+      cwd = self.depot_manager.GetDepotDir(depot)
       revision_work_list = self.source_control.GetRevisionList(bad_revision,
           good_revision, cwd=cwd)
 
     return revision_work_list
 
   def _GetV8BleedingEdgeFromV8TrunkIfMappable(self, revision):
     commit_position = self.source_control.GetCommitPosition(revision)
 
     if bisect_utils.IsStringInt(commit_position):
       # V8 is tricky to bisect, in that there are only a few instances when
       # we can dive into bleeding_edge and get back a meaningful result.
       # Try to detect a V8 "business as usual" case, which is when:
       #  1. trunk revision N has description "Version X.Y.Z"
       #  2. bleeding_edge revision (N-1) has description "Prepare push to
       #     trunk. Now working on X.Y.(Z+1)."
       #
       # As of 01/24/2014, V8 trunk descriptions are formatted:
       # "Version 3.X.Y (based on bleeding_edge revision rZ)"
       # So we can just try parsing that out first and fall back to the old way.
-      v8_dir = self._GetDepotDirectory('v8')
-      v8_bleeding_edge_dir = self._GetDepotDirectory('v8_bleeding_edge')
+      v8_dir = self.depot_manager.GetDepotDir('v8')
+      v8_bleeding_edge_dir = self.depot_manager.GetDepotDir('v8_bleeding_edge')
 
       revision_info = self.source_control.QueryRevisionInfo(revision,
           cwd=v8_dir)
 
       version_re = re.compile("Version (?P<values>[0-9,.]+)")
 
       regex_results = version_re.search(revision_info['subject'])
 
       if regex_results:
         git_revision = None
@@ skipping 19 matching lines @@
 
           if git_revision:
             revision_info = self.source_control.QueryRevisionInfo(git_revision,
                 cwd=v8_bleeding_edge_dir)
 
             if 'Prepare push to trunk' in revision_info['subject']:
               return git_revision
     return None
 
   def _GetNearestV8BleedingEdgeFromTrunk(self, revision, search_forward=True):
-    cwd = self._GetDepotDirectory('v8')
+    cwd = self.depot_manager.GetDepotDir('v8')
     cmd = ['log', '--format=%ct', '-1', revision]
     output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
     commit_time = int(output)
     commits = []
 
     if search_forward:
       cmd = ['log', '--format=%H', '-10', '--after=%d' % commit_time,
           'origin/master']
       output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
       output = output.split()
@@ skipping 41 matching lines @@
       results = {}
       for depot_name, depot_data in DEPOT_DEPS_NAME.iteritems():
         if (depot_data.get('platform') and
             depot_data.get('platform') != os.name):
           continue
 
         if (depot_data.get('recurse') and depot in depot_data.get('from')):
           depot_data_src = depot_data.get('src') or depot_data.get('src_old')
           src_dir = deps_data.get(depot_data_src)
           if src_dir:
-            self.depot_cwd[depot_name] = os.path.join(
-                SRC_DIR, depot_data_src[4:])
+            self.depot_manager.AddDepot(
+              depot_name, os.path.join(SRC_DIR, depot_data_src[4:]))
             re_results = rxp.search(src_dir)
             if re_results:
               results[depot_name] = re_results.group('revision')
             else:
               warning_text = ('Could not parse revision for %s while bisecting '
                               '%s' % (depot_name, depot))
               if not warning_text in self.warnings:
                 self.warnings.append(warning_text)
           else:
             results[depot_name] = None
@@ skipping 16 matching lines @@
   def _Get3rdPartyRevisions(self, depot):
     """Parses the DEPS file to determine WebKit/v8/etc... versions.
 
     Args:
       depot: A depot name. Should be in the DEPOT_NAMES list.
 
     Returns:
       A dict in the format {depot: revision} if successful, otherwise None.
     """
     cwd = os.getcwd()
-    self.ChangeToDepotWorkingDirectory(depot)
+    self.depot_manager.ChangeToDepotDir(depot)
 
     results = {}
 
     if depot == 'chromium' or depot == 'android-chrome':
       results = self._ParseRevisionsFromDEPSFile(depot)
       os.chdir(cwd)
 
     if depot == 'cros':
       cmd = [
           bisect_utils.CROS_SDK_PATH,
@@ skipping 20 matching lines @@
 
           version = contents[2]
 
           if contents[3] != '0':
             warningText = ('Chrome version: %s.%s but using %s.0 to bisect.' %
                            (version, contents[3], version))
             if not warningText in self.warnings:
               self.warnings.append(warningText)
 
           cwd = os.getcwd()
-          self.ChangeToDepotWorkingDirectory('chromium')
+          self.depot_manager.ChangeToDepotDir('chromium')
           cmd = ['log', '-1', '--format=%H',
                  '--author=chrome-release@google.com',
                  '--grep=to %s' % version, 'origin/master']
           return_code = bisect_utils.CheckRunGit(cmd)
           os.chdir(cwd)
 
           results['chromium'] = output.strip()
 
     if depot == 'v8':
       # We can't try to map the trunk revision to bleeding edge yet, because
@@ skipping 217 matching lines @@
     Returns:
       Updated DEPS content as string if deps key is found, otherwise None.
     """
     # Check whether the depot and revision pattern in DEPS file vars
     # e.g. for webkit the format is "webkit_revision": "12345".
     deps_revision = re.compile(r'(?<="%s": ")([0-9]+)(?=")' % deps_key,
                                re.MULTILINE)
     new_data = None
     if re.search(deps_revision, deps_contents):
       commit_position = self.source_control.GetCommitPosition(
-          git_revision, self._GetDepotDirectory(depot))
+          git_revision, self.depot_manager.GetDepotDir(depot))
       if not commit_position:
         print 'Could not determine commit position for %s' % git_revision
         return None
       # Update the revision information for the given depot
       new_data = re.sub(deps_revision, str(commit_position), deps_contents)
     else:
       # Check whether the depot and revision pattern in DEPS file vars
       # e.g. for webkit the format is "webkit_revision": "559a6d4ab7a84c539..".
       deps_revision = re.compile(
           r'(?<=["\']%s["\']: ["\'])([a-fA-F0-9]{40})(?=["\'])' % deps_key,
@@ skipping 343 matching lines @@
     # Some SVN depots were split into multiple git depots, so we need to
     # figure out for each mirror which git revision to grab. There's no
     # guarantee that the SVN revision will exist for each of the dependent
     # depots, so we have to grep the git logs and grab the next earlier one.
     if (not is_base
         and DEPOT_DEPS_NAME[depot]['depends']
         and self.source_control.IsGit()):
       commit_position = self.source_control.GetCommitPosition(revision)
 
       for d in DEPOT_DEPS_NAME[depot]['depends']:
-        self.ChangeToDepotWorkingDirectory(d)
+        self.depot_manager.ChangeToDepotDir(d)
 
         dependant_rev = self.source_control.ResolveToRevision(
             commit_position, d, DEPOT_DEPS_NAME, -1000)
 
         if dependant_rev:
           revisions_to_sync.append([d, dependant_rev])
 
       num_resolved = len(revisions_to_sync)
       num_needed = len(DEPOT_DEPS_NAME[depot]['depends'])
 
-      self.ChangeToDepotWorkingDirectory(depot)
+      self.depot_manager.ChangeToDepotDir(depot)
 
       if not ((num_resolved - 1) == num_needed):
         return None
 
     return revisions_to_sync
 
   @staticmethod
   def PerformPreBuildCleanup():
     """Performs cleanup between runs."""
     print 'Cleaning up between runs.'
     print
 
     # Leaving these .pyc files around between runs may disrupt some perf tests.
     for (path, _, files) in os.walk(SRC_DIR):
       for cur_file in files:
         if cur_file.endswith('.pyc'):
           path_to_file = os.path.join(path, cur_file)
           os.remove(path_to_file)
 
   def PerformCrosChrootCleanup(self):
     """Deletes the chroot.
 
     Returns:
       True if successful.
     """
     cwd = os.getcwd()
-    self.ChangeToDepotWorkingDirectory('cros')
+    self.depot_manager.ChangeToDepotDir('cros')
     cmd = [bisect_utils.CROS_SDK_PATH, '--delete']
     return_code = bisect_utils.RunProcess(cmd)
     os.chdir(cwd)
     return not return_code
 
   def CreateCrosChroot(self):
     """Creates a new chroot.
 
     Returns:
       True if successful.
     """
     cwd = os.getcwd()
-    self.ChangeToDepotWorkingDirectory('cros')
+    self.depot_manager.ChangeToDepotDir('cros')
     cmd = [bisect_utils.CROS_SDK_PATH, '--create']
     return_code = bisect_utils.RunProcess(cmd)
     os.chdir(cwd)
     return not return_code
 
   def _PerformPreSyncCleanup(self, depot):
     """Performs any necessary cleanup before syncing.
 
     Args:
       depot: Depot name.
@@ skipping 141 matching lines @@
     """Syncs multiple depots to particular revisions.
 
     Args:
       revisions_to_sync: A list of (depot, revision) pairs to be synced.
       sync_client: Program used to sync, e.g. "gclient", "repo". Can be None.
 
     Returns:
       True if successful, False otherwise.
     """
     for depot, revision in revisions_to_sync:
-      self.ChangeToDepotWorkingDirectory(depot)
+      self.depot_manager.ChangeToDepotDir(depot)
 
       if sync_client:
         self.PerformPreBuildCleanup()
 
       # When using gclient to sync, you need to specify the depot you
       # want so that all the dependencies sync properly as well.
       # i.e. gclient sync src@<SHA1>
       if sync_client == 'gclient':
         revision = '%s@%s' % (DEPOT_DEPS_NAME[depot]['src'], revision)
 
@@ skipping 20 matching lines @@
       dist_to_good_value = abs(current_value['std_dev'] -
           known_good_value['std_dev'])
       dist_to_bad_value = abs(current_value['std_dev'] -
           known_bad_value['std_dev'])
     else:
       dist_to_good_value = abs(current_value['mean'] - known_good_value['mean'])
       dist_to_bad_value = abs(current_value['mean'] - known_bad_value['mean'])
 
     return dist_to_good_value < dist_to_bad_value
 
-  def _GetDepotDirectory(self, depot_name):
-    if depot_name == 'chromium':
-      return SRC_DIR
-    elif depot_name == 'cros':
-      return self.cros_cwd
-    elif depot_name in DEPOT_NAMES:
-      return self.depot_cwd[depot_name]
-    else:
-      assert False, ('Unknown depot [ %s ] encountered. Possibly a new one '
-                     'was added without proper support?' % depot_name)
-
-  def ChangeToDepotWorkingDirectory(self, depot_name):
-    """Given a depot, changes to the appropriate working directory.
-
-    Args:
-      depot_name: The name of the depot (see DEPOT_NAMES).
-    """
-    os.chdir(self._GetDepotDirectory(depot_name))
-
   def _FillInV8BleedingEdgeInfo(self, min_revision_data, max_revision_data):
     r1 = self._GetNearestV8BleedingEdgeFromTrunk(min_revision_data['revision'],
         search_forward=True)
     r2 = self._GetNearestV8BleedingEdgeFromTrunk(max_revision_data['revision'],
         search_forward=False)
     min_revision_data['external']['v8_bleeding_edge'] = r1
     max_revision_data['external']['v8_bleeding_edge'] = r2
 
     if (not self._GetV8BleedingEdgeFromV8TrunkIfMappable(
             min_revision_data['revision'])
@@ skipping 55 matching lines @@
       end_revision: End of the revision range.
       start_revision: Start of the revision range.
       previous_revision: The last revision we synced to on |previous_depot|.
 
     Returns:
       A list containing the revisions between |start_revision| and
       |end_revision| inclusive.
     """
     # Change into working directory of external library to run
     # subsequent commands.
-    self.ChangeToDepotWorkingDirectory(current_depot)
+    self.depot_manager.ChangeToDepotDir(current_depot)
 
     # V8 (and possibly others) is merged in periodically. Bisecting
     # this directory directly won't give much good info.
     if DEPOT_DEPS_NAME[current_depot].has_key('custom_deps'):
       config_path = os.path.join(SRC_DIR, '..')
       if bisect_utils.RunGClientAndCreateConfig(self.opts,
           DEPOT_DEPS_NAME[current_depot]['custom_deps'], cwd=config_path):
         return []
       if bisect_utils.RunGClient(
           ['sync', '--revision', previous_revision], cwd=SRC_DIR):
         return []
 
     if current_depot == 'v8_bleeding_edge':
-      self.ChangeToDepotWorkingDirectory('chromium')
+      self.depot_manager.ChangeToDepotDir('chromium')
 
       shutil.move('v8', 'v8.bak')
       shutil.move('v8_bleeding_edge', 'v8')
 
       self.cleanup_commands.append(['mv', 'v8', 'v8_bleeding_edge'])
       self.cleanup_commands.append(['mv', 'v8.bak', 'v8'])
 
-      self.depot_cwd['v8_bleeding_edge'] = os.path.join(SRC_DIR, 'v8')
-      self.depot_cwd['v8'] = os.path.join(SRC_DIR, 'v8.bak')
+      self.depot_manager.AddDepot(
+          'v8_bleeding_edge', os.path.join(SRC_DIR, 'v8'))
+      self.depot_manager.AddDepot('v8', os.path.join(SRC_DIR, 'v8.bak'))
 
-      self.ChangeToDepotWorkingDirectory(current_depot)
+      self.depot_manager.ChangeToDepotDir(current_depot)
 
     depot_revision_list = self.GetRevisionList(current_depot,
                                                end_revision,
                                                start_revision)
 
-    self.ChangeToDepotWorkingDirectory('chromium')
+    self.depot_manager.ChangeToDepotDir('chromium')
 
     return depot_revision_list
 
   def GatherReferenceValues(self, good_rev, bad_rev, cmd, metric, target_depot):
     """Gathers reference values by running the performance tests on the
     known good and bad revisions.
 
     Args:
       good_rev: The last known good revision where the performance regression
         has not occurred yet.
@@ skipping 86 matching lines @@
 
     Args:
       good_revision: Number/tag of the known good revision.
       bad_revision: Number/tag of the known bad revision.
 
     Returns:
       True if the revisions are in the proper order (good earlier than bad).
     """
     if self.source_control.IsGit() and target_depot != 'cros':
       cmd = ['log', '--format=%ct', '-1', good_revision]
-      cwd = self._GetDepotDirectory(target_depot)
+      cwd = self.depot_manager.GetDepotDir(target_depot)
 
       output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
       good_commit_time = int(output)
 
       cmd = ['log', '--format=%ct', '-1', bad_revision]
       output = bisect_utils.CheckRunGit(cmd, cwd=cwd)
       bad_commit_time = int(output)
 
       return good_commit_time <= bad_commit_time
     else:
@@ skipping 45 matching lines @@
     intermediate revisions to determine the CL where the performance regression
     occurred.
 
     Args:
       command_to_run: Specify the command to execute the performance test.
       good_revision: Number/tag of the known good revision.
       bad_revision: Number/tag of the known bad revision.
       metric: The performance metric to monitor.
 
     Returns:
-      A dict with 2 members, 'revision_data' and 'error'. On success,
-      'revision_data' will contain a dict mapping revision ids to
-      data about that revision. Each piece of revision data consists of a
-      dict with the following keys:
-
-      'passed': Represents whether the performance test was successful at
-          that revision. Possible values include: 1 (passed), 0 (failed),
-          '?' (skipped), 'F' (build failed).
-      'depot': The depot that this revision is from (i.e. WebKit)
-      'external': If the revision is a 'src' revision, 'external' contains
-          the revisions of each of the external libraries.
-      'sort': A sort value for sorting the dict in order of commits.
-
-      For example:
-      {
-        'error':None,
-        'revision_data':
-        {
-          'CL #1':
-          {
-            'passed': False,
-            'depot': 'chromium',
-            'external': None,
-            'sort': 0
-          }
-        }
-      }
-
-      If an error occurred, the 'error' field will contain the message and
-      'revision_data' will be empty.
+      A BisectResults object.
     """
-    results = {
-        'revision_data' : {},
-        'error' : None,
-    }
+    results = BisectResults(self.depot_manager, self.source_control)
 
     # Choose depot to bisect first
     target_depot = 'chromium'
     if self.opts.target_platform == 'cros':
       target_depot = 'cros'
     elif self.opts.target_platform == 'android-chrome':
       target_depot = 'android-chrome'
 
     cwd = os.getcwd()
-    self.ChangeToDepotWorkingDirectory(target_depot)
+    self.depot_manager.ChangeToDepotDir(target_depot)
 
     # If they passed SVN revisions, we can try match them to git SHA1 hashes.
     bad_revision = self.source_control.ResolveToRevision(
         bad_revision_in, target_depot, DEPOT_DEPS_NAME, 100)
     good_revision = self.source_control.ResolveToRevision(
         good_revision_in, target_depot, DEPOT_DEPS_NAME, -100)
 
     os.chdir(cwd)
     if bad_revision is None:
-      results['error'] = 'Couldn\'t resolve [%s] to SHA1.' % bad_revision_in
+      results.error = 'Couldn\'t resolve [%s] to SHA1.' % bad_revision_in
       return results
 
     if good_revision is None:
-      results['error'] = 'Couldn\'t resolve [%s] to SHA1.' % good_revision_in
+      results.error = 'Couldn\'t resolve [%s] to SHA1.' % good_revision_in
       return results
 
     # Check that they didn't accidentally swap good and bad revisions.
     if not self.CheckIfRevisionsInProperOrder(
         target_depot, good_revision, bad_revision):
-      results['error'] = ('bad_revision < good_revision, did you swap these '
-                          'by mistake?')
+      results.error = ('bad_revision < good_revision, did you swap these '
+                       'by mistake?')
       return results
     bad_revision, good_revision = self.NudgeRevisionsIfDEPSChange(
         bad_revision, good_revision, good_revision_in)
     if self.opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepStart('Gathering Revisions')
 
     cannot_bisect = self.CanPerformBisect(good_revision, bad_revision)
     if cannot_bisect:
-      results['error'] = cannot_bisect.get('error')
+      results.error = cannot_bisect.get('error')
       return results
 
     print 'Gathering revision range for bisection.'
     # Retrieve a list of revisions to do bisection on.
     src_revision_list = self.GetRevisionList(
         target_depot, bad_revision, good_revision)
 
     if self.opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
 
     if src_revision_list:
       # revision_data will store information about a revision such as the
       # depot it came from, the webkit/V8 revision at that time,
       # performance timing, build state, etc...
-      revision_data = results['revision_data']
+      revision_data = results.revision_data
 
       # revision_list is the list we're binary searching through at the moment.
       revision_list = []
 
       sort_key_ids = 0
 
       for current_revision_id in src_revision_list:
         sort_key_ids += 1
 
         revision_data[current_revision_id] = {
@@ skipping 22 matching lines @@
       bad_results, good_results = self.GatherReferenceValues(good_revision,
                                                                bad_revision,
                                                                command_to_run,
                                                                metric,
                                                                target_depot)
 
       if self.opts.output_buildbot_annotations:
         bisect_utils.OutputAnnotationStepClosed()
 
       if bad_results[1]:
-        results['error'] = ('An error occurred while building and running '
+        results.error = ('An error occurred while building and running '
             'the \'bad\' reference value. The bisect cannot continue without '
             'a working \'bad\' revision to start from.\n\nError: %s' %
-                bad_results[0])
+            bad_results[0])
         return results
 
       if good_results[1]:
-        results['error'] = ('An error occurred while building and running '
+        results.error = ('An error occurred while building and running '
             'the \'good\' reference value. The bisect cannot continue without '
             'a working \'good\' revision to start from.\n\nError: %s' %
-                good_results[0])
+            good_results[0])
         return results
 
 
       # We need these reference values to determine if later runs should be
       # classified as pass or fail.
       known_bad_value = bad_results[0]
       known_good_value = good_results[0]
 
       # Can just mark the good and bad revisions explicitly here since we
       # already know the results.
@@ skipping 44 matching lines @@
               break
 
             earliest_revision = max_revision_data['external'][external_depot]
             latest_revision = min_revision_data['external'][external_depot]
 
             new_revision_list = self.PrepareToBisectOnDepot(
                 external_depot, latest_revision, earliest_revision,
                 previous_revision)
 
             if not new_revision_list:
-              results['error'] = ('An error occurred attempting to retrieve '
-                                  'revision range: [%s..%s]' %
-                                  (earliest_revision, latest_revision))
+              results.error = ('An error occurred attempting to retrieve '
+                               'revision range: [%s..%s]' %
+                               (earliest_revision, latest_revision))
               return results
 
             _AddRevisionsIntoRevisionData(
                 new_revision_list, external_depot, min_revision_data['sort'],
                 revision_data)
 
             # Reset the bisection and perform it on the newly inserted
             # changelists.
             revision_list = new_revision_list
             min_revision = 0
             max_revision = len(revision_list) - 1
             sort_key_ids += len(revision_list)
 
             print ('Regression in metric %s appears to be the result of '
                    'changes in [%s].' % (metric, external_depot))
 
             self.PrintRevisionsToBisectMessage(revision_list, external_depot)
 
             continue
           else:
             break
         else:
           next_revision_index = (int((max_revision - min_revision) / 2) +
                                  min_revision)
 
         next_revision_id = revision_list[next_revision_index]
         next_revision_data = revision_data[next_revision_id]
         next_revision_depot = next_revision_data['depot']
 
-        self.ChangeToDepotWorkingDirectory(next_revision_depot)
+        self.depot_manager.ChangeToDepotDir(next_revision_depot)
 
         if self.opts.output_buildbot_annotations:
           step_name = 'Working on [%s]' % next_revision_id
           bisect_utils.OutputAnnotationStepStart(step_name)
 
         print 'Working on revision: [%s]' % next_revision_id
 
         run_results = self.RunTest(
             next_revision_id, next_revision_depot, command_to_run, metric,
             skippable=True)
@@ skipping 28 matching lines @@
           # If the build is broken, remove it and redo search.
           revision_list.pop(next_revision_index)
 
           max_revision -= 1
 
         if self.opts.output_buildbot_annotations:
           self._PrintPartialResults(results)
           bisect_utils.OutputAnnotationStepClosed()
     else:
       # Weren't able to sync and retrieve the revision range.
-      results['error'] = ('An error occurred attempting to retrieve revision '
-                          'range: [%s..%s]' % (good_revision, bad_revision))
+      results.error = ('An error occurred attempting to retrieve revision '
+                       'range: [%s..%s]' % (good_revision, bad_revision))
 
     return results
 
-  def _PrintPartialResults(self, results_dict):
-    revision_data = results_dict['revision_data']
-    revision_data_sorted = sorted(revision_data.iteritems(),
-                                  key = lambda x: x[1]['sort'])
-    results_dict = self._GetResultsDict(revision_data, revision_data_sorted)
-
-    self._PrintTestedCommitsTable(revision_data_sorted,
+  def _PrintPartialResults(self, results):
+    results_dict = results.GetResultsDict()
+    self._PrintTestedCommitsTable(results_dict['revision_data_sorted'],
                                   results_dict['first_working_revision'],
                                   results_dict['last_broken_revision'],
                                   100, final_step=False)
 
   def _ConfidenceLevelStatus(self, results_dict):
     if not results_dict['confidence']:
       return None
     confidence_status = 'Successful with %(level)s confidence%(warning)s.'
     if results_dict['confidence'] >= HIGH_CONFIDENCE:
       level = 'high'
     else:
       level = 'low'
     warning = ' and warnings'
     if not self.warnings:
       warning = ''
     return confidence_status % {'level': level, 'warning': warning}
 
   def _GetViewVCLinkFromDepotAndHash(self, cl, depot):
     info = self.source_control.QueryRevisionInfo(cl,
-        self._GetDepotDirectory(depot))
+        self.depot_manager.GetDepotDir(depot))
     if depot and DEPOT_DEPS_NAME[depot].has_key('viewvc'):
       try:
         # Format is "git-svn-id: svn://....@123456 <other data>"
         svn_line = [i for i in info['body'].splitlines() if 'git-svn-id:' in i]
         svn_revision = svn_line[0].split('@')
         svn_revision = svn_revision[1].split(' ')[0]
         return DEPOT_DEPS_NAME[depot]['viewvc'] + svn_revision
       except IndexError:
         return ''
     return ''
@@ skipping 132 matching lines @@
       if not previous_link:
         previous_link = previous_id
 
       print '  %8s  %70s %s' % (
           current_data['depot'], current_link,
           ('%d%%' % confidence).center(10, ' '))
       print '  %8s  %70s' % (
           previous_data['depot'], previous_link)
       print
 
-  def _GetResultsDict(self, revision_data, revision_data_sorted):
-    # Find range where it possibly broke.
-    first_working_revision = None
-    first_working_revision_index = -1
-    last_broken_revision = None
-    last_broken_revision_index = -1
-
-    culprit_revisions = []
-    other_regressions = []
-    regression_size = 0.0
-    regression_std_err = 0.0
-    confidence = 0.0
-
-    for i in xrange(len(revision_data_sorted)):
-      k, v = revision_data_sorted[i]
-      if v['passed'] == 1:
-        if not first_working_revision:
-          first_working_revision = k
-          first_working_revision_index = i
-
-      if not v['passed']:
-        last_broken_revision = k
-        last_broken_revision_index = i
-
-    if last_broken_revision != None and first_working_revision != None:
-      broken_means = []
-      for i in xrange(0, last_broken_revision_index + 1):
-        if revision_data_sorted[i][1]['value']:
-          broken_means.append(revision_data_sorted[i][1]['value']['values'])
-
-      working_means = []
-      for i in xrange(first_working_revision_index, len(revision_data_sorted)):
-        if revision_data_sorted[i][1]['value']:
-          working_means.append(revision_data_sorted[i][1]['value']['values'])
-
-      # Flatten the lists to calculate mean of all values.
-      working_mean = sum(working_means, [])
-      broken_mean = sum(broken_means, [])
-
-      # Calculate the approximate size of the regression
-      mean_of_bad_runs = math_utils.Mean(broken_mean)
-      mean_of_good_runs = math_utils.Mean(working_mean)
-
-      regression_size = 100 * math_utils.RelativeChange(mean_of_good_runs,
-                                                      mean_of_bad_runs)
-      if math.isnan(regression_size):
-        regression_size = 'zero-to-nonzero'
-
-      regression_std_err = math.fabs(math_utils.PooledStandardError(
-          [working_mean, broken_mean]) /
-          max(0.0001, min(mean_of_good_runs, mean_of_bad_runs))) * 100.0
-
-      # Give a "confidence" in the bisect. At the moment we use how distinct the
-      # values are before and after the last broken revision, and how noisy the
-      # overall graph is.
-      confidence = ConfidenceScore(working_means, broken_means)
-
-      culprit_revisions = []
-
-      cwd = os.getcwd()
-      self.ChangeToDepotWorkingDirectory(
-          revision_data[last_broken_revision]['depot'])
-
-      if revision_data[last_broken_revision]['depot'] == 'cros':
-        # Want to get a list of all the commits and what depots they belong
-        # to so that we can grab info about each.
-        cmd = ['repo', 'forall', '-c',
-            'pwd ; git log --pretty=oneline --before=%d --after=%d' % (
-            last_broken_revision, first_working_revision + 1)]
-        output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd)
-
-        changes = []
-        assert not return_code, ('An error occurred while running '
-                                 '"%s"' % ' '.join(cmd))
-        last_depot = None
-        cwd = os.getcwd()
-        for l in output.split('\n'):
-          if l:
-            # Output will be in form:
-            # /path_to_depot
-            # /path_to_other_depot
-            # <SHA1>
-            # /path_again
-            # <SHA1>
-            # etc.
-            if l[0] == '/':
-              last_depot = l
-            else:
-              contents = l.split(' ')
-              if len(contents) > 1:
-                changes.append([last_depot, contents[0]])
-        for c in changes:
-          os.chdir(c[0])
-          info = self.source_control.QueryRevisionInfo(c[1])
-          culprit_revisions.append((c[1], info, None))
-      else:
-        for i in xrange(last_broken_revision_index, len(revision_data_sorted)):
-          k, v = revision_data_sorted[i]
-          if k == first_working_revision:
-            break
-          self.ChangeToDepotWorkingDirectory(v['depot'])
-          info = self.source_control.QueryRevisionInfo(k)
-          culprit_revisions.append((k, info, v['depot']))
-      os.chdir(cwd)
-
-      # Check for any other possible regression ranges.
-      other_regressions = _FindOtherRegressions(
-          revision_data_sorted, mean_of_bad_runs > mean_of_good_runs)
-
-    return {
-        'first_working_revision': first_working_revision,
-        'last_broken_revision': last_broken_revision,
-        'culprit_revisions': culprit_revisions,
-        'other_regressions': other_regressions,
-        'regression_size': regression_size,
-        'regression_std_err': regression_std_err,
-        'confidence': confidence,
-    }
-
   def _CheckForWarnings(self, results_dict):
     if len(results_dict['culprit_revisions']) > 1:
       self.warnings.append('Due to build errors, regression range could '
                            'not be narrowed down to a single commit.')
     if self.opts.repeat_test_count == 1:
       self.warnings.append('Tests were only set to run once. This may '
                            'be insufficient to get meaningful results.')
     if 0 < results_dict['confidence'] < HIGH_CONFIDENCE:
       self.warnings.append('Confidence is not high. Try bisecting again '
                            'with increased repeat_count, larger range, or '
                            'on another metric.')
     if not results_dict['confidence']:
       self.warnings.append('Confidence score is 0%. Try bisecting again on '
                            'another platform or another metric.')
 
   def FormatAndPrintResults(self, bisect_results):
     """Prints the results from a bisection run in a readable format.
 
     Args:
       bisect_results: The results from a bisection test run.
     """
-    revision_data = bisect_results['revision_data']
-    revision_data_sorted = sorted(revision_data.iteritems(),
-                                  key = lambda x: x[1]['sort'])
-    results_dict = self._GetResultsDict(revision_data, revision_data_sorted)
+    results_dict = bisect_results.GetResultsDict()
 
     self._CheckForWarnings(results_dict)
 
     if self.opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepStart('Build Status Per Revision')
 
     print
     print 'Full results of bisection:'
-    for current_id, current_data  in revision_data_sorted:
+    for current_id, current_data  in results_dict['revision_data_sorted']:
       build_status = current_data['passed']
 
       if type(build_status) is bool:
         if build_status:
           build_status = 'Good'
         else:
           build_status = 'Bad'
 
       print '  %20s  %40s  %s' % (current_data['depot'],
                                   current_id, build_status)
     print
 
     if self.opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
       # The perf dashboard scrapes the "results" step in order to comment on
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
 
     self._PrintBanner(results_dict)
     self._PrintWarnings()
 
     if results_dict['culprit_revisions'] and results_dict['confidence']:
       for culprit in results_dict['culprit_revisions']:
         cl, info, depot = culprit
         self._PrintRevisionInfo(cl, info, depot)
       if results_dict['other_regressions']:
         self._PrintOtherRegressions(results_dict['other_regressions'],
-                                    revision_data)
-    self._PrintTestedCommitsTable(revision_data_sorted,
+                                    results_dict['revision_data'])
+    self._PrintTestedCommitsTable(results_dict['revision_data_sorted'],
                                   results_dict['first_working_revision'],
                                   results_dict['last_broken_revision'],
                                   results_dict['confidence'])
-    _PrintStepTime(revision_data_sorted)
+    _PrintStepTime(results_dict['revision_data_sorted'])
     self._PrintReproSteps()
     _PrintThankYou()
     if self.opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
 
   def _PrintBanner(self, results_dict):
     if self._IsBisectModeReturnCode():
       metrics = 'N/A'
       change = 'Yes'
     else:
@@ skipping 390 matching lines @@
     if (not source_control.IsInProperBranch() and
         not opts.debug_ignore_sync and
         not opts.working_directory):
       raise RuntimeError('You must switch to master branch to run bisection.')
     bisect_test = BisectPerformanceMetrics(source_control, opts)
     try:
       bisect_results = bisect_test.Run(opts.command,
                                        opts.bad_revision,
                                        opts.good_revision,
                                        opts.metric)
-      if bisect_results['error']:
-        raise RuntimeError(bisect_results['error'])
+      if bisect_results.error:
+        raise RuntimeError(bisect_results.error)
       bisect_test.FormatAndPrintResults(bisect_results)
       return 0
     finally:
       bisect_test.PerformCleanup()
   except RuntimeError, e:
     if opts.output_buildbot_annotations:
       # The perf dashboard scrapes the "results" step in order to comment on
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())