Extract Builder and subclasses to separate module.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 34 matching lines @@
 import shlex
 import shutil
 import StringIO
 import sys
 import time
 import zipfile
 
 sys.path.append(os.path.join(os.path.dirname(__file__), 'telemetry'))
 
 from auto_bisect import bisect_utils
+from auto_bisect import builder
 from auto_bisect import math_utils
 from auto_bisect import post_perf_builder_job as bisect_builder
 from auto_bisect import source_control as source_control_module
 from auto_bisect import ttest
 from telemetry.util import cloud_storage
 
 # Below is the map of "depot" names to information about each depot. Each depot
 # is a repository, and in the process of bisecting, revision ranges in these
 # repositories may also be bisected.
 #
@@ skipping 79 matching lines @@
         'svn': 'http://skia.googlecode.com/svn/trunk/gyp',
         'depends': None,
         'from': ['chromium'],
         'viewvc': 'https://code.google.com/p/skia/source/detail?r=',
         'deps_var': 'None'
     }
 }
 
 DEPOT_NAMES = DEPOT_DEPS_NAME.keys()
 
-CROS_SDK_PATH = os.path.join('..', 'cros', 'chromite', 'bin', 'cros_sdk')
 CROS_CHROMEOS_PATTERN = 'chromeos-base/chromeos-chrome'
-CROS_TEST_KEY_PATH = os.path.join('..', 'cros', 'chromite', 'ssh_keys',
-                                  'testing_rsa')
-CROS_SCRIPT_KEY_PATH = os.path.join('..', 'cros', 'src', 'scripts',
-                                    'mod_for_test_scripts', 'ssh_keys',
-                                    'testing_rsa')
 
 # Possible return values from BisectPerformanceMetrics.SyncBuildAndRunRevision.
 BUILD_RESULT_SUCCEED = 0
 BUILD_RESULT_FAIL = 1
 BUILD_RESULT_SKIPPED = 2
 
 # Maximum time in seconds to wait after posting build request to tryserver.
 # TODO: Change these values based on the actual time taken by buildbots on
 # the tryserver.
 MAX_MAC_BUILD_TIME = 14400
@@ skipping 266 matching lines @@
     for name in zf.namelist():
       if verbose:
         print 'Extracting %s' % name
       zf.extract(name, output_dir)
       if bisect_utils.IsMacHost():
         # Restore permission bits.
         os.chmod(os.path.join(output_dir, name),
                  zf.getinfo(name).external_attr >> 16L)
 
 
-def SetBuildSystemDefault(build_system, use_goma, goma_dir):
-  """Sets up any environment variables needed to build with the specified build
-  system.
-
-  Args:
-    build_system: A string specifying build system. Currently only 'ninja' or
-        'make' are supported.
-  """
-  if build_system == 'ninja':
-    gyp_var = os.getenv('GYP_GENERATORS', default='')
-
-    if not gyp_var or not 'ninja' in gyp_var:
-      if gyp_var:
-        os.environ['GYP_GENERATORS'] = gyp_var + ',ninja'
-      else:
-        os.environ['GYP_GENERATORS'] = 'ninja'
-
-      if bisect_utils.IsWindowsHost():
-        os.environ['GYP_DEFINES'] = ('component=shared_library '
-                                     'incremental_chrome_dll=1 '
-                                     'disable_nacl=1 fastbuild=1 '
-                                     'chromium_win_pch=0')
-
-  elif build_system == 'make':
-    os.environ['GYP_GENERATORS'] = 'make'
-  else:
-    raise RuntimeError('%s build not supported.' % build_system)
-
-  if use_goma:
-    os.environ['GYP_DEFINES'] = '%s %s' % (os.getenv('GYP_DEFINES', default=''),
-                                           'use_goma=1')
-    if goma_dir:
-      os.environ['GYP_DEFINES'] += ' gomadir=%s' % goma_dir
-
-
-def BuildWithMake(threads, targets, build_type='Release'):
-  cmd = ['make', 'BUILDTYPE=%s' % build_type]
-
-  if threads:
-    cmd.append('-j%d' % threads)
-
-  cmd += targets
-
-  return_code = bisect_utils.RunProcess(cmd)
-
-  return not return_code
-
-
-def BuildWithNinja(threads, targets, build_type='Release'):
-  cmd = ['ninja', '-C', os.path.join('out', build_type)]
-
-  if threads:
-    cmd.append('-j%d' % threads)
-
-  cmd += targets
-
-  return_code = bisect_utils.RunProcess(cmd)
-
-  return not return_code
-
-
-def BuildWithVisualStudio(targets, build_type='Release'):
-  path_to_devenv = os.path.abspath(
-      os.path.join(os.environ['VS100COMNTOOLS'], '..', 'IDE', 'devenv.com'))
-  path_to_sln = os.path.join(os.getcwd(), 'chrome', 'chrome.sln')
-  cmd = [path_to_devenv, '/build', build_type, path_to_sln]
-
-  for t in targets:
-    cmd.extend(['/Project', t])
-
-  return_code = bisect_utils.RunProcess(cmd)
-
-  return not return_code
-
-
 def WriteStringToFile(text, file_name):
   try:
     with open(file_name, 'wb') as f:
       f.write(text)
   except IOError:
     raise RuntimeError('Error writing to file [%s]' % file_name )
 
 
 def ReadStringFromFile(file_name):
   try:
     with open(file_name) as f:
       return f.read()
   except IOError:
     raise RuntimeError('Error reading file [%s]' % file_name )
 
 
 def ChangeBackslashToSlashInPatch(diff_text):
   """Formats file paths in the given text to unix-style paths."""
   if diff_text:
     diff_lines = diff_text.split('\n')
     for i in range(len(diff_lines)):
       if (diff_lines[i].startswith('--- ') or
           diff_lines[i].startswith('+++ ')):
         diff_lines[i] = diff_lines[i].replace('\\', '/')
     return '\n'.join(diff_lines)
   return None
 
 
-class Builder(object):
-  """Builder is used by the bisect script to build relevant targets and deploy.
-  """
-  def __init__(self, opts):
-    """Performs setup for building with target build system.
-
-    Args:
-        opts: Options parsed from command line.
-    """
-    if bisect_utils.IsWindowsHost():
-      if not opts.build_preference:
-        opts.build_preference = 'msvs'
-
-      if opts.build_preference == 'msvs':
-        if not os.getenv('VS100COMNTOOLS'):
-          raise RuntimeError(
-              'Path to visual studio could not be determined.')
-      else:
-        SetBuildSystemDefault(opts.build_preference, opts.use_goma,
-                              opts.goma_dir)
-    else:
-      if not opts.build_preference:
-        if 'ninja' in os.getenv('GYP_GENERATORS', default=''):
-          opts.build_preference = 'ninja'
-        else:
-          opts.build_preference = 'make'
-
-      SetBuildSystemDefault(opts.build_preference, opts.use_goma, opts.goma_dir)
-
-    if not bisect_utils.SetupPlatformBuildEnvironment(opts):
-      raise RuntimeError('Failed to set platform environment.')
-
-  @staticmethod
-  def FromOpts(opts):
-    builder = None
-    if opts.target_platform == 'cros':
-      builder = CrosBuilder(opts)
-    elif opts.target_platform == 'android':
-      builder = AndroidBuilder(opts)
-    elif opts.target_platform == 'android-chrome':
-      builder = AndroidChromeBuilder(opts)
-    else:
-      builder = DesktopBuilder(opts)
-    return builder
-
-  def Build(self, depot, opts):
-    raise NotImplementedError()
-
-  def GetBuildOutputDirectory(self, opts, src_dir=None):
-    """Returns the path to the build directory, relative to the checkout root.
-
-      Assumes that the current working directory is the checkout root.
-    """
-    src_dir = src_dir or 'src'
-    if opts.build_preference == 'ninja' or bisect_utils.IsLinuxHost():
-      return os.path.join(src_dir, 'out')
-    if bisect_utils.IsMacHost():
-      return os.path.join(src_dir, 'xcodebuild')
-    if bisect_utils.IsWindowsHost():
-      return os.path.join(src_dir, 'build')
-    raise NotImplementedError('Unexpected platform %s' % sys.platform)
-
-
-class DesktopBuilder(Builder):
-  """DesktopBuilder is used to build Chromium on linux/mac/windows."""
-  def __init__(self, opts):
-    super(DesktopBuilder, self).__init__(opts)
-
-  def Build(self, depot, opts):
-    """Builds chromium_builder_perf target using options passed into
-    the script.
-
-    Args:
-        depot: Current depot being bisected.
-        opts: The options parsed from the command line.
-
-    Returns:
-        True if build was successful.
-    """
-    targets = ['chromium_builder_perf']
-
-    threads = None
-    if opts.use_goma:
-      threads = 64
-
-    build_success = False
-    if opts.build_preference == 'make':
-      build_success = BuildWithMake(threads, targets, opts.target_build_type)
-    elif opts.build_preference == 'ninja':
-      build_success = BuildWithNinja(threads, targets, opts.target_build_type)
-    elif opts.build_preference == 'msvs':
-      assert bisect_utils.IsWindowsHost(), 'msvs is only supported on Windows.'
-      build_success = BuildWithVisualStudio(targets, opts.target_build_type)
-    else:
-      assert False, 'No build system defined.'
-    return build_success
-
-
-class AndroidBuilder(Builder):
-  """AndroidBuilder is used to build on android."""
-  def __init__(self, opts):
-    super(AndroidBuilder, self).__init__(opts)
-
-  def _GetTargets(self):
-    return ['chrome_shell_apk', 'cc_perftests_apk', 'android_tools']
-
-  def Build(self, depot, opts):
-    """Builds the android content shell and other necessary tools using options
-    passed into the script.
-
-    Args:
-        depot: Current depot being bisected.
-        opts: The options parsed from the command line.
-
-    Returns:
-        True if build was successful.
-    """
-    threads = None
-    if opts.use_goma:
-      threads = 64
-
-    build_success = False
-    if opts.build_preference == 'ninja':
-      build_success = BuildWithNinja(
-          threads, self._GetTargets(), opts.target_build_type)
-    else:
-      assert False, 'No build system defined.'
-
-    return build_success
-
-
-class AndroidChromeBuilder(AndroidBuilder):
-  """AndroidBuilder is used to build on android's chrome."""
-  def __init__(self, opts):
-    super(AndroidChromeBuilder, self).__init__(opts)
-
-  def _GetTargets(self):
-    return AndroidBuilder._GetTargets(self) + ['chrome_apk']
-
-
-class CrosBuilder(Builder):
-  """CrosBuilder is used to build and image ChromeOS/Chromium when cros is the
-  target platform."""
-  def __init__(self, opts):
-    super(CrosBuilder, self).__init__(opts)
-
-  def ImageToTarget(self, opts):
-    """Installs latest image to target specified by opts.cros_remote_ip.
-
-    Args:
-        opts: Program options containing cros_board and cros_remote_ip.
-
-    Returns:
-        True if successful.
-    """
-    try:
-      # Keys will most likely be set to 0640 after wiping the chroot.
-      os.chmod(CROS_SCRIPT_KEY_PATH, 0600)
-      os.chmod(CROS_TEST_KEY_PATH, 0600)
-      cmd = [CROS_SDK_PATH, '--', './bin/cros_image_to_target.py',
-             '--remote=%s' % opts.cros_remote_ip,
-             '--board=%s' % opts.cros_board, '--test', '--verbose']
-
-      return_code = bisect_utils.RunProcess(cmd)
-      return not return_code
-    except OSError:
-      return False
-
-  def BuildPackages(self, opts, depot):
-    """Builds packages for cros.
-
-    Args:
-        opts: Program options containing cros_board.
-        depot: The depot being bisected.
-
-    Returns:
-        True if successful.
-    """
-    cmd = [CROS_SDK_PATH]
-
-    if depot != 'cros':
-      path_to_chrome = os.path.join(os.getcwd(), '..')
-      cmd += ['--chrome_root=%s' % path_to_chrome]
-
-    cmd += ['--']
-
-    if depot != 'cros':
-      cmd += ['CHROME_ORIGIN=LOCAL_SOURCE']
-
-    cmd += ['BUILDTYPE=%s' % opts.target_build_type, './build_packages',
-        '--board=%s' % opts.cros_board]
-    return_code = bisect_utils.RunProcess(cmd)
-
-    return not return_code
-
-  def BuildImage(self, opts, depot):
-    """Builds test image for cros.
-
-    Args:
-        opts: Program options containing cros_board.
-        depot: The depot being bisected.
-
-    Returns:
-        True if successful.
-    """
-    cmd = [CROS_SDK_PATH]
-
-    if depot != 'cros':
-      path_to_chrome = os.path.join(os.getcwd(), '..')
-      cmd += ['--chrome_root=%s' % path_to_chrome]
-
-    cmd += ['--']
-
-    if depot != 'cros':
-      cmd += ['CHROME_ORIGIN=LOCAL_SOURCE']
-
-    cmd += ['BUILDTYPE=%s' % opts.target_build_type, '--', './build_image',
-        '--board=%s' % opts.cros_board, 'test']
-
-    return_code = bisect_utils.RunProcess(cmd)
-
-    return not return_code
-
-  def Build(self, depot, opts):
-    """Builds targets using options passed into the script.
-
-    Args:
-        depot: Current depot being bisected.
-        opts: The options parsed from the command line.
-
-    Returns:
-        True if build was successful.
-    """
-    if self.BuildPackages(opts, depot):
-      if self.BuildImage(opts, depot):
-        return self.ImageToTarget(opts)
-    return False
-
-
 def _ParseRevisionsFromDEPSFileManually(deps_file_contents):
   """Parses the vars section of the DEPS file with regex.
 
   Args:
     deps_file_contents: The DEPS file contents as a string.
 
   Returns:
     A dict in the format {depot:revision} if successful, otherwise None.
   """
   # We'll parse the "vars" section of the DEPS file.
@@ skipping 365 matching lines @@
       step_count += 1
   if step_count:
     step_perf_time_avg = step_perf_time_avg / step_count
     step_build_time_avg = step_build_time_avg / step_count
   print
   print 'Average build time : %s' % datetime.timedelta(
       seconds=int(step_build_time_avg))
   print 'Average test time  : %s' % datetime.timedelta(
       seconds=int(step_perf_time_avg))
 
+
 def _FindOtherRegressions(revision_data_sorted, bad_greater_than_good):
   """Compiles a list of other possible regressions from the revision data.
 
   Args:
     revision_data_sorted: Sorted list of (revision, revision data dict) pairs.
     bad_greater_than_good: Whether the result value at the "bad" revision is
         numerically greater than the result value at the "good" revision.
 
   Returns:
     A list of [current_rev, previous_rev, confidence] for other places where
@@ skipping 19 matching lines @@
         is_same_direction = (prev_less_than_current if
             bad_greater_than_good else not prev_less_than_current)
 
         # Only report potential regressions with high confidence.
         if is_same_direction and confidence > 50:
           other_regressions.append([current_id, previous_id, confidence])
       previous_values.append(current_values)
       previous_id = current_id
   return other_regressions
 
+
 class BisectPerformanceMetrics(object):
   """This class contains functionality to perform a bisection of a range of
   revisions to narrow down where performance regressions may have occurred.
 
   The main entry-point is the Run method.
   """
 
   def __init__(self, source_control, opts):
     super(BisectPerformanceMetrics, self).__init__()
 
     self.opts = opts
     self.source_control = source_control
     self.src_cwd = os.getcwd()
     self.cros_cwd = os.path.join(os.getcwd(), '..', 'cros')
     self.depot_cwd = {}
     self.cleanup_commands = []
     self.warnings = []
-    self.builder = Builder.FromOpts(opts)
+    self.builder = builder.Builder.FromOpts(opts)
 
     # This always starts true since the script grabs latest first.
     self.was_blink = True
 
     for d in DEPOT_NAMES:
       # The working directory of each depot is just the path to the depot, but
       # since we're already in 'src', we can skip that part.
 
       self.depot_cwd[d] = os.path.join(
           self.src_cwd, DEPOT_DEPS_NAME[d]['src'][4:])
@@ skipping 192 matching lines @@
       A dict in the format {depot:revision} if successful, otherwise None.
     """
     cwd = os.getcwd()
     self.ChangeToDepotWorkingDirectory(depot)
 
     results = {}
 
     if depot == 'chromium' or depot == 'android-chrome':
       results = self._ParseRevisionsFromDEPSFile(depot)
       os.chdir(cwd)
-    elif depot == 'cros':
-      cmd = [CROS_SDK_PATH, '--', 'portageq-%s' % self.opts.cros_board,
-             'best_visible', '/build/%s' % self.opts.cros_board, 'ebuild',
-             CROS_CHROMEOS_PATTERN]
+
+    if depot == 'cros':
+      cmd = [
+          bisect_utils.CROS_SDK_PATH,
+          '--',
+          'portageq-%s' % self.opts.cros_board,
+          'best_visible',
+          '/build/%s' % self.opts.cros_board,
+          'ebuild',
+          CROS_CHROMEOS_PATTERN
+      ]
       output, return_code = bisect_utils.RunProcessAndRetrieveOutput(cmd)
 
       assert not return_code, ('An error occurred while running '
                                '"%s"' % ' '.join(cmd))
 
       if len(output) > CROS_CHROMEOS_PATTERN:
         output = output[len(CROS_CHROMEOS_PATTERN):]
 
       if len(output) > 1:
         output = output.split('_')[0]
@@ skipping 11 matching lines @@
 
           cwd = os.getcwd()
           self.ChangeToDepotWorkingDirectory('chromium')
           cmd = ['log', '-1', '--format=%H',
                  '--author=chrome-release@google.com',
                  '--grep=to %s' % version, 'origin/master']
           return_code = bisect_utils.CheckRunGit(cmd)
           os.chdir(cwd)
 
           results['chromium'] = output.strip()
-    elif depot == 'v8':
+
+    if depot == 'v8':
       # We can't try to map the trunk revision to bleeding edge yet, because
       # we don't know which direction to try to search in. Have to wait until
       # the bisect has narrowed the results down to 2 v8 rolls.
       results['v8_bleeding_edge'] = None
 
     return results
 
   def BackupOrRestoreOutputdirectory(self, restore=False, build_type='Release'):
     """Backs up or restores build output directory based on restore argument.
 
     Args:
       restore: Indicates whether to restore or backup. Default is False(Backup)
       build_type: Target build type ('Release', 'Debug', 'Release_x64' etc.)
 
     Returns:
       Path to backup or restored location as string. otherwise None if it fails.
     """
     build_dir = os.path.abspath(
-        self.builder.GetBuildOutputDirectory(self.opts, self.src_cwd))
+        builder.GetBuildOutputDirectory(self.opts, self.src_cwd))
     source_dir = os.path.join(build_dir, build_type)
     destination_dir = os.path.join(build_dir, '%s.bak' % build_type)
     if restore:
       source_dir, destination_dir = destination_dir, source_dir
     if os.path.exists(source_dir):
       RmTreeAndMkDir(destination_dir, skip_makedir=True)
       shutil.move(source_dir, destination_dir)
       return destination_dir
     return None
 
@@ skipping 43 matching lines @@
     if patch:
       # Get the SHA of the DEPS changes patch.
       patch_sha = GetSHA1HexDigest(patch)
 
       # Update the DEPS changes patch with a patch to create a new file named
       # 'DEPS.sha' and add patch_sha evaluated above to it.
       patch = '%s\n%s' % (patch, DEPS_SHA_PATCH % {'deps_sha': patch_sha})
 
     # Get Build output directory
     abs_build_dir = os.path.abspath(
-        self.builder.GetBuildOutputDirectory(self.opts, self.src_cwd))
+        builder.GetBuildOutputDirectory(self.opts, self.src_cwd))
 
     fetch_build_func = lambda: self.GetBuildArchiveForRevision(
       revision, self.opts.gs_bucket, self.opts.target_arch,
       patch_sha, abs_build_dir)
 
     # Downloaded archive file path, downloads build archive for given revision.
     downloaded_file = fetch_build_func()
 
     # When build archive doesn't exists, post a build request to tryserver
     # and wait for the build to be produced.
@@ skipping 337 matching lines @@
 
     if not _GenerateProfileIfNecessary(args):
       err_text = 'Failed to generate profile for performance test.'
       return (err_text, failure_code)
 
     # If running a Telemetry test for Chrome OS, insert the remote IP and
     # identity parameters.
     is_telemetry = bisect_utils.IsTelemetryCommand(command_to_run)
     if self.opts.target_platform == 'cros' and is_telemetry:
       args.append('--remote=%s' % self.opts.cros_remote_ip)
-      args.append('--identity=%s' % CROS_TEST_KEY_PATH)
+      args.append('--identity=%s' % bisect_utils.CROS_TEST_KEY_PATH)
 
     start_time = time.time()
 
     metric_values = []
     output_of_all_runs = ''
     for i in xrange(self.opts.repeat_test_count):
       # Can ignore the return code since if the tests fail, it won't return 0.
       current_args = copy.copy(args)
       if is_telemetry:
         if i == 0 and reset_on_first_run:
@@ skipping 173 matching lines @@
     return True
 
   def PerformCrosChrootCleanup(self):
     """Deletes the chroot.
 
     Returns:
         True if successful.
     """
     cwd = os.getcwd()
     self.ChangeToDepotWorkingDirectory('cros')
-    cmd = [CROS_SDK_PATH, '--delete']
+    cmd = [bisect_utils.CROS_SDK_PATH, '--delete']
     return_code = bisect_utils.RunProcess(cmd)
     os.chdir(cwd)
     return not return_code
 
   def CreateCrosChroot(self):
     """Creates a new chroot.
 
     Returns:
         True if successful.
     """
     cwd = os.getcwd()
     self.ChangeToDepotWorkingDirectory('cros')
-    cmd = [CROS_SDK_PATH, '--create']
+    cmd = [bisect_utils.CROS_SDK_PATH, '--create']
     return_code = bisect_utils.RunProcess(cmd)
     os.chdir(cwd)
     return not return_code
 
   def PerformPreSyncCleanup(self, revision, depot):
     """Performs any necessary cleanup before syncing.
 
     Returns:
       True if successful.
     """
@@ skipping 1528 matching lines @@
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())