Remove Skia's forked buildbot code

slave/skia_slave_scripts/build_step.py

-# Copyright (c) 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Base class for all slave-side build steps. """
-
-import config
-# pylint: disable=W0611
-import flavor_utils
-import imp
-import multiprocessing
-import os
-import shlex
-import signal
-import subprocess
-import sys
-import time
-import traceback
-
-from playback_dirs import LocalSkpPlaybackDirs
-from playback_dirs import StorageSkpPlaybackDirs
-
-BUILDBOT_PATH = os.path.realpath(os.path.join(
-    os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir))
-
-# Add important directories to the PYTHONPATH
-sys.path.append(os.path.join(BUILDBOT_PATH))
-sys.path.append(os.path.join(BUILDBOT_PATH, 'site_config'))
-sys.path.append(os.path.join(BUILDBOT_PATH, 'master'))
-sys.path.insert(0, os.path.join(BUILDBOT_PATH, 'common'))
-
-import builder_name_schema
-import slave_hosts_cfg
-import slaves_cfg
-
-from py.utils import misc
-
-
-DEFAULT_TIMEOUT = 4800
-DEFAULT_NO_OUTPUT_TIMEOUT = 3600
-DEFAULT_NUM_CORES = 2
-
-
-GM_EXPECTATIONS_FILENAME = 'expected-results.json'
-GM_IGNORE_FAILURES_FILE = 'ignored-tests.txt'
-
-
-# multiprocessing.Value doesn't accept boolean types, so we have to use an int.
-INT_TRUE = 1
-INT_FALSE = 0
-build_step_stdout_has_written = multiprocessing.Value('i', INT_FALSE)
-
-
-class BuildStepWarning(Exception):
-  pass
-
-
-class BuildStepFailure(Exception):
-  pass
-
-
-class BuildStepTimeout(Exception):
-  pass
-
-
-class BuildStepLogger(object):
-  """ Override stdout so that we can keep track of when anything has been
-  logged.  This enables timeouts based on how long the process has gone without
-  writing output.
-  """
-  def __init__(self):
-    self.stdout = sys.stdout
-    sys.stdout = self
-    build_step_stdout_has_written.value = INT_FALSE
-
-  def __del__(self):
-    sys.stdout = self.stdout
-
-  def fileno(self):
-    return self.stdout.fileno()
-
-  def write(self, data):
-    build_step_stdout_has_written.value = INT_TRUE
-    self.stdout.write(data)
-
-  def flush(self):
-    self.stdout.flush()
-
-
-def _GetBuildSlaveID(desired_slave_name):
-  """ Returns the index of the build slave in the list of build slaves running
-  on this machine, in the form of a string. """
-  for host_dict in slave_hosts_cfg.SLAVE_HOSTS.itervalues():
-    for slave_name, slave_id, _ in host_dict.slaves:
-      if slave_name == desired_slave_name:
-        return slave_id
-  raise Exception('No build slave found with name %s' % desired_slave_name)
-
-
-class BuildStep(multiprocessing.Process):
-
-  def __init__(self, args, attempts=1, timeout=DEFAULT_TIMEOUT,
-               no_output_timeout=DEFAULT_NO_OUTPUT_TIMEOUT):
-    """ Constructs a BuildStep instance.
-
-    args: dictionary containing arguments to this BuildStep.
-    attempts: how many times to try this BuildStep before giving up.
-    timeout: maximum time allowed for this BuildStep.
-    no_output_timeout: maximum time allowed for this BuildStep to run without
-        any output.
-    """
-    multiprocessing.Process.__init__(self)
-
-    self._args = dict(args)
-
-    self.timeout = timeout
-    self.no_output_timeout = no_output_timeout
-    self.attempts = attempts
-
-    self._builder_name = args['builder_name']
-    self._build_number = args['build_number']
-    self._slavename = os.environ['TESTING_SLAVENAME']
-
-    # Change to the correct working directory. This is needed on Windows, where
-    # our path lengths would otherwise be too long.
-    if os.name == 'nt':
-      curdir = os.getcwd()
-      # The buildslave name and builder name combo is too long.  Instead, obtain
-      # a unique buildslave ID.
-      buildslave_id = _GetBuildSlaveID(self._slavename)
-      workdir = os.path.join('C:\\', buildslave_id, self._builder_name,
-                             curdir[curdir.rfind('build'):])
-      print 'chdir to %s' % workdir
-      if not os.path.isdir(workdir):
-        os.makedirs(workdir)
-      os.chdir(workdir)
-
-    # Add CWD to the PYTHONPATH
-    sys.path.append(os.getcwd())
-
-    self._configuration = args['configuration']
-    if os.name == 'nt' and 'x86_64' in self._builder_name:
-      self._configuration += '_x64'
-
-    self._target_platform = args['target_platform']
-    self._deps_target_os = \
-        None if args['deps_target_os'] == 'None' else args['deps_target_os']
-    self._revision = \
-        None if args['revision'] == 'None' or args['revision'] == 'HEAD' \
-        else args['revision']
-    self._got_revision = \
-        None if args['got_revision'] == 'None' else args['got_revision']
-
-    # Import the flavor-specific build step utils module.
-    flavor = args.get('flavor', 'default')
-    try:
-      flavor_utils_module_name = '%s_build_step_utils' % flavor
-      flavor_utils_path = os.path.join(os.path.dirname(__file__),
-                                       'flavor_utils',
-                                       '%s.py' % flavor_utils_module_name)
-      flavor_utils_module = imp.load_source(
-          'flavor_utils.%s' % flavor_utils_module_name, flavor_utils_path)
-      flavor_utils_class_name = ''.join([part.title() for part in
-                                         flavor.split('_')])
-      flavor_utils_class = getattr(flavor_utils_module,
-                                   '%sBuildStepUtils' % flavor_utils_class_name)
-      self._flavor_utils = flavor_utils_class(self)
-    except (ImportError, IOError) as e:
-      raise Exception('Unrecognized build flavor: %s\n%s' % (flavor, e))
-
-    # Trybots should use expectations from the corresponding waterfall bot.
-    # This fixes https://code.google.com/p/skia/issues/detail?id=1552
-    gm_expected_subdir = builder_name_schema.GetWaterfallBot(
-        self._builder_name)
-
-    # Figure out where we are going to store images generated by GM.
-    self._gm_actual_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'actual')
-    self._gm_expected_dir = os.path.join('expectations', 'gm',
-                                         gm_expected_subdir)
-    self._gm_actual_dir = os.path.join(self._gm_actual_basedir,
-                                       self._builder_name)
-    self._dm_dir = os.path.join(os.pardir, os.pardir, 'dm')
-
-    self._resource_dir = 'resources'
-    self._make_flags = shlex.split(args['make_flags'].replace('"', ''))
-    self._test_args = shlex.split(args['test_args'].replace('"', ''))
-    self._gm_args = shlex.split(args['gm_args'].replace('"', ''))
-    self._bench_args = shlex.split(args['bench_args'].replace('"', ''))
-    self._is_try = args['is_try'] == 'True'
-
-    self._default_make_flags = []
-    self._default_ninja_flags = []
-
-    # TODO(epoger): Throughout the buildbot code, we use various terms to refer
-    # to the same thing: "skps", "pictures", "replay", "playback".
-    # We should pick one of those terms, and rename things so that we are
-    # consistent.
-    # See https://codereview.chromium.org/295753002/ for additional discussion.
-
-    # Adding the playback directory transfer objects.
-    self._local_playback_dirs = LocalSkpPlaybackDirs(
-        self._builder_name,
-        None if args['perf_output_basedir'] == 'None'
-            else args['perf_output_basedir'])
-    self._storage_playback_dirs = StorageSkpPlaybackDirs(
-        self._builder_name,
-        None if args['perf_output_basedir'] == 'None'
-            else args['perf_output_basedir'])
-
-    self.skp_dir = self._local_playback_dirs.PlaybackSkpDir()
-    self.playback_actual_images_dir = (
-        self._local_playback_dirs.PlaybackActualImagesDir())
-    self.playback_actual_summaries_dir = (
-        self._local_playback_dirs.PlaybackActualSummariesDir())
-    self.playback_expected_summaries_dir = (
-        self._local_playback_dirs.PlaybackExpectedSummariesDir())
-
-    # Figure out where we are going to store performance related data.
-    if args['perf_output_basedir'] != 'None':
-      self._perf_data_dir = os.path.join(args['perf_output_basedir'],
-                                         self._builder_name, 'data')
-      self._perf_graphs_dir = os.path.join(args['perf_output_basedir'],
-                                           self._builder_name, 'graphs')
-      self._perf_range_input_dir = os.path.join(
-          args['perf_output_basedir'], self._builder_name, 'expectations')
-    else:
-      self._perf_data_dir = None
-      self._perf_graphs_dir = None
-      self._perf_range_input_dir = None
-    self._skimage_in_dir = os.path.join(os.pardir, 'skimage_in')
-
-    self._skimage_expected_dir = os.path.join('expectations', 'skimage')
-
-    self._skimage_out_dir = os.path.join('out', self._configuration,
-                                         'skimage_out')
-
-    self._device_dirs = self._flavor_utils.GetDeviceDirs()
-
-  @property
-  def configuration(self):
-    return self._configuration
-
-  @property
-  def builder_name(self):
-    return self._builder_name
-
-  @property
-  def args(self):
-    return self._args
-
-  # TODO(epoger): remove default_make_flags property once all builds use ninja
-  @property
-  def default_make_flags(self):
-    return self._default_make_flags
-
-  @property
-  def default_ninja_flags(self):
-    return self._default_ninja_flags
-
-  @property
-  def make_flags(self):
-    return self._make_flags
-
-  @property
-  def perf_data_dir(self):
-    return self._perf_data_dir
-
-  @property
-  def resource_dir(self):
-    return self._resource_dir
-
-  @property
-  def skimage_in_dir(self):
-    return self._skimage_in_dir
-
-  @property
-  def skimage_expected_dir(self):
-    return self._skimage_expected_dir
-
-  @property
-  def skimage_out_dir(self):
-    return self._skimage_out_dir
-
-  @property
-  def local_playback_dirs(self):
-    return self._local_playback_dirs
-
-  def _PreRun(self):
-    """ Optional preprocessing step defined in the BuildStepUtils. """
-    self._flavor_utils.PreRun()
-
-  def _Run(self):
-    """ Code to be run in a given BuildStep.  No return value; throws exception
-    on failure.  Override this method in subclasses.
-    """
-    raise Exception('Cannot instantiate abstract BuildStep')
-
-  def run(self):
-    """ Internal method used by multiprocess.Process. _Run is provided to be
-    overridden instead of this method to ensure that this implementation always
-    runs.
-    """
-    # If a BuildStep has exceeded its allotted time, the parent process needs to
-    # be able to kill the BuildStep process AND any which it has spawned,
-    # without harming itself. On posix platforms, the terminate() method is
-    # insufficient; it fails to kill the subprocesses launched by this process.
-    # So, we use use the setpgrp() function to set a new process group for the
-    # BuildStep process and its children and call os.killpg() to kill the group.
-    if os.name == 'posix':
-      os.setpgrp()
-    try:
-      self._Run()
-    except BuildStepWarning as e:
-      print e
-      sys.exit(config.Master.retcode_warnings)
-
-  def _WaitFunc(self, attempt):
-    """ Waits a number of seconds depending upon the attempt number of a
-    retry-able BuildStep before making the next attempt.  This can be overridden
-    by subclasses and should be defined for attempt in [0, self.attempts - 1]
-
-    This default implementation is exponential; we double the wait time with
-    each attempt, starting with a 15-second pause between the first and second
-    attempts.
-    """
-    base_secs = 15
-    wait = base_secs * (2 ** attempt)
-    print 'Retrying in %d seconds...' % wait
-    time.sleep(wait)
-
-  @staticmethod
-  def KillBuildStep(step):
-    """ Kills a running BuildStep.
-
-    step: the running BuildStep instance to kill.
-    """
-    # On posix platforms, the terminate() method is insufficient; it fails to
-    # kill the subprocesses launched by this process. So, we use use the
-    # setpgrp() function to set a new process group for the BuildStep process
-    # and its children and call os.killpg() to kill the group.
-    if os.name == 'posix':
-      os.killpg(os.getpgid(step.pid), signal.SIGTERM)
-    elif os.name == 'nt':
-      subprocess.call(['taskkill', '/F', '/T', '/PID', str(step.pid)])
-    else:
-      step.terminate()
-
-  @staticmethod
-  def RunBuildStep(StepType):
-    """ Run a BuildStep, possibly making multiple attempts and handling
-    timeouts.
-
-    StepType: class type which subclasses BuildStep, indicating what step should
-        be run. StepType should override _Run().
-    """
-    # pylint: disable=W0612
-    logger = BuildStepLogger()
-    args = misc.ArgsToDict(sys.argv)
-    attempt = 0
-    while True:
-      step = StepType(args=args)
-      try:
-        start_time = time.time()
-        last_written_time = start_time
-        # pylint: disable=W0212
-        step._PreRun()
-        step.start()
-        while step.is_alive():
-          current_time = time.time()
-          if current_time - start_time > step.timeout:
-            BuildStep.KillBuildStep(step)
-            raise BuildStepTimeout('Build step exceeded timeout of %d seconds' %
-                                   step.timeout)
-          elif current_time - last_written_time > step.no_output_timeout:
-            BuildStep.KillBuildStep(step)
-            raise BuildStepTimeout(
-                'Build step exceeded %d seconds with no output' %
-                step.no_output_timeout)
-          time.sleep(1)
-          if build_step_stdout_has_written.value == INT_TRUE:
-            last_written_time = time.time()
-        print 'Build Step Finished.'
-        if step.exitcode == 0:
-          return 0
-        elif step.exitcode == config.Master.retcode_warnings:
-          # A warning is considered to be an acceptable finishing state.
-          return config.Master.retcode_warnings
-        else:
-          raise BuildStepFailure('Build step failed.')
-      except Exception:
-        print traceback.format_exc()
-        if attempt + 1 >= step.attempts:
-          raise
-      # pylint: disable=W0212
-      step._WaitFunc(attempt)
-      attempt += 1
-      print '**** %s, attempt %d ****' % (StepType.__name__, attempt + 1)