Fixes for Swarming recipes

infra/bots/common.py

 #!/usr/bin/env python

[borenet, 2016/03/30 14:02:09]

This file can be deleted entirely once download_skps.py and download_images.py
are deleted (next CL).

 #
 # Copyright 2016 Google Inc.
 #
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
-import contextlib
-import glob
-import math
 import os
-import psutil
 import shutil
-import socket
 import subprocess
-import sys
-import time
-import urllib2
-
-from flavor import android_flavor
-from flavor import chromeos_flavor
-from flavor import cmake_flavor
-from flavor import coverage_flavor
-from flavor import default_flavor
-from flavor import ios_flavor
-from flavor import valgrind_flavor
-from flavor import xsan_flavor
 
 
-CONFIG_COVERAGE = 'Coverage'
-CONFIG_DEBUG = 'Debug'
-CONFIG_RELEASE = 'Release'
-VALID_CONFIGS = (CONFIG_COVERAGE, CONFIG_DEBUG, CONFIG_RELEASE)
-
-BUILD_PRODUCTS_WHITELIST = [
-  'dm',
-  'dm.exe',
-  'nanobench',
-  'nanobench.exe',
-  '*.so',
-  '*.dll',
-]
-
-GM_ACTUAL_FILENAME = 'actual-results.json'
-GM_EXPECTATIONS_FILENAME = 'expected-results.json'
-GM_IGNORE_TESTS_FILENAME = 'ignored-tests.txt'
-
-GOLD_UNINTERESTING_HASHES_URL = 'https://gold.skia.org/_/hashes'
-
 GS_GM_BUCKET = 'chromium-skia-gm'
-GS_SUMMARIES_BUCKET = 'chromium-skia-gm-summaries'
 
 GS_SUBDIR_TMPL_SK_IMAGE = 'skimage/v%s'
 GS_SUBDIR_TMPL_SKP = 'playback_%s/skps'
 
-SKIA_REPO = 'https://skia.googlesource.com/skia.git'
-INFRA_REPO = 'https://skia.googlesource.com/buildbot.git'
-
-SERVICE_ACCOUNT_FILE = 'service-account-skia.json'
-SERVICE_ACCOUNT_INTERNAL_FILE = 'service-account-skia-internal.json'
-
 VERSION_FILE_SK_IMAGE = 'SK_IMAGE_VERSION'
 VERSION_FILE_SKP = 'SKP_VERSION'
 
 
-def is_android(bot_cfg):
-  """Determine whether the given bot is an Android bot."""
-  return ('Android' in bot_cfg.get('extra_config', '') or
-          bot_cfg.get('os') == 'Android')
-
-def is_chromeos(bot_cfg):
-  return ('CrOS' in bot_cfg.get('extra_config', '') or
-          bot_cfg.get('os') == 'ChromeOS')
-
-def is_cmake(bot_cfg):
-  return 'CMake' in bot_cfg.get('extra_config', '')
-
-def is_ios(bot_cfg):
-  return ('iOS' in bot_cfg.get('extra_config', '') or
-          bot_cfg.get('os') == 'iOS')
-
-
-def is_valgrind(bot_cfg):
-  return 'Valgrind' in bot_cfg.get('extra_config', '')
-
-
-def is_xsan(bot_cfg):
-  return (bot_cfg.get('extra_config') == 'ASAN' or
-          bot_cfg.get('extra_config') == 'MSAN' or
-          bot_cfg.get('extra_config') == 'TSAN')
-
-
 def download_dir(skia_dir, tmp_dir, version_file, gs_path_tmpl, dst_dir):
   # Ensure that the tmp_dir exists.
   if not os.path.isdir(tmp_dir):
     os.makedirs(tmp_dir)
 
   # Get the expected version.
   with open(os.path.join(skia_dir, version_file)) as f:
     expected_version = f.read().rstrip()
 
   print 'Expected %s = %s' % (version_file, expected_version)
@@ skipping 13 matching lines @@
     if actual_version != -1:
       os.remove(actual_version_file)
     if os.path.isdir(dst_dir):
       shutil.rmtree(dst_dir)
     os.makedirs(dst_dir)
     gs_path = 'gs://%s/%s/*' % (GS_GM_BUCKET, gs_path_tmpl % expected_version)
     print 'Downloading from %s' % gs_path
     subprocess.check_call(['gsutil', 'cp', '-R', gs_path, dst_dir])
     with open(actual_version_file, 'w') as f:
       f.write(expected_version)
-
-
-def get_uninteresting_hashes(hashes_file):
-  retries = 5
-  timeout = 60
-  wait_base = 15
-
-  socket.setdefaulttimeout(timeout)
-  for retry in range(retries):
-    try:
-      with contextlib.closing(
-          urllib2.urlopen(GOLD_UNINTERESTING_HASHES_URL, timeout=timeout)) as w:
-        hashes = w.read()
-        with open(hashes_file, 'w') as f:
-          f.write(hashes)
-          break
-    except Exception as e:
-      print >> sys.stderr, 'Failed to get uninteresting hashes from %s:\n%s' % (
-          GOLD_UNINTERESTING_HASHES_URL, e)
-      if retry == retries:
-        raise
-      waittime = wait_base * math.pow(2, retry)
-      print 'Retry in %d seconds.' % waittime
-      time.sleep(waittime)
-
-
-class BotInfo(object):
-  def __init__(self, bot_name, swarm_out_dir):
-    """Initialize the bot, given its name.
-
-    Assumes that CWD is the directory containing this file.
-    """
-    self.name = bot_name
-    self.skia_dir = os.path.abspath(os.path.join(
-        os.path.dirname(os.path.realpath(__file__)),
-        os.pardir, os.pardir))
-    self.swarm_out_dir = swarm_out_dir
-    os.chdir(self.skia_dir)
-    self.build_dir = os.path.abspath(os.path.join(self.skia_dir, os.pardir))
-    self.infrabots_dir = os.path.join(self.skia_dir, 'infra', 'bots')
-    self.home_dir = os.path.expanduser('~')
-
-    self.spec = self.get_bot_spec(bot_name)
-    self.bot_cfg = self.spec['builder_cfg']
-    self.out_dir = os.path.join(os.pardir, 'out')
-    self.configuration = self.spec['configuration']
-    self.default_env = {
-      'CHROME_HEADLESS': '1',
-      'SKIA_OUT': self.out_dir,
-      'BUILDTYPE': self.configuration,
-      'PATH': os.environ['PATH'],
-    }
-    if 'Win' in self.bot_cfg['os']:
-      self.default_env['SystemRoot'] = 'C:\\Windows'
-      self.default_env['TEMP'] = os.path.join(
-          self.home_dir, 'AppData', 'Local', 'Temp')
-      self.default_env['TMP'] = self.default_env['TEMP']
-    self.default_env.update(self.spec['env'])
-    self.build_targets = [str(t) for t in self.spec['build_targets']]
-    self.is_trybot = self.bot_cfg['is_trybot']
-    self.upload_dm_results = self.spec['upload_dm_results']
-    self.upload_perf_results = self.spec['upload_perf_results']
-    self.perf_data_dir = os.path.join(self.swarm_out_dir, 'perfdata',
-                                      self.name, 'data')
-    self.resource_dir = os.path.join(self.skia_dir, 'resources')
-    self.images_dir = os.path.join(self.build_dir, 'images')
-    self.local_skp_dir = os.path.join(self.build_dir, 'playback', 'skps')
-    self.dm_flags = self.spec['dm_flags']
-    self.nanobench_flags = self.spec['nanobench_flags']
-    self._ccache = None
-    self._checked_for_ccache = False
-    self._already_ran = {}
-    self.tmp_dir = os.path.join(self.build_dir, 'tmp')
-    self.flavor = self.get_flavor(self.bot_cfg)
-
-    # These get filled in during subsequent steps.
-    self.device_dirs = None
-    self.build_number = None
-    self.got_revision = None
-    self.master_name = None
-    self.slave_name = None
-
-  @property
-  def ccache(self):
-    if not self._checked_for_ccache:
-      self._checked_for_ccache = True
-      if sys.platform != 'win32':
-        try:
-          result = subprocess.check_output(['which', 'ccache'])
-          self._ccache = result.rstrip()
-        except subprocess.CalledProcessError:
-          pass
-
-    return self._ccache
-
-  def get_bot_spec(self, bot_name):
-    """Retrieve the bot spec for this bot."""
-    sys.path.append(self.skia_dir)
-    from tools import buildbot_spec
-    return buildbot_spec.get_builder_spec(bot_name)
-
-  def get_flavor(self, bot_cfg):
-    """Return a flavor utils object specific to the given bot."""
-    if is_android(bot_cfg):
-      return android_flavor.AndroidFlavorUtils(self)
-    elif is_chromeos(bot_cfg):
-      return chromeos_flavor.ChromeOSFlavorUtils(self)
-    elif is_cmake(bot_cfg):
-      return cmake_flavor.CMakeFlavorUtils(self)
-    elif is_ios(bot_cfg):
-      return ios_flavor.iOSFlavorUtils(self)
-    elif is_valgrind(bot_cfg):
-      return valgrind_flavor.ValgrindFlavorUtils(self)
-    elif is_xsan(bot_cfg):
-      return xsan_flavor.XSanFlavorUtils(self)
-    elif bot_cfg.get('configuration') == CONFIG_COVERAGE:
-      return coverage_flavor.CoverageFlavorUtils(self)
-    else:
-      return default_flavor.DefaultFlavorUtils(self)
-
-  def run(self, cmd, env=None, cwd=None):
-    _env = {}
-    _env.update(self.default_env)
-    _env.update(env or {})
-    cwd = cwd or self.skia_dir
-    print '============'
-    print 'CMD: %s' % cmd
-    print 'CWD: %s' % cwd
-    print 'ENV: %s' % _env
-    print '============'
-    subprocess.check_call(cmd, env=_env, cwd=cwd)
-
-  def compile_steps(self):
-    for t in self.build_targets:
-      self.flavor.compile(t)
-    dst = os.path.join(self.swarm_out_dir, 'out', self.configuration)
-    os.makedirs(dst)
-    for pattern in BUILD_PRODUCTS_WHITELIST:
-      path = os.path.join(self.out_dir, self.configuration, pattern)
-      for f in glob.glob(path):
-          print 'Copying build product %s' % f
-          shutil.copy(f, dst)
-    self.cleanup()
-
-  def _run_once(self, fn, *args, **kwargs):
-    if not fn.__name__ in self._already_ran:
-      self._already_ran[fn.__name__] = True
-      fn(*args, **kwargs)
-
-  def install(self):
-    """Copy the required executables and files to the device."""
-    self.device_dirs = self.flavor.get_device_dirs()
-
-    # Run any device-specific installation.
-    self.flavor.install()
-
-    # TODO(borenet): Only copy files which have changed.
-    # Resources
-    self.flavor.copy_directory_contents_to_device(self.resource_dir,
-                                                  self.device_dirs.resource_dir)
-
-  def _key_params(self):
-    """Build a unique key from the builder name (as a list).
-
-    E.g.  arch x86 gpu GeForce320M mode MacMini4.1 os Mac10.6
-    """
-    # Don't bother to include role, which is always Test.
-    # TryBots are uploaded elsewhere so they can use the same key.
-    blacklist = ['role', 'is_trybot']
-
-    flat = []
-    for k in sorted(self.bot_cfg.keys()):
-      if k not in blacklist:
-        flat.append(k)
-        flat.append(self.bot_cfg[k])
-    return flat
-
-  def test_steps(self, got_revision, master_name, slave_name, build_number,
-                 issue=None, patchset=None):
-    """Run the DM test."""
-    self.build_number = build_number
-    self.got_revision = got_revision
-    self.master_name = master_name
-    self.slave_name = slave_name
-    self._run_once(self.install)
-
-    use_hash_file = False
-    if self.upload_dm_results:
-      # This must run before we write anything into self.device_dirs.dm_dir
-      # or we may end up deleting our output on machines where they're the same.
-      host_dm_dir = os.path.join(self.swarm_out_dir, 'dm')
-      print 'host dm dir: %s' % host_dm_dir
-      self.flavor.create_clean_host_dir(host_dm_dir)
-      if str(host_dm_dir) != str(self.device_dirs.dm_dir):
-        self.flavor.create_clean_device_dir(self.device_dirs.dm_dir)
-
-      # Obtain the list of already-generated hashes.
-      if not os.path.isdir(self.tmp_dir):
-        os.makedirs(self.tmp_dir)
-      hash_filename = 'uninteresting_hashes.txt'
-      host_hashes_file = os.path.join(self.tmp_dir, hash_filename)
-      hashes_file = self.flavor.device_path_join(
-          self.device_dirs.tmp_dir, hash_filename)
-
-      try:
-        get_uninteresting_hashes(host_hashes_file)
-      except Exception:
-        pass
-
-      if os.path.exists(host_hashes_file):
-        self.flavor.copy_file_to_device(host_hashes_file, hashes_file)
-        use_hash_file = True
-
-    # Run DM.
-    properties = [
-      'gitHash',      self.got_revision,
-      'master',       self.master_name,
-      'builder',      self.name,
-      'build_number', self.build_number,
-    ]
-    if self.is_trybot:
-      if not issue:
-        raise Exception('issue is required for trybots.')
-      if not patchset:
-        raise Exception('patchset is required for trybots.')
-      properties.extend([
-        'issue',    issue,
-        'patchset', patchset,
-      ])
-
-    args = [
-      'dm',
-      '--undefok',   # This helps branches that may not know new flags.
-      '--resourcePath', self.device_dirs.resource_dir,
-      '--skps', self.device_dirs.skp_dir,
-      '--images', self.flavor.device_path_join(
-          self.device_dirs.images_dir, 'dm'),
-      '--nameByHash',
-      '--properties'
-    ] + properties
-
-    args.append('--key')
-    args.extend(self._key_params())
-    if use_hash_file:
-      args.extend(['--uninterestingHashesFile', hashes_file])
-    if self.upload_dm_results:
-      args.extend(['--writePath', self.device_dirs.dm_dir])
-
-    skip_flag = None
-    if self.bot_cfg.get('cpu_or_gpu') == 'CPU':
-      skip_flag = '--nogpu'
-    elif self.bot_cfg.get('cpu_or_gpu') == 'GPU':
-      skip_flag = '--nocpu'
-    if skip_flag:
-      args.append(skip_flag)
-    args.extend(self.dm_flags)
-
-    self.flavor.run(args, env=self.default_env)
-
-    if self.upload_dm_results:
-      # Copy images and JSON to host machine if needed.
-      self.flavor.copy_directory_contents_to_host(self.device_dirs.dm_dir,
-                                                  host_dm_dir)
-
-    # See skia:2789.
-    if ('Valgrind' in self.name and
-        self.bot_cfg.get('cpu_or_gpu') == 'GPU'):
-      abandonGpuContext = list(args)
-      abandonGpuContext.append('--abandonGpuContext')
-      self.flavor.run(abandonGpuContext)
-      preAbandonGpuContext = list(args)
-      preAbandonGpuContext.append('--preAbandonGpuContext')
-      self.flavor.run(preAbandonGpuContext)
-
-    self.cleanup()
-
-  def perf_steps(self, got_revision, master_name, slave_name, build_number,
-                 issue=None, patchset=None):
-    """Run Skia benchmarks."""
-    self.build_number = build_number
-    self.got_revision = got_revision
-    self.master_name = master_name
-    self.slave_name = slave_name
-    self._run_once(self.install)
-    if self.upload_perf_results:
-      self.flavor.create_clean_device_dir(self.device_dirs.perf_data_dir)
-
-    # Run nanobench.
-    properties = [
-      '--properties',
-      'gitHash',      self.got_revision,
-      'build_number', self.build_number,
-    ]
-    if self.is_trybot:
-      if not issue:
-        raise Exception('issue is required for trybots.')
-      if not patchset:
-        raise Exception('patchset is required for trybots.')
-      properties.extend([
-        'issue',    issue,
-        'patchset', patchset,
-      ])
-
-    target = 'nanobench'
-    if 'VisualBench' in self.name:
-      target = 'visualbench'
-    args = [
-        target,
-        '--undefok',   # This helps branches that may not know new flags.
-        '-i',       self.device_dirs.resource_dir,
-        '--skps',   self.device_dirs.skp_dir,
-        '--images', self.flavor.device_path_join(
-            self.device_dirs.images_dir, 'dm'),  # Using DM images for now.
-    ]
-
-    skip_flag = None
-    if self.bot_cfg.get('cpu_or_gpu') == 'CPU':
-      skip_flag = '--nogpu'
-    elif self.bot_cfg.get('cpu_or_gpu') == 'GPU':
-      skip_flag = '--nocpu'
-    if skip_flag:
-      args.append(skip_flag)
-    args.extend(self.nanobench_flags)
-
-    if self.upload_perf_results:
-      json_path = self.flavor.device_path_join(
-          self.device_dirs.perf_data_dir,
-          'nanobench_%s.json' % self.got_revision)
-      args.extend(['--outResultsFile', json_path])
-      args.extend(properties)
-
-      keys_blacklist = ['configuration', 'role', 'is_trybot']
-      args.append('--key')
-      for k in sorted(self.bot_cfg.keys()):
-        if not k in keys_blacklist:
-          args.extend([k, self.bot_cfg[k]])
-
-    self.flavor.run(args, env=self.default_env)
-
-    # See skia:2789.
-    if ('Valgrind' in self.name and
-        self.bot_cfg.get('cpu_or_gpu') == 'GPU'):
-      abandonGpuContext = list(args)
-      abandonGpuContext.extend(['--abandonGpuContext', '--nocpu'])
-      self.flavor.run(abandonGpuContext, env=self.default_env)
-
-    # Copy results to host.
-    if self.upload_perf_results:
-      if not os.path.exists(self.perf_data_dir):
-        os.makedirs(self.perf_data_dir)
-      self.flavor.copy_directory_contents_to_host(
-          self.device_dirs.perf_data_dir, self.perf_data_dir)
-
-    self.cleanup()
-
-  def cleanup(self):
-    if sys.platform == 'win32':
-      # Kill mspdbsrv.exe, which tends to hang around after the build finishes.
-      for p in psutil.process_iter():
-        try:
-          if p.name == 'mspdbsrv.exe':
-            p.kill()
-        except psutil._error.AccessDenied:
-          pass
-    self.flavor.cleanup_steps()