Remove Skia's forked buildbot code

master/skia_master_scripts/utils.py

Index: master/skia_master_scripts/utils.py
diff --git a/master/skia_master_scripts/utils.py b/master/skia_master_scripts/utils.py
deleted file mode 100644
index 55fed8205e0ca85a0d8cf5ac232b63b4cb1754e4..0000000000000000000000000000000000000000
--- a/master/skia_master_scripts/utils.py
+++ /dev/null
@@ -1,529 +0,0 @@
-# Copyright (c) 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-"""Miscellaneous utilities needed by the Skia buildbot master."""
-
-
-import difflib
-import httplib2
-import json
-import os
-import re
-
-# requires Google APIs client library for Python; see
-# https://code.google.com/p/google-api-python-client/wiki/Installation
-from apiclient.discovery import build
-from buildbot.scheduler import Dependent
-from buildbot.scheduler import Scheduler
-from buildbot.schedulers import timed
-from buildbot.schedulers.filter import ChangeFilter
-from config_private import TRY_SVN_BASEURL
-from master import try_job_svn
-from master import try_job_rietveld
-from master.builders_pools import BuildersPools
-from oauth2client.client import SignedJwtAssertionCredentials
-
-import builder_name_schema
-import config_private
-import os
-import skia_vars
-import subprocess
-
-
-GATEKEEPER_NAME = 'GateKeeper'
-
-TRY_SCHEDULER_SVN = 'skia_try_svn'
-TRY_SCHEDULER_RIETVELD = 'skia_try_rietveld'
-TRY_SCHEDULERS = [TRY_SCHEDULER_SVN, TRY_SCHEDULER_RIETVELD]
-TRY_SCHEDULERS_STR = '|'.join(TRY_SCHEDULERS)
-
-
-def GetListFromEnvVar(name, splitstring=','):
-  """ Returns contents of an environment variable, as a list.
-
-  If the environment variable is unset or set to empty-string, this returns
-  an empty list.
-
-  name: string; name of the environment variable to read
-  splitstring: string with which to split the env var into list items
-  """
-  unsplit = os.environ.get(name, None)
-  if unsplit:
-    return unsplit.split(',')
-  else:
-    return []
-
-
-def StringDiff(expected, actual):
-  """ Returns the diff between two multiline strings, as a multiline string."""
-  return ''.join(difflib.unified_diff(expected.splitlines(1),
-                                      actual.splitlines(1)))
-
-
-def ToString(obj):
-  """ Returns a string representation of the given object. This differs from the
-  built-in string function in that it does not give memory locations.
-
-  obj: the object to print.
-  """
-  def sanitize(obj):
-    if isinstance(obj, list) or isinstance(obj, tuple):
-      return [sanitize(sub_obj) for sub_obj in obj]
-    elif isinstance(obj, dict):
-      rv = {}
-      for k, v in obj.iteritems():
-        rv[k] = sanitize(v)
-      return rv
-    elif isinstance(obj, str) or obj is None:
-      return obj
-    else:
-      return '<Object>'
-  return json.dumps(sanitize(obj), indent=4, sort_keys=True)
-
-
-def FixGitSvnEmail(addr):
-  """ Git-svn tacks a git-svn-id onto email addresses. This function removes it.
-
-  For example, "skia.buildbots@gmail.com@2bbb7eff-a529-9590-31e7-b0007b416f81"
-  becomes, "skia.buildbots@gmail.com". Addresses containing a single '@' will be
-  unchanged.
-  """
-  return '@'.join(addr.split('@')[:2])
-
-
-class SkiaChangeFilter(ChangeFilter):
-  """Skia specific subclass of ChangeFilter."""
-
-  def __init__(self, builders, **kwargs):
-    self._builders = builders
-    ChangeFilter.__init__(self, **kwargs)
-
-  def filter_change(self, change):
-    """Overrides ChangeFilter.filter_change to pass builders to filter_fn.
-
-    The code has been copied from
-    http://buildbot.net/buildbot/docs/0.8.3/reference/buildbot.schedulers.filter-pysrc.html#ChangeFilter
-    with one change: We pass a sequence of builders to the filter function.
-    """
-    if self.filter_fn is not None and not self.filter_fn(change,
-                                                         self._builders):
-      return False
-    for (filt_list, filt_re, filt_fn, chg_attr) in self.checks:
-      chg_val = getattr(change, chg_attr, '')
-      if filt_list is not None and chg_val not in filt_list:
-        return False
-      if filt_re is not None and (
-          chg_val is None or not filt_re.match(chg_val)):
-        return False
-      if filt_fn is not None and not filt_fn(chg_val):
-        return False
-    return True
-
-
-def _AssertValidString(var, varName='[unknown]'):
-  """Raises an exception if a var is not a valid string.
-  
-  A string is considered valid if it is not None, is not the empty string and is
-  not just whitespace.
-
-  Args:
-    var: the variable to validate
-    varName: name of the variable, for error reporting
-  """
-  if not isinstance(var, str):
-    raise Exception('variable "%s" is not a string' % varName)
-  if not var:
-    raise Exception('variable "%s" is empty' % varName)
-  if var.isspace():
-    raise Exception('variable "%s" is whitespace' % varName)
-
-
-def _AssertValidStringList(var, varName='[unknown]'):
-  """Raises an exception if var is not a list of valid strings.
-  
-  A list is considered valid if it is either empty or if it contains at
-  least one item and each item it contains is also a valid string.
-
-  Args:
-    var: the variable to validate
-    varName: name of the variable, for error reporting
-  """
-  if not isinstance(var, list):
-    raise Exception('variable "%s" is not a list' % varName)
-  for index, item in zip(range(len(var)), var):
-    _AssertValidString(item, '%s[%d]' % (varName, index))
-
-
-def FileBug(summary, description, owner=None, ccs=None, labels=None):
-  """Files a bug to the Skia issue tracker.
-
-  Args:
-    summary: a single-line string to use as the issue summary
-    description: a multiline string to use as the issue description
-    owner: email address of the issue owner (as a string), or None if unknown
-    ccs: email addresses (list of strings) to CC on the bug
-    labels: labels (list of strings) to apply to the bug
-
-  Returns: 
-    A representation of the issue tracker issue that was filed or raises an
-    exception if there was a problem.
-  """
-  project_id = 'skia' # This is the project name: skia
-  key_file = 'key.p12' # Key file from the API console, renamed to key.p12
-  service_acct = ('352371350305-b3u8jq5sotdh964othi9ntg9d0pelu77'
-                  '@developer.gserviceaccount.com') # Created with the key
-  result = {}
-  if not ccs:
-    ccs = []
-  if not labels:
-    labels = []
-  
-  if owner is not None:  # owner can be None
-    _AssertValidString(owner, 'owner')
-  _AssertValidString(summary, 'summary')
-  _AssertValidString(description, 'description')
-  _AssertValidStringList(ccs, 'ccs')
-  _AssertValidStringList(labels, 'labels')
-
-  f = file(key_file, 'rb')
-  key = f.read()
-  f.close()
-
-  # Create an httplib2.Http object to handle the HTTP requests and authorize
-  # it with the credentials.
-  credentials = SignedJwtAssertionCredentials(
-      service_acct,
-      key,
-      scope='https://www.googleapis.com/auth/projecthosting')
-  http = httplib2.Http()
-  http = credentials.authorize(http)
-
-  service = build("projecthosting", "v2", http=http)
-
-  # Insert a new issue into the project.
-  body = {
-    'summary': summary,
-    'description': description
-  }
-  
-  insertparams = {
-    'projectId': project_id,
-    'sendEmail': 'true'
-  }
-
-  if owner is not None:
-    owner_value = {
-      'name': owner
-    }
-    body['owner'] = owner_value
-
-  cc_values = []
-  for cc in ccs:
-    cc_values.append({'name': cc})  
-  body['cc'] = cc_values
-  
-  body['labels'] = labels
-
-  insertparams['body'] = body
-   
-  request = service.issues().insert(**insertparams)
-  result = request.execute()
-
-  return result
-
-
-# Skip buildbot runs of a CL if its commit log message contains the following
-# substring.
-SKIP_BUILDBOT_SUBSTRING = '(SkipBuildbotRuns)'
-
-# If the below regex is found in a CL's commit log message, only run the
-# builders specified therein.
-RUN_BUILDERS_REGEX = '\(RunBuilders:(.+)\)'
-RUN_BUILDERS_RE_COMPILED = re.compile(RUN_BUILDERS_REGEX)
-
-
-def CapWordsToUnderscores(string):
-  """ Converts a string containing capitalized words to one in which all
-  characters are lowercase and words are separated by underscores.
-
-  Example:
-    'Nexus10' becomes 'nexus_10'
-
-  string: string; string to manipulate.
-  """
-  name_parts = []
-  for part in re.split('(\d+)', string):
-    if re.match('(\d+)', part):
-      name_parts.append(part)
-    else:
-      name_parts.extend(re.findall('[A-Z][a-z]*', part))
-  return '_'.join([part.lower() for part in name_parts])
-
-
-def UnderscoresToCapWords(string):
-  """ Converts a string lowercase words separated by underscores to one in which
-  words are capitalized and not separated by underscores.
-
-  Example:
-    'nexus_10' becomes 'Nexus10'
-
-  string: string; string to manipulate.
-  """
-  name_parts = string.split('_')
-  return ''.join([part.title() for part in name_parts])
-
-
-# Since we can't modify the existing Helper class, we subclass it here,
-# overriding the necessary parts to get things working as we want.
-class SkiaHelper(object):
-  def __init__(self, defaults):
-    self._defaults = defaults
-    self._builders = []
-    self._factories = {}
-    self._schedulers = {}
-
-  def Builder(self, name, factory, gatekeeper=None, scheduler=None,
-              builddir=None, auto_reboot=False, notify_on_missing=False):
-    # Override the category with the first two parts of the builder name.
-    name_parts = name.split(builder_name_schema.BUILDER_NAME_SEP)
-    category = name_parts[0]
-    subcategory = name_parts[1] if len(name_parts) > 1 else 'default'
-    full_category = '|'.join((category, subcategory))
-    self._builders.append({'name': name,
-                           'factory': factory,
-                           'gatekeeper': gatekeeper,
-                           'schedulers': scheduler.split('|'),
-                           'builddir': builddir,
-                           'category': full_category,
-                           'auto_reboot': auto_reboot,
-                           'notify_on_missing': notify_on_missing})
-
-  def PeriodicScheduler(self, name, minute=0, hour='*', dayOfMonth='*',
-                        month='*', dayOfWeek='*'):
-    """Helper method for the Periodic scheduler."""
-    if name in self._schedulers:
-      raise ValueError('Scheduler %s already exists' % name)
-    self._schedulers[name] = {'type': 'PeriodicScheduler',
-                              'builders': [],
-                              'minute': minute,
-                              'hour': hour,
-                              'dayOfMonth': dayOfMonth,
-                              'month': month,
-                              'dayOfWeek': dayOfWeek}
-
-  def Dependent(self, name, parent):
-    if name in self._schedulers:
-      raise ValueError('Scheduler %s already exists' % name)
-    self._schedulers[name] = {'type': 'Dependent',
-                              'parent': parent,
-                              'builders': []}
-
-  def Factory(self, name, factory):
-    if name in self._factories:
-      raise ValueError('Factory %s already exists' % name)
-    self._factories[name] = factory
-
-  def Scheduler(self, name, treeStableTimer=60, categories=None):
-    if name in self._schedulers:
-      raise ValueError('Scheduler %s already exists' % name)
-    self._schedulers[name] = {'type': 'Scheduler',
-                              'treeStableTimer': treeStableTimer,
-                              'builders': [],
-                              'categories': categories}
-
-  def TryJobSubversion(self, name):
-    """ Adds a Subversion-based try scheduler. """
-    if name in self._schedulers:
-      raise ValueError('Scheduler %s already exists' % name)
-    self._schedulers[name] = {'type': 'TryJobSubversion', 'builders': []}
-
-  def TryJobRietveld(self, name):
-    """ Adds a Rietveld-based try scheduler. """
-    if name in self._schedulers:
-      raise ValueError('Scheduler %s already exists' % name)
-    self._schedulers[name] = {'type': 'TryJobRietveld', 'builders': []}
-
-  def Update(self, c):
-    for builder in self._builders:
-      # Update the schedulers with the builder.
-      schedulers = builder['schedulers']
-      if schedulers:
-        for scheduler in schedulers:
-          self._schedulers[scheduler]['builders'].append(builder['name'])
-
-      # Construct the category.
-      categories = []
-      if builder.get('category', None):
-        categories.append(builder['category'])
-      if builder.get('gatekeeper', None):
-        categories.extend(builder['gatekeeper'].split('|'))
-      category = '|'.join(categories)
-
-      # Append the builder to the list.
-      new_builder = {'name': builder['name'],
-                     'factory': self._factories[builder['factory']],
-                     'category': category,
-                     'auto_reboot': builder['auto_reboot']}
-      if builder['builddir']:
-        new_builder['builddir'] = builder['builddir']
-      c['builders'].append(new_builder)
-
-    c['builders'].sort(key=lambda builder: builder['name'])
-
-    # Process the main schedulers.
-    for s_name in self._schedulers:
-      scheduler = self._schedulers[s_name]
-      if scheduler['type'] == 'Scheduler':
-        def filter_fn(change, builders):
-          """Filters out if change.comments contains certain keywords.
-
-          The change is filtered out if the commit message contains:
-          * SKIP_BUILDBOT_SUBSTRING or
-          * RUN_BUILDERS_REGEX when the scheduler does not contain any of the
-            specified builders
-
-          Args:
-            change: An instance of changes.Change.
-            builders: Sequence of strings. The builders that are run by this
-              scheduler.
-
-          Returns:
-            If the change should be filtered out (i.e. not run by the buildbot
-            code) then False is returned else True is returned.
-          """
-          if SKIP_BUILDBOT_SUBSTRING in change.comments:
-            return False
-          match_obj = RUN_BUILDERS_RE_COMPILED.search(change.comments)
-          if builders and match_obj:
-            for builder_to_run in match_obj.group(1).split(','):
-              if builder_to_run.strip() in builders:
-                break
-            else:
-              return False
-          return True
-
-        skia_change_filter = SkiaChangeFilter(
-            builders=scheduler['builders'],
-            branch=skia_vars.GetGlobalVariable('master_branch_name'),
-            filter_fn=filter_fn)
-
-        instance = Scheduler(name=s_name,
-                             treeStableTimer=scheduler['treeStableTimer'],
-                             builderNames=scheduler['builders'],
-                             change_filter=skia_change_filter)
-        c['schedulers'].append(instance)
-        self._schedulers[s_name]['instance'] = instance
-
-    # Process the periodic schedulers.
-    for s_name in self._schedulers:
-      scheduler = self._schedulers[s_name]
-      if scheduler['type'] == 'PeriodicScheduler':
-        instance = timed.Nightly(
-            name=s_name,
-            branch=skia_vars.GetGlobalVariable('master_branch_name'),
-            builderNames=scheduler['builders'],
-            minute=scheduler['minute'],
-            hour=scheduler['hour'],
-            dayOfMonth=scheduler['dayOfMonth'],
-            month=scheduler['month'],
-            dayOfWeek=scheduler['dayOfWeek'])
-        c['schedulers'].append(instance)
-        self._schedulers[s_name]['instance'] = instance
-
-    # Process the Rietveld-based try schedulers.
-    for s_name in self._schedulers:
-      scheduler = self._schedulers[s_name]
-      if scheduler['type'] == 'TryJobRietveld':
-        pools = BuildersPools(s_name)
-        pools[s_name].extend(scheduler['builders'])
-        instance = try_job_rietveld.TryJobRietveld(
-            name=s_name,
-            pools=pools,
-            last_good_urls={'skia': None},
-            code_review_sites={'skia': config_private.CODE_REVIEW_SITE},
-            project='skia')
-        c['schedulers'].append(instance)
-        self._schedulers[s_name]['instance'] = instance
-
-    # Process the svn-based try schedulers.
-    for s_name in self._schedulers:
-      scheduler = self._schedulers[s_name]
-      if scheduler['type'] == 'TryJobSubversion':
-        pools = BuildersPools(s_name)
-        pools[s_name].extend(scheduler['builders'])
-        instance = try_job_svn.TryJobSubversion(
-            name=s_name,
-            svn_url=TRY_SVN_BASEURL,
-            last_good_urls={'skia': None},
-            code_review_sites={'skia': config_private.CODE_REVIEW_SITE},
-            pools=pools)
-        c['schedulers'].append(instance)
-        self._schedulers[s_name]['instance'] = instance
-
-    # Process the dependent schedulers.
-    for s_name in self._schedulers:
-      scheduler = self._schedulers[s_name]
-      if scheduler['type'] == 'Dependent':
-        instance = Dependent(
-            s_name,
-            self._schedulers[scheduler['parent']]['instance'],
-            scheduler['builders'])
-        c['schedulers'].append(instance)
-        self._schedulers[s_name]['instance'] = instance
-
-
-def CanMergeBuildRequests(req1, req2):
-  """ Determine whether or not two BuildRequests can be merged. Note that the
-  call to buildbot.sourcestamp.SourceStamp.canBeMergedWith() is conspicuously
-  missing. This is because that method verifies that:
-  1. req1.source.repository == req2.source.repository
-  2. req1.source.project == req2.source.project
-  3. req1.source.branch == req2.source.branch
-  4. req1.patch == None and req2.patch = None
-  5. (req1.source.changes and req2.source.changes) or \
-     (not req1.source.changes and not req2.source.changes and \
-      req1.source.revision == req2.source.revision) 
-
-  Of the above, we want 1, 2, 3, and 5.
-  Instead of 4, we want to make sure that neither request is a Trybot request.
-  """
-  # Verify that the repositories are the same (#1 above).
-  if req1.source.repository != req2.source.repository:
-    return False
-
-  # Verify that the projects are the same (#2 above).
-  if req1.source.project != req2.source.project:
-    return False
-
-  # Verify that the branches are the same (#3 above).
-  if req1.source.branch != req2.source.branch:
-    return False
-
-  # If either is a try request, don't merge (#4 above).
-  if (builder_name_schema.IsTrybot(req1.buildername) or
-      builder_name_schema.IsTrybot(req2.buildername)):
-    return False
-
-  # Verify that either: both requests are associated with changes OR neither
-  # request is associated with a change but the revisions match (#5 above).
-  if req1.source.changes and not req2.source.changes:
-    return False
-  if not req1.source.changes and req2.source.changes:
-    return False
-  if not (req1.source.changes and req2.source.changes):
-    if req1.source.revision != req2.source.revision:
-      return False
-  
-  return True
-
-
-def get_current_revision():
-  """Obtain the checked-out buildbot code revision."""
-  checkout_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
-  if os.path.isdir(os.path.join(checkout_dir, '.git')):
-    return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
-  elif os.path.isdir(os.path.join(checkout_dir, '.svn')):
-    return subprocess.check_output(['svnversion', '.']).strip()
-  raise Exception('Unable to determine version control system.')