master/skia_master_scripts/monkeypatches.py - Issue 648353002: Remove Skia's forked buildbot code

Unified Diff: master/skia_master_scripts/monkeypatches.py

Issue 648353002: Remove Skia's forked buildbot code (Closed) Base URL: https://skia.googlesource.com/buildbot.git@master

Patch Set: Fix launch_slaves, remove more stuff Created 6 years, 2 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Download patch

Index: master/skia_master_scripts/monkeypatches.py

diff --git a/master/skia_master_scripts/monkeypatches.py b/master/skia_master_scripts/monkeypatches.py

deleted file mode 100644

index 4d4a473f9c38c049e67168908571bdb3566b81ba..0000000000000000000000000000000000000000

--- a/master/skia_master_scripts/monkeypatches.py

+++ /dev/null

@@ -1,585 +0,0 @@

-# Use of this source code is governed by a BSD-style license that can be

-# found in the LICENSE file.

-""" Monkeypatches to override upstream code. """

-from buildbot.changes.gitpoller import GitPoller

-from buildbot.process.properties import Properties

-from buildbot.schedulers.trysched import BadJobfile

-from buildbot.status.builder import EXCEPTION, FAILURE

-from buildbot.status.web import base as webstatus_base

-from buildbot.status.web.status_json import BuilderJsonResource

-from buildbot.status.web.status_json import BuildersJsonResource

-from buildbot.status.web.status_json import ChangeSourcesJsonResource

-from buildbot.status.web.status_json import JsonResource

-from buildbot.status.web.status_json import JsonStatusResource

-from buildbot.status.web.status_json import MetricsJsonResource

-from buildbot.status.web.status_json import ProjectJsonResource

-from buildbot.status.web.status_json import SlavesJsonResource

-from master import build_utils

-from master import chromium_notifier

-from master import gatekeeper

-from master import try_job_base

-from master import try_job_rietveld

-from master import try_job_svn

-from master.try_job_base import text_to_dict

-from twisted.internet import defer

-from twisted.python import log

-from twisted.web import server

-from webstatus import builder_statuses

-import builder_name_schema

-import config_private

-import json

-import master_revision

-import re

-import slave_hosts_cfg

-import slaves_cfg

-import skia_vars

-import utils

-# The following users are allowed to run trybots even though they do not have

-# accounts in google.com or chromium.org

-TRYBOTS_REQUESTER_WHITELIST = [

- 'henrik.smiding@intel.com',

- 'kkinnunen@nvidia.com',

- 'cdalton@nvidia.com',

- 'ravimist@gmail.com'

-################################################################################

-############################# Trybot Monkeypatches #############################

-################################################################################

-@defer.deferredGenerator

-def SubmitTryJobChanges(self, changes):

- """ Override of SVNPoller.submit_changes:

- http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_svn.py?view=markup

- We modify it so that the patch file url is added to the build properties.

- This allows the slave to download the patch directly rather than receiving

- it from the master.

- """

- for chdict in changes:

- # pylint: disable=E1101

- parsed = self.parent.parse_options(text_to_dict(chdict['comments']))

- # 'fix' revision.

- # LKGR must be known before creating the change object.

- wfd = defer.waitForDeferred(self.parent.get_lkgr(parsed))

- yield wfd

- wfd.getResult()

- wfd = defer.waitForDeferred(self.master.addChange(

- author=','.join(parsed['email']),

- revision=parsed['revision'],

- comments='',

- properties={'patch_file_url': chdict['repository'] + '/' + \

- chdict['files'][0]}))

- yield wfd

- change = wfd.getResult()

- self.parent.addChangeInner(chdict['files'], parsed, change.number)

-try_job_svn.SVNPoller.submit_changes = SubmitTryJobChanges

-def TryJobCreateBuildset(self, ssid, parsed_job):

- """ Override of TryJobBase.create_buildset:

- http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_base.py?view=markup

- We modify it to verify that the requested builders are in the builder pool for

- this try scheduler. This prevents try requests from running on builders which

- are not registered as trybots. This apparently isn't a problem for Chromium

- since they use a separate try master.

- """

- log.msg('Creating try job(s) %s' % ssid)

- result = None

- for builder in parsed_job['bot']:

- if builder in self.pools[self.name]:

- result = self.addBuildsetForSourceStamp(ssid=ssid,

- reason=parsed_job['name'],

- external_idstring=parsed_job['name'],

- builderNames=[builder],

- properties=self.get_props(builder, parsed_job))

- else:

- log.msg('Scheduler: %s rejecting try job for builder: %s not in %s' % (

- self.name,

- builder,

- self.pools[self.name]))

- log.msg('Returning buildset: %s' % result)

- return result

-try_job_base.TryJobBase.create_buildset = TryJobCreateBuildset

-def HtmlResourceRender(self, request):

- """ Override of buildbot.status.web.base.HtmlResource.render:

- http://src.chromium.org/viewvc/chrome/trunk/tools/build/third_party/buildbot_8_4p1/buildbot/status/web/base.py?view=markup

- We modify it to pass additional variables on to the web status pages, and

- remove the "if False" section.

- """

- # tell the WebStatus about the HTTPChannel that got opened, so they

- # can close it if we get reconfigured and the WebStatus goes away.

- # They keep a weakref to this, since chances are good that it will be

- # closed by the browser or by us before we get reconfigured. See

- # ticket #102 for details.

- if hasattr(request, "channel"):

- # web.distrib.Request has no .channel

- request.site.buildbot_service.registerChannel(request.channel)

- ctx = self.getContext(request)

- ############################## Added by borenet ##############################

- status = self.getStatus(request)

- all_builders = status.getBuilderNames()

- all_full_category_names = set()

- all_categories = set()

- all_subcategories = set()

- subcategories_by_category = {}

- for builder_name in all_builders:

- category_full = status.getBuilder(builder_name).category or 'default'

- all_full_category_names.add(category_full)

- category_split = category_full.split('|')

- category = category_split[0]

- subcategory = category_split[1] if len(category_split) > 1 else 'default'

- all_categories.add(category)

- all_subcategories.add(subcategory)

- if not subcategories_by_category.get(category):

- subcategories_by_category[category] = []

- if not subcategory in subcategories_by_category[category]:

- subcategories_by_category[category].append(subcategory)

- ctx['tree_status_baseurl'] = \

- skia_vars.GetGlobalVariable('tree_status_baseurl')

- ctx['all_full_category_names'] = sorted(list(all_full_category_names))

- ctx['all_categories'] = sorted(list(all_categories))

- ctx['all_subcategories'] = sorted(list(all_subcategories))

- ctx['subcategories_by_category'] = subcategories_by_category

- ctx['default_refresh'] = \

- skia_vars.GetGlobalVariable('default_webstatus_refresh')

- ctx['skia_repo'] = config_private.SKIA_GIT_URL

- active_master = config_private.Master.get_active_master()

- ctx['internal_port'] = active_master.master_port

- ctx['external_port'] = active_master.master_port_alt

- ctx['title_url'] = config_private.Master.Skia.project_url

- ctx['slave_hosts_cfg'] = slave_hosts_cfg.SLAVE_HOSTS

- ctx['slaves_cfg'] = slaves_cfg.SLAVES

- ctx['active_master_name'] = active_master.project_name

- ctx['master_revision'] = utils.get_current_revision()

- ctx['master_running_revision'] = active_master.running_revision

- ctx['master_launch_datetime'] = active_master.launch_datetime

- ctx['is_internal_view'] = request.host.port == ctx['internal_port']

- ctx['masters'] = []

- for master in config_private.Master.valid_masters:

- ctx['masters'].append({

- 'name': master.project_name,

- 'host': master.master_host,

- 'internal_port': master.master_port,

- 'external_port': master.master_port_alt,

- })

- ##############################################################################

- d = defer.maybeDeferred(lambda : self.content(request, ctx))

- def handle(data):

- if isinstance(data, unicode):

- data = data.encode("utf-8")

- request.setHeader("content-type", self.contentType)

- if request.method == "HEAD":

- request.setHeader("content-length", len(data))

- return ''

- return data

- d.addCallback(handle)

- def ok(data):

- request.write(data)

- request.finish()

- def fail(f):

- request.processingFailed(f)

- return None # processingFailed will log this for us

- d.addCallbacks(ok, fail)

- return server.NOT_DONE_YET

-webstatus_base.HtmlResource.render = HtmlResourceRender

-class TryBuildersJsonResource(JsonResource):

- """ Clone of buildbot.status.web.status_json.BuildersJsonResource:

- http://src.chromium.org/viewvc/chrome/trunk/tools/build/third_party/buildbot_8_4p1/buildbot/status/web/status_json.py?view=markup

- We add filtering to display only the try builders.

- """

- help = """List of all the try builders defined on a master."""

- pageTitle = 'Builders'

- def __init__(self, status, include_only_cq_trybots=False):

- JsonResource.__init__(self, status)

- for builder_name in self.status.getBuilderNames():

- if builder_name_schema.IsTrybot(builder_name) and (

- not include_only_cq_trybots or builder_name in slaves_cfg.CQ_TRYBOTS):

- self.putChild(builder_name,

- BuilderJsonResource(status,

- status.getBuilder(builder_name)))

-class CQRequiredStepsJsonResource(JsonResource):

- help = 'List the steps which cannot fail on the commit queue.'

- pageTitle = 'CQ Required Steps'

- def asDict(self, request):

- return {'cq_required_steps':

- skia_vars.GetGlobalVariable('cq_required_steps')}

-def JsonStatusResourceInit(self, status):

- """ Override of buildbot.status.web.status_json.JsonStatusResource.__init__:

- http://src.chromium.org/viewvc/chrome/trunk/tools/build/third_party/buildbot_8_4p1/buildbot/status/web/status_json.py?view=markup

- We add trybots, cqtrybots, cq_required_steps (details below).

- """

- JsonResource.__init__(self, status)

- self.level = 1

- self.putChild('builders', BuildersJsonResource(status))

- self.putChild('change_sources', ChangeSourcesJsonResource(status))

- self.putChild('project', ProjectJsonResource(status))

- self.putChild('slaves', SlavesJsonResource(status))

- self.putChild('metrics', MetricsJsonResource(status))

- ############################## Added by borenet ##############################

- # Added to address: https://code.google.com/p/skia/issues/detail?id=1134

- self.putChild('trybots', TryBuildersJsonResource(status))

- ##############################################################################

- ############################## Added by rmistry ##############################

- # Added to have a place to get the list of trybots run by the CQ.

- self.putChild('cqtrybots',

- TryBuildersJsonResource(status, include_only_cq_trybots=True))

- ##############################################################################

- ############################## Added by borenet ##############################

- # Added to have a place to get the list of steps which cannot fail on the CQ.

- self.putChild('cq_required_steps', CQRequiredStepsJsonResource(status))

- ##############################################################################

- ############################## Added by borenet ##############################

- # Added to have a way to determine which code revision the master is running.

- self.putChild('master_revision',

- master_revision.MasterCheckedOutRevisionJsonResource(status))

- running_rev = config_private.Master.get_active_master().running_revision

- self.putChild('master_running_revision',

- master_revision.MasterRunningRevisionJsonResource(

- status=status, running_revision=running_rev))

- # This page gives the result of the most recent build for each builder.

- self.putChild('builder_statuses',

- builder_statuses.BuilderStatusesJsonResource(status))

- ##############################################################################

- # This needs to be called before the first HelpResource().body call.

- self.hackExamples()

-JsonStatusResource.__init__ = JsonStatusResourceInit

-@defer.deferredGenerator

-def TryJobRietveldSubmitJobs(self, jobs):

- """ Override of master.try_job_rietveld.TryJobRietveld.SubmitJobs:

- http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_rietveld.py?view=markup

- We modify it to include "baseurl" as a build property.

- """

- log.msg('TryJobRietveld.SubmitJobs: %s' % json.dumps(jobs, indent=2))

- for job in jobs:

- try:

- # Gate the try job on the user that requested the job, not the one that

- # authored the CL.

- # pylint: disable=W0212

- ########################## Added by rmistry ##########################

- if (job.get('requester') and not job['requester'].endswith('@google.com')

- and not job['requester'].endswith('@chromium.org')

- and not job['requester'] in TRYBOTS_REQUESTER_WHITELIST):

- # Reject the job only if the requester has an email not ending in

- # google.com or chromium.org

- raise BadJobfile(

- 'TryJobRietveld rejecting job from %s' % job['requester'])

- ######################################################################

- ########################## Added by borenet ##########################

- if not (job.get('baseurl') and

- config_private.Master.Skia.project_name.lower() in

- job['baseurl']):

- raise BadJobfile('TryJobRietveld rejecting job with unknown baseurl: %s'

- % job.get('baseurl'))

- ######################################################################

- if job['email'] != job['requester']:

- # Note the fact the try job was requested by someone else in the

- # 'reason'.

- job['reason'] = job.get('reason') or ''

- if job['reason']:

- job['reason'] += '; '

- job['reason'] += "This CL was triggered by %s" % job['requester']

- options = {

- 'bot': {job['builder']: job['tests']},

- 'email': [job['email']],

- 'project': [self._project],

- 'try_job_key': job['key'],

- }

- # Transform some properties as is expected by parse_options().

- for key in (

- ########################## Added by borenet ##########################

- 'baseurl',

- ######################################################################

- 'name', 'user', 'root', 'reason', 'clobber', 'patchset',

- 'issue', 'requester', 'revision'):

- options[key] = [job[key]]

- # Now cleanup the job dictionary and submit it.

- cleaned_job = self.parse_options(options)

- wfd = defer.waitForDeferred(self.get_lkgr(cleaned_job))

- yield wfd

- wfd.getResult()

- wfd = defer.waitForDeferred(self.master.addChange(

- author=','.join(cleaned_job['email']),

- # TODO(maruel): Get patchset properties to get the list of files.

- # files=[],

- revision=cleaned_job['revision'],

- comments=''))

- yield wfd

- changeids = [wfd.getResult().number]

- wfd = defer.waitForDeferred(self.SubmitJob(cleaned_job, changeids))

- yield wfd

- wfd.getResult()

- except BadJobfile, e:

- # We need to mark it as failed otherwise it'll stay in the pending

- # state. Simulate a buildFinished event on the build.

- if not job.get('key'):

- log.err(

- 'Got %s for issue %s but not key, not updating Rietveld' %

- (e, job.get('issue')))

- continue

- log.err(

- 'Got %s for issue %s, updating Rietveld' % (e, job.get('issue')))

- for service in self.master.services:

- if service.__class__.__name__ == 'TryServerHttpStatusPush':

- # pylint: disable=W0212,W0612

- build = {

- 'properties': [

- ('buildername', job.get('builder'), None),

- ('buildnumber', -1, None),

- ('issue', job['issue'], None),

- ('patchset', job['patchset'], None),

- ('project', self._project, None),

- ('revision', '', None),

- ('slavename', '', None),

- ('try_job_key', job['key'], None),

- ],

- 'reason': job.get('reason', ''),

- # Use EXCEPTION until SKIPPED results in a non-green try job

- # results on Rietveld.

- 'results': EXCEPTION,

- }

- ########################## Added by rmistry #########################

- # Do not update Rietveld to mark the try job request as failed.

- # See https://code.google.com/p/chromium/issues/detail?id=224014 for

- # more context.

- # service.push('buildFinished', build=build)

- #####################################################################

- break

-try_job_rietveld.TryJobRietveld.SubmitJobs = TryJobRietveldSubmitJobs

-def TryJobBaseGetProps(self, builder, options):

- """ Override of try_job_base.TryJobBase.get_props:

- http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_base.py?view=markup

- We modify it to add "baseurl".

- """

- keys = (

-############################### Added by borenet ###############################

- 'baseurl',

-################################################################################

- 'clobber',

- 'issue',

- 'patchset',

- 'requester',

- 'rietveld',

- 'root',

- 'try_job_key',

- )

- # All these settings have no meaning when False or not set, so don't set

- # them in that case.

- properties = dict((i, options[i]) for i in keys if options.get(i))

- properties['testfilter'] = options['bot'].get(builder, None)

- # pylint: disable=W0212

- props = Properties()

- props.updateFromProperties(self.properties)

- props.update(properties, self._PROPERTY_SOURCE)

- return props

-try_job_base.TryJobBase.get_props = TryJobBaseGetProps

-def TryJobRietveldConstructor(

- self, name, pools, properties=None, last_good_urls=None,

- code_review_sites=None, project=None):

- try_job_base.TryJobBase.__init__(self, name, pools, properties,

- last_good_urls, code_review_sites)

- # pylint: disable=W0212

- endpoint = self._GetRietveldEndPointForProject(code_review_sites, project)

-############################### Added by rmistry ###############################

- # rmistry: Adding '&master=tryserver.skia' to the endpoint to help filter the

- # number of pending try patchsets returned. More details are in

- # https://code.google.com/p/skia/issues/detail?id=2659

- endpoint += '&master=tryserver.skia'

- # rmistry: Increased the polling time from 10 seconds to 1 min because 10

- # seconds is too short for us. The RietveldPoller stops working if the time is

- # too short.

- # pylint: disable=W0212

- self._poller = try_job_rietveld._RietveldPoller(endpoint, interval=60)

-################################################################################

- # pylint: disable=W0212

- self._valid_users = try_job_rietveld._ValidUserPoller(interval=12 * 60 * 60)

- self._project = project

- log.msg('TryJobRietveld created, get_pending_endpoint=%s '

- 'project=%s' % (endpoint, project))

-try_job_rietveld.TryJobRietveld.__init__ = TryJobRietveldConstructor

-class SkiaGateKeeper(gatekeeper.GateKeeper):

- def isInterestingBuilder(self, builder_status):

- """ Override of gatekeeper.GateKeeper.isInterestingBuilder:

- http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/gatekeeper.py?view=markup

- We modify it to actually check whether the builder should be considered by

- the GateKeeper, as indicated in its category name.

- """

- ret = (not builder_name_schema.IsTrybot(builder_status.getName()) and

- chromium_notifier.ChromiumNotifier.isInterestingBuilder(self,

- builder_status))

- log.msg('[gatekeeper-debug2] ======================')

- log.msg('[gatekeeper-debug2] is not trybot: %s' % (

- not builder_name_schema.IsTrybot(builder_status.getName())))

- log.msg('[gatekeeper-debug2] isInterestingBuilder: %s' % (

- chromium_notifier.ChromiumNotifier.isInterestingBuilder(

- self, builder_status)))

- log.msg('[gatekeeper-debug2] builder_status.getName(): %s' % (

- builder_status.getName()))

- log.msg('[gatekeeper-debug2] ret: %s' % ret)

- log.msg('[gatekeeper-debug2] ======================')

- return ret

- def isInterestingStep(self, build_status, step_status, results):

- """ Override of gatekeeper.GateKeeper.isInterestingStep:

- http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/gatekeeper.py?view=markup

- We modify it to comment out the SVN revision comparision to determine if the

- current build is older because Skia uses commit hashes.

- """

- # If we have not failed, or are not interested in this builder,

- # then we have nothing to do.

- if results[0] != FAILURE:

- return False

- # Check if the slave is still alive. We should not close the tree for

- # inactive slaves.

- slave_name = build_status.getSlavename()

- if slave_name in self.master_status.getSlaveNames():

- # @type self.master_status: L{buildbot.status.builder.Status}

- # @type self.parent: L{buildbot.master.BuildMaster}

- # @rtype getSlave(): L{buildbot.status.builder.SlaveStatus}

- slave_status = self.master_status.getSlave(slave_name)

- if slave_status and not slave_status.isConnected():

- log.msg('[gatekeeper] Slave %s was disconnected, '

- 'not closing the tree' % slave_name)

- return False

- # If the previous build step failed with the same result, we don't care

- # about this step.

- previous_build_status = build_status.getPreviousBuild()

- if previous_build_status:

- step_name = self.getName(step_status)

- step_type = self.getGenericName(step_name)

- previous_steps = [step for step in previous_build_status.getSteps()

- if self.getGenericName(self.getName(step)) == step_type]

- if len(previous_steps) == 1:

- if previous_steps[0].getResults()[0] == FAILURE:

- log.msg('[gatekeeper] Slave %s failed, but previously failed on '

- 'the same step (%s). So not closing tree.' % (

- (step_name, slave_name)))

- return False

- else:

- log.msg('[gatekeeper] len(previous_steps) == %d which is weird' %

- len(previous_steps))

- # If check_revisions=False that means that the tree closure request is

- # coming from nightly scheduled bots, that need not necessarily have the

- # revision info.

- if not self.check_revisions:

- return True

- # If we don't have a version stamp nor a blame list, then this is most

- # likely a build started manually, and we don't want to close the

- # tree.

- latest_revision = build_utils.getLatestRevision(build_status)

- if not latest_revision or not build_status.getResponsibleUsers():

- log.msg('[gatekeeper] Slave %s failed, but no version stamp, '

- 'so skipping.' % slave_name)

- return False

- # If the tree is open, we don't want to close it again for the same

- # revision, or an earlier one in case the build that just finished is a

- # slow one and we already fixed the problem and manually opened the tree.

- ############################### Added by rmistry ###########################

- # rmistry: Commenting out the below SVN revision comparision because Skia

- # uses commit hashes.

- # TODO(rmistry): Figure out how to ensure that previous builds do not close

- # the tree again.

- #

- # if latest_revision <= self._last_closure_revision:

- # log.msg('[gatekeeper] Slave %s failed, but we already closed it '

- # 'for a previous revision (old=%s, new=%s)' % (

- # slave_name, str(self._last_closure_revision),

- # str(latest_revision)))

- # return False

- ###########################################################################

- log.msg('[gatekeeper] Decided to close tree because of slave %s '

- 'on revision %s' % (slave_name, str(latest_revision)))

- # Up to here, in theory we'd check if the tree is closed but this is too

- # slow to check here. Instead, take a look only when we want to close the

- # tree.

- return True

-# Fix try_job_base.TryJobBase._EMAIL_VALIDATOR to handle *.info. This was fixed

-# in https://codereview.chromium.org/216293005 but we need this monkeypatch to

-# pick it up without a DEPS roll.

-try_job_base.TryJobBase._EMAIL_VALIDATOR = re.compile(

- r'[a-zA-Z0-9][a-zA-Z0-9\.\+\-\_]*@[a-zA-Z0-9\.\-]+\.[a-zA-Z]{2,}$')

-# Add logging to GitPoller._stop_on_failure

-def _stop_on_failure(self, f):

- "utility method to stop the service when a failure occurs"

- log.err('GitPoller stopping due to failure: %s' % str(f))

- if self.running:

- d = defer.maybeDeferred(lambda : self.stopService())

- d.addErrback(log.err, 'while stopping broken GitPoller service')

- return f

-GitPoller._stop_on_failure = _stop_on_failure

« no previous file with comments | « master/skia_master_scripts/master_revision.py ('k') | master/skia_master_scripts/nacl_factory.py » ('j') | scripts/launch_slaves.py » ('J')