CrOS: Switch tryserver over to recipes, Gitiles.

scripts/master/cros_try_job_git.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import base64
 import json
 import os
 import re
 import shutil
 
 from StringIO import StringIO
 
 try:
   # Create a block to work around evil sys.modules manipulation in
   # email/__init__.py that triggers pylint false positives.
   # pylint: disable=E0611,F0401
   from email.Message import Message
   from email.Utils import formatdate
 except ImportError:
   raise
 
 from buildbot.process.properties import Properties
 from buildbot.schedulers.trysched import TryBase
 
 from twisted.internet import defer, reactor, utils
 from twisted.mail.smtp import SMTPSenderFactory
 from twisted.python import log
 
+from common.twisted_util.response import StringResponse
+from master import gitiles_poller
 from master.try_job_base import BadJobfile
 
 
 class InvalidEtcBuild(BadJobfile):
   pass
 
 
-def get_file_contents(poller, branch, file_path):
-  """Returns a Deferred to returns the file's content."""
-  return utils.getProcessOutput(
-      poller.gitbin,
-      ['show', 'origin/%s:%s' % (branch, file_path)],
-      path=poller.workdir,
-      )
-
-
 def translate_v1_to_v2(parsed_job):
   """Translate tryjob desc from V1 to V2."""
   parsed_job.setdefault('extra_args', []).append('--remote-trybot')
   parsed_job['version'] = 2
 
 
 def translate_v2_to_v3(parsed_job):
   """Translate tryjob desc from V2 to V3."""
   # V3 --remote-patches format is not backwards compatible.
   if any(a.startswith('--remote-patches')
@@ skipping 15 matching lines @@
   _TRANSLATION_FUNCS = {
       1 : translate_v1_to_v2,
       2 : translate_v2_to_v3,
   }
 
   # Valid 'etc' builder targets. Specifically, this ensures:
   # - The build name doesn't begin with a flag ('--')
   # - The build name doesn't contain spaces (to spill into extra args).
   ETC_TARGET_RE = re.compile(r'^[a-zA-Z][\w-]+\w$')
 
+  # Template path URL component to retrieve the Base64 contents of a file from
+  # Gitiles.
+  _GITILES_PATH_TMPL = '%(repo)s/+/%(revision)s/%(path)s?format=text'
+
   @classmethod
   def updateJobDesc(cls, parsed_job):
     """Ensure job description is in the format we expect."""
     while parsed_job['version'] < cls._TRYJOB_FORMAT_VERSION:
       prev_ver = parsed_job['version']
       translation_func = cls._TRANSLATION_FUNCS[parsed_job['version']]
       translation_func(parsed_job)
       if parsed_job['version'] <= prev_ver:
         raise AssertionError('translation function %s not incrementing version!'
                              % str(translation_func))
@@ skipping 103 matching lines @@
     # Calculate the buildroot for builds.
     config_dict = self.cbuildbot_configs.get(config)
     if config_dict is None:
       buildroot = '/b/cbuild/etc_master'
     elif config_dict['internal']:
       buildroot = '/b/cbuild/internal_master'
     else:
       buildroot = '/b/cbuild/external_master'
     props.setProperty('buildroot', buildroot, self._PROPERTY_SOURCE)
 
-    props.setProperty('extra_args', options.get('extra_args', []),
-                      self._PROPERTY_SOURCE)
     props.setProperty('slaves_request', options.get('slaves_request', []),
                       self._PROPERTY_SOURCE)
-    props.setProperty('chromeos_config', config, self._PROPERTY_SOURCE)
+    props.setProperty('cbb_config', config, self._PROPERTY_SOURCE)
 
     return props
 
   def create_buildset(self, ssid, parsed_job):
     """Overriding base class method."""
     dlist = []
     buildset_name = '%s:%s' % (parsed_job['user'], parsed_job['name'])
     for bot in parsed_job['bot']:
       config = self.cbuildbot_configs.get(bot)
       if config:
@@ skipping 27 matching lines @@
     m.set_type("text/html")
     m['Date'] = formatdate(localtime=True)
     m['Subject'] = 'Tryjob failed validation'
     m['From'] = self.from_addr
     m['Reply-To'] = self.reply_to
     result = defer.Deferred()
     sender_factory = SMTPSenderFactory(self.from_addr, emails,
                                        StringIO(m.as_string()), result)
     reactor.connectTCP(self.smtp_host, 25, sender_factory)
 
-  @defer.deferredGenerator
+  @defer.inlineCallbacks
   def gotChange(self, change, important):
     """Process the received data and send the queue buildset."""
-    # Implicitly skips over non-files like directories.
-    if len(change.files) != 1:
-      # We only accept changes with 1 diff file.
-      raise BadJobfile(
-          'Try job with too many files %s' % (','.join(change.files)))
-
     # Find poller that this change came from.
     for poller in self.pollers:
-      if poller.repourl == change.repository:
+      if not isinstance(poller, gitiles_poller.GitilesPoller):
+        continue
+      if poller.repo_url == change.repository:
         break
     else:
       raise BadJobfile(
           'Received tryjob from unsupported repository %s' % change.repository)
 
     # pylint: disable=W0631
-    wfd = defer.waitForDeferred(
-        get_file_contents(poller, change.branch, change.files[0]))
-    yield wfd
+    file_contents = yield self.loadGitilesChangeFile(poller, change)
 
-    parsed = None
+    parsed = {}
     try:
-      parsed = self.load_job(wfd.getResult())
+      parsed = self.load_job(file_contents)
       self.validate_job(parsed)
       self.updateJobDesc(parsed)
     except BadJobfile as e:
       self.send_validation_fail_email(parsed.setdefault('name', ''),
                                       parsed['email'], str(e))
       raise
     except Exception as e:
       print 'EXCEPTION:', e
       import traceback
       traceback.print_exc()
       raise
 
     # The sourcestamp/buildsets created will be merge-able.
-    d = self.master.db.sourcestamps.addSourceStamp(
+    ssid = yield self.master.db.sourcestamps.addSourceStamp(
         branch=change.branch,
         revision=change.revision,
         project=change.project,
         repository=change.repository,
         changeids=[change.number])
-    d.addCallback(self.create_buildset, parsed)
-    d.addErrback(log.err, "Failed to queue a try job!")
+    yield self.create_buildset(ssid, parsed)
+
+  @defer.inlineCallbacks
+  def loadGitilesChangeFile(self, poller, change):
+    if len(change.files) != 1:
+      # We only accept changes with 1 diff file.
+      raise BadJobfile(

[David James, 2015/09/15 19:35:51]

What happens when somebody has a commit that affects >1 file? Does it just
ignore that commit and continue? Or does it halt on that commit?

[phobbs, 2015/09/15 21:27:49]

I think there would still be only one diff in that case.

[David James, 2015/09/15 21:34:37]

I mean that somebody did a manual commit in the trybot repository and (for
example) deleted a bunch of files. In that case, change.files would be >1, no?

[dnj, 2015/09/15 21:55:52]

Yes, we expect a push that has been formulated by cbuildbot. However, note that
this criteria has always been applied (see #258 in before-diff).

+          'Try job with too many files %s' % (','.join(change.files)))
+
+    # Load the contents of the modified file.
+    path = self._GITILES_PATH_TMPL % {
+        'repo': poller.repo_path,
+        'revision': change.revision,
+        'path': change.files[0],
+    }
+    contents_b64 = yield poller.agent.request('GET', path, retry=5,
+                                              protocol=StringResponse.Get)
+    defer.returnValue(base64.b64decode(contents_b64))