Rietveld: schedule builds on buildbucket

appengine/chromium_rietveld/codereview/buildbucket.py

 # Copyright 2008 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """Rietveld-BuildBucket integration module."""
 
 import datetime
 import json
 import logging
 import os
 import urllib
+import uuid
 
 from google.appengine.api import app_identity
 from google.appengine.api import memcache
 from google.appengine.api import users
 from google.appengine.ext import ndb
 
 from django.conf import settings
 
 from codereview import common
 from codereview import models
 from codereview import net
 
 EPOCH = datetime.datetime.utcfromtimestamp(0)
 BUILDBUCKET_APP_ID = (
     'cr-buildbucket-test' if common.IS_DEV else 'cr-buildbucket')
 BUILDBUCKET_API_ROOT = (
     'https://%s.appspot.com/_ah/api/buildbucket/v1' % BUILDBUCKET_APP_ID)
-# See tag conventions http://cr-buildbucket.appspot.com/#docs/conventions
+# See the convention
+# https://chromium.googlesource.com/infra/infra/+/master/appengine/cr-buildbucket/doc/index.md#buildset-tag
 BUILDSET_TAG_FORMAT = 'patch/rietveld/{hostname}/{issue}/{patch}'
 
 AUTH_SERVICE_ID = 'chrome-infra-auth'
 IMPERSONATION_TOKEN_MINT_URL = (
     'https://%s.appspot.com/auth_service/api/v1/delegation/token/create' %
     AUTH_SERVICE_ID)
 IMPERSONATION_TOKEN_CACHE_KEY_FORMAT = 'impersonation_token/v1/%s'
 
 
 class BuildBucketError(Exception):
@@ skipping 13 matching lines @@
   @property
   def is_from_buildbucket(self):
     # Used in build_result.html template.
     return True
 
   @classmethod
   def convert_status_to_result(cls, build):
     """Converts build status to TryJobResult.result.
 
     See buildbucket docs here:
-    https://cr-buildbucket.appspot.com/#/docs/build
+    https://chromium.googlesource.com/infra/infra/+/master/appengine/cr-buildbucket/doc/index.md#Build
     """
     status = build.get('status')
     if status == 'SCHEDULED':
       return cls.TRYPENDING
 
     if status == 'COMPLETED':
       if build.get('result') == 'SUCCESS':
         return cls.SUCCESS
       if build.get('result') == 'FAILURE':
         if build.get('failure_reason') == 'BUILD_FAILURE':
@@ skipping 69 matching lines @@
 def get_builds_for_patchset_async(project, issue_id, patchset_id):
   """Queries BuildBucket for builds associated with the patchset.
 
   Requests for max 500 builds and does not check "next_cursor". Currently if
   more than 100 builds are requested, only 100 are returned. Presumably there
   will be no patchsets with >100 builds.
 
   Returns:
     A list of buildbucket build dicts.
   """
-  # See tag conventions http://cr-buildbucket.appspot.com/#docs/conventions .
-  hostname = common.get_preferred_domain(project, default_to_appid=False)
-  if not hostname:
-    logging.error(
-        'Preferred domain name for this app is not set. '
-        'See PREFERRED_DOMAIN_NAMES in settings.py: %r', hostname)
-    raise ndb.Return([])
-
-  buildset_tag = BUILDSET_TAG_FORMAT.format(
-      hostname=hostname,
-      issue=issue_id,
-      patch=patchset_id,
-  )
+  buildset_tag = get_buildset_for(project, issue_id, patchset_id)
   params = {
       'max_builds': 500,
       'tag': 'buildset:%s' % buildset_tag,
   }
 
   logging.info(
       'Fetching builds for patchset %s/%s. Buildset: %s',
       issue_id, patchset_id, buildset_tag)
   try:
     resp = yield rpc_async('GET', 'search', params=params)
@@ skipping 19 matching lines @@
     try_job_result = BuildbucketTryJobResult.from_build(build)
     if not try_job_result.builder:
       logging.info(
           'Build %s does not have a builder' % try_job_result.build_id)
       continue
     results.append(try_job_result)
   raise ndb.Return(results)
 
 
 ################################################################################
+## Scheduling builds.
+
+
+def schedule(issue, patchset_id, builds):
+  """Schedules builds on buildbucket.
+
+  |builds| is a list of (master, builder) tuples, where master must not have
+  '.master' prefix.
+  """
+  account = models.Account.current_user_account
+  assert account, 'User is not logged in; cannot schedule builds.'
+
+  if not builds:
+    return
+
+  self_hostname = common.get_preferred_domain(issue.project)
+
+  req = {'builds':[]}
+  opid = uuid.uuid4()
+
+  for i, (master, builder) in enumerate(builds):
+    # Build definitions are similar to what CQ produces:
+    # https://chrome-internal.googlesource.com/infra/infra_internal/+/c3092da98975c7a3e083093f21f0f4130c66a51c/commit_queue/buildbucket_util.py#171
+    req['builds'].append({
+        'bucket': 'master.%s' % master,
+        'parameters_json': json.dumps({
+          'builder_name': builder,
+          'changes': [{
+              'author': {'email': issue.owner.email()},
+              'url': 'https://%s/%s/%s/' % (
+                  self_hostname, issue.key.id(), patchset_id)
+          }],
+          'properties': {
+            'issue': issue.key.id(),
+            'master': master,
+            'patch_project': issue.project,
+            'patch_storage': 'rietveld',
+            'patchset': patchset_id,
+            'project': issue.project,
+            'rietveld': self_hostname,
+          },
+        }),
+        'tags': [
+            'builder:%s' % builder,
+            'buildset:%s' % get_buildset_for(
+                issue.project, issue.key.id(), patchset_id),
+            'master:%s' % master,
+            'user_agent:rietveld',
+        ],
+        'client_operation_id': '%s:%s' % (opid, i),
+    })
+
+  logging.debug(
+      'Scheduling %d builds on behalf of %s', len(req['builds']), account.email)
+  res = rpc_async('PUT', 'builds/batch', payload=req).get_result()
+  for r in res['results']:
+    error = r.get('error')
+    if error:
+      logging.error('Build scheduling failed. Response: %r', res)
+      raise BuildBucketError('Could not schedule build(s): %r' % error)
+
+  actual_builds = [r['build'] for r in res['results']]
+  logging.info(
+      'Scheduled buildbucket builds: %r',
+      ', '.join([str(b['id']) for b in actual_builds]))
+  return actual_builds
+
+
+################################################################################
 ## Buildbucket RPC.
 
 
 @ndb.tasklet
 def _mint_delegation_token_async():
   """Generates an access token to impersonate the current user, if any.
 
   Memcaches the token.
   """
   account = models.Account.current_user_account
@@ skipping 94 matching lines @@
     return None
   return value
 
 
 def timestamp_to_datetime(timestamp):
   if timestamp is None:
     return None
   if isinstance(timestamp, basestring):
     timestamp = int(timestamp)
   return EPOCH + datetime.timedelta(microseconds=timestamp)
+
+
+def get_buildset_for(project, issue_id, patchset_id):
+  # See the convention
+  # https://chromium.googlesource.com/infra/infra/+/master/appengine/cr-buildbucket/doc/index.md#buildset-tag
+  hostname = common.get_preferred_domain(project, default_to_appid=False)
+  if not hostname:
+    logging.error(
+        'Preferred domain name for this app is not set. '
+        'See PREFERRED_DOMAIN_NAMES in settings.py: %r', hostname)
+    raise ndb.Return([])
+
+  return BUILDSET_TAG_FORMAT.format(
+      hostname=hostname,
+      issue=issue_id,
+      patch=patchset_id,
+  )