swarming: add support for cipd on the server side

appengine/swarming/server/task_request.py

Index: appengine/swarming/server/task_request.py
diff --git a/appengine/swarming/server/task_request.py b/appengine/swarming/server/task_request.py
index 35088c0f86a2882b23e147d16dfa27d79896d368..8b1cead5ba8c76ea0142ae653306c5a1414ea23b 100644
--- a/appengine/swarming/server/task_request.py
+++ b/appengine/swarming/server/task_request.py
@@ -61,6 +61,7 @@ from components import datastore_utils
 from components import pubsub
 from components import utils
 from server import task_pack
+import cipd
 
 
 # Maximum acceptable priority value, which is effectively the lowest priority.
@@ -223,6 +224,28 @@ class FilesRef(ndb.Model):
           'isolated requires server and namespace')
 
 
+class CipdPackage(ndb.Model):
+  """A CIPD package to install in $CIPD_PATH and $PATH before task execution.
+
+  A part of TaskProperties.
+  """
+  package_name = ndb.StringProperty(indexed=False)
+  version = ndb.StringProperty(indexed=False)
+
+  def _pre_put_hook(self):
+    super(CipdPackage, self)._pre_put_hook()
+    if not self.package_name:
+      raise datastore_errors.BadValueError('CIPD package name is required')
+    if not cipd.is_valid_package_name(self.package_name):

[M-A Ruel, 2016/04/25 19:09:45]

Could you use validator=wrapper_function instead?

You can then just keep lines 237-238 and 242-243, I prefer this to required=True
because required=True breaks loading. :/

[nodir, 2016/04/25 20:27:16]

Done.

+      raise datastore_errors.BadValueError(
+          'malformed package name "%s"' % self.package_name)
+    if not self.version:
+      raise datastore_errors.BadValueError('CIPD package version is required')
+    if not cipd.is_valid_version(self.version):

[M-A Ruel, 2016/04/25 19:09:45]

Same

[nodir, 2016/04/25 20:27:15]

Done.

+      raise datastore_errors.BadValueError(
+          'malformed package version "%s"' % self.version)
+
+
 class TaskProperties(ndb.Model):
   """Defines all the properties of a task to be run on the Swarming
   infrastructure.
@@ -251,6 +274,10 @@ class TaskProperties(ndb.Model):
   # File inputs of the task. Only inputs_ref or command&data can be specified.
   inputs_ref = ndb.LocalStructuredProperty(FilesRef)
 
+  # A list of CIPD packages to install $CIPD_PATH and $PATH before task
+  # execution.
+  packages = ndb.LocalStructuredProperty(CipdPackage, repeated=True)
+
   # Filter to use to determine the required properties on the bot to run on. For
   # example, Windows or hostname. Encoded as json. Optional but highly
   # recommended.
@@ -332,6 +359,15 @@ class TaskProperties(ndb.Model):
         raise datastore_errors.BadValueError('extra_args require inputs_ref')
       if self.inputs_ref:
         self.inputs_ref._pre_put_hook()
+      for p in self.packages:

[M-A Ruel, 2016/04/25 19:09:45]

Can you also sort it? It's important that listing in arbitrary orders still
result in the same hash digest.

[nodir, 2016/04/25 20:27:16]

good point, done
also add a check that only one version of a package is specified

+        p._pre_put_hook()
+      if self.idempotent:
+        pinned = lambda p: cipd.is_pinned_version(p.version)
+        if self.packages and any(not pinned(p) for p in self.packages):
+          raise datastore_errors.BadValueError(
+            'an idempotent task cannot have unpinned packages; '
+            'use instance IDs or tags as package versions')
+
 
 
 class TaskRequest(ndb.Model):
@@ -565,7 +601,7 @@ def make_request(request, is_bot_or_admin):
 
   If parent_task_id is set, properties for the parent are used:
   - priority: defaults to parent.priority - 1
-  - user: overriden by parent.user
+  - user: overridden by parent.user
 
   """
   assert request.__class__ is TaskRequest