swarming: add support for cipd on the server side

appengine/swarming/server/task_request.py

Index: appengine/swarming/server/task_request.py
diff --git a/appengine/swarming/server/task_request.py b/appengine/swarming/server/task_request.py
index 35088c0f86a2882b23e147d16dfa27d79896d368..e768bcbaadd6a7b0867edeca52a07bbf167c50ef 100644
--- a/appengine/swarming/server/task_request.py
+++ b/appengine/swarming/server/task_request.py
@@ -61,6 +61,7 @@ from components import datastore_utils
 from components import pubsub
 from components import utils
 from server import task_pack
+import cipd
 
 
 # Maximum acceptable priority value, which is effectively the lowest priority.
@@ -203,6 +204,20 @@ def _validate_tags(prop, value):
         '%s must be key:value form, not %s' % (prop._name, value))
 
 
+def _validate_package_name(prop, value):
+  """Validates a CIPD package name."""
+  if not cipd.is_valid_package_name(value):
+    raise datastore_errors.BadValueError(
+      '%s must be a valid CIPD package name "%s"' % (prop._name, value))
+
+
+def _validate_package_version(prop, value):
+  """Validates a CIPD package version."""
+  if not cipd.is_valid_version(value):
+    raise datastore_errors.BadValueError(
+      '%s must be a valid package version "%s"' % (prop._name, value))
+
+
 ### Models.
 
 
@@ -223,6 +238,24 @@ class FilesRef(ndb.Model):
           'isolated requires server and namespace')
 
 
+class CipdPackage(ndb.Model):
+  """A CIPD package to install in $CIPD_PATH and $PATH before task execution.
+
+  A part of TaskProperties.
+  """
+  package_name = ndb.StringProperty(
+      indexed=False, validator=_validate_package_name)
+  version = ndb.StringProperty(
+      indexed=False, validator=_validate_package_version)
+
+  def _pre_put_hook(self):
+    super(CipdPackage, self)._pre_put_hook()
+    if not self.package_name:
+      raise datastore_errors.BadValueError('CIPD package name is required')
+    if not self.version:
+      raise datastore_errors.BadValueError('CIPD package version is required')
+
+
 class TaskProperties(ndb.Model):
   """Defines all the properties of a task to be run on the Swarming
   infrastructure.
@@ -251,6 +284,10 @@ class TaskProperties(ndb.Model):
   # File inputs of the task. Only inputs_ref or command&data can be specified.
   inputs_ref = ndb.LocalStructuredProperty(FilesRef)
 
+  # A list of CIPD packages to install $CIPD_PATH and $PATH before task
+  # execution.
+  packages = ndb.LocalStructuredProperty(CipdPackage, repeated=True)
+
   # Filter to use to determine the required properties on the bot to run on. For
   # example, Windows or hostname. Encoded as json. Optional but highly
   # recommended.
@@ -333,6 +370,24 @@ class TaskProperties(ndb.Model):
       if self.inputs_ref:
         self.inputs_ref._pre_put_hook()
 
+      package_names = set()
+      for p in self.packages:
+        p._pre_put_hook()
+        if p.package_name in package_names:
+          # We don't support different versions of the same package yet.

[M-A Ruel, 2016/04/27 17:43:56]

"yet"? Ever? I don't think this comment is necessary

[nodir, 2016/04/29 22:28:08]

removed comment

+          raise datastore_errors.BadValueError(
+              'package %s is specified more than once' % p.package_name)
+        package_names.add(p.package_name)
+      self.packages = sorted(self.packages, key=lambda p: p.package_name)

[M-A Ruel, 2016/04/27 17:43:56]

self.packages.sort(key=lambda p: p.package_name)
?

[nodir, 2016/04/29 22:28:08]

Done.

+
+      if self.idempotent:
+        pinned = lambda p: cipd.is_pinned_version(p.version)
+        if self.packages and any(not pinned(p) for p in self.packages):
+          raise datastore_errors.BadValueError(
+            'an idempotent task cannot have unpinned packages; '
+            'use instance IDs or tags as package versions')
+
+
 
 class TaskRequest(ndb.Model):
   """Contains a user request.
@@ -565,7 +620,7 @@ def make_request(request, is_bot_or_admin):
 
   If parent_task_id is set, properties for the parent are used:
   - priority: defaults to parent.priority - 1
-  - user: overriden by parent.user
+  - user: overridden by parent.user
 
   """
   assert request.__class__ is TaskRequest