swarming: custom cipd package paths

appengine/swarming/server/task_request.py

 # coding: utf-8
 # Copyright 2014 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Tasks definition.
 
 Each user request creates a new TaskRequest. The TaskRequest instance saves the
 metadata of the request, e.g. who requested it, when why, etc. It links to the
 actual data of the request in a TaskProperties. The TaskProperties represents
@@ skipping 28 matching lines @@
     +---------------------+
                ^
                |
     <See task_to_run.py and task_result.py>
 
 TaskProperties is embedded in TaskRequest. TaskProperties is still declared as a
 separate entity to clearly declare the boundary for task request deduplication.
 """
 
 
+import collections
 import datetime
 import hashlib
 import random
 import re
 import urlparse
 
 from google.appengine.api import datastore_errors
 from google.appengine.ext import ndb
 
 from components import auth
@@ skipping 154 matching lines @@
               prop._name, value))
 
 
 def _validate_package_version(prop, value):
   """Validates a CIPD package version."""
   if not cipd.is_valid_version(value):
     raise datastore_errors.BadValueError(
         '%s must be a valid package version "%s"' % (prop._name, value))
 
 
+def _validate_package_path(_prop, path):
+  """Validates a CIPD installation path."""
+  if not path:
+    raise datastore_errors.BadValueError(
+        'CIPD package path is required. Use "." to install to run dir.')
+  if '\\' in path:
+    raise datastore_errors.BadValueError(
+        'CIPD package path cannot contain \\. On Windows forward-slashes '
+        'will be replaced with back-slashes.')
+  if '..' in path.split('/'):
+    raise datastore_errors.BadValueError(
+        'CIPD package path cannot contain "..".')
+  if path.startswith('/'):
+    raise datastore_errors.BadValueError(
+        'CIPD package path cannot start with "/".')
+
+
 ### Models.
 
 
 class FilesRef(ndb.Model):
   """Defines a data tree reference, normally a reference to a .isolated file."""
 
   # TODO(maruel): make this class have one responsibility. Currently it is used
   # in two modes:
   # - a reference to a tree, as class docstring says.
   # - input/output settings in TaskProperties.
@@ skipping 19 matching lines @@
   A part of TaskProperties.
   """
   # Package name template. May use cipd.ALL_PARAMS.
   # Most users will specify ${platform} parameter.
   package_name = ndb.StringProperty(
       indexed=False, validator=_validate_package_name_template)
   # Package version that is valid for all packages matched by package_name.
   # Most users will specify tags.
   version = ndb.StringProperty(
       indexed=False, validator=_validate_package_version)
+  # Path to dir, relative to the run dir, where to install the package.
+  # If empty, the package will be installed in run dir.
+  path = ndb.StringProperty(indexed=False, validator=_validate_package_path)
 
   def __str__(self):
     return '%s:%s' % (self.package_name, self.version)
 
   def _pre_put_hook(self):
     super(CipdPackage, self)._pre_put_hook()
     if not self.package_name:
       raise datastore_errors.BadValueError('CIPD package name is required')
     if not self.version:
       raise datastore_errors.BadValueError('CIPD package version is required')
 
 
 class CipdInput(ndb.Model):
   """Specifies which CIPD client and packages to install, from which server.
 
   A part of TaskProperties.
   """
   # URL of the CIPD server. Must start with "https://" or "http://".
   server = ndb.StringProperty(indexed=False, validator=_validate_url)
 
   # CIPD package of CIPD client to use.
   # client_package.version is required.
+  # client_package.path must be None.
   client_package = ndb.LocalStructuredProperty(CipdPackage)
 
   # List of packages to install in $CIPD_PATH prior task execution.
   packages = ndb.LocalStructuredProperty(CipdPackage, repeated=True)
 
   def _pre_put_hook(self):
     if not self.server:
       raise datastore_errors.BadValueError('cipd server is required')
     if not self.client_package:
       raise datastore_errors.BadValueError('client_package is required')
+    if self.client_package.path:
+      raise datastore_errors.BadValueError('client_package.path must be unset')
     self.client_package._pre_put_hook()
 
     if not self.packages:
       raise datastore_errors.BadValueError(
           'cipd_input cannot have an empty package list')
 
     package_names = set()
     for p in self.packages:
       p._pre_put_hook()
       if p.package_name in package_names:
         raise datastore_errors.BadValueError(
            'package %s is specified more than once' % p.package_name)
       package_names.add(p.package_name)
     self.packages.sort(key=lambda p: p.package_name)
 
+  def packages_grouped_by_path(self):
+    """Returns sorted [(path), [package]) list. Used by user_task.html."""
+    packages = collections.defaultdict(list)
+    for p in self.packages:
+      packages[p.path].append(p)
+    for pkgs in packages.itervalues():
+      pkgs.sort()
+    return sorted(packages.iteritems())
+
 
 class TaskProperties(ndb.Model):
   """Defines all the properties of a task to be run on the Swarming
   infrastructure.
 
   This entity is not saved in the DB as a standalone entity, instead it is
   embedded in a TaskRequest.
 
   This model is immutable.
 
@@ skipping 450 matching lines @@
   _put_request(request)
   return request
 
 
 def validate_priority(priority):
   """Throws ValueError if priority is not a valid value."""
   if 0 > priority or MAXIMUM_PRIORITY < priority:
     raise datastore_errors.BadValueError(
         'priority (%d) must be between 0 and %d (inclusive)' %
         (priority, MAXIMUM_PRIORITY))