swarming: custom cipd package paths

appengine/swarming/server/task_request.py

 # coding: utf-8
 # Copyright 2014 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Tasks definition.
 
 Each user request creates a new TaskRequest. The TaskRequest instance saves the
 metadata of the request, e.g. who requested it, when why, etc. It links to the
 actual data of the request in a TaskProperties. The TaskProperties represents
@@ skipping 28 matching lines @@
     +---------------------+
                ^
                |
     <See task_to_run.py and task_result.py>
 
 TaskProperties is embedded in TaskRequest. TaskProperties is still declared as a
 separate entity to clearly declare the boundary for task request deduplication.
 """
 
 
+import collections
 import datetime
 import hashlib
 import random
 import re
 import urlparse
 
 from google.appengine.api import datastore_errors
 from google.appengine.ext import ndb
 
 from components import auth
@@ skipping 193 matching lines @@
   A part of TaskProperties.
   """
   # Package name template. May use cipd.ALL_PARAMS.
   # Most users will specify ${platform} parameter.
   package_name = ndb.StringProperty(
       indexed=False, validator=_validate_package_name_template)
   # Package version that is valid for all packages matched by package_name.
   # Most users will specify tags.
   version = ndb.StringProperty(
       indexed=False, validator=_validate_package_version)
+  # Path to dir, relative to the run dir, where to install the package.
+  # If empty, the package will be installed in run dir.
+  path = ndb.StringProperty(indexed=False)
 
   def __str__(self):
     return '%s:%s' % (self.package_name, self.version)
 
   def _pre_put_hook(self):
     super(CipdPackage, self)._pre_put_hook()
     if not self.package_name:
       raise datastore_errors.BadValueError('CIPD package name is required')
     if not self.version:
       raise datastore_errors.BadValueError('CIPD package version is required')
 
+    if self.path:

[M-A Ruel, 2016/06/15 17:28:27]

Use a validator instead.

[nodir, 2016/06/15 17:53:42]

Done.

+      if '\\' in self.path:
+        raise datastore_errors.BadValueError(
+            'CIPD package path cannot contain \\. On Windows forward-slashes '
+            'will be replaced with back-slashes.')
+      if '..' in self.path.split('/'):
+        raise datastore_errors.BadValueError(
+            'CIPD package path cannot contain "..".')
+      if self.path.startswith('/'):
+        raise datastore_errors.BadValueError(
+            'CIPD package path cannot start with "/".')
+
 
 class CipdInput(ndb.Model):
   """Specifies which CIPD client and packages to install, from which server.
 
   A part of TaskProperties.
   """
   # URL of the CIPD server. Must start with "https://" or "http://".
   server = ndb.StringProperty(indexed=False, validator=_validate_url)
 
   # CIPD package of CIPD client to use.
   # client_package.version is required.
+  # client_package.path must be None.
   client_package = ndb.LocalStructuredProperty(CipdPackage)
 
   # List of packages to install in $CIPD_PATH prior task execution.
   packages = ndb.LocalStructuredProperty(CipdPackage, repeated=True)
 
   def _pre_put_hook(self):
     if not self.server:
       raise datastore_errors.BadValueError('cipd server is required')
     if not self.client_package:
       raise datastore_errors.BadValueError('client_package is required')
+    if self.client_package.path:
+      raise datastore_errors.BadValueError('client_package.path must be unset')
     self.client_package._pre_put_hook()
 
     if not self.packages:
       raise datastore_errors.BadValueError(
           'cipd_input cannot have an empty package list')
 
     package_names = set()
     for p in self.packages:
       p._pre_put_hook()
       if p.package_name in package_names:
         raise datastore_errors.BadValueError(
            'package %s is specified more than once' % p.package_name)
       package_names.add(p.package_name)
     self.packages.sort(key=lambda p: p.package_name)
 
+  def packages_grouped_by_path(self):
+    """Returns sorted [(path), [package]) list. Used by user_task.html."""
+    packages = collections.defaultdict(list)
+    for p in self.packages:
+      packages[p.path or '.'].append(p)

[M-A Ruel, 2016/06/15 17:28:27]

I don't like that path=None and path='.' have the same outcome. This means one
can specify the same task in two different ways and get different
properties_hash. I prefer to unconditionally store '.', never None. This
simplify the code, as in user_task.html.

[nodir, 2016/06/15 17:53:41]

Good point, done.

+    for pkgs in packages.values():

[M-A Ruel, 2016/06/15 17:28:27]

itervalues()

[nodir, 2016/06/15 17:53:42]

Done.

+      pkgs.sort()
+    return sorted(packages.iteritems())
 

[M-A Ruel, 2016/06/15 17:28:27]

2 lines

[nodir, 2016/06/15 17:53:42]

Done.

 class TaskProperties(ndb.Model):
   """Defines all the properties of a task to be run on the Swarming
   infrastructure.
 
   This entity is not saved in the DB as a standalone entity, instead it is
   embedded in a TaskRequest.
 
   This model is immutable.
 
   New-style TaskProperties supports invocation of run_isolated. When this
@@ skipping 449 matching lines @@
   _put_request(request)
   return request
 
 
 def validate_priority(priority):
   """Throws ValueError if priority is not a valid value."""
   if 0 > priority or MAXIMUM_PRIORITY < priority:
     raise datastore_errors.BadValueError(
         'priority (%d) must be between 0 and %d (inclusive)' %
         (priority, MAXIMUM_PRIORITY))