swarming: Fix named cache support.

client/swarming.py

 #!/usr/bin/env python
 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Client tool to trigger tasks or retrieve results from a Swarming server."""
 
-__version__ = '0.8.8'
+__version__ = '0.8.9'
 
 import collections
 import datetime
 import json
 import logging
 import optparse
 import os
 import subprocess
 import sys
 import threading
@@ skipping 139 matching lines @@
       'isolated',
       'isolatedserver',
       'namespace',
     ])
 
 
 # See ../appengine/swarming/swarming_rpcs.py.
 TaskProperties = collections.namedtuple(
     'TaskProperties',
     [
+      'caches',
       'cipd_input',
       'command',
       'dimensions',
       'env',
       'execution_timeout_secs',
       'extra_args',
       'grace_period_secs',
       'idempotent',
       'inputs_ref',
       'io_timeout_secs',
@@ skipping 702 matching lines @@
 
 ### Commands.
 
 
 def abort_task(_swarming, _manifest):
   """Given a task manifest that was triggered, aborts its execution."""
   # TODO(vadimsh): No supported by the server yet.
 
 
 def add_filter_options(parser):
-  parser.filter_group = optparse.OptionGroup(parser, 'Filtering slaves')
+  parser.filter_group = optparse.OptionGroup(parser, 'Bot selection')
   parser.filter_group.add_option(
       '-d', '--dimension', default=[], action='append', nargs=2,
       dest='dimensions', metavar='FOO bar',
       help='dimension to filter on')
   parser.add_option_group(parser.filter_group)
 
 
 def add_sharding_options(parser):
   parser.sharding_group = optparse.OptionGroup(parser, 'Sharding options')
   parser.sharding_group.add_option(
       '--shards', type='int', default=1,
       help='Number of shards to trigger and collect.')
   parser.add_option_group(parser.sharding_group)
 
 
 def add_trigger_options(parser):
   """Adds all options to trigger a task on Swarming."""
   isolateserver.add_isolate_server_options(parser)
   add_filter_options(parser)
 
-  parser.task_group = optparse.OptionGroup(parser, 'Task properties')
-  parser.task_group.add_option(
+  group = optparse.OptionGroup(parser, 'Task properties')
+  group.add_option(
       '-s', '--isolated',
       help='Hash of the .isolated to grab from the isolate server')
-  parser.task_group.add_option(
+  group.add_option(
       '-e', '--env', default=[], action='append', nargs=2, metavar='FOO bar',
       help='Environment variables to set')
-  parser.task_group.add_option(
+  group.add_option(
+      '--idempotent', action='store_true', default=False,
+      help='When set, the server will actively try to find a previous task '
+           'with the same parameter and return this result instead if possible')
+  group.add_option(
+      '--secret-bytes-path',
+      help='The optional path to a file containing the secret_bytes to use with'
+           'this task.')
+  group.add_option(
+      '--hard-timeout', type='int', default=60*60,
+      help='Seconds to allow the task to complete.')
+  group.add_option(
+      '--io-timeout', type='int', default=20*60,
+      help='Seconds to allow the task to be silent.')
+  group.add_option(
+      '--raw-cmd', action='store_true', default=False,
+      help='When set, the command after -- is used as-is without run_isolated. '
+           'In this case, no .isolated file is expected.')
+  group.add_option(
+      '--cipd-package', action='append', default=[],
+      help='CIPD packages to install on the Swarming bot.  Uses the format: '
+           'path:package_name:version')
+  group.add_option(
+      '--named-cache', action='append', nargs=2, default=[],
+      help='"<name> <relpath>" items to keep a persistent bot managed cache')
+  group.add_option(
+      '--service-account',
+      help='Name of a service account to run the task as. Only literal "bot" '
+           'string can be specified currently (to run the task under bot\'s '
+           'account). Don\'t use task service accounts if not given '
+           '(default).')
+  group.add_option(
+      '-o', '--output', action='append', default=[],
+      help='A list of files to return in addition to those written to'
+           '$(ISOLATED_OUTDIR). An error will occur if a file specified by'
+           'this option is also written directly to $(ISOLATED_OUTDIR).')
+  parser.add_option_group(group)
+
+  group = optparse.OptionGroup(parser, 'Task request')
+  group.add_option(
       '--priority', type='int', default=100,
       help='The lower value, the more important the task is')
-  parser.task_group.add_option(
+  group.add_option(
       '-T', '--task-name',
       help='Display name of the task. Defaults to '
            '<base_name>/<dimensions>/<isolated hash>/<timestamp> if an '
            'isolated file is provided, if a hash is provided, it defaults to '
            '<user>/<dimensions>/<isolated hash>/<timestamp>')
-  parser.task_group.add_option(
+  group.add_option(
       '--tags', action='append', default=[],
       help='Tags to assign to the task.')
-  parser.task_group.add_option(
+  group.add_option(
       '--user', default='',
       help='User associated with the task. Defaults to authenticated user on '
            'the server.')
-  parser.task_group.add_option(
-      '--idempotent', action='store_true', default=False,
-      help='When set, the server will actively try to find a previous task '
-           'with the same parameter and return this result instead if possible')
-  parser.task_group.add_option(
-      '--secret-bytes-path',
-      help='The optional path to a file containing the secret_bytes to use with'
-           'this task.')
-  parser.task_group.add_option(
+  group.add_option(
       '--expiration', type='int', default=6*60*60,
       help='Seconds to allow the task to be pending for a bot to run before '
            'this task request expires.')
-  parser.task_group.add_option(
+  group.add_option(
       '--deadline', type='int', dest='expiration',
       help=optparse.SUPPRESS_HELP)
-  parser.task_group.add_option(
-      '--hard-timeout', type='int', default=60*60,
-      help='Seconds to allow the task to complete.')
-  parser.task_group.add_option(
-      '--io-timeout', type='int', default=20*60,
-      help='Seconds to allow the task to be silent.')
-  parser.task_group.add_option(
-      '--raw-cmd', action='store_true', default=False,
-      help='When set, the command after -- is used as-is without run_isolated. '
-           'In this case, no .isolated file is expected.')
-  parser.task_group.add_option(
-      '--cipd-package', action='append', default=[],
-      help='CIPD packages to install on the Swarming bot.  Uses the format: '
-           'path:package_name:version')
-  parser.task_group.add_option(
-      '--service-account',
-      help='Name of a service account to run the task as. Only literal "bot" '
-           'string can be specified currently (to run the task under bot\'s '
-           'account). Don\'t use task service accounts if not given '
-           '(default).')
-  parser.task_group.add_option(
-      '-o', '--output', action='append', default=[],
-      help='A list of files to return in addition to those written to'
-           '$(ISOLATED_OUTDIR). An error will occur if a file specified by'
-           'this option is also written directly to $(ISOLATED_OUTDIR).')
-  parser.add_option_group(parser.task_group)
+  parser.add_option_group(group)
 
 
 def process_trigger_options(parser, options, args):
   """Processes trigger options and does preparatory steps.
 
   Uploads files to isolate server and generates service account tokens if
   necessary.
   """
   options.dimensions = dict(options.dimensions)
   options.env = dict(options.env)
@@ skipping 37 matching lines @@
     cipd_input = CipdInput(
         client_package=None,
         packages=cipd_packages,
         server=None)
 
   secret_bytes = None
   if options.secret_bytes_path:
     with open(options.secret_bytes_path, 'r') as f:
       secret_bytes = f.read().encode('base64')
 
+  caches = [
+    {u'name': unicode(i[0]), u'path': unicode(i[1])}
+    for i in options.named_cache
+  ]
   # If inputs_ref.isolated is used, command is actually extra_args.
   # Otherwise it's an actual command to run.
   isolated_input = inputs_ref and inputs_ref.isolated
   properties = TaskProperties(
+      caches=caches,
       cipd_input=cipd_input,
       command=None if isolated_input else command,
       dimensions=options.dimensions,
       env=options.env,
       execution_timeout_secs=options.hard_timeout,
       extra_args=command if isolated_input else None,
       grace_period_secs=30,
       idempotent=options.idempotent,
       inputs_ref=inputs_ref,
       io_timeout_secs=options.io_timeout,
@@ skipping 609 matching lines @@
   dispatcher = subcommand.CommandDispatcher(__name__)
   return dispatcher.execute(OptionParserSwarming(version=__version__), args)
 
 
 if __name__ == '__main__':
   subprocess42.inhibit_os_error_reporting()
   fix_encoding.fix_encoding()
   tools.disable_buffering()
   colorama.init()
   sys.exit(main(sys.argv[1:]))