swarming: custom cipd package paths

appengine/swarming/swarming_bot/bot_code/task_runner.py

 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Runs a Swarming task.
 
 Downloads all the necessary files to run the task, executes the command and
 streams results back to the Swarming server.
 
 The process exit code is 0 when the task was executed, even if the task itself
 failed. If there's any failure in the setup or teardown, like invalid packet
 response, failure to contact the server, etc, a non zero exit code is used. It's
 up to the calling process (bot_main.py) to signal that there was an internal
 failure and to cancel this task run and ask the server to retry it.
 """
 
 import base64
 import json
 import logging
 import optparse
 import os
 import signal
 import sys
 import time
 
+from utils import file_path
 from utils import net
 from utils import on_error
 from utils import subprocess42
 from utils import zip_package
 
 import bot_auth
 import file_reader
 
 
 # Path to this file or the zip containing this file.
@@ skipping 38 matching lines @@
 
 def get_run_isolated():
   """Returns the path to itself to run run_isolated.
 
   Mocked in test to point to the real run_isolated.py script.
   """
   return [sys.executable, THIS_FILE, 'run_isolated']
 
 
 def get_isolated_cmd(
-    work_dir, task_details, isolated_result, min_free_space, bot_file):
+    work_dir, task_details, isolated_result, bot_file, package_list,
+    min_free_space):
   """Returns the command to call run_isolated. Mocked in tests."""
   assert (bool(task_details.command) !=
           bool(task_details.isolated and task_details.isolated.get('input')))
   bot_dir = os.path.dirname(work_dir)
   if os.path.isfile(isolated_result):
     os.remove(isolated_result)
   cmd = get_run_isolated()
 
   if task_details.isolated:
     cmd.extend(
         [
           '-I', task_details.isolated['server'].encode('utf-8'),
           '--namespace', task_details.isolated['namespace'].encode('utf-8'),
         ])
     isolated_input = task_details.isolated.get('input')
     if isolated_input:
       cmd.extend(
           [
             '--isolated', isolated_input,
           ])
 
   if task_details.cipd_input and task_details.cipd_input.get('packages'):
-    to_pkg = lambda p: '%s:%s' % (p['package_name'], p['version'])
+    package_json = {
+      # Package JSON object format matches.

[M-A Ruel, 2016/06/15 17:28:27]

Can't parse this sentence

[nodir, 2016/06/15 17:53:42]

Done.

+      'packages': task_details.cipd_input['packages'],
+    }
+    with open(package_list, 'w') as f:

[M-A Ruel, 2016/06/15 17:28:27]

'wb'

[nodir, 2016/06/15 17:53:42]

Done.

+      json.dump(package_json, f)
     cmd.extend(
         [
           '--cipd-cache', os.path.join(bot_dir, 'cipd_cache'),
           '--cipd-client-package',
-          to_pkg(task_details.cipd_input.get('client_package')),
+          task_details.cipd_input['client_package']['package_name'],
+          '--cipd-client-version',
+          task_details.cipd_input['client_package']['version'],
+          '--cipd-package-list', package_list,
           '--cipd-server', task_details.cipd_input.get('server'),
         ])
-    for p in task_details.cipd_input['packages']:
-      cmd.extend(['--cipd-package', to_pkg(p)])
 
   cmd.extend(
       [
         '--json', isolated_result,
         '--log-file', os.path.join(bot_dir, 'logs', 'run_isolated.log'),
         '--cache', os.path.join(bot_dir, 'isolated_cache'),
         '--root-dir', os.path.join(work_dir, 'isolated'),
       ])
   if min_free_space:
     cmd.extend(('--min-free-space', str(min_free_space)))
@@ skipping 235 matching lines @@
   # Signal the command is about to be started.
   last_packet = start = now = monotonic_time()
   params = {
     'cost_usd': cost_usd_hour * (now - task_start) / 60. / 60.,
     'id': task_details.bot_id,
     'task_id': task_details.task_id,
   }
   post_update(swarming_server, headers_cb(), params, None, '', 0)
 
   isolated_result = os.path.join(work_dir, 'isolated_result.json')
+  package_list = os.path.join(work_dir, 'package_list.json')
   cmd = get_isolated_cmd(
-      work_dir, task_details, isolated_result, min_free_space, bot_file)
+      work_dir, task_details, isolated_result, bot_file, package_list,
+      min_free_space)
   # Hard timeout enforcement is deferred to run_isolated. Grace is doubled to
   # give one 'grace_period' slot to the child process and one slot to upload
   # the results back.
   task_details.hard_timeout = 0
   if task_details.grace_period:
     task_details.grace_period *= 2
 
   try:
     # TODO(maruel): Support both channels independently and display stderr in
     # red.
@@ skipping 173 matching lines @@
         swarming_server, headers_cb(), params, exit_code,
         stdout, output_chunk_start)
     return {
       u'exit_code': exit_code,
       u'hard_timeout': had_hard_timeout,
       u'io_timeout': had_io_timeout,
       u'must_signal_internal_failure': must_signal_internal_failure,
       u'version': OUT_VERSION,
     }
   finally:
-    try:
-      os.remove(isolated_result)
-    except OSError:
-      pass
+    file_path.try_remove(unicode(isolated_result))
+    file_path.try_remove(unicode(package_list))
     stop_headers_reader()
 
 
 def main(args):
   subprocess42.inhibit_os_error_reporting()
   parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
   parser.add_option('--in-file', help='Name of the request file')
   parser.add_option(
       '--out-file', help='Name of the JSON file to write a task summary to')
   parser.add_option(
@@ skipping 19 matching lines @@
     options.start = now
 
   try:
     load_and_run(
         options.in_file, options.swarming_server, options.cost_usd_hour,
         options.start, options.out_file, options.min_free_space,
         options.bot_file)
     return 0
   finally:
     logging.info('quitting')