Complete CIPD integration

appengine/swarming/swarming_bot/bot_code/task_runner.py

 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Runs a Swarming task.
 
 Downloads all the necessary files to run the task, executes the command and
 streams results back to the Swarming server.
 
 The process exit code is 0 when the task was executed, even if the task itself
@@ skipping 82 matching lines @@
           '-I', task_details.isolated['server'].encode('utf-8'),
           '--namespace', task_details.isolated['namespace'].encode('utf-8'),
         ])
     isolated_input = task_details.isolated.get('input')
     if isolated_input:
       cmd.extend(
           [
             '--isolated', isolated_input,
           ])
 
+  if task_details.cipd_input and task_details.cipd_input.get('packages'):
+    to_pkg = lambda p: '%s:%s' % (p['package_name'], p['version'])
+    cmd.extend(
+        [
+          '--cipd-cache', os.path.join(bot_dir, 'cipd_cache'),
+          '--cipd-client-package', to_pkg(

[M-A Ruel, 2016/06/06 23:54:28]

I'd prefer the to_pkg( on the next line then the function call wouldn't be
split.

[nodir, 2016/06/09 23:36:28]

Done.

+              task_details.cipd_input.get('client_package')),
+          '--cipd-server', task_details.cipd_input.get('server'),
+        ])
+    for p in task_details.cipd_input['packages']:
+      cmd.extend(['--cipd-package', to_pkg(p)])
+
   cmd.extend(
       [
         '--json', isolated_result,
         '--log-file', os.path.join(bot_dir, 'logs', 'run_isolated.log'),
-        '--cache', os.path.join(bot_dir, 'cache'),
+        '--cache', os.path.join(bot_dir, 'isolated_cache'),
         '--root-dir', os.path.join(work_dir, 'isolated'),
       ])
   if min_free_space:
     cmd.extend(('--min-free-space', str(min_free_space)))
 
   if task_details.hard_timeout:
     cmd.extend(('--hard-timeout', str(task_details.hard_timeout)))
   if task_details.grace_period:
     cmd.extend(('--grace-period', str(task_details.grace_period)))
 
@@ skipping 15 matching lines @@
     # Get all the data first so it fails early if the task details is invalid.
     self.bot_id = data['bot_id']
 
     # Raw command. Only self.command or self.isolated.input can be set.
     self.command = data['command'] or []
 
     # Isolated command. Is a serialized version of task_request.FilesRef.
     self.isolated = data['isolated']
     self.extra_args = data['extra_args']
 
+    self.cipd_input = data.get('cipd_input')
+
     self.env = {
       k.encode('utf-8'): v.encode('utf-8') for k, v in data['env'].iteritems()
     }
     self.grace_period = data['grace_period']
     self.hard_timeout = data['hard_timeout']
     self.io_timeout = data['io_timeout']
     self.task_id = data['task_id']
 
 
 class MustExit(Exception):
@@ skipping 306 matching lines @@
         if run_isolated_result.get('duration') is not None:
           # Calculate the real task duration as measured by run_isolated and
           # calculate the remaining overhead.
           params['bot_overhead'] = params['duration']
           params['duration'] = run_isolated_result['duration']
           params['bot_overhead'] -= params['duration']
           params['bot_overhead'] -= run_isolated_result.get(
               'download', {}).get('duration', 0)
           params['bot_overhead'] -= run_isolated_result.get(
               'upload', {}).get('duration', 0)
+          params['bot_overhead'] -= run_isolated_result.get(
+              'cipd', {}).get('duration', 0)
           if params['bot_overhead'] < 0:
             params['bot_overhead'] = 0
         isolated_stats = run_isolated_result.get('stats', {}).get('isolated')
         if isolated_stats:
           params['isolated_stats'] = isolated_stats
+        cipd_stats = run_isolated_result.get('stats', {}).get('cipd')
+        if cipd_stats:
+          params['cipd_stats'] = cipd_stats
     except (IOError, OSError, ValueError) as e:
       logging.error('Swallowing error: %s', e)
       if not must_signal_internal_failure:
         must_signal_internal_failure = str(e)
     # TODO(maruel): Send the internal failure here instead of sending it through
     # bot_main, this causes a race condition.
     if exit_code is None:
       exit_code = -1
     params['hard_timeout'] = had_hard_timeout
     post_update(swarming_server, params, exit_code, stdout, output_chunk_start)
@@ skipping 37 matching lines @@
   if options.start > now:
     options.start = now
 
   try:
     load_and_run(
         options.in_file, options.swarming_server, options.cost_usd_hour,
         options.start, options.out_file, options.min_free_space)
     return 0
   finally:
     logging.info('quitting')