Complete CIPD integration

appengine/swarming/swarming_bot/bot_code/task_runner.py

 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Runs a Swarming task.
 
 Downloads all the necessary files to run the task, executes the command and
 streams results back to the Swarming server.
 
 The process exit code is 0 when the task was executed, even if the task itself
@@ skipping 85 matching lines @@
           '-I', task_details.isolated['server'].encode('utf-8'),
           '--namespace', task_details.isolated['namespace'].encode('utf-8'),
         ])
     isolated_input = task_details.isolated.get('input')
     if isolated_input:
       cmd.extend(
           [
             '--isolated', isolated_input,
           ])
 
+  if task_details.cipd_input and task_details.cipd_input.get('packages'):
+    to_pkg = lambda p: '%s:%s' % (p['package_name'], p['version'])
+    cmd.extend(
+        [
+          '--cipd-cache', os.path.join(bot_dir, 'cipd_cache'),
+          '--cipd-client-package',
+          to_pkg(task_details.cipd_input.get('client_package')),
+          '--cipd-server', task_details.cipd_input.get('server'),
+        ])
+    for p in task_details.cipd_input['packages']:
+      cmd.extend(['--cipd-package', to_pkg(p)])

[M-A Ruel, 2016/06/10 02:06:42]

(FYI only) Eventually we may have to pass this information through a file as we
may blow the 8kb limit (Thanks Windows). Should be fine initially but this could
blow up as people starts adding more packages with long URLs or whatnot. We
should be fine for a while.

+
   cmd.extend(
       [
         '--json', isolated_result,
         '--log-file', os.path.join(bot_dir, 'logs', 'run_isolated.log'),
-        '--cache', os.path.join(bot_dir, 'cache'),
+        '--cache', os.path.join(bot_dir, 'isolated_cache'),

[M-A Ruel, 2016/06/17 17:38:32]

Oh, this is problematic, now all the bots have a stale 'cache' directory and
this is getting filled up. It's kind of too late already as this change was
deployed.

[nodir, 2016/06/17 18:58:57]

I should not have done that

         '--root-dir', os.path.join(work_dir, 'isolated'),
       ])
   if min_free_space:
     cmd.extend(('--min-free-space', str(min_free_space)))
 
   if task_details.hard_timeout:
     cmd.extend(('--hard-timeout', str(task_details.hard_timeout)))
   if task_details.grace_period:
     cmd.extend(('--grace-period', str(task_details.grace_period)))
 
@@ skipping 15 matching lines @@
     # Get all the data first so it fails early if the task details is invalid.
     self.bot_id = data['bot_id']
 
     # Raw command. Only self.command or self.isolated.input can be set.
     self.command = data['command'] or []
 
     # Isolated command. Is a serialized version of task_request.FilesRef.
     self.isolated = data['isolated']
     self.extra_args = data['extra_args']
 
+    self.cipd_input = data.get('cipd_input')
+
     self.env = {
       k.encode('utf-8'): v.encode('utf-8') for k, v in data['env'].iteritems()
     }
     self.grace_period = data['grace_period']
     self.hard_timeout = data['hard_timeout']
     self.io_timeout = data['io_timeout']
     self.task_id = data['task_id']
 
 
 class MustExit(Exception):
@@ skipping 354 matching lines @@
         if run_isolated_result.get('duration') is not None:
           # Calculate the real task duration as measured by run_isolated and
           # calculate the remaining overhead.
           params['bot_overhead'] = params['duration']
           params['duration'] = run_isolated_result['duration']
           params['bot_overhead'] -= params['duration']
           params['bot_overhead'] -= run_isolated_result.get(
               'download', {}).get('duration', 0)
           params['bot_overhead'] -= run_isolated_result.get(
               'upload', {}).get('duration', 0)
+          params['bot_overhead'] -= run_isolated_result.get(
+              'cipd', {}).get('duration', 0)
           if params['bot_overhead'] < 0:
             params['bot_overhead'] = 0
         isolated_stats = run_isolated_result.get('stats', {}).get('isolated')
         if isolated_stats:
           params['isolated_stats'] = isolated_stats
+        cipd_stats = run_isolated_result.get('stats', {}).get('cipd')
+        if cipd_stats:
+          params['cipd_stats'] = cipd_stats
     except (IOError, OSError, ValueError) as e:
       logging.error('Swallowing error: %s', e)
       if not must_signal_internal_failure:
         must_signal_internal_failure = str(e)
     # TODO(maruel): Send the internal failure here instead of sending it through
     # bot_main, this causes a race condition.
     if exit_code is None:
       exit_code = -1
     params['hard_timeout'] = had_hard_timeout
     post_update(
@@ skipping 40 matching lines @@
   if options.start > now:
     options.start = now
 
   try:
     load_and_run(
         options.in_file, options.swarming_server, options.cost_usd_hour,
         options.start, options.out_file, options.min_free_space)
     return 0
   finally:
     logging.info('quitting')