Expose named caches as dimensions.

appengine/swarming/local_smoke_test.py

 #!/usr/bin/env python
 # Copyright 2014 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Integration test for the Swarming server, Swarming bot and Swarming client.
 
 It starts both a Swarming server and a Swarming bot and triggers tasks with the
 Swarming client to ensure the system works end to end.
 """
 
 import base64
 import json
 import logging
 import os
 import re
 import signal
 import socket
 import sys
 import tempfile
 import textwrap
+import time
 import unittest
 import urllib
 
 APP_DIR = os.path.dirname(os.path.abspath(__file__))
 BOT_DIR = os.path.join(APP_DIR, 'swarming_bot')
 CLIENT_DIR = os.path.join(APP_DIR, '..', '..', 'client')
 
 from tools import start_bot
 from tools import start_servers
 
@@ skipping 169 matching lines @@
       os.remove(tmp)
 
   def terminate(self, bot_id):
     return self._run('terminate', ['--wait', bot_id])
 
   def cleanup(self):
     if self._tmpdir:
       file_path.rmtree(self._tmpdir)
       self._tmpdir = None
 
+  def query_bot(self):
+    """Returns the bot's properties."""
+    data = json.loads(self._capture('query', ['bots/list', '--limit', '10']))
+    if not data.get('items'):
+      return None
+    assert len(data['items']) == 1
+    return data['items'][0]
+
   def dump_log(self):
     print >> sys.stderr, '-' * 60
     print >> sys.stderr, 'Client calls'
     print >> sys.stderr, '-' * 60
     for i in xrange(self._index):
       with open(os.path.join(self._tmpdir, 'client_%d.log' % i), 'rb') as f:
         log = f.read().strip('\n')
       for l in log.splitlines():
         sys.stderr.write('  %s\n' % l)
 
@@ skipping 13 matching lines @@
       '--verbose',
     ] + args
     with open(name, 'wb') as f:
       f.write('\nRunning: %s\n' % ' '.join(cmd))
       f.flush()
       p = subprocess42.Popen(
           cmd, stdout=f, stderr=subprocess42.STDOUT, cwd=CLIENT_DIR)
       p.communicate()
       return p.returncode
 
+  def _capture(self, command, args):
+    cmd = [
+      sys.executable, 'swarming.py', command, '-S', self._swarming_server,
+    ] + args
+    p = subprocess42.Popen(cmd, stdout=subprocess42.PIPE, cwd=CLIENT_DIR)
+    return p.communicate()[0]
+
 
 def gen_expected(**kwargs):
   expected = {
     u'abandoned_ts': None,
     u'bot_dimensions': None,
     u'bot_id': unicode(socket.getfqdn().split('.', 1)[0]),
     u'children_task_ids': [],
     u'cost_saved_usd': None,
     u'deduped_from': None,
     u'exit_codes': [0],
@@ skipping 375 matching lines @@
       self._run_isolated(
           hello_world, 'secret_bytes',
           ['--secret-bytes-path', tmp, '--', '${ISOLATED_OUTDIR}'],
           expected_summary, {os.path.join('0', 'sekret'): 'foobar\n'})
     finally:
       os.remove(tmp)
 
   def test_local_cache(self):
     # First task creates the cache, second copy the content to the output
     # directory. Each time it's the exact same script.
+    dimensions = {
+        i['key']: i['value'] for i in self.client.query_bot()['dimensions']}
+    self.assertEqual(set(self.dimensions), set(dimensions))
+    self.assertNotIn(u'cache', set(dimensions))
     script = '\n'.join((
       'import os, shutil, sys',
       'p = "p/b/a.txt"',
       'if not os.path.isfile(p):',
       '  with open(p, "wb") as f:',
       '    f.write("Yo!")',
       'else:',
       '  shutil.copy(p, sys.argv[1])',
       'print "hi"'))
     sizes = sorted([len(script), 200])
@@ skipping 36 matching lines @@
           u'initial_size': unicode(sum(sizes)),
           u'items_cold': [],
           u'items_hot': sizes,
         },
         u'isolated_upload': {
           u'items_cold': [3, 110],
           u'items_hot': [],
         },
       },
     )
+    # The previous task caused the bot to have a named cache.
+    expected_summary['bot_dimensions'][u'caches'] = [u'fuu']
     self._run_isolated(
         script, 'cache_second',
         ['--named-cache', 'fuu', 'p/b', '--', '${ISOLATED_OUTDIR}/yo'],
         expected_summary,
         {'0/yo': 'Yo!'})
 
+    # Check that the bot now has a cache dimension by independently querying.
+    expected = set(self.dimensions)
+    expected.add(u'caches')
+    dimensions = {
+        i['key']: i['value'] for i in self.client.query_bot()['dimensions']}
+    self.assertEqual(expected, set(dimensions))
+
   def _run_isolated(self, hello_world, name, args, expected_summary,
       expected_files, deduped=False, isolated_content=None):
     """Runs hello_world.py as an isolated file."""
     # Shared code for all test_isolated_* test cases.
     tmpdir = tempfile.mkdtemp(prefix='swarming_smoke')
     try:
       isolate_path = os.path.join(tmpdir, 'i.isolate')
       isolated_path = os.path.join(tmpdir, 'i.isolated')
       with open(isolate_path, 'wb') as f:
         json.dump(isolated_content or ISOLATE_HELLO_WORLD, f)
@@ skipping 132 matching lines @@
     servers.start()
     bot = start_bot.LocalBot(servers.swarming_server.url)
     Test.bot = bot
     bot.start()
     client = SwarmingClient(
         servers.swarming_server.url, servers.isolate_server.url)
     # Test cases only interract with the client; except for test_update_continue
     # which mutates the bot.
     Test.client = client
     Test.servers = servers
+    while not client.query_bot():
+      # Wait for the bot to come online helps when the unit test cases query the
+      # bot. It may takes a few loop.
+      time.sleep(0.1)
     failed = not unittest.main(exit=False).result.wasSuccessful()
 
     # Then try to terminate the bot sanely. After the terminate request
     # completed, the bot process should have terminated. Give it a few
     # seconds due to delay between sending the event that the process is
     # shutting down vs the process is shut down.
     if client.terminate(bot.bot_id) is not 0:
       print >> sys.stderr, 'swarming.py terminate failed'
       failed = True
     try:
       bot.wait(10)
     except subprocess42.TimeoutExpired:
       print >> sys.stderr, 'Bot is still alive after swarming.py terminate'
       failed = True
   except KeyboardInterrupt:
     print >> sys.stderr, '<Ctrl-C>'
     failed = True
     if bot is not None and bot.poll() is None:
       bot.kill()
       bot.wait()
   finally:
     cleanup(bot, client, servers, failed or verbose, leak)
   return int(failed)
 
 
 if __name__ == '__main__':
   sys.exit(main())