Replace custom blob gRPC API with ByteStream

appengine/swarming/server/bot_code_test.py

 #!/usr/bin/env python
 # Copyright 2014 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import StringIO
 import logging
 import os
 import re
 import subprocess
 import sys
 import tempfile
+import time
 import unittest
 import zipfile
 
 import test_env
 test_env.setup_test_env()
 
 from components import auth
 from test_support import test_case
 
 from server import bot_archive
@@ skipping 10 matching lines @@
 
 class BotManagementTest(test_case.TestCase):
   def setUp(self):
     super(BotManagementTest, self).setUp()
     self.testbed.init_user_stub()
 
     self.mock(
         auth, 'get_current_identity',
         lambda: auth.Identity(auth.IDENTITY_USER, 'joe@localhost'))
 
+
   def test_get_bootstrap(self):
     def get_self_config_mock(path, revision=None, store_last_good=False):
       self.assertEqual('scripts/bootstrap.py', path)
       self.assertEqual(None, revision)
       self.assertEqual(True, store_last_good)
       return None, 'foo bar'
     self.mock(config, 'get_self_config', get_self_config_mock)
     f = bot_code.get_bootstrap('localhost', 'token', None)
     expected = (
       '#!/usr/bin/env python\n'
@@ skipping 34 matching lines @@
 
   def test_get_bot_version(self):
     actual, additionals = bot_code.get_bot_version('http://localhost')
     self.assertTrue(re.match(r'^[0-9a-f]{64}$', actual), actual)
     expected = {
       'config/bot_config.py': bot_code.get_bot_config().content,
     }
     self.assertEqual(expected, additionals)
 
   def test_get_swarming_bot_zip(self):
+    local_mc = {'store': {}, 'reads': 0, 'writes': 0}
+
+    def mock_memcache_get(version, desc):
+      result = local_mc['store'].get(bot_code.bot_key(version, desc))
+      if result is not None:
+        local_mc['reads'] += 1
+      return result
+
+    def mock_memcache_async_get(version, desc, part=None):

[Vadim Sh., 2017/06/27 19:12:49]

this will be more correct mock:

@ndb.tasklet
def mock_memcache_get_async(....):
  raise ndb.Return(local_mc['store'].get(....))

Or equivalently:

def mock_memcache_get_async(....):
  fut = ndb.Future()
  fut.set_result(local_mc['store'].get(....))
  return fut

[aludwin, 2017/06/27 20:10:17]

How would I do the counting in this case? I'd like to delay the counts until
someone says "get_result".

[Vadim Sh., 2017/06/27 21:14:29]

yeah, this will not be possible.

But why counting number of initiated calls is any worse? e.g 

@ndb.tasklet
def mock_memcache_get(...):
  local_mc['reads'] += 1
  raise ndb.Return(local_mc['store'].get(....))

[aludwin, 2017/06/28 13:23:53]

Just thought it would be nice to ensure that everything happened at the right
time - but I've changed it to look like this.

+      class Future:
+        def __init__(self, value):
+          self._value = value
+        def get_result(self):
+          if self._value is not None:
+            local_mc['reads'] += 1
+          return self._value
+      value = local_mc['store'].get(bot_code.bot_key(version, desc, part))
+      return Future(value)
+
+    def mock_memcache_async_set(value, version, desc, part=None):
+      class Future:
+        def __init__(self, key, value):
+          self._key = key
+          self._value = value
+        @property
+        def state(self):
+          local_mc['writes'] += 1
+          local_mc['store'][key] = value
+          return bot_code.ndb.Future.FINISHING
+      key = bot_code.bot_key(version, desc, part)
+      return Future(key, value)
+
+    self.mock(bot_code, 'bot_memcache_async_set', mock_memcache_async_set)
+    self.mock(bot_code, 'bot_memcache_async_get', mock_memcache_async_get)
+    self.mock(bot_code, 'bot_memcache_get', mock_memcache_get)
+    self.mock(bot_code, 'MAX_MEMCACHED_SIZE_BYTES', 100000)
+
+    self.assertEqual(0, local_mc['writes'])
     zipped_code = bot_code.get_swarming_bot_zip('http://localhost')
+    self.assertEqual(0, local_mc['reads'])
+    self.assertNotEqual(0, local_mc['writes'])
+
+    # Make sure that we read from memcached if we get it again
+    zipped_code_copy = bot_code.get_swarming_bot_zip('http://localhost')
+    self.assertEqual(local_mc['writes'], local_mc['reads'])
+    # Why not assertEqual? Don't want to dump ~1MB of data if this fails.
+    self.assertTrue(zipped_code == zipped_code_copy)
+
     # Ensure the zip is valid and all the expected files are present.
     with zipfile.ZipFile(StringIO.StringIO(zipped_code), 'r') as zip_file:
       for i in bot_archive.FILES:
         with zip_file.open(i) as f:
           content = f.read()
           if os.path.basename(i) != '__init__.py':
             self.assertTrue(content, i)
 
     temp_dir = tempfile.mkdtemp(prefix='swarming')
     try:
       # Try running the bot and ensure it can import the required files. (It
       # would crash if it failed to import them).
       bot_path = os.path.join(temp_dir, 'swarming_bot.zip')
       with open(bot_path, 'wb') as f:
         f.write(zipped_code)
       proc = subprocess.Popen(
           [sys.executable, bot_path, 'start_bot', '-h'],
           cwd=temp_dir,
           stdout=subprocess.PIPE,
           stderr=subprocess.STDOUT)
       out = proc.communicate()[0]
       self.assertEqual(0, proc.returncode, out)
     finally:
       file_path.rmtree(temp_dir)
 
+
   def test_bootstrap_token(self):
     tok = bot_code.generate_bootstrap_token()
     self.assertEqual(
         {'for': 'user:joe@localhost'}, bot_code.validate_bootstrap_token(tok))
 
 
 if __name__ == '__main__':
   fix_encoding.fix_encoding()
   logging.basicConfig(
       level=logging.DEBUG if '-v' in sys.argv else logging.ERROR)
   if '-v' in sys.argv:
     unittest.TestCase.maxDiff = None
   unittest.main()