Replace custom blob gRPC API with ByteStream

appengine/swarming/server/bot_code_test.py

Index: appengine/swarming/server/bot_code_test.py
diff --git a/appengine/swarming/server/bot_code_test.py b/appengine/swarming/server/bot_code_test.py
index 46218e519aea147fe0f27195af850d718d31d1c7..8ffeea3efc2c6fc947b20647340987b4e42539f9 100755
--- a/appengine/swarming/server/bot_code_test.py
+++ b/appengine/swarming/server/bot_code_test.py
@@ -10,6 +10,7 @@ import re
 import subprocess
 import sys
 import tempfile
+import time
 import unittest
 import zipfile
 
@@ -40,6 +41,7 @@ class BotManagementTest(test_case.TestCase):
         auth, 'get_current_identity',
         lambda: auth.Identity(auth.IDENTITY_USER, 'joe@localhost'))
 
+
   def test_get_bootstrap(self):
     def get_self_config_mock(path, revision=None, store_last_good=False):
       self.assertEqual('scripts/bootstrap.py', path)
@@ -94,7 +96,54 @@ class BotManagementTest(test_case.TestCase):
     self.assertEqual(expected, additionals)
 
   def test_get_swarming_bot_zip(self):
+    local_mc = {'store': {}, 'reads': 0, 'writes': 0}
+
+    def mock_memcache_get(version, desc):
+      result = local_mc['store'].get(bot_code.bot_key(version, desc))
+      if result is not None:
+        local_mc['reads'] += 1
+      return result
+
+    def mock_memcache_async_get(version, desc, part=None):

[Vadim Sh., 2017/06/27 19:12:49]

this will be more correct mock:

@ndb.tasklet
def mock_memcache_get_async(....):
  raise ndb.Return(local_mc['store'].get(....))

Or equivalently:

def mock_memcache_get_async(....):
  fut = ndb.Future()
  fut.set_result(local_mc['store'].get(....))
  return fut

[aludwin, 2017/06/27 20:10:17]

How would I do the counting in this case? I'd like to delay the counts until
someone says "get_result".

[Vadim Sh., 2017/06/27 21:14:29]

yeah, this will not be possible.

But why counting number of initiated calls is any worse? e.g 

@ndb.tasklet
def mock_memcache_get(...):
  local_mc['reads'] += 1
  raise ndb.Return(local_mc['store'].get(....))

[aludwin, 2017/06/28 13:23:53]

Just thought it would be nice to ensure that everything happened at the right
time - but I've changed it to look like this.

+      class Future:
+        def __init__(self, value):
+          self._value = value
+        def get_result(self):
+          if self._value is not None:
+            local_mc['reads'] += 1
+          return self._value
+      value = local_mc['store'].get(bot_code.bot_key(version, desc, part))
+      return Future(value)
+
+    def mock_memcache_async_set(value, version, desc, part=None):
+      class Future:
+        def __init__(self, key, value):
+          self._key = key
+          self._value = value
+        @property
+        def state(self):
+          local_mc['writes'] += 1
+          local_mc['store'][key] = value
+          return bot_code.ndb.Future.FINISHING
+      key = bot_code.bot_key(version, desc, part)
+      return Future(key, value)
+
+    self.mock(bot_code, 'bot_memcache_async_set', mock_memcache_async_set)
+    self.mock(bot_code, 'bot_memcache_async_get', mock_memcache_async_get)
+    self.mock(bot_code, 'bot_memcache_get', mock_memcache_get)
+    self.mock(bot_code, 'MAX_MEMCACHED_SIZE_BYTES', 100000)
+
+    self.assertEqual(0, local_mc['writes'])
     zipped_code = bot_code.get_swarming_bot_zip('http://localhost')
+    self.assertEqual(0, local_mc['reads'])
+    self.assertNotEqual(0, local_mc['writes'])
+
+    # Make sure that we read from memcached if we get it again
+    zipped_code_copy = bot_code.get_swarming_bot_zip('http://localhost')
+    self.assertEqual(local_mc['writes'], local_mc['reads'])
+    # Why not assertEqual? Don't want to dump ~1MB of data if this fails.
+    self.assertTrue(zipped_code == zipped_code_copy)
+
     # Ensure the zip is valid and all the expected files are present.
     with zipfile.ZipFile(StringIO.StringIO(zipped_code), 'r') as zip_file:
       for i in bot_archive.FILES:
@@ -120,6 +169,7 @@ class BotManagementTest(test_case.TestCase):
     finally:
       file_path.rmtree(temp_dir)
 
+
   def test_bootstrap_token(self):
     tok = bot_code.generate_bootstrap_token()
     self.assertEqual(