Add ability to linkify hashes on isolate server

appengine/isolate/model.py

 # Copyright 2014 The LUCI Authors. All rights reserved.
 # Use of this source code is governed by the Apache v2.0 license that can be
 # found in the LICENSE file.
 
 """This module defines Isolate Server model(s)."""
 
 import datetime
 import hashlib
 import logging
 import random
@@ skipping 117 matching lines @@
 
 def entry_key_from_id(key_id):
   """Returns the ndb.Key for the key_id."""
   hash_key = key_id.rsplit('/', 1)[1]
   N = config.settings().sharding_letters
   return ndb.Key(
       ContentEntry, key_id,
       parent=datastore_utils.shard_key(hash_key, N, 'ContentShard'))
 
 
+def get_content(namespace, hash_key):
+  """Returns the content from either memcache or datastore.

[M-A Ruel, 2016/04/14 19:22:36]

Returns the content from either memcache or datastore, when stored inline.

[kjlubick, 2016/04/14 19:59:07]

Done.

+
+  This does NOT return data from GCS, it is up to the client to do that.
+
+  The first argument in the tuple is the content, the second is either

[M-A Ruel, 2016/04/14 19:22:36]

Returns:
  tuple(content, ContentEntry)
  At most only one of the two is set.

[kjlubick, 2016/04/14 19:59:08]

Done.

+  None, if loaded from memcache, or the entity, if loaded from datastore.
+
+  Raises LookupError if the content cannot be found.
+  Raises ValueError if the hash_key is invalid.
+  """
+  memcache_entry = memcache.get(hash_key, namespace='table_%s' % namespace)
+  if memcache_entry is not None:
+    return (memcache_entry, None)
+  else:
+    # Raises ValueError
+    key = get_entry_key(namespace, hash_key)
+    entity = key.get()
+    if entity is None:
+      raise LookupError("namespace %s, key %s does not refer to anything" %
+        (namespace, hash_key))
+    return (entity.content, entity)
+
+
 def expiration_jitter(now, expiration):
   """Returns expiration/next_tag pair to set in a ContentEntry."""
   jittered = random.uniform(1, 1.2) * expiration
   expiration = now + datetime.timedelta(seconds=jittered)
   next_tag = now + datetime.timedelta(seconds=jittered*0.1)
   return expiration, next_tag
 
 
 def expand_content(namespace, source):
   """Yields expanded data from source."""
@@ skipping 63 matching lines @@
   ndb.delete_multi(keys_to_delete)
   # Note that some content entries may NOT have corresponding GS files. That
   # happens for small entries stored inline in the datastore or memcache. Since
   # this function operates only on keys, it can't distinguish "large" entries
   # stored in GS from "small" ones stored inline. So instead it tries to delete
   # all corresponding GS files, silently skipping ones that are not there.
   gcs.delete_files(
       config.settings().gs_bucket,
       (i.id() for i in keys_to_delete),
       ignore_missing=True)