[Extensions DocServer] Update Server for new Datastore API

chrome/common/extensions/docs/server2/datastore_util.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import cPickle
 import googledatastore as datastore
 import logging
 
 from future import Future
 
 # N.B.: In order to use this module you should have a working cloud development
 # environment configured with the googledatastore module installed.
 #
 # Please see https://cloud.google.com/datastore/docs/getstarted/start_python/
 
 
 _DATASET_NAME = 'chrome-apps-doc'
 _PERSISTENT_OBJECT_KIND = 'PersistentObjectStoreItem'
 _VALUE_PROPERTY_NAME = 'pickled_value'

[asargent_no_longer_on_chrome, 2016/09/14 04:57:57]

random aside: this style of having variables for these 3 instead of just
directly using the string seems kind of strange to me. Especially since 2 of
them only appear once in the file. 

Unless there's some compelling reason to have them that I'm not seeing, a future
cleanup could improve readability IMHO by inlining these strings. 
 

[Devlin, 2016/09/14 15:09:24]

For this particular case (property name), the reason to have a variable is to
avoid a typo in one of the three occurrences, which I think is pretty valid. 
For the other two, I would normally think it's a way of having self-documenting
variables (e.g. bool flag_x = false; DoFoo(flag_x) is more readable than
DoFoo(false)), but in both of these cases, it's pretty obvious from the context
what it is, so I agree that inlining sg.  I'll do that in a followup.

 
 # The max number of entities to include in a single request. This is capped at
 # 500 by the service. In practice we may send fewer due to _MAX_REQUEST_SIZE
 _MAX_BATCH_SIZE = 500
 
 
 # The maximum entity size allowed by Datastore.
 _MAX_ENTITY_SIZE = 1024*1024
 
 
 # The maximum request size (in bytes) to send Datastore. This is an approximate
 # size based on the sum of entity blob_value sizes.
 _MAX_REQUEST_SIZE = 5*1024*1024
 
 
 def _CreateEntity(name, value):
-  entity = datastore.Entity()
-  path = entity.key.path_element.add()
+  entity = datastore.Entity(exclude_from_indexes=[_VALUE_PROPERTY_NAME])
+  path = entity.key.path.add()
   path.kind = _PERSISTENT_OBJECT_KIND
   path.name = name
-  pickled_value_property = entity.property.add()
-  pickled_value_property.name = _VALUE_PROPERTY_NAME
-  pickled_value_property.value.indexed = False
-  pickled_value_property.value.blob_value = value
+  entity.update({_VALUE_PROPERTY_NAME: value})
   return entity
 
 
 def _CreateBatches(data):
   '''Constructs batches of at most _MAX_BATCH_SIZE entities to cover all
   entities defined in |data| without exceeding the transaction size limit.
   This is a generator emitting lists of entities.
   '''
   def get_size(entity):
-    return len(entity.property[0].value.blob_value)
+    return len(entity.properties[_VALUE_PROPERTY_NAME].value.blob_value)
 
   entities = [_CreateEntity(name, value) for name, value in data.iteritems()]
   batch_start = 0
   batch_end = 1
   batch_size = get_size(entities[0])
   while batch_end < len(entities):
     next_size = get_size(entities[batch_end])
     if (batch_size + next_size > _MAX_REQUEST_SIZE or
         batch_end - batch_start >= _MAX_BATCH_SIZE):
       yield entities[batch_start:batch_end], batch_end, len(entities)
@@ skipping 58 matching lines @@
 
   logging.info('Culling new data...')
   for k in data.keys():
     if ((k in original_data and original_data[k] == data[k]) or
         (len(data[k]) > _MAX_ENTITY_SIZE)):
       del data[k]
 
   for batch, n, total in _CreateBatches(data):
     commit_request = datastore.CommitRequest()
     commit_request.mode = datastore.CommitRequest.NON_TRANSACTIONAL
-    commit_request.mutation.upsert.extend(list(batch))
+    commit_request.mutations.upsert.extend(list(batch))
 
     logging.info('Committing %s/%s entities...' % (n, total))
     datastore.commit(commit_request)