Implement support for converting db values to datastore values with db.Query.filter

test/db/e2e/db_test_impl.dart

 // Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library db_test;
 
 /// NOTE: In order to run these tests, the following datastore indices must
 /// exist:
 /// $ cat index.yaml
 /// indexes:
@@ skipping 43 matching lines @@
 import '../../common_e2e.dart';
 
 @db.Kind()
 class Person extends db.Model {
   @db.StringProperty()
   String name;
 
   @db.IntProperty()
   int age;
 
-  @db.ModelKeyProperty()
+  @db.ModelKeyProperty(propertyName: 'mangledWife')
   db.Key wife;
 
   operator==(Object other) => sameAs(other);
 
   sameAs(Object other) {
     return other is Person &&
         id == other.id &&
         parentKey == other.parentKey &&
         name == other.name &&
         age == other.age &&
@@ skipping 56 matching lines @@
           return false;
         }
       }
       return true;
     }
     return false;
   }
 }
 
 
-Future sleep(Duration duration) {
-  var completer = new Completer();
-  new Timer(duration, completer.complete);
-  return completer.future;
-}
+Future sleep(Duration duration) => new Future.delayed(duration);
 
 runTests(db.DatastoreDB store, String namespace) {
   var partition = store.newPartition(namespace);
 
   void compareModels(List<db.Model> expectedModels,
                      List<db.Model> models,
                      {bool anyOrder: false}) {
     expect(models.length, equals(expectedModels.length));
     if (anyOrder) {
       // Do expensive O(n^2) search.
@@ skipping 232 matching lines @@
       for (var i = 1; i <= 10; i++) {
         var languages = [];
         if (i == 9) {
           languages = ['foo'];
         } else if (i == 10) {
           languages = ['foo', 'bar'];
         }
         users.add(new User()
             ..id = i
             ..parentKey = root
+            ..wife = root.append(User, id: 42 + i)
             ..age = 42 + i
             ..name = 'user$i'
             ..nickname = 'nickname${i%3}'
             ..languages = languages);
       }
 
       var expandoPersons = [];
       for (var i = 1; i <= 3; i++) {
         var expandoPerson = new ExpandoPerson()
             ..parentKey = root
@@ skipping 30 matching lines @@
 
       var usersSortedAndFilteredNameDescNicknameDesc =
           usersSortedNameDescNicknameDesc.where((User u) {
         return LOWER_BOUND.compareTo(u.name) <= 0;
       }).toList();
 
       var fooUsers = users.where(
           (User u) => u.languages.contains('foo')).toList();
       var barUsers = users.where(
           (User u) => u.languages.contains('bar')).toList();
+      var usersWithWife = users.where(
+          (User u) => u.wife == root.append(User, id: 42 + 3));
 
       var allInserts = []
           ..addAll(users)
           ..addAll(expandoPersons);
       var allKeys = allInserts.map((db.Model model) => model.key).toList();
       return store.commit(inserts: allInserts).then((_) {
         return waitUntilEntitiesReady(store, allKeys, partition).then((_) {
           var tests = [
             // Queries for [Person] return no results, we only have [User]
             // objects.
             () {
               return store.query(Person, partition: partition).run().toList()
                   .then((List<db.Model> models) {
                 compareModels([], models);
               });
             },
 
             // All users query
             () {
               return store.query(User, partition: partition).run().toList()
                   .then((List<db.Model> models) {
                 compareModels(users, models, anyOrder: true);
               });
             },
 
             // Sorted query
-            () {
-              return store.query(User, partition: partition)
+            () async {
+              var query = store.query(User, partition: partition)
                   ..order('-name')
-                  ..order('nickname')
-                  ..run().toList().then((List<db.Model> models) {
-                compareModels(
-                    usersSortedNameDescNicknameAsc, models);
-              });
+                  ..order('nickname');
+              var models = await runQueryWithExponentialBackoff(
+                  query, usersSortedNameDescNicknameAsc.length);

[kustermann, 2015/12/11 11:08:44]

There is still some flakiness on the buildbot. I decided to make it more robust
and make all the query tests use exponential backoff. Hopefully that removes
this last bit of flakiness.

(The reason why I belief that the [waitUntilEntitiesReady] is not enough is that
the backend has different tables for Entities, OnePropertyIndex,
ComposedPropertyIndex, so even if the first table is ready, the other two are
still eventually consistent)

+              compareModels(
+                  usersSortedNameDescNicknameAsc, models);
             },
-            () {
-              return store.query(User, partition: partition)
+            () async {
+              var query = store.query(User, partition: partition)
                   ..order('-name')
                   ..order('-nickname')
-                  ..run().toList().then((List<db.Model> models) {
-                compareModels(
-                    usersSortedNameDescNicknameDesc, models);
-              });
+                  ..run();
+              var models = await runQueryWithExponentialBackoff(
+                  query, usersSortedNameDescNicknameDesc.length);
+              compareModels(
+                  usersSortedNameDescNicknameDesc, models);
             },
 
             // Sorted query with filter
-            () {
-              return store.query(User, partition: partition)
+            () async {
+              var query = store.query(User, partition: partition)
                   ..filter('name >=', LOWER_BOUND)
                   ..order('-name')
-                  ..order('nickname')
-                  ..run().toList().then((List<db.Model> models) {
-                compareModels(usersSortedAndFilteredNameDescNicknameAsc,
-                    models);
-              });
+                  ..order('nickname');
+              var models = await runQueryWithExponentialBackoff(
+                  query, usersSortedAndFilteredNameDescNicknameAsc.length);
+              compareModels(usersSortedAndFilteredNameDescNicknameAsc,
+                  models);
             },
-            () {
-              return store.query(User, partition: partition)
+            () async {
+              var query = store.query(User, partition: partition)
                   ..filter('name >=', LOWER_BOUND)
                   ..order('-name')
                   ..order('-nickname')
-                  ..run().toList().then((List<db.Model> models) {
-                compareModels(usersSortedAndFilteredNameDescNicknameDesc,
-                    models);
-              });
+                  ..run();
+              var models = await runQueryWithExponentialBackoff(
+                  query, usersSortedAndFilteredNameDescNicknameDesc.length);
+              compareModels(usersSortedAndFilteredNameDescNicknameDesc,
+                  models);
             },
 
             // Filter lists
             /* FIXME: TODO: FIXME: "IN" not supported in public proto/apiary */
-            () {
-              return store.query(User, partition: partition)
+            () async {
+              var query = store.query(User, partition: partition)
                   ..filter('languages IN', ['foo'])
                   ..order('name')
-                  ..run().toList().then((List<db.Model> models) {
-                compareModels(fooUsers, models, anyOrder: true);
-              });
+                  ..run();
+              var models = await runQueryWithExponentialBackoff(query, fooUsers.length);

[Søren Gjesse, 2015/12/14 07:53:24]

Long line - and a few more below.

[kustermann, 2015/12/14 09:41:16]

Done.

+              compareModels(fooUsers, models, anyOrder: true);
             },
-            () {
-              return store.query(User, partition: partition)
+            () async {
+              var query = store.query(User, partition: partition)
                   ..filter('languages IN', ['bar'])
                   ..order('name')
-                  ..run().toList().then((List<db.Model> models) {
-                compareModels(barUsers, models, anyOrder: true);
-              });
+                  ..run();
+              var models = await runQueryWithExponentialBackoff(query, barUsers.length);
+              compareModels(barUsers, models, anyOrder: true);
+            },
+
+            // Filter equals
+            () async {
+              var wifeKey = root.append(User, id: usersWithWife.first.wife.id);
+              var query = store.query(User, partition: partition)
+                  ..filter('wife =', wifeKey)
+                  ..run();
+              var models = await runQueryWithExponentialBackoff(query, usersWithWife.length);
+              compareModels(usersWithWife, models, anyOrder: true);
             },
 
             // Simple limit/offset test.
-            () {
-              return store.query(User, partition: partition)
+            () async {
+              var query = store.query(User, partition: partition)
                   ..order('-name')
                   ..order('nickname')
                   ..offset(3)
-                  ..limit(4)
-                  ..run().toList().then((List<db.Model> models) {
-                var expectedModels =
-                    usersSortedAndFilteredNameDescNicknameAsc.sublist(3, 7);
-                compareModels(expectedModels, models);
-              });
+                  ..limit(4);
+              var expectedModels =
+                  usersSortedAndFilteredNameDescNicknameAsc.sublist(3, 7);
+              var models = await runQueryWithExponentialBackoff(
+                  query, expectedModels.length);
+              compareModels(expectedModels, models);
             },
 
             // Expando queries: Filter on normal property.
-            () {
-              return store.query(ExpandoPerson, partition: partition)
+            () async {
+              var query = store.query(ExpandoPerson, partition: partition)
                   ..filter('name =', expandoPersons.last.name)
-                  ..run().toList().then((List<db.Model> models) {
-                compareModels([expandoPersons.last], models);
-              });
+                  ..run();
+              var models = await runQueryWithExponentialBackoff(query, 1);
+              compareModels([expandoPersons.last], models);
             },
             // Expando queries: Filter on expanded String property
-            () {
-              return store.query(ExpandoPerson, partition: partition)
+            () async {
+              var query = store.query(ExpandoPerson, partition: partition)
                   ..filter('foo =', expandoPersons.last.foo)
-                  ..run().toList().then((List<db.Model> models) {
-                compareModels([expandoPersons.last], models);
-              });
+                  ..run();
+              var models = await runQueryWithExponentialBackoff(query, 1);
+              compareModels([expandoPersons.last], models);
             },
             // Expando queries: Filter on expanded int property
-            () {
-              return store.query(ExpandoPerson, partition: partition)
+            () async {
+              var query = store.query(ExpandoPerson, partition: partition)
                   ..filter('bar =', expandoPersons.last.bar)
-                  ..run().toList().then((List<db.Model> models) {
-                compareModels([expandoPersons.last], models);
-              });
+                  ..run();
+              var models = await runQueryWithExponentialBackoff(query, 1);
+              compareModels([expandoPersons.last], models);
             },
             // Expando queries: Filter normal property with different
             // propertyName (datastore name is 'NN').
-            () {
-              return store.query(ExpandoPerson, partition: partition)
+            () async {
+              var query = store.query(ExpandoPerson, partition: partition)
                   ..filter('nickname =', expandoPersons.last.nickname)
-                  ..run().toList().then((List<db.Model> models) {
-                compareModels([expandoPersons.last], models);
-              });
+                  ..run();
+              var models = await runQueryWithExponentialBackoff(query, 1);
+              compareModels([expandoPersons.last], models);
             },
 
             // Delete results
             () => store.commit(deletes: allKeys),
 
             // Wait until the entity deletes are reflected in the indices.
             () => waitUntilEntitiesGone(store, allKeys, partition),
 
             // Make sure queries don't return results
             () => store.lookup(allKeys).then((List<db.Model> models) {
               expect(models.length, equals(allKeys.length));
               for (var model in models) {
                 expect(model, isNull);
               }
             }),
           ];
           return Future.forEach(tests, (f) => f());
         });
       });
     });
   });
 }
 
+Future<List<db.Model>> runQueryWithExponentialBackoff(

[Søren Gjesse, 2015/12/14 07:53:24]

This looks like a good idea.

+    db.Query query, int expectedResults) async {
+  for (int i = 0; i <= 6; i++) {
+    if (i > 0) {
+      // Wait for 0.1s, 0.2s, ..., 12.8s
+      var duration = new Duration(milliseconds: 100 * (2 << i));
+      print("Running query did return less results than expected."
+            "Using exponential backoff: Sleeping for $duration.");
+      await sleep(duration);
+    }
+
+    List<db.Model> models = await query.run().toList();
+    if (models.length >= expectedResults) {
+      return models;
+    }
+  }
+
+  throw new Exception(
+      "Tried running a query with exponential backoff, giving up now.");
+}
+
 Future waitUntilEntitiesReady(db.DatastoreDB mdb,
                               List<db.Key> keys,
                               db.Partition partition) {
   return waitUntilEntitiesHelper(mdb, keys, true, partition);
 }
 
 Future waitUntilEntitiesGone(db.DatastoreDB mdb,
                              List<db.Key> keys,
                              db.Partition partition) {
   return waitUntilEntitiesHelper(mdb, keys, false, partition);
@@ skipping 35 matching lines @@
   var scopes = datastore_impl.DatastoreImpl.SCOPES;
 
   withAuthClient(scopes, (String project, httpClient) {
     var datastore = new datastore_impl.DatastoreImpl(httpClient, 's~$project');
     return datastore_test.cleanupDB(datastore, null).then((_) {
       return runE2EUnittest(
           () => runTests(new db.DatastoreDB(datastore), null));
     });
   });
 }