Fix a bug where deletions weren't updating the raw Kind index.

impl/memory/datastore_query_execution_test.go

 // Copyright 2015 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package memory
 
 import (
         "fmt"
         "strings"
         "testing"
         "time"
 
         "github.com/luci/gae/service/blobstore"
         ds "github.com/luci/gae/service/datastore"
         "github.com/luci/gae/service/info"
         "golang.org/x/net/context"
 
         . "github.com/luci/luci-go/common/testing/assertions"
         . "github.com/smartystreets/goconvey/convey"
 )
 
 func mkKey(appID, namespace string, elems ...interface{}) *ds.Key {
         return ds.MkKeyContext(appID, namespace).MakeKey(elems...)
 }
 
+var enableOnly = false
+
+func enable() bool {

[dnj, 2016/11/11 08:37:53]

This is something useful for debugging specific test cases. Basically add this
to your query and the code will *only* execute that query. Much easier to
isolate than massive comment blocks.

+        enableOnly = true
+        return true
+}
+
 type qExpect struct {
         q     *ds.Query
         inTxn bool
 
         get   []ds.PropertyMap
         keys  []*ds.Key
         count int
+
+        enable bool
 }
 
 type qExStage struct {
         addIdxs []*ds.IndexDefinition
         putEnts []ds.PropertyMap
         delEnts []*ds.Key
 
         expect []qExpect
 
         extraFns []func(context.Context)
@@ skipping 202 matching lines @@
 
                                 {q: nq("Kind").Gt("Val", 2).Lte("Val", 5), get: []ds.PropertyMap{
                                         stage1Data[0], stage1Data[3],
                                 }},
 
                                 {q: nq("Kind").Gt("Val", 2).Lte("Val", 5).Order("-Val"), get: []ds.PropertyMap{
                                         stage1Data[3], stage1Data[0],
                                 }},
 
                                 {q: nq("").Gt("__key__", key("Kind", 2)),
-                                        // count counts from the index with KeysOnly and so counts the deleted

[dnj, 2016/11/11 08:37:53]

(This is no longer true, since we don't auto-KeysOnly queries for Count).

-                                        // entity Unique/1.
-                                        count: 8,
+                                        count: 7,
                                         get: []ds.PropertyMap{
                                                 // TODO(riannucci): determine if the real datastore shows metadata
                                                 // during kindless queries. The documentation seems to imply so, but
                                                 // I'd like to be sure.
                                                 pmap("$key", key("Kind", 2, "__entity_group__", 1), Next,
                                                         "__version__", 1),
                                                 stage1Data[2],
                                                 stage1Data[4],
                                                 // this is 5 because the value is retrieved from HEAD and not from
                                                 // the index snapshot!
@@ skipping 148 matching lines @@
                                                 "  - name: Val",
                                         }, "\n"))
                                 },
                         },
                 },
 
                 {
                         expect: []qExpect{
                                 // eventual consistency; Unique/1 is deleted at HEAD. Keysonly finds it,
                                 // but 'normal' doesn't.
+                                {q: nq("Unique").Gt("__key__", key("AKind", 5)).Lte("__key__", key("Zeta", "prime")).KeysOnly(true),
+                                        keys: []*ds.Key{key("Unique", 1)}},
                                 {q: nq("Unique").Gt("__key__", key("AKind", 5)).Lte("__key__", key("Zeta", "prime")),
-                                        keys: []*ds.Key{key("Unique", 1)},
-                                        get:  []ds.PropertyMap{}},
+                                        get: []ds.PropertyMap{}},
 
                                 {q: nq("Kind").Eq("Val", 1, 3), get: []ds.PropertyMap{
                                         stage1Data[0], stage2Data[2],
                                 }},
                         },
                 },
         }},
         {"collapsed types", []qExStage{
                 {
                         putEnts: collapsedData,
@@ skipping 37 matching lines @@
                                 },
                                 {
                                         q: nq("Kind").Eq("Val", []byte("uwutm8")),
                                         get: []ds.PropertyMap{
                                                 collapsedData[5],
                                         },
                                 },
                         },
                 },
         }},
+
+        {"regression: tombstones and limit/offset queries", []qExStage{
+                {
+                        putEnts: []ds.PropertyMap{
+                                pmap("$key", key("Kind", 1)),
+                                pmap("$key", key("Kind", 2)),
+                                pmap("$key", key("Kind", 3)),
+                        },
+                        delEnts: []*ds.Key{key("Kind", 2)},
+                },
+                {
+                        expect: []qExpect{
+                                {
+                                        q: nq("Kind").Limit(2),
+                                        get: []ds.PropertyMap{
+                                                pmap("$key", key("Kind", 1)),
+                                                pmap("$key", key("Kind", 3)),
+                                        },
+                                },
+
+                                {
+                                        q:   nq("Kind").Offset(2),
+                                        get: []ds.PropertyMap{},
+                                },
+                        },
+                },
+        }},
 }
 
 func TestQueryExecution(t *testing.T) {
         t.Parallel()
 
         Convey("Test query execution", t, func() {
                 c, err := info.Namespace(Use(context.Background()), "ns")
                 if err != nil {
                         panic(err)
                 }
@@ skipping 23 matching lines @@
                                                         panic(err)
                                                 }
                                         }
 
                                         if err := ds.Delete(c, stage.delEnts); err != nil {
                                                 panic(err)
                                         }
 
                                         Convey(fmt.Sprintf("stage %d", i), func() {
                                                 for j, expect := range stage.expect {
+                                                        if enableOnly && !expect.enable {
+                                                                continue
+                                                        }
                                                         runner := func(c context.Context, f func(ic context.Context) error, _ *ds.TransactionOptions) error {
                                                                 return f(c)
                                                         }
                                                         if expect.inTxn {
                                                                 runner = ds.RunInTransaction
                                                         }
 
                                                         if expect.count == 0 {
                                                                 if len(expect.keys) > 0 {
                                                                         expect.count = len(expect.keys)
@@ skipping 94 matching lines @@
 
         v, ok := actual.(error)
         if !ok {
                 return fmt.Sprintf("type of 'actual' must be error, not %T", actual)
         }
         if v == nil || v == ds.Stop {
                 return ""
         }
         return fmt.Sprintf("expected success value, not %v", v)
 }