Refactor keys and queries in datastore service and implementation.

impl/memory/datastore_test.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package memory
 
 import (
         "fmt"
         "testing"
         "time"
 
         dsS "github.com/luci/gae/service/datastore"
-        "github.com/luci/gae/service/datastore/dskey"
         "github.com/luci/gae/service/datastore/serialize"
         infoS "github.com/luci/gae/service/info"
         . "github.com/smartystreets/goconvey/convey"
         "golang.org/x/net/context"
 )
 
-func TestDatastoreKinder(t *testing.T) {
-        t.Parallel()
-
-        Convey("Datastore keys", t, func() {
-                c := Use(context.Background())
-                ds := dsS.Get(c)
-                So(ds, ShouldNotBeNil)
-
-                Convey("implements DSNewKeyer", func() {
-                        Convey("NewKey", func() {
-                                key := ds.NewKey("nerd", "stringID", 0, nil)
-                                So(key, ShouldNotBeNil)
-                                So(key.Kind(), ShouldEqual, "nerd")
-                                So(key.StringID(), ShouldEqual, "stringID")
-                                So(key.IntID(), ShouldEqual, 0)
-                                So(key.Parent(), ShouldBeNil)
-                                So(key.AppID(), ShouldEqual, "dev~app")
-                                So(key.Namespace(), ShouldEqual, "")
-                                So(key.String(), ShouldEqual, "/nerd,stringID")
-                                So(key.Incomplete(), ShouldBeFalse)
-                                So(key.Valid(false, "dev~app", ""), ShouldBeTrue)
-                        })
-                })
-
-        })
-}
-
 type MetaGroup struct {
-        _id    int64   `gae:"$id,1"`
-        _kind  string  `gae:"$kind,__entity_group__"`
-        Parent dsS.Key `gae:"$parent"`
+        _id    int64    `gae:"$id,1"`
+        _kind  string   `gae:"$kind,__entity_group__"`
+        Parent *dsS.Key `gae:"$parent"`
 
         Version int64 `gae:"__version__"`
 }
 
-func testGetMeta(c context.Context, k dsS.Key) int64 {
+func testGetMeta(c context.Context, k *dsS.Key) int64 {
         ds := dsS.Get(c)
-        mg := &MetaGroup{Parent: dskey.Root(k)}
+        mg := &MetaGroup{Parent: k.Root()}
         if err := ds.Get(mg); err != nil {
                 panic(err)
         }
         return mg.Version
 }
 
 var pls = dsS.GetPLS
 
 type Foo struct {
-        Id     int64   `gae:"$id"`
-        Parent dsS.Key `gae:"$parent"`
+        ID     int64    `gae:"$id"`
+        Parent *dsS.Key `gae:"$parent"`
 
         Val int
 }
 
 func TestDatastoreSingleReadWriter(t *testing.T) {
         t.Parallel()
 
         Convey("Datastore single reads and writes", t, func() {
                 c := Use(context.Background())
                 ds := dsS.Get(c)
                 So(ds, ShouldNotBeNil)
 
                 Convey("getting objects that DNE is an error", func() {
-                        So(ds.Get(&Foo{Id: 1}), ShouldEqual, dsS.ErrNoSuchEntity)
+                        So(ds.Get(&Foo{ID: 1}), ShouldEqual, dsS.ErrNoSuchEntity)
                 })
 
                 Convey("bad namespaces fail", func() {
                         _, err := infoS.Get(c).Namespace("$$blzyall")
                         So(err.Error(), ShouldContainSubstring, "namespace \"$$blzyall\" does not match")
                 })
 
                 Convey("Can Put stuff", func() {
                         // with an incomplete key!
                         f := &Foo{Val: 10}
                         So(ds.Put(f), ShouldBeNil)
                         k := ds.KeyForObj(f)
-                        So(k.String(), ShouldEqual, "/Foo,1")
+                        So(k.String(), ShouldEqual, "dev~app::/Foo,1")
 
                         Convey("and Get it back", func() {
-                                newFoo := &Foo{Id: 1}
+                                newFoo := &Foo{ID: 1}
                                 So(ds.Get(newFoo), ShouldBeNil)
                                 So(newFoo, ShouldResemble, f)
 
                                 Convey("but it's hidden from a different namespace", func() {
                                         c, err := infoS.Get(c).Namespace("whombat")
                                         So(err, ShouldBeNil)
                                         ds = dsS.Get(c)
                                         So(ds.Get(f), ShouldEqual, dsS.ErrNoSuchEntity)
                                 })
 
@@ skipping 14 matching lines @@
                                 So(testGetMeta(c, k), ShouldEqual, 1)
 
                                 foos := make([]Foo, 10)
                                 for i := range foos {
                                         foos[i].Val = 10
                                         foos[i].Parent = k
                                 }
                                 So(ds.PutMulti(foos), ShouldBeNil)
                                 So(testGetMeta(c, k), ShouldEqual, 11)
 
-                                keys := make([]dsS.Key, len(foos))
+                                keys := make([]*dsS.Key, len(foos))
                                 for i, f := range foos {
                                         keys[i] = ds.KeyForObj(&f)
                                 }
 
                                 Convey("ensure that group versions persist across deletes", func() {
                                         So(ds.DeleteMulti(append(keys, k)), ShouldBeNil)
 
                                         // TODO(riannucci): replace with a Count query instead of this cast
                                         /*
                                                 ents := ds.(*dsImpl).data.head.GetCollection("ents:")
                                                 num, _ := ents.GetTotals()
                                                 // /__entity_root_ids__,Foo
                                                 // /Foo,1/__entity_group__,1
                                                 // /Foo,1/__entity_group_ids__,1
                                                 So(num, ShouldEqual, 3)
                                         */
 
                                         So(testGetMeta(c, k), ShouldEqual, 22)
 
-                                        So(ds.Put(&Foo{Id: 1}), ShouldBeNil)
+                                        So(ds.Put(&Foo{ID: 1}), ShouldBeNil)
                                         So(testGetMeta(c, k), ShouldEqual, 23)
                                 })
 
                                 Convey("can Get", func() {
                                         vals := make([]dsS.PropertyMap, len(keys))
                                         for i := range vals {
                                                 vals[i] = dsS.PropertyMap{}
-                                                vals[i].SetMeta("key", keys[i])
+                                                So(vals[i].SetMeta("key", keys[i]), ShouldBeNil)
                                         }
                                         So(ds.GetMulti(vals), ShouldBeNil)
 
                                         for i, val := range vals {
                                                 So(val, ShouldResemble, dsS.PropertyMap{
                                                         "Val":  {dsS.MkProperty(10)},
                                                         "$key": {dsS.MkPropertyNI(keys[i])},
                                                 })
                                         }
                                 })
 
                         })
                 })
 
                 Convey("implements DSTransactioner", func() {
                         Convey("Put", func() {
                                 f := &Foo{Val: 10}
                                 So(ds.Put(f), ShouldBeNil)
                                 k := ds.KeyForObj(f)
-                                So(k.String(), ShouldEqual, "/Foo,1")
+                                So(k.String(), ShouldEqual, "dev~app::/Foo,1")
 
                                 Convey("can Put new entity groups", func() {
                                         err := ds.RunInTransaction(func(c context.Context) error {
                                                 ds := dsS.Get(c)
 
                                                 f := &Foo{Val: 100}
                                                 So(ds.Put(f), ShouldBeNil)
-                                                So(f.Id, ShouldEqual, 2)
+                                                So(f.ID, ShouldEqual, 2)
 
-                                                f.Id = 0
+                                                f.ID = 0
                                                 f.Val = 200
                                                 So(ds.Put(f), ShouldBeNil)
-                                                So(f.Id, ShouldEqual, 3)
+                                                So(f.ID, ShouldEqual, 3)
 
                                                 return nil
                                         }, &dsS.TransactionOptions{XG: true})
                                         So(err, ShouldBeNil)
 
-                                        f := &Foo{Id: 2}
+                                        f := &Foo{ID: 2}
                                         So(ds.Get(f), ShouldBeNil)
                                         So(f.Val, ShouldEqual, 100)
 
-                                        f.Id = 3
+                                        f.ID = 3
                                         So(ds.Get(f), ShouldBeNil)
                                         So(f.Val, ShouldEqual, 200)
                                 })
 
                                 Convey("can Put new entities in a current group", func() {
                                         err := ds.RunInTransaction(func(c context.Context) error {
                                                 ds := dsS.Get(c)
 
                                                 f := &Foo{Val: 100, Parent: k}
                                                 So(ds.Put(f), ShouldBeNil)
-                                                So(ds.KeyForObj(f).String(), ShouldEqual, "/Foo,1/Foo,1")
+                                                So(ds.KeyForObj(f).String(), ShouldEqual, "dev~app::/Foo,1/Foo,1")
 
-                                                f.Id = 0
+                                                f.ID = 0
                                                 f.Val = 200
                                                 So(ds.Put(f), ShouldBeNil)
-                                                So(ds.KeyForObj(f).String(), ShouldEqual, "/Foo,1/Foo,2")
+                                                So(ds.KeyForObj(f).String(), ShouldEqual, "dev~app::/Foo,1/Foo,2")
 
                                                 return nil
                                         }, nil)
                                         So(err, ShouldBeNil)
 
-                                        f := &Foo{Id: 1, Parent: k}
+                                        f := &Foo{ID: 1, Parent: k}
                                         So(ds.Get(f), ShouldBeNil)
                                         So(f.Val, ShouldEqual, 100)
 
-                                        f.Id = 2
+                                        f.ID = 2
                                         So(ds.Get(f), ShouldBeNil)
                                         So(f.Val, ShouldEqual, 200)
                                 })
 
                                 Convey("Deletes work too", func() {
                                         err := ds.RunInTransaction(func(c context.Context) error {
                                                 return dsS.Get(c).Delete(k)
                                         }, nil)
                                         So(err, ShouldBeNil)
-                                        So(ds.Get(&Foo{Id: 1}), ShouldEqual, dsS.ErrNoSuchEntity)
+                                        So(ds.Get(&Foo{ID: 1}), ShouldEqual, dsS.ErrNoSuchEntity)
                                 })
 
                                 Convey("A Get counts against your group count", func() {
                                         err := ds.RunInTransaction(func(c context.Context) error {
                                                 ds := dsS.Get(c)
 
                                                 pm := dsS.PropertyMap{}
-                                                pm.SetMeta("key", ds.NewKey("Foo", "", 20, nil))
+                                                So(pm.SetMeta("key", ds.NewKey("Foo", "", 20, nil)), ShouldBeNil)
                                                 So(ds.Get(pm), ShouldEqual, dsS.ErrNoSuchEntity)
 
-                                                pm.SetMeta("key", k)
+                                                So(pm.SetMeta("key", k), ShouldBeNil)
                                                 So(ds.Get(pm).Error(), ShouldContainSubstring, "cross-group")
                                                 return nil
                                         }, nil)
                                         So(err, ShouldBeNil)
                                 })
 
                                 Convey("Get takes a snapshot", func() {
                                         err := ds.RunInTransaction(func(c context.Context) error {
                                                 txnDS := dsS.Get(c)
 
                                                 So(txnDS.Get(f), ShouldBeNil)
                                                 So(f.Val, ShouldEqual, 10)
 
                                                 // Don't ever do this in a real program unless you want to guarantee
                                                 // a failed transaction :)
                                                 f.Val = 11
                                                 So(ds.Put(f), ShouldBeNil)
 
                                                 So(txnDS.Get(f), ShouldBeNil)
                                                 So(f.Val, ShouldEqual, 10)
 
                                                 return nil
                                         }, nil)
                                         So(err, ShouldBeNil)
 
-                                        f := &Foo{Id: 1}
+                                        f := &Foo{ID: 1}
                                         So(ds.Get(f), ShouldBeNil)
                                         So(f.Val, ShouldEqual, 11)
                                 })
 
                                 Convey("and snapshots are consistent even after Puts", func() {
                                         err := ds.RunInTransaction(func(c context.Context) error {
                                                 txnDS := dsS.Get(c)
 
-                                                f := &Foo{Id: 1}
+                                                f := &Foo{ID: 1}
                                                 So(txnDS.Get(f), ShouldBeNil)
                                                 So(f.Val, ShouldEqual, 10)
 
                                                 // Don't ever do this in a real program unless you want to guarantee
                                                 // a failed transaction :)
                                                 f.Val = 11
                                                 So(ds.Put(f), ShouldBeNil)
 
                                                 So(txnDS.Get(f), ShouldBeNil)
                                                 So(f.Val, ShouldEqual, 10)
 
                                                 f.Val = 20
                                                 So(txnDS.Put(f), ShouldBeNil)
 
                                                 So(txnDS.Get(f), ShouldBeNil)
                                                 So(f.Val, ShouldEqual, 10) // still gets 10
 
                                                 return nil
                                         }, &dsS.TransactionOptions{Attempts: 1})
                                         So(err.Error(), ShouldContainSubstring, "concurrent")
 
-                                        f := &Foo{Id: 1}
+                                        f := &Foo{ID: 1}
                                         So(ds.Get(f), ShouldBeNil)
                                         So(f.Val, ShouldEqual, 11)
                                 })
 
                                 Convey("Reusing a transaction context is bad news", func() {
                                         txnDS := dsS.Interface(nil)
                                         err := ds.RunInTransaction(func(c context.Context) error {
                                                 txnDS = dsS.Get(c)
                                                 So(txnDS.Get(f), ShouldBeNil)
                                                 return nil
@@ skipping 15 matching lines @@
 
                                 Convey("Concurrent transactions only accept one set of changes", func() {
                                         // Note: I think this implementation is actually /slightly/ wrong.
                                         // According to my read of the docs for appengine, when you open a
                                         // transaction it actually (essentially) holds a reference to the
                                         // entire datastore. Our implementation takes a snapshot of the
                                         // entity group as soon as something observes/affects it.
                                         //
                                         // That said... I'm not sure if there's really a semantic difference.
                                         err := ds.RunInTransaction(func(c context.Context) error {
-                                                So(dsS.Get(c).Put(&Foo{Id: 1, Val: 21}), ShouldBeNil)
+                                                So(dsS.Get(c).Put(&Foo{ID: 1, Val: 21}), ShouldBeNil)
 
                                                 err := ds.RunInTransaction(func(c context.Context) error {
-                                                        So(dsS.Get(c).Put(&Foo{Id: 1, Val: 27}), ShouldBeNil)
+                                                        So(dsS.Get(c).Put(&Foo{ID: 1, Val: 27}), ShouldBeNil)
                                                         return nil
                                                 }, nil)
                                                 So(err, ShouldBeNil)
 
                                                 return nil
                                         }, nil)
                                         So(err.Error(), ShouldContainSubstring, "concurrent")
 
-                                        f := &Foo{Id: 1}
+                                        f := &Foo{ID: 1}
                                         So(ds.Get(f), ShouldBeNil)
                                         So(f.Val, ShouldEqual, 27)
                                 })
 
                                 Convey("XG", func() {
                                         Convey("Modifying two groups with XG=false is invalid", func() {
                                                 err := ds.RunInTransaction(func(c context.Context) error {
                                                         ds := dsS.Get(c)
-                                                        f := &Foo{Id: 1, Val: 200}
+                                                        f := &Foo{ID: 1, Val: 200}
                                                         So(ds.Put(f), ShouldBeNil)
 
-                                                        f.Id = 2
+                                                        f.ID = 2
                                                         err := ds.Put(f)
                                                         So(err.Error(), ShouldContainSubstring, "cross-group")
                                                         return err
                                                 }, nil)
                                                 So(err.Error(), ShouldContainSubstring, "cross-group")
                                         })
 
                                         Convey("Modifying >25 groups with XG=true is invald", func() {
                                                 err := ds.RunInTransaction(func(c context.Context) error {
                                                         ds := dsS.Get(c)
                                                         foos := make([]Foo, 25)
                                                         for i := int64(1); i < 26; i++ {
-                                                                foos[i-1].Id = i
+                                                                foos[i-1].ID = i
                                                                 foos[i-1].Val = 200
                                                         }
                                                         So(ds.PutMulti(foos), ShouldBeNil)
-                                                        err := ds.Put(&Foo{Id: 26})
+                                                        err := ds.Put(&Foo{ID: 26})
                                                         So(err.Error(), ShouldContainSubstring, "too many entity groups")
                                                         return err
                                                 }, &dsS.TransactionOptions{XG: true})
                                                 So(err.Error(), ShouldContainSubstring, "too many entity groups")
                                         })
                                 })
 
                                 Convey("Errors and panics", func() {
                                         Convey("returning an error aborts", func() {
                                                 err := ds.RunInTransaction(func(c context.Context) error {
                                                         ds := dsS.Get(c)
-                                                        So(ds.Put(&Foo{Id: 1, Val: 200}), ShouldBeNil)
+                                                        So(ds.Put(&Foo{ID: 1, Val: 200}), ShouldBeNil)
                                                         return fmt.Errorf("thingy")
                                                 }, nil)
                                                 So(err.Error(), ShouldEqual, "thingy")
 
-                                                f := &Foo{Id: 1}
+                                                f := &Foo{ID: 1}
                                                 So(ds.Get(f), ShouldBeNil)
                                                 So(f.Val, ShouldEqual, 10)
                                         })
 
                                         Convey("panicing aborts", func() {
                                                 So(func() {
-                                                        ds.RunInTransaction(func(c context.Context) error {
+                                                        So(ds.RunInTransaction(func(c context.Context) error {
                                                                 ds := dsS.Get(c)
                                                                 So(ds.Put(&Foo{Val: 200}), ShouldBeNil)
                                                                 panic("wheeeeee")
-                                                        }, nil)
+                                                        }, nil), ShouldBeNil)
                                                 }, ShouldPanic)
 
-                                                f := &Foo{Id: 1}
+                                                f := &Foo{ID: 1}
                                                 So(ds.Get(f), ShouldBeNil)
                                                 So(f.Val, ShouldEqual, 10)
                                         })
                                 })
 
                                 Convey("Transaction retries", func() {
                                         tst := ds.Testable()
                                         Reset(func() { tst.SetTransactionRetryCount(0) })
 
                                         Convey("SetTransactionRetryCount set to zere", func() {
@@ skipping 118 matching lines @@
                 ds := dsS.Get(Use(context.Background()))
                 m := Model{ID: 1}
                 So(ds.Put(&m), ShouldBeNil)
 
                 // Reset to something non zero to ensure zero is fetched.
                 m.Time = time.Now().UTC()
                 So(ds.Get(&m), ShouldBeNil)
                 So(m.Time.IsZero(), ShouldBeTrue)
         })
 }