| OLD | NEW |
| 1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
| 4 | 4 |
| 5 package dscache | 5 package dscache |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "bytes" | 8 "bytes" |
| 9 "encoding/binary" | 9 "encoding/binary" |
| 10 "errors" | 10 "errors" |
| 11 "math/rand" | 11 "math/rand" |
| 12 "testing" | 12 "testing" |
| 13 "time" | 13 "time" |
| 14 | 14 |
| 15 "github.com/luci/gae/filter/featureBreaker" | 15 "github.com/luci/gae/filter/featureBreaker" |
| 16 "github.com/luci/gae/impl/memory" | 16 "github.com/luci/gae/impl/memory" |
| 17 » "github.com/luci/gae/service/datastore" | 17 » ds "github.com/luci/gae/service/datastore" |
| 18 "github.com/luci/gae/service/datastore/serialize" | 18 "github.com/luci/gae/service/datastore/serialize" |
| 19 » "github.com/luci/gae/service/memcache" | 19 » mc "github.com/luci/gae/service/memcache" |
| 20 |
| 20 "github.com/luci/luci-go/common/clock" | 21 "github.com/luci/luci-go/common/clock" |
| 21 "github.com/luci/luci-go/common/clock/testclock" | 22 "github.com/luci/luci-go/common/clock/testclock" |
| 22 "github.com/luci/luci-go/common/data/rand/mathrand" | 23 "github.com/luci/luci-go/common/data/rand/mathrand" |
| 24 |
| 25 "golang.org/x/net/context" |
| 26 |
| 23 . "github.com/smartystreets/goconvey/convey" | 27 . "github.com/smartystreets/goconvey/convey" |
| 24 "golang.org/x/net/context" | |
| 25 ) | 28 ) |
| 26 | 29 |
| 27 type object struct { | 30 type object struct { |
| 28 ID int64 `gae:"$id"` | 31 ID int64 `gae:"$id"` |
| 29 | 32 |
| 30 Value string | 33 Value string |
| 31 BigData []byte | 34 BigData []byte |
| 32 } | 35 } |
| 33 | 36 |
| 34 type shardObj struct { // see shardsForKey() at top | 37 type shardObj struct { // see shardsForKey() at top |
| (...skipping 21 matching lines...) Expand all Loading... |
| 56 if err != nil { | 59 if err != nil { |
| 57 panic(err) | 60 panic(err) |
| 58 } | 61 } |
| 59 | 62 |
| 60 Convey("Test dscache", t, func() { | 63 Convey("Test dscache", t, func() { |
| 61 c := mathrand.Set(context.Background(), rand.New(rand.NewSource(
1))) | 64 c := mathrand.Set(context.Background(), rand.New(rand.NewSource(
1))) |
| 62 clk := testclock.New(zeroTime) | 65 clk := testclock.New(zeroTime) |
| 63 c = clock.Set(c, clk) | 66 c = clock.Set(c, clk) |
| 64 c = memory.Use(c) | 67 c = memory.Use(c) |
| 65 | 68 |
| 66 » » dsUnder := datastore.Get(c) | 69 » » underCtx := c |
| 67 » » mc := memcache.Get(c) | |
| 68 | 70 |
| 69 » » shardsForKey := func(k *datastore.Key) int { | 71 » » shardsForKey := func(k *ds.Key) int { |
| 70 last := k.LastTok() | 72 last := k.LastTok() |
| 71 if last.Kind == "shardObj" { | 73 if last.Kind == "shardObj" { |
| 72 return int(last.IntID) | 74 return int(last.IntID) |
| 73 } | 75 } |
| 74 if last.Kind == "noCacheObj" { | 76 if last.Kind == "noCacheObj" { |
| 75 return 0 | 77 return 0 |
| 76 } | 78 } |
| 77 return DefaultShards | 79 return DefaultShards |
| 78 } | 80 } |
| 79 | 81 |
| 80 numMemcacheItems := func() uint64 { | 82 numMemcacheItems := func() uint64 { |
| 81 » » » stats, err := mc.Stats() | 83 » » » stats, err := mc.Stats(c) |
| 82 So(err, ShouldBeNil) | 84 So(err, ShouldBeNil) |
| 83 return stats.Items | 85 return stats.Items |
| 84 } | 86 } |
| 85 | 87 |
| 86 Convey("enabled cases", func() { | 88 Convey("enabled cases", func() { |
| 87 c = FilterRDS(c, shardsForKey) | 89 c = FilterRDS(c, shardsForKey) |
| 88 ds := datastore.Get(c) | |
| 89 So(dsUnder, ShouldNotBeNil) | |
| 90 So(ds, ShouldNotBeNil) | |
| 91 So(mc, ShouldNotBeNil) | |
| 92 | 90 |
| 93 Convey("basically works", func() { | 91 Convey("basically works", func() { |
| 94 » » » » pm := datastore.PropertyMap{ | 92 » » » » pm := ds.PropertyMap{ |
| 95 » » » » » "BigData": datastore.MkProperty([]byte("
")), | 93 » » » » » "BigData": ds.MkProperty([]byte("")), |
| 96 » » » » » "Value": datastore.MkProperty("hi"), | 94 » » » » » "Value": ds.MkProperty("hi"), |
| 97 } | 95 } |
| 98 encoded := append([]byte{0}, serialize.ToBytes(p
m)...) | 96 encoded := append([]byte{0}, serialize.ToBytes(p
m)...) |
| 99 | 97 |
| 100 o := object{ID: 1, Value: "hi"} | 98 o := object{ID: 1, Value: "hi"} |
| 101 » » » » So(ds.Put(&o), ShouldBeNil) | 99 » » » » So(ds.Put(c, &o), ShouldBeNil) |
| 102 | 100 |
| 103 o = object{ID: 1} | 101 o = object{ID: 1} |
| 104 » » » » So(dsUnder.Get(&o), ShouldBeNil) | 102 » » » » So(ds.Get(underCtx, &o), ShouldBeNil) |
| 105 So(o.Value, ShouldEqual, "hi") | 103 So(o.Value, ShouldEqual, "hi") |
| 106 | 104 |
| 107 » » » » itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForO
bj(&o))) | 105 » » » » itm, err := mc.GetKey(c, MakeMemcacheKey(0, ds.K
eyForObj(c, &o))) |
| 108 » » » » So(err, ShouldEqual, memcache.ErrCacheMiss) | 106 » » » » So(err, ShouldEqual, mc.ErrCacheMiss) |
| 109 | 107 |
| 110 o = object{ID: 1} | 108 o = object{ID: 1} |
| 111 » » » » So(ds.Get(&o), ShouldBeNil) | 109 » » » » So(ds.Get(c, &o), ShouldBeNil) |
| 112 So(o.Value, ShouldEqual, "hi") | 110 So(o.Value, ShouldEqual, "hi") |
| 113 | 111 |
| 114 » » » » itm, err = mc.Get(itm.Key()) | 112 » » » » itm, err = mc.GetKey(c, itm.Key()) |
| 115 So(err, ShouldBeNil) | 113 So(err, ShouldBeNil) |
| 116 So(itm.Value(), ShouldResemble, encoded) | 114 So(itm.Value(), ShouldResemble, encoded) |
| 117 | 115 |
| 118 Convey("now we don't need the datastore!", func(
) { | 116 Convey("now we don't need the datastore!", func(
) { |
| 119 o := object{ID: 1} | 117 o := object{ID: 1} |
| 120 | 118 |
| 121 // delete it, bypassing the cache filter
. Don't do this in production | 119 // delete it, bypassing the cache filter
. Don't do this in production |
| 122 // unless you want a crappy cache. | 120 // unless you want a crappy cache. |
| 123 » » » » » So(dsUnder.Delete(ds.KeyForObj(&o)), Sho
uldBeNil) | 121 » » » » » So(ds.Delete(underCtx, ds.KeyForObj(unde
rCtx, &o)), ShouldBeNil) |
| 124 | 122 |
| 125 » » » » » itm, err := mc.Get(MakeMemcacheKey(0, ds
.KeyForObj(&o))) | 123 » » » » » itm, err := mc.GetKey(c, MakeMemcacheKey
(0, ds.KeyForObj(c, &o))) |
| 126 So(err, ShouldBeNil) | 124 So(err, ShouldBeNil) |
| 127 So(itm.Value(), ShouldResemble, encoded) | 125 So(itm.Value(), ShouldResemble, encoded) |
| 128 | 126 |
| 129 » » » » » So(ds.Get(&o), ShouldBeNil) | 127 » » » » » So(ds.Get(c, &o), ShouldBeNil) |
| 130 So(o.Value, ShouldEqual, "hi") | 128 So(o.Value, ShouldEqual, "hi") |
| 131 }) | 129 }) |
| 132 | 130 |
| 133 Convey("deleting it properly records that fact,
however", func() { | 131 Convey("deleting it properly records that fact,
however", func() { |
| 134 o := object{ID: 1} | 132 o := object{ID: 1} |
| 135 » » » » » So(ds.Delete(ds.KeyForObj(&o)), ShouldBe
Nil) | 133 » » » » » So(ds.Delete(c, ds.KeyForObj(c, &o)), Sh
ouldBeNil) |
| 136 | 134 |
| 137 » » » » » itm, err := mc.Get(MakeMemcacheKey(0, ds
.KeyForObj(&o))) | 135 » » » » » itm, err := mc.GetKey(c, MakeMemcacheKey
(0, ds.KeyForObj(c, &o))) |
| 138 » » » » » So(err, ShouldEqual, memcache.ErrCacheMi
ss) | 136 » » » » » So(err, ShouldEqual, mc.ErrCacheMiss) |
| 139 » » » » » So(ds.Get(&o), ShouldEqual, datastore.Er
rNoSuchEntity) | 137 » » » » » So(ds.Get(c, &o), ShouldEqual, ds.ErrNoS
uchEntity) |
| 140 | 138 |
| 141 » » » » » itm, err = mc.Get(itm.Key()) | 139 » » » » » itm, err = mc.GetKey(c, itm.Key()) |
| 142 So(err, ShouldBeNil) | 140 So(err, ShouldBeNil) |
| 143 So(itm.Value(), ShouldResemble, []byte{}
) | 141 So(itm.Value(), ShouldResemble, []byte{}
) |
| 144 | 142 |
| 145 // this one hits memcache | 143 // this one hits memcache |
| 146 » » » » » So(ds.Get(&o), ShouldEqual, datastore.Er
rNoSuchEntity) | 144 » » » » » So(ds.Get(c, &o), ShouldEqual, ds.ErrNoS
uchEntity) |
| 147 }) | 145 }) |
| 148 }) | 146 }) |
| 149 | 147 |
| 150 Convey("compression works", func() { | 148 Convey("compression works", func() { |
| 151 o := object{ID: 2, Value: `¯\_(ツ)_/¯`} | 149 o := object{ID: 2, Value: `¯\_(ツ)_/¯`} |
| 152 data := make([]byte, 4000) | 150 data := make([]byte, 4000) |
| 153 for i := range data { | 151 for i := range data { |
| 154 const alpha = "ABCDEFGHIJKLMNOPQRSTUVWXY
Zabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()" | 152 const alpha = "ABCDEFGHIJKLMNOPQRSTUVWXY
Zabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()" |
| 155 data[i] = alpha[i%len(alpha)] | 153 data[i] = alpha[i%len(alpha)] |
| 156 } | 154 } |
| 157 o.BigData = data | 155 o.BigData = data |
| 158 | 156 |
| 159 » » » » So(ds.Put(&o), ShouldBeNil) | 157 » » » » So(ds.Put(c, &o), ShouldBeNil) |
| 160 » » » » So(ds.Get(&o), ShouldBeNil) | 158 » » » » So(ds.Get(c, &o), ShouldBeNil) |
| 161 | 159 |
| 162 » » » » itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForO
bj(&o))) | 160 » » » » itm, err := mc.GetKey(c, MakeMemcacheKey(0, ds.K
eyForObj(c, &o))) |
| 163 So(err, ShouldBeNil) | 161 So(err, ShouldBeNil) |
| 164 | 162 |
| 165 So(itm.Value()[0], ShouldEqual, ZlibCompression) | 163 So(itm.Value()[0], ShouldEqual, ZlibCompression) |
| 166 So(len(itm.Value()), ShouldEqual, 653) // a bit
smaller than 4k | 164 So(len(itm.Value()), ShouldEqual, 653) // a bit
smaller than 4k |
| 167 | 165 |
| 168 // ensure the next Get comes from the cache | 166 // ensure the next Get comes from the cache |
| 169 » » » » So(dsUnder.Delete(ds.KeyForObj(&o)), ShouldBeNil
) | 167 » » » » So(ds.Delete(underCtx, ds.KeyForObj(underCtx, &o
)), ShouldBeNil) |
| 170 | 168 |
| 171 o = object{ID: 2} | 169 o = object{ID: 2} |
| 172 » » » » So(ds.Get(&o), ShouldBeNil) | 170 » » » » So(ds.Get(c, &o), ShouldBeNil) |
| 173 So(o.Value, ShouldEqual, `¯\_(ツ)_/¯`) | 171 So(o.Value, ShouldEqual, `¯\_(ツ)_/¯`) |
| 174 So(o.BigData, ShouldResemble, data) | 172 So(o.BigData, ShouldResemble, data) |
| 175 }) | 173 }) |
| 176 | 174 |
| 177 Convey("transactions", func() { | 175 Convey("transactions", func() { |
| 178 Convey("work", func() { | 176 Convey("work", func() { |
| 179 // populate an object @ ID1 | 177 // populate an object @ ID1 |
| 180 » » » » » So(ds.Put(&object{ID: 1, Value: "somethi
ng"}), ShouldBeNil) | 178 » » » » » So(ds.Put(c, &object{ID: 1, Value: "some
thing"}), ShouldBeNil) |
| 181 » » » » » So(ds.Get(&object{ID: 1}), ShouldBeNil) | 179 » » » » » So(ds.Get(c, &object{ID: 1}), ShouldBeNi
l) |
| 182 | 180 |
| 183 » » » » » So(ds.Put(&object{ID: 2, Value: "nurbs"}
), ShouldBeNil) | 181 » » » » » So(ds.Put(c, &object{ID: 2, Value: "nurb
s"}), ShouldBeNil) |
| 184 » » » » » So(ds.Get(&object{ID: 2}), ShouldBeNil) | 182 » » » » » So(ds.Get(c, &object{ID: 2}), ShouldBeNi
l) |
| 185 | 183 |
| 186 // memcache now has the wrong value (sim
ulated race) | 184 // memcache now has the wrong value (sim
ulated race) |
| 187 » » » » » So(dsUnder.Put(&object{ID: 1, Value: "el
se"}), ShouldBeNil) | 185 » » » » » So(ds.Put(underCtx, &object{ID: 1, Value
: "else"}), ShouldBeNil) |
| 188 » » » » » So(ds.RunInTransaction(func(c context.Co
ntext) error { | 186 » » » » » So(ds.RunInTransaction(c, func(c context
.Context) error { |
| 189 » » » » » » ds := datastore.Get(c) | |
| 190 o := &object{ID: 1} | 187 o := &object{ID: 1} |
| 191 » » » » » » So(ds.Get(o), ShouldBeNil) | 188 » » » » » » So(ds.Get(c, o), ShouldBeNil) |
| 192 So(o.Value, ShouldEqual, "else") | 189 So(o.Value, ShouldEqual, "else") |
| 193 o.Value = "txn" | 190 o.Value = "txn" |
| 194 » » » » » » So(ds.Put(o), ShouldBeNil) | 191 » » » » » » So(ds.Put(c, o), ShouldBeNil) |
| 195 | 192 |
| 196 » » » » » » So(ds.Delete(ds.KeyForObj(&objec
t{ID: 2})), ShouldBeNil) | 193 » » » » » » So(ds.Delete(c, ds.KeyForObj(c,
&object{ID: 2})), ShouldBeNil) |
| 197 return nil | 194 return nil |
| 198 » » » » » }, &datastore.TransactionOptions{XG: tru
e}), ShouldBeNil) | 195 » » » » » }, &ds.TransactionOptions{XG: true}), Sh
ouldBeNil) |
| 199 | 196 |
| 200 » » » » » _, err := mc.Get(MakeMemcacheKey(0, ds.K
eyForObj(&object{ID: 1}))) | 197 » » » » » _, err := mc.GetKey(c, MakeMemcacheKey(0
, ds.KeyForObj(c, &object{ID: 1}))) |
| 201 » » » » » So(err, ShouldEqual, memcache.ErrCacheMi
ss) | 198 » » » » » So(err, ShouldEqual, mc.ErrCacheMiss) |
| 202 » » » » » _, err = mc.Get(MakeMemcacheKey(0, ds.Ke
yForObj(&object{ID: 2}))) | 199 » » » » » _, err = mc.GetKey(c, MakeMemcacheKey(0,
ds.KeyForObj(c, &object{ID: 2}))) |
| 203 » » » » » So(err, ShouldEqual, memcache.ErrCacheMi
ss) | 200 » » » » » So(err, ShouldEqual, mc.ErrCacheMiss) |
| 204 o := &object{ID: 1} | 201 o := &object{ID: 1} |
| 205 » » » » » So(ds.Get(o), ShouldBeNil) | 202 » » » » » So(ds.Get(c, o), ShouldBeNil) |
| 206 So(o.Value, ShouldEqual, "txn") | 203 So(o.Value, ShouldEqual, "txn") |
| 207 }) | 204 }) |
| 208 | 205 |
| 209 Convey("errors don't invalidate", func() { | 206 Convey("errors don't invalidate", func() { |
| 210 // populate an object @ ID1 | 207 // populate an object @ ID1 |
| 211 » » » » » So(ds.Put(&object{ID: 1, Value: "somethi
ng"}), ShouldBeNil) | 208 » » » » » So(ds.Put(c, &object{ID: 1, Value: "some
thing"}), ShouldBeNil) |
| 212 » » » » » So(ds.Get(&object{ID: 1}), ShouldBeNil) | 209 » » » » » So(ds.Get(c, &object{ID: 1}), ShouldBeNi
l) |
| 213 So(numMemcacheItems(), ShouldEqual, 1) | 210 So(numMemcacheItems(), ShouldEqual, 1) |
| 214 | 211 |
| 215 » » » » » So(ds.RunInTransaction(func(c context.Co
ntext) error { | 212 » » » » » So(ds.RunInTransaction(c, func(c context
.Context) error { |
| 216 » » » » » » ds := datastore.Get(c) | |
| 217 o := &object{ID: 1} | 213 o := &object{ID: 1} |
| 218 » » » » » » So(ds.Get(o), ShouldBeNil) | 214 » » » » » » So(ds.Get(c, o), ShouldBeNil) |
| 219 So(o.Value, ShouldEqual, "someth
ing") | 215 So(o.Value, ShouldEqual, "someth
ing") |
| 220 o.Value = "txn" | 216 o.Value = "txn" |
| 221 » » » » » » So(ds.Put(o), ShouldBeNil) | 217 » » » » » » So(ds.Put(c, o), ShouldBeNil) |
| 222 return errors.New("OH NOES") | 218 return errors.New("OH NOES") |
| 223 }, nil).Error(), ShouldContainSubstring,
"OH NOES") | 219 }, nil).Error(), ShouldContainSubstring,
"OH NOES") |
| 224 | 220 |
| 225 // memcache still has the original | 221 // memcache still has the original |
| 226 So(numMemcacheItems(), ShouldEqual, 1) | 222 So(numMemcacheItems(), ShouldEqual, 1) |
| 227 » » » » » So(dsUnder.Delete(ds.KeyForObj(&object{I
D: 1})), ShouldBeNil) | 223 » » » » » So(ds.Delete(underCtx, ds.KeyForObj(unde
rCtx, &object{ID: 1})), ShouldBeNil) |
| 228 o := &object{ID: 1} | 224 o := &object{ID: 1} |
| 229 » » » » » So(ds.Get(o), ShouldBeNil) | 225 » » » » » So(ds.Get(c, o), ShouldBeNil) |
| 230 So(o.Value, ShouldEqual, "something") | 226 So(o.Value, ShouldEqual, "something") |
| 231 }) | 227 }) |
| 232 }) | 228 }) |
| 233 | 229 |
| 234 Convey("control", func() { | 230 Convey("control", func() { |
| 235 Convey("per-model bypass", func() { | 231 Convey("per-model bypass", func() { |
| 236 type model struct { | 232 type model struct { |
| 237 » » » » » » ID string `gae
:"$id"` | 233 » » » » » » ID string `gae:"$id"` |
| 238 » » » » » » UseDSCache datastore.Toggle `gae
:"$dscache.enable,false"` | 234 » » » » » » UseDSCache ds.Toggle `gae:"$dsca
che.enable,false"` |
| 239 | 235 |
| 240 Value string | 236 Value string |
| 241 } | 237 } |
| 242 | 238 |
| 243 itms := []model{ | 239 itms := []model{ |
| 244 {ID: "hi", Value: "something"}, | 240 {ID: "hi", Value: "something"}, |
| 245 » » » » » » {ID: "there", Value: "else", Use
DSCache: datastore.On}, | 241 » » » » » » {ID: "there", Value: "else", Use
DSCache: ds.On}, |
| 246 } | 242 } |
| 247 | 243 |
| 248 » » » » » So(ds.PutMulti(itms), ShouldBeNil) | 244 » » » » » So(ds.Put(c, itms), ShouldBeNil) |
| 249 » » » » » So(ds.GetMulti(itms), ShouldBeNil) | 245 » » » » » So(ds.Get(c, itms), ShouldBeNil) |
| 250 | 246 |
| 251 So(numMemcacheItems(), ShouldEqual, 1) | 247 So(numMemcacheItems(), ShouldEqual, 1) |
| 252 }) | 248 }) |
| 253 | 249 |
| 254 Convey("per-key shard count", func() { | 250 Convey("per-key shard count", func() { |
| 255 s := &shardObj{ID: 4, Value: "hi"} | 251 s := &shardObj{ID: 4, Value: "hi"} |
| 256 » » » » » So(ds.Put(s), ShouldBeNil) | 252 » » » » » So(ds.Put(c, s), ShouldBeNil) |
| 257 » » » » » So(ds.Get(s), ShouldBeNil) | 253 » » » » » So(ds.Get(c, s), ShouldBeNil) |
| 258 | 254 |
| 259 So(numMemcacheItems(), ShouldEqual, 1) | 255 So(numMemcacheItems(), ShouldEqual, 1) |
| 260 for i := 0; i < 20; i++ { | 256 for i := 0; i < 20; i++ { |
| 261 » » » » » » So(ds.Get(s), ShouldBeNil) | 257 » » » » » » So(ds.Get(c, s), ShouldBeNil) |
| 262 } | 258 } |
| 263 So(numMemcacheItems(), ShouldEqual, 4) | 259 So(numMemcacheItems(), ShouldEqual, 4) |
| 264 }) | 260 }) |
| 265 | 261 |
| 266 Convey("per-key cache disablement", func() { | 262 Convey("per-key cache disablement", func() { |
| 267 n := &noCacheObj{ID: "nurbs", Value: tru
e} | 263 n := &noCacheObj{ID: "nurbs", Value: tru
e} |
| 268 » » » » » So(ds.Put(n), ShouldBeNil) | 264 » » » » » So(ds.Put(c, n), ShouldBeNil) |
| 269 » » » » » So(ds.Get(n), ShouldBeNil) | 265 » » » » » So(ds.Get(c, n), ShouldBeNil) |
| 270 So(numMemcacheItems(), ShouldEqual, 0) | 266 So(numMemcacheItems(), ShouldEqual, 0) |
| 271 }) | 267 }) |
| 272 | 268 |
| 273 Convey("per-model expiration", func() { | 269 Convey("per-model expiration", func() { |
| 274 type model struct { | 270 type model struct { |
| 275 ID int64 `gae:"$id"` | 271 ID int64 `gae:"$id"` |
| 276 DSCacheExp int64 `gae:"$dscache.
expiration,7"` | 272 DSCacheExp int64 `gae:"$dscache.
expiration,7"` |
| 277 | 273 |
| 278 Value string | 274 Value string |
| 279 } | 275 } |
| 280 | 276 |
| 281 » » » » » So(ds.Put(&model{ID: 1, Value: "mooo"}),
ShouldBeNil) | 277 » » » » » So(ds.Put(c, &model{ID: 1, Value: "mooo"
}), ShouldBeNil) |
| 282 » » » » » So(ds.Get(&model{ID: 1}), ShouldBeNil) | 278 » » » » » So(ds.Get(c, &model{ID: 1}), ShouldBeNil
) |
| 283 | 279 |
| 284 » » » » » itm, err := mc.Get(MakeMemcacheKey(0, ds
.KeyForObj(&model{ID: 1}))) | 280 » » » » » itm, err := mc.GetKey(c, MakeMemcacheKey
(0, ds.KeyForObj(c, &model{ID: 1}))) |
| 285 So(err, ShouldBeNil) | 281 So(err, ShouldBeNil) |
| 286 | 282 |
| 287 clk.Add(10 * time.Second) | 283 clk.Add(10 * time.Second) |
| 288 » » » » » _, err = mc.Get(itm.Key()) | 284 » » » » » _, err = mc.GetKey(c, itm.Key()) |
| 289 » » » » » So(err, ShouldEqual, memcache.ErrCacheMi
ss) | 285 » » » » » So(err, ShouldEqual, mc.ErrCacheMiss) |
| 290 }) | 286 }) |
| 291 }) | 287 }) |
| 292 | 288 |
| 293 Convey("screw cases", func() { | 289 Convey("screw cases", func() { |
| 294 Convey("memcache contains bogus value (simulated
failed AddMulti)", func() { | 290 Convey("memcache contains bogus value (simulated
failed AddMulti)", func() { |
| 295 o := &object{ID: 1, Value: "spleen"} | 291 o := &object{ID: 1, Value: "spleen"} |
| 296 » » » » » So(ds.Put(o), ShouldBeNil) | 292 » » » » » So(ds.Put(c, o), ShouldBeNil) |
| 297 | 293 |
| 298 sekret := []byte("I am a banana") | 294 sekret := []byte("I am a banana") |
| 299 » » » » » itm := mc.NewItem(MakeMemcacheKey(0, ds.
KeyForObj(o))).SetValue(sekret) | 295 » » » » » itm := mc.NewItem(c, MakeMemcacheKey(0,
ds.KeyForObj(c, o))).SetValue(sekret) |
| 300 » » » » » So(mc.Set(itm), ShouldBeNil) | 296 » » » » » So(mc.Set(c, itm), ShouldBeNil) |
| 301 | 297 |
| 302 o = &object{ID: 1} | 298 o = &object{ID: 1} |
| 303 » » » » » So(ds.Get(o), ShouldBeNil) | 299 » » » » » So(ds.Get(c, o), ShouldBeNil) |
| 304 So(o.Value, ShouldEqual, "spleen") | 300 So(o.Value, ShouldEqual, "spleen") |
| 305 | 301 |
| 306 » » » » » itm, err := mc.Get(itm.Key()) | 302 » » » » » itm, err := mc.GetKey(c, itm.Key()) |
| 307 So(err, ShouldBeNil) | 303 So(err, ShouldBeNil) |
| 308 So(itm.Flags(), ShouldEqual, ItemUKNONWN
) | 304 So(itm.Flags(), ShouldEqual, ItemUKNONWN
) |
| 309 So(itm.Value(), ShouldResemble, sekret) | 305 So(itm.Value(), ShouldResemble, sekret) |
| 310 }) | 306 }) |
| 311 | 307 |
| 312 Convey("memcache contains bogus value (corrupt e
ntry)", func() { | 308 Convey("memcache contains bogus value (corrupt e
ntry)", func() { |
| 313 o := &object{ID: 1, Value: "spleen"} | 309 o := &object{ID: 1, Value: "spleen"} |
| 314 » » » » » So(ds.Put(o), ShouldBeNil) | 310 » » » » » So(ds.Put(c, o), ShouldBeNil) |
| 315 | 311 |
| 316 sekret := []byte("I am a banana") | 312 sekret := []byte("I am a banana") |
| 317 » » » » » itm := (mc.NewItem(MakeMemcacheKey(0, ds
.KeyForObj(o))). | 313 » » » » » itm := (mc.NewItem(c, MakeMemcacheKey(0,
ds.KeyForObj(c, o))). |
| 318 SetValue(sekret). | 314 SetValue(sekret). |
| 319 SetFlags(uint32(ItemHasData))) | 315 SetFlags(uint32(ItemHasData))) |
| 320 » » » » » So(mc.Set(itm), ShouldBeNil) | 316 » » » » » So(mc.Set(c, itm), ShouldBeNil) |
| 321 | 317 |
| 322 o = &object{ID: 1} | 318 o = &object{ID: 1} |
| 323 » » » » » So(ds.Get(o), ShouldBeNil) | 319 » » » » » So(ds.Get(c, o), ShouldBeNil) |
| 324 So(o.Value, ShouldEqual, "spleen") | 320 So(o.Value, ShouldEqual, "spleen") |
| 325 | 321 |
| 326 » » » » » itm, err := mc.Get(itm.Key()) | 322 » » » » » itm, err := mc.GetKey(c, itm.Key()) |
| 327 So(err, ShouldBeNil) | 323 So(err, ShouldBeNil) |
| 328 So(itm.Flags(), ShouldEqual, ItemHasData
) | 324 So(itm.Flags(), ShouldEqual, ItemHasData
) |
| 329 So(itm.Value(), ShouldResemble, sekret) | 325 So(itm.Value(), ShouldResemble, sekret) |
| 330 }) | 326 }) |
| 331 | 327 |
| 332 Convey("other entity has the lock", func() { | 328 Convey("other entity has the lock", func() { |
| 333 o := &object{ID: 1, Value: "spleen"} | 329 o := &object{ID: 1, Value: "spleen"} |
| 334 » » » » » So(ds.Put(o), ShouldBeNil) | 330 » » » » » So(ds.Put(c, o), ShouldBeNil) |
| 335 | 331 |
| 336 sekret := []byte("r@vmarod!#)%9T") | 332 sekret := []byte("r@vmarod!#)%9T") |
| 337 » » » » » itm := (mc.NewItem(MakeMemcacheKey(0, ds
.KeyForObj(o))). | 333 » » » » » itm := (mc.NewItem(c, MakeMemcacheKey(0,
ds.KeyForObj(c, o))). |
| 338 SetValue(sekret). | 334 SetValue(sekret). |
| 339 SetFlags(uint32(ItemHasLock))) | 335 SetFlags(uint32(ItemHasLock))) |
| 340 » » » » » So(mc.Set(itm), ShouldBeNil) | 336 » » » » » So(mc.Set(c, itm), ShouldBeNil) |
| 341 | 337 |
| 342 o = &object{ID: 1} | 338 o = &object{ID: 1} |
| 343 » » » » » So(ds.Get(o), ShouldBeNil) | 339 » » » » » So(ds.Get(c, o), ShouldBeNil) |
| 344 So(o.Value, ShouldEqual, "spleen") | 340 So(o.Value, ShouldEqual, "spleen") |
| 345 | 341 |
| 346 » » » » » itm, err := mc.Get(itm.Key()) | 342 » » » » » itm, err := mc.GetKey(c, itm.Key()) |
| 347 So(err, ShouldBeNil) | 343 So(err, ShouldBeNil) |
| 348 So(itm.Flags(), ShouldEqual, ItemHasLock
) | 344 So(itm.Flags(), ShouldEqual, ItemHasLock
) |
| 349 So(itm.Value(), ShouldResemble, sekret) | 345 So(itm.Value(), ShouldResemble, sekret) |
| 350 }) | 346 }) |
| 351 | 347 |
| 352 Convey("massive entities can't be cached", func(
) { | 348 Convey("massive entities can't be cached", func(
) { |
| 353 o := &object{ID: 1, Value: "spleen"} | 349 o := &object{ID: 1, Value: "spleen"} |
| 354 mr := mathrand.Get(c) | 350 mr := mathrand.Get(c) |
| 355 numRounds := (internalValueSizeLimit / 8
) * 2 | 351 numRounds := (internalValueSizeLimit / 8
) * 2 |
| 356 buf := bytes.Buffer{} | 352 buf := bytes.Buffer{} |
| 357 for i := 0; i < numRounds; i++ { | 353 for i := 0; i < numRounds; i++ { |
| 358 So(binary.Write(&buf, binary.Lit
tleEndian, mr.Int63()), ShouldBeNil) | 354 So(binary.Write(&buf, binary.Lit
tleEndian, mr.Int63()), ShouldBeNil) |
| 359 } | 355 } |
| 360 o.BigData = buf.Bytes() | 356 o.BigData = buf.Bytes() |
| 361 » » » » » So(ds.Put(o), ShouldBeNil) | 357 » » » » » So(ds.Put(c, o), ShouldBeNil) |
| 362 | 358 |
| 363 o.BigData = nil | 359 o.BigData = nil |
| 364 » » » » » So(ds.Get(o), ShouldBeNil) | 360 » » » » » So(ds.Get(c, o), ShouldBeNil) |
| 365 | 361 |
| 366 » » » » » itm, err := mc.Get(MakeMemcacheKey(0, ds
.KeyForObj(o))) | 362 » » » » » itm, err := mc.GetKey(c, MakeMemcacheKey
(0, ds.KeyForObj(c, o))) |
| 367 So(err, ShouldBeNil) | 363 So(err, ShouldBeNil) |
| 368 | 364 |
| 369 // Is locked until the next put, forcing
all access to the datastore. | 365 // Is locked until the next put, forcing
all access to the datastore. |
| 370 So(itm.Value(), ShouldResemble, []byte{}
) | 366 So(itm.Value(), ShouldResemble, []byte{}
) |
| 371 So(itm.Flags(), ShouldEqual, ItemHasLock
) | 367 So(itm.Flags(), ShouldEqual, ItemHasLock
) |
| 372 | 368 |
| 373 o.BigData = []byte("hi :)") | 369 o.BigData = []byte("hi :)") |
| 374 » » » » » So(ds.Put(o), ShouldBeNil) | 370 » » » » » So(ds.Put(c, o), ShouldBeNil) |
| 375 » » » » » So(ds.Get(o), ShouldBeNil) | 371 » » » » » So(ds.Get(c, o), ShouldBeNil) |
| 376 | 372 |
| 377 » » » » » itm, err = mc.Get(itm.Key()) | 373 » » » » » itm, err = mc.GetKey(c, itm.Key()) |
| 378 So(err, ShouldBeNil) | 374 So(err, ShouldBeNil) |
| 379 So(itm.Flags(), ShouldEqual, ItemHasData
) | 375 So(itm.Flags(), ShouldEqual, ItemHasData
) |
| 380 }) | 376 }) |
| 381 | 377 |
| 382 Convey("failure on Setting memcache locks is a h
ard stop", func() { | 378 Convey("failure on Setting memcache locks is a h
ard stop", func() { |
| 383 c, fb := featureBreaker.FilterMC(c, nil) | 379 c, fb := featureBreaker.FilterMC(c, nil) |
| 384 fb.BreakFeatures(nil, "SetMulti") | 380 fb.BreakFeatures(nil, "SetMulti") |
| 385 » » » » » ds := datastore.Get(c) | 381 » » » » » So(ds.Put(c, &object{ID: 1}).Error(), Sh
ouldContainSubstring, "SetMulti") |
| 386 » » » » » So(ds.Put(&object{ID: 1}).Error(), Shoul
dContainSubstring, "SetMulti") | |
| 387 }) | 382 }) |
| 388 | 383 |
| 389 Convey("failure on Setting memcache locks in a t
ransaction is a hard stop", func() { | 384 Convey("failure on Setting memcache locks in a t
ransaction is a hard stop", func() { |
| 390 c, fb := featureBreaker.FilterMC(c, nil) | 385 c, fb := featureBreaker.FilterMC(c, nil) |
| 391 fb.BreakFeatures(nil, "SetMulti") | 386 fb.BreakFeatures(nil, "SetMulti") |
| 392 » » » » » ds := datastore.Get(c) | 387 » » » » » So(ds.RunInTransaction(c, func(c context
.Context) error { |
| 393 » » » » » So(ds.RunInTransaction(func(c context.Co
ntext) error { | 388 » » » » » » So(ds.Put(c, &object{ID: 1}), Sh
ouldBeNil) |
| 394 » » » » » » So(datastore.Get(c).Put(&object{
ID: 1}), ShouldBeNil) | |
| 395 // no problems here... memcache
operations happen after the function | 389 // no problems here... memcache
operations happen after the function |
| 396 // body quits. | 390 // body quits. |
| 397 return nil | 391 return nil |
| 398 }, nil).Error(), ShouldContainSubstring,
"SetMulti") | 392 }, nil).Error(), ShouldContainSubstring,
"SetMulti") |
| 399 }) | 393 }) |
| 400 | 394 |
| 401 }) | 395 }) |
| 402 | 396 |
| 403 Convey("misc", func() { | 397 Convey("misc", func() { |
| 404 Convey("verify numShards caps at MaxShards", fun
c() { | 398 Convey("verify numShards caps at MaxShards", fun
c() { |
| 405 sc := supportContext{shardsForKey: shard
sForKey} | 399 sc := supportContext{shardsForKey: shard
sForKey} |
| 406 » » » » » So(sc.numShards(ds.KeyForObj(&shardObj{I
D: 9001})), ShouldEqual, MaxShards) | 400 » » » » » So(sc.numShards(ds.KeyForObj(c, &shardOb
j{ID: 9001})), ShouldEqual, MaxShards) |
| 407 }) | 401 }) |
| 408 | 402 |
| 409 Convey("CompressionType.String", func() { | 403 Convey("CompressionType.String", func() { |
| 410 So(NoCompression.String(), ShouldEqual,
"NoCompression") | 404 So(NoCompression.String(), ShouldEqual,
"NoCompression") |
| 411 So(ZlibCompression.String(), ShouldEqual
, "ZlibCompression") | 405 So(ZlibCompression.String(), ShouldEqual
, "ZlibCompression") |
| 412 So(CompressionType(100).String(), Should
Equal, "UNKNOWN_CompressionType(100)") | 406 So(CompressionType(100).String(), Should
Equal, "UNKNOWN_CompressionType(100)") |
| 413 }) | 407 }) |
| 414 }) | 408 }) |
| 415 }) | 409 }) |
| 416 | 410 |
| 417 Convey("disabled cases", func() { | 411 Convey("disabled cases", func() { |
| 418 defer func() { | 412 defer func() { |
| 419 globalEnabled = true | 413 globalEnabled = true |
| 420 }() | 414 }() |
| 421 | 415 |
| 422 So(IsGloballyEnabled(c), ShouldBeTrue) | 416 So(IsGloballyEnabled(c), ShouldBeTrue) |
| 423 | 417 |
| 424 So(SetGlobalEnable(c, false), ShouldBeNil) | 418 So(SetGlobalEnable(c, false), ShouldBeNil) |
| 425 // twice is a nop | 419 // twice is a nop |
| 426 So(SetGlobalEnable(c, false), ShouldBeNil) | 420 So(SetGlobalEnable(c, false), ShouldBeNil) |
| 427 | 421 |
| 428 // but it takes 5 minutes to kick in | 422 // but it takes 5 minutes to kick in |
| 429 So(IsGloballyEnabled(c), ShouldBeTrue) | 423 So(IsGloballyEnabled(c), ShouldBeTrue) |
| 430 clk.Add(time.Minute*5 + time.Second) | 424 clk.Add(time.Minute*5 + time.Second) |
| 431 So(IsGloballyEnabled(c), ShouldBeFalse) | 425 So(IsGloballyEnabled(c), ShouldBeFalse) |
| 432 | 426 |
| 433 » » » So(mc.Set(mc.NewItem("test").SetValue([]byte("hi"))), Sh
ouldBeNil) | 427 » » » So(mc.Set(c, mc.NewItem(c, "test").SetValue([]byte("hi")
)), ShouldBeNil) |
| 434 So(numMemcacheItems(), ShouldEqual, 1) | 428 So(numMemcacheItems(), ShouldEqual, 1) |
| 435 So(SetGlobalEnable(c, true), ShouldBeNil) | 429 So(SetGlobalEnable(c, true), ShouldBeNil) |
| 436 // memcache gets flushed as a side effect | 430 // memcache gets flushed as a side effect |
| 437 So(numMemcacheItems(), ShouldEqual, 0) | 431 So(numMemcacheItems(), ShouldEqual, 0) |
| 438 | 432 |
| 439 // Still takes 5 minutes to kick in | 433 // Still takes 5 minutes to kick in |
| 440 So(IsGloballyEnabled(c), ShouldBeFalse) | 434 So(IsGloballyEnabled(c), ShouldBeFalse) |
| 441 clk.Add(time.Minute*5 + time.Second) | 435 clk.Add(time.Minute*5 + time.Second) |
| 442 So(IsGloballyEnabled(c), ShouldBeTrue) | 436 So(IsGloballyEnabled(c), ShouldBeTrue) |
| 443 }) | 437 }) |
| 444 }) | 438 }) |
| 445 } | 439 } |
| 446 | 440 |
| 447 func TestStaticEnable(t *testing.T) { | 441 func TestStaticEnable(t *testing.T) { |
| 448 // intentionally not parallel b/c deals with global variable | 442 // intentionally not parallel b/c deals with global variable |
| 449 // t.Parallel() | 443 // t.Parallel() |
| 450 | 444 |
| 451 Convey("Test InstanceEnabledStatic", t, func() { | 445 Convey("Test InstanceEnabledStatic", t, func() { |
| 452 InstanceEnabledStatic = false | 446 InstanceEnabledStatic = false |
| 453 defer func() { | 447 defer func() { |
| 454 InstanceEnabledStatic = true | 448 InstanceEnabledStatic = true |
| 455 }() | 449 }() |
| 456 | 450 |
| 457 c := context.Background() | 451 c := context.Background() |
| 458 newC := FilterRDS(c, nil) | 452 newC := FilterRDS(c, nil) |
| 459 So(newC, ShouldEqual, c) | 453 So(newC, ShouldEqual, c) |
| 460 }) | 454 }) |
| 461 } | 455 } |
| OLD | NEW |