Add server/cache support to gaeconfig.

appengine/gaeconfig/cache_test.go

 // Copyright 2016 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package gaeconfig
 
 import (
+        "math/rand"
         "testing"
         "time"
 
         "golang.org/x/net/context"
 
         "github.com/luci/gae/filter/featureBreaker"
-        "github.com/luci/gae/impl/memory"
         mc "github.com/luci/gae/service/memcache"
         "github.com/luci/luci-go/common/clock/testclock"
-        "github.com/luci/luci-go/common/data/caching/proccache"
+        memconfig "github.com/luci/luci-go/common/config/impl/memory"
+        "github.com/luci/luci-go/common/errors"
+        "github.com/luci/luci-go/server/config"
+        "github.com/luci/luci-go/server/config/caching"
+        "github.com/luci/luci-go/server/config/testconfig"
 
-        . "github.com/luci/luci-go/common/testing/assertions"
+        "github.com/luci/gae/impl/memory"
+
         . "github.com/smartystreets/goconvey/convey"
 )
 
-func TestCache(t *testing.T) {
+func TestMemcache(t *testing.T) {
         t.Parallel()
 
         Convey("Test cache", t, func() {
                 c := context.Background()
                 c = memory.Use(c)
 
-                pc := &proccache.Cache{}
-                c = proccache.Use(c, pc)
                 c, clk := testclock.UseTime(c, testclock.TestTimeUTC)
                 _ = clk
 
                 c, mcFB := featureBreaker.FilterMC(c, nil)
 
-                cache := &cache{}
+                getMCStats := func(c context.Context) *mc.Statistics {
+                        st, err := mc.Stats(c)
+                        if err != nil {
+                                panic(err)
+                        }
+                        return st
+                }
 
-                Convey("Should be able to store stuff", func() {
-                        Convey("memcache+proccache", func() {
-                                cache.Store(c, "item", time.Second, []byte("foobar"))
-                                itm, err := mc.GetKey(c, string(cacheKey("item")))
-                                So(err, ShouldBeNil)
-                                So(itm.Value(), ShouldResemble, []byte("\xff\xf4\xea\x9c\xf7\xd8\xde\xdc\x01foobar"))
-                                val, ok := proccache.Get(c, cacheKey("item"))
-                                So(ok, ShouldBeTrue)
-                                So(val, ShouldResemble, []byte("foobar"))
+                configDB := map[string]memconfig.ConfigSet{
+                        "projects/foo": {
+                                "test.cfg": "foo",
+                        },
+                }
 
-                                Convey("and get it back", func() {
-                                        So(cache.Retrieve(c, "item"), ShouldResemble, []byte("foobar"))
+                var backend config.Backend
+                backend = &config.ClientBackend{
+                        Provider: &testconfig.LocalClientProvider{
+                                Base: memconfig.New(configDB),
+                        },
+                }
+                backend = memcacheBackend(backend, time.Minute)
+                c = config.WithBackend(c, backend)
 
-                                        Convey("unless it's expired", func() {
-                                                clk.Add(time.Second * 2)
-                                                So(cache.Retrieve(c, "item"), ShouldBeNil)
-                                        })
+                Convey(`Pulling items from memcache`, func() {
+                        var content string
 
-                                        Convey("when missing from proccache", func() {
-                                                proccache.GetCache(c).Mutate(cacheKey("item"), func(*proccache.Entry) *proccache.Entry {
-                                                        return nil
-                                                })
-                                                So(cache.Retrieve(c, "item"), ShouldResemble, []byte("foobar"))
-                                        })
+                        Convey(`Should be able to fetch a config as service`, func() {
+                                So(config.Get(c, config.AsService, "projects/foo", "test.cfg", config.String(&content), nil), ShouldBeNil)
+                                So(content, ShouldEqual, "foo")
+                                So(getMCStats(c).Misses, ShouldEqual, 1)
+                                So(getMCStats(c).Hits, ShouldEqual, 0)
+                                So(getMCStats(c).Items, ShouldEqual, 1)
 
-                                        Convey("unless memcache is corrupt", func() {
-                                                proccache.GetCache(c).Mutate(cacheKey("item"), func(*proccache.Entry) *proccache.Entry {
-                                                        return nil
-                                                })
-                                                err := mc.Set(c, mc.NewItem(c, string(cacheKey("item"))).SetValue([]byte("\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff")))
-                                                So(err, ShouldBeNil)
+                                // A second fetch hits memcache.
+                                So(config.Get(c, config.AsService, "projects/foo", "test.cfg", config.String(&content), nil), ShouldBeNil)
+                                So(content, ShouldEqual, "foo")
+                                So(getMCStats(c).Misses, ShouldEqual, 1)
+                                So(getMCStats(c).Hits, ShouldEqual, 1)
+                                So(getMCStats(c).Items, ShouldEqual, 1)
 
-                                                So(cache.Retrieve(c, "item"), ShouldBeNil)
-                                        })
+                                // Memcache expires, full reload.
+                                clk.Add(time.Minute)
 
-                                })
+                                So(config.Get(c, config.AsService, "projects/foo", "test.cfg", config.String(&content), nil), ShouldBeNil)
+                                So(content, ShouldEqual, "foo")
+                                So(getMCStats(c).Misses, ShouldEqual, 2)
+                                So(getMCStats(c).Hits, ShouldEqual, 1)
+                                So(getMCStats(c).Items, ShouldEqual, 1)
                         })
 
-                        Convey("if memcache is down", func() {
-                                mcFB.BreakFeatures(nil, "SetMulti")
+                        Convey(`When memcache is broken`, func() {
+                                testErr := errors.New("test error")
+                                mcFB.BreakFeatures(testErr, "GetMulti")
 
-                                cache.Store(c, "item", time.Second, []byte("foobar"))
-                                _, err := mc.GetKey(c, string(cacheKey("item")))
-                                So(err, ShouldErrLike, mc.ErrCacheMiss)
+                                So(config.Get(c, config.AsService, "projects/foo", "test.cfg", config.String(&content), nil), ShouldBeNil)
+                                So(content, ShouldEqual, "foo")
+                                So(getMCStats(c).Misses, ShouldEqual, 0)
+                                So(getMCStats(c).Hits, ShouldEqual, 0)
+                                So(getMCStats(c).Items, ShouldEqual, 0)
+                        })
 
-                                val, ok := proccache.Get(c, cacheKey("item"))
-                                So(ok, ShouldBeTrue)
-                                So(val, ShouldResemble, []byte("foobar"))
+                        Convey(`When a cached entry is corrupted`, func() {
+                                cacheKey := caching.Key{
+                                        Schema:    caching.Schema,
+                                        Op:        caching.OpGet,
+                                        ConfigSet: "projects/foo",
+                                        Path:      "test.cfg",
+                                        Content:   true,
+                                        Authority: config.AsService,
+                                }
+                                So(mc.Set(c, mc.NewItem(c, memcacheKey(&cacheKey)).SetValue([]byte("!!! trash !!!"))), ShouldBeNil)
+                                So(config.Get(c, config.AsService, "projects/foo", "test.cfg", config.String(&content), nil), ShouldBeNil)
+                                So(content, ShouldEqual, "foo")
+                                So(getMCStats(c).Misses, ShouldEqual, 0)
+                                So(getMCStats(c).Hits, ShouldEqual, 1)
+                                So(getMCStats(c).Items, ShouldEqual, 1)
+                        })
 
-                                Convey("and still get it back", func() {
-                                        So(cache.Retrieve(c, "item"), ShouldResemble, []byte("foobar"))
+                        Convey(`Will skip the cache if an item is too large.`, func() {
+                                // We need to use a pseudo-random string b/c we compress it.
+                                buf := make([]byte, maxMemCacheSize*2)
+                                rng := rand.New(rand.NewSource(0))
+                                _, _ = rng.Read(buf)
 
-                                        Convey("unless it's expired", func() {
-                                                clk.Add(time.Second * 2)
-                                                So(cache.Retrieve(c, "item"), ShouldBeNil)
-                                        })
-                                })
+                                orig := string(buf)
+                                configDB["projects/foo"]["test.cfg"] = orig
+
+                                So(config.Get(c, config.AsService, "projects/foo", "test.cfg", config.String(&content), nil), ShouldBeNil)
+                                So(content, ShouldEqual, orig)
+                                So(getMCStats(c).Misses, ShouldEqual, 1)
+                                So(getMCStats(c).Hits, ShouldEqual, 0)
+                                So(getMCStats(c).Items, ShouldEqual, 0)
                         })
                 })
         })
 }