Implement config service cache on top of datastore

appengine/gaeconfig/ds_test.go

+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package gaeconfig
+
+import (
+        "fmt"
+        "net/http"
+        "net/http/httptest"
+        "sort"
+        "strings"
+        "testing"
+        "time"
+
+        "github.com/luci/luci-go/appengine/datastorecache"
+        "github.com/luci/luci-go/common/clock/testclock"
+        commonConfig "github.com/luci/luci-go/common/config"
+        memConfig "github.com/luci/luci-go/common/config/impl/memory"
+        "github.com/luci/luci-go/common/errors"
+        configPB "github.com/luci/luci-go/common/proto/config"
+        "github.com/luci/luci-go/server/auth"
+        "github.com/luci/luci-go/server/auth/authtest"
+        "github.com/luci/luci-go/server/config"
+        "github.com/luci/luci-go/server/config/caching"
+        "github.com/luci/luci-go/server/config/testconfig"
+        "github.com/luci/luci-go/server/config/textproto"
+        "github.com/luci/luci-go/server/router"
+        "github.com/luci/luci-go/server/settings"
+
+        "github.com/luci/gae/filter/count"
+        "github.com/luci/gae/impl/memory"
+        "github.com/luci/gae/service/datastore"
+
+        "github.com/golang/protobuf/proto"
+        "golang.org/x/net/context"
+
+        . "github.com/luci/luci-go/common/testing/assertions"
+        . "github.com/smartystreets/goconvey/convey"
+)
+
+// testCache is a generic Cache testing layer.
+type testCache interface {
+        dsCacheBackend
+
+        setCacheErr(err error)
+        setProjectDNE(project string)
+        addConfig(configSet, path, content string) *config.Item
+        addProjectConfig(name commonConfig.ProjectName, access string)
+        addConfigSets(path string, configSets ...string) []string
+        addConfigSetURL(configSet string) string
+}
+
+func projectConfigWithAccess(name commonConfig.ProjectName, access ...string) *configPB.ProjectCfg {
+        return &configPB.ProjectCfg{
+                Name:   proto.String(string(name)),
+                Access: access,
+        }
+}
+
+// fakeCache is a pure in-memory testCache implementation. It is very simple,
+// storing only raw cache key/value pairs.
+type fakeCache struct {
+        d   map[string]datastorecache.Value
+        err error
+}
+
+func mkFakeCache() *fakeCache {
+        return &fakeCache{
+                d: make(map[string]datastorecache.Value),
+        }
+}
+
+func (fc *fakeCache) Get(c context.Context, key []byte) (v datastorecache.Value, err error) {
+        if err = fc.err; err != nil {
+                return
+        }
+
+        var k caching.Key
+        caching.Decode(key, &k)
+
+        var ok bool
+        if v, ok = fc.d[string(key)]; ok {
+                return
+        }
+
+        err = datastorecache.ErrCacheExpired
+        return
+}
+
+func (fc *fakeCache) setCacheData(key caching.Key, d []byte) {
+}
+
+func (fc *fakeCache) set(key caching.Key, v *caching.Value) {
+        encKey, err := caching.Encode(&key)
+        if err != nil {
+                panic(fmt.Errorf("failed to encode key: %s", err))
+        }
+
+        if v == nil {
+                delete(fc.d, string(encKey))
+                return
+        }
+
+        encValue, err := v.Encode()
+        if err != nil {
+                panic(fmt.Errorf("failed to encode cache value: %s", err))
+        }
+
+        fc.d[string(encKey)] = datastorecache.Value{
+                Schema:      dsCacheSchema,
+                Data:        encValue,
+                Description: key.String(),
+        }
+}
+
+func (fc *fakeCache) setCacheErr(err error) { fc.err = err }
+
+func (fc *fakeCache) setProjectDNE(project string) {
+        // Get for this project config will fail.
+        fc.set(caching.Key{
+                Schema:    caching.Schema,
+                Op:        caching.OpGet,
+                ConfigSet: config.ProjectConfigSet(commonConfig.ProjectName(project)),
+                Path:      config.ProjectConfigPath,
+        }, nil)
+}
+
+func (fc *fakeCache) addConfigImpl(configSet, path, format, formatData, content string) *config.Item {
+        var (
+                item *config.Item
+                cv   caching.Value
+        )
+        if content != "" {
+                item = &config.Item{
+                        Meta: config.Meta{
+                                ConfigSet:   configSet,
+                                Path:        path,
+                                ContentHash: "hash",
+                        },
+                        Content:    content,
+                        Format:     format,
+                        FormatData: formatData,
+                }
+                cv.LoadItems(item)
+        }
+
+        fc.set(caching.Key{
+                Schema:     caching.Schema,
+                Authority:  config.AsService,
+                Op:         caching.OpGet,
+                ConfigSet:  configSet,
+                Path:       path,
+                Content:    true,
+                Format:     format,
+                FormatData: formatData,
+        }, &cv)
+
+        return item
+}
+
+func (fc *fakeCache) addConfig(configSet, path, content string) *config.Item {
+        return fc.addConfigImpl(configSet, path, "", "", content)
+}
+
+// addProjectConfig caches a "project.cfg" file for the specified project with
+// the specified access string.
+func (fc *fakeCache) addProjectConfig(name commonConfig.ProjectName, access string) {
+        // We're loading the resolved version of this cache item.
+        pcfg := projectConfigWithAccess(name, access)
+        pcfgName := proto.MessageName(pcfg)
+
+        f := textproto.Formatter{}
+        formattedData, err := f.FormatItem(proto.MarshalTextString(pcfg), pcfgName)
+        if err != nil {
+                panic(err)
+        }
+
+        fc.addConfigImpl(config.ProjectConfigSet(name), config.ProjectConfigPath,
+                textproto.BinaryFormat, pcfgName, formattedData)
+}
+
+func (fc *fakeCache) addConfigSets(path string, configSets ...string) []string {
+        items := make([]*config.Item, len(configSets))
+        contents := make([]string, len(configSets))
+        for i, cs := range configSets {
+                contents[i] = cs
+                items[i] = &config.Item{
+                        Meta: config.Meta{
+                                ConfigSet:   cs,
+                                Path:        path,
+                                ContentHash: "hash",
+                        },
+                        Content:    contents[i],
+                        Format:     "",
+                        FormatData: "",
+                }
+        }
+
+        for _, t := range []config.GetAllType{config.Project, config.Ref} {
+                var cv caching.Value
+                cv.LoadItems(items...)
+
+                fc.set(caching.Key{
+                        Schema:     caching.Schema,
+                        Authority:  config.AsService,
+                        Op:         caching.OpGetAll,
+                        Content:    true,
+                        Path:       path,
+                        GetAllType: t,
+                        Format:     "",
+                        FormatData: "",
+                }, &cv)
+        }
+        return contents
+}
+
+func (fc *fakeCache) addConfigSetURL(configSet string) string {
+        u := fmt.Sprintf("https://exmaple.com/config-sets/%s", configSet)
+        fc.set(caching.Key{
+                Schema:    caching.Schema,
+                Authority: config.AsService,
+                Op:        caching.OpConfigSetURL,
+                ConfigSet: configSet,
+        }, &caching.Value{
+                URL: u,
+        })
+        return u
+}
+
+// fullStackCache is a testCache implementation built on top of an in-memory
+// base config.Interface with the datastore Cache layer on top of it.
+type fullStackCache struct {
+        cache *datastorecache.Cache
+        err   error
+
+        data    map[string]memConfig.ConfigSet
+        backend config.Backend
+        junkIdx int
+}
+
+func (fsc *fullStackCache) Get(c context.Context, key []byte) (datastorecache.Value, error) {
+        if err := fsc.err; err != nil {
+                return datastorecache.Value{}, err
+        }
+        return fsc.cache.Get(c, key)
+}
+
+func (fsc *fullStackCache) setCacheErr(err error) { fsc.err = err }
+
+func (fsc *fullStackCache) setProjectDNE(project string) {
+        key := "projects/" + project
+        for k := range fsc.data {
+                if k == key || strings.HasPrefix(k, key+"/") {
+                        delete(fsc.data, k)
+                }
+        }
+}
+
+func (fsc *fullStackCache) addConfig(configSet, path, content string) *config.Item {
+        cset := fsc.data[configSet]
+        if cset == nil {
+                cset = memConfig.ConfigSet{}
+                fsc.data[configSet] = cset
+        }
+        if content == "" {
+                delete(cset, path)
+                return nil
+        }
+        cset[path] = content
+
+        // Pull the config right back out of the base service.
+        item, err := fsc.backend.Get(context.Background(), configSet, path, config.Params{
+                Authority: config.AsService,
+        })
+        if err != nil {
+                panic(err)
+        }
+        return item
+}
+
+// addProjectConfig caches a "project.cfg" file for the specified project with
+// the specified access string.
+func (fsc *fullStackCache) addProjectConfig(name commonConfig.ProjectName, access string) {
+        fsc.addConfig(config.ProjectConfigSet(name), config.ProjectConfigPath,
+                proto.MarshalTextString(projectConfigWithAccess(name, access)))
+}
+
+func (fsc *fullStackCache) addConfigSets(path string, configSets ...string) []string {
+        sort.Strings(configSets)
+        items := make([]*config.Item, len(configSets))
+        for i, cs := range configSets {
+                items[i] = fsc.addConfig(cs, path, cs)
+        }
+        return configSets
+}
+
+func (fsc *fullStackCache) addConfigSetURL(configSet string) string {
+        if _, ok := fsc.data[configSet]; !ok {
+                fsc.data[configSet] = memConfig.ConfigSet{}
+        }
+
+        // We're pretty rigid here. Whatever our backend returns is all we can
+        // return. We will just assert that anything more flexible has to conform to
+        // this.
+        v, err := fsc.backend.ConfigSetURL(context.Background(), config.AsService, configSet)
+        if err != nil {
+                panic(err)
+        }
+        return v.String()
+}
+
+// stripMeta strips cache-specific identifying information from a set of Metas.
+func stripMeta(metas []*config.Meta) []*config.Meta {
+        for _, meta := range metas {
+                meta.ContentHash = ""
+                meta.Revision = ""
+        }
+        return metas
+}
+
+func testDatastoreCacheImpl(c context.Context, backend config.Backend, cache testCache) {
+        // Install fake auth state.
+        var authState authtest.FakeState
+        c = auth.WithState(c, &authState)
+        authState.Identity = "user:person@example.com"
+        authState.IdentityGroups = []string{"users"}
+
+        dsc := datastoreCache{
+                refreshInterval: 1 * time.Hour,
+                failOpen:        false,
+                cache:           cache,
+        }
+        c = config.WithBackend(c, dsc.getBackend(backend))
+
+        testErr := errors.New("test error")
+
+        Convey(`Test Get`, func() {
+                var v string
+
+                Convey(`Config missing`, func() {
+                        cache.addConfig("projects/test", "foo", "")
+
+                        So(config.Get(c, config.AsService, "projects/test", "foo", config.String(&v), nil),
+                                ShouldEqual, config.ErrNoConfig)
+                })
+
+                Convey(`Config is present`, func() {
+                        cache.addConfig("projects/test", "foo", "bar")
+                        cache.addProjectConfig("test", "group:privileged")
+
+                        Convey(`As service`, func() {
+                                So(config.Get(c, config.AsService, "projects/test", "foo", config.String(&v), nil), ShouldBeNil)
+                                So(v, ShouldEqual, "bar")
+                        })
+
+                        Convey(`As user, when not a project group member, fails with ErrNoConfig`, func() {
+                                So(config.Get(c, config.AsUser, "projects/test", "foo", config.String(&v), nil),
+                                        ShouldEqual, config.ErrNoConfig)
+                        })
+
+                        Convey(`As user, when a project group member, succeeds.`, func() {
+                                authState.IdentityGroups = append(authState.IdentityGroups, "privileged")
+                                So(config.Get(c, config.AsUser, "projects/test", "foo", config.String(&v), nil), ShouldBeNil)
+                                So(v, ShouldEqual, "bar")
+                        })
+
+                        Convey(`As anonymous, fails with ErrNoConfig`, func() {
+                                So(config.Get(c, config.AsAnonymous, "projects/test", "foo", config.String(&v), nil),
+                                        ShouldEqual, config.ErrNoConfig)
+                        })
+                })
+        })
+
+        Convey(`Test GetAll with projects`, func() {
+                var v []string
+                var meta []*config.Meta
+
+                Convey(`When cache returns an error`, func() {
+                        cache.setCacheErr(testErr)
+
+                        So(config.GetAll(c, config.AsService, config.Project, "test.cfg", config.StringSlice(&v), nil),
+                                ShouldUnwrapTo, testErr)
+                        So(config.GetAll(c, config.AsUser, config.Project, "test.cfg", config.StringSlice(&v), nil),
+                                ShouldUnwrapTo, testErr)
+                        So(config.GetAll(c, config.AsAnonymous, config.Project, "test.cfg", config.StringSlice(&v), nil),
+                                ShouldUnwrapTo, testErr)
+                })
+
+                Convey(`With project configs installed`, func() {
+                        allConfigs := cache.addConfigSets("test.cfg",
+                                "projects/bar",
+                                "projects/baz",
+                                "projects/foo")
+
+                        Convey(`As service, retrieves all configs.`, func() {
+                                So(config.GetAll(c, config.AsService, config.Project, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+                                So(v, ShouldResemble, allConfigs)
+                                So(stripMeta(meta), ShouldResemble, []*config.Meta{
+                                        {ConfigSet: "projects/bar", Path: "test.cfg"},
+                                        {ConfigSet: "projects/baz", Path: "test.cfg"},
+                                        {ConfigSet: "projects/foo", Path: "test.cfg"},
+                                })
+                        })
+
+                        Convey(`As user`, func() {
+                                Convey(`Not a member of any projects, receives empty slice.`, func() {
+                                        cache.addProjectConfig("foo", "group:someone")
+
+                                        So(config.GetAll(c, config.AsUser, config.Project, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+                                        So(v, ShouldResemble, []string(nil))
+                                        So(stripMeta(meta), ShouldResemble, []*config.Meta{})
+                                })
+
+                                Convey(`Member of "foo", gets only "foo".`, func() {
+                                        cache.addProjectConfig("foo", "group:users")
+
+                                        So(config.GetAll(c, config.AsUser, config.Project, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+                                        So(v, ShouldResemble, allConfigs[2:3])
+                                        So(stripMeta(meta), ShouldResemble, []*config.Meta{
+                                                {ConfigSet: "projects/foo", Path: "test.cfg"},
+                                        })
+                                })
+
+                                Convey(`Member of all projects, gets all projects.`, func() {
+                                        cache.addProjectConfig("foo", "group:users")
+                                        cache.addProjectConfig("bar", "group:users")
+                                        cache.addProjectConfig("baz", "group:users")
+
+                                        So(config.GetAll(c, config.AsUser, config.Project, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+                                        So(v, ShouldResemble, allConfigs)
+                                        So(stripMeta(meta), ShouldResemble, []*config.Meta{
+                                                {ConfigSet: "projects/bar", Path: "test.cfg"},
+                                                {ConfigSet: "projects/baz", Path: "test.cfg"},
+                                                {ConfigSet: "projects/foo", Path: "test.cfg"},
+                                        })
+                                })
+                        })
+                })
+        })
+
+        Convey(`Test GetAll with refs`, func() {
+                var v []string
+                var meta []*config.Meta
+
+                Convey(`When cache returns an error`, func() {
+                        cache.setCacheErr(testErr)
+
+                        So(config.GetAll(c, config.AsService, config.Ref, "test.cfg", config.StringSlice(&v), nil),
+                                ShouldUnwrapTo, testErr)
+                        So(config.GetAll(c, config.AsUser, config.Ref, "test.cfg", config.StringSlice(&v), nil),
+                                ShouldUnwrapTo, testErr)
+                        So(config.GetAll(c, config.AsAnonymous, config.Ref, "test.cfg", config.StringSlice(&v), nil),
+                                ShouldUnwrapTo, testErr)
+                })
+
+                Convey(`With ref configs installed`, func() {
+                        allConfigs := cache.addConfigSets("test.cfg",
+                                "projects/bar/refs/branches/mybranch",
+                                "projects/bar/refs/heads/master",
+                                "projects/foo/refs/branches/mybranch",
+                                "projects/foo/refs/heads/master")
+
+                        Convey(`As service, retrieves all configs.`, func() {
+                                So(config.GetAll(c, config.AsService, config.Ref, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+                                So(v, ShouldResemble, allConfigs)
+                                So(stripMeta(meta), ShouldResemble, []*config.Meta{
+                                        {ConfigSet: "projects/bar/refs/branches/mybranch", Path: "test.cfg"},
+                                        {ConfigSet: "projects/bar/refs/heads/master", Path: "test.cfg"},
+                                        {ConfigSet: "projects/foo/refs/branches/mybranch", Path: "test.cfg"},
+                                        {ConfigSet: "projects/foo/refs/heads/master", Path: "test.cfg"},
+                                })
+                        })
+
+                        Convey(`As user`, func() {
+                                Convey(`Not a member of any projects, receives empty slice.`, func() {
+                                        cache.addProjectConfig("foo", "group:someone")
+
+                                        So(config.GetAll(c, config.AsUser, config.Ref, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+                                        So(v, ShouldResemble, []string(nil))
+                                        So(stripMeta(meta), ShouldResemble, []*config.Meta{})
+                                })
+
+                                Convey(`Member of "foo", gets only "foo".`, func() {
+                                        cache.addProjectConfig("foo", "group:users")
+
+                                        So(config.GetAll(c, config.AsUser, config.Ref, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+                                        So(v, ShouldResemble, allConfigs[2:4])
+                                        So(stripMeta(meta), ShouldResemble, []*config.Meta{
+                                                {ConfigSet: "projects/foo/refs/branches/mybranch", Path: "test.cfg"},
+                                                {ConfigSet: "projects/foo/refs/heads/master", Path: "test.cfg"},
+                                        })
+                                })
+
+                                Convey(`Member of all projects, gets all projects.`, func() {
+                                        cache.addProjectConfig("foo", "group:users")
+                                        cache.addProjectConfig("bar", "group:users")
+
+                                        So(config.GetAll(c, config.AsUser, config.Ref, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+                                        So(v, ShouldResemble, allConfigs)
+                                        So(stripMeta(meta), ShouldResemble, []*config.Meta{
+                                                {ConfigSet: "projects/bar/refs/branches/mybranch", Path: "test.cfg"},
+                                                {ConfigSet: "projects/bar/refs/heads/master", Path: "test.cfg"},
+                                                {ConfigSet: "projects/foo/refs/branches/mybranch", Path: "test.cfg"},
+                                                {ConfigSet: "projects/foo/refs/heads/master", Path: "test.cfg"},
+                                        })
+                                })
+                        })
+                })
+        })
+
+        Convey(`Test ConfigSetURL`, func() {
+                cache.addProjectConfig("foo", "group:someone")
+                csURL := cache.addConfigSetURL("projects/foo")
+
+                Convey(`AsService`, func() {
+                        u, err := config.GetConfigSetURL(c, config.AsService, "projects/foo")
+                        So(err, ShouldBeNil)
+                        So(u.String(), ShouldEqual, csURL)
+                })
+
+                Convey(`AsUser`, func() {
+
+                        Convey(`When not a member of the group.`, func() {
+                                _, err := config.GetConfigSetURL(c, config.AsUser, "projects/foo")
+                                So(err, ShouldEqual, config.ErrNoConfig)
+                        })
+
+                        Convey(`When a member of the group.`, func() {
+                                authState.IdentityGroups = append(authState.IdentityGroups, "someone")
+
+                                u, err := config.GetConfigSetURL(c, config.AsUser, "projects/foo")
+                                So(err, ShouldBeNil)
+                                So(u.String(), ShouldEqual, csURL)
+                        })
+                })
+
+                Convey(`AsAnonymous`, func() {
+                        // Log in just to show that, while AsUser could access, AsAnonymous
+                        // cannot.
+                        authState.IdentityGroups = append(authState.IdentityGroups, "someone")
+                        _, err := config.GetConfigSetURL(c, config.AsUser, "projects/foo")
+                        So(err, ShouldBeNil)
+
+                        _, err = config.GetConfigSetURL(c, config.AsAnonymous, "projects/foo")
+                        So(err, ShouldEqual, config.ErrNoConfig)
+                })
+        })
+}
+
+func TestDatastoreCache(t *testing.T) {
+        t.Parallel()
+
+        Convey(`Testing with in-memory stub cache`, t, func() {
+                c := context.Background()
+                fc := mkFakeCache()
+
+                var backend config.Backend
+                backend = &config.ClientBackend{
+                        Provider: &testconfig.LocalClientProvider{
+                                Base: memConfig.New(nil),
+                        },
+                }
+
+                fr := defaultFormatterRegistry()
+                backend = &config.FormatBackend{
+                        Backend:     backend,
+                        GetRegistry: func(context.Context) *config.FormatterRegistry { return fr },
+                }
+
+                Convey(`Standard datastore tests`, func() {
+                        testDatastoreCacheImpl(c, backend, fc)
+                })
+
+                Convey(`A testing setup built around the fake cache`, func() {
+                        dsc := datastoreCache{
+                                refreshInterval: 1 * time.Hour,
+                                failOpen:        false,
+                                cache:           fc,
+                        }
+                        c = config.WithBackend(c, dsc.getBackend(backend))
+
+                        Convey(`Errors with different schema.`, func() {
+                                fc.addConfig("foo", "bar", "value")
+                                for k, v := range fc.d {
+                                        v.Schema = "unknown"
+                                        fc.d[k] = v
+                                }
+
+                                var v string
+                                So(config.Get(c, config.AsService, "foo", "bar", config.String(&v), nil),
+                                        ShouldErrLike, `response schema ("unknown") doesn't match current`)
+                        })
+                })
+        })
+}
+
+func TestDatastoreCacheFullStack(t *testing.T) {
+        t.Parallel()
+
+        Convey(`Testing full-stack datastore cache`, t, func() {
+                c := memory.Use(context.Background())
+
+                data := map[string]memConfig.ConfigSet{}
+
+                var backend config.Backend
+                backend = &config.ClientBackend{
+                        Provider: &testconfig.LocalClientProvider{
+                                Base: memConfig.New(data),
+                        },
+                }
+
+                fr := defaultFormatterRegistry()
+                backend = &config.FormatBackend{
+                        Backend:     backend,
+                        GetRegistry: func(context.Context) *config.FormatterRegistry { return fr },
+                }
+
+                fsc := fullStackCache{
+                        cache:   &dsCache,
+                        data:    data,
+                        backend: backend,
+                }
+                testDatastoreCacheImpl(c, backend, &fsc)
+        })
+}
+
+func TestDatastoreCacheIntegration(t *testing.T) {
+        t.Parallel()
+
+        Convey(`A testing environment with fake cache and config service`, t, func() {
+                c, clk := testclock.UseTime(context.Background(), datastore.RoundTime(testclock.TestTimeUTC))
+
+                c = memory.Use(c)
+
+                // Install fake auth state.
+                var authState authtest.FakeState
+                c = auth.WithState(c, &authState)
+                authState.Identity = "user:person@example.com"
+                authState.IdentityGroups = []string{"all", "users"}
+
+                // Use a memory-backed Config service instance.
+                baseMap := map[string]memConfig.ConfigSet{
+                        "projects/open": map[string]string{
+                                "project.cfg": proto.MarshalTextString(projectConfigWithAccess("foo", "group:all")),
+                                "test.cfg":    "Test Config Content",
+                        },
+                        "projects/foo": map[string]string{
+                                "project.cfg": proto.MarshalTextString(projectConfigWithAccess("foo", "group:restricted")),
+                        },
+                        "projects/noconfig": map[string]string{},
+                        "projects/invalidconfig": map[string]string{
+                                "project.cfg": "!!! not a valid config !!!",
+                        },
+                        "projects/foo/refs/heads/master": map[string]string{
+                                "ref.cfg": "foo",
+                        },
+                        "projects/foo/refs/branches/bar": map[string]string{
+                                "ref.cfg": "foo/bar",
+                        },
+                        "projects/bar/refs/heads/master": map[string]string{
+                                "ref.cfg": "bar",
+                        },
+                }
+                base := memConfig.New(baseMap)
+                backend := &config.ClientBackend{
+                        Provider: &testconfig.LocalClientProvider{
+                                Base: base,
+                        },
+                }
+
+                // Install our settings into a memory-backed Settings storage.
+                memSettings := settings.MemoryStorage{}
+                c = settings.Use(c, settings.New(&memSettings))
+                s := Settings{
+                        CacheExpirationSec: 10,
+                        DatastoreCacheMode: dsCacheStrict,
+                }
+                putSettings := func() {
+                        if err := settings.GetSettings(c).Set(c, settingsKey, &s, "test harness", "initial settings"); err != nil {
+                                panic(err)
+                        }
+                }
+                putSettings()
+
+                // Install our middleware chain into our Router.
+                baseMW := router.NewMiddlewareChain(func(ctx *router.Context, next router.Handler) {
+                        ctx.Context = c
+                        next(ctx)
+                })
+                rtr := router.New()
+                server := httptest.NewServer(rtr)
+                defer server.Close()
+
+                // Install our Manager cron handlers.
+                installCacheCronHandlerImpl(rtr, baseMW, backend)
+
+                runCron := func() int {
+                        datastore.GetTestable(c).CatchupIndexes()
+                        resp, err := http.Get(fmt.Sprintf("%s/admin/config/cache/manager", server.URL))
+                        if err != nil {
+                                panic(fmt.Errorf("failed to GET: %s", err))
+                        }
+                        return resp.StatusCode
+                }
+                So(runCron(), ShouldEqual, http.StatusOK)
+
+                // Called when all parameters are in place to install our config service
+                // layers into "c".
+                installConfig := func(c context.Context) context.Context { return useImpl(c, backend) }
+
+                loadProjectConfigs := func(c context.Context, a config.Authority) (projs []string, meta []*config.Meta, err error) {
+                        err = config.GetAll(c, a, config.Project, "project.cfg",
+                                config.StringSlice(&projs), &meta)
+                        return
+                }
+
+                Convey(`Cache modes`, func() {
+                        // Load all of our project configs and metadata. Load directly from our
+                        // non-datastore Backend, since this data is not part of the test.
+                        allProjs, _, err := loadProjectConfigs(config.WithBackend(c, backend), config.AsService)
+                        if err != nil {
+                                panic(err)
+                        }
+
+                        Convey(`Disabled, skips datastore cache.`, func() {
+                                s.DatastoreCacheMode = dsCacheDisabled
+                                putSettings()
+
+                                c = installConfig(c)
+                                c, cnt := count.FilterRDS(c)
+
+                                projs, _, err := loadProjectConfigs(c, config.AsService)
+                                So(err, ShouldBeNil)
+                                So(projs, ShouldResemble, allProjs)
+
+                                // No datastore operations should have been performed.
+                                So(cnt.GetMulti.Total(), ShouldEqual, 0)
+                                So(cnt.PutMulti.Total(), ShouldEqual, 0)
+                        })
+
+                        Convey(`Enabled`, func() {
+                                s.DatastoreCacheMode = dsCacheEnabled
+                                putSettings()
+
+                                c = installConfig(c)
+                                c, cnt := count.FilterRDS(c)
+                                projs, _, err := loadProjectConfigs(c, config.AsService)
+                                So(err, ShouldBeNil)
+                                So(projs, ShouldResemble, allProjs)
+
+                                So(cnt.GetMulti.Total(), ShouldEqual, 1)
+                                So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+                                // Break our backing store. This forces lookups to load from cache.
+                                memConfig.SetError(base, errors.New("config is broken"))
+
+                                projs, _, err = loadProjectConfigs(c, config.AsService)
+                                So(err, ShouldBeNil)
+                                So(projs, ShouldResemble, allProjs)
+
+                                So(cnt.GetMulti.Total(), ShouldEqual, 2)
+                                So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+                                // Expire the cache.
+                                clk.Add(time.Hour)
+
+                                projs, _, err = loadProjectConfigs(c, config.AsService)
+                                So(err, ShouldBeNil)
+                                So(projs, ShouldResemble, allProjs)
+
+                                So(cnt.GetMulti.Total(), ShouldEqual, 3)
+                                So(cnt.PutMulti.Total(), ShouldEqual, 1)
+                        })
+
+                        Convey(`Strict`, func() {
+                                s.DatastoreCacheMode = dsCacheStrict
+                                putSettings()
+
+                                c = installConfig(c)
+                                c, cnt := count.FilterRDS(c)
+                                projs, _, err := loadProjectConfigs(c, config.AsService)
+                                So(err, ShouldBeNil)
+                                So(projs, ShouldResemble, allProjs)
+
+                                So(cnt.GetMulti.Total(), ShouldEqual, 1)
+                                So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+                                // Break our backing store. This forces lookups to load from cache.
+                                memConfig.SetError(base, errors.New("config is broken"))
+
+                                projs, _, err = loadProjectConfigs(c, config.AsService)
+                                So(err, ShouldBeNil)
+                                So(projs, ShouldResemble, allProjs)
+
+                                So(cnt.GetMulti.Total(), ShouldEqual, 2)
+                                So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+                                // Expire the cache. Now that the entries are expired, we will
+                                // fail-closed to the backing store and return "config is broken".
+                                clk.Add(time.Hour)
+
+                                projs, _, err = loadProjectConfigs(c, config.AsService)
+                                So(err, ShouldUnwrapTo, datastorecache.ErrCacheExpired)
+
+                                So(cnt.GetMulti.Total(), ShouldEqual, 3)
+                                So(cnt.PutMulti.Total(), ShouldEqual, 1)
+                        })
+                })
+
+                Convey(`A user, denied access to a project, gains access once project config refreshes.`, func() {
+                        c = installConfig(c)
+
+                        _, metas, err := loadProjectConfigs(c, config.AsUser)
+                        So(err, ShouldBeNil)
+                        So(stripMeta(metas), ShouldResemble, []*config.Meta{
+                                {ConfigSet: "projects/open", Path: "project.cfg"},
+                        })
+
+                        // Update "foo" project ACLs to include the "users" group.
+                        baseMap["projects/foo"]["project.cfg"] = proto.MarshalTextString(projectConfigWithAccess("foo", "group:users"))
+
+                        // Still denied access (cached).
+                        c = installConfig(c)
+                        _, metas, err = loadProjectConfigs(c, config.AsUser)
+                        So(err, ShouldBeNil)
+                        So(stripMeta(metas), ShouldResemble, []*config.Meta{
+                                {ConfigSet: "projects/open", Path: "project.cfg"},
+                        })
+
+                        // Expire cache.
+                        clk.Add(time.Hour)
+                        _, _, err = loadProjectConfigs(c, config.AsUser)
+                        So(err, ShouldUnwrapTo, datastorecache.ErrCacheExpired)
+
+                        // Update our cache entries.
+                        So(runCron(), ShouldEqual, http.StatusOK)
+
+                        // Granted access!
+                        c = installConfig(c)
+                        _, metas, err = loadProjectConfigs(c, config.AsUser)
+                        So(err, ShouldBeNil)
+                        So(stripMeta(metas), ShouldResemble, []*config.Meta{
+                                {ConfigSet: "projects/foo", Path: "project.cfg"},
+                                {ConfigSet: "projects/open", Path: "project.cfg"},
+                        })
+                })
+
+                Convey(`GetConfig iterative updates`, func() {
+                        c = installConfig(c)
+
+                        get := func(c context.Context) (v string, meta config.Meta, err error) {
+                                err = config.Get(c, config.AsService, "projects/open", "test.cfg", config.String(&v), &meta)
+                                return
+                        }
+
+                        baseContentHash := func(configSet, config string) string {
+                                cfg, err := base.GetConfig(c, configSet, config, true)
+                                if err != nil {
+                                        panic(err)
+                                }
+                                return cfg.ContentHash
+                        }
+
+                        _, meta, err := get(c)
+                        So(err, ShouldBeNil)
+                        So(meta.ContentHash, ShouldEqual, baseContentHash("projects/open", "test.cfg"))
+
+                        Convey(`When the content changes, the hash is updated.`, func() {
+                                // Change content, hash should not have changed.
+                                baseMap["projects/open"]["test.cfg"] = "New Content!"
+                                _, meta2, err := get(c)
+                                So(err, ShouldBeNil)
+                                So(meta2.ContentHash, ShouldEqual, meta.ContentHash)
+
+                                // Expire the cache entry and refresh.
+                                clk.Add(time.Hour)
+                                So(runCron(), ShouldEqual, http.StatusOK)
+
+                                // Re-get the config. The content hash should have changed.
+                                _, meta, err = get(c)
+                                So(err, ShouldBeNil)
+                                So(meta.ContentHash, ShouldEqual, baseContentHash("projects/open", "test.cfg"))
+                                So(meta.ContentHash, ShouldNotEqual, meta2.ContentHash)
+                        })
+
+                        Convey(`When the content is deleted, returns ErrNoConfig.`, func() {
+                                // Change content, hash should not have changed.
+                                delete(baseMap["projects/open"], "test.cfg")
+                                _, meta2, err := get(c)
+                                So(err, ShouldBeNil)
+                                So(meta2.ContentHash, ShouldEqual, meta.ContentHash)
+
+                                // Expire the cache entry and refresh.
+                                clk.Add(time.Hour)
+                                So(runCron(), ShouldEqual, http.StatusOK)
+
+                                _, _, err = get(c)
+                                So(err, ShouldEqual, config.ErrNoConfig)
+                        })
+                })
+
+                // Load project config, return a slice of hashes.
+                getHashes := func(c context.Context, t config.GetAllType, path string) ([]string, error) {
+                        var (
+                                content []string
+                                metas   []*config.Meta
+                        )
+                        err := config.GetAll(c, config.AsService, t, path, config.StringSlice(&content), &metas)
+                        if err != nil {
+                                return nil, err
+                        }
+
+                        hashes := make([]string, len(metas))
+                        for i, meta := range metas {
+                                hashes[i] = meta.ContentHash
+                        }
+                        return hashes, nil
+                }
+
+                // Return project config hashes from backing memory config store.
+                baseContentHashes := func(t config.GetAllType, path string) []string {
+                        var fn func(context.Context, string, bool) ([]commonConfig.Config, error)
+                        switch t {
+                        case config.Project:
+                                fn = base.GetProjectConfigs
+                        case config.Ref:
+                                fn = base.GetRefConfigs
+                        }
+                        cfgs, err := fn(c, path, true)
+                        if err != nil {
+                                panic(err)
+                        }
+                        hashes := make([]string, len(cfgs))
+                        for i := range cfgs {
+                                hashes[i] = cfgs[i].ContentHash
+                        }
+                        return hashes
+                }
+
+                Convey(`GetAll for project iterative updates`, func() {
+                        c = installConfig(c)
+
+                        hashes, err := getHashes(c, config.Project, "project.cfg")
+                        So(err, ShouldBeNil)
+                        So(hashes, ShouldResemble, baseContentHashes(config.Project, "project.cfg"))
+
+                        Convey(`When the project list doesn't change, is not re-fetched.`, func() {
+                                // Expire the cache entry and refresh.
+                                clk.Add(time.Hour)
+                                So(runCron(), ShouldEqual, http.StatusOK)
+
+                                hashes2, err := getHashes(c, config.Project, "project.cfg")
+                                So(err, ShouldBeNil)
+                                So(hashes2, ShouldResemble, hashes)
+                        })
+
+                        Convey(`When the project changes, is re-fetched.`, func() {
+                                delete(baseMap["projects/invalidconfig"], "project.cfg")
+
+                                // Expire the cache entry and refresh.
+                                clk.Add(time.Hour)
+                                So(runCron(), ShouldEqual, http.StatusOK)
+
+                                hashes2, err := getHashes(c, config.Project, "project.cfg")
+                                So(err, ShouldBeNil)
+                                So(hashes2, ShouldResemble, baseContentHashes(config.Project, "project.cfg"))
+                                So(hashes2, ShouldNotResemble, hashes)
+                        })
+                })
+
+                Convey(`GetAll for ref iterative updates`, func() {
+                        c = installConfig(c)
+
+                        hashes, err := getHashes(c, config.Ref, "ref.cfg")
+                        So(err, ShouldBeNil)
+                        So(hashes, ShouldResemble, baseContentHashes(config.Ref, "ref.cfg"))
+
+                        Convey(`When the ref list doesn't change, is not re-fetched.`, func() {
+                                // Expire the cache entry and refresh.
+                                clk.Add(time.Hour)
+                                So(runCron(), ShouldEqual, http.StatusOK)
+
+                                hashes2, err := getHashes(c, config.Ref, "ref.cfg")
+                                So(err, ShouldBeNil)
+                                So(hashes2, ShouldResemble, hashes)
+                        })
+
+                        Convey(`When the ref changes, is re-fetched.`, func() {
+                                delete(baseMap["projects/foo/refs/branches/bar"], "ref.cfg")
+
+                                // Expire the cache entry and refresh.
+                                clk.Add(time.Hour)
+                                So(runCron(), ShouldEqual, http.StatusOK)
+
+                                hashes2, err := getHashes(c, config.Ref, "ref.cfg")
+                                So(err, ShouldBeNil)
+                                So(hashes2, ShouldResemble, baseContentHashes(config.Ref, "ref.cfg"))
+                                So(hashes2, ShouldNotResemble, hashes)
+                        })
+                })
+
+                Convey(`Test GetConfigSetURL user fetch`, func() {
+                        c = installConfig(c)
+
+                        // Project does not exist, missing for user and service.
+                        _, err := config.GetConfigSetURL(c, config.AsUser, "projects/urltest")
+                        So(err, ShouldEqual, config.ErrNoConfig)
+                        _, err = config.GetConfigSetURL(c, config.AsService, "projects/urltest")
+                        So(err, ShouldEqual, config.ErrNoConfig)
+
+                        // Add the project, still missing b/c of cache.
+                        baseMap["projects/urltest"] = memConfig.ConfigSet{
+                                "project.cfg": proto.MarshalTextString(projectConfigWithAccess("foo", "group:exclusive")),
+                        }
+
+                        _, err = config.GetConfigSetURL(c, config.AsUser, "projects/urltest")
+                        So(err, ShouldEqual, config.ErrNoConfig)
+                        _, err = config.GetConfigSetURL(c, config.AsService, "projects/urltest")
+                        So(err, ShouldEqual, config.ErrNoConfig)
+
+                        // Expire the cache entry and refresh. Next project load will pick up
+                        // the new project.
+                        clk.Add(time.Hour)
+                        So(runCron(), ShouldEqual, http.StatusOK)
+
+                        // User still cannot access (not member of group).
+                        _, err = config.GetConfigSetURL(c, config.AsUser, "projects/urltest")
+                        So(err, ShouldEqual, config.ErrNoConfig)
+
+                        // Service can access.
+                        u, err := config.GetConfigSetURL(c, config.AsService, "projects/urltest")
+                        So(err, ShouldBeNil)
+                        So(u.String(), ShouldEqual, "https://example.com/fake-config/projects/urltest")
+
+                        // User joins group, immediately gets access b/c the cached entry is the
+                        // service response, and so was soft-forbidden before.
+                        authState.IdentityGroups = append(authState.IdentityGroups, "exclusive")
+
+                        _, err = config.GetConfigSetURL(c, config.AsUser, "projects/urltest")
+                        So(err, ShouldBeNil)
+                        So(u.String(), ShouldEqual, "https://example.com/fake-config/projects/urltest")
+                })
+        })
+}