Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(110)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: appengine/gaeconfig/ds_test.go

Issue 2576923003: Implement config service cache on top of datastore (Closed)
Patch Set: Created 4 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« appengine/gaeconfig/ds.go ('K') | « appengine/gaeconfig/ds.go ('k') | common/testing/assertions/error_tests.go » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: appengine/gaeconfig/ds_test.go
diff --git a/appengine/gaeconfig/ds_test.go b/appengine/gaeconfig/ds_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..238787f3ad901883762136aa4c928a126c316dcc
--- /dev/null
+++ b/appengine/gaeconfig/ds_test.go
@@ -0,0 +1,1045 @@
+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package gaeconfig
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "sort"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/luci/luci-go/appengine/datastorecache"
+ "github.com/luci/luci-go/common/clock/testclock"
+ commonConfig "github.com/luci/luci-go/common/config"
+ memConfig "github.com/luci/luci-go/common/config/impl/memory"
+ "github.com/luci/luci-go/common/errors"
+ configPB "github.com/luci/luci-go/common/proto/config"
+ "github.com/luci/luci-go/server/auth"
+ "github.com/luci/luci-go/server/auth/authtest"
+ "github.com/luci/luci-go/server/config"
+ "github.com/luci/luci-go/server/config/caching"
+ "github.com/luci/luci-go/server/config/testconfig"
+ "github.com/luci/luci-go/server/config/textproto"
+ "github.com/luci/luci-go/server/router"
+ "github.com/luci/luci-go/server/settings"
+
+ "github.com/luci/gae/filter/count"
+ "github.com/luci/gae/impl/memory"
+ "github.com/luci/gae/service/datastore"
+
+ "github.com/golang/protobuf/proto"
+ "golang.org/x/net/context"
+
+ . "github.com/luci/luci-go/common/testing/assertions"
+ . "github.com/smartystreets/goconvey/convey"
+)
+
+// testCache is a generic Cache testing layer.
+type testCache interface {
+ dsCacheBackend
+
+ setCacheErr(err error)
+ setProjectDNE(project string)
+ addConfig(configSet, path, content string) *config.Item
+ addProjectConfig(name commonConfig.ProjectName, access string)
+ addConfigSets(path string, configSets ...string) []string
+ addConfigSetURL(configSet string) string
+}
+
+func projectConfigWithAccess(name commonConfig.ProjectName, access ...string) *configPB.ProjectCfg {
+ return &configPB.ProjectCfg{
+ Name: proto.String(string(name)),
+ Access: access,
+ }
+}
+
+// fakeCache is a pure in-memory testCache implementation. It is very simple,
+// storing only raw cache key/value pairs.
+type fakeCache struct {
+ d map[string]datastorecache.Value
+ err error
+}
+
+func mkFakeCache() *fakeCache {
+ return &fakeCache{
+ d: make(map[string]datastorecache.Value),
+ }
+}
+
+func (fc *fakeCache) Get(c context.Context, key []byte) (v datastorecache.Value, err error) {
+ if err = fc.err; err != nil {
+ return
+ }
+
+ var k caching.Key
+ caching.Decode(key, &k)
+
+ var ok bool
+ if v, ok = fc.d[string(key)]; ok {
+ return
+ }
+
+ err = datastorecache.ErrCacheExpired
+ return
+}
+
+func (fc *fakeCache) setCacheData(key caching.Key, d []byte) {
+}
+
+func (fc *fakeCache) set(key caching.Key, v *caching.Value) {
+ encKey, err := caching.Encode(&key)
+ if err != nil {
+ panic(fmt.Errorf("failed to encode key: %s", err))
+ }
+
+ if v == nil {
+ delete(fc.d, string(encKey))
+ return
+ }
+
+ encValue, err := v.Encode()
+ if err != nil {
+ panic(fmt.Errorf("failed to encode cache value: %s", err))
+ }
+
+ fc.d[string(encKey)] = datastorecache.Value{
+ Schema: dsCacheSchema,
+ Data: encValue,
+ Description: key.String(),
+ }
+}
+
+func (fc *fakeCache) setCacheErr(err error) { fc.err = err }
+
+func (fc *fakeCache) setProjectDNE(project string) {
+ // Get for this project config will fail.
+ fc.set(caching.Key{
+ Schema: caching.Schema,
+ Op: caching.OpGet,
+ ConfigSet: config.ProjectConfigSet(commonConfig.ProjectName(project)),
+ Path: config.ProjectConfigPath,
+ }, nil)
+}
+
+func (fc *fakeCache) addConfigImpl(configSet, path, format, formatData, content string) *config.Item {
+ var (
+ item *config.Item
+ cv caching.Value
+ )
+ if content != "" {
+ item = &config.Item{
+ Meta: config.Meta{
+ ConfigSet: configSet,
+ Path: path,
+ ContentHash: "hash",
+ },
+ Content: content,
+ Format: format,
+ FormatData: formatData,
+ }
+ cv.LoadItems(item)
+ }
+
+ fc.set(caching.Key{
+ Schema: caching.Schema,
+ Authority: config.AsService,
+ Op: caching.OpGet,
+ ConfigSet: configSet,
+ Path: path,
+ Content: true,
+ Format: format,
+ FormatData: formatData,
+ }, &cv)
+
+ return item
+}
+
+func (fc *fakeCache) addConfig(configSet, path, content string) *config.Item {
+ return fc.addConfigImpl(configSet, path, "", "", content)
+}
+
+// addProjectConfig caches a "project.cfg" file for the specified project with
+// the specified access string.
+func (fc *fakeCache) addProjectConfig(name commonConfig.ProjectName, access string) {
+ // We're loading the resolved version of this cache item.
+ pcfg := projectConfigWithAccess(name, access)
+ pcfgName := proto.MessageName(pcfg)
+
+ f := textproto.Formatter{}
+ formattedData, err := f.FormatItem(proto.MarshalTextString(pcfg), pcfgName)
+ if err != nil {
+ panic(err)
+ }
+
+ fc.addConfigImpl(config.ProjectConfigSet(name), config.ProjectConfigPath,
+ textproto.BinaryFormat, pcfgName, formattedData)
+}
+
+func (fc *fakeCache) addConfigSets(path string, configSets ...string) []string {
+ items := make([]*config.Item, len(configSets))
+ contents := make([]string, len(configSets))
+ for i, cs := range configSets {
+ contents[i] = cs
+ items[i] = &config.Item{
+ Meta: config.Meta{
+ ConfigSet: cs,
+ Path: path,
+ ContentHash: "hash",
+ },
+ Content: contents[i],
+ Format: "",
+ FormatData: "",
+ }
+ }
+
+ for _, t := range []config.GetAllType{config.Project, config.Ref} {
+ var cv caching.Value
+ cv.LoadItems(items...)
+
+ fc.set(caching.Key{
+ Schema: caching.Schema,
+ Authority: config.AsService,
+ Op: caching.OpGetAll,
+ Content: true,
+ Path: path,
+ GetAllType: t,
+ Format: "",
+ FormatData: "",
+ }, &cv)
+ }
+ return contents
+}
+
+func (fc *fakeCache) addConfigSetURL(configSet string) string {
+ u := fmt.Sprintf("https://exmaple.com/config-sets/%s", configSet)
+ fc.set(caching.Key{
+ Schema: caching.Schema,
+ Authority: config.AsService,
+ Op: caching.OpConfigSetURL,
+ ConfigSet: configSet,
+ }, &caching.Value{
+ URL: u,
+ })
+ return u
+}
+
+// fullStackCache is a testCache implementation built on top of an in-memory
+// base config.Interface with the datastore Cache layer on top of it.
+type fullStackCache struct {
+ cache *datastorecache.Cache
+ err error
+
+ data map[string]memConfig.ConfigSet
+ backend config.Backend
+ junkIdx int
+}
+
+func (fsc *fullStackCache) Get(c context.Context, key []byte) (datastorecache.Value, error) {
+ if err := fsc.err; err != nil {
+ return datastorecache.Value{}, err
+ }
+ return fsc.cache.Get(c, key)
+}
+
+func (fsc *fullStackCache) setCacheErr(err error) { fsc.err = err }
+
+func (fsc *fullStackCache) setProjectDNE(project string) {
+ key := "projects/" + project
+ for k := range fsc.data {
+ if k == key || strings.HasPrefix(k, key+"/") {
+ delete(fsc.data, k)
+ }
+ }
+}
+
+func (fsc *fullStackCache) addConfig(configSet, path, content string) *config.Item {
+ cset := fsc.data[configSet]
+ if cset == nil {
+ cset = memConfig.ConfigSet{}
+ fsc.data[configSet] = cset
+ }
+ if content == "" {
+ delete(cset, path)
+ return nil
+ }
+ cset[path] = content
+
+ // Pull the config right back out of the base service.
+ item, err := fsc.backend.Get(context.Background(), configSet, path, config.Params{
+ Authority: config.AsService,
+ })
+ if err != nil {
+ panic(err)
+ }
+ return item
+}
+
+// addProjectConfig caches a "project.cfg" file for the specified project with
+// the specified access string.
+func (fsc *fullStackCache) addProjectConfig(name commonConfig.ProjectName, access string) {
+ fsc.addConfig(config.ProjectConfigSet(name), config.ProjectConfigPath,
+ proto.MarshalTextString(projectConfigWithAccess(name, access)))
+}
+
+func (fsc *fullStackCache) addConfigSets(path string, configSets ...string) []string {
+ sort.Strings(configSets)
+ items := make([]*config.Item, len(configSets))
+ for i, cs := range configSets {
+ items[i] = fsc.addConfig(cs, path, cs)
+ }
+ return configSets
+}
+
+func (fsc *fullStackCache) addConfigSetURL(configSet string) string {
+ if _, ok := fsc.data[configSet]; !ok {
+ fsc.data[configSet] = memConfig.ConfigSet{}
+ }
+
+ // We're pretty rigid here. Whatever our backend returns is all we can
+ // return. We will just assert that anything more flexible has to conform to
+ // this.
+ v, err := fsc.backend.ConfigSetURL(context.Background(), config.AsService, configSet)
+ if err != nil {
+ panic(err)
+ }
+ return v.String()
+}
+
+// stripMeta strips cache-specific identifying information from a set of Metas.
+func stripMeta(metas []*config.Meta) []*config.Meta {
+ for _, meta := range metas {
+ meta.ContentHash = ""
+ meta.Revision = ""
+ }
+ return metas
+}
+
+func testDatastoreCacheImpl(c context.Context, backend config.Backend, cache testCache) {
+ // Install fake auth state.
+ var authState authtest.FakeState
+ c = auth.WithState(c, &authState)
+ authState.Identity = "user:person@example.com"
+ authState.IdentityGroups = []string{"users"}
+
+ dsc := datastoreCache{
+ refreshInterval: 1 * time.Hour,
+ failOpen: false,
+ cache: cache,
+ }
+ c = config.WithBackend(c, dsc.getBackend(backend))
+
+ testErr := errors.New("test error")
+
+ Convey(`Test Get`, func() {
+ var v string
+
+ Convey(`Config missing`, func() {
+ cache.addConfig("projects/test", "foo", "")
+
+ So(config.Get(c, config.AsService, "projects/test", "foo", config.String(&v), nil),
+ ShouldEqual, config.ErrNoConfig)
+ })
+
+ Convey(`Config is present`, func() {
+ cache.addConfig("projects/test", "foo", "bar")
+ cache.addProjectConfig("test", "group:privileged")
+
+ Convey(`As service`, func() {
+ So(config.Get(c, config.AsService, "projects/test", "foo", config.String(&v), nil), ShouldBeNil)
+ So(v, ShouldEqual, "bar")
+ })
+
+ Convey(`As user, when not a project group member, fails with ErrNoConfig`, func() {
+ So(config.Get(c, config.AsUser, "projects/test", "foo", config.String(&v), nil),
+ ShouldEqual, config.ErrNoConfig)
+ })
+
+ Convey(`As user, when a project group member, succeeds.`, func() {
+ authState.IdentityGroups = append(authState.IdentityGroups, "privileged")
+ So(config.Get(c, config.AsUser, "projects/test", "foo", config.String(&v), nil), ShouldBeNil)
+ So(v, ShouldEqual, "bar")
+ })
+
+ Convey(`As anonymous, fails with ErrNoConfig`, func() {
+ So(config.Get(c, config.AsAnonymous, "projects/test", "foo", config.String(&v), nil),
+ ShouldEqual, config.ErrNoConfig)
+ })
+ })
+ })
+
+ Convey(`Test GetAll with projects`, func() {
+ var v []string
+ var meta []*config.Meta
+
+ Convey(`When cache returns an error`, func() {
+ cache.setCacheErr(testErr)
+
+ So(config.GetAll(c, config.AsService, config.Project, "test.cfg", config.StringSlice(&v), nil),
+ ShouldUnwrapTo, testErr)
+ So(config.GetAll(c, config.AsUser, config.Project, "test.cfg", config.StringSlice(&v), nil),
+ ShouldUnwrapTo, testErr)
+ So(config.GetAll(c, config.AsAnonymous, config.Project, "test.cfg", config.StringSlice(&v), nil),
+ ShouldUnwrapTo, testErr)
+ })
+
+ Convey(`With project configs installed`, func() {
+ allConfigs := cache.addConfigSets("test.cfg",
+ "projects/bar",
+ "projects/baz",
+ "projects/foo")
+
+ Convey(`As service, retrieves all configs.`, func() {
+ So(config.GetAll(c, config.AsService, config.Project, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+ So(v, ShouldResemble, allConfigs)
+ So(stripMeta(meta), ShouldResemble, []*config.Meta{
+ {ConfigSet: "projects/bar", Path: "test.cfg"},
+ {ConfigSet: "projects/baz", Path: "test.cfg"},
+ {ConfigSet: "projects/foo", Path: "test.cfg"},
+ })
+ })
+
+ Convey(`As user`, func() {
+ Convey(`Not a member of any projects, receives empty slice.`, func() {
+ cache.addProjectConfig("foo", "group:someone")
+
+ So(config.GetAll(c, config.AsUser, config.Project, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+ So(v, ShouldResemble, []string(nil))
+ So(stripMeta(meta), ShouldResemble, []*config.Meta{})
+ })
+
+ Convey(`Member of "foo", gets only "foo".`, func() {
+ cache.addProjectConfig("foo", "group:users")
+
+ So(config.GetAll(c, config.AsUser, config.Project, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+ So(v, ShouldResemble, allConfigs[2:3])
+ So(stripMeta(meta), ShouldResemble, []*config.Meta{
+ {ConfigSet: "projects/foo", Path: "test.cfg"},
+ })
+ })
+
+ Convey(`Member of all projects, gets all projects.`, func() {
+ cache.addProjectConfig("foo", "group:users")
+ cache.addProjectConfig("bar", "group:users")
+ cache.addProjectConfig("baz", "group:users")
+
+ So(config.GetAll(c, config.AsUser, config.Project, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+ So(v, ShouldResemble, allConfigs)
+ So(stripMeta(meta), ShouldResemble, []*config.Meta{
+ {ConfigSet: "projects/bar", Path: "test.cfg"},
+ {ConfigSet: "projects/baz", Path: "test.cfg"},
+ {ConfigSet: "projects/foo", Path: "test.cfg"},
+ })
+ })
+ })
+ })
+ })
+
+ Convey(`Test GetAll with refs`, func() {
+ var v []string
+ var meta []*config.Meta
+
+ Convey(`When cache returns an error`, func() {
+ cache.setCacheErr(testErr)
+
+ So(config.GetAll(c, config.AsService, config.Ref, "test.cfg", config.StringSlice(&v), nil),
+ ShouldUnwrapTo, testErr)
+ So(config.GetAll(c, config.AsUser, config.Ref, "test.cfg", config.StringSlice(&v), nil),
+ ShouldUnwrapTo, testErr)
+ So(config.GetAll(c, config.AsAnonymous, config.Ref, "test.cfg", config.StringSlice(&v), nil),
+ ShouldUnwrapTo, testErr)
+ })
+
+ Convey(`With ref configs installed`, func() {
+ allConfigs := cache.addConfigSets("test.cfg",
+ "projects/bar/refs/branches/mybranch",
+ "projects/bar/refs/heads/master",
+ "projects/foo/refs/branches/mybranch",
+ "projects/foo/refs/heads/master")
+
+ Convey(`As service, retrieves all configs.`, func() {
+ So(config.GetAll(c, config.AsService, config.Ref, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+ So(v, ShouldResemble, allConfigs)
+ So(stripMeta(meta), ShouldResemble, []*config.Meta{
+ {ConfigSet: "projects/bar/refs/branches/mybranch", Path: "test.cfg"},
+ {ConfigSet: "projects/bar/refs/heads/master", Path: "test.cfg"},
+ {ConfigSet: "projects/foo/refs/branches/mybranch", Path: "test.cfg"},
+ {ConfigSet: "projects/foo/refs/heads/master", Path: "test.cfg"},
+ })
+ })
+
+ Convey(`As user`, func() {
+ Convey(`Not a member of any projects, receives empty slice.`, func() {
+ cache.addProjectConfig("foo", "group:someone")
+
+ So(config.GetAll(c, config.AsUser, config.Ref, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+ So(v, ShouldResemble, []string(nil))
+ So(stripMeta(meta), ShouldResemble, []*config.Meta{})
+ })
+
+ Convey(`Member of "foo", gets only "foo".`, func() {
+ cache.addProjectConfig("foo", "group:users")
+
+ So(config.GetAll(c, config.AsUser, config.Ref, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+ So(v, ShouldResemble, allConfigs[2:4])
+ So(stripMeta(meta), ShouldResemble, []*config.Meta{
+ {ConfigSet: "projects/foo/refs/branches/mybranch", Path: "test.cfg"},
+ {ConfigSet: "projects/foo/refs/heads/master", Path: "test.cfg"},
+ })
+ })
+
+ Convey(`Member of all projects, gets all projects.`, func() {
+ cache.addProjectConfig("foo", "group:users")
+ cache.addProjectConfig("bar", "group:users")
+
+ So(config.GetAll(c, config.AsUser, config.Ref, "test.cfg", config.StringSlice(&v), &meta), ShouldBeNil)
+ So(v, ShouldResemble, allConfigs)
+ So(stripMeta(meta), ShouldResemble, []*config.Meta{
+ {ConfigSet: "projects/bar/refs/branches/mybranch", Path: "test.cfg"},
+ {ConfigSet: "projects/bar/refs/heads/master", Path: "test.cfg"},
+ {ConfigSet: "projects/foo/refs/branches/mybranch", Path: "test.cfg"},
+ {ConfigSet: "projects/foo/refs/heads/master", Path: "test.cfg"},
+ })
+ })
+ })
+ })
+ })
+
+ Convey(`Test ConfigSetURL`, func() {
+ cache.addProjectConfig("foo", "group:someone")
+ csURL := cache.addConfigSetURL("projects/foo")
+
+ Convey(`AsService`, func() {
+ u, err := config.GetConfigSetURL(c, config.AsService, "projects/foo")
+ So(err, ShouldBeNil)
+ So(u.String(), ShouldEqual, csURL)
+ })
+
+ Convey(`AsUser`, func() {
+
+ Convey(`When not a member of the group.`, func() {
+ _, err := config.GetConfigSetURL(c, config.AsUser, "projects/foo")
+ So(err, ShouldEqual, config.ErrNoConfig)
+ })
+
+ Convey(`When a member of the group.`, func() {
+ authState.IdentityGroups = append(authState.IdentityGroups, "someone")
+
+ u, err := config.GetConfigSetURL(c, config.AsUser, "projects/foo")
+ So(err, ShouldBeNil)
+ So(u.String(), ShouldEqual, csURL)
+ })
+ })
+
+ Convey(`AsAnonymous`, func() {
+ // Log in just to show that, while AsUser could access, AsAnonymous
+ // cannot.
+ authState.IdentityGroups = append(authState.IdentityGroups, "someone")
+ _, err := config.GetConfigSetURL(c, config.AsUser, "projects/foo")
+ So(err, ShouldBeNil)
+
+ _, err = config.GetConfigSetURL(c, config.AsAnonymous, "projects/foo")
+ So(err, ShouldEqual, config.ErrNoConfig)
+ })
+ })
+}
+
+func TestDatastoreCache(t *testing.T) {
+ t.Parallel()
+
+ Convey(`Testing with in-memory stub cache`, t, func() {
+ c := context.Background()
+ fc := mkFakeCache()
+
+ var backend config.Backend
+ backend = &config.ClientBackend{
+ Provider: &testconfig.LocalClientProvider{
+ Base: memConfig.New(nil),
+ },
+ }
+
+ fr := defaultFormatterRegistry()
+ backend = &config.FormatBackend{
+ Backend: backend,
+ GetRegistry: func(context.Context) *config.FormatterRegistry { return fr },
+ }
+
+ Convey(`Standard datastore tests`, func() {
+ testDatastoreCacheImpl(c, backend, fc)
+ })
+
+ Convey(`A testing setup built around the fake cache`, func() {
+ dsc := datastoreCache{
+ refreshInterval: 1 * time.Hour,
+ failOpen: false,
+ cache: fc,
+ }
+ c = config.WithBackend(c, dsc.getBackend(backend))
+
+ Convey(`Errors with different schema.`, func() {
+ fc.addConfig("foo", "bar", "value")
+ for k, v := range fc.d {
+ v.Schema = "unknown"
+ fc.d[k] = v
+ }
+
+ var v string
+ So(config.Get(c, config.AsService, "foo", "bar", config.String(&v), nil),
+ ShouldErrLike, `response schema ("unknown") doesn't match current`)
+ })
+ })
+ })
+}
+
+func TestDatastoreCacheFullStack(t *testing.T) {
+ t.Parallel()
+
+ Convey(`Testing full-stack datastore cache`, t, func() {
+ c := memory.Use(context.Background())
+
+ data := map[string]memConfig.ConfigSet{}
+
+ var backend config.Backend
+ backend = &config.ClientBackend{
+ Provider: &testconfig.LocalClientProvider{
+ Base: memConfig.New(data),
+ },
+ }
+
+ fr := defaultFormatterRegistry()
+ backend = &config.FormatBackend{
+ Backend: backend,
+ GetRegistry: func(context.Context) *config.FormatterRegistry { return fr },
+ }
+
+ fsc := fullStackCache{
+ cache: &dsCache,
+ data: data,
+ backend: backend,
+ }
+ testDatastoreCacheImpl(c, backend, &fsc)
+ })
+}
+
+func TestDatastoreCacheIntegration(t *testing.T) {
+ t.Parallel()
+
+ Convey(`A testing environment with fake cache and config service`, t, func() {
+ c, clk := testclock.UseTime(context.Background(), datastore.RoundTime(testclock.TestTimeUTC))
+
+ c = memory.Use(c)
+
+ // Install fake auth state.
+ var authState authtest.FakeState
+ c = auth.WithState(c, &authState)
+ authState.Identity = "user:person@example.com"
+ authState.IdentityGroups = []string{"all", "users"}
+
+ // Use a memory-backed Config service instance.
+ baseMap := map[string]memConfig.ConfigSet{
+ "projects/open": map[string]string{
+ "project.cfg": proto.MarshalTextString(projectConfigWithAccess("foo", "group:all")),
+ "test.cfg": "Test Config Content",
+ },
+ "projects/foo": map[string]string{
+ "project.cfg": proto.MarshalTextString(projectConfigWithAccess("foo", "group:restricted")),
+ },
+ "projects/noconfig": map[string]string{},
+ "projects/invalidconfig": map[string]string{
+ "project.cfg": "!!! not a valid config !!!",
+ },
+ "projects/foo/refs/heads/master": map[string]string{
+ "ref.cfg": "foo",
+ },
+ "projects/foo/refs/branches/bar": map[string]string{
+ "ref.cfg": "foo/bar",
+ },
+ "projects/bar/refs/heads/master": map[string]string{
+ "ref.cfg": "bar",
+ },
+ }
+ base := memConfig.New(baseMap)
+ backend := &config.ClientBackend{
+ Provider: &testconfig.LocalClientProvider{
+ Base: base,
+ },
+ }
+
+ // Install our settings into a memory-backed Settings storage.
+ memSettings := settings.MemoryStorage{}
+ c = settings.Use(c, settings.New(&memSettings))
+ s := Settings{
+ CacheExpirationSec: 10,
+ DatastoreCacheMode: dsCacheStrict,
+ }
+ putSettings := func() {
+ if err := settings.GetSettings(c).Set(c, settingsKey, &s, "test harness", "initial settings"); err != nil {
+ panic(err)
+ }
+ }
+ putSettings()
+
+ // Install our middleware chain into our Router.
+ baseMW := router.NewMiddlewareChain(func(ctx *router.Context, next router.Handler) {
+ ctx.Context = c
+ next(ctx)
+ })
+ rtr := router.New()
+ server := httptest.NewServer(rtr)
+ defer server.Close()
+
+ // Install our Manager cron handlers.
+ installCacheCronHandlerImpl(rtr, baseMW, backend)
+
+ runCron := func() int {
+ datastore.GetTestable(c).CatchupIndexes()
+ resp, err := http.Get(fmt.Sprintf("%s/admin/config/cache/manager", server.URL))
+ if err != nil {
+ panic(fmt.Errorf("failed to GET: %s", err))
+ }
+ return resp.StatusCode
+ }
+ So(runCron(), ShouldEqual, http.StatusOK)
+
+ // Called when all parameters are in place to install our config service
+ // layers into "c".
+ installConfig := func(c context.Context) context.Context { return useImpl(c, backend) }
+
+ loadProjectConfigs := func(c context.Context, a config.Authority) (projs []string, meta []*config.Meta, err error) {
+ err = config.GetAll(c, a, config.Project, "project.cfg",
+ config.StringSlice(&projs), &meta)
+ return
+ }
+
+ Convey(`Cache modes`, func() {
+ // Load all of our project configs and metadata. Load directly from our
+ // non-datastore Backend, since this data is not part of the test.
+ allProjs, _, err := loadProjectConfigs(config.WithBackend(c, backend), config.AsService)
+ if err != nil {
+ panic(err)
+ }
+
+ Convey(`Disabled, skips datastore cache.`, func() {
+ s.DatastoreCacheMode = dsCacheDisabled
+ putSettings()
+
+ c = installConfig(c)
+ c, cnt := count.FilterRDS(c)
+
+ projs, _, err := loadProjectConfigs(c, config.AsService)
+ So(err, ShouldBeNil)
+ So(projs, ShouldResemble, allProjs)
+
+ // No datastore operations should have been performed.
+ So(cnt.GetMulti.Total(), ShouldEqual, 0)
+ So(cnt.PutMulti.Total(), ShouldEqual, 0)
+ })
+
+ Convey(`Enabled`, func() {
+ s.DatastoreCacheMode = dsCacheEnabled
+ putSettings()
+
+ c = installConfig(c)
+ c, cnt := count.FilterRDS(c)
+ projs, _, err := loadProjectConfigs(c, config.AsService)
+ So(err, ShouldBeNil)
+ So(projs, ShouldResemble, allProjs)
+
+ So(cnt.GetMulti.Total(), ShouldEqual, 1)
+ So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+ // Break our backing store. This forces lookups to load from cache.
+ memConfig.SetError(base, errors.New("config is broken"))
+
+ projs, _, err = loadProjectConfigs(c, config.AsService)
+ So(err, ShouldBeNil)
+ So(projs, ShouldResemble, allProjs)
+
+ So(cnt.GetMulti.Total(), ShouldEqual, 2)
+ So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+ // Expire the cache.
+ clk.Add(time.Hour)
+
+ projs, _, err = loadProjectConfigs(c, config.AsService)
+ So(err, ShouldBeNil)
+ So(projs, ShouldResemble, allProjs)
+
+ So(cnt.GetMulti.Total(), ShouldEqual, 3)
+ So(cnt.PutMulti.Total(), ShouldEqual, 1)
+ })
+
+ Convey(`Strict`, func() {
+ s.DatastoreCacheMode = dsCacheStrict
+ putSettings()
+
+ c = installConfig(c)
+ c, cnt := count.FilterRDS(c)
+ projs, _, err := loadProjectConfigs(c, config.AsService)
+ So(err, ShouldBeNil)
+ So(projs, ShouldResemble, allProjs)
+
+ So(cnt.GetMulti.Total(), ShouldEqual, 1)
+ So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+ // Break our backing store. This forces lookups to load from cache.
+ memConfig.SetError(base, errors.New("config is broken"))
+
+ projs, _, err = loadProjectConfigs(c, config.AsService)
+ So(err, ShouldBeNil)
+ So(projs, ShouldResemble, allProjs)
+
+ So(cnt.GetMulti.Total(), ShouldEqual, 2)
+ So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+ // Expire the cache. Now that the entries are expired, we will
+ // fail-closed to the backing store and return "config is broken".
+ clk.Add(time.Hour)
+
+ projs, _, err = loadProjectConfigs(c, config.AsService)
+ So(err, ShouldUnwrapTo, datastorecache.ErrCacheExpired)
+
+ So(cnt.GetMulti.Total(), ShouldEqual, 3)
+ So(cnt.PutMulti.Total(), ShouldEqual, 1)
+ })
+ })
+
+ Convey(`A user, denied access to a project, gains access once project config refreshes.`, func() {
+ c = installConfig(c)
+
+ _, metas, err := loadProjectConfigs(c, config.AsUser)
+ So(err, ShouldBeNil)
+ So(stripMeta(metas), ShouldResemble, []*config.Meta{
+ {ConfigSet: "projects/open", Path: "project.cfg"},
+ })
+
+ // Update "foo" project ACLs to include the "users" group.
+ baseMap["projects/foo"]["project.cfg"] = proto.MarshalTextString(projectConfigWithAccess("foo", "group:users"))
+
+ // Still denied access (cached).
+ c = installConfig(c)
+ _, metas, err = loadProjectConfigs(c, config.AsUser)
+ So(err, ShouldBeNil)
+ So(stripMeta(metas), ShouldResemble, []*config.Meta{
+ {ConfigSet: "projects/open", Path: "project.cfg"},
+ })
+
+ // Expire cache.
+ clk.Add(time.Hour)
+ _, _, err = loadProjectConfigs(c, config.AsUser)
+ So(err, ShouldUnwrapTo, datastorecache.ErrCacheExpired)
+
+ // Update our cache entries.
+ So(runCron(), ShouldEqual, http.StatusOK)
+
+ // Granted access!
+ c = installConfig(c)
+ _, metas, err = loadProjectConfigs(c, config.AsUser)
+ So(err, ShouldBeNil)
+ So(stripMeta(metas), ShouldResemble, []*config.Meta{
+ {ConfigSet: "projects/foo", Path: "project.cfg"},
+ {ConfigSet: "projects/open", Path: "project.cfg"},
+ })
+ })
+
+ Convey(`GetConfig iterative updates`, func() {
+ c = installConfig(c)
+
+ get := func(c context.Context) (v string, meta config.Meta, err error) {
+ err = config.Get(c, config.AsService, "projects/open", "test.cfg", config.String(&v), &meta)
+ return
+ }
+
+ baseContentHash := func(configSet, config string) string {
+ cfg, err := base.GetConfig(c, configSet, config, true)
+ if err != nil {
+ panic(err)
+ }
+ return cfg.ContentHash
+ }
+
+ _, meta, err := get(c)
+ So(err, ShouldBeNil)
+ So(meta.ContentHash, ShouldEqual, baseContentHash("projects/open", "test.cfg"))
+
+ Convey(`When the content changes, the hash is updated.`, func() {
+ // Change content, hash should not have changed.
+ baseMap["projects/open"]["test.cfg"] = "New Content!"
+ _, meta2, err := get(c)
+ So(err, ShouldBeNil)
+ So(meta2.ContentHash, ShouldEqual, meta.ContentHash)
+
+ // Expire the cache entry and refresh.
+ clk.Add(time.Hour)
+ So(runCron(), ShouldEqual, http.StatusOK)
+
+ // Re-get the config. The content hash should have changed.
+ _, meta, err = get(c)
+ So(err, ShouldBeNil)
+ So(meta.ContentHash, ShouldEqual, baseContentHash("projects/open", "test.cfg"))
+ So(meta.ContentHash, ShouldNotEqual, meta2.ContentHash)
+ })
+
+ Convey(`When the content is deleted, returns ErrNoConfig.`, func() {
+ // Change content, hash should not have changed.
+ delete(baseMap["projects/open"], "test.cfg")
+ _, meta2, err := get(c)
+ So(err, ShouldBeNil)
+ So(meta2.ContentHash, ShouldEqual, meta.ContentHash)
+
+ // Expire the cache entry and refresh.
+ clk.Add(time.Hour)
+ So(runCron(), ShouldEqual, http.StatusOK)
+
+ _, _, err = get(c)
+ So(err, ShouldEqual, config.ErrNoConfig)
+ })
+ })
+
+ // Load project config, return a slice of hashes.
+ getHashes := func(c context.Context, t config.GetAllType, path string) ([]string, error) {
+ var (
+ content []string
+ metas []*config.Meta
+ )
+ err := config.GetAll(c, config.AsService, t, path, config.StringSlice(&content), &metas)
+ if err != nil {
+ return nil, err
+ }
+
+ hashes := make([]string, len(metas))
+ for i, meta := range metas {
+ hashes[i] = meta.ContentHash
+ }
+ return hashes, nil
+ }
+
+ // Return project config hashes from backing memory config store.
+ baseContentHashes := func(t config.GetAllType, path string) []string {
+ var fn func(context.Context, string, bool) ([]commonConfig.Config, error)
+ switch t {
+ case config.Project:
+ fn = base.GetProjectConfigs
+ case config.Ref:
+ fn = base.GetRefConfigs
+ }
+ cfgs, err := fn(c, path, true)
+ if err != nil {
+ panic(err)
+ }
+ hashes := make([]string, len(cfgs))
+ for i := range cfgs {
+ hashes[i] = cfgs[i].ContentHash
+ }
+ return hashes
+ }
+
+ Convey(`GetAll for project iterative updates`, func() {
+ c = installConfig(c)
+
+ hashes, err := getHashes(c, config.Project, "project.cfg")
+ So(err, ShouldBeNil)
+ So(hashes, ShouldResemble, baseContentHashes(config.Project, "project.cfg"))
+
+ Convey(`When the project list doesn't change, is not re-fetched.`, func() {
+ // Expire the cache entry and refresh.
+ clk.Add(time.Hour)
+ So(runCron(), ShouldEqual, http.StatusOK)
+
+ hashes2, err := getHashes(c, config.Project, "project.cfg")
+ So(err, ShouldBeNil)
+ So(hashes2, ShouldResemble, hashes)
+ })
+
+ Convey(`When the project changes, is re-fetched.`, func() {
+ delete(baseMap["projects/invalidconfig"], "project.cfg")
+
+ // Expire the cache entry and refresh.
+ clk.Add(time.Hour)
+ So(runCron(), ShouldEqual, http.StatusOK)
+
+ hashes2, err := getHashes(c, config.Project, "project.cfg")
+ So(err, ShouldBeNil)
+ So(hashes2, ShouldResemble, baseContentHashes(config.Project, "project.cfg"))
+ So(hashes2, ShouldNotResemble, hashes)
+ })
+ })
+
+ Convey(`GetAll for ref iterative updates`, func() {
+ c = installConfig(c)
+
+ hashes, err := getHashes(c, config.Ref, "ref.cfg")
+ So(err, ShouldBeNil)
+ So(hashes, ShouldResemble, baseContentHashes(config.Ref, "ref.cfg"))
+
+ Convey(`When the ref list doesn't change, is not re-fetched.`, func() {
+ // Expire the cache entry and refresh.
+ clk.Add(time.Hour)
+ So(runCron(), ShouldEqual, http.StatusOK)
+
+ hashes2, err := getHashes(c, config.Ref, "ref.cfg")
+ So(err, ShouldBeNil)
+ So(hashes2, ShouldResemble, hashes)
+ })
+
+ Convey(`When the ref changes, is re-fetched.`, func() {
+ delete(baseMap["projects/foo/refs/branches/bar"], "ref.cfg")
+
+ // Expire the cache entry and refresh.
+ clk.Add(time.Hour)
+ So(runCron(), ShouldEqual, http.StatusOK)
+
+ hashes2, err := getHashes(c, config.Ref, "ref.cfg")
+ So(err, ShouldBeNil)
+ So(hashes2, ShouldResemble, baseContentHashes(config.Ref, "ref.cfg"))
+ So(hashes2, ShouldNotResemble, hashes)
+ })
+ })
+
+ Convey(`Test GetConfigSetURL user fetch`, func() {
+ c = installConfig(c)
+
+ // Project does not exist, missing for user and service.
+ _, err := config.GetConfigSetURL(c, config.AsUser, "projects/urltest")
+ So(err, ShouldEqual, config.ErrNoConfig)
+ _, err = config.GetConfigSetURL(c, config.AsService, "projects/urltest")
+ So(err, ShouldEqual, config.ErrNoConfig)
+
+ // Add the project, still missing b/c of cache.
+ baseMap["projects/urltest"] = memConfig.ConfigSet{
+ "project.cfg": proto.MarshalTextString(projectConfigWithAccess("foo", "group:exclusive")),
+ }
+
+ _, err = config.GetConfigSetURL(c, config.AsUser, "projects/urltest")
+ So(err, ShouldEqual, config.ErrNoConfig)
+ _, err = config.GetConfigSetURL(c, config.AsService, "projects/urltest")
+ So(err, ShouldEqual, config.ErrNoConfig)
+
+ // Expire the cache entry and refresh. Next project load will pick up
+ // the new project.
+ clk.Add(time.Hour)
+ So(runCron(), ShouldEqual, http.StatusOK)
+
+ // User still cannot access (not member of group).
+ _, err = config.GetConfigSetURL(c, config.AsUser, "projects/urltest")
+ So(err, ShouldEqual, config.ErrNoConfig)
+
+ // Service can access.
+ u, err := config.GetConfigSetURL(c, config.AsService, "projects/urltest")
+ So(err, ShouldBeNil)
+ So(u.String(), ShouldEqual, "https://example.com/fake-config/projects/urltest")
+
+ // User joins group, immediately gets access b/c the cached entry is the
+ // service response, and so was soft-forbidden before.
+ authState.IdentityGroups = append(authState.IdentityGroups, "exclusive")
+
+ _, err = config.GetConfigSetURL(c, config.AsUser, "projects/urltest")
+ So(err, ShouldBeNil)
+ So(u.String(), ShouldEqual, "https://example.com/fake-config/projects/urltest")
+ })
+ })
+}
« appengine/gaeconfig/ds.go ('K') | « appengine/gaeconfig/ds.go ('k') | common/testing/assertions/error_tests.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698