Implement config service cache on top of datastore

luci_config/appengine/gaeconfig/default_test.go

Index: luci_config/appengine/gaeconfig/default_test.go
diff --git a/luci_config/appengine/gaeconfig/default_test.go b/luci_config/appengine/gaeconfig/default_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..e419170882291c243e5b051923ff0396e02ee9ac
--- /dev/null
+++ b/luci_config/appengine/gaeconfig/default_test.go
@@ -0,0 +1,468 @@
+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package gaeconfig
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/luci/luci-go/appengine/datastorecache"
+	"github.com/luci/luci-go/common/clock/testclock"
+	commonConfig "github.com/luci/luci-go/common/config"
+	memConfig "github.com/luci/luci-go/common/config/impl/memory"
+	configPB "github.com/luci/luci-go/common/proto/config"
+	"github.com/luci/luci-go/luci_config/common/cfgtypes"
+	"github.com/luci/luci-go/luci_config/server/cfgclient"
+	"github.com/luci/luci-go/luci_config/server/cfgclient/backend"
+	"github.com/luci/luci-go/luci_config/server/cfgclient/backend/client"
+	"github.com/luci/luci-go/luci_config/server/cfgclient/backend/testconfig"
+	"github.com/luci/luci-go/server/auth"
+	"github.com/luci/luci-go/server/auth/authtest"
+	"github.com/luci/luci-go/server/router"
+	"github.com/luci/luci-go/server/settings"
+
+	"github.com/luci/gae/filter/count"
+	"github.com/luci/gae/impl/memory"
+	ds "github.com/luci/gae/service/datastore"
+
+	"github.com/golang/protobuf/proto"
+	"golang.org/x/net/context"
+
+	. "github.com/luci/luci-go/common/testing/assertions"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func projectConfigWithAccess(name cfgtypes.ProjectName, access ...string) *configPB.ProjectCfg {
+	return &configPB.ProjectCfg{
+		Name:   proto.String(string(name)),
+		Access: access,
+	}
+}
+
+// stripMeta strips cache-specific identifying information from a set of Metas.
+func stripMeta(metas []*cfgclient.Meta) []*cfgclient.Meta {
+	for _, meta := range metas {
+		meta.ContentHash = ""
+		meta.Revision = ""
+	}
+	return metas
+}
+
+func TestDatastoreCacheIntegration(t *testing.T) {
+	t.Parallel()
+
+	Convey(`A testing environment with fake cache and config service`, t, func() {
+		c, clk := testclock.UseTime(context.Background(), ds.RoundTime(testclock.TestTimeUTC))
+
+		c = memory.Use(c)
+
+		// Install fake auth state.
+		var authState authtest.FakeState
+		c = auth.WithState(c, &authState)
+		authState.Identity = "user:person@example.com"
+		authState.IdentityGroups = []string{"all", "users"}
+
+		// Use a memory-backed Config service instance.
+		baseMap := map[string]memConfig.ConfigSet{
+			"projects/open": map[string]string{
+				"project.cfg": proto.MarshalTextString(projectConfigWithAccess("foo", "group:all")),
+				"test.cfg":    "Test Config Content",
+			},
+			"projects/foo": map[string]string{
+				"project.cfg": proto.MarshalTextString(projectConfigWithAccess("foo", "group:restricted")),
+			},
+			"projects/noconfig": map[string]string{},
+			"projects/invalidconfig": map[string]string{
+				"project.cfg": "!!! not a valid config !!!",
+			},
+			"projects/foo/refs/heads/master": map[string]string{
+				"ref.cfg": "foo",
+			},
+			"projects/foo/refs/branches/bar": map[string]string{
+				"ref.cfg": "foo/bar",
+			},
+			"projects/bar/refs/heads/master": map[string]string{
+				"ref.cfg": "bar",
+			},
+		}
+		base := memConfig.New(baseMap)
+		be := &client.Backend{
+			Provider: &testconfig.Provider{
+				Base: base,
+			},
+		}
+
+		// Install our settings into a memory-backed Settings storage.
+		memSettings := settings.MemoryStorage{}
+		c = settings.Use(c, settings.New(&memSettings))
+		s := Settings{
+			CacheExpirationSec: 10,
+			DatastoreCacheMode: dsCacheStrict,
+		}
+		putSettings := func() {
+			if err := settings.GetSettings(c).Set(c, settingsKey, &s, "test harness", "initial settings"); err != nil {
+				panic(err)
+			}
+		}
+		putSettings()
+
+		// Install our middleware chain into our Router.
+		baseMW := router.NewMiddlewareChain(func(ctx *router.Context, next router.Handler) {
+			ctx.Context = c
+			next(ctx)
+		})
+		rtr := router.New()
+		server := httptest.NewServer(rtr)
+		defer server.Close()
+
+		// Install our Manager cron handlers.
+		installCacheCronHandlerImpl(rtr, baseMW, be)
+
+		runCron := func() int {
+			ds.GetTestable(c).CatchupIndexes()
+			resp, err := http.Get(fmt.Sprintf("%s/admin/config/cache/manager", server.URL))
+			if err != nil {
+				panic(fmt.Errorf("failed to GET: %s", err))
+			}
+			return resp.StatusCode
+		}
+		So(runCron(), ShouldEqual, http.StatusOK)
+
+		// Called when all parameters are in place to install our config service
+		// layers into "c".
+		installConfig := func(c context.Context) context.Context { return useImpl(c, be) }
+
+		loadProjectConfigs := func(c context.Context, a cfgclient.Authority) (projs []string, meta []*cfgclient.Meta, err error) {
+			err = cfgclient.Projects(c, a, "project.cfg", cfgclient.StringSlice(&projs), &meta)
+			return
+		}
+
+		Convey(`Cache modes`, func() {
+			// Load all of our project configs and metadata. Load directly from our
+			// non-datastore Backend, since this data is not part of the test.
+			allProjs, _, err := loadProjectConfigs(backend.WithBackend(c, be), cfgclient.AsService)
+			if err != nil {
+				panic(err)
+			}
+
+			Convey(`Disabled, skips datastore cache.`, func() {
+				s.DatastoreCacheMode = dsCacheDisabled
+				putSettings()
+
+				c = installConfig(c)
+				c, cnt := count.FilterRDS(c)
+
+				projs, _, err := loadProjectConfigs(c, cfgclient.AsService)
+				So(err, ShouldBeNil)
+				So(projs, ShouldResemble, allProjs)
+
+				// No datastore operations should have been performed.
+				So(cnt.GetMulti.Total(), ShouldEqual, 0)
+				So(cnt.PutMulti.Total(), ShouldEqual, 0)
+			})
+
+			Convey(`Enabled`, func() {
+				s.DatastoreCacheMode = dsCacheEnabled
+				putSettings()
+
+				c = installConfig(c)
+				c, cnt := count.FilterRDS(c)
+				projs, _, err := loadProjectConfigs(c, cfgclient.AsService)
+				So(err, ShouldBeNil)
+				So(projs, ShouldResemble, allProjs)
+
+				So(cnt.GetMulti.Total(), ShouldEqual, 1)
+				So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+				// Break our backing store. This forces lookups to load from cache.
+				memConfig.SetError(base, errors.New("config is broken"))
+
+				projs, _, err = loadProjectConfigs(c, cfgclient.AsService)
+				So(err, ShouldBeNil)
+				So(projs, ShouldResemble, allProjs)
+
+				So(cnt.GetMulti.Total(), ShouldEqual, 2)
+				So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+				// Expire the cache.
+				clk.Add(time.Hour)
+
+				projs, _, err = loadProjectConfigs(c, cfgclient.AsService)
+				So(err, ShouldBeNil)
+				So(projs, ShouldResemble, allProjs)
+
+				So(cnt.GetMulti.Total(), ShouldEqual, 3)
+				So(cnt.PutMulti.Total(), ShouldEqual, 1)
+			})
+
+			Convey(`Strict`, func() {
+				s.DatastoreCacheMode = dsCacheStrict
+				putSettings()
+
+				c = installConfig(c)
+				c, cnt := count.FilterRDS(c)
+				projs, _, err := loadProjectConfigs(c, cfgclient.AsService)
+				So(err, ShouldBeNil)
+				So(projs, ShouldResemble, allProjs)
+
+				So(cnt.GetMulti.Total(), ShouldEqual, 1)
+				So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+				// Break our backing store. This forces lookups to load from cache.
+				memConfig.SetError(base, errors.New("config is broken"))
+
+				projs, _, err = loadProjectConfigs(c, cfgclient.AsService)
+				So(err, ShouldBeNil)
+				So(projs, ShouldResemble, allProjs)
+
+				So(cnt.GetMulti.Total(), ShouldEqual, 2)
+				So(cnt.PutMulti.Total(), ShouldEqual, 1)
+
+				// Expire the cache. Now that the entries are expired, we will
+				// fail-closed to the backing store and return "config is broken".
+				clk.Add(time.Hour)
+
+				projs, _, err = loadProjectConfigs(c, cfgclient.AsService)
+				So(err, ShouldUnwrapTo, datastorecache.ErrCacheExpired)
+
+				So(cnt.GetMulti.Total(), ShouldEqual, 3)
+				So(cnt.PutMulti.Total(), ShouldEqual, 1)
+			})
+		})
+
+		Convey(`A user, denied access to a project, gains access once project config refreshes.`, func() {
+			c = installConfig(c)
+
+			_, metas, err := loadProjectConfigs(c, cfgclient.AsUser)
+			So(err, ShouldBeNil)
+			So(stripMeta(metas), ShouldResemble, []*cfgclient.Meta{
+				{ConfigSet: "projects/open", Path: "project.cfg"},
+			})
+
+			// Update "foo" project ACLs to include the "users" group.
+			baseMap["projects/foo"]["project.cfg"] = proto.MarshalTextString(projectConfigWithAccess("foo", "group:users"))
+
+			// Still denied access (cached).
+			c = installConfig(c)
+			_, metas, err = loadProjectConfigs(c, cfgclient.AsUser)
+			So(err, ShouldBeNil)
+			So(stripMeta(metas), ShouldResemble, []*cfgclient.Meta{
+				{ConfigSet: "projects/open", Path: "project.cfg"},
+			})
+
+			// Expire cache.
+			clk.Add(time.Hour)
+			_, _, err = loadProjectConfigs(c, cfgclient.AsUser)
+			So(err, ShouldUnwrapTo, datastorecache.ErrCacheExpired)
+
+			// Update our cache entries.
+			So(runCron(), ShouldEqual, http.StatusOK)
+
+			// Granted access!
+			c = installConfig(c)
+			_, metas, err = loadProjectConfigs(c, cfgclient.AsUser)
+			So(err, ShouldBeNil)
+			So(stripMeta(metas), ShouldResemble, []*cfgclient.Meta{
+				{ConfigSet: "projects/foo", Path: "project.cfg"},
+				{ConfigSet: "projects/open", Path: "project.cfg"},
+			})
+		})
+
+		Convey(`GetConfig iterative updates`, func() {
+			c = installConfig(c)
+
+			get := func(c context.Context) (v string, meta cfgclient.Meta, err error) {
+				err = cfgclient.Get(c, cfgclient.AsService, "projects/open", "test.cfg", cfgclient.String(&v), &meta)
+				return
+			}
+
+			baseContentHash := func(configSet, config string) string {
+				cfg, err := base.GetConfig(c, configSet, config, true)
+				if err != nil {
+					panic(err)
+				}
+				return cfg.ContentHash
+			}
+
+			_, meta, err := get(c)
+			So(err, ShouldBeNil)
+			So(meta.ContentHash, ShouldEqual, baseContentHash("projects/open", "test.cfg"))
+
+			Convey(`When the content changes, the hash is updated.`, func() {
+				// Change content, hash should not have changed.
+				baseMap["projects/open"]["test.cfg"] = "New Content!"
+				_, meta2, err := get(c)
+				So(err, ShouldBeNil)
+				So(meta2.ContentHash, ShouldEqual, meta.ContentHash)
+
+				// Expire the cache entry and refresh.
+				clk.Add(time.Hour)
+				So(runCron(), ShouldEqual, http.StatusOK)
+
+				// Re-get the config. The content hash should have changed.
+				_, meta, err = get(c)
+				So(err, ShouldBeNil)
+				So(meta.ContentHash, ShouldEqual, baseContentHash("projects/open", "test.cfg"))
+				So(meta.ContentHash, ShouldNotEqual, meta2.ContentHash)
+			})
+
+			Convey(`When the content is deleted, returns ErrNoConfig.`, func() {
+				// Change content, hash should not have changed.
+				delete(baseMap["projects/open"], "test.cfg")
+				_, meta2, err := get(c)
+				So(err, ShouldBeNil)
+				So(meta2.ContentHash, ShouldEqual, meta.ContentHash)
+
+				// Expire the cache entry and refresh.
+				clk.Add(time.Hour)
+				So(runCron(), ShouldEqual, http.StatusOK)
+
+				_, _, err = get(c)
+				So(err, ShouldEqual, cfgclient.ErrNoConfig)
+			})
+		})
+
+		// Load project config, return a slice of hashes.
+		type getAllFn func(context.Context, cfgclient.Authority, string, cfgclient.MultiResolver, *[]*cfgclient.Meta) error
+		getHashes := func(c context.Context, gaFn getAllFn, path string) ([]string, error) {
+			var (
+				content []string
+				metas   []*cfgclient.Meta
+			)
+			err := gaFn(c, cfgclient.AsService, path, cfgclient.StringSlice(&content), &metas)
+			if err != nil {
+				return nil, err
+			}
+
+			hashes := make([]string, len(metas))
+			for i, meta := range metas {
+				hashes[i] = meta.ContentHash
+			}
+			return hashes, nil
+		}
+
+		// Return project config hashes from backing memory config store.
+		type baseConfigsFn func(context.Context, string, bool) ([]commonConfig.Config, error)
+		baseContentHashes := func(fn baseConfigsFn, path string) []string {
+			cfgs, err := fn(c, path, true)
+			if err != nil {
+				panic(err)
+			}
+			hashes := make([]string, len(cfgs))
+			for i := range cfgs {
+				hashes[i] = cfgs[i].ContentHash
+			}
+			return hashes
+		}
+
+		Convey(`Projects for iterative updates`, func() {
+			c = installConfig(c)
+
+			hashes, err := getHashes(c, cfgclient.Projects, "project.cfg")
+			So(err, ShouldBeNil)
+			So(hashes, ShouldResemble, baseContentHashes(base.GetProjectConfigs, "project.cfg"))
+
+			Convey(`When the project list doesn't change, is not re-fetched.`, func() {
+				// Expire the cache entry and refresh.
+				clk.Add(time.Hour)
+				So(runCron(), ShouldEqual, http.StatusOK)
+
+				hashes2, err := getHashes(c, cfgclient.Projects, "project.cfg")
+				So(err, ShouldBeNil)
+				So(hashes2, ShouldResemble, hashes)
+			})
+
+			Convey(`When the project changes, is re-fetched.`, func() {
+				delete(baseMap["projects/invalidconfig"], "project.cfg")
+
+				// Expire the cache entry and refresh.
+				clk.Add(time.Hour)
+				So(runCron(), ShouldEqual, http.StatusOK)
+
+				hashes2, err := getHashes(c, cfgclient.Projects, "project.cfg")
+				So(err, ShouldBeNil)
+				So(hashes2, ShouldResemble, baseContentHashes(base.GetProjectConfigs, "project.cfg"))
+				So(hashes2, ShouldNotResemble, hashes)
+			})
+		})
+
+		Convey(`Ref for iterative updates`, func() {
+			c = installConfig(c)
+
+			hashes, err := getHashes(c, cfgclient.Refs, "ref.cfg")
+			So(err, ShouldBeNil)
+			So(hashes, ShouldResemble, baseContentHashes(base.GetRefConfigs, "ref.cfg"))
+
+			Convey(`When the ref list doesn't change, is not re-fetched.`, func() {
+				// Expire the cache entry and refresh.
+				clk.Add(time.Hour)
+				So(runCron(), ShouldEqual, http.StatusOK)
+
+				hashes2, err := getHashes(c, cfgclient.Refs, "ref.cfg")
+				So(err, ShouldBeNil)
+				So(hashes2, ShouldResemble, hashes)
+			})
+
+			Convey(`When the ref changes, is re-fetched.`, func() {
+				delete(baseMap["projects/foo/refs/branches/bar"], "ref.cfg")
+
+				// Expire the cache entry and refresh.
+				clk.Add(time.Hour)
+				So(runCron(), ShouldEqual, http.StatusOK)
+
+				hashes2, err := getHashes(c, cfgclient.Refs, "ref.cfg")
+				So(err, ShouldBeNil)
+				So(hashes2, ShouldResemble, baseContentHashes(base.GetRefConfigs, "ref.cfg"))
+				So(hashes2, ShouldNotResemble, hashes)
+			})
+		})
+
+		Convey(`Test GetConfigSetURL user fetch`, func() {
+			c = installConfig(c)
+
+			// Project does not exist, missing for user and service.
+			_, err := cfgclient.GetConfigSetURL(c, cfgclient.AsUser, "projects/urltest")
+			So(err, ShouldEqual, cfgclient.ErrNoConfig)
+			_, err = cfgclient.GetConfigSetURL(c, cfgclient.AsService, "projects/urltest")
+			So(err, ShouldEqual, cfgclient.ErrNoConfig)
+
+			// Add the project, still missing b/c of cache.
+			baseMap["projects/urltest"] = memConfig.ConfigSet{
+				"project.cfg": proto.MarshalTextString(projectConfigWithAccess("foo", "group:exclusive")),
+			}
+
+			_, err = cfgclient.GetConfigSetURL(c, cfgclient.AsUser, "projects/urltest")
+			So(err, ShouldEqual, cfgclient.ErrNoConfig)
+			_, err = cfgclient.GetConfigSetURL(c, cfgclient.AsService, "projects/urltest")
+			So(err, ShouldEqual, cfgclient.ErrNoConfig)
+
+			// Expire the cache entry and refresh. Next project load will pick up
+			// the new project.
+			clk.Add(time.Hour)
+			So(runCron(), ShouldEqual, http.StatusOK)
+
+			// User still cannot access (not member of group).
+			_, err = cfgclient.GetConfigSetURL(c, cfgclient.AsUser, "projects/urltest")
+			So(err, ShouldEqual, cfgclient.ErrNoConfig)
+
+			// Service can access.
+			u, err := cfgclient.GetConfigSetURL(c, cfgclient.AsService, "projects/urltest")
+			So(err, ShouldBeNil)
+			So(u.String(), ShouldEqual, "https://example.com/fake-config/projects/urltest")
+
+			// User joins group, immediately gets access b/c the cached entry is the
+			// service response, and so was soft-forbidden before.
+			authState.IdentityGroups = append(authState.IdentityGroups, "exclusive")
+
+			_, err = cfgclient.GetConfigSetURL(c, cfgclient.AsUser, "projects/urltest")
+			So(err, ShouldBeNil)
+			So(u.String(), ShouldEqual, "https://example.com/fake-config/projects/urltest")
+		})
+	})
+}