Implement config service cache on top of datastore

luci_config/appengine/backend/datastore/ds_test.go

Index: luci_config/appengine/backend/datastore/ds_test.go
diff --git a/luci_config/appengine/backend/datastore/ds_test.go b/luci_config/appengine/backend/datastore/ds_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..001f8ac0b125a0d6b95f21f31170ac5451e97221
--- /dev/null
+++ b/luci_config/appengine/backend/datastore/ds_test.go
@@ -0,0 +1,627 @@
+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package datastore
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/luci/luci-go/appengine/datastorecache"
+	memConfig "github.com/luci/luci-go/common/config/impl/memory"
+	"github.com/luci/luci-go/common/errors"
+	configPB "github.com/luci/luci-go/common/proto/config"
+	gaeformat "github.com/luci/luci-go/luci_config/appengine/format"
+	"github.com/luci/luci-go/luci_config/common/cfgtypes"
+	"github.com/luci/luci-go/luci_config/server/cfgclient"
+	"github.com/luci/luci-go/luci_config/server/cfgclient/backend"
+	"github.com/luci/luci-go/luci_config/server/cfgclient/backend/caching"
+	"github.com/luci/luci-go/luci_config/server/cfgclient/backend/client"
+	"github.com/luci/luci-go/luci_config/server/cfgclient/backend/format"
+	"github.com/luci/luci-go/luci_config/server/cfgclient/backend/testconfig"
+	"github.com/luci/luci-go/luci_config/server/cfgclient/textproto"
+	"github.com/luci/luci-go/server/auth"
+	"github.com/luci/luci-go/server/auth/authtest"
+
+	"github.com/luci/gae/impl/memory"
+
+	"github.com/golang/protobuf/proto"
+	"golang.org/x/net/context"
+
+	. "github.com/luci/luci-go/common/testing/assertions"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+// testCache is a generic Cache testing layer.
+type testCache interface {
+	dsCacheBackend
+
+	setCacheErr(err error)
+	setProjectDNE(project string)
+	addConfig(configSet cfgtypes.ConfigSet, path, content string) *backend.Item
+	addProjectConfig(name cfgtypes.ProjectName, access string)
+	addConfigSets(path string, configSets ...cfgtypes.ConfigSet) []string
+	addConfigSetURL(configSet cfgtypes.ConfigSet) string
+}
+
+func projectConfigWithAccess(name cfgtypes.ProjectName, access ...string) *configPB.ProjectCfg {
+	return &configPB.ProjectCfg{
+		Name:   proto.String(string(name)),
+		Access: access,
+	}
+}
+
+// fakeCache is a pure in-memory testCache implementation. It is very simple,
+// storing only raw cache key/value pairs.
+type fakeCache struct {
+	d   map[string]datastorecache.Value
+	err error
+}
+
+func mkFakeCache() *fakeCache {
+	return &fakeCache{
+		d: make(map[string]datastorecache.Value),
+	}
+}
+
+func (fc *fakeCache) Get(c context.Context, key []byte) (v datastorecache.Value, err error) {
+	if err = fc.err; err != nil {
+		return
+	}
+
+	var k caching.Key
+	caching.Decode(key, &k)
+
+	var ok bool
+	if v, ok = fc.d[string(key)]; ok {
+		return
+	}
+
+	err = datastorecache.ErrCacheExpired
+	return
+}
+
+func (fc *fakeCache) setCacheData(key caching.Key, d []byte) {
+}
+
+func (fc *fakeCache) set(key caching.Key, v *caching.Value) {
+	encKey, err := caching.Encode(&key)
+	if err != nil {
+		panic(fmt.Errorf("failed to encode key: %s", err))
+	}
+
+	if v == nil {
+		delete(fc.d, string(encKey))
+		return
+	}
+
+	encValue, err := v.Encode()
+	if err != nil {
+		panic(fmt.Errorf("failed to encode cache value: %s", err))
+	}
+
+	fc.d[string(encKey)] = datastorecache.Value{
+		Schema:      dsCacheSchema,
+		Data:        encValue,
+		Description: key.String(),
+	}
+}
+
+func (fc *fakeCache) setCacheErr(err error) { fc.err = err }
+
+func (fc *fakeCache) setProjectDNE(project string) {
+	// Get for this project config will fail.
+	fc.set(caching.Key{
+		Schema:    caching.Schema,
+		Op:        caching.OpGet,
+		ConfigSet: string(cfgtypes.ProjectConfigSet(cfgtypes.ProjectName(project))),
+		Path:      cfgclient.ProjectConfigPath,
+	}, nil)
+}
+
+func (fc *fakeCache) addConfigImpl(cs cfgtypes.ConfigSet, path, formatter, formatData, content string) *backend.Item {
+	var (
+		item *backend.Item
+		cv   caching.Value
+	)
+	if content != "" {
+		item = &backend.Item{
+			Meta: backend.Meta{
+				ConfigSet:   string(cs),
+				Path:        path,
+				ContentHash: "hash",
+			},
+			Content:    content,
+			FormatSpec: backend.FormatSpec{formatter, formatData},
+		}
+		cv.LoadItems(item)
+	}
+
+	fc.set(caching.Key{
+		Schema:     caching.Schema,
+		Authority:  backend.AsService,
+		Op:         caching.OpGet,
+		ConfigSet:  string(cs),
+		Path:       path,
+		Content:    true,
+		Formatter:  formatter,
+		FormatData: formatData,
+	}, &cv)
+
+	return item
+}
+
+func (fc *fakeCache) addConfig(cs cfgtypes.ConfigSet, path, content string) *backend.Item {
+	return fc.addConfigImpl(cs, path, "", "", content)
+}
+
+// addProjectConfig caches a "project.cfg" file for the specified project with
+// the specified access string.
+func (fc *fakeCache) addProjectConfig(name cfgtypes.ProjectName, access string) {
+	// We're loading the resolved version of this cache item.
+	pcfg := projectConfigWithAccess(name, access)
+	pcfgName := proto.MessageName(pcfg)
+
+	f := textproto.Formatter{}
+	formattedData, err := f.FormatItem(proto.MarshalTextString(pcfg), pcfgName)
+	if err != nil {
+		panic(err)
+	}
+
+	fc.addConfigImpl(cfgtypes.ProjectConfigSet(name), cfgclient.ProjectConfigPath,
+		textproto.BinaryFormat, pcfgName, formattedData)
+}
+
+func (fc *fakeCache) addConfigSets(path string, configSets ...cfgtypes.ConfigSet) []string {
+	items := make([]*backend.Item, len(configSets))
+	contents := make([]string, len(configSets))
+	for i, cs := range configSets {
+		contents[i] = string(cs)
+		items[i] = &backend.Item{
+			Meta: backend.Meta{
+				ConfigSet:   string(cs),
+				Path:        path,
+				ContentHash: "hash",
+			},
+			Content: contents[i],
+		}
+	}
+
+	for _, t := range []backend.GetAllTarget{backend.GetAllProject, backend.GetAllRef} {
+		var cv caching.Value
+		cv.LoadItems(items...)
+
+		fc.set(caching.Key{
+			Schema:       caching.Schema,
+			Authority:    backend.AsService,
+			Op:           caching.OpGetAll,
+			Content:      true,
+			Path:         path,
+			GetAllTarget: t,
+		}, &cv)
+	}
+	return contents
+}
+
+func (fc *fakeCache) addConfigSetURL(configSet cfgtypes.ConfigSet) string {
+	u := fmt.Sprintf("https://exmaple.com/config-sets/%s", configSet)
+	fc.set(caching.Key{
+		Schema:    caching.Schema,
+		Authority: backend.AsService,
+		Op:        caching.OpConfigSetURL,
+		ConfigSet: string(configSet),
+	}, &caching.Value{
+		URL: u,
+	})
+	return u
+}
+
+// fullStackCache is a testCache implementation built on top of an in-memory
+// base backend.B with the datastore Cache layer on top of it.
+type fullStackCache struct {
+	cache *datastorecache.Cache
+	err   error
+
+	data    map[string]memConfig.ConfigSet
+	backend backend.B
+	junkIdx int
+}
+
+func (fsc *fullStackCache) Get(c context.Context, key []byte) (datastorecache.Value, error) {
+	if err := fsc.err; err != nil {
+		return datastorecache.Value{}, err
+	}
+	return fsc.cache.Get(c, key)
+}
+
+func (fsc *fullStackCache) setCacheErr(err error) { fsc.err = err }
+
+func (fsc *fullStackCache) setProjectDNE(project string) {
+	key := "projects/" + project
+	for k := range fsc.data {
+		if k == key || strings.HasPrefix(k, key+"/") {
+			delete(fsc.data, k)
+		}
+	}
+}
+
+func (fsc *fullStackCache) addConfig(cs cfgtypes.ConfigSet, path, content string) *backend.Item {
+	cset := fsc.data[string(cs)]
+	if cset == nil {
+		cset = memConfig.ConfigSet{}
+		fsc.data[string(cs)] = cset
+	}
+	if content == "" {
+		delete(cset, path)
+		return nil
+	}
+	cset[path] = content
+
+	// Pull the config right back out of the base service.
+	item, err := fsc.backend.Get(context.Background(), string(cs), path, backend.Params{
+		Authority: backend.AsService,
+	})
+	if err != nil {
+		panic(err)
+	}
+	return item
+}
+
+// addProjectConfig caches a "project.cfg" file for the specified project with
+// the specified access string.
+func (fsc *fullStackCache) addProjectConfig(name cfgtypes.ProjectName, access string) {
+	fsc.addConfig(cfgtypes.ProjectConfigSet(name), cfgclient.ProjectConfigPath,
+		proto.MarshalTextString(projectConfigWithAccess(name, access)))
+}
+
+func (fsc *fullStackCache) addConfigSets(path string, configSets ...cfgtypes.ConfigSet) []string {
+	// Sort the config sets list, then put it back.
+	cstr := make([]string, len(configSets))
+	for i, cs := range configSets {
+		cstr[i] = string(cs)
+	}
+	sort.Strings(cstr)
+	for i, cs := range cstr {
+		configSets[i] = cfgtypes.ConfigSet(cs)
+	}
+
+	items := make([]*backend.Item, len(configSets))
+	for i, cs := range configSets {
+		items[i] = fsc.addConfig(cs, path, string(cs))
+	}
+	return cstr
+}
+
+func (fsc *fullStackCache) addConfigSetURL(configSet cfgtypes.ConfigSet) string {
+	if _, ok := fsc.data[string(configSet)]; !ok {
+		fsc.data[string(configSet)] = memConfig.ConfigSet{}
+	}
+
+	// We're pretty rigid here. Whatever our backend returns is all we can
+	// return. We will just assert that anything more flexible has to conform to
+	// this.
+	v, err := fsc.backend.ConfigSetURL(context.Background(), string(configSet),
+		backend.Params{Authority: backend.AsService})
+	if err != nil {
+		panic(err)
+	}
+	return v.String()
+}
+
+// stripMeta strips cache-specific identifying information from a set of Metas.
+func stripMeta(metas []*cfgclient.Meta) []*cfgclient.Meta {
+	for _, meta := range metas {
+		meta.ContentHash = ""
+		meta.Revision = ""
+	}
+	return metas
+}
+
+func testDatastoreCacheImpl(c context.Context, be backend.B, cache testCache) {
+	// Install fake auth state.
+	var authState authtest.FakeState
+	c = auth.WithState(c, &authState)
+	authState.Identity = "user:person@example.com"
+	authState.IdentityGroups = []string{"users"}
+
+	dsc := Config{
+		RefreshInterval: 1 * time.Hour,
+		FailOpen:        false,
+		cache:           cache,
+	}
+	c = backend.WithBackend(c, dsc.Backend(be))
+
+	testErr := errors.New("test error")
+
+	Convey(`Test Get`, func() {
+		var v string
+
+		Convey(`Config missing`, func() {
+			cache.addConfig("projects/test", "foo", "")
+
+			So(cfgclient.Get(c, cfgclient.AsService, "projects/test", "foo", cfgclient.String(&v), nil),
+				ShouldEqual, cfgclient.ErrNoConfig)
+		})
+
+		Convey(`Config is present`, func() {
+			cache.addConfig("projects/test", "foo", "bar")
+			cache.addProjectConfig("test", "group:privileged")
+
+			Convey(`As service`, func() {
+				So(cfgclient.Get(c, cfgclient.AsService, "projects/test", "foo", cfgclient.String(&v), nil), ShouldBeNil)
+				So(v, ShouldEqual, "bar")
+			})
+
+			Convey(`As user, when not a project group member, fails with ErrNoConfig`, func() {
+				So(cfgclient.Get(c, cfgclient.AsUser, "projects/test", "foo", cfgclient.String(&v), nil),
+					ShouldEqual, cfgclient.ErrNoConfig)
+			})
+
+			Convey(`As user, when a project group member, succeeds.`, func() {
+				authState.IdentityGroups = append(authState.IdentityGroups, "privileged")
+				So(cfgclient.Get(c, cfgclient.AsUser, "projects/test", "foo", cfgclient.String(&v), nil), ShouldBeNil)
+				So(v, ShouldEqual, "bar")
+			})
+
+			Convey(`As anonymous, fails with ErrNoConfig`, func() {
+				So(cfgclient.Get(c, cfgclient.AsAnonymous, "projects/test", "foo", cfgclient.String(&v), nil),
+					ShouldEqual, cfgclient.ErrNoConfig)
+			})
+		})
+	})
+
+	Convey(`Test Projects`, func() {
+		var v []string
+		var meta []*cfgclient.Meta
+
+		Convey(`When cache returns an error`, func() {
+			cache.setCacheErr(testErr)
+
+			So(cfgclient.Projects(c, cfgclient.AsService, "test.cfg", cfgclient.StringSlice(&v), nil),
+				ShouldUnwrapTo, testErr)
+			So(cfgclient.Projects(c, cfgclient.AsUser, "test.cfg", cfgclient.StringSlice(&v), nil),
+				ShouldUnwrapTo, testErr)
+			So(cfgclient.Projects(c, cfgclient.AsAnonymous, "test.cfg", cfgclient.StringSlice(&v), nil),
+				ShouldUnwrapTo, testErr)
+		})
+
+		Convey(`With project configs installed`, func() {
+			allConfigs := cache.addConfigSets("test.cfg",
+				"projects/bar",
+				"projects/baz",
+				"projects/foo")
+
+			Convey(`As service, retrieves all configs.`, func() {
+				So(cfgclient.Projects(c, cfgclient.AsService, "test.cfg", cfgclient.StringSlice(&v), &meta), ShouldBeNil)
+				So(v, ShouldResemble, allConfigs)
+				So(stripMeta(meta), ShouldResemble, []*cfgclient.Meta{
+					{ConfigSet: "projects/bar", Path: "test.cfg"},
+					{ConfigSet: "projects/baz", Path: "test.cfg"},
+					{ConfigSet: "projects/foo", Path: "test.cfg"},
+				})
+			})
+
+			Convey(`As user`, func() {
+				Convey(`Not a member of any projects, receives empty slice.`, func() {
+					cache.addProjectConfig("foo", "group:someone")
+
+					So(cfgclient.Projects(c, cfgclient.AsUser, "test.cfg", cfgclient.StringSlice(&v), &meta), ShouldBeNil)
+					So(v, ShouldResemble, []string(nil))
+					So(stripMeta(meta), ShouldResemble, []*cfgclient.Meta{})
+				})
+
+				Convey(`Member of "foo", gets only "foo".`, func() {
+					cache.addProjectConfig("foo", "group:users")
+
+					So(cfgclient.Projects(c, cfgclient.AsUser, "test.cfg", cfgclient.StringSlice(&v), &meta), ShouldBeNil)
+					So(v, ShouldResemble, allConfigs[2:3])
+					So(stripMeta(meta), ShouldResemble, []*cfgclient.Meta{
+						{ConfigSet: "projects/foo", Path: "test.cfg"},
+					})
+				})
+
+				Convey(`Member of all projects, gets all projects.`, func() {
+					cache.addProjectConfig("foo", "group:users")
+					cache.addProjectConfig("bar", "group:users")
+					cache.addProjectConfig("baz", "group:users")
+
+					So(cfgclient.Projects(c, cfgclient.AsUser, "test.cfg", cfgclient.StringSlice(&v), &meta), ShouldBeNil)
+					So(v, ShouldResemble, allConfigs)
+					So(stripMeta(meta), ShouldResemble, []*cfgclient.Meta{
+						{ConfigSet: "projects/bar", Path: "test.cfg"},
+						{ConfigSet: "projects/baz", Path: "test.cfg"},
+						{ConfigSet: "projects/foo", Path: "test.cfg"},
+					})
+				})
+			})
+		})
+	})
+
+	Convey(`Test Refs`, func() {
+		var v []string
+		var meta []*cfgclient.Meta
+
+		Convey(`When cache returns an error`, func() {
+			cache.setCacheErr(testErr)
+
+			So(cfgclient.Refs(c, cfgclient.AsService, "test.cfg", cfgclient.StringSlice(&v), nil),
+				ShouldUnwrapTo, testErr)
+			So(cfgclient.Refs(c, cfgclient.AsUser, "test.cfg", cfgclient.StringSlice(&v), nil),
+				ShouldUnwrapTo, testErr)
+			So(cfgclient.Refs(c, cfgclient.AsAnonymous, "test.cfg", cfgclient.StringSlice(&v), nil),
+				ShouldUnwrapTo, testErr)
+		})
+
+		Convey(`With ref configs installed`, func() {
+			allConfigs := cache.addConfigSets("test.cfg",
+				"projects/bar/refs/branches/mybranch",
+				"projects/bar/refs/heads/master",
+				"projects/foo/refs/branches/mybranch",
+				"projects/foo/refs/heads/master")
+
+			Convey(`As service, retrieves all configs.`, func() {
+				So(cfgclient.Refs(c, cfgclient.AsService, "test.cfg", cfgclient.StringSlice(&v), &meta), ShouldBeNil)
+				So(v, ShouldResemble, allConfigs)
+				So(stripMeta(meta), ShouldResemble, []*cfgclient.Meta{
+					{ConfigSet: "projects/bar/refs/branches/mybranch", Path: "test.cfg"},
+					{ConfigSet: "projects/bar/refs/heads/master", Path: "test.cfg"},
+					{ConfigSet: "projects/foo/refs/branches/mybranch", Path: "test.cfg"},
+					{ConfigSet: "projects/foo/refs/heads/master", Path: "test.cfg"},
+				})
+			})
+
+			Convey(`As user`, func() {
+				Convey(`Not a member of any projects, receives empty slice.`, func() {
+					cache.addProjectConfig("foo", "group:someone")
+
+					So(cfgclient.Refs(c, cfgclient.AsUser, "test.cfg", cfgclient.StringSlice(&v), &meta), ShouldBeNil)
+					So(v, ShouldResemble, []string(nil))
+					So(stripMeta(meta), ShouldResemble, []*cfgclient.Meta{})
+				})
+
+				Convey(`Member of "foo", gets only "foo".`, func() {
+					cache.addProjectConfig("foo", "group:users")
+
+					So(cfgclient.Refs(c, cfgclient.AsUser, "test.cfg", cfgclient.StringSlice(&v), &meta), ShouldBeNil)
+					So(v, ShouldResemble, allConfigs[2:4])
+					So(stripMeta(meta), ShouldResemble, []*cfgclient.Meta{
+						{ConfigSet: "projects/foo/refs/branches/mybranch", Path: "test.cfg"},
+						{ConfigSet: "projects/foo/refs/heads/master", Path: "test.cfg"},
+					})
+				})
+
+				Convey(`Member of all projects, gets all projects.`, func() {
+					cache.addProjectConfig("foo", "group:users")
+					cache.addProjectConfig("bar", "group:users")
+
+					So(cfgclient.Refs(c, cfgclient.AsUser, "test.cfg", cfgclient.StringSlice(&v), &meta), ShouldBeNil)
+					So(v, ShouldResemble, allConfigs)
+					So(stripMeta(meta), ShouldResemble, []*cfgclient.Meta{
+						{ConfigSet: "projects/bar/refs/branches/mybranch", Path: "test.cfg"},
+						{ConfigSet: "projects/bar/refs/heads/master", Path: "test.cfg"},
+						{ConfigSet: "projects/foo/refs/branches/mybranch", Path: "test.cfg"},
+						{ConfigSet: "projects/foo/refs/heads/master", Path: "test.cfg"},
+					})
+				})
+			})
+		})
+	})
+
+	Convey(`Test ConfigSetURL`, func() {
+		cache.addProjectConfig("foo", "group:someone")
+		csURL := cache.addConfigSetURL("projects/foo")
+
+		Convey(`AsService`, func() {
+			u, err := cfgclient.GetConfigSetURL(c, cfgclient.AsService, "projects/foo")
+			So(err, ShouldBeNil)
+			So(u.String(), ShouldEqual, csURL)
+		})
+
+		Convey(`AsUser`, func() {
+
+			Convey(`When not a member of the group.`, func() {
+				_, err := cfgclient.GetConfigSetURL(c, cfgclient.AsUser, "projects/foo")
+				So(err, ShouldEqual, cfgclient.ErrNoConfig)
+			})
+
+			Convey(`When a member of the group.`, func() {
+				authState.IdentityGroups = append(authState.IdentityGroups, "someone")
+
+				u, err := cfgclient.GetConfigSetURL(c, cfgclient.AsUser, "projects/foo")
+				So(err, ShouldBeNil)
+				So(u.String(), ShouldEqual, csURL)
+			})
+		})
+
+		Convey(`AsAnonymous`, func() {
+			_, err := cfgclient.GetConfigSetURL(c, cfgclient.AsAnonymous, "projects/foo")
+			So(err, ShouldEqual, cfgclient.ErrNoConfig)
+
+			// With credentials, can access.
+			authState.IdentityGroups = append(authState.IdentityGroups, "someone")
+			_, err = cfgclient.GetConfigSetURL(c, cfgclient.AsAnonymous, "projects/foo")
+			So(err, ShouldBeNil)
+		})
+	})
+}
+
+func TestDatastoreCache(t *testing.T) {
+	t.Parallel()
+
+	Convey(`Testing with in-memory stub cache`, t, func() {
+		c := context.Background()
+		fc := mkFakeCache()
+
+		var be backend.B
+		be = &client.Backend{
+			Provider: &testconfig.Provider{
+				Base: memConfig.New(nil),
+			},
+		}
+
+		fr := gaeformat.Default()
+		be = &format.Backend{
+			B:           be,
+			GetRegistry: func(context.Context) *cfgclient.FormatterRegistry { return fr },
+		}
+
+		Convey(`Standard datastore tests`, func() {
+			testDatastoreCacheImpl(c, be, fc)
+		})
+
+		Convey(`A testing setup built around the fake cache`, func() {
+			dsc := Config{
+				RefreshInterval: 1 * time.Hour,
+				FailOpen:        false,
+				cache:           fc,
+			}
+			c = backend.WithBackend(c, dsc.Backend(be))
+
+			Convey(`Errors with different schema.`, func() {
+				fc.addConfig("foo", "bar", "value")
+				for k, v := range fc.d {
+					v.Schema = "unknown"
+					fc.d[k] = v
+				}
+
+				var v string
+				So(cfgclient.Get(c, cfgclient.AsService, "foo", "bar", cfgclient.String(&v), nil),
+					ShouldErrLike, `response schema ("unknown") doesn't match current`)
+			})
+		})
+	})
+}
+
+func TestDatastoreCacheFullStack(t *testing.T) {
+	t.Parallel()
+
+	Convey(`Testing full-stack datastore cache`, t, func() {
+		c := memory.Use(context.Background())
+
+		data := map[string]memConfig.ConfigSet{}
+
+		var be backend.B
+		be = &client.Backend{
+			Provider: &testconfig.Provider{
+				Base: memConfig.New(data),
+			},
+		}
+
+		fr := gaeformat.Default()
+		be = &format.Backend{
+			B:           be,
+			GetRegistry: func(context.Context) *cfgclient.FormatterRegistry { return fr },
+		}
+
+		fsc := fullStackCache{
+			cache:   &Cache,
+			data:    data,
+			backend: be,
+		}
+		testDatastoreCacheImpl(c, be, &fsc)
+	})
+}