LogDog: Add Storage-layer data caching.

logdog/appengine/coordinator/storage_cache_test.go

Index: logdog/appengine/coordinator/storage_cache_test.go
diff --git a/logdog/appengine/coordinator/storage_cache_test.go b/logdog/appengine/coordinator/storage_cache_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..c931dc8ed87df5bcc2cf521c6cf3a15356ba117f
--- /dev/null
+++ b/logdog/appengine/coordinator/storage_cache_test.go
@@ -0,0 +1,127 @@
+// Copyright 2015 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package coordinator
+
+import (
+	"testing"
+	"time"
+
+	"github.com/luci/luci-go/common/clock/testclock"
+	"github.com/luci/luci-go/common/errors"
+	"github.com/luci/luci-go/logdog/common/storage/caching"
+
+	"github.com/luci/gae/filter/featureBreaker"
+	"github.com/luci/gae/impl/memory"
+	"github.com/luci/gae/service/memcache"
+
+	"golang.org/x/net/context"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func testStorageCache(t *testing.T, compress bool) {
+	t.Parallel()
+
+	cloneItemsWithData := func(src []*caching.Item, data []byte) []*caching.Item {
+		dst := make([]*caching.Item, len(src))
+		for i, itm := range src {
+			clone := *itm
+			clone.Data = data
+			dst[i] = &clone
+		}
+		return dst
+	}
+
+	Convey(`Testing storage cache in a testing envrionment`, t, func() {
+		c := memory.Use(context.Background())
+		c, tc := testclock.UseTime(c, testclock.TestTimeLocal)
+		c, fb := featureBreaker.FilterMC(c, nil)
+
+		var cache StorageCache
+		if compress {
+			cache.compressionThreshold = 1
+		}
+
+		items := []*caching.Item{
+			{Schema: "test", Type: "type", Key: "foo", Data: []byte("foo")},
+			{Schema: "test", Type: "type", Key: "bar", Data: []byte("bar")},
+			{Schema: "test", Type: "othertype", Key: "foo", Data: []byte("foo2")},
+			{Schema: "otherschema", Type: "othertype", Key: "foo", Data: []byte("foo3")},
+		}
+
+		Convey(`Can load those items into cache`, func() {
+			cache.Put(c, time.Minute, items...)
+
+			stats, err := memcache.Stats(c)
+			So(err, ShouldBeNil)
+			So(stats.Items, ShouldEqual, 4)
+
+			Convey(`And retrieve those items from cache.`, func() {
+				oit := cloneItemsWithData(items, []byte("junk"))
+				cache.Get(c, oit...)
+				So(oit, ShouldResemble, items)
+			})
+
+			Convey(`Returns nil data if memcache.GetMulti is broken.`, func() {
+				fb.BreakFeatures(errors.New("test error"), "GetMulti")
+
+				cache.Get(c, items...)
+				So(items, ShouldResemble, cloneItemsWithData(items, nil))
+			})
+		})
+
+		Convey(`Does not load items into cache if memcache.SetMulti is broken.`, func() {
+			fb.BreakFeatures(errors.New("test error"), "SetMulti")
+
+			cache.Put(c, time.Minute, items...)
+			cache.Get(c, items...)
+			So(items, ShouldResemble, cloneItemsWithData(items, nil))
+		})
+
+		Convey(`Get on missing item returns nil data.`, func() {
+			cache.Get(c, items...)
+			So(items, ShouldResemble, cloneItemsWithData(items, nil))
+		})
+
+		Convey(`Will replace existing item value.`, func() {
+			cache.Put(c, time.Minute, items...)
+
+			other := cloneItemsWithData(items, []byte("ohaithere"))
+			cache.Put(c, time.Minute, other...)
+
+			cache.Get(c, items...)
+			So(items, ShouldResemble, other)
+		})
+
+		Convey(`Applies expiration (or lack thereof).`, func() {
+			cache.Put(c, time.Minute, items[0])
+			cache.Put(c, 0, items[1])
+
+			// items[0]
+			itm, err := memcache.GetKey(c, cache.mkCacheKey(items[0]))
+			So(err, ShouldBeNil)
+			So(itm.Key(), ShouldEqual, cache.mkCacheKey(items[0]))
+			So(len(itm.Value()), ShouldNotEqual, 0)
+
+			tc.Add(time.Minute + 1) // Expires items[0].
+			_, err = memcache.GetKey(c, cache.mkCacheKey(items[0]))
+			So(err, ShouldEqual, memcache.ErrCacheMiss)
+
+			// items[1]
+			itm, err = memcache.GetKey(c, cache.mkCacheKey(items[1]))
+			So(err, ShouldBeNil)
+			So(itm.Key(), ShouldEqual, cache.mkCacheKey(items[1]))
+			So(len(itm.Value()), ShouldNotEqual, 0)
+		})
+	})
+}
+
+func TestStorageCacheWithoutCompression(t *testing.T) {
+	testStorageCache(t, false)
+}
+
+func TestStorageCacheWithCompression(t *testing.T) {
+	testStorageCache(t, true)
+}