impl/cloud: Add support for "memcached" memcache.

impl/cloud/memcache_test.go

Index: impl/cloud/memcache_test.go
diff --git a/impl/cloud/memcache_test.go b/impl/cloud/memcache_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..4411ae544e5e6b9a61f8e3579eaf20d2a2dd2a09
--- /dev/null
+++ b/impl/cloud/memcache_test.go
@@ -0,0 +1,336 @@
+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package cloud
+
+import (
+	"flag"
+	"fmt"
+	"math"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/luci/luci-go/common/errors"
+
+	"github.com/luci/gae/service/info"
+	mc "github.com/luci/gae/service/memcache"
+
+	"github.com/bradfitz/gomemcache/memcache"
+	"golang.org/x/net/context"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var memcacheServer = flag.String("test.memcache-server", "",
+	"[<addr>]:<port> of memcached service to test against. THIS WILL FLUSH THE CACHE.")
+
+// TestMemcache tests the memcache implementation against a live "memcached"
+// instance. The test assumes ownership of the instance, and will flush all of
+// its keys in between test suites, so DO NOT connect this to a production
+// memcached cluster!
+//
+// The memcache host is passed to this test suite via the "-test.memcache-host"
+// flag. If the flag is not provided, this test suite will be skipped.
+//
+// Starting a local memcached server (on default port 11211) can be done with:
+//	$ memcached -l localhost -vvv
+//
+// Testing against this service can be done using the flag:
+// "-test.memcache-server localhost:11211
+func TestMemcache(t *testing.T) {
+	t.Parallel()
+
+	// See if a memcache server is configured. If no server is configured, we will
+	// skip this test suite.
+	if *memcacheServer == "" {
+		t.Logf("No memcache server detected (-test.memcache-server). Skipping test suite.")
+		return
+	}
+
+	Convey(fmt.Sprintf(`A memcache instance bound to %q`, *memcacheServer), t, func() {
+		client := memcache.New(*memcacheServer)
+		if err := client.DeleteAll(); err != nil {
+			t.Fatalf("failed to flush memcache before running test suite: %s", err)
+		}
+
+		cfg := Config{MC: client}
+		c := cfg.Use(context.Background())
+
+		get := func(c context.Context, keys ...string) []string {
+			bmc := bindMemcacheClient(nil, info.GetNamespace(c))
+
+			v := make([]string, len(keys))
+			for i, k := range keys {
+				itm, err := client.Get(bmc.makeKey(k))
+				switch err {
+				case nil:
+					v[i] = string(itm.Value)
+
+				case memcache.ErrCacheMiss:
+					break
+
+				default:
+					t.Fatalf("item %q could not be loaded: %s", itm.Key, err)
+				}
+			}
+			return v
+		}
+
+		Convey(`AddMulti`, func() {
+			items := []mc.Item{
+				mc.NewItem(c, "foo").SetValue([]byte("FOO")),
+				mc.NewItem(c, "bar").SetValue([]byte("BAR")),
+				mc.NewItem(c, "baz").SetValue([]byte("BAZ")),
+			}
+			So(mc.Add(c, items...), ShouldBeNil)
+			So(get(c, "foo", "bar", "baz"), ShouldResemble, []string{"FOO", "BAR", "BAZ"})
+
+			// Namespaced.
+			oc := info.MustNamespace(c, "other")
+			oitems := make([]mc.Item, len(items))
+			for i, itm := range items {
+				oitems[i] = mc.NewItem(oc, itm.Key())
+				oitems[i].SetValue([]byte("OTHER_" + string(itm.Value())))
+			}
+			So(mc.Add(oc, oitems...), ShouldBeNil)
+			So(get(oc, "foo", "bar", "baz"), ShouldResemble, []string{"OTHER_FOO", "OTHER_BAR", "OTHER_BAZ"})
+
+			Convey(`Returns ErrNotStored if the item is already added.`, func() {
+				So(mc.Add(c, items...), ShouldResemble, errors.MultiError{
+					mc.ErrNotStored, mc.ErrNotStored, mc.ErrNotStored})
+			})
+
+			Convey(`GetMulti`, func() {
+				getItems := []mc.Item{
+					mc.NewItem(c, "foo"),
+					mc.NewItem(c, "bar"),
+					mc.NewItem(c, "baz"),
+				}
+				So(mc.Get(c, getItems...), ShouldBeNil)
+				So(getItems[0].Value(), ShouldResemble, []byte("FOO"))
+				So(getItems[1].Value(), ShouldResemble, []byte("BAR"))
+				So(getItems[2].Value(), ShouldResemble, []byte("BAZ"))
+
+				// Namespaced.
+				So(mc.Get(oc, getItems...), ShouldBeNil)
+				So(oitems[0].Value(), ShouldResemble, []byte("OTHER_FOO"))
+				So(oitems[1].Value(), ShouldResemble, []byte("OTHER_BAR"))
+				So(oitems[2].Value(), ShouldResemble, []byte("OTHER_BAZ"))
+			})
+
+			Convey(`SetMulti`, func() {
+				items[2].SetValue([]byte("NEWBAZ"))
+				items = append(items, mc.NewItem(c, "qux").SetValue([]byte("QUX")))
+
+				oitems[2].SetValue([]byte("OTHER_NEWBAZ"))
+				oitems = append(oitems, mc.NewItem(c, "qux").SetValue([]byte("OTHER_QUX")))
+
+				So(mc.Set(c, items...), ShouldBeNil)
+				So(mc.Set(oc, oitems...), ShouldBeNil)
+
+				So(get(c, "foo", "bar", "baz", "qux"), ShouldResemble, []string{"FOO", "BAR", "NEWBAZ", "QUX"})
+				So(get(oc, "foo", "bar", "baz", "qux"), ShouldResemble,
+					[]string{"OTHER_FOO", "OTHER_BAR", "OTHER_NEWBAZ", "OTHER_QUX"})
+			})
+
+			Convey(`DeleteMulti`, func() {
+				So(mc.Delete(c, "foo", "bar"), ShouldBeNil)
+				So(get(c, "foo", "bar", "baz"), ShouldResemble, []string{"", "", "BAZ"})
+
+				So(mc.Delete(oc, "foo", "bar"), ShouldBeNil)
+				So(get(oc, "foo", "bar", "baz"), ShouldResemble, []string{"", "", "OTHER_BAZ"})
+			})
+
+			Convey(`CompareAndSwapMulti`, func() {
+				// Get all of the values.
+				So(mc.Get(c, items...), ShouldBeNil)
+
+				// Change "foo"'s value.
+				newFoo := mc.NewItem(c, "foo")
+				newFoo.SetAll(items[0])
+				newFoo.SetValue([]byte("NEWFOO"))
+				So(mc.Set(c, newFoo), ShouldBeNil)
+
+				Convey(`Works`, func() {
+					// Do CAS on foo, bar. "foo" should fail because it's changed.
+					items[0].SetValue([]byte("CASFOO"))
+					items[1].SetValue([]byte("CASBAR"))
+					So(mc.CompareAndSwap(c, items...), ShouldResemble, errors.MultiError{mc.ErrCASConflict, nil, nil})
+
+					So(get(c, "foo", "bar", "baz"), ShouldResemble, []string{"NEWFOO", "CASBAR", "BAZ"})
+				})
+
+				Convey(`SetAll transfers CAS ID`, func() {
+					// Do CAS on foo, bar. "foo" should fail because it's changed.
+					foo := mc.NewItem(c, "foo")
+					foo.SetAll(items[0])
+					foo.SetValue([]byte("CASFOO"))
+
+					bar := mc.NewItem(c, "bar")
+					bar.SetAll(items[1])
+					bar.SetValue([]byte("CASBAR"))
+
+					So(mc.CompareAndSwap(c, foo, bar), ShouldResemble, errors.MultiError{mc.ErrCASConflict, nil})
+
+					So(get(c, "foo", "bar"), ShouldResemble, []string{"NEWFOO", "CASBAR"})
+				})
+			})
+
+			Convey(`Flush`, func() {
+				So(mc.Flush(c), ShouldBeNil)
+				So(get(c, "foo", "bar", "baz"), ShouldResemble, []string{"", "", ""})
+			})
+		})
+
+		Convey(`Increment`, func() {
+
+			Convey(`Missing`, func() {
+				// IncrementExisting on non-existent item will return ErrCacheMiss.
+				_, err := mc.IncrementExisting(c, "foo", 1)
+				So(err, ShouldEqual, mc.ErrCacheMiss)
+
+				// IncrementExisting with delta of 0 on non-existent item will return
+				// ErrCacheMiss.
+				_, err = mc.IncrementExisting(c, "foo", 1)
+				So(err, ShouldEqual, mc.ErrCacheMiss)
+			})
+
+			Convey(`Initial value (positive)`, func() {
+				// Can set the initial value.
+				nv, err := mc.Increment(c, "foo", 1, 1336)
+				So(err, ShouldBeNil)
+				So(nv, ShouldEqual, 1337)
+				So(get(c, "foo"), ShouldResemble, []string{"1337"})
+
+				// Followup increment (initial value is ignored).
+				nv, err = mc.Increment(c, "foo", 1, 20000)
+				So(err, ShouldBeNil)
+				So(nv, ShouldEqual, 1338)
+			})
+
+			Convey(`Initial value (positive, overflows)`, func() {
+				// Can set the initial value.
+				nv, err := mc.Increment(c, "foo", 10, math.MaxUint64)
+				So(err, ShouldBeNil)
+				So(nv, ShouldEqual, 9)
+				So(get(c, "foo"), ShouldResemble, []string{"9"})
+			})
+
+			Convey(`Initial value (negative)`, func() {
+				// Can set the initial value.
+				nv, err := mc.Increment(c, "foo", -1, 1338)
+				So(err, ShouldBeNil)
+				So(nv, ShouldEqual, 1337)
+				So(get(c, "foo"), ShouldResemble, []string{"1337"})
+			})
+
+			Convey(`Initial value (negative, underflows to zero)`, func() {
+				// Can set the initial value.
+				nv, err := mc.Increment(c, "foo", -1337, 2)
+				So(err, ShouldBeNil)
+				So(nv, ShouldEqual, 0)
+				So(get(c, "foo"), ShouldResemble, []string{"0"})
+			})
+
+			Convey(`With large initial value (zero delta)`, func() {
+				const iv = uint64(math.MaxUint64 - 1)
+
+				// Can set the initial value.
+				nv, err := mc.Increment(c, "foo", 0, iv)
+				So(err, ShouldBeNil)
+				So(nv, ShouldEqual, uint64(math.MaxUint64-1))
+
+				Convey(`Can increment`, func() {
+					nv, err := mc.IncrementExisting(c, "foo", 1)
+					So(err, ShouldBeNil)
+					So(nv, ShouldEqual, iv+1)
+				})
+
+				Convey(`Can increment (overflow, wraps)`, func() {
+					nv, err := mc.IncrementExisting(c, "foo", 10)
+					So(err, ShouldBeNil)
+					So(nv, ShouldEqual, 8)
+				})
+
+				Convey(`Can decrement`, func() {
+					nv, err := mc.IncrementExisting(c, "foo", -123)
+					So(err, ShouldBeNil)
+					So(nv, ShouldEqual, iv-123)
+				})
+			})
+
+			Convey(`With small initial value (zero delta)`, func() {
+				// Can set the initial value.
+				nv, err := mc.Increment(c, "foo", 0, 1337)
+				So(err, ShouldBeNil)
+				So(nv, ShouldEqual, 1337)
+
+				Convey(`Can decrement (underflow to zero)`, func() {
+					nv, err := mc.IncrementExisting(c, "foo", -2000)
+					So(err, ShouldBeNil)
+					So(nv, ShouldEqual, 0)
+				})
+			})
+
+			Convey(`Namespaced`, func() {
+				oc := info.MustNamespace(c, "other")
+
+				// Initial (non-namespaced).
+				nv, err := mc.Increment(c, "foo", 1, 100)
+				So(err, ShouldBeNil)
+				So(nv, ShouldEqual, 101)
+
+				// Initial (namespaced).
+				_, err = mc.IncrementExisting(oc, "foo", -1)
+				So(err, ShouldEqual, mc.ErrCacheMiss)
+
+				nv, err = mc.Increment(oc, "foo", -1, 20000)
+				So(err, ShouldBeNil)
+				So(nv, ShouldEqual, 19999)
+
+				// Increment (namespaced).
+				nv, err = mc.IncrementExisting(oc, "foo", 1)
+				So(err, ShouldBeNil)
+				So(nv, ShouldEqual, 20000)
+
+				// Increment (non-namespaced).
+				nv, err = mc.IncrementExisting(c, "foo", 1)
+				So(err, ShouldBeNil)
+				So(nv, ShouldEqual, 102)
+			})
+		})
+
+		Convey(`Stats returns ErrNoStats`, func() {
+			_, err := mc.Stats(c)
+			So(err, ShouldEqual, mc.ErrNoStats)
+		})
+
+		Convey(`A really long key gets hashed.`, func() {
+			key := strings.Repeat("X", keyHashSizeThreshold+1)
+			item := mc.NewItem(c, key)
+			item.SetValue([]byte("ohaithere"))
+			So(mc.Add(c, item), ShouldBeNil)
+
+			// Retrieve the item. The memcache layer doesn't change the key, so we
+			// will assert this by building the hash key ourselves and asserting that
+			// it also retrieves the item.
+			hashedKey := hashBytes([]byte(key))
+			So(get(c, hashedKey), ShouldResemble, []string{"ohaithere"})
+		})
+
+		Convey(`Items will expire (sleeps 1 second).`, func() {
+			item := mc.NewItem(c, "foo").SetValue([]byte("FOO")).SetExpiration(1 * time.Second)
+			So(mc.Add(c, item), ShouldBeNil)
+
+			// Immediate Get.
+			So(mc.Get(c, item), ShouldBeNil)
+			So(item.Value(), ShouldResemble, []byte("FOO"))
+
+			// Expire.
+			time.Sleep(1 * time.Second)
+			So(mc.Get(c, item), ShouldEqual, mc.ErrCacheMiss)
+		})
+	})
+}