A transparent cache for datastore, backed by memcache.

filter/dscache/dscache.go

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package dscache
+
+import (
+        "bytes"
+        "crypto/sha1"
+        "encoding/base64"
+        "fmt"
+
+        ds "github.com/luci/gae/service/datastore"
+)
+
+var (
+        // LockTimeSeconds is the number of seconds that a "lock" memcache entry will
+        // have its expiration set to. It's set to just over half of the frontend
+        // request handler timeout (currently 60 seconds).
+        LockTimeSeconds = 31
+
+        // CacheTimeSeconds is the default number of seconds that a positively cached
+        // entity will be retained (memcache contention notwithstanding). A value of
+        // 0 is infinite.
+        CacheTimeSeconds int64 = 0

[Vadim Sh., 2015/08/06 01:23:34]

I vote for "1 day" or something. To keep memcache clean and tidy. I don't think
1 day vs infinite makes much difference performance-wise.

[iannucci, 2015/08/06 02:37:33]

Sure. Done.

+
+        // CompressionThreshold is the number of bytes of entity value after which
+        // compression kicks in.
+        CompressionThreshold = 860
+
+        // DefaultShards is the default number of key sharding to do.
+        DefaultShards int = 1
+
+        // DefaultEnable indicates whether or not caching is globally enabled or
+        // disabled by default. Can still be overridden by CacheEnableMeta.
+        DefaultEnabled = true
+)
+
+const (
+        MemcacheVersion = "1"

[dnj, 2015/08/05 18:32:18]

Do all of these need to be exported?

[iannucci, 2015/08/06 01:54:01]

no :D but I like them to be :P

+
+        // KeyFormat is the format string used to generate memcache keys. It's
+        //   gae:<version>:<shard#>:<base64_std_nopad(sha1(datastore.Key))>
+        KeyFormat      = "gae:" + MemcacheVersion + ":%x:%s"
+        Sha1B64Padding = 1
+        Sha1B64Size    = 28 - Sha1B64Padding
+
+        MaxShards          = 256
+        MaxShardsLen       = len("ff")
+        InternalGAEPadding = 96
+        ValueSizeLimit     = (1000 * 1000) - InternalGAEPadding - MaxShardsLen
+
+        CacheEnableMeta     = "dscache.enable"
+        CacheExpirationMeta = "dscache.expiration"
+
+        // NonceUint32s is the number of 32 bit uints to use in the 'lock' nonce.
+        NonceUint32s = 2
+)
+
+// internalValueSizeLimit is a var for testing purposes.
+var internalValueSizeLimit = ValueSizeLimit
+
+type CompressionType byte
+
+const (
+        NoCompression CompressionType = iota
+        ZlibCompression
+)
+
+func (c CompressionType) String() string {
+        switch c {
+        case NoCompression:
+                return "NoCompression"
+        case ZlibCompression:
+                return "ZlibCompression"
+        default:
+                return fmt.Sprintf("UNKNOWN_CompressionType(%d)", c)
+        }
+}
+
+// FlagValue is used to indicate if a memcache entry currently contains an
+// item or a lock.
+type FlagValue uint32
+
+const (
+        ItemUKNONWN FlagValue = iota
+        ItemHasData
+        ItemHasLock
+)
+
+func MakeMemcacheKey(shard int, k ds.Key) string {
+        return fmt.Sprintf(KeyFormat, shard, HashKey(k))
+}
+
+func HashKey(k ds.Key) string {
+        // errs can't happen, since we're using a byte buffer.
+        buf := bytes.Buffer{}
+        _ = ds.WriteKey(&buf, ds.WithoutContext, k)
+        dgst := sha1.Sum(buf.Bytes())
+        buf.Reset()
+        enc := base64.NewEncoder(base64.StdEncoding, &buf)
+        _, _ = enc.Write(dgst[:])
+        enc.Close()
+        return buf.String()[:buf.Len()-Sha1B64Padding]
+}
+
+// TODO(riannucci): Should there be a way to purge the cache entries for a range
+// of datastore keys?

[Vadim Sh., 2015/08/06 01:23:34]

why?

[iannucci, 2015/08/06 02:37:33]

Not sure... that's why a todo question :D

The thought was to possibly manually evict some, but not all, of the cache.
Though I think that just flushing the entire memcache is probably better/safer.

Removed