server/config: Generic caching backend.

server/config/caching/config.go

+// Copyright 2015 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+package caching
+
+import (
+        "fmt"
+        "net/url"
+
+        "github.com/luci/luci-go/common/errors"
+        log "github.com/luci/luci-go/common/logging"
+        "github.com/luci/luci-go/server/config"
+
+        "golang.org/x/net/context"
+)
+
+// Schema is the current package's cache schema.
+const Schema = "v2"
+
+// Operation is a cache entry operation. Cache entries are all stored in the
+// same object, with different parameters filled based on the operation that
+// they represent.
+type Operation string
+
+const (
+        // OpGet is the Get operation.
+        OpGet = Operation("Get")
+        // OpGetAll is the GetAll operation.
+        OpGetAll = Operation("GetAll")
+        // OpConfigSetURL is the ConfigSetURL operation.
+        OpConfigSetURL = Operation("ConfigSetURL")
+)
+
+// Key is a cache key.
+type Key struct {
+        // Schema is the schema for this key/value. If schema changes in a backwards-
+        // incompatible way, this must also change.
+        Schema string `json:"s,omitempty"`
+
+        // Authority is the config authority to use.
+        Authority config.Authority `json:"a,omitempty"`
+
+        // Op is the operation that is being cached.
+        Op Operation `json:"op,omitempty"`
+
+        // Content is true if this request asked for content.
+        Content bool `json:"c,omitempty"`
+
+        // Format is the requesting Key's Format parameter.
+        Format string `json:"f,omitempty"`
+        // FormatData is the requesting Key's FormatData parameter.
+        FormatData string `json:"fd,omitempty"`
+
+        // ConfigSet is the config set parameter. This is valid for "OpGet".
+        ConfigSet string `json:"cs,omitempty"`
+        // Path is the path parameters. This is valid for "OpGet" and "OpGetAll".
+        Path string `json:"p,omitempty"`
+
+        // GetAllType is the "GetAll" operation type. This is valid for "OpGetAll".
+        GetAllType config.GetAllType `json:"gat,omitempty"`
+}
+
+// ParamHash returns a deterministic hash of all of the key parameters.
+func (k *Key) ParamHash() []byte {
+        cstr := ""
+        if k.Content {
+                cstr = "y"
+        }
+        return HashParams(k.Schema, string(k.Authority), string(k.Op), cstr, k.Format, k.FormatData,
+                k.ConfigSet, k.Path, string(k.GetAllType))
+}
+
+// String prints a text representation of the key. No effort is made to ensure
+// that this representation is consistent or deterministic, and it is not
+// bound to the cache schema.
+func (k *Key) String() string { return fmt.Sprintf("CacheKey[%#v]", k) }
+
+// Params returns the config.Params that are encoded in this Key.
+func (k *Key) Params() config.Params {
+        return config.Params{
+                Content:    k.Content,
+                Authority:  k.Authority,
+                Format:     k.Format,
+                FormatData: k.FormatData,
+        }
+}
+
+// ValueItem is a cache-optimized config.Item projection.

[iannucci, 2017/01/07 20:53:17]

I would explain that this means that it stores the result of being transformed
by the format operation, with a link to that chunk of docs.

[dnj, 2017/01/10 03:29:17]

Done.

+type ValueItem struct {
+        ConfigSet string `json:"cs,omitempty"`
+        Path      string `json:"p,omitempty"`
+
+        ContentHash string `json:"ch,omitempty"`
+        Revision    string `json:"r,omitempty"`
+
+        Content []byte `json:"c,omitempty"`
+
+        Format     string `json:"f,omitempty"`
+        FormatData []byte `json:"fd,omitempty"`

[iannucci, 2017/01/07 20:53:16]

why is formatdata needed again? couldn't it just be stored in Content with
cmpbin?

[dnj, 2017/01/10 03:29:17]

b/c we have a single registered Formatter instance (TextProto) that then has to
know *which* textproto to unmarshal into. Could combine, but I think this is
simpler. FormatData is welcome to be structured, but doesn't need to be ATM with
this scheme.

+}
+
+// MakeValueItem builds a caching ValueItem from a config.Item.
+func MakeValueItem(it *config.Item) ValueItem {
+        return ValueItem{
+                ConfigSet:   it.ConfigSet,
+                Path:        it.Path,
+                ContentHash: it.ContentHash,
+                Revision:    it.Revision,
+                Content:     []byte(it.Content),
+                Format:      it.Format,
+                FormatData:  []byte(it.FormatData),
+        }
+}
+
+// ConfigItem returns the config.Item equivalent of vi.
+func (vi *ValueItem) ConfigItem() *config.Item {
+        return &config.Item{
+                Meta: config.Meta{
+                        ConfigSet:   vi.ConfigSet,
+                        Path:        vi.Path,
+                        ContentHash: vi.ContentHash,
+                        Revision:    vi.Revision,
+                },
+                Content:    string(vi.Content),
+                Format:     vi.Format,
+                FormatData: string(vi.FormatData),
+        }
+}
+
+// Value is a cache value.
+type Value struct {
+        // Items is the cached set of config response items.
+        //
+        // For Get, this will either be empty (cached miss) or have a single Item
+        // in it (cache hit).
+        Items []ValueItem `json:"i,omitempty"`
+
+        // URL is a URL string.
+        //
+        // Used with GetConfigSetURL.
+        URL string `json:"u,omitempty"`
+}
+
+// LoadItems loads a set of config.Item into v's Items field. If items is nil,
+// v.Items will be nil.
+func (v *Value) LoadItems(items ...*config.Item) {
+        if len(items) == 0 {
+                v.Items = nil
+                return
+        }
+
+        v.Items = make([]ValueItem, len(items))
+        for i, it := range items {
+                v.Items[i] = MakeValueItem(it)
+        }
+}
+
+// SingleItem returns the first config.Item in v's Items slice. If the Items
+// slice is empty, SingleItem will return nil.
+func (v *Value) SingleItem() *config.Item {
+        if len(v.Items) == 0 {
+                return nil
+        }
+        return v.Items[0].ConfigItem()
+}
+
+// ConfigItems returns the config.Item projection of v's Items slice.
+func (v *Value) ConfigItems() []*config.Item {
+        if len(v.Items) == 0 {
+                return nil
+        }
+
+        res := make([]*config.Item, len(v.Items))
+        for i := range v.Items {
+                res[i] = v.Items[i].ConfigItem()
+        }
+        return res
+}
+
+// DecodeValue loads a Value from is encoded representation.
+func DecodeValue(d []byte) (*Value, error) {
+        var v Value
+        if err := Decode(d, &v); err != nil {
+                return nil, errors.Annotate(err).Err()
+        }
+        return &v, nil
+}
+
+// Encode encodes this Valu.

[iannucci, 2017/01/07 20:53:16]

Value

[dnj, 2017/01/10 03:29:17]

Cool kids say "valu".

+//
+// This is offered for convenience, but caches aren't required to use this
+// encoding.
+//
+// The format stores the Value as compressed JSON.
+func (v *Value) Encode() ([]byte, error) { return Encode(v) }
+
+// Loader retrieves a Value by consulting the backing backend.
+//
+// The input Value is the current cached Value, or nil if there is no current
+// cached Value. The output Value is the cached Value, if one exists. It is
+// acceptable to return mutate "v" and/or return it as the output Value.
+type Loader func(context.Context, Key, *Value) (*Value, error)
+
+// Backend is a config.Backend implementation that caches responses.
+//
+// All cached values are full-content regardless of whether or not full content
+// was requested.
+//
+// Backend caches content and no-content requests as separate cache entries.
+// This enables one cache to do low-overhead updating against another cache
+// implementation.
+type Backend struct {
+        // Backend is the backing Backend.
+        config.Backend
+
+        // HardFailure, if true, means that a failure to retrieve a cached item will
+        // be propagated. If false, a cache error will result in a fall-through to
+        // Backend.
+        HardFailure bool

[iannucci, 2017/01/07 20:53:16]

I would explain that you'd set this to true if the 'fallback' is unacceptably
slow for your application.

I would also maybe call it FailFast or something like that.
https://en.wikipedia.org/wiki/Fail-fast

[dnj, 2017/01/10 03:29:17]

I'll rename to FailOnError. "Fail fast" means something different here, I think.
Namely, FailOnError says, "expect to work and die if you don't", and "fail fast"
is done with an intention of not propagating failures downstream, which isn't
what would happen here.

+
+        // CacheGet retrieves the cached value associated with key. If no such cached
+        // value exists, CacheGet is responsible for resolving the cache value using
+        // the supplied Loaer.

[iannucci, 2017/01/07 20:53:16]

Loader

[dnj, 2017/01/10 03:29:17]

Done.

+        CacheGet func(context.Context, Key, Loader) (*Value, error)
+}
+
+// Get implements config.Backend.
+func (b *Backend) Get(c context.Context, configSet, path string, p config.Params) (*config.Item, error) {
+        key := Key{
+                Schema:     Schema,
+                Authority:  p.Authority,
+                Op:         OpGet,
+                Content:    p.Content,
+                ConfigSet:  configSet,
+                Path:       path,
+                Format:     p.Format,
+                FormatData: p.FormatData,
+        }
+        value, err := b.CacheGet(c, key, b.loader)
+        if err != nil {
+                if b.HardFailure {
+                        log.Fields{
+                                log.ErrorKey: err,
+                                "authority":  p.Authority,
+                                "configSet":  configSet,
+                                "path":       path,
+                        }.Errorf(c, "(Hard Failure) failed to load cache value.")
+                        return nil, errors.Annotate(err).Err()
+                }
+
+                log.Fields{
+                        log.ErrorKey: err,
+                        "authority":  p.Authority,
+                        "configSet":  configSet,
+                        "path":       path,
+                }.Warningf(c, "Failed to load cache value.")
+                return b.Backend.Get(c, configSet, path, p)
+        }
+
+        it := value.SingleItem()
+        if it == nil {
+                // Sentinel for no config.
+                return nil, config.ErrNoConfig
+        }
+        return it, nil
+}
+
+// GetAll implements config.Backend.
+func (b *Backend) GetAll(c context.Context, t config.GetAllType, path string, p config.Params) ([]*config.Item, error) {
+        key := Key{
+                Schema:     Schema,
+                Authority:  p.Authority,
+                Op:         OpGetAll,
+                Content:    p.Content,
+                Path:       path,
+                GetAllType: t,
+                Format:     p.Format,
+                FormatData: p.FormatData,
+        }
+        value, err := b.CacheGet(c, key, b.loader)
+        if err != nil {
+                if b.HardFailure {
+                        log.Fields{
+                                log.ErrorKey: err,
+                                "authority":  p.Authority,
+                                "type":       t,
+                                "path":       path,
+                        }.Errorf(c, "(Hard Failure) failed to load cache value.")
+                        return nil, errors.Annotate(err).Err()
+                }
+
+                log.Fields{
+                        log.ErrorKey: err,
+                        "authority":  p.Authority,
+                        "type":       t,
+                        "path":       path,
+                }.Warningf(c, "Failed to load cache value.")
+                return b.Backend.GetAll(c, t, path, p)
+        }
+        return value.ConfigItems(), nil
+}
+
+// ConfigSetURL implements config.Backend.
+func (b *Backend) ConfigSetURL(c context.Context, a config.Authority, configSet string) (u url.URL, err error) {
+        key := Key{
+                Schema:    Schema,
+                Authority: a,
+                Op:        OpConfigSetURL,
+                ConfigSet: configSet,
+        }
+
+        var value *Value
+        if value, err = b.CacheGet(c, key, b.loader); err != nil {
+                if b.HardFailure {
+                        log.Fields{
+                                log.ErrorKey: err,
+                                "authority":  a,
+                                "configSet":  configSet,
+                        }.Errorf(c, "(Hard Failure) failed to load cache value.")
+                        err = errors.Annotate(err).Err()
+                        return
+                }
+
+                log.Fields{
+                        log.ErrorKey: err,
+                        "authority":  a,
+                        "configSet":  configSet,
+                }.Warningf(c, "Failed to load cache value.")
+                return b.Backend.ConfigSetURL(c, a, configSet)
+        }
+
+        if value.URL == "" {
+                // Sentinel for no config.
+                err = config.ErrNoConfig
+                return
+        }
+
+        up, err := url.Parse(value.URL)
+        if err != nil {
+                err = errors.Annotate(err).Reason("failed to parse cached URL: %(value)q").D("value", value.URL).Err()
+                return
+        }
+
+        u = *up
+        return
+}
+
+// loader runs a cache get against the configured Base backend.
+//
+// This should be used by caches that do not have the cached value.
+func (b *Backend) loader(c context.Context, k Key, v *Value) (*Value, error) {
+        return CacheLoad(c, b.Backend, k, v)
+}
+
+// CacheLoad loads k from backend b.
+//
+// If an existing cache value is known, it should be supplied as v. Otherwise,

[iannucci, 2017/01/07 20:53:17]

It would be good to explain how the provided v will be used... is it going to be
saved to the cache? Or something else?

+// v should be nil.
+//
+// This is effectively a Loader function that is detached from a given cache
+// instance.
+func CacheLoad(c context.Context, b config.Backend, k Key, v *Value) (rv *Value, err error) {
+        switch k.Op {
+        case OpGet:
+                rv, err = doGet(c, b, k.ConfigSet, k.Path, v, k.Params())
+        case OpGetAll:
+                rv, err = doGetAll(c, b, k.GetAllType, k.Path, v, k.Params())
+        case OpConfigSetURL:
+                rv, err = doConfigSetURL(c, b, k.Authority, k.ConfigSet)
+        default:
+                return nil, errors.Reason("unknown operation: %(op)v").D("op", k.Op).Err()
+        }
+        if err != nil {
+                return nil, err
+        }
+        return
+}
+
+func doGet(c context.Context, b config.Backend, configSet, path string, v *Value, p config.Params) (*Value, error) {
+        hadItem := (v != nil && len(v.Items) > 0)
+        if !hadItem {
+                // Initialize empty "v".
+                v = &Value{}
+        }
+
+        // If we have a current item, or if we are requesting "no-content", then
+        // perform a "no-content" lookup.
+        if hadItem || !p.Content {
+                noContentP := p
+                noContentP.Content = false
+
+                item, err := b.Get(c, configSet, path, noContentP)
+                switch err {
+                case nil:
+                        if hadItem && (item.ContentHash == v.Items[0].ContentHash) {
+                                // Nothing changed.
+                                return v, nil
+                        }
+
+                        // If our "Get" is, itself, no-content, then this is our actual result.
+                        if !p.Content {
+                                v.LoadItems(item)
+                                return v, nil
+                        }
+
+                        // If we get here, we are requesting full-content and our hash check
+                        // showed that the current content has a different hash.
+                        break
+
+                case config.ErrNoConfig:
+                        v.LoadItems()
+                        return v, nil
+
+                default:
+                        return nil, errors.Annotate(err).Err()
+                }
+        }
+
+        // Perform a full content request.
+        switch item, err := b.Get(c, configSet, path, p); err {
+        case nil:
+                v.LoadItems(item)
+                return v, nil
+
+        case config.ErrNoConfig:
+                // Empty "config missing" item.
+                v.LoadItems()
+                return v, nil
+
+        default:
+                return nil, errors.Annotate(err).Err()
+        }
+}
+
+func doGetAll(c context.Context, b config.Backend, t config.GetAllType, path string, v *Value, p config.Params) (*Value, error) {
+        // If we already have a cached value, or if we're requesting no-content, do a
+        // no-content refresh to see if anything has changed.
+        //
+        // Response values are in order, so this is a simple traversal.
+        if v != nil || !p.Content {
+                noContentP := p
+                noContentP.Content = false
+
+                items, err := b.GetAll(c, t, path, noContentP)
+                if err != nil {
+                        return nil, errors.Annotate(err).Reason("failed RPC (hash-only)").Err()
+                }
+
+                // If we already have a cached item, validate it.
+                if v != nil && len(items) == len(v.Items) {
+                        match := true
+                        for i, other := range items {
+                                cur := v.Items[i]
+                                if cur.ConfigSet == other.ConfigSet && cur.Path == other.Path &&
+                                        cur.ContentHash == other.ContentHash {
+                                        continue
+                                }
+
+                                match = false
+                                break
+                        }
+
+                        // If all configs match, our response hasn't changed.
+                        if match {
+                                return v, nil
+                        }
+                }
+
+                // If we requested no-content, then this is our result.
+                if !p.Content {
+                        var retV Value
+                        retV.LoadItems(items...)
+                        return &retV, nil
+                }
+        }
+
+        // Perform a full-content request.
+        items, err := b.GetAll(c, t, path, p)
+        if err != nil {
+                return nil, errors.Annotate(err).Err()
+        }
+        var retV Value
+        retV.LoadItems(items...)
+        return &retV, nil
+}
+
+func doConfigSetURL(c context.Context, b config.Backend, a config.Authority, configSet string) (*Value, error) {
+        u, err := b.ConfigSetURL(c, a, configSet)
+        switch err {
+        case nil:
+                return &Value{
+                        URL: u.String(),
+                }, nil
+
+        case config.ErrNoConfig:
+                return &Value{}, nil
+
+        default:
+                return nil, err
+        }
+}