server/config: Add text protobuf support.

server/config/textproto/resolver.go

+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+// Package textproto implements a textproto config service Resolver.

[iannucci, 2017/01/07 20:22:11]

I would mention that it actually stores the binary proto. Something like:

"This takes config files in textproto format, and caches them in binary proto
format."

[dnj, 2017/01/10 03:26:37]

Done.

+//
+// This has some advantages over raw text content caching:
+//      - More space-efficient.
+//      - Decodes faster.
+//      - If the config service protobuf schema differs from the application's
+//        compiled schema, and the schema change is responsible (adding, renaming,
+//        repurposing) the binary cache value will continue to load where the text
+//        protobuf would fail.
+//
+// Additionally, it uses the "luci-go" text protobuf multi-line extension. For
+// more information, see: github.com/luci/luci-go/common/proto
+package textproto
+
+import (
+        "bytes"
+        "reflect"
+        "strings"
+
+        "github.com/luci/luci-go/common/data/cmpbin"
+        "github.com/luci/luci-go/common/errors"
+        luciProto "github.com/luci/luci-go/common/proto"
+        "github.com/luci/luci-go/server/config"
+
+        "github.com/golang/protobuf/proto"
+)
+
+var typeOfProtoMessage = reflect.TypeOf((*proto.Message)(nil)).Elem()

[iannucci, 2017/01/07 20:22:11]

really dumb that the proto package doesn't have this as an exported var.

[dnj, 2017/01/10 03:26:37]

Acknowledged.

+
+// BinaryFormat is the resolver's binary protobuf format string.

[iannucci, 2017/01/07 20:22:11]

I would make sure that Format is documented so that it suggests this format. In
fact, if Format is a type, maybe it could be `struct { package, subformat string
}` ?

[dnj, 2017/01/10 03:26:37]

The "subformat" part is variable, dependent on the message type, so this can't
be constant. But otherwise yes.

+const BinaryFormat = "github.com/luci/luci-go/server/config/TextProto:binary"
+
+// Message is a config.Resolver that resolves config data into proto.Message
+// instances by parsing the config data as a text protobuf.
+func Message(out proto.Message) interface {
+        config.Resolver
+        config.FormattingResolver
+} {
+        if out == nil {
+                panic("cannot pass a nil proto.Message instance")
+        }
+        return &resolver{
+                messageName: proto.MessageName(out),
+                single:      out,
+                singleType:  reflect.TypeOf(out),
+        }
+}
+
+// Slice is a config.MultiResolver which resolves a slice of configurations
+// into a TextProto.
+//
+// out must be a pointer to a slice of some proto.Message implementation. If it
+// isn't, this function will panic.
+//
+// For example:
+//
+//      var out []*MyProtoMessages
+//      TextProtoSlice(&out)
+func Slice(out interface{}) interface {
+        config.MultiResolver
+        config.FormattingResolver
+} {
+        r := resolver{}
+
+        v := reflect.ValueOf(out)
+        if !r.loadMulti(v) {
+                panic(errors.Reason("%(type)s is not a pointer to a slice of protobuf message types").D("type", v.Type()).Err())
+        }
+        return &r
+}
+
+type resolver struct {
+        messageName string
+
+        single     proto.Message
+        singleType reflect.Type
+
+        multiDest reflect.Value
+}
+
+func (r *resolver) loadMulti(v reflect.Value) bool {
+        t := v.Type()
+        if t.Kind() != reflect.Ptr {
+                return false
+        }
+        valElem := t.Elem()
+        if valElem.Kind() != reflect.Slice {
+                return false
+        }
+        sliceContent := valElem.Elem()
+        if sliceContent.Kind() != reflect.Ptr {
+                return false
+        }
+        if !sliceContent.Implements(typeOfProtoMessage) {
+                return false
+        }
+
+        r.messageName = proto.MessageName(archetypeMessage(sliceContent))
+        r.singleType = sliceContent
+        r.multiDest = v.Elem()
+        return true
+}
+
+// Resolve implements config.MultiResolver.
+func (r *resolver) Resolve(it *config.Item) error {
+        return r.resolveItem(r.single, it.Content, it.Format)
+}
+
+// PrepareMulti implements config.MultiResolver.
+func (r *resolver) PrepareMulti(size int) {
+        slice := reflect.MakeSlice(r.multiDest.Type(), size, size)
+        r.multiDest.Set(slice)
+}
+
+// ResolveItemAt implements config.MultiResolver.
+func (r *resolver) ResolveItemAt(i int, it *config.Item) error {
+        msgV := archetypeInstance(r.singleType)
+        if err := r.resolveItem(msgV.Interface().(proto.Message), it.Content, it.Format); err != nil {
+                return err
+        }
+        r.multiDest.Index(i).Set(msgV)
+        return nil
+}
+
+func (r *resolver) resolveItem(out proto.Message, content string, format string) error {
+        switch format {
+        case "":
+                // Not formatted (text protobuf).
+                if err := luciProto.UnmarshalTextML(content, out); err != nil {
+                        return errors.Annotate(err).Reason("failed to unmarshal text protobuf").Err()
+                }
+                return nil
+
+        case BinaryFormat:
+                if err := parseBinaryContent(content, out); err != nil {
+                        return errors.Annotate(err).Reason("failed to unmarshal binary protobuf").Err()
+                }
+                return nil
+
+        default:
+                return errors.Reason("unsupported content format: %(format)q").D("format", format).Err()
+        }
+}
+
+func (r *resolver) Format() (string, string) { return BinaryFormat, r.messageName }
+
+// RegisterFormatter registers the textproto Formatter with fr.
+func RegisterFormatter(fr *config.FormatterRegistry) {
+        fr.Register(BinaryFormat, &Formatter{})
+}
+
+// Formatter is a config.Formatter implementation bound to a specific
+// protobuf message.
+//
+// It takes a text protobuf representation of that message as input and returns
+// a binary protobuf representation as output.
+type Formatter struct{}
+
+// FormatItem implements config.Formatter.
+func (f *Formatter) FormatItem(c, fd string) (string, error) {
+        archetype := proto.MessageType(fd)
+        if archetype == nil {
+                return "", errors.Reason("unknown proto.Message type %(type)q in formatter data").
+                        D("type", fd).Err()
+        }
+        msg := archetypeMessage(archetype)
+
+        // Convert from config to protobuf.
+        if err := luciProto.UnmarshalTextML(c, msg); err != nil {
+                return "", errors.Annotate(err).Reason("failed to unmarshal text protobuf content").Err()
+        }
+
+        // Binary format.
+        bc, err := makeBinaryContent(msg)
+        if err != nil {
+                return "", errors.Annotate(err).Err()
+        }
+        return bc, nil
+}
+
+// t is a pointer to a proto.Message instance.
+func archetypeInstance(t reflect.Type) reflect.Value {
+        return reflect.New(t.Elem())
+}
+
+func archetypeMessage(t reflect.Type) proto.Message {
+        return archetypeInstance(t).Interface().(proto.Message)
+}
+
+// makeBinaryContent constructs a binary content string from text protobuf
+// content.
+//
+// The binary content is formatted by concatenating two "cmpbin" binary values
+// together:
+// [Message Name] | [Marshaled Message Data]
+func makeBinaryContent(msg proto.Message) (string, error) {
+        d, err := proto.Marshal(msg)
+        if err != nil {
+                return "", errors.Annotate(err).Reason("failed to marshal message").Err()
+        }
+
+        var buf bytes.Buffer
+        if _, err := cmpbin.WriteString(&buf, proto.MessageName(msg)); err != nil {
+                return "", errors.Annotate(err).Reason("failed to write message name").Err()
+        }
+        if _, err := cmpbin.WriteBytes(&buf, d); err != nil {
+                return "", errors.Annotate(err).Reason("failed to write binary message content").Err()
+        }
+        return buf.String(), nil
+}
+
+// parseBinaryContent parses a binary content string, pulling out the message
+// type and marshalled message data. It then unmarshals the specified type into
+// a new message based on the archetype.
+//
+// If the binary content's declared type doesn't match the archetype, or if the
+// binary content is invalid, an error will be returned.
+func parseBinaryContent(v string, msg proto.Message) error {
+        r := strings.NewReader(v)
+        encName, _, err := cmpbin.ReadString(r)
+        if err != nil {
+                return errors.Annotate(err).Reason("failed to read message name").Err()
+        }
+
+        // Construct a message for this.
+        if name := proto.MessageName(msg); name != encName {
+                return errors.Reason("message name %(name)q doesn't match encoded name %(enc)q").
+                        D("name", name).D("enc", encName).Err()
+        }
+
+        // We have the right message, unmarshal.
+        d, _, err := cmpbin.ReadBytes(r)
+        if err != nil {
+                return errors.Annotate(err).Reason("failed to read binary message content").Err()
+        }
+        if err := proto.Unmarshal(d, msg); err != nil {
+                return errors.Annotate(err).Reason("failed to unmarshal message").Err()
+        }
+        return nil
+}