server/config: Add text protobuf support.

server/config/textproto/resolver.go

Index: server/config/textproto/resolver.go
diff --git a/server/config/textproto/resolver.go b/server/config/textproto/resolver.go
new file mode 100644
index 0000000000000000000000000000000000000000..ea71cbf4f30f453585ef0d972635b1bb67a4a347
--- /dev/null
+++ b/server/config/textproto/resolver.go
@@ -0,0 +1,244 @@
+// Copyright 2016 The LUCI Authors. All rights reserved.
+// Use of this source code is governed under the Apache License, Version 2.0
+// that can be found in the LICENSE file.
+
+// Package textproto implements a textproto config service Resolver.

[iannucci, 2017/01/07 20:22:11]

I would mention that it actually stores the binary proto. Something like:

"This takes config files in textproto format, and caches them in binary proto
format."

[dnj, 2017/01/10 03:26:37]

Done.

+//
+// This has some advantages over raw text content caching:
+//	- More space-efficient.
+//	- Decodes faster.
+//	- If the config service protobuf schema differs from the application's
+//	  compiled schema, and the schema change is responsible (adding, renaming,
+//	  repurposing) the binary cache value will continue to load where the text
+//	  protobuf would fail.
+//
+// Additionally, it uses the "luci-go" text protobuf multi-line extension. For
+// more information, see: github.com/luci/luci-go/common/proto
+package textproto
+
+import (
+	"bytes"
+	"reflect"
+	"strings"
+
+	"github.com/luci/luci-go/common/data/cmpbin"
+	"github.com/luci/luci-go/common/errors"
+	luciProto "github.com/luci/luci-go/common/proto"
+	"github.com/luci/luci-go/server/config"
+
+	"github.com/golang/protobuf/proto"
+)
+
+var typeOfProtoMessage = reflect.TypeOf((*proto.Message)(nil)).Elem()

[iannucci, 2017/01/07 20:22:11]

really dumb that the proto package doesn't have this as an exported var.

[dnj, 2017/01/10 03:26:37]

Acknowledged.

+
+// BinaryFormat is the resolver's binary protobuf format string.

[iannucci, 2017/01/07 20:22:11]

I would make sure that Format is documented so that it suggests this format. In
fact, if Format is a type, maybe it could be `struct { package, subformat string
}` ?

[dnj, 2017/01/10 03:26:37]

The "subformat" part is variable, dependent on the message type, so this can't
be constant. But otherwise yes.

+const BinaryFormat = "github.com/luci/luci-go/server/config/TextProto:binary"
+
+// Message is a config.Resolver that resolves config data into proto.Message
+// instances by parsing the config data as a text protobuf.
+func Message(out proto.Message) interface {
+	config.Resolver
+	config.FormattingResolver
+} {
+	if out == nil {
+		panic("cannot pass a nil proto.Message instance")
+	}
+	return &resolver{
+		messageName: proto.MessageName(out),
+		single:      out,
+		singleType:  reflect.TypeOf(out),
+	}
+}
+
+// Slice is a config.MultiResolver which resolves a slice of configurations
+// into a TextProto.
+//
+// out must be a pointer to a slice of some proto.Message implementation. If it
+// isn't, this function will panic.
+//
+// For example:
+//
+//	var out []*MyProtoMessages
+//	TextProtoSlice(&out)
+func Slice(out interface{}) interface {
+	config.MultiResolver
+	config.FormattingResolver
+} {
+	r := resolver{}
+
+	v := reflect.ValueOf(out)
+	if !r.loadMulti(v) {
+		panic(errors.Reason("%(type)s is not a pointer to a slice of protobuf message types").D("type", v.Type()).Err())
+	}
+	return &r
+}
+
+type resolver struct {
+	messageName string
+
+	single     proto.Message
+	singleType reflect.Type
+
+	multiDest reflect.Value
+}
+
+func (r *resolver) loadMulti(v reflect.Value) bool {
+	t := v.Type()
+	if t.Kind() != reflect.Ptr {
+		return false
+	}
+	valElem := t.Elem()
+	if valElem.Kind() != reflect.Slice {
+		return false
+	}
+	sliceContent := valElem.Elem()
+	if sliceContent.Kind() != reflect.Ptr {
+		return false
+	}
+	if !sliceContent.Implements(typeOfProtoMessage) {
+		return false
+	}
+
+	r.messageName = proto.MessageName(archetypeMessage(sliceContent))
+	r.singleType = sliceContent
+	r.multiDest = v.Elem()
+	return true
+}
+
+// Resolve implements config.MultiResolver.
+func (r *resolver) Resolve(it *config.Item) error {
+	return r.resolveItem(r.single, it.Content, it.Format)
+}
+
+// PrepareMulti implements config.MultiResolver.
+func (r *resolver) PrepareMulti(size int) {
+	slice := reflect.MakeSlice(r.multiDest.Type(), size, size)
+	r.multiDest.Set(slice)
+}
+
+// ResolveItemAt implements config.MultiResolver.
+func (r *resolver) ResolveItemAt(i int, it *config.Item) error {
+	msgV := archetypeInstance(r.singleType)
+	if err := r.resolveItem(msgV.Interface().(proto.Message), it.Content, it.Format); err != nil {
+		return err
+	}
+	r.multiDest.Index(i).Set(msgV)
+	return nil
+}
+
+func (r *resolver) resolveItem(out proto.Message, content string, format string) error {
+	switch format {
+	case "":
+		// Not formatted (text protobuf).
+		if err := luciProto.UnmarshalTextML(content, out); err != nil {
+			return errors.Annotate(err).Reason("failed to unmarshal text protobuf").Err()
+		}
+		return nil
+
+	case BinaryFormat:
+		if err := parseBinaryContent(content, out); err != nil {
+			return errors.Annotate(err).Reason("failed to unmarshal binary protobuf").Err()
+		}
+		return nil
+
+	default:
+		return errors.Reason("unsupported content format: %(format)q").D("format", format).Err()
+	}
+}
+
+func (r *resolver) Format() (string, string) { return BinaryFormat, r.messageName }
+
+// RegisterFormatter registers the textproto Formatter with fr.
+func RegisterFormatter(fr *config.FormatterRegistry) {
+	fr.Register(BinaryFormat, &Formatter{})
+}
+
+// Formatter is a config.Formatter implementation bound to a specific
+// protobuf message.
+//
+// It takes a text protobuf representation of that message as input and returns
+// a binary protobuf representation as output.
+type Formatter struct{}
+
+// FormatItem implements config.Formatter.
+func (f *Formatter) FormatItem(c, fd string) (string, error) {
+	archetype := proto.MessageType(fd)
+	if archetype == nil {
+		return "", errors.Reason("unknown proto.Message type %(type)q in formatter data").
+			D("type", fd).Err()
+	}
+	msg := archetypeMessage(archetype)
+
+	// Convert from config to protobuf.
+	if err := luciProto.UnmarshalTextML(c, msg); err != nil {
+		return "", errors.Annotate(err).Reason("failed to unmarshal text protobuf content").Err()
+	}
+
+	// Binary format.
+	bc, err := makeBinaryContent(msg)
+	if err != nil {
+		return "", errors.Annotate(err).Err()
+	}
+	return bc, nil
+}
+
+// t is a pointer to a proto.Message instance.
+func archetypeInstance(t reflect.Type) reflect.Value {
+	return reflect.New(t.Elem())
+}
+
+func archetypeMessage(t reflect.Type) proto.Message {
+	return archetypeInstance(t).Interface().(proto.Message)
+}
+
+// makeBinaryContent constructs a binary content string from text protobuf
+// content.
+//
+// The binary content is formatted by concatenating two "cmpbin" binary values
+// together:
+// [Message Name] | [Marshaled Message Data]
+func makeBinaryContent(msg proto.Message) (string, error) {
+	d, err := proto.Marshal(msg)
+	if err != nil {
+		return "", errors.Annotate(err).Reason("failed to marshal message").Err()
+	}
+
+	var buf bytes.Buffer
+	if _, err := cmpbin.WriteString(&buf, proto.MessageName(msg)); err != nil {
+		return "", errors.Annotate(err).Reason("failed to write message name").Err()
+	}
+	if _, err := cmpbin.WriteBytes(&buf, d); err != nil {
+		return "", errors.Annotate(err).Reason("failed to write binary message content").Err()
+	}
+	return buf.String(), nil
+}
+
+// parseBinaryContent parses a binary content string, pulling out the message
+// type and marshalled message data. It then unmarshals the specified type into
+// a new message based on the archetype.
+//
+// If the binary content's declared type doesn't match the archetype, or if the
+// binary content is invalid, an error will be returned.
+func parseBinaryContent(v string, msg proto.Message) error {
+	r := strings.NewReader(v)
+	encName, _, err := cmpbin.ReadString(r)
+	if err != nil {
+		return errors.Annotate(err).Reason("failed to read message name").Err()
+	}
+
+	// Construct a message for this.
+	if name := proto.MessageName(msg); name != encName {
+		return errors.Reason("message name %(name)q doesn't match encoded name %(enc)q").
+			D("name", name).D("enc", encName).Err()
+	}
+
+	// We have the right message, unmarshal.
+	d, _, err := cmpbin.ReadBytes(r)
+	if err != nil {
+		return errors.Annotate(err).Reason("failed to read binary message content").Err()
+	}
+	if err := proto.Unmarshal(d, msg); err != nil {
+		return errors.Annotate(err).Reason("failed to unmarshal message").Err()
+	}
+	return nil
+}