logdog: Add bundler library.

client/internal/logdog/butler/bundler/sizer_fast.go

Index: client/internal/logdog/butler/bundler/sizer_fast.go
diff --git a/client/internal/logdog/butler/bundler/sizer_fast.go b/client/internal/logdog/butler/bundler/sizer_fast.go
new file mode 100644
index 0000000000000000000000000000000000000000..97732eaa96cd6d094b811ec9dd64f1bde36f8e93
--- /dev/null
+++ b/client/internal/logdog/butler/bundler/sizer_fast.go
@@ -0,0 +1,146 @@
+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package bundler
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/luci/luci-go/common/logdog/protocol"
+	"github.com/luci/luci-go/common/logdog/protocol/protoutil"
+	"github.com/luci/luci-go/common/logdog/types"
+)
+
+var (
+	bundleEntryTagSize int
+	logEntryTagSize    int
+)
+
+func init() {
+	bundleEntryTagSize = mustCalculateTagSize(&protocol.ButlerLogBundle{}, "Entries")
+	logEntryTagSize = mustCalculateTagSize(&protocol.ButlerLogBundle_Entry{}, "Logs")
+}
+
+func mustCalculateTagSize(i interface{}, field string) int {
+	value, err := calculateTagSize(i, field)
+	if err != nil {
+		panic(err)
+	}
+	return value
+}
+
+func calculateTagSize(i interface{}, field string) (int, error) {
+	v := reflect.TypeOf(i)
+	if v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+	if v.Kind() != reflect.Struct {
+		return 0, fmt.Errorf("sizer: %s is not a struct", v)
+	}
+
+	f, ok := v.FieldByName(field)
+	if !ok {
+		return 0, fmt.Errorf("sizer: could not find field %s.%s", v, field)
+	}
+
+	tag := protobufTag(f)
+	if tag == -1 {
+		return 0, fmt.Errorf("sizer: field %s.%s has no protobuf tag", v, field)
+	}
+
+	return varintLength(uint64(tag) << 3), nil

[tandrii(chromium), 2015/08/11 14:06:08]

I guess that's how protobuf encodes stuff. Still would be nice to have a url to
the doc and maybe short explanation.

[dnj (Google), 2015/08/11 16:24:01]

Done.

+}
+
+func varintLength(val uint64) int {
+	switch {
+	case val < 0x80:
+		return 1
+	case val < 0x4000:
+		return 2
+	case val < 0x200000:
+		return 3
+	case val < 0x10000000:
+		return 4
+	case val < 0x800000000:
+		return 5
+	case val < 0x40000000000:
+		return 6
+	case val < 0x2000000000000:
+		return 7
+	case val < 0x100000000000000:
+		return 8
+	case val < 0x8000000000000000:
+		return 9
+	default:
+		// Maximum uvarint size.
+		return 10
+	}
+}
+
+func protobufTag(f reflect.StructField) int {
+	// If this field doesn't have a "protobuf" tag, ignore it.
+	value := f.Tag.Get("protobuf")
+	parts := strings.Split(value, ",")
+	if len(parts) < 2 {
+		return -1

[tandrii(chromium), 2015/08/11 14:06:08]

Special return values => who are you, a C programmer? :)
This func should return (int, error) to be more Go-like and it'd be line-neutral
change :)

[dnj (Google), 2015/08/11 16:24:01]

Done.

+	}
+	tag, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return -1
+	}
+	return tag
+}
+
+type fastSizer struct {
+	size int
+
+	// As we add LogEntry to a ButlerLogBundle_Entry, the bundle entry's size (and
+	// therefore it's protobuf size prefix) will grow. We account for the growth
+	// by tracking the size of each ButlerLogBundle_Entry and updating it when
+	// we add a LogEntry to it. This is stored independently from the cumulative
+	// size and factored in when Size() is calculated.
+	beSize map[types.StreamPath]int
+}
+
+// NewFastSizer is a Sizer that is optimized for LogDog protobufs.

[tandrii(chromium), 2015/08/11 14:06:08]

nit: i prefer to put public stuff at the top

[dnj (Google), 2015/08/11 16:24:01]

Done.

+//
+// In exchange for rapid size calculation, it performs worst-case estimates on
+// the unknown protocol overheads, leading to potential size overestimation.
+func NewFastSizer(b *protocol.ButlerLogBundle) Sizer {
+	return &fastSizer{
+		size:   proto.Size(b),
+		beSize: make(map[types.StreamPath]int),
+	}
+}
+
+func (b *fastSizer) Size() int {
+	size := b.size
+	for _, v := range b.beSize {
+		size += varintLength(uint64(v))
+	}
+	return size
+}
+
+func (b *fastSizer) AppendBundleEntry(e *protocol.ButlerLogBundle_Entry) {
+	ps := proto.Size(e)
+	s := bundleEntryTagSize + ps
+	b.size += s
+	b.addBundleEntrySize(e, s)
+}
+
+func (b *fastSizer) AppendLogEntry(be *protocol.ButlerLogBundle_Entry, e *protocol.LogEntry) {
+	ps := proto.Size(e)
+	s := logEntryTagSize + varintLength(uint64(ps)) + ps
+	b.size += s
+	b.addBundleEntrySize(be, s)
+}
+
+func (b *fastSizer) addBundleEntrySize(e *protocol.ButlerLogBundle_Entry, count int) {
+	path := protoutil.DescriptorPath(e.GetDesc())
+	b.beSize[path] += count
+}