logdog: Add bundler library.

client/internal/logdog/butler/bundler/sizer_fast.go

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package bundler
+
+import (
+        "fmt"
+        "reflect"
+        "strconv"
+        "strings"
+
+        "github.com/golang/protobuf/proto"
+        "github.com/luci/luci-go/common/logdog/protocol"
+        "github.com/luci/luci-go/common/logdog/protocol/protoutil"
+        "github.com/luci/luci-go/common/logdog/types"
+)
+
+var (
+        bundleEntryTagSize int
+        logEntryTagSize    int
+)
+
+func init() {
+        bundleEntryTagSize = mustCalculateTagSize(&protocol.ButlerLogBundle{}, "Entries")
+        logEntryTagSize = mustCalculateTagSize(&protocol.ButlerLogBundle_Entry{}, "Logs")
+}
+
+func mustCalculateTagSize(i interface{}, field string) int {
+        value, err := calculateTagSize(i, field)
+        if err != nil {
+                panic(err)
+        }
+        return value
+}
+
+func calculateTagSize(i interface{}, field string) (int, error) {
+        v := reflect.TypeOf(i)
+        if v.Kind() == reflect.Ptr {
+                v = v.Elem()
+        }
+        if v.Kind() != reflect.Struct {
+                return 0, fmt.Errorf("sizer: %s is not a struct", v)
+        }
+
+        f, ok := v.FieldByName(field)
+        if !ok {
+                return 0, fmt.Errorf("sizer: could not find field %s.%s", v, field)
+        }
+
+        tag := protobufTag(f)
+        if tag == -1 {
+                return 0, fmt.Errorf("sizer: field %s.%s has no protobuf tag", v, field)
+        }
+
+        return varintLength(uint64(tag) << 3), nil

[tandrii(chromium), 2015/08/11 14:06:08]

I guess that's how protobuf encodes stuff. Still would be nice to have a url to
the doc and maybe short explanation.

[dnj (Google), 2015/08/11 16:24:01]

Done.

+}
+
+func varintLength(val uint64) int {
+        switch {
+        case val < 0x80:
+                return 1
+        case val < 0x4000:
+                return 2
+        case val < 0x200000:
+                return 3
+        case val < 0x10000000:
+                return 4
+        case val < 0x800000000:
+                return 5
+        case val < 0x40000000000:
+                return 6
+        case val < 0x2000000000000:
+                return 7
+        case val < 0x100000000000000:
+                return 8
+        case val < 0x8000000000000000:
+                return 9
+        default:
+                // Maximum uvarint size.
+                return 10
+        }
+}
+
+func protobufTag(f reflect.StructField) int {
+        // If this field doesn't have a "protobuf" tag, ignore it.
+        value := f.Tag.Get("protobuf")
+        parts := strings.Split(value, ",")
+        if len(parts) < 2 {
+                return -1

[tandrii(chromium), 2015/08/11 14:06:08]

Special return values => who are you, a C programmer? :)
This func should return (int, error) to be more Go-like and it'd be line-neutral
change :)

[dnj (Google), 2015/08/11 16:24:01]

Done.

+        }
+        tag, err := strconv.Atoi(parts[1])
+        if err != nil {
+                return -1
+        }
+        return tag
+}
+
+type fastSizer struct {
+        size int
+
+        // As we add LogEntry to a ButlerLogBundle_Entry, the bundle entry's size (and
+        // therefore it's protobuf size prefix) will grow. We account for the growth
+        // by tracking the size of each ButlerLogBundle_Entry and updating it when
+        // we add a LogEntry to it. This is stored independently from the cumulative
+        // size and factored in when Size() is calculated.
+        beSize map[types.StreamPath]int
+}
+
+// NewFastSizer is a Sizer that is optimized for LogDog protobufs.

[tandrii(chromium), 2015/08/11 14:06:08]

nit: i prefer to put public stuff at the top

[dnj (Google), 2015/08/11 16:24:01]

Done.

+//
+// In exchange for rapid size calculation, it performs worst-case estimates on
+// the unknown protocol overheads, leading to potential size overestimation.
+func NewFastSizer(b *protocol.ButlerLogBundle) Sizer {
+        return &fastSizer{
+                size:   proto.Size(b),
+                beSize: make(map[types.StreamPath]int),
+        }
+}
+
+func (b *fastSizer) Size() int {
+        size := b.size
+        for _, v := range b.beSize {
+                size += varintLength(uint64(v))
+        }
+        return size
+}
+
+func (b *fastSizer) AppendBundleEntry(e *protocol.ButlerLogBundle_Entry) {
+        ps := proto.Size(e)
+        s := bundleEntryTagSize + ps
+        b.size += s
+        b.addBundleEntrySize(e, s)
+}
+
+func (b *fastSizer) AppendLogEntry(be *protocol.ButlerLogBundle_Entry, e *protocol.LogEntry) {
+        ps := proto.Size(e)
+        s := logEntryTagSize + varintLength(uint64(ps)) + ps
+        b.size += s
+        b.addBundleEntrySize(be, s)
+}
+
+func (b *fastSizer) addBundleEntrySize(e *protocol.ButlerLogBundle_Entry, count int) {
+        path := protoutil.DescriptorPath(e.GetDesc())
+        b.beSize[path] += count
+}