Fix sparse index handling, add index params.

logdog/common/archive/archive_test.go

 // Copyright 2015 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package archive
 
 import (
         "bytes"
         "fmt"
         "io"
@@ skipping 65 matching lines @@
         err error
 }
 
 func (w *errWriter) Write(d []byte) (int, error) {
         if w.err != nil {
                 return 0, w.err
         }
         return w.Writer.Write(d)
 }
 
+// indexParams umarshals an index from a byte stream and removes any entries.

[dnj, 2016/10/17 22:12:12]

(We now test that the generated index has the right parameters)

+func indexParams(d []byte) logpb.LogIndex {
+        var index logpb.LogIndex
+        if err := proto.Unmarshal(d, &index); err != nil {
+                panic(fmt.Errorf("failed to unmarshal index protobuf: %v", err))
+        }
+        index.Entries = nil
+        return index
+}
+
 type indexChecker struct {
         fixedSize int
 }
 
 func (ic *indexChecker) size(pb proto.Message) int {
         if ic.fixedSize > 0 {
                 return ic.fixedSize
         }
         return proto.Size(pb)
 }
@@ skipping 126 matching lines @@
                         LogWriter:   &logB,
                         IndexWriter: &indexB,
                         DataWriter:  &dataB,
                 }
 
                 Convey(`A sequence of logs will build a complete index.`, func() {
                         ts.add(0, 1, 2, 3, 4, 5, 6)
                         So(Archive(m), ShouldBeNil)
 
                         So(&indexB, ic.shouldContainIndexFor, desc, &logB)
+                        So(indexParams(indexB.Bytes()), ShouldResemble, logpb.LogIndex{
+                                Desc:            desc,
+                                LastPrefixIndex: 12,
+                                LastStreamIndex: 6,
+                                LogEntryCount:   7,
+                        })
                         So(dataB.String(), ShouldEqual, "0\n1\n2\n3\n4\n5\n6\n")
                 })
 
                 Convey(`A sequence of non-contiguous logs will build a complete index.`, func() {
                         ts.add(0, 1, 3, 6)
                         So(Archive(m), ShouldBeNil)
 
                         So(&indexB, ic.shouldContainIndexFor, desc, &logB, 0, 1, 3, 6)
+                        So(indexParams(indexB.Bytes()), ShouldResemble, logpb.LogIndex{
+                                Desc:            desc,
+                                LastPrefixIndex: 12,
+                                LastStreamIndex: 6,
+                                LogEntryCount:   4,
+                        })
                         So(dataB.String(), ShouldEqual, "0\n1\n3\n6\n")
                 })
 
                 Convey(`Out of order logs are ignored`, func() {
                         Convey(`When StreamIndex is out of order.`, func() {
                                 ts.add(0, 2, 1, 3)
                                 So(Archive(m), ShouldBeNil)
 
                                 So(&indexB, ic.shouldContainIndexFor, desc, &logB, 0, 2, 3)
+                                So(indexParams(indexB.Bytes()), ShouldResemble, logpb.LogIndex{
+                                        Desc:            desc,
+                                        LastPrefixIndex: 6,
+                                        LastStreamIndex: 3,
+                                        LogEntryCount:   3,
+                                })
                         })
 
                         Convey(`When PrefixIndex is out of order.`, func() {
                                 ts.add(0, 1)
                                 le := gen(2)
                                 le.PrefixIndex = 1
                                 ts.addEntries(le)
                                 ts.add(3, 4)
                                 So(Archive(m), ShouldBeNil)
 
                                 So(&indexB, ic.shouldContainIndexFor, desc, &logB, 0, 1, 3, 4)
+                                So(indexParams(indexB.Bytes()), ShouldResemble, logpb.LogIndex{
+                                        Desc:            desc,
+                                        LastPrefixIndex: 8,
+                                        LastStreamIndex: 4,
+                                        LogEntryCount:   4,
+                                })
                         })
 
                         Convey(`When Sequence is out of order.`, func() {
                                 ts.add(0, 1)
                                 le := gen(2)
                                 le.Sequence = 0
                                 ts.addEntries(le)
                                 ts.add(3, 4)
                                 So(Archive(m), ShouldBeNil)
 
                                 So(&indexB, ic.shouldContainIndexFor, desc, &logB, 0, 1, 3, 4)
+                                So(indexParams(indexB.Bytes()), ShouldResemble, logpb.LogIndex{
+                                        Desc:            desc,
+                                        LastPrefixIndex: 8,
+                                        LastStreamIndex: 4,
+                                        LogEntryCount:   4,
+                                })
                         })
                 })
 
                 Convey(`When TimeOffset is out of order, it is bumped to max.`, func() {
                         ts.add(0, 1)
                         le := gen(2)
                         le.TimeOffset = nil // 0
                         ts.addEntries(le)
                         ts.add(3, 4)
                         So(Archive(m), ShouldBeNil)
 
                         So(&indexB, ic.shouldContainIndexFor, desc, &logB, 0, 1, 2, 3, 4)
+                        So(indexParams(indexB.Bytes()), ShouldResemble, logpb.LogIndex{
+                                Desc:            desc,
+                                LastPrefixIndex: 8,
+                                LastStreamIndex: 4,
+                                LogEntryCount:   5,
+                        })
                 })
 
                 Convey(`Source errors will be returned`, func() {
                         Convey(`nil LogEntry`, func() {
                                 ts.addLogEntry(nil)
                                 So(Archive(m), ShouldErrLike, "nil LogEntry")
                         })
 
                         Convey(`Error returned`, func() {
                                 ts.add(0, 1, 2, 3, 4, 5)
@@ skipping 29 matching lines @@
                 })
 
                 Convey(`When building sparse index`, func() {
                         ts.add(0, 1, 2, 3, 4, 5)
 
                         Convey(`Can build an index for every 3 StreamIndex.`, func() {
                                 m.StreamIndexRange = 3
                                 So(Archive(m), ShouldBeNil)
 
                                 So(&indexB, ic.shouldContainIndexFor, desc, &logB, 0, 3)
+                                So(indexParams(indexB.Bytes()), ShouldResemble, logpb.LogIndex{
+                                        Desc:            desc,
+                                        LastPrefixIndex: 10,
+                                        LastStreamIndex: 5,
+                                        LogEntryCount:   6,
+                                })
                         })
 
                         Convey(`Can build an index for every 3 PrefixIndex.`, func() {
                                 m.PrefixIndexRange = 3
                                 So(Archive(m), ShouldBeNil)
 
                                 // Note that in our generated logs, PrefixIndex = 2*StreamIndex.
                                 So(&indexB, ic.shouldContainIndexFor, desc, &logB, 0, 2, 4)
+                                So(indexParams(indexB.Bytes()), ShouldResemble, logpb.LogIndex{
+                                        Desc:            desc,
+                                        LastPrefixIndex: 10,
+                                        LastStreamIndex: 5,
+                                        LogEntryCount:   6,
+                                })
                         })
 
                         Convey(`Can build an index for every 13 bytes.`, func() {
                                 ic.fixedSize = 5
                                 m.ByteRange = 13
                                 m.sizeFunc = func(pb proto.Message) int {
                                         // Stub all LogEntry to be 5 bytes.
                                         return 5
                                 }
                                 So(Archive(m), ShouldBeNil)
 
                                 So(&indexB, ic.shouldContainIndexFor, desc, &logB, 0, 2, 5)
+                                So(indexParams(indexB.Bytes()), ShouldResemble, logpb.LogIndex{
+                                        Desc:            desc,
+                                        LastPrefixIndex: 10,
+                                        LastStreamIndex: 5,
+                                        LogEntryCount:   6,
+                                })
                         })
                 })
         })
 }