Fix sparse index handling, add index params.

logdog/common/storage/archive/storage.go

Index: logdog/common/storage/archive/storage.go
diff --git a/logdog/common/storage/archive/storage.go b/logdog/common/storage/archive/storage.go
index be8c18dffd21f9b5e255849082b4ba756c56239b..6800adfb8bb50d2ce01a0ecff9ec59268e441fc0 100644
--- a/logdog/common/storage/archive/storage.go
+++ b/logdog/common/storage/archive/storage.go
@@ -115,8 +115,9 @@ func (s *storageImpl) Get(req storage.GetRequest, cb storage.GetCallback) error
 
 	// Identify the byte offsets that we want to fetch from the entries stream.
 	st := s.buildGetStrategy(&req, idx)
-	if st.lastIndex == -1 || req.Index > st.lastIndex {
-		// No records to read.
+	if st.lastIndex >= 0 && req.Index > st.lastIndex {
+		// We know the last index, and the user requested logs past it, so there are
+		// no records to read.
 		return nil
 	}
 
@@ -312,14 +313,19 @@ func (gs *getStrategy) setEndOffset(v uint64) {
 }
 
 func (s *storageImpl) buildGetStrategy(req *storage.GetRequest, idx *logpb.LogIndex) *getStrategy {
-	st := getStrategy{}
+	st := getStrategy{
+		lastIndex: -1,
+	}
 
 	if len(idx.Entries) == 0 {
-		st.lastIndex = -1
 		return &st
 	}
 
-	st.lastIndex = types.MessageIndex(idx.Entries[len(idx.Entries)-1].StreamIndex)

[dnj, 2016/10/17 22:12:12]

The actual bug: if there is a sparse index whose last entry, X, ends before the
end of the stream, for a stream with N (>X) entries, and an offset, >X, is
requested, this will falsely assume that the index is contiguous and will say,
"hey, X is the last entry, so there are no logs past X", even though there's
X+1..N.

Now, we only do this optimization if the index contains the cool new "last
stream index" entry.

+	// If we have a log entry count, mark the last log index.
+	if idx.LogEntryCount > 0 {
+		st.lastIndex = types.MessageIndex(idx.Entries[len(idx.Entries)-1].StreamIndex)
+	}
+
 	startIdx := indexEntryFor(idx.Entries, req.Index)
 	if startIdx < 0 {
 		startIdx = 0