LogDog: Handle archive failures.

appengine/logdog/coordinator/logStream.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package coordinator
 
 import (
         "crypto/sha256"
         "encoding/hex"
         "errors"
@@ skipping 96 matching lines @@
         // can be queried against.
         //
         // The serialization/deserialization is handled manually in order to enable
         // key/value queries.
         Tags TagMap `gae:"-"`
 
         // Source is the set of source strings sent by the Butler.
         Source []string
 
         // TerminalIndex is the log stream index of the last log entry in the stream.
-        // If the value is -1, the log is still streaming.
+        //
+        // If the value is <0:

[Vadim Sh., 2016/03/31 22:29:59]

typo: "If" is not needed here

[dnj, 2016/04/01 22:57:04]

Acknowledged.

+        //      - If the log is in the LSPending state, the stream is still streaming
+        //        without a known terminal index.
+        //      - If the log is in LSArchived state, it means that the stream's archival
+        //        has completed, but had no log entries.
         TerminalIndex int64 `gae:",noindex"`
 
         // ArchiveIndexURL is the Google Storage URL where the log stream's index is
         // archived.
         ArchiveIndexURL string `gae:",noindex"`
         // ArchiveIndexSize is the size, in bytes, of the archived Index. It will be
         // zero if the file is not archived.
         ArchiveIndexSize int64 `gae:",noindex"`
         // ArchiveStreamURL is the Google Storage URL where the log stream's raw
         // stream data is archived. If this is not empty, the log stream is considered
         // archived.
         ArchiveStreamURL string `gae:",noindex"`
         // ArchiveStreamSize is the size, in bytes, of the archived stream. It will be
         // zero if the file is not archived.
         ArchiveStreamSize int64 `gae:",noindex"`
         // ArchiveDataURL is the Google Storage URL where the log stream's assembled
         // data is archived. If this is not empty, the log stream is considered
         // archived.
         ArchiveDataURL string `gae:",noindex"`
         // ArchiveDataSize is the size, in bytes, of the archived data. It will be
         // zero if the file is not archived.
         ArchiveDataSize int64 `gae:",noindex"`
         // ArchiveWhole is true if archival is complete and the archived log stream
         // was not missing any entries.
         ArchiveWhole bool
+        // ArchiveErrors is the number of errors encountered during archival. This
+        // will be incremented when incomplete archival is permitted, but the archival
+        // still failed due to an error (e.g., data corruption). If this exceeds a
+        // threshold, the stream may be marked archived even if the archival failed.
+        ArchiveErrors int
 
         // _ causes datastore to ignore unrecognized fields and strip them in future
         // writes.
         _ ds.PropertyMap `gae:"-,extra"`
 
         // hashID is the cached generated ID from the stream's Prefix/Name fields. If
         // this is populated, ID metadata will be retrieved from this field instead of
         // generated.
         hashID string
 }
@@ skipping 193 matching lines @@
                 return nil, err
         }
         return &pb, nil
 }
 
 // Terminated returns true if this stream has been terminated.
 func (s *LogStream) Terminated() bool {
         return s.State >= LSTerminated
 }
 
+// Streaming returns true if this log stream is still streaming without a known
+// terminal index.
+func (s *LogStream) Streaming() bool {
+        return (s.State == LSPending && s.TerminalIndex < 0)
+}
+
 // Archived returns true if this stream has been archived. A stream is archived
 // if it has any of its archival properties set.
 func (s *LogStream) Archived() bool {
         return s.State >= LSArchived
 }
 
 // ArchiveMatches tests if the supplied Stream, Index, and Data archival URLs
 // match the current values.
 func (s *LogStream) ArchiveMatches(sURL, iURL, dURL string) bool {
         return (s.ArchiveStreamURL == sURL && s.ArchiveIndexURL == iURL && s.ArchiveDataURL == dURL)
@@ skipping 208 matching lines @@
 // were created before the supplied time.
 func AddOlderFilter(q *ds.Query, t time.Time) *ds.Query {
         return q.Lt("Created", t.UTC()).Order("-Created")
 }
 
 // AddNewerFilter adds a filter to queries that restricts them to results that
 // were created after the supplied time.
 func AddNewerFilter(q *ds.Query, t time.Time) *ds.Query {
         return q.Gt("Created", t.UTC()).Order("-Created")
 }