LogDog: Add log rendering view.

appengine/logdog/coordinator/logView/view.go

+// Copyright 2015 The Chromium Authors. All rights reserved.

[Ryan Tseng, 2016/02/08 22:56:55]

2016  /happy new year

[dnj (Google), 2016/02/09 02:50:03]

Done.

+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package logView
+
+import (
+        "io"
+        "net/http"
+        "strings"
+        "time"
+
+        "github.com/golang/protobuf/proto"
+        "github.com/julienschmidt/httprouter"
+        ds "github.com/luci/gae/service/datastore"
+        "github.com/luci/luci-go/appengine/logdog/coordinator"
+        "github.com/luci/luci-go/appengine/logdog/coordinator/config"
+        "github.com/luci/luci-go/common/clock"
+        "github.com/luci/luci-go/common/logdog/fetcher"
+        "github.com/luci/luci-go/common/logdog/renderer"
+        "github.com/luci/luci-go/common/logdog/types"
+        log "github.com/luci/luci-go/common/logging"
+        "github.com/luci/luci-go/common/proto/logdog/logpb"
+        "github.com/luci/luci-go/server/logdog/storage"
+        "golang.org/x/net/context"
+)
+
+const (
+        // DefaultViewTimeout is the default value for the Handler's Timeout.
+        //
+        // It targets the managed VM environment, which allows 24 hours. We stop
+        // 10 minutes short of that to avoid conflict.
+        DefaultViewTimeout = (24 * time.Hour) - (10 * time.Minute)
+
+        // viewLogDelay is the amount of time to wait in between polls when no more
+        // log entries are available during streaming.
+        viewLogDelay = (5 * time.Second)
+)
+
+// Handler offers an HTTP handler that will return the rendered contents of a
+// log stream.
+type Handler struct {
+        coordinator.Service
+
+        // timeout is the maximum amount of time that we are willing to handle a
+        // View request.
+        //
+        // This should be short of the maximum request time allowed by an AppEngine
+        // handler, else the instance may be terminated.
+        timeout time.Duration
+}
+
+// Handle will render the contents of a log stream to the supplied
+// ResponseWriter.
+//
+// It expects the stream hash/path to be embedded in p as "path".
+func (h *Handler) Handle(c context.Context, w http.ResponseWriter, req *http.Request, p httprouter.Params) {
+        rc := h.handleImpl(c, w, req, p)
+        w.WriteHeader(rc)
+}
+
+func (h *Handler) handleImpl(c context.Context, w http.ResponseWriter, req *http.Request, p httprouter.Params) int {
+        // Get the requested log path/hash.
+        path := strings.TrimLeft(p.ByName("path"), "/")
+        log.Fields{
+                "path": path,
+        }.Infof(c, "Request to view log stream.")
+
+        ls, err := coordinator.NewLogStream(path)
+        if err != nil {
+                log.WithError(err).Errorf(c, "Invalid log sream path/hash value supplied.")
+                return http.StatusBadRequest
+        }
+
+        // Set a timeout on our Context.
+        timeout := h.timeout
+        if timeout <= 0 {
+                timeout = DefaultViewTimeout
+        }
+        c, cancelFunc := context.WithTimeout(c, timeout)
+        defer cancelFunc()
+
+        // Load the log stream metadata.
+        if err := ds.Get(c).Get(&ls); err != nil {
+                log.WithError(err).Errorf(c, "Failed to load LogStream.")
+
+                switch err {
+                case ds.ErrNoSuchEntity:
+                        return http.StatusNotFound
+                default:
+                        return http.StatusInternalServerError
+                }
+        }
+
+        // The user can stream purged logs only if they're an administrator. If not,
+        // pretend that the stream does not exist.
+        if ls.Purged {
+                if err := config.IsAdminUser(c); err != nil {
+                        log.WithError(err).Warningf(c, "Non-admin user requested purged log stream.")
+                        return http.StatusNotFound
+                }
+        }
+
+        // Get our Storage instance.
+        st, err := h.GetLogStreamStorage(c, ls)
+        if err != nil {
+                log.WithError(err).Errorf(c, "Failed to get Storage instance.")
+                return http.StatusInternalServerError
+        }
+        defer st.Close()
+
+        // Fetch the requested log stream!
+        w.Header().Add("Content-Type", ls.ContentType)
+
+        fetchCtx, cancelFunc := context.WithCancel(c)

[Ryan Tseng, 2016/02/08 22:56:55]

cancelFunc isn't used anywhere

[dnj (Google), 2016/02/09 02:50:03]

Done.

+        f := fetcher.New(fetchCtx, fetcher.Options{
+                Source: &storageLogSource{
+                        ls: ls,
+                        st: st,
+                },
+                Delay: viewLogDelay,
+        })
+        r := renderer.Renderer{
+                Source:    f,
+                Reproduce: true,
+        }
+
+        // Render log data to our HTTP response.
+        count, err := io.Copy(w, &r)
+        if err != nil {
+                log.WithError(err).Errorf(c, "Failed to output log stream.")
+                return http.StatusInternalServerError
+        }
+
+        log.Fields{
+                "bytes": count,
+        }.Infof(c, "Successfully wrote log stream.")
+        return http.StatusOK
+}
+
+// storageLogSource is a stateful incremental implementation of the
+// fetcher.Source interface which uses direct storage access to pull log entries.
+//
+// The Fetcher will respond to Context cancellation, so we don't have to worry
+// about that here.
+type storageLogSource struct {
+        ls *coordinator.LogStream
+        st storage.Storage
+}
+
+func (f *storageLogSource) LogEntries(c context.Context, req *fetcher.LogRequest) (

[Ryan Tseng, 2016/02/08 22:56:55]

docstring (lint should catch this)

[dnj (Google), 2016/02/09 02:50:03]

It's a public method on a private struct, so that's not necessary. In this case,
it's implementing an interface, so the interface's documentation should be
sufficient. No need to repeat.

+        logs []*logpb.LogEntry, tidx types.MessageIndex, err error) {
+        // If this log stream hasn't been terminated yet, re-load it and check if
+        // there is a terminal index.
+        tidx = types.MessageIndex(f.ls.TerminalIndex)
+        if tidx < 0 {
+                log.Debugf(c, "Log stream is non-terminal, refreshing.")
+                if err := ds.Get(c).Get(f.ls); err != nil {
+                        log.WithError(err).Errorf(c, "Failed to refresh log stream.")
+                }
+
+                tidx = types.MessageIndex(f.ls.TerminalIndex)
+                if tidx >= 0 {
+                        log.Fields{
+                                "terminalIndex": tidx,
+                        }.Debugf(c, "Log stream has been terminated.")
+                }
+        }
+
+        sreq := storage.GetRequest{
+                Path:  f.ls.Path(),
+                Index: types.MessageIndex(req.Index),
+        }
+
+        // If we have a terminal index, set a limit. This is probably unnecessary, but
+        // will handle the error case where logs past the terminal index are written
+        // to intermediate storage before a terminal index is registered.
+        if tidx >= 0 {
+                sreq.Limit = int64(tidx - req.Index + 1)
+                if sreq.Limit <= 0 {
+                        // No more log records in the stream.
+                        return
+                }
+        }
+
+        // Issue the Get request.

[Ryan Tseng, 2016/02/08 22:56:55]

presumably for all chunks of a logstream?

[dnj (Google), 2016/02/09 02:50:03]

Well, for the ones requested via "req".

+        var logData [][]byte
+        bytes := int64(0)
+        err = f.st.Get(&sreq, func(idx types.MessageIndex, ld []byte) bool {
+                // Enforce contiguous log records.
+                if idx != sreq.Index {
+                        return false
+                }
+                logData = append(logData, ld)
+                sreq.Index++
+                bytes += int64(len(ld))
+
+                // Apply byte/count constraints.
+                return (!(req.Count > 0 && int64(len(logData)) >= req.Count) && !(req.Bytes > 0 && bytes >= req.Bytes))
+        })
+        if err != nil {
+                log.Fields{
+                        log.ErrorKey:   err,
+                        "initialIndex": req.Index,
+                        "nextIndex":    sreq.Index,
+                        "count":        len(logData),
+                }.Errorf(c, "Failed to fetch logs.")
+                return
+        }
+
+        // Convert the logs to protobufs.
+        logs = make([]*logpb.LogEntry, len(logData))
+        for i, ld := range logData {
+                le := logpb.LogEntry{}
+                if err = proto.Unmarshal(ld, &le); err != nil {
+                        log.Fields{
+                                log.ErrorKey: err,
+                                "logIndex":   i,
+                        }.Errorf(c, "Failed to unmarshal LogEntry from log data.")
+                        return
+                }
+                logs[i] = &le
+        }
+        return
+}
+
+// cancelSleep will block for the specified duration, or until the supplied
+// Context is done.
+func cancelSleep(c context.Context, d time.Duration) error {
+        t := clock.NewTimer(c)
+        t.Reset(d)
+        defer t.Stop()
+
+        select {
+        case <-c.Done():
+                return c.Err()
+        case <-t.GetC():
+                return nil
+        }
+}