LogDog: Add Storage-layer data caching.

logdog/appengine/coordinator/endpoints/logs/get.go

 // Copyright 2015 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.
 
 package logs
 
 import (
         "time"
 
         ds "github.com/luci/gae/service/datastore"
@@ skipping 150 matching lines @@
                 defer func() {
                         if err := gs.Close(); err != nil {
                                 log.WithError(err).Warningf(c, "Failed to close Google Storage client.")
                         }
                 }()
 
                 st, err = archive.New(c, archive.Options{
                         IndexURL:  lst.ArchiveIndexURL,
                         StreamURL: lst.ArchiveStreamURL,
                         Client:    gs,
+                        Cache:     svc.StorageCache(),
                 })
                 if err != nil {
                         log.WithError(err).Errorf(c, "Failed to create Google Storage storage instance.")
                         return nil, err
                 }
         }
         defer st.Close()
 
         project, path := coordinator.Project(c), ls.Path()
 
@@ skipping 31 matching lines @@
         }
         logs := make([]*logpb.LogEntry, 0, asz)
 
         sreq := storage.GetRequest{
                 Project: project,
                 Path:    path,
                 Index:   types.MessageIndex(req.Index),
                 Limit:   logCount,
         }
 
+        var ierr error
         count := 0
-        var ierr error
         err := retry.Retry(c, retry.TransientOnly(retry.Default), func() error {
                 // Issue the Get request. This may return a transient error, in which case
                 // we will retry.
                 return st.Get(sreq, func(e *storage.Entry) bool {
                         var le *logpb.LogEntry
                         if le, ierr = e.GetLogEntry(); ierr != nil {
                                 return false
                         }
 
                         if count > 0 && byteLimit-len(e.D) < 0 {
                                 // Not the first log, and we've exceeded our byte limit.
                                 return false
                         }
-                        byteLimit -= len(e.D)
 
                         sidx, _ := e.GetStreamIndex() // GetLogEntry succeeded, so this must.
                         if !(req.NonContiguous || sidx == sreq.Index) {
                                 return false
                         }
+
                         logs = append(logs, le)
                         sreq.Index = sidx + 1
+                        byteLimit -= len(e.D)
                         count++
                         return !(logCount > 0 && count >= logCount)
                 })
         }, func(err error, delay time.Duration) {
                 log.Fields{
                         log.ErrorKey:   err,
                         "delay":        delay,
                         "initialIndex": req.Index,
                         "nextIndex":    sreq.Index,
                         "count":        len(logs),
@@ skipping 48 matching lines @@
                 return []*logpb.LogEntry{le}, nil
 
         case storage.ErrDoesNotExist:
                 return nil, nil
 
         default:
                 log.WithError(err).Errorf(c, "Failed to fetch tail log.")
                 return nil, err
         }
 }