| OLD | NEW |
| 1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
| 4 | 4 |
| 5 // Package archive implements a storage.Storage instance that retrieves logs | 5 // Package archive implements a storage.Storage instance that retrieves logs |
| 6 // from a Google Storage archive. | 6 // from a Google Storage archive. |
| 7 // | 7 // |
| 8 // This is a special implementation of storage.Storage, and does not fully | 8 // This is a special implementation of storage.Storage, and does not fully |
| 9 // conform to the API expecations. Namely: | 9 // conform to the API expecations. Namely: |
| 10 // - It is read-only. Mutation methods will return storage.ErrReadOnly. | 10 // - It is read-only. Mutation methods will return storage.ErrReadOnly. |
| 11 // - Storage methods ignore the supplied Path argument, instead opting for | 11 // - Storage methods ignore the supplied Path argument, instead opting for |
| 12 // the archive configured in its Options. | 12 // the archive configured in its Options. |
| 13 package archive | 13 package archive |
| 14 | 14 |
| 15 import ( | 15 import ( |
| 16 "bytes" | 16 "bytes" |
| 17 "fmt" | 17 "fmt" |
| 18 "io" | 18 "io" |
| 19 "io/ioutil" | 19 "io/ioutil" |
| 20 "sort" | 20 "sort" |
| 21 "sync/atomic" | 21 "sync/atomic" |
| 22 | 22 |
| 23 "github.com/luci/luci-go/common/config" | |
| 24 "github.com/luci/luci-go/common/data/recordio" | 23 "github.com/luci/luci-go/common/data/recordio" |
| 25 "github.com/luci/luci-go/common/errors" | 24 "github.com/luci/luci-go/common/errors" |
| 26 "github.com/luci/luci-go/common/gcloud/gs" | 25 "github.com/luci/luci-go/common/gcloud/gs" |
| 27 "github.com/luci/luci-go/common/iotools" | 26 "github.com/luci/luci-go/common/iotools" |
| 28 log "github.com/luci/luci-go/common/logging" | 27 log "github.com/luci/luci-go/common/logging" |
| 29 "github.com/luci/luci-go/logdog/api/logpb" | 28 "github.com/luci/luci-go/logdog/api/logpb" |
| 30 "github.com/luci/luci-go/logdog/common/storage" | 29 "github.com/luci/luci-go/logdog/common/storage" |
| 31 "github.com/luci/luci-go/logdog/common/storage/caching" | 30 "github.com/luci/luci-go/logdog/common/storage/caching" |
| 32 "github.com/luci/luci-go/logdog/common/types" | 31 "github.com/luci/luci-go/logdog/common/types" |
| 32 "github.com/luci/luci-go/luci_config/common/cfgtypes" |
| 33 | 33 |
| 34 cloudStorage "cloud.google.com/go/storage" | 34 cloudStorage "cloud.google.com/go/storage" |
| 35 "github.com/golang/protobuf/proto" | 35 "github.com/golang/protobuf/proto" |
| 36 "golang.org/x/net/context" | 36 "golang.org/x/net/context" |
| 37 ) | 37 ) |
| 38 | 38 |
| 39 const ( | 39 const ( |
| 40 // maxStreamRecordSize is the maximum record size we're willing to read
from | 40 // maxStreamRecordSize is the maximum record size we're willing to read
from |
| 41 // our archived log stream. This will help prevent out-of-memory errors
if the | 41 // our archived log stream. This will help prevent out-of-memory errors
if the |
| 42 // arhived log stream is malicious or corrupt. | 42 // arhived log stream is malicious or corrupt. |
| (...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 218 // Enforce our limit, if one is supplied. | 218 // Enforce our limit, if one is supplied. |
| 219 if remaining > 0 { | 219 if remaining > 0 { |
| 220 remaining-- | 220 remaining-- |
| 221 if remaining == 0 { | 221 if remaining == 0 { |
| 222 return nil | 222 return nil |
| 223 } | 223 } |
| 224 } | 224 } |
| 225 } | 225 } |
| 226 } | 226 } |
| 227 | 227 |
| 228 func (s *storageImpl) Tail(project config.ProjectName, path types.StreamPath) (*
storage.Entry, error) { | 228 func (s *storageImpl) Tail(project cfgtypes.ProjectName, path types.StreamPath)
(*storage.Entry, error) { |
| 229 idx, err := s.getIndex() | 229 idx, err := s.getIndex() |
| 230 if err != nil { | 230 if err != nil { |
| 231 return nil, err | 231 return nil, err |
| 232 } | 232 } |
| 233 | 233 |
| 234 // Get the offset that is as close to our tail record as possible. If we
know | 234 // Get the offset that is as close to our tail record as possible. If we
know |
| 235 // what that index is (from "idx"), we can request it directly. Otherwis
e, we | 235 // what that index is (from "idx"), we can request it directly. Otherwis
e, we |
| 236 // will get as close as possible and read forwards from there. | 236 // will get as close as possible and read forwards from there. |
| 237 req := storage.GetRequest{} | 237 req := storage.GetRequest{} |
| 238 switch { | 238 switch { |
| (...skipping 229 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 468 ui := uint64(i) | 468 ui := uint64(i) |
| 469 s := sort.Search(len(entries), func(i int) bool { | 469 s := sort.Search(len(entries), func(i int) bool { |
| 470 return entries[i].StreamIndex > ui | 470 return entries[i].StreamIndex > ui |
| 471 }) | 471 }) |
| 472 | 472 |
| 473 // The returned index is the one immediately after the index that we wan
t. If | 473 // The returned index is the one immediately after the index that we wan
t. If |
| 474 // our search returned 0, the first index entry is > our search entry, a
nd we | 474 // our search returned 0, the first index entry is > our search entry, a
nd we |
| 475 // will return nil. | 475 // will return nil. |
| 476 return s - 1 | 476 return s - 1 |
| 477 } | 477 } |
| OLD | NEW |