| OLD | NEW |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 // Package archive implements a storage.Storage instance that retrieves logs | 5 // Package archive implements a storage.Storage instance that retrieves logs |
| 6 // from a Google Storage archive. | 6 // from a Google Storage archive. |
| 7 // | 7 // |
| 8 // This is a special implementation of storage.Storage, and does not fully | 8 // This is a special implementation of storage.Storage, and does not fully |
| 9 // conform to the API expecations. Namely: | 9 // conform to the API expecations. Namely: |
| 10 // - It is read-only. Mutation methods will return storage.ErrReadOnly. | 10 // - It is read-only. Mutation methods will return storage.ErrReadOnly. |
| 11 // - Storage methods ignore the supplied Path argument, instead opting for | 11 // - Storage methods ignore the supplied Path argument, instead opting for |
| 12 // the archive configured in its Options. | 12 // the archive configured in its Options. |
| 13 package archive | 13 package archive |
| 14 | 14 |
| 15 import ( | 15 import ( |
| 16 "bytes" | 16 "bytes" |
| 17 "fmt" | 17 "fmt" |
| 18 "io" | 18 "io" |
| 19 "io/ioutil" | 19 "io/ioutil" |
| 20 "sort" | 20 "sort" |
| 21 "sync" | 21 "sync" |
| 22 | 22 |
| 23 "golang.org/x/net/context" | 23 "golang.org/x/net/context" |
| 24 | 24 |
| 25 "github.com/golang/protobuf/proto" | 25 "github.com/golang/protobuf/proto" |
| 26 "github.com/luci/luci-go/common/config" |
| 26 "github.com/luci/luci-go/common/gcloud/gs" | 27 "github.com/luci/luci-go/common/gcloud/gs" |
| 27 "github.com/luci/luci-go/common/iotools" | 28 "github.com/luci/luci-go/common/iotools" |
| 28 "github.com/luci/luci-go/common/logdog/types" | 29 "github.com/luci/luci-go/common/logdog/types" |
| 29 log "github.com/luci/luci-go/common/logging" | 30 log "github.com/luci/luci-go/common/logging" |
| 30 "github.com/luci/luci-go/common/proto/logdog/logpb" | 31 "github.com/luci/luci-go/common/proto/logdog/logpb" |
| 31 "github.com/luci/luci-go/common/recordio" | 32 "github.com/luci/luci-go/common/recordio" |
| 32 "github.com/luci/luci-go/server/logdog/storage" | 33 "github.com/luci/luci-go/server/logdog/storage" |
| 33 ) | 34 ) |
| 34 | 35 |
| 35 const ( | 36 const ( |
| 36 // maxStreamRecordSize is the maximum record size we're willing to read
from | 37 // maxStreamRecordSize is the maximum record size we're willing to read
from |
| 37 // our archived log stream. This will help prevent out-of-memory errors
if the | 38 // our archived log stream. This will help prevent out-of-memory errors
if the |
| 38 // arhived log stream is malicious or corrupt. | 39 // arhived log stream is malicious or corrupt. |
| 39 maxStreamRecordSize = 16 * 1024 * 1024 | 40 maxStreamRecordSize = 16 * 1024 * 1024 |
| 40 ) | 41 ) |
| 41 | 42 |
| 42 // Options is the set of configuration options for this Storage instance. | 43 // Options is the set of configuration options for this Storage instance. |
| 43 // | 44 // |
| 44 // Unlike other Storage instances, this is bound to a single archived stream. | 45 // Unlike other Storage instances, this is bound to a single archived stream. |
| 45 // Path parameters in requests will be ignored in favor of the Google Storage | 46 // Project and Path parameters in requests will be ignored in favor of the |
| 46 // URLs. | 47 // Google Storage URLs. |
| 47 type Options struct { | 48 type Options struct { |
| 48 // IndexURL is the Google Storage URL for the stream's index. | 49 // IndexURL is the Google Storage URL for the stream's index. |
| 49 IndexURL string | 50 IndexURL string |
| 50 // StreamURL is the Google Storage URL for the stream's entries. | 51 // StreamURL is the Google Storage URL for the stream's entries. |
| 51 StreamURL string | 52 StreamURL string |
| 52 | 53 |
| 53 // Client is the HTTP client to use for authentication. | 54 // Client is the HTTP client to use for authentication. |
| 54 // | 55 // |
| 55 // Closing this Storage instance does not close the underlying Client. | 56 // Closing this Storage instance does not close the underlying Client. |
| 56 Client gs.Client | 57 Client gs.Client |
| (...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 197 if max > 0 { | 198 if max > 0 { |
| 198 max-- | 199 max-- |
| 199 if max == 0 { | 200 if max == 0 { |
| 200 break | 201 break |
| 201 } | 202 } |
| 202 } | 203 } |
| 203 } | 204 } |
| 204 return nil | 205 return nil |
| 205 } | 206 } |
| 206 | 207 |
| 207 func (s *storageImpl) Tail(path types.StreamPath) ([]byte, types.MessageIndex, e
rror) { | 208 func (s *storageImpl) Tail(project config.ProjectName, path types.StreamPath) ([
]byte, types.MessageIndex, error) { |
| 208 idx, err := s.getIndex() | 209 idx, err := s.getIndex() |
| 209 if err != nil { | 210 if err != nil { |
| 210 return nil, 0, err | 211 return nil, 0, err |
| 211 } | 212 } |
| 212 | 213 |
| 213 // Get the offset of the last record. | 214 // Get the offset of the last record. |
| 214 if len(idx.Entries) == 0 { | 215 if len(idx.Entries) == 0 { |
| 215 return nil, 0, nil | 216 return nil, 0, nil |
| 216 } | 217 } |
| 217 lle := idx.Entries[len(idx.Entries)-1] | 218 lle := idx.Entries[len(idx.Entries)-1] |
| (...skipping 156 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 374 ui := uint64(i) | 375 ui := uint64(i) |
| 375 s := sort.Search(len(entries), func(i int) bool { | 376 s := sort.Search(len(entries), func(i int) bool { |
| 376 return entries[i].StreamIndex > ui | 377 return entries[i].StreamIndex > ui |
| 377 }) | 378 }) |
| 378 | 379 |
| 379 // The returned index is the one immediately after the index that we wan
t. If | 380 // The returned index is the one immediately after the index that we wan
t. If |
| 380 // our search returned 0, the first index entry is > our search entry, a
nd we | 381 // our search returned 0, the first index entry is > our search entry, a
nd we |
| 381 // will return nil. | 382 // will return nil. |
| 382 return s - 1 | 383 return s - 1 |
| 383 } | 384 } |
| OLD | NEW |