Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(472)

Side by Side Diff: logdog/common/storage/archive/storage.go

Issue 2626433004: Move "common/config" common types into cfgtypes. (Closed)
Patch Set: Created 3 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « logdog/client/coordinator/stream.go ('k') | logdog/common/storage/bigtable/cache.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2015 The LUCI Authors. All rights reserved. 1 // Copyright 2015 The LUCI Authors. All rights reserved.
2 // Use of this source code is governed under the Apache License, Version 2.0 2 // Use of this source code is governed under the Apache License, Version 2.0
3 // that can be found in the LICENSE file. 3 // that can be found in the LICENSE file.
4 4
5 // Package archive implements a storage.Storage instance that retrieves logs 5 // Package archive implements a storage.Storage instance that retrieves logs
6 // from a Google Storage archive. 6 // from a Google Storage archive.
7 // 7 //
8 // This is a special implementation of storage.Storage, and does not fully 8 // This is a special implementation of storage.Storage, and does not fully
9 // conform to the API expecations. Namely: 9 // conform to the API expecations. Namely:
10 // - It is read-only. Mutation methods will return storage.ErrReadOnly. 10 // - It is read-only. Mutation methods will return storage.ErrReadOnly.
11 // - Storage methods ignore the supplied Path argument, instead opting for 11 // - Storage methods ignore the supplied Path argument, instead opting for
12 // the archive configured in its Options. 12 // the archive configured in its Options.
13 package archive 13 package archive
14 14
15 import ( 15 import (
16 "bytes" 16 "bytes"
17 "fmt" 17 "fmt"
18 "io" 18 "io"
19 "io/ioutil" 19 "io/ioutil"
20 "sort" 20 "sort"
21 "sync/atomic" 21 "sync/atomic"
22 22
23 "github.com/luci/luci-go/common/config"
24 "github.com/luci/luci-go/common/data/recordio" 23 "github.com/luci/luci-go/common/data/recordio"
25 "github.com/luci/luci-go/common/errors" 24 "github.com/luci/luci-go/common/errors"
26 "github.com/luci/luci-go/common/gcloud/gs" 25 "github.com/luci/luci-go/common/gcloud/gs"
27 "github.com/luci/luci-go/common/iotools" 26 "github.com/luci/luci-go/common/iotools"
28 log "github.com/luci/luci-go/common/logging" 27 log "github.com/luci/luci-go/common/logging"
29 "github.com/luci/luci-go/logdog/api/logpb" 28 "github.com/luci/luci-go/logdog/api/logpb"
30 "github.com/luci/luci-go/logdog/common/storage" 29 "github.com/luci/luci-go/logdog/common/storage"
31 "github.com/luci/luci-go/logdog/common/storage/caching" 30 "github.com/luci/luci-go/logdog/common/storage/caching"
32 "github.com/luci/luci-go/logdog/common/types" 31 "github.com/luci/luci-go/logdog/common/types"
32 "github.com/luci/luci-go/luci_config/common/cfgtypes"
33 33
34 cloudStorage "cloud.google.com/go/storage" 34 cloudStorage "cloud.google.com/go/storage"
35 "github.com/golang/protobuf/proto" 35 "github.com/golang/protobuf/proto"
36 "golang.org/x/net/context" 36 "golang.org/x/net/context"
37 ) 37 )
38 38
39 const ( 39 const (
40 // maxStreamRecordSize is the maximum record size we're willing to read from 40 // maxStreamRecordSize is the maximum record size we're willing to read from
41 // our archived log stream. This will help prevent out-of-memory errors if the 41 // our archived log stream. This will help prevent out-of-memory errors if the
42 // arhived log stream is malicious or corrupt. 42 // arhived log stream is malicious or corrupt.
(...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after
218 // Enforce our limit, if one is supplied. 218 // Enforce our limit, if one is supplied.
219 if remaining > 0 { 219 if remaining > 0 {
220 remaining-- 220 remaining--
221 if remaining == 0 { 221 if remaining == 0 {
222 return nil 222 return nil
223 } 223 }
224 } 224 }
225 } 225 }
226 } 226 }
227 227
228 func (s *storageImpl) Tail(project config.ProjectName, path types.StreamPath) (* storage.Entry, error) { 228 func (s *storageImpl) Tail(project cfgtypes.ProjectName, path types.StreamPath) (*storage.Entry, error) {
229 idx, err := s.getIndex() 229 idx, err := s.getIndex()
230 if err != nil { 230 if err != nil {
231 return nil, err 231 return nil, err
232 } 232 }
233 233
234 // Get the offset that is as close to our tail record as possible. If we know 234 // Get the offset that is as close to our tail record as possible. If we know
235 // what that index is (from "idx"), we can request it directly. Otherwis e, we 235 // what that index is (from "idx"), we can request it directly. Otherwis e, we
236 // will get as close as possible and read forwards from there. 236 // will get as close as possible and read forwards from there.
237 req := storage.GetRequest{} 237 req := storage.GetRequest{}
238 switch { 238 switch {
(...skipping 229 matching lines...) Expand 10 before | Expand all | Expand 10 after
468 ui := uint64(i) 468 ui := uint64(i)
469 s := sort.Search(len(entries), func(i int) bool { 469 s := sort.Search(len(entries), func(i int) bool {
470 return entries[i].StreamIndex > ui 470 return entries[i].StreamIndex > ui
471 }) 471 })
472 472
473 // The returned index is the one immediately after the index that we wan t. If 473 // The returned index is the one immediately after the index that we wan t. If
474 // our search returned 0, the first index entry is > our search entry, a nd we 474 // our search returned 0, the first index entry is > our search entry, a nd we
475 // will return nil. 475 // will return nil.
476 return s - 1 476 return s - 1
477 } 477 }
OLDNEW
« no previous file with comments | « logdog/client/coordinator/stream.go ('k') | logdog/common/storage/bigtable/cache.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698