Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(28)

Side by Side Diff: server/logdog/storage/archive/storage.go

Issue 1909943003: LogDog: Add project support to Storage. (Closed) Base URL: https://github.com/luci/luci-go@logdog-project-coordinator-services
Patch Set: Rebase? Created 4 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2015 The Chromium Authors. All rights reserved. 1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 // Package archive implements a storage.Storage instance that retrieves logs 5 // Package archive implements a storage.Storage instance that retrieves logs
6 // from a Google Storage archive. 6 // from a Google Storage archive.
7 // 7 //
8 // This is a special implementation of storage.Storage, and does not fully 8 // This is a special implementation of storage.Storage, and does not fully
9 // conform to the API expecations. Namely: 9 // conform to the API expecations. Namely:
10 // - It is read-only. Mutation methods will return storage.ErrReadOnly. 10 // - It is read-only. Mutation methods will return storage.ErrReadOnly.
11 // - Storage methods ignore the supplied Path argument, instead opting for 11 // - Storage methods ignore the supplied Path argument, instead opting for
12 // the archive configured in its Options. 12 // the archive configured in its Options.
13 package archive 13 package archive
14 14
15 import ( 15 import (
16 "bytes" 16 "bytes"
17 "fmt" 17 "fmt"
18 "io" 18 "io"
19 "io/ioutil" 19 "io/ioutil"
20 "sort" 20 "sort"
21 "sync" 21 "sync"
22 22
23 "golang.org/x/net/context" 23 "golang.org/x/net/context"
24 24
25 "github.com/golang/protobuf/proto" 25 "github.com/golang/protobuf/proto"
26 "github.com/luci/luci-go/common/config"
26 "github.com/luci/luci-go/common/gcloud/gs" 27 "github.com/luci/luci-go/common/gcloud/gs"
27 "github.com/luci/luci-go/common/iotools" 28 "github.com/luci/luci-go/common/iotools"
28 "github.com/luci/luci-go/common/logdog/types" 29 "github.com/luci/luci-go/common/logdog/types"
29 log "github.com/luci/luci-go/common/logging" 30 log "github.com/luci/luci-go/common/logging"
30 "github.com/luci/luci-go/common/proto/logdog/logpb" 31 "github.com/luci/luci-go/common/proto/logdog/logpb"
31 "github.com/luci/luci-go/common/recordio" 32 "github.com/luci/luci-go/common/recordio"
32 "github.com/luci/luci-go/server/logdog/storage" 33 "github.com/luci/luci-go/server/logdog/storage"
33 ) 34 )
34 35
35 const ( 36 const (
36 // maxStreamRecordSize is the maximum record size we're willing to read from 37 // maxStreamRecordSize is the maximum record size we're willing to read from
37 // our archived log stream. This will help prevent out-of-memory errors if the 38 // our archived log stream. This will help prevent out-of-memory errors if the
38 // arhived log stream is malicious or corrupt. 39 // arhived log stream is malicious or corrupt.
39 maxStreamRecordSize = 16 * 1024 * 1024 40 maxStreamRecordSize = 16 * 1024 * 1024
40 ) 41 )
41 42
42 // Options is the set of configuration options for this Storage instance. 43 // Options is the set of configuration options for this Storage instance.
43 // 44 //
44 // Unlike other Storage instances, this is bound to a single archived stream. 45 // Unlike other Storage instances, this is bound to a single archived stream.
45 // Path parameters in requests will be ignored in favor of the Google Storage 46 // Project and Path parameters in requests will be ignored in favor of the
46 // URLs. 47 // Google Storage URLs.
47 type Options struct { 48 type Options struct {
48 // IndexURL is the Google Storage URL for the stream's index. 49 // IndexURL is the Google Storage URL for the stream's index.
49 IndexURL string 50 IndexURL string
50 // StreamURL is the Google Storage URL for the stream's entries. 51 // StreamURL is the Google Storage URL for the stream's entries.
51 StreamURL string 52 StreamURL string
52 53
53 // Client is the HTTP client to use for authentication. 54 // Client is the HTTP client to use for authentication.
54 // 55 //
55 // Closing this Storage instance does not close the underlying Client. 56 // Closing this Storage instance does not close the underlying Client.
56 Client gs.Client 57 Client gs.Client
(...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after
197 if max > 0 { 198 if max > 0 {
198 max-- 199 max--
199 if max == 0 { 200 if max == 0 {
200 break 201 break
201 } 202 }
202 } 203 }
203 } 204 }
204 return nil 205 return nil
205 } 206 }
206 207
207 func (s *storageImpl) Tail(path types.StreamPath) ([]byte, types.MessageIndex, e rror) { 208 func (s *storageImpl) Tail(project config.ProjectName, path types.StreamPath) ([ ]byte, types.MessageIndex, error) {
208 idx, err := s.getIndex() 209 idx, err := s.getIndex()
209 if err != nil { 210 if err != nil {
210 return nil, 0, err 211 return nil, 0, err
211 } 212 }
212 213
213 // Get the offset of the last record. 214 // Get the offset of the last record.
214 if len(idx.Entries) == 0 { 215 if len(idx.Entries) == 0 {
215 return nil, 0, nil 216 return nil, 0, nil
216 } 217 }
217 lle := idx.Entries[len(idx.Entries)-1] 218 lle := idx.Entries[len(idx.Entries)-1]
(...skipping 156 matching lines...) Expand 10 before | Expand all | Expand 10 after
374 ui := uint64(i) 375 ui := uint64(i)
375 s := sort.Search(len(entries), func(i int) bool { 376 s := sort.Search(len(entries), func(i int) bool {
376 return entries[i].StreamIndex > ui 377 return entries[i].StreamIndex > ui
377 }) 378 })
378 379
379 // The returned index is the one immediately after the index that we wan t. If 380 // The returned index is the one immediately after the index that we wan t. If
380 // our search returned 0, the first index entry is > our search entry, a nd we 381 // our search returned 0, the first index entry is > our search entry, a nd we
381 // will return nil. 382 // will return nil.
382 return s - 1 383 return s - 1
383 } 384 }
OLDNEW
« no previous file with comments | « appengine/logdog/coordinator/endpoints/logs/get.go ('k') | server/logdog/storage/bigtable/rowKey.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698