OLD | NEW |
1 // Copyright 2015 The Chromium Authors. All rights reserved. | 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Package archive implements a storage.Storage instance that retrieves logs | 5 // Package archive implements a storage.Storage instance that retrieves logs |
6 // from a Google Storage archive. | 6 // from a Google Storage archive. |
7 // | 7 // |
8 // This is a special implementation of storage.Storage, and does not fully | 8 // This is a special implementation of storage.Storage, and does not fully |
9 // conform to the API expecations. Namely: | 9 // conform to the API expecations. Namely: |
10 // - It is read-only. Mutation methods will return storage.ErrReadOnly. | 10 // - It is read-only. Mutation methods will return storage.ErrReadOnly. |
11 // - Storage methods ignore the supplied Path argument, instead opting for | 11 // - Storage methods ignore the supplied Path argument, instead opting for |
12 // the archive configured in its Options. | 12 // the archive configured in its Options. |
13 package archive | 13 package archive |
14 | 14 |
15 import ( | 15 import ( |
16 "bytes" | 16 "bytes" |
17 "fmt" | 17 "fmt" |
18 "io" | 18 "io" |
19 "io/ioutil" | 19 "io/ioutil" |
20 "sort" | 20 "sort" |
21 "sync" | 21 "sync" |
22 | 22 |
23 "golang.org/x/net/context" | 23 "golang.org/x/net/context" |
24 | 24 |
25 "github.com/golang/protobuf/proto" | 25 "github.com/golang/protobuf/proto" |
| 26 "github.com/luci/luci-go/common/config" |
26 "github.com/luci/luci-go/common/gcloud/gs" | 27 "github.com/luci/luci-go/common/gcloud/gs" |
27 "github.com/luci/luci-go/common/iotools" | 28 "github.com/luci/luci-go/common/iotools" |
28 "github.com/luci/luci-go/common/logdog/types" | 29 "github.com/luci/luci-go/common/logdog/types" |
29 log "github.com/luci/luci-go/common/logging" | 30 log "github.com/luci/luci-go/common/logging" |
30 "github.com/luci/luci-go/common/proto/logdog/logpb" | 31 "github.com/luci/luci-go/common/proto/logdog/logpb" |
31 "github.com/luci/luci-go/common/recordio" | 32 "github.com/luci/luci-go/common/recordio" |
32 "github.com/luci/luci-go/server/logdog/storage" | 33 "github.com/luci/luci-go/server/logdog/storage" |
33 ) | 34 ) |
34 | 35 |
35 const ( | 36 const ( |
36 // maxStreamRecordSize is the maximum record size we're willing to read
from | 37 // maxStreamRecordSize is the maximum record size we're willing to read
from |
37 // our archived log stream. This will help prevent out-of-memory errors
if the | 38 // our archived log stream. This will help prevent out-of-memory errors
if the |
38 // arhived log stream is malicious or corrupt. | 39 // arhived log stream is malicious or corrupt. |
39 maxStreamRecordSize = 16 * 1024 * 1024 | 40 maxStreamRecordSize = 16 * 1024 * 1024 |
40 ) | 41 ) |
41 | 42 |
42 // Options is the set of configuration options for this Storage instance. | 43 // Options is the set of configuration options for this Storage instance. |
43 // | 44 // |
44 // Unlike other Storage instances, this is bound to a single archived stream. | 45 // Unlike other Storage instances, this is bound to a single archived stream. |
45 // Path parameters in requests will be ignored in favor of the Google Storage | 46 // Project and Path parameters in requests will be ignored in favor of the |
46 // URLs. | 47 // Google Storage URLs. |
47 type Options struct { | 48 type Options struct { |
48 // IndexURL is the Google Storage URL for the stream's index. | 49 // IndexURL is the Google Storage URL for the stream's index. |
49 IndexURL string | 50 IndexURL string |
50 // StreamURL is the Google Storage URL for the stream's entries. | 51 // StreamURL is the Google Storage URL for the stream's entries. |
51 StreamURL string | 52 StreamURL string |
52 | 53 |
53 // Client is the HTTP client to use for authentication. | 54 // Client is the HTTP client to use for authentication. |
54 // | 55 // |
55 // Closing this Storage instance does not close the underlying Client. | 56 // Closing this Storage instance does not close the underlying Client. |
56 Client gs.Client | 57 Client gs.Client |
(...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
197 if max > 0 { | 198 if max > 0 { |
198 max-- | 199 max-- |
199 if max == 0 { | 200 if max == 0 { |
200 break | 201 break |
201 } | 202 } |
202 } | 203 } |
203 } | 204 } |
204 return nil | 205 return nil |
205 } | 206 } |
206 | 207 |
207 func (s *storageImpl) Tail(path types.StreamPath) ([]byte, types.MessageIndex, e
rror) { | 208 func (s *storageImpl) Tail(project config.ProjectName, path types.StreamPath) ([
]byte, types.MessageIndex, error) { |
208 idx, err := s.getIndex() | 209 idx, err := s.getIndex() |
209 if err != nil { | 210 if err != nil { |
210 return nil, 0, err | 211 return nil, 0, err |
211 } | 212 } |
212 | 213 |
213 // Get the offset of the last record. | 214 // Get the offset of the last record. |
214 if len(idx.Entries) == 0 { | 215 if len(idx.Entries) == 0 { |
215 return nil, 0, nil | 216 return nil, 0, nil |
216 } | 217 } |
217 lle := idx.Entries[len(idx.Entries)-1] | 218 lle := idx.Entries[len(idx.Entries)-1] |
(...skipping 156 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
374 ui := uint64(i) | 375 ui := uint64(i) |
375 s := sort.Search(len(entries), func(i int) bool { | 376 s := sort.Search(len(entries), func(i int) bool { |
376 return entries[i].StreamIndex > ui | 377 return entries[i].StreamIndex > ui |
377 }) | 378 }) |
378 | 379 |
379 // The returned index is the one immediately after the index that we wan
t. If | 380 // The returned index is the one immediately after the index that we wan
t. If |
380 // our search returned 0, the first index entry is > our search entry, a
nd we | 381 // our search returned 0, the first index entry is > our search entry, a
nd we |
381 // will return nil. | 382 // will return nil. |
382 return s - 1 | 383 return s - 1 |
383 } | 384 } |
OLD | NEW |