| OLD | NEW |
| 1 // Copyright 2016 The LUCI Authors. All rights reserved. | 1 // Copyright 2016 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
| 4 | 4 |
| 5 // Package main implements a simple CLI tool to load and interact with storage | 5 // Package main implements a simple CLI tool to load and interact with storage |
| 6 // data in Google BigTable data. | 6 // data in Google BigTable data. |
| 7 package main | 7 package main |
| 8 | 8 |
| 9 import ( | 9 import ( |
| 10 "bytes" | |
| 11 "flag" | 10 "flag" |
| 12 "io" | 11 "io" |
| 13 "os" | 12 "os" |
| 13 "strings" |
| 14 | 14 |
| 15 "github.com/luci/luci-go/client/authcli" | 15 "github.com/luci/luci-go/client/authcli" |
| 16 "github.com/luci/luci-go/common/auth" | 16 "github.com/luci/luci-go/common/auth" |
| 17 "github.com/luci/luci-go/common/cli" | 17 "github.com/luci/luci-go/common/cli" |
| 18 "github.com/luci/luci-go/common/data/rand/mathrand" | 18 "github.com/luci/luci-go/common/data/rand/mathrand" |
| 19 "github.com/luci/luci-go/common/errors" | 19 "github.com/luci/luci-go/common/errors" |
| 20 log "github.com/luci/luci-go/common/logging" | 20 log "github.com/luci/luci-go/common/logging" |
| 21 "github.com/luci/luci-go/common/logging/gologger" | 21 "github.com/luci/luci-go/common/logging/gologger" |
| 22 "github.com/luci/luci-go/logdog/common/storage" | 22 "github.com/luci/luci-go/logdog/common/storage" |
| 23 "github.com/luci/luci-go/logdog/common/storage/bigtable" | 23 "github.com/luci/luci-go/logdog/common/storage/bigtable" |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 55 func (app *application) addFlags(fs *flag.FlagSet) { | 55 func (app *application) addFlags(fs *flag.FlagSet) { |
| 56 fs.StringVar(&app.btProject, "bt-project", "", "BigTable project name (r
equired)") | 56 fs.StringVar(&app.btProject, "bt-project", "", "BigTable project name (r
equired)") |
| 57 fs.StringVar(&app.btInstance, "bt-instance", "", "BigTable instance name
(required)") | 57 fs.StringVar(&app.btInstance, "bt-instance", "", "BigTable instance name
(required)") |
| 58 fs.StringVar(&app.btLogTable, "bt-log-table", "", "BigTable log table na
me (required)") | 58 fs.StringVar(&app.btLogTable, "bt-log-table", "", "BigTable log table na
me (required)") |
| 59 } | 59 } |
| 60 | 60 |
| 61 func (app *application) getBigTableClient(c context.Context) (storage.Storage, e
rror) { | 61 func (app *application) getBigTableClient(c context.Context) (storage.Storage, e
rror) { |
| 62 a := auth.NewAuthenticator(c, auth.SilentLogin, app.authOpts) | 62 a := auth.NewAuthenticator(c, auth.SilentLogin, app.authOpts) |
| 63 tsrc, err := a.TokenSource() | 63 tsrc, err := a.TokenSource() |
| 64 if err != nil { | 64 if err != nil { |
| 65 » » return nil, errors.Annotate(err).Reason("failed to get token sou
rce").Err() | 65 » » return nil, errors.Annotate(err, "failed to get token source").E
rr() |
| 66 } | 66 } |
| 67 | 67 |
| 68 return bigtable.New(c, bigtable.Options{ | 68 return bigtable.New(c, bigtable.Options{ |
| 69 Project: app.btProject, | 69 Project: app.btProject, |
| 70 Instance: app.btInstance, | 70 Instance: app.btInstance, |
| 71 LogTable: app.btLogTable, | 71 LogTable: app.btLogTable, |
| 72 ClientOptions: []option.ClientOption{ | 72 ClientOptions: []option.ClientOption{ |
| 73 option.WithTokenSource(tsrc), | 73 option.WithTokenSource(tsrc), |
| 74 }, | 74 }, |
| 75 Cache: &memory.Cache{}, | 75 Cache: &memory.Cache{}, |
| (...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 140 // Execute our subcommand. | 140 // Execute our subcommand. |
| 141 return subcommands.Run(&app, fs.Args()) | 141 return subcommands.Run(&app, fs.Args()) |
| 142 } | 142 } |
| 143 | 143 |
| 144 func main() { | 144 func main() { |
| 145 mathrand.SeedRandomly() | 145 mathrand.SeedRandomly() |
| 146 os.Exit(mainImpl(context.Background(), chromeinfra.DefaultAuthOptions(),
os.Args[1:])) | 146 os.Exit(mainImpl(context.Background(), chromeinfra.DefaultAuthOptions(),
os.Args[1:])) |
| 147 } | 147 } |
| 148 | 148 |
| 149 func renderErr(c context.Context, err error) { | 149 func renderErr(c context.Context, err error) { |
| 150 » rerr := errors.RenderStack(err) | 150 » log.Errorf(c, "Error encountered during operation: %s\n%s", err, |
| 151 | 151 » » strings.Join(errors.RenderStack(err), "\n")) |
| 152 » var buf bytes.Buffer | |
| 153 » rerr.DumpTo(&buf) | |
| 154 » log.Errorf(c, "Error encountered during operation: %s\n%s", err, buf.Byt
es()) | |
| 155 } | 152 } |
| 156 | 153 |
| 157 func unmarshalAndDump(c context.Context, out io.Writer, data []byte, msg proto.M
essage) error { | 154 func unmarshalAndDump(c context.Context, out io.Writer, data []byte, msg proto.M
essage) error { |
| 158 if data != nil { | 155 if data != nil { |
| 159 if err := proto.Unmarshal(data, msg); err != nil { | 156 if err := proto.Unmarshal(data, msg); err != nil { |
| 160 log.WithError(err).Errorf(c, "Failed to unmarshal protob
uf.") | 157 log.WithError(err).Errorf(c, "Failed to unmarshal protob
uf.") |
| 161 return err | 158 return err |
| 162 } | 159 } |
| 163 } | 160 } |
| 164 | 161 |
| (...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 207 case cmd.project == "": | 204 case cmd.project == "": |
| 208 log.Errorf(c, "Missing required argument (-project).") | 205 log.Errorf(c, "Missing required argument (-project).") |
| 209 return 1 | 206 return 1 |
| 210 case cmd.path == "": | 207 case cmd.path == "": |
| 211 log.Errorf(c, "Missing required argument (-path).") | 208 log.Errorf(c, "Missing required argument (-path).") |
| 212 return 1 | 209 return 1 |
| 213 } | 210 } |
| 214 | 211 |
| 215 stClient, err := app.getBigTableClient(c) | 212 stClient, err := app.getBigTableClient(c) |
| 216 if err != nil { | 213 if err != nil { |
| 217 » » renderErr(c, errors.Annotate(err).Reason("failed to create stora
ge client").Err()) | 214 » » renderErr(c, errors.Annotate(err, "failed to create storage clie
nt").Err()) |
| 218 return 1 | 215 return 1 |
| 219 } | 216 } |
| 220 defer stClient.Close() | 217 defer stClient.Close() |
| 221 | 218 |
| 222 for round := 0; round < cmd.rounds; round++ { | 219 for round := 0; round < cmd.rounds; round++ { |
| 223 log.Infof(c, "Get round %d.", round+1) | 220 log.Infof(c, "Get round %d.", round+1) |
| 224 | 221 |
| 225 var innerErr error | 222 var innerErr error |
| 226 err = stClient.Get(storage.GetRequest{ | 223 err = stClient.Get(storage.GetRequest{ |
| 227 Project: cfgtypes.ProjectName(cmd.project), | 224 Project: cfgtypes.ProjectName(cmd.project), |
| (...skipping 11 matching lines...) Expand all Loading... |
| 239 "index": le.StreamIndex, | 236 "index": le.StreamIndex, |
| 240 }.Infof(c, "Fetched log entry.") | 237 }.Infof(c, "Fetched log entry.") |
| 241 | 238 |
| 242 if innerErr = unmarshalAndDump(c, os.Stdout, nil, le); i
nnerErr != nil { | 239 if innerErr = unmarshalAndDump(c, os.Stdout, nil, le); i
nnerErr != nil { |
| 243 return false | 240 return false |
| 244 } | 241 } |
| 245 return true | 242 return true |
| 246 }) | 243 }) |
| 247 switch { | 244 switch { |
| 248 case innerErr != nil: | 245 case innerErr != nil: |
| 249 » » » renderErr(c, errors.Annotate(err).Reason("failed to proc
ess fetched log entries").Err()) | 246 » » » renderErr(c, errors.Annotate(err, "failed to process fet
ched log entries").Err()) |
| 250 return 1 | 247 return 1 |
| 251 | 248 |
| 252 case err != nil: | 249 case err != nil: |
| 253 » » » renderErr(c, errors.Annotate(err).Reason("Failed to Get
log entries.").Err()) | 250 » » » renderErr(c, errors.Annotate(err, "Failed to Get log ent
ries.").Err()) |
| 254 return 1 | 251 return 1 |
| 255 } | 252 } |
| 256 } | 253 } |
| 257 | 254 |
| 258 return 0 | 255 return 0 |
| 259 } | 256 } |
| 260 | 257 |
| 261 //////////////////////////////////////////////////////////////////////////////// | 258 //////////////////////////////////////////////////////////////////////////////// |
| 262 // Subcommand: tail | 259 // Subcommand: tail |
| 263 //////////////////////////////////////////////////////////////////////////////// | 260 //////////////////////////////////////////////////////////////////////////////// |
| (...skipping 27 matching lines...) Expand all Loading... |
| 291 case cmd.project == "": | 288 case cmd.project == "": |
| 292 log.Errorf(c, "Missing required argument (-project).") | 289 log.Errorf(c, "Missing required argument (-project).") |
| 293 return 1 | 290 return 1 |
| 294 case cmd.path == "": | 291 case cmd.path == "": |
| 295 log.Errorf(c, "Missing required argument (-path).") | 292 log.Errorf(c, "Missing required argument (-path).") |
| 296 return 1 | 293 return 1 |
| 297 } | 294 } |
| 298 | 295 |
| 299 stClient, err := app.getBigTableClient(c) | 296 stClient, err := app.getBigTableClient(c) |
| 300 if err != nil { | 297 if err != nil { |
| 301 » » renderErr(c, errors.Annotate(err).Reason("failed to create stora
ge client").Err()) | 298 » » renderErr(c, errors.Annotate(err, "failed to create storage clie
nt").Err()) |
| 302 return 1 | 299 return 1 |
| 303 } | 300 } |
| 304 defer stClient.Close() | 301 defer stClient.Close() |
| 305 | 302 |
| 306 for round := 0; round < cmd.rounds; round++ { | 303 for round := 0; round < cmd.rounds; round++ { |
| 307 log.Infof(c, "Tail round %d.", round+1) | 304 log.Infof(c, "Tail round %d.", round+1) |
| 308 e, err := stClient.Tail(cfgtypes.ProjectName(cmd.project), types
.StreamPath(cmd.path)) | 305 e, err := stClient.Tail(cfgtypes.ProjectName(cmd.project), types
.StreamPath(cmd.path)) |
| 309 if err != nil { | 306 if err != nil { |
| 310 » » » renderErr(c, errors.Annotate(err).Reason("failed to tail
log entries").Err()) | 307 » » » renderErr(c, errors.Annotate(err, "failed to tail log en
tries").Err()) |
| 311 return 1 | 308 return 1 |
| 312 } | 309 } |
| 313 | 310 |
| 314 if e == nil { | 311 if e == nil { |
| 315 log.Infof(c, "No log data to tail.") | 312 log.Infof(c, "No log data to tail.") |
| 316 continue | 313 continue |
| 317 } | 314 } |
| 318 | 315 |
| 319 le, err := e.GetLogEntry() | 316 le, err := e.GetLogEntry() |
| 320 if err != nil { | 317 if err != nil { |
| 321 » » » renderErr(c, errors.Annotate(err).Reason("failed to unma
rshal log entry").Err()) | 318 » » » renderErr(c, errors.Annotate(err, "failed to unmarshal l
og entry").Err()) |
| 322 return 1 | 319 return 1 |
| 323 } | 320 } |
| 324 | 321 |
| 325 log.Fields{ | 322 log.Fields{ |
| 326 "index": le.StreamIndex, | 323 "index": le.StreamIndex, |
| 327 "size": len(e.D), | 324 "size": len(e.D), |
| 328 }.Debugf(c, "Dumping tail entry.") | 325 }.Debugf(c, "Dumping tail entry.") |
| 329 if err := unmarshalAndDump(c, os.Stdout, nil, le); err != nil { | 326 if err := unmarshalAndDump(c, os.Stdout, nil, le); err != nil { |
| 330 » » » renderErr(c, errors.Annotate(err).Reason("failed to dump
log entry").Err()) | 327 » » » renderErr(c, errors.Annotate(err, "failed to dump log en
try").Err()) |
| 331 return 1 | 328 return 1 |
| 332 } | 329 } |
| 333 } | 330 } |
| 334 | 331 |
| 335 return 0 | 332 return 0 |
| 336 } | 333 } |
| OLD | NEW |