| OLD | NEW |
| 1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
| 4 | 4 |
| 5 package main | 5 package main |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "encoding/json" | 8 "encoding/json" |
| 9 "errors" | 9 "errors" |
| 10 "fmt" | 10 "fmt" |
| 11 "io/ioutil" | 11 "io/ioutil" |
| 12 "log" | 12 "log" |
| 13 "os" | 13 "os" |
| 14 "path/filepath" | 14 "path/filepath" |
| 15 "strings" | 15 "strings" |
| 16 "time" | 16 "time" |
| 17 | 17 |
| 18 humanize "github.com/dustin/go-humanize" | 18 humanize "github.com/dustin/go-humanize" |
| 19 "github.com/golang/protobuf/proto" | 19 "github.com/golang/protobuf/proto" |
| 20 "github.com/maruel/subcommands" | 20 "github.com/maruel/subcommands" |
| 21 "golang.org/x/net/context" | 21 "golang.org/x/net/context" |
| 22 | 22 |
| 23 "github.com/luci/luci-go/client/isolate" | 23 "github.com/luci/luci-go/client/isolate" |
| 24 "github.com/luci/luci-go/common/auth" | 24 "github.com/luci/luci-go/common/auth" |
| 25 "github.com/luci/luci-go/common/eventlog" | |
| 26 logpb "github.com/luci/luci-go/common/eventlog/proto" | 25 logpb "github.com/luci/luci-go/common/eventlog/proto" |
| 27 "github.com/luci/luci-go/common/isolated" | 26 "github.com/luci/luci-go/common/isolated" |
| 28 "github.com/luci/luci-go/common/isolatedclient" | 27 "github.com/luci/luci-go/common/isolatedclient" |
| 29 ) | 28 ) |
| 30 | 29 |
| 31 const ( | 30 const ( |
| 32 // archiveThreshold is the size (in bytes) used to determine whether to
add | 31 // archiveThreshold is the size (in bytes) used to determine whether to
add |
| 33 // files to a tar archive before uploading. Files smaller than this size
will | 32 // files to a tar archive before uploading. Files smaller than this size
will |
| 34 // be combined into archives before being uploaded to the server. | 33 // be combined into archives before being uploaded to the server. |
| 35 archiveThreshold = 100e3 // 100kB | 34 archiveThreshold = 100e3 // 100kB |
| (...skipping 237 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 273 if err != nil { | 272 if err != nil { |
| 274 return err | 273 return err |
| 275 } | 274 } |
| 276 if err := ioutil.WriteFile(c.dumpJSON, j, 0644); err != nil { | 275 if err := ioutil.WriteFile(c.dumpJSON, j, 0644); err != nil { |
| 277 return err | 276 return err |
| 278 } | 277 } |
| 279 } | 278 } |
| 280 | 279 |
| 281 end := time.Now() | 280 end := time.Now() |
| 282 | 281 |
| 283 » if endpoint := eventlogEndpoint(c.isolateFlags.EventlogEndpoint); endpoi
nt != "" { | 282 » archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{ |
| 284 » » logger := eventlog.NewClient(ctx, endpoint) | 283 » » HitCount: proto.Int64(int64(checker.Hit.Count)), |
| 285 | 284 » » MissCount: proto.Int64(int64(checker.Miss.Count)), |
| 286 » » // TODO(mcgreevy): fill out more stats in archiveDetails. | 285 » » HitBytes: &checker.Hit.Bytes, |
| 287 » » archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{ | 286 » » MissBytes: &checker.Miss.Bytes, |
| 288 » » » HitCount: proto.Int64(int64(checker.Hit.Count)), | 287 » » IsolateHash: []string{string(isolItem.Digest)}, |
| 289 » » » MissCount: proto.Int64(int64(checker.Miss.Count)), | 288 » } |
| 290 » » » HitBytes: &checker.Hit.Bytes, | 289 » eventlogger := NewLogger(ctx, c.isolateFlags.EventlogEndpoint) |
| 291 » » » MissBytes: &checker.Miss.Bytes, | 290 » op := logpb.IsolateClientEvent_ARCHIVE.Enum() |
| 292 » » » IsolateHash: []string{string(isolItem.Digest)}, | 291 » if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err
!= nil { |
| 293 » » } | 292 » » log.Printf("Failed to log to eventlog: %v", err) |
| 294 » » if err := logStats(ctx, logger, start, end, archiveDetails, getB
uildbotInfo()); err != nil { | |
| 295 » » » log.Printf("Failed to log to eventlog: %v", err) | |
| 296 » » } | |
| 297 } | 293 } |
| 298 | 294 |
| 299 return nil | 295 return nil |
| 300 } | 296 } |
| 301 | 297 |
| 302 func (c *expArchiveRun) parseFlags(args []string) error { | 298 func (c *expArchiveRun) parseFlags(args []string) error { |
| 303 if len(args) != 0 { | 299 if len(args) != 0 { |
| 304 return errors.New("position arguments not expected") | 300 return errors.New("position arguments not expected") |
| 305 } | 301 } |
| 306 if err := c.commonServerFlags.Parse(); err != nil { | 302 if err := c.commonServerFlags.Parse(); err != nil { |
| (...skipping 27 matching lines...) Expand all Loading... |
| 334 } | 330 } |
| 335 | 331 |
| 336 func hashFile(path string) (isolated.HexDigest, error) { | 332 func hashFile(path string) (isolated.HexDigest, error) { |
| 337 f, err := os.Open(path) | 333 f, err := os.Open(path) |
| 338 if err != nil { | 334 if err != nil { |
| 339 return "", err | 335 return "", err |
| 340 } | 336 } |
| 341 defer f.Close() | 337 defer f.Close() |
| 342 return isolated.Hash(f) | 338 return isolated.Hash(f) |
| 343 } | 339 } |
| 344 | |
| 345 // buildbotInfo contains information about the build in which this command was r
un. | |
| 346 type buildbotInfo struct { | |
| 347 // Variables which are not present in the environment are nil. | |
| 348 master, builder, buildID, slave *string | |
| 349 } | |
| 350 | |
| 351 // getBuildbotInfo poulates a buildbotInfo with information from the environment
. | |
| 352 func getBuildbotInfo() *buildbotInfo { | |
| 353 getEnvValue := func(key string) *string { | |
| 354 if val, ok := os.LookupEnv(key); ok { | |
| 355 return &val | |
| 356 } | |
| 357 return nil | |
| 358 } | |
| 359 | |
| 360 return &buildbotInfo{ | |
| 361 master: getEnvValue("BUILDBOT_MASTERNAME"), | |
| 362 builder: getEnvValue("BUILDBOT_BUILDERNAME"), | |
| 363 buildID: getEnvValue("BUILDBOT_BUILDNUMBER"), | |
| 364 slave: getEnvValue("BUILDBOT_SLAVENAME"), | |
| 365 } | |
| 366 } | |
| 367 | |
| 368 func logStats(ctx context.Context, logger *eventlog.Client, start, end time.Time
, archiveDetails *logpb.IsolateClientEvent_ArchiveDetails, bi *buildbotInfo) err
or { | |
| 369 event := logger.NewLogEvent(ctx, eventlog.Point()) | |
| 370 event.InfraEvent.IsolateClientEvent = &logpb.IsolateClientEvent{ | |
| 371 Binary: &logpb.Binary{ | |
| 372 Name: proto.String("isolate"), | |
| 373 VersionNumber: proto.String(version), | |
| 374 }, | |
| 375 Operation: logpb.IsolateClientEvent_ARCHIVE.Enum(), | |
| 376 ArchiveDetails: archiveDetails, | |
| 377 Master: bi.master, | |
| 378 Builder: bi.builder, | |
| 379 BuildId: bi.buildID, | |
| 380 Slave: bi.slave, | |
| 381 StartTsUsec: proto.Int64(int64(start.UnixNano() / 1e3)), | |
| 382 EndTsUsec: proto.Int64(int64(end.UnixNano() / 1e3)), | |
| 383 } | |
| 384 return logger.LogSync(ctx, event) | |
| 385 } | |
| 386 | |
| 387 func eventlogEndpoint(endpointFlag string) string { | |
| 388 switch endpointFlag { | |
| 389 case "test": | |
| 390 return eventlog.TestEndpoint | |
| 391 case "prod": | |
| 392 return eventlog.ProdEndpoint | |
| 393 default: | |
| 394 return endpointFlag | |
| 395 } | |
| 396 } | |
| OLD | NEW |