| OLD | NEW |
| 1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
| 4 | 4 |
| 5 package main | 5 package main |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "encoding/json" | 8 "encoding/json" |
| 9 "errors" | 9 "errors" |
| 10 "fmt" | 10 "fmt" |
| (...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 94 ctx, cancel := context.WithCancel(context.Background()) | 94 ctx, cancel := context.WithCancel(context.Background()) |
| 95 defer cancel() | 95 defer cancel() |
| 96 | 96 |
| 97 // Create the isolated client which connects to the isolate server. | 97 // Create the isolated client which connects to the isolate server. |
| 98 authCl, err := c.createAuthClient() | 98 authCl, err := c.createAuthClient() |
| 99 if err != nil { | 99 if err != nil { |
| 100 return err | 100 return err |
| 101 } | 101 } |
| 102 client := isolatedclient.New(nil, authCl, c.isolatedFlags.ServerURL, c.i
solatedFlags.Namespace, nil, nil) | 102 client := isolatedclient.New(nil, authCl, c.isolatedFlags.ServerURL, c.i
solatedFlags.Namespace, nil, nil) |
| 103 | 103 |
| 104 » // Set up a checker an uploader. We limit the uploader to one concurrent | 104 » // Set up a checker and uploader. We limit the uploader to one concurren
t |
| 105 // upload, since the uploads are all coming from disk (with the exceptio
n of | 105 // upload, since the uploads are all coming from disk (with the exceptio
n of |
| 106 // the isolated JSON itself) and we only want a single goroutine reading
from | 106 // the isolated JSON itself) and we only want a single goroutine reading
from |
| 107 // disk at once. | 107 // disk at once. |
| 108 checker := NewChecker(ctx, client) | 108 checker := NewChecker(ctx, client) |
| 109 uploader := NewUploader(ctx, client, 1) | 109 uploader := NewUploader(ctx, client, 1) |
| 110 | 110 |
| 111 // Walk each of the deps, partioning the results into symlinks and files
categorised by size. | 111 // Walk each of the deps, partioning the results into symlinks and files
categorised by size. |
| 112 var links, archiveFiles, indivFiles []*Item | 112 var links, archiveFiles, indivFiles []*Item |
| 113 var archiveSize, indivSize int64 // Cumulative size of archived/individu
al files. | 113 var archiveSize, indivSize int64 // Cumulative size of archived/individu
al files. |
| 114 for _, dep := range deps { | 114 for _, dep := range deps { |
| (...skipping 169 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 284 logger := eventlog.NewClient(ctx, endpoint) | 284 logger := eventlog.NewClient(ctx, endpoint) |
| 285 | 285 |
| 286 // TODO(mcgreevy): fill out more stats in archiveDetails. | 286 // TODO(mcgreevy): fill out more stats in archiveDetails. |
| 287 archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{ | 287 archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{ |
| 288 HitCount: proto.Int64(int64(checker.Hit.Count)), | 288 HitCount: proto.Int64(int64(checker.Hit.Count)), |
| 289 MissCount: proto.Int64(int64(checker.Miss.Count)), | 289 MissCount: proto.Int64(int64(checker.Miss.Count)), |
| 290 HitBytes: &checker.Hit.Bytes, | 290 HitBytes: &checker.Hit.Bytes, |
| 291 MissBytes: &checker.Miss.Bytes, | 291 MissBytes: &checker.Miss.Bytes, |
| 292 IsolateHash: []string{string(isolItem.Digest)}, | 292 IsolateHash: []string{string(isolItem.Digest)}, |
| 293 } | 293 } |
| 294 » » if err := logStats(ctx, logger, start, end, archiveDetails); err
!= nil { | 294 » » if err := logStats(ctx, logger, start, end, archiveDetails, getB
uildbotInfo()); err != nil { |
| 295 log.Printf("Failed to log to eventlog: %v", err) | 295 log.Printf("Failed to log to eventlog: %v", err) |
| 296 } | 296 } |
| 297 } | 297 } |
| 298 | 298 |
| 299 return nil | 299 return nil |
| 300 } | 300 } |
| 301 | 301 |
| 302 func (c *expArchiveRun) parseFlags(args []string) error { | 302 func (c *expArchiveRun) parseFlags(args []string) error { |
| 303 if len(args) != 0 { | 303 if len(args) != 0 { |
| 304 return errors.New("position arguments not expected") | 304 return errors.New("position arguments not expected") |
| (...skipping 30 matching lines...) Expand all Loading... |
| 335 | 335 |
| 336 func hashFile(path string) (isolated.HexDigest, error) { | 336 func hashFile(path string) (isolated.HexDigest, error) { |
| 337 f, err := os.Open(path) | 337 f, err := os.Open(path) |
| 338 if err != nil { | 338 if err != nil { |
| 339 return "", err | 339 return "", err |
| 340 } | 340 } |
| 341 defer f.Close() | 341 defer f.Close() |
| 342 return isolated.Hash(f) | 342 return isolated.Hash(f) |
| 343 } | 343 } |
| 344 | 344 |
| 345 func logStats(ctx context.Context, logger *eventlog.Client, start, end time.Time
, archiveDetails *logpb.IsolateClientEvent_ArchiveDetails) error { | 345 // buildbotInfo contains information about the build in which this command was r
un. |
| 346 type buildbotInfo struct { |
| 347 » // Variables which are not present in the environment are nil. |
| 348 » master, builder, buildID, slave *string |
| 349 } |
| 350 |
| 351 // getBuildbotInfo poulates a buildbotInfo with information from the environment
. |
| 352 func getBuildbotInfo() *buildbotInfo { |
| 353 » getEnvValue := func(key string) *string { |
| 354 » » if val, ok := os.LookupEnv(key); ok { |
| 355 » » » return &val |
| 356 » » } |
| 357 » » return nil |
| 358 » } |
| 359 |
| 360 » return &buildbotInfo{ |
| 361 » » master: getEnvValue("BUILDBOT_MASTERNAME"), |
| 362 » » builder: getEnvValue("BUILDBOT_BUILDERNAME"), |
| 363 » » buildID: getEnvValue("BUILDBOT_BUILDNUMBER"), |
| 364 » » slave: getEnvValue("BUILDBOT_SLAVENAME"), |
| 365 » } |
| 366 } |
| 367 |
| 368 func logStats(ctx context.Context, logger *eventlog.Client, start, end time.Time
, archiveDetails *logpb.IsolateClientEvent_ArchiveDetails, bi *buildbotInfo) err
or { |
| 346 event := logger.NewLogEvent(ctx, eventlog.Point()) | 369 event := logger.NewLogEvent(ctx, eventlog.Point()) |
| 347 event.InfraEvent.IsolateClientEvent = &logpb.IsolateClientEvent{ | 370 event.InfraEvent.IsolateClientEvent = &logpb.IsolateClientEvent{ |
| 348 Binary: &logpb.Binary{ | 371 Binary: &logpb.Binary{ |
| 349 Name: proto.String("isolate"), | 372 Name: proto.String("isolate"), |
| 350 VersionNumber: proto.String(version), | 373 VersionNumber: proto.String(version), |
| 351 }, | 374 }, |
| 352 Operation: logpb.IsolateClientEvent_ARCHIVE.Enum(), | 375 Operation: logpb.IsolateClientEvent_ARCHIVE.Enum(), |
| 353 ArchiveDetails: archiveDetails, | 376 ArchiveDetails: archiveDetails, |
| 354 » » // TODO(mcgreevy): fill out Master, Builder, BuildId, Slave. | 377 » » Master: bi.master, |
| 355 » » StartTsUsec: proto.Int64(int64(start.UnixNano() / 1e3)), | 378 » » Builder: bi.builder, |
| 356 » » EndTsUsec: proto.Int64(int64(end.UnixNano() / 1e3)), | 379 » » BuildId: bi.buildID, |
| 380 » » Slave: bi.slave, |
| 381 » » StartTsUsec: proto.Int64(int64(start.UnixNano() / 1e3)), |
| 382 » » EndTsUsec: proto.Int64(int64(end.UnixNano() / 1e3)), |
| 357 } | 383 } |
| 358 return logger.LogSync(ctx, event) | 384 return logger.LogSync(ctx, event) |
| 359 } | 385 } |
| 360 | 386 |
| 361 func eventlogEndpoint(endpointFlag string) string { | 387 func eventlogEndpoint(endpointFlag string) string { |
| 362 switch endpointFlag { | 388 switch endpointFlag { |
| 363 case "test": | 389 case "test": |
| 364 return eventlog.TestEndpoint | 390 return eventlog.TestEndpoint |
| 365 case "prod": | 391 case "prod": |
| 366 return eventlog.ProdEndpoint | 392 return eventlog.ProdEndpoint |
| 367 default: | 393 default: |
| 368 return endpointFlag | 394 return endpointFlag |
| 369 } | 395 } |
| 370 } | 396 } |
| OLD | NEW |