Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(60)

Side by Side Diff: client/cmd/isolate/batch_archive.go

Issue 2923133002: Add eventlogging to batcharchive. (Closed)
Patch Set: rebase Created 3 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2015 The LUCI Authors. All rights reserved. 1 // Copyright 2015 The LUCI Authors. All rights reserved.
2 // Use of this source code is governed under the Apache License, Version 2.0 2 // Use of this source code is governed under the Apache License, Version 2.0
3 // that can be found in the LICENSE file. 3 // that can be found in the LICENSE file.
4 4
5 package main 5 package main
6 6
7 import ( 7 import (
8 "encoding/json" 8 "encoding/json"
9 "errors" 9 "errors"
10 "fmt" 10 "fmt"
11 "io" 11 "io"
12 "log"
12 "os" 13 "os"
13 "path/filepath" 14 "path/filepath"
14 "strings" 15 "strings"
15 "sync" 16 "sync"
16 "time" 17 "time"
17 18
19 "github.com/golang/protobuf/proto"
18 "github.com/maruel/subcommands" 20 "github.com/maruel/subcommands"
19 21
20 "github.com/luci/luci-go/client/archiver" 22 "github.com/luci/luci-go/client/archiver"
21 "github.com/luci/luci-go/client/isolate" 23 "github.com/luci/luci-go/client/isolate"
22 "github.com/luci/luci-go/common/auth" 24 "github.com/luci/luci-go/common/auth"
23 "github.com/luci/luci-go/common/data/text/units" 25 "github.com/luci/luci-go/common/data/text/units"
26 logpb "github.com/luci/luci-go/common/eventlog/proto"
24 "github.com/luci/luci-go/common/isolated" 27 "github.com/luci/luci-go/common/isolated"
25 "github.com/luci/luci-go/common/isolatedclient" 28 "github.com/luci/luci-go/common/isolatedclient"
26 ) 29 )
27 30
28 func cmdBatchArchive(defaultAuthOpts auth.Options) *subcommands.Command { 31 func cmdBatchArchive(defaultAuthOpts auth.Options) *subcommands.Command {
29 return &subcommands.Command{ 32 return &subcommands.Command{
30 UsageLine: "batcharchive <options> file1 file2 ...", 33 UsageLine: "batcharchive <options> file1 file2 ...",
31 ShortDesc: "archives multiple isolated trees at once.", 34 ShortDesc: "archives multiple isolated trees at once.",
32 LongDesc: `Archives multiple isolated trees at once. 35 LongDesc: `Archives multiple isolated trees at once.
33 36
34 Using single command instead of multiple sequential invocations allows to cut 37 Using single command instead of multiple sequential invocations allows to cut
35 redundant work when isolated trees share common files (e.g. file hashes are 38 redundant work when isolated trees share common files (e.g. file hashes are
36 checked only once, their presence on the server is checked only once, and 39 checked only once, their presence on the server is checked only once, and
37 so on). 40 so on).
38 41
39 Takes a list of paths to *.isolated.gen.json files that describe what trees to 42 Takes a list of paths to *.isolated.gen.json files that describe what trees to
40 isolate. Format of files is: 43 isolate. Format of files is:
41 { 44 {
42 "version": 1, 45 "version": 1,
43 "dir": <absolute path to a directory all other paths are relative to>, 46 "dir": <absolute path to a directory all other paths are relative to>,
44 "args": [list of command line arguments for single 'archive' command] 47 "args": [list of command line arguments for single 'archive' command]
45 }`, 48 }`,
46 CommandRun: func() subcommands.CommandRun { 49 CommandRun: func() subcommands.CommandRun {
47 c := batchArchiveRun{} 50 c := batchArchiveRun{}
48 c.commonServerFlags.Init(defaultAuthOpts) 51 c.commonServerFlags.Init(defaultAuthOpts)
52 c.loggingFlags.Init(&c.Flags)
49 c.Flags.StringVar(&c.dumpJSON, "dump-json", "", 53 c.Flags.StringVar(&c.dumpJSON, "dump-json", "",
50 "Write isolated digests of archived trees to thi s file as JSON") 54 "Write isolated digests of archived trees to thi s file as JSON")
51 return &c 55 return &c
52 }, 56 },
53 } 57 }
54 } 58 }
55 59
56 type batchArchiveRun struct { 60 type batchArchiveRun struct {
57 commonServerFlags 61 commonServerFlags
58 » dumpJSON string 62 » loggingFlags loggingFlags
63 » dumpJSON string
59 } 64 }
60 65
61 func (c *batchArchiveRun) Parse(a subcommands.Application, args []string) error { 66 func (c *batchArchiveRun) Parse(a subcommands.Application, args []string) error {
62 if err := c.commonServerFlags.Parse(); err != nil { 67 if err := c.commonServerFlags.Parse(); err != nil {
63 return err 68 return err
64 } 69 }
65 if len(args) == 0 { 70 if len(args) == 0 {
66 return errors.New("at least one isolate file required") 71 return errors.New("at least one isolate file required")
67 } 72 }
68 return nil 73 return nil
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after
155 } 160 }
156 } 161 }
157 }(arg) 162 }(arg)
158 } 163 }
159 go func() { 164 go func() {
160 wg.Wait() 165 wg.Wait()
161 close(items) 166 close(items)
162 }() 167 }()
163 168
164 data := map[string]isolated.HexDigest{} 169 data := map[string]isolated.HexDigest{}
170 var digests []string
165 for item := range items { 171 for item := range items {
166 item.WaitForHashed() 172 item.WaitForHashed()
167 if item.Error() == nil { 173 if item.Error() == nil {
168 » » » data[item.name] = item.Digest() 174 » » » d := item.Digest()
169 » » » fmt.Printf("%s%s %s\n", prefix, item.Digest(), item.nam e) 175 » » » data[item.name] = d
176 » » » digests = append(digests, string(d))
177 » » » fmt.Printf("%s%s %s\n", prefix, d, item.name)
170 } else { 178 } else {
171 fmt.Fprintf(os.Stderr, "%s%s %s\n", prefix, item.name, item.Error()) 179 fmt.Fprintf(os.Stderr, "%s%s %s\n", prefix, item.name, item.Error())
172 } 180 }
173 } 181 }
174 err = arch.Close() 182 err = arch.Close()
175 duration := time.Since(start) 183 duration := time.Since(start)
176 // Only write the file once upload is confirmed. 184 // Only write the file once upload is confirmed.
177 if err == nil && c.dumpJSON != "" { 185 if err == nil && c.dumpJSON != "" {
178 err = writeJSONDigestFile(c.dumpJSON, data) 186 err = writeJSONDigestFile(c.dumpJSON, data)
179 } 187 }
188
189 stats := arch.Stats()
180 if !c.defaultFlags.Quiet { 190 if !c.defaultFlags.Quiet {
181 stats := arch.Stats()
182 fmt.Fprintf(os.Stderr, "Hits : %5d (%s)\n", stats.TotalHits() , stats.TotalBytesHits()) 191 fmt.Fprintf(os.Stderr, "Hits : %5d (%s)\n", stats.TotalHits() , stats.TotalBytesHits())
183 fmt.Fprintf(os.Stderr, "Misses : %5d (%s)\n", stats.TotalMisses (), stats.TotalBytesPushed()) 192 fmt.Fprintf(os.Stderr, "Misses : %5d (%s)\n", stats.TotalMisses (), stats.TotalBytesPushed())
184 fmt.Fprintf(os.Stderr, "Duration: %s\n", units.Round(duration, t ime.Millisecond)) 193 fmt.Fprintf(os.Stderr, "Duration: %s\n", units.Round(duration, t ime.Millisecond))
185 } 194 }
195
196 end := time.Now()
197 archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{
198 HitCount: proto.Int64(int64(stats.TotalHits())),
199 MissCount: proto.Int64(int64(stats.TotalMisses())),
200 HitBytes: proto.Int64(int64(stats.TotalBytesHits())),
201 MissBytes: proto.Int64(int64(stats.TotalBytesPushed())),
202 IsolateHash: digests,
203 }
204
205 eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint)
206 op := logpb.IsolateClientEvent_BATCH_ARCHIVE.Enum()
207 if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err != nil {
208 log.Printf("Failed to log to eventlog: %v", err)
209 }
210
186 return err 211 return err
187 } 212 }
188 213
189 // processGenJSON validates a genJSON file and returns the contents. 214 // processGenJSON validates a genJSON file and returns the contents.
190 func processGenJSON(genJSONPath string) (*isolate.ArchiveOptions, error) { 215 func processGenJSON(genJSONPath string) (*isolate.ArchiveOptions, error) {
191 f, err := os.Open(genJSONPath) 216 f, err := os.Open(genJSONPath)
192 if err != nil { 217 if err != nil {
193 return nil, fmt.Errorf("opening %s: %s", genJSONPath, err) 218 return nil, fmt.Errorf("opening %s: %s", genJSONPath, err)
194 } 219 }
195 defer f.Close() 220 defer f.Close()
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
273 fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err) 298 fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
274 return 1 299 return 1
275 } 300 }
276 defer cl.Close() 301 defer cl.Close()
277 if err := c.main(a, args); err != nil { 302 if err := c.main(a, args); err != nil {
278 fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err) 303 fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
279 return 1 304 return 1
280 } 305 }
281 return 0 306 return 0
282 } 307 }
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698