| OLD | NEW |
| 1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
| 4 | 4 |
| 5 package main | 5 package main |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "encoding/json" | 8 "encoding/json" |
| 9 "errors" | 9 "errors" |
| 10 "fmt" | 10 "fmt" |
| 11 "io" | 11 "io" |
| 12 "log" |
| 12 "os" | 13 "os" |
| 13 "path/filepath" | 14 "path/filepath" |
| 14 "strings" | 15 "strings" |
| 15 "sync" | 16 "sync" |
| 16 "time" | 17 "time" |
| 17 | 18 |
| 19 "github.com/golang/protobuf/proto" |
| 18 "github.com/maruel/subcommands" | 20 "github.com/maruel/subcommands" |
| 19 | 21 |
| 20 "github.com/luci/luci-go/client/archiver" | 22 "github.com/luci/luci-go/client/archiver" |
| 21 "github.com/luci/luci-go/client/isolate" | 23 "github.com/luci/luci-go/client/isolate" |
| 22 "github.com/luci/luci-go/common/auth" | 24 "github.com/luci/luci-go/common/auth" |
| 23 "github.com/luci/luci-go/common/data/text/units" | 25 "github.com/luci/luci-go/common/data/text/units" |
| 26 logpb "github.com/luci/luci-go/common/eventlog/proto" |
| 24 "github.com/luci/luci-go/common/isolated" | 27 "github.com/luci/luci-go/common/isolated" |
| 25 "github.com/luci/luci-go/common/isolatedclient" | 28 "github.com/luci/luci-go/common/isolatedclient" |
| 26 ) | 29 ) |
| 27 | 30 |
| 28 func cmdBatchArchive(defaultAuthOpts auth.Options) *subcommands.Command { | 31 func cmdBatchArchive(defaultAuthOpts auth.Options) *subcommands.Command { |
| 29 return &subcommands.Command{ | 32 return &subcommands.Command{ |
| 30 UsageLine: "batcharchive <options> file1 file2 ...", | 33 UsageLine: "batcharchive <options> file1 file2 ...", |
| 31 ShortDesc: "archives multiple isolated trees at once.", | 34 ShortDesc: "archives multiple isolated trees at once.", |
| 32 LongDesc: `Archives multiple isolated trees at once. | 35 LongDesc: `Archives multiple isolated trees at once. |
| 33 | 36 |
| 34 Using single command instead of multiple sequential invocations allows to cut | 37 Using single command instead of multiple sequential invocations allows to cut |
| 35 redundant work when isolated trees share common files (e.g. file hashes are | 38 redundant work when isolated trees share common files (e.g. file hashes are |
| 36 checked only once, their presence on the server is checked only once, and | 39 checked only once, their presence on the server is checked only once, and |
| 37 so on). | 40 so on). |
| 38 | 41 |
| 39 Takes a list of paths to *.isolated.gen.json files that describe what trees to | 42 Takes a list of paths to *.isolated.gen.json files that describe what trees to |
| 40 isolate. Format of files is: | 43 isolate. Format of files is: |
| 41 { | 44 { |
| 42 "version": 1, | 45 "version": 1, |
| 43 "dir": <absolute path to a directory all other paths are relative to>, | 46 "dir": <absolute path to a directory all other paths are relative to>, |
| 44 "args": [list of command line arguments for single 'archive' command] | 47 "args": [list of command line arguments for single 'archive' command] |
| 45 }`, | 48 }`, |
| 46 CommandRun: func() subcommands.CommandRun { | 49 CommandRun: func() subcommands.CommandRun { |
| 47 c := batchArchiveRun{} | 50 c := batchArchiveRun{} |
| 48 c.commonServerFlags.Init(defaultAuthOpts) | 51 c.commonServerFlags.Init(defaultAuthOpts) |
| 52 c.loggingFlags.Init(&c.Flags) |
| 49 c.Flags.StringVar(&c.dumpJSON, "dump-json", "", | 53 c.Flags.StringVar(&c.dumpJSON, "dump-json", "", |
| 50 "Write isolated digests of archived trees to thi
s file as JSON") | 54 "Write isolated digests of archived trees to thi
s file as JSON") |
| 51 return &c | 55 return &c |
| 52 }, | 56 }, |
| 53 } | 57 } |
| 54 } | 58 } |
| 55 | 59 |
| 56 type batchArchiveRun struct { | 60 type batchArchiveRun struct { |
| 57 commonServerFlags | 61 commonServerFlags |
| 58 » dumpJSON string | 62 » loggingFlags loggingFlags |
| 63 » dumpJSON string |
| 59 } | 64 } |
| 60 | 65 |
| 61 func (c *batchArchiveRun) Parse(a subcommands.Application, args []string) error
{ | 66 func (c *batchArchiveRun) Parse(a subcommands.Application, args []string) error
{ |
| 62 if err := c.commonServerFlags.Parse(); err != nil { | 67 if err := c.commonServerFlags.Parse(); err != nil { |
| 63 return err | 68 return err |
| 64 } | 69 } |
| 65 if len(args) == 0 { | 70 if len(args) == 0 { |
| 66 return errors.New("at least one isolate file required") | 71 return errors.New("at least one isolate file required") |
| 67 } | 72 } |
| 68 return nil | 73 return nil |
| (...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 155 } | 160 } |
| 156 } | 161 } |
| 157 }(arg) | 162 }(arg) |
| 158 } | 163 } |
| 159 go func() { | 164 go func() { |
| 160 wg.Wait() | 165 wg.Wait() |
| 161 close(items) | 166 close(items) |
| 162 }() | 167 }() |
| 163 | 168 |
| 164 data := map[string]isolated.HexDigest{} | 169 data := map[string]isolated.HexDigest{} |
| 170 var digests []string |
| 165 for item := range items { | 171 for item := range items { |
| 166 item.WaitForHashed() | 172 item.WaitForHashed() |
| 167 if item.Error() == nil { | 173 if item.Error() == nil { |
| 168 » » » data[item.name] = item.Digest() | 174 » » » d := item.Digest() |
| 169 » » » fmt.Printf("%s%s %s\n", prefix, item.Digest(), item.nam
e) | 175 » » » data[item.name] = d |
| 176 » » » digests = append(digests, string(d)) |
| 177 » » » fmt.Printf("%s%s %s\n", prefix, d, item.name) |
| 170 } else { | 178 } else { |
| 171 fmt.Fprintf(os.Stderr, "%s%s %s\n", prefix, item.name,
item.Error()) | 179 fmt.Fprintf(os.Stderr, "%s%s %s\n", prefix, item.name,
item.Error()) |
| 172 } | 180 } |
| 173 } | 181 } |
| 174 err = arch.Close() | 182 err = arch.Close() |
| 175 duration := time.Since(start) | 183 duration := time.Since(start) |
| 176 // Only write the file once upload is confirmed. | 184 // Only write the file once upload is confirmed. |
| 177 if err == nil && c.dumpJSON != "" { | 185 if err == nil && c.dumpJSON != "" { |
| 178 err = writeJSONDigestFile(c.dumpJSON, data) | 186 err = writeJSONDigestFile(c.dumpJSON, data) |
| 179 } | 187 } |
| 188 |
| 189 stats := arch.Stats() |
| 180 if !c.defaultFlags.Quiet { | 190 if !c.defaultFlags.Quiet { |
| 181 stats := arch.Stats() | |
| 182 fmt.Fprintf(os.Stderr, "Hits : %5d (%s)\n", stats.TotalHits()
, stats.TotalBytesHits()) | 191 fmt.Fprintf(os.Stderr, "Hits : %5d (%s)\n", stats.TotalHits()
, stats.TotalBytesHits()) |
| 183 fmt.Fprintf(os.Stderr, "Misses : %5d (%s)\n", stats.TotalMisses
(), stats.TotalBytesPushed()) | 192 fmt.Fprintf(os.Stderr, "Misses : %5d (%s)\n", stats.TotalMisses
(), stats.TotalBytesPushed()) |
| 184 fmt.Fprintf(os.Stderr, "Duration: %s\n", units.Round(duration, t
ime.Millisecond)) | 193 fmt.Fprintf(os.Stderr, "Duration: %s\n", units.Round(duration, t
ime.Millisecond)) |
| 185 } | 194 } |
| 195 |
| 196 end := time.Now() |
| 197 archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{ |
| 198 HitCount: proto.Int64(int64(stats.TotalHits())), |
| 199 MissCount: proto.Int64(int64(stats.TotalMisses())), |
| 200 HitBytes: proto.Int64(int64(stats.TotalBytesHits())), |
| 201 MissBytes: proto.Int64(int64(stats.TotalBytesPushed())), |
| 202 IsolateHash: digests, |
| 203 } |
| 204 |
| 205 eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint) |
| 206 op := logpb.IsolateClientEvent_BATCH_ARCHIVE.Enum() |
| 207 if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err
!= nil { |
| 208 log.Printf("Failed to log to eventlog: %v", err) |
| 209 } |
| 210 |
| 186 return err | 211 return err |
| 187 } | 212 } |
| 188 | 213 |
| 189 // processGenJSON validates a genJSON file and returns the contents. | 214 // processGenJSON validates a genJSON file and returns the contents. |
| 190 func processGenJSON(genJSONPath string) (*isolate.ArchiveOptions, error) { | 215 func processGenJSON(genJSONPath string) (*isolate.ArchiveOptions, error) { |
| 191 f, err := os.Open(genJSONPath) | 216 f, err := os.Open(genJSONPath) |
| 192 if err != nil { | 217 if err != nil { |
| 193 return nil, fmt.Errorf("opening %s: %s", genJSONPath, err) | 218 return nil, fmt.Errorf("opening %s: %s", genJSONPath, err) |
| 194 } | 219 } |
| 195 defer f.Close() | 220 defer f.Close() |
| (...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 273 fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err) | 298 fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err) |
| 274 return 1 | 299 return 1 |
| 275 } | 300 } |
| 276 defer cl.Close() | 301 defer cl.Close() |
| 277 if err := c.main(a, args); err != nil { | 302 if err := c.main(a, args); err != nil { |
| 278 fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err) | 303 fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err) |
| 279 return 1 | 304 return 1 |
| 280 } | 305 } |
| 281 return 0 | 306 return 0 |
| 282 } | 307 } |
| OLD | NEW |