| OLD | NEW |
| 1 // Copyright 2015 The LUCI Authors. | 1 // Copyright 2015 The LUCI Authors. |
| 2 // | 2 // |
| 3 // Licensed under the Apache License, Version 2.0 (the "License"); | 3 // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 // you may not use this file except in compliance with the License. | 4 // you may not use this file except in compliance with the License. |
| 5 // You may obtain a copy of the License at | 5 // You may obtain a copy of the License at |
| 6 // | 6 // |
| 7 // http://www.apache.org/licenses/LICENSE-2.0 | 7 // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 // | 8 // |
| 9 // Unless required by applicable law or agreed to in writing, software | 9 // Unless required by applicable law or agreed to in writing, software |
| 10 // distributed under the License is distributed on an "AS IS" BASIS, | 10 // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 // See the License for the specific language governing permissions and | 12 // See the License for the specific language governing permissions and |
| 13 // limitations under the License. | 13 // limitations under the License. |
| 14 | 14 |
| 15 package main | 15 package main |
| 16 | 16 |
| 17 import ( | 17 import ( |
| 18 "encoding/json" | |
| 19 "errors" | 18 "errors" |
| 20 "fmt" | 19 "fmt" |
| 21 "io/ioutil" | |
| 22 "log" | 20 "log" |
| 23 "os" | 21 "os" |
| 24 "path/filepath" | 22 "path/filepath" |
| 25 "strings" | |
| 26 "time" | 23 "time" |
| 27 | 24 |
| 28 humanize "github.com/dustin/go-humanize" | 25 humanize "github.com/dustin/go-humanize" |
| 29 "github.com/golang/protobuf/proto" | 26 "github.com/golang/protobuf/proto" |
| 30 "github.com/maruel/subcommands" | 27 "github.com/maruel/subcommands" |
| 31 "golang.org/x/net/context" | 28 "golang.org/x/net/context" |
| 32 | 29 |
| 33 "github.com/luci/luci-go/client/internal/common" | 30 "github.com/luci/luci-go/client/internal/common" |
| 34 "github.com/luci/luci-go/client/isolate" | 31 "github.com/luci/luci-go/client/isolate" |
| 35 "github.com/luci/luci-go/common/auth" | 32 "github.com/luci/luci-go/common/auth" |
| (...skipping 174 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 210 filesSize := uint64(parts.filesToArchive.totalSize + parts.indivFiles.to
talSize) | 207 filesSize := uint64(parts.filesToArchive.totalSize + parts.indivFiles.to
talSize) |
| 211 log.Printf("Isolate expanded to %d files (total size %s) and %d symlinks
", numFiles, humanize.Bytes(filesSize), len(parts.links.items)) | 208 log.Printf("Isolate expanded to %d files (total size %s) and %d symlinks
", numFiles, humanize.Bytes(filesSize), len(parts.links.items)) |
| 212 log.Printf("\t%d files (%s) to be isolated individually", len(parts.indi
vFiles.items), humanize.Bytes(uint64(parts.indivFiles.totalSize))) | 209 log.Printf("\t%d files (%s) to be isolated individually", len(parts.indi
vFiles.items), humanize.Bytes(uint64(parts.indivFiles.totalSize))) |
| 213 log.Printf("\t%d files (%s) to be isolated in archives", len(parts.files
ToArchive.items), humanize.Bytes(uint64(parts.filesToArchive.totalSize))) | 210 log.Printf("\t%d files (%s) to be isolated in archives", len(parts.files
ToArchive.items), humanize.Bytes(uint64(parts.filesToArchive.totalSize))) |
| 214 | 211 |
| 215 tracker := NewUploadTracker(checker, uploader, isol) | 212 tracker := NewUploadTracker(checker, uploader, isol) |
| 216 if err := tracker.UploadDeps(parts); err != nil { | 213 if err := tracker.UploadDeps(parts); err != nil { |
| 217 return err | 214 return err |
| 218 } | 215 } |
| 219 | 216 |
| 220 » isolItem, isolJSON, err := tracker.Finalize(archiveOpts.Isolated) | 217 » isolDigest, err := tracker.Finalize(archiveOpts.Isolated, c.dumpJSON) |
| 221 if err != nil { | 218 if err != nil { |
| 222 return err | 219 return err |
| 223 } | 220 } |
| 224 | 221 |
| 225 // Make sure that all pending items have been checked. | |
| 226 if err := checker.Close(); err != nil { | |
| 227 return err | |
| 228 } | |
| 229 | |
| 230 // Make sure that all the uploads have completed successfully. | |
| 231 if err := uploader.Close(); err != nil { | |
| 232 return err | |
| 233 } | |
| 234 | |
| 235 // Write the isolated file, and emit its digest to stdout. | |
| 236 if err := ioutil.WriteFile(archiveOpts.Isolated, isolJSON, 0644); err !=
nil { | |
| 237 return err | |
| 238 } | |
| 239 fmt.Printf("%s\t%s\n", isolItem.Digest, filepath.Base(archiveOpts.Isolat
ed)) | |
| 240 | |
| 241 // Optionally, write the digest of the isolated file as JSON (in the sam
e | |
| 242 // format as batch_archive). | |
| 243 if c.dumpJSON != "" { | |
| 244 // The name is the base name of the isolated file, extension str
ipped. | |
| 245 name := filepath.Base(archiveOpts.Isolated) | |
| 246 if i := strings.LastIndex(name, "."); i != -1 { | |
| 247 name = name[:i] | |
| 248 } | |
| 249 j, err := json.Marshal(map[string]isolated.HexDigest{ | |
| 250 name: isolItem.Digest, | |
| 251 }) | |
| 252 if err != nil { | |
| 253 return err | |
| 254 } | |
| 255 if err := ioutil.WriteFile(c.dumpJSON, j, 0644); err != nil { | |
| 256 return err | |
| 257 } | |
| 258 } | |
| 259 | |
| 260 end := time.Now() | 222 end := time.Now() |
| 261 | 223 |
| 262 archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{ | 224 archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{ |
| 263 HitCount: proto.Int64(int64(checker.Hit.Count)), | 225 HitCount: proto.Int64(int64(checker.Hit.Count)), |
| 264 MissCount: proto.Int64(int64(checker.Miss.Count)), | 226 MissCount: proto.Int64(int64(checker.Miss.Count)), |
| 265 HitBytes: &checker.Hit.Bytes, | 227 HitBytes: &checker.Hit.Bytes, |
| 266 MissBytes: &checker.Miss.Bytes, | 228 MissBytes: &checker.Miss.Bytes, |
| 267 » » IsolateHash: []string{string(isolItem.Digest)}, | 229 » » IsolateHash: []string{string(isolDigest)}, |
| 268 } | 230 } |
| 269 eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint) | 231 eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint) |
| 270 op := logpb.IsolateClientEvent_ARCHIVE.Enum() | 232 op := logpb.IsolateClientEvent_ARCHIVE.Enum() |
| 271 if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err
!= nil { | 233 if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err
!= nil { |
| 272 log.Printf("Failed to log to eventlog: %v", err) | 234 log.Printf("Failed to log to eventlog: %v", err) |
| 273 } | 235 } |
| 274 | 236 |
| 275 return nil | 237 return nil |
| 276 } | 238 } |
| 277 | 239 |
| (...skipping 28 matching lines...) Expand all Loading... |
| 306 } | 268 } |
| 307 | 269 |
| 308 func hashFile(path string) (isolated.HexDigest, error) { | 270 func hashFile(path string) (isolated.HexDigest, error) { |
| 309 f, err := os.Open(path) | 271 f, err := os.Open(path) |
| 310 if err != nil { | 272 if err != nil { |
| 311 return "", err | 273 return "", err |
| 312 } | 274 } |
| 313 defer f.Close() | 275 defer f.Close() |
| 314 return isolated.Hash(f) | 276 return isolated.Hash(f) |
| 315 } | 277 } |
| OLD | NEW |