OLD | NEW |
1 // Copyright 2015 The LUCI Authors. | 1 // Copyright 2015 The LUCI Authors. |
2 // | 2 // |
3 // Licensed under the Apache License, Version 2.0 (the "License"); | 3 // Licensed under the Apache License, Version 2.0 (the "License"); |
4 // you may not use this file except in compliance with the License. | 4 // you may not use this file except in compliance with the License. |
5 // You may obtain a copy of the License at | 5 // You may obtain a copy of the License at |
6 // | 6 // |
7 // http://www.apache.org/licenses/LICENSE-2.0 | 7 // http://www.apache.org/licenses/LICENSE-2.0 |
8 // | 8 // |
9 // Unless required by applicable law or agreed to in writing, software | 9 // Unless required by applicable law or agreed to in writing, software |
10 // distributed under the License is distributed on an "AS IS" BASIS, | 10 // distributed under the License is distributed on an "AS IS" BASIS, |
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 // See the License for the specific language governing permissions and | 12 // See the License for the specific language governing permissions and |
13 // limitations under the License. | 13 // limitations under the License. |
14 | 14 |
15 package main | 15 package main |
16 | 16 |
17 import ( | 17 import ( |
18 "encoding/json" | |
19 "errors" | 18 "errors" |
20 "fmt" | 19 "fmt" |
21 "io/ioutil" | |
22 "log" | 20 "log" |
23 "os" | 21 "os" |
24 "path/filepath" | 22 "path/filepath" |
25 "strings" | |
26 "time" | 23 "time" |
27 | 24 |
28 humanize "github.com/dustin/go-humanize" | 25 humanize "github.com/dustin/go-humanize" |
29 "github.com/golang/protobuf/proto" | 26 "github.com/golang/protobuf/proto" |
30 "github.com/maruel/subcommands" | 27 "github.com/maruel/subcommands" |
31 "golang.org/x/net/context" | 28 "golang.org/x/net/context" |
32 | 29 |
33 "github.com/luci/luci-go/client/internal/common" | 30 "github.com/luci/luci-go/client/internal/common" |
34 "github.com/luci/luci-go/client/isolate" | 31 "github.com/luci/luci-go/client/isolate" |
35 "github.com/luci/luci-go/common/auth" | 32 "github.com/luci/luci-go/common/auth" |
(...skipping 174 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
210 filesSize := uint64(parts.filesToArchive.totalSize + parts.indivFiles.to
talSize) | 207 filesSize := uint64(parts.filesToArchive.totalSize + parts.indivFiles.to
talSize) |
211 log.Printf("Isolate expanded to %d files (total size %s) and %d symlinks
", numFiles, humanize.Bytes(filesSize), len(parts.links.items)) | 208 log.Printf("Isolate expanded to %d files (total size %s) and %d symlinks
", numFiles, humanize.Bytes(filesSize), len(parts.links.items)) |
212 log.Printf("\t%d files (%s) to be isolated individually", len(parts.indi
vFiles.items), humanize.Bytes(uint64(parts.indivFiles.totalSize))) | 209 log.Printf("\t%d files (%s) to be isolated individually", len(parts.indi
vFiles.items), humanize.Bytes(uint64(parts.indivFiles.totalSize))) |
213 log.Printf("\t%d files (%s) to be isolated in archives", len(parts.files
ToArchive.items), humanize.Bytes(uint64(parts.filesToArchive.totalSize))) | 210 log.Printf("\t%d files (%s) to be isolated in archives", len(parts.files
ToArchive.items), humanize.Bytes(uint64(parts.filesToArchive.totalSize))) |
214 | 211 |
215 tracker := NewUploadTracker(checker, uploader, isol) | 212 tracker := NewUploadTracker(checker, uploader, isol) |
216 if err := tracker.UploadDeps(parts); err != nil { | 213 if err := tracker.UploadDeps(parts); err != nil { |
217 return err | 214 return err |
218 } | 215 } |
219 | 216 |
220 » isolItem, isolJSON, err := tracker.Finalize(archiveOpts.Isolated) | 217 » isolDigest, err := tracker.Finalize(archiveOpts.Isolated, c.dumpJSON) |
221 if err != nil { | 218 if err != nil { |
222 return err | 219 return err |
223 } | 220 } |
224 | 221 |
225 // Make sure that all pending items have been checked. | |
226 if err := checker.Close(); err != nil { | |
227 return err | |
228 } | |
229 | |
230 // Make sure that all the uploads have completed successfully. | |
231 if err := uploader.Close(); err != nil { | |
232 return err | |
233 } | |
234 | |
235 // Write the isolated file, and emit its digest to stdout. | |
236 if err := ioutil.WriteFile(archiveOpts.Isolated, isolJSON, 0644); err !=
nil { | |
237 return err | |
238 } | |
239 fmt.Printf("%s\t%s\n", isolItem.Digest, filepath.Base(archiveOpts.Isolat
ed)) | |
240 | |
241 // Optionally, write the digest of the isolated file as JSON (in the sam
e | |
242 // format as batch_archive). | |
243 if c.dumpJSON != "" { | |
244 // The name is the base name of the isolated file, extension str
ipped. | |
245 name := filepath.Base(archiveOpts.Isolated) | |
246 if i := strings.LastIndex(name, "."); i != -1 { | |
247 name = name[:i] | |
248 } | |
249 j, err := json.Marshal(map[string]isolated.HexDigest{ | |
250 name: isolItem.Digest, | |
251 }) | |
252 if err != nil { | |
253 return err | |
254 } | |
255 if err := ioutil.WriteFile(c.dumpJSON, j, 0644); err != nil { | |
256 return err | |
257 } | |
258 } | |
259 | |
260 end := time.Now() | 222 end := time.Now() |
261 | 223 |
262 archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{ | 224 archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{ |
263 HitCount: proto.Int64(int64(checker.Hit.Count)), | 225 HitCount: proto.Int64(int64(checker.Hit.Count)), |
264 MissCount: proto.Int64(int64(checker.Miss.Count)), | 226 MissCount: proto.Int64(int64(checker.Miss.Count)), |
265 HitBytes: &checker.Hit.Bytes, | 227 HitBytes: &checker.Hit.Bytes, |
266 MissBytes: &checker.Miss.Bytes, | 228 MissBytes: &checker.Miss.Bytes, |
267 » » IsolateHash: []string{string(isolItem.Digest)}, | 229 » » IsolateHash: []string{string(isolDigest)}, |
268 } | 230 } |
269 eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint) | 231 eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint) |
270 op := logpb.IsolateClientEvent_ARCHIVE.Enum() | 232 op := logpb.IsolateClientEvent_ARCHIVE.Enum() |
271 if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err
!= nil { | 233 if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err
!= nil { |
272 log.Printf("Failed to log to eventlog: %v", err) | 234 log.Printf("Failed to log to eventlog: %v", err) |
273 } | 235 } |
274 | 236 |
275 return nil | 237 return nil |
276 } | 238 } |
277 | 239 |
(...skipping 28 matching lines...) Expand all Loading... |
306 } | 268 } |
307 | 269 |
308 func hashFile(path string) (isolated.HexDigest, error) { | 270 func hashFile(path string) (isolated.HexDigest, error) { |
309 f, err := os.Open(path) | 271 f, err := os.Open(path) |
310 if err != nil { | 272 if err != nil { |
311 return "", err | 273 return "", err |
312 } | 274 } |
313 defer f.Close() | 275 defer f.Close() |
314 return isolated.Hash(f) | 276 return isolated.Hash(f) |
315 } | 277 } |
OLD | NEW |