Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(634)

Side by Side Diff: client/cmd/isolate/exp_archive.go

Issue 2991503002: isolate: Finalize the isolate in UploadTracker (Closed)
Patch Set: Created 3 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | client/cmd/isolate/upload_tracker.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2015 The LUCI Authors. 1 // Copyright 2015 The LUCI Authors.
2 // 2 //
3 // Licensed under the Apache License, Version 2.0 (the "License"); 3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License. 4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at 5 // You may obtain a copy of the License at
6 // 6 //
7 // http://www.apache.org/licenses/LICENSE-2.0 7 // http://www.apache.org/licenses/LICENSE-2.0
8 // 8 //
9 // Unless required by applicable law or agreed to in writing, software 9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS, 10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and 12 // See the License for the specific language governing permissions and
13 // limitations under the License. 13 // limitations under the License.
14 14
15 package main 15 package main
16 16
17 import ( 17 import (
18 "encoding/json"
19 "errors" 18 "errors"
20 "fmt" 19 "fmt"
21 "io/ioutil"
22 "log" 20 "log"
23 "os" 21 "os"
24 "path/filepath" 22 "path/filepath"
25 "strings"
26 "time" 23 "time"
27 24
28 humanize "github.com/dustin/go-humanize" 25 humanize "github.com/dustin/go-humanize"
29 "github.com/golang/protobuf/proto" 26 "github.com/golang/protobuf/proto"
30 "github.com/maruel/subcommands" 27 "github.com/maruel/subcommands"
31 "golang.org/x/net/context" 28 "golang.org/x/net/context"
32 29
33 "github.com/luci/luci-go/client/internal/common" 30 "github.com/luci/luci-go/client/internal/common"
34 "github.com/luci/luci-go/client/isolate" 31 "github.com/luci/luci-go/client/isolate"
35 "github.com/luci/luci-go/common/auth" 32 "github.com/luci/luci-go/common/auth"
(...skipping 174 matching lines...) Expand 10 before | Expand all | Expand 10 after
210 filesSize := uint64(parts.filesToArchive.totalSize + parts.indivFiles.to talSize) 207 filesSize := uint64(parts.filesToArchive.totalSize + parts.indivFiles.to talSize)
211 log.Printf("Isolate expanded to %d files (total size %s) and %d symlinks ", numFiles, humanize.Bytes(filesSize), len(parts.links.items)) 208 log.Printf("Isolate expanded to %d files (total size %s) and %d symlinks ", numFiles, humanize.Bytes(filesSize), len(parts.links.items))
212 log.Printf("\t%d files (%s) to be isolated individually", len(parts.indi vFiles.items), humanize.Bytes(uint64(parts.indivFiles.totalSize))) 209 log.Printf("\t%d files (%s) to be isolated individually", len(parts.indi vFiles.items), humanize.Bytes(uint64(parts.indivFiles.totalSize)))
213 log.Printf("\t%d files (%s) to be isolated in archives", len(parts.files ToArchive.items), humanize.Bytes(uint64(parts.filesToArchive.totalSize))) 210 log.Printf("\t%d files (%s) to be isolated in archives", len(parts.files ToArchive.items), humanize.Bytes(uint64(parts.filesToArchive.totalSize)))
214 211
215 tracker := NewUploadTracker(checker, uploader, isol) 212 tracker := NewUploadTracker(checker, uploader, isol)
216 if err := tracker.UploadDeps(parts); err != nil { 213 if err := tracker.UploadDeps(parts); err != nil {
217 return err 214 return err
218 } 215 }
219 216
220 » isolItem, isolJSON, err := tracker.Finalize(archiveOpts.Isolated) 217 » isolDigest, err := tracker.Finalize(archiveOpts.Isolated, c.dumpJSON)
221 if err != nil { 218 if err != nil {
222 return err 219 return err
223 } 220 }
224 221
225 // Make sure that all pending items have been checked.
226 if err := checker.Close(); err != nil {
227 return err
228 }
229
230 // Make sure that all the uploads have completed successfully.
231 if err := uploader.Close(); err != nil {
232 return err
233 }
234
235 // Write the isolated file, and emit its digest to stdout.
236 if err := ioutil.WriteFile(archiveOpts.Isolated, isolJSON, 0644); err != nil {
237 return err
238 }
239 fmt.Printf("%s\t%s\n", isolItem.Digest, filepath.Base(archiveOpts.Isolat ed))
240
241 // Optionally, write the digest of the isolated file as JSON (in the sam e
242 // format as batch_archive).
243 if c.dumpJSON != "" {
244 // The name is the base name of the isolated file, extension str ipped.
245 name := filepath.Base(archiveOpts.Isolated)
246 if i := strings.LastIndex(name, "."); i != -1 {
247 name = name[:i]
248 }
249 j, err := json.Marshal(map[string]isolated.HexDigest{
250 name: isolItem.Digest,
251 })
252 if err != nil {
253 return err
254 }
255 if err := ioutil.WriteFile(c.dumpJSON, j, 0644); err != nil {
256 return err
257 }
258 }
259
260 end := time.Now() 222 end := time.Now()
261 223
262 archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{ 224 archiveDetails := &logpb.IsolateClientEvent_ArchiveDetails{
263 HitCount: proto.Int64(int64(checker.Hit.Count)), 225 HitCount: proto.Int64(int64(checker.Hit.Count)),
264 MissCount: proto.Int64(int64(checker.Miss.Count)), 226 MissCount: proto.Int64(int64(checker.Miss.Count)),
265 HitBytes: &checker.Hit.Bytes, 227 HitBytes: &checker.Hit.Bytes,
266 MissBytes: &checker.Miss.Bytes, 228 MissBytes: &checker.Miss.Bytes,
267 » » IsolateHash: []string{string(isolItem.Digest)}, 229 » » IsolateHash: []string{string(isolDigest)},
268 } 230 }
269 eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint) 231 eventlogger := NewLogger(ctx, c.loggingFlags.EventlogEndpoint)
270 op := logpb.IsolateClientEvent_ARCHIVE.Enum() 232 op := logpb.IsolateClientEvent_ARCHIVE.Enum()
271 if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err != nil { 233 if err := eventlogger.logStats(ctx, op, start, end, archiveDetails); err != nil {
272 log.Printf("Failed to log to eventlog: %v", err) 234 log.Printf("Failed to log to eventlog: %v", err)
273 } 235 }
274 236
275 return nil 237 return nil
276 } 238 }
277 239
(...skipping 28 matching lines...) Expand all
306 } 268 }
307 269
308 func hashFile(path string) (isolated.HexDigest, error) { 270 func hashFile(path string) (isolated.HexDigest, error) {
309 f, err := os.Open(path) 271 f, err := os.Open(path)
310 if err != nil { 272 if err != nil {
311 return "", err 273 return "", err
312 } 274 }
313 defer f.Close() 275 defer f.Close()
314 return isolated.Hash(f) 276 return isolated.Hash(f)
315 } 277 }
OLDNEW
« no previous file with comments | « no previous file | client/cmd/isolate/upload_tracker.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698