Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(135)

Side by Side Diff: client/cmd/isolate/upload_tracker.go

Issue 2985203002: isolate: Move json dumping back into exparchive main (Closed)
Patch Set: Created 3 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « client/cmd/isolate/exp_archive.go ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2017 The LUCI Authors. 1 // Copyright 2017 The LUCI Authors.
2 // 2 //
3 // Licensed under the Apache License, Version 2.0 (the "License"); 3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License. 4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at 5 // You may obtain a copy of the License at
6 // 6 //
7 // http://www.apache.org/licenses/LICENSE-2.0 7 // http://www.apache.org/licenses/LICENSE-2.0
8 // 8 //
9 // Unless required by applicable law or agreed to in writing, software 9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS, 10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and 12 // See the License for the specific language governing permissions and
13 // limitations under the License. 13 // limitations under the License.
14 14
15 package main 15 package main
16 16
17 import ( 17 import (
18 "encoding/json" 18 "encoding/json"
19 "fmt" 19 "fmt"
20 "io"
21 "io/ioutil" 20 "io/ioutil"
22 "log" 21 "log"
23 "os" 22 "os"
24 "path/filepath" 23 "path/filepath"
25 "strings" 24 "strings"
26 25
27 humanize "github.com/dustin/go-humanize" 26 humanize "github.com/dustin/go-humanize"
28 "github.com/luci/luci-go/common/isolated" 27 "github.com/luci/luci-go/common/isolated"
29 "github.com/luci/luci-go/common/isolatedclient" 28 "github.com/luci/luci-go/common/isolatedclient"
30 ) 29 )
31 30
32 // UploadTracker uploads and keeps track of files. 31 // UploadTracker uploads and keeps track of files.
33 type UploadTracker struct { 32 type UploadTracker struct {
34 checker *Checker 33 checker *Checker
35 uploader *Uploader 34 uploader *Uploader
36 isol *isolated.Isolated 35 isol *isolated.Isolated
37 } 36 }
38 37
39 // NewUploadTracker constructs an UploadTracker. It tracks uploaded files in is ol.Files. 38 // NewUploadTracker constructs an UploadTracker. It tracks uploaded files in is ol.Files.
40 func NewUploadTracker(checker *Checker, uploader *Uploader, isol *isolated.Isola ted) *UploadTracker { 39 func NewUploadTracker(checker *Checker, uploader *Uploader, isol *isolated.Isola ted) *UploadTracker {
40 // TODO: share a Checker and Uploader with other UploadTrackers
41 // when batch uploading.
41 isol.Files = make(map[string]isolated.File) 42 isol.Files = make(map[string]isolated.File)
42 return &UploadTracker{ 43 return &UploadTracker{
43 checker: checker, 44 checker: checker,
44 uploader: uploader, 45 uploader: uploader,
45 isol: isol, 46 isol: isol,
46 } 47 }
47 } 48 }
48 49
49 // UploadDeps uploads all of the items in parts. 50 // UploadDeps uploads all of the items in parts.
50 func (ut *UploadTracker) UploadDeps(parts partitionedDeps) error { 51 func (ut *UploadTracker) UploadDeps(parts partitionedDeps) error {
(...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after
134 } 135 }
135 log.Printf("QUEUED %q for upload", item.RelPath) 136 log.Printf("QUEUED %q for upload", item.RelPath)
136 ut.uploader.UploadFile(item, ps, func() { 137 ut.uploader.UploadFile(item, ps, func() {
137 log.Printf("UPLOADED %q", item.RelPath) 138 log.Printf("UPLOADED %q", item.RelPath)
138 }) 139 })
139 }) 140 })
140 } 141 }
141 return nil 142 return nil
142 } 143 }
143 144
145 // IsolatedSummary contains an isolate name and its digest.
146 type IsolatedSummary struct {
147 // Name is the base name an isolated file with any extension stripped
148 Name string
149 Digest isolated.HexDigest
150 }
151
144 // Finalize creates and uploads the isolate JSON at the isolatePath, and closes the checker and uploader. 152 // Finalize creates and uploads the isolate JSON at the isolatePath, and closes the checker and uploader.
145 // It returns the isolate digest. 153 // It returns the isolate name and digest.
146 // If dumpJSONPath is non-empty, the digest is also written to that path as
147 // JSON (in the same format as batch_archive).
148 // Finalize should only be called after UploadDeps. 154 // Finalize should only be called after UploadDeps.
149 func (ut *UploadTracker) Finalize(isolatedPath string, dumpJSONWriter io.Writer) (isolated.HexDigest, error) { 155 func (ut *UploadTracker) Finalize(isolatedPath string) (IsolatedSummary, error) {
150 isolFile, err := newIsolatedFile(ut.isol, isolatedPath) 156 isolFile, err := newIsolatedFile(ut.isol, isolatedPath)
151 if err != nil { 157 if err != nil {
152 » » return "", err 158 » » return IsolatedSummary{}, err
153 } 159 }
154 160
155 // Check and upload isolate JSON. 161 // Check and upload isolate JSON.
156 ut.checker.AddItem(isolFile.item(), true, func(item *Item, ps *isolatedc lient.PushState) { 162 ut.checker.AddItem(isolFile.item(), true, func(item *Item, ps *isolatedc lient.PushState) {
157 if ps == nil { 163 if ps == nil {
158 return 164 return
159 } 165 }
160 log.Printf("QUEUED %q for upload", item.RelPath) 166 log.Printf("QUEUED %q for upload", item.RelPath)
161 ut.uploader.UploadBytes(item.RelPath, isolFile.contents(), ps, f unc() { 167 ut.uploader.UploadBytes(item.RelPath, isolFile.contents(), ps, f unc() {
162 log.Printf("UPLOADED %q", item.RelPath) 168 log.Printf("UPLOADED %q", item.RelPath)
163 }) 169 })
164 }) 170 })
165 171
166 // Make sure that all pending items have been checked. 172 // Make sure that all pending items have been checked.
167 if err := ut.checker.Close(); err != nil { 173 if err := ut.checker.Close(); err != nil {
168 » » return "", err 174 » » return IsolatedSummary{}, err
169 } 175 }
170 176
171 // Make sure that all the uploads have completed successfully. 177 // Make sure that all the uploads have completed successfully.
172 if err := ut.uploader.Close(); err != nil { 178 if err := ut.uploader.Close(); err != nil {
173 » » return "", err 179 » » return IsolatedSummary{}, err
174 } 180 }
175 181
176 » // Write the isolated file, and emit its digest to stdout. 182 » // Write the isolated file...
177 if err := isolFile.writeJSONFile(); err != nil { 183 if err := isolFile.writeJSONFile(); err != nil {
178 » » return "", err 184 » » return IsolatedSummary{}, err
179 } 185 }
180 186
181 » fmt.Printf("%s\t%s\n", isolFile.item().Digest, filepath.Base(isolatedPat h)) 187 » return IsolatedSummary{
182 188 » » Name: isolFile.name(),
183 » if err := dumpJSON(isolFile.name(), dumpJSONWriter, isolFile.item()); er r != nil { 189 » » Digest: isolFile.item().Digest,
184 » » return "", err 190 » }, nil
185 » }
186
187 » return isolFile.item().Digest, nil
188 }
189
190 func dumpJSON(isolName string, dumpJSONWriter io.Writer, isolItem *Item) error {
191 » enc := json.NewEncoder(dumpJSONWriter)
192 » return enc.Encode(map[string]isolated.HexDigest{
193 » » isolName: isolItem.Digest,
194 » })
195 } 191 }
196 192
197 // isolatedFile is an isolated file which is stored in memory. 193 // isolatedFile is an isolated file which is stored in memory.
198 // It can produce a corresponding Item, for upload to the server, 194 // It can produce a corresponding Item, for upload to the server,
199 // and also write its contents to the filesystem. 195 // and also write its contents to the filesystem.
200 type isolatedFile struct { 196 type isolatedFile struct {
201 json []byte 197 json []byte
202 path string 198 path string
203 } 199 }
204 200
(...skipping 25 matching lines...) Expand all
230 } 226 }
231 227
232 // name returns the base name of the isolated file, extension stripped. 228 // name returns the base name of the isolated file, extension stripped.
233 func (ij *isolatedFile) name() string { 229 func (ij *isolatedFile) name() string {
234 name := filepath.Base(ij.path) 230 name := filepath.Base(ij.path)
235 if i := strings.LastIndex(name, "."); i != -1 { 231 if i := strings.LastIndex(name, "."); i != -1 {
236 name = name[:i] 232 name = name[:i]
237 } 233 }
238 return name 234 return name
239 } 235 }
OLDNEW
« no previous file with comments | « client/cmd/isolate/exp_archive.go ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698