| OLD | NEW |
| 1 // Copyright 2017 The LUCI Authors. | 1 // Copyright 2017 The LUCI Authors. |
| 2 // | 2 // |
| 3 // Licensed under the Apache License, Version 2.0 (the "License"); | 3 // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 // you may not use this file except in compliance with the License. | 4 // you may not use this file except in compliance with the License. |
| 5 // You may obtain a copy of the License at | 5 // You may obtain a copy of the License at |
| 6 // | 6 // |
| 7 // http://www.apache.org/licenses/LICENSE-2.0 | 7 // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 // | 8 // |
| 9 // Unless required by applicable law or agreed to in writing, software | 9 // Unless required by applicable law or agreed to in writing, software |
| 10 // distributed under the License is distributed on an "AS IS" BASIS, | 10 // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 // See the License for the specific language governing permissions and | 12 // See the License for the specific language governing permissions and |
| 13 // limitations under the License. | 13 // limitations under the License. |
| 14 | 14 |
| 15 package main | 15 package main |
| 16 | 16 |
| 17 import ( | 17 import ( |
| 18 "encoding/json" | 18 "encoding/json" |
| 19 "fmt" | 19 "fmt" |
| 20 "io" | |
| 21 "io/ioutil" | 20 "io/ioutil" |
| 22 "log" | 21 "log" |
| 23 "os" | 22 "os" |
| 24 "path/filepath" | 23 "path/filepath" |
| 25 "strings" | 24 "strings" |
| 26 | 25 |
| 27 humanize "github.com/dustin/go-humanize" | 26 humanize "github.com/dustin/go-humanize" |
| 28 "github.com/luci/luci-go/common/isolated" | 27 "github.com/luci/luci-go/common/isolated" |
| 29 "github.com/luci/luci-go/common/isolatedclient" | 28 "github.com/luci/luci-go/common/isolatedclient" |
| 30 ) | 29 ) |
| 31 | 30 |
| 32 // UploadTracker uploads and keeps track of files. | 31 // UploadTracker uploads and keeps track of files. |
| 33 type UploadTracker struct { | 32 type UploadTracker struct { |
| 34 checker *Checker | 33 checker *Checker |
| 35 uploader *Uploader | 34 uploader *Uploader |
| 36 isol *isolated.Isolated | 35 isol *isolated.Isolated |
| 37 } | 36 } |
| 38 | 37 |
| 39 // NewUploadTracker constructs an UploadTracker. It tracks uploaded files in is
ol.Files. | 38 // NewUploadTracker constructs an UploadTracker. It tracks uploaded files in is
ol.Files. |
| 40 func NewUploadTracker(checker *Checker, uploader *Uploader, isol *isolated.Isola
ted) *UploadTracker { | 39 func NewUploadTracker(checker *Checker, uploader *Uploader, isol *isolated.Isola
ted) *UploadTracker { |
| 40 // TODO: share a Checker and Uploader with other UploadTrackers |
| 41 // when batch uploading. |
| 41 isol.Files = make(map[string]isolated.File) | 42 isol.Files = make(map[string]isolated.File) |
| 42 return &UploadTracker{ | 43 return &UploadTracker{ |
| 43 checker: checker, | 44 checker: checker, |
| 44 uploader: uploader, | 45 uploader: uploader, |
| 45 isol: isol, | 46 isol: isol, |
| 46 } | 47 } |
| 47 } | 48 } |
| 48 | 49 |
| 49 // UploadDeps uploads all of the items in parts. | 50 // UploadDeps uploads all of the items in parts. |
| 50 func (ut *UploadTracker) UploadDeps(parts partitionedDeps) error { | 51 func (ut *UploadTracker) UploadDeps(parts partitionedDeps) error { |
| (...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 134 } | 135 } |
| 135 log.Printf("QUEUED %q for upload", item.RelPath) | 136 log.Printf("QUEUED %q for upload", item.RelPath) |
| 136 ut.uploader.UploadFile(item, ps, func() { | 137 ut.uploader.UploadFile(item, ps, func() { |
| 137 log.Printf("UPLOADED %q", item.RelPath) | 138 log.Printf("UPLOADED %q", item.RelPath) |
| 138 }) | 139 }) |
| 139 }) | 140 }) |
| 140 } | 141 } |
| 141 return nil | 142 return nil |
| 142 } | 143 } |
| 143 | 144 |
| 145 // IsolatedSummary contains an isolate name and its digest. |
| 146 type IsolatedSummary struct { |
| 147 // Name is the base name an isolated file with any extension stripped |
| 148 Name string |
| 149 Digest isolated.HexDigest |
| 150 } |
| 151 |
| 144 // Finalize creates and uploads the isolate JSON at the isolatePath, and closes
the checker and uploader. | 152 // Finalize creates and uploads the isolate JSON at the isolatePath, and closes
the checker and uploader. |
| 145 // It returns the isolate digest. | 153 // It returns the isolate name and digest. |
| 146 // If dumpJSONPath is non-empty, the digest is also written to that path as | |
| 147 // JSON (in the same format as batch_archive). | |
| 148 // Finalize should only be called after UploadDeps. | 154 // Finalize should only be called after UploadDeps. |
| 149 func (ut *UploadTracker) Finalize(isolatedPath string, dumpJSONWriter io.Writer)
(isolated.HexDigest, error) { | 155 func (ut *UploadTracker) Finalize(isolatedPath string) (IsolatedSummary, error)
{ |
| 150 isolFile, err := newIsolatedFile(ut.isol, isolatedPath) | 156 isolFile, err := newIsolatedFile(ut.isol, isolatedPath) |
| 151 if err != nil { | 157 if err != nil { |
| 152 » » return "", err | 158 » » return IsolatedSummary{}, err |
| 153 } | 159 } |
| 154 | 160 |
| 155 // Check and upload isolate JSON. | 161 // Check and upload isolate JSON. |
| 156 ut.checker.AddItem(isolFile.item(), true, func(item *Item, ps *isolatedc
lient.PushState) { | 162 ut.checker.AddItem(isolFile.item(), true, func(item *Item, ps *isolatedc
lient.PushState) { |
| 157 if ps == nil { | 163 if ps == nil { |
| 158 return | 164 return |
| 159 } | 165 } |
| 160 log.Printf("QUEUED %q for upload", item.RelPath) | 166 log.Printf("QUEUED %q for upload", item.RelPath) |
| 161 ut.uploader.UploadBytes(item.RelPath, isolFile.contents(), ps, f
unc() { | 167 ut.uploader.UploadBytes(item.RelPath, isolFile.contents(), ps, f
unc() { |
| 162 log.Printf("UPLOADED %q", item.RelPath) | 168 log.Printf("UPLOADED %q", item.RelPath) |
| 163 }) | 169 }) |
| 164 }) | 170 }) |
| 165 | 171 |
| 166 // Make sure that all pending items have been checked. | 172 // Make sure that all pending items have been checked. |
| 167 if err := ut.checker.Close(); err != nil { | 173 if err := ut.checker.Close(); err != nil { |
| 168 » » return "", err | 174 » » return IsolatedSummary{}, err |
| 169 } | 175 } |
| 170 | 176 |
| 171 // Make sure that all the uploads have completed successfully. | 177 // Make sure that all the uploads have completed successfully. |
| 172 if err := ut.uploader.Close(); err != nil { | 178 if err := ut.uploader.Close(); err != nil { |
| 173 » » return "", err | 179 » » return IsolatedSummary{}, err |
| 174 } | 180 } |
| 175 | 181 |
| 176 » // Write the isolated file, and emit its digest to stdout. | 182 » // Write the isolated file... |
| 177 if err := isolFile.writeJSONFile(); err != nil { | 183 if err := isolFile.writeJSONFile(); err != nil { |
| 178 » » return "", err | 184 » » return IsolatedSummary{}, err |
| 179 } | 185 } |
| 180 | 186 |
| 181 » fmt.Printf("%s\t%s\n", isolFile.item().Digest, filepath.Base(isolatedPat
h)) | 187 » return IsolatedSummary{ |
| 182 | 188 » » Name: isolFile.name(), |
| 183 » if err := dumpJSON(isolFile.name(), dumpJSONWriter, isolFile.item()); er
r != nil { | 189 » » Digest: isolFile.item().Digest, |
| 184 » » return "", err | 190 » }, nil |
| 185 » } | |
| 186 | |
| 187 » return isolFile.item().Digest, nil | |
| 188 } | |
| 189 | |
| 190 func dumpJSON(isolName string, dumpJSONWriter io.Writer, isolItem *Item) error { | |
| 191 » enc := json.NewEncoder(dumpJSONWriter) | |
| 192 » return enc.Encode(map[string]isolated.HexDigest{ | |
| 193 » » isolName: isolItem.Digest, | |
| 194 » }) | |
| 195 } | 191 } |
| 196 | 192 |
| 197 // isolatedFile is an isolated file which is stored in memory. | 193 // isolatedFile is an isolated file which is stored in memory. |
| 198 // It can produce a corresponding Item, for upload to the server, | 194 // It can produce a corresponding Item, for upload to the server, |
| 199 // and also write its contents to the filesystem. | 195 // and also write its contents to the filesystem. |
| 200 type isolatedFile struct { | 196 type isolatedFile struct { |
| 201 json []byte | 197 json []byte |
| 202 path string | 198 path string |
| 203 } | 199 } |
| 204 | 200 |
| (...skipping 25 matching lines...) Expand all Loading... |
| 230 } | 226 } |
| 231 | 227 |
| 232 // name returns the base name of the isolated file, extension stripped. | 228 // name returns the base name of the isolated file, extension stripped. |
| 233 func (ij *isolatedFile) name() string { | 229 func (ij *isolatedFile) name() string { |
| 234 name := filepath.Base(ij.path) | 230 name := filepath.Base(ij.path) |
| 235 if i := strings.LastIndex(name, "."); i != -1 { | 231 if i := strings.LastIndex(name, "."); i != -1 { |
| 236 name = name[:i] | 232 name = name[:i] |
| 237 } | 233 } |
| 238 return name | 234 return name |
| 239 } | 235 } |
| OLD | NEW |