OLD | NEW |
1 // Google Storage utility that contains methods for both CT master and worker | 1 // Google Storage utility that contains methods for both CT master and worker |
2 // scripts. | 2 // scripts. |
3 package util | 3 package util |
4 | 4 |
5 import ( | 5 import ( |
6 "fmt" | 6 "fmt" |
7 "io" | 7 "io" |
8 "io/ioutil" | 8 "io/ioutil" |
9 "net/http" | 9 "net/http" |
10 "os" | 10 "os" |
(...skipping 122 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
133 req := gs.service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/") | 133 req := gs.service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/") |
134 for req != nil { | 134 for req != nil { |
135 resp, err := req.Do() | 135 resp, err := req.Do() |
136 if err != nil { | 136 if err != nil { |
137 return fmt.Errorf("Error occured while listing %s: %s",
gsDir, err) | 137 return fmt.Errorf("Error occured while listing %s: %s",
gsDir, err) |
138 } | 138 } |
139 for _, result := range resp.Items { | 139 for _, result := range resp.Items { |
140 fileName := filepath.Base(result.Name) | 140 fileName := filepath.Base(result.Name) |
141 | 141 |
142 wg.Add(1) | 142 wg.Add(1) |
143 » » » go func() { | 143 » » » go func(result *storage.Object) { |
144 defer wg.Done() | 144 defer wg.Done() |
145 respBody, err := getRespBody(result, gs.client) | 145 respBody, err := getRespBody(result, gs.client) |
146 if err != nil { | 146 if err != nil { |
147 glog.Errorf("Could not fetch %s: %s", re
sult.MediaLink, err) | 147 glog.Errorf("Could not fetch %s: %s", re
sult.MediaLink, err) |
148 return | 148 return |
149 } | 149 } |
150 defer respBody.Close() | 150 defer respBody.Close() |
151 outputFile := filepath.Join(localDir, fileName) | 151 outputFile := filepath.Join(localDir, fileName) |
152 out, err := os.Create(outputFile) | 152 out, err := os.Create(outputFile) |
153 if err != nil { | 153 if err != nil { |
154 glog.Errorf("Unable to create file %s: %
s", outputFile, err) | 154 glog.Errorf("Unable to create file %s: %
s", outputFile, err) |
155 return | 155 return |
156 } | 156 } |
157 defer out.Close() | 157 defer out.Close() |
158 if _, err = io.Copy(out, respBody); err != nil { | 158 if _, err = io.Copy(out, respBody); err != nil { |
159 glog.Error(err) | 159 glog.Error(err) |
160 return | 160 return |
161 } | 161 } |
162 glog.Infof("Downloaded gs://%s/%s to %s", GS_BUC
KET_NAME, result.Name, outputFile) | 162 glog.Infof("Downloaded gs://%s/%s to %s", GS_BUC
KET_NAME, result.Name, outputFile) |
163 » » » }() | 163 » » » }(result) |
164 } | 164 } |
165 if len(resp.NextPageToken) > 0 { | 165 if len(resp.NextPageToken) > 0 { |
166 req.PageToken(resp.NextPageToken) | 166 req.PageToken(resp.NextPageToken) |
167 } else { | 167 } else { |
168 req = nil | 168 req = nil |
169 } | 169 } |
170 } | 170 } |
171 wg.Wait() | 171 wg.Wait() |
172 return nil | 172 return nil |
173 } | 173 } |
(...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
240 if err != nil { | 240 if err != nil { |
241 glog.Errorf("Objects.Insert failed: %s", err) | 241 glog.Errorf("Objects.Insert failed: %s", err) |
242 return | 242 return |
243 } | 243 } |
244 glog.Infof("Created object %s at location %s", res.Name,
res.SelfLink) | 244 glog.Infof("Created object %s at location %s", res.Name,
res.SelfLink) |
245 }() | 245 }() |
246 } | 246 } |
247 wg.Wait() | 247 wg.Wait() |
248 return nil | 248 return nil |
249 } | 249 } |
OLD | NEW |