| OLD | NEW |
| 1 // Google Storage utility that contains methods for both CT master and worker | 1 // Google Storage utility that contains methods for both CT master and worker |
| 2 // scripts. | 2 // scripts. |
| 3 package util | 3 package util |
| 4 | 4 |
| 5 import ( | 5 import ( |
| 6 "fmt" | 6 "fmt" |
| 7 "io" | 7 "io" |
| 8 "io/ioutil" | 8 "io/ioutil" |
| 9 "net/http" | 9 "net/http" |
| 10 "os" | 10 "os" |
| (...skipping 122 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 133 req := gs.service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/") | 133 req := gs.service.Objects.List(GS_BUCKET_NAME).Prefix(gsDir + "/") |
| 134 for req != nil { | 134 for req != nil { |
| 135 resp, err := req.Do() | 135 resp, err := req.Do() |
| 136 if err != nil { | 136 if err != nil { |
| 137 return fmt.Errorf("Error occured while listing %s: %s",
gsDir, err) | 137 return fmt.Errorf("Error occured while listing %s: %s",
gsDir, err) |
| 138 } | 138 } |
| 139 for _, result := range resp.Items { | 139 for _, result := range resp.Items { |
| 140 fileName := filepath.Base(result.Name) | 140 fileName := filepath.Base(result.Name) |
| 141 | 141 |
| 142 wg.Add(1) | 142 wg.Add(1) |
| 143 » » » go func() { | 143 » » » go func(result *storage.Object) { |
| 144 defer wg.Done() | 144 defer wg.Done() |
| 145 respBody, err := getRespBody(result, gs.client) | 145 respBody, err := getRespBody(result, gs.client) |
| 146 if err != nil { | 146 if err != nil { |
| 147 glog.Errorf("Could not fetch %s: %s", re
sult.MediaLink, err) | 147 glog.Errorf("Could not fetch %s: %s", re
sult.MediaLink, err) |
| 148 return | 148 return |
| 149 } | 149 } |
| 150 defer respBody.Close() | 150 defer respBody.Close() |
| 151 outputFile := filepath.Join(localDir, fileName) | 151 outputFile := filepath.Join(localDir, fileName) |
| 152 out, err := os.Create(outputFile) | 152 out, err := os.Create(outputFile) |
| 153 if err != nil { | 153 if err != nil { |
| 154 glog.Errorf("Unable to create file %s: %
s", outputFile, err) | 154 glog.Errorf("Unable to create file %s: %
s", outputFile, err) |
| 155 return | 155 return |
| 156 } | 156 } |
| 157 defer out.Close() | 157 defer out.Close() |
| 158 if _, err = io.Copy(out, respBody); err != nil { | 158 if _, err = io.Copy(out, respBody); err != nil { |
| 159 glog.Error(err) | 159 glog.Error(err) |
| 160 return | 160 return |
| 161 } | 161 } |
| 162 glog.Infof("Downloaded gs://%s/%s to %s", GS_BUC
KET_NAME, result.Name, outputFile) | 162 glog.Infof("Downloaded gs://%s/%s to %s", GS_BUC
KET_NAME, result.Name, outputFile) |
| 163 » » » }() | 163 » » » }(result) |
| 164 } | 164 } |
| 165 if len(resp.NextPageToken) > 0 { | 165 if len(resp.NextPageToken) > 0 { |
| 166 req.PageToken(resp.NextPageToken) | 166 req.PageToken(resp.NextPageToken) |
| 167 } else { | 167 } else { |
| 168 req = nil | 168 req = nil |
| 169 } | 169 } |
| 170 } | 170 } |
| 171 wg.Wait() | 171 wg.Wait() |
| 172 return nil | 172 return nil |
| 173 } | 173 } |
| (...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 240 if err != nil { | 240 if err != nil { |
| 241 glog.Errorf("Objects.Insert failed: %s", err) | 241 glog.Errorf("Objects.Insert failed: %s", err) |
| 242 return | 242 return |
| 243 } | 243 } |
| 244 glog.Infof("Created object %s at location %s", res.Name,
res.SelfLink) | 244 glog.Infof("Created object %s at location %s", res.Name,
res.SelfLink) |
| 245 }() | 245 }() |
| 246 } | 246 } |
| 247 wg.Wait() | 247 wg.Wait() |
| 248 return nil | 248 return nil |
| 249 } | 249 } |
| OLD | NEW |