| OLD | NEW |
| 1 // Application that captures webpage archives on a CT worker and uploads it to | 1 // Application that captures webpage archives on a CT worker and uploads it to |
| 2 // Google Storage. | 2 // Google Storage. |
| 3 package main | 3 package main |
| 4 | 4 |
| 5 import ( | 5 import ( |
| 6 "flag" | 6 "flag" |
| 7 "fmt" | 7 "fmt" |
| 8 "io/ioutil" | 8 "io/ioutil" |
| 9 "path/filepath" | 9 "path/filepath" |
| 10 | 10 |
| (...skipping 20 matching lines...) Expand all Loading... |
| 31 | 31 |
| 32 // Create the task file so that the master knows this worker is still bu
sy. | 32 // Create the task file so that the master knows this worker is still bu
sy. |
| 33 skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES)) | 33 skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES)) |
| 34 defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES) | 34 defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES) |
| 35 | 35 |
| 36 if *chromiumBuild == "" { | 36 if *chromiumBuild == "" { |
| 37 glog.Error("Must specify --chromium_build") | 37 glog.Error("Must specify --chromium_build") |
| 38 return | 38 return |
| 39 } | 39 } |
| 40 | 40 |
| 41 // Reset the local chromium checkout. |
| 42 if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil { |
| 43 glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err) |
| 44 return |
| 45 } |
| 41 // Sync the local chromium checkout. | 46 // Sync the local chromium checkout. |
| 42 if err := util.SyncDir(util.ChromiumSrcDir); err != nil { | 47 if err := util.SyncDir(util.ChromiumSrcDir); err != nil { |
| 43 glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir
, err) | 48 glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir
, err) |
| 44 return | 49 return |
| 45 } | 50 } |
| 46 | 51 |
| 47 // Delete and remake the local webpage archives directory. | 52 // Delete and remake the local webpage archives directory. |
| 48 pathToArchives := filepath.Join(util.WebArchivesDir, *pagesetType) | 53 pathToArchives := filepath.Join(util.WebArchivesDir, *pagesetType) |
| 49 skutil.RemoveAll(pathToArchives) | 54 skutil.RemoveAll(pathToArchives) |
| 50 skutil.MkdirAll(pathToArchives, 0700) | 55 skutil.MkdirAll(pathToArchives, 0700) |
| (...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 106 | 111 |
| 107 // Write timestamp to the webpage archives dir. | 112 // Write timestamp to the webpage archives dir. |
| 108 skutil.LogErr(util.CreateTimestampFile(pathToArchives)) | 113 skutil.LogErr(util.CreateTimestampFile(pathToArchives)) |
| 109 | 114 |
| 110 // Upload webpage archives dir to Google Storage. | 115 // Upload webpage archives dir to Google Storage. |
| 111 if err := gs.UploadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetT
ype, *workerNum); err != nil { | 116 if err := gs.UploadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetT
ype, *workerNum); err != nil { |
| 112 glog.Error(err) | 117 glog.Error(err) |
| 113 return | 118 return |
| 114 } | 119 } |
| 115 } | 120 } |
| OLD | NEW |