| Index: ct/go/master_scripts/create_pagesets_on_workers/main.go
|
| diff --git a/ct/go/master_scripts/create_pagesets_on_workers/main.go b/ct/go/master_scripts/create_pagesets_on_workers/main.go
|
| index b48cbe4b4bba2da6cd6890dc3b44155911767ccd..d18f2bc6cf0825a8f32d7495df796a9cb4ff7079 100644
|
| --- a/ct/go/master_scripts/create_pagesets_on_workers/main.go
|
| +++ b/ct/go/master_scripts/create_pagesets_on_workers/main.go
|
| @@ -6,7 +6,7 @@ package main
|
| import (
|
| "flag"
|
| "fmt"
|
| - "strings"
|
| + "path/filepath"
|
| "time"
|
|
|
| "github.com/skia-dev/glog"
|
| @@ -15,9 +15,14 @@ import (
|
| "go.skia.org/infra/ct/go/master_scripts/master_common"
|
| "go.skia.org/infra/ct/go/util"
|
| "go.skia.org/infra/go/common"
|
| + "go.skia.org/infra/go/swarming"
|
| skutil "go.skia.org/infra/go/util"
|
| )
|
|
|
| +const (
|
| + MAX_PAGES_PER_SWARMING_BOT = 1000
|
| +)
|
| +
|
| var (
|
| emails = flag.String("emails", "", "The comma separated email addresses to notify when the task is picked up and completes.")
|
| gaeTaskID = flag.Int64("gae_task_id", -1, "The key of the App Engine task. This task will be updated when the task is completed.")
|
| @@ -71,10 +76,6 @@ func main() {
|
| // Ensure webapp is updated and completion email is sent even if task fails.
|
| defer updateWebappTask()
|
| defer sendEmail(emailsArr)
|
| - if !*master_common.Local {
|
| - // Cleanup tmp files after the run.
|
| - defer util.CleanTmpDir()
|
| - }
|
| // Finish with glog flush and how long the task took.
|
| defer util.TimeTrack(time.Now(), "Creating Pagesets on Workers")
|
| defer glog.Flush()
|
| @@ -84,16 +85,20 @@ func main() {
|
| return
|
| }
|
|
|
| - cmd := append(master_common.WorkerSetupCmds(),
|
| - // The main command that runs create_pagesets on all workers.
|
| - fmt.Sprintf(
|
| - "create_pagesets --worker_num=%s --log_dir=%s --log_id=%s --pageset_type=%s --local=%t;",
|
| - util.WORKER_NUM_KEYWORD, util.GLogDir, *runID, *pagesetType, *master_common.Local))
|
| -
|
| - _, err := util.SSH(strings.Join(cmd, " "), util.Slaves, util.CREATE_PAGESETS_TIMEOUT)
|
| + // Empty the remote dir before the workers upload to it.
|
| + gs, err := util.NewGsUtil(nil)
|
| if err != nil {
|
| - glog.Errorf("Error while running cmd %s: %s", cmd, err)
|
| + glog.Error(err)
|
| return
|
| }
|
| + gsBaseDir := filepath.Join(util.SWARMING_DIR_NAME, util.PAGESETS_DIR_NAME, *pagesetType)
|
| + skutil.LogErr(gs.DeleteRemoteDir(gsBaseDir))
|
| +
|
| + // Archive, trigger and collect swarming tasks.
|
| + if err := util.TriggerSwarmingTask(*pagesetType, "create_pagesets", util.CREATE_PAGESETS_ISOLATE, swarming.RECOMMENDED_HARD_TIMEOUT, swarming.RECOMMENDED_IO_TIMEOUT, MAX_PAGES_PER_SWARMING_BOT, map[string]string{}); err != nil {
|
| + glog.Errorf("Error encountered when swarming tasks: %s", err)
|
| + return
|
| + }
|
| +
|
| *taskCompletedSuccessfully = true
|
| }
|
|
|