Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(37)

Unified Diff: ct/go/master_scripts/create_pagesets_on_workers/main.go

Issue 1990873002: Use swarming in create_pagesets CT task (Closed) Base URL: https://skia.googlesource.com/buildbot@ct-3-swarming-timeouts
Patch Set: Address comment Created 4 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « no previous file | ct/go/util/constants.go » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: ct/go/master_scripts/create_pagesets_on_workers/main.go
diff --git a/ct/go/master_scripts/create_pagesets_on_workers/main.go b/ct/go/master_scripts/create_pagesets_on_workers/main.go
index b48cbe4b4bba2da6cd6890dc3b44155911767ccd..d18f2bc6cf0825a8f32d7495df796a9cb4ff7079 100644
--- a/ct/go/master_scripts/create_pagesets_on_workers/main.go
+++ b/ct/go/master_scripts/create_pagesets_on_workers/main.go
@@ -6,7 +6,7 @@ package main
import (
"flag"
"fmt"
- "strings"
+ "path/filepath"
"time"
"github.com/skia-dev/glog"
@@ -15,9 +15,14 @@ import (
"go.skia.org/infra/ct/go/master_scripts/master_common"
"go.skia.org/infra/ct/go/util"
"go.skia.org/infra/go/common"
+ "go.skia.org/infra/go/swarming"
skutil "go.skia.org/infra/go/util"
)
+const (
+ MAX_PAGES_PER_SWARMING_BOT = 1000
+)
+
var (
emails = flag.String("emails", "", "The comma separated email addresses to notify when the task is picked up and completes.")
gaeTaskID = flag.Int64("gae_task_id", -1, "The key of the App Engine task. This task will be updated when the task is completed.")
@@ -71,10 +76,6 @@ func main() {
// Ensure webapp is updated and completion email is sent even if task fails.
defer updateWebappTask()
defer sendEmail(emailsArr)
- if !*master_common.Local {
- // Cleanup tmp files after the run.
- defer util.CleanTmpDir()
- }
// Finish with glog flush and how long the task took.
defer util.TimeTrack(time.Now(), "Creating Pagesets on Workers")
defer glog.Flush()
@@ -84,16 +85,20 @@ func main() {
return
}
- cmd := append(master_common.WorkerSetupCmds(),
- // The main command that runs create_pagesets on all workers.
- fmt.Sprintf(
- "create_pagesets --worker_num=%s --log_dir=%s --log_id=%s --pageset_type=%s --local=%t;",
- util.WORKER_NUM_KEYWORD, util.GLogDir, *runID, *pagesetType, *master_common.Local))
-
- _, err := util.SSH(strings.Join(cmd, " "), util.Slaves, util.CREATE_PAGESETS_TIMEOUT)
+ // Empty the remote dir before the workers upload to it.
+ gs, err := util.NewGsUtil(nil)
if err != nil {
- glog.Errorf("Error while running cmd %s: %s", cmd, err)
+ glog.Error(err)
return
}
+ gsBaseDir := filepath.Join(util.SWARMING_DIR_NAME, util.PAGESETS_DIR_NAME, *pagesetType)
+ skutil.LogErr(gs.DeleteRemoteDir(gsBaseDir))
+
+ // Archive, trigger and collect swarming tasks.
+ if err := util.TriggerSwarmingTask(*pagesetType, "create_pagesets", util.CREATE_PAGESETS_ISOLATE, swarming.RECOMMENDED_HARD_TIMEOUT, swarming.RECOMMENDED_IO_TIMEOUT, MAX_PAGES_PER_SWARMING_BOT, map[string]string{}); err != nil {
+ glog.Errorf("Error encountered when swarming tasks: %s", err)
+ return
+ }
+
*taskCompletedSuccessfully = true
}
« no previous file with comments | « no previous file | ct/go/util/constants.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698