Use swarming in capture_archives CT task and capture archives in parallel

ct/go/util/constants.go

 package util
 
 import (
         "fmt"
         "path/filepath"
         "time"
 )
 
 const (
         NUM_WORKERS_PROD      int = 100
@@ skipping 168 matching lines @@
         // Fix Archives
         // Setting a 1 day timeout since it may take a while to validate archives.
         FIX_ARCHIVES_TIMEOUT = 1 * 24 * time.Hour
 
         // Poller
         MAKE_ALL_TIMEOUT = 15 * time.Minute
 
         WEBHOOK_SALT_MSG = `For prod, set this file to the value of GCE metadata key webhook_request_salt or call webhook.MustInitRequestSaltFromMetadata() if running in GCE. For testing, run 'echo -n "notverysecret" | base64 -w 0 > /b/storage/webhook_salt.data' or call frontend.InitForTesting().`
 
         // Swarming constants.
-        SWARMING_DIR_NAME    = "swarming"
-        SWARMING_POOL        = "CT"
+        SWARMING_DIR_NAME = "swarming"
+        SWARMING_POOL     = "CT"
+        // Timeouts.
         BATCHARCHIVE_TIMEOUT = 10 * time.Minute
+        XVFB_TIMEOUT         = 5 * time.Minute
         // Isolate files.
-        CREATE_PAGESETS_ISOLATE = "create_pagesets.isolate"
+        CREATE_PAGESETS_ISOLATE  = "create_pagesets.isolate"
+        CAPTURE_ARCHIVES_ISOLATE = "capture_archives.isolate"
 )
 
 type PagesetTypeInfo struct {
         NumPages                   int
         CSVSource                  string
         UserAgent                  string
         CaptureArchivesTimeoutSecs int
         CreatePagesetsTimeoutSecs  int
         CaptureSKPsTimeoutSecs     int
         RunChromiumPerfTimeoutSecs int
         Description                string
 }
 
 var (
         Master       = fmt.Sprintf(WORKER_NAME_TEMPLATE, 101)
         CtUser       = "chrome-bot"
         Slaves       = GetCTWorkersProd()
         GSBucketName = "cluster-telemetry"
 
         // Email address of cluster telemetry admins. They will be notified everytime
         // a task has started and completed.
         CtAdmins = []string{"rmistry@google.com", "benjaminwagner@google.com"}
 
         // Names of local directories and files.
         StorageDir           = filepath.Join("/", "b", STORAGE_DIR_NAME)
         RepoDir              = filepath.Join("/", "b", REPO_DIR_NAME)
+        DepotToolsDir        = filepath.Join("/", "b", "depot_tools")
         GomaDir              = filepath.Join("/", "b", "build", "goma")
         ChromiumBuildsDir    = filepath.Join(StorageDir, CHROMIUM_BUILDS_DIR_NAME)
         ChromiumSrcDir       = filepath.Join(StorageDir, "chromium", "src")
         TelemetryBinariesDir = filepath.Join(ChromiumSrcDir, "tools", "perf")
         TelemetrySrcDir      = filepath.Join(ChromiumSrcDir, "tools", "telemetry")
         CatapultSrcDir       = filepath.Join(ChromiumSrcDir, "third_party", "catapult")
         TaskFileDir          = filepath.Join(StorageDir, "current_task")
         ClientSecretPath     = filepath.Join(StorageDir, "client_secret.json")
         GSTokenPath          = filepath.Join(StorageDir, "google_storage_token.data")
         EmailTokenPath       = filepath.Join(StorageDir, "email.data")
@@ skipping 101 matching lines @@
 
         SupportedPageSetsToDesc = map[string]string{
                 PLATFORM_LINUX:   "Linux (100 Ubuntu12.04 machines)",
                 PLATFORM_ANDROID: "Android (100 N5 devices)",
         }
 )
 
 func NumWorkers() int {
         return len(Slaves)
 }