OLD | NEW |
1 // Utility that contains methods for both CT master and worker scripts. | 1 // Utility that contains methods for both CT master and worker scripts. |
2 package util | 2 package util |
3 | 3 |
4 import ( | 4 import ( |
5 "bufio" | 5 "bufio" |
6 "encoding/json" | 6 "encoding/json" |
7 "fmt" | 7 "fmt" |
8 "io" | 8 "io" |
9 "io/ioutil" | 9 "io/ioutil" |
10 "os" | 10 "os" |
11 "path" | 11 "path" |
12 "path/filepath" | 12 "path/filepath" |
13 "runtime" | 13 "runtime" |
14 "strconv" | 14 "strconv" |
15 "sync" | 15 "sync" |
16 "time" | 16 "time" |
17 | 17 |
18 "go.skia.org/infra/go/exec" | 18 "go.skia.org/infra/go/exec" |
| 19 "go.skia.org/infra/go/swarming" |
19 "go.skia.org/infra/go/util" | 20 "go.skia.org/infra/go/util" |
20 | 21 |
21 "github.com/skia-dev/glog" | 22 "github.com/skia-dev/glog" |
22 ) | 23 ) |
23 | 24 |
24 const ( | 25 const ( |
25 MAX_SYNC_TRIES = 3 | 26 MAX_SYNC_TRIES = 3 |
26 | 27 |
27 TS_FORMAT = "20060102150405" | 28 TS_FORMAT = "20060102150405" |
28 | 29 |
(...skipping 354 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
383 })) | 384 })) |
384 } | 385 } |
385 }(i) | 386 }(i) |
386 } | 387 } |
387 | 388 |
388 // Wait for all spawned goroutines to complete. | 389 // Wait for all spawned goroutines to complete. |
389 wg.Wait() | 390 wg.Wait() |
390 | 391 |
391 return nil | 392 return nil |
392 } | 393 } |
| 394 |
| 395 // GetStartRange returns the range worker should start processing at based on it
s num and how many |
| 396 // artifacts it is allowed to process. |
| 397 func GetStartRange(workerNum, artifactsPerWorker int) int { |
| 398 return ((workerNum - 1) * artifactsPerWorker) + 1 |
| 399 } |
| 400 |
| 401 // ArchiveTriggerCollectSwarmingTasks is a convenience function for CT scripts t
o call to |
| 402 // batch archive isolates, trigger swarming tasks, collect the triggered swarmin
g tasks. |
| 403 func ArchiveTriggerCollectSwarmingTask(s *swarming.SwarmingClient, taskNames, ge
nJSONs []string, hardTimeout, ioTimeout time.Duration) error { |
| 404 // Batcharchive the tasks. |
| 405 tasksToHashes, err := s.BatchArchiveTargets(genJSONs, BATCHARCHIVE_TIMEO
UT) |
| 406 if err != nil { |
| 407 return fmt.Errorf("Could not batch archive targets: %s", err) |
| 408 } |
| 409 if len(taskNames) != len(tasksToHashes) { |
| 410 return fmt.Errorf("len(taskNames) was %d and len(tasksToHashes)
was %d", len(taskNames), len(tasksToHashes)) |
| 411 } |
| 412 // Trigger swarming using the isolate hashes. |
| 413 dimensions := map[string]string{"pool": SWARMING_POOL} |
| 414 tasks, err := s.TriggerSwarmingTasks(tasksToHashes, dimensions, swarming
.RECOMMENDED_PRIORITY, swarming.RECOMMENDED_EXPIRATION, hardTimeout, ioTimeout,
false) |
| 415 if err != nil { |
| 416 return fmt.Errorf("Could not trigger swarming task: %s", err) |
| 417 } |
| 418 // Collect all tasks and log the ones that fail. |
| 419 for _, task := range tasks { |
| 420 if _, _, err := task.Collect(s); err != nil { |
| 421 glog.Errorf("task %s failed: %s", task.Title, err) |
| 422 continue |
| 423 } |
| 424 } |
| 425 return nil |
| 426 } |
| 427 |
| 428 // GetPathToPyFiles returns the location of CT's python scripts. |
| 429 func GetPathToPyFiles(runOnSwarming bool) string { |
| 430 var pathToPyFiles string |
| 431 if runOnSwarming { |
| 432 pathToPyFiles = filepath.Join(filepath.Dir(filepath.Dir(os.Args[
0])), "src", "go.skia.org", "infra", "ct", "py") |
| 433 } else { |
| 434 _, currentFile, _, _ := runtime.Caller(0) |
| 435 pathToPyFiles = filepath.Join(filepath.Dir(filepath.Dir(filepath
.Dir(currentFile))), "py") |
| 436 } |
| 437 return pathToPyFiles |
| 438 } |
OLD | NEW |