| OLD | NEW |
| 1 package main | 1 package main |
| 2 | 2 |
| 3 /* | 3 /* |
| 4 Performance test for TaskScheduler. | 4 Performance test for TaskScheduler. |
| 5 */ | 5 */ |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "encoding/json" | 8 "encoding/json" |
| 9 "fmt" | 9 "fmt" |
| 10 "io/ioutil" | 10 "io/ioutil" |
| 11 "math" | 11 "math" |
| 12 "net/http" | 12 "net/http" |
| 13 _ "net/http/pprof" | 13 _ "net/http/pprof" |
| 14 "os" | 14 "os" |
| 15 "path" | 15 "path" |
| 16 "reflect" | 16 "reflect" |
| 17 "time" | 17 "time" |
| 18 | 18 |
| 19 "github.com/davecgh/go-spew/spew" | 19 "github.com/davecgh/go-spew/spew" |
| 20 swarming_api "github.com/luci/luci-go/common/api/swarming/swarming/v1" | 20 swarming_api "github.com/luci/luci-go/common/api/swarming/swarming/v1" |
| 21 "github.com/skia-dev/glog" | 21 "github.com/skia-dev/glog" |
| 22 "go.skia.org/infra/build_scheduler/go/db" | |
| 23 "go.skia.org/infra/build_scheduler/go/db/local_db" | |
| 24 "go.skia.org/infra/build_scheduler/go/task_scheduler" | |
| 25 "go.skia.org/infra/go/common" | 22 "go.skia.org/infra/go/common" |
| 26 "go.skia.org/infra/go/exec" | 23 "go.skia.org/infra/go/exec" |
| 27 "go.skia.org/infra/go/gitinfo" | 24 "go.skia.org/infra/go/gitinfo" |
| 28 "go.skia.org/infra/go/isolate" | 25 "go.skia.org/infra/go/isolate" |
| 29 "go.skia.org/infra/go/swarming" | 26 "go.skia.org/infra/go/swarming" |
| 27 "go.skia.org/infra/task_scheduler/go/db" |
| 28 "go.skia.org/infra/task_scheduler/go/db/local_db" |
| 29 "go.skia.org/infra/task_scheduler/go/scheduling" |
| 30 ) | 30 ) |
| 31 | 31 |
| 32 func assertNoError(err error) { | 32 func assertNoError(err error) { |
| 33 if err != nil { | 33 if err != nil { |
| 34 glog.Fatalf("Expected no error but got: %s", err.Error()) | 34 glog.Fatalf("Expected no error but got: %s", err.Error()) |
| 35 } | 35 } |
| 36 } | 36 } |
| 37 | 37 |
| 38 func assertEqual(a, b interface{}) { | 38 func assertEqual(a, b interface{}) { |
| 39 if a != b { | 39 if a != b { |
| (...skipping 128 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 168 'command': [ | 168 'command': [ |
| 169 'python', 'recipes.py', 'run', | 169 'python', 'recipes.py', 'run', |
| 170 ], | 170 ], |
| 171 'files': [ | 171 'files': [ |
| 172 '../../somefile.txt', | 172 '../../somefile.txt', |
| 173 ], | 173 ], |
| 174 }, | 174 }, |
| 175 }`) | 175 }`) |
| 176 | 176 |
| 177 // Add tasks to the repo. | 177 // Add tasks to the repo. |
| 178 » var tasks = map[string]*task_scheduler.TaskSpec{ | 178 » var tasks = map[string]*scheduling.TaskSpec{ |
| 179 » » "Build-Ubuntu-GCC-Arm7-Release-Android": &task_scheduler.TaskSpe
c{ | 179 » » "Build-Ubuntu-GCC-Arm7-Release-Android": &scheduling.TaskSpec{ |
| 180 » » » CipdPackages: []*task_scheduler.CipdPackage{}, | 180 » » » CipdPackages: []*scheduling.CipdPackage{}, |
| 181 Dependencies: []string{}, | 181 Dependencies: []string{}, |
| 182 Dimensions: []string{"pool:Skia", "os:Ubuntu"}, | 182 Dimensions: []string{"pool:Skia", "os:Ubuntu"}, |
| 183 Isolate: "compile_skia.isolate", | 183 Isolate: "compile_skia.isolate", |
| 184 Priority: 0.9, | 184 Priority: 0.9, |
| 185 }, | 185 }, |
| 186 » » "Test-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &task_schedul
er.TaskSpec{ | 186 » » "Test-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &scheduling.T
askSpec{ |
| 187 » » » CipdPackages: []*task_scheduler.CipdPackage{}, | 187 » » » CipdPackages: []*scheduling.CipdPackage{}, |
| 188 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An
droid"}, | 188 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An
droid"}, |
| 189 Dimensions: []string{"pool:Skia", "os:Android", "devic
e_type:grouper"}, | 189 Dimensions: []string{"pool:Skia", "os:Android", "devic
e_type:grouper"}, |
| 190 Isolate: "test_skia.isolate", | 190 Isolate: "test_skia.isolate", |
| 191 Priority: 0.9, | 191 Priority: 0.9, |
| 192 }, | 192 }, |
| 193 » » "Perf-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &task_schedul
er.TaskSpec{ | 193 » » "Perf-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &scheduling.T
askSpec{ |
| 194 » » » CipdPackages: []*task_scheduler.CipdPackage{}, | 194 » » » CipdPackages: []*scheduling.CipdPackage{}, |
| 195 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An
droid"}, | 195 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An
droid"}, |
| 196 Dimensions: []string{"pool:Skia", "os:Android", "devic
e_type:grouper"}, | 196 Dimensions: []string{"pool:Skia", "os:Android", "devic
e_type:grouper"}, |
| 197 Isolate: "perf_skia.isolate", | 197 Isolate: "perf_skia.isolate", |
| 198 Priority: 0.9, | 198 Priority: 0.9, |
| 199 }, | 199 }, |
| 200 } | 200 } |
| 201 » moarTasks := map[string]*task_scheduler.TaskSpec{} | 201 » moarTasks := map[string]*scheduling.TaskSpec{} |
| 202 for name, task := range tasks { | 202 for name, task := range tasks { |
| 203 for i := 0; i < 100; i++ { | 203 for i := 0; i < 100; i++ { |
| 204 newName := fmt.Sprintf("%s%d", name, i) | 204 newName := fmt.Sprintf("%s%d", name, i) |
| 205 deps := make([]string, 0, len(task.Dependencies)) | 205 deps := make([]string, 0, len(task.Dependencies)) |
| 206 for _, d := range task.Dependencies { | 206 for _, d := range task.Dependencies { |
| 207 deps = append(deps, fmt.Sprintf("%s%d", d, i)) | 207 deps = append(deps, fmt.Sprintf("%s%d", d, i)) |
| 208 } | 208 } |
| 209 » » » newTask := &task_scheduler.TaskSpec{ | 209 » » » newTask := &scheduling.TaskSpec{ |
| 210 CipdPackages: task.CipdPackages, | 210 CipdPackages: task.CipdPackages, |
| 211 Dependencies: deps, | 211 Dependencies: deps, |
| 212 Dimensions: task.Dimensions, | 212 Dimensions: task.Dimensions, |
| 213 Isolate: task.Isolate, | 213 Isolate: task.Isolate, |
| 214 Priority: task.Priority, | 214 Priority: task.Priority, |
| 215 } | 215 } |
| 216 moarTasks[newName] = newTask | 216 moarTasks[newName] = newTask |
| 217 } | 217 } |
| 218 } | 218 } |
| 219 » cfg := task_scheduler.TasksCfg{ | 219 » cfg := scheduling.TasksCfg{ |
| 220 Tasks: moarTasks, | 220 Tasks: moarTasks, |
| 221 } | 221 } |
| 222 » f, err := os.Create(path.Join(repoDir, task_scheduler.TASKS_CFG_FILE)) | 222 » f, err := os.Create(path.Join(repoDir, scheduling.TASKS_CFG_FILE)) |
| 223 assertNoError(err) | 223 assertNoError(err) |
| 224 assertNoError(json.NewEncoder(f).Encode(&cfg)) | 224 assertNoError(json.NewEncoder(f).Encode(&cfg)) |
| 225 assertNoError(f.Close()) | 225 assertNoError(f.Close()) |
| 226 » run(repoDir, "git", "add", task_scheduler.TASKS_CFG_FILE) | 226 » run(repoDir, "git", "add", scheduling.TASKS_CFG_FILE) |
| 227 commit(repoDir, "Add more tasks!") | 227 commit(repoDir, "Add more tasks!") |
| 228 run(repoDir, "git", "push", "origin", "master") | 228 run(repoDir, "git", "push", "origin", "master") |
| 229 run(repoDir, "git", "branch", "-u", "origin/master") | 229 run(repoDir, "git", "branch", "-u", "origin/master") |
| 230 | 230 |
| 231 // Create a bunch of bots. | 231 // Create a bunch of bots. |
| 232 bots := make([]*swarming_api.SwarmingRpcsBotInfo, 100) | 232 bots := make([]*swarming_api.SwarmingRpcsBotInfo, 100) |
| 233 for idx, _ := range bots { | 233 for idx, _ := range bots { |
| 234 dims := map[string]string{ | 234 dims := map[string]string{ |
| 235 "pool": "Skia", | 235 "pool": "Skia", |
| 236 } | 236 } |
| (...skipping 13 matching lines...) Expand all Loading... |
| 250 assertNoError(err) | 250 assertNoError(err) |
| 251 | 251 |
| 252 commits, err := repo.RevList(head) | 252 commits, err := repo.RevList(head) |
| 253 assertNoError(err) | 253 assertNoError(err) |
| 254 assertDeepEqual([]string{head}, commits) | 254 assertDeepEqual([]string{head}, commits) |
| 255 | 255 |
| 256 isolateClient, err := isolate.NewClient(workdir) | 256 isolateClient, err := isolate.NewClient(workdir) |
| 257 assertNoError(err) | 257 assertNoError(err) |
| 258 isolateClient.ServerUrl = isolate.FAKE_SERVER_URL | 258 isolateClient.ServerUrl = isolate.FAKE_SERVER_URL |
| 259 swarmingClient := swarming.NewTestClient() | 259 swarmingClient := swarming.NewTestClient() |
| 260 » s, err := task_scheduler.NewTaskScheduler(d, cache, time.Duration(math.M
axInt64), workdir, []string{"skia.git"}, isolateClient, swarmingClient) | 260 » s, err := scheduling.NewTaskScheduler(d, cache, time.Duration(math.MaxIn
t64), workdir, []string{"skia.git"}, isolateClient, swarmingClient) |
| 261 assertNoError(err) | 261 assertNoError(err) |
| 262 | 262 |
| 263 runTasks := func(bots []*swarming_api.SwarmingRpcsBotInfo) { | 263 runTasks := func(bots []*swarming_api.SwarmingRpcsBotInfo) { |
| 264 swarmingClient.MockBots(bots) | 264 swarmingClient.MockBots(bots) |
| 265 assertNoError(s.MainLoop()) | 265 assertNoError(s.MainLoop()) |
| 266 assertNoError(cache.Update()) | 266 assertNoError(cache.Update()) |
| 267 tasks, err := cache.GetTasksForCommits(repoName, commits) | 267 tasks, err := cache.GetTasksForCommits(repoName, commits) |
| 268 assertNoError(err) | 268 assertNoError(err) |
| 269 newTasks := map[string]*db.Task{} | 269 newTasks := map[string]*db.Task{} |
| 270 for _, v := range tasks { | 270 for _, v := range tasks { |
| (...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 311 // Actually run the test. | 311 // Actually run the test. |
| 312 i := 0 | 312 i := 0 |
| 313 for ; ; i++ { | 313 for ; ; i++ { |
| 314 runTasks(bots) | 314 runTasks(bots) |
| 315 if s.QueueLen() == 0 { | 315 if s.QueueLen() == 0 { |
| 316 break | 316 break |
| 317 } | 317 } |
| 318 } | 318 } |
| 319 glog.Infof("Finished in %d iterations.", i) | 319 glog.Infof("Finished in %d iterations.", i) |
| 320 } | 320 } |
| OLD | NEW |