OLD | NEW |
1 package main | 1 package main |
2 | 2 |
3 /* | 3 /* |
4 Performance test for TaskScheduler. | 4 Performance test for TaskScheduler. |
5 */ | 5 */ |
6 | 6 |
7 import ( | 7 import ( |
8 "encoding/json" | 8 "encoding/json" |
9 "fmt" | 9 "fmt" |
10 "io/ioutil" | 10 "io/ioutil" |
11 "math" | 11 "math" |
12 "net/http" | 12 "net/http" |
13 _ "net/http/pprof" | 13 _ "net/http/pprof" |
14 "os" | 14 "os" |
15 "path" | 15 "path" |
16 "reflect" | 16 "reflect" |
17 "time" | 17 "time" |
18 | 18 |
19 "github.com/davecgh/go-spew/spew" | 19 "github.com/davecgh/go-spew/spew" |
20 swarming_api "github.com/luci/luci-go/common/api/swarming/swarming/v1" | 20 swarming_api "github.com/luci/luci-go/common/api/swarming/swarming/v1" |
21 "github.com/skia-dev/glog" | 21 "github.com/skia-dev/glog" |
22 "go.skia.org/infra/build_scheduler/go/db" | |
23 "go.skia.org/infra/build_scheduler/go/db/local_db" | |
24 "go.skia.org/infra/build_scheduler/go/task_scheduler" | |
25 "go.skia.org/infra/go/common" | 22 "go.skia.org/infra/go/common" |
26 "go.skia.org/infra/go/exec" | 23 "go.skia.org/infra/go/exec" |
27 "go.skia.org/infra/go/gitinfo" | 24 "go.skia.org/infra/go/gitinfo" |
28 "go.skia.org/infra/go/isolate" | 25 "go.skia.org/infra/go/isolate" |
29 "go.skia.org/infra/go/swarming" | 26 "go.skia.org/infra/go/swarming" |
| 27 "go.skia.org/infra/task_scheduler/go/db" |
| 28 "go.skia.org/infra/task_scheduler/go/db/local_db" |
| 29 "go.skia.org/infra/task_scheduler/go/scheduling" |
30 ) | 30 ) |
31 | 31 |
32 func assertNoError(err error) { | 32 func assertNoError(err error) { |
33 if err != nil { | 33 if err != nil { |
34 glog.Fatalf("Expected no error but got: %s", err.Error()) | 34 glog.Fatalf("Expected no error but got: %s", err.Error()) |
35 } | 35 } |
36 } | 36 } |
37 | 37 |
38 func assertEqual(a, b interface{}) { | 38 func assertEqual(a, b interface{}) { |
39 if a != b { | 39 if a != b { |
(...skipping 128 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
168 'command': [ | 168 'command': [ |
169 'python', 'recipes.py', 'run', | 169 'python', 'recipes.py', 'run', |
170 ], | 170 ], |
171 'files': [ | 171 'files': [ |
172 '../../somefile.txt', | 172 '../../somefile.txt', |
173 ], | 173 ], |
174 }, | 174 }, |
175 }`) | 175 }`) |
176 | 176 |
177 // Add tasks to the repo. | 177 // Add tasks to the repo. |
178 » var tasks = map[string]*task_scheduler.TaskSpec{ | 178 » var tasks = map[string]*scheduling.TaskSpec{ |
179 » » "Build-Ubuntu-GCC-Arm7-Release-Android": &task_scheduler.TaskSpe
c{ | 179 » » "Build-Ubuntu-GCC-Arm7-Release-Android": &scheduling.TaskSpec{ |
180 » » » CipdPackages: []*task_scheduler.CipdPackage{}, | 180 » » » CipdPackages: []*scheduling.CipdPackage{}, |
181 Dependencies: []string{}, | 181 Dependencies: []string{}, |
182 Dimensions: []string{"pool:Skia", "os:Ubuntu"}, | 182 Dimensions: []string{"pool:Skia", "os:Ubuntu"}, |
183 Isolate: "compile_skia.isolate", | 183 Isolate: "compile_skia.isolate", |
184 Priority: 0.9, | 184 Priority: 0.9, |
185 }, | 185 }, |
186 » » "Test-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &task_schedul
er.TaskSpec{ | 186 » » "Test-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &scheduling.T
askSpec{ |
187 » » » CipdPackages: []*task_scheduler.CipdPackage{}, | 187 » » » CipdPackages: []*scheduling.CipdPackage{}, |
188 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An
droid"}, | 188 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An
droid"}, |
189 Dimensions: []string{"pool:Skia", "os:Android", "devic
e_type:grouper"}, | 189 Dimensions: []string{"pool:Skia", "os:Android", "devic
e_type:grouper"}, |
190 Isolate: "test_skia.isolate", | 190 Isolate: "test_skia.isolate", |
191 Priority: 0.9, | 191 Priority: 0.9, |
192 }, | 192 }, |
193 » » "Perf-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &task_schedul
er.TaskSpec{ | 193 » » "Perf-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &scheduling.T
askSpec{ |
194 » » » CipdPackages: []*task_scheduler.CipdPackage{}, | 194 » » » CipdPackages: []*scheduling.CipdPackage{}, |
195 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An
droid"}, | 195 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An
droid"}, |
196 Dimensions: []string{"pool:Skia", "os:Android", "devic
e_type:grouper"}, | 196 Dimensions: []string{"pool:Skia", "os:Android", "devic
e_type:grouper"}, |
197 Isolate: "perf_skia.isolate", | 197 Isolate: "perf_skia.isolate", |
198 Priority: 0.9, | 198 Priority: 0.9, |
199 }, | 199 }, |
200 } | 200 } |
201 » moarTasks := map[string]*task_scheduler.TaskSpec{} | 201 » moarTasks := map[string]*scheduling.TaskSpec{} |
202 for name, task := range tasks { | 202 for name, task := range tasks { |
203 for i := 0; i < 100; i++ { | 203 for i := 0; i < 100; i++ { |
204 newName := fmt.Sprintf("%s%d", name, i) | 204 newName := fmt.Sprintf("%s%d", name, i) |
205 deps := make([]string, 0, len(task.Dependencies)) | 205 deps := make([]string, 0, len(task.Dependencies)) |
206 for _, d := range task.Dependencies { | 206 for _, d := range task.Dependencies { |
207 deps = append(deps, fmt.Sprintf("%s%d", d, i)) | 207 deps = append(deps, fmt.Sprintf("%s%d", d, i)) |
208 } | 208 } |
209 » » » newTask := &task_scheduler.TaskSpec{ | 209 » » » newTask := &scheduling.TaskSpec{ |
210 CipdPackages: task.CipdPackages, | 210 CipdPackages: task.CipdPackages, |
211 Dependencies: deps, | 211 Dependencies: deps, |
212 Dimensions: task.Dimensions, | 212 Dimensions: task.Dimensions, |
213 Isolate: task.Isolate, | 213 Isolate: task.Isolate, |
214 Priority: task.Priority, | 214 Priority: task.Priority, |
215 } | 215 } |
216 moarTasks[newName] = newTask | 216 moarTasks[newName] = newTask |
217 } | 217 } |
218 } | 218 } |
219 » cfg := task_scheduler.TasksCfg{ | 219 » cfg := scheduling.TasksCfg{ |
220 Tasks: moarTasks, | 220 Tasks: moarTasks, |
221 } | 221 } |
222 » f, err := os.Create(path.Join(repoDir, task_scheduler.TASKS_CFG_FILE)) | 222 » f, err := os.Create(path.Join(repoDir, scheduling.TASKS_CFG_FILE)) |
223 assertNoError(err) | 223 assertNoError(err) |
224 assertNoError(json.NewEncoder(f).Encode(&cfg)) | 224 assertNoError(json.NewEncoder(f).Encode(&cfg)) |
225 assertNoError(f.Close()) | 225 assertNoError(f.Close()) |
226 » run(repoDir, "git", "add", task_scheduler.TASKS_CFG_FILE) | 226 » run(repoDir, "git", "add", scheduling.TASKS_CFG_FILE) |
227 commit(repoDir, "Add more tasks!") | 227 commit(repoDir, "Add more tasks!") |
228 run(repoDir, "git", "push", "origin", "master") | 228 run(repoDir, "git", "push", "origin", "master") |
229 run(repoDir, "git", "branch", "-u", "origin/master") | 229 run(repoDir, "git", "branch", "-u", "origin/master") |
230 | 230 |
231 // Create a bunch of bots. | 231 // Create a bunch of bots. |
232 bots := make([]*swarming_api.SwarmingRpcsBotInfo, 100) | 232 bots := make([]*swarming_api.SwarmingRpcsBotInfo, 100) |
233 for idx, _ := range bots { | 233 for idx, _ := range bots { |
234 dims := map[string]string{ | 234 dims := map[string]string{ |
235 "pool": "Skia", | 235 "pool": "Skia", |
236 } | 236 } |
(...skipping 13 matching lines...) Expand all Loading... |
250 assertNoError(err) | 250 assertNoError(err) |
251 | 251 |
252 commits, err := repo.RevList(head) | 252 commits, err := repo.RevList(head) |
253 assertNoError(err) | 253 assertNoError(err) |
254 assertDeepEqual([]string{head}, commits) | 254 assertDeepEqual([]string{head}, commits) |
255 | 255 |
256 isolateClient, err := isolate.NewClient(workdir) | 256 isolateClient, err := isolate.NewClient(workdir) |
257 assertNoError(err) | 257 assertNoError(err) |
258 isolateClient.ServerUrl = isolate.FAKE_SERVER_URL | 258 isolateClient.ServerUrl = isolate.FAKE_SERVER_URL |
259 swarmingClient := swarming.NewTestClient() | 259 swarmingClient := swarming.NewTestClient() |
260 » s, err := task_scheduler.NewTaskScheduler(d, cache, time.Duration(math.M
axInt64), workdir, []string{"skia.git"}, isolateClient, swarmingClient) | 260 » s, err := scheduling.NewTaskScheduler(d, cache, time.Duration(math.MaxIn
t64), workdir, []string{"skia.git"}, isolateClient, swarmingClient) |
261 assertNoError(err) | 261 assertNoError(err) |
262 | 262 |
263 runTasks := func(bots []*swarming_api.SwarmingRpcsBotInfo) { | 263 runTasks := func(bots []*swarming_api.SwarmingRpcsBotInfo) { |
264 swarmingClient.MockBots(bots) | 264 swarmingClient.MockBots(bots) |
265 assertNoError(s.MainLoop()) | 265 assertNoError(s.MainLoop()) |
266 assertNoError(cache.Update()) | 266 assertNoError(cache.Update()) |
267 tasks, err := cache.GetTasksForCommits(repoName, commits) | 267 tasks, err := cache.GetTasksForCommits(repoName, commits) |
268 assertNoError(err) | 268 assertNoError(err) |
269 newTasks := map[string]*db.Task{} | 269 newTasks := map[string]*db.Task{} |
270 for _, v := range tasks { | 270 for _, v := range tasks { |
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
311 // Actually run the test. | 311 // Actually run the test. |
312 i := 0 | 312 i := 0 |
313 for ; ; i++ { | 313 for ; ; i++ { |
314 runTasks(bots) | 314 runTasks(bots) |
315 if s.QueueLen() == 0 { | 315 if s.QueueLen() == 0 { |
316 break | 316 break |
317 } | 317 } |
318 } | 318 } |
319 glog.Infof("Finished in %d iterations.", i) | 319 glog.Infof("Finished in %d iterations.", i) |
320 } | 320 } |
OLD | NEW |