Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(23)

Side by Side Diff: task_scheduler/go/scheduling/perftest/perftest.go

Issue 2296763008: [task scheduler] Move files from build_scheduler/ to task_scheduler/ (Closed) Base URL: https://skia.googlesource.com/buildbot@master
Patch Set: Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « task_scheduler/go/scheduling/cache_wrapper.go ('k') | task_scheduler/go/scheduling/specs.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 package main 1 package main
2 2
3 /* 3 /*
4 Performance test for TaskScheduler. 4 Performance test for TaskScheduler.
5 */ 5 */
6 6
7 import ( 7 import (
8 "encoding/json" 8 "encoding/json"
9 "fmt" 9 "fmt"
10 "io/ioutil" 10 "io/ioutil"
11 "math" 11 "math"
12 "net/http" 12 "net/http"
13 _ "net/http/pprof" 13 _ "net/http/pprof"
14 "os" 14 "os"
15 "path" 15 "path"
16 "reflect" 16 "reflect"
17 "time" 17 "time"
18 18
19 "github.com/davecgh/go-spew/spew" 19 "github.com/davecgh/go-spew/spew"
20 swarming_api "github.com/luci/luci-go/common/api/swarming/swarming/v1" 20 swarming_api "github.com/luci/luci-go/common/api/swarming/swarming/v1"
21 "github.com/skia-dev/glog" 21 "github.com/skia-dev/glog"
22 "go.skia.org/infra/build_scheduler/go/db"
23 "go.skia.org/infra/build_scheduler/go/db/local_db"
24 "go.skia.org/infra/build_scheduler/go/task_scheduler"
25 "go.skia.org/infra/go/common" 22 "go.skia.org/infra/go/common"
26 "go.skia.org/infra/go/exec" 23 "go.skia.org/infra/go/exec"
27 "go.skia.org/infra/go/gitinfo" 24 "go.skia.org/infra/go/gitinfo"
28 "go.skia.org/infra/go/isolate" 25 "go.skia.org/infra/go/isolate"
29 "go.skia.org/infra/go/swarming" 26 "go.skia.org/infra/go/swarming"
27 "go.skia.org/infra/task_scheduler/go/db"
28 "go.skia.org/infra/task_scheduler/go/db/local_db"
29 "go.skia.org/infra/task_scheduler/go/scheduling"
30 ) 30 )
31 31
32 func assertNoError(err error) { 32 func assertNoError(err error) {
33 if err != nil { 33 if err != nil {
34 glog.Fatalf("Expected no error but got: %s", err.Error()) 34 glog.Fatalf("Expected no error but got: %s", err.Error())
35 } 35 }
36 } 36 }
37 37
38 func assertEqual(a, b interface{}) { 38 func assertEqual(a, b interface{}) {
39 if a != b { 39 if a != b {
(...skipping 128 matching lines...) Expand 10 before | Expand all | Expand 10 after
168 'command': [ 168 'command': [
169 'python', 'recipes.py', 'run', 169 'python', 'recipes.py', 'run',
170 ], 170 ],
171 'files': [ 171 'files': [
172 '../../somefile.txt', 172 '../../somefile.txt',
173 ], 173 ],
174 }, 174 },
175 }`) 175 }`)
176 176
177 // Add tasks to the repo. 177 // Add tasks to the repo.
178 » var tasks = map[string]*task_scheduler.TaskSpec{ 178 » var tasks = map[string]*scheduling.TaskSpec{
179 » » "Build-Ubuntu-GCC-Arm7-Release-Android": &task_scheduler.TaskSpe c{ 179 » » "Build-Ubuntu-GCC-Arm7-Release-Android": &scheduling.TaskSpec{
180 » » » CipdPackages: []*task_scheduler.CipdPackage{}, 180 » » » CipdPackages: []*scheduling.CipdPackage{},
181 Dependencies: []string{}, 181 Dependencies: []string{},
182 Dimensions: []string{"pool:Skia", "os:Ubuntu"}, 182 Dimensions: []string{"pool:Skia", "os:Ubuntu"},
183 Isolate: "compile_skia.isolate", 183 Isolate: "compile_skia.isolate",
184 Priority: 0.9, 184 Priority: 0.9,
185 }, 185 },
186 » » "Test-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &task_schedul er.TaskSpec{ 186 » » "Test-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &scheduling.T askSpec{
187 » » » CipdPackages: []*task_scheduler.CipdPackage{}, 187 » » » CipdPackages: []*scheduling.CipdPackage{},
188 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An droid"}, 188 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An droid"},
189 Dimensions: []string{"pool:Skia", "os:Android", "devic e_type:grouper"}, 189 Dimensions: []string{"pool:Skia", "os:Android", "devic e_type:grouper"},
190 Isolate: "test_skia.isolate", 190 Isolate: "test_skia.isolate",
191 Priority: 0.9, 191 Priority: 0.9,
192 }, 192 },
193 » » "Perf-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &task_schedul er.TaskSpec{ 193 » » "Perf-Android-GCC-Nexus7-GPU-Tegra3-Arm7-Release": &scheduling.T askSpec{
194 » » » CipdPackages: []*task_scheduler.CipdPackage{}, 194 » » » CipdPackages: []*scheduling.CipdPackage{},
195 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An droid"}, 195 Dependencies: []string{"Build-Ubuntu-GCC-Arm7-Release-An droid"},
196 Dimensions: []string{"pool:Skia", "os:Android", "devic e_type:grouper"}, 196 Dimensions: []string{"pool:Skia", "os:Android", "devic e_type:grouper"},
197 Isolate: "perf_skia.isolate", 197 Isolate: "perf_skia.isolate",
198 Priority: 0.9, 198 Priority: 0.9,
199 }, 199 },
200 } 200 }
201 » moarTasks := map[string]*task_scheduler.TaskSpec{} 201 » moarTasks := map[string]*scheduling.TaskSpec{}
202 for name, task := range tasks { 202 for name, task := range tasks {
203 for i := 0; i < 100; i++ { 203 for i := 0; i < 100; i++ {
204 newName := fmt.Sprintf("%s%d", name, i) 204 newName := fmt.Sprintf("%s%d", name, i)
205 deps := make([]string, 0, len(task.Dependencies)) 205 deps := make([]string, 0, len(task.Dependencies))
206 for _, d := range task.Dependencies { 206 for _, d := range task.Dependencies {
207 deps = append(deps, fmt.Sprintf("%s%d", d, i)) 207 deps = append(deps, fmt.Sprintf("%s%d", d, i))
208 } 208 }
209 » » » newTask := &task_scheduler.TaskSpec{ 209 » » » newTask := &scheduling.TaskSpec{
210 CipdPackages: task.CipdPackages, 210 CipdPackages: task.CipdPackages,
211 Dependencies: deps, 211 Dependencies: deps,
212 Dimensions: task.Dimensions, 212 Dimensions: task.Dimensions,
213 Isolate: task.Isolate, 213 Isolate: task.Isolate,
214 Priority: task.Priority, 214 Priority: task.Priority,
215 } 215 }
216 moarTasks[newName] = newTask 216 moarTasks[newName] = newTask
217 } 217 }
218 } 218 }
219 » cfg := task_scheduler.TasksCfg{ 219 » cfg := scheduling.TasksCfg{
220 Tasks: moarTasks, 220 Tasks: moarTasks,
221 } 221 }
222 » f, err := os.Create(path.Join(repoDir, task_scheduler.TASKS_CFG_FILE)) 222 » f, err := os.Create(path.Join(repoDir, scheduling.TASKS_CFG_FILE))
223 assertNoError(err) 223 assertNoError(err)
224 assertNoError(json.NewEncoder(f).Encode(&cfg)) 224 assertNoError(json.NewEncoder(f).Encode(&cfg))
225 assertNoError(f.Close()) 225 assertNoError(f.Close())
226 » run(repoDir, "git", "add", task_scheduler.TASKS_CFG_FILE) 226 » run(repoDir, "git", "add", scheduling.TASKS_CFG_FILE)
227 commit(repoDir, "Add more tasks!") 227 commit(repoDir, "Add more tasks!")
228 run(repoDir, "git", "push", "origin", "master") 228 run(repoDir, "git", "push", "origin", "master")
229 run(repoDir, "git", "branch", "-u", "origin/master") 229 run(repoDir, "git", "branch", "-u", "origin/master")
230 230
231 // Create a bunch of bots. 231 // Create a bunch of bots.
232 bots := make([]*swarming_api.SwarmingRpcsBotInfo, 100) 232 bots := make([]*swarming_api.SwarmingRpcsBotInfo, 100)
233 for idx, _ := range bots { 233 for idx, _ := range bots {
234 dims := map[string]string{ 234 dims := map[string]string{
235 "pool": "Skia", 235 "pool": "Skia",
236 } 236 }
(...skipping 13 matching lines...) Expand all
250 assertNoError(err) 250 assertNoError(err)
251 251
252 commits, err := repo.RevList(head) 252 commits, err := repo.RevList(head)
253 assertNoError(err) 253 assertNoError(err)
254 assertDeepEqual([]string{head}, commits) 254 assertDeepEqual([]string{head}, commits)
255 255
256 isolateClient, err := isolate.NewClient(workdir) 256 isolateClient, err := isolate.NewClient(workdir)
257 assertNoError(err) 257 assertNoError(err)
258 isolateClient.ServerUrl = isolate.FAKE_SERVER_URL 258 isolateClient.ServerUrl = isolate.FAKE_SERVER_URL
259 swarmingClient := swarming.NewTestClient() 259 swarmingClient := swarming.NewTestClient()
260 » s, err := task_scheduler.NewTaskScheduler(d, cache, time.Duration(math.M axInt64), workdir, []string{"skia.git"}, isolateClient, swarmingClient) 260 » s, err := scheduling.NewTaskScheduler(d, cache, time.Duration(math.MaxIn t64), workdir, []string{"skia.git"}, isolateClient, swarmingClient)
261 assertNoError(err) 261 assertNoError(err)
262 262
263 runTasks := func(bots []*swarming_api.SwarmingRpcsBotInfo) { 263 runTasks := func(bots []*swarming_api.SwarmingRpcsBotInfo) {
264 swarmingClient.MockBots(bots) 264 swarmingClient.MockBots(bots)
265 assertNoError(s.MainLoop()) 265 assertNoError(s.MainLoop())
266 assertNoError(cache.Update()) 266 assertNoError(cache.Update())
267 tasks, err := cache.GetTasksForCommits(repoName, commits) 267 tasks, err := cache.GetTasksForCommits(repoName, commits)
268 assertNoError(err) 268 assertNoError(err)
269 newTasks := map[string]*db.Task{} 269 newTasks := map[string]*db.Task{}
270 for _, v := range tasks { 270 for _, v := range tasks {
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after
311 // Actually run the test. 311 // Actually run the test.
312 i := 0 312 i := 0
313 for ; ; i++ { 313 for ; ; i++ {
314 runTasks(bots) 314 runTasks(bots)
315 if s.QueueLen() == 0 { 315 if s.QueueLen() == 0 {
316 break 316 break
317 } 317 }
318 } 318 }
319 glog.Infof("Finished in %d iterations.", i) 319 glog.Infof("Finished in %d iterations.", i)
320 } 320 }
OLDNEW
« no previous file with comments | « task_scheduler/go/scheduling/cache_wrapper.go ('k') | task_scheduler/go/scheduling/specs.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698