Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(39)

Side by Side Diff: ct/go/worker_scripts/capture_skps/main.go

Issue 1392173005: [CT] Update worker scripts to use new benchmarks and delete old benchmarks (Closed) Base URL: https://skia.googlesource.com/buildbot@master
Patch Set: Remove staging code Created 5 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Application that captures SKPs from CT's webpage archives. 1 // Application that captures SKPs from CT's webpage archives.
2 package main 2 package main
3 3
4 import ( 4 import (
5 "encoding/csv" 5 "encoding/csv"
6 "encoding/json"
6 "flag" 7 "flag"
7 "fmt" 8 "fmt"
8 "io/ioutil" 9 "io/ioutil"
9 "os" 10 "os"
10 "path/filepath" 11 "path/filepath"
11 "runtime" 12 "runtime"
12 "sync" 13 "sync"
13 "time" 14 "time"
14 15
15 "github.com/skia-dev/glog" 16 "github.com/skia-dev/glog"
16 17
17 "strings"
18
19 "go.skia.org/infra/ct/go/util" 18 "go.skia.org/infra/ct/go/util"
20 "go.skia.org/infra/ct/go/worker_scripts/worker_common" 19 "go.skia.org/infra/ct/go/worker_scripts/worker_common"
21 "go.skia.org/infra/go/common" 20 "go.skia.org/infra/go/common"
22 skutil "go.skia.org/infra/go/util" 21 skutil "go.skia.org/infra/go/util"
23 ) 22 )
24 23
25 const ( 24 const (
26 // The number of goroutines that will run in parallel to capture SKPs. 25 // The number of goroutines that will run in parallel to capture SKPs.
27 WORKER_POOL_SIZE = 10 26 WORKER_POOL_SIZE = 10
28 ) 27 )
(...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after
152 151
153 // Create and run a goroutine closure that captures SKPs. 152 // Create and run a goroutine closure that captures SKPs.
154 go func() { 153 go func() {
155 // Decrement the WaitGroup counter when the goroutine co mpletes. 154 // Decrement the WaitGroup counter when the goroutine co mpletes.
156 defer wg.Done() 155 defer wg.Done()
157 156
158 for pagesetName := range pagesetRequests { 157 for pagesetName := range pagesetRequests {
159 158
160 mutex.RLock() 159 mutex.RLock()
161 160
162 pagesetBaseName := filepath.Base(pagesetName)
163 // Convert the filename into a format consumable by the run_benchmarks 161 // Convert the filename into a format consumable by the run_benchmarks
164 // binary. 162 // binary.
165 pagesetNameNoExt := strings.TrimSuffix(pagesetBa seName, filepath.Ext(pagesetBaseName))
166 pagesetPath := filepath.Join(pathToPagesets, pag esetName) 163 pagesetPath := filepath.Join(pathToPagesets, pag esetName)
167 164
165 pagesetContent, err := os.Open(pagesetPath)
166 if err != nil {
167 glog.Errorf("Could not read %s: %s", pag esetPath, err)
168 continue
169 }
170 decodedPageset := util.PagesetVars{}
171 if err := json.NewDecoder(pagesetContent).Decode (&decodedPageset); err != nil {
172 glog.Errorf("Could not JSON decode %s: % s", pagesetPath, err)
173 continue
174 }
175
168 glog.Infof("===== Processing %s =====", pagesetP ath) 176 glog.Infof("===== Processing %s =====", pagesetP ath)
169 177
170 skutil.LogErr(os.Chdir(pathToPyFiles)) 178 skutil.LogErr(os.Chdir(pathToPyFiles))
171 args := []string{ 179 args := []string{
172 » » » » » util.BINARY_RUN_BENCHMARK, 180 » » » » » filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
173 » » » » » fmt.Sprintf("%s.%s", util.BENCHMARK_SKPI CTURE_PRINTER, util.BenchmarksToPagesetName[util.BENCHMARK_SKPICTURE_PRINTER]), 181 » » » » » util.BenchmarksToPagesetName[util.BENCHM ARK_SKPICTURE_PRINTER],
174 » » » » » "--page-set-name=" + pagesetNameNoExt,
175 » » » » » "--page-set-base-dir=" + pathToPagesets,
176 "--also-run-disabled-tests", 182 "--also-run-disabled-tests",
177 "--page-repeat=1", // Only need one run for SKPs. 183 "--page-repeat=1", // Only need one run for SKPs.
178 "--skp-outdir=" + pathToSkps, 184 "--skp-outdir=" + pathToSkps,
179 "--extra-browser-args=" + util.DEFAULT_B ROWSER_ARGS, 185 "--extra-browser-args=" + util.DEFAULT_B ROWSER_ARGS,
186 "--user-agent=" + decodedPageset.UserAge nt,
187 "--urls-list=" + decodedPageset.UrlsList ,
188 "--archive-data-file=" + decodedPageset. ArchiveDataFile,
180 } 189 }
181 // Figure out which browser should be used. 190 // Figure out which browser should be used.
182 if *targetPlatform == util.PLATFORM_ANDROID { 191 if *targetPlatform == util.PLATFORM_ANDROID {
183 args = append(args, "--browser=android-c hromium") 192 args = append(args, "--browser=android-c hromium")
184 } else { 193 } else {
185 args = append(args, "--browser=exact", " --browser-executable="+chromiumBinary) 194 args = append(args, "--browser=exact", " --browser-executable="+chromiumBinary)
186 } 195 }
187 // Set the PYTHONPATH to the pagesets and the te lemetry dirs. 196 // Set the PYTHONPATH to the pagesets and the te lemetry dirs.
188 env := []string{ 197 env := []string{
189 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), 198 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir),
(...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after
297 } 306 }
298 writer := csv.NewWriter(csvFile) 307 writer := csv.NewWriter(csvFile)
299 defer writer.Flush() 308 defer writer.Flush()
300 for _, row := range [][]string{headers, values} { 309 for _, row := range [][]string{headers, values} {
301 if err := writer.Write(row); err != nil { 310 if err := writer.Write(row); err != nil {
302 return fmt.Errorf("Could not write to %s: %s", csvPath, err) 311 return fmt.Errorf("Could not write to %s: %s", csvPath, err)
303 } 312 }
304 } 313 }
305 return nil 314 return nil
306 } 315 }
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698