Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(652)

Side by Side Diff: ct/go/worker_scripts/capture_skps/main.go

Issue 1392173005: [CT] Update worker scripts to use new benchmarks and delete old benchmarks (Closed) Base URL: https://skia.googlesource.com/buildbot@master
Patch Set: Cleanup capture_archives Created 5 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Application that captures SKPs from CT's webpage archives. 1 // Application that captures SKPs from CT's webpage archives.
2 package main 2 package main
3 3
4 import ( 4 import (
5 "encoding/csv" 5 "encoding/csv"
6 "flag" 6 "flag"
7 "fmt" 7 "fmt"
8 "io/ioutil" 8 "io/ioutil"
9 "os" 9 "os"
10 "path/filepath" 10 "path/filepath"
11 "runtime" 11 "runtime"
12 "sync" 12 "sync"
13 "time" 13 "time"
14 14
15 "github.com/skia-dev/glog" 15 "github.com/skia-dev/glog"
16 16
17 "strings"
18
19 "go.skia.org/infra/ct/go/util" 17 "go.skia.org/infra/ct/go/util"
20 "go.skia.org/infra/ct/go/worker_scripts/worker_common" 18 "go.skia.org/infra/ct/go/worker_scripts/worker_common"
21 "go.skia.org/infra/go/common" 19 "go.skia.org/infra/go/common"
22 skutil "go.skia.org/infra/go/util" 20 skutil "go.skia.org/infra/go/util"
23 ) 21 )
24 22
25 const ( 23 const (
26 // The number of goroutines that will run in parallel to capture SKPs. 24 // The number of goroutines that will run in parallel to capture SKPs.
27 WORKER_POOL_SIZE = 10 25 WORKER_POOL_SIZE = 10
28 ) 26 )
(...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after
152 150
153 // Create and run a goroutine closure that captures SKPs. 151 // Create and run a goroutine closure that captures SKPs.
154 go func() { 152 go func() {
155 // Decrement the WaitGroup counter when the goroutine co mpletes. 153 // Decrement the WaitGroup counter when the goroutine co mpletes.
156 defer wg.Done() 154 defer wg.Done()
157 155
158 for pagesetName := range pagesetRequests { 156 for pagesetName := range pagesetRequests {
159 157
160 mutex.RLock() 158 mutex.RLock()
161 159
162 » » » » pagesetBaseName := filepath.Base(pagesetName) 160 » » » » // Read the pageset.
163 » » » » // Convert the filename into a format consumable by the run_benchmarks
164 » » » » // binary.
165 » » » » pagesetNameNoExt := strings.TrimSuffix(pagesetBa seName, filepath.Ext(pagesetBaseName))
166 pagesetPath := filepath.Join(pathToPagesets, pag esetName) 161 pagesetPath := filepath.Join(pathToPagesets, pag esetName)
162 decodedPageset, err := util.ReadPageset(pagesetP ath)
163 if err != nil {
164 glog.Errorf("Could not read %s: %s", pag esetPath, err)
165 continue
166 }
167 167
168 glog.Infof("===== Processing %s =====", pagesetP ath) 168 glog.Infof("===== Processing %s =====", pagesetP ath)
169 169
170 skutil.LogErr(os.Chdir(pathToPyFiles)) 170 skutil.LogErr(os.Chdir(pathToPyFiles))
171 args := []string{ 171 args := []string{
172 » » » » » util.BINARY_RUN_BENCHMARK, 172 » » » » » filepath.Join(util.TelemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
173 » » » » » fmt.Sprintf("%s.%s", util.BENCHMARK_SKPI CTURE_PRINTER, util.BenchmarksToPagesetName[util.BENCHMARK_SKPICTURE_PRINTER]), 173 » » » » » util.BenchmarksToPagesetName[util.BENCHM ARK_SKPICTURE_PRINTER],
174 » » » » » "--page-set-name=" + pagesetNameNoExt,
175 » » » » » "--page-set-base-dir=" + pathToPagesets,
176 "--also-run-disabled-tests", 174 "--also-run-disabled-tests",
177 "--page-repeat=1", // Only need one run for SKPs. 175 "--page-repeat=1", // Only need one run for SKPs.
178 "--skp-outdir=" + pathToSkps, 176 "--skp-outdir=" + pathToSkps,
179 "--extra-browser-args=" + util.DEFAULT_B ROWSER_ARGS, 177 "--extra-browser-args=" + util.DEFAULT_B ROWSER_ARGS,
178 "--user-agent=" + decodedPageset.UserAge nt,
179 "--urls-list=" + decodedPageset.UrlsList ,
180 "--archive-data-file=" + decodedPageset. ArchiveDataFile,
180 } 181 }
181 // Figure out which browser should be used. 182 // Figure out which browser should be used.
182 if *targetPlatform == util.PLATFORM_ANDROID { 183 if *targetPlatform == util.PLATFORM_ANDROID {
183 args = append(args, "--browser=android-c hromium") 184 args = append(args, "--browser=android-c hromium")
184 } else { 185 } else {
185 args = append(args, "--browser=exact", " --browser-executable="+chromiumBinary) 186 args = append(args, "--browser=exact", " --browser-executable="+chromiumBinary)
186 } 187 }
187 // Set the PYTHONPATH to the pagesets and the te lemetry dirs. 188 // Set the PYTHONPATH to the pagesets and the te lemetry dirs.
188 env := []string{ 189 env := []string{
189 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), 190 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir),
(...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after
297 } 298 }
298 writer := csv.NewWriter(csvFile) 299 writer := csv.NewWriter(csvFile)
299 defer writer.Flush() 300 defer writer.Flush()
300 for _, row := range [][]string{headers, values} { 301 for _, row := range [][]string{headers, values} {
301 if err := writer.Write(row); err != nil { 302 if err := writer.Write(row); err != nil {
302 return fmt.Errorf("Could not write to %s: %s", csvPath, err) 303 return fmt.Errorf("Could not write to %s: %s", csvPath, err)
303 } 304 }
304 } 305 }
305 return nil 306 return nil
306 } 307 }
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698