| OLD | NEW |
| 1 // Application that captures SKPs from CT's webpage archives. | 1 // Application that captures SKPs from CT's webpage archives. |
| 2 package main | 2 package main |
| 3 | 3 |
| 4 import ( | 4 import ( |
| 5 "encoding/csv" | 5 "encoding/csv" |
| 6 "encoding/json" |
| 6 "flag" | 7 "flag" |
| 7 "fmt" | 8 "fmt" |
| 8 "io/ioutil" | 9 "io/ioutil" |
| 9 "os" | 10 "os" |
| 10 "path/filepath" | 11 "path/filepath" |
| 11 "runtime" | 12 "runtime" |
| 12 "sync" | 13 "sync" |
| 13 "time" | 14 "time" |
| 14 | 15 |
| 15 "github.com/skia-dev/glog" | 16 "github.com/skia-dev/glog" |
| 16 | 17 |
| 17 "strings" | |
| 18 | |
| 19 "go.skia.org/infra/ct/go/util" | 18 "go.skia.org/infra/ct/go/util" |
| 20 "go.skia.org/infra/ct/go/worker_scripts/worker_common" | 19 "go.skia.org/infra/ct/go/worker_scripts/worker_common" |
| 21 "go.skia.org/infra/go/common" | 20 "go.skia.org/infra/go/common" |
| 22 skutil "go.skia.org/infra/go/util" | 21 skutil "go.skia.org/infra/go/util" |
| 23 ) | 22 ) |
| 24 | 23 |
| 25 const ( | 24 const ( |
| 26 // The number of goroutines that will run in parallel to capture SKPs. | 25 // The number of goroutines that will run in parallel to capture SKPs. |
| 27 WORKER_POOL_SIZE = 10 | 26 WORKER_POOL_SIZE = 10 |
| 28 ) | 27 ) |
| (...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 152 | 151 |
| 153 // Create and run a goroutine closure that captures SKPs. | 152 // Create and run a goroutine closure that captures SKPs. |
| 154 go func() { | 153 go func() { |
| 155 // Decrement the WaitGroup counter when the goroutine co
mpletes. | 154 // Decrement the WaitGroup counter when the goroutine co
mpletes. |
| 156 defer wg.Done() | 155 defer wg.Done() |
| 157 | 156 |
| 158 for pagesetName := range pagesetRequests { | 157 for pagesetName := range pagesetRequests { |
| 159 | 158 |
| 160 mutex.RLock() | 159 mutex.RLock() |
| 161 | 160 |
| 162 pagesetBaseName := filepath.Base(pagesetName) | |
| 163 // Convert the filename into a format consumable
by the run_benchmarks | 161 // Convert the filename into a format consumable
by the run_benchmarks |
| 164 // binary. | 162 // binary. |
| 165 pagesetNameNoExt := strings.TrimSuffix(pagesetBa
seName, filepath.Ext(pagesetBaseName)) | |
| 166 pagesetPath := filepath.Join(pathToPagesets, pag
esetName) | 163 pagesetPath := filepath.Join(pathToPagesets, pag
esetName) |
| 167 | 164 |
| 165 pagesetContent, err := os.Open(pagesetPath) |
| 166 if err != nil { |
| 167 glog.Errorf("Could not read %s: %s", pag
esetPath, err) |
| 168 continue |
| 169 } |
| 170 decodedPageset := util.PagesetVars{} |
| 171 if err := json.NewDecoder(pagesetContent).Decode
(&decodedPageset); err != nil { |
| 172 glog.Errorf("Could not JSON decode %s: %
s", pagesetPath, err) |
| 173 continue |
| 174 } |
| 175 |
| 168 glog.Infof("===== Processing %s =====", pagesetP
ath) | 176 glog.Infof("===== Processing %s =====", pagesetP
ath) |
| 169 | 177 |
| 170 skutil.LogErr(os.Chdir(pathToPyFiles)) | 178 skutil.LogErr(os.Chdir(pathToPyFiles)) |
| 171 args := []string{ | 179 args := []string{ |
| 172 » » » » » util.BINARY_RUN_BENCHMARK, | 180 » » » » » filepath.Join(util.TelemetryBinariesDir,
util.BINARY_RUN_BENCHMARK), |
| 173 » » » » » fmt.Sprintf("%s.%s", util.BENCHMARK_SKPI
CTURE_PRINTER, util.BenchmarksToPagesetName[util.BENCHMARK_SKPICTURE_PRINTER]), | 181 » » » » » util.BenchmarksToPagesetName[util.BENCHM
ARK_SKPICTURE_PRINTER], |
| 174 » » » » » "--page-set-name=" + pagesetNameNoExt, | |
| 175 » » » » » "--page-set-base-dir=" + pathToPagesets, | |
| 176 "--also-run-disabled-tests", | 182 "--also-run-disabled-tests", |
| 177 "--page-repeat=1", // Only need one run
for SKPs. | 183 "--page-repeat=1", // Only need one run
for SKPs. |
| 178 "--skp-outdir=" + pathToSkps, | 184 "--skp-outdir=" + pathToSkps, |
| 179 "--extra-browser-args=" + util.DEFAULT_B
ROWSER_ARGS, | 185 "--extra-browser-args=" + util.DEFAULT_B
ROWSER_ARGS, |
| 186 "--user-agent=" + decodedPageset.UserAge
nt, |
| 187 "--urls-list=" + decodedPageset.UrlsList
, |
| 188 "--archive-data-file=" + decodedPageset.
ArchiveDataFile, |
| 180 } | 189 } |
| 181 // Figure out which browser should be used. | 190 // Figure out which browser should be used. |
| 182 if *targetPlatform == util.PLATFORM_ANDROID { | 191 if *targetPlatform == util.PLATFORM_ANDROID { |
| 183 args = append(args, "--browser=android-c
hromium") | 192 args = append(args, "--browser=android-c
hromium") |
| 184 } else { | 193 } else { |
| 185 args = append(args, "--browser=exact", "
--browser-executable="+chromiumBinary) | 194 args = append(args, "--browser=exact", "
--browser-executable="+chromiumBinary) |
| 186 } | 195 } |
| 187 // Set the PYTHONPATH to the pagesets and the te
lemetry dirs. | 196 // Set the PYTHONPATH to the pagesets and the te
lemetry dirs. |
| 188 env := []string{ | 197 env := []string{ |
| 189 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON
PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), | 198 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON
PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), |
| (...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 297 } | 306 } |
| 298 writer := csv.NewWriter(csvFile) | 307 writer := csv.NewWriter(csvFile) |
| 299 defer writer.Flush() | 308 defer writer.Flush() |
| 300 for _, row := range [][]string{headers, values} { | 309 for _, row := range [][]string{headers, values} { |
| 301 if err := writer.Write(row); err != nil { | 310 if err := writer.Write(row); err != nil { |
| 302 return fmt.Errorf("Could not write to %s: %s", csvPath,
err) | 311 return fmt.Errorf("Could not write to %s: %s", csvPath,
err) |
| 303 } | 312 } |
| 304 } | 313 } |
| 305 return nil | 314 return nil |
| 306 } | 315 } |
| OLD | NEW |