| OLD | NEW |
| 1 // Application that captures SKPs from CT's webpage archives. | 1 // Application that captures SKPs from CT's webpage archives. |
| 2 package main | 2 package main |
| 3 | 3 |
| 4 import ( | 4 import ( |
| 5 "encoding/csv" | 5 "encoding/csv" |
| 6 "flag" | 6 "flag" |
| 7 "fmt" | 7 "fmt" |
| 8 "io/ioutil" | 8 "io/ioutil" |
| 9 "os" | 9 "os" |
| 10 "path/filepath" | 10 "path/filepath" |
| 11 "runtime" | 11 "runtime" |
| 12 "sync" | 12 "sync" |
| 13 "time" | 13 "time" |
| 14 | 14 |
| 15 "github.com/skia-dev/glog" | 15 "github.com/skia-dev/glog" |
| 16 | 16 |
| 17 "strings" | |
| 18 | |
| 19 "go.skia.org/infra/ct/go/util" | 17 "go.skia.org/infra/ct/go/util" |
| 20 "go.skia.org/infra/ct/go/worker_scripts/worker_common" | 18 "go.skia.org/infra/ct/go/worker_scripts/worker_common" |
| 21 "go.skia.org/infra/go/common" | 19 "go.skia.org/infra/go/common" |
| 22 skutil "go.skia.org/infra/go/util" | 20 skutil "go.skia.org/infra/go/util" |
| 23 ) | 21 ) |
| 24 | 22 |
| 25 const ( | 23 const ( |
| 26 // The number of goroutines that will run in parallel to capture SKPs. | 24 // The number of goroutines that will run in parallel to capture SKPs. |
| 27 WORKER_POOL_SIZE = 10 | 25 WORKER_POOL_SIZE = 10 |
| 28 ) | 26 ) |
| (...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 152 | 150 |
| 153 // Create and run a goroutine closure that captures SKPs. | 151 // Create and run a goroutine closure that captures SKPs. |
| 154 go func() { | 152 go func() { |
| 155 // Decrement the WaitGroup counter when the goroutine co
mpletes. | 153 // Decrement the WaitGroup counter when the goroutine co
mpletes. |
| 156 defer wg.Done() | 154 defer wg.Done() |
| 157 | 155 |
| 158 for pagesetName := range pagesetRequests { | 156 for pagesetName := range pagesetRequests { |
| 159 | 157 |
| 160 mutex.RLock() | 158 mutex.RLock() |
| 161 | 159 |
| 162 » » » » pagesetBaseName := filepath.Base(pagesetName) | 160 » » » » // Read the pageset. |
| 163 » » » » // Convert the filename into a format consumable
by the run_benchmarks | |
| 164 » » » » // binary. | |
| 165 » » » » pagesetNameNoExt := strings.TrimSuffix(pagesetBa
seName, filepath.Ext(pagesetBaseName)) | |
| 166 pagesetPath := filepath.Join(pathToPagesets, pag
esetName) | 161 pagesetPath := filepath.Join(pathToPagesets, pag
esetName) |
| 162 decodedPageset, err := util.ReadPageset(pagesetP
ath) |
| 163 if err != nil { |
| 164 glog.Errorf("Could not read %s: %s", pag
esetPath, err) |
| 165 continue |
| 166 } |
| 167 | 167 |
| 168 glog.Infof("===== Processing %s =====", pagesetP
ath) | 168 glog.Infof("===== Processing %s =====", pagesetP
ath) |
| 169 | 169 |
| 170 skutil.LogErr(os.Chdir(pathToPyFiles)) | 170 skutil.LogErr(os.Chdir(pathToPyFiles)) |
| 171 args := []string{ | 171 args := []string{ |
| 172 » » » » » util.BINARY_RUN_BENCHMARK, | 172 » » » » » filepath.Join(util.TelemetryBinariesDir,
util.BINARY_RUN_BENCHMARK), |
| 173 » » » » » fmt.Sprintf("%s.%s", util.BENCHMARK_SKPI
CTURE_PRINTER, util.BenchmarksToPagesetName[util.BENCHMARK_SKPICTURE_PRINTER]), | 173 » » » » » util.BenchmarksToPagesetName[util.BENCHM
ARK_SKPICTURE_PRINTER], |
| 174 » » » » » "--page-set-name=" + pagesetNameNoExt, | |
| 175 » » » » » "--page-set-base-dir=" + pathToPagesets, | |
| 176 "--also-run-disabled-tests", | 174 "--also-run-disabled-tests", |
| 177 "--page-repeat=1", // Only need one run
for SKPs. | 175 "--page-repeat=1", // Only need one run
for SKPs. |
| 178 "--skp-outdir=" + pathToSkps, | 176 "--skp-outdir=" + pathToSkps, |
| 179 "--extra-browser-args=" + util.DEFAULT_B
ROWSER_ARGS, | 177 "--extra-browser-args=" + util.DEFAULT_B
ROWSER_ARGS, |
| 178 "--user-agent=" + decodedPageset.UserAge
nt, |
| 179 "--urls-list=" + decodedPageset.UrlsList
, |
| 180 "--archive-data-file=" + decodedPageset.
ArchiveDataFile, |
| 180 } | 181 } |
| 181 // Figure out which browser should be used. | 182 // Figure out which browser should be used. |
| 182 if *targetPlatform == util.PLATFORM_ANDROID { | 183 if *targetPlatform == util.PLATFORM_ANDROID { |
| 183 args = append(args, "--browser=android-c
hromium") | 184 args = append(args, "--browser=android-c
hromium") |
| 184 } else { | 185 } else { |
| 185 args = append(args, "--browser=exact", "
--browser-executable="+chromiumBinary) | 186 args = append(args, "--browser=exact", "
--browser-executable="+chromiumBinary) |
| 186 } | 187 } |
| 187 // Set the PYTHONPATH to the pagesets and the te
lemetry dirs. | 188 // Set the PYTHONPATH to the pagesets and the te
lemetry dirs. |
| 188 env := []string{ | 189 env := []string{ |
| 189 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON
PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), | 190 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON
PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), |
| (...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 297 } | 298 } |
| 298 writer := csv.NewWriter(csvFile) | 299 writer := csv.NewWriter(csvFile) |
| 299 defer writer.Flush() | 300 defer writer.Flush() |
| 300 for _, row := range [][]string{headers, values} { | 301 for _, row := range [][]string{headers, values} { |
| 301 if err := writer.Write(row); err != nil { | 302 if err := writer.Write(row); err != nil { |
| 302 return fmt.Errorf("Could not write to %s: %s", csvPath,
err) | 303 return fmt.Errorf("Could not write to %s: %s", csvPath,
err) |
| 303 } | 304 } |
| 304 } | 305 } |
| 305 return nil | 306 return nil |
| 306 } | 307 } |
| OLD | NEW |