OLD | NEW |
1 // Application that captures SKPs from CT's webpage archives. | 1 // Application that captures SKPs from CT's webpage archives. |
2 package main | 2 package main |
3 | 3 |
4 import ( | 4 import ( |
5 "encoding/csv" | 5 "encoding/csv" |
6 "flag" | 6 "flag" |
7 "fmt" | 7 "fmt" |
8 "io/ioutil" | 8 "io/ioutil" |
9 "os" | 9 "os" |
10 "path/filepath" | 10 "path/filepath" |
11 "runtime" | 11 "runtime" |
12 "sync" | 12 "sync" |
13 "time" | 13 "time" |
14 | 14 |
15 "github.com/skia-dev/glog" | 15 "github.com/skia-dev/glog" |
16 | 16 |
17 "strings" | |
18 | |
19 "go.skia.org/infra/ct/go/util" | 17 "go.skia.org/infra/ct/go/util" |
20 "go.skia.org/infra/ct/go/worker_scripts/worker_common" | 18 "go.skia.org/infra/ct/go/worker_scripts/worker_common" |
21 "go.skia.org/infra/go/common" | 19 "go.skia.org/infra/go/common" |
22 skutil "go.skia.org/infra/go/util" | 20 skutil "go.skia.org/infra/go/util" |
23 ) | 21 ) |
24 | 22 |
25 const ( | 23 const ( |
26 // The number of goroutines that will run in parallel to capture SKPs. | 24 // The number of goroutines that will run in parallel to capture SKPs. |
27 WORKER_POOL_SIZE = 10 | 25 WORKER_POOL_SIZE = 10 |
28 ) | 26 ) |
(...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
152 | 150 |
153 // Create and run a goroutine closure that captures SKPs. | 151 // Create and run a goroutine closure that captures SKPs. |
154 go func() { | 152 go func() { |
155 // Decrement the WaitGroup counter when the goroutine co
mpletes. | 153 // Decrement the WaitGroup counter when the goroutine co
mpletes. |
156 defer wg.Done() | 154 defer wg.Done() |
157 | 155 |
158 for pagesetName := range pagesetRequests { | 156 for pagesetName := range pagesetRequests { |
159 | 157 |
160 mutex.RLock() | 158 mutex.RLock() |
161 | 159 |
162 » » » » pagesetBaseName := filepath.Base(pagesetName) | 160 » » » » // Read the pageset. |
163 » » » » // Convert the filename into a format consumable
by the run_benchmarks | |
164 » » » » // binary. | |
165 » » » » pagesetNameNoExt := strings.TrimSuffix(pagesetBa
seName, filepath.Ext(pagesetBaseName)) | |
166 pagesetPath := filepath.Join(pathToPagesets, pag
esetName) | 161 pagesetPath := filepath.Join(pathToPagesets, pag
esetName) |
| 162 decodedPageset, err := util.ReadPageset(pagesetP
ath) |
| 163 if err != nil { |
| 164 glog.Errorf("Could not read %s: %s", pag
esetPath, err) |
| 165 continue |
| 166 } |
167 | 167 |
168 glog.Infof("===== Processing %s =====", pagesetP
ath) | 168 glog.Infof("===== Processing %s =====", pagesetP
ath) |
169 | 169 |
170 skutil.LogErr(os.Chdir(pathToPyFiles)) | 170 skutil.LogErr(os.Chdir(pathToPyFiles)) |
171 args := []string{ | 171 args := []string{ |
172 » » » » » util.BINARY_RUN_BENCHMARK, | 172 » » » » » filepath.Join(util.TelemetryBinariesDir,
util.BINARY_RUN_BENCHMARK), |
173 » » » » » fmt.Sprintf("%s.%s", util.BENCHMARK_SKPI
CTURE_PRINTER, util.BenchmarksToPagesetName[util.BENCHMARK_SKPICTURE_PRINTER]), | 173 » » » » » util.BenchmarksToPagesetName[util.BENCHM
ARK_SKPICTURE_PRINTER], |
174 » » » » » "--page-set-name=" + pagesetNameNoExt, | |
175 » » » » » "--page-set-base-dir=" + pathToPagesets, | |
176 "--also-run-disabled-tests", | 174 "--also-run-disabled-tests", |
177 "--page-repeat=1", // Only need one run
for SKPs. | 175 "--page-repeat=1", // Only need one run
for SKPs. |
178 "--skp-outdir=" + pathToSkps, | 176 "--skp-outdir=" + pathToSkps, |
179 "--extra-browser-args=" + util.DEFAULT_B
ROWSER_ARGS, | 177 "--extra-browser-args=" + util.DEFAULT_B
ROWSER_ARGS, |
| 178 "--user-agent=" + decodedPageset.UserAge
nt, |
| 179 "--urls-list=" + decodedPageset.UrlsList
, |
| 180 "--archive-data-file=" + decodedPageset.
ArchiveDataFile, |
180 } | 181 } |
181 // Figure out which browser should be used. | 182 // Figure out which browser should be used. |
182 if *targetPlatform == util.PLATFORM_ANDROID { | 183 if *targetPlatform == util.PLATFORM_ANDROID { |
183 args = append(args, "--browser=android-c
hromium") | 184 args = append(args, "--browser=android-c
hromium") |
184 } else { | 185 } else { |
185 args = append(args, "--browser=exact", "
--browser-executable="+chromiumBinary) | 186 args = append(args, "--browser=exact", "
--browser-executable="+chromiumBinary) |
186 } | 187 } |
187 // Set the PYTHONPATH to the pagesets and the te
lemetry dirs. | 188 // Set the PYTHONPATH to the pagesets and the te
lemetry dirs. |
188 env := []string{ | 189 env := []string{ |
189 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON
PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), | 190 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON
PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), |
(...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
297 } | 298 } |
298 writer := csv.NewWriter(csvFile) | 299 writer := csv.NewWriter(csvFile) |
299 defer writer.Flush() | 300 defer writer.Flush() |
300 for _, row := range [][]string{headers, values} { | 301 for _, row := range [][]string{headers, values} { |
301 if err := writer.Write(row); err != nil { | 302 if err := writer.Write(row); err != nil { |
302 return fmt.Errorf("Could not write to %s: %s", csvPath,
err) | 303 return fmt.Errorf("Could not write to %s: %s", csvPath,
err) |
303 } | 304 } |
304 } | 305 } |
305 return nil | 306 return nil |
306 } | 307 } |
OLD | NEW |