OLD | NEW |
1 // Application that captures SKPs from CT's webpage archives. | 1 // Application that captures SKPs from CT's webpage archives. |
2 package main | 2 package main |
3 | 3 |
4 import ( | 4 import ( |
5 "encoding/csv" | 5 "encoding/csv" |
| 6 "encoding/json" |
6 "flag" | 7 "flag" |
7 "fmt" | 8 "fmt" |
8 "io/ioutil" | 9 "io/ioutil" |
9 "os" | 10 "os" |
10 "path/filepath" | 11 "path/filepath" |
11 "runtime" | 12 "runtime" |
12 "sync" | 13 "sync" |
13 "time" | 14 "time" |
14 | 15 |
15 "github.com/skia-dev/glog" | 16 "github.com/skia-dev/glog" |
16 | 17 |
17 "strings" | |
18 | |
19 "go.skia.org/infra/ct/go/util" | 18 "go.skia.org/infra/ct/go/util" |
20 "go.skia.org/infra/ct/go/worker_scripts/worker_common" | 19 "go.skia.org/infra/ct/go/worker_scripts/worker_common" |
21 "go.skia.org/infra/go/common" | 20 "go.skia.org/infra/go/common" |
22 skutil "go.skia.org/infra/go/util" | 21 skutil "go.skia.org/infra/go/util" |
23 ) | 22 ) |
24 | 23 |
25 const ( | 24 const ( |
26 // The number of goroutines that will run in parallel to capture SKPs. | 25 // The number of goroutines that will run in parallel to capture SKPs. |
27 WORKER_POOL_SIZE = 10 | 26 WORKER_POOL_SIZE = 10 |
28 ) | 27 ) |
(...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
152 | 151 |
153 // Create and run a goroutine closure that captures SKPs. | 152 // Create and run a goroutine closure that captures SKPs. |
154 go func() { | 153 go func() { |
155 // Decrement the WaitGroup counter when the goroutine co
mpletes. | 154 // Decrement the WaitGroup counter when the goroutine co
mpletes. |
156 defer wg.Done() | 155 defer wg.Done() |
157 | 156 |
158 for pagesetName := range pagesetRequests { | 157 for pagesetName := range pagesetRequests { |
159 | 158 |
160 mutex.RLock() | 159 mutex.RLock() |
161 | 160 |
162 pagesetBaseName := filepath.Base(pagesetName) | |
163 // Convert the filename into a format consumable
by the run_benchmarks | 161 // Convert the filename into a format consumable
by the run_benchmarks |
164 // binary. | 162 // binary. |
165 pagesetNameNoExt := strings.TrimSuffix(pagesetBa
seName, filepath.Ext(pagesetBaseName)) | |
166 pagesetPath := filepath.Join(pathToPagesets, pag
esetName) | 163 pagesetPath := filepath.Join(pathToPagesets, pag
esetName) |
167 | 164 |
| 165 pagesetContent, err := os.Open(pagesetPath) |
| 166 if err != nil { |
| 167 glog.Errorf("Could not read %s: %s", pag
esetPath, err) |
| 168 continue |
| 169 } |
| 170 decodedPageset := util.PagesetVars{} |
| 171 if err := json.NewDecoder(pagesetContent).Decode
(&decodedPageset); err != nil { |
| 172 glog.Errorf("Could not JSON decode %s: %
s", pagesetPath, err) |
| 173 continue |
| 174 } |
| 175 |
168 glog.Infof("===== Processing %s =====", pagesetP
ath) | 176 glog.Infof("===== Processing %s =====", pagesetP
ath) |
169 | 177 |
170 skutil.LogErr(os.Chdir(pathToPyFiles)) | 178 skutil.LogErr(os.Chdir(pathToPyFiles)) |
171 args := []string{ | 179 args := []string{ |
172 » » » » » util.BINARY_RUN_BENCHMARK, | 180 » » » » » filepath.Join(util.TelemetryBinariesDir,
util.BINARY_RUN_BENCHMARK), |
173 » » » » » fmt.Sprintf("%s.%s", util.BENCHMARK_SKPI
CTURE_PRINTER, util.BenchmarksToPagesetName[util.BENCHMARK_SKPICTURE_PRINTER]), | 181 » » » » » util.BenchmarksToPagesetName[util.BENCHM
ARK_SKPICTURE_PRINTER], |
174 » » » » » "--page-set-name=" + pagesetNameNoExt, | |
175 » » » » » "--page-set-base-dir=" + pathToPagesets, | |
176 "--also-run-disabled-tests", | 182 "--also-run-disabled-tests", |
177 "--page-repeat=1", // Only need one run
for SKPs. | 183 "--page-repeat=1", // Only need one run
for SKPs. |
178 "--skp-outdir=" + pathToSkps, | 184 "--skp-outdir=" + pathToSkps, |
179 "--extra-browser-args=" + util.DEFAULT_B
ROWSER_ARGS, | 185 "--extra-browser-args=" + util.DEFAULT_B
ROWSER_ARGS, |
| 186 "--user-agent=" + decodedPageset.UserAge
nt, |
| 187 "--urls-list=" + decodedPageset.UrlsList
, |
| 188 "--archive-data-file=" + decodedPageset.
ArchiveDataFile, |
180 } | 189 } |
181 // Figure out which browser should be used. | 190 // Figure out which browser should be used. |
182 if *targetPlatform == util.PLATFORM_ANDROID { | 191 if *targetPlatform == util.PLATFORM_ANDROID { |
183 args = append(args, "--browser=android-c
hromium") | 192 args = append(args, "--browser=android-c
hromium") |
184 } else { | 193 } else { |
185 args = append(args, "--browser=exact", "
--browser-executable="+chromiumBinary) | 194 args = append(args, "--browser=exact", "
--browser-executable="+chromiumBinary) |
186 } | 195 } |
187 // Set the PYTHONPATH to the pagesets and the te
lemetry dirs. | 196 // Set the PYTHONPATH to the pagesets and the te
lemetry dirs. |
188 env := []string{ | 197 env := []string{ |
189 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON
PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), | 198 fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHON
PATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir), |
(...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
297 } | 306 } |
298 writer := csv.NewWriter(csvFile) | 307 writer := csv.NewWriter(csvFile) |
299 defer writer.Flush() | 308 defer writer.Flush() |
300 for _, row := range [][]string{headers, values} { | 309 for _, row := range [][]string{headers, values} { |
301 if err := writer.Write(row); err != nil { | 310 if err := writer.Write(row); err != nil { |
302 return fmt.Errorf("Could not write to %s: %s", csvPath,
err) | 311 return fmt.Errorf("Could not write to %s: %s", csvPath,
err) |
303 } | 312 } |
304 } | 313 } |
305 return nil | 314 return nil |
306 } | 315 } |
OLD | NEW |