OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Archives or replays webpages and creates SKPs in a Google Storage location. | 6 """Archives or replays webpages and creates SKPs in a Google Storage location. |
7 | 7 |
8 To archive webpages and store SKP files (archives should be rarely updated): | 8 To archive webpages and store SKP files (archives should be rarely updated): |
9 | 9 |
10 cd ../buildbot/slave/skia_slave_scripts | 10 cd ../buildbot/slave/skia_slave_scripts |
(...skipping 184 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
195 | 195 |
196 # Create the required local storage directories. | 196 # Create the required local storage directories. |
197 self._CreateLocalStorageDirs() | 197 self._CreateLocalStorageDirs() |
198 | 198 |
199 # Start the timer. | 199 # Start the timer. |
200 start_time = time.time() | 200 start_time = time.time() |
201 | 201 |
202 # Loop through all page_sets. | 202 # Loop through all page_sets. |
203 for page_set in self._page_sets: | 203 for page_set in self._page_sets: |
204 | 204 |
205 page_set_basename = os.path.basename(page_set).split('.')[0] + '.json' | 205 page_set_basename = os.path.basename(page_set).split('.')[0] |
| 206 page_set_json_name = page_set_basename + '.json' |
206 wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr' | 207 wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr' |
| 208 page_set_dir = os.path.dirname(page_set) |
207 | 209 |
208 if self._record: | 210 if self._record: |
209 # Create an archive of the specified webpages if '--record=True' is | 211 # Create an archive of the specified webpages if '--record=True' is |
210 # specified. | 212 # specified. |
211 record_wpr_cmd = ( | 213 record_wpr_cmd = ( |
| 214 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, |
212 'DISPLAY=%s' % X11_DISPLAY, | 215 'DISPLAY=%s' % X11_DISPLAY, |
213 os.path.join(self._telemetry_binaries_dir, 'record_wpr'), | 216 os.path.join(self._telemetry_binaries_dir, 'record_wpr'), |
214 '--extra-browser-args=--disable-setuid-sandbox', | 217 '--extra-browser-args=--disable-setuid-sandbox', |
215 '--browser=exact', | 218 '--browser=exact', |
216 '--browser-executable=%s' % self._browser_executable, | 219 '--browser-executable=%s' % self._browser_executable, |
217 page_set | 220 '%s_page_set' % page_set_basename, |
| 221 '--page-set-base-dir=%s' % page_set_dir |
218 ) | 222 ) |
219 for _ in range(RETRY_RECORD_WPR_COUNT): | 223 for _ in range(RETRY_RECORD_WPR_COUNT): |
220 output = shell_utils.run(' '.join(record_wpr_cmd), shell=True) | 224 output = shell_utils.run(' '.join(record_wpr_cmd), shell=True) |
221 if RECORD_FAILURE_MSG in output: | 225 if RECORD_FAILURE_MSG in output: |
222 print output | 226 print output |
223 else: | 227 else: |
224 # Break out of the retry loop since there were no errors. | 228 # Break out of the retry loop since there were no errors. |
225 break | 229 break |
226 else: | 230 else: |
227 # If we get here then record_wpr did not succeed and thus did not | 231 # If we get here then record_wpr did not succeed and thus did not |
228 # break out of the loop. | 232 # break out of the loop. |
229 raise Exception('record_wpr failed for page_set: %s' % page_set) | 233 raise Exception('record_wpr failed for page_set: %s' % page_set) |
230 | 234 |
231 else: | 235 else: |
232 if not self._skip_all_gs_access: | 236 if not self._skip_all_gs_access: |
233 # Get the webpages archive so that it can be replayed. | 237 # Get the webpages archive so that it can be replayed. |
234 self._DownloadWebpagesArchive(wpr_data_file, page_set_basename) | 238 self._DownloadWebpagesArchive(wpr_data_file, page_set_json_name) |
235 | 239 |
236 page_set_name = os.path.basename(page_set).split('.')[0] | |
237 page_set_dir = os.path.dirname(page_set) | |
238 run_benchmark_cmd = ( | 240 run_benchmark_cmd = ( |
239 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, | 241 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, |
240 'DISPLAY=%s' % X11_DISPLAY, | 242 'DISPLAY=%s' % X11_DISPLAY, |
241 'timeout', '300', | 243 'timeout', '300', |
242 os.path.join(self._telemetry_binaries_dir, 'run_benchmark'), | 244 os.path.join(self._telemetry_binaries_dir, 'run_benchmark'), |
243 '--extra-browser-args=--disable-setuid-sandbox', | 245 '--extra-browser-args=--disable-setuid-sandbox', |
244 '--browser=exact', | 246 '--browser=exact', |
245 '--browser-executable=%s' % self._browser_executable, | 247 '--browser-executable=%s' % self._browser_executable, |
246 SKP_BENCHMARK, | 248 SKP_BENCHMARK, |
247 '--page-set-name=%s' % page_set_name, | 249 '--page-set-name=%s' % page_set_basename, |
248 '--page-set-base-dir=%s' % page_set_dir, | 250 '--page-set-base-dir=%s' % page_set_dir, |
249 '--skp-outdir=%s' % TMP_SKP_DIR, | 251 '--skp-outdir=%s' % TMP_SKP_DIR, |
250 '--also-run-disabled-tests' | 252 '--also-run-disabled-tests' |
251 ) | 253 ) |
252 | 254 |
253 for _ in range(RETRY_RUN_MEASUREMENT_COUNT): | 255 for _ in range(RETRY_RUN_MEASUREMENT_COUNT): |
254 try: | 256 try: |
255 print '\n\n=======Capturing SKP of %s=======\n\n' % page_set | 257 print '\n\n=======Capturing SKP of %s=======\n\n' % page_set |
256 shell_utils.run(' '.join(run_benchmark_cmd), shell=True) | 258 shell_utils.run(' '.join(run_benchmark_cmd), shell=True) |
257 except shell_utils.CommandFailedException: | 259 except shell_utils.CommandFailedException: |
258 # skpicture_printer sometimes fails with AssertionError but the | 260 # skpicture_printer sometimes fails with AssertionError but the |
259 # captured SKP is still valid. This is a known issue. | 261 # captured SKP is still valid. This is a known issue. |
260 pass | 262 pass |
261 | 263 |
262 if self._record: | 264 if self._record: |
263 # Move over the created archive into the local webpages archive | 265 # Move over the created archive into the local webpages archive |
264 # directory. | 266 # directory. |
265 shutil.move( | 267 shutil.move( |
266 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file), | 268 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file), |
267 self._local_record_webpages_archive_dir) | 269 self._local_record_webpages_archive_dir) |
268 shutil.move( | 270 shutil.move( |
269 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, | 271 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, |
270 page_set_basename), | 272 page_set_json_name), |
271 self._local_record_webpages_archive_dir) | 273 self._local_record_webpages_archive_dir) |
272 | 274 |
273 # Rename generated SKP files into more descriptive names. | 275 # Rename generated SKP files into more descriptive names. |
274 try: | 276 try: |
275 self._RenameSkpFiles(page_set) | 277 self._RenameSkpFiles(page_set) |
276 # Break out of the retry loop since there were no errors. | 278 # Break out of the retry loop since there were no errors. |
277 break | 279 break |
278 except Exception: | 280 except Exception: |
279 # There was a failure continue with the loop. | 281 # There was a failure continue with the loop. |
280 traceback.print_exc() | 282 traceback.print_exc() |
(...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
368 shutil.rmtree(site) | 370 shutil.rmtree(site) |
369 | 371 |
370 def _CreateLocalStorageDirs(self): | 372 def _CreateLocalStorageDirs(self): |
371 """Creates required local storage directories for this script.""" | 373 """Creates required local storage directories for this script.""" |
372 for d in (self._local_record_webpages_archive_dir, | 374 for d in (self._local_record_webpages_archive_dir, |
373 self._local_skp_dir): | 375 self._local_skp_dir): |
374 if os.path.exists(d): | 376 if os.path.exists(d): |
375 shutil.rmtree(d) | 377 shutil.rmtree(d) |
376 os.makedirs(d) | 378 os.makedirs(d) |
377 | 379 |
378 def _DownloadWebpagesArchive(self, wpr_data_file, page_set_basename): | 380 def _DownloadWebpagesArchive(self, wpr_data_file, page_set_json_name): |
379 """Downloads the webpages archive and its required page set from GS.""" | 381 """Downloads the webpages archive and its required page set from GS.""" |
380 wpr_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', | 382 wpr_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', |
381 wpr_data_file) | 383 wpr_data_file) |
382 page_set_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, | 384 page_set_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, |
383 'webpages_archive', | 385 'webpages_archive', |
384 page_set_basename) | 386 page_set_json_name) |
385 gs = gs_utils.GSUtils() | 387 gs = gs_utils.GSUtils() |
386 gs_bucket = self._dest_gsbase.lstrip(gs_utils.GS_PREFIX) | 388 gs_bucket = self._dest_gsbase.lstrip(gs_utils.GS_PREFIX) |
387 if (gs.does_storage_object_exist(gs_bucket, wpr_source) and | 389 if (gs.does_storage_object_exist(gs_bucket, wpr_source) and |
388 gs.does_storage_object_exist(gs_bucket, page_set_source)): | 390 gs.does_storage_object_exist(gs_bucket, page_set_source)): |
389 gs.download_file(gs_bucket, wpr_source, | 391 gs.download_file(gs_bucket, wpr_source, |
390 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, | 392 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, |
391 wpr_data_file)) | 393 wpr_data_file)) |
392 gs.download_file(gs_bucket, page_set_source, | 394 gs.download_file(gs_bucket, page_set_source, |
393 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, | 395 os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, |
394 page_set_basename)) | 396 page_set_json_name)) |
395 else: | 397 else: |
396 raise Exception('%s and %s do not exist in Google Storage!' % ( | 398 raise Exception('%s and %s do not exist in Google Storage!' % ( |
397 wpr_source, page_set_source)) | 399 wpr_source, page_set_source)) |
398 | 400 |
399 | 401 |
400 if '__main__' == __name__: | 402 if '__main__' == __name__: |
401 option_parser = optparse.OptionParser() | 403 option_parser = optparse.OptionParser() |
402 option_parser.add_option( | 404 option_parser.add_option( |
403 '', '--page_sets', | 405 '', '--page_sets', |
404 help='Specifies the page sets to use to archive. Supports globs.', | 406 help='Specifies the page sets to use to archive. Supports globs.', |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
450 default=None) | 452 default=None) |
451 option_parser.add_option( | 453 option_parser.add_option( |
452 '', '--non-interactive', action='store_true', | 454 '', '--non-interactive', action='store_true', |
453 help='Runs the script without any prompts. If this flag is specified and ' | 455 help='Runs the script without any prompts. If this flag is specified and ' |
454 '--skia_tools is specified then the debugger is not run.', | 456 '--skia_tools is specified then the debugger is not run.', |
455 default=False) | 457 default=False) |
456 options, unused_args = option_parser.parse_args() | 458 options, unused_args = option_parser.parse_args() |
457 | 459 |
458 playback = SkPicturePlayback(options) | 460 playback = SkPicturePlayback(options) |
459 sys.exit(playback.Run()) | 461 sys.exit(playback.Run()) |
OLD | NEW |